From fc4329d444cf5330d3de309c6621d9f062256394 Mon Sep 17 00:00:00 2001 From: Daoyuan Li Date: Wed, 29 Jul 2026 16:51:16 -0700 Subject: [PATCH 1/3] Harden v0.2.3 clarity and artifact safety --- CHANGELOG.md | 24 ++++- README.md | 62 ++++++------ README.zh-CN.md | 45 +++++---- docs/banner.svg | 22 ++--- docs/gpu-calc-hard-equal-update-v2.svg | 50 +++++----- docs/limitations.md | 23 +++-- docs/single-gpu-guide.md | 13 ++- scripts/publish_benchmark_artifacts.py | 126 ++++++++++++++----------- src/miniverl/cache/store.py | 55 ++++++++--- src/miniverl/environments/base.py | 24 +++-- src/miniverl/models/tokenizers.py | 53 +++++++++-- src/miniverl/training/trainer.py | 32 ++++++- tests/unit/test_cache.py | 98 +++++++++++++++++++ tests/unit/test_packaging.py | 27 ++++-- tests/unit/test_protocol.py | 87 +++++++++++++++-- tests/unit/test_tokenizer_identity.py | 12 +++ tests/unit/test_trainer_lifecycle.py | 28 ++++++ 17 files changed, 583 insertions(+), 198 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9f1b8da..f1bdae6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,23 @@ All notable changes to miniVERL are recorded here. The format follows ## [Unreleased] +### Changed + +- The generated GPU benchmark figure now reports measured 0% negative controls, + separates protocol qualification from the quantitative axis, and distinguishes + continuation time from the reusable protocol-teacher preparation cost. +- The banner and bilingual onboarding now describe the device-name-agnostic + single-GPU CUDA path and install CUDA PyTorch before optional training extras. +- Protocol-v2 prompt examples are generated from each environment's active + `ToolSpec`; the immutable protocol-v1 prompt is unchanged. + +### Fixed + +- Legacy teacher caches can no longer bypass adapter or structural-tokenizer + identity checks, and cache shard/index publication is crash-safe. +- Tokenizer structural digests ignore source-location metadata, and failed + model construction cannot leave an orphan partial run directory. + ## [0.2.2] - 2026-07-29 Single-GPU portability and presentation release. @@ -291,7 +308,8 @@ Same-tokenizer only; one trajectory per forward pass; `swap` unavailable for quantized models; only Qwen3 and Qwen2 architectures tested; single-seed GPU results. The full list is in `docs/limitations.md`. -[Unreleased]: https://github.com/DaoyuanLi2816/mini-verl/compare/v0.2.1...HEAD +[Unreleased]: https://github.com/DaoyuanLi2816/mini-verl/compare/v0.2.2...HEAD +[0.2.2]: https://github.com/DaoyuanLi2816/mini-verl/compare/v0.2.1...v0.2.2 [0.2.1]: https://github.com/DaoyuanLi2816/mini-verl/compare/v0.2.0...v0.2.1 -[0.2.0]: https://github.com/DaoyuanLi2816/mini-verl/compare/v0.1.0...v0.2.0 -[0.1.0]: https://github.com/DaoyuanLi2816/mini-verl/releases/tag/v0.1.0 +[0.2.0]: https://github.com/DaoyuanLi2816/mini-verl/compare/37781ef0b00f3346d4b7b40fbe4d1c0ce1355063...v0.2.0 +[0.1.0]: https://github.com/DaoyuanLi2816/mini-verl/tree/37781ef0b00f3346d4b7b40fbe4d1c0ce1355063 diff --git a/README.md b/README.md index 0ba7f19..325fd40 100644 --- a/README.md +++ b/README.md @@ -87,7 +87,7 @@ keeps the whole lifecycle in one readable single-GPU process. | Compressed `top-k + tail` KL and JSD | yes; the unsmoothed coarse-graining has a proven lower-bound relationship to the exact loss | | Privileged-context teacher with an explicit alignment map | yes | | Frozen standard PEFT teacher adapters with provenance and competence gates | yes | -| Single-GPU CUDA path with automatic bf16/fp16 selection | yes; model-agnostic code path, measured reference on an RTX 4080 | +| Single-GPU CUDA path with automatic bf16/fp16 selection | yes; device-name-agnostic CUDA path, measured reference on an RTX 4080 | | `resident` and `swap` memory strategies, `auto` resolution | yes, with an equivalence test | | Versioned, checksummed, pickle-free teacher-target cache | yes | | SFT / offline KD / strict OPD / explicitly labeled replay behind one trainer | yes | @@ -115,33 +115,39 @@ keeps the whole lifecycle in one readable single-GPU process. The [public, immutable protocol-teacher adapter](https://huggingface.co/DaoyuanLi/mini-verl-qwen3-1.7b-protocol-teacher) is the default in the single-GPU recipe. It passed an independently -prespecified policy-competence gate before this benchmark was inspected. The -two controls intentionally remove that guarantee: their loss decreased -normally, but they taught the student an incompatible tool policy. This is a -measured teacher-competence failure, not a trainer crash. - -The historical teacher-selection gate and the downstream benchmark both used -the same 24-task v0.2 `test` set. Candidate A was prespecified and passed on the -first attempt, so no fallback tuning occurred, but the final set was not a -completely untouched test set. Future teacher selection uses `eval`; downstream -reporting uses `test`. See [limitations](docs/limitations.md). - -OPD still only ties SFT and takes about 6x as much training time here -(523.8 s versus 86.4 s on average). The task saturates; two seeds do not support -a significance claim, and no general OPD advantage is claimed. See the +prespecified policy-competence gate before this benchmark was inspected. Both +controls completed normally and measured 0% in both seeds; they were neither +configuration failures nor crashes. Both used the ambiguous historical +protocol-v1 prompt, so 0% cannot be attributed solely to intrinsic teacher +behaviour; it diagnoses the missing qualification gate in that setup. + +The historical gate and benchmark reused the same 24-task v0.2 `test` set. +Candidate A was prespecified and passed first try (no fallback tuning), but the +set was not untouched. Future selection uses `eval`; reporting uses `test`. +See [limitations](docs/limitations.md). + +OPD only ties SFT and takes 6.1× as much continuation time here (523.8 s versus +86.4 s). The task saturates; two seeds support neither significance nor a +general OPD advantage. See the [full result and legacy transcript diagnosis](docs/rtx4080-baselines.md). ![Two-seed protocol-teacher benchmark](docs/gpu-calc-hard-equal-update-v2.svg) -An older RTX 4080 pipeline smoke run used the now-preserved -[raw-teacher control recipe](recipes/qwen_consumer_gpu_calc_raw_teacher.yaml) -to train **Qwen3-0.6B** from **Qwen3-1.7B** in **481 s / 16 optimizer steps**, -peaking at **4.25 GiB -allocated / 4.76 GiB reserved**, and moved held-out greedy success from -**0.0% to 100.0%** on 12 tasks. The 8-cycle supervised cold start did most of -that work: the first OPD rollout batch already scored 83.3%. It demonstrates -the pipeline end to end, not an OPD-over-SFT result. Every number is traceable -to [`docs/rtx4080-baselines.md`](docs/rtx4080-baselines.md). +| Artifact | Role | +| --- | --- | +| [Default recipe](recipes/qwen_consumer_gpu_calc.yaml) | protocol-qualified default | +| [Schema-v2 benchmark](benchmarks/results/gpu-calc-hard-equal-update-v2.json) | frozen five-arm result | +| [Raw-teacher recipe](recipes/qwen_consumer_gpu_calc_raw_teacher.yaml) | historical control; not default | + +
+Historical 481-second raw-teacher smoke (schema v1) + +On RTX 4080, 16 updates took 481 s, peaked at **4.25/4.76 GiB +allocated/reserved**, and moved 12-task success from **0% to 100%**. Cold start +did most of it (first OPD batch: 83.3%); this proves the pipeline, not OPD over +SFT. [Trace](docs/rtx4080-baselines.md). + +
## Local toy demo @@ -239,10 +245,10 @@ The recipe pins both revisions: | teacher | `Qwen/Qwen3-1.7B` | `70d244cc86ccca08cf5af4e1e306ecf908b1ad5e` | Apache-2.0 | Their `tokenizer.json` files are byte-identical -(`sha256 aeb13307a71acd8fe81861d94ad54ab689df773318809eed3cbe794b4492dae4`), -which is what makes the same-tokenizer contract hold. miniVERL verifies it at -load time by behavioural fingerprint and refuses to run otherwise. The recipe -also pins the [protocol-teacher adapter](https://huggingface.co/DaoyuanLi/mini-verl-qwen3-1.7b-protocol-teacher) +(`sha256 aeb13307a71acd8fe81861d94ad54ab689df773318809eed3cbe794b4492dae4`). +New runs compare structural identity; old artifacts use the legacy fixed-probe +behavioural fingerprint. +The recipe also pins the [protocol-teacher adapter](https://huggingface.co/DaoyuanLi/mini-verl-qwen3-1.7b-protocol-teacher) at revision `23323751318135484c06c043b1f9b9e7016dd89f` and requires its recorded strict policy success to be at least 50% before allocating the teacher. diff --git a/README.zh-CN.md b/README.zh-CN.md index 265e5f8..c14741a 100644 --- a/README.zh-CN.md +++ b/README.zh-CN.md @@ -74,7 +74,7 @@ miniVERL 把上面每一条都变成**被代码检查的性质**，而不是注 | 压缩的 `top-k + tail` KL / JSD | 支持；未平滑粗粒化与精确散度的下界关系有严格证明 | | 特权上下文教师模式，带显式对齐表 | 支持 | | 标准冻结 PEFT 教师适配器，带来源记录与能力门禁 | 支持 | -| 自动选择 bf16/fp16 的单卡 CUDA 路径 | 支持；代码路径不绑定型号，实测参考为 RTX 4080 | +| 自动选择 bf16/fp16 的单卡 CUDA 路径 | 支持；CUDA 路径不绑定设备名称，实测参考为 RTX 4080 | | `resident` / `swap` 显存策略与 `auto` 解析 | 支持，并有等价性测试 | | 带版本号与校验和、完全不用 pickle 的教师目标缓存 | 支持 | | SFT / 离线 KD / 严格 OPD / 显式标注 replay 统一在一个 trainer 中 | 支持 | @@ -101,25 +101,31 @@ miniVERL 把上面每一条都变成**被代码检查的性质**，而不是注 [公开且固定版本的协议教师适配器](https://huggingface.co/DaoyuanLi/mini-verl-qwen3-1.7b-protocol-teacher) 现在是单卡配方的默认教师。它在下游 benchmark 被查看之前，已经通过 -预先指定、独立评估的策略能力门槛。两个负对照故意去掉了这个保证：loss 正常 -下降，但教师把不兼容的工具策略教给了学生。这是实测的教师能力问题，不是 -trainer 崩溃。 +预先指定的能力门槛。两个负对照均正常完成且两个种子都是 0%；它们不是配置 +失败或崩溃。两者使用含歧义的历史 protocol-v1 prompt，故不能把 0% 只归因 +于教师内在行为；它诊断的是该设置缺少资格门禁。 -OPD 在这里仍然只是追平 SFT，而且平均训练时间约为 SFT 的 6 倍 -（523.8 秒对 86.4 秒）。任务已经饱和；两个种子不足以支持显著性结论，也不 -构成 OPD 普遍更优的证据。完整结果和旧实验的逐轨迹诊断见 +OPD 在这里仍然只是追平 SFT，继续训练耗时是 SFT 的 6.1 倍（523.8 秒对 +86.4 秒）。任务已经饱和；两个种子既不支持显著性结论，也不构成 OPD +普遍更优的证据。完整结果和旧实验的逐轨迹诊断见 [`docs/rtx4080-baselines.md`](docs/rtx4080-baselines.md)。 ![双种子协议教师对照](docs/gpu-calc-hard-equal-update-v2.svg) -更早的一次 RTX 4080 流水线 smoke 运行使用了现在保留的 -[原始教师对照配方](recipes/qwen_consumer_gpu_calc_raw_teacher.yaml)，用 -**Qwen3-1.7B** 蒸馏 **Qwen3-0.6B**，耗时 **481 秒 / 16 个优化步**，显存峰值为 -**4.25 GiB 已分配 / 4.76 GiB 已保留**，并在 12 个留出任务上把贪心成功率 -从 **0.0% 提升到 100.0%**。其中 8 个 cycle 的监督冷启动完成了大部分工作： -第一批 OPD rollout 已经达到 83.3%。它证明流水线能端到端运行，而不是证明 -OPD 优于 SFT。所有数字都可追溯到 -[`docs/rtx4080-baselines.md`](docs/rtx4080-baselines.md)。 +| 产物 | 定位 | +| --- | --- | +| [默认配方](recipes/qwen_consumer_gpu_calc.yaml) | 协议合格 | +| [Schema-v2 结果](benchmarks/results/gpu-calc-hard-equal-update-v2.json) | 冻结五臂对照 | +| [Raw-teacher](recipes/qwen_consumer_gpu_calc_raw_teacher.yaml) | 历史对照；非默认 | + +
+481 秒 smoke（v1） + +16 步 481 秒，峰值 **4.25/4.76 GiB 已分配/保留**，12 题 +从 **0% 到 100%**。冷启动完成了大部分工作（首批 OPD：83.3%）；这证明 +流水线，而非 OPD 优于 SFT。[追溯](docs/rtx4080-baselines.md)。 + +
## 本地玩具演示 @@ -175,10 +181,9 @@ tokens by span type (only assistant_* can enter the loss) ## 个人单卡快速上手 -默认配方使用 `device: auto` 与 `dtype: auto`：支持 bf16 的显卡自动使用 bf16， -Titan V 等较老 CUDA 显卡自动使用 fp16。RTX 3070、Titan V、RTX 4080 和 -RTX 5090 级别显卡都走同一条代码路径；本仓库目前只有 RTX 4080 的实测结果。 -能否装下取决于显存、模型大小、驱动和 token 预算，而不是显卡的商品名。修改配方前请阅读 +默认 `device: auto` / `dtype: auto`：新卡用 bf16，Titan V 等旧卡用 fp16。 +RTX 3070、Titan V、RTX 4080、RTX 5090 走同一 CUDA 路径，但仅 4080 有实测。 +能否装下取决于显存、模型、驱动和 token 预算。修改前请阅读 [`单卡适配指南`](docs/single-gpu-guide.md)。 ```bash @@ -201,7 +206,7 @@ miniverl report runs/ --out runs//report.html | 学生 | `Qwen/Qwen3-0.6B` | `c1899de289a04d12100db370d81485cdf75e47ca` | Apache-2.0 | | 教师 | `Qwen/Qwen3-1.7B` | `70d244cc86ccca08cf5af4e1e306ecf908b1ad5e` | Apache-2.0 | -两者的 `tokenizer.json` **逐字节相同**（`sha256 aeb13307a71acd8fe81861d94ad54ab689df773318809eed3cbe794b4492dae4`），这正是"同一分词器"契约成立的依据。miniVERL 在加载时用行为指纹核对，不一致就直接报错退出。 +两者的 `tokenizer.json` **逐字节相同**（`sha256 aeb13307a71acd8fe81861d94ad54ab689df773318809eed3cbe794b4492dae4`）。新运行首查结构身份；旧产物回退到固定探针行为指纹。 配方还把[协议教师适配器](https://huggingface.co/DaoyuanLi/mini-verl-qwen3-1.7b-protocol-teacher) 锁定在 revision `23323751318135484c06c043b1f9b9e7016dd89f`，并在分配教师模型 diff --git a/docs/banner.svg b/docs/banner.svg index 565e354..3347229 100644 --- a/docs/banner.svg +++ b/docs/banner.svg @@ -1,7 +1,7 @@ - + @@ -40,7 +40,7 @@ - PERSONAL SINGLE-GPU TRAINING + SINGLE-GPU LLM POST-TRAINING miniVERL @@ -56,22 +56,22 @@ - - one card - 1× CUDA GPU + - GPU adaptive - BF16 / FP16 auto + - audit first + typed provenance - pip install "miniverl[train,cuda]" + font-size="11.3" fill="#dbeafe">pip install miniverl @@ -112,7 +112,7 @@ Distill - Use BF16 or FP16 on the GPU you have + Exact or top-k + tail teacher targets diff --git a/docs/gpu-calc-hard-equal-update-v2.svg b/docs/gpu-calc-hard-equal-update-v2.svg index a7e0a90..9aac355 100644 --- a/docs/gpu-calc-hard-equal-update-v2.svg +++ b/docs/gpu-calc-hard-equal-update-v2.svg @@ -1,17 +1,17 @@ miniVERL protocol-teacher benchmark -Strict held-out success and training time for equal-optimizer-update arms over 2 prespecified seeds. 2 protocol-incompatible negative controls are shown separately from the supported comparison. The cold-start baseline is labeled no training. - +Strict held-out success and continuation train time for equal-update arms over 2 prespecified seeds. 2 completed negative-control arms without a protocol qualification gate are shown separately. The cold-start baseline has zero continuation updates. + -Protocol-aligned OPD matches continued SFT -schema v2 · 2 prespecified seeds · budget axis: optimizer_steps +Same strict success; protocol OPD uses 6.1× more continuation time +Schema v2 · 2 prespecified seeds · equal budget: 12 continuation optimizer updates Strict held-out success -Training time +Continuation train time 0% 50% @@ -24,7 +24,7 @@ 600s - + Cold start @@ -32,10 +32,10 @@ 75% - -NO TRAINING + +0 CONTINUATION UPDATES - + Continued SFT @@ -49,7 +49,7 @@ 86s - + OPD · protocol-aligned teacher @@ -64,29 +64,37 @@ 524s -DIAGNOSTIC CONTROLS · INTENTIONALLY PROTOCOL-INCOMPATIBLE TEACHERS - +DIAGNOSTIC NEGATIVE CONTROLS · TEACHERS NOT PROTOCOL-QUALIFIED + -Raw teacher (control) - -PROTOCOL MISMATCH +Raw teacher (control) + +NEGATIVE CONTROL · NO PROTOCOL GATE + + + +0% 444s - + -Privileged context (control) - -PROTOCOL MISMATCH +Privileged context (control) + +NEGATIVE CONTROL · NO PROTOCOL GATE + + + +0% 532s -Diagnostic controls intentionally use protocol-incompatible teachers; strict success was 0% in every seed; cold start records setup only (0.06s), not a training phase. -Bars are seed means; hollow dots are individual seeds [1234, 20260727]. Source JSON SHA-256 53fc1d4d5b7adee0 +Protocol-teacher preparation: ~555 s once, excluded from continuation bars. +Bars = seed means · dots = seeds [1234, 20260727] · Source JSON SHA-256 53fc1d4d5b7adee0 diff --git a/docs/limitations.md b/docs/limitations.md index 78f6dd1..468def3 100644 --- a/docs/limitations.md +++ b/docs/limitations.md @@ -21,22 +21,21 @@ code rather than documented and hoped for: - `miniverl.models.factory.build_tokenizer` returns **one** tokenizer object, which both backends then share. When the teacher declares a different - tokenizer id, that tokenizer is loaded and its behavioural fingerprint is - compared against the student's; any difference raises - `TokenizerMismatchError` before a single weight is downloaded. + tokenizer id, that tokenizer is loaded and + `miniverl.models.tokenizers.assert_same_tokenizer` is called before either + model is constructed. New runs compare structural identity first; any + difference raises `TokenizerMismatchError`. - `miniverl.trajectory.alignment.build_alignment_map` and `miniverl.teachers.local.LocalTeacherScorer.score` both re-check the fingerprint on the teacher render used by `privileged_context` mode. -`miniverl.models.tokenizers.assert_same_tokenizer` also exists, but it has no -call sites in `src/` or `tests/` and is not in the module's `__all__`. Do not -rely on it; the two checks above are the ones that run. - -The fingerprint is a SHA-256 over the tokenizer class name, `len(tokenizer)`, -the EOS and PAD ids, the sorted additional special tokens, and the token ids -produced for a fixed probe string containing chat and tool markup -(`miniverl.models.tokenizers.PROBE_TEXT`). Two tokenizers that agree on -metadata but disagree on the probe are still rejected. +Structural identity hashes the full vocabulary, added vocabulary, special-token +map, backend tokenizer and behaviour-relevant tokenizer configuration. Local +source paths are canonicalized away. The legacy fingerprint is a SHA-256 over +the tokenizer class, length, special-token metadata and token ids produced for +one fixed string (`miniverl.models.tokenizers.PROBE_TEXT`). It remains a +compatibility fallback when an old artifact has no structural digest; agreement +on that probe is not proof that two tokenizer structures are identical. The reason is structural, not incidental. A bucketed teacher target is a set of vocabulary **indices** plus log-probabilities; the student gathers its own diff --git a/docs/single-gpu-guide.md b/docs/single-gpu-guide.md index b547ce7..e75014e 100644 --- a/docs/single-gpu-guide.md +++ b/docs/single-gpu-guide.md @@ -1,6 +1,6 @@ # Bring your own GPU -miniVERL is a **personal single-GPU** training stack. It has no GPU model +miniVERL is a **single-GPU CUDA LLM post-training** stack. It has no GPU model allowlist and no multi-GPU launcher: one process uses one CUDA device, while the model pair and sequence budget determine whether a recipe fits. @@ -18,15 +18,22 @@ RTX 4080. Its portable defaults are: ## Start here -Install the training stack and the CUDA quantization backend: +First use the [PyTorch install selector](https://pytorch.org/get-started/locally/) +to install the wheel matching your CUDA driver. The measured stack used the +following channel; choose a different supported channel when your system needs +one: ```bash +python -m pip install torch --index-url https://download.pytorch.org/whl/cu130 python -m pip install "miniverl[train,cuda]" miniverl doctor miniverl validate recipes/qwen_consumer_gpu_calc.yaml miniverl train recipes/qwen_consumer_gpu_calc.yaml --dry-run ``` +The `train,cuda` extra adds the training and quantization dependencies; it does +not select a CUDA-enabled PyTorch build on its own. + Then watch free memory in another terminal before the real run: ```bash @@ -45,7 +52,7 @@ These are starting points, not benchmark claims: | Your card | Sensible first move | Evidence status | | --- | --- | --- | -| 8–12 GiB, such as an RTX 3070 or Titan V | Try the shipped pair, but be ready to shorten token budgets or select a smaller teacher. Keep `dtype: auto`; Titan V-class hardware resolves to fp16 because it has no bf16 path. | Supported by model-agnostic code paths; not measured in this repository | +| 8–12 GiB, such as an RTX 3070 or Titan V | Try the shipped pair, but be ready to shorten token budgets or select a smaller teacher. Keep `dtype: auto`; Titan V-class hardware resolves to fp16 because it has no bf16 path. | Supported by the device-name-agnostic CUDA path; not measured in this repository | | 16–24 GiB | Start with the shipped recipe unchanged. Increase budgets only after recording a successful baseline. | The exact default pair is measured on one RTX 4080 16 GiB | | 24–32+ GiB, such as RTX 3090/4090/5090-class cards | Use the same recipe first; extra headroom can support longer contexts, larger models, or less quantization. Change one variable at a time. | Expected from the same single-device path; not measured here | diff --git a/scripts/publish_benchmark_artifacts.py b/scripts/publish_benchmark_artifacts.py index 9ac024e..dc2ce72 100644 --- a/scripts/publish_benchmark_artifacts.py +++ b/scripts/publish_benchmark_artifacts.py @@ -84,6 +84,7 @@ def render_svg(result: BenchmarkResult, source_sha256: str) -> str: arms = grouped[name] success = [arm.strict_task_success_rate or 0.0 for arm in arms] seconds = [arm.train_seconds or 0.0 for arm in arms] + optimizer_steps = [arm.optimizer_steps or 0 for arm in arms] rows.append( { "name": name, @@ -91,6 +92,7 @@ def render_svg(result: BenchmarkResult, source_sha256: str) -> str: "success_mean": sum(success) / len(success), "seconds": seconds, "seconds_mean": sum(seconds) / len(seconds), + "optimizer_steps": optimizer_steps, } ) display_order = { @@ -111,6 +113,33 @@ def render_svg(result: BenchmarkResult, source_sha256: str) -> str: (row["seconds_mean"] for row in rows if row["name"] == "cold-start-only"), None, ) + row_by_name = {row["name"]: row for row in rows} + sft_row = row_by_name.get("sft-continued") + protocol_opd_row = row_by_name.get("opd-protocol-sft-teacher") + if ( + sft_row is not None + and protocol_opd_row is not None + and math.isclose(sft_row["success_mean"], protocol_opd_row["success_mean"]) + and sft_row["seconds_mean"] > 0 + ): + continuation_ratio = protocol_opd_row["seconds_mean"] / sft_row["seconds_mean"] + title = ( + "Same strict success; protocol OPD uses " + f"{continuation_ratio:.1f}× more continuation time" + ) + else: + title = "Strict held-out success and continuation train time" + trained_step_counts = { + int(step) + for row in rows + if row["name"] != "cold-start-only" + for step in row["optimizer_steps"] + } + if len(trained_step_counts) == 1: + (trained_steps,) = trained_step_counts + budget_description = f"equal budget: {trained_steps} continuation optimizer updates" + else: + budget_description = "equal continuation budget by optimizer updates" width = 1120 diagnostic_gap = 36 if diagnostic_start is not None else 0 @@ -147,16 +176,16 @@ def line(text: str) -> str: ) svg += line('miniVERL protocol-teacher benchmark') desc = ( - "Strict held-out success and training time for equal-optimizer-update arms " + "Strict held-out success and continuation train time for equal-update arms " f"over {len(result.seeds)} prespecified seeds." ) if diagnostic_count: desc += ( - f" {diagnostic_count} protocol-incompatible negative controls are " - "shown separately from the supported comparison." + f" {diagnostic_count} completed negative-control arms without a protocol " + "qualification gate are shown separately." ) if cold_start_seconds is not None: - desc += " The cold-start baseline is labeled no training." + desc += " The cold-start baseline has zero continuation updates." svg += line(f'{desc}') svg += line( " -Same strict success; protocol OPD uses 6.1× more continuation time +Saturated calculator task · same success, 6.1× OPD continuation time Schema v2 · 2 prespecified seeds · equal budget: 12 continuation optimizer updates -Strict held-out success +Strict success on v0.2 test set Continuation train time 0% @@ -68,8 +68,8 @@ Raw teacher (control) - -NEGATIVE CONTROL · NO PROTOCOL GATE + +NEGATIVE CONTROL · UNGATED @@ -83,8 +83,8 @@ Privileged context (control) - -NEGATIVE CONTROL · NO PROTOCOL GATE + +NEGATIVE CONTROL · UNGATED @@ -95,6 +95,6 @@ 532s -Protocol-teacher preparation: ~555 s once, excluded from continuation bars. +Teacher preparation is not included in continuation bars. Bars = seed means · dots = seeds [1234, 20260727] · Source JSON SHA-256 53fc1d4d5b7adee0 diff --git a/scripts/publish_benchmark_artifacts.py b/scripts/publish_benchmark_artifacts.py index dc2ce72..a1ff5a3 100644 --- a/scripts/publish_benchmark_artifacts.py +++ b/scripts/publish_benchmark_artifacts.py @@ -124,11 +124,11 @@ def render_svg(result: BenchmarkResult, source_sha256: str) -> str: ): continuation_ratio = protocol_opd_row["seconds_mean"] / sft_row["seconds_mean"] title = ( - "Same strict success; protocol OPD uses " - f"{continuation_ratio:.1f}× more continuation time" + "Saturated calculator task · same success, " + f"{continuation_ratio:.1f}× OPD continuation time" ) else: - title = "Strict held-out success and continuation train time" + title = "Calculator task · strict success and continuation train time" trained_step_counts = { int(step) for row in rows @@ -176,8 +176,8 @@ def line(text: str) -> str: ) svg += line('miniVERL protocol-teacher benchmark') desc = ( - "Strict held-out success and continuation train time for equal-update arms " - f"over {len(result.seeds)} prespecified seeds." + "Strict success on the v0.2 calculator test set and continuation train time " + f"for equal-update arms over {len(result.seeds)} prespecified seeds." ) if diagnostic_count: desc += ( @@ -238,7 +238,9 @@ def line(text: str) -> str: svg += line( f'' ) - svg += line(f'Strict held-out success') + svg += line( + f'Strict success on v0.2 test set' + ) svg += line(f'Continuation train time') for fraction in (0.0, 0.5, 1.0): x = success_x + success_w * fraction @@ -280,12 +282,12 @@ def line(text: str) -> str: svg += line(f'{display_name}') if is_diagnostic: svg += line( - f'' ) svg += line( - f'NEGATIVE CONTROL · NO PROTOCOL GATE' + f'NEGATIVE CONTROL · UNGATED' ) success_end = success_x + success_w * row["success_mean"] @@ -341,8 +343,8 @@ def line(text: str) -> str: svg += line("") svg += line( - f'Protocol-teacher preparation: ' - "~555 s once, excluded from continuation bars." + f'' + "Teacher preparation is not included in continuation bars." ) svg += line( f'Bars = seed means · dots = seeds ' diff --git a/src/miniverl/cache/store.py b/src/miniverl/cache/store.py index c1e859a..720a3d6 100644 --- a/src/miniverl/cache/store.py +++ b/src/miniverl/cache/store.py @@ -28,6 +28,7 @@ import hashlib import json +import re import struct import uuid from datetime import datetime, timezone @@ -44,11 +45,13 @@ CacheShardMeta, TeacherTargetBatch, ) +from miniverl.utils.logging import get_logger from miniverl.utils.runs import write_json_atomic __all__ = ["TeacherCache", "read_safetensors_header", "sha256_file"] _INDEX_NAME = "index.json" +_SHARD_NAME = re.compile(r"^shard-(\d+)\.safetensors$") _TENSOR_FIELDS = ( "positions", "topk_indices", @@ -57,12 +60,28 @@ "target_token_ids", "weights", ) +logger = get_logger("cache") def _replace_shard_file(source: Path, target: Path) -> None: source.replace(target) +def _delete_shard_file(path: Path) -> None: + path.unlink() + + +def _next_shard_id(path: Path, index: CacheIndex) -> int: + names = set(index.shards) + names.update(shard.filename for shard in index.shards.values()) + if path.is_dir(): + names.update(child.name for child in path.iterdir() if child.is_file()) + suffixes = [ + int(match.group(1)) for name in names if (match := _SHARD_NAME.fullmatch(name)) is not None + ] + return max(suffixes, default=-1) + 1 + + def sha256_file(path: Path) -> tuple[str, int]: """Return ``(hex digest, size in bytes)`` of a file, streamed.""" digest = hashlib.sha256() @@ -108,13 +127,13 @@ def read_safetensors_header(path: Path) -> dict[str, Any]: class TeacherCache: """Append-only, shard-based store for compressed teacher targets.""" - def __init__(self, path: str | Path, index: CacheIndex, *, entries_per_shard: int = 32) -> None: + def __init__(self, path: str | Path, index: CacheIndex) -> None: self.path = Path(path) self.index = index - self.entries_per_shard = entries_per_shard + self.entries_per_shard = index.entries_per_shard self._pending: dict[str, dict[str, Any]] = {} self._pending_order: list[str] = [] - self._shard_counter = len(index.shards) + self._shard_counter = _next_shard_id(self.path, index) # Cumulative I/O accounting is intentionally independent of the current # footprint: pruning may shrink ``total_bytes()`` but cannot un-write the # shards that consumed bandwidth earlier in the run. @@ -182,8 +201,9 @@ def create( temperature=temperature, loss_mode=loss_mode, dtype=dtype, + entries_per_shard=entries_per_shard, ) - cache = cls(target, index, entries_per_shard=entries_per_shard) + cache = cls(target, index) cache._write_index() return cache @@ -567,25 +587,42 @@ def stats(self) -> CacheCompressionStats: def prune_before(self, policy_version: int) -> int: """Drop entries older than ``policy_version``; returns the number removed. - Whole shards are deleted only when every entry they hold is dropped, so - the index and the files stay consistent. + The next index is published before its now-unreferenced shards are + deleted. An interrupted cleanup can therefore leave an inert orphan + shard, but never an index that points at a shard already removed. """ stale = [ traj_id for traj_id, entry in self.index.entries.items() if entry.policy_version < policy_version ] + if not stale: + return 0 + + next_index = self.index.model_copy(deep=True) for traj_id in stale: - del self.index.entries[traj_id] - live_shards = {e.shard for e in self.index.entries.values()} - for name in list(self.index.shards): + del next_index.entries[traj_id] + live_shards = {entry.shard for entry in next_index.entries.values()} + unreferenced_shards = [] + for name in list(next_index.shards): if name not in live_shards: - shard_path = self.path / name - if shard_path.is_file(): - shard_path.unlink() - del self.index.shards[name] - if stale: - self._write_index() + unreferenced_shards.append(name) + del next_index.shards[name] + + self._write_index(next_index) + self.index = next_index + for name in unreferenced_shards: + shard_path = self.path / name + if not shard_path.is_file(): + continue + try: + _delete_shard_file(shard_path) + except OSError as exc: + logger.warning( + "teacher-cache prune left unreferenced shard %s: %s", + shard_path, + exc, + ) return len(stale) diff --git a/src/miniverl/schemas/cache.py b/src/miniverl/schemas/cache.py index b27a8d2..67b6d7c 100644 --- a/src/miniverl/schemas/cache.py +++ b/src/miniverl/schemas/cache.py @@ -75,6 +75,7 @@ class CacheIndex(BaseModel): temperature: float = Field(gt=0.0) loss_mode: str dtype: str = "float32" + entries_per_shard: int = Field(default=32, ge=1, le=4096) entries: dict[str, CacheEntryMeta] = Field(default_factory=dict) shards: dict[str, CacheShardMeta] = Field(default_factory=dict) diff --git a/tests/unit/test_cache.py b/tests/unit/test_cache.py index 4f49942..e6f7847 100644 --- a/tests/unit/test_cache.py +++ b/tests/unit/test_cache.py @@ -345,6 +345,55 @@ def test_sharding_is_deterministic_and_index_stays_consistent(tmp_path: Path): assert cache.validate() == [] +def test_reopen_after_pruning_allocates_after_the_highest_surviving_shard( + tmp_path: Path, +) -> None: + cache_path = tmp_path / "tc" + cache = _cache(cache_path, entries_per_shard=1) + for policy_version in range(4): + cache.write( + _batch(f"t{policy_version}", policy_version=policy_version), + selector="hybrid", + ) + + surviving_bytes = { + name: (cache_path / name).read_bytes() + for name in ("shard-00002.safetensors", "shard-00003.safetensors") + } + assert cache.prune_before(2) == 2 + + reopened = TeacherCache.open(cache_path) + assert reopened.entries_per_shard == 1 + assert reopened.read("t2", expect_policy_version=2).trajectory_id == "t2" + assert reopened.read("t3", expect_policy_version=3).trajectory_id == "t3" + + reopened.write(_batch("t4", policy_version=4), selector="hybrid") + + assert sorted(reopened.index.shards) == [ + "shard-00002.safetensors", + "shard-00003.safetensors", + "shard-00004.safetensors", + ] + assert reopened.read("t4", expect_policy_version=4).trajectory_id == "t4" + for name, original in surviving_bytes.items(): + assert (cache_path / name).read_bytes() == original + + +def test_reopen_allocates_after_the_highest_on_disk_orphan_shard(tmp_path: Path) -> None: + cache_path = tmp_path / "tc" + cache = _cache(cache_path, entries_per_shard=1) + cache.write(_batch("t0"), selector="hybrid") + (cache_path / "shard-00007.safetensors").write_bytes( + (cache_path / "shard-00000.safetensors").read_bytes() + ) + + reopened = TeacherCache.open(cache_path) + reopened.write(_batch("t1"), selector="hybrid") + + assert reopened.index.entries["t1"].shard == "shard-00008.safetensors" + assert (cache_path / "shard-00007.safetensors").is_file() + + def test_shard_is_written_to_a_temporary_name_before_publication( tmp_path: Path, monkeypatch ) -> None: @@ -478,6 +527,68 @@ def test_prune_before_removes_old_policy_versions_and_their_shards(tmp_path: Pat assert "new" in cache +def test_failed_prune_index_publication_preserves_old_index_and_shards( + tmp_path: Path, monkeypatch +) -> None: + import miniverl.cache.store as store_module + + cache_path = tmp_path / "tc" + cache = _cache(cache_path, entries_per_shard=1) + cache.write(_batch("old", policy_version=1), selector="hybrid") + cache.write(_batch("new", policy_version=5), selector="hybrid") + index_path = cache_path / "index.json" + original_index_bytes = index_path.read_bytes() + original_index = cache.index.model_dump(mode="json") + original_shards = { + path.name: path.read_bytes() for path in cache_path.glob("shard-*.safetensors") + } + + def fail_index_publication(*_args, **_kwargs): # type: ignore[no-untyped-def] + raise OSError("injected prune index replacement failure") + + monkeypatch.setattr(store_module, "write_json_atomic", fail_index_publication) + with pytest.raises(OSError, match="injected prune index"): + cache.prune_before(5) + + assert cache.index.model_dump(mode="json") == original_index + assert index_path.read_bytes() == original_index_bytes + assert { + path.name: path.read_bytes() for path in cache_path.glob("shard-*.safetensors") + } == original_shards + assert cache.read("old", expect_policy_version=1).trajectory_id == "old" + assert cache.read("new", expect_policy_version=5).trajectory_id == "new" + + +def test_prune_deletion_failure_leaves_a_harmless_orphan_after_index_publication( + tmp_path: Path, monkeypatch +) -> None: + import miniverl.cache.store as store_module + + cache_path = tmp_path / "tc" + cache = _cache(cache_path, entries_per_shard=1) + cache.write(_batch("old", policy_version=1), selector="hybrid") + cache.write(_batch("new", policy_version=5), selector="hybrid") + old_shard = cache.index.entries["old"].shard + deletion_attempts: list[Path] = [] + + def fail_deletion(path: Path) -> None: + published = json.loads((cache_path / "index.json").read_text(encoding="utf-8")) + assert old_shard not in published["shards"] + deletion_attempts.append(path) + raise OSError("injected orphan cleanup failure") + + monkeypatch.setattr(store_module, "_delete_shard_file", fail_deletion, raising=False) + assert cache.prune_before(5) == 1 + + assert deletion_attempts == [cache_path / old_shard] + assert old_shard not in cache.index.shards + assert (cache_path / old_shard).is_file() + reopened = TeacherCache.open(cache_path) + assert "old" not in reopened + assert reopened.read("new", expect_policy_version=5).trajectory_id == "new" + assert reopened.validate() == [] + + def test_pruning_does_not_reduce_cumulative_bytes_written(tmp_path: Path): cache = _cache(tmp_path / "tc", entries_per_shard=1) cache.write(_batch("old", policy_version=1), selector="hybrid") diff --git a/tests/unit/test_packaging.py b/tests/unit/test_packaging.py index 5ef7656..3918c42 100644 --- a/tests/unit/test_packaging.py +++ b/tests/unit/test_packaging.py @@ -269,18 +269,20 @@ def test_published_gpu_visualization_matches_its_source_result(): text_nodes = [text.strip() for text in root.itertext() if text.strip()] visible_text = " ".join(text_nodes) assert f"Source JSON SHA-256 {digest[:16]}" in visible_text - assert "Same strict success; protocol OPD uses 6.1× more continuation time" in visible_text + assert "Saturated calculator task · same success, 6.1× OPD continuation time" in visible_text assert "DIAGNOSTIC NEGATIVE CONTROLS · TEACHERS NOT PROTOCOL-QUALIFIED" in visible_text assert "intentionally incompatible" not in visible_text.lower() + assert "Strict success on v0.2 test set" in visible_text assert "Continuation train time" in visible_text - assert "Protocol-teacher preparation: ~555 s once, excluded" in visible_text + assert "Teacher preparation is not included in continuation bars." in visible_text + assert "~555" not in visible_text assert "optimizer_steps" not in visible_text cold_text = [text.strip() for text in groups["cold-start-only"].itertext() if text.strip()] assert "0 CONTINUATION UPDATES" in cold_text assert "0s" not in cold_text for name in ("opd-raw-teacher", "opd-privileged-context"): arm_text = [text.strip() for text in groups[name].itertext() if text.strip()] - assert "NEGATIVE CONTROL · NO PROTOCOL GATE" in arm_text + assert "NEGATIVE CONTROL · UNGATED" in arm_text assert "0%" in arm_text assert "PROTOCOL MISMATCH" not in arm_text assert "COLLAPSED" not in visible_text