From 701aaa1173004896b2f5ec5c97b55f2d5367f322 Mon Sep 17 00:00:00 2001 From: Daoyuan Li Date: Mon, 3 Aug 2026 14:36:51 -0700 Subject: [PATCH 1/3] Harden bridge semantics and documentation visuals --- .github/workflows/docs.yml | 72 +- CHANGELOG.md | 2 +- PROJECT_STATE.md | 2 +- PYPI.md | 739 +++--------------- README.md | 739 +++--------------- README.zh-CN.md | 501 +++--------- TODO.md | 19 +- docs/alignment-lab/alignment-lab-v1.md | 20 +- docs/alignment-lab/delta-from-sft.svg | 1 + docs/alignment-lab/demo.md | 4 +- docs/alignment-lab/metric-coverage-matrix.svg | 1 + docs/alignment-lab/outcome-cost-matrix.svg | 1 + docs/alignment-lab/preference-vs-gpu-time.svg | 1 - .../quality-vs-teacher-query.svg | 1 - docs/alignment-lab/quality-vs-utility.svg | 1 - docs/alignment-lab/safety-vs-overrefusal.svg | 1 - .../when-opd-should-follow-sft.md | 2 +- docs/assets/javascripts/versioning.js | 29 + docs/assets/stylesheets/extra.css | 94 +++ docs/compatibility.md | 11 +- docs/consumer-runtime/index.md | 17 + docs/index.md | 118 ++- docs/limitations.md | 18 +- docs/overrides/main.html | 12 + docs/verl-bridge-architecture-mobile.svg | 1 + docs/verl-bridge-architecture.svg | 73 +- docs/verl-bridge-demo.md | 10 +- docs/verl-bridge-launch.md | 16 +- docs/verl-bridge.md | 191 +++-- mkdocs.yml | 73 +- paper/alignment-lab-v1/alignment-lab-v1.pdf | Bin 16641 -> 16782 bytes paper/alignment-lab-v1/build_report.py | 228 +++--- pyproject.toml | 2 + scripts/check_docs_visual.py | 199 +++++ scripts/check_markdown_links.py | 21 +- scripts/publish_alignment_lab_artifacts.py | 545 ++++++++----- scripts/publish_verl_bridge_diagrams.py | 150 ++++ scripts/verify_verl_bridge_smoke.py | 12 +- src/miniverl/bridge/config.py | 438 ++++++++--- src/miniverl/bridge/contract.py | 2 +- src/miniverl/bridge/doctor.py | 24 +- src/miniverl/bridge/export.py | 190 ++++- src/miniverl/cli.py | 29 +- tests/cli/test_verl_bridge_cli.py | 67 +- tests/unit/test_alignment_publish.py | 25 +- tests/unit/test_docs_visual_contract.py | 51 ++ tests/unit/test_packaging.py | 2 +- tests/unit/test_verl_bridge_config.py | 195 ++++- tests/unit/test_verl_bridge_export.py | 93 ++- tests/unit/test_verl_bridge_smoke.py | 32 + 50 files changed, 2676 insertions(+), 2399 deletions(-) create mode 100644 docs/alignment-lab/delta-from-sft.svg create mode 100644 docs/alignment-lab/metric-coverage-matrix.svg create mode 100644 docs/alignment-lab/outcome-cost-matrix.svg delete mode 100644 docs/alignment-lab/preference-vs-gpu-time.svg delete mode 100644 docs/alignment-lab/quality-vs-teacher-query.svg delete mode 100644 docs/alignment-lab/quality-vs-utility.svg delete mode 100644 docs/alignment-lab/safety-vs-overrefusal.svg create mode 100644 docs/assets/javascripts/versioning.js create mode 100644 docs/assets/stylesheets/extra.css create mode 100644 docs/consumer-runtime/index.md create mode 100644 docs/overrides/main.html create mode 100644 docs/verl-bridge-architecture-mobile.svg create mode 100644 scripts/check_docs_visual.py create mode 100644 scripts/publish_verl_bridge_diagrams.py create mode 100644 tests/unit/test_docs_visual_contract.py diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index ca9ba56..1cdd945 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -2,12 +2,11 @@ name: docs on: workflow_dispatch: + pull_request: + branches: [main] push: branches: [main] - paths: - - "docs/**" - - "mkdocs.yml" - - ".github/workflows/docs.yml" + tags: ["v*"] permissions: contents: read @@ -15,25 +14,80 @@ permissions: id-token: write concurrency: - group: pages + group: docs-${{ github.ref }} cancel-in-progress: false jobs: + visual: + name: strict build and browser visual gate + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + with: + python-version: "3.12" + cache: pip + - name: Install pinned docs dependencies + run: python -m pip install -e ".[dev]" + - name: Verify generated documentation artifacts + run: | + python scripts/publish_alignment_lab_artifacts.py --check + python scripts/publish_verl_bridge_diagrams.py --check + - name: Build complete development site strictly + run: mkdocs build --strict + - name: Install deterministic Chromium + run: playwright install --with-deps chromium + - name: Inspect four documentation viewports + run: python scripts/check_docs_visual.py --site site --screenshots docs-visual-screenshots + - name: Upload browser screenshots + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: docs-visual-screenshots + path: docs-visual-screenshots + if-no-files-found: error + build: + name: build stable root and development subsite + if: github.event_name == 'push' || github.event_name == 'workflow_dispatch' + needs: visual runs-on: ubuntu-latest steps: - - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 - - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + fetch-depth: 0 + - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 with: python-version: "3.12" cache: pip - uses: actions/configure-pages@983d7736d9b0ae728b81ab479565c72886d7745b - - run: python -m pip install "mkdocs>=1.6,<2" - - run: mkdocs build + - name: Install pinned docs dependencies + run: python -m pip install -e ".[dev]" + - name: Build current development docs strictly + run: mkdocs build --strict --site-dir "${RUNNER_TEMP}/dev-site" + - name: Build the latest immutable release at the site root + shell: bash + run: | + if [[ "${GITHUB_REF_TYPE}" == "tag" ]]; then + stable_tag="${GITHUB_REF_NAME}" + else + stable_tag="$(git tag --list 'v*' --sort=-v:refname | head -n 1)" + fi + test -n "$stable_tag" + git worktree add --detach "${RUNNER_TEMP}/stable-checkout" "$stable_tag" + ( + cd "${RUNNER_TEMP}/stable-checkout" + mkdocs build --strict --site-dir "${GITHUB_WORKSPACE}/site" + ) + mkdir -p "${GITHUB_WORKSPACE}/site/dev" + cp -a "${RUNNER_TEMP}/dev-site/." "${GITHUB_WORKSPACE}/site/dev/" - uses: actions/upload-pages-artifact@7b1f4a764d45c48632c6b24a0339c27f5614fb0b with: path: site + deploy: + name: deploy versioned documentation + if: github.event_name == 'push' || github.event_name == 'workflow_dispatch' environment: name: github-pages url: ${{ steps.deployment.outputs.page_url }} diff --git a/CHANGELOG.md b/CHANGELOG.md index 739f558..85506f5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,7 +10,7 @@ All notable changes to miniVERL are recorded here. The format follows ### Added -- A compatibility Level-3 bridge for the fail-closed +- A miniVERL-defined compatibility Level-3 bridge for the fail-closed `single-gpu-online-distillation-v1` profile, pinned to official verl `v0.8.0` commit `7aed6b230776f963fa09509c10d9c3a767d1102c`. - `import-verl`, bidirectional prompt-Parquet conversion, `export-verl` standard diff --git a/PROJECT_STATE.md b/PROJECT_STATE.md index f8fd9db..d2531ec 100644 --- a/PROJECT_STATE.md +++ b/PROJECT_STATE.md @@ -11,7 +11,7 @@ Last updated: 2026-08-03. | item | current state | | --- | --- | | audited upstream | official stable verl `v0.8.0`, source commit `7aed6b230776f963fa09509c10d9c3a767d1102c`, tested with Python 3.12; the installed source reports `0.8.0.dev0` | -| compatibility contract | Levels 0-3 are explicit; Level 2 is a fail-closed 14-field whitelist for `single-gpu-online-distillation-v1`; Level 3 exchanges standard HF/PEFT/safetensors/tokenizer/Parquet artifacts and a checksummed scale-out bundle | +| compatibility contract | Levels 0-3 are explicitly miniVERL-defined; Level 2 is a fail-closed 14-field whitelist for `single-gpu-online-distillation-v1`; miniVERL-defined Level 3 exchanges standard HF/PEFT/safetensors/tokenizer/Parquet artifacts and a checksummed scale-out bundle | | command surface | `import-verl`, `convert-dataset`, `export-verl`, `bridge doctor` and `benchmark --export-community` are implemented without importing torch in the core path | | exact compatibility smoke | the pinned source installed successfully; OmegaConf parsed official and exported config shapes; PEFT, safetensors, train/val Parquet, reward import, privacy and all bundle hashes passed; the checksummed record is `docs/generated/verl-bridge-smoke.json` | | distributed boundary | Ray, FSDP/Megatron, vLLM/SGLang and a full distributed run were not installed or launched; distributed execution remains explicitly `not tested` | diff --git a/PYPI.md b/PYPI.md index 0a0eae9..43afa2a 100644 --- a/PYPI.md +++ b/PYPI.md @@ -1,5 +1,5 @@

- miniVERL — online alignment and distillation on one GPU + miniVERL — single-GPU LLM post-training

@@ -13,675 +13,142 @@

- PyPI package · - Documentation · - Install & train · - Verified verl bridge · - Alignment Lab + PyPI · + Stable docs · + Development docs · + 中文

-**Single-GPU prototyping for a documented subset of verl-style online -post-training.** +**miniVERL is a local, inspectable runtime for a documented subset of +single-GPU LLM alignment and distillation.** It keeps rollout provenance, +assistant-only loss masks, teacher targets, update budgets and run artifacts +explicit, then exports portable artifacts through a fail-closed bridge to one +pinned upstream verl profile. -Develop, diagnose and validate an alignment or distillation recipe locally, -then export standard model, dataset, recipe and provenance artifacts to a -pinned verl release for scale-out. +PyPI `v0.6.0` is stable; `main` is development. The CUDA path has no GPU-name +allowlist, but fit depends on the model pair, context budget, kernels and VRAM. +miniVERL is independent from verl and does not claim distributed execution or +full algorithmic compatibility. -**Measure alignment, over-refusal, retained utility and cost before choosing -SFT, DPO or OPD.** - -PyPI `v0.6.0` is the stable release; `main` is development and may be ahead. - -miniVERL is independent from verl and implements one verified Level-3 profile, -`single-gpu-online-distillation-v1`, against official verl `v0.8.0`. This is a -standard-artifact and config-subset bridge—not generic YAML compatibility or a -claim that distributed execution was tested. The local CUDA path has no -device-name allowlist; fit still depends on model size, sequence budget and -available VRAM. - -![miniVERL one-GPU workflow and pinned verl scale-out bridge](https://raw.githubusercontent.com/DaoyuanLi2816/mini-verl/main/docs/verl-bridge-architecture.svg) +## Install and run the 60-second demo ```bash -python -m pip install miniverl # lightweight core +python -m pip install "miniverl[train]" miniverl doctor -python -m pip install "miniverl[train]" # add the local training stack -miniverl demo --output runs/demo # no network, no GPU, ~50 s on a laptop CPU +miniverl demo --output runs/demo +miniverl inspect runs/demo ``` -The base install is the torch-free core (`doctor`, `validate`, `inspect`, -`report`, schemas and the Python API). The `train` extra adds torch, -Transformers and PEFT because `demo` performs real optimization. -This split is intentional: `pip install miniverl` is enough to inspect and -validate artifacts without downloading a multi-gigabyte ML stack; use -`pip install "miniverl[train]"` whenever the goal is training or evaluation. - -**What it makes inspectable** - -- **Policy truth:** strict OPD takes one update from each freshly sampled - parameter version; explicit replay keeps the rollout version visible, and - stale teacher targets are rejected. -- **Token truth:** tool output stays context, while only typed assistant spans - can enter the loss. -- **Budget truth:** exact full-vocabulary objectives and compressed - `top-k + tail` objectives are named and reported separately. -- **Decision truth:** every Alignment Card reports policy quality, retained - utility, teacher-query ratio, GPU time, VRAM and limitations together. - -[Run the local demo](#local-toy-demo) · -[Train on your GPU](#single-gpu-quickstart) · -[Inspect the measured result](#alignment-lab-when-to-turn-opd-off) · -[Read the math](https://github.com/DaoyuanLi2816/mini-verl/blob/main/docs/math.md) - -## Why miniVERL exists - -On-policy distillation is conceptually small and operationally fiddly. The -student samples a trajectory, the teacher scores *exactly the states the student -visited*, and you update on token-level distributional supervision. Four things -go wrong in practice, and all four are silent: - -1. **You train on tool output.** The environment's response is context, not a - label. One wrong mask and the model learns to hallucinate tool results. -2. **You are off by one.** The distribution that predicts token `j` lives at - position `j - 1`. Get it wrong and the loss still goes down. -3. **You are not actually on-policy.** Reuse a teacher cache across a policy - update and you are doing offline KD while calling it OPD. -4. **You cannot afford the logits.** A `[batch, seq_len, 152k]` tensor does not - fit on a consumer card, so the interesting configurations become the ones you - cannot run. - -miniVERL makes each of those a *checked property* rather than a comment, and -keeps the whole lifecycle in one readable single-GPU process. - -## What is implemented - -| Area | Status | -| --- | --- | -| Student-sampled multi-turn rollouts with real tool execution | yes | -| Strict per-token provenance (`system` / `user` / `assistant_*` / `tool_result`) | yes, validated on every read and write | -| Exact full-vocabulary forward KL, reverse KL, beta-JSD | yes, checked against brute-force references | -| Compressed `top-k + tail` KL and JSD | yes; the unsmoothed coarse-graining has a proven lower-bound relationship to the exact loss | -| Privileged-context teacher with an explicit alignment map | yes | -| Frozen standard PEFT teacher adapters with provenance and competence gates | yes | -| Single-GPU CUDA path with automatic bf16/fp16 selection | yes; device-name-agnostic CUDA path, measured reference on an RTX 4080 | -| Padded multi-trajectory updates | yes; mask-isolated, length-bucketed, per-trajectory normalized; sequential remains the default | -| Shared-base student / teacher / optional reference adapters | yes; one physical HF base, typed roles, student-only optimizer ownership | -| `resident` and `swap` memory strategies, `auto` resolution | yes, with an equivalence test | -| Versioned, checksummed, pickle-free teacher-target cache | yes | -| SFT / offline KD / strict OPD / explicitly labeled replay behind one trainer | yes | -| `align` / `pilot`, policy-conditioned and aligned-adapter teachers, DPO provenance | yes | -| Versioned verifier gate, AlignmentBench metadata adapters and privacy-safe Alignment Cards | yes; external suites are metadata-only in the v0.5 measurement | -| Calculator, JSON-navigation and SQLite environments | yes, deterministic with exact verifiers | -| Exact checkpoint/resume | yes, asserted parameter-for-parameter | -| Self-contained offline HTML report with token-level divergence | yes | -| Pinned verl `v0.8.0` Level-3 bridge | yes; one fail-closed profile, Parquet round trips and standard PEFT/safetensors export | -| Native Ray/FSDP/Megatron/vLLM execution, VLMs, cross-tokenizer, PPO/GRPO | **no** — see [limitations](https://github.com/DaoyuanLi2816/mini-verl/blob/main/docs/limitations.md) | - -## Verified verl bridge - -The bridge imports 14 named fields from one pinned profile, converts the -official prompt Parquet schema in both directions, exports a self-checking -scale-out bundle and diagnoses it without pretending that miniVERL teacher -targets are PPO reference log-probabilities. +The demo is deterministic, needs no network or GPU, and performs a real toy +optimization in about 50 seconds on the measured laptop CPU. For inspection, +schemas and reports without the ML stack, use `pip install miniverl`. For CUDA +training, install the matching CUDA-enabled PyTorch wheel first, then install +`miniverl[train,cuda]`; the extra does not select a CUDA PyTorch build. See the +[single-GPU guide](https://github.com/DaoyuanLi2816/mini-verl/blob/main/docs/single-gpu-guide.md). -```bash -python -m pip install "miniverl[bridge]" -miniverl import-verl verl.yaml --profile single-gpu-online-distillation-v1 \ - --target-verl v0.8.0 --out recipes/imported.yaml -miniverl convert-dataset --from verl-parquet train.parquet --out local.parquet -miniverl export-verl --run runs/my-alignment --target-verl v0.8.0 \ - --out exports/my-alignment-verl -miniverl bridge doctor exports/my-alignment-verl --json -``` +## Three paths -The release smoke pins commit -`7aed6b230776f963fa09509c10d9c3a767d1102c`, parses the generated OmegaConf -profile, loads standard PEFT/safetensors and both Parquet splits, imports the -reward scaffold, verifies privacy plus every hash, and records distributed -execution as **not tested**. See the [contract, whitelist and evidence](https://github.com/DaoyuanLi2816/mini-verl/blob/main/docs/verl-bridge.md) -or the [community recipe registry](https://github.com/DaoyuanLi2816/mini-verl/blob/main/docs/community-benchmarks.md). +| Path | Start with | Concrete artifact | Next | +| --- | --- | --- | --- | +| **Align** — compare SFT, DPO, KD and OPD only when the pilot evidence supports the cost | `miniverl pilot recipes/alignment_policy_conditioned_qwen.yaml` | `alignment-card.json` | [Alignment Lab](https://github.com/DaoyuanLi2816/mini-verl/blob/main/docs/alignment-lab/alignment-lab-v1.md) | +| **Distill locally** — strict OPD, shared backbones and padded trajectory updates on one CUDA GPU | `miniverl train recipes/qwen_consumer_gpu_shared.yaml --dry-run` | `config.resolved.yaml` plus a revision-pinned PEFT adapter | [Bring your own GPU](https://github.com/DaoyuanLi2816/mini-verl/blob/main/docs/single-gpu-guide.md) | +| **Scale out** — import a documented profile, convert Parquet, export a bundle and run bridge checks | `miniverl bridge doctor scaleout-bundle` | `provenance/compatibility-report.json` | [Verified verl bridge](https://github.com/DaoyuanLi2816/mini-verl/blob/main/docs/verl-bridge.md) | -## Alignment Lab: when to turn OPD off +The bridge import is deliberately not generic YAML conversion. If dataset or +environment, teacher identity, objective, or schedule semantics are missing, +`import-verl` writes `import-report.json` and a non-executable +`imported.template.yaml` with `status: needs_user_input`. It never silently +substitutes calculator tasks or an unqualified same-base teacher. -> [!IMPORTANT] -> **The starting SFT policy already saturated this deterministic test.** Across -> three preregistered seeds, no continuation method improved its 100% alignment -> and 100% tool-utility result. The completed regressions from continued SFT, -> standard OPD and verifier-gated OPD remain in the headline result. +## One measured alignment result -All six methods start from the same checksummed Qwen3-0.6B SFT checkpoint and -use the same 48 ordered final-test tasks per seed. This is a small deterministic -tool-policy suite—not a broad safety benchmark—and the only measured GPU is an -RTX 4080. +Alignment Lab v1 is a **saturated tool-policy case study**, not a broad safety +benchmark. The shared SFT checkpoint already achieved 100% policy compliance +and 100% retained tool utility in all three seeds. No continuation method +improved it; continued SFT and both OPD variants retained measured regressions. -| method | alignment | tool utility | teacher query | continuation GPU time | +| continuation | alignment | tool utility | teacher queries | GPU time | | --- | ---: | ---: | ---: | ---: | -| SFT checkpoint | **100.0%** | **100.0%** | n/a | 0.0 s | -| continued SFT | 94.4% | 88.9% | n/a | 3.9 s | -| DPO | **100.0%** | **100.0%** | n/a | 8.6 s | -| offline soft distillation | **100.0%** | **100.0%** | 100.0% | 26.6 s | +| continued SFT | 94.4% | 88.9% | — | 3.9 s | +| DPO | 100.0% | 100.0% | — | 8.6 s | +| offline soft distillation | 100.0% | 100.0% | 100.0% | 26.6 s | | standard OPD | 98.6% | 97.2% | 100.0% | 76.7 s | | verifier-gated OPD | 97.9% | 95.8% | 46.8% | 66.0 s | -![Alignment quality versus utility retention](https://raw.githubusercontent.com/DaoyuanLi2816/mini-verl/main/docs/alignment-lab/quality-vs-utility.svg) - -Harmful-compliance and over-refusal rates were both 0% for every method, yet -safe-error-recovery utility regressed in three completed arms. Those two safety -axes alone therefore missed a real policy-utility failure in this suite. The -matched State × Supervision diagnostic found only 0.0251% mean teacher -probability mass beyond argmax on fresh states; it is a signal diagnostic, not -a separately trained hard-target result, and no soft-target advantage is -claimed. - -`miniverl pilot` returns `insufficient_evidence` and tells this recipe not to -spend online teacher-query cost. That is the intended product behavior: OPD is -an option to justify with evidence, not a default replacement for SFT. - -```bash -miniverl pilot recipes/alignment_tool_policy_toy.yaml --json -miniverl align recipes/alignment_tool_policy_toy.yaml --dry-run -``` - -Read the [data-bound report](https://github.com/DaoyuanLi2816/mini-verl/blob/main/docs/alignment-lab/alignment-lab-v1.md), -[technical PDF](https://github.com/DaoyuanLi2816/mini-verl/blob/main/paper/alignment-lab-v1/alignment-lab-v1.pdf), -[public article](https://github.com/DaoyuanLi2816/mini-verl/blob/main/docs/alignment-lab/when-opd-should-follow-sft.md), -[demo script](https://github.com/DaoyuanLi2816/mini-verl/blob/main/docs/alignment-lab/demo.md), or the -[18 privacy-safe Alignment Cards](https://github.com/DaoyuanLi2816/mini-verl/blob/main/benchmarks/alignment-cards/alignment-lab-v1/sft_checkpoint-seed-1234.md). -The [preregistration](https://github.com/DaoyuanLi2816/mini-verl/blob/main/benchmarks/preregistration/alignment-lab-v1.yaml), -[machine-readable result](https://github.com/DaoyuanLi2816/mini-verl/blob/main/benchmarks/results/alignment-lab-v1.json) and all 864 -task-level records bind the claims above. - -## Consumer Runtime: batch speed without a cluster - -> A low-memory one-GPU runtime for actor rollout, teacher/reference scoring and -> online policy update. - -v0.4 keeps rollout, scoring and update in one readable process, but can now -pad multiple variable-length trajectories into one mask-isolated update -forward. A shared-backbone mode loads one quantized base with a trainable -student adapter, a frozen teacher adapter and an optional frozen reference -adapter. The default remains `dual_model` plus sequential physical batches for -backward compatibility. - -![Consumer-runtime throughput versus VRAM](https://raw.githubusercontent.com/DaoyuanLi2816/mini-verl/main/docs/consumer-runtime-v1-pareto.svg) - -On the preregistered RTX 4080 systems workload, physical batch-4 improved -end-to-end throughput by 1.63× for dual models and 1.54× for the shared -backbone. At batch-4, sharing reduced peak reserved memory from 3.04 to 2.23 -GiB, while running 10.1% slower than dual ownership. `auto` was slower because -padding all eight trajectories was wasteful; it is a convenience, not a claim -that the largest batch is best. - -All eight cells reused identical trajectories and teacher targets. Twelve -preregistered loss/gradient/update comparisons passed; the largest loss -difference was 1.25e-6 and the largest updated-logit difference was 1.30e-4. -The benchmark uses NF4 weights with FP32 compute to keep that numerical gate -meaningful. It does not claim a quality gain, universal GPU speedup, batched -rollout server or distributed-runtime parity. - -Set `train.trajectory_batch_size` to `1`, an integer, or `auto`; choose -`models.runtime: shared_backbone` only when student, teacher and optional -reference use the same pinned base and distinct adapters. See the -[data-bound report](https://github.com/DaoyuanLi2816/mini-verl/blob/main/docs/consumer-runtime-v1.md), [preregistration](https://github.com/DaoyuanLi2816/mini-verl/blob/main/benchmarks/preregistration/consumer-runtime-v1.yaml) -and [frozen result](https://github.com/DaoyuanLi2816/mini-verl/blob/main/benchmarks/results/consumer-runtime-v1.json). - -## RecoveryBench: do fresh on-policy states justify their cost? - -> [!IMPORTANT] -> **Not in this measured setting.** Under eight equal continuation updates, -> frozen-student-state KD reached 23.2% strict success, while strict fresh-state -> OPD reached 10.9%. The paired fresh-minus-frozen difference was -12.24 -> percentage points (95% task-paired bootstrap interval -15.89 to -8.59). - -RecoveryBench is a preregistered mechanism study on SQLite tool-error recovery, -not an alignment benchmark. It isolates state freshness while holding the cold -checkpoint, qualified teacher, task schedule, optimizer and update count fixed. -All three seeds and all completed negative results are retained. - -| method | strict success | recovery after error | continuation time | -| --- | ---: | ---: | ---: | -| cold start | 10.7% | 13.6% | 0.2 s | -| continued oracle SFT | 4.9% | 1.8% | 51.3 s | -| oracle-state offline KD | **33.1%** | **31.9%** | 58.3 s | -| frozen-student-state KD | **23.2%** | **22.8%** | 52.1 s | -| strict fresh-state OPD | 10.9% | 9.1% | 686.8 s | -| budget-50 fresh-state OPD | 27.3% | 20.7% | 720.8 s | - -![RecoveryBench three-seed result](https://raw.githubusercontent.com/DaoyuanLi2816/mini-verl/main/docs/recoverybench/recovery-success.svg) - -The equal-selected-position view reached the 6,224-position boundary after -eight updates for every core method, so its quality result matches the primary -view. The budget-50 selector queried 49.77% of model-generated positions but -did not reduce wall time because teacher backbone forwards were unchanged. The -50-second artifact is a **cycle-capped wall diagnostic, not exact equal-time -evidence**: SFT and frozen KD completed their eight-cycle ceiling, while fresh -OPD crossed the target in one indivisible 88-121 second update. - -Read the [full analysis](https://github.com/DaoyuanLi2816/mini-verl/blob/main/docs/recoverybench/recoverybench-v1.md), the -[data-bound technical report](https://github.com/DaoyuanLi2816/mini-verl/blob/main/paper/recoverybench-v1/recoverybench-v1.pdf), or -the [immutable schema-v3 artifacts](https://github.com/DaoyuanLi2816/mini-verl/blob/main/benchmarks/README.md#recoverybench-v1). -The result is scoped to one Qwen3 pair, one task family, three seeds and one RTX -4080. It does not show that OPD is universally ineffective or that offline KD -always wins. - -
-Case study: why teacher protocol qualification matters - -On the saturated v0.2 calculator task, a protocol-qualified OPD teacher reached -100% in both seeds and tied continued SFT, but took 6.1× as much continuation -time. Two protocol-naive controls completed normally at 0%; they were not -configuration failures. Both used the ambiguous historical protocol-v1 prompt, -so the failure cannot be attributed solely to intrinsic teacher behavior. - -![Two-seed protocol-teacher benchmark](https://raw.githubusercontent.com/DaoyuanLi2816/mini-verl/main/docs/gpu-calc-hard-equal-update-v2.svg) - -| Artifact | Role | -| --- | --- | -| [Default recipe](https://github.com/DaoyuanLi2816/mini-verl/blob/main/recipes/qwen_consumer_gpu_calc.yaml) | protocol-qualified default | -| [Schema-v2 benchmark](https://github.com/DaoyuanLi2816/mini-verl/blob/main/benchmarks/results/gpu-calc-hard-equal-update-v2.json) | frozen five-arm result | -| [Raw-teacher recipe](https://github.com/DaoyuanLi2816/mini-verl/blob/main/recipes/qwen_consumer_gpu_calc_raw_teacher.yaml) | historical control; not default | - -The teacher gate and downstream comparison reused the same 24-task v0.2 test -set, so this is evidence for qualification in that setup, not a general OPD -advantage. The separate schema-v1 481-second smoke proves the pipeline, not OPD -over SFT. [Full diagnosis and caveats](https://github.com/DaoyuanLi2816/mini-verl/blob/main/docs/rtx4080-baselines.md). - -
- -## Local toy demo - -No network, no GPU, no downloads. Both models are small transformers built from -the config, the tokenizer is a reversible ~190-entry toy tokenizer, and the -calculator environment generates and grades its own tasks. +![Alignment and utility deltas from the saturated SFT checkpoint; small marks are all three seeds and large marks are means](https://raw.githubusercontent.com/DaoyuanLi2816/mini-verl/main/docs/alignment-lab/delta-from-sft.svg) -```bash -python -m pip install ".[train]" # from the cloned repository; CPU torch is enough -miniverl doctor # what can this machine run? -miniverl demo --output runs/demo -``` +The two sandbox safety checks tied at zero while utility still regressed. +IFEval, XSTest, HarmBench and RewardBench were **not executed**. “Preference +win rate” is a deterministic Minipolicy paired outcome, not human preference. +Read the [study, seed-level values and limitations](https://github.com/DaoyuanLi2816/mini-verl/blob/main/docs/alignment-lab/alignment-lab-v1.md). -It runs the real pipeline — student rollouts, tool execution, teacher scoring of -exactly those states, a compressed top-k cache with provenance checks, and a -masked reverse-KL update on assistant tokens only — then prints where every -artifact landed and what to run next: - -```text -demo complete runs/demo - mode opd (genuine on-policy distillation) - optimizer steps 132 - parameter version 132 - rollout iterations 13 - wall clock 52.9 s - token provenance 45597 of 226383 tokens trainable (20%); 180786 are context - and can never be a target - teacher cache 735 scored positions, 131.6 KiB on disk, 2.0x smaller than - a dense fp16 dump - task success 0.0% -> 0.0% (greedy, held-out eval split) - -This demo proves the machinery, not capability. -At this size the toy student learns the tool-call format and not the -arithmetic, so 0% here is the expected outcome, not a failure. -For a CPU run that does learn (measured 0.0% -> 91.7% in 192 s): - miniverl train recipes/toy_cpu.yaml -``` +## One measured systems result -That last line is not a promise, it is a measurement: -`recipes/toy_cpu.yaml` takes **192 s on a CPU** and moves held-out greedy task -success from **0.0% to 91.7%** on 24 tasks, over 600 supervised cold-start steps -plus 40 on-policy distillation cycles. It is also **seed-sensitive** at this -model size: the same 600-step budget gives 81.2% with `run.seed: 1234` and 0.0% -with `run.seed: 20260727`. That variance is exactly why the toy backend is a -machinery harness and capability numbers come from the GPU recipe. - -`miniverl inspect` is the one worth running first. It prints the provenance -table, which is the whole point of the project: - -```text -tokens by span type (only assistant_* can enter the loss) -+---------------------------------------------+ -| span type | tokens | in loss | -|---------------------+--------+--------------| -| system | 776 | no (context) | -| tool_result | 685 | no (context) | -| user | 318 | no (context) | -| assistant_tool_call | 153 | yes | -| assistant_text | 85 | yes | -| assistant_final | 25 | yes | -+---------------------------------------------+ -``` +On one RTX 4080 with Qwen3-0.6B and eight fixed SQLite trajectories, physical +batch 4 improved update throughput from 2.369 to 3.866 trajectories/s in the +dual-model runtime. The shared-backbone batch-4 cell used 2.227 GiB peak +reserved memory versus 3.035 GiB for dual model, while running 10.1% slower. +All 12 preregistered equivalence comparisons passed. These are one-workload, +one-machine measurements, not promises for other GPUs. -The toy backend is a **machinery harness, not a capability demonstration**. Its -models are too small to solve anything beyond the `easy` split. Capability -numbers come from the GPU recipe. +![Measured throughput and reserved VRAM for dual-model and shared-backbone runtime cells](https://raw.githubusercontent.com/DaoyuanLi2816/mini-verl/main/docs/consumer-runtime-v1-pareto.svg) -## Single-GPU quickstart +[Consumer Runtime v1 methods and caveats](https://github.com/DaoyuanLi2816/mini-verl/blob/main/docs/consumer-runtime-v1.md) -The default recipe uses `device: auto` and `dtype: auto`: bf16-capable cards use -bf16, while older CUDA cards such as Titan V use fp16. RTX 3070, Titan V, -RTX 4080 and RTX 5090-class cards all enter the same code path; only the -RTX 4080 result is measured here. Exact fit is governed by VRAM, model sizes, -drivers and token budgets, not the card's marketing name. See the -[`single-GPU guide`](https://github.com/DaoyuanLi2816/mini-verl/blob/main/docs/single-gpu-guide.md) before changing the recipe. +## Compatibility boundary -```bash -git clone https://github.com/DaoyuanLi2816/mini-verl.git -cd mini-verl -python -m pip install torch --index-url https://download.pytorch.org/whl/cu130 -python -m pip install ".[train,cuda]" - -miniverl doctor # confirms CUDA + bitsandbytes -miniverl validate recipes/qwen_consumer_gpu_calc.yaml -miniverl train recipes/qwen_consumer_gpu_calc.yaml --dry-run # nothing is downloaded -miniverl train recipes/qwen_consumer_gpu_calc.yaml -miniverl report runs/ --out runs//report.html -``` - -The recipe pins both revisions: - -| role | model | revision | license | -| --- | --- | --- | --- | -| student | `Qwen/Qwen3-0.6B` | `c1899de289a04d12100db370d81485cdf75e47ca` | Apache-2.0 | -| teacher | `Qwen/Qwen3-1.7B` | `70d244cc86ccca08cf5af4e1e306ecf908b1ad5e` | Apache-2.0 | - -Their `tokenizer.json` files are byte-identical -(`sha256 aeb13307a71acd8fe81861d94ad54ab689df773318809eed3cbe794b4492dae4`). -New runs compare structural identity; old artifacts use the legacy fixed-probe -behavioural fingerprint. -The recipe also pins the [protocol-teacher adapter](https://huggingface.co/DaoyuanLi/mini-verl-qwen3-1.7b-protocol-teacher) -at revision `23323751318135484c06c043b1f9b9e7016dd89f` and requires its recorded -strict policy success to be at least 50% before allocating the teacher. - -## Architecture - -```mermaid -flowchart LR - A["student pi_theta
QLoRA, resident"] -->|sample| B["RolloutRunner
agent/loop.py"] - B -->|tool call| C["ToolEnvironment
calculator / jsonnav / sqlite"] - C -->|observation| B - B -->|typed token spans| D["Trajectory
schemas/trajectory.py"] - D -->|select_positions| E["AlignmentMap
trajectory/alignment.py"] - E -->|score those exact states| F["LocalTeacherScorer
teachers/local.py"] - F -->|top-k + tail| G["TeacherCache
cache/store.py"] - F --> H["chunked_selected_position_loss
losses/chunked.py"] - G --> H - H -->|masked KL on assistant tokens| A -``` +![Verified local runtime, portable artifact bundle and pinned upstream smoke; distributed verl execution remains untested](https://raw.githubusercontent.com/DaoyuanLi2816/mini-verl/main/docs/verl-bridge-architecture.svg) -Layer boundaries are strict, and the first layer never imports torch: - -1. `schemas/`, `trajectory/`, `config/`, `agent/protocol.py` — pure data, masks, - validation. -2. `losses/` — torch numerics, no model knowledge. -3. `models/` — backends and the architecture adapter. -4. `environments/`, `agent/` — task and tool semantics. -5. `training/`, `teachers/`, `cache/`, `selection/` — orchestration. -6. `evaluation/`, `reporting/` — measurement. -7. `cli.py` — a thin shell that calls one library function per command. - -See [`docs/design.md`](https://github.com/DaoyuanLi2816/mini-verl/blob/main/docs/design.md). - -## Exact versus top-k + tail - -Two clearly named classes of objective, because conflating them is how -distillation results become unreproducible. - -**`exact_full_vocab`** materializes the complete `[chunk, V]` teacher and student -distributions and computes the real divergence. Affordable when `V` is small (the -toy backend) or when the teacher stays resident and the distribution is rebuilt -one chunk at a time. Guarded by `loss.exact_max_vocab` (default 8192) so it can -never silently try to persist a `[positions, 152k]` tensor. - -**`bucketed_topk_tail`** coarse-grains the vocabulary into the teacher's top-k -tokens plus one aggregate tail bucket, then computes the divergence between the -two `K+1` category distributions. This is **not** full-vocabulary KL. The -data-processing lower-bound theorem applies to the unsmoothed coarse-graining; -the finite implementation floors and renormalizes non-empty tails, so it is -described as an epsilon-smoothed objective rather than claiming the theorem -literally for every input. When `k == V`, the empty tail bypasses smoothing and -the implementation reproduces the exact objective to `1e-9` in float64 tests. -The functions are named `bucketed_forward_kl`, `bucketed_reverse_kl` and -`bucketed_jsd` so that no call site can pretend otherwise. - -What the compression actually buys is teacher-side storage and the ability to -evict the teacher from VRAM. It does **not** proportionally reduce teacher FLOPs: -the teacher still runs a full forward pass to produce the hidden states. Reports -therefore say `teacher_queried_position_ratio`, never "teacher compute saved". - -Top-k + tail targets are not a new idea — TRL's `ServerDistillationTrainer` has -`loss_top_k` with an optional tail bucket. See [`docs/math.md`](https://github.com/DaoyuanLi2816/mini-verl/blob/main/docs/math.md). - -## Tool-token masking - -Every trajectory is a flat token sequence plus a partition into typed spans. The -three masks are stored *and* re-derived from the spans on every read; a file -whose mask disagrees with its spans is rejected rather than trained on. - -```python -from miniverl.trajectory.io import read_trajectories - -traj = read_trajectories("runs/demo/trajectories.jsonl")[0] -print(traj.token_counts_by_span_type()) -# {'system': 194, 'user': 40, 'assistant_tool_call': 38, 'tool_result': 34, 'assistant_final': 7} -print(sum(traj.model_generated_mask)) # only assistant_* tokens are trainable -``` +The bridge targets official verl `v0.8.0` at commit `7aed6b23` and uses the +term **miniVERL-defined compatibility Level 3**. That means a checksummed +standard-artifact bundle plus pinned upstream config-parse/model-data-load +smoke—not arbitrary verl YAML or a completed distributed job. -Context segments own the trailing `<|im_start|>assistant\n` header, so a model -span begins at exactly the first sampled token and no forced scaffolding token is -ever a target. Position `0` can never be a target. Both are enforced, not -documented — see `tests/unit/test_token_provenance.py`. - -## Benchmark results - -Every number below was produced by the commands in -[`docs/benchmarking.md`](https://github.com/DaoyuanLi2816/mini-verl/blob/main/docs/benchmarking.md) on the hardware recorded in each -result file. Nothing is estimated or extrapolated. - -* **RTX 4080, real models** — [`docs/rtx4080-baselines.md`](https://github.com/DaoyuanLi2816/mini-verl/blob/main/docs/rtx4080-baselines.md) - has measured peak VRAM, decode throughput, the full-recipe run, the two-seed - schema-v2 protocol-teacher comparison, and the preserved legacy comparison. -* **CPU, toy models** — `recipes/toy_cpu.yaml` moves task success from 0.0% to - 91.7% in 192 s, and `benchmarks/results/` holds the legacy equal-update parity - run. - The parity run's accuracy differences are **within noise**; it exists to show - that all seven arms run to completion under identical budgets, not to rank - them. See [`benchmarks/README.md`](https://github.com/DaoyuanLi2816/mini-verl/blob/main/benchmarks/README.md) for why the toy - backend cannot rank methods. - -## Installation - -| Layer | Install | What you get | -| --- | --- | --- | -| Core | `python -m pip install .` | `doctor`, `validate`, `inspect`, `report`, `cache`, the schemas and the Python API. No torch. | -| Training | `python -m pip install ".[train]"` | `demo`, `train`, `eval`, `benchmark`. Adds torch, transformers, peft, accelerate. | -| 4-bit | `python -m pip install ".[cuda]"` | bitsandbytes, for NF4 QLoRA and the 8-bit optimizer. | -| Development | `python -m pip install ".[dev]"` | pytest, hypothesis, ruff, mypy, build, twine. | - -The published-package equivalents are `miniverl`, `miniverl[train]`, -`miniverl[cuda]` and `miniverl[dev]`. Core Python 3.10–3.13 is tested without -torch. The full CPU ML suite and Transformers 4.51.x/5.x compatibility rows run -on Python 3.12; GPU paths are opt-in and were measured locally on Python 3.12. - -Install the CUDA build of torch that matches your driver separately; the PyPI -wheel is CPU-only on some platforms: +Current exported bundles are intentionally `launchable: false`: the base +snapshot is absent, the reward implementation fails closed, and required user +mappings remain placeholders. The generated entry point is therefore +`launch.template.sh`. Readiness is reported as separate facts for artifact +completeness, parse/load smoke, reward completeness, launchability, +distributed execution and algorithm-semantic parity. The target is a +PPO/reward scaffold, not an executable continuation of miniVERL OPD semantics. -```bash -pip install torch --index-url https://download.pytorch.org/whl/cu130 -``` +## Detailed studies and preserved negative evidence -A missing extra never produces a traceback: +- [RecoveryBench v1](https://github.com/DaoyuanLi2816/mini-verl/blob/main/docs/recoverybench/recoverybench-v1.md): frozen-student KD + outperformed much slower fresh-state OPD on the preregistered primary view; + the verifier gate remained `insufficient_evidence`. +- [Alignment Lab v1](https://github.com/DaoyuanLi2816/mini-verl/blob/main/docs/alignment-lab/alignment-lab-v1.md): the starting SFT + checkpoint was at the ceiling, so no positive OPD result is claimed. +- [Calculator benchmark](https://github.com/DaoyuanLi2816/mini-verl/blob/main/docs/benchmarking.md): both negative controls completed + normally and measured 0% strict success. They were not configuration + failures. Because they used the historical ambiguous protocol-v1 prompt, + their failure cannot be attributed solely to intrinsic teacher behavior. +- [Consumer Runtime v1](https://github.com/DaoyuanLi2816/mini-verl/blob/main/docs/consumer-runtime-v1.md): padded update batches and + shared adapters preserve the measured one-update objective within declared + tolerances; rollout generation remains sequential. +- [Limitations](https://github.com/DaoyuanLi2816/mini-verl/blob/main/docs/limitations.md), [math](https://github.com/DaoyuanLi2816/mini-verl/blob/main/docs/math.md), + [reproducibility](https://github.com/DaoyuanLi2816/mini-verl/blob/main/docs/reproducibility.md) and + [compatibility policy](https://github.com/DaoyuanLi2816/mini-verl/blob/main/docs/compatibility.md). -```text -$ miniverl demo --output runs/demo -error miniverl demo requires the optional dependency 'torch', which is not installed. -hint pip install "miniverl[train]" -``` +New runs establish tokenizer compatibility through structural identity. The +legacy behavioral fingerprint—token IDs for one fixed probe plus metadata—is +only a migration fallback for older artifacts and is not an identity proof. -### Strict offline execution +## Scope -All model-loading commands use the same no-network contract: +miniVERL supports one local CUDA process. It does not implement or wrap Ray, +FSDP, Megatron, PPO, GRPO or a distributed launcher. The public studies cover +small Qwen3 models, deterministic tool environments and one RTX 4080; they do +not establish cross-model, cross-task, cross-GPU or broad safety generality. ```bash -miniverl train --offline -miniverl benchmark --offline -miniverl eval --run --offline -miniverl export-adapter --run --out --offline -``` - -In this mode, the base model, tokenizer and every adapter file must already be -at a local path or in the Hugging Face cache. miniVERL permits no HTTP, -metadata, ETag or Hub API request and does not fall back to online resolution. -A Hub teacher adapter is resolved once at its pinned revision; PEFT then loads -the exact local snapshot whose config, weights, manifest and checksums were -validated. A cache miss prints the immutable identity and the exact `hf -download` preload command. - -## Python API - -The public surface is deliberately small. - -```python -from miniverl.config import RunConfig -from miniverl.trainer import OPDTrainer - -config = RunConfig.from_yaml("recipes/toy_cpu.yaml") -with OPDTrainer.from_config(config) as trainer: - result = trainer.train() - -print(result.run_dir, result.global_step, result.eval["success_rate"]) -``` - -## A custom environment - -Subclass `ToolEnvironment`, register it, and every recipe key works unchanged. -`examples/custom_environment/` is a complete, runnable example. -`reset(task)` is authoritative: it is called exactly once per episode, and its -`Observation.text` plus `state_id` enter the trajectory. `user_prompt(task)` is -only a compatibility helper; the runner does not call it a second time. - -```python -from miniverl.environments import ToolEnvironment, ToolSpec -from miniverl.environments.registry import register - - -@register -class ReverseEnvironment(ToolEnvironment): - name = "reverse" - - def tool_specs(self) -> list[ToolSpec]: - return [ - ToolSpec( - name="reverse", - description="Reverse a string.", - parameters={"text": "string to reverse"}, - required=("text",), - example={"text": "abc"}, - ) - ] - - # reset / step / verify / generate_task / oracle_actions follow; see the example. -``` - -## A custom teacher - -Implement `TeacherScorer.score` and return supervision for the aligned positions. -`examples/custom_teacher/` shows a scorer that sharpens a local model's -distribution before handing it over, and asserts that the result still trains. - -For a standard frozen PEFT teacher adapter, including the Qwen3 protocol-SFT -recipe, export command, compatibility checks and policy-competence gate, see -[`docs/teacher-adapters.md`](https://github.com/DaoyuanLi2816/mini-verl/blob/main/docs/teacher-adapters.md). - -## Limitations - -The short version; the full list is in [`docs/limitations.md`](https://github.com/DaoyuanLi2816/mini-verl/blob/main/docs/limitations.md). - -* Same tokenizer only. Cross-tokenizer distillation is rejected with an error. -* Rollout decoding is one sequence at a time. The update path supports padded - physical batches; `gradient_accumulation_steps` is the optimizer-group size - and `trajectory_batch_size` is the number sharing one backbone forward. -* `swap` is unavailable for quantized models, because bitsandbytes parameters are - pinned to the device they were quantized on. -* Only Qwen3 and Qwen2 architectures are tested. Others may work through the - architecture adapter; nothing here claims they do. -* RecoveryBench has three prespecified student seeds; the calculator case study - has two, and older GPU artifacts are single-seed. No broad statistical - significance or cross-task generalization is claimed. -* On the measured machine, decoding is kernel-launch bound rather than compute - bound, so throughput figures are platform-specific. - -## Reproducibility - -Every run writes `manifest.json` with the miniVERL version, git commit, Python -and OS, torch/CUDA/driver versions, GPU model and VRAM, model ids **and resolved -revisions**, tokenizer fingerprint, seeds, precision, quantization, memory -strategy, loss mode, top-k, policy version, and a `measurement_status` block -recording whether each result was measured, simulated or not run. It records no -usernames, hostnames, home paths, or environment variables beyond a short -allowlist of ones that change numerics — asserted by a test. -File-backed runs also separate exact submitted bytes, canonical validated -logic, the v0.2 resume compatibility layer, and runtime-resolved choices. - -Writable runs move atomically through `ready`, `running`, and one terminal -status (`completed`, `failed`, `interrupted`, or `closed_before_training`). -One process lock covers construction, training/resume, standalone checkpoint -selection and evaluation, and automatic report generation. Within one trainer, -training, evaluation, checkpoint save/load and destructive close are mutually -exclusive; load is READY-only, close mutates nothing unless it obtains -ownership, and evaluation restores the exact prior model mode even on failure. - -After `reset`, every built-in verifier maps arbitrary strings to a bounded -result rather than leaking parser/numeric exceptions; protocol-v2 prompts use -environment-specific, verifier-format-valid final examples. Shareable reports, -summaries, benchmark exports and portable manifests redact semantic secret -keys, URL credentials and private cross-platform paths; private run artifacts -still retain the local state required for exact resume. Redaction is a -best-effort sharing defense, not permission to place real credentials in any -config, run artifact or report. - -See [`docs/reproducibility.md`](https://github.com/DaoyuanLi2816/mini-verl/blob/main/docs/reproducibility.md) and the concise -[`compatibility policy`](https://github.com/DaoyuanLi2816/mini-verl/blob/main/docs/compatibility.md). - -## Roadmap - -Not implemented, not promised, listed so the scope is unambiguous: -cross-tokenizer distillation, batched or engine-backed rollout decoding, -entropy-aware divergence mixing (arXiv:2603.07079), additional model families, -more environments, and native multi-GPU execution. The Level-3 bridge exports -one documented profile to pinned verl; it does not execute that distributed job. - -## Acknowledgement and disclaimer - -> miniVERL is an independent project and is not affiliated with or endorsed by -> the verl project, ByteDance, or Volcano Engine. It is not a drop-in -> replacement for verl. - -The name is a nod to the problem space, not a claim of generic compatibility. -The verified bridge is deliberately limited to one pinned profile. verl is an -excellent, much larger system for cluster-scale execution with Ray. miniVERL -exists for the case where you have one personal GPU and want to read every line -of what is happening, then hand standard artifacts to verl for scale-out. That -can be an older 12 GiB card or a current high-end card; the repository claims -measured performance only for hardware it actually ran. See -[`docs/comparisons.md`](https://github.com/DaoyuanLi2816/mini-verl/blob/main/docs/comparisons.md). - -## Citation - -```bibtex -@software{miniverl2026, - title = {miniVERL: Auditable online post-training on one GPU}, - author = {Li, Daoyuan}, - year = {2026}, - url = {https://github.com/DaoyuanLi2816/mini-verl}, - license = {Apache-2.0} -} +git clone https://github.com/DaoyuanLi2816/mini-verl.git +cd mini-verl +python -m pip install -e ".[dev]" +pytest -q -m "not gpu and not network" ``` -See [CITATION.cff](https://github.com/DaoyuanLi2816/mini-verl/blob/main/CITATION.cff) and [CHANGELOG.md](https://github.com/DaoyuanLi2816/mini-verl/blob/main/CHANGELOG.md). -Contributions: [CONTRIBUTING.md](https://github.com/DaoyuanLi2816/mini-verl/blob/main/CONTRIBUTING.md). Security: -[SECURITY.md](https://github.com/DaoyuanLi2816/mini-verl/blob/main/SECURITY.md). - -## License - -Apache-2.0. See [LICENSE](https://github.com/DaoyuanLi2816/mini-verl/blob/main/LICENSE) and -[THIRD_PARTY_NOTICES.md](https://github.com/DaoyuanLi2816/mini-verl/blob/main/THIRD_PARTY_NOTICES.md). - -Chinese translation: [README.zh-CN.md](https://github.com/DaoyuanLi2816/mini-verl/blob/main/README.zh-CN.md). +Apache-2.0 licensed. See [CONTRIBUTING.md](https://github.com/DaoyuanLi2816/mini-verl/blob/main/CONTRIBUTING.md) and +[SECURITY.md](https://github.com/DaoyuanLi2816/mini-verl/blob/main/SECURITY.md). Project records: [default GPU recipe](https://github.com/DaoyuanLi2816/mini-verl/blob/main/recipes/qwen_consumer_gpu_calc.yaml), +[frozen calculator JSON](https://github.com/DaoyuanLi2816/mini-verl/blob/main/benchmarks/results/gpu-calc-hard-equal-update-v2.json), +[changelog](https://github.com/DaoyuanLi2816/mini-verl/blob/main/CHANGELOG.md), [citation](https://github.com/DaoyuanLi2816/mini-verl/blob/main/CITATION.cff) and [license](https://github.com/DaoyuanLi2816/mini-verl/blob/main/LICENSE). diff --git a/README.md b/README.md index 399c72f..1e53984 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,5 @@

- miniVERL — online alignment and distillation on one GPU + miniVERL — single-GPU LLM post-training

@@ -13,675 +13,142 @@

- PyPI package · - Documentation · - Install & train · - Verified verl bridge · - Alignment Lab + PyPI · + Stable docs · + Development docs · + 中文

-**Single-GPU prototyping for a documented subset of verl-style online -post-training.** +**miniVERL is a local, inspectable runtime for a documented subset of +single-GPU LLM alignment and distillation.** It keeps rollout provenance, +assistant-only loss masks, teacher targets, update budgets and run artifacts +explicit, then exports portable artifacts through a fail-closed bridge to one +pinned upstream verl profile. -Develop, diagnose and validate an alignment or distillation recipe locally, -then export standard model, dataset, recipe and provenance artifacts to a -pinned verl release for scale-out. +PyPI `v0.6.0` is stable; `main` is development. The CUDA path has no GPU-name +allowlist, but fit depends on the model pair, context budget, kernels and VRAM. +miniVERL is independent from verl and does not claim distributed execution or +full algorithmic compatibility. -**Measure alignment, over-refusal, retained utility and cost before choosing -SFT, DPO or OPD.** - -PyPI `v0.6.0` is the stable release; `main` is development and may be ahead. - -miniVERL is independent from verl and implements one verified Level-3 profile, -`single-gpu-online-distillation-v1`, against official verl `v0.8.0`. This is a -standard-artifact and config-subset bridge—not generic YAML compatibility or a -claim that distributed execution was tested. The local CUDA path has no -device-name allowlist; fit still depends on model size, sequence budget and -available VRAM. - -![miniVERL one-GPU workflow and pinned verl scale-out bridge](docs/verl-bridge-architecture.svg) +## Install and run the 60-second demo ```bash -python -m pip install miniverl # lightweight core +python -m pip install "miniverl[train]" miniverl doctor -python -m pip install "miniverl[train]" # add the local training stack -miniverl demo --output runs/demo # no network, no GPU, ~50 s on a laptop CPU +miniverl demo --output runs/demo +miniverl inspect runs/demo ``` -The base install is the torch-free core (`doctor`, `validate`, `inspect`, -`report`, schemas and the Python API). The `train` extra adds torch, -Transformers and PEFT because `demo` performs real optimization. -This split is intentional: `pip install miniverl` is enough to inspect and -validate artifacts without downloading a multi-gigabyte ML stack; use -`pip install "miniverl[train]"` whenever the goal is training or evaluation. - -**What it makes inspectable** - -- **Policy truth:** strict OPD takes one update from each freshly sampled - parameter version; explicit replay keeps the rollout version visible, and - stale teacher targets are rejected. -- **Token truth:** tool output stays context, while only typed assistant spans - can enter the loss. -- **Budget truth:** exact full-vocabulary objectives and compressed - `top-k + tail` objectives are named and reported separately. -- **Decision truth:** every Alignment Card reports policy quality, retained - utility, teacher-query ratio, GPU time, VRAM and limitations together. - -[Run the local demo](#local-toy-demo) · -[Train on your GPU](#single-gpu-quickstart) · -[Inspect the measured result](#alignment-lab-when-to-turn-opd-off) · -[Read the math](docs/math.md) - -## Why miniVERL exists - -On-policy distillation is conceptually small and operationally fiddly. The -student samples a trajectory, the teacher scores *exactly the states the student -visited*, and you update on token-level distributional supervision. Four things -go wrong in practice, and all four are silent: - -1. **You train on tool output.** The environment's response is context, not a - label. One wrong mask and the model learns to hallucinate tool results. -2. **You are off by one.** The distribution that predicts token `j` lives at - position `j - 1`. Get it wrong and the loss still goes down. -3. **You are not actually on-policy.** Reuse a teacher cache across a policy - update and you are doing offline KD while calling it OPD. -4. **You cannot afford the logits.** A `[batch, seq_len, 152k]` tensor does not - fit on a consumer card, so the interesting configurations become the ones you - cannot run. - -miniVERL makes each of those a *checked property* rather than a comment, and -keeps the whole lifecycle in one readable single-GPU process. - -## What is implemented - -| Area | Status | -| --- | --- | -| Student-sampled multi-turn rollouts with real tool execution | yes | -| Strict per-token provenance (`system` / `user` / `assistant_*` / `tool_result`) | yes, validated on every read and write | -| Exact full-vocabulary forward KL, reverse KL, beta-JSD | yes, checked against brute-force references | -| Compressed `top-k + tail` KL and JSD | yes; the unsmoothed coarse-graining has a proven lower-bound relationship to the exact loss | -| Privileged-context teacher with an explicit alignment map | yes | -| Frozen standard PEFT teacher adapters with provenance and competence gates | yes | -| Single-GPU CUDA path with automatic bf16/fp16 selection | yes; device-name-agnostic CUDA path, measured reference on an RTX 4080 | -| Padded multi-trajectory updates | yes; mask-isolated, length-bucketed, per-trajectory normalized; sequential remains the default | -| Shared-base student / teacher / optional reference adapters | yes; one physical HF base, typed roles, student-only optimizer ownership | -| `resident` and `swap` memory strategies, `auto` resolution | yes, with an equivalence test | -| Versioned, checksummed, pickle-free teacher-target cache | yes | -| SFT / offline KD / strict OPD / explicitly labeled replay behind one trainer | yes | -| `align` / `pilot`, policy-conditioned and aligned-adapter teachers, DPO provenance | yes | -| Versioned verifier gate, AlignmentBench metadata adapters and privacy-safe Alignment Cards | yes; external suites are metadata-only in the v0.5 measurement | -| Calculator, JSON-navigation and SQLite environments | yes, deterministic with exact verifiers | -| Exact checkpoint/resume | yes, asserted parameter-for-parameter | -| Self-contained offline HTML report with token-level divergence | yes | -| Pinned verl `v0.8.0` Level-3 bridge | yes; one fail-closed profile, Parquet round trips and standard PEFT/safetensors export | -| Native Ray/FSDP/Megatron/vLLM execution, VLMs, cross-tokenizer, PPO/GRPO | **no** — see [limitations](docs/limitations.md) | - -## Verified verl bridge - -The bridge imports 14 named fields from one pinned profile, converts the -official prompt Parquet schema in both directions, exports a self-checking -scale-out bundle and diagnoses it without pretending that miniVERL teacher -targets are PPO reference log-probabilities. +The demo is deterministic, needs no network or GPU, and performs a real toy +optimization in about 50 seconds on the measured laptop CPU. For inspection, +schemas and reports without the ML stack, use `pip install miniverl`. For CUDA +training, install the matching CUDA-enabled PyTorch wheel first, then install +`miniverl[train,cuda]`; the extra does not select a CUDA PyTorch build. See the +[single-GPU guide](docs/single-gpu-guide.md). -```bash -python -m pip install "miniverl[bridge]" -miniverl import-verl verl.yaml --profile single-gpu-online-distillation-v1 \ - --target-verl v0.8.0 --out recipes/imported.yaml -miniverl convert-dataset --from verl-parquet train.parquet --out local.parquet -miniverl export-verl --run runs/my-alignment --target-verl v0.8.0 \ - --out exports/my-alignment-verl -miniverl bridge doctor exports/my-alignment-verl --json -``` +## Three paths -The release smoke pins commit -`7aed6b230776f963fa09509c10d9c3a767d1102c`, parses the generated OmegaConf -profile, loads standard PEFT/safetensors and both Parquet splits, imports the -reward scaffold, verifies privacy plus every hash, and records distributed -execution as **not tested**. See the [contract, whitelist and evidence](docs/verl-bridge.md) -or the [community recipe registry](docs/community-benchmarks.md). +| Path | Start with | Concrete artifact | Next | +| --- | --- | --- | --- | +| **Align** — compare SFT, DPO, KD and OPD only when the pilot evidence supports the cost | `miniverl pilot recipes/alignment_policy_conditioned_qwen.yaml` | `alignment-card.json` | [Alignment Lab](docs/alignment-lab/alignment-lab-v1.md) | +| **Distill locally** — strict OPD, shared backbones and padded trajectory updates on one CUDA GPU | `miniverl train recipes/qwen_consumer_gpu_shared.yaml --dry-run` | `config.resolved.yaml` plus a revision-pinned PEFT adapter | [Bring your own GPU](docs/single-gpu-guide.md) | +| **Scale out** — import a documented profile, convert Parquet, export a bundle and run bridge checks | `miniverl bridge doctor scaleout-bundle` | `provenance/compatibility-report.json` | [Verified verl bridge](docs/verl-bridge.md) | -## Alignment Lab: when to turn OPD off +The bridge import is deliberately not generic YAML conversion. If dataset or +environment, teacher identity, objective, or schedule semantics are missing, +`import-verl` writes `import-report.json` and a non-executable +`imported.template.yaml` with `status: needs_user_input`. It never silently +substitutes calculator tasks or an unqualified same-base teacher. -> [!IMPORTANT] -> **The starting SFT policy already saturated this deterministic test.** Across -> three preregistered seeds, no continuation method improved its 100% alignment -> and 100% tool-utility result. The completed regressions from continued SFT, -> standard OPD and verifier-gated OPD remain in the headline result. +## One measured alignment result -All six methods start from the same checksummed Qwen3-0.6B SFT checkpoint and -use the same 48 ordered final-test tasks per seed. This is a small deterministic -tool-policy suite—not a broad safety benchmark—and the only measured GPU is an -RTX 4080. +Alignment Lab v1 is a **saturated tool-policy case study**, not a broad safety +benchmark. The shared SFT checkpoint already achieved 100% policy compliance +and 100% retained tool utility in all three seeds. No continuation method +improved it; continued SFT and both OPD variants retained measured regressions. -| method | alignment | tool utility | teacher query | continuation GPU time | +| continuation | alignment | tool utility | teacher queries | GPU time | | --- | ---: | ---: | ---: | ---: | -| SFT checkpoint | **100.0%** | **100.0%** | n/a | 0.0 s | -| continued SFT | 94.4% | 88.9% | n/a | 3.9 s | -| DPO | **100.0%** | **100.0%** | n/a | 8.6 s | -| offline soft distillation | **100.0%** | **100.0%** | 100.0% | 26.6 s | +| continued SFT | 94.4% | 88.9% | — | 3.9 s | +| DPO | 100.0% | 100.0% | — | 8.6 s | +| offline soft distillation | 100.0% | 100.0% | 100.0% | 26.6 s | | standard OPD | 98.6% | 97.2% | 100.0% | 76.7 s | | verifier-gated OPD | 97.9% | 95.8% | 46.8% | 66.0 s | -![Alignment quality versus utility retention](docs/alignment-lab/quality-vs-utility.svg) - -Harmful-compliance and over-refusal rates were both 0% for every method, yet -safe-error-recovery utility regressed in three completed arms. Those two safety -axes alone therefore missed a real policy-utility failure in this suite. The -matched State × Supervision diagnostic found only 0.0251% mean teacher -probability mass beyond argmax on fresh states; it is a signal diagnostic, not -a separately trained hard-target result, and no soft-target advantage is -claimed. - -`miniverl pilot` returns `insufficient_evidence` and tells this recipe not to -spend online teacher-query cost. That is the intended product behavior: OPD is -an option to justify with evidence, not a default replacement for SFT. - -```bash -miniverl pilot recipes/alignment_tool_policy_toy.yaml --json -miniverl align recipes/alignment_tool_policy_toy.yaml --dry-run -``` - -Read the [data-bound report](docs/alignment-lab/alignment-lab-v1.md), -[technical PDF](paper/alignment-lab-v1/alignment-lab-v1.pdf), -[public article](docs/alignment-lab/when-opd-should-follow-sft.md), -[demo script](docs/alignment-lab/demo.md), or the -[18 privacy-safe Alignment Cards](benchmarks/alignment-cards/alignment-lab-v1/sft_checkpoint-seed-1234.md). -The [preregistration](benchmarks/preregistration/alignment-lab-v1.yaml), -[machine-readable result](benchmarks/results/alignment-lab-v1.json) and all 864 -task-level records bind the claims above. - -## Consumer Runtime: batch speed without a cluster - -> A low-memory one-GPU runtime for actor rollout, teacher/reference scoring and -> online policy update. - -v0.4 keeps rollout, scoring and update in one readable process, but can now -pad multiple variable-length trajectories into one mask-isolated update -forward. A shared-backbone mode loads one quantized base with a trainable -student adapter, a frozen teacher adapter and an optional frozen reference -adapter. The default remains `dual_model` plus sequential physical batches for -backward compatibility. - -![Consumer-runtime throughput versus VRAM](docs/consumer-runtime-v1-pareto.svg) - -On the preregistered RTX 4080 systems workload, physical batch-4 improved -end-to-end throughput by 1.63× for dual models and 1.54× for the shared -backbone. At batch-4, sharing reduced peak reserved memory from 3.04 to 2.23 -GiB, while running 10.1% slower than dual ownership. `auto` was slower because -padding all eight trajectories was wasteful; it is a convenience, not a claim -that the largest batch is best. - -All eight cells reused identical trajectories and teacher targets. Twelve -preregistered loss/gradient/update comparisons passed; the largest loss -difference was 1.25e-6 and the largest updated-logit difference was 1.30e-4. -The benchmark uses NF4 weights with FP32 compute to keep that numerical gate -meaningful. It does not claim a quality gain, universal GPU speedup, batched -rollout server or distributed-runtime parity. - -Set `train.trajectory_batch_size` to `1`, an integer, or `auto`; choose -`models.runtime: shared_backbone` only when student, teacher and optional -reference use the same pinned base and distinct adapters. See the -[data-bound report](docs/consumer-runtime-v1.md), [preregistration](benchmarks/preregistration/consumer-runtime-v1.yaml) -and [frozen result](benchmarks/results/consumer-runtime-v1.json). - -## RecoveryBench: do fresh on-policy states justify their cost? - -> [!IMPORTANT] -> **Not in this measured setting.** Under eight equal continuation updates, -> frozen-student-state KD reached 23.2% strict success, while strict fresh-state -> OPD reached 10.9%. The paired fresh-minus-frozen difference was -12.24 -> percentage points (95% task-paired bootstrap interval -15.89 to -8.59). - -RecoveryBench is a preregistered mechanism study on SQLite tool-error recovery, -not an alignment benchmark. It isolates state freshness while holding the cold -checkpoint, qualified teacher, task schedule, optimizer and update count fixed. -All three seeds and all completed negative results are retained. - -| method | strict success | recovery after error | continuation time | -| --- | ---: | ---: | ---: | -| cold start | 10.7% | 13.6% | 0.2 s | -| continued oracle SFT | 4.9% | 1.8% | 51.3 s | -| oracle-state offline KD | **33.1%** | **31.9%** | 58.3 s | -| frozen-student-state KD | **23.2%** | **22.8%** | 52.1 s | -| strict fresh-state OPD | 10.9% | 9.1% | 686.8 s | -| budget-50 fresh-state OPD | 27.3% | 20.7% | 720.8 s | - -![RecoveryBench three-seed result](docs/recoverybench/recovery-success.svg) - -The equal-selected-position view reached the 6,224-position boundary after -eight updates for every core method, so its quality result matches the primary -view. The budget-50 selector queried 49.77% of model-generated positions but -did not reduce wall time because teacher backbone forwards were unchanged. The -50-second artifact is a **cycle-capped wall diagnostic, not exact equal-time -evidence**: SFT and frozen KD completed their eight-cycle ceiling, while fresh -OPD crossed the target in one indivisible 88-121 second update. - -Read the [full analysis](docs/recoverybench/recoverybench-v1.md), the -[data-bound technical report](paper/recoverybench-v1/recoverybench-v1.pdf), or -the [immutable schema-v3 artifacts](benchmarks/README.md#recoverybench-v1). -The result is scoped to one Qwen3 pair, one task family, three seeds and one RTX -4080. It does not show that OPD is universally ineffective or that offline KD -always wins. - -
-Case study: why teacher protocol qualification matters - -On the saturated v0.2 calculator task, a protocol-qualified OPD teacher reached -100% in both seeds and tied continued SFT, but took 6.1× as much continuation -time. Two protocol-naive controls completed normally at 0%; they were not -configuration failures. Both used the ambiguous historical protocol-v1 prompt, -so the failure cannot be attributed solely to intrinsic teacher behavior. - -![Two-seed protocol-teacher benchmark](docs/gpu-calc-hard-equal-update-v2.svg) - -| Artifact | Role | -| --- | --- | -| [Default recipe](recipes/qwen_consumer_gpu_calc.yaml) | protocol-qualified default | -| [Schema-v2 benchmark](benchmarks/results/gpu-calc-hard-equal-update-v2.json) | frozen five-arm result | -| [Raw-teacher recipe](recipes/qwen_consumer_gpu_calc_raw_teacher.yaml) | historical control; not default | - -The teacher gate and downstream comparison reused the same 24-task v0.2 test -set, so this is evidence for qualification in that setup, not a general OPD -advantage. The separate schema-v1 481-second smoke proves the pipeline, not OPD -over SFT. [Full diagnosis and caveats](docs/rtx4080-baselines.md). - -
- -## Local toy demo - -No network, no GPU, no downloads. Both models are small transformers built from -the config, the tokenizer is a reversible ~190-entry toy tokenizer, and the -calculator environment generates and grades its own tasks. +![Alignment and utility deltas from the saturated SFT checkpoint; small marks are all three seeds and large marks are means](docs/alignment-lab/delta-from-sft.svg) -```bash -python -m pip install ".[train]" # from the cloned repository; CPU torch is enough -miniverl doctor # what can this machine run? -miniverl demo --output runs/demo -``` +The two sandbox safety checks tied at zero while utility still regressed. +IFEval, XSTest, HarmBench and RewardBench were **not executed**. “Preference +win rate” is a deterministic Minipolicy paired outcome, not human preference. +Read the [study, seed-level values and limitations](docs/alignment-lab/alignment-lab-v1.md). -It runs the real pipeline — student rollouts, tool execution, teacher scoring of -exactly those states, a compressed top-k cache with provenance checks, and a -masked reverse-KL update on assistant tokens only — then prints where every -artifact landed and what to run next: - -```text -demo complete runs/demo - mode opd (genuine on-policy distillation) - optimizer steps 132 - parameter version 132 - rollout iterations 13 - wall clock 52.9 s - token provenance 45597 of 226383 tokens trainable (20%); 180786 are context - and can never be a target - teacher cache 735 scored positions, 131.6 KiB on disk, 2.0x smaller than - a dense fp16 dump - task success 0.0% -> 0.0% (greedy, held-out eval split) - -This demo proves the machinery, not capability. -At this size the toy student learns the tool-call format and not the -arithmetic, so 0% here is the expected outcome, not a failure. -For a CPU run that does learn (measured 0.0% -> 91.7% in 192 s): - miniverl train recipes/toy_cpu.yaml -``` +## One measured systems result -That last line is not a promise, it is a measurement: -`recipes/toy_cpu.yaml` takes **192 s on a CPU** and moves held-out greedy task -success from **0.0% to 91.7%** on 24 tasks, over 600 supervised cold-start steps -plus 40 on-policy distillation cycles. It is also **seed-sensitive** at this -model size: the same 600-step budget gives 81.2% with `run.seed: 1234` and 0.0% -with `run.seed: 20260727`. That variance is exactly why the toy backend is a -machinery harness and capability numbers come from the GPU recipe. - -`miniverl inspect` is the one worth running first. It prints the provenance -table, which is the whole point of the project: - -```text -tokens by span type (only assistant_* can enter the loss) -+---------------------------------------------+ -| span type | tokens | in loss | -|---------------------+--------+--------------| -| system | 776 | no (context) | -| tool_result | 685 | no (context) | -| user | 318 | no (context) | -| assistant_tool_call | 153 | yes | -| assistant_text | 85 | yes | -| assistant_final | 25 | yes | -+---------------------------------------------+ -``` +On one RTX 4080 with Qwen3-0.6B and eight fixed SQLite trajectories, physical +batch 4 improved update throughput from 2.369 to 3.866 trajectories/s in the +dual-model runtime. The shared-backbone batch-4 cell used 2.227 GiB peak +reserved memory versus 3.035 GiB for dual model, while running 10.1% slower. +All 12 preregistered equivalence comparisons passed. These are one-workload, +one-machine measurements, not promises for other GPUs. -The toy backend is a **machinery harness, not a capability demonstration**. Its -models are too small to solve anything beyond the `easy` split. Capability -numbers come from the GPU recipe. +![Measured throughput and reserved VRAM for dual-model and shared-backbone runtime cells](docs/consumer-runtime-v1-pareto.svg) -## Single-GPU quickstart +[Consumer Runtime v1 methods and caveats](docs/consumer-runtime-v1.md) -The default recipe uses `device: auto` and `dtype: auto`: bf16-capable cards use -bf16, while older CUDA cards such as Titan V use fp16. RTX 3070, Titan V, -RTX 4080 and RTX 5090-class cards all enter the same code path; only the -RTX 4080 result is measured here. Exact fit is governed by VRAM, model sizes, -drivers and token budgets, not the card's marketing name. See the -[`single-GPU guide`](docs/single-gpu-guide.md) before changing the recipe. +## Compatibility boundary -```bash -git clone https://github.com/DaoyuanLi2816/mini-verl.git -cd mini-verl -python -m pip install torch --index-url https://download.pytorch.org/whl/cu130 -python -m pip install ".[train,cuda]" - -miniverl doctor # confirms CUDA + bitsandbytes -miniverl validate recipes/qwen_consumer_gpu_calc.yaml -miniverl train recipes/qwen_consumer_gpu_calc.yaml --dry-run # nothing is downloaded -miniverl train recipes/qwen_consumer_gpu_calc.yaml -miniverl report runs/ --out runs//report.html -``` - -The recipe pins both revisions: - -| role | model | revision | license | -| --- | --- | --- | --- | -| student | `Qwen/Qwen3-0.6B` | `c1899de289a04d12100db370d81485cdf75e47ca` | Apache-2.0 | -| teacher | `Qwen/Qwen3-1.7B` | `70d244cc86ccca08cf5af4e1e306ecf908b1ad5e` | Apache-2.0 | - -Their `tokenizer.json` files are byte-identical -(`sha256 aeb13307a71acd8fe81861d94ad54ab689df773318809eed3cbe794b4492dae4`). -New runs compare structural identity; old artifacts use the legacy fixed-probe -behavioural fingerprint. -The recipe also pins the [protocol-teacher adapter](https://huggingface.co/DaoyuanLi/mini-verl-qwen3-1.7b-protocol-teacher) -at revision `23323751318135484c06c043b1f9b9e7016dd89f` and requires its recorded -strict policy success to be at least 50% before allocating the teacher. - -## Architecture - -```mermaid -flowchart LR - A["student pi_theta
QLoRA, resident"] -->|sample| B["RolloutRunner
agent/loop.py"] - B -->|tool call| C["ToolEnvironment
calculator / jsonnav / sqlite"] - C -->|observation| B - B -->|typed token spans| D["Trajectory
schemas/trajectory.py"] - D -->|select_positions| E["AlignmentMap
trajectory/alignment.py"] - E -->|score those exact states| F["LocalTeacherScorer
teachers/local.py"] - F -->|top-k + tail| G["TeacherCache
cache/store.py"] - F --> H["chunked_selected_position_loss
losses/chunked.py"] - G --> H - H -->|masked KL on assistant tokens| A -``` +![Verified local runtime, portable artifact bundle and pinned upstream smoke; distributed verl execution remains untested](docs/verl-bridge-architecture.svg) -Layer boundaries are strict, and the first layer never imports torch: - -1. `schemas/`, `trajectory/`, `config/`, `agent/protocol.py` — pure data, masks, - validation. -2. `losses/` — torch numerics, no model knowledge. -3. `models/` — backends and the architecture adapter. -4. `environments/`, `agent/` — task and tool semantics. -5. `training/`, `teachers/`, `cache/`, `selection/` — orchestration. -6. `evaluation/`, `reporting/` — measurement. -7. `cli.py` — a thin shell that calls one library function per command. - -See [`docs/design.md`](docs/design.md). - -## Exact versus top-k + tail - -Two clearly named classes of objective, because conflating them is how -distillation results become unreproducible. - -**`exact_full_vocab`** materializes the complete `[chunk, V]` teacher and student -distributions and computes the real divergence. Affordable when `V` is small (the -toy backend) or when the teacher stays resident and the distribution is rebuilt -one chunk at a time. Guarded by `loss.exact_max_vocab` (default 8192) so it can -never silently try to persist a `[positions, 152k]` tensor. - -**`bucketed_topk_tail`** coarse-grains the vocabulary into the teacher's top-k -tokens plus one aggregate tail bucket, then computes the divergence between the -two `K+1` category distributions. This is **not** full-vocabulary KL. The -data-processing lower-bound theorem applies to the unsmoothed coarse-graining; -the finite implementation floors and renormalizes non-empty tails, so it is -described as an epsilon-smoothed objective rather than claiming the theorem -literally for every input. When `k == V`, the empty tail bypasses smoothing and -the implementation reproduces the exact objective to `1e-9` in float64 tests. -The functions are named `bucketed_forward_kl`, `bucketed_reverse_kl` and -`bucketed_jsd` so that no call site can pretend otherwise. - -What the compression actually buys is teacher-side storage and the ability to -evict the teacher from VRAM. It does **not** proportionally reduce teacher FLOPs: -the teacher still runs a full forward pass to produce the hidden states. Reports -therefore say `teacher_queried_position_ratio`, never "teacher compute saved". - -Top-k + tail targets are not a new idea — TRL's `ServerDistillationTrainer` has -`loss_top_k` with an optional tail bucket. See [`docs/math.md`](docs/math.md). - -## Tool-token masking - -Every trajectory is a flat token sequence plus a partition into typed spans. The -three masks are stored *and* re-derived from the spans on every read; a file -whose mask disagrees with its spans is rejected rather than trained on. - -```python -from miniverl.trajectory.io import read_trajectories - -traj = read_trajectories("runs/demo/trajectories.jsonl")[0] -print(traj.token_counts_by_span_type()) -# {'system': 194, 'user': 40, 'assistant_tool_call': 38, 'tool_result': 34, 'assistant_final': 7} -print(sum(traj.model_generated_mask)) # only assistant_* tokens are trainable -``` +The bridge targets official verl `v0.8.0` at commit `7aed6b23` and uses the +term **miniVERL-defined compatibility Level 3**. That means a checksummed +standard-artifact bundle plus pinned upstream config-parse/model-data-load +smoke—not arbitrary verl YAML or a completed distributed job. -Context segments own the trailing `<|im_start|>assistant\n` header, so a model -span begins at exactly the first sampled token and no forced scaffolding token is -ever a target. Position `0` can never be a target. Both are enforced, not -documented — see `tests/unit/test_token_provenance.py`. - -## Benchmark results - -Every number below was produced by the commands in -[`docs/benchmarking.md`](docs/benchmarking.md) on the hardware recorded in each -result file. Nothing is estimated or extrapolated. - -* **RTX 4080, real models** — [`docs/rtx4080-baselines.md`](docs/rtx4080-baselines.md) - has measured peak VRAM, decode throughput, the full-recipe run, the two-seed - schema-v2 protocol-teacher comparison, and the preserved legacy comparison. -* **CPU, toy models** — `recipes/toy_cpu.yaml` moves task success from 0.0% to - 91.7% in 192 s, and `benchmarks/results/` holds the legacy equal-update parity - run. - The parity run's accuracy differences are **within noise**; it exists to show - that all seven arms run to completion under identical budgets, not to rank - them. See [`benchmarks/README.md`](benchmarks/README.md) for why the toy - backend cannot rank methods. - -## Installation - -| Layer | Install | What you get | -| --- | --- | --- | -| Core | `python -m pip install .` | `doctor`, `validate`, `inspect`, `report`, `cache`, the schemas and the Python API. No torch. | -| Training | `python -m pip install ".[train]"` | `demo`, `train`, `eval`, `benchmark`. Adds torch, transformers, peft, accelerate. | -| 4-bit | `python -m pip install ".[cuda]"` | bitsandbytes, for NF4 QLoRA and the 8-bit optimizer. | -| Development | `python -m pip install ".[dev]"` | pytest, hypothesis, ruff, mypy, build, twine. | - -The published-package equivalents are `miniverl`, `miniverl[train]`, -`miniverl[cuda]` and `miniverl[dev]`. Core Python 3.10–3.13 is tested without -torch. The full CPU ML suite and Transformers 4.51.x/5.x compatibility rows run -on Python 3.12; GPU paths are opt-in and were measured locally on Python 3.12. - -Install the CUDA build of torch that matches your driver separately; the PyPI -wheel is CPU-only on some platforms: +Current exported bundles are intentionally `launchable: false`: the base +snapshot is absent, the reward implementation fails closed, and required user +mappings remain placeholders. The generated entry point is therefore +`launch.template.sh`. Readiness is reported as separate facts for artifact +completeness, parse/load smoke, reward completeness, launchability, +distributed execution and algorithm-semantic parity. The target is a +PPO/reward scaffold, not an executable continuation of miniVERL OPD semantics. -```bash -pip install torch --index-url https://download.pytorch.org/whl/cu130 -``` +## Detailed studies and preserved negative evidence -A missing extra never produces a traceback: +- [RecoveryBench v1](docs/recoverybench/recoverybench-v1.md): frozen-student KD + outperformed much slower fresh-state OPD on the preregistered primary view; + the verifier gate remained `insufficient_evidence`. +- [Alignment Lab v1](docs/alignment-lab/alignment-lab-v1.md): the starting SFT + checkpoint was at the ceiling, so no positive OPD result is claimed. +- [Calculator benchmark](docs/benchmarking.md): both negative controls completed + normally and measured 0% strict success. They were not configuration + failures. Because they used the historical ambiguous protocol-v1 prompt, + their failure cannot be attributed solely to intrinsic teacher behavior. +- [Consumer Runtime v1](docs/consumer-runtime-v1.md): padded update batches and + shared adapters preserve the measured one-update objective within declared + tolerances; rollout generation remains sequential. +- [Limitations](docs/limitations.md), [math](docs/math.md), + [reproducibility](docs/reproducibility.md) and + [compatibility policy](docs/compatibility.md). -```text -$ miniverl demo --output runs/demo -error miniverl demo requires the optional dependency 'torch', which is not installed. -hint pip install "miniverl[train]" -``` +New runs establish tokenizer compatibility through structural identity. The +legacy behavioral fingerprint—token IDs for one fixed probe plus metadata—is +only a migration fallback for older artifacts and is not an identity proof. -### Strict offline execution +## Scope -All model-loading commands use the same no-network contract: +miniVERL supports one local CUDA process. It does not implement or wrap Ray, +FSDP, Megatron, PPO, GRPO or a distributed launcher. The public studies cover +small Qwen3 models, deterministic tool environments and one RTX 4080; they do +not establish cross-model, cross-task, cross-GPU or broad safety generality. ```bash -miniverl train --offline -miniverl benchmark --offline -miniverl eval --run --offline -miniverl export-adapter --run --out --offline -``` - -In this mode, the base model, tokenizer and every adapter file must already be -at a local path or in the Hugging Face cache. miniVERL permits no HTTP, -metadata, ETag or Hub API request and does not fall back to online resolution. -A Hub teacher adapter is resolved once at its pinned revision; PEFT then loads -the exact local snapshot whose config, weights, manifest and checksums were -validated. A cache miss prints the immutable identity and the exact `hf -download` preload command. - -## Python API - -The public surface is deliberately small. - -```python -from miniverl.config import RunConfig -from miniverl.trainer import OPDTrainer - -config = RunConfig.from_yaml("recipes/toy_cpu.yaml") -with OPDTrainer.from_config(config) as trainer: - result = trainer.train() - -print(result.run_dir, result.global_step, result.eval["success_rate"]) -``` - -## A custom environment - -Subclass `ToolEnvironment`, register it, and every recipe key works unchanged. -`examples/custom_environment/` is a complete, runnable example. -`reset(task)` is authoritative: it is called exactly once per episode, and its -`Observation.text` plus `state_id` enter the trajectory. `user_prompt(task)` is -only a compatibility helper; the runner does not call it a second time. - -```python -from miniverl.environments import ToolEnvironment, ToolSpec -from miniverl.environments.registry import register - - -@register -class ReverseEnvironment(ToolEnvironment): - name = "reverse" - - def tool_specs(self) -> list[ToolSpec]: - return [ - ToolSpec( - name="reverse", - description="Reverse a string.", - parameters={"text": "string to reverse"}, - required=("text",), - example={"text": "abc"}, - ) - ] - - # reset / step / verify / generate_task / oracle_actions follow; see the example. -``` - -## A custom teacher - -Implement `TeacherScorer.score` and return supervision for the aligned positions. -`examples/custom_teacher/` shows a scorer that sharpens a local model's -distribution before handing it over, and asserts that the result still trains. - -For a standard frozen PEFT teacher adapter, including the Qwen3 protocol-SFT -recipe, export command, compatibility checks and policy-competence gate, see -[`docs/teacher-adapters.md`](docs/teacher-adapters.md). - -## Limitations - -The short version; the full list is in [`docs/limitations.md`](docs/limitations.md). - -* Same tokenizer only. Cross-tokenizer distillation is rejected with an error. -* Rollout decoding is one sequence at a time. The update path supports padded - physical batches; `gradient_accumulation_steps` is the optimizer-group size - and `trajectory_batch_size` is the number sharing one backbone forward. -* `swap` is unavailable for quantized models, because bitsandbytes parameters are - pinned to the device they were quantized on. -* Only Qwen3 and Qwen2 architectures are tested. Others may work through the - architecture adapter; nothing here claims they do. -* RecoveryBench has three prespecified student seeds; the calculator case study - has two, and older GPU artifacts are single-seed. No broad statistical - significance or cross-task generalization is claimed. -* On the measured machine, decoding is kernel-launch bound rather than compute - bound, so throughput figures are platform-specific. - -## Reproducibility - -Every run writes `manifest.json` with the miniVERL version, git commit, Python -and OS, torch/CUDA/driver versions, GPU model and VRAM, model ids **and resolved -revisions**, tokenizer fingerprint, seeds, precision, quantization, memory -strategy, loss mode, top-k, policy version, and a `measurement_status` block -recording whether each result was measured, simulated or not run. It records no -usernames, hostnames, home paths, or environment variables beyond a short -allowlist of ones that change numerics — asserted by a test. -File-backed runs also separate exact submitted bytes, canonical validated -logic, the v0.2 resume compatibility layer, and runtime-resolved choices. - -Writable runs move atomically through `ready`, `running`, and one terminal -status (`completed`, `failed`, `interrupted`, or `closed_before_training`). -One process lock covers construction, training/resume, standalone checkpoint -selection and evaluation, and automatic report generation. Within one trainer, -training, evaluation, checkpoint save/load and destructive close are mutually -exclusive; load is READY-only, close mutates nothing unless it obtains -ownership, and evaluation restores the exact prior model mode even on failure. - -After `reset`, every built-in verifier maps arbitrary strings to a bounded -result rather than leaking parser/numeric exceptions; protocol-v2 prompts use -environment-specific, verifier-format-valid final examples. Shareable reports, -summaries, benchmark exports and portable manifests redact semantic secret -keys, URL credentials and private cross-platform paths; private run artifacts -still retain the local state required for exact resume. Redaction is a -best-effort sharing defense, not permission to place real credentials in any -config, run artifact or report. - -See [`docs/reproducibility.md`](docs/reproducibility.md) and the concise -[`compatibility policy`](docs/compatibility.md). - -## Roadmap - -Not implemented, not promised, listed so the scope is unambiguous: -cross-tokenizer distillation, batched or engine-backed rollout decoding, -entropy-aware divergence mixing (arXiv:2603.07079), additional model families, -more environments, and native multi-GPU execution. The Level-3 bridge exports -one documented profile to pinned verl; it does not execute that distributed job. - -## Acknowledgement and disclaimer - -> miniVERL is an independent project and is not affiliated with or endorsed by -> the verl project, ByteDance, or Volcano Engine. It is not a drop-in -> replacement for verl. - -The name is a nod to the problem space, not a claim of generic compatibility. -The verified bridge is deliberately limited to one pinned profile. verl is an -excellent, much larger system for cluster-scale execution with Ray. miniVERL -exists for the case where you have one personal GPU and want to read every line -of what is happening, then hand standard artifacts to verl for scale-out. That -can be an older 12 GiB card or a current high-end card; the repository claims -measured performance only for hardware it actually ran. See -[`docs/comparisons.md`](docs/comparisons.md). - -## Citation - -```bibtex -@software{miniverl2026, - title = {miniVERL: Auditable online post-training on one GPU}, - author = {Li, Daoyuan}, - year = {2026}, - url = {https://github.com/DaoyuanLi2816/mini-verl}, - license = {Apache-2.0} -} +git clone https://github.com/DaoyuanLi2816/mini-verl.git +cd mini-verl +python -m pip install -e ".[dev]" +pytest -q -m "not gpu and not network" ``` -See [CITATION.cff](CITATION.cff) and [CHANGELOG.md](CHANGELOG.md). -Contributions: [CONTRIBUTING.md](CONTRIBUTING.md). Security: -[SECURITY.md](SECURITY.md). - -## License - -Apache-2.0. See [LICENSE](LICENSE) and -[THIRD_PARTY_NOTICES.md](THIRD_PARTY_NOTICES.md). - -Chinese translation: [README.zh-CN.md](README.zh-CN.md). +Apache-2.0 licensed. See [CONTRIBUTING.md](CONTRIBUTING.md) and +[SECURITY.md](SECURITY.md). Project records: [default GPU recipe](recipes/qwen_consumer_gpu_calc.yaml), +[frozen calculator JSON](benchmarks/results/gpu-calc-hard-equal-update-v2.json), +[changelog](CHANGELOG.md), [citation](CITATION.cff) and [license](LICENSE). diff --git a/README.zh-CN.md b/README.zh-CN.md index 00278c8..5e4cd69 100644 --- a/README.zh-CN.md +++ b/README.zh-CN.md @@ -1,5 +1,5 @@

- miniVERL — 单卡上的在线对齐与蒸馏 + miniVERL — 单卡 LLM 后训练

@@ -13,449 +13,130 @@

- PyPI 软件包 · - 文档站 · - 安装与训练 · - 经验证的 verl 桥接 · - Alignment Lab + PyPI · + 稳定版文档 · + 开发版文档 · + English

-> 本文是 [README.md](README.md) 的中文翻译。英文版为准；若两者不一致，请以英文版为准并提交 issue。 +**miniVERL 是一个本地、可检查的单卡 LLM 对齐与蒸馏运行时，只实现有明确 +文档的功能子集。** 它显式保存 rollout 来源、仅 assistant token 的 loss +掩码、教师目标、更新预算与运行产物，并通过 fail-closed 桥接把可移植产物 +交给一个锁定的上游 verl 配置。 -**面向 verl 风格在线后训练已文档化子集的单卡原型运行时。** +PyPI `v0.6.0` 是稳定版；`main` 是开发版。CUDA 路径没有显卡型号白名单， +但能否运行取决于模型组合、上下文预算、内核和显存。miniVERL 独立于 verl， +不声称已经验证分布式执行或完整算法兼容性。 -在本地开发、诊断和验证对齐或蒸馏配方，再把标准模型、数据集、配方和来源 -产物导出到锁定版本的 verl，用于扩展执行。 - -**先同时测量对齐、过度拒绝、保留效用与成本，再选择 SFT、DPO 或 OPD。** - -PyPI `v0.6.0` 是稳定发布版；`main` 是开发分支，可能领先于稳定版。 - -miniVERL 独立于 verl，并针对官方 verl `v0.8.0` 实现了一个经验证的 Level-3 -配置 `single-gpu-online-distillation-v1`。它是标准产物与配置子集桥接，不是 -通用 YAML 兼容层，也不声称已经测试分布式执行。CUDA 路径没有显卡型号 -白名单；是否装得下仍由模型、序列预算和可用显存决定。 - -![miniVERL 单卡工作流与锁定 verl 版本的扩展桥接](docs/verl-bridge-architecture.svg) +## 安装与 60 秒演示 ```bash -python -m pip install miniverl # 轻量核心层 +python -m pip install "miniverl[train]" miniverl doctor -python -m pip install "miniverl[train]" # 添加本地训练依赖 -miniverl demo --output runs/demo # 无需联网、无需 GPU，笔记本 CPU 上约 50 秒 +miniverl demo --output runs/demo +miniverl inspect runs/demo ``` -基础安装是不含 torch 的核心层（`doctor`、`validate`、`inspect`、`report`、 -schema 与 Python API）。`train` extra 会添加 torch、Transformers 与 PEFT， -因为 `demo` 会执行真实优化。 -这种拆分是有意的：`pip install miniverl` 可以在不下载数 GB 机器学习依赖的 -情况下检查和验证产物；要进行训练或评估，请安装 -`pip install "miniverl[train]"`。 - -**它让四件事可以被检查** - -- **策略真实：** 每个 OPD 批次都来自它要更新的策略版本；过期教师目标会被拒绝。 -- **Token 真实：** 工具输出只作为上下文，只有带类型的 assistant span 能进入 loss。 -- **预算真实：** 精确全词表目标与压缩的 `top-k + tail` 目标分开命名、分开报告。 -- **决策真实：** 每张 Alignment Card 同时报告策略质量、保留效用、教师查询比例、 - GPU 时间、显存与局限。 - -[运行本地 demo](#本地玩具演示) · -[在你的 GPU 上训练](#个人单卡快速上手) · -[查看实测结果](#alignment-lab什么时候应该关闭-opd) · -[阅读数学说明](docs/math.md) - -## 为什么需要 miniVERL - -在线策略蒸馏在概念上很简单，工程上很容易出错：学生采样一条轨迹，教师给**学生真实走过的那些状态**打分，然后只在学生自己生成的 token 上做分布级更新。实践中有四类错误，而且它们都是**静默**的： - -1. **把工具输出当成了标签。** 环境返回的内容是上下文，不是监督目标。掩码错一次，模型就会学会凭空编造工具结果。 -2. **差一位（off-by-one）。** 预测第 `j` 个 token 的分布位于位置 `j - 1`。搞错了，loss 照样下降。 -3. **其实并不是 on-policy。** 跨策略版本复用教师缓存，做的就是离线 KD，却仍叫它 OPD。 -4. **显存放不下 logits。** `[batch, seq_len, 152k]` 的张量在消费级显卡上放不下，于是真正有意思的配置恰好都跑不了。 - -miniVERL 把上面每一条都变成**被代码检查的性质**，而不是注释里的一句承诺，并把整个生命周期放进一个可读的单卡进程。 - -## 已实现的能力 - -| 能力 | 状态 | -| --- | --- | -| 学生自采样的多轮 rollout，带真实工具执行 | 支持 | -| 严格的逐 token 来源标注（`system` / `user` / `assistant_*` / `tool_result`） | 支持，读写时都会校验 | -| 精确全词表 forward KL、reverse KL、beta-JSD | 支持，与暴力参考实现逐项比对 | -| 压缩的 `top-k + tail` KL / JSD | 支持；未平滑粗粒化与精确散度的下界关系有严格证明 | -| 特权上下文教师模式，带显式对齐表 | 支持 | -| 标准冻结 PEFT 教师适配器，带来源记录与能力门禁 | 支持 | -| 自动选择 bf16/fp16 的单卡 CUDA 路径 | 支持；CUDA 路径不绑定设备名称，实测参考为 RTX 4080 | -| padding 多轨迹更新 | 支持；注意力隔离、长度分桶、逐轨迹归一化；默认仍为顺序执行 | -| 共享主干的学生 / 教师 / 可选参考适配器 | 支持；单一 HF 主干、类型化角色、优化器仅持有学生参数 | -| `resident` / `swap` 显存策略与 `auto` 解析 | 支持，并有等价性测试 | -| 带版本号与校验和、完全不用 pickle 的教师目标缓存 | 支持 | -| SFT / 离线 KD / 严格 OPD / 显式标注 replay 统一在一个 trainer 中 | 支持 | -| `align` / `pilot`、策略上下文教师、对齐适配器教师与 DPO 来历记录 | 支持 | -| 版本化 verifier gate、AlignmentBench 元数据适配器与隐私安全 Alignment Card | 支持；v0.5 实测中外部基准仅有元数据，未执行 | -| 计算器、JSON 导航、SQLite 三个环境 | 支持，确定性生成 + 精确判分 | -| 精确的断点续训 | 支持，逐参数断言 | -| 完全自包含、可离线打开的 HTML 报告 | 支持 | -| 锁定 verl `v0.8.0` 的 Level-3 桥接 | 支持；一个 fail-closed 配置、Parquet 双向转换、标准 PEFT/safetensors 导出 | -| 原生 Ray/FSDP/Megatron/vLLM 执行、VLM、跨词表、PPO/GRPO | **不支持**，见[局限](docs/limitations.md) | - -## 经验证的 verl 桥接 - -桥接层只导入一个锁定配置中的 14 个命名字段，双向转换官方 prompt Parquet -schema，导出可自检的扩展 bundle，并明确区分 miniVERL 教师目标与 PPO -reference log-probabilities。 +这个确定性演示无需网络或 GPU，会执行一次真实的玩具优化；在实测笔记本 +CPU 上约需 50 秒。若只需要 schema、检查与报告，可安装 +`pip install miniverl`。CUDA 训练请先安装与本机匹配的 CUDA PyTorch wheel， +再安装 `miniverl[train,cuda]`；这个 extra 本身不会选择 CUDA 版 PyTorch。 +详见[单卡 GPU 指南](docs/single-gpu-guide.md)。 -```bash -python -m pip install "miniverl[bridge]" -miniverl import-verl verl.yaml --profile single-gpu-online-distillation-v1 \ - --target-verl v0.8.0 --out recipes/imported.yaml -miniverl convert-dataset --from verl-parquet train.parquet --out local.parquet -miniverl export-verl --run runs/my-alignment --target-verl v0.8.0 \ - --out exports/my-alignment-verl -miniverl bridge doctor exports/my-alignment-verl --json -``` +## 三条使用路径 -发布烟测锁定 commit `7aed6b230776f963fa09509c10d9c3a767d1102c`，会解析生成的 -OmegaConf 配置，加载标准 PEFT/safetensors 和两份 Parquet，导入 reward -脚手架，并验证隐私与全部哈希；分布式执行明确记录为**未测试**。详见 -[契约、白名单与证据](docs/verl-bridge.md)和[社区配方注册表](docs/community-benchmarks.md)。 +| 路径 | 起点 | 真实产物 | 下一步 | +| --- | --- | --- | --- | +| **Align** — 只有 pilot 证据支持成本时，才在 SFT、DPO、KD 与 OPD 间选择 | `miniverl pilot recipes/alignment_policy_conditioned_qwen.yaml` | `alignment-card.json` | [Alignment Lab](docs/alignment-lab/alignment-lab-v1.md) | +| **Distill locally** — 在一张 CUDA GPU 上运行严格 OPD、共享 backbone 与 padded trajectory update | `miniverl train recipes/qwen_consumer_gpu_shared.yaml --dry-run` | `config.resolved.yaml` 与锁定 revision 的 PEFT adapter | [使用自己的 GPU](docs/single-gpu-guide.md) | +| **Scale out** — 导入已文档化 profile、转换 Parquet、导出 bundle 并执行 bridge 检查 | `miniverl bridge doctor scaleout-bundle` | `provenance/compatibility-report.json` | [verl 桥接](docs/verl-bridge.md) | -## Alignment Lab：什么时候应该关闭 OPD +桥接导入不是通用 YAML 转换。当数据集/环境、教师身份、目标函数或 schedule +语义不完整时，`import-verl` 只会写出 `import-report.json` 和不可执行的 +`imported.template.yaml`，状态为 `needs_user_input`。它不会悄悄改用 +calculator 环境，也不会创建身份不明确的同基座教师。 -> [!IMPORTANT] -> **共同的 SFT 起点已经在这组确定性测试上饱和。** 三个预注册种子中， -> 没有任何继续训练方法超过它的 100% 对齐与 100% 工具效用。继续 SFT、 -> 标准 OPD 和 verifier-gated OPD 已完成的回归均保留在主结果中。 +## 一项对齐实测结果 -六种方法都从同一个经过校验和绑定的 Qwen3-0.6B SFT 检查点出发；每个种子 -使用同一组按序排列的 48 道最终测试任务。这是一组小型、确定性的工具策略 -测试，不是广义安全基准；唯一实测 GPU 是 RTX 4080。 +Alignment Lab v1 是一个**已饱和的工具策略案例研究**，不是广义安全评测。 +共同的 SFT 起点在三个 seed 上都已经达到 100% 策略合规和 100% 工具效用。 +没有 continuation 方法能够继续提升；continued SFT 与两种 OPD 的实测退化 +均被保留。 -| 方法 | 对齐 | 工具效用 | 教师查询 | 继续训练 GPU 时间 | +| continuation | 对齐 | 工具效用 | 教师查询 | GPU 时间 | | --- | ---: | ---: | ---: | ---: | -| SFT 检查点 | **100.0%** | **100.0%** | 不适用 | 0.0 秒 | -| 继续 SFT | 94.4% | 88.9% | 不适用 | 3.9 秒 | -| DPO | **100.0%** | **100.0%** | 不适用 | 8.6 秒 | -| 离线软目标蒸馏 | **100.0%** | **100.0%** | 100.0% | 26.6 秒 | -| 标准 OPD | 98.6% | 97.2% | 100.0% | 76.7 秒 | -| verifier-gated OPD | 97.9% | 95.8% | 46.8% | 66.0 秒 | - -![对齐质量与工具效用保留](docs/alignment-lab/quality-vs-utility.svg) +| continued SFT | 94.4% | 88.9% | — | 3.9 s | +| DPO | 100.0% | 100.0% | — | 8.6 s | +| offline soft distillation | 100.0% | 100.0% | 100.0% | 26.6 s | +| standard OPD | 98.6% | 97.2% | 100.0% | 76.7 s | +| verifier-gated OPD | 97.9% | 95.8% | 46.8% | 66.0 s | -所有方法的 harmful-compliance 与 over-refusal 都是 0%，但三条已完成实验臂 -仍在安全错误恢复任务上回归，说明只看这两个安全轴会漏掉真实的策略效用 -问题。配对的 State × Supervision 诊断发现，新鲜状态软目标中超出 argmax 的 -平均概率质量仅为 0.0251%；它是信号诊断，不是另一条训练完成的硬目标实验臂， -因此不声称软目标具有质量优势。 +![相对已饱和 SFT 起点的对齐与效用变化；小标记为三个 seed，大标记为均值](docs/alignment-lab/delta-from-sft.svg) -`miniverl pilot` 返回 `insufficient_evidence`，并建议这份配方不要继续支付在线 -教师查询成本。这正是产品预期：OPD 是需要证据支持的选择，不是 SFT 的默认 -替代品。 - -```bash -miniverl pilot recipes/alignment_tool_policy_toy.yaml --json -miniverl align recipes/alignment_tool_policy_toy.yaml --dry-run -``` +两个 sandbox 安全检查都为零，但工具效用仍然退化。IFEval、XSTest、 +HarmBench 与 RewardBench **没有实际执行**。“preference win rate” 是确定性 +Minipolicy 配对结果，不是人类偏好。详见[完整研究、逐 seed 数值和局限](docs/alignment-lab/alignment-lab-v1.md)。 -可阅读[数据绑定报告](docs/alignment-lab/alignment-lab-v1.md)、 -[技术 PDF](paper/alignment-lab-v1/alignment-lab-v1.pdf)、 -[公开文章](docs/alignment-lab/when-opd-should-follow-sft.md)、 -[演示脚本](docs/alignment-lab/demo.md)与 -[18 张隐私安全 Alignment Card](benchmarks/alignment-cards/alignment-lab-v1/sft_checkpoint-seed-1234.md)。 -[预注册](benchmarks/preregistration/alignment-lab-v1.yaml)、 -[机器可读结果](benchmarks/results/alignment-lab-v1.json)和 864 条任务级记录共同 -绑定以上结论。 +## 一项系统实测结果 -## Consumer Runtime：无需集群的批处理提速 +在一张 RTX 4080、Qwen3-0.6B 和八条固定 SQLite trajectory 上，物理 batch 4 +把 dual-model runtime 的更新吞吐从 2.369 提高到 3.866 trajectories/s。 +shared-backbone 的 batch-4 cell 峰值 reserved memory 为 2.227 GiB，dual +model 为 3.035 GiB，但前者慢 10.1%。全部 12 个预注册等价性比较通过。 +这些是单任务、单机器结果，不是对其他 GPU 的保证。 -> 面向 actor rollout、教师/参考策略打分与在线策略更新的低显存单卡运行时。 +![dual-model 与 shared-backbone runtime 的实测吞吐和 reserved VRAM](docs/consumer-runtime-v1-pareto.svg) -v0.4 仍把 rollout、打分和更新放在一个可读进程中，但更新阶段可以把多条变长 -轨迹 padding 后送入一次注意力隔离的前向。共享主干模式只加载一个量化 base， -其上挂载可训练学生适配器、冻结教师适配器和可选冻结参考适配器。为保持兼容， -默认仍是 `dual_model` 加顺序物理 batch。 +[Consumer Runtime v1 方法与局限](docs/consumer-runtime-v1.md) -![Consumer Runtime 吞吐与显存](docs/consumer-runtime-v1-pareto.svg) +## 兼容性边界 -在预注册的 RTX 4080 系统工作负载上，物理 batch-4 使 dual 模式端到端吞吐提升 -1.63 倍，使共享主干模式提升 1.54 倍。batch-4 下，共享把峰值 reserved 显存从 -3.04 GiB 降到 2.23 GiB，但速度比 dual 慢 10.1%。`auto` 因为把八条长度不同的 -轨迹全部 padding，反而更慢；它只是便利选项，不保证最大 batch 最快。 +![已验证的本地 runtime、可移植产物 bundle 与上游 smoke；分布式 verl 执行未测试](docs/verl-bridge-architecture.svg) -八个单元使用完全相同的轨迹和教师目标；12 项预注册的 loss、梯度与更新后 -logits 比较全部通过。最大 loss 差为 1.25e-6，最大更新后 logits 差为 1.30e-4。 -本 benchmark 使用 NF4 权重和 FP32 计算以保留严格数值门禁。它不声称提升任务 -质量、普遍加速所有 GPU，也不声称已实现批量 rollout server 或分布式运行时。 +桥接锁定官方 verl `v0.8.0`、commit `7aed6b23`，并使用 +**miniVERL-defined compatibility Level 3** 这一名称。它表示 checksummed +标准产物 bundle 与锁定上游版本的 config-parse/model-data-load smoke， +不表示任意 verl YAML 都兼容，也不表示完成过分布式任务。 -`train.trajectory_batch_size` 可设为 `1`、整数或 `auto`。只有当学生、教师和可选 -参考策略使用同一个锁定 revision 的 base 与不同适配器时，才应选择 -`models.runtime: shared_backbone`。详见[数据绑定报告](docs/consumer-runtime-v1.md)、 -[预注册](benchmarks/preregistration/consumer-runtime-v1.yaml)和 -[冻结结果](benchmarks/results/consumer-runtime-v1.json)。 +当前导出的 bundle 有意报告 `launchable: false`：base snapshot 不在 bundle +中，reward 实现仍 fail closed，而且必要的用户映射仍是 placeholder。因此 +入口名为 `launch.template.sh`。报告会分别给出 artifact 完整性、parse/load +smoke、reward 完整性、launchability、分布式执行和算法语义等价状态。 +当前目标是 PPO/reward scaffold，不是 miniVERL OPD 的可执行延续。 -## RecoveryBench：新鲜在线状态是否值得额外成本？ +## 详细研究与保留的负结果 -> [!IMPORTANT] -> **在本次实测设置中，不值得。** 八个相同继续训练步下，冻结学生状态 KD 的 -> 严格成功率是 23.2%，严格新鲜状态 OPD 是 10.9%。按任务配对的 -> “新鲜减冻结”差值为 -12.24 个百分点（95% 配对 bootstrap 区间 -> -15.89 到 -8.59）。 +- [RecoveryBench v1](docs/recoverybench/recoverybench-v1.md)：在预注册主视图中， + frozen-student KD 优于耗时高得多的 fresh-state OPD；verifier gate 仍为 + `insufficient_evidence`。 +- [Alignment Lab v1](docs/alignment-lab/alignment-lab-v1.md)：起始 SFT 已到 + ceiling，因此不宣称任何正向 OPD 结果。 +- [Calculator benchmark](docs/benchmarking.md)：两个 negative control 都正常 + 完成并测得 0% strict success，不是配置失败。它们使用了历史上有歧义的 + protocol-v1 prompt，因此不能把失败完全归因于教师的内在行为。 +- [Consumer Runtime v1](docs/consumer-runtime-v1.md)：padded update batch 与 + shared adapter 在既定容差内保持单次更新目标；rollout 生成仍是逐条执行。 +- [局限](docs/limitations.md)、[数学](docs/math.md)、 + [可复现性](docs/reproducibility.md)与[兼容策略](docs/compatibility.md)。 -RecoveryBench 是一个预注册的 SQLite 工具错误恢复机制研究，不是 alignment -benchmark。它固定冷启动检查点、合格教师、任务顺序、优化器和更新步数，单独 -考察状态新鲜度。三个种子和所有已完成的负结果都被保留。 +新运行以 tokenizer 结构身份作为主要兼容性检查。旧版 behavioral +fingerprint 只对一个固定 probe 的 token ID 与元数据做摘要，仅用于旧产物迁移， +不能证明两个 tokenizer 的身份相同。 -| 方法 | 严格成功率 | 出错后恢复率 | 继续训练耗时 | -| --- | ---: | ---: | ---: | -| 冷启动 | 10.7% | 13.6% | 0.2 秒 | -| 继续 oracle SFT | 4.9% | 1.8% | 51.3 秒 | -| oracle 状态离线 KD | **33.1%** | **31.9%** | 58.3 秒 | -| 冻结学生状态 KD | **23.2%** | **22.8%** | 52.1 秒 | -| 严格新鲜状态 OPD | 10.9% | 9.1% | 686.8 秒 | -| budget-50 新鲜状态 OPD | 27.3% | 20.7% | 720.8 秒 | +## 范围 -![RecoveryBench 三种子结果](docs/recoverybench/recovery-success.svg) - -等选中位置视图中，三个核心方法都在八步后越过 6,224 位置边界，因此质量 -结果与主视图相同。budget-50 只查询了模型生成位置的 49.77%，但没有减少教师 -主干前向，故 wall time 没有下降。50 秒产物是**受 cycle 上限约束的 wall-time -诊断，不是精确等时间证据**：SFT 与冻结 KD 完成八个 cycle，而新鲜 OPD 在 -一个不可再分的 88–121 秒更新中越过目标。 - -可阅读[完整分析](docs/recoverybench/recoverybench-v1.md)、 -[数据绑定技术报告](paper/recoverybench-v1/recoverybench-v1.pdf)和 -[不可变 schema-v3 产物](benchmarks/README.md#recoverybench-v1)。结论仅适用于 -一个 Qwen3 师生组合、一个任务族、三个种子和一张 RTX 4080；它不证明 OPD -普遍无效，也不证明离线 KD 总是胜出。 - -
-案例：为什么必须验证教师的工具协议能力 - -在已饱和的 v0.2 计算器任务上，协议合格的 OPD 教师在两个种子上都达到 -100%，与继续 SFT 持平，但继续训练耗时为后者的 6.1 倍。两个协议不合格的 -负对照都正常完成且为 0%，并非配置失败。两者使用有歧义的历史 protocol-v1 -prompt，因此不能把失败完全归因于教师自身行为。 - -![双种子协议教师对照](docs/gpu-calc-hard-equal-update-v2.svg) - -| 产物 | 定位 | -| --- | --- | -| [默认配方](recipes/qwen_consumer_gpu_calc.yaml) | 协议合格 | -| [Schema-v2 结果](benchmarks/results/gpu-calc-hard-equal-update-v2.json) | 冻结五臂对照 | -| [Raw-teacher](recipes/qwen_consumer_gpu_calc_raw_teacher.yaml) | 历史对照；非默认 | - -教师门禁与下游对照复用了同一组 24 道 v0.2 test 任务，因此这里只能支持该 -设置下“教师资格很重要”，不能支持 OPD 普遍优越。另一个 schema-v1 的 -481 秒 smoke 证明的是流水线，不是 OPD 胜过 SFT。 -[完整诊断与限制](docs/rtx4080-baselines.md)。 - -
- -## 本地玩具演示 - -不联网、不用 GPU、不下载任何权重。师生两个模型都是由配置直接构建的小型 transformer，分词器是一个可逆的约 190 词条玩具分词器，计算器环境自己生成并判分。 - -```bash -python -m pip install ".[train]" # 在克隆后的仓库中执行；CPU 版 torch 就够 -miniverl doctor # 这台机器能跑什么？ -miniverl demo --output runs/demo -``` - -它跑的是**真实**流水线——学生 rollout、工具执行、教师对这些状态打分、写入带来源校验的压缩 top-k 缓存、只在 assistant token 上做掩码 reverse-KL 更新——然后打印它到底证明了什么： - -```text -demo complete runs/demo - mode opd (genuine on-policy distillation) - optimizer steps 132 - parameter version 132 - rollout iterations 13 - wall clock 52.9 s - token provenance 45597 of 226383 tokens trainable (20%); 180786 are context - and can never be a target - teacher cache 735 scored positions, 131.6 KiB on disk, 2.0x smaller than - a dense fp16 dump - task success 0.0% -> 0.0% (greedy, held-out eval split) -``` - -demo 证明的是**机制**，不是能力：在这个规模下玩具学生只学会了工具调用**格式**，学不会算术复制，所以这里的 0% 是预期结果而非失败。想看真正学起来的 CPU 运行（实测 0.0% → 91.7%，192 秒）： - -```bash -miniverl train recipes/toy_cpu.yaml -``` - -这不是承诺而是实测：`recipes/toy_cpu.yaml` 在 CPU 上耗时 **192 秒**，在 24 个留出任务上把贪心成功率从 **0.0% 提升到 91.7%**（600 步监督冷启动 + 40 个在线策略蒸馏 cycle）。它在这个模型规模下**对随机种子敏感**：同样 600 步预算，`run.seed: 1234` 得到 81.2%，`run.seed: 20260727` 得到 0.0%。这个方差正是"玩具后端只是机制验证台、能力数字必须来自 GPU 配方"的原因。 - -最值得先跑的是 `miniverl inspect`，它打印的来源表就是这个项目的核心： - -```text -tokens by span type (only assistant_* can enter the loss) -+---------------------------------------------+ -| span type | tokens | in loss | -|---------------------+--------+--------------| -| system | 776 | no (context) | -| tool_result | 685 | no (context) | -| user | 318 | no (context) | -| assistant_tool_call | 153 | yes | -| assistant_text | 85 | yes | -| assistant_final | 25 | yes | -+---------------------------------------------+ -``` - -玩具后端是**机制验证台，不是能力展示**。它的模型太小，除了 `easy` 难度之外什么都做不了。能力数字来自 GPU 配方。 - -## 个人单卡快速上手 - -默认 `device: auto` / `dtype: auto`：新卡用 bf16，Titan V 等旧卡用 fp16。 -RTX 3070、Titan V、RTX 4080、RTX 5090 走同一 CUDA 路径，但仅 4080 有实测。 -能否装下取决于显存、模型、驱动和 token 预算。修改前请阅读 -[`单卡适配指南`](docs/single-gpu-guide.md)。 +miniVERL 只支持单个本地 CUDA 进程，不实现或包装 Ray、FSDP、Megatron、 +PPO、GRPO 或分布式 launcher。公开研究只覆盖小型 Qwen3、确定性工具环境与 +一张 RTX 4080，不能推出跨模型、跨任务、跨 GPU 或广义安全结论。 ```bash git clone https://github.com/DaoyuanLi2816/mini-verl.git cd mini-verl -python -m pip install torch --index-url https://download.pytorch.org/whl/cu130 -python -m pip install ".[train,cuda]" - -miniverl doctor # 确认 CUDA 与 bitsandbytes -miniverl validate recipes/qwen_consumer_gpu_calc.yaml -miniverl train recipes/qwen_consumer_gpu_calc.yaml --dry-run # 不下载任何东西 -miniverl train recipes/qwen_consumer_gpu_calc.yaml -miniverl report runs/ --out runs//report.html +python -m pip install -e ".[dev]" +pytest -q -m "not gpu and not network" ``` -配方中两个模型都锁定了 revision： - -| 角色 | 模型 | revision | 许可证 | -| --- | --- | --- | --- | -| 学生 | `Qwen/Qwen3-0.6B` | `c1899de289a04d12100db370d81485cdf75e47ca` | Apache-2.0 | -| 教师 | `Qwen/Qwen3-1.7B` | `70d244cc86ccca08cf5af4e1e306ecf908b1ad5e` | Apache-2.0 | - -两者的 `tokenizer.json` **逐字节相同**（`sha256 aeb13307a71acd8fe81861d94ad54ab689df773318809eed3cbe794b4492dae4`）。新运行首查结构身份；旧产物回退到固定探针行为指纹。 - -配方还把[协议教师适配器](https://huggingface.co/DaoyuanLi/mini-verl-qwen3-1.7b-protocol-teacher) -锁定在 revision `23323751318135484c06c043b1f9b9e7016dd89f`，并在分配教师模型 -之前要求其已记录的严格策略成功率至少达到 50%。 - -## 精确 vs. top-k + tail - -两类目标函数被明确区分命名，因为把它们混为一谈正是蒸馏结果无法复现的常见原因。 - -**`exact_full_vocab`** 会构造完整的 `[chunk, V]` 师生分布并计算真实散度。适用于词表很小（玩具后端），或教师常驻显存、分布按 chunk 现算的情况。由 `loss.exact_max_vocab`（默认 8192）兜底，避免静默地尝试持久化 `[positions, 152k]` 张量。 - -**`bucketed_topk_tail`** 把词表粗粒化为「教师 top-k 个 token + 一个聚合尾桶」，再在两个 `K+1` 类分布之间算散度。这**不是**全词表 KL。数据处理不等式严格适用于未平滑的粗粒化；实际实现会对非空尾桶做 epsilon 下限和重新归一化，因此文档把它称为 epsilon 平滑目标，不宣称每个输入上仍严格满足该定理。当 `k == V` 时空尾桶绕过平滑，float64 测试确认实现与精确目标在 `1e-9` 内一致。函数名就叫 `bucketed_forward_kl` / `bucketed_reverse_kl` / `bucketed_jsd`，任何调用点都无法把它伪装成精确 KL。 - -压缩真正省下的是**教师侧的存储**，以及把教师从显存中换出的能力。它**不会**按比例减少教师的 FLOPs——教师仍要跑一次完整前向来产生 hidden states。因此报告里写的是 `teacher_queried_position_ratio`，绝不写"节省了教师算力"。 - -top-k + tail 目标本身并不新颖：TRL 的 `ServerDistillationTrainer` 就有 `loss_top_k` 和可选的尾桶。详见 [`docs/math.md`](docs/math.md)。 - -## 工具 token 掩码 - -每条轨迹都是「一维 token 序列 + 类型化 span 划分」。三个掩码既被存储，**也**在每次读取时从 span 重新推导；一旦掩码与 span 不一致，文件会被拒绝而不是拿去训练。 - -上下文 span 会包含结尾的 `<|im_start|>assistant\n` 头部，因此模型 span 恰好从第一个采样 token 开始，任何被强制写入的脚手架 token 都不会成为监督目标。位置 `0` 永远不能作为目标。这两点都由 `tests/unit/test_token_provenance.py` 强制执行。 - -## 基准结果 - -下面所有数字都由 [`docs/benchmarking.md`](docs/benchmarking.md) 中的命令、在对应结果文件记录的硬件上跑出来。没有估算，没有外推。 - -* **RTX 4080，真实模型** —— [`docs/rtx4080-baselines.md`](docs/rtx4080-baselines.md) 记录了实测显存峰值、解码吞吐、完整配方运行以及旧版等优化器更新对照，同时也记录了尝试过的配置和**未运行**的配置。 -* **CPU，玩具模型** —— `recipes/toy_cpu.yaml` 在 192 秒内把成功率从 0.0% 提到 91.7%；`benchmarks/results/` 中还有旧版等优化器更新机制对照。后者的准确率差异**在噪声范围内**，作用是证明所有分支在同一比较轴下都能跑完，而不是给它们排名。原因见 [`benchmarks/README.md`](benchmarks/README.md)。 - -## 安装分层 - -| 层次 | 安装命令 | 得到什么 | -| --- | --- | --- | -| 核心 | `python -m pip install .` | `doctor`、`validate`、`inspect`、`report`、`cache`，以及 schema 和 Python API。**不含 torch**。 | -| 训练 | `python -m pip install ".[train]"` | `demo`、`train`、`eval`、`benchmark`。加入 torch、transformers、peft、accelerate。 | -| 4-bit | `python -m pip install ".[cuda]"` | bitsandbytes，用于 NF4 QLoRA 与 8-bit 优化器。 | -| 开发 | `python -m pip install ".[dev]"` | pytest、hypothesis、ruff、mypy、build、twine。 | - -缺少可选依赖时不会抛出裸异常： - -```text -$ miniverl demo --output runs/demo -error miniverl demo requires the optional dependency 'torch', which is not installed. -hint pip install "miniverl[train]" -``` - -### 严格离线执行 - -所有会加载模型的命令共享同一份“零网络”契约： - -```bash -miniverl train --offline -miniverl benchmark --offline -miniverl eval --run --offline -miniverl export-adapter --run --out --offline -``` - -此模式要求基础模型、分词器和每个适配器文件已经位于本地路径或 Hugging -Face 缓存中；miniVERL 不允许 HTTP、metadata、ETag 或 Hub API 请求，也不会 -静默退回在线解析。Hub 教师适配器只按固定 revision 解析一次，PEFT 随后加载 -刚刚验证过配置、权重、manifest 与校验和的同一个本地 snapshot。缓存缺失时， -错误会给出不可变身份以及精确的 `hf download` 预加载命令。 - -## Python API - -对外暴露的接口刻意保持很小： - -```python -from miniverl.config import RunConfig -from miniverl.trainer import OPDTrainer - -config = RunConfig.from_yaml("recipes/toy_cpu.yaml") -with OPDTrainer.from_config(config) as trainer: - result = trainer.train() - -print(result.run_dir, result.global_step, result.eval["success_rate"]) -``` - -自定义环境见 `examples/custom_environment/`，自定义教师见 `examples/custom_teacher/`，两个例子都可直接运行。 - -标准冻结 PEFT 教师适配器、Qwen3 协议 SFT 配方、导出命令、兼容性检查和教师策略能力门禁见 -[`docs/teacher-adapters.md`](docs/teacher-adapters.md)。 - -## 局限 - -简版如下，完整清单见 [`docs/limitations.md`](docs/limitations.md)。 - -* 仅支持师生同一分词器；跨词表蒸馏会直接报错。 -* rollout 解码仍逐条执行；更新路径支持 padding 物理 batch。 - `gradient_accumulation_steps` 是优化器组大小，`trajectory_batch_size` 是一次主干前向共享的轨迹数。 -* 量化模型不能用 `swap`，因为 bitsandbytes 的参数绑定在量化时所在的设备上。 -* 只测试过 Qwen3 与 Qwen2 架构。其他架构可能能通过架构适配器工作，但本项目不作任何声明。 -* RecoveryBench 使用三个预先指定的学生种子；计算器案例使用两个，更早的 GPU - 产物为单种子。不声称广泛统计显著性或跨任务泛化。 -* 在实测机器上，解码是 kernel 启动开销受限而非算力受限，因此吞吐数字与平台强相关。 - -## 可复现性 - -每次运行都会写出 `manifest.json`，记录 miniVERL 版本、git commit、Python 与操作系统、torch/CUDA/驱动版本、GPU 型号与显存、模型 id **及解析后的 revision**、分词器指纹、随机种子、精度、量化、显存策略、损失模式、top-k、策略版本，以及一个 `measurement_status` 块，说明每项结果是实测、模拟还是未运行。 - -可写运行会原子地经过 `ready`、`running`，再进入 `completed`、`failed`、`interrupted` 或 `closed_before_training` 终态。同一把跨进程锁覆盖构造、训练/续训、独立评估的 checkpoint 选择与加载，以及自动报告。在同一个 trainer 内，训练、评估、checkpoint 保存/加载和破坏性关闭互斥；加载仅允许在 READY 状态执行，`close()` 只有取得操作所有权后才会修改资源，评估即使失败也会恢复此前的精确模型模式。 - -每个内置环境在 `reset` 后都会把任意字符串变成有界验证结果，不向外泄漏解析或数值异常；protocol-v2 使用各环境可被验证器接受的 final 格式示例。可分享的 HTML、Markdown、JSON、benchmark 导出和 portable manifest 会遮蔽语义化密钥、URL 凭证及跨平台私有路径，而私有运行目录仍保留精确续训所需的本地状态。脱敏只是尽力而为的分享防线，不代表可以把真实凭据写进配置、运行产物或报告。 - -它**不**记录用户名、主机名、家目录，也不记录白名单之外的任何环境变量（白名单只包含会影响数值结果的少数几个）——这一点有测试断言。 -来自文件的运行还会分别保存原始提交字节、规范化验证配置、v0.2 -续训兼容层和运行时解析后的选择。 - -详见 [`docs/reproducibility.md`](docs/reproducibility.md) 和 -[`兼容性策略`](docs/compatibility.md)。 - -## 路线图 - -以下均**未实现**、也不作承诺，仅为明确边界：跨词表蒸馏、批量或引擎化 rollout 解码、熵感知散度混合（arXiv:2603.07079）、更多模型族、更多环境、原生多卡执行。Level-3 桥接只把一个已文档化配置导出到锁定版本的 verl，并不执行分布式作业。 - -## 致谢与声明 - -> miniVERL 是一个独立项目，与 verl 项目、字节跳动（ByteDance）或火山引擎（Volcano Engine）没有隶属关系，也未获得其背书。它**不是** verl 的直接替代品。 - -这个名字只是对问题领域的致意，不代表通用兼容性；经验证的桥接被明确限制在一个锁定配置内。verl 是面向 Ray 集群扩展的更大系统。miniVERL 面向的是「只有一块个人显卡、并且希望把每一行发生的事都读懂，之后再把标准产物交给 verl 扩展」的场景：它可以是较老的 12 GiB 显卡，也可以是当前的高端显卡；仓库只对实际跑过的硬件声明实测性能。对比见 [`docs/comparisons.md`](docs/comparisons.md)。 - -## 引用与许可证 - -引用格式见 [CITATION.cff](CITATION.cff)，变更记录见 [CHANGELOG.md](CHANGELOG.md)，贡献指南见 [CONTRIBUTING.md](CONTRIBUTING.md)，安全策略见 [SECURITY.md](SECURITY.md)。 - -Apache-2.0，见 [LICENSE](LICENSE) 与 [THIRD_PARTY_NOTICES.md](THIRD_PARTY_NOTICES.md)。 +项目使用 Apache-2.0 许可证。参见 [CONTRIBUTING.md](CONTRIBUTING.md) 与 +[SECURITY.md](SECURITY.md)。项目记录：[默认 GPU 配方](recipes/qwen_consumer_gpu_calc.yaml)、 +[冻结的 calculator JSON](benchmarks/results/gpu-calc-hard-equal-update-v2.json)、 +[变更记录](CHANGELOG.md)、[引用信息](CITATION.cff)与[许可证](LICENSE)。 diff --git a/TODO.md b/TODO.md index b228e9c..b22eed6 100644 --- a/TODO.md +++ b/TODO.md @@ -1,17 +1,18 @@ # TODO -This is the post-v0.2.1 research and engineering backlog. Completed release, -publication, protocol-teacher and two-seed benchmark work lives in -`PROJECT_STATE.md` and `CHANGELOG.md`, not in this active list. +This is the post-v0.6 roadmap. Completed RecoveryBench, Consumer Runtime, +Alignment Lab and pinned verl-bridge work lives in `PROJECT_STATE.md` and +`CHANGELOG.md`, not in this active list. ## Scientific follow-up -- [ ] Select future protocol-teacher candidates on the `eval` split and reserve - `test` for downstream reporting. -- [ ] Evaluate on a less saturated task family with at least three prespecified - seeds; retain SFT, raw-teacher and protocol-aligned controls. -- [ ] Run JSON-navigation and SQLite real-model comparisons with the same - matched-budget and immutable-artifact discipline. +- [ ] [Preregister and execute real external alignment endpoints](https://github.com/DaoyuanLi2816/mini-verl/issues/39) (for example + IFEval, XSTest, HarmBench and RewardBench) in a future release; keep them + separate from Alignment Lab v1's sandbox-policy checks. +- [ ] Evaluate a less saturated task family with at least three prespecified + seeds, eval-only selection and a reserved one-read test split. +- [ ] Extend matched-budget JSON-navigation and SQLite evidence only when the + design adds information beyond the published RecoveryBench study. - [ ] Repeat the measured GPU recipes on Linux; do not treat the expected throughput improvement as measured until those artifacts exist. diff --git a/docs/alignment-lab/alignment-lab-v1.md b/docs/alignment-lab/alignment-lab-v1.md index 5707735..1f45f1a 100644 --- a/docs/alignment-lab/alignment-lab-v1.md +++ b/docs/alignment-lab/alignment-lab-v1.md @@ -1,4 +1,4 @@ -# Online Policy Distillation After SFT on One Consumer GPU +# Alignment Lab v1: a saturated tool-policy case study ## Abstract @@ -30,6 +30,9 @@ student never receives that context. All actions are synthetic sandbox actions; no real destructive operation is executed. IFEval, XSTest, HarmBench and RewardBench are represented by pinned metadata adapters only and are **not** measured endpoints in this artifact. +“Preference win rate” is the deterministic Minipolicy paired outcome, not a +human-preference measurement. Harmful compliance and over-refusal are sandbox +policy checks, not a broad safety result. ## Final result @@ -42,13 +45,20 @@ measured endpoints in this artifact. | standard OPD | 98.6% | 0.0% | 0.0% | 97.2% | 100.0% | 76.7 s | 0.94 GiB | | verifier-gated OPD | 97.9% | 0.0% | 0.0% | 95.8% | 46.8% | 66.0 s | 0.87 GiB | -![Alignment quality versus utility retention](quality-vs-utility.svg) +![Forest chart of alignment and tool-utility deltas from the saturated SFT checkpoint, with every seed and the three-seed means](delta-from-sft.svg) -![Safety-policy outcome versus over-refusal](safety-vs-overrefusal.svg) +![Row matrix of alignment, retained tool utility, teacher-query ratio, continuation GPU time and peak VRAM, including every measured seed](outcome-cost-matrix.svg) -![Preference outcome versus continuation GPU time](preference-vs-gpu-time.svg) +![Coverage matrix showing tied zero sandbox safety checks, utility regressions and external safety benchmarks not executed](metric-coverage-matrix.svg) -![Alignment quality versus teacher-query ratio](quality-vs-teacher-query.svg) +
+Figure provenance + +- Result SHA-256: `584752dccb91654109c357b8ebb12681a12a9c1476a9ba539dd35e4d860a22ef` +- Task-level result SHA-256: `8d7fc723436d7377d196fc44046d960e3cb7f0aa81e03d49ef05b627eb84630f` +- Three seed identities: `1234`, `20260727`, `20260801` + +
The starting checkpoint defines a ceiling; overlapping continuation points are not evidence of algorithmic equivalence, and every non-overlapping regression diff --git a/docs/alignment-lab/delta-from-sft.svg b/docs/alignment-lab/delta-from-sft.svg new file mode 100644 index 0000000..6d21242 --- /dev/null +++ b/docs/alignment-lab/delta-from-sft.svg @@ -0,0 +1 @@ +Continuation could not improve the saturated SFT checkpointForest chart of alignment-score and retained-tool-utility percentage-point deltas for five continuation methods. Every seed is shown at its exact x value.Continuation could not improve the saturated SFT checkpoint48 paired sandbox tasks per seed · mean marks plus all three measured seeds · zero = starting SFTContinuation methoddelta from the same-seed SFT checkpoint (percentage points)alignment meanutility-40-30-20-10+0+5zero baselinecontinued SFTA seeds: 0.0 / 0.0 / -16.7U seeds: 0.0 / 0.0 / -33.3A -5.6U -11.1DPOA seeds: 0.0 / 0.0 / 0.0U seeds: 0.0 / 0.0 / 0.0A 0.0U 0.0offline soft distillationA seeds: 0.0 / 0.0 / 0.0U seeds: 0.0 / 0.0 / 0.0A 0.0U 0.0standard OPDA seeds: 0.0 / 0.0 / -4.2U seeds: 0.0 / 0.0 / -8.3A -1.4U -2.8verifier-gated OPDA seeds: 0.0 / -6.2 / 0.0U seeds: 0.0 / -12.5 / 0.0A -2.1U -4.2Seed shapes: ● 1234 · ◆ 20260727 · × 20260801. A = alignment; U = retained tool utility. diff --git a/docs/alignment-lab/demo.md b/docs/alignment-lab/demo.md index 5f14c5f..cad6907 100644 --- a/docs/alignment-lab/demo.md +++ b/docs/alignment-lab/demo.md @@ -1,4 +1,4 @@ -# 75-second Alignment Lab demo +# Alignment Lab demo recording script This is a reproducible recording plan, not a fabricated video. The short run uses the CPU toy backend and proves the workflow and artifact surfaces; its 0% @@ -43,6 +43,6 @@ CPU. Do not edit out the card's toy-backend limitation or 0% result. - [Formal benchmark pilot decision](https://github.com/DaoyuanLi2816/mini-verl/blob/main/examples/alignment-lab/pilot.json) - [Reviewed toy Alignment Card](https://github.com/DaoyuanLi2816/mini-verl/blob/main/examples/alignment-lab/alignment-card.json) -- [Policy-sensitive query figure](quality-vs-teacher-query.svg) +- [Outcome and cost matrix](outcome-cost-matrix.svg) No user quote, adoption number or unmeasured GPU claim is part of this demo. diff --git a/docs/alignment-lab/metric-coverage-matrix.svg b/docs/alignment-lab/metric-coverage-matrix.svg new file mode 100644 index 0000000..180401e --- /dev/null +++ b/docs/alignment-lab/metric-coverage-matrix.svg @@ -0,0 +1 @@ +Metric coverage: zero sandbox failures did not imply full utility retentionCoverage matrix showing zero harmful-compliance and over-refusal rates, observed tool-utility deltas, measured sandbox endpoints and unexecuted external benchmarks.Metric coverage: zero sandbox failures did not imply full utility retentionAll three measured seeds are printed · deterministic Minipolicy checks onlyMethodHarmful complianceseed valuesOver-refusalseed valuesTool utility Δmean · seeds (pp)Sandbox endpointmeasured?External safetyexecuted?SFT checkpoint0% · 0% · 0%0% · 0% · 0%0.00.0 / 0.0 / 0.0YESNOT RUNcontinued SFT0% · 0% · 0%0% · 0% · 0%-11.10.0 / 0.0 / -33.3YESNOT RUNDPO0% · 0% · 0%0% · 0% · 0%0.00.0 / 0.0 / 0.0YESNOT RUNoffline soft distillation0% · 0% · 0%0% · 0% · 0%0.00.0 / 0.0 / 0.0YESNOT RUNstandard OPD0% · 0% · 0%0% · 0% · 0%-2.80.0 / 0.0 / -8.3YESNOT RUNverifier-gated OPD0% · 0% · 0%0% · 0% · 0%-4.20.0 / -12.5 / 0.0YESNOT RUNThe two sandbox safety checks tied at zero while utility still regressed.IFEval, XSTest, HarmBench and RewardBench were not executed; this is not a broad safety benchmark. diff --git a/docs/alignment-lab/outcome-cost-matrix.svg b/docs/alignment-lab/outcome-cost-matrix.svg new file mode 100644 index 0000000..b0baccc --- /dev/null +++ b/docs/alignment-lab/outcome-cost-matrix.svg @@ -0,0 +1 @@ +Outcome and continuation-cost matrixRow matrix of alignment, retained tool utility, teacher-query ratio, continuation GPU time and peak VRAM for every method and seed.Outcome and continuation-cost matrixDirect labels + seed marks · non-teacher methods remain not applicable, never zeroMethodAlignment0–100%Tool utility0–100%Teacher query0–100%GPU time0–100 secondsPeak VRAM0–2 GiBSFT checkpoint100.0%100.0%— not applicable0.0s0.63 GiBcontinued SFT94.4%88.9%— not applicable3.9s0.96 GiBDPO100.0%100.0%— not applicable8.6s1.64 GiBoffline soft distillation100.0%100.0%100.0%26.6s0.94 GiBstandard OPD98.6%97.2%100.0%76.7s0.94 GiBverifier-gated OPD97.9%95.8%46.8%66.0s0.87 GiBBars show the three-seed mean except VRAM, whose main bar is the observed maximum; seed shapes show every run.Query ratio is selected target positions, not teacher FLOPs. DPO time includes its pinned TRL job. diff --git a/docs/alignment-lab/preference-vs-gpu-time.svg b/docs/alignment-lab/preference-vs-gpu-time.svg deleted file mode 100644 index ec5ba41..0000000 --- a/docs/alignment-lab/preference-vs-gpu-time.svg +++ /dev/null @@ -1 +0,0 @@ -No method beats the starting policy; continuation cost differsPreference outcome versus continuation GPU time for six methods.No method beats the starting policy; continuation cost differsThree-seed means · four continuation updates except the frozen SFT checkpoint018355371880.80.91.0continuation GPU time (seconds, mean)preference win rate0.0s3.9s8.6s26.6s76.7s66.0sSFT checkpointcontinued SFTDPOoffline soft distillationstandard OPDverifier-gated OPDDPO includes external TRL training; evaluation excluded from GPU-time axis · source 584752dccb916541 diff --git a/docs/alignment-lab/quality-vs-teacher-query.svg b/docs/alignment-lab/quality-vs-teacher-query.svg deleted file mode 100644 index 8c4fc2a..0000000 --- a/docs/alignment-lab/quality-vs-teacher-query.svg +++ /dev/null @@ -1 +0,0 @@ -Fewer teacher targets do not guarantee better alignmentAlignment outcome versus measured teacher queried-position ratio.Fewer teacher targets do not guarantee better alignmentThree-seed means · ratios are measured selected positions / generated positions0.00.00.20.20.40.40.60.60.80.81.01.0teacher queried-position ratioalignment scoreSFT checkpointcontinued SFTDPOoffline soft distillationstandard OPDverifier-gated OPDno teacher callsQuery ratio counts selected positions, not teacher backbone FLOPs · source 584752dccb916541 diff --git a/docs/alignment-lab/quality-vs-utility.svg b/docs/alignment-lab/quality-vs-utility.svg deleted file mode 100644 index 4c51f66..0000000 --- a/docs/alignment-lab/quality-vs-utility.svg +++ /dev/null @@ -1 +0,0 @@ -SFT starts at the ceiling; continuation adds no gainMean alignment score versus retained tool utility for six methods across three seeds.SFT starts at the ceiling; continuation adds no gainQwen3-0.6B · 48 paired test tasks · 3 seeds · deterministic Minipolicy v10.00.00.20.20.40.40.60.60.80.81.01.0tool utility retentionalignment scoreSFT checkpointcontinued SFTDPOoffline soft distillationstandard OPDverifier-gated OPD3 / 6 methodsat the 1.0 / 1.0 ceilingConcentric rings denote exact overlap · source SHA-256 584752dccb916541 diff --git a/docs/alignment-lab/safety-vs-overrefusal.svg b/docs/alignment-lab/safety-vs-overrefusal.svg deleted file mode 100644 index 0944ea4..0000000 --- a/docs/alignment-lab/safety-vs-overrefusal.svg +++ /dev/null @@ -1 +0,0 @@ -Safety-policy checks can pass while benign utility regressesHarmful compliance and over-refusal do not capture every policy or utility failure.Safety-policy checks can pass while benign utility regressesExact validators · 48 paired tasks per seed · no real destructive actions0.00.00.20.20.40.40.60.60.80.81.01.0over-refusal rateharmful-compliance rateSFT checkpointcontinued SFTDPOoffline soft distillationstandard OPDverifier-gated OPD0% / 0%harmful / over-refusalDeterministic sandbox policy checks, not a broad safety benchmark · source 584752dccb916541 diff --git a/docs/alignment-lab/when-opd-should-follow-sft.md b/docs/alignment-lab/when-opd-should-follow-sft.md index 838fd24..8a41eae 100644 --- a/docs/alignment-lab/when-opd-should-follow-sft.md +++ b/docs/alignment-lab/when-opd-should-follow-sft.md @@ -70,7 +70,7 @@ averaged 94.4% alignment and 88.9% retained utility. Standard OPD averaged 98.6% and 97.2%; verifier-gated OPD averaged 97.9% and 95.8%. Every completed regression is preserved. -![Alignment quality versus retained tool utility](quality-vs-utility.svg) +![Continuation-method alignment and retained-tool-utility deltas from the saturated SFT checkpoint](delta-from-sft.svg) Harmful-compliance and over-refusal rates were both 0% for every method. Those two axes alone therefore missed the safe-error-recovery regressions. Alignment diff --git a/docs/assets/javascripts/versioning.js b/docs/assets/javascripts/versioning.js new file mode 100644 index 0000000..d241a8d --- /dev/null +++ b/docs/assets/javascripts/versioning.js @@ -0,0 +1,29 @@ +(() => { + const banner = document.querySelector(".docs-channel"); + const selector = document.getElementById("docs-version-selector"); + const label = document.getElementById("docs-channel-label"); + if (!banner || !selector || !label) return; + + const pathname = window.location.pathname; + const devMarker = "/dev/"; + const isDev = pathname.includes(devMarker); + const scriptPath = document.currentScript + ? new URL(document.currentScript.src).pathname + : pathname; + const assetsMarker = "/assets/javascripts/"; + let base = scriptPath.includes(assetsMarker) + ? `${scriptPath.slice(0, scriptPath.indexOf(assetsMarker))}/` + : pathname.replace(/[^/]*$/, ""); + if (base.endsWith(devMarker)) base = base.slice(0, -devMarker.length + 1); + const stableVersion = banner.dataset.stableVersion; + const devVersion = banner.dataset.devVersion; + + selector.value = isDev ? "dev" : "stable"; + label.textContent = isDev + ? `Development documentation · ${devVersion}` + : `Stable documentation · ${stableVersion}`; + + selector.addEventListener("change", () => { + window.location.assign(selector.value === "dev" ? `${base}dev/` : base); + }); +})(); diff --git a/docs/assets/stylesheets/extra.css b/docs/assets/stylesheets/extra.css new file mode 100644 index 0000000..3fe25d9 --- /dev/null +++ b/docs/assets/stylesheets/extra.css @@ -0,0 +1,94 @@ +:root { + --md-text-font: "DejaVu Sans", "Segoe UI", sans-serif; + --md-code-font: "DejaVu Sans Mono", Consolas, monospace; +} + +.docs-channel { + align-items: center; + display: flex; + flex-wrap: wrap; + gap: 0.65rem 1rem; + justify-content: center; + min-height: 2rem; +} + +.docs-channel label { + font-size: 0.72rem; + opacity: 0.85; +} + +.docs-channel select { + background: color-mix(in srgb, var(--md-primary-fg-color) 75%, black); + border: 1px solid rgba(255, 255, 255, 0.55); + border-radius: 0.3rem; + color: white; + font: inherit; + padding: 0.2rem 0.45rem; +} + +.md-typeset img, +.md-typeset picture, +.md-typeset picture img { + height: auto; + max-width: 100%; +} + +.md-typeset picture { + display: block; + margin: 1.2rem auto; +} + +.md-typeset .bridge-architecture img { + border-radius: 0.75rem; + display: block; + margin: 0 auto; + width: 100%; +} + +.md-typeset__scrollwrap { + overflow-x: auto; + overscroll-behavior-inline: contain; +} + +.md-typeset table:not([class]) { + max-width: 100%; +} + +.path-grid { + display: grid; + gap: 1rem; + grid-template-columns: repeat(3, minmax(0, 1fr)); + margin: 1.4rem 0; +} + +.path-card { + border: 1px solid var(--md-default-fg-color--lightest); + border-radius: 0.7rem; + min-width: 0; + padding: 1rem; +} + +.path-card h2 { + margin-top: 0; +} + +.path-card pre { + white-space: pre-wrap; + word-break: break-word; +} + +@media (max-width: 820px) { + .path-grid { + grid-template-columns: 1fr; + } +} + +@media (max-width: 600px) { + .docs-channel { + gap: 0.35rem 0.65rem; + } + + .md-typeset { + font-size: 0.78rem; + } +} diff --git a/docs/compatibility.md b/docs/compatibility.md index 8cbb734..afe8d13 100644 --- a/docs/compatibility.md +++ b/docs/compatibility.md @@ -25,14 +25,17 @@ regeneration hint. Compatibility Level 1 covers standard Hugging Face, PEFT, safetensors, tokenizer and Parquet artifacts. Level 2 is a fail-closed 14-field config -whitelist. Level 3 adds the generated bundle, reward scaffold, hashes and an -exact-source smoke. Unknown, algorithm-changing or distributed-only verl -fields are rejected instead of guessed. +whitelist. **miniVERL-defined compatibility Level 3** adds the generated +bundle, reward scaffold, hashes and an exact-source smoke. Unknown, +algorithm-changing or distributed-only verl fields are rejected instead of +guessed. -These levels do not make miniVERL a verl runtime. Optimizer state, distributed +These miniVERL-defined levels do not make miniVERL a verl runtime. Optimizer state, distributed RNG, FSDP/Megatron native checkpoints, Ray runtime state and teacher-cache to PPO-reference-cache conversion are unsupported. The release smoke validates artifacts and configuration; distributed execution is recorded as not tested. +Current bundles contain a fail-closed reward scaffold and an absent base +snapshot, so they use `launch.template.sh` and report `launchable: false`. See the [bridge contract](verl-bridge.md). ## Versioned but extensible diff --git a/docs/consumer-runtime/index.md b/docs/consumer-runtime/index.md new file mode 100644 index 0000000..765b6de --- /dev/null +++ b/docs/consumer-runtime/index.md @@ -0,0 +1,17 @@ +# Consumer Runtime + +The single-GPU runtime supports a conservative dual-model path and a +shared-backbone path that switches standard PEFT roles on one resident base. +Padded trajectory updates improve update throughput without changing the +effective optimizer batch or the strict-OPD freshness contract. + +![Measured continuation-time and peak-memory Pareto view for the frozen Consumer Runtime matrix](../consumer-runtime-v1-pareto.svg) + +The figure is a systems result for one RTX 4080 workload, not a cross-GPU speed +forecast or a new quality experiment. Read the +[full data-bound Consumer Runtime v1 report](../consumer-runtime-v1.md) for the +matrix, profiler evidence, equivalence gate and hardware limits. + +Next: configure a matching PyTorch build and recipe with the +[single-GPU guide](../single-gpu-guide.md), or inspect failure recovery in +[RecoveryBench](../recoverybench/recoverybench-v1.md). diff --git a/docs/index.md b/docs/index.md index dfebad6..c4f05e3 100644 --- a/docs/index.md +++ b/docs/index.md @@ -1,24 +1,98 @@ # miniVERL -Single-GPU prototyping for a documented subset of verl-style online -post-training. Develop, diagnose and validate an alignment or distillation -recipe locally, then export standard model, dataset, recipe and provenance -artifacts to a pinned verl release for scale-out. - -miniVERL is an independent project. Its verified bridge targets only -`single-gpu-online-distillation-v1` on `verl v0.8.0`; distributed execution is -not tested. - -## Start here - -- [Why OPD after SFT?](alignment-lab/when-opd-should-follow-sft.md) -- [Choose SFT vs DPO vs OPD](alignment-lab/alignment-lab-v1.md) -- [One-GPU alignment quickstart](single-gpu-guide.md) -- [Shared-backbone dual-adapter runtime](consumer-runtime-v1.md) -- [Batched runtime](benchmarking.md) -- [RecoveryBench](recoverybench/recoverybench-v1.md) -- [Verified verl bridge](verl-bridge.md) -- [90-second verified-bridge demo](verl-bridge-demo.md) -- [Artifact and schema reference](trajectory-schema.md) -- [Community recipes and submissions](community-benchmarks.md) -- [v0.6 launch story and card copy](verl-bridge-launch.md) +Auditable single-GPU LLM post-training for choosing, running and inspecting +SFT, DPO, knowledge distillation and strict OPD—plus a bounded artifact bridge +to one pinned verl profile. miniVERL is independent; no upstream endorsement is +implied, and distributed execution is not tested. + +[Install and run locally](single-gpu-guide.md){ .md-button .md-button--primary } +[Read the compatibility boundary](verl-bridge.md){ .md-button } + +## Install and verify in about a minute + +Install the PyTorch build that matches your CPU or CUDA system first, then the +training extra. This CPU example is deterministic and downloads no model: + +```bash +python -m pip install torch --index-url https://download.pytorch.org/whl/cpu +python -m pip install "miniverl[train]" +miniverl demo --fast --output runs/quickstart +miniverl inspect runs/quickstart/trajectories.jsonl +``` + +The result is a typed trajectory log, checksummed teacher cache, manifest and +self-contained report. For CUDA wheels and memory-aware recipes, use the +[single-GPU guide](single-gpu-guide.md). + +## Choose a path + +
+ +
+ +## Align + +Choose SFT, DPO, offline KD or OPD from explicit pilot evidence. + +```bash +miniverl pilot recipes/alignment_tool_policy_toy.yaml --json +``` + +**Artifact:** an [Alignment Card](alignment-lab/alignment-lab-v1.md#reproducibility-and-artifacts) +with starting checkpoint, metrics, cost and limitations. + +**Next:** [When should OPD follow SFT?](alignment-lab/when-opd-should-follow-sft.md) + +
+ +
+ +## Distill locally + +Use strict OPD, shared-backbone role switching and padded trajectory updates on +one CUDA GPU. + +```bash +miniverl train recipes/qwen_consumer_gpu_shared.yaml --dry-run --json +``` + +**Artifact:** a resolved recipe and typed provenance plan before model loading. + +**Next:** [Consumer-GPU shared runtime](consumer-runtime/index.md) + +
+ +
+ +## Scale out + +Import only the documented profile, convert Parquet, export standard artifacts +and inspect the unsupported boundary. + +```bash +miniverl bridge doctor exports/my-bundle --json +``` + +**Artifact:** `provenance/compatibility-report.json` with separate readiness +flags; current bundles are not launchable. + +**Next:** [verl bridge contract](verl-bridge.md) + +
+ +
+ +## Measured evidence, kept scoped + +The Alignment Lab case study starts from an SFT checkpoint already at 100% +alignment and 100% retained tool utility on its deterministic sandbox suite. +No continuation method improves it; completed regressions remain visible. +External IFEval, XSTest, HarmBench and RewardBench endpoints were not executed. + +![Forest chart of continuation-method alignment and tool-utility deltas from the saturated SFT checkpoint](alignment-lab/delta-from-sft.svg) + +The consumer runtime result is a systems result, not a new quality claim: +shared-backbone role switching and padded trajectory updates reduce measured +memory/runtime overhead while preserving the tested local objective. See the +[Consumer Runtime report](consumer-runtime/index.md) and +[RecoveryBench](recoverybench/recoverybench-v1.md) for full evidence and limits. diff --git a/docs/limitations.md b/docs/limitations.md index b9cf47a..c149e39 100644 --- a/docs/limitations.md +++ b/docs/limitations.md @@ -441,18 +441,20 @@ call. ### The verified verl bridge does not execute verl -The v0.6 Level-3 bridge targets official verl `v0.8.0` at one exact commit and -one named profile. It validates standard PEFT/safetensors/tokenizer artifacts, -Parquet prompt data, 14 whitelisted config fields, a safe reward scaffold and -bundle hashes. The recorded smoke installs that exact source and parses or -loads each exchange surface. +The v0.6 **miniVERL-defined compatibility Level 3** bridge targets official verl +`v0.8.0` at one exact commit and one named profile. It validates standard +PEFT/safetensors/tokenizer artifacts, Parquet prompt data, 14 whitelisted +config fields, a safe reward scaffold and bundle hashes. The recorded smoke +installs that exact source and parses or loads each exchange surface. It does **not** launch Ray, FSDP/Megatron, vLLM/SGLang or a distributed training job. It does not convert optimizer state, distributed RNG, native sharded checkpoints, Ray runtime state or teacher caches into PPO reference caches. -Unknown and distributed-only config fields fail by default. “Level 3” therefore -means a validated scale-out bundle, not runtime parity or generic verl YAML -support. See the [exact contract and evidence](verl-bridge.md). +Unknown and distributed-only config fields fail by default. The +miniVERL-defined label therefore means a validated scale-out bundle, not +runtime parity or generic verl YAML support. Current bundles are not +launchable: the base snapshot is absent, reward logic fails closed and user +mappings remain placeholders. See the [exact contract and evidence](verl-bridge.md). `models.teacher.mode: privileged_context` works only with environments that implement `privileged_context()`. All three built-in environments do; diff --git a/docs/overrides/main.html b/docs/overrides/main.html new file mode 100644 index 0000000..f1cc49c --- /dev/null +++ b/docs/overrides/main.html @@ -0,0 +1,12 @@ +{% extends "base.html" %} + +{% block announce %} +
+ Stable documentation + + +
+{% endblock %} diff --git a/docs/verl-bridge-architecture-mobile.svg b/docs/verl-bridge-architecture-mobile.svg new file mode 100644 index 0000000..69a2bf6 --- /dev/null +++ b/docs/verl-bridge-architecture-mobile.svg @@ -0,0 +1 @@ +miniVERL → verl bridgeVertical mobile diagram with local runtime, portable bundle and pinned upstream smoke as verified layers, followed by Distributed execution: NOT TESTED.miniVERL → verl bridgev0.8.0 · 7aed6b23independent project; no endorsement1 · miniVERL local runtimesingle-GPU training + evaluationteacher role · targetsreference role · DPOreward role · verifierstudent · local updates2 · portable artifact bundlestandard, reviewable interchangePEFTsafetensorsParquetconfigtyped provenance3 · pinned upstream smokeverl v0.8.0 at 7aed6b23config parse + artifact load checksverified: documented profile subsetDistributed execution:NOT TESTEDNo Ray / FSDP / vLLM job ranNo OPD-to-PPO semantic parityunverified execution layer diff --git a/docs/verl-bridge-architecture.svg b/docs/verl-bridge-architecture.svg index 1333ce1..a3c2bac 100644 --- a/docs/verl-bridge-architecture.svg +++ b/docs/verl-bridge-architecture.svg @@ -1,72 +1 @@ - - miniVERL to verl scale-out architecture - A one-GPU miniVERL workflow exports standard model, dataset, recipe and provenance artifacts to a pinned documented subset of verl for distributed scale-out. Distributed execution is not tested by miniVERL. - - - - - - - - - - - - - - - - - - - - - - - One GPU to a pinned scale-out bundle - Documented subset · standard artifacts · explicit unsupported semantics - - - MINIVERL · ONE CUDA GPU - develop · diagnose · validate - - - - actor - - rollout - - teacher / ref - reward - - update - - eval - - - - - - typed provenance · PEFT / safetensors · Parquet · checksummed manifests - - - - export-verl - - - VERL · SCALE-OUT - v0.8.0 · exact commit pin - - - Ray orchestration - - FSDP / Megatron - - vLLM / SGLang - - Independent project · no endorsement implied · distributed execution status: not tested - miniVERL - - - +Scale-out bridge: verified artifacts, bounded claimsThree verified layers connect miniVERL local runtime artifacts to a pinned verl parse and load smoke. A dashed arrow leads to distributed execution marked not tested.Scale-out bridge: verified artifacts, bounded claimspinned upstream v0.8.0 · 7aed6b23 · independent project; no endorsement1 · miniVERL local runtimesingle-GPU training, evaluation and portable provenanceteacher role · targetsreference role · DPOreward role · verifierstudent · local updates2 · portable artifact bundlestandard formats with explicit config and provenance boundariesPEFTsafetensorsParquetresolved configtyped provenance3 · pinned upstream parse/load smokeverl v0.8.0 at 7aed6b23 · config parse + PEFT/safetensors/Parquet structural load checksverified boundary: artifact interchange and the documented profile subsetDistributed execution: NOT TESTEDNo Ray / FSDP / vLLM job ran · no OPD-to-PPO semantic-parity claimunverified execution layer diff --git a/docs/verl-bridge-demo.md b/docs/verl-bridge-demo.md index 078e354..d503781 100644 --- a/docs/verl-bridge-demo.md +++ b/docs/verl-bridge-demo.md @@ -1,4 +1,4 @@ -# 90-second verified-bridge demo +# Verified-bridge demo recording script This recording is an artifact-only CPU demo. It needs the `bridge` extra but does not need CUDA, model downloads, Ray or a verl installation. @@ -29,15 +29,15 @@ miniverl bridge doctor _demo-bundle --json # 65–80 s: show the pin and the unsupported-semantics boundary cat _demo-bundle/recipe/REQUIRED_VERL.txt -python -c "import json; p=json.load(open('_demo-bundle/provenance/compatibility-report.json')); print(p['unsupported_semantics']); print('distributed:', p['distributed_execution_status'])" +python -c "import json; p=json.load(open('_demo-bundle/provenance/compatibility-report.json')); print(p['unsupported_semantics']); print('launchable:', p['launchable']); print('distributed tested:', p['distributed_execution_tested'])" # 80–90 s: show the portable tree python -c "from pathlib import Path; print('\n'.join(p.as_posix() for p in sorted(Path('_demo-bundle').rglob('*')) if p.is_file()))" ``` -Narration: “This proves that the handoff is pinned, standard, loadable, -privacy-checked and checksummed. It does not prove a distributed verl job ran; -that status is deliberately `not tested`.” +Narration: “This proves that the handoff is pinned, structurally checked and +checksummed. It does not prove a distributed verl job ran: the reward scaffold +still fails closed and the bundle is not launchable.” The release's stronger compatibility evidence additionally installs the exact official verl commit and uses OmegaConf plus PEFT to load the relevant surfaces. diff --git a/docs/verl-bridge-launch.md b/docs/verl-bridge-launch.md index ece3782..cf6546a 100644 --- a/docs/verl-bridge-launch.md +++ b/docs/verl-bridge-launch.md @@ -16,10 +16,11 @@ quietly changes algorithm semantics. miniVERL keeps the local regime small: actor → rollout → teacher/reference or reward → update → evaluation, in one process on one CUDA device. v0.6 then adds -four explicit compatibility levels. Level 1 exchanges standard Hugging Face, -PEFT, safetensors, tokenizer and Parquet artifacts. Level 2 imports exactly 14 -documented fields from one fail-closed profile. Level 3 generates a checksummed -bundle for official verl `v0.8.0` at commit +four explicit miniVERL-defined compatibility levels. Level 1 exchanges standard +Hugging Face, PEFT, safetensors, tokenizer and Parquet artifacts. Level 2 +imports exactly 14 documented fields from one fail-closed profile. +**miniVERL-defined compatibility Level 3** generates a checksummed bundle for +official verl `v0.8.0` at commit `7aed6b230776f963fa09509c10d9c3a767d1102c`. The importer rejects critic, PPO/GRPO, Ray resources, FSDP/Megatron placement, @@ -27,15 +28,16 @@ vLLM/SGLang placement, asynchronous rollout and unknown fields. The dataset converter preserves chat messages, ground truth, extension data, rejection reasons and hashes without silently truncating. The exporter writes standard adapter, tokenizer and Parquet files, a pinned override recipe, a safe reward -scaffold, source manifests and `SHA256SUMS`. `miniverl bridge doctor` checks the -whole handoff before the user chooses to launch anything. +scaffold, source manifests and `SHA256SUMS`. Because the base snapshot, reward +implementation and user mappings are not complete, the entry point is named +`launch.template.sh` and doctor reports `launchable: false`. The release smoke installed the exact official source under Python 3.12. It parsed the official and exported OmegaConf shapes; loaded a standard PEFT LoRA config, safetensors header and both Parquet splits; imported the reward scaffold; and verified privacy plus every artifact hash. It did not install or run Ray, FSDP/Megatron or vLLM/SGLang. Distributed execution is therefore -recorded as **not tested**, not implied by the Level-3 label. +recorded as **not tested**, not implied by the miniVERL-defined label. This bridge also preserves miniVERL's negative results. RecoveryBench did not show a general fresh-state advantage, and the Alignment Lab began from a diff --git a/docs/verl-bridge.md b/docs/verl-bridge.md index db6a8d0..f55ba26 100644 --- a/docs/verl-bridge.md +++ b/docs/verl-bridge.md @@ -1,129 +1,126 @@ -# Verified verl bridge +# verl bridge: portable artifacts, bounded semantics -miniVERL implements compatibility Level 3 for one named profile: -`single-gpu-online-distillation-v1`. It exchanges standard artifacts with the -official [`verl v0.8.0`](https://github.com/verl-project/verl/releases/tag/v0.8.0) -source at commit `7aed6b230776f963fa09509c10d9c3a767d1102c`. The tested compatibility -environment is Python 3.12; the package built from that tag reports version -`0.8.0.dev0`. +miniVERL is an independent project; no endorsement by the verl project is +implied. The bridge targets the documented +`single-gpu-online-distillation-v1` profile subset of +[`verl v0.8.0`](https://github.com/verl-project/verl/tree/v0.8.0), pinned to +commit `7aed6b230776f963fa09509c10d9c3a767d1102c` (`7aed6b23`). It is +**miniVERL-defined compatibility Level 3**, not full verl compatibility. -![miniVERL to verl architecture](verl-bridge-architecture.svg) + + + Three verified bridge layers—miniVERL local runtime, a portable artifact bundle, and a pinned upstream parse/load smoke—followed by a dashed arrow to distributed execution marked NOT TESTED. + -This is a bridge to a documented subset, not generic verl YAML support and not -distributed-runtime parity. miniVERL is independent from the verl project; no -endorsement or upstream compatibility guarantee is implied. +The solid arrows cover local artifact production, the portable bundle, and the +pinned parse/load smoke. The dashed arrow is deliberate: no Ray, FSDP, vLLM or +distributed verl job ran, and no miniVERL-OPD-to-verl-PPO semantic parity is +claimed. -## Compatibility levels +## Compatibility state -| Level | Contract | v0.6 status | +| State | Current value | Meaning | | --- | --- | --- | -| 0 | prompt/data → rollout → scoring → target/advantage → update → evaluation | documented | -| 1 | Hugging Face, PEFT, safetensors, tokenizer, Parquet and provenance artifacts | validated | -| 2 | named config-field whitelist for a pinned source | validated, fail-closed | -| 3 | generated bundle, exact pin, config/Parquet/adapter/scaffold/hash smoke | validated | +| `artifact_bundle_complete` | `true` | PEFT, safetensors, Parquet, config and provenance are present and hashed. | +| `upstream_config_parse_passed` | `false` in a new bundle | Set only by a separate pinned upstream smoke record, never inferred at export time. | +| `model_data_load_smoke_passed` | `false` in a new bundle | The export itself does not load the base snapshot or execute a model. | +| `reward_implementation_complete` | `false` | The generated reward function deliberately fails closed. | +| `launchable` | `false` | Base weights, reward logic and confirmed mappings are incomplete. | +| `distributed_execution_tested` | `false` | No distributed job ran. | +| `algorithm_semantic_parity` | `false` | The target is a PPO/reward scaffold, not a continuation of miniVERL OPD. | -The Level-3 claim means the bundle is structurally loadable and bound to the -pin. It does not mean a distributed training job ran. +The committed [pinned smoke record](generated/verl-bridge-smoke.json) verifies a +specific artifact-only upstream parse/load exercise. It remains separate from +the readiness state of a newly exported bundle and from any execution claim. -## Import the narrow config profile +## Import a resolved profile subset + +`import-verl` accepts the documented, resolved field subset—not arbitrary +Hydra/OmegaConf or verl YAML. With only a source profile, it writes +`import-report.json` and a non-executable `imported.template.yaml`: ```bash -miniverl import-verl path/to/verl.yaml \ +miniverl import-verl resolved-verl.yaml \ --profile single-gpu-online-distillation-v1 \ --target-verl v0.8.0 \ --out recipes/imported.yaml ``` -`import-report.json` records the source digest, every mapped field, -informational ignores, unsupported fields, conflicts, inserted defaults and -the generated recipe digest. A rejected import still writes the report but -never writes a partial recipe. - -| Accepted verl field | miniVERL disposition | -| --- | --- | -| `data.train_files`, `data.val_files`, `data.prompt_key` | retained as bridge metadata in the report; use `convert-dataset` for data | -| `data.max_prompt_length` | contributes to `rollout.max_total_tokens` | -| `data.max_response_length` | `rollout.max_new_tokens_per_turn` and total-token bound | -| `data.seed` | `run.seed` and deterministic split seed | -| `actor_rollout_ref.model.path` | same-base student and policy-conditioned teacher scaffold | -| `actor_rollout_ref.model.enable_gradient_checkpointing` | student gradient checkpointing | -| `actor_rollout_ref.actor.optim.lr` | `train.learning_rate` | -| `trainer.save_freq`, `trainer.test_freq` | checkpoint/evaluation cycle frequencies | -| `trainer.project_name`, `trainer.experiment_name` | portable run name | -| `trainer.total_epochs` | `train.cycles` | - -Critic, advantage-estimator, PPO clipping, GRPO grouping, Ray resources, -FSDP/Megatron placement, tensor/pipeline parallelism, vLLM/SGLang placement, -async rollout and multi-node fields fail by default. Unknown fields fail too. - -## Convert prompt datasets +The status is `needs_user_input` until the source or command determines the +training environment, qualified teacher, objective and schedule interpretation. +Parquet paths never silently select the calculator environment, and a same-base +standard teacher without a distinct model or adapter is never invented. + +To deliberately produce a runnable recipe, supply the missing contract: ```bash -python -m pip install "miniverl[bridge]" -miniverl convert-dataset --from verl-parquet input.parquet --out miniverl.parquet -miniverl convert-dataset --to verl-parquet miniverl.parquet --out export.parquet +miniverl import-verl resolved-verl.yaml \ + --profile single-gpu-online-distillation-v1 \ + --target-verl v0.8.0 \ + --environment jsonnav \ + --teacher-model Qwen/Qwen3-1.7B \ + --loss-profile topk-tail-reverse-kl \ + --schedule-mapping epochs-as-cycles \ + --out recipes/imported.yaml ``` -Both directions validate `data_source`, structured chat `prompt`, `ability`, -`reward_model.ground_truth` and `extra_info`; report accepted/rejected rows, -truncation risk and input/output digests; and never truncate silently. -miniVERL token provenance and teacher targets live only in -`extra_info.miniverl` or its checksummed sidecar. They are distillation targets, -never relabeled as PPO reference log-probabilities. +The explicit schedule option acknowledges that verl epochs/save/test frequency +units are not proven equivalent to miniVERL cycles. Every source field is +classified as `exact`, `derived`, `informational_only`, +`requires_user_confirmation` or `unsupported`. In particular: -## Export and inspect a bundle +| Source field | Classification | Treatment | +| --- | --- | --- | +| `data.train_files`, `data.val_files`, `data.prompt_key` | `informational_only` | Recorded in the report; never substituted for a `ToolEnvironment`. | +| `data.max_response_length` | `exact` | Copied to the per-turn response bound. | +| `data.max_prompt_length` | `derived` | Combined with response length for miniVERL's total trajectory bound. | +| optimizer learning rate and seed | `exact` | Copied after finite numeric validation. | +| `trainer.total_epochs`, `save_freq`, `test_freq` | `requires_user_confirmation` | Copied only after the explicit schedule mapping. | +| algorithm, distributed or unknown fields | `unsupported` | Rejected with a report. | + +Finite scientific-notation strings such as `1e-5` are accepted. NaN, infinity +and unresolved `${...}` interpolations are rejected with an actionable error. +Every runnable output passes `RunConfig` validation before atomic publication. + +## Export a portable bundle ```bash -miniverl export-verl --run runs/my-alignment \ +miniverl export-verl --run runs/ \ --target-verl v0.8.0 \ - --out exports/my-alignment-verl -miniverl bridge doctor exports/my-alignment-verl --json + --out exports/ + +miniverl bridge doctor exports/ --require-verl ``` -The source run must contain a standard adapter under `model/` and official -prompt-schema `data/train.parquet` plus `data/val.parquet`. The exporter writes: +The bundle contains: ```text model/ adapter_config.json, adapter_model.safetensors, tokenizer metadata, - base-model.json (exact identity; base weights are not bundled) + base-model.json (identity only; base snapshot is not bundled) data/ train.parquet, val.parquet -recipe/ verl-overrides.yaml, launch.sh, REQUIRED_VERL.txt -reward/ reward_or_verifier_scaffold.py -provenance/ source manifests, compatibility report, SHA256SUMS +recipe/ verl-overrides.yaml, launch.template.sh, REQUIRED_VERL.txt +reward/ reward_or_verifier_scaffold.py (fails closed) +provenance/ source manifest/result, compatibility-report.json, SHA256SUMS README.md ``` -`bridge doctor` checks the exact target, PEFT config and safetensors structure, -tokenizer structural digest, both Parquet schemas, OmegaConf-compatible recipe -shape, side-effect-free reward import, unsupported semantics, privacy and every -artifact hash. Add `--require-verl` to require a VCS installation whose -`direct_url.json` resolves to the pinned commit. - -The generated override points verl at `model/base` and the adapter at `model/`. -Before launch, materialize the exact model id and 40-character revision from -`model/base-model.json`; `launch.sh` fails closed and prints the corresponding -`hf download` command if `model/base/config.json` is absent. It also refuses to -run while the reward scaffold still contains its generated fail-closed body. - -## Recorded smoke - -The release candidate installed the official commit without its distributed -dependency stack, using Python 3.12. The first Windows build needed -`PYTHONUTF8=1` because the upstream setup reads its UTF-8 README with the local -code page; the same exact commit then built successfully. OmegaConf parsed the -official generated PPO config, found all 14 import-whitelist fields plus the six -export handoff fields, and structurally merged the exported overrides. A -standard `LoraConfig`, safetensors header, both Parquet splits and the -fail-closed reward scaffold loaded; privacy and 14 artifact hashes passed. The checksummed record is -[`generated/verl-bridge-smoke.json`](generated/verl-bridge-smoke.json). - -The tiny CPU dry run is intentionally artifact-only. Installing and launching -Ray, FSDP/Megatron and vLLM/SGLang was outside this test, so distributed -execution remains **not tested**. - -## Unsupported semantic conversions - -The bridge does not convert optimizer state, distributed RNG, FSDP or Megatron -native checkpoints, Ray runtime state, or teacher cache semantics. It does not -claim that a miniVERL teacher cache is a PPO reference cache. Review and test -the generated reward scaffold before any scale-out launch. +Available source-run response length and learning rate are preserved in the +override file. The miniVERL total-token bound, cycle schedule and environment +identity are preserved in `source_run_values`; they are not relabelled as +equivalent verl intent. Any prompt limit or schedule value inserted for the PPO +scaffold appears in `placeholder_defaults` with `source_run_intent: false`. + +`bridge doctor` verifies pins, standard adapter structure, tokenizer metadata, +Parquet schema, override structure, reward importability, privacy and hashes. +An `ok` verdict means the artifact checks passed; it still returns +`launchable: false` while the fail-closed reward scaffold remains. The template +script also refuses to proceed without the immutable base snapshot and a +completed reward implementation. + +## Unsupported boundary + +The bridge does not translate optimizer state, distributed RNG, FSDP or +Megatron checkpoints, Ray state, PPO advantage/clipping semantics, GRPO group +semantics, or a miniVERL teacher cache into PPO reference log-probabilities. +See [compatibility](compatibility.md), [launch requirements](verl-bridge-launch.md) +and the [demo recording script](verl-bridge-demo.md). diff --git a/mkdocs.yml b/mkdocs.yml index e9a16f6..12aa79e 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -1,29 +1,74 @@ site_name: miniVERL -site_description: Auditable one-GPU online post-training and a pinned verl scale-out bridge +site_description: Auditable single-GPU post-training with a bounded verl artifact bridge site_url: https://daoyuanli2816.github.io/mini-verl/ repo_url: https://github.com/DaoyuanLi2816/mini-verl repo_name: DaoyuanLi2816/mini-verl +edit_uri: edit/main/docs/ theme: - name: readthedocs + name: material + custom_dir: docs/overrides + language: en + font: false + icon: + repo: fontawesome/brands/github + palette: + - media: "(prefers-color-scheme: light)" + scheme: default + primary: indigo + accent: cyan + toggle: + icon: material/weather-night + name: Switch to dark mode + - media: "(prefers-color-scheme: dark)" + scheme: slate + primary: indigo + accent: cyan + toggle: + icon: material/weather-sunny + name: Switch to light mode + features: + - content.code.copy + - content.code.annotate + - navigation.footer + - navigation.indexes + - navigation.sections + - navigation.top + - search.highlight + - search.suggest +plugins: + - search nav: - Home: index.md - - One-GPU quickstart: single-gpu-guide.md - - Alignment Lab: - - Why OPD after SFT?: alignment-lab/when-opd-should-follow-sft.md - - SFT vs DPO vs OPD: alignment-lab/alignment-lab-v1.md - - Consumer runtime: consumer-runtime-v1.md - - RecoveryBench: recoverybench/recoverybench-v1.md - - Verified verl bridge: verl-bridge.md - - 90-second bridge demo: verl-bridge-demo.md + - Start on one GPU: single-gpu-guide.md + - Align: + - When OPD should follow SFT: alignment-lab/when-opd-should-follow-sft.md + - "Alignment Lab v1: saturated case study": alignment-lab/alignment-lab-v1.md + - Distill locally: + - Consumer runtime: consumer-runtime/index.md + - Runtime benchmarking: benchmarking.md + - RecoveryBench: recoverybench/recoverybench-v1.md + - Scale out: + - verl bridge boundary: verl-bridge.md + - Launch requirements: verl-bridge-launch.md + - Demo recording script: verl-bridge-demo.md - Community benchmarks: community-benchmarks.md - - v0.6 launch story: verl-bridge-launch.md - Schemas: - Trajectories: trajectory-schema.md - - Benchmarking: benchmarking.md + - Benchmark results: benchmarking.md - Limitations: limitations.md markdown_extensions: - - tables - - fenced_code - admonition + - attr_list + - md_in_html + - tables - toc: permalink: true + - pymdownx.details + - pymdownx.highlight: + anchor_linenums: true + - pymdownx.superfences +extra_css: + - assets/stylesheets/extra.css +extra_javascript: + - assets/javascripts/versioning.js +copyright: miniVERL is an independent project; no endorsement by upstream projects is implied. diff --git a/paper/alignment-lab-v1/alignment-lab-v1.pdf b/paper/alignment-lab-v1/alignment-lab-v1.pdf index 6b26fdfbf10f2d4d0ead121183f619b50b603050..3edc34d88c96fe5f54e1f9813c2ed716d33b52cb 100644 GIT binary patch delta 5551 zcmaiX$?p2hdZx}Q$tPuTlw=X97k|J8o5^^dFqqK>JTMp!fG04baFd=6MY5wCA*wWYE^pmsHA%9t!MaO|N9UB^nd>FFaPhah+dwau9xHW{Ywe= zzy2r6{R)4*!uRJHzK_wb_wAN9?zcQW{OQ*rd>wzi-k<1CzyHHu|J#?q+21;y-(K(M zF}(aHg>SIP7?w4vIH4@L%IA8~nuo@ombW%`>Bn)(BO@oFR=tnKeCF+GVSrRkN9R4e z)2Q>dXb%q2zN>ou2uF4&FrH^wEg3Vn(>h&*%I7B4O7H7)$F1YX@-Ux%o9o89Z*!Q| zZBg7d%ZH=Rr9FHk_d`X)=tdQ;y$!1g8vJ^^(y&hOvYdrTVd^nhel&*?G;O{PZiO(I zp{CcZ4-2RwDCF6eYvOAuluGj;fxo(Vk6*P*@U^_br`*TV_F?fRIpoVeu0nLK)5viR1)3XtA5 zU>4Aqxjw2fE%GyGhJ(2InFtnO8zo|}3M{u0q_4EaEvJz%uXDkOs25Fk*emvDU@>tT zT*n%Bh0o$fCu+xVm_8qKvj;@W6=cvj;`l%srMX)@OnHW;iG9Vb4`=)DBPL}aS`h70 z3n32MfcEv#G8l%PMeP{t%j-YvLIU`fZ?pc|IUHtN z=H;2&gsUSW%gU?xX|9=?beV+bI(Itu&@NffO0AS+oJ)$0jIgDwf1bIV;IMjHdsPYD z@}zaEStI$_9^r+F*0y`M`gA+%^_`Z)7uOoHhXwFsV<{ozk~v5d(kTn(v|Abj8LoWyQZGh8v5RNuHKhaiUOhf;Y-nDz#gV( z@%a?i`LuglqC2)g!gp)Diu~SCwtvR71di2Psy38D;9;-lA|Y8Z!-kFVx7W(=T5D7} zP4|*tWxN%ApNxC-V8YElWYh!sX5CiQ<*SV4kU4wx>*MkGO7iDYGi5f~YS(&{N6#G; z@5cdm@{lXvToOLY6Q^F}@2uS%sXUj&qHGQyBV<^P6FbO2?y;ha!5yk`?l)x;fL9yq z=82sKmsxkaJWtp`tH5`gchBFQj~%G9lPvw`Lr&@qy^w0`Nt?+;S{OO}JB~q_N-EEwqw()!ir$e@(YRX*itq;@Oq3et7d(g+Fy!`F1gVs>{ ztah#o(6|yAiGOHc%q5c@S&J4Z~tLq)|+LX5JA1x*7jj z0PoZ2BD0;|d7Jc+MbLVjhXnI5`^QP#%S&*|)>1_)MJC+tmkEyg&YK&kx1g8P1s)m2 zp1i5%cvjdN&@F+PO4FZdMOgqe#xwyidpl>d`vfmH0;+OV<;<9*jX)7yO9SuKq7QZG z)Dk^M(6BP1vA*(pP?KlzVy-Iaz9^z4v%^S%+dp?3>EeYvQ@r>^};5bEd+9L{<5Lvjne~Dswq!b^W|NSTTj7%OVFxmPZY0mdY1QJjz3kpHY#kM=7JJ~ zr+(&R9XM~AT4iE&CVD7QKvTUli5?3uKPOMDZm!A)Fo^kTArqFGiQBo%77H9Aaczgp*>l8#4-G{H;&sCqpQlbYWX-O|jcQP}ioF!Rh{ zKi$ojNVZuUDcl=BGRnsCPN`_X4<=~HT`>vD!zz&_$MjYgns~e!Up?HKLfmCwc_j+Y5;*XoE$sm)%_E!Xyxt1Xt9f<~1KP`KA z(%)V_H%MbgbU}b@M?z6oxbkwTB$n86uZcBBwF)Tu?3Ai5%t2)TL z+T?b*V1tHk&?P|%kJN{JHK>>k!;03E){o|Nc)VhKGIi>$FRa-QMZIZFyW#+!`S~R7 zV@rZHqQ%cv&>j_N7!CRLU=+VfLq-%q6Q-Kp$F(wU;h?3-qffkom+8TDD`NZE?KAWg zpbIr-+t1RrEC;dAm1dJvUU7B(c^r*3p1KdCP$Bf=(Gr9&XS{1rG$Im&A8&?8eL?nb z_qeqhVNd6J0xtWz=~B&=PjuF4h*mpCAUlmDw(Zh0EIBv2rCStvxcM55C%WPR?S2a% ziJL07Hz;&CFb3i_O>nwgD$JQd%d}!hYOZQ1i*77%>~GmGV?+h9&O#pMPO_X^sNl05 zo%-B~>0m>^CVI>GG1EjNPxQB5^_h&w`qFOIA0mNUAdrZ!%&XNp1qHk67+ixYy#QcG zGpWnzc$7MC44d_`K2&1ZxXr5V=mu6@5|MjjvNFEw-TQa5e`J39VpW8ikab+qZ=9rt z_dH#z5G&1bgJYxX@n)K|)#tOXCzX@0{ zjjSKLuoi*6(^>J2PEi@W!!|vpC2-Q`u-+0A408vH1OL0mIxt1=BI)s?iK9I7f0+4w z&TW&;yCBoFdth%#SHlTM`Z%$>u0elR3uxx>4g;r*_vSw=;022hSEAM>X!ek^HX#=A z<>{QU{j)jzC{2KON!2$Oqc&d{Pv3)jwXZ52N!a7XckPRfXtNB%_1N4RLa z)rJJY?Hd(2hjv%JQm%I0#vSApM7f4aGiXWMi*3emnK$E}2;y1Abj<-2C3yBA03)4XiTv*kBYy3Wbu^ z;5NE>1Ax?%i@>NPW5XU1KFT}_dTY-Y+1akqT1^U-%XRtHf+qswl+)$PM|BEO75EIa z+kAVGT#wsnwfRKV?)*W*{dy7VTcR$n`AlMp<#%5(^9IfBML0N!_{E&K5?Kp~5({r1ez3*_^ALuGC4~XJKifS2|rV zGw|Mc6xb()c&v~bn;R{Z7oOUGKiw+RDV>(sY&mQ2*w^X=9)j|#zI$0tD&(X%$}}!) z1VaCAEgxxDrN<-MBfEp_3BqsQ%GXNVU*s1V+RauG9km+5WOuV(=nB+xv^hJ=)}B7N zQLQ%b=jIlb`T_~TquJ~%Mzt!|>Qjp|Az;pRG@cFE&o>Gm6x(Iq#!Q9C7@KRmO9l>O zsy|;wI~U(08@jMxd{l)ET6f-tQOO}rwY5FI3n$5+Z#!h)=Xn9{dyyecY7wAOoAw})pbn5e)hT$Yg@tedm6lp*K@RTBeJGNBMe zLL`Yz(9>Zk9~HCd)tFD}yn;?~;~-x&rhCUdJbWuuOV_XLbvMzr3#~>nnhC zw56JXt!wu46Jq$VaxTwj%mp|_?HLWf9=nZ^e<1fd=btp#KPeO0PVnaRIolob?(|iJ zJ9gKecx_bg41VGw+-f~V7enCNvECuXOPdU}Ltgogusm{&N9og@$>xX&(vp%ME@kO) zDRlI*^19c5!uL>adz?TX%7I+U;?gx>(!2J#xyNvP(6eBAn3&Cc1DXkNwN3NjP5E1? zHEd5xrdAnJGZg9Ut(9lgmm=3MD&kMv$Z|tj5S!u`N9~I*RX!RAll^AVD;+{2XO8IT z^!iK}*7b>B^l^tt5|8)5q6w09{({HS#NC)d$3lOMeuw{fTqTkGp>8{#QzD8Rr&9Ox z;oC^=-IE+=(cRAr?y}|h@O2yq-KO;!hq9951LwP-goVC(Jq^%9!$oOxLcEIa6x5*e zTqte0oGa?p6)u_IqS!t4zRa$LmsZVbNT2k>HNi6gG#%O_auTEXV(DFvBsZBbBVTRg zcIntq1}a|QTAH#Ry7rJEyh={h2Appwqn%+-)$jPsd5ysqnaKEE{JHe8TAsSkb}x$s zYOn#5e0!-|$}9>sHd!L77cadM(P;1wU|XINH#u{`ODs>-_G(fyWZ+0d7MpK=JW}ZvlU)h2MvsC191I93Tn$Duw>H36 z)#aGppG3H1+Kp|+TPMw-^pZOk{N8vS+~kfo>DhjCWkU=41W;>pa%l2UL0IuOlW z!|I~Qh!wa_{rQjOfBwfm;J^QC_wSCxuhxI8U)Nv1{pY5I(JD_LxPPC^ns!A)*cISnxW8EjFarK)Bf)ezyDwH{u@eA Lzk%RLAOH66C)8;=)Wh~z#tYk9&v~^^)6MzSqVN-z9STvuFpfHQ_~mB=d|HB-blOrXMdswG z(_u!Nmm_pH^C$(H-GTpTj-tjhI&0N=U)U1GX1lThaV8}2A$ML;AkT%z1k9r0aM4d@ z8#p)<+lsMVoS0jCgWRBo)i<_x+o?`^en-ANT`kyF-RO?hPG3VRq6b)ntJ(H!txV+o zlFVB3^TM+5-B9*8w{+yP>KV_Xa-K@1Sl^aDy>;Q?o~?YFtIt)|y_kbk*{i+9Q@D*g zSoKrhNd{Fb_hZ+rUYT2d8qEi%QFFwwZaKQE4Fq_2Ma-9%yz{N-M0G9F+Z3!i_p4Ne zx4R3@8$}?Kp5f*Un6-X`Og-$B8_bwQSokjId(K=Ia|JA%`zNzmsqfLM@(Qg{<>P0* z(BnE&S%P=dqBE+l@sQLDasAmk*px z?D?eI=b(ZT!aD=iN#?A@^-_Uw?I9dkKa z%;t%%^j&`|6DzjXVn)%_m?tFX=-gOi47SK?)ma3f`tVS}VXMOxeYDcPx<1pkKUyee<@-VFsCX*n9gPM_f

SKAGcTdClraaX{U|ps+0}*2w2;i?$y-D!f#ece!*vXX$g)qpfHTDWCBKX42d`xwlVMls#+? zcz2^~MCFRubO*UlB7t6X{Kz{cqd<|7=?U*lnRjCdr_5IzPpiv!D*9Vxa-;7igFI3E z)?IA^NP8;FopX8Gn0tK>3+#$+q7ScyaN-@|i_6h-h*9LJ!JLbo!}O|@!iSc%&TL`7 zNQ0qSncIZ{CoLZFCVp%L1B{A7*)-d$TCJ$vxASLz{6Tkrw}35Ssa0wvx;K-=uyW0T z^l=%P_dtCz7W?nXU)}!l`+q6&)0&Z_QKwLZj5G9^y!5^W_%2F!Nppc zscQ*a0Ii`cF3csc+B4us7q$4AU&l-vT3B7mtdweW$1lhPf6tVYRNTv(EpZ5VGo)c1 zhSm0~KQO*qSR|YJ$W!OAy=XQhK8k4qz2_9Ywom)!_%<4oq;K~3WoSGi=sh$CTv^|_ zcK~P)I`?OKl&NF&C30aE=7v7g0nJ4?=y}36YFeyk`@Kfx+1_jP3&U-n;?IsdpDhjD z+q1`R+W4xtKgoPpll^Ee4d3R1Ug_9rS?9+40@$FzIh=Joy?=L2pA#UvziXJ*Nfc-x z-W!Tg-&MwtEiA7O6})d=*vswqbEIGkwqL%gsUMsdC5tw7O+%mP1k#yPwcTRE`rujv zJgKvEzVB6?TkwN@U{&U>A6EVeq+$$sx1=I{O(23Mdxs@+yy z=Q)@`&+odJJbIIv{njX@H-4aeDLk};*J4>EKUlEyy=Zlw7UMHG6^g#1Ua0+my4{%O zElBz4Md$9_V9oRlt)#t@VTe6~0g0zk`~<4C(4GlGZ5!m}*3}SR>;z4LdPfX>{-~_1 zCunS|tuIQq&vmA4!+8U#&UWvv?yPMz+hNwW&uB-}HE|SU{F)Gq+J#!O5 z;r6-mjjG4qPUWtJ>Kz@jyT#!Em0wISYmSf`?}^$=J2Xq;sWR_nrNiBg$ZB4qRGThT zmrI2g=lw7#bk34B1t^|ASHni$$3i~`9*4ZiF zg^)p?RaG6-C!e+u=9uNNr1~gdp+fUsaLTBiiNo!Q-{c+Rhpy@JBOD#m15nxG-f}H3 zl*rt~-wP4GDEuyWYJk$-F4mh(v6xiWNf)1$dvu{k+66fkWM@6qC+vX}soPE>&M_RU zm~(RPe`M~`^_~VnwT^a(1<=6fB=56@38%yjpptsuy_=xhq>56ItbPo&)qS#ozsWSN zxv~R(QO9uMW81is`mqj{Omkb_?5^6xo)2L{^7MxGTDPV^*^gKJeDH8bxfVdg%&=wZQF-h~s;Y ztd=cCY%P^7>>OfOdlbiTUaOPTiQCSGp8MWj*eX}Q9@Jws8Zkpw5e{vETF&U_#^vht z;$!5mFQXgn@U$3nsfw|d4~A>Ac}`h+(}i%UKf8|R1wcO)?c+WLkrXlnfmmv}zj}#b z&bCx$5OHl1Z!f9xpx7K^QmOJoSdZ1gkcJd^sGo_!({>IU5Lu6uc-Q=3C`os4S~jVE zujxeJ`3~p-EC?T^4(~h@U}){=0;mrNOuIj8mOx^aBF`FkVBO~(MLM##k$DD@(c#FY zC3k%hPmwDktr6OvL-o;qdZ^N6W$jNM)Z9o3@x_|!W4#l-{CC&Ri^%0rnUz&MTdu@a z-VJ)+=h?h>0T^Gwv*!X9fa;Ytr@=8FYF(Y2l_o-AQ}pfr$QJ0>P z+Gtmfi|f~ zIDbF_*w^I|xPw1dBH_D)+!JVpFy<)}=SC@R@493H5p^tU&l3ZiVdoP&RJuhBEQOQjj zRG22?=lb%HGfQ1pV%oYnaK`k|GqeH!o77vQ(7vD1KFi(M1@ zKIL`gUR$Tq>zP1r9lEu16S!={XtTJ?9?L2bTm!7B1xBSE6ob05TPUw$A$cg=v;afy zU?G4r*3=&Lh`YRAbZHP6`ll)(=w|nZ&!fs4sm01Y>To0ZKBsvn7n`@!6`Tn2AdJk; zSHRMi*9S_a=Kb9TV*?$=b193tZubBlPGf%p)P@fm=`YrU5Pu(?=4P@lQ|*0DvWkSX z8=RL_Ew&*b6&kIE>QB{vDsGJA_&_EHnH0>h&#^zotPRiP^xUPH(eCiF7`nE zuS1dv-DM;9Cw5DQubN7)9B1#TeFv@0&o_eJtL|g^&T2z&Dk4A-7LNVOJ&F43Mnv2Y zZ`vco&ihbeCc9i3LuE zi*9+{tkSRaUf15}tV$c70e57a{rC`18cd-<_NEWY;^g_Ih&atTCL;|m#MTOeaTA)p zep^2^vdvm#|Fw9qyH3xA@hKI~R9i}R5=Korv{N}1fU=O^)K1=adRcnr)bzoVdobbc zlX~iBN_#(fgq8goDh+{ntrb2*d$*ADgLr6-i?}Pr&HW5Xd;BLHuaabCm*SIBYv_$u z|BIpJO5OR)A9ZR5^?`)&8{;+plY^e8aUB*Z^}M*h`rs_9A0I!dw>B&UoUXN+1M;lx z_~R3>l39Ks4<#NZe4vA`WiZ*84^)*GL<-$M*?z^=6g0G|JOy z(r$d)_iS9gF4D&WyghRP7yCu8@;bsRuxEk~W`ZAqgPB%>Od4-Xu74P3J^gWA*hTuR zxW(bJvOZXNt2jC&sn^d39nd3lx`3f6nU@Q0>0|be{dTpnZcV%HweAe?yCo-s=ejhB z6Mthq#t9c+!<);BRtg-_@PZ0a^z%e${Q*-&Jiem9<6KU6^KKqyDwn6FSg7jfn}3t% zXG)vyV&>4|2f*7}mes>CVdyI`-Xq~y=<8GI;Fu6)*}>Dx zeo@q~CO`T8OQ*=I>kwV%*1ENO_>U&sBXIKOcuuc_dq3Tl(f=#)r@vv7pBr_5LC{p5 G{rWFlT|T-1 diff --git a/paper/alignment-lab-v1/build_report.py b/paper/alignment-lab-v1/build_report.py index fe6b5a4..c45e551 100644 --- a/paper/alignment-lab-v1/build_report.py +++ b/paper/alignment-lab-v1/build_report.py @@ -49,12 +49,12 @@ RED = colors.HexColor("#be123c") LIGHT = colors.HexColor("#f1f5f9") PALETTE = [ - colors.HexColor("#94a3b8"), - colors.HexColor("#60a5fa"), - colors.HexColor("#a78bfa"), - colors.HexColor("#fbbf24"), - colors.HexColor("#fb7185"), - colors.HexColor("#34d399"), + colors.HexColor("#A7A9AC"), + colors.HexColor("#0072B2"), + colors.HexColor("#CC79A7"), + colors.HexColor("#E69F00"), + colors.HexColor("#D55E00"), + colors.HexColor("#009E73"), ] LABELS = { @@ -220,91 +220,111 @@ def _table(rows: list[list[Any]], widths: list[float], *, header: bool = True) - return table -class QualityUtilityPlot(Flowable): - """Small vector quality-versus-utility plot bound to method means.""" +class DeltaForestPlot(Flowable): + """Vector forest plot with exact seed values and data-bound mean marks.""" - def __init__(self, summary: list[dict[str, Any]], width: float = 6.7 * inch): + def __init__(self, result: dict[str, Any], width: float = 6.7 * inch): super().__init__() - self.summary = summary + self.result = result self.width = width - self.height = 3.0 * inch + self.height = 3.15 * inch + + @staticmethod + def _x(value: float, left: float, chart_w: float) -> float: + if value < -40 or value > 5: + raise ValueError(f"Alignment Lab delta outside forest domain: {value}") + return left + chart_w * (value + 40) / 45 + + @staticmethod + def _mark(canvas: Canvas, x: float, y: float, seed: int, color: colors.Color) -> None: + canvas.setStrokeColor(color) + canvas.setFillColor(color) + if seed == 1234: + canvas.circle(x, y, 2.2, fill=1, stroke=0) + elif seed == 20260727: + canvas.saveState() + canvas.translate(x, y) + canvas.rotate(45) + canvas.rect(-2.1, -2.1, 4.2, 4.2, fill=1, stroke=0) + canvas.restoreState() + else: + canvas.setLineWidth(1.5) + canvas.line(x - 2.2, y - 2.2, x + 2.2, y + 2.2) + canvas.line(x + 2.2, y - 2.2, x - 2.2, y + 2.2) def draw(self) -> None: canvas = self.canv - left, bottom = 0.62 * inch, 0.48 * inch - chart_w, chart_h = 4.65 * inch, 2.18 * inch - canvas.setStrokeColor(colors.HexColor("#cbd5e1")) - canvas.rect(left, bottom, chart_w, chart_h, fill=0, stroke=1) - canvas.setFont("Helvetica", 7) - canvas.setFillColor(MUTED) - for index in range(6): - value = 0.8 + 0.04 * index - x = left + chart_w * (value - 0.8) / 0.2 - y = bottom + chart_h * (value - 0.8) / 0.2 - canvas.setStrokeColor(colors.HexColor("#e2e8f0")) - canvas.line(x, bottom, x, bottom + chart_h) - canvas.line(left, y, left + chart_w, y) + left, chart_w = 2.05 * inch, 4.45 * inch + bottom, top = 0.34 * inch, self.height - 0.22 * inch + baselines = { + int(arm["seed"]): arm + for arm in self.result["arms"] + if arm["method"] == "sft_checkpoint" + } + for tick in (-40, -30, -20, -10, 0, 5): + x = self._x(float(tick), left, chart_w) + canvas.setStrokeColor(INK if tick == 0 else colors.HexColor("#dbe3ee")) + canvas.setLineWidth(1.2 if tick == 0 else 0.45) + canvas.line(x, bottom, x, top) canvas.setFillColor(MUTED) - canvas.drawCentredString(x, bottom - 11, f"{value:.2f}") - canvas.drawRightString(left - 5, y - 2, f"{value:.2f}") - for index, row in enumerate(self.summary): - utility = float(row["tool_utility_retention_mean"]) - quality = float(row["alignment_score_mean"]) - x = left + chart_w * (utility - 0.8) / 0.2 - y = bottom + chart_h * (quality - 0.8) / 0.2 - canvas.setStrokeColor(PALETTE[index]) - canvas.setLineWidth(2) - canvas.circle(x, y, 4 + index * 1.4, fill=0, stroke=1) - legend_y = bottom + chart_h - index * 20 - canvas.setFillColor(PALETTE[index]) - canvas.circle(left + chart_w + 26, legend_y, 3.4, fill=1, stroke=0) - canvas.setFillColor(INK) - canvas.setFont("Helvetica", 7.2) - canvas.drawString(left + chart_w + 35, legend_y - 2.5, LABELS[row["method"]]) - canvas.setFont("Helvetica", 7.2) - canvas.setFillColor(MUTED) - canvas.drawCentredString(left + chart_w / 2, 3, "tool utility retention") - canvas.saveState() - canvas.translate(8, bottom + chart_h / 2) - canvas.rotate(90) - canvas.drawCentredString(0, 0, "alignment score") - canvas.restoreState() - - -class QueryCostBars(Flowable): - """Compact teacher-query and GPU-time comparison.""" - - def __init__(self, summary: list[dict[str, Any]], width: float = 6.7 * inch): - super().__init__() - self.summary = summary - self.width = width - self.height = 2.25 * inch - - def draw(self) -> None: - canvas = self.canv - label_w = 1.55 * inch - bar_w = 3.7 * inch - max_time = max(float(row["gpu_seconds_mean"]) for row in self.summary) or 1 - for index, row in enumerate(self.summary): - y = self.height - 17 - index * 24 - canvas.setFillColor(INK) - canvas.setFont("Helvetica", 7.2) - canvas.drawString(0, y + 2, LABELS[row["method"]]) - canvas.setFillColor(PALETTE[index]) - time = float(row["gpu_seconds_mean"]) - canvas.roundRect(label_w, y, bar_w * time / max_time, 7, 3, fill=1, stroke=0) - ratio = row.get("teacher_query_ratio_mean") - ratio_text = "n/a" if ratio is None else f"{100 * float(ratio):.1f}%" + canvas.setFont("Helvetica", 6.8) + canvas.drawCentredString(x, bottom - 10, f"{tick:+d}") + methods = tuple(method for method in LABELS if method != "sft_checkpoint") + for index, method in enumerate(methods): + y = top - 20 - index * 34 + arms = sorted( + (arm for arm in self.result["arms"] if arm["method"] == method), + key=lambda arm: (1234, 20260727, 20260801).index(int(arm["seed"])), + ) + alignment = [ + 100 + * ( + float(arm["metrics"]["alignment_score"]) + - float(baselines[int(arm["seed"])]["metrics"]["alignment_score"]) + ) + for arm in arms + ] + utility = [ + 100 + * ( + float(arm["metrics"]["tool_utility_retention"]) + - float(baselines[int(arm["seed"])]["metrics"]["tool_utility_retention"]) + ) + for arm in arms + ] + color = PALETTE[index + 1] canvas.setFillColor(INK) - canvas.setFont("Helvetica-Bold", 7.2) - canvas.drawRightString(self.width, y + 1, f"{time:.1f}s · query {ratio_text}") + canvas.setFont("Helvetica-Bold", 6.9) + canvas.drawString(0, y + 3, LABELS[method]) + canvas.setFont("Helvetica", 5.8) + canvas.setFillColor(MUTED) + canvas.drawString( + 0, + y - 7, + "A " + "/".join(f"{value:+.1f}" for value in alignment), + ) + canvas.drawString( + 0.93 * inch, + y - 7, + "U " + "/".join(f"{value:+.1f}" for value in utility), + ) + for arm, value in zip(arms, alignment, strict=True): + self._mark(canvas, self._x(value, left, chart_w), y + 4, int(arm["seed"]), color) + for arm, value in zip(arms, utility, strict=True): + self._mark(canvas, self._x(value, left, chart_w), y - 4, int(arm["seed"]), color) + alignment_mean = sum(alignment) / len(alignment) + utility_mean = sum(utility) / len(utility) + canvas.setStrokeColor(color) + canvas.setLineWidth(1.5) + canvas.circle(self._x(alignment_mean, left, chart_w), y + 4, 4.2, fill=0, stroke=1) + ux = self._x(utility_mean, left, chart_w) + canvas.rect(ux - 4.2, y - 8.2, 8.4, 8.4, fill=0, stroke=1) canvas.setFillColor(MUTED) - canvas.setFont("Helvetica", 7) + canvas.setFont("Helvetica", 6.4) canvas.drawString( - label_w, - 2, - "continuation GPU time; query ratio is selected positions / generated positions", + 0, 2, "A = alignment; U = tool utility; seed shapes: circle / diamond / cross" ) + canvas.drawCentredString(left + chart_w / 2, 2, "delta from same-seed SFT checkpoint (pp)") def _page(canvas: Canvas, doc: SimpleDocTemplate) -> None: @@ -332,9 +352,9 @@ def build(output: Path = OUTPUT) -> None: story.extend( [ Spacer(1, 0.48 * inch), - _p("Online Policy Distillation After SFT on One Consumer GPU", styles["title"]), + _p("Alignment Lab v1: a saturated tool-policy case study", styles["title"]), _p( - "Alignment Lab v1 technical report · miniVERL v0.5.0 · Daoyuan Li", + "Alignment Lab v1 technical report · miniVERL v0.6.1 · Daoyuan Li", styles["subtitle"], ), _p( @@ -357,6 +377,12 @@ def build(output: Path = OUTPUT) -> None: "a scoped no-continuation decision, not a broad claim that OPD is ineffective.", styles["body"], ), + _p( + "Preference win rate is the deterministic Minipolicy paired outcome, not human " + "preference. The two zero-valued safety-policy metrics are sandbox checks, not " + "a broad safety benchmark; IFEval, XSTest, HarmBench and RewardBench were not run.", + styles["body"], + ), _p( "Keywords: alignment, on-policy distillation, DPO, retained utility, verifier gating, consumer GPU", styles["small"], @@ -448,6 +474,7 @@ def build(output: Path = OUTPUT) -> None: if row["teacher_query_ratio_mean"] is None else f"{100 * row['teacher_query_ratio_mean']:.1f}%", f"{row['gpu_seconds_mean']:.1f}s", + f"{row['peak_vram_bytes_max'] / 2**30:.2f} GiB", ] ) story.extend( @@ -455,32 +482,37 @@ def build(output: Path = OUTPUT) -> None: PageBreak(), _p("4. Final three-seed result", styles["h1"]), _table( - [["Method", "Align", "Harm", "Over", "Utility", "Query", "GPU"], *result_rows], [ - 1.44 * inch, - 0.62 * inch, - 0.58 * inch, - 0.58 * inch, - 0.66 * inch, - 0.64 * inch, + ["Method", "Align", "Harm", "Over", "Utility", "Query", "GPU", "VRAM"], + *result_rows, + ], + [ + 1.25 * inch, + 0.55 * inch, + 0.52 * inch, + 0.52 * inch, + 0.59 * inch, 0.62 * inch, + 0.57 * inch, + 0.72 * inch, ], ), Spacer(1, 0.15 * inch), - QualityUtilityPlot(summary), + DeltaForestPlot(result), _p( - "Concentric points at 1.0 / 1.0 denote exact mean overlap, not algorithmic " - "equivalence. Every non-overlapping regression remains in the primary table.", + "The zero line is the same-seed starting SFT checkpoint. Outlined circle/square " + "marks are three-seed alignment/utility means; the smaller seed shapes remain at " + "their exact values. Every regression remains visible.", styles["center"], ), Spacer(1, 0.1 * inch), _p("Cost and query accounting", styles["h2"]), - QueryCostBars(summary), _p( - "DPO time includes its external pinned TRL training. Evaluation is excluded from " - "the continuation-GPU-time axis. Query ratio counts selected positions and does " - "not imply proportional teacher-backbone FLOP savings.", - styles["small"], + "The outcome-and-cost table is the PDF matrix: alignment, utility, query, GPU " + "time and peak VRAM share one row per method. Non-teacher query cells are n/a, " + "never zero. DPO time includes its pinned TRL training; evaluation is excluded. " + "Query ratio counts selected positions, not teacher-backbone FLOPs.", + styles["body"], ), ] ) @@ -654,7 +686,7 @@ def build(output: Path = OUTPUT) -> None: leftMargin=0.66 * inch, topMargin=0.62 * inch, bottomMargin=0.7 * inch, - title="Online Policy Distillation After SFT on One Consumer GPU", + title="Alignment Lab v1: a saturated tool-policy case study", author="Daoyuan Li", subject="miniVERL Alignment Lab v1", ) diff --git a/pyproject.toml b/pyproject.toml index cce4b3c..3b78c27 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -75,6 +75,8 @@ dev = [ "twine>=5.1", "pyarrow>=15", "mkdocs>=1.6,<2", + "mkdocs-material==9.7.7", + "playwright==1.62.0", ] [project.scripts] diff --git a/scripts/check_docs_visual.py b/scripts/check_docs_visual.py new file mode 100644 index 0000000..d37ba69 --- /dev/null +++ b/scripts/check_docs_visual.py @@ -0,0 +1,199 @@ +#!/usr/bin/env python3 +"""Build-time browser assertions for documentation layout and generated SVGs.""" + +from __future__ import annotations + +import argparse +import contextlib +import functools +import http.server +import re +import threading +from pathlib import Path +from typing import Any + +VIEWPORTS = ((1440, 900), (1024, 768), (820, 1000), (390, 844)) +PAGES = ( + "/", + "/alignment-lab/alignment-lab-v1/", + "/consumer-runtime/", + "/recoverybench/recoverybench-v1/", + "/verl-bridge/", +) + + +class _QuietHandler(http.server.SimpleHTTPRequestHandler): + def log_message(self, format: str, *args: Any) -> None: + del format, args + + +@contextlib.contextmanager +def _server(site: Path): + handler = functools.partial(_QuietHandler, directory=str(site)) + server = http.server.ThreadingHTTPServer(("127.0.0.1", 0), handler) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + try: + yield f"http://127.0.0.1:{server.server_port}" + finally: + server.shutdown() + thread.join(timeout=5) + server.server_close() + + +def _assert_page(page: Any, *, route: str, width: int) -> list[str]: + overflow = page.evaluate( + "() => ({client: document.documentElement.clientWidth, scroll: document.documentElement.scrollWidth})" + ) + if overflow["scroll"] > overflow["client"] + 1: + raise AssertionError(f"horizontal document overflow at {route}: {overflow}") + + figure_problems = page.evaluate( + """() => { + const content = document.querySelector('.md-content__inner'); + if (!content) return ['missing content column']; + const cr = content.getBoundingClientRect(); + return [...document.querySelectorAll('.md-typeset img, .md-typeset picture')] + .map((node) => ({node, rect: node.getBoundingClientRect()})) + .filter(({rect}) => rect.width > cr.width + 1 || rect.left < cr.left - 1 || rect.right > cr.right + 1) + .map(({node, rect}) => `${node.tagName}:${node.getAttribute('src') || ''}:${rect.width}/${cr.width}`); + }""" + ) + if figure_problems: + raise AssertionError(f"figure exceeds content column at {route}: {figure_problems}") + + table_problems = page.evaluate( + """() => [...document.querySelectorAll('.md-typeset table')].filter((table) => { + if (table.scrollWidth <= table.clientWidth + 1) return false; + const wrapper = table.closest('.md-typeset__scrollwrap'); + if (!wrapper) return true; + const overflow = getComputedStyle(wrapper).overflowX; + return !['auto', 'scroll'].includes(overflow); + }).map((table) => table.textContent.slice(0, 80))""" + ) + if table_problems: + raise AssertionError( + f"table is neither wrapped nor horizontally scrollable at {route}: {table_problems}" + ) + + if route == "/verl-bridge/" and width == 390: + current = page.locator("picture.bridge-architecture img").evaluate("img => img.currentSrc") + if not current.endswith("verl-bridge-architecture-mobile.svg"): + raise AssertionError(f"mobile bridge did not select the vertical layout: {current}") + + return page.locator('img[src*=".svg"], picture img').evaluate_all( + "nodes => nodes.map(node => node.currentSrc || node.src).filter(src => src.endsWith('.svg'))" + ) + + +def _assert_svg(page: Any, url: str) -> None: + page.set_viewport_size({"width": 1400, "height": 1200}) + page.goto(url, wait_until="load") + result = page.evaluate( + """() => { + const svg = document.querySelector('svg'); + if (!svg || !svg.viewBox || !svg.viewBox.baseVal) return {error: 'missing SVG viewBox'}; + const root = svg.getBoundingClientRect(); + const vb = svg.viewBox.baseVal; + const scale = Math.min(820, vb.width) / vb.width; + const outside = []; + for (const node of svg.querySelectorAll('text, [data-role]')) { + const rect = node.getBoundingClientRect(); + if (rect.left < root.left - 1 || rect.top < root.top - 1 || + rect.right > root.right + 1 || rect.bottom > root.bottom + 1) { + outside.push(`${node.tagName}:${(node.textContent || '').trim().slice(0, 60)}`); + } + } + const labels = [...svg.querySelectorAll('[data-role="chart-label"], [data-role="diagram-label"]')]; + const overlap = []; + for (let i = 0; i < labels.length; i += 1) { + const a = labels[i].getBoundingClientRect(); + for (let j = i + 1; j < labels.length; j += 1) { + const b = labels[j].getBoundingClientRect(); + const area = Math.max(0, Math.min(a.right, b.right) - Math.max(a.left, b.left)) * + Math.max(0, Math.min(a.bottom, b.bottom) - Math.max(a.top, b.top)); + if (area > 1) overlap.push(`${i}:${j}`); + } + } + const legends = [...svg.querySelectorAll('[data-role="legend"]')]; + const plots = [...svg.querySelectorAll('[data-role="plot-region"]')]; + const legendPlotOverlap = []; + for (const legend of legends) for (const plot of plots) { + const a = legend.getBoundingClientRect(); const b = plot.getBoundingClientRect(); + if (Math.max(0, Math.min(a.right, b.right) - Math.max(a.left, b.left)) * + Math.max(0, Math.min(a.bottom, b.bottom) - Math.max(a.top, b.top)) > 1) { + legendPlotOverlap.push('legend overlaps plotting region'); + } + } + const tooSmall = labels.filter((node) => { + const size = parseFloat(getComputedStyle(node).fontSize || '0') * scale; + return size > 0 && size < 10.5; + }).map((node) => `${getComputedStyle(node).fontSize}:${(node.textContent || '').trim().slice(0, 50)}`); + return {outside, overlap, legendPlotOverlap, tooSmall}; + }""" + ) + if result.get("error"): + raise AssertionError(result["error"]) + if result["outside"]: + raise AssertionError(f"SVG label outside viewBox in {url}: {result['outside']}") + if result["overlap"]: + raise AssertionError(f"chart label overlap in {url}: {result['overlap']}") + if result["legendPlotOverlap"]: + raise AssertionError(f"legend overlaps the plotting region in {url}") + if result["tooSmall"]: + raise AssertionError( + f"chart label is smaller than 10.5 px at 820 px in {url}: {result['tooSmall']}" + ) + + +def check(site: Path, screenshots: Path) -> None: + try: + from playwright.sync_api import sync_playwright + except ImportError as exc: # pragma: no cover - exercised in CI setup + raise SystemExit("install the pinned Playwright dev dependency first") from exc + + screenshots.mkdir(parents=True, exist_ok=True) + with _server(site) as base_url, sync_playwright() as playwright: + browser = playwright.chromium.launch() + svg_urls: set[str] = set() + try: + for width, height in VIEWPORTS: + context = browser.new_context( + viewport={"width": width, "height": height}, + color_scheme="dark", + device_scale_factor=1, + locale="en-US", + reduced_motion="reduce", + ) + page = context.new_page() + for route in PAGES: + page.goto(f"{base_url}{route}", wait_until="networkidle") + svg_urls.update(_assert_page(page, route=route, width=width)) + slug = "home" if route == "/" else re.sub(r"[^a-z0-9]+", "-", route).strip("-") + page.screenshot( + path=str(screenshots / f"{width}x{height}-{slug}.png"), + full_page=True, + animations="disabled", + ) + context.close() + svg_page = browser.new_page() + for url in sorted(svg_urls): + _assert_svg(svg_page, url) + svg_page.close() + finally: + browser.close() + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--site", type=Path, default=Path("site")) + parser.add_argument("--screenshots", type=Path, default=Path("docs-visual-screenshots")) + args = parser.parse_args() + if not args.site.is_dir(): + parser.error(f"site directory does not exist: {args.site}") + check(args.site.resolve(), args.screenshots.resolve()) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/check_markdown_links.py b/scripts/check_markdown_links.py index 405e0ec..728613e 100644 --- a/scripts/check_markdown_links.py +++ b/scripts/check_markdown_links.py @@ -61,7 +61,7 @@ def _anchors(path: Path) -> set[str]: def check_markdown_links(root: Path) -> list[str]: """Return every broken repository-local target.""" - ignored_parts = {".git", ".venv", "dist", "build", "runs"} + ignored_parts = {".artifacts", ".git", ".venv", "dist", "build", "runs", "site"} files = [ path for path in root.rglob("*.md") @@ -73,10 +73,10 @@ def check_markdown_links(root: Path) -> list[str]: lines = _active_lines(source) for number, line in enumerate(lines, start=1): targets = [ - *_MARKDOWN_TARGET.findall(line), - *_HTML_TARGET.findall(line), + *((target, False) for target in _MARKDOWN_TARGET.findall(line)), + *((target, True) for target in _HTML_TARGET.findall(line)), ] - for raw_target in targets: + for raw_target, is_raw_html in targets: if raw_target.startswith(("http://", "https://", "mailto:", "data:")): continue if raw_target.startswith("#"): @@ -84,7 +84,18 @@ def check_markdown_links(root: Path) -> list[str]: fragment = raw_target[1:] else: path_text, separator, fragment = raw_target.partition("#") - target_path = (source.parent / unquote(path_text)).resolve() + relative_base = source.parent + if ( + is_raw_html + and source.suffix.lower() == ".md" + and (root.resolve() / "docs") in source.parents + ): + relative_base = ( + source.parent + if source.name.lower() == "index.md" + else source.parent / source.stem + ) + target_path = (relative_base / unquote(path_text)).resolve() if not separator: fragment = "" try: diff --git a/scripts/publish_alignment_lab_artifacts.py b/scripts/publish_alignment_lab_artifacts.py index c648460..601fd25 100644 --- a/scripts/publish_alignment_lab_artifacts.py +++ b/scripts/publish_alignment_lab_artifacts.py @@ -50,12 +50,12 @@ "verifier_gated_opd": "verifier-gated OPD", } COLORS = { - "sft_checkpoint": "#94a3b8", - "continued_sft": "#60a5fa", - "dpo": "#a78bfa", - "offline_distillation": "#fbbf24", - "standard_opd": "#fb7185", - "verifier_gated_opd": "#34d399", + "sft_checkpoint": "#A7A9AC", + "continued_sft": "#0072B2", + "dpo": "#CC79A7", + "offline_distillation": "#E69F00", + "standard_opd": "#D55E00", + "verifier_gated_opd": "#009E73", } @@ -571,240 +571,415 @@ def _load_result(path: Path) -> dict[str, Any]: return payload -def _svg_shell(title: str, description: str, subtitle: str, body: list[str]) -> str: +def _svg_shell( + title: str, description: str, subtitle: str, body: list[str], *, height: int = 720 +) -> str: style = ( - "text{font-family:Inter,'Segoe UI',sans-serif;fill:#edf4ff}" - ".title{font-size:29px;font-weight:760}.sub{font-size:16px;fill:#9fb0cc}" - ".axis{font-size:14px;fill:#8fa2bf}.label{font-size:15px;font-weight:650}" - ".value{font-size:15px;font-weight:760}.foot{font-size:15px;fill:#91a1bd}" + "text{font-family:'DejaVu Sans','Segoe UI',sans-serif;fill:#edf4ff}" + ".title{font-size:31px;font-weight:760}.sub{font-size:17px;fill:#aebbd2}" + ".axis{font-size:17px;fill:#aebbd2}.label{font-size:18px;font-weight:650}" + ".value{font-size:17px;font-weight:760}.small{font-size:16px;fill:#b9c5d8}" + ".header{font-size:17px;font-weight:700;fill:#dce7f8}" ) return "".join( [ - '', + f'', f'{escape(title)}', f'{escape(description)}', - '', - '', + f'', + f'', f"", - f'{escape(title)}', - f'{escape(subtitle)}', + f'{escape(title)}', + f'{escape(subtitle)}', *body, "\n", ] ) -def _scatter_axes(*, x_label: str, y_label: str) -> list[str]: - body: list[str] = [] - left, right, top, bottom = 110, 840, 155, 500 - for tick in range(6): - px = left + tick * (right - left) / 5 - py = bottom - tick * (bottom - top) / 5 - value = tick / 5 - body.extend( - [ - f'', - f'{value:.1f}', - f'', - f'{value:.1f}', - ] - ) - body.extend( - [ - f'{escape(x_label)}', - f'{escape(y_label)}', - ] +def _scale(value: float, *, low: float, high: float, start: float, end: float) -> float: + if value < low - 1e-9 or value > high + 1e-9: + raise ValueError(f"quantitative value {value} lies outside [{low}, {high}]") + return start + (value - low) * (end - start) / (high - low) + + +def _seed_mark(*, x: float, y: float, method: str, seed: int, value: float) -> str: + color = COLORS[method] + common = ( + f'data-encoding="seed-point" data-seed="{seed}" data-value="{value:.10g}" ' + f'fill="{color}" stroke="#07111f" stroke-width="1.5"' + ) + if seed == SEEDS[0]: + return f'' + if seed == SEEDS[1]: + points = f"{x:.1f},{y - 5:.1f} {x + 5:.1f},{y:.1f} {x:.1f},{y + 5:.1f} {x - 5:.1f},{y:.1f}" + return f'' + return ( + f'' ) - return body -def _legend() -> list[str]: - body: list[str] = [] - for index, method in enumerate(METHODS): - y = 171 + index * 48 - body.extend( - [ - f'', - f'{escape(LABELS[method])}', - ] - ) - return body +def _method_arms(payload: dict[str, Any], method: str) -> list[dict[str, Any]]: + rows = sorted( + (arm for arm in payload["arms"] if arm["method"] == method), + key=lambda arm: SEEDS.index(int(arm["seed"])), + ) + if [int(row["seed"]) for row in rows] != list(SEEDS): + raise ValueError(f"method {method} does not contain the three measured seeds") + return rows -def _xy(value: float, *, low: float, high: float, start: float, end: float) -> float: - if math.isclose(low, high): - return (start + end) / 2 - return start + (value - low) * (end - start) / (high - low) +def _pp(value: float) -> str: + return "0.0" if math.isclose(value, 0.0, abs_tol=5e-5) else f"{value:+.1f}" -def _quality_utility(payload: dict[str, Any], source: str) -> str: - body = _scatter_axes(x_label="tool utility retention", y_label="alignment score") - for index, row in enumerate(payload["method_summary"]): - method = row["method"] - x = 110 + float(row["tool_utility_retention_mean"]) * 730 - y = 500 - float(row["alignment_score_mean"]) * 345 - # Concentric rings keep methods visible when their means overlap exactly. - radius = 9 + index * 4 - body.append( - f'' +def _delta_from_sft(payload: dict[str, Any]) -> str: + body: list[str] = [ + 'Continuation method', + 'delta from the same-seed SFT checkpoint (percentage points)', + '', + 'alignment mean', + '', + 'utility', + ] + left, right = 370.0, 1040.0 + low, high = -40.0, 5.0 + for tick in (-40, -30, -20, -10, 0, 5): + x = _scale(float(tick), low=low, high=high, start=left, end=right) + stroke = "#f4f7fb" if tick == 0 else "#233553" + width = 2.5 if tick == 0 else 1 + body.extend( + [ + f'', + f'{tick:+d}', + ] ) - body.extend(_legend()) - perfect = sum( - 1 - for row in payload["method_summary"] - if math.isclose(float(row["alignment_score_mean"]), 1.0) - and math.isclose(float(row["tool_utility_retention_mean"]), 1.0) + body.append( + f'zero baseline' ) - body.extend( - [ - '', - f'{perfect} / 6 methods', - 'at the 1.0 / 1.0 ceiling', - '', - f'Concentric rings denote exact overlap · source SHA-256 {source[:16]}', + baseline = {int(arm["seed"]): arm for arm in _method_arms(payload, "sft_checkpoint")} + for index, method in enumerate(METHODS[1:]): + y = 210 + index * 86 + arms = _method_arms(payload, method) + alignment = [ + 100 + * ( + float(arm["metrics"]["alignment_score"]) + - float(baseline[int(arm["seed"])]["metrics"]["alignment_score"]) + ) + for arm in arms ] + utility = [ + 100 + * ( + float(arm["metrics"]["tool_utility_retention"]) + - float(baseline[int(arm["seed"])]["metrics"]["tool_utility_retention"]) + ) + for arm in arms + ] + align_mean = statistics.fmean(alignment) + utility_mean = statistics.fmean(utility) + body.extend( + [ + f'{escape(LABELS[method])}', + f'A seeds: {" / ".join(_pp(v) for v in alignment)}', + f'U seeds: {" / ".join(_pp(v) for v in utility)}', + f'', + ] + ) + for arm, value in zip(arms, alignment, strict=True): + body.append( + _seed_mark( + x=_scale(value, low=low, high=high, start=left, end=right), + y=y - 11, + method=method, + seed=int(arm["seed"]), + value=value, + ) + ) + for arm, value in zip(arms, utility, strict=True): + body.append( + _seed_mark( + x=_scale(value, low=low, high=high, start=left, end=right), + y=y + 11, + method=method, + seed=int(arm["seed"]), + value=value, + ) + ) + ax = _scale(align_mean, low=low, high=high, start=left, end=right) + ux = _scale(utility_mean, low=low, high=high, start=left, end=right) + body.extend( + [ + f'', + f'', + f'A {_pp(align_mean)}', + f'U {_pp(utility_mean)}', + ] + ) + body.append( + 'Seed shapes: ● 1234 · ◆ 20260727 · × 20260801. A = alignment; U = retained tool utility.' ) return _svg_shell( - "SFT starts at the ceiling; continuation adds no gain", - "Mean alignment score versus retained tool utility for six methods across three seeds.", - "Qwen3-0.6B · 48 paired test tasks · 3 seeds · deterministic Minipolicy v1", + "Continuation could not improve the saturated SFT checkpoint", + ( + "Forest chart of alignment-score and retained-tool-utility percentage-point deltas " + "for five continuation methods. Every seed is shown at its exact x value." + ), + "48 paired sandbox tasks per seed · mean marks plus all three measured seeds · zero = starting SFT", body, ) -def _safety_overrefusal(payload: dict[str, Any], source: str) -> str: - body = _scatter_axes(x_label="over-refusal rate", y_label="harmful-compliance rate") - for index, row in enumerate(payload["method_summary"]): - method = row["method"] - x = 110 + float(row["over_refusal_rate_mean"]) * 730 - y = 500 - float(row["harmful_compliance_rate_mean"]) * 345 - radius = 9 + index * 4 +def _matrix_cell( + *, + payload: dict[str, Any], + x: float, + y: float, + values: list[float], + mean: float, + domain_high: float, + method: str, + formatter: Any, + main: float | None = None, +) -> list[str]: + bar_width = 84.0 + main_value = mean if main is None else main + main_x = _scale(main_value, low=0.0, high=domain_high, start=x, end=x + bar_width) + body = [ + f'', + f'', + ] + for arm, value in zip(_method_arms(payload, method), values, strict=True): body.append( - f'' + _seed_mark( + x=_scale(value, low=0.0, high=domain_high, start=x, end=x + bar_width), + y=y, + method=method, + seed=int(arm["seed"]), + value=value, + ) ) - body.extend(_legend()) - body.extend( - [ - '', - '0% / 0%', - 'harmful / over-refusal', - '', - f'Deterministic sandbox policy checks, not a broad safety benchmark · source {source[:16]}', - ] - ) - return _svg_shell( - "Safety-policy checks can pass while benign utility regresses", - "Harmful compliance and over-refusal do not capture every policy or utility failure.", - "Exact validators · 48 paired tasks per seed · no real destructive actions", - body, + body.append( + f'{escape(formatter(main_value))}' ) + return body -def _preference_cost(payload: dict[str, Any], source: str) -> str: - rows = payload["method_summary"] - max_cost = max(float(row["gpu_seconds_mean"]) for row in rows) * 1.15 - body: list[str] = [] - left, right, top, bottom = 110, 840, 155, 500 - for tick in range(6): - value = max_cost * tick / 5 - px = left + tick * (right - left) / 5 +def _outcome_cost_matrix(payload: dict[str, Any]) -> str: + columns = (250.0, 415.0, 580.0, 745.0, 910.0) + body: list[str] = [ + 'Method', + 'Alignment', + '0–100%', + 'Tool utility', + '0–100%', + 'Teacher query', + '0–100%', + 'GPU time', + '0–100 seconds', + 'Peak VRAM', + '0–2 GiB', + ] + summary = {row["method"]: row for row in payload["method_summary"]} + for index, method in enumerate(METHODS): + y = 195 + index * 78 + arms = _method_arms(payload, method) + row = summary[method] + alignment = [100 * float(arm["metrics"]["alignment_score"]) for arm in arms] + utility = [100 * float(arm["metrics"]["tool_utility_retention"]) for arm in arms] + time_values = [float(arm["cost"]["gpu_seconds"]) for arm in arms] + vram_values = [float(arm["cost"]["peak_vram_bytes"]) / 2**30 for arm in arms] body.extend( [ - f'', - f'{value:.0f}', + f'', + f'', + f'{escape(LABELS[method])}', ] ) - for value in (0.8, 0.9, 1.0): - py = _xy(value, low=0.8, high=1.0, start=bottom, end=top) body.extend( - [ - f'', - f'{value:.1f}', - ] + _matrix_cell( + payload=payload, + x=columns[0], + y=y, + values=alignment, + mean=statistics.fmean(alignment), + domain_high=100, + method=method, + formatter=lambda value: f"{value:.1f}%", + ) ) - body.extend( - [ - 'continuation GPU time (seconds, mean)', - 'preference win rate', - ] - ) - for index, row in enumerate(rows): - method = row["method"] - px = _xy(float(row["gpu_seconds_mean"]), low=0, high=max_cost, start=left, end=right) - py = ( - _xy( - float(row["preference_win_rate_mean"]), - low=0.8, - high=1.0, - start=bottom, - end=top, + body.extend( + _matrix_cell( + payload=payload, + x=columns[1], + y=y, + values=utility, + mean=statistics.fmean(utility), + domain_high=100, + method=method, + formatter=lambda value: f"{value:.1f}%", ) - + (index - 2.5) * 5 ) + query_values = [arm["metrics"]["teacher_query_ratio"] for arm in arms] + if all(value is None for value in query_values): + body.append( + f'— not applicable' + ) + elif any(value is None for value in query_values): + raise ValueError(f"method {method} mixes applicable and non-applicable query ratios") + else: + query_percent = [100 * float(value) for value in query_values] + body.extend( + _matrix_cell( + payload=payload, + x=columns[2], + y=y, + values=query_percent, + mean=statistics.fmean(query_percent), + domain_high=100, + method=method, + formatter=lambda value: f"{value:.1f}%", + ) + ) body.extend( - [ - f'', - f'{float(row["gpu_seconds_mean"]):.1f}s', - ] + _matrix_cell( + payload=payload, + x=columns[3], + y=y, + values=time_values, + mean=float(row["gpu_seconds_mean"]), + domain_high=100, + method=method, + formatter=lambda value: f"{value:.1f}s", + ) + ) + body.extend( + _matrix_cell( + payload=payload, + x=columns[4], + y=y, + values=vram_values, + mean=statistics.fmean(vram_values), + main=float(row["peak_vram_bytes_max"]) / 2**30, + domain_high=2, + method=method, + formatter=lambda value: f"{value:.2f} GiB", + ) ) - body.extend(_legend()) body.extend( [ - '', - f'DPO includes external TRL training; evaluation excluded from GPU-time axis · source {source[:16]}', + 'Bars show the three-seed mean except VRAM, whose main bar is the observed maximum; seed shapes show every run.', + 'Query ratio is selected target positions, not teacher FLOPs. DPO time includes its pinned TRL job.', ] ) return _svg_shell( - "No method beats the starting policy; continuation cost differs", - "Preference outcome versus continuation GPU time for six methods.", - "Three-seed means · four continuation updates except the frozen SFT checkpoint", + "Outcome and continuation-cost matrix", + ( + "Row matrix of alignment, retained tool utility, teacher-query ratio, continuation " + "GPU time and peak VRAM for every method and seed." + ), + "Direct labels + seed marks · non-teacher methods remain not applicable, never zero", body, + height=740, ) -def _quality_query(payload: dict[str, Any], source: str) -> str: - rows = payload["method_summary"] - body = _scatter_axes(x_label="teacher queried-position ratio", y_label="alignment score") - non_teacher_y = 1.0 - for index, row in enumerate(rows): - method = row["method"] - ratio = row["teacher_query_ratio_mean"] - if ratio is None: - ratio = 0.0 - px = 110 + float(ratio) * 730 - py = 500 - float(row["alignment_score_mean"]) * 345 + (index - 2.5) * 5 - body.append( - f'' +def _metric_coverage_matrix(payload: dict[str, Any]) -> str: + summary = {row["method"]: row for row in payload["method_summary"]} + baseline = {int(arm["seed"]): arm for arm in _method_arms(payload, "sft_checkpoint")} + body: list[str] = [ + 'Method', + 'Harmful compliance', + 'seed values', + 'Over-refusal', + 'seed values', + 'Tool utility Δ', + 'mean · seeds (pp)', + 'Sandbox endpoint', + 'measured?', + 'External safety', + 'executed?', + ] + for index, method in enumerate(METHODS): + y = 196 + index * 67 + arms = _method_arms(payload, method) + harmful = [100 * float(arm["metrics"]["harmful_compliance_rate"]) for arm in arms] + refusal = [100 * float(arm["metrics"]["over_refusal_rate"]) for arm in arms] + utility_delta = [ + 100 + * ( + float(arm["metrics"]["tool_utility_retention"]) + - float(baseline[int(arm["seed"])]["metrics"]["tool_utility_retention"]) + ) + for arm in arms + ] + body.extend( + [ + f'', + f'', + f'{escape(LABELS[method])}', + f'{" · ".join(f"{v:.0f}%" for v in harmful)}', + f'{" · ".join(f"{v:.0f}%" for v in refusal)}', + f'{_pp(100 * (float(summary[method]["tool_utility_retention_mean"]) - 1.0))}', + f'{" / ".join(_pp(v) for v in utility_delta)}', + f'YES', + f'NOT RUN', + ] ) - if row["teacher_query_ratio_mean"] is None: - non_teacher_y = py - body.extend(_legend()) body.extend( [ - f'no teacher calls', - '', - f'Query ratio counts selected positions, not teacher backbone FLOPs · source {source[:16]}', + '', + 'The two sandbox safety checks tied at zero while utility still regressed.', + 'IFEval, XSTest, HarmBench and RewardBench were not executed; this is not a broad safety benchmark.', ] ) return _svg_shell( - "Fewer teacher targets do not guarantee better alignment", - "Alignment outcome versus measured teacher queried-position ratio.", - "Three-seed means · ratios are measured selected positions / generated positions", + "Metric coverage: zero sandbox failures did not imply full utility retention", + ( + "Coverage matrix showing zero harmful-compliance and over-refusal rates, observed " + "tool-utility deltas, measured sandbox endpoints and unexecuted external benchmarks." + ), + "All three measured seeds are printed · deterministic Minipolicy checks only", body, ) +def assert_chart_suitability(figures: dict[str, str]) -> None: + """Reject known misleading fallbacks in generated Alignment Lab figures.""" + expected = { + "delta-from-sft.svg", + "outcome-cost-matrix.svg", + "metric-coverage-matrix.svg", + } + if set(figures) != expected: + raise ValueError(f"Alignment Lab must publish exactly {sorted(expected)}") + combined = "\n".join(figures.values()).lower() + if "concentric" in combined or 'data-encoding="jitter"' in combined: + raise ValueError("method-order rings and unlabelled jitter are forbidden") + outcome = figures["outcome-cost-matrix.svg"] + if "— not applicable" not in outcome or 'data-applicable="false"' not in outcome: + raise ValueError("non-teacher query ratios must remain explicitly not applicable") + if 'data-applicable="false" data-value="0"' in outcome: + raise ValueError("not-applicable query ratios must never be coerced to zero") + if 'data-encoding="seed-point"' not in combined: + raise ValueError("every chart set must expose the measured seed values") + + def render_figures(payload: dict[str, Any], source_sha256: str) -> dict[str, str]: - return { - "quality-vs-utility.svg": _quality_utility(payload, source_sha256), - "safety-vs-overrefusal.svg": _safety_overrefusal(payload, source_sha256), - "preference-vs-gpu-time.svg": _preference_cost(payload, source_sha256), - "quality-vs-teacher-query.svg": _quality_query(payload, source_sha256), + del source_sha256 # hashes live in the report provenance block, never in the plot canvas + rendered = { + "delta-from-sft.svg": _delta_from_sft(payload), + "outcome-cost-matrix.svg": _outcome_cost_matrix(payload), + "metric-coverage-matrix.svg": _metric_coverage_matrix(payload), } + assert_chart_suitability(rendered) + return rendered def render_cards(payload: dict[str, Any], source_sha256: str) -> dict[str, str]: @@ -937,7 +1112,7 @@ def render_report(payload: dict[str, Any], source_sha256: str) -> str: ) else: negative_rows = "- None; all continuation arms tied the saturated baseline." - return f"""# Online Policy Distillation After SFT on One Consumer GPU + return f"""# Alignment Lab v1: a saturated tool-policy case study ## Abstract @@ -969,6 +1144,9 @@ def render_report(payload: dict[str, Any], source_sha256: str) -> str: no real destructive operation is executed. IFEval, XSTest, HarmBench and RewardBench are represented by pinned metadata adapters only and are **not** measured endpoints in this artifact. +“Preference win rate” is the deterministic Minipolicy paired outcome, not a +human-preference measurement. Harmful compliance and over-refusal are sandbox +policy checks, not a broad safety result. ## Final result @@ -976,13 +1154,20 @@ def render_report(payload: dict[str, Any], source_sha256: str) -> str: | --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | {chr(10).join(rows)} -![Alignment quality versus utility retention](quality-vs-utility.svg) +![Forest chart of alignment and tool-utility deltas from the saturated SFT checkpoint, with every seed and the three-seed means](delta-from-sft.svg) + +![Row matrix of alignment, retained tool utility, teacher-query ratio, continuation GPU time and peak VRAM, including every measured seed](outcome-cost-matrix.svg) + +![Coverage matrix showing tied zero sandbox safety checks, utility regressions and external safety benchmarks not executed](metric-coverage-matrix.svg) -![Safety-policy outcome versus over-refusal](safety-vs-overrefusal.svg) +
+Figure provenance -![Preference outcome versus continuation GPU time](preference-vs-gpu-time.svg) +- Result SHA-256: `{source_sha256}` +- Task-level result SHA-256: `{payload["task_results_sha256"]}` +- Three seed identities: `1234`, `20260727`, `20260801` -![Alignment quality versus teacher-query ratio](quality-vs-teacher-query.svg) +
The starting checkpoint defines a ceiling; overlapping continuation points are not evidence of algorithmic equivalence, and every non-overlapping regression diff --git a/scripts/publish_verl_bridge_diagrams.py b/scripts/publish_verl_bridge_diagrams.py new file mode 100644 index 0000000..f9630ee --- /dev/null +++ b/scripts/publish_verl_bridge_diagrams.py @@ -0,0 +1,150 @@ +#!/usr/bin/env python3 +"""Generate responsive, claim-bounded verl bridge diagrams.""" + +from __future__ import annotations + +import argparse +from pathlib import Path +from xml.sax.saxutils import escape + +from miniverl.bridge.contract import VERL_COMMIT, VERL_TAG + +ROOT = Path(__file__).resolve().parents[1] +DOCS = ROOT / "docs" +SHORT_COMMIT = VERL_COMMIT[:8] + + +def _shell(*, width: int, height: int, title: str, description: str, body: list[str]) -> str: + style = ( + "text{font-family:'DejaVu Sans','Segoe UI',sans-serif;fill:#edf4ff}" + ".title{font-size:30px;font-weight:760}.sub{font-size:17px;fill:#aebbd2}" + ".layer{font-size:20px;font-weight:760}.body{font-size:16px;fill:#c2cee1}" + ".role{font-size:15px;font-weight:700}.status{font-size:22px;font-weight:800}" + ".foot{font-size:15px;fill:#9fb0ca}" + ) + return "".join( + [ + f'', + f'{escape(title)}', + f'{escape(description)}', + f'', + f"", + *body, + "\n", + ] + ) + + +def _desktop() -> str: + title = "Scale-out bridge: verified artifacts, bounded claims" + description = ( + "Three verified layers connect miniVERL local runtime artifacts to a pinned verl " + "parse and load smoke. A dashed arrow leads to distributed execution marked not tested." + ) + body = [ + f'{title}', + f'pinned upstream {VERL_TAG} · {SHORT_COMMIT} · independent project; no endorsement', + '', + '1 · miniVERL local runtime', + 'single-GPU training, evaluation and portable provenance', + '', + 'teacher role · targets', + '', + 'reference role · DPO', + '', + 'reward role · verifier', + '', + 'student · local updates', + '', + '', + '', + '2 · portable artifact bundle', + 'standard formats with explicit config and provenance boundaries', + 'PEFT', + 'safetensors', + 'Parquet', + 'resolved config', + 'typed provenance', + '', + '', + '', + '3 · pinned upstream parse/load smoke', + f'verl {VERL_TAG} at {SHORT_COMMIT} · config parse + PEFT/safetensors/Parquet structural load checks', + 'verified boundary: artifact interchange and the documented profile subset', + '', + '', + '', + 'Distributed execution: NOT TESTED', + 'No Ray / FSDP / vLLM job ran · no OPD-to-PPO semantic-parity claim', + 'unverified execution layer', + ] + return _shell(width=1120, height=760, title=title, description=description, body=body) + + +def _mobile() -> str: + title = "miniVERL → verl bridge" + description = ( + "Vertical mobile diagram with local runtime, portable bundle and pinned upstream smoke " + "as verified layers, followed by Distributed execution: NOT TESTED." + ) + body = [ + f'{title}', + f'{VERL_TAG} · {SHORT_COMMIT}', + 'independent project; no endorsement', + '', + '1 · miniVERL local runtime', + 'single-GPU training + evaluation', + 'teacher role · targets', + 'reference role · DPO', + 'reward role · verifier', + 'student · local updates', + '', + '', + '2 · portable artifact bundle', + 'standard, reviewable interchange', + 'PEFT', + 'safetensors', + 'Parquet', + 'config', + 'typed provenance', + '', + '', + '3 · pinned upstream smoke', + f'verl {VERL_TAG} at {SHORT_COMMIT}', + 'config parse + artifact load checks', + 'verified: documented profile subset', + '', + '', + 'Distributed execution:', + 'NOT TESTED', + 'No Ray / FSDP / vLLM job ran', + 'No OPD-to-PPO semantic parity', + 'unverified execution layer', + ] + return _shell(width=390, height=1048, title=title, description=description, body=body) + + +def render_diagrams() -> dict[str, str]: + return { + "verl-bridge-architecture.svg": _desktop(), + "verl-bridge-architecture-mobile.svg": _mobile(), + } + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--check", action="store_true") + args = parser.parse_args() + for name, content in render_diagrams().items(): + path = DOCS / name + if args.check: + if not path.is_file() or path.read_text(encoding="utf-8") != content: + raise SystemExit(f"generated bridge diagram is stale: {path}") + else: + path.write_text(content, encoding="utf-8", newline="\n") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/verify_verl_bridge_smoke.py b/scripts/verify_verl_bridge_smoke.py index 1e7056b..50ecd2f 100644 --- a/scripts/verify_verl_bridge_smoke.py +++ b/scripts/verify_verl_bridge_smoke.py @@ -1,4 +1,4 @@ -"""Verify a Level-3 bundle against the installed exact verl source snapshot.""" +"""Verify a miniVERL-defined Level-3 bundle against the pinned verl snapshot.""" from __future__ import annotations @@ -112,6 +112,16 @@ def verify_smoke(bundle: str | Path, *, out: str | Path) -> dict[str, Any]: "model_or_adapter_load": diagnosis["model_adapter_loadability"]["peft_config_load"], "parquet_load": "train and val passed", "reward_scaffold_import": "passed; fail-closed scorer not executed", + "artifact_bundle_complete": diagnosis["artifact_bundle_complete"], + "upstream_config_parse_passed": True, + "model_data_load_smoke_passed": True, + "model_data_load_smoke_scope": ( + "PEFT config, safetensors structure and both Parquet splits; base weights not loaded" + ), + "reward_implementation_complete": False, + "launchable": False, + "distributed_execution_tested": False, + "algorithm_semantic_parity": False, "tiny_cpu_dry_run": { "status": "artifact-only", "reason": "full PPO execution requires the excluded distributed inference stack", diff --git a/src/miniverl/bridge/config.py b/src/miniverl/bridge/config.py index 0e70826..05fb969 100644 --- a/src/miniverl/bridge/config.py +++ b/src/miniverl/bridge/config.py @@ -1,12 +1,13 @@ -"""Fail-closed import for the pinned verl single-GPU distillation profile.""" +"""Fail-closed import for the pinned, documented verl profile subset.""" from __future__ import annotations import hashlib +import math import uuid from collections.abc import Mapping from pathlib import Path -from typing import Any +from typing import Any, Literal import yaml @@ -23,31 +24,103 @@ __all__ = ["import_verl_config"] -_MAPPED: dict[str, tuple[str | None, str]] = { - "data.train_files": (None, "bridge_metadata"), - "data.val_files": (None, "bridge_metadata"), - "data.prompt_key": (None, "bridge_metadata"), - "data.max_prompt_length": ("rollout.max_total_tokens", "mapped"), - "data.max_response_length": ("rollout.max_new_tokens_per_turn", "mapped"), - "data.seed": ("run.seed", "mapped"), - "actor_rollout_ref.model.path": ("models.student.model_id", "mapped"), +FieldClass = Literal[ + "exact", + "derived", + "informational_only", + "requires_user_confirmation", + "unsupported", +] + +_FIELD_RULES: dict[str, tuple[str | None, FieldClass, str]] = { + "data.train_files": ( + None, + "informational_only", + "miniVERL trains against an explicitly selected local ToolEnvironment, not this Parquet path", + ), + "data.val_files": ( + None, + "informational_only", + "miniVERL evaluates an explicitly selected local ToolEnvironment, not this Parquet path", + ), + "data.prompt_key": ( + None, + "informational_only", + "the selected miniVERL environment owns prompt construction", + ), + "data.max_prompt_length": ( + "rollout.max_total_tokens", + "derived", + "combined with max_response_length; miniVERL has a total trajectory-token bound", + ), + "data.max_response_length": ( + "rollout.max_new_tokens_per_turn", + "exact", + "copied as the per-turn generation bound", + ), + "data.seed": ("run.seed", "exact", "copied without a unit change"), + "actor_rollout_ref.model.path": ( + "models.student.model_id", + "exact", + "copied as the student model identity", + ), "actor_rollout_ref.model.enable_gradient_checkpointing": ( "models.student.gradient_checkpointing", - "mapped", + "exact", + "copied as a model construction option", + ), + "actor_rollout_ref.actor.optim.lr": ( + "train.learning_rate", + "exact", + "copied as the optimizer learning rate", + ), + "trainer.save_freq": ( + "train.save_every_cycles", + "requires_user_confirmation", + "verl frequency units are not proven equivalent to miniVERL cycles", + ), + "trainer.test_freq": ( + "train.eval_every_cycles", + "requires_user_confirmation", + "verl frequency units are not proven equivalent to miniVERL cycles", + ), + "trainer.project_name": ( + "run.name", + "derived", + "combined with trainer.experiment_name", + ), + "trainer.experiment_name": ( + "run.name", + "derived", + "combined with trainer.project_name", + ), + "trainer.total_epochs": ( + "train.cycles", + "requires_user_confirmation", + "epochs and miniVERL continuation cycles are not proven equivalent", + ), + "trainer.logger": (None, "informational_only", "not used by the local runtime"), + "trainer.resume_mode": (None, "informational_only", "not used by the import"), + "trainer.default_local_dir": ( + None, + "informational_only", + "not copied because output provenance is rooted at the requested destination", ), - "actor_rollout_ref.actor.optim.lr": ("train.learning_rate", "mapped"), - "trainer.save_freq": ("train.save_every_cycles", "mapped"), - "trainer.test_freq": ("train.eval_every_cycles", "mapped"), - "trainer.project_name": ("run.name", "mapped"), - "trainer.experiment_name": ("run.name", "mapped"), - "trainer.total_epochs": ("train.cycles", "mapped"), } -_IGNORED_INFORMATIONAL = { - "trainer.logger", - "trainer.resume_mode", - "trainer.default_local_dir", +_LOSS_PROFILES: dict[str, dict[str, str]] = { + "topk-tail-reverse-kl": { + "mode": "bucketed_topk_tail", + "divergence": "reverse_kl", + }, + "topk-tail-forward-kl": { + "mode": "bucketed_topk_tail", + "divergence": "forward_kl", + }, + "exact-reverse-kl": {"mode": "exact_full_vocab", "divergence": "reverse_kl"}, + "exact-forward-kl": {"mode": "exact_full_vocab", "divergence": "forward_kl"}, } +_SCHEDULE_MAPPING = "epochs-as-cycles" def _digest_bytes(value: bytes) -> str: @@ -81,9 +154,26 @@ def _integer(value: Any, field: str, *, minimum: int = 0) -> int: def _positive_number(value: Any, field: str) -> float: - if isinstance(value, bool) or not isinstance(value, (int, float)) or float(value) <= 0: - raise ConfigError(f"verl field {field} must be a positive number") - return float(value) + if isinstance(value, str): + stripped = value.strip() + if "${" in stripped: + raise ConfigError( + f"verl field {field} contains an unresolved interpolation", + hint="pass a fully resolved, documented profile before importing", + ) + try: + parsed = float(stripped) + except ValueError as exc: + raise ConfigError( + f"verl field {field} must be a finite positive number; got {value!r}" + ) from exc + elif isinstance(value, bool) or not isinstance(value, (int, float)): + raise ConfigError(f"verl field {field} must be a finite positive number") + else: + parsed = float(value) + if not math.isfinite(parsed) or parsed <= 0: + raise ConfigError(f"verl field {field} must be a finite positive number") + return parsed def _atomic_yaml(path: Path, payload: dict[str, Any]) -> bytes: @@ -97,10 +187,107 @@ def _atomic_yaml(path: Path, payload: dict[str, Any]) -> bytes: return rendered -def _generated_recipe(source: Mapping[str, Any]) -> tuple[dict[str, Any], list[dict[str, Any]]]: +def _source_identity() -> dict[str, str]: + return {"repository": VERL_REPOSITORY, "tag": VERL_TAG, "commit": VERL_COMMIT} + + +def _classify(flat: Mapping[str, Any]) -> dict[str, dict[str, Any]]: + classified: dict[str, dict[str, Any]] = {} + for path, value in sorted(flat.items()): + target, classification, reason = _FIELD_RULES.get( + path, + (None, "unsupported", "outside the documented resolved-profile subset"), + ) + classified[path] = { + "target": target, + "classification": classification, + "value": portable_payload(value), + "reason": reason, + } + return classified + + +def _required_inputs( + *, + student_model: str, + environment: str | None, + teacher_model: str | None, + teacher_adapter: str | None, + loss_profile: str | None, + schedule_mapping: str | None, +) -> list[dict[str, str]]: + required: list[dict[str, str]] = [] + if not environment: + required.append( + { + "field": "environment", + "reason": "Parquet file names do not identify a miniVERL ToolEnvironment", + "supply": "--environment ", + } + ) + teacher_is_unqualified_same_base = teacher_model == student_model and not teacher_adapter + if (not teacher_model and not teacher_adapter) or teacher_is_unqualified_same_base: + required.append( + { + "field": "teacher_identity", + "reason": ( + "the source does not establish a distinct teacher or a same-base teacher adapter" + ), + "supply": "--teacher-model and/or --teacher-adapter ", + } + ) + if not loss_profile: + required.append( + { + "field": "loss_profile", + "reason": "the source profile does not determine a miniVERL distillation objective", + "supply": f"--loss-profile <{'|'.join(sorted(_LOSS_PROFILES))}>", + } + ) + if not schedule_mapping: + required.append( + { + "field": "schedule_mapping", + "reason": "verl epochs/frequencies are not proven equivalent to miniVERL cycles", + "supply": f"--schedule-mapping {_SCHEDULE_MAPPING}", + } + ) + return required + + +def _template(source: Mapping[str, Any], required: list[dict[str, str]]) -> dict[str, Any]: + return { + "schema_version": 1, + "status": "needs_user_input", + "note": "This is a non-executable import template, not a miniVERL RunConfig.", + "source_profile": BRIDGE_PROFILE, + "source_values": portable_payload(dict(source)), + "required_user_input": required, + } + + +def _generated_recipe( + source: Mapping[str, Any], + *, + environment: str, + teacher_model: str | None, + teacher_adapter: str | None, + loss_profile: str, + schedule_mapping: str, +) -> tuple[dict[str, Any], list[dict[str, Any]]]: model_id = _get(source, "actor_rollout_ref.model.path") if not isinstance(model_id, str) or not model_id.strip(): raise ConfigError("verl field actor_rollout_ref.model.path is required") + if loss_profile not in _LOSS_PROFILES: + raise ConfigError( + f"unsupported loss profile {loss_profile!r}", + hint=f"choose one of {', '.join(sorted(_LOSS_PROFILES))}", + ) + if schedule_mapping != _SCHEDULE_MAPPING: + raise ConfigError( + f"unsupported schedule mapping {schedule_mapping!r}", + hint=f"use --schedule-mapping {_SCHEDULE_MAPPING} to explicitly accept the unit change", + ) project = _get(source, "trainer.project_name", "verl-import") experiment = _get(source, "trainer.experiment_name", "profile") if not isinstance(project, str) or not isinstance(experiment, str): @@ -127,6 +314,16 @@ def _generated_recipe(source: Mapping[str, Any]) -> tuple[dict[str, Any], list[d "verl field actor_rollout_ref.model.enable_gradient_checkpointing must be boolean" ) + resolved_teacher = teacher_model or model_id + teacher: dict[str, Any] = { + "model_id": resolved_teacher, + "dtype": "auto", + "quantization": "none", + "mode": "standard", + } + if teacher_adapter: + teacher["adapter"] = {"path": teacher_adapter, "source": "local"} + recipe: dict[str, Any] = { "schema_version": 1, "run": { @@ -148,16 +345,11 @@ def _generated_recipe(source: Mapping[str, Any]) -> tuple[dict[str, Any], list[d "gradient_checkpointing": gradient_checkpointing, "lora": {"enabled": True}, }, - "teacher": { - "model_id": model_id, - "dtype": "auto", - "quantization": "none", - "mode": "standard", - }, + "teacher": teacher, }, "environment": { - "name": "calculator", - "params": {"protocol_version": "v2", "prompt_style": "compact"}, + "name": environment, + "params": {}, "train_tasks": 64, "eval_tasks": 32, "test_tasks": 32, @@ -168,11 +360,7 @@ def _generated_recipe(source: Mapping[str, Any]) -> tuple[dict[str, Any], list[d "max_total_tokens": prompt_length + response_length, }, "selection": {"selector": "all_model_tokens"}, - "loss": { - "mode": "bucketed_topk_tail", - "divergence": "reverse_kl", - "top_k": 64, - }, + "loss": {**_LOSS_PROFILES[loss_profile], "top_k": 64}, "train": { "cycles": cycles, "learning_rate": learning_rate, @@ -180,26 +368,26 @@ def _generated_recipe(source: Mapping[str, Any]) -> tuple[dict[str, Any], list[d "eval_every_cycles": test_freq, "opd_freshness": "strict", }, - "cache": { - "strict_policy_version": True, - "reuse_across_policy_versions": False, - }, + "cache": {"strict_policy_version": True, "reuse_across_policy_versions": False}, } defaults = [ { - "field": "models.teacher", - "value": "policy-conditioned same-base teacher", - "reason": "the profile imports no separate teacher identity", + "field": "environment split sizes", + "value": {"train": 64, "eval": 32, "test": 32}, + "reason": "the verl subset has file paths but no miniVERL task-pool sizes", + "source_run_intent": False, }, { - "field": "environment", - "value": "calculator protocol-v2 scaffold", - "reason": "verl prompt data does not identify a miniVERL tool environment", + "field": "selection.selector", + "value": "all_model_tokens", + "reason": "the verl subset does not encode miniVERL token provenance selection", + "source_run_intent": False, }, { - "field": "loss", - "value": "reverse_kl top-k-plus-tail", - "reason": "PPO/GRPO semantics are intentionally outside this profile", + "field": "loss.top_k", + "value": 64, + "reason": "profile constant used only for a top-k + tail loss profile", + "source_run_intent": False, }, ] return recipe, defaults @@ -211,8 +399,13 @@ def import_verl_config( profile: str, target_verl: str, out: str | Path, + environment: str | None = None, + teacher_model: str | None = None, + teacher_adapter: str | None = None, + loss_profile: str | None = None, + schedule_mapping: str | None = None, ) -> dict[str, Any]: - """Import only the documented profile and emit a complete decision report.""" + """Import the resolved profile subset or emit a non-executable template.""" if profile != BRIDGE_PROFILE: raise ConfigError( f"unsupported verl bridge profile {profile!r}", hint=f"use --profile {BRIDGE_PROFILE}" @@ -228,75 +421,122 @@ def import_verl_config( raise ConfigError("verl config must contain one YAML mapping") flat = _flatten(payload) + classification = _classify(flat) unsupported = sorted( - path for path in flat if path not in _MAPPED and path not in _IGNORED_INFORMATIONAL + path + for path, decision in classification.items() + if decision["classification"] == "unsupported" ) + report_path = Path(out).parent / "import-report.json" + common: dict[str, Any] = { + "schema_version": 2, + "source_verl": _source_identity(), + "profile": BRIDGE_PROFILE, + "input_contract": "resolved documented profile subset; not arbitrary verl YAML", + "source_config_sha256": _digest_bytes(source_bytes), + "field_classification": classification, + "unsupported_fields": unsupported, + "semantic_conflicts": [], + } if unsupported: rejection_report = { - "schema_version": 1, - "source_verl": { - "repository": VERL_REPOSITORY, - "tag": VERL_TAG, - "commit": VERL_COMMIT, - }, - "profile": BRIDGE_PROFILE, - "source_config_sha256": _digest_bytes(source_bytes), - "mapped_fields": {}, - "ignored_informational_fields": [], - "unsupported_fields": unsupported, - "semantic_conflicts": [], + **common, "inserted_defaults": [], + "required_user_input": [], "generated_miniverl_sha256": None, + "generated_recipe_validated": False, + "generated_path": None, "status": "rejected", } - write_json_atomic(Path(out).parent / "import-report.json", rejection_report) + write_json_atomic(report_path, rejection_report) raise ConfigError( f"unsupported verl field {unsupported[0]!r} for profile {BRIDGE_PROFILE}", - hint="remove algorithm, distributed, rollout-runtime or unknown fields; inspect the documented whitelist", + hint=( + "remove algorithm, distributed, rollout-runtime or unknown fields; " + "inspect the documented resolved-profile whitelist" + ), ) - recipe, inserted_defaults = _generated_recipe(payload) + student_model = _get(payload, "actor_rollout_ref.model.path") + if not isinstance(student_model, str) or not student_model.strip(): + raise ConfigError("verl field actor_rollout_ref.model.path is required") + required = _required_inputs( + student_model=student_model, + environment=environment, + teacher_model=teacher_model, + teacher_adapter=teacher_adapter, + loss_profile=loss_profile, + schedule_mapping=schedule_mapping, + ) destination = Path(out) + if required: + template_path = destination.parent / "imported.template.yaml" + template = _template(payload, required) + rendered = yaml.safe_dump(template, sort_keys=False, allow_unicode=True, width=100).encode( + "utf-8" + ) + report = { + **common, + "inserted_defaults": [], + "required_user_input": required, + "user_confirmations": {}, + "generated_miniverl_sha256": _digest_bytes(rendered), + "generated_recipe_validated": False, + "generated_path": template_path.name, + "status": "needs_user_input", + "claim": "No runnable miniVERL recipe was generated.", + } + _atomic_yaml(template_path, template) + try: + write_json_atomic(report_path, report) + except BaseException: + template_path.unlink(missing_ok=True) + raise + return report + + assert environment is not None + assert loss_profile is not None + assert schedule_mapping is not None + recipe, inserted_defaults = _generated_recipe( + payload, + environment=environment, + teacher_model=teacher_model, + teacher_adapter=teacher_adapter, + loss_profile=loss_profile, + schedule_mapping=schedule_mapping, + ) + try: + from miniverl.config import RunConfig + + RunConfig.from_mapping(recipe) + except Exception as exc: + raise ConfigError( + f"generated miniVERL recipe failed RunConfig validation: {exc}", + hint="check the explicit environment, teacher and loss-profile arguments", + ) from exc rendered = yaml.safe_dump(recipe, sort_keys=False, allow_unicode=True, width=100).encode( "utf-8" ) - mapped_fields = { - path: { - "target": target, - "disposition": disposition, - "value": portable_payload(flat[path]), - } - for path, (target, disposition) in _MAPPED.items() - if path in flat - } - ignored = [ - {"field": path, "value": portable_payload(flat[path])} - for path in sorted(_IGNORED_INFORMATIONAL.intersection(flat)) - ] - report: dict[str, Any] = { - "schema_version": 1, - "source_verl": { - "repository": VERL_REPOSITORY, - "tag": VERL_TAG, - "commit": VERL_COMMIT, - }, - "profile": BRIDGE_PROFILE, - "source_config_sha256": _digest_bytes(source_bytes), - "mapped_fields": mapped_fields, - "ignored_informational_fields": ignored, - "unsupported_fields": [], - "semantic_conflicts": [], + report = { + **common, "inserted_defaults": inserted_defaults, + "required_user_input": [], + "user_confirmations": { + "environment": environment, + "teacher_model": teacher_model, + "teacher_adapter": teacher_adapter, + "loss_profile": loss_profile, + "schedule_mapping": schedule_mapping, + }, "generated_miniverl_sha256": _digest_bytes(rendered), + "generated_recipe_validated": True, + "generated_path": destination.name, "status": "accepted", - "claim": ( - "Imports the documented single-gpu-online-distillation-v1 subset of pinned verl v0.8.0." - ), + "claim": "Imports only the resolved documented profile subset for pinned verl v0.8.0.", } - _atomic_yaml(destination, recipe) try: - write_json_atomic(destination.parent / "import-report.json", report) + write_json_atomic(report_path, report) except BaseException: destination.unlink(missing_ok=True) raise diff --git a/src/miniverl/bridge/contract.py b/src/miniverl/bridge/contract.py index 0d96405..2ee9768 100644 --- a/src/miniverl/bridge/contract.py +++ b/src/miniverl/bridge/contract.py @@ -27,7 +27,7 @@ 0: "conceptual post-training flow", 1: "standard artifact interoperability", 2: "versioned config-field whitelist", - 3: "validated pinned scale-out bundle", + 3: "miniVERL-defined validated pinned artifact bundle", } diff --git a/src/miniverl/bridge/doctor.py b/src/miniverl/bridge/doctor.py index 44b749a..0f2cef9 100644 --- a/src/miniverl/bridge/doctor.py +++ b/src/miniverl/bridge/doctor.py @@ -208,9 +208,12 @@ def _check_reward(root: Path) -> dict[str, Any]: raise ImportError("compute_score is not callable") except Exception as exc: return {"status": "fail", "detail": str(exc)} + source = path.read_text(encoding="utf-8") + implementation_complete = "complete and test reward_or_verifier_scaffold" not in source return { "status": "ok", "detail": "side-effect-free import; scaffold intentionally not executed", + "implementation_complete": implementation_complete, } @@ -294,10 +297,10 @@ def inspect_bridge_bundle(root: str | Path, *, require_verl: bool = False) -> di except (OSError, json.JSONDecodeError): compatibility = {} checks = (target, model, tokenizer, parquet, config, reward, hashes, privacy) - failed = any(check.get("status") != "ok" for check in checks) - if require_verl and installed.get("status") != "ok": - failed = True - local_smoke = "failed" if failed else "passed" + artifact_failed = any(check.get("status") != "ok" for check in checks) + pinned_verl_failed = require_verl and installed.get("status") != "ok" + failed = artifact_failed or pinned_verl_failed + local_smoke = "failed" if artifact_failed else "passed" return { "target_verl": target, "installed_verl": installed, @@ -310,6 +313,19 @@ def inspect_bridge_bundle(root: str | Path, *, require_verl: bool = False) -> di "artifact_hashes": hashes, "privacy": privacy, "local_smoke_status": local_smoke, + "artifact_bundle_complete": not artifact_failed, + "upstream_config_parse_passed": bool( + compatibility.get("upstream_config_parse_passed", False) + ), + "model_data_load_smoke_passed": bool( + compatibility.get("model_data_load_smoke_passed", False) + ), + "reward_implementation_complete": bool(reward.get("implementation_complete", False)), + "launchable": False, + "distributed_execution_tested": bool( + compatibility.get("distributed_execution_tested", False) + ), + "algorithm_semantic_parity": bool(compatibility.get("algorithm_semantic_parity", False)), "distributed_execution_status": compatibility.get( "distributed_execution_status", "not tested" ), diff --git a/src/miniverl/bridge/export.py b/src/miniverl/bridge/export.py index 40f01a3..76edbf5 100644 --- a/src/miniverl/bridge/export.py +++ b/src/miniverl/bridge/export.py @@ -1,9 +1,10 @@ -"""Transactional Level-3 verl bundle export from standard miniVERL artifacts.""" +"""Transactional miniVERL-defined Level-3 artifact-bundle export.""" from __future__ import annotations import hashlib import json +import math import re import shlex import shutil @@ -133,15 +134,150 @@ def _adapter_contract(source: Path) -> dict[str, Any]: } -def _verl_overrides(adapter: dict[str, Any]) -> dict[str, Any]: +def _get(payload: dict[str, Any], path: str) -> Any: + current: Any = payload + for part in path.split("."): + if not isinstance(current, dict) or part not in current: + return None + current = current[part] + return current + + +def _source_config(run: Path) -> tuple[dict[str, Any], str | None]: + for name in ("config.resolved.yaml", "config.original.yaml"): + path = run / name + if not path.is_file(): + continue + try: + payload = yaml.safe_load(path.read_text(encoding="utf-8")) + except (OSError, yaml.YAMLError) as exc: + raise ConfigError(f"cannot read source-run {name}: {exc}") from exc + if not isinstance(payload, dict): + raise ConfigError(f"source-run {name} must contain a YAML mapping") + return payload, name + return {}, None + + +def _source_run_values(config: dict[str, Any]) -> dict[str, Any]: + fields = ( + "run.name", + "run.seed", + "environment.name", + "rollout.max_total_tokens", + "rollout.max_new_tokens_per_turn", + "train.learning_rate", + "train.cycles", + "train.save_every_cycles", + "train.eval_every_cycles", + ) return { + field: portable_payload(value) + for field in fields + if (value := _get(config, field)) is not None + } + + +def _positive_source_number(config: dict[str, Any], path: str) -> float | None: + value = _get(config, path) + if isinstance(value, bool) or not isinstance(value, (int, float)): + return None + number = float(value) + return number if math.isfinite(number) and number > 0 else None + + +def _positive_source_integer(config: dict[str, Any], path: str) -> int | None: + value = _get(config, path) + return value if isinstance(value, int) and not isinstance(value, bool) and value > 0 else None + + +def _verl_overrides( + adapter: dict[str, Any], source_config: dict[str, Any] +) -> tuple[dict[str, Any], list[dict[str, Any]]]: + response_length = _positive_source_integer(source_config, "rollout.max_new_tokens_per_turn") + learning_rate = _positive_source_number(source_config, "train.learning_rate") + seed = _get(source_config, "run.seed") + if isinstance(seed, bool) or not isinstance(seed, int) or seed < 0: + seed = None + run_name = _get(source_config, "run.name") + if not isinstance(run_name, str) or not run_name.strip(): + run_name = None + placeholders: list[dict[str, Any]] = [ + { + "field": "data.max_prompt_length", + "value": 512, + "reason": ( + "miniVERL records max_total_tokens, not an equivalent standalone prompt limit" + ), + "source_run_intent": False, + }, + { + "field": "trainer.total_epochs", + "value": 1, + "reason": "miniVERL continuation cycles are not proven equivalent to verl epochs", + "source_run_intent": False, + }, + { + "field": "trainer.save_freq", + "value": 1, + "reason": "miniVERL cycle frequency units are not proven equivalent", + "source_run_intent": False, + }, + { + "field": "trainer.test_freq", + "value": 1, + "reason": "miniVERL cycle frequency units are not proven equivalent", + "source_run_intent": False, + }, + ] + if response_length is None: + response_length = 128 + placeholders.append( + { + "field": "data.max_response_length", + "value": response_length, + "reason": "no validated source-run response bound was available", + "source_run_intent": False, + } + ) + if learning_rate is None: + learning_rate = 1e-5 + placeholders.append( + { + "field": "actor_rollout_ref.actor.optim.lr", + "value": learning_rate, + "reason": "no validated source-run learning rate was available", + "source_run_intent": False, + } + ) + if seed is None: + seed = 1234 + placeholders.append( + { + "field": "data.seed", + "value": seed, + "reason": "no validated source-run seed was available", + "source_run_intent": False, + } + ) + if run_name is None: + run_name = "exported-profile" + placeholders.append( + { + "field": "trainer.experiment_name", + "value": run_name, + "reason": "no validated source-run name was available", + "source_run_intent": False, + } + ) + + overrides = { "data": { "train_files": ["data/train.parquet"], "val_files": ["data/val.parquet"], "prompt_key": "prompt", "max_prompt_length": 512, - "max_response_length": 128, - "seed": 1234, + "max_response_length": response_length, + "seed": seed, }, "actor_rollout_ref": { "model": { @@ -152,7 +288,7 @@ def _verl_overrides(adapter: dict[str, Any]) -> dict[str, Any]: "target_modules": adapter["target_modules"], "lora_adapter_path": "model", }, - "actor": {"optim": {"lr": 1e-5}}, + "actor": {"optim": {"lr": learning_rate}}, }, "custom_reward_function": { "path": "reward/reward_or_verifier_scaffold.py", @@ -162,16 +298,20 @@ def _verl_overrides(adapter: dict[str, Any]) -> dict[str, Any]: "save_freq": 1, "test_freq": 1, "project_name": "miniverl-bridge", - "experiment_name": "exported-profile", + "experiment_name": run_name, "total_epochs": 1, }, } + return overrides, placeholders -def _launch_script(adapter: dict[str, Any]) -> str: +def _launch_script(adapter: dict[str, Any], overrides: dict[str, Any]) -> str: base_model = shlex.quote(adapter["base_model"]) revision = shlex.quote(adapter["revision"]) targets = shlex.quote(json.dumps(adapter["target_modules"], separators=(",", ":"))) + data = overrides["data"] + trainer = overrides["trainer"] + learning_rate = overrides["actor_rollout_ref"]["actor"]["optim"]["lr"] return f"""#!/usr/bin/env bash set -euo pipefail @@ -191,18 +331,19 @@ def _launch_script(adapter: dict[str, Any]) -> str: data.train_files="['$BUNDLE_ROOT/data/train.parquet']" \\ data.val_files="['$BUNDLE_ROOT/data/val.parquet']" \\ data.prompt_key=prompt \\ - data.max_prompt_length=512 \\ - data.max_response_length=128 \\ + data.max_prompt_length={data["max_prompt_length"]} \\ + data.max_response_length={data["max_response_length"]} \\ actor_rollout_ref.model.path="$BUNDLE_ROOT/model/base" \\ actor_rollout_ref.model.enable_gradient_checkpointing=true \\ actor_rollout_ref.model.lora_rank={adapter["rank"]} \\ actor_rollout_ref.model.lora_alpha={adapter["alpha"]} \\ actor_rollout_ref.model.target_modules={targets} \\ actor_rollout_ref.model.lora_adapter_path="$BUNDLE_ROOT/model" \\ - actor_rollout_ref.actor.optim.lr=1e-5 \\ + actor_rollout_ref.actor.optim.lr={learning_rate} \\ custom_reward_function.path="$BUNDLE_ROOT/reward/reward_or_verifier_scaffold.py" \\ custom_reward_function.name=compute_score \\ - trainer.total_epochs=1 trainer.save_freq=1 trainer.test_freq=1 + trainer.total_epochs={trainer["total_epochs"]} \\ + trainer.save_freq={trainer["save_freq"]} trainer.test_freq={trainer["test_freq"]} """ @@ -244,7 +385,9 @@ def _bundle_readme() -> str: The generated reward scaffold fails closed until domain logic is supplied and tested. Materialize `model/base` from the exact identity in `model/base-model.json` before launch; the adapter directory alone is not a -base-model checkpoint. The bundle has **not** executed a distributed job. It does not convert +base-model checkpoint. `recipe/launch.template.sh` is not launch-ready. This is +a PPO/reward scaffold, not an executable continuation of miniVERL OPD +semantics. The bundle has **not** executed a distributed job. It does not convert optimizer state, distributed RNG, FSDP/Megatron checkpoints, Ray state, or a miniVERL teacher cache into PPO reference log-probabilities. """ @@ -264,7 +407,7 @@ def export_verl_bundle( target_verl: str, out: str | Path, ) -> dict[str, Any]: - """Export one immutable, self-checking Level-3 artifact bundle.""" + """Export one immutable, self-checking artifact bundle.""" validate_target_verl(target_verl) run_path = Path(run) manifest_path = run_path / "manifest.json" @@ -272,6 +415,9 @@ def export_verl_bundle( raise ConfigError(f"miniVERL run is missing manifest.json: {run_path}") model_source = _model_source(run_path) adapter = _adapter_contract(model_source) + source_config, source_config_file = _source_config(run_path) + source_run_values = _source_run_values(source_config) + overrides, placeholder_defaults = _verl_overrides(adapter, source_config) data_source = run_path / "data" for split in ("train.parquet", "val.parquet"): if not (data_source / split).is_file(): @@ -289,6 +435,7 @@ def export_verl_bundle( report: dict[str, Any] = { "schema_version": 1, "compatibility_level": 3, + "compatibility_level_name": "miniVERL-defined compatibility Level 3", "compatibility_levels": COMPATIBILITY_LEVELS, "profile": BRIDGE_PROFILE, "target_verl": { @@ -311,6 +458,17 @@ def export_verl_bundle( "result/provenance manifests", ], "unsupported_semantics": list(_UNSUPPORTED), + "source_config_file": source_config_file, + "source_run_values": source_run_values, + "placeholder_defaults": placeholder_defaults, + "artifact_bundle_complete": True, + "upstream_config_parse_passed": False, + "model_data_load_smoke_passed": False, + "reward_implementation_complete": False, + "launchable": False, + "distributed_execution_tested": False, + "algorithm_semantic_parity": False, + "target_semantics": "PPO/reward scaffold", "local_smoke_status": "generated; run bridge doctor", "distributed_execution_status": "not tested", "claim": ( @@ -337,11 +495,9 @@ def export_verl_bundle( recipe.mkdir() write_text( recipe / "verl-overrides.yaml", - yaml.safe_dump( - _verl_overrides(adapter), sort_keys=False, allow_unicode=True, width=100 - ), + yaml.safe_dump(overrides, sort_keys=False, allow_unicode=True, width=100), ) - write_text(recipe / "launch.sh", _launch_script(adapter)) + write_text(recipe / "launch.template.sh", _launch_script(adapter, overrides)) write_text(recipe / "REQUIRED_VERL.txt", required_verl_text()) reward = temporary / "reward" diff --git a/src/miniverl/cli.py b/src/miniverl/cli.py index 5a7ae44..6cade5e 100644 --- a/src/miniverl/cli.py +++ b/src/miniverl/cli.py @@ -966,6 +966,21 @@ def import_verl_command( profile: str = typer.Option(..., "--profile", help="Documented bridge profile."), target_verl: str = typer.Option(..., "--target-verl", help="Pinned verl tag or commit."), out: Path = typer.Option(..., "--out", help="New miniVERL recipe path."), + environment: Optional[str] = typer.Option( + None, "--environment", help="Explicit registered miniVERL environment." + ), + teacher_model: Optional[str] = typer.Option( + None, "--teacher-model", help="Explicit frozen teacher model identity." + ), + teacher_adapter: Optional[str] = typer.Option( + None, "--teacher-adapter", help="Optional local teacher adapter path." + ), + loss_profile: Optional[str] = typer.Option( + None, "--loss-profile", help="Explicit miniVERL distillation objective profile." + ), + schedule_mapping: Optional[str] = typer.Option( + None, "--schedule-mapping", help="Explicit schedule-unit mapping." + ), as_json: bool = typer.Option(False, "--json", help="Emit machine-readable JSON."), ) -> None: """Import the documented whitelist, never generic verl YAML.""" @@ -977,15 +992,22 @@ def import_verl_command( profile=profile, target_verl=target_verl, out=out, + environment=environment, + teacher_model=teacher_model, + teacher_adapter=teacher_adapter, + loss_profile=loss_profile, + schedule_mapping=schedule_mapping, ) except MiniVerlError as exc: _fail(exc) return - payload = {"written": str(out), "report": str(out.parent / "import-report.json"), **report} + written = out.parent / str(report["generated_path"]) + payload = {"written": str(written), "report": str(out.parent / "import-report.json"), **report} if as_json: _emit_json(payload) return - console.print(f"[green]verl profile imported[/green] {_esc(out)}") + style = "green" if report["status"] == "accepted" else "yellow" + console.print(f"[{style}]verl profile {report['status']}[/{style}] {_esc(written)}") console.print(f" profile {_esc(profile)}") console.print(f" report {_esc(out.parent / 'import-report.json')}") @@ -1044,7 +1066,7 @@ def export_verl_command( out: Path = typer.Option(..., "--out", help="New scale-out bundle directory."), as_json: bool = typer.Option(False, "--json", help="Emit machine-readable JSON."), ) -> None: - """Export a self-checking Level-3 bundle of standard artifacts.""" + """Export a self-checking miniVERL-defined Level-3 artifact bundle.""" try: from miniverl.bridge.export import export_verl_bundle @@ -1057,6 +1079,7 @@ def export_verl_command( return console.print(f"[green]verl bundle exported[/green] {_esc(out)}") console.print(f" profile {_esc(report['profile'])}") + console.print(" launchable: false") console.print(" distributed execution: not tested") diff --git a/tests/cli/test_verl_bridge_cli.py b/tests/cli/test_verl_bridge_cli.py index a61b944..086e98d 100644 --- a/tests/cli/test_verl_bridge_cli.py +++ b/tests/cli/test_verl_bridge_cli.py @@ -10,7 +10,7 @@ from miniverl.cli import app -def test_import_verl_cli_writes_recipe_and_report(tmp_path: Path) -> None: +def test_import_verl_cli_defaults_to_a_needs_input_template(tmp_path: Path) -> None: source = tmp_path / "verl.yaml" source.write_text( yaml.safe_dump( @@ -59,10 +59,68 @@ def test_import_verl_cli_writes_recipe_and_report(tmp_path: Path) -> None: assert result.exit_code == 0, result.output payload = json.loads(result.stdout) assert payload["profile"] == BRIDGE_PROFILE - assert out.is_file() + assert payload["status"] == "needs_user_input" + assert not out.exists() + assert (tmp_path / "imported.template.yaml").is_file() assert (tmp_path / "import-report.json").is_file() +def test_import_verl_cli_explicit_contract_writes_a_valid_recipe(tmp_path: Path) -> None: + source = tmp_path / "verl.yaml" + source.write_text( + yaml.safe_dump( + { + "data": { + "train_files": ["train.parquet"], + "val_files": ["val.parquet"], + "prompt_key": "prompt", + "max_prompt_length": 64, + "max_response_length": 32, + "seed": 7, + }, + "actor_rollout_ref": { + "model": {"path": "Qwen/Qwen3-0.6B"}, + "actor": {"optim": {"lr": "1e-5"}}, + }, + "trainer": { + "save_freq": 1, + "test_freq": 1, + "project_name": "test", + "experiment_name": "bridge", + "total_epochs": 1, + }, + } + ), + encoding="utf-8", + ) + out = tmp_path / "imported.yaml" + result = CliRunner().invoke( + app, + [ + "import-verl", + str(source), + "--profile", + BRIDGE_PROFILE, + "--target-verl", + VERL_TAG, + "--out", + str(out), + "--environment", + "calculator", + "--teacher-model", + "Qwen/Qwen3-1.7B", + "--loss-profile", + "topk-tail-reverse-kl", + "--schedule-mapping", + "epochs-as-cycles", + "--json", + ], + ) + assert result.exit_code == 0, result.output + assert json.loads(result.stdout)["status"] == "accepted" + assert out.is_file() + + def test_benchmark_export_community_exact_command_needs_no_training_stack( tmp_path: Path, ) -> None: @@ -93,13 +151,16 @@ def test_export_and_bridge_doctor_cli_round_trip(tmp_path: Path) -> None: ], ) assert exported.exit_code == 0, exported.output - assert json.loads(exported.stdout)["distributed_execution_status"] == "not tested" + exported_payload = json.loads(exported.stdout) + assert exported_payload["distributed_execution_status"] == "not tested" + assert exported_payload["launchable"] is False diagnosed = CliRunner().invoke(app, ["bridge", "doctor", str(bundle), "--json"]) assert diagnosed.exit_code == 0, diagnosed.output payload = json.loads(diagnosed.stdout) assert payload["verdict"] == "ok" assert payload["config_profile"]["model_handoff_problems"] == [] + assert payload["launchable"] is False def test_convert_dataset_cli_preserves_the_official_chat_schema(tmp_path: Path) -> None: diff --git a/tests/unit/test_alignment_publish.py b/tests/unit/test_alignment_publish.py index 2502dba..6ee8fc6 100644 --- a/tests/unit/test_alignment_publish.py +++ b/tests/unit/test_alignment_publish.py @@ -53,6 +53,12 @@ def test_alignment_result_schema_and_paired_task_evidence() -> None: (ROOT / "benchmarks/schema/alignment-lab-result.schema.json").read_text(encoding="utf-8") ) result = json.loads(result_path.read_text(encoding="utf-8")) + assert hashlib.sha256(result_path.read_bytes()).hexdigest() == ( + "584752dccb91654109c357b8ebb12681a12a9c1476a9ba539dd35e4d860a22ef" + ) + assert hashlib.sha256(task_path.read_bytes()).hexdigest() == ( + "8d7fc723436d7377d196fc44046d960e3cb7f0aa81e03d49ef05b627eb84630f" + ) jsonschema.validate(result, schema) assert result["measurement_status"] == "measured_final" assert len(result["arms"]) == 18 @@ -116,6 +122,23 @@ def test_alignment_figures_are_exactly_generated_and_privacy_safe() -> None: payload = publisher._load_result(result_path) source_digest = hashlib.sha256(result_path.read_bytes()).hexdigest() rendered = publisher.render_figures(payload, source_digest) + assert set(rendered) == { + "delta-from-sft.svg", + "outcome-cost-matrix.svg", + "metric-coverage-matrix.svg", + } + publisher.assert_chart_suitability(rendered) + combined_svg = "\n".join(rendered.values()) + assert "concentric" not in combined_svg.lower() + assert "jitter" not in combined_svg.lower() + assert source_digest not in combined_svg + assert "— not applicable" in rendered["outcome-cost-matrix.svg"] + assert 'data-applicable="false"' in rendered["outcome-cost-matrix.svg"] + assert 'data-encoding="seed-point"' in combined_svg + assert ( + "The two sandbox safety checks tied at zero while utility still regressed." + in (rendered["metric-coverage-matrix.svg"]) + ) for name, content in rendered.items(): target = ROOT / "docs/alignment-lab" / name assert target.read_text(encoding="utf-8") == content @@ -138,7 +161,7 @@ def test_alignment_figures_are_exactly_generated_and_privacy_safe() -> None: pdf_path = ROOT / "paper" / "alignment-lab-v1" / "alignment-lab-v1.pdf" assert hashlib.sha256(pdf_path.read_bytes()).hexdigest() == ( - "adbffa967f6b9a25d2cdb0cc4464a93c13db4615a1e91499585fb199285d980b" + "db4aeb2507839ce200cb5c6d93855fd687b9bb8b978fe76b089c5f1993af78a5" ) diff --git a/tests/unit/test_docs_visual_contract.py b/tests/unit/test_docs_visual_contract.py new file mode 100644 index 0000000..03c5223 --- /dev/null +++ b/tests/unit/test_docs_visual_contract.py @@ -0,0 +1,51 @@ +from __future__ import annotations + +from pathlib import Path + +import yaml + +ROOT = Path(__file__).resolve().parents[2] + + +def test_docs_use_the_pinned_modern_responsive_theme() -> None: + config = yaml.safe_load((ROOT / "mkdocs.yml").read_text(encoding="utf-8")) + assert config["theme"]["name"] == "material" + assert "content.code.copy" in config["theme"]["features"] + assert "navigation.footer" in config["theme"]["features"] + assert config["plugins"] == ["search"] + assert "assets/stylesheets/extra.css" in config["extra_css"] + assert "assets/javascripts/versioning.js" in config["extra_javascript"] + + pyproject = (ROOT / "pyproject.toml").read_text(encoding="utf-8") + assert "mkdocs-material==9.7.7" in pyproject + assert "playwright==1.62.0" in pyproject + + +def test_versioned_docs_and_browser_visual_gate_are_wired() -> None: + workflow = (ROOT / ".github/workflows/docs.yml").read_text(encoding="utf-8") + assert "mkdocs build --strict" in workflow + assert "site/dev" in workflow + assert "playwright install --with-deps chromium" in workflow + assert "docs-visual-screenshots" in workflow + + script = (ROOT / "scripts/check_docs_visual.py").read_text(encoding="utf-8") + for viewport in ("1440, 900", "1024, 768", "820, 1000", "390, 844"): + assert viewport in script + for page in ( + '"/"', + '"/alignment-lab/alignment-lab-v1/"', + '"/consumer-runtime/"', + '"/recoverybench/recoverybench-v1/"', + '"/verl-bridge/"', + ): + assert page in script + for contract in ( + "horizontal document overflow", + "figure exceeds content column", + "SVG label outside viewBox", + "chart label overlap", + "table is neither wrapped nor horizontally scrollable", + "mobile bridge did not select the vertical layout", + "chart label is smaller than", + ): + assert contract in script diff --git a/tests/unit/test_packaging.py b/tests/unit/test_packaging.py index 88a66b7..b129350 100644 --- a/tests/unit/test_packaging.py +++ b/tests/unit/test_packaging.py @@ -376,7 +376,7 @@ def test_verl_bridge_visual_and_launch_assets_are_accessible_and_honest() -> Non assert "Flagship article" in launch assert "Hugging Face card addenda" in launch assert "Distributed execution" in launch - assert "90-second verified-bridge demo" in demo + assert "Verified-bridge demo recording script" in demo assert "does not prove a distributed verl job ran" in demo diff --git a/tests/unit/test_verl_bridge_config.py b/tests/unit/test_verl_bridge_config.py index 56c312f..f521158 100644 --- a/tests/unit/test_verl_bridge_config.py +++ b/tests/unit/test_verl_bridge_config.py @@ -5,11 +5,12 @@ import pytest import yaml +from pydantic import ValidationError from miniverl.errors import ConfigError -def _source() -> dict[str, object]: +def _source(*, learning_rate: object = 2.0e-5) -> dict[str, object]: return { "data": { "train_files": ["train.parquet"], @@ -24,7 +25,7 @@ def _source() -> dict[str, object]: "path": "Qwen/Qwen3-0.6B", "enable_gradient_checkpointing": True, }, - "actor": {"optim": {"lr": 2.0e-5}}, + "actor": {"optim": {"lr": learning_rate}}, }, "trainer": { "save_freq": 2, @@ -36,39 +37,32 @@ def _source() -> dict[str, object]: } -def test_import_verl_maps_only_the_pinned_profile_and_writes_a_report(tmp_path: Path) -> None: +def _write_source(tmp_path: Path, payload: dict[str, object] | None = None) -> Path: + source = tmp_path / "verl.yaml" + source.write_text(yaml.safe_dump(payload or _source(), sort_keys=False), encoding="utf-8") + return source + + +def test_import_verl_defaults_to_a_non_executable_needs_input_template( + tmp_path: Path, +) -> None: from miniverl.bridge.config import import_verl_config from miniverl.bridge.contract import VERL_COMMIT, VERL_REPOSITORY, VERL_TAG + from miniverl.config import RunConfig - source = tmp_path / "verl.yaml" - source.write_text(yaml.safe_dump(_source(), sort_keys=False), encoding="utf-8") out = tmp_path / "recipes" / "imported.yaml" - report = import_verl_config( - source, + _write_source(tmp_path), profile="single-gpu-online-distillation-v1", target_verl=VERL_TAG, out=out, ) - generated = yaml.safe_load(out.read_text(encoding="utf-8")) - assert generated["run"] == { - "name": "bridge-smoke-strict-profile", - "mode": "opd", - "seed": 77, - "output_dir": "runs", - "deterministic": True, - "tags": ["verl-bridge", "single-gpu-online-distillation-v1"], - } - assert generated["models"]["student"]["model_id"] == "Qwen/Qwen3-0.6B" - assert generated["models"]["student"]["gradient_checkpointing"] is True - assert generated["models"]["teacher"]["model_id"] == "Qwen/Qwen3-0.6B" - assert generated["rollout"]["max_new_tokens_per_turn"] == 128 - assert generated["rollout"]["max_total_tokens"] == 640 - assert generated["train"]["learning_rate"] == 2.0e-5 - assert generated["train"]["cycles"] == 3 - assert generated["train"]["save_every_cycles"] == 2 - assert generated["train"]["eval_every_cycles"] == 1 + template = out.parent / "imported.template.yaml" + assert not out.exists() + assert template.is_file() + with pytest.raises(ValidationError): + RunConfig.from_yaml(template) on_disk = json.loads((out.parent / "import-report.json").read_text(encoding="utf-8")) assert on_disk == report @@ -77,14 +71,140 @@ def test_import_verl_maps_only_the_pinned_profile_and_writes_a_report(tmp_path: "tag": VERL_TAG, "commit": VERL_COMMIT, } - assert report["profile"] == "single-gpu-online-distillation-v1" - assert report["unsupported_fields"] == [] - assert report["semantic_conflicts"] == [] - assert report["source_config_sha256"] - assert report["generated_miniverl_sha256"] + assert report["status"] == "needs_user_input" + assert report["generated_path"] == "imported.template.yaml" + assert {item["field"] for item in report["required_user_input"]} == { + "environment", + "teacher_identity", + "loss_profile", + "schedule_mapping", + } + classifications = report["field_classification"] + assert classifications["data.train_files"]["classification"] == "informational_only" + assert classifications["data.val_files"]["classification"] == "informational_only" + assert classifications["data.prompt_key"]["classification"] == "informational_only" + assert classifications["data.max_prompt_length"]["classification"] == "derived" + assert classifications["trainer.total_epochs"]["classification"] == ( + "requires_user_confirmation" + ) + assert classifications["trainer.save_freq"]["classification"] == ("requires_user_confirmation") + assert classifications["trainer.test_freq"]["classification"] == ("requires_user_confirmation") + assert all( + item["classification"] + in { + "exact", + "derived", + "informational_only", + "requires_user_confirmation", + "unsupported", + } + for item in classifications.values() + ) + assert "mapped_fields" not in report + rendered = template.read_text(encoding="utf-8") + assert "calculator" not in rendered + assert "teacher" not in rendered.lower() or "required" in rendered.lower() + + +def test_import_verl_explicit_contract_produces_a_valid_recipe(tmp_path: Path) -> None: + from miniverl.bridge.config import import_verl_config + from miniverl.bridge.contract import VERL_TAG + from miniverl.config import RunConfig + + out = tmp_path / "recipes" / "imported.yaml" + report = import_verl_config( + _write_source(tmp_path), + profile="single-gpu-online-distillation-v1", + target_verl=VERL_TAG, + out=out, + environment="jsonnav", + teacher_model="Qwen/Qwen3-1.7B", + loss_profile="topk-tail-reverse-kl", + schedule_mapping="epochs-as-cycles", + ) + + generated = yaml.safe_load(out.read_text(encoding="utf-8")) + validated = RunConfig.from_yaml(out) + assert generated["run"]["name"] == "bridge-smoke-strict-profile" + assert generated["run"]["mode"] == "opd" + assert generated["models"]["student"]["model_id"] == "Qwen/Qwen3-0.6B" + assert generated["models"]["teacher"]["model_id"] == "Qwen/Qwen3-1.7B" + assert generated["models"]["teacher"]["mode"] == "standard" + assert generated["environment"]["name"] == "jsonnav" + assert generated["rollout"]["max_new_tokens_per_turn"] == 128 + assert generated["rollout"]["max_total_tokens"] == 640 + assert generated["train"]["learning_rate"] == 2.0e-5 + assert generated["train"]["cycles"] == 3 + assert validated.environment.name == "jsonnav" + assert report["status"] == "accepted" + assert report["generated_path"] == "imported.yaml" + assert report["generated_recipe_validated"] is True + assert report["user_confirmations"]["schedule_mapping"] == "epochs-as-cycles" + + +def test_import_verl_accepts_finite_scientific_notation_strings(tmp_path: Path) -> None: + from miniverl.bridge.config import import_verl_config + from miniverl.bridge.contract import VERL_TAG + + source = tmp_path / "verl.yaml" + source.write_text( + yaml.safe_dump(_source(), sort_keys=False).replace("2.0e-05", "1e-5"), + encoding="utf-8", + ) + out = tmp_path / "imported.yaml" + report = import_verl_config( + source, + profile="single-gpu-online-distillation-v1", + target_verl=VERL_TAG, + out=out, + environment="calculator", + teacher_model="Qwen/Qwen3-1.7B", + loss_profile="topk-tail-reverse-kl", + schedule_mapping="epochs-as-cycles", + ) assert report["status"] == "accepted" - assert "data.train_files" in report["mapped_fields"] - assert report["mapped_fields"]["data.train_files"]["disposition"] == "bridge_metadata" + assert yaml.safe_load(out.read_text(encoding="utf-8"))["train"]["learning_rate"] == 1e-5 + + +@pytest.mark.parametrize("learning_rate", ["nan", ".inf", "-inf", "${actor.lr}"]) +def test_import_verl_rejects_non_finite_or_unresolved_numeric_values( + tmp_path: Path, learning_rate: str +) -> None: + from miniverl.bridge.config import import_verl_config + from miniverl.bridge.contract import VERL_TAG + + with pytest.raises(ConfigError, match=r"finite|interpolation"): + import_verl_config( + _write_source(tmp_path, _source(learning_rate=learning_rate)), + profile="single-gpu-online-distillation-v1", + target_verl=VERL_TAG, + out=tmp_path / "imported.yaml", + environment="calculator", + teacher_model="Qwen/Qwen3-1.7B", + loss_profile="topk-tail-reverse-kl", + schedule_mapping="epochs-as-cycles", + ) + + +def test_import_verl_does_not_qualify_a_same_base_teacher_without_an_adapter( + tmp_path: Path, +) -> None: + from miniverl.bridge.config import import_verl_config + from miniverl.bridge.contract import VERL_TAG + + report = import_verl_config( + _write_source(tmp_path), + profile="single-gpu-online-distillation-v1", + target_verl=VERL_TAG, + out=tmp_path / "imported.yaml", + environment="calculator", + teacher_model="Qwen/Qwen3-0.6B", + loss_profile="topk-tail-reverse-kl", + schedule_mapping="epochs-as-cycles", + ) + assert report["status"] == "needs_user_input" + assert any(item["field"] == "teacher_identity" for item in report["required_user_input"]) + assert not (tmp_path / "imported.yaml").exists() @pytest.mark.parametrize( @@ -109,30 +229,25 @@ def test_import_verl_fails_closed_on_algorithm_or_scale_out_fields( assert isinstance(child, dict) cursor = child cursor[path[-1]] = value - source = tmp_path / "verl.yaml" - source.write_text(yaml.safe_dump(payload), encoding="utf-8") with pytest.raises(ConfigError, match="unsupported verl field"): import_verl_config( - source, + _write_source(tmp_path, payload), profile="single-gpu-online-distillation-v1", target_verl=VERL_TAG, out=tmp_path / "imported.yaml", ) - assert not (tmp_path / "imported.yaml").exists() rejection = json.loads((tmp_path / "import-report.json").read_text(encoding="utf-8")) assert rejection["status"] == "rejected" - assert ".".join(path) in rejection["unsupported_fields"] + assert rejection["field_classification"][".".join(path)]["classification"] == "unsupported" def test_import_verl_rejects_a_moving_or_unverified_target(tmp_path: Path) -> None: from miniverl.bridge.config import import_verl_config - source = tmp_path / "verl.yaml" - source.write_text(yaml.safe_dump(_source()), encoding="utf-8") with pytest.raises(ConfigError, match="pinned verl target"): import_verl_config( - source, + _write_source(tmp_path), profile="single-gpu-online-distillation-v1", target_verl="main", out=tmp_path / "imported.yaml", diff --git a/tests/unit/test_verl_bridge_export.py b/tests/unit/test_verl_bridge_export.py index b86e909..c974516 100644 --- a/tests/unit/test_verl_bridge_export.py +++ b/tests/unit/test_verl_bridge_export.py @@ -63,7 +63,7 @@ def _run(tmp_path: Path) -> Path: return run -def test_export_verl_emits_the_exact_level3_bundle_and_doctor_verifies_it( +def test_export_verl_emits_a_fail_closed_bundle_and_doctor_verifies_artifacts( tmp_path: Path, ) -> None: from miniverl.bridge.contract import VERL_COMMIT, VERL_TAG @@ -81,7 +81,7 @@ def test_export_verl_emits_the_exact_level3_bundle_and_doctor_verifies_it( "data/train.parquet", "data/val.parquet", "recipe/verl-overrides.yaml", - "recipe/launch.sh", + "recipe/launch.template.sh", "recipe/REQUIRED_VERL.txt", "reward/reward_or_verifier_scaffold.py", "provenance/miniverl-manifest.json", @@ -99,7 +99,17 @@ def test_export_verl_emits_the_exact_level3_bundle_and_doctor_verifies_it( assert "verl-project/verl" in requirement assert "main" not in requirement assert report["compatibility_level"] == 3 + assert report["compatibility_level_name"] == "miniVERL-defined compatibility Level 3" + assert report["artifact_bundle_complete"] is True + assert report["upstream_config_parse_passed"] is False + assert report["model_data_load_smoke_passed"] is False + assert report["reward_implementation_complete"] is False + assert report["launchable"] is False + assert report["distributed_execution_tested"] is False + assert report["algorithm_semantic_parity"] is False assert report["distributed_execution_status"] == "not tested" + assert report["target_semantics"] == "PPO/reward scaffold" + assert report["placeholder_defaults"] overrides = yaml.safe_load((out / "recipe" / "verl-overrides.yaml").read_text()) model = overrides["actor_rollout_ref"]["model"] @@ -115,7 +125,7 @@ def test_export_verl_emits_the_exact_level3_bundle_and_doctor_verifies_it( "revision": "c1899de289a04d12100db370d81485cdf75e47ca", "status": "not bundled; materialize the exact snapshot before launch", } - launch = (out / "recipe" / "launch.sh").read_text(encoding="utf-8") + launch = (out / "recipe" / "launch.template.sh").read_text(encoding="utf-8") assert "model/base/config.json" in launch assert "lora_adapter_path" in launch assert "hf download Qwen/Qwen3-0.6B --revision c1899de" in launch @@ -127,6 +137,83 @@ def test_export_verl_emits_the_exact_level3_bundle_and_doctor_verifies_it( assert diagnosis["reward_scaffold_importability"]["status"] == "ok" assert diagnosis["artifact_hashes"]["status"] == "ok" assert diagnosis["distributed_execution_status"] == "not tested" + assert diagnosis["artifact_bundle_complete"] is True + assert diagnosis["reward_implementation_complete"] is False + assert diagnosis["launchable"] is False + assert diagnosis["distributed_execution_tested"] is False + assert diagnosis["algorithm_semantic_parity"] is False + + +def test_export_preserves_available_source_values_without_claiming_schedule_parity( + tmp_path: Path, +) -> None: + from miniverl.bridge.contract import VERL_TAG + from miniverl.bridge.export import export_verl_bundle + + run = _run(tmp_path) + (run / "config.resolved.yaml").write_text( + yaml.safe_dump( + { + "rollout": {"max_total_tokens": 777, "max_new_tokens_per_turn": 96}, + "train": { + "cycles": 4, + "learning_rate": 3e-5, + "save_every_cycles": 2, + "eval_every_cycles": 3, + }, + "environment": {"name": "json_navigation"}, + }, + sort_keys=False, + ), + encoding="utf-8", + ) + out = tmp_path / "export" + report = export_verl_bundle(run, target_verl=VERL_TAG, out=out) + + overrides = yaml.safe_load((out / "recipe" / "verl-overrides.yaml").read_text()) + assert overrides["data"]["max_response_length"] == 96 + assert overrides["actor_rollout_ref"]["actor"]["optim"]["lr"] == 3e-5 + assert report["source_run_values"]["rollout.max_total_tokens"] == 777 + assert report["source_run_values"]["train.cycles"] == 4 + assert report["source_run_values"]["environment.name"] == "json_navigation" + placeholders = {item["field"]: item for item in report["placeholder_defaults"]} + assert "data.max_prompt_length" in placeholders + assert "trainer.total_epochs" in placeholders + assert placeholders["trainer.total_epochs"]["source_run_intent"] is False + + +def test_current_reward_scaffold_can_never_report_ready_to_launch(tmp_path: Path) -> None: + from miniverl.bridge.contract import VERL_TAG + from miniverl.bridge.doctor import inspect_bridge_bundle + from miniverl.bridge.export import export_verl_bundle + + out = tmp_path / "export" + report = export_verl_bundle(_run(tmp_path), target_verl=VERL_TAG, out=out) + diagnosis = inspect_bridge_bundle(out) + + assert report["reward_implementation_complete"] is False + assert report["launchable"] is False + assert diagnosis["reward_implementation_complete"] is False + assert diagnosis["launchable"] is False + assert not (out / "recipe" / "launch.sh").exists() + + +def test_missing_pinned_verl_does_not_relabel_a_complete_artifact_bundle( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + from miniverl.bridge import doctor + from miniverl.bridge.contract import VERL_TAG + from miniverl.bridge.export import export_verl_bundle + + out = tmp_path / "export" + export_verl_bundle(_run(tmp_path), target_verl=VERL_TAG, out=out) + monkeypatch.setattr(doctor, "_installed_verl", lambda: {"status": "not installed"}) + + diagnosis = doctor.inspect_bridge_bundle(out, require_verl=True) + + assert diagnosis["verdict"] == "fail" + assert diagnosis["artifact_bundle_complete"] is True + assert diagnosis["upstream_config_parse_passed"] is False def test_bridge_doctor_detects_tampering(tmp_path: Path) -> None: diff --git a/tests/unit/test_verl_bridge_smoke.py b/tests/unit/test_verl_bridge_smoke.py index 94e7e27..a180f22 100644 --- a/tests/unit/test_verl_bridge_smoke.py +++ b/tests/unit/test_verl_bridge_smoke.py @@ -1,5 +1,6 @@ from __future__ import annotations +import importlib.util import json from pathlib import Path @@ -43,3 +44,34 @@ def test_committed_exact_source_smoke_is_pin_bound_and_does_not_claim_scale_out( assert record["bundle_doctor"]["artifact_hashes"]["files"] == 14 assert record["distributed_execution_status"] == "not tested" assert record["tiny_cpu_dry_run"]["status"] == "artifact-only" + + +def test_bridge_diagrams_are_generated_responsive_and_fail_closed() -> None: + root = Path(__file__).resolve().parents[2] + script = root / "scripts" / "publish_verl_bridge_diagrams.py" + spec = importlib.util.spec_from_file_location("publish_verl_bridge_diagrams", script) + assert spec and spec.loader + publisher = importlib.util.module_from_spec(spec) + spec.loader.exec_module(publisher) + + rendered = publisher.render_diagrams() + assert set(rendered) == { + "verl-bridge-architecture.svg", + "verl-bridge-architecture-mobile.svg", + } + for name, content in rendered.items(): + assert (root / "docs" / name).read_text(encoding="utf-8") == content + assert "v0.8.0" in content + assert "7aed6b23" in content + assert "independent project; no endorsement" in content + assert "Distributed execution: NOT TESTED" in content + assert "stroke-dasharray" in content + assert "teacher role" in content + assert "reference role" in content + assert "reward role" in content + assert "export-verl" not in content + + page = (root / "docs" / "verl-bridge.md").read_text(encoding="utf-8") + assert " Date: Mon, 3 Aug 2026 14:42:30 -0700 Subject: [PATCH 2/3] Shorten portable chart titles --- docs/alignment-lab/delta-from-sft.svg | 2 +- docs/alignment-lab/metric-coverage-matrix.svg | 2 +- scripts/publish_alignment_lab_artifacts.py | 33 +++++++++++++++++-- tests/unit/test_alignment_publish.py | 7 ++++ 4 files changed, 39 insertions(+), 5 deletions(-) diff --git a/docs/alignment-lab/delta-from-sft.svg b/docs/alignment-lab/delta-from-sft.svg index 6d21242..63a5ccb 100644 --- a/docs/alignment-lab/delta-from-sft.svg +++ b/docs/alignment-lab/delta-from-sft.svg @@ -1 +1 @@ -Continuation could not improve the saturated SFT checkpointForest chart of alignment-score and retained-tool-utility percentage-point deltas for five continuation methods. Every seed is shown at its exact x value.Continuation could not improve the saturated SFT checkpoint48 paired sandbox tasks per seed · mean marks plus all three measured seeds · zero = starting SFTContinuation methoddelta from the same-seed SFT checkpoint (percentage points)alignment meanutility-40-30-20-10+0+5zero baselinecontinued SFTA seeds: 0.0 / 0.0 / -16.7U seeds: 0.0 / 0.0 / -33.3A -5.6U -11.1DPOA seeds: 0.0 / 0.0 / 0.0U seeds: 0.0 / 0.0 / 0.0A 0.0U 0.0offline soft distillationA seeds: 0.0 / 0.0 / 0.0U seeds: 0.0 / 0.0 / 0.0A 0.0U 0.0standard OPDA seeds: 0.0 / 0.0 / -4.2U seeds: 0.0 / 0.0 / -8.3A -1.4U -2.8verifier-gated OPDA seeds: 0.0 / -6.2 / 0.0U seeds: 0.0 / -12.5 / 0.0A -2.1U -4.2Seed shapes: ● 1234 · ◆ 20260727 · × 20260801. A = alignment; U = retained tool utility. +No continuation improved saturated SFTForest chart of alignment-score and retained-tool-utility percentage-point deltas for five continuation methods. Every seed is shown at its exact x value.No continuation improved saturated SFT48 paired sandbox tasks per seed · mean marks plus all three measured seeds · zero = starting SFTContinuation methoddelta from the same-seed SFT checkpoint (percentage points)alignment meanutility-40-30-20-10+0+5zero baselinecontinued SFTA seeds: 0.0 / 0.0 / -16.7U seeds: 0.0 / 0.0 / -33.3A -5.6U -11.1DPOA seeds: 0.0 / 0.0 / 0.0U seeds: 0.0 / 0.0 / 0.0A 0.0U 0.0offline soft distillationA seeds: 0.0 / 0.0 / 0.0U seeds: 0.0 / 0.0 / 0.0A 0.0U 0.0standard OPDA seeds: 0.0 / 0.0 / -4.2U seeds: 0.0 / 0.0 / -8.3A -1.4U -2.8verifier-gated OPDA seeds: 0.0 / -6.2 / 0.0U seeds: 0.0 / -12.5 / 0.0A -2.1U -4.2Seed shapes: ● 1234 · ◆ 20260727 · × 20260801. A = alignment; U = retained tool utility. diff --git a/docs/alignment-lab/metric-coverage-matrix.svg b/docs/alignment-lab/metric-coverage-matrix.svg index 180401e..7f2f422 100644 --- a/docs/alignment-lab/metric-coverage-matrix.svg +++ b/docs/alignment-lab/metric-coverage-matrix.svg @@ -1 +1 @@ -Metric coverage: zero sandbox failures did not imply full utility retentionCoverage matrix showing zero harmful-compliance and over-refusal rates, observed tool-utility deltas, measured sandbox endpoints and unexecuted external benchmarks.Metric coverage: zero sandbox failures did not imply full utility retentionAll three measured seeds are printed · deterministic Minipolicy checks onlyMethodHarmful complianceseed valuesOver-refusalseed valuesTool utility Δmean · seeds (pp)Sandbox endpointmeasured?External safetyexecuted?SFT checkpoint0% · 0% · 0%0% · 0% · 0%0.00.0 / 0.0 / 0.0YESNOT RUNcontinued SFT0% · 0% · 0%0% · 0% · 0%-11.10.0 / 0.0 / -33.3YESNOT RUNDPO0% · 0% · 0%0% · 0% · 0%0.00.0 / 0.0 / 0.0YESNOT RUNoffline soft distillation0% · 0% · 0%0% · 0% · 0%0.00.0 / 0.0 / 0.0YESNOT RUNstandard OPD0% · 0% · 0%0% · 0% · 0%-2.80.0 / 0.0 / -8.3YESNOT RUNverifier-gated OPD0% · 0% · 0%0% · 0% · 0%-4.20.0 / -12.5 / 0.0YESNOT RUNThe two sandbox safety checks tied at zero while utility still regressed.IFEval, XSTest, HarmBench and RewardBench were not executed; this is not a broad safety benchmark. +Zero sandbox checks did not preserve utilityCoverage matrix showing zero harmful-compliance and over-refusal rates, observed tool-utility deltas, measured sandbox endpoints and unexecuted external benchmarks.Zero sandbox checks did not preserve utilityAll three measured seeds are printed · deterministic Minipolicy checks onlyMethodHarmful complianceseed valuesOver-refusalseed valuesTool utility Δmean · seeds (pp)Sandbox endpointmeasured?External safetyexecuted?SFT checkpoint0% · 0% · 0%0% · 0% · 0%0.00.0 / 0.0 / 0.0YESNOT RUNcontinued SFT0% · 0% · 0%0% · 0% · 0%-11.10.0 / 0.0 / -33.3YESNOT RUNDPO0% · 0% · 0%0% · 0% · 0%0.00.0 / 0.0 / 0.0YESNOT RUNoffline soft distillation0% · 0% · 0%0% · 0% · 0%0.00.0 / 0.0 / 0.0YESNOT RUNstandard OPD0% · 0% · 0%0% · 0% · 0%-2.80.0 / 0.0 / -8.3YESNOT RUNverifier-gated OPD0% · 0% · 0%0% · 0% · 0%-4.20.0 / -12.5 / 0.0YESNOT RUNThe two sandbox safety checks tied at zero while utility still regressed.IFEval, XSTest, HarmBench and RewardBench were not executed; this is not a broad safety benchmark. diff --git a/scripts/publish_alignment_lab_artifacts.py b/scripts/publish_alignment_lab_artifacts.py index 601fd25..7b5f917 100644 --- a/scripts/publish_alignment_lab_artifacts.py +++ b/scripts/publish_alignment_lab_artifacts.py @@ -574,6 +574,8 @@ def _load_result(path: Path) -> dict[str, Any]: def _svg_shell( title: str, description: str, subtitle: str, body: list[str], *, height: int = 720 ) -> str: + if len(title) > 48: + raise ValueError("SVG title must remain portable across deterministic fallback fonts") style = ( "text{font-family:'DejaVu Sans','Segoe UI',sans-serif;fill:#edf4ff}" ".title{font-size:31px;font-weight:760}.sub{font-size:17px;fill:#aebbd2}" @@ -726,7 +728,7 @@ def _delta_from_sft(payload: dict[str, Any]) -> str: 'Seed shapes: ● 1234 · ◆ 20260727 · × 20260801. A = alignment; U = retained tool utility.' ) return _svg_shell( - "Continuation could not improve the saturated SFT checkpoint", + "No continuation improved saturated SFT", ( "Forest chart of alignment-score and retained-tool-utility percentage-point deltas " "for five continuation methods. Every seed is shown at its exact x value." @@ -940,7 +942,7 @@ def _metric_coverage_matrix(payload: dict[str, Any]) -> str: ] ) return _svg_shell( - "Metric coverage: zero sandbox failures did not imply full utility retention", + "Zero sandbox checks did not preserve utility", ( "Coverage matrix showing zero harmful-compliance and over-refusal rates, observed " "tool-utility deltas, measured sandbox endpoints and unexecuted external benchmarks." @@ -1290,6 +1292,26 @@ def _check(result_path: Path, task_path: Path) -> None: raise ValueError(f"generated artifact is stale: {PILOT_EXAMPLE}") +def _refresh_derived(result_path: Path, task_path: Path) -> None: + """Rewrite public derived artifacts without touching frozen result evidence.""" + payload = _load_result(result_path) + if payload.get("task_results_sha256") != _sha256(task_path): + raise ValueError("Alignment Lab task-results digest mismatch") + source_digest = _sha256(result_path) + for name, content in render_figures(payload, source_digest).items(): + (DOCS / name).write_text(content, encoding="utf-8", newline="\n") + for name, content in render_cards(payload, source_digest).items(): + (CARDS / name).write_text(content, encoding="utf-8", newline="\n") + (DOCS / "alignment-lab-v1.md").write_text( + render_report(payload, source_digest), encoding="utf-8", newline="\n" + ) + PILOT_EXAMPLE.write_text( + json.dumps(payload["pilot"], indent=2, sort_keys=True) + "\n", + encoding="utf-8", + newline="\n", + ) + + def main() -> int: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--run-root", type=Path) @@ -1298,11 +1320,16 @@ def main() -> int: parser.add_argument("--dpo-root", type=Path, default=ROOT / "artifacts") parser.add_argument("--result", type=Path, default=RESULT) parser.add_argument("--task-results", type=Path, default=TASK_RESULTS) - parser.add_argument("--check", action="store_true") + mode = parser.add_mutually_exclusive_group() + mode.add_argument("--check", action="store_true") + mode.add_argument("--refresh-derived", action="store_true") args = parser.parse_args() if args.check: _check(args.result, args.task_results) return 0 + if args.refresh_derived: + _refresh_derived(args.result, args.task_results) + return 0 missing = [ name for name in ("run_root", "baseline_prefix", "baseline_recovery") diff --git a/tests/unit/test_alignment_publish.py b/tests/unit/test_alignment_publish.py index 6ee8fc6..85427db 100644 --- a/tests/unit/test_alignment_publish.py +++ b/tests/unit/test_alignment_publish.py @@ -9,6 +9,7 @@ from types import ModuleType import jsonschema +import pytest import yaml ROOT = Path(__file__).resolve().parents[2] @@ -165,6 +166,12 @@ def test_alignment_figures_are_exactly_generated_and_privacy_safe() -> None: ) +def test_alignment_svg_titles_are_portable_across_fallback_fonts() -> None: + publisher = _publisher() + with pytest.raises(ValueError, match="portable across deterministic fallback fonts"): + publisher._svg_shell("x" * 49, "description", "subtitle", []) + + def test_dpo_external_training_cost_is_included() -> None: result = json.loads( (ROOT / "benchmarks/results/alignment-lab-v1.json").read_text(encoding="utf-8") From 639492b25bd1195099299f0afca4a9052de20503 Mon Sep 17 00:00:00 2001 From: Daoyuan Li Date: Mon, 3 Aug 2026 14:59:45 -0700 Subject: [PATCH 3/3] Finalize v0.6.1 release metadata --- CHANGELOG.md | 44 ++++++++++++++- CITATION.cff | 4 +- PROJECT_STATE.md | 15 ++++++ PYPI.md | 48 ++++++++--------- README.md | 2 +- README.zh-CN.md | 2 +- .../results/gpu-calc-hard-equal-update-v2.md | 2 +- docs/generated/quality.json | 20 +++---- docs/overrides/main.html | 6 +-- docs/release-checklist.md | 53 +++++++++++++++++++ src/miniverl/__init__.py | 2 +- tests/unit/test_packaging.py | 2 +- 12 files changed, 155 insertions(+), 45 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 85506f5..4202bc9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,47 @@ All notable changes to miniVERL are recorded here. The format follows ## [Unreleased] +## [0.6.1] - 2026-08-03 + +### Added + +- Deterministic real-browser documentation gates across five representative + pages and four viewports, with overflow, SVG bounds, label collision, + readability, table and responsive-bridge assertions plus screenshot + artifacts. +- Responsive desktop and mobile bridge diagrams that separate the verified + local runtime, portable bundle and pinned upstream smoke from explicitly + untested distributed execution. + +### Changed + +- The Alignment Lab case study now uses three data-bound forest/matrix figures + that show every measured seed, preserve not-applicable query ratios and make + the limited sandbox-safety coverage explicit without changing frozen data. +- The documentation uses pinned Material 9.7.7 with stable/development paths, + search, dark/light modes, copy controls and a task-oriented landing page. +- The English and Chinese READMEs are shorter product guides with one scoped + evidence summary and direct Align, Distill locally and Scale out paths. + +### Fixed + +- `import-verl` now classifies field semantics, fails closed with a + non-executable template when data, teacher, objective or schedule choices are + unresolved, accepts finite scientific-notation strings and validates every + runnable recipe before atomic publication. +- `export-verl` now reports artifact completeness, upstream parse/load smoke, + reward implementation, launchability, distributed execution and algorithm + parity independently; its fail-closed scaffold emits `launch.template.sh` + and is never described as ready to launch. + +### Verified + +- Official verl `v0.8.0` commit `7aed6b230776f963fa09509c10d9c3a767d1102c` + still passes the bounded parse/load smoke. Distributed execution and + miniVERL OPD-to-PPO semantic parity remain untested and unclaimed. +- Every frozen calculator, RecoveryBench, Consumer Runtime, Alignment Lab and + bridge-smoke JSON/JSONL artifact remains byte-identical. + ## [0.6.0] - 2026-08-03 ### Added @@ -556,7 +597,8 @@ Same-tokenizer only; one trajectory per forward pass; `swap` unavailable for quantized models; only Qwen3 and Qwen2 architectures tested; single-seed GPU results. The full list is in `docs/limitations.md`. -[Unreleased]: https://github.com/DaoyuanLi2816/mini-verl/compare/v0.6.0...HEAD +[Unreleased]: https://github.com/DaoyuanLi2816/mini-verl/compare/v0.6.1...HEAD +[0.6.1]: https://github.com/DaoyuanLi2816/mini-verl/compare/v0.6.0...v0.6.1 [0.6.0]: https://github.com/DaoyuanLi2816/mini-verl/compare/v0.5.0...v0.6.0 [0.5.0]: https://github.com/DaoyuanLi2816/mini-verl/compare/v0.4.0...v0.5.0 [0.4.0]: https://github.com/DaoyuanLi2816/mini-verl/compare/v0.3.0...v0.4.0 diff --git a/CITATION.cff b/CITATION.cff index 44d5bd0..0eb4c7d 100644 --- a/CITATION.cff +++ b/CITATION.cff @@ -2,7 +2,7 @@ cff-version: 1.2.0 title: "miniVERL: On-policy distillation for tool-using agents on one GPU" message: "If you use miniVERL in your work, please cite it as below." type: software -version: 0.6.0 +version: 0.6.1 date-released: 2026-08-03 license: Apache-2.0 repository-code: "https://github.com/DaoyuanLi2816/mini-verl" @@ -98,4 +98,4 @@ references: url: "https://arxiv.org/abs/2603.07079" notes: >- Motivates recording per-token teacher entropy. Entropy-aware divergence - mixing is a roadmap item and is not implemented in miniVERL v0.6.0. + mixing is a roadmap item and is not implemented in miniVERL v0.6.1. diff --git a/PROJECT_STATE.md b/PROJECT_STATE.md index d2531ec..6bf1a0d 100644 --- a/PROJECT_STATE.md +++ b/PROJECT_STATE.md @@ -6,6 +6,21 @@ and what it printed. Last updated: 2026-08-03. +## v0.6.1 Visual integrity and bridge correctness release candidate + +| item | current state | +| --- | --- | +| scope | representation and documentation UX were rebuilt without rerunning a benchmark; import/export bridge semantics now fail closed instead of implying runnable or equivalent execution | +| Alignment Lab visuals | the four non-data-bound scatterplots are replaced by a three-seed delta forest, an outcome/cost matrix and a metric-coverage matrix; not-applicable teacher ratios remain N/A, and the zero-variance sandbox checks are not plotted as a two-dimensional safety result | +| bridge diagram | responsive desktop/mobile layouts show solid arrows only across the verified local runtime, portable bundle and pinned `v0.8.0` / `7aed6b23` parse-load smoke, followed by a dashed arrow to prominent `Distributed execution: NOT TESTED` | +| import contract | every source field is classified as exact, derived, informational only, requiring user confirmation or unsupported; unresolved data/environment, teacher, objective or schedule semantics produce `needs_user_input` plus a non-executable template | +| export status | artifact completeness, upstream parse/load, model/data smoke, reward implementation, launchability, distributed testing and algorithm parity are independent flags; the fail-closed reward scaffold remains non-launchable and uses `launch.template.sh` | +| documentation | pinned Material 9.7.7 supplies stable/dev navigation, search, dark/light modes, command-copy controls and responsive tables/images; the landing page exposes Align, Distill locally and Scale out paths | +| visual gate | Playwright checks five pages at 1440x900, 1024x768, 820x1000 and 390x844 for overflow, SVG bounds, label collisions, readable labels, tables and the responsive bridge; docs run [`30855762827`](https://github.com/DaoyuanLi2816/mini-verl/actions/runs/30855762827) is green and its 20 Linux screenshots were manually inspected | +| validation | local ruff, format, mypy, actionlint, full non-GPU/non-network, available GPU/network, strict MkDocs, browser visual, package/Twine, clean-install, extracted-sdist, bridge end-to-end, privacy, Markdown/link and generated-byte gates pass; PR-head CI [`30855762617`](https://github.com/DaoyuanLi2816/mini-verl/actions/runs/30855762617), build [`30855763132`](https://github.com/DaoyuanLi2816/mini-verl/actions/runs/30855763132) and pinned bridge [`30855762724`](https://github.com/DaoyuanLi2816/mini-verl/actions/runs/30855762724) are green | +| immutable evidence | no result JSON/JSONL or frozen bridge-smoke record changed; calculator SHA-256 remains `53fc1d4d5b7adee09618d77ad62d4086ba56b78569832d6fc7c3bcd5c2695bbc` and Alignment Lab result/task SHA-256 values remain `584752dccb91654109c357b8ebb12681a12a9c1476a9ba539dd35e4d860a22ef` / `8d7fc723436d7377d196fc44046d960e3cb7f0aa81e03d49ef05b627eb84630f` | +| release state | exact `0.6.1` metadata is being finalized on focused PR [#40](https://github.com/DaoyuanLi2816/mini-verl/pull/40); merge, tag and OIDC publication remain gated on the final head being green | + ## v0.6.0 Verified verl Bridge release | item | current state | diff --git a/PYPI.md b/PYPI.md index 43afa2a..c5032ab 100644 --- a/PYPI.md +++ b/PYPI.md @@ -1,5 +1,5 @@

- miniVERL — single-GPU LLM post-training + miniVERL — single-GPU LLM post-training

@@ -8,7 +8,7 @@ [![Build](https://github.com/DaoyuanLi2816/mini-verl/actions/workflows/build.yml/badge.svg)](https://github.com/DaoyuanLi2816/mini-verl/actions/workflows/build.yml) [![PyPI](https://img.shields.io/pypi/v/miniverl.svg)](https://pypi.org/project/miniverl/) [![Python](https://img.shields.io/badge/python-3.10%20%7C%203.11%20%7C%203.12%20%7C%203.13-blue)](https://www.python.org) -[![License](https://img.shields.io/badge/license-Apache--2.0-blue)](https://github.com/DaoyuanLi2816/mini-verl/blob/main/LICENSE) +[![License](https://img.shields.io/badge/license-Apache--2.0-blue)](https://github.com/DaoyuanLi2816/mini-verl/blob/v0.6.1/LICENSE)
@@ -16,7 +16,7 @@ PyPI · Stable docs · Development docs · - 中文 + 中文

**miniVERL is a local, inspectable runtime for a documented subset of @@ -25,7 +25,7 @@ assistant-only loss masks, teacher targets, update budgets and run artifacts explicit, then exports portable artifacts through a fail-closed bridge to one pinned upstream verl profile. -PyPI `v0.6.0` is stable; `main` is development. The CUDA path has no GPU-name +PyPI `v0.6.1` is stable; `main` is development. The CUDA path has no GPU-name allowlist, but fit depends on the model pair, context budget, kernels and VRAM. miniVERL is independent from verl and does not claim distributed execution or full algorithmic compatibility. @@ -44,15 +44,15 @@ optimization in about 50 seconds on the measured laptop CPU. For inspection, schemas and reports without the ML stack, use `pip install miniverl`. For CUDA training, install the matching CUDA-enabled PyTorch wheel first, then install `miniverl[train,cuda]`; the extra does not select a CUDA PyTorch build. See the -[single-GPU guide](https://github.com/DaoyuanLi2816/mini-verl/blob/main/docs/single-gpu-guide.md). +[single-GPU guide](https://github.com/DaoyuanLi2816/mini-verl/blob/v0.6.1/docs/single-gpu-guide.md). ## Three paths | Path | Start with | Concrete artifact | Next | | --- | --- | --- | --- | -| **Align** — compare SFT, DPO, KD and OPD only when the pilot evidence supports the cost | `miniverl pilot recipes/alignment_policy_conditioned_qwen.yaml` | `alignment-card.json` | [Alignment Lab](https://github.com/DaoyuanLi2816/mini-verl/blob/main/docs/alignment-lab/alignment-lab-v1.md) | -| **Distill locally** — strict OPD, shared backbones and padded trajectory updates on one CUDA GPU | `miniverl train recipes/qwen_consumer_gpu_shared.yaml --dry-run` | `config.resolved.yaml` plus a revision-pinned PEFT adapter | [Bring your own GPU](https://github.com/DaoyuanLi2816/mini-verl/blob/main/docs/single-gpu-guide.md) | -| **Scale out** — import a documented profile, convert Parquet, export a bundle and run bridge checks | `miniverl bridge doctor scaleout-bundle` | `provenance/compatibility-report.json` | [Verified verl bridge](https://github.com/DaoyuanLi2816/mini-verl/blob/main/docs/verl-bridge.md) | +| **Align** — compare SFT, DPO, KD and OPD only when the pilot evidence supports the cost | `miniverl pilot recipes/alignment_policy_conditioned_qwen.yaml` | `alignment-card.json` | [Alignment Lab](https://github.com/DaoyuanLi2816/mini-verl/blob/v0.6.1/docs/alignment-lab/alignment-lab-v1.md) | +| **Distill locally** — strict OPD, shared backbones and padded trajectory updates on one CUDA GPU | `miniverl train recipes/qwen_consumer_gpu_shared.yaml --dry-run` | `config.resolved.yaml` plus a revision-pinned PEFT adapter | [Bring your own GPU](https://github.com/DaoyuanLi2816/mini-verl/blob/v0.6.1/docs/single-gpu-guide.md) | +| **Scale out** — import a documented profile, convert Parquet, export a bundle and run bridge checks | `miniverl bridge doctor scaleout-bundle` | `provenance/compatibility-report.json` | [Verified verl bridge](https://github.com/DaoyuanLi2816/mini-verl/blob/v0.6.1/docs/verl-bridge.md) | The bridge import is deliberately not generic YAML conversion. If dataset or environment, teacher identity, objective, or schedule semantics are missing, @@ -75,12 +75,12 @@ improved it; continued SFT and both OPD variants retained measured regressions. | standard OPD | 98.6% | 97.2% | 100.0% | 76.7 s | | verifier-gated OPD | 97.9% | 95.8% | 46.8% | 66.0 s | -![Alignment and utility deltas from the saturated SFT checkpoint; small marks are all three seeds and large marks are means](https://raw.githubusercontent.com/DaoyuanLi2816/mini-verl/main/docs/alignment-lab/delta-from-sft.svg) +![Alignment and utility deltas from the saturated SFT checkpoint; small marks are all three seeds and large marks are means](https://raw.githubusercontent.com/DaoyuanLi2816/mini-verl/v0.6.1/docs/alignment-lab/delta-from-sft.svg) The two sandbox safety checks tied at zero while utility still regressed. IFEval, XSTest, HarmBench and RewardBench were **not executed**. “Preference win rate” is a deterministic Minipolicy paired outcome, not human preference. -Read the [study, seed-level values and limitations](https://github.com/DaoyuanLi2816/mini-verl/blob/main/docs/alignment-lab/alignment-lab-v1.md). +Read the [study, seed-level values and limitations](https://github.com/DaoyuanLi2816/mini-verl/blob/v0.6.1/docs/alignment-lab/alignment-lab-v1.md). ## One measured systems result @@ -91,13 +91,13 @@ reserved memory versus 3.035 GiB for dual model, while running 10.1% slower. All 12 preregistered equivalence comparisons passed. These are one-workload, one-machine measurements, not promises for other GPUs. -![Measured throughput and reserved VRAM for dual-model and shared-backbone runtime cells](https://raw.githubusercontent.com/DaoyuanLi2816/mini-verl/main/docs/consumer-runtime-v1-pareto.svg) +![Measured throughput and reserved VRAM for dual-model and shared-backbone runtime cells](https://raw.githubusercontent.com/DaoyuanLi2816/mini-verl/v0.6.1/docs/consumer-runtime-v1-pareto.svg) -[Consumer Runtime v1 methods and caveats](https://github.com/DaoyuanLi2816/mini-verl/blob/main/docs/consumer-runtime-v1.md) +[Consumer Runtime v1 methods and caveats](https://github.com/DaoyuanLi2816/mini-verl/blob/v0.6.1/docs/consumer-runtime-v1.md) ## Compatibility boundary -![Verified local runtime, portable artifact bundle and pinned upstream smoke; distributed verl execution remains untested](https://raw.githubusercontent.com/DaoyuanLi2816/mini-verl/main/docs/verl-bridge-architecture.svg) +![Verified local runtime, portable artifact bundle and pinned upstream smoke; distributed verl execution remains untested](https://raw.githubusercontent.com/DaoyuanLi2816/mini-verl/v0.6.1/docs/verl-bridge-architecture.svg) The bridge targets official verl `v0.8.0` at commit `7aed6b23` and uses the term **miniVERL-defined compatibility Level 3**. That means a checksummed @@ -114,21 +114,21 @@ PPO/reward scaffold, not an executable continuation of miniVERL OPD semantics. ## Detailed studies and preserved negative evidence -- [RecoveryBench v1](https://github.com/DaoyuanLi2816/mini-verl/blob/main/docs/recoverybench/recoverybench-v1.md): frozen-student KD +- [RecoveryBench v1](https://github.com/DaoyuanLi2816/mini-verl/blob/v0.6.1/docs/recoverybench/recoverybench-v1.md): frozen-student KD outperformed much slower fresh-state OPD on the preregistered primary view; the verifier gate remained `insufficient_evidence`. -- [Alignment Lab v1](https://github.com/DaoyuanLi2816/mini-verl/blob/main/docs/alignment-lab/alignment-lab-v1.md): the starting SFT +- [Alignment Lab v1](https://github.com/DaoyuanLi2816/mini-verl/blob/v0.6.1/docs/alignment-lab/alignment-lab-v1.md): the starting SFT checkpoint was at the ceiling, so no positive OPD result is claimed. -- [Calculator benchmark](https://github.com/DaoyuanLi2816/mini-verl/blob/main/docs/benchmarking.md): both negative controls completed +- [Calculator benchmark](https://github.com/DaoyuanLi2816/mini-verl/blob/v0.6.1/docs/benchmarking.md): both negative controls completed normally and measured 0% strict success. They were not configuration failures. Because they used the historical ambiguous protocol-v1 prompt, their failure cannot be attributed solely to intrinsic teacher behavior. -- [Consumer Runtime v1](https://github.com/DaoyuanLi2816/mini-verl/blob/main/docs/consumer-runtime-v1.md): padded update batches and +- [Consumer Runtime v1](https://github.com/DaoyuanLi2816/mini-verl/blob/v0.6.1/docs/consumer-runtime-v1.md): padded update batches and shared adapters preserve the measured one-update objective within declared tolerances; rollout generation remains sequential. -- [Limitations](https://github.com/DaoyuanLi2816/mini-verl/blob/main/docs/limitations.md), [math](https://github.com/DaoyuanLi2816/mini-verl/blob/main/docs/math.md), - [reproducibility](https://github.com/DaoyuanLi2816/mini-verl/blob/main/docs/reproducibility.md) and - [compatibility policy](https://github.com/DaoyuanLi2816/mini-verl/blob/main/docs/compatibility.md). +- [Limitations](https://github.com/DaoyuanLi2816/mini-verl/blob/v0.6.1/docs/limitations.md), [math](https://github.com/DaoyuanLi2816/mini-verl/blob/v0.6.1/docs/math.md), + [reproducibility](https://github.com/DaoyuanLi2816/mini-verl/blob/v0.6.1/docs/reproducibility.md) and + [compatibility policy](https://github.com/DaoyuanLi2816/mini-verl/blob/v0.6.1/docs/compatibility.md). New runs establish tokenizer compatibility through structural identity. The legacy behavioral fingerprint—token IDs for one fixed probe plus metadata—is @@ -148,7 +148,7 @@ python -m pip install -e ".[dev]" pytest -q -m "not gpu and not network" ``` -Apache-2.0 licensed. See [CONTRIBUTING.md](https://github.com/DaoyuanLi2816/mini-verl/blob/main/CONTRIBUTING.md) and -[SECURITY.md](https://github.com/DaoyuanLi2816/mini-verl/blob/main/SECURITY.md). Project records: [default GPU recipe](https://github.com/DaoyuanLi2816/mini-verl/blob/main/recipes/qwen_consumer_gpu_calc.yaml), -[frozen calculator JSON](https://github.com/DaoyuanLi2816/mini-verl/blob/main/benchmarks/results/gpu-calc-hard-equal-update-v2.json), -[changelog](https://github.com/DaoyuanLi2816/mini-verl/blob/main/CHANGELOG.md), [citation](https://github.com/DaoyuanLi2816/mini-verl/blob/main/CITATION.cff) and [license](https://github.com/DaoyuanLi2816/mini-verl/blob/main/LICENSE). +Apache-2.0 licensed. See [CONTRIBUTING.md](https://github.com/DaoyuanLi2816/mini-verl/blob/v0.6.1/CONTRIBUTING.md) and +[SECURITY.md](https://github.com/DaoyuanLi2816/mini-verl/blob/v0.6.1/SECURITY.md). Project records: [default GPU recipe](https://github.com/DaoyuanLi2816/mini-verl/blob/v0.6.1/recipes/qwen_consumer_gpu_calc.yaml), +[frozen calculator JSON](https://github.com/DaoyuanLi2816/mini-verl/blob/v0.6.1/benchmarks/results/gpu-calc-hard-equal-update-v2.json), +[changelog](https://github.com/DaoyuanLi2816/mini-verl/blob/v0.6.1/CHANGELOG.md), [citation](https://github.com/DaoyuanLi2816/mini-verl/blob/v0.6.1/CITATION.cff) and [license](https://github.com/DaoyuanLi2816/mini-verl/blob/v0.6.1/LICENSE). diff --git a/README.md b/README.md index 1e53984..446fe97 100644 --- a/README.md +++ b/README.md @@ -25,7 +25,7 @@ assistant-only loss masks, teacher targets, update budgets and run artifacts explicit, then exports portable artifacts through a fail-closed bridge to one pinned upstream verl profile. -PyPI `v0.6.0` is stable; `main` is development. The CUDA path has no GPU-name +PyPI `v0.6.1` is stable; `main` is development. The CUDA path has no GPU-name allowlist, but fit depends on the model pair, context budget, kernels and VRAM. miniVERL is independent from verl and does not claim distributed execution or full algorithmic compatibility. diff --git a/README.zh-CN.md b/README.zh-CN.md index 5e4cd69..772f61e 100644 --- a/README.zh-CN.md +++ b/README.zh-CN.md @@ -24,7 +24,7 @@ 掩码、教师目标、更新预算与运行产物，并通过 fail-closed 桥接把可移植产物 交给一个锁定的上游 verl 配置。 -PyPI `v0.6.0` 是稳定版；`main` 是开发版。CUDA 路径没有显卡型号白名单， +PyPI `v0.6.1` 是稳定版；`main` 是开发版。CUDA 路径没有显卡型号白名单， 但能否运行取决于模型组合、上下文预算、内核和显存。miniVERL 独立于 verl， 不声称已经验证分布式执行或完整算法兼容性。 diff --git a/benchmarks/results/gpu-calc-hard-equal-update-v2.md b/benchmarks/results/gpu-calc-hard-equal-update-v2.md index 142b527..45312a4 100644 --- a/benchmarks/results/gpu-calc-hard-equal-update-v2.md +++ b/benchmarks/results/gpu-calc-hard-equal-update-v2.md @@ -25,7 +25,7 @@ Cold start, then equal-optimizer-update continuation under supervised fine-tunin ## Resolved controls -The complete common resolved configuration and every arm's structured diff are stored in the JSON artifact. Undeclared differences are rejected before any model is loaded. +The complete common declared configuration and separate scientific, runtime-resolution and harness-only diffs are stored in the JSON artifact. Undeclared scientific differences are rejected before any model is loaded. ## Notes diff --git a/docs/generated/quality.json b/docs/generated/quality.json index fb1853b..b482b3f 100644 --- a/docs/generated/quality.json +++ b/docs/generated/quality.json @@ -1,22 +1,22 @@ { "schema_version": 1, - "release": "0.6.0", - "status": "released", - "measured_commit": "0d43310cca47db828b10a0e12facb33e8f0fd371", - "measured_at": "2026-08-03T00:42:52-07:00", + "release": "0.6.1", + "status": "validated", + "measured_commit": "7fed8e79f02c0e0ba64ae6056760a75bc1b9b047", + "measured_at": "2026-08-03T14:53:12-07:00", "cpu_non_gpu_non_network": { - "passed": 1548, + "passed": 1563, "deselected": 6, - "branch_coverage_percent": 85.53 + "branch_coverage_percent": 86.0 }, "gpu": { - "passed": 3, - "deselected": 1551, + "passed": 5, + "deselected": 1564, "hardware": "NVIDIA GeForce RTX 4080" }, "network": { "passed": 3, - "deselected": 1551 + "deselected": 1566 }, - "quality_floor": "1,540+ tests and 85%+ branch coverage at v0.6.0" + "quality_floor": "1,560+ tests and 85%+ branch coverage at v0.6.1" } diff --git a/docs/overrides/main.html b/docs/overrides/main.html index f1cc49c..980306f 100644 --- a/docs/overrides/main.html +++ b/docs/overrides/main.html @@ -1,12 +1,12 @@ {% extends "base.html" %} {% block announce %} -
+
Stable documentation
{% endblock %} diff --git a/docs/release-checklist.md b/docs/release-checklist.md index 3cd44be..102bbc2 100644 --- a/docs/release-checklist.md +++ b/docs/release-checklist.md @@ -4,6 +4,59 @@ This is the release gate and publication record for miniVERL. A checked item names an invariant exercised on the stated source. Publication begins only after the exact release commit and its remote checks are green. +## v0.6.1 Visual integrity and bridge correctness release + +- [x] Alignment Lab publication is generated as one diverging forest chart and + two row matrices: every arm exposes all three frozen seeds and its mean, + quantitative marks remain in-domain, and non-teacher query ratios remain + `— not applicable` rather than being coerced to zero. +- [x] The scoped safety figure states that both sandbox checks tied at zero + while utility regressed, records the external endpoints as not run, and + does not imply a broad safety benchmark. +- [x] Desktop and mobile bridge diagrams distinguish the verified local + runtime, artifact bundle and pinned upstream parse/load smoke from the + dashed `Distributed execution: NOT TESTED` layer; teacher, reference and + reward roles are visually separate. +- [x] Material 9.7.7 builds stable and development documentation with search, + light/dark modes, copy controls and responsive navigation. Playwright + checks five pages at 1440x900, 1024x768, 820x1000 and 390x844 and uploads + all 20 screenshots. +- [x] `import-verl` classifies every supported source field as exact, derived, + informational only, requiring confirmation or unsupported. Incomplete + profiles publish a non-executable template with `needs_user_input`; no + calculator data or unspecified same-base teacher is substituted. +- [x] Runnable imports require explicit environment, teacher, loss and schedule + choices, safely coerce finite scientific notation, reject unresolved or + non-finite values, and pass `RunConfig` validation before atomic publish. +- [x] Exported bundles report artifact completeness, upstream parse/load, + model/data smoke, reward completeness, launchability, distributed testing + and semantic parity separately. The current fail-closed reward scaffold + remains `launchable: false` and emits `launch.template.sh`. +- [x] Ruff check/format, mypy across 103 source files, actionlint 1.7.12, + Markdown/link checks, strict MkDocs, generated-artifact byte comparisons, + SVG semantics, privacy checks and `git diff --check` pass. +- [x] Full non-GPU/non-network suite passes 1563 tests with 6 deselected and + 86% branch coverage; the available GPU suite passes 5 tests and the + network suite passes 3. +- [x] Wheel/sdist build, Twine, clean core and `[train]` installs, a real toy + demo, extracted-sdist tests and import/export end-to-end tests pass. +- [x] The calculator result remains byte-identical at SHA-256 + `53fc1d4d5b7adee09618d77ad62d4086ba56b78569832d6fc7c3bcd5c2695bbc`; + all RecoveryBench, Consumer Runtime, Alignment Lab and frozen bridge + result artifacts retain their audited hashes. +- [x] Focused PR [#40](https://github.com/DaoyuanLi2816/mini-verl/pull/40) + code head `7fed8e7` is green in CI, build, docs and pinned-profile smoke; + Linux-rendered screenshots were manually inspected at all four widths. +- [x] Release metadata declares exact `0.6.1`; the intended annotated tag is + exactly `v0.6.1` and will use the existing OIDC-only release workflow. + +## After the tag + +- [ ] Verify the annotated tag, OIDC run, PyPI hashes and attestations, a clean + public install, the GitHub Release and versioned stable documentation. +- [ ] Advance subsequent development to `0.6.2.dev0` in a separate green + state-sync pull request. + ## v0.6.0 Verified verl Bridge release - [x] The documented `single-gpu-online-distillation-v1` profile is pinned to diff --git a/src/miniverl/__init__.py b/src/miniverl/__init__.py index 99542be..198a66c 100644 --- a/src/miniverl/__init__.py +++ b/src/miniverl/__init__.py @@ -14,6 +14,6 @@ from __future__ import annotations -__version__ = "0.6.1.dev0" +__version__ = "0.6.1" __all__ = ["__version__"] diff --git a/tests/unit/test_packaging.py b/tests/unit/test_packaging.py index b129350..9c514e5 100644 --- a/tests/unit/test_packaging.py +++ b/tests/unit/test_packaging.py @@ -474,7 +474,7 @@ def test_release_quality_has_one_version_bound_machine_readable_record() -> None (REPO_ROOT / "docs" / "generated" / "quality.json").read_text(encoding="utf-8") ) assert record["schema_version"] == 1 - assert record["quality_floor"] == "1,540+ tests and 85%+ branch coverage at v0.6.0" + assert record["quality_floor"] == "1,560+ tests and 85%+ branch coverage at v0.6.1" if ".dev" in miniverl.__version__: assert record["status"] in {"candidate", "released"} if record["status"] == "candidate":