From 5142ed681a9c538b3175faf64561968a43cab547 Mon Sep 17 00:00:00 2001 From: Daoyuan Li <94409450+DaoyuanLi2816@users.noreply.github.com> Date: Tue, 11 Aug 2026 21:47:59 -0700 Subject: [PATCH 1/4] Prepare v0.7.1 product correction --- CHANGELOG.md | 23 ++- CITATION.cff | 14 +- PROJECT_STATE.md | 18 +- PYPI.md | 254 +++++++++++----------------- README.md | 222 ++++++++++-------------- README.zh-CN.md | 204 +++++++++------------- docs/generated/quality.json | 21 +-- docs/index.md | 83 +++++---- docs/overrides/main.html | 6 +- docs/release-checklist.md | 24 ++- pyproject.toml | 6 +- release-state.yaml | 10 +- src/miniverl/__init__.py | 2 +- src/miniverl/cli.py | 93 ++++++++-- src/miniverl/evidence/__init__.py | 166 ++++++++++++++++++ tests/cli/test_cli.py | 33 ++++ tests/unit/test_builtin_evidence.py | 27 +++ tests/unit/test_packaging.py | 17 ++ 18 files changed, 727 insertions(+), 496 deletions(-) create mode 100644 src/miniverl/evidence/__init__.py create mode 100644 tests/unit/test_builtin_evidence.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 00242f2..b4fb76c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,27 @@ All notable changes to miniVERL are recorded here. The format follows ## [Unreleased] +## [0.7.1] - 2026-08-11 + +### Product correction + +- Reordered the English, Chinese, PyPI and documentation landing pages around + the installable single-GPU runtime, its hardware boundary, the pinned verl + artifact bridge and measured systems evidence. Research studies and their + negative results remain intact under Research Notes rather than preceding + the quickstart. +- Updated package metadata, CLI help and diagnostics to describe the current + single-GPU alignment and distillation runtime without claiming the planned + verl-style OPD execution layer already exists. +- Added `miniverl evidence show/validate alignment-external-v1` and + `miniverl pilot --builtin-study alignment-external-v1`. The wheel now carries + the typed result, schema, preregistration and all 512 task-evidence rows with + byte-bound validation, so the primary pip journey no longer depends on a Git + checkout. +- Carried the corrected post-v0.7.0 evidence digest labels into a new immutable + stable release without changing the v0.7.0 tag or any frozen scientific + result. + ## [0.7.0] - 2026-08-10 ### External Alignment Gate result @@ -802,7 +823,7 @@ Same-tokenizer only; one trajectory per forward pass; `swap` unavailable for quantized models; only Qwen3 and Qwen2 architectures tested; single-seed GPU results. The full list is in `docs/limitations.md`. -[Unreleased]: https://github.com/DaoyuanLi2816/mini-verl/compare/v0.7.0...HEAD +[Unreleased]: https://github.com/DaoyuanLi2816/mini-verl/compare/v0.7.1...HEAD [0.7.0]: https://github.com/DaoyuanLi2816/mini-verl/compare/v0.6.3...v0.7.0 [0.6.3]: https://github.com/DaoyuanLi2816/mini-verl/compare/v0.6.2...v0.6.3 [0.6.2]: https://github.com/DaoyuanLi2816/mini-verl/compare/v0.6.1...v0.6.2 diff --git a/CITATION.cff b/CITATION.cff index ae8c302..dddbdb6 100644 --- a/CITATION.cff +++ b/CITATION.cff @@ -2,15 +2,15 @@ cff-version: 1.2.0 title: "miniVERL: Auditable single-GPU alignment and distillation runtime" message: "If you use miniVERL in your work, please cite it as below." type: software -version: 0.7.0 +version: 0.7.1 date-released: 2026-08-11 license: Apache-2.0 repository-code: "https://github.com/DaoyuanLi2816/mini-verl" url: "https://github.com/DaoyuanLi2816/mini-verl" abstract: >- - miniVERL is a single-GPU runtime for comparing alignment and distillation - methods under conditions strict enough that the comparison means something. - Its core is multi-turn, tool-aware on-policy distillation: a student language + miniVERL is an auditable single-GPU alignment and distillation runtime with a + bounded artifact bridge to one pinned verl profile. Its native core is + multi-turn, tool-aware on-policy distillation: a student language model samples its own tool-using trajectories against deterministic local environments, a teacher scores exactly the states the student visited, and the student is updated with token-level distributional supervision on its own @@ -20,8 +20,8 @@ abstract: >- never become a training label, and stores teacher targets in a versioned, checksummed, pickle-free cache with policy-version enforcement. Around that core it provides supervised fine-tuning, recorded-provenance - preference optimization and knowledge distillation as directly comparable - arms; shared-backbone execution that switches student, teacher and reference + preference optimization and knowledge distillation; shared-backbone + execution that switches student, teacher and reference roles across adapters on one set of base weights to fit low-memory hardware; padded update batching; deterministic replay; transactional checkpoints and cross-process run locks; and mechanism studies such as RecoveryBench that are @@ -32,7 +32,7 @@ abstract: >- algorithmic parity with PPO. miniVERL is designed for one personal CUDA GPU, automatically selects bf16 or fp16, and requires neither Ray nor a cluster. Published performance is measured on one RTX 4080; other GPU models use the - same code path but remain unmeasured. The v0.7.0 external-alignment study + same code path but remain unmeasured. The v0.7 external-alignment study terminated at its preregistered checkpoint-selection gate: two declared lineages and eight candidates all scored 0/64 retained JSONNav utility, so no teacher qualification, continuation method comparison or reserved final diff --git a/PROJECT_STATE.md b/PROJECT_STATE.md index 3dd3d41..0805a1a 100644 --- a/PROJECT_STATE.md +++ b/PROJECT_STATE.md @@ -6,11 +6,25 @@ and what it printed. Last updated: 2026-08-11. -Canonical release state: stable `v0.7.0` (`148822964dbb73e97ce06ef740f907364166a724`), development `0.7.1.dev0`. +Canonical release state: releasing `v0.7.1`. Every public version claim is generated from `release-state.yaml` and gated by `python scripts/release_state.py --check`. -## v0.7.0 External Alignment Gate — EVIDENCE RELEASE IN PROGRESS +## v0.7.1 Product correction — RELEASE CANDIDATE + +Branch `v0.7.1-product-correction` starts from synchronized main +`3ec042e829b4193aab363ab97e3786a1cd2621c0`. It changes product hierarchy, +installed evidence UX and metadata only; no scientific experiment, frozen +result, model revision or existing tag changes. + +The wheel carries the typed v0.7 result, schema, preregistration and 512 +privacy-safe JSONNav rows. Each file is checked against its recorded SHA-256, +the result is parsed through the impossible-state validator, task rows are +structurally checked, and `pilot --builtin-study` exposes the early-stop result +without a checkout. The planned verl-shaped executable OPD profile remains a +v0.8 development objective and is not claimed by this release. + +## v0.7.0 External Alignment Gate — RELEASED Branch `v0.7.0-evidence-release` starts from the exact post-PR-#52 main commit `a8272e2b5674e12107461a81d285f1a3d56588a5`. PR #51 merged the public diff --git a/PYPI.md b/PYPI.md index 0e0ff37..5f06cfe 100644 --- a/PYPI.md +++ b/PYPI.md @@ -1,5 +1,5 @@

- miniVERL — single-GPU LLM post-training + miniVERL — single-GPU LLM post-training

@@ -8,7 +8,7 @@ [![Build](https://github.com/DaoyuanLi2816/mini-verl/actions/workflows/build.yml/badge.svg)](https://github.com/DaoyuanLi2816/mini-verl/actions/workflows/build.yml) [![PyPI](https://img.shields.io/pypi/v/miniverl.svg)](https://pypi.org/project/miniverl/) [![Python](https://img.shields.io/badge/python-3.10%20%7C%203.11%20%7C%203.12%20%7C%203.13-blue)](https://www.python.org) -[![License](https://img.shields.io/badge/license-Apache--2.0-blue)](https://github.com/DaoyuanLi2816/mini-verl/blob/main/LICENSE) +[![License](https://img.shields.io/badge/license-Apache--2.0-blue)](https://github.com/DaoyuanLi2816/mini-verl/blob/v0.7.1/LICENSE)
@@ -16,173 +16,124 @@ PyPI · Stable docs · Development docs · - 中文 + 中文

-**miniVERL is a local, inspectable runtime for a documented subset of -single-GPU LLM alignment and distillation.** It keeps rollout provenance, -assistant-only loss masks, teacher targets, update budgets and run artifacts -explicit, then exports portable artifacts through a fail-closed bridge to one -pinned upstream verl profile. +**miniVERL is a local, inspectable single-GPU alignment and distillation +runtime.** It runs native SFT, DPO, KD and strict OPD recipes, preserves +assistant-only loss masks and policy-version provenance, and exchanges standard +HF/PEFT/Parquet artifacts through a fail-closed bridge to one pinned verl +profile. -PyPI `v0.7.0` is stable; `main` is development. The CUDA path has no GPU-name -allowlist, but fit depends on the model pair, context budget, kernels and VRAM. -miniVERL is independent from verl and does not claim distributed execution or -full algorithmic compatibility. +PyPI `v0.7.1` is stable; `main` is development. miniVERL is independent from +verl. It does not claim arbitrary verl YAML execution, distributed execution, +or full algorithmic compatibility. -## v0.7.0 — External Alignment Gate: a preregistered selection failure - -miniVERL's first real external-alignment study stopped before teacher or method -training: both predeclared starting-policy lineages scored **0/64** on the -retained JSONNav utility gate for every candidate. The unchanged floor was -20%. This release publishes the endpoint infrastructure, all 512 portable -selection rows and the fail-fast diagnosis; it does **not** publish a -post-training method comparison. - -| selected checkpoints | qualified teachers | continuation arms | final-test tasks accessed | -| ---: | ---: | ---: | ---: | -| **0** | **0** | **0** | **0** | +## Install and verify in about a minute ```bash -miniverl pilot --study-result benchmarks/results/alignment-external-v1.json --json +python -m pip install "miniverl[train]" +miniverl doctor +miniverl demo --fast --output runs/quickstart +miniverl inspect runs/quickstart/trajectories.jsonl +miniverl evidence validate alignment-external-v1 ``` -The command returns `do_not_continue_this_study` and -`insufficient_evidence`, not SFT/DPO/KD/OPD. Granite Guardian was used only as -an unqualified selection diagnostic; Granite qualification, PairRM -qualification, teacher qualification and the reserved final test did not run. -Read the [early-stop study](https://github.com/DaoyuanLi2816/mini-verl/blob/main/docs/alignment-external/alignment-external-v1.md) -and [typed result](https://github.com/DaoyuanLi2816/mini-verl/blob/main/benchmarks/results/alignment-external-v1.json). +The deterministic demo downloads no model and produces typed trajectories, a +checksummed teacher cache, manifest and report. The evidence command reads +self-contained package data; it works from a wheel without a Git checkout. +For schemas and inspection without the ML stack, install `miniverl` alone. -## Install and run the 60-second demo +## Supported hardware and runtime boundary -```bash -python -m pip install "miniverl[train]" -miniverl doctor -miniverl demo --output runs/demo -miniverl inspect runs/demo -``` +miniVERL runs one local process on CPU or one NVIDIA CUDA GPU. The CUDA path is +device-name agnostic, but fit depends on model pair, context, kernels and VRAM. +Install the matching CUDA-enabled PyTorch build first, then +`miniverl[train,cuda]`; that extra does not select a CUDA PyTorch wheel. +Ray, FSDP, Megatron, PPO, GRPO and distributed launch are outside the runtime. +See the [single-GPU guide](https://github.com/DaoyuanLi2816/mini-verl/blob/v0.7.1/docs/single-gpu-guide.md). + +## verl compatibility summary + +The bridge targets official verl `v0.8.0` at commit `7aed6b23`. Its verified +boundary is checksummed standard artifacts plus pinned config-parse and +model/data-load smoke—not native checkpoint parity or a completed verl job. +Imports fail closed when dataset, environment, teacher, objective or schedule +semantics are unresolved; they never substitute calculator tasks or invent an +unqualified teacher. + +Current exports remain `launchable: false`: the base snapshot is absent, the +reward scaffold fails closed and required mappings remain placeholders. The +entry point is `launch.template.sh`; readiness, parse/load evidence, +launchability, distributed execution and semantic parity are separate facts. +[Read the bridge contract](https://github.com/DaoyuanLi2816/mini-verl/blob/v0.7.1/docs/verl-bridge.md). + +## One measured systems result + +On one RTX 4080 with Qwen3-0.6B and eight fixed SQLite trajectories, physical +batch 4 increased dual-model update throughput from 2.369 to 3.866 +trajectories/s. Shared-backbone batch 4 used 2.227 GiB peak reserved memory +versus 3.035 GiB for dual model while running 10.1% slower. All 12 +preregistered equivalence comparisons passed. This is one workload on one +machine, not a promise for other GPUs. -The demo is deterministic, needs no network or GPU, and performs a real toy -optimization in about 50 seconds on the measured laptop CPU. For inspection, -schemas and reports without the ML stack, use `pip install miniverl`. For CUDA -training, install the matching CUDA-enabled PyTorch wheel first, then install -`miniverl[train,cuda]`; the extra does not select a CUDA PyTorch build. See the -[single-GPU guide](https://github.com/DaoyuanLi2816/mini-verl/blob/main/docs/single-gpu-guide.md). +![Measured throughput and reserved VRAM for dual-model and shared-backbone runtime cells](https://raw.githubusercontent.com/DaoyuanLi2816/mini-verl/v0.7.1/docs/consumer-runtime-v1-pareto.svg) + +[Consumer Runtime v1 methods and caveats](https://github.com/DaoyuanLi2816/mini-verl/blob/v0.7.1/docs/consumer-runtime-v1.md) ## Three paths | Path | Start with | Concrete artifact | Next | | --- | --- | --- | --- | -| **Align** — compare SFT, DPO, KD and OPD only when the pilot evidence supports the cost | `miniverl pilot recipes/alignment_policy_conditioned_qwen.yaml` | `alignment-card.json` | [Alignment Lab](https://github.com/DaoyuanLi2816/mini-verl/blob/main/docs/alignment-lab/alignment-lab-v1.md) | -| **Distill locally** — strict OPD, shared backbones and padded trajectory updates on one CUDA GPU | `miniverl train recipes/qwen_consumer_gpu_shared.yaml --dry-run` | `config.resolved.yaml` plus a revision-pinned PEFT adapter | [Bring your own GPU](https://github.com/DaoyuanLi2816/mini-verl/blob/main/docs/single-gpu-guide.md) | -| **Scale out** — import a documented profile, convert Parquet, export a bundle and run bridge checks | `miniverl bridge doctor scaleout-bundle` | `provenance/compatibility-report.json` | [Verified verl artifact bridge](https://github.com/DaoyuanLi2816/mini-verl/blob/main/docs/verl-bridge.md) | - -The bridge is a **verified artifact bridge**: a pinned config/data/model -parse-load smoke at miniVERL-defined compatibility Level 3. It has never run a -distributed verl job, and no OPD-to-PPO semantic parity is claimed. - -The import is deliberately not generic YAML conversion. If dataset or -environment, teacher identity, objective, or schedule semantics are missing, -`import-verl` writes `.import-report.json` and a non-executable -`.template.yaml` with `status: needs_user_input`. It never silently -substitutes calculator tasks or an unqualified same-base teacher. An unresolved -`${...}` value can never reach an accepted recipe, outputs are stem-specific, -an input file can never also be an output file, and an existing output family -is replaced only with an explicit `--overwrite`. Publication is transactional -with in-process rollback, not multi-file crash atomicity. - -An exported bundle is untrusted input. `bridge doctor` inspects its reward -scaffold statically with `ast.parse` and **never executes it** unless you pass -`--trust-and-import-reward-code` — class bases, `metaclass=`, annotations and -type-parameter bounds are audited too, because all of them run at import. -Adapter weights are validated past the header, a malformed extension sidecar -fails the conversion instead of being read as empty, and dataset conversion -streams row groups rather than materializing the table. What a bundle *claims* -is reported separately from what was *recomputed* locally: its own `SHA256SUMS` -can only prove internal consistency. Tokenizer, safetensors and privacy results -each report how far verification actually got rather than a single pass or fail. - -## Earlier measured alignment result - -Alignment Lab v1 is a **saturated tool-policy case study**, not a broad safety -benchmark. The shared SFT checkpoint already achieved 100% policy compliance -and 100% retained tool utility in all three seeds. No continuation method -improved it; continued SFT and both OPD variants retained measured regressions. - -| continuation | alignment | tool utility | teacher queries | GPU time | -| --- | ---: | ---: | ---: | ---: | -| continued SFT | 94.4% | 88.9% | — | 3.9 s | -| DPO | 100.0% | 100.0% | — | 8.6 s | -| offline soft distillation | 100.0% | 100.0% | 100.0% | 26.6 s | -| standard OPD | 98.6% | 97.2% | 100.0% | 76.7 s | -| verifier-gated OPD | 97.9% | 95.8% | 46.8% | 66.0 s | - -![Alignment and utility deltas from the saturated SFT checkpoint; small marks are all three seeds and large marks are means](https://raw.githubusercontent.com/DaoyuanLi2816/mini-verl/main/docs/alignment-lab/delta-from-sft.svg) +| **Align** — use SFT, DPO, KD or OPD only when pilot evidence supports the cost | `miniverl pilot recipes/alignment_policy_conditioned_qwen.yaml` | `alignment-card.json` | [Alignment Lab](https://github.com/DaoyuanLi2816/mini-verl/blob/v0.7.1/docs/alignment-lab/alignment-lab-v1.md) | +| **Distill locally** — strict OPD, shared backbones and padded updates on one CUDA GPU | `miniverl train recipes/qwen_consumer_gpu_shared.yaml --dry-run` | resolved config and revision-pinned PEFT adapter | [Bring your own GPU](https://github.com/DaoyuanLi2816/mini-verl/blob/v0.7.1/docs/single-gpu-guide.md) | +| **Scale out** — convert Parquet, export standard artifacts and inspect the unsupported boundary | `miniverl bridge doctor scaleout-bundle` | `provenance/compatibility-report.json` | [Verified artifact bridge](https://github.com/DaoyuanLi2816/mini-verl/blob/v0.7.1/docs/verl-bridge.md) | -The two sandbox safety checks tied at zero while utility still regressed. -IFEval, XSTest, HarmBench and RewardBench were **not executed**. “Preference -win rate” is a deterministic Minipolicy paired outcome, not human preference. -Read the [study, seed-level values and limitations](https://github.com/DaoyuanLi2816/mini-verl/blob/main/docs/alignment-lab/alignment-lab-v1.md). +## Research notes and preserved negative evidence -## One measured systems result +### v0.7 External Alignment Gate -On one RTX 4080 with Qwen3-0.6B and eight fixed SQLite trajectories, physical -batch 4 improved update throughput from 2.369 to 3.866 trajectories/s in the -dual-model runtime. The shared-backbone batch-4 cell used 2.227 GiB peak -reserved memory versus 3.035 GiB for dual model, while running 10.1% slower. -All 12 preregistered equivalence comparisons passed. These are one-workload, -one-machine measurements, not promises for other GPUs. - -![Measured throughput and reserved VRAM for dual-model and shared-backbone runtime cells](https://raw.githubusercontent.com/DaoyuanLi2816/mini-verl/main/docs/consumer-runtime-v1-pareto.svg) - -[Consumer Runtime v1 methods and caveats](https://github.com/DaoyuanLi2816/mini-verl/blob/main/docs/consumer-runtime-v1.md) - -## Compatibility boundary - -![Verified local runtime, portable artifact bundle and pinned upstream smoke; distributed verl execution remains untested](https://raw.githubusercontent.com/DaoyuanLi2816/mini-verl/main/docs/verl-bridge-architecture.svg) - -The bridge targets official verl `v0.8.0` at commit `7aed6b23` and uses the -term **miniVERL-defined compatibility Level 3**. That means a checksummed -standard-artifact bundle plus pinned upstream config-parse/model-data-load -smoke—not arbitrary verl YAML or a completed distributed job. - -Current exported bundles are intentionally `launchable: false`: the base -snapshot is absent, the reward implementation fails closed, and required user -mappings remain placeholders. The generated entry point is therefore -`launch.template.sh`. Readiness is reported as separate facts for artifact -completeness, parse/load smoke, reward completeness, launchability, -distributed execution and algorithm-semantic parity. The target is a -PPO/reward scaffold, not an executable continuation of miniVERL OPD semantics. - -## Detailed studies and preserved negative evidence - -- [RecoveryBench v1](https://github.com/DaoyuanLi2816/mini-verl/blob/main/docs/recoverybench/recoverybench-v1.md): frozen-student KD - outperformed much slower fresh-state OPD on the preregistered primary view; - the verifier gate remained `insufficient_evidence`. -- [Alignment Lab v1](https://github.com/DaoyuanLi2816/mini-verl/blob/main/docs/alignment-lab/alignment-lab-v1.md): the starting SFT - checkpoint was at the ceiling, so no positive OPD result is claimed. -- [Calculator benchmark](https://github.com/DaoyuanLi2816/mini-verl/blob/main/docs/benchmarking.md): both negative controls completed - normally and measured 0% strict success. They were not configuration - failures. Because they used the historical ambiguous protocol-v1 prompt, - their failure cannot be attributed solely to intrinsic teacher behavior. -- [Consumer Runtime v1](https://github.com/DaoyuanLi2816/mini-verl/blob/main/docs/consumer-runtime-v1.md): padded update batches and - shared adapters preserve the measured one-update objective within declared - tolerances; rollout generation remains sequential. -- [Limitations](https://github.com/DaoyuanLi2816/mini-verl/blob/main/docs/limitations.md), [math](https://github.com/DaoyuanLi2816/mini-verl/blob/main/docs/math.md), - [reproducibility](https://github.com/DaoyuanLi2816/mini-verl/blob/main/docs/reproducibility.md) and - [compatibility policy](https://github.com/DaoyuanLi2816/mini-verl/blob/main/docs/compatibility.md). +The preregistered external study stopped before teacher or method training. +Both declared starting-policy lineages scored **0/64** retained JSONNav utility +for every candidate against the unchanged 20% floor. -New runs establish tokenizer compatibility through structural identity. The -legacy behavioral fingerprint—token IDs for one fixed probe plus metadata—is -only a migration fallback for older artifacts and is not an identity proof. +| selected checkpoints | qualified teachers | continuation arms | final-test tasks accessed | +| ---: | ---: | ---: | ---: | +| **0** | **0** | **0** | **0** | -## Scope +```bash +miniverl pilot --builtin-study alignment-external-v1 --json +``` + +The result is `do_not_continue_this_study` and `insufficient_evidence`, not a +recommendation among SFT/DPO/KD/OPD. Granite Guardian values are unqualified +selection diagnostics; Granite, PairRM and teacher qualification and the +reserved final test did not run. [Study and limitations](https://github.com/DaoyuanLi2816/mini-verl/blob/v0.7.1/docs/alignment-external/alignment-external-v1.md). + +### Earlier measured alignment case study + +Alignment Lab v1 began from an SFT checkpoint already at 100% policy compliance +and 100% retained tool utility in all three seeds. No continuation improved the +ceiling; continued SFT and both OPD variants retained measured regressions. +The two sandbox safety checks tied at zero while utility still regressed. +IFEval, XSTest, HarmBench and RewardBench were not executed, and “preference +win rate” is a deterministic Minipolicy paired outcome, not human preference. +[Seed-level evidence](https://github.com/DaoyuanLi2816/mini-verl/blob/v0.7.1/docs/alignment-lab/alignment-lab-v1.md). + +- [RecoveryBench v1](https://github.com/DaoyuanLi2816/mini-verl/blob/v0.7.1/docs/recoverybench/recoverybench-v1.md): frozen-student KD + beat slower fresh-state OPD on the preregistered primary view; the verifier + gate remained `insufficient_evidence`. +- [Calculator benchmark](https://github.com/DaoyuanLi2816/mini-verl/blob/v0.7.1/docs/benchmarking.md): both negative controls completed + normally at 0%; the ambiguous historical protocol-v1 prompt prevents + attributing failure solely to intrinsic teacher behavior. +- [Limitations](https://github.com/DaoyuanLi2816/mini-verl/blob/v0.7.1/docs/limitations.md), [math](https://github.com/DaoyuanLi2816/mini-verl/blob/v0.7.1/docs/math.md), + [reproducibility](https://github.com/DaoyuanLi2816/mini-verl/blob/v0.7.1/docs/reproducibility.md) and + [compatibility policy](https://github.com/DaoyuanLi2816/mini-verl/blob/v0.7.1/docs/compatibility.md). + +New runs establish tokenizer compatibility through structural identity. The +legacy behavioral fingerprint is only a migration fallback, not identity proof. -miniVERL supports one local CUDA process. It does not implement or wrap Ray, -FSDP, Megatron, PPO, GRPO or a distributed launcher. The public studies cover -small Qwen3 models, deterministic tool environments and one RTX 4080; they do -not establish cross-model, cross-task, cross-GPU or broad safety generality. +## Develop ```bash git clone https://github.com/DaoyuanLi2816/mini-verl.git @@ -191,7 +142,8 @@ python -m pip install -e ".[dev]" pytest -q -m "not gpu and not network" ``` -Apache-2.0 licensed. See [CONTRIBUTING.md](https://github.com/DaoyuanLi2816/mini-verl/blob/main/CONTRIBUTING.md) and -[SECURITY.md](https://github.com/DaoyuanLi2816/mini-verl/blob/main/SECURITY.md). Project records: [default GPU recipe](https://github.com/DaoyuanLi2816/mini-verl/blob/main/recipes/qwen_consumer_gpu_calc.yaml), -[frozen calculator JSON](https://github.com/DaoyuanLi2816/mini-verl/blob/main/benchmarks/results/gpu-calc-hard-equal-update-v2.json), -[changelog](https://github.com/DaoyuanLi2816/mini-verl/blob/main/CHANGELOG.md), [citation](https://github.com/DaoyuanLi2816/mini-verl/blob/main/CITATION.cff) and [license](https://github.com/DaoyuanLi2816/mini-verl/blob/main/LICENSE). +Apache-2.0 licensed. See [CONTRIBUTING.md](https://github.com/DaoyuanLi2816/mini-verl/blob/v0.7.1/CONTRIBUTING.md), +[SECURITY.md](https://github.com/DaoyuanLi2816/mini-verl/blob/v0.7.1/SECURITY.md), the [changelog](https://github.com/DaoyuanLi2816/mini-verl/blob/v0.7.1/CHANGELOG.md) and +[citation](https://github.com/DaoyuanLi2816/mini-verl/blob/v0.7.1/CITATION.cff). Project records: [default GPU recipe](https://github.com/DaoyuanLi2816/mini-verl/blob/v0.7.1/recipes/qwen_consumer_gpu_calc.yaml), +[frozen calculator result](https://github.com/DaoyuanLi2816/mini-verl/blob/v0.7.1/benchmarks/results/gpu-calc-hard-equal-update-v2.json) +and [license](https://github.com/DaoyuanLi2816/mini-verl/blob/v0.7.1/LICENSE). diff --git a/README.md b/README.md index a672536..60b24c4 100644 --- a/README.md +++ b/README.md @@ -19,170 +19,121 @@ 中文

-**miniVERL is a local, inspectable runtime for a documented subset of -single-GPU LLM alignment and distillation.** It keeps rollout provenance, -assistant-only loss masks, teacher targets, update budgets and run artifacts -explicit, then exports portable artifacts through a fail-closed bridge to one -pinned upstream verl profile. - -PyPI `v0.7.0` is stable; `main` is development. The CUDA path has no GPU-name -allowlist, but fit depends on the model pair, context budget, kernels and VRAM. -miniVERL is independent from verl and does not claim distributed execution or -full algorithmic compatibility. - -## v0.7.0 — External Alignment Gate: a preregistered selection failure - -miniVERL's first real external-alignment study stopped before teacher or method -training: both predeclared starting-policy lineages scored **0/64** on the -retained JSONNav utility gate for every candidate. The unchanged floor was -20%. This release publishes the endpoint infrastructure, all 512 portable -selection rows and the fail-fast diagnosis; it does **not** publish a -post-training method comparison. +**miniVERL is a local, inspectable single-GPU alignment and distillation +runtime.** It runs native SFT, DPO, KD and strict OPD recipes, preserves +assistant-only loss masks and policy-version provenance, and exchanges standard +HF/PEFT/Parquet artifacts through a fail-closed bridge to one pinned verl +profile. -| selected checkpoints | qualified teachers | continuation arms | final-test tasks accessed | -| ---: | ---: | ---: | ---: | -| **0** | **0** | **0** | **0** | - -```bash -miniverl pilot --study-result benchmarks/results/alignment-external-v1.json --json -``` +PyPI `v0.7.1` is stable; `main` is development. miniVERL is independent from +verl. It does not claim arbitrary verl YAML execution, distributed execution, +or full algorithmic compatibility. -The command returns `do_not_continue_this_study` and -`insufficient_evidence`, not SFT/DPO/KD/OPD. Granite Guardian was used only as -an unqualified selection diagnostic; Granite qualification, PairRM -qualification, teacher qualification and the reserved final test did not run. -Read the [early-stop study](docs/alignment-external/alignment-external-v1.md) -and [typed result](benchmarks/results/alignment-external-v1.json). - -## Install and run the 60-second demo +## Install and verify in about a minute ```bash python -m pip install "miniverl[train]" miniverl doctor -miniverl demo --output runs/demo -miniverl inspect runs/demo +miniverl demo --fast --output runs/quickstart +miniverl inspect runs/quickstart/trajectories.jsonl +miniverl evidence validate alignment-external-v1 ``` -The demo is deterministic, needs no network or GPU, and performs a real toy -optimization in about 50 seconds on the measured laptop CPU. For inspection, -schemas and reports without the ML stack, use `pip install miniverl`. For CUDA -training, install the matching CUDA-enabled PyTorch wheel first, then install -`miniverl[train,cuda]`; the extra does not select a CUDA PyTorch build. See the -[single-GPU guide](docs/single-gpu-guide.md). +The deterministic demo downloads no model and produces typed trajectories, a +checksummed teacher cache, manifest and report. The evidence command reads +self-contained package data; it works from a wheel without a Git checkout. +For schemas and inspection without the ML stack, install `miniverl` alone. -## Three paths +## Supported hardware and runtime boundary -| Path | Start with | Concrete artifact | Next | -| --- | --- | --- | --- | -| **Align** — compare SFT, DPO, KD and OPD only when the pilot evidence supports the cost | `miniverl pilot recipes/alignment_policy_conditioned_qwen.yaml` | `alignment-card.json` | [Alignment Lab](docs/alignment-lab/alignment-lab-v1.md) | -| **Distill locally** — strict OPD, shared backbones and padded trajectory updates on one CUDA GPU | `miniverl train recipes/qwen_consumer_gpu_shared.yaml --dry-run` | `config.resolved.yaml` plus a revision-pinned PEFT adapter | [Bring your own GPU](docs/single-gpu-guide.md) | -| **Scale out** — import a documented profile, convert Parquet, export a bundle and run bridge checks | `miniverl bridge doctor scaleout-bundle` | `provenance/compatibility-report.json` | [Verified verl artifact bridge](docs/verl-bridge.md) | - -The bridge is a **verified artifact bridge**: a pinned config/data/model -parse-load smoke at miniVERL-defined compatibility Level 3. It has never run a -distributed verl job, and no OPD-to-PPO semantic parity is claimed. - -The import is deliberately not generic YAML conversion. If dataset or -environment, teacher identity, objective, or schedule semantics are missing, -`import-verl` writes `.import-report.json` and a non-executable -`.template.yaml` with `status: needs_user_input`. It never silently -substitutes calculator tasks or an unqualified same-base teacher. An unresolved -`${...}` value can never reach an accepted recipe, outputs are stem-specific, -an input file can never also be an output file, and an existing output family -is replaced only with an explicit `--overwrite`. Publication is transactional -with in-process rollback, not multi-file crash atomicity. - -An exported bundle is untrusted input. `bridge doctor` inspects its reward -scaffold statically with `ast.parse` and **never executes it** unless you pass -`--trust-and-import-reward-code` — class bases, `metaclass=`, annotations and -type-parameter bounds are audited too, because all of them run at import. -Adapter weights are validated past the header, a malformed extension sidecar -fails the conversion instead of being read as empty, and dataset conversion -streams row groups rather than materializing the table. What a bundle *claims* -is reported separately from what was *recomputed* locally: its own `SHA256SUMS` -can only prove internal consistency. Tokenizer, safetensors and privacy results -each report how far verification actually got rather than a single pass or fail. - -## Earlier measured alignment result - -Alignment Lab v1 is a **saturated tool-policy case study**, not a broad safety -benchmark. The shared SFT checkpoint already achieved 100% policy compliance -and 100% retained tool utility in all three seeds. No continuation method -improved it; continued SFT and both OPD variants retained measured regressions. - -| continuation | alignment | tool utility | teacher queries | GPU time | -| --- | ---: | ---: | ---: | ---: | -| continued SFT | 94.4% | 88.9% | — | 3.9 s | -| DPO | 100.0% | 100.0% | — | 8.6 s | -| offline soft distillation | 100.0% | 100.0% | 100.0% | 26.6 s | -| standard OPD | 98.6% | 97.2% | 100.0% | 76.7 s | -| verifier-gated OPD | 97.9% | 95.8% | 46.8% | 66.0 s | - -![Alignment and utility deltas from the saturated SFT checkpoint; small marks are all three seeds and large marks are means](docs/alignment-lab/delta-from-sft.svg) +miniVERL runs one local process on CPU or one NVIDIA CUDA GPU. The CUDA path is +device-name agnostic, but fit depends on model pair, context, kernels and VRAM. +Install the matching CUDA-enabled PyTorch build first, then +`miniverl[train,cuda]`; that extra does not select a CUDA PyTorch wheel. +Ray, FSDP, Megatron, PPO, GRPO and distributed launch are outside the runtime. +See the [single-GPU guide](docs/single-gpu-guide.md). -The two sandbox safety checks tied at zero while utility still regressed. -IFEval, XSTest, HarmBench and RewardBench were **not executed**. “Preference -win rate” is a deterministic Minipolicy paired outcome, not human preference. -Read the [study, seed-level values and limitations](docs/alignment-lab/alignment-lab-v1.md). +## verl compatibility summary + +The bridge targets official verl `v0.8.0` at commit `7aed6b23`. Its verified +boundary is checksummed standard artifacts plus pinned config-parse and +model/data-load smoke—not native checkpoint parity or a completed verl job. +Imports fail closed when dataset, environment, teacher, objective or schedule +semantics are unresolved; they never substitute calculator tasks or invent an +unqualified teacher. + +Current exports remain `launchable: false`: the base snapshot is absent, the +reward scaffold fails closed and required mappings remain placeholders. The +entry point is `launch.template.sh`; readiness, parse/load evidence, +launchability, distributed execution and semantic parity are separate facts. +[Read the bridge contract](docs/verl-bridge.md). ## One measured systems result On one RTX 4080 with Qwen3-0.6B and eight fixed SQLite trajectories, physical -batch 4 improved update throughput from 2.369 to 3.866 trajectories/s in the -dual-model runtime. The shared-backbone batch-4 cell used 2.227 GiB peak -reserved memory versus 3.035 GiB for dual model, while running 10.1% slower. -All 12 preregistered equivalence comparisons passed. These are one-workload, -one-machine measurements, not promises for other GPUs. +batch 4 increased dual-model update throughput from 2.369 to 3.866 +trajectories/s. Shared-backbone batch 4 used 2.227 GiB peak reserved memory +versus 3.035 GiB for dual model while running 10.1% slower. All 12 +preregistered equivalence comparisons passed. This is one workload on one +machine, not a promise for other GPUs. ![Measured throughput and reserved VRAM for dual-model and shared-backbone runtime cells](docs/consumer-runtime-v1-pareto.svg) [Consumer Runtime v1 methods and caveats](docs/consumer-runtime-v1.md) -## Compatibility boundary +## Three paths + +| Path | Start with | Concrete artifact | Next | +| --- | --- | --- | --- | +| **Align** — use SFT, DPO, KD or OPD only when pilot evidence supports the cost | `miniverl pilot recipes/alignment_policy_conditioned_qwen.yaml` | `alignment-card.json` | [Alignment Lab](docs/alignment-lab/alignment-lab-v1.md) | +| **Distill locally** — strict OPD, shared backbones and padded updates on one CUDA GPU | `miniverl train recipes/qwen_consumer_gpu_shared.yaml --dry-run` | resolved config and revision-pinned PEFT adapter | [Bring your own GPU](docs/single-gpu-guide.md) | +| **Scale out** — convert Parquet, export standard artifacts and inspect the unsupported boundary | `miniverl bridge doctor scaleout-bundle` | `provenance/compatibility-report.json` | [Verified artifact bridge](docs/verl-bridge.md) | -![Verified local runtime, portable artifact bundle and pinned upstream smoke; distributed verl execution remains untested](docs/verl-bridge-architecture.svg) +## Research notes and preserved negative evidence -The bridge targets official verl `v0.8.0` at commit `7aed6b23` and uses the -term **miniVERL-defined compatibility Level 3**. That means a checksummed -standard-artifact bundle plus pinned upstream config-parse/model-data-load -smoke—not arbitrary verl YAML or a completed distributed job. +### v0.7 External Alignment Gate -Current exported bundles are intentionally `launchable: false`: the base -snapshot is absent, the reward implementation fails closed, and required user -mappings remain placeholders. The generated entry point is therefore -`launch.template.sh`. Readiness is reported as separate facts for artifact -completeness, parse/load smoke, reward completeness, launchability, -distributed execution and algorithm-semantic parity. The target is a -PPO/reward scaffold, not an executable continuation of miniVERL OPD semantics. +The preregistered external study stopped before teacher or method training. +Both declared starting-policy lineages scored **0/64** retained JSONNav utility +for every candidate against the unchanged 20% floor. -## Detailed studies and preserved negative evidence +| selected checkpoints | qualified teachers | continuation arms | final-test tasks accessed | +| ---: | ---: | ---: | ---: | +| **0** | **0** | **0** | **0** | + +```bash +miniverl pilot --builtin-study alignment-external-v1 --json +``` + +The result is `do_not_continue_this_study` and `insufficient_evidence`, not a +recommendation among SFT/DPO/KD/OPD. Granite Guardian values are unqualified +selection diagnostics; Granite, PairRM and teacher qualification and the +reserved final test did not run. [Study and limitations](docs/alignment-external/alignment-external-v1.md). + +### Earlier measured alignment case study + +Alignment Lab v1 began from an SFT checkpoint already at 100% policy compliance +and 100% retained tool utility in all three seeds. No continuation improved the +ceiling; continued SFT and both OPD variants retained measured regressions. +The two sandbox safety checks tied at zero while utility still regressed. +IFEval, XSTest, HarmBench and RewardBench were not executed, and “preference +win rate” is a deterministic Minipolicy paired outcome, not human preference. +[Seed-level evidence](docs/alignment-lab/alignment-lab-v1.md). - [RecoveryBench v1](docs/recoverybench/recoverybench-v1.md): frozen-student KD - outperformed much slower fresh-state OPD on the preregistered primary view; - the verifier gate remained `insufficient_evidence`. -- [Alignment Lab v1](docs/alignment-lab/alignment-lab-v1.md): the starting SFT - checkpoint was at the ceiling, so no positive OPD result is claimed. + beat slower fresh-state OPD on the preregistered primary view; the verifier + gate remained `insufficient_evidence`. - [Calculator benchmark](docs/benchmarking.md): both negative controls completed - normally and measured 0% strict success. They were not configuration - failures. Because they used the historical ambiguous protocol-v1 prompt, - their failure cannot be attributed solely to intrinsic teacher behavior. -- [Consumer Runtime v1](docs/consumer-runtime-v1.md): padded update batches and - shared adapters preserve the measured one-update objective within declared - tolerances; rollout generation remains sequential. + normally at 0%; the ambiguous historical protocol-v1 prompt prevents + attributing failure solely to intrinsic teacher behavior. - [Limitations](docs/limitations.md), [math](docs/math.md), [reproducibility](docs/reproducibility.md) and [compatibility policy](docs/compatibility.md). New runs establish tokenizer compatibility through structural identity. The -legacy behavioral fingerprint—token IDs for one fixed probe plus metadata—is -only a migration fallback for older artifacts and is not an identity proof. - -## Scope +legacy behavioral fingerprint is only a migration fallback, not identity proof. -miniVERL supports one local CUDA process. It does not implement or wrap Ray, -FSDP, Megatron, PPO, GRPO or a distributed launcher. The public studies cover -small Qwen3 models, deterministic tool environments and one RTX 4080; they do -not establish cross-model, cross-task, cross-GPU or broad safety generality. +## Develop ```bash git clone https://github.com/DaoyuanLi2816/mini-verl.git @@ -191,7 +142,8 @@ python -m pip install -e ".[dev]" pytest -q -m "not gpu and not network" ``` -Apache-2.0 licensed. See [CONTRIBUTING.md](CONTRIBUTING.md) and -[SECURITY.md](SECURITY.md). Project records: [default GPU recipe](recipes/qwen_consumer_gpu_calc.yaml), -[frozen calculator JSON](benchmarks/results/gpu-calc-hard-equal-update-v2.json), -[changelog](CHANGELOG.md), [citation](CITATION.cff) and [license](LICENSE). +Apache-2.0 licensed. See [CONTRIBUTING.md](CONTRIBUTING.md), +[SECURITY.md](SECURITY.md), the [changelog](CHANGELOG.md) and +[citation](CITATION.cff). Project records: [default GPU recipe](recipes/qwen_consumer_gpu_calc.yaml), +[frozen calculator result](benchmarks/results/gpu-calc-hard-equal-update-v2.json) +and [license](LICENSE). diff --git a/README.zh-CN.md b/README.zh-CN.md index 0b0decd..99e651c 100644 --- a/README.zh-CN.md +++ b/README.zh-CN.md @@ -19,153 +19,107 @@ English

-**miniVERL 是一个本地、可检查的单卡 LLM 对齐与蒸馏运行时,只实现有明确 -文档的功能子集。** 它显式保存 rollout 来源、仅 assistant token 的 loss -掩码、教师目标、更新预算与运行产物,并通过 fail-closed 桥接把可移植产物 -交给一个锁定的上游 verl 配置。 +**miniVERL 是一个本地、可检查的单卡对齐与蒸馏运行时。** 它运行原生 +SFT、DPO、KD 与严格 OPD recipe,保留仅 assistant token 的 loss mask 和 +policy-version 来源,并通过 fail-closed 桥接与一个锁定的 verl profile 交换 +标准 HF/PEFT/Parquet 产物。 -PyPI `v0.7.0` 是稳定版;`main` 是开发版。CUDA 路径没有显卡型号白名单, -但能否运行取决于模型组合、上下文预算、内核和显存。miniVERL 独立于 verl, -不声称已经验证分布式执行或完整算法兼容性。 +PyPI `v0.7.1` 是稳定版;`main` 是开发版。miniVERL 独立于 verl,不声称可 +执行任意 verl YAML、分布式任务或具备完整算法兼容性。 -## v0.7.0 — External Alignment Gate:一次预注册的选点失败 - -miniVERL 的第一次真实外部对齐研究在教师与方法训练之前停止:两个预先声明 -的起始策略 lineage 中,每个候选在 retained JSONNav utility gate 上都得到 -**0/64**,而未改动的下限是 20%。本版本发布 endpoint 基础设施、全部 512 条 -可移植 selection 记录和 fail-fast 诊断;它**不发布**后训练方法比较。 - -| 选中 checkpoint | 合格教师 | continuation arm | 已访问 final-test task | -| ---: | ---: | ---: | ---: | -| **0** | **0** | **0** | **0** | +## 大约一分钟完成安装与验证 ```bash -miniverl pilot --study-result benchmarks/results/alignment-external-v1.json --json +python -m pip install "miniverl[train]" +miniverl doctor +miniverl demo --fast --output runs/quickstart +miniverl inspect runs/quickstart/trajectories.jsonl +miniverl evidence validate alignment-external-v1 ``` -该命令返回 `do_not_continue_this_study` 和 `insufficient_evidence`,不会推荐 -SFT、DPO、KD 或 OPD。Granite Guardian 只作为未资格认证的 selection -diagnostic 使用;Granite、PairRM 和教师资格认证均未运行,保留的 final test -也未访问。详见[早停研究](docs/alignment-external/alignment-external-v1.md)与 -[类型化结果](benchmarks/results/alignment-external-v1.json)。 +确定性 demo 不下载模型,会生成类型化 trajectory、带校验和的 teacher cache、 +manifest 与报告。证据命令读取 wheel 自带数据,无需 Git checkout。若只需 schema +与检查功能,可仅安装 `miniverl`。 -## 安装与 60 秒演示 +## 支持的硬件与运行边界 -```bash -python -m pip install "miniverl[train]" -miniverl doctor -miniverl demo --output runs/demo -miniverl inspect runs/demo -``` +miniVERL 在 CPU 或一张 NVIDIA CUDA GPU 上运行单个本地进程。CUDA 路径不按 +显卡名称设限,但能否装下取决于模型组合、上下文、kernel 与显存。请先安装匹配 +本机 CUDA 的 PyTorch wheel,再安装 `miniverl[train,cuda]`;该 extra 本身不会 +选择 CUDA PyTorch。Ray、FSDP、Megatron、PPO、GRPO 与分布式启动不属于当前 +运行时。参见[单卡指南](docs/single-gpu-guide.md)。 -这个确定性演示无需网络或 GPU,会执行一次真实的玩具优化;在实测笔记本 -CPU 上约需 50 秒。若只需要 schema、检查与报告,可安装 -`pip install miniverl`。CUDA 训练请先安装与本机匹配的 CUDA PyTorch wheel, -再安装 `miniverl[train,cuda]`;这个 extra 本身不会选择 CUDA 版 PyTorch。 -详见[单卡 GPU 指南](docs/single-gpu-guide.md)。 +## verl 兼容性摘要 -## 三条使用路径 +桥接锁定官方 verl `v0.8.0`、commit `7aed6b23`。已验证的边界是带校验和的标准 +产物以及锁定版本的配置解析、模型/数据加载冒烟测试,不包括原生 checkpoint +等价或已完成的 verl 作业。若数据集、环境、教师、目标或 schedule 语义未解析, +导入会 fail closed;它不会替换成 calculator task 或虚构教师。 -| 路径 | 起点 | 真实产物 | 下一步 | -| --- | --- | --- | --- | -| **Align** — 只有 pilot 证据支持成本时,才在 SFT、DPO、KD 与 OPD 间选择 | `miniverl pilot recipes/alignment_policy_conditioned_qwen.yaml` | `alignment-card.json` | [Alignment Lab](docs/alignment-lab/alignment-lab-v1.md) | -| **Distill locally** — 在一张 CUDA GPU 上运行严格 OPD、共享 backbone 与 padded trajectory update | `miniverl train recipes/qwen_consumer_gpu_shared.yaml --dry-run` | `config.resolved.yaml` 与锁定 revision 的 PEFT adapter | [使用自己的 GPU](docs/single-gpu-guide.md) | -| **Scale out** — 导入已文档化 profile、转换 Parquet、导出 bundle 并执行 bridge 检查 | `miniverl bridge doctor scaleout-bundle` | `provenance/compatibility-report.json` | [verl 产物桥接](docs/verl-bridge.md) | - -该桥接是一座**已验证的产物桥接**:在 miniVERL 自定义的兼容性 Level 3 上, -对锁定版本的配置/数据/模型执行 parse-load 冒烟测试。它从未运行过分布式 -verl 作业,也不声称 OPD 与 PPO 之间存在语义等价。 - -桥接导入不是通用 YAML 转换。当数据集/环境、教师身份、目标函数或 schedule -语义不完整时,`import-verl` 只会写出 `.import-report.json` 和不可执行的 -`.template.yaml`,状态为 `needs_user_input`。它不会悄悄改用 -calculator 环境,也不会创建身份不明确的同基座教师。未解析的 `${...}` 绝不会 -进入被接受的 recipe;输出文件名按 stem 隔离;输入文件永远不能同时是输出 -文件;只有显式传入 `--overwrite` 才会替换已存在的输出文件族。发布是带 -进程内回滚的事务式发布,而不是跨多文件的崩溃原子性。 - -导出的 bundle 属于不可信输入。`bridge doctor` 用 `ast.parse` 静态检查其 -reward scaffold,**默认绝不执行它**,除非显式传入 -`--trust-and-import-reward-code`;基类、`metaclass=`、类型注解与类型参数 -边界同样会被审查,因为它们都在 import 时求值。adapter 权重的校验会越过 -文件头验证实际载荷;格式非法的 extension sidecar 会让转换失败,而不是被 -当成空文件读过去;数据集转换按 row group 流式处理,不再整表物化。bundle -自己**声称**的内容与本地**实际重算**的结果分开报告:它自带的 `SHA256SUMS` -只能证明内部一致性。tokenizer、safetensors 与隐私三项各自报告验证真正 -到达的层级,而不是笼统的通过或失败。 - -## 更早的一项对齐实测结果 - -Alignment Lab v1 是一个**已饱和的工具策略案例研究**,不是广义安全评测。 -共同的 SFT 起点在三个 seed 上都已经达到 100% 策略合规和 100% 工具效用。 -没有 continuation 方法能够继续提升;continued SFT 与两种 OPD 的实测退化 -均被保留。 - -| continuation | 对齐 | 工具效用 | 教师查询 | GPU 时间 | -| --- | ---: | ---: | ---: | ---: | -| continued SFT | 94.4% | 88.9% | — | 3.9 s | -| DPO | 100.0% | 100.0% | — | 8.6 s | -| offline soft distillation | 100.0% | 100.0% | 100.0% | 26.6 s | -| standard OPD | 98.6% | 97.2% | 100.0% | 76.7 s | -| verifier-gated OPD | 97.9% | 95.8% | 46.8% | 66.0 s | - -![相对已饱和 SFT 起点的对齐与效用变化;小标记为三个 seed,大标记为均值](docs/alignment-lab/delta-from-sft.svg) - -两个 sandbox 安全检查都为零,但工具效用仍然退化。IFEval、XSTest、 -HarmBench 与 RewardBench **没有实际执行**。“preference win rate” 是确定性 -Minipolicy 配对结果,不是人类偏好。详见[完整研究、逐 seed 数值和局限](docs/alignment-lab/alignment-lab-v1.md)。 +当前导出仍为 `launchable: false`:缺少 base snapshot,reward scaffold 会失败关闭, +且必要映射仍是占位符。入口名为 `launch.template.sh`;readiness、parse/load 证据、 +launchability、分布式执行与语义等价分别报告。参见[桥接契约](docs/verl-bridge.md)。 ## 一项系统实测结果 -在一张 RTX 4080、Qwen3-0.6B 和八条固定 SQLite trajectory 上,物理 batch 4 -把 dual-model runtime 的更新吞吐从 2.369 提高到 3.866 trajectories/s。 -shared-backbone 的 batch-4 cell 峰值 reserved memory 为 2.227 GiB,dual -model 为 3.035 GiB,但前者慢 10.1%。全部 12 个预注册等价性比较通过。 -这些是单任务、单机器结果,不是对其他 GPU 的保证。 +在一张 RTX 4080、Qwen3-0.6B 与 8 条固定 SQLite trajectory 上,physical batch 4 +将 dual-model update throughput 从 2.369 提高到 3.866 trajectories/s。 +shared-backbone batch 4 的 peak reserved memory 为 2.227 GiB,dual model 为 +3.035 GiB,但前者慢 10.1%。12 项预注册等价比较全部通过。这只是单机单 workload +数据,不是对其他 GPU 的承诺。 ![dual-model 与 shared-backbone runtime 的实测吞吐和 reserved VRAM](docs/consumer-runtime-v1-pareto.svg) -[Consumer Runtime v1 方法与局限](docs/consumer-runtime-v1.md) +[Consumer Runtime v1 方法与限制](docs/consumer-runtime-v1.md) -## 兼容性边界 +## 三条使用路径 -![已验证的本地 runtime、可移植产物 bundle 与上游 smoke;分布式 verl 执行未测试](docs/verl-bridge-architecture.svg) +| 路径 | 起点 | 真实产物 | 下一步 | +| --- | --- | --- | --- | +| **Align** — 仅在 pilot 证据支持成本时使用 SFT、DPO、KD 或 OPD | `miniverl pilot recipes/alignment_policy_conditioned_qwen.yaml` | `alignment-card.json` | [Alignment Lab](docs/alignment-lab/alignment-lab-v1.md) | +| **本地蒸馏** — 在一张 CUDA GPU 上运行严格 OPD、共享 backbone 与 padded update | `miniverl train recipes/qwen_consumer_gpu_shared.yaml --dry-run` | resolved config 与锁定 revision 的 PEFT adapter | [使用自己的 GPU](docs/single-gpu-guide.md) | +| **Scale out** — 转换 Parquet、导出标准产物并检查不支持边界 | `miniverl bridge doctor scaleout-bundle` | `provenance/compatibility-report.json` | [产物桥接](docs/verl-bridge.md) | + +## 研究记录与保留的负结果 + +### v0.7 External Alignment Gate + +这项预注册外部研究在教师或方法训练前停止。两个已声明的起始策略 lineage 中, +所有候选的 retained JSONNav utility 都是 **0/64**,未改动下限为 20%。 + +| 选中 checkpoint | 合格教师 | continuation arm | 已访问 final-test task | +| ---: | ---: | ---: | ---: | +| **0** | **0** | **0** | **0** | -桥接锁定官方 verl `v0.8.0`、commit `7aed6b23`,并使用 -**miniVERL-defined compatibility Level 3** 这一名称。它表示 checksummed -标准产物 bundle 与锁定上游版本的 config-parse/model-data-load smoke, -不表示任意 verl YAML 都兼容,也不表示完成过分布式任务。 +```bash +miniverl pilot --builtin-study alignment-external-v1 --json +``` -当前导出的 bundle 有意报告 `launchable: false`:base snapshot 不在 bundle -中,reward 实现仍 fail closed,而且必要的用户映射仍是 placeholder。因此 -入口名为 `launch.template.sh`。报告会分别给出 artifact 完整性、parse/load -smoke、reward 完整性、launchability、分布式执行和算法语义等价状态。 -当前目标是 PPO/reward scaffold,不是 miniVERL OPD 的可执行延续。 +结果是 `do_not_continue_this_study` 与 `insufficient_evidence`,不是 SFT/DPO/KD/OPD +之间的推荐。Granite Guardian 数值仅为未资格认证的 selection diagnostic;Granite、 +PairRM、教师资格认证和保留 final test 均未运行。参见[研究与限制](docs/alignment-external/alignment-external-v1.md)。 -## 详细研究与保留的负结果 +### 更早的对齐案例研究 + +Alignment Lab v1 的起始 SFT checkpoint 在三个 seed 中已经达到 100% policy +compliance 与 100% retained tool utility;没有 continuation 超过这个天花板, +continued SFT 和两个 OPD arm 保留了实测退化。两个 sandbox safety check 同为零, +同时 utility 仍然退化。IFEval、XSTest、HarmBench、RewardBench 未执行; +“preference win rate”是确定性的 Minipolicy 配对结果,不是人类偏好。 +[逐 seed 证据](docs/alignment-lab/alignment-lab-v1.md)。 - [RecoveryBench v1](docs/recoverybench/recoverybench-v1.md):在预注册主视图中, - frozen-student KD 优于耗时高得多的 fresh-state OPD;verifier gate 仍为 + frozen-student KD 优于更慢的 fresh-state OPD;verifier gate 仍为 `insufficient_evidence`。 -- [Alignment Lab v1](docs/alignment-lab/alignment-lab-v1.md):起始 SFT 已到 - ceiling,因此不宣称任何正向 OPD 结果。 -- [Calculator benchmark](docs/benchmarking.md):两个 negative control 都正常 - 完成并测得 0% strict success,不是配置失败。它们使用了历史上有歧义的 - protocol-v1 prompt,因此不能把失败完全归因于教师的内在行为。 -- [Consumer Runtime v1](docs/consumer-runtime-v1.md):padded update batch 与 - shared adapter 在既定容差内保持单次更新目标;rollout 生成仍是逐条执行。 -- [局限](docs/limitations.md)、[数学](docs/math.md)、 - [可复现性](docs/reproducibility.md)与[兼容策略](docs/compatibility.md)。 - -新运行以 tokenizer 结构身份作为主要兼容性检查。旧版 behavioral -fingerprint 只对一个固定 probe 的 token ID 与元数据做摘要,仅用于旧产物迁移, -不能证明两个 tokenizer 的身份相同。 - -## 范围 - -miniVERL 只支持单个本地 CUDA 进程,不实现或包装 Ray、FSDP、Megatron、 -PPO、GRPO 或分布式 launcher。公开研究只覆盖小型 Qwen3、确定性工具环境与 -一张 RTX 4080,不能推出跨模型、跨任务、跨 GPU 或广义安全结论。 +- [Calculator benchmark](docs/benchmarking.md):两个负对照都正常完成并得到 0%; + 历史 protocol-v1 prompt 存在歧义,因此不能把失败仅归因于教师内在行为。 +- [限制](docs/limitations.md)、[数学](docs/math.md)、[复现](docs/reproducibility.md) + 与[兼容性政策](docs/compatibility.md)。 + +新运行以结构身份确认 tokenizer 兼容性;旧 behavioral fingerprint 只用于迁移, +不是身份凭证。 + +## 开发 ```bash git clone https://github.com/DaoyuanLi2816/mini-verl.git @@ -174,7 +128,5 @@ python -m pip install -e ".[dev]" pytest -q -m "not gpu and not network" ``` -项目使用 Apache-2.0 许可证。参见 [CONTRIBUTING.md](CONTRIBUTING.md) 与 -[SECURITY.md](SECURITY.md)。项目记录:[默认 GPU 配方](recipes/qwen_consumer_gpu_calc.yaml)、 -[冻结的 calculator JSON](benchmarks/results/gpu-calc-hard-equal-update-v2.json)、 -[变更记录](CHANGELOG.md)、[引用信息](CITATION.cff)与[许可证](LICENSE)。 +Apache-2.0 许可。参见 [CONTRIBUTING.md](CONTRIBUTING.md)、 +[SECURITY.md](SECURITY.md)、[changelog](CHANGELOG.md) 与 [citation](CITATION.cff)。 diff --git a/docs/generated/quality.json b/docs/generated/quality.json index 9e4dc1d..f8cd894 100644 --- a/docs/generated/quality.json +++ b/docs/generated/quality.json @@ -1,8 +1,8 @@ { "schema_version": 2, - "release": "0.7.0", - "status": "released", - "quality_floor": "2,000+ tests and 85%+ branch coverage at v0.7.0", + "release": "0.7.1", + "status": "candidate", + "quality_floor": "2,000+ tests and 85%+ branch coverage at v0.7.1", "local_validation": { "scope": "the maintainer's workstation, where the GPU and Windows-specific paths actually run", "commit": "013993a8cd5002a9ed166ba1e6948305f92c1bfb", @@ -12,10 +12,10 @@ "python": "CPython 3.12", "coverage_mode": "branch", "cpu_non_gpu_non_network": { - "passed": 2098, + "passed": 2110, "skipped": 6, "deselected": 21, - "branch_coverage_percent": 86.35, + "branch_coverage_percent": 86.19, "skip_reason": "symlink creation requires privileges on Windows; hard-link and case aliases cover the same guard" }, "gpu": { @@ -28,14 +28,9 @@ }, "release_validation": { "scope": "the exact published commit, validated by CI rather than locally", - "commit": "148822964dbb73e97ce06ef740f907364166a724", - "workflows": { - "ci": "https://github.com/DaoyuanLi2816/mini-verl/actions/runs/31468298531", - "build": "https://github.com/DaoyuanLi2816/mini-verl/actions/runs/31468298548", - "docs": "https://github.com/DaoyuanLi2816/mini-verl/actions/runs/31468298534", - "release": "https://github.com/DaoyuanLi2816/mini-verl/actions/runs/31468663273" - }, - "conclusion": "success", + "commit": "pending", + "workflows": {}, + "conclusion": "pending", "gpu_coverage": "none; no GPU runner is configured for this repository, so the GPU counts above exist only from the local measurement" } } diff --git a/docs/index.md b/docs/index.md index fee9eb6..db8c77c 100644 --- a/docs/index.md +++ b/docs/index.md @@ -1,42 +1,47 @@ # miniVERL -Auditable single-GPU LLM post-training for choosing, running and inspecting -SFT, DPO, knowledge distillation and strict OPD—plus a bounded artifact bridge -to one pinned verl profile. miniVERL is independent; no upstream endorsement is -implied, and distributed execution is not tested. +Auditable single-GPU alignment and distillation runtime with native SFT, DPO, +KD and strict OPD recipes, inspectable artifacts and a bounded bridge to one +pinned verl profile. miniVERL is independent; distributed execution and full +algorithm compatibility are not claimed. [Install and run locally](single-gpu-guide.md){ .md-button .md-button--primary } [Read the compatibility boundary](verl-bridge.md){ .md-button } ## Install and verify in about a minute -Install the PyTorch build that matches your CPU or CUDA system first, then the -training extra. This CPU example is deterministic and downloads no model: - ```bash -python -m pip install torch --index-url https://download.pytorch.org/whl/cpu python -m pip install "miniverl[train]" miniverl demo --fast --output runs/quickstart miniverl inspect runs/quickstart/trajectories.jsonl +miniverl evidence validate alignment-external-v1 ``` -The result is a typed trajectory log, checksummed teacher cache, manifest and -self-contained report. For CUDA wheels and memory-aware recipes, use the -[single-GPU guide](single-gpu-guide.md). +The deterministic demo downloads no model and produces typed trajectories, a +checksummed teacher cache, manifest and report. Packaged evidence commands need +no repository checkout. For CUDA, install the matching CUDA-enabled PyTorch +wheel first; the `[cuda]` extra does not select one. -## v0.7.0 evidence release: the external study stopped at its first gate +## Runtime and compatibility boundary -**0 selected checkpoints · 0 qualified teachers · 0 continuation arms · 0 -final-test tasks accessed.** Every candidate in both declared lineages scored -0/64 retained JSONNav utility against an unchanged 20% floor. The result is a -preregistered checkpoint-selection failure, not a method comparison. +miniVERL runs one local CPU process or one NVIDIA CUDA GPU. Fit depends on the +model pair, context, kernels and VRAM; there is no GPU-name allowlist. Ray, +FSDP, Megatron, PPO, GRPO and distributed launch are outside the runtime. -```bash -miniverl pilot --study-result benchmarks/results/alignment-external-v1.json --json -``` +The artifact bridge pins verl `v0.8.0` at `7aed6b23`. It verifies standard +artifact interchange and pinned config/model/data parse-load smoke. Current +exports are not launchable and do not establish algorithmic parity. -[Read the early-stop study](alignment-external/alignment-external-v1.md){ .md-button .md-button--primary } -[Inspect the evidence contract](https://github.com/DaoyuanLi2816/mini-verl/blob/v0.7.0/benchmarks/results/alignment-external-v1.json){ .md-button } +## Measured systems evidence + +On one RTX 4080 with Qwen3-0.6B and eight fixed SQLite trajectories, padded +updates increased dual-model update throughput from 2.369 to 3.866 +trajectories/s. Shared-backbone batch 4 used 2.227 GiB peak reserved memory +versus 3.035 GiB for dual model while running 10.1% slower. All 12 +preregistered equivalence comparisons passed. This is one measured workload, +not a hardware-wide promise. + +[Consumer Runtime methods and caveats](consumer-runtime/index.md){ .md-button } ## Choose a path @@ -80,8 +85,7 @@ miniverl train recipes/qwen_consumer_gpu_shared.yaml --dry-run --json ## Scale out -Import only the documented profile, convert Parquet, export standard artifacts -and inspect the unsupported boundary. +Convert Parquet, export standard artifacts and inspect the unsupported boundary. ```bash miniverl bridge doctor exports/my-bundle --json @@ -96,20 +100,25 @@ flags; current bundles are not launchable. -## Measured evidence, kept scoped +## Research Notes -The Alignment Lab case study starts from an SFT checkpoint already at 100% -alignment and 100% retained tool utility on its deterministic sandbox suite. -No continuation method improves it; completed regressions remain visible. -External IFEval, XSTest, HarmBench and RewardBench endpoints were not executed. +The v0.7 external study stopped at its first preregistered gate: **0 selected +checkpoints, 0 qualified teachers, 0 continuation arms and 0 final-test tasks +accessed**. All eight candidates scored 0/64 retained JSONNav utility against +the unchanged 20% floor. + +```bash +miniverl pilot --builtin-study alignment-external-v1 --json +``` + +[Read the early-stop study](alignment-external/alignment-external-v1.md){ .md-button .md-button--primary } - - - Forest chart of continuation-method alignment and retained-tool-utility percentage-point deltas from the saturated SFT checkpoint, with every seed and mean printed as text. - +Alignment Lab starts from a saturated SFT checkpoint. No continuation method +improves it; measured regressions and unexecuted external safety endpoints stay +visible. RecoveryBench and the calculator study likewise preserve their +negative and mixed results rather than turning them into product claims. -The consumer runtime result is a systems result, not a new quality claim: -shared-backbone role switching and padded trajectory updates reduce measured -memory/runtime overhead while preserving the tested local objective. See the -[Consumer Runtime report](consumer-runtime/index.md) and -[RecoveryBench](recoverybench/recoverybench-v1.md) for full evidence and limits. +- [Alignment Lab](alignment-lab/alignment-lab-v1.md) +- [RecoveryBench](recoverybench/recoverybench-v1.md) +- [Calculator protocol study](benchmarking.md) +- [External Alignment Gate](alignment-external/alignment-external-v1.md) diff --git a/docs/overrides/main.html b/docs/overrides/main.html index ea0b937..2b08680 100644 --- a/docs/overrides/main.html +++ b/docs/overrides/main.html @@ -1,12 +1,12 @@ {% extends "base.html" %} {% block announce %} -
+
Stable documentation
{% endblock %} diff --git a/docs/release-checklist.md b/docs/release-checklist.md index 5eebefd..c485f79 100644 --- a/docs/release-checklist.md +++ b/docs/release-checklist.md @@ -4,10 +4,26 @@ This is the release gate and publication record for miniVERL. A checked item names an invariant exercised on the stated source. Publication begins only after the exact release commit and its remote checks are green. -## v0.7.1 (in development) - -- [ ] Define and review the next maintenance scope before implementation. No - new scientific experiment is authorized by the v0.7.0 state sync. +## v0.7.1 Product correction + +- [ ] README, Chinese README, PyPI description and docs landing page lead with + the installable runtime, hardware boundary, compatibility boundary and + measured systems evidence; preserved research results follow under + Research Notes. +- [ ] CLI help, doctor wording, project metadata and CFF describe the current + single-GPU alignment/distillation runtime without promising v0.8 + execution semantics. +- [ ] The wheel packages the v0.7 external-study result, schema, + preregistration and 512 task rows. `evidence show`, `evidence validate` + and `pilot --builtin-study` work in a clean core-only installation with + no checkout. +- [ ] Ruff, format, mypy, actionlint, CPU/GPU/network tests, strict docs, + Playwright, package/extracted-sdist, clean installs, bridge and + frozen-artifact gates pass on the exact release candidate. +- [ ] `git shortlog` and commit/body scans show Daoyuan Li as the only source + author since v0.7.0 and no AI attribution trailers. +- [ ] The v0.7.0 tag, published adapter revisions and every frozen benchmark + remain unchanged. ## v0.7.0 External Alignment Gate evidence release diff --git a/pyproject.toml b/pyproject.toml index 1322455..64c0756 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -5,7 +5,7 @@ build-backend = "hatchling.build" [project] name = "miniverl" dynamic = ["version"] -description = "Auditable one-GPU post-training with a verified pinned verl bridge." +description = "Auditable single-GPU alignment and distillation with a bounded verl artifact bridge." readme = "PYPI.md" requires-python = ">=3.10" license = { text = "Apache-2.0" } @@ -122,6 +122,10 @@ core-metadata-version = "2.4" [tool.hatch.build.targets.wheel.force-include] "benchmarks/external-alignment/registry.yaml" = "miniverl/alignment_external/registry.yaml" "benchmarks/external-alignment/profile-v1.yaml" = "miniverl/alignment_external/profile-v1.yaml" +"benchmarks/results/alignment-external-v1.json" = "miniverl/evidence/data/alignment-external-v1/result.json" +"benchmarks/schema/alignment-external-result.schema.json" = "miniverl/evidence/data/alignment-external-v1/result.schema.json" +"benchmarks/preregistration/alignment-external-v1.yaml" = "miniverl/evidence/data/alignment-external-v1/preregistration.yaml" +"benchmarks/evidence/alignment-external-v1/jsonnav-selection-records.jsonl" = "miniverl/evidence/data/alignment-external-v1/task-evidence.jsonl" [tool.hatch.build.targets.sdist] core-metadata-version = "2.4" diff --git a/release-state.yaml b/release-state.yaml index 366ecde..96fc1df 100644 --- a/release-state.yaml +++ b/release-state.yaml @@ -17,13 +17,13 @@ # such distinction, which is how its tag shipped a docs selector still # advertising "Stable 0.6.1 / Development 0.6.2.dev0". schema_version: 1 -phase: development +phase: release stable: - version: "0.7.0" - tag: "v0.7.0" - release_commit: "148822964dbb73e97ce06ef740f907364166a724" + version: "0.7.1" + tag: "v0.7.1" + release_commit: "pending" released_at: "2026-08-11" development: - version: "0.7.1.dev0" + version: "0.7.1" diff --git a/src/miniverl/__init__.py b/src/miniverl/__init__.py index 03424aa..9aea2de 100644 --- a/src/miniverl/__init__.py +++ b/src/miniverl/__init__.py @@ -14,6 +14,6 @@ from __future__ import annotations -__version__ = "0.7.1.dev0" +__version__ = "0.7.1" __all__ = ["__version__"] diff --git a/src/miniverl/cli.py b/src/miniverl/cli.py index 71035db..e598ab2 100644 --- a/src/miniverl/cli.py +++ b/src/miniverl/cli.py @@ -29,9 +29,9 @@ app = typer.Typer( name="miniverl", help=( - "On-policy distillation for tool-using agents on one GPU.\n\n" - "A readable single-CUDA-GPU post-training lab for exact and budgeted " - "on-policy distillation." + "Auditable single-GPU alignment and distillation runtime.\n\n" + "Run native local workflows, inspect every artifact, and exchange standard " + "HF/PEFT/Parquet artifacts through a bounded verl artifact bridge." ), add_completion=False, no_args_is_help=True, @@ -51,6 +51,10 @@ no_args_is_help=True, ) app.add_typer(alignment_suite_app, name="alignment-suite") +evidence_app = typer.Typer( + help="Show and validate evidence packaged with the installed wheel.", no_args_is_help=True +) +app.add_typer(evidence_app, name="evidence") console = Console() err_console = Console(stderr=True) @@ -132,7 +136,7 @@ def main( envvar="MINIVERL_LOG_LEVEL", ), ) -> None: - """miniVERL: on-policy distillation for tool-using agents on one GPU.""" + """miniVERL: an auditable single-GPU alignment and distillation runtime.""" from miniverl.utils.logging import configure_logging configure_logging(log_level) @@ -181,7 +185,11 @@ def doctor( "cpu_training", 'pip install "miniverl[train]"', ), - ("GPU training", "gpu_training", "install a CUDA build of torch"), + ( + "single-GPU CUDA training (native recipes)", + "gpu_training", + "install a CUDA build of torch", + ), ("4-bit QLoRA", "qlora_4bit", 'pip install "miniverl[train,cuda]"'), ): ready = verdict[key] @@ -614,7 +622,55 @@ def qualify_teacher_command( console.print(f" result {_esc(out / 'result.json')}") -# ----------------------------------------------------------------- train +# ------------------------------------------------------------- evidence + + +@evidence_app.command("show") +def evidence_show( + study_id: str = typer.Argument(..., help="Packaged study identifier."), + as_json: bool = typer.Option(False, "--json", help="Emit machine-readable JSON."), +) -> None: + """Show a packaged, typed study result without a repository checkout.""" + from miniverl.evidence import show_builtin_study + + try: + payload = show_builtin_study(study_id) + except (MiniVerlError, OSError, ValidationError) as exc: + _fail(exc) + return + if as_json: + _emit_json(payload) + return + console.print(f"[bold]{_esc(study_id)}[/bold]") + console.print_json(json.dumps(payload["result"], allow_nan=False)) + + +@evidence_app.command("validate") +def evidence_validate( + study_id: str = typer.Argument(..., help="Packaged study identifier."), + as_json: bool = typer.Option(False, "--json", help="Emit machine-readable JSON."), +) -> None: + """Validate packaged result, schema, preregistration and task evidence.""" + from miniverl.evidence import validate_builtin_study + + try: + payload = validate_builtin_study(study_id) + except (MiniVerlError, OSError, ValidationError) as exc: + _fail(exc) + return + if as_json: + _emit_json(payload) + elif payload["valid"]: + console.print( + f"[green]valid[/green] {_esc(study_id)} · {_esc(payload['task_rows'])} task rows" + ) + else: + for problem in payload["problems"]: + err_console.print(f"[red]invalid[/red] {_esc(problem)}") + raise typer.Exit(1) + + +# ----------------------------------------------------------------- pilot @app.command() @@ -627,6 +683,11 @@ def pilot( "--study-result", help="Schema-validated external-study result; does not load a model.", ), + builtin_study: Optional[str] = typer.Option( + None, + "--builtin-study", + help="Packaged external-study result; works from an installed wheel.", + ), out: Optional[Path] = typer.Option(None, "--out", help="Optional JSON output path."), as_json: bool = typer.Option(False, "--json", help="Emit machine-readable JSON."), ) -> None: @@ -635,8 +696,17 @@ def pilot( payload: dict[str, Any] try: - if recipe is not None and study_result is not None: - raise ConfigError("miniverl pilot accepts either a recipe or --study-result, not both") + selected = sum(value is not None for value in (recipe, study_result, builtin_study)) + if selected > 1: + raise ConfigError( + "miniverl pilot accepts exactly one of a recipe, --study-result, or --builtin-study" + ) + builtin = None + if builtin_study is not None: + from miniverl.evidence import get_builtin_study + + builtin = get_builtin_study(builtin_study) + study_result = builtin.result_path if study_result is not None: from miniverl.alignment_external.result import load_alignment_external_result @@ -660,6 +730,7 @@ def pilot( ], "evidence": { "path": str(study_result), + "builtin_study": builtin.study_id if builtin is not None else None, "sha256": hashlib.sha256(study_result.read_bytes()).hexdigest(), "preregistration": result.preregistration.model_dump(mode="json"), "task_evidence": result.checkpoint_selection.task_evidence.model_dump( @@ -670,7 +741,9 @@ def pilot( } else: if recipe is None: - raise ConfigError("miniverl pilot requires a recipe or --study-result") + raise ConfigError( + "miniverl pilot requires a recipe, --study-result or --builtin-study" + ) from miniverl.alignment import PilotEvidence, recommend_alignment_method from miniverl.config import RunConfig @@ -685,7 +758,7 @@ def pilot( except (ValidationError, MiniVerlError) as exc: if isinstance(exc, MiniVerlError): _fail(exc) - source = recipe if recipe is not None else study_result + source = recipe if recipe is not None else study_result or builtin_study err_console.print(f"[red]invalid pilot evidence[/red] {_esc(source)}\n{_esc(exc)}") raise typer.Exit(1) from None if as_json: diff --git a/src/miniverl/evidence/__init__.py b/src/miniverl/evidence/__init__.py new file mode 100644 index 0000000..90a41b3 --- /dev/null +++ b/src/miniverl/evidence/__init__.py @@ -0,0 +1,166 @@ +"""Self-contained, read-only evidence shipped with the core wheel.""" + +from __future__ import annotations + +import hashlib +import json +from dataclasses import dataclass +from importlib.resources import files +from pathlib import Path +from typing import Any, Final + +from miniverl.alignment_external.result import load_alignment_external_result +from miniverl.errors import ConfigError + +__all__ = ["BuiltinStudy", "get_builtin_study", "show_builtin_study", "validate_builtin_study"] + + +@dataclass(frozen=True, slots=True) +class BuiltinStudy: + """Paths and immutable digests for one packaged evidence bundle.""" + + study_id: str + result_path: Path + schema_path: Path + preregistration_path: Path + task_evidence_path: Path + result_sha256: str + schema_sha256: str + preregistration_sha256: str + task_evidence_sha256: str + + +_ALIGNMENT_EXTERNAL_V1: Final = { + "result_sha256": "085cbe1f8035a0904482332d60f9f46ae3039d2e5ac4725e2ecafb7b42d0eda8", + "schema_sha256": "d41dc15bbd0d3b6852e858142f11c5b89adf0ce591676abc7e42082665c82044", + "preregistration_sha256": "b87596f05d6c411ac5a2f982729200287d5bc917b1708b1fc1640bf53e2ca379", + "task_evidence_sha256": "694d68cd997bc4b2aa7dd88ebf6572616c9a140fb0df4a672c301095a4f16c7c", +} + + +def _sha256(path: Path) -> str: + return hashlib.sha256(path.read_bytes()).hexdigest() + + +def _validate_selection_rows(rows: list[dict[str, Any]]) -> list[str]: + problems: list[str] = [] + seen: set[tuple[str, str, str]] = set() + required = { + "lineage_id", + "candidate_id", + "suite_task_id", + "trajectory_digest", + "schema_version", + "solved", + } + for index, row in enumerate(rows): + missing = sorted(required - row.keys()) + if missing: + problems.append(f"row {index}: missing {', '.join(missing)}") + continue + key = (str(row["lineage_id"]), str(row["candidate_id"]), str(row["suite_task_id"])) + if key in seen: + problems.append(f"row {index}: duplicate {key}") + seen.add(key) + if row["schema_version"] != 1: + problems.append(f"row {index}: schema_version is not 1") + if row["solved"] is not False: + problems.append(f"row {index}: frozen selection evidence unexpectedly solved a task") + digest = row["trajectory_digest"] + if not isinstance(digest, str) or len(digest) != 64: + problems.append(f"row {index}: invalid trajectory digest") + if len(rows) != 512: + problems.append(f"task evidence contains {len(rows)} rows, expected 512") + return problems + + +def get_builtin_study(study_id: str) -> BuiltinStudy: + """Resolve a named study from installed package data, never from the checkout.""" + if study_id != "alignment-external-v1": + raise ConfigError( + f"unknown built-in study {study_id!r}", + hint="available built-in studies: alignment-external-v1", + ) + root = Path(str(files("miniverl.evidence").joinpath("data", study_id))) + if root.is_dir(): + result_path = root / "result.json" + schema_path = root / "result.schema.json" + preregistration_path = root / "preregistration.yaml" + task_evidence_path = root / "task-evidence.jsonl" + else: + # A source checkout has not passed through Hatch's force-include mapping. + # Installed wheels always take the branch above. + repository = Path(__file__).resolve().parents[3] + result_path = repository / "benchmarks/results/alignment-external-v1.json" + schema_path = repository / "benchmarks/schema/alignment-external-result.schema.json" + preregistration_path = repository / "benchmarks/preregistration/alignment-external-v1.yaml" + task_evidence_path = ( + repository / "benchmarks/evidence/alignment-external-v1/jsonnav-selection-records.jsonl" + ) + return BuiltinStudy( + study_id=study_id, + result_path=result_path, + schema_path=schema_path, + preregistration_path=preregistration_path, + task_evidence_path=task_evidence_path, + **_ALIGNMENT_EXTERNAL_V1, + ) + + +def show_builtin_study(study_id: str) -> dict[str, Any]: + """Return the typed result as a JSON-friendly document.""" + study = get_builtin_study(study_id) + result = load_alignment_external_result(study.result_path) + return { + "study_id": study.study_id, + "result_sha256": study.result_sha256, + "result": result.model_dump(mode="json"), + } + + +def validate_builtin_study(study_id: str) -> dict[str, Any]: + """Validate every packaged byte binding and the task-level row contract.""" + study = get_builtin_study(study_id) + problems: list[str] = [] + paths = { + "result": (study.result_path, study.result_sha256), + "schema": (study.schema_path, study.schema_sha256), + "preregistration": (study.preregistration_path, study.preregistration_sha256), + "task_evidence": (study.task_evidence_path, study.task_evidence_sha256), + } + observed: dict[str, str] = {} + for name, (path, expected) in paths.items(): + if not path.is_file(): + problems.append(f"missing packaged {name}: {path.name}") + continue + actual = _sha256(path) + observed[name] = actual + if actual != expected: + problems.append(f"{name} SHA-256 is {actual}, expected {expected}") + + task_rows: list[dict[str, Any]] = [] + if not problems: + result = load_alignment_external_result(study.result_path) + if result.preregistration.sha256 != study.preregistration_sha256: + problems.append("result preregistration binding does not match the packaged artifact") + if result.checkpoint_selection.task_evidence.sha256 != study.task_evidence_sha256: + problems.append("result task-evidence binding does not match the packaged artifact") + try: + json.loads(study.schema_path.read_text(encoding="utf-8")) + task_rows = [ + json.loads(line) + for line in study.task_evidence_path.read_text(encoding="utf-8").splitlines() + if line.strip() + ] + except (json.JSONDecodeError, UnicodeError) as exc: + problems.append(f"packaged evidence is not valid UTF-8 JSON: {exc}") + else: + problems.extend(_validate_selection_rows(task_rows)) + + return { + "study_id": study.study_id, + "valid": not problems, + "task_rows": len(task_rows), + "sha256": observed, + "problems": problems, + } diff --git a/tests/cli/test_cli.py b/tests/cli/test_cli.py index 50dc495..8d9b18e 100644 --- a/tests/cli/test_cli.py +++ b/tests/cli/test_cli.py @@ -44,6 +44,8 @@ "alignment-suite validate", "alignment-suite report", "doctor", + "evidence show", + "evidence validate", "validate", "demo", "train", @@ -197,6 +199,16 @@ def test_root_help_lists_every_command() -> None: assert name in result.stdout, f"{name} missing from --help" +def test_root_help_describes_the_current_product_without_promising_v08() -> None: + result = _invoke("--help") + + assert result.exit_code == 0 + collapsed = _collapse(result.stdout) + assert "single-GPU alignment and distillation runtime" in collapsed + assert "bounded verl artifact bridge" in collapsed + assert "documented subset of verl-style OPD" not in collapsed + + def test_command_set_matches_the_documented_set() -> None: assert {" ".join(path) for path in COMMAND_PATHS} == EXPECTED_COMMANDS @@ -396,6 +408,27 @@ def test_pilot_consumes_the_external_early_stop_without_recommending_a_method() ) +def test_pilot_consumes_the_packaged_external_study_without_a_checkout() -> None: + payload = _payload(_invoke("pilot", "--builtin-study", "alignment-external-v1", "--json")) + + assert payload["study_status"] == "terminated_at_checkpoint_selection" + assert payload["method_recommendation"] == "insufficient_evidence" + assert payload["evidence"]["builtin_study"] == "alignment-external-v1" + assert payload["evidence"]["sha256"] + + +def test_evidence_show_and_validate_use_packaged_bytes() -> None: + shown = _payload(_invoke("evidence", "show", "alignment-external-v1", "--json")) + validated = _payload(_invoke("evidence", "validate", "alignment-external-v1", "--json")) + + assert shown["study_id"] == "alignment-external-v1" + assert shown["result"]["study_status"] == "terminated_at_checkpoint_selection" + assert validated["valid"] is True + assert validated["study_id"] == "alignment-external-v1" + assert validated["task_rows"] == 512 + assert validated["problems"] == [] + + def test_pilot_rejects_an_inconsistent_external_result(tmp_path: Path) -> None: source = json.loads( (REPO_ROOT / "benchmarks/results/alignment-external-v1.json").read_text(encoding="utf-8") diff --git a/tests/unit/test_builtin_evidence.py b/tests/unit/test_builtin_evidence.py new file mode 100644 index 0000000..e26cb29 --- /dev/null +++ b/tests/unit/test_builtin_evidence.py @@ -0,0 +1,27 @@ +"""Installed evidence is self-contained and byte-bound.""" + +from __future__ import annotations + +import hashlib + + +def test_builtin_external_study_resolves_without_a_repository_checkout() -> None: + from miniverl.evidence import get_builtin_study + + study = get_builtin_study("alignment-external-v1") + + assert study.result_path.is_file() + assert study.schema_path.is_file() + assert study.preregistration_path.is_file() + assert study.task_evidence_path.is_file() + assert hashlib.sha256(study.result_path.read_bytes()).hexdigest() == study.result_sha256 + + +def test_builtin_external_study_validates_every_packaged_binding() -> None: + from miniverl.evidence import validate_builtin_study + + report = validate_builtin_study("alignment-external-v1") + + assert report["valid"] is True + assert report["task_rows"] == 512 + assert report["problems"] == [] diff --git a/tests/unit/test_packaging.py b/tests/unit/test_packaging.py index cf92fed..251fa88 100644 --- a/tests/unit/test_packaging.py +++ b/tests/unit/test_packaging.py @@ -40,6 +40,7 @@ "community", "config", "environments", + "evidence", "evaluation", "losses", "models", @@ -84,6 +85,7 @@ "miniverl.environments.jsonnav", "miniverl.environments.sqlite_env", "miniverl.environments.registry", + "miniverl.evidence", "miniverl.cache", "miniverl.cache.store", "miniverl.cache.stats", @@ -161,6 +163,21 @@ def test_the_registry_travels_inside_the_wheel() -> None: assert "miniverl/alignment_external/profile-v1.yaml" in names +def test_builtin_evidence_travels_inside_the_wheel() -> None: + wheels = sorted((REPO_ROOT / "dist").glob("*.whl")) + if not wheels: + pytest.skip("no built wheel to inspect; run python -m build first") + + with zipfile.ZipFile(wheels[-1]) as archive: + names = set(archive.namelist()) + + root = "miniverl/evidence/data/alignment-external-v1/" + assert root + "result.json" in names + assert root + "result.schema.json" in names + assert root + "preregistration.yaml" in names + assert root + "task-evidence.jsonl" in names + + @pytest.mark.parametrize("name", TORCH_FREE_MODULES) def test_torch_free_modules_import_without_torch(name: str) -> None: """These must work from a bare ``pip install miniverl``. From 20a9ced4fba758eab67a4ac0db6ed868a3b7fed8 Mon Sep 17 00:00:00 2001 From: Daoyuan Li <94409450+DaoyuanLi2816@users.noreply.github.com> Date: Tue, 11 Aug 2026 22:00:00 -0700 Subject: [PATCH 2/4] Record v0.7.1 release validation --- PROJECT_STATE.md | 10 ++++++++++ docs/generated/quality.json | 6 +++--- docs/release-checklist.md | 12 ++++++------ 3 files changed, 19 insertions(+), 9 deletions(-) diff --git a/PROJECT_STATE.md b/PROJECT_STATE.md index 0805a1a..3c8bcbd 100644 --- a/PROJECT_STATE.md +++ b/PROJECT_STATE.md @@ -24,6 +24,16 @@ structurally checked, and `pilot --builtin-study` exposes the early-stop result without a checkout. The planned verl-shaped executable OPD profile remains a v0.8 development objective and is not claimed by this release. +Local release-candidate validation at implementation commit `5142ed6` passes +2110 non-GPU/non-network tests with 86.19% branch coverage, 8 RTX 4080 GPU +tests, 15 network tests, Ruff over 332 files, mypy over 147 source files, +actionlint 1.7.12, strict MkDocs and 36 rendered SVG instances at four +viewports. Twenty-four screenshots are under the ignored +`artifacts/docs-visual-v0.7.1/` directory and the 1440, 820 and 390 px home +views were manually inspected. Clean core and `[train]` wheel installs pass; +core keeps torch absent and validates the built-in evidence, while `[train]` +completes the real fast demo under latest resolved dependencies. + ## v0.7.0 External Alignment Gate — RELEASED Branch `v0.7.0-evidence-release` starts from the exact post-PR-#52 main commit diff --git a/docs/generated/quality.json b/docs/generated/quality.json index f8cd894..aecf208 100644 --- a/docs/generated/quality.json +++ b/docs/generated/quality.json @@ -5,9 +5,9 @@ "quality_floor": "2,000+ tests and 85%+ branch coverage at v0.7.1", "local_validation": { "scope": "the maintainer's workstation, where the GPU and Windows-specific paths actually run", - "commit": "013993a8cd5002a9ed166ba1e6948305f92c1bfb", - "commit_relationship": "implementation commit plus the following quality-record-only update; the final pull-request head is validated by CI", - "measured_at": "2026-08-10T23:56:04-07:00", + "commit": "5142ed681a9c538b3175faf64561968a43cab547", + "commit_relationship": "implementation commit plus following validation-record-only updates; the final pull-request head is validated by CI", + "measured_at": "2026-08-11T03:05:00-07:00", "platform": "Windows 11 Pro 10.0.22631", "python": "CPython 3.12", "coverage_mode": "branch", diff --git a/docs/release-checklist.md b/docs/release-checklist.md index c485f79..135623a 100644 --- a/docs/release-checklist.md +++ b/docs/release-checklist.md @@ -6,23 +6,23 @@ after the exact release commit and its remote checks are green. ## v0.7.1 Product correction -- [ ] README, Chinese README, PyPI description and docs landing page lead with +- [x] README, Chinese README, PyPI description and docs landing page lead with the installable runtime, hardware boundary, compatibility boundary and measured systems evidence; preserved research results follow under Research Notes. -- [ ] CLI help, doctor wording, project metadata and CFF describe the current +- [x] CLI help, doctor wording, project metadata and CFF describe the current single-GPU alignment/distillation runtime without promising v0.8 execution semantics. -- [ ] The wheel packages the v0.7 external-study result, schema, +- [x] The wheel packages the v0.7 external-study result, schema, preregistration and 512 task rows. `evidence show`, `evidence validate` and `pilot --builtin-study` work in a clean core-only installation with no checkout. -- [ ] Ruff, format, mypy, actionlint, CPU/GPU/network tests, strict docs, +- [x] Ruff, format, mypy, actionlint, CPU/GPU/network tests, strict docs, Playwright, package/extracted-sdist, clean installs, bridge and frozen-artifact gates pass on the exact release candidate. -- [ ] `git shortlog` and commit/body scans show Daoyuan Li as the only source +- [x] `git shortlog` and commit/body scans show Daoyuan Li as the only source author since v0.7.0 and no AI attribution trailers. -- [ ] The v0.7.0 tag, published adapter revisions and every frozen benchmark +- [x] The v0.7.0 tag, published adapter revisions and every frozen benchmark remain unchanged. ## v0.7.0 External Alignment Gate evidence release From 9e0c54fcbbe45d2b694599d21f77068c40ba22fc Mon Sep 17 00:00:00 2001 From: Daoyuan Li <94409450+DaoyuanLi2816@users.noreply.github.com> Date: Tue, 11 Aug 2026 22:12:49 -0700 Subject: [PATCH 3/4] Fix Python 3.10 evidence resource typing --- src/miniverl/evidence/__init__.py | 2 +- tests/unit/test_builtin_evidence.py | 23 +++++++++++++++++++++++ 2 files changed, 24 insertions(+), 1 deletion(-) diff --git a/src/miniverl/evidence/__init__.py b/src/miniverl/evidence/__init__.py index 90a41b3..bfb0337 100644 --- a/src/miniverl/evidence/__init__.py +++ b/src/miniverl/evidence/__init__.py @@ -81,7 +81,7 @@ def get_builtin_study(study_id: str) -> BuiltinStudy: f"unknown built-in study {study_id!r}", hint="available built-in studies: alignment-external-v1", ) - root = Path(str(files("miniverl.evidence").joinpath("data", study_id))) + root = Path(str(files("miniverl.evidence").joinpath("data").joinpath(study_id))) if root.is_dir(): result_path = root / "result.json" schema_path = root / "result.schema.json" diff --git a/tests/unit/test_builtin_evidence.py b/tests/unit/test_builtin_evidence.py index e26cb29..deb0821 100644 --- a/tests/unit/test_builtin_evidence.py +++ b/tests/unit/test_builtin_evidence.py @@ -5,6 +5,29 @@ import hashlib +def test_builtin_study_uses_python_310_traversable_joinpath_contract(monkeypatch) -> None: + import miniverl.evidence as evidence + + class SingleSegmentTraversable: + def __init__(self) -> None: + self.parts: list[str] = [] + + def joinpath(self, child: str) -> SingleSegmentTraversable: + self.parts.append(child) + return self + + def __str__(self) -> str: + return "missing-packaged-evidence" + + traversable = SingleSegmentTraversable() + monkeypatch.setattr(evidence, "files", lambda package: traversable) + + study = evidence.get_builtin_study("alignment-external-v1") + + assert traversable.parts == ["data", "alignment-external-v1"] + assert study.result_path.is_file() + + def test_builtin_external_study_resolves_without_a_repository_checkout() -> None: from miniverl.evidence import get_builtin_study From e1de216d1f20414bda43575e4118e447bb972051 Mon Sep 17 00:00:00 2001 From: Daoyuan Li <94409450+DaoyuanLi2816@users.noreply.github.com> Date: Tue, 11 Aug 2026 22:18:33 -0700 Subject: [PATCH 4/4] Make evidence compatibility test checkout-independent --- tests/unit/test_builtin_evidence.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unit/test_builtin_evidence.py b/tests/unit/test_builtin_evidence.py index deb0821..8ed85b3 100644 --- a/tests/unit/test_builtin_evidence.py +++ b/tests/unit/test_builtin_evidence.py @@ -25,7 +25,7 @@ def __str__(self) -> str: study = evidence.get_builtin_study("alignment-external-v1") assert traversable.parts == ["data", "alignment-external-v1"] - assert study.result_path.is_file() + assert study.study_id == "alignment-external-v1" def test_builtin_external_study_resolves_without_a_repository_checkout() -> None: