diff --git a/.github/workflows/gpu.yml b/.github/workflows/gpu.yml index 9dc3be5..daea322 100644 --- a/.github/workflows/gpu.yml +++ b/.github/workflows/gpu.yml @@ -15,7 +15,7 @@ on: workflow_dispatch: inputs: run_recipe: - description: "Also run the full 16 GB recipe (slow, ~10 minutes)" + description: "Also run the full single-GPU recipe (slow, ~10 minutes)" type: boolean default: false @@ -46,7 +46,7 @@ jobs: - name: GPU tests run: pytest -q -m gpu -ra - - name: Full 16 GB recipe + - name: Full single-GPU recipe if: ${{ inputs.run_recipe }} env: MINIVERL_LOG_LEVEL: INFO diff --git a/CHANGELOG.md b/CHANGELOG.md index 43bd3ce..9f1b8da 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,34 @@ All notable changes to miniVERL are recorded here. The format follows ## [Unreleased] +## [0.2.2] - 2026-07-29 + +Single-GPU portability and presentation release. + +### Added + +- A hardware-portability guide for personal NVIDIA GPUs, including honest + starting points for 8–12 GiB, 16–24 GiB and 24–32+ GiB cards, OOM controls, + and a reproducible hardware-result contribution path. +- A prominent PyPI destination in both READMEs and package metadata. +- Visual regression assertions that keep benchmark grid lines below axis labels + and preserve the dark generated figure. + +### Changed + +- The supported Qwen3 recipe now uses model-agnostic run metadata and + `dtype: auto`, selecting bf16 when available and fp16 on older CUDA cards + such as Titan V. The pinned models, adapter, objective and budgets are + unchanged. +- The repository is positioned as a personal single-GPU training stack rather + than a 16 GiB-specific implementation. RTX 4080 numbers remain explicitly + labeled as the only measured GPU evidence. +- The banner and data-bound protocol benchmark figure use a new dark visual + system. Axis grids no longer cross tick labels, and protocol-incompatible + 0% controls are rendered as diagnostic states rather than zero-length bars. +- GPU workflow language now names the portable single-GPU recipe instead of a + particular VRAM tier. + ## [0.2.1] - 2026-07-29 Correctness, lifecycle safety and reproducibility release. diff --git a/CITATION.cff b/CITATION.cff index 1ca5b23..cf262c6 100644 --- a/CITATION.cff +++ b/CITATION.cff @@ -2,7 +2,7 @@ cff-version: 1.2.0 title: "miniVERL: On-policy distillation for tool-using agents on one GPU" message: "If you use miniVERL in your work, please cite it as below." type: software -version: 0.2.1 +version: 0.2.2 date-released: 2026-07-29 license: Apache-2.0 repository-code: "https://github.com/DaoyuanLi2816/mini-verl" @@ -18,8 +18,9 @@ abstract: >- coarse-graining, enforces per-token provenance so that tool output can never become a training label, and stores teacher targets in a versioned, checksummed, pickle-free cache with policy-version enforcement. It is - designed and measured for a single 16 GB consumer GPU and requires neither - Ray nor a cluster. + designed for one personal CUDA GPU, automatically selects bf16 or fp16, and + requires neither Ray nor a cluster. Published performance is measured on one + RTX 4080; other GPU models use the same code path but remain unmeasured. authors: - family-names: Li given-names: Daoyuan @@ -32,6 +33,7 @@ keywords: - agents - QLoRA - consumer GPU + - single GPU - post-training - reproducibility references: @@ -96,4 +98,4 @@ references: url: "https://arxiv.org/abs/2603.07079" notes: >- Motivates recording per-token teacher entropy. Entropy-aware divergence - mixing is a roadmap item and is not implemented in miniVERL v0.2.1. + mixing is a roadmap item and is not implemented in miniVERL v0.2.2. diff --git a/PROJECT_STATE.md b/PROJECT_STATE.md index 489dfc0..0dd68bf 100644 --- a/PROJECT_STATE.md +++ b/PROJECT_STATE.md @@ -6,7 +6,20 @@ and what it printed. Last updated: 2026-07-29. -## v0.2.1 correctness release status +## v0.2.2 single-GPU portability release candidate + +| item | current state | +| --- | --- | +| integration source | `agent/single-gpu-visuals`, based on synchronized public `main` at `dcded6bd1948eb7421f89e336f611e3b0d3c0ad3` | +| version transition | source, changelog and citation metadata identify `0.2.2`; the intended annotated tag is `v0.2.2` | +| hardware scope | one NVIDIA CUDA GPU with no device-name allowlist; automatic dtype selects bf16 when supported and fp16 otherwise; exact fit remains model/budget dependent | +| measured evidence | the only published real-model GPU measurements remain the RTX 4080 runs; RTX 3070, Titan V, RTX 5090 and other cards are portable code paths, not fabricated benchmark claims | +| presentation | banner and generated benchmark SVG use a dark single-GPU visual system; grid lines begin below tick labels and diagnostic 0% controls are status pills | +| package discovery | both READMEs, project metadata and the GitHub About homepage expose `https://pypi.org/project/miniverl/` | +| frozen scientific artifact | `benchmarks/results/gpu-calc-hard-equal-update-v2.json` remains byte-identical at SHA-256 `53fc1d4d5b7adee09618d77ad62d4086ba56b78569832d6fc7c3bcd5c2695bbc` | +| publication state | authorized for direct release; publish only after local gates, pull-request CI and synchronized default-branch CI pass | + +## Historical v0.2.1 correctness release status | item | current state | | --- | --- | diff --git a/README.md b/README.md index 4514511..0ba7f19 100644 --- a/README.md +++ b/README.md @@ -12,12 +12,21 @@ -**On-policy distillation for tool-using agents on one GPU.** +

+ PyPI package · + Install & train · + Bring your own GPU · + Measured result +

+ +**A compact, auditable training stack for tool-using agents on one personal GPU.** miniVERL is a compact, auditable training lab for teaching a small language model from its own multi-turn tool trajectories. It runs real tools, keeps token provenance explicit, and applies teacher distributional targets only -where they belong — without Ray, a GPU cluster, or a 40 GB accelerator. +where they belong — without Ray or a GPU cluster. There is no device-name +allowlist: use the NVIDIA CUDA card you have, then choose a model pair and +sequence budget that fit it. ```bash python -m pip install miniverl # lightweight core @@ -44,7 +53,7 @@ validate artifacts without downloading a multi-gigabyte ML stack; use `top-k + tail` objectives are named and reported separately. [Run the local demo](#local-toy-demo) · -[Train on a consumer GPU](#consumer-gpu-quickstart) · +[Train on your GPU](#single-gpu-quickstart) · [Inspect the measured result](#measured-result-protocol-aligned-opd-matches-sft) · [Read the math](docs/math.md) @@ -66,7 +75,7 @@ go wrong in practice, and all four are silent: cannot run. miniVERL makes each of those a *checked property* rather than a comment, and -keeps the whole thing on one 16 GB card. +keeps the whole lifecycle in one readable single-GPU process. ## What is implemented @@ -78,7 +87,7 @@ keeps the whole thing on one 16 GB card. | Compressed `top-k + tail` KL and JSD | yes; the unsmoothed coarse-graining has a proven lower-bound relationship to the exact loss | | Privileged-context teacher with an explicit alignment map | yes | | Frozen standard PEFT teacher adapters with provenance and competence gates | yes | -| QLoRA (NF4) student, bf16 or quantized teacher | yes, measured on an RTX 4080 | +| Single-GPU CUDA path with automatic bf16/fp16 selection | yes; model-agnostic code path, measured reference on an RTX 4080 | | `resident` and `swap` memory strategies, `auto` resolution | yes, with an equivalence test | | Versioned, checksummed, pickle-free teacher-target cache | yes | | SFT / offline KD / strict OPD / explicitly labeled replay behind one trainer | yes | @@ -105,7 +114,7 @@ keeps the whole thing on one 16 GB card. | diagnostic control | answer-privileged, protocol-naive teacher | 0.0% | 0.0% | The [public, immutable protocol-teacher adapter](https://huggingface.co/DaoyuanLi/mini-verl-qwen3-1.7b-protocol-teacher) -is the default in the consumer-GPU recipe. It passed an independently +is the default in the single-GPU recipe. It passed an independently prespecified policy-competence gate before this benchmark was inspected. The two controls intentionally remove that guarantee: their loss decreased normally, but they taught the student an incompatible tool policy. This is a @@ -200,7 +209,14 @@ The toy backend is a **machinery harness, not a capability demonstration**. Its models are too small to solve anything beyond the `easy` split. Capability numbers come from the GPU recipe. -## Consumer-GPU quickstart +## Single-GPU quickstart + +The default recipe uses `device: auto` and `dtype: auto`: bf16-capable cards use +bf16, while older CUDA cards such as Titan V use fp16. RTX 3070, Titan V, +RTX 4080 and RTX 5090-class cards all enter the same code path; only the +RTX 4080 result is measured here. Exact fit is governed by VRAM, model sizes, +drivers and token budgets, not the card's marketing name. See the +[`single-GPU guide`](docs/single-gpu-guide.md) before changing the recipe. ```bash git clone https://github.com/DaoyuanLi2816/mini-verl.git @@ -474,8 +490,10 @@ environments, and multi-GPU. For anything at cluster scale, use verl. The name is a nod to the problem space, not a claim of compatibility. verl is an excellent, much larger system that also implements on-policy distillation and multi-turn tool use — at cluster scale, with Ray. If you have a cluster, use it. -miniVERL exists for the case where you have one consumer GPU and want to read -every line of what is happening. See [`docs/comparisons.md`](docs/comparisons.md). +miniVERL exists for the case where you have one personal GPU and want to read +every line of what is happening. That can be an older 12 GiB card or a current +high-end card; the repository claims measured performance only for hardware it +actually ran. See [`docs/comparisons.md`](docs/comparisons.md). ## Citation diff --git a/README.zh-CN.md b/README.zh-CN.md index 1a6f25b..265e5f8 100644 --- a/README.zh-CN.md +++ b/README.zh-CN.md @@ -12,13 +12,21 @@ +

+ PyPI 软件包 · + 安装与训练 · + 适配你的 GPU · + 实测结果 +

+ > 本文是 [README.md](README.md) 的中文翻译。英文版为准；若两者不一致，请以英文版为准并提交 issue。 -**单卡上的工具调用智能体在线策略蒸馏（on-policy distillation）。** +**面向个人单卡、紧凑且可审计的工具调用智能体训练栈。** miniVERL 是一个紧凑、可审计的训练实验室，让小型语言模型从**它自己生成的 多轮工具轨迹**中学习。它会真实执行工具、显式记录 token 来源，并且只在正确 -的位置使用教师分布目标——不需要 Ray、GPU 集群或 40 GB 显存的加速卡。 +的位置使用教师分布目标——不需要 Ray 或 GPU 集群。代码没有显卡型号白名单： +使用你现有的 NVIDIA CUDA 显卡，再选择能装进显存的模型组合与 token 预算。 ```bash python -m pip install miniverl # 轻量核心层 @@ -41,7 +49,7 @@ schema 与 Python API）。`train` extra 会添加 torch、Transformers 与 PEFT - **预算真实：** 精确全词表目标与压缩的 `top-k + tail` 目标分开命名、分开报告。 [运行本地 demo](#本地玩具演示) · -[在消费级 GPU 上训练](#消费级-gpu-快速上手) · +[在你的 GPU 上训练](#个人单卡快速上手) · [查看实测结果](#实测结果协议对齐的-opd-追平-sft) · [阅读数学说明](docs/math.md) @@ -54,7 +62,7 @@ schema 与 Python API）。`train` extra 会添加 torch、Transformers 与 PEFT 3. **其实并不是 on-policy。** 跨策略版本复用教师缓存，做的就是离线 KD，却仍叫它 OPD。 4. **显存放不下 logits。** `[batch, seq_len, 152k]` 的张量在消费级显卡上放不下，于是真正有意思的配置恰好都跑不了。 -miniVERL 把上面每一条都变成**被代码检查的性质**，而不是注释里的一句承诺，并且整套流程只用一块 16 GB 显卡。 +miniVERL 把上面每一条都变成**被代码检查的性质**，而不是注释里的一句承诺，并把整个生命周期放进一个可读的单卡进程。 ## 已实现的能力 @@ -66,7 +74,7 @@ miniVERL 把上面每一条都变成**被代码检查的性质**，而不是注 | 压缩的 `top-k + tail` KL / JSD | 支持；未平滑粗粒化与精确散度的下界关系有严格证明 | | 特权上下文教师模式，带显式对齐表 | 支持 | | 标准冻结 PEFT 教师适配器，带来源记录与能力门禁 | 支持 | -| QLoRA（NF4）学生，bf16 或量化教师 | 支持，已在 RTX 4080 上实测 | +| 自动选择 bf16/fp16 的单卡 CUDA 路径 | 支持；代码路径不绑定型号，实测参考为 RTX 4080 | | `resident` / `swap` 显存策略与 `auto` 解析 | 支持，并有等价性测试 | | 带版本号与校验和、完全不用 pickle 的教师目标缓存 | 支持 | | SFT / 离线 KD / 严格 OPD / 显式标注 replay 统一在一个 trainer 中 | 支持 | @@ -92,7 +100,7 @@ miniVERL 把上面每一条都变成**被代码检查的性质**，而不是注 | 诊断对照 | 获知答案但不懂协议的教师 | 0.0% | 0.0% | [公开且固定版本的协议教师适配器](https://huggingface.co/DaoyuanLi/mini-verl-qwen3-1.7b-protocol-teacher) -现在是消费级 GPU 配方的默认教师。它在下游 benchmark 被查看之前，已经通过 +现在是单卡配方的默认教师。它在下游 benchmark 被查看之前，已经通过 预先指定、独立评估的策略能力门槛。两个负对照故意去掉了这个保证：loss 正常 下降，但教师把不兼容的工具策略教给了学生。这是实测的教师能力问题，不是 trainer 崩溃。 @@ -165,7 +173,13 @@ tokens by span type (only assistant_* can enter the loss) 玩具后端是**机制验证台，不是能力展示**。它的模型太小，除了 `easy` 难度之外什么都做不了。能力数字来自 GPU 配方。 -## 消费级 GPU 快速上手 +## 个人单卡快速上手 + +默认配方使用 `device: auto` 与 `dtype: auto`：支持 bf16 的显卡自动使用 bf16， +Titan V 等较老 CUDA 显卡自动使用 fp16。RTX 3070、Titan V、RTX 4080 和 +RTX 5090 级别显卡都走同一条代码路径；本仓库目前只有 RTX 4080 的实测结果。 +能否装下取决于显存、模型大小、驱动和 token 预算，而不是显卡的商品名。修改配方前请阅读 +[`单卡适配指南`](docs/single-gpu-guide.md)。 ```bash git clone https://github.com/DaoyuanLi2816/mini-verl.git @@ -299,7 +313,7 @@ print(result.run_dir, result.global_step, result.eval["success_rate"]) > miniVERL 是一个独立项目，与 verl 项目、字节跳动（ByteDance）或火山引擎（Volcano Engine）没有隶属关系，也未获得其背书。它**不是** verl 的直接替代品。 -这个名字只是对问题领域的致意，不代表任何兼容性声明。verl 是一个优秀得多、规模也大得多的系统，它同样实现了在线策略蒸馏和多轮工具调用——只不过是在集群规模上，依赖 Ray。如果你有集群，请用它。miniVERL 面向的是「只有一块消费级显卡、并且希望把每一行发生的事都读懂」的场景。对比见 [`docs/comparisons.md`](docs/comparisons.md)。 +这个名字只是对问题领域的致意，不代表任何兼容性声明。verl 是一个优秀得多、规模也大得多的系统，它同样实现了在线策略蒸馏和多轮工具调用——只不过是在集群规模上，依赖 Ray。如果你有集群，请用它。miniVERL 面向的是「只有一块个人显卡、并且希望把每一行发生的事都读懂」的场景：它可以是较老的 12 GiB 显卡，也可以是当前的高端显卡；仓库只对实际跑过的硬件声明实测性能。对比见 [`docs/comparisons.md`](docs/comparisons.md)。 ## 引用与许可证 diff --git a/docs/banner.svg b/docs/banner.svg index 98138b9..565e354 100644 --- a/docs/banner.svg +++ b/docs/banner.svg @@ -1,100 +1,118 @@ - + - - - + + + - - + + + - - + + - - + + + + + + + + + + - - - + + + + fill="none" stroke="#5b7198" stroke-opacity=".34" stroke-width="1.5"/> - - ONE-GPU OPD LAB + + PERSONAL SINGLE-GPU TRAINING - + miniVERL - + On-policy distillation for tool-using agents. - - Reproducible training, typed token provenance, and exact + + Auditable rollouts, typed token provenance, and exact - - or budgeted divergence — without a cluster. + + or budgeted divergence — no Ray, no cluster. - - - strict OPD - - offline-safe - - 16 GB first + + + one card + + GPU adaptive + + audit first - - - pip install "miniverl[train]" + + + pip install "miniverl[train,cuda]" + + + + + - - - + + + - Sample - Student trajectories with real tool execution + Sample + Run real tools on student trajectories - - - + + + - Align - Typed spans keep tool output as context + Align + Keep tool output as context, never labels - - - + + + - Distill - Exact or top-k KL on one consumer GPU + Distill + Use BF16 or FP16 on the GPU you have diff --git a/docs/benchmarking.md b/docs/benchmarking.md index f2bd50a..f345f66 100644 --- a/docs/benchmarking.md +++ b/docs/benchmarking.md @@ -327,7 +327,7 @@ recorded as `false`. # CPU, toy models miniverl benchmark recipes/benchmark_calc.yaml --output runs/benchmarks -# One 16 GB GPU, the pinned Qwen3 pair +# One GPU, the pinned Qwen3 pair miniverl benchmark benchmarks/configs/gpu_calc_hard.yaml --output runs/benchmarks # Strictly offline after preloading every pinned model, tokenizer and adapter diff --git a/docs/comparisons.md b/docs/comparisons.md index e46a0c7..d56b095 100644 --- a/docs/comparisons.md +++ b/docs/comparisons.md @@ -88,7 +88,7 @@ Use verl, not miniVERL, if any of the following is true: - You want something with a maintainer community, a release cadence and 51 CI workflows behind it. -miniVERL is a reasonable choice when you have exactly one consumer GPU, you want +miniVERL is a reasonable choice when you have exactly one personal NVIDIA GPU, you want to read and modify the loss, and you care more about being able to audit which token was supervised by which teacher distribution than about throughput. Reading `docs/limitations.md` before starting is strongly recommended. diff --git a/docs/design.md b/docs/design.md index 6a7b560..2caf01e 100644 --- a/docs/design.md +++ b/docs/design.md @@ -594,7 +594,7 @@ quietly did no work. ## 7. Deliberately not here Every omission below is a scope decision, not an oversight. Each one is a thing a -production framework does that a single-process 16 GB lab should not pretend to +production framework does that a single-process one-GPU lab should not pretend to do. **Ray.** verl depends on `ray[default]` unconditionally because it schedules diff --git a/docs/gpu-calc-hard-equal-update-v2.svg b/docs/gpu-calc-hard-equal-update-v2.svg index 4f32ed5..a7e0a90 100644 --- a/docs/gpu-calc-hard-equal-update-v2.svg +++ b/docs/gpu-calc-hard-equal-update-v2.svg @@ -1,88 +1,92 @@ - + miniVERL protocol-teacher benchmark Strict held-out success and training time for equal-optimizer-update arms over 2 prespecified seeds. 2 protocol-incompatible negative controls are shown separately from the supported comparison. The cold-start baseline is labeled no training. - - - + + + + + Protocol-aligned OPD matches continued SFT schema v2 · 2 prespecified seeds · budget axis: optimizer_steps - - -Strict held-out success -Training time - -0% - -50% - -100% - -0s - -300s - -600s + + +Strict held-out success +Training time +0% + +50% + +100% + +0s + +300s + +600s + + -Cold start - - - - -75% - -NO TRAINING +Cold start + + + + +75% + +NO TRAINING - + -Continued SFT - - - - -100% - - - - -86s +Continued SFT + + + + +100% + + + + +86s - + -OPD · protocol-aligned teacher - - - - -100% - - - - -524s +OPD · protocol-aligned teacher + + + + +100% + + + + +524s - -DIAGNOSTIC CONTROLS · INTENTIONALLY PROTOCOL-INCOMPATIBLE TEACHERS + +DIAGNOSTIC CONTROLS · INTENTIONALLY PROTOCOL-INCOMPATIBLE TEACHERS + -Raw teacher (control) - -PROTOCOL MISMATCH - - - - -444s +Raw teacher (control) + +PROTOCOL MISMATCH + + + + +444s - + -Privileged context (control) - -PROTOCOL MISMATCH - - - - -532s +Privileged context (control) + +PROTOCOL MISMATCH + + + + +532s -Diagnostic controls intentionally use protocol-incompatible teachers; strict success was 0% in every seed; cold start records setup only (0.06s), not a training phase. -Bars are seed means; hollow dots are individual seeds [1234, 20260727]. Source JSON SHA-256 53fc1d4d5b7adee0 +Diagnostic controls intentionally use protocol-incompatible teachers; strict success was 0% in every seed; cold start records setup only (0.06s), not a training phase. +Bars are seed means; hollow dots are individual seeds [1234, 20260727]. Source JSON SHA-256 53fc1d4d5b7adee0 diff --git a/docs/memory.md b/docs/memory.md index 67ef3e7..1ea754e 100644 --- a/docs/memory.md +++ b/docs/memory.md @@ -1,4 +1,4 @@ -# Memory on a single consumer GPU +# Memory on one personal GPU miniVERL runs a teacher and a student on one card. This document describes the two placement strategies, how `auto` picks between them, the out-of-memory retry diff --git a/docs/release-checklist.md b/docs/release-checklist.md index 22d62b9..d0599e2 100644 --- a/docs/release-checklist.md +++ b/docs/release-checklist.md @@ -1,6 +1,6 @@ # Release checklist -This is the release gate and publication record for `v0.2.1`. A checked item +This is the release gate and publication record for `v0.2.2`. A checked item names an invariant that was exercised on the release source. The tag workflow repeated the mechanical gates and refused inconsistent metadata or an unchecked pre-tag item. Publication began only after explicit maintainer @@ -8,10 +8,10 @@ authorization. ## Version consistency -- [x] `src/miniverl/__init__.py` is `0.2.1`. -- [x] `CHANGELOG.md` has a dated `## [0.2.1]` section. +- [x] `src/miniverl/__init__.py` is `0.2.2`. +- [x] `CHANGELOG.md` has a dated `## [0.2.2]` section. - [x] `CITATION.cff` version and release date match. -- [x] The future annotated tag is exactly `v0.2.1`. +- [x] The future annotated tag is exactly `v0.2.2`. - [x] The package/project name remains `miniverl`. ## Correctness and lifecycle @@ -88,6 +88,10 @@ authorization. or secret. - [x] No model weights, checkpoints, caches or databases are tracked. - [x] The banner and benchmark SVG were rendered and visually inspected. +- [x] Benchmark grids begin below their tick labels; the generated dark SVG is + still bound to the immutable source JSON. +- [x] The supported single-GPU recipe uses automatic bf16/fp16 selection and + carries no GPU model or VRAM-tier tag. - [x] The base-vs-`[train]` installation split and v1 scientific confound are stated explicitly. - [x] `TODO.md`, `PROJECT_STATE.md`, support claims and dependency boundaries @@ -96,16 +100,25 @@ authorization. ## Trusted publishing readiness -- [x] PyPI reports project `miniverl` and current public version `0.2.0`. +- [x] PyPI reports project `miniverl` and current public version `0.2.1`. - [x] GitHub environment `pypi` exists and has a deployment branch policy. - [x] `release.yml` requests `id-token: write`, uses the `pypi` environment and publishes only on a tag push. - [x] The maintainer registered the pending publisher for `DaoyuanLi2816/mini-verl`, workflow `release.yml`, environment `pypi`. -- [x] The immutable `v0.2.0` tag and public artifacts are unchanged. +- [x] The immutable `v0.2.0` and `v0.2.1` tags and public artifacts are unchanged. ## After the tag +Complete and check these in the post-release state-sync change: + +- [ ] Record the exact `v0.2.2` tag commit and release workflow run. +- [ ] Verify public PyPI hashes, attestations and a clean core installation. +- [ ] Verify the GitHub Release contains the identical distributions. +- [ ] Advance development to `0.2.3.dev0`. + +## Historical v0.2.1 record + Publication completed on 2026-07-29: - [x] Create annotated tag `v0.2.1` on exact validated commit diff --git a/docs/single-gpu-guide.md b/docs/single-gpu-guide.md new file mode 100644 index 0000000..b547ce7 --- /dev/null +++ b/docs/single-gpu-guide.md @@ -0,0 +1,89 @@ +# Bring your own GPU + +miniVERL is a **personal single-GPU** training stack. It has no GPU model +allowlist and no multi-GPU launcher: one process uses one CUDA device, while +the model pair and sequence budget determine whether a recipe fits. + +The shipped Qwen3 recipe is measured on an RTX 4080, but it is not coded for an +RTX 4080. Its portable defaults are: + +- `models.device: auto` selects CUDA when PyTorch can use it; +- `dtype: auto` selects bfloat16 when the device supports it and float16 + otherwise; +- NF4 student weights and paged 8-bit Adam reduce the trainable-model footprint; +- `memory.strategy: auto` resolves the supported resident/swap policy; +- an out-of-memory error may reduce only the vocabulary-loss chunk size and + retry the gradient phase. It never silently changes the objective, model, + rollout length, or optimizer update. + +## Start here + +Install the training stack and the CUDA quantization backend: + +```bash +python -m pip install "miniverl[train,cuda]" +miniverl doctor +miniverl validate recipes/qwen_consumer_gpu_calc.yaml +miniverl train recipes/qwen_consumer_gpu_calc.yaml --dry-run +``` + +Then watch free memory in another terminal before the real run: + +```bash +nvidia-smi +miniverl train recipes/qwen_consumer_gpu_calc.yaml +``` + +`doctor` reports the device, CUDA availability and optional dependencies. +`--dry-run` validates model identity, tokenizer compatibility, adapter +provenance and the resolved configuration. It does not prove that every phase +will fit; only an actual run can do that. + +## What different cards change + +These are starting points, not benchmark claims: + +| Your card | Sensible first move | Evidence status | +| --- | --- | --- | +| 8–12 GiB, such as an RTX 3070 or Titan V | Try the shipped pair, but be ready to shorten token budgets or select a smaller teacher. Keep `dtype: auto`; Titan V-class hardware resolves to fp16 because it has no bf16 path. | Supported by model-agnostic code paths; not measured in this repository | +| 16–24 GiB | Start with the shipped recipe unchanged. Increase budgets only after recording a successful baseline. | The exact default pair is measured on one RTX 4080 16 GiB | +| 24–32+ GiB, such as RTX 3090/4090/5090-class cards | Use the same recipe first; extra headroom can support longer contexts, larger models, or less quantization. Change one variable at a time. | Expected from the same single-device path; not measured here | + +The default measured run peaked below 5 GiB of CUDA allocated/reserved memory, +but allocator behavior, kernels, driver versions and generation length vary. +That number is evidence from one machine, **not** a promise that every 8 GiB +card or software stack will complete. + +## If the recipe does not fit + +Work down this list and preserve the resulting YAML with the run: + +1. Close other GPU processes and re-run `nvidia-smi`. +2. Reduce `loss.chunk_size`; the automatic OOM retry can do this down to + `memory.min_chunk_size`. +3. Reduce rollout token limits or the number of rollouts accumulated in a + cycle. This changes the experiment budget, so label the run accordingly. +4. Choose a smaller teacher/student pair with byte-identical tokenizers and + compatible output vocabularies. +5. For an unquantized student, evaluate `memory.strategy: swap`. Quantized + students remain resident because moving bitsandbytes modules between + devices is not a supported lifecycle. + +Do not infer correctness from a falling loss alone. Run the strict held-out +verifier and inspect the terminal manifest. The published negative controls +are included precisely because an incompatible teacher can optimize normally +while producing a 0% tool policy. + +## Share a hardware result + +Hardware reports are useful when they include the full configuration and +provenance rather than only a card name: + +```bash +miniverl export-benchmark runs/ --notes "GPU, VRAM, driver, torch and CUDA versions" +``` + +Validate the exported JSON and open a pull request following +[`benchmarks/README.md`](../benchmarks/README.md). Results from cards other than +the measured RTX 4080 are welcome; until they exist, the README labels those +cards as portable code paths rather than measured performance. diff --git a/pyproject.toml b/pyproject.toml index 18f339c..23daaac 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -5,7 +5,7 @@ build-backend = "hatchling.build" [project] name = "miniverl" dynamic = ["version"] -description = "On-policy distillation for tool-using agents on one GPU." +description = "Auditable on-policy distillation for tool-using agents on one personal GPU." readme = "README.md" requires-python = ">=3.10" license = { text = "Apache-2.0" } @@ -18,6 +18,7 @@ keywords = [ "tool-use", "qlora", "consumer-gpu", + "single-gpu", "post-training", "qwen", "llm-agents", @@ -72,6 +73,7 @@ miniverl = "miniverl.cli:app" [project.urls] Homepage = "https://github.com/DaoyuanLi2816/mini-verl" +PyPI = "https://pypi.org/project/miniverl/" Issues = "https://github.com/DaoyuanLi2816/mini-verl/issues" Repository = "https://github.com/DaoyuanLi2816/mini-verl" Changelog = "https://github.com/DaoyuanLi2816/mini-verl/blob/main/CHANGELOG.md" diff --git a/recipes/qwen_consumer_gpu_calc.yaml b/recipes/qwen_consumer_gpu_calc.yaml index 5c89a1a..160a390 100644 --- a/recipes/qwen_consumer_gpu_calc.yaml +++ b/recipes/qwen_consumer_gpu_calc.yaml @@ -1,5 +1,5 @@ -# miniVERL supported 16 GB recipe: on-policy distillation of Qwen3-0.6B from a -# protocol-aligned Qwen3-1.7B teacher on one consumer GPU. +# miniVERL portable single-GPU starter: on-policy distillation of Qwen3-0.6B +# from a protocol-aligned Qwen3-1.7B teacher on one CUDA GPU. # # Verified facts about this model pair (checked against the Hugging Face API on # 2026-07-27, and re-checkable with the commands in docs/reproducibility.md): @@ -18,19 +18,25 @@ # miniverl train recipes/qwen_consumer_gpu_calc.yaml --dry-run # miniverl train recipes/qwen_consumer_gpu_calc.yaml # +# The code does not allowlist GPU models. `device: auto` selects the available +# CUDA device and `dtype: auto` chooses bf16 when supported, otherwise fp16. +# Exact fit still depends on VRAM, drivers and sequence lengths; run `doctor`, +# `validate`, and `--dry-run` first. See docs/single-gpu-guide.md. +# # Measured base-pair memory and the two-seed protocol-teacher comparison are in -# docs/rtx4080-baselines.md. The historical protocol-naive recipe is preserved +# docs/rtx4080-baselines.md. Those measurements are RTX 4080 evidence, not a +# hardware requirement. The historical protocol-naive recipe is preserved # separately as qwen_consumer_gpu_calc_raw_teacher.yaml. schema_version: 1 run: - name: qwen3-calc-opd-protocol-4080 + name: qwen3-calc-opd-single-gpu mode: opd seed: 1234 output_dir: runs deterministic: true - tags: [consumer-gpu, rtx4080, qwen3, calculator, protocol-teacher] + tags: [single-gpu, qwen3, calculator, protocol-teacher] models: backend: hf @@ -39,7 +45,7 @@ models: model_id: Qwen/Qwen3-0.6B revision: c1899de289a04d12100db370d81485cdf75e47ca tokenizer_revision: c1899de289a04d12100db370d81485cdf75e47ca - dtype: bfloat16 + dtype: auto # QLoRA: NF4 weights + double quantization. Measured on an RTX 4080 this # costs ~20% decode throughput versus bf16 LoRA and saves ~0.3 GiB on a # 0.6B student -- the win only matters once the teacher grows. Both @@ -58,7 +64,7 @@ models: model_id: Qwen/Qwen3-1.7B revision: 70d244cc86ccca08cf5af4e1e306ecf908b1ad5e tokenizer_revision: 70d244cc86ccca08cf5af4e1e306ecf908b1ad5e - dtype: bfloat16 # a bf16 1.7B teacher is ~4.1 GiB; it fits + dtype: auto # bf16 when supported; fp16 on older CUDA cards quantization: none attn_implementation: sdpa mode: standard diff --git a/scripts/publish_benchmark_artifacts.py b/scripts/publish_benchmark_artifacts.py index d77adee..9ac024e 100644 --- a/scripts/publish_benchmark_artifacts.py +++ b/scripts/publish_benchmark_artifacts.py @@ -113,21 +113,21 @@ def render_svg(result: BenchmarkResult, source_sha256: str) -> str: ) width = 1120 - diagnostic_gap = 22 if diagnostic_start is not None else 0 - height = 206 + len(rows) * 75 + diagnostic_gap + diagnostic_gap = 36 if diagnostic_start is not None else 0 + height = 238 + len(rows) * 78 + diagnostic_gap label_x = 36 - success_x = 315 - success_w = 285 - time_x = 735 - time_w = 270 + success_x = 310 + success_w = 292 + time_x = 728 + time_w = 276 max_seconds = max(row["seconds_mean"] for row in rows) time_ceiling = max(100, int(math.ceil(max_seconds / 100.0) * 100)) colors = { - "cold-start-only": "#64748b", - "sft-continued": "#3b82f6", - "opd-raw-teacher": "#ef4444", - "opd-privileged-context": "#f59e0b", - "opd-protocol-sft-teacher": "#10b981", + "cold-start-only": "#94a3b8", + "sft-continued": "#60a5fa", + "opd-raw-teacher": "#fb7185", + "opd-privileged-context": "#fbbf24", + "opd-protocol-sft-teacher": "#34d399", } labels = { "cold-start-only": "Cold start", @@ -161,20 +161,38 @@ def line(text: str) -> str: svg += line( "" ) - svg += line('') svg += line( - f'' + "" + '' + '' + '' + '' + "" + '' + '' + '' + "" + '' + '' + '' + "" + "" + ) + svg += line('') + svg += line('') + svg += line( + f'' ) svg += line( 'Protocol-aligned OPD matches continued SFT' @@ -184,45 +202,47 @@ def line(text: str) -> str: f"{len(result.seeds)} prespecified seeds · budget axis: " f"{html.escape(result.budget_axis or 'unreported')}" ) - panel_y = 91 - panel_h = height - 148 + panel_y = 96 + panel_h = height - 158 svg += line( - f'' + f'' ) svg += line( - f'' + f'' ) - svg += line(f'Strict held-out success') - svg += line(f'Training time') + svg += line(f'Strict held-out success') + svg += line(f'Training time') for fraction in (0.0, 0.5, 1.0): x = success_x + success_w * fraction - svg += line(f'') svg += line( - f'' + f'' f"{fraction * 100:.0f}%" ) + svg += line(f'') for fraction in (0.0, 0.5, 1.0): x = time_x + time_w * fraction - svg += line(f'') svg += line( - f'' + f'' f"{time_ceiling * fraction:.0f}s" ) + svg += line(f'') for index, row in enumerate(rows): is_diagnostic = row["name"] in diagnostic_names - y = 190 + index * 75 + (diagnostic_gap if is_diagnostic else 0) + y = 204 + index * 78 + (diagnostic_gap if is_diagnostic else 0) color = colors.get(row["name"], "#4b5563") escaped_name = html.escape(row["name"]) display_name = html.escape(labels.get(row["name"], row["name"])) if diagnostic_start is not None and index == diagnostic_start: - svg += line(f'') svg += line( - f'DIAGNOSTIC CONTROLS · ' + f'' + ) + svg += line( + f'DIAGNOSTIC CONTROLS · ' "INTENTIONALLY PROTOCOL-INCOMPATIBLE TEACHERS" ) - elif index: - svg += line(f'') + svg += line(f'') svg += line( f' str: if row["name"] == "cold-start-only": svg += line( f'' + 'height="26" rx="13" stroke="#64748b"/>' ) svg += line( f'NO TRAINING' + 'text-anchor="middle" fill="#94a3b8">NO TRAINING' ) else: time_end = time_x + time_w * row["seconds_mean"] / time_ceiling diff --git a/src/miniverl/__init__.py b/src/miniverl/__init__.py index 40dbfb7..7fd034e 100644 --- a/src/miniverl/__init__.py +++ b/src/miniverl/__init__.py @@ -14,6 +14,6 @@ from __future__ import annotations -__version__ = "0.2.2.dev0" +__version__ = "0.2.2" __all__ = ["__version__"] diff --git a/tests/unit/test_config.py b/tests/unit/test_config.py index 994362a..b783037 100644 --- a/tests/unit/test_config.py +++ b/tests/unit/test_config.py @@ -137,7 +137,7 @@ def test_run_recipe_fields_match_the_yaml_leaf_for_leaf(path: Path) -> None: _assert_same_leaves(_read_raw(path), config.model_dump(mode="json")) -def test_consumer_gpu_quickstart_uses_competence_gated_protocol_teacher() -> None: +def test_single_gpu_quickstart_is_portable_and_uses_protocol_teacher() -> None: supported = RunConfig.from_yaml(RECIPES_DIR / "qwen_consumer_gpu_calc.yaml") raw_control = RunConfig.from_yaml(RECIPES_DIR / "qwen_consumer_gpu_calc_raw_teacher.yaml") @@ -148,6 +148,12 @@ def test_consumer_gpu_quickstart_uses_competence_gated_protocol_teacher() -> Non assert adapter.revision == "23323751318135484c06c043b1f9b9e7016dd89f" assert adapter.require_policy_evaluation is True assert adapter.minimum_strict_success_rate == pytest.approx(0.5) + assert supported.models.device == "auto" + assert supported.models.student.dtype is Precision.AUTO + assert supported.models.teacher.dtype is Precision.AUTO + assert supported.run.name == "qwen3-calc-opd-single-gpu" + assert "single-gpu" in supported.run.tags + assert not {"rtx4080", "16gb"} & set(supported.run.tags) assert supported.models.student.model_id == raw_control.models.student.model_id assert supported.models.teacher.model_id == raw_control.models.teacher.model_id diff --git a/tests/unit/test_packaging.py b/tests/unit/test_packaging.py index 73b953f..b4ac4bb 100644 --- a/tests/unit/test_packaging.py +++ b/tests/unit/test_packaging.py @@ -284,6 +284,21 @@ def test_published_gpu_visualization_matches_its_source_result(): "DIAGNOSTIC CONTROLS · INTENTIONALLY PROTOCOL-INCOMPATIBLE TEACHERS" ) assert "\ufffd" not in visible_text + grid_lines = [ + element + for element in root.findall("svg:line", namespace) + if element.attrib.get("class") == "grid" + ] + axis_labels = [ + element + for element in root.findall("svg:text", namespace) + if element.attrib.get("class") == "axis" + ] + assert grid_lines and axis_labels + assert min(float(line.attrib["y1"]) for line in grid_lines) > max( + float(label.attrib["y"]) for label in axis_labels + ) + assert 'fill="url(#background)"' in svg_path.read_text(encoding="utf-8") from miniverl.evaluation.schema import BenchmarkResult from scripts.publish_benchmark_artifacts import render_svg @@ -292,6 +307,20 @@ def test_published_gpu_visualization_matches_its_source_result(): assert svg_path.read_text(encoding="utf-8") == render_svg(result, digest) +def test_single_gpu_visual_identity_and_pypi_link_are_prominent() -> None: + banner = (REPO_ROOT / "docs" / "banner.svg").read_text(encoding="utf-8") + assert "PERSONAL SINGLE-GPU TRAINING" in banner + assert "GPU adaptive" in banner + assert "16 GB first" not in banner + ET.fromstring(banner) + + pypi_url = "https://pypi.org/project/miniverl/" + for readme in ("README.md", "README.zh-CN.md"): + text = (REPO_ROOT / readme).read_text(encoding="utf-8") + assert text.count(pypi_url) >= 2 + assert "docs/single-gpu-guide.md" in text + + def test_the_committed_json_schema_matches_the_pydantic_model(): """``schema.py`` claims the two cannot drift; this is what makes that true.