From eef98e9318d1c134429c3e603c88a316d4bd58e2 Mon Sep 17 00:00:00 2001 From: Daoyuan Li <94409450+DaoyuanLi2816@users.noreply.github.com> Date: Fri, 14 Aug 2026 00:28:41 -0700 Subject: [PATCH 1/3] Add full SmolLM2 OPD workload --- .../verl-opd-v0.8-single-gpu-smollm2.yaml | 98 ++++++++ scripts/run_smollm2_opd_reference_workload.py | 223 ++++++++++++++++++ src/miniverl/bridge/opd_runtime.py | 5 +- tests/unit/test_opd_reference_workload.py | 49 ++++ 4 files changed, 373 insertions(+), 2 deletions(-) create mode 100644 examples/verl-opd-v0.8-single-gpu-smollm2.yaml create mode 100644 scripts/run_smollm2_opd_reference_workload.py diff --git a/examples/verl-opd-v0.8-single-gpu-smollm2.yaml b/examples/verl-opd-v0.8-single-gpu-smollm2.yaml new file mode 100644 index 0000000..6be0ffb --- /dev/null +++ b/examples/verl-opd-v0.8-single-gpu-smollm2.yaml @@ -0,0 +1,98 @@ +# Fully pinned SmolLM2 recipe for the direct-GKD v0.8 profile. +data: + train_files: [data/train.parquet] + val_files: [] + prompt_key: prompt + train_batch_size: 4 + max_prompt_length: 128 + max_response_length: 64 + filter_overlong_prompts: false + truncation: error + shuffle: true + seed: 17 +actor_rollout_ref: + model: + path: HuggingFaceTB/SmolLM2-360M-Instruct + enable_gradient_checkpointing: true + lora_rank: 8 + lora_alpha: 16 + target_modules: [q_proj, v_proj] + lora_adapter_path: null + actor: + optim: + lr: 1e-5 + weight_decay: 0.01 + lr_warmup_steps: 0 + loss_agg_mode: token-mean + use_kl_loss: false + ppo_mini_batch_size: 4 + ppo_max_token_len_per_gpu: 768 + use_dynamic_bsz: true + rollout: + name: vllm + n: 1 + temperature: 1.0 + top_p: 0.95 + tensor_model_parallel_size: 1 + gpu_memory_utilization: 0.5 + max_model_len: 192 + max_num_batched_tokens: 768 + max_num_seqs: 4 +algorithm: + use_kl_in_reward: false +distillation: + enabled: true + teacher_key: data_source + n_gpus_per_node: 1 + nnodes: 1 + teacher_models: + teacher_model: + model_path: HuggingFaceTB/SmolLM2-1.7B-Instruct + num_replicas: 1 + inference: + name: vllm + dtype: bfloat16 + tensor_model_parallel_size: 1 + data_parallel_size: 1 + pipeline_model_parallel_size: 1 + gpu_memory_utilization: 0.5 + max_model_len: 193 + distillation_loss: + loss_mode: forward_kl_topk + topk: 32 + use_task_rewards: false + distillation_loss_coef: 1.0 + loss_max_clamp: null + log_prob_min_clamp: -10.0 + use_policy_gradient: false +trainer: + project_name: mini-verl + experiment_name: smollm2-opd-developer-workload + save_freq: 4 + test_freq: 8 + total_epochs: 1 + total_training_steps: 8 + n_gpus_per_node: 1 + nnodes: 1 +miniverl: + student_revision: a10cc1512eabd3dde888204e902eca88bddb4951 + teacher_revision: 31b70e2e869a7173562077fd711b654946d38674 + actor_runtime: + dtype: auto + quantization: nf4 + attn_implementation: sdpa + teacher_runtime: + quantization: nf4 + attn_implementation: sdpa + runtime: + mode: dual_model_resident + memory: + vram_limit_gib: 16 + headroom_gib: 1.5 + batching: + rollout_batch_size: 4 + teacher_score_batch_size: 4 + update_trajectory_batch_size: 1 + teacher_adapter: + path: null + revision: null diff --git a/scripts/run_smollm2_opd_reference_workload.py b/scripts/run_smollm2_opd_reference_workload.py new file mode 100644 index 0000000..f10033c --- /dev/null +++ b/scripts/run_smollm2_opd_reference_workload.py @@ -0,0 +1,223 @@ +"""Run the bounded SmolLM2 direct-GKD developer workload on one CUDA GPU. + +This is systems evidence only. It executes eight fresh-policy updates both +uninterrupted and with an interruption after update four, then verifies PEFT +export and a materialized pinned-verl handoff bundle. It does not score model +quality. +""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import subprocess +import time +from datetime import datetime, timezone +from pathlib import Path +from typing import Any + +from run_verl_opd_reference_workload import ( + INTERRUPT_AFTER, + LOGICAL_BATCH, + PROMPT_LIMIT, + RESPONSE_LIMIT, + TOP_K, + UPDATES, + _equivalence, + _package_version, + _train_resumed, + _train_uninterrupted, + _tree_bytes, + summarize_run, + write_dataset, +) + +from miniverl import __version__ +from miniverl.bridge.export import export_verl_bundle +from miniverl.bridge.materialize import materialize_verl_bundle +from miniverl.bridge.opd_plan import build_immutable_opd_plan, write_immutable_opd_plan +from miniverl.bridge.opd_runtime import build_system_plan +from miniverl.bridge.opd_v08 import VERL_OPD_V08_PROFILE, load_verl_opd_v08_source +from miniverl.utils.runs import write_json_atomic + +EXAMPLE = Path("examples/verl-opd-v0.8-single-gpu-smollm2.yaml") + + +def _sha256(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as handle: + for chunk in iter(lambda: handle.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def _driver() -> str: + return ( + subprocess.run( + ["nvidia-smi", "--query-gpu=driver_version", "--format=csv,noheader"], + check=True, + capture_output=True, + text=True, + ) + .stdout.splitlines()[0] + .strip() + ) + + +def _overrides(dataset: Path) -> list[str]: + return [ + f'data.train_files=["{dataset.resolve().as_posix()}"]', + "data.val_files=[]", + ] + + +def build_plan(dataset: Path, path: Path) -> tuple[Any, Any]: + compiled = load_verl_opd_v08_source( + EXAMPLE, + overrides=_overrides(dataset), + accept_local_reinterpretations=True, + ) + system = build_system_plan(compiled) + plan = build_immutable_opd_plan(compiled, source=str(EXAMPLE), system_plan=system) + write_immutable_opd_plan(path, plan) + from miniverl.config import RunConfig + + return plan, RunConfig.model_validate(plan.resolved_native_config) + + +def _materialize(run: Path, out: Path, *, offline: bool) -> dict[str, Any]: + bundle = out / "verl-bundle" + exported = export_verl_bundle(run, target_verl="v0.8.0", out=bundle) + if exported["launchable"] is not False: + raise RuntimeError("an unmaterialized bundle must fail closed") + materialized = materialize_verl_bundle(bundle, download=not offline, offline=offline) + if materialized["launchable"] is not True: + raise RuntimeError("the exact materialized bundle did not become launchable") + if materialized["distributed_execution_tested"] is not False: + raise RuntimeError("materialization cannot imply distributed execution") + return { + "artifact_bundle_complete": materialized["artifact_bundle_complete"], + "upstream_config_parse_passed": materialized["upstream_config_parse_passed"], + "model_data_load_smoke_passed": materialized["model_data_load_smoke_passed"], + "launchable": materialized["launchable"], + "distributed_execution_tested": materialized["distributed_execution_tested"], + "bundle_bytes": _tree_bytes(bundle), + } + + +def run_workload(out: Path, result_path: Path, *, offline: bool) -> dict[str, Any]: + if out.exists(): + raise FileExistsError(f"output directory already exists: {out}") + out.mkdir(parents=True) + started = time.perf_counter() + dataset = out / "data/smollm2-workload.parquet" + dataset_identity = write_dataset(dataset) + plan, native = build_plan(dataset, out / "plan.json") + reference_run = _train_uninterrupted(native, plan, out / "runs", offline=offline) + reference = summarize_run( + reference_run["run"], construction_seconds=reference_run["construction_seconds"] + ) + resumed_run = _train_resumed(native, plan, out / "runs", offline=offline) + resumed = summarize_run(resumed_run["run"], construction_seconds=0.0) + equivalence = _equivalence(reference, resumed) + scaleout = _materialize(reference_run["run"], out, offline=offline) + + import torch + + device = torch.cuda.get_device_properties(torch.cuda.current_device()) + payload: dict[str, Any] = { + "schema_version": 1, + "kind": "single_gpu_smollm2_direct_gkd_developer_workload", + "status": "maintainer_measured", + "measured_at": datetime.now(timezone.utc) + .replace(microsecond=0) + .isoformat() + .replace("+00:00", "Z"), + "miniverl_version": __version__, + "source_commit": subprocess.run( + ["git", "rev-parse", "HEAD"], check=True, capture_output=True, text=True + ).stdout.strip(), + "workload_driver_sha256": _sha256(Path(__file__)), + "profile": VERL_OPD_V08_PROFILE, + "profile_identity": plan.profile_identity, + "verl": {**plan.pinned_verl, "distributed_execution_tested": False}, + "hardware": { + "gpu": device.name, + "gpu_count": 1, + "vram_gib": round(int(device.total_memory) / 1024**3, 3), + "driver": _driver(), + "torch": torch.__version__, + "cuda_runtime": str(torch.version.cuda), + "transformers": _package_version("transformers"), + "peft": _package_version("peft"), + "bitsandbytes": _package_version("bitsandbytes"), + }, + "models": { + "student": native.models.student.model_dump(mode="json"), + "teacher": native.models.teacher.model_dump(mode="json"), + }, + "recipe": { + "prompts_available": dataset_identity["distinct_prompts"], + "prompts_consumed": reference["distinct_prompts_consumed"], + "prompt_limit": PROMPT_LIMIT, + "response_limit": RESPONSE_LIMIT, + "logical_batch": LOGICAL_BATCH, + "optimizer_updates": UPDATES, + "interrupt_after_updates": INTERRUPT_AFTER, + "teacher_target": "top_k_ids_log_probs_and_mass", + "top_k": TOP_K, + "compiled_plan_sha256": plan.plan_digest, + "input_parquet_sha256": dataset_identity["sha256"], + }, + "measurements": { + key: value + for key, value in reference.items() + if key not in {"checkpoint_hashes", "checkpoint_state", "trajectory_sha256"} + }, + "resume": { + "resume_load_seconds": round(resumed_run["resume_load_seconds"], 4), + "interrupt_checkpoint_bytes": resumed_run["interrupt_checkpoint_bytes"], + **equivalence, + }, + "artifacts": { + "checkpoint_hashes": reference["checkpoint_hashes"], + "trajectory_sha256": reference["trajectory_sha256"], + "standard_peft_adapter_sha256": reference_run["adapter_manifest"]["checksums"][ + "adapter_model.safetensors" + ], + "standard_peft_load_verified": True, + "total_workload_bytes": _tree_bytes(out), + }, + "scaleout": scaleout, + "resource_contract": { + "peak_reserved_limit_gib": 14.5, + "peak_reserved_within_limit": reference["peak_reserved_gib"] <= 14.5, + "total_gpu_workload_seconds": round(time.perf_counter() - started, 2), + "gpu_hour_limit": 3.0, + }, + "scientific_scope": { + "runtime_correctness_only": True, + "alignment_quality_evaluated": False, + "task_quality_evaluated": False, + "algorithm_comparison": False, + }, + } + if not payload["resource_contract"]["peak_reserved_within_limit"]: + raise RuntimeError("SmolLM2 workload exceeded the 14.5 GiB reserved-memory limit") + result_path.parent.mkdir(parents=True, exist_ok=True) + write_json_atomic(result_path, payload) + return payload + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--out", required=True, type=Path) + parser.add_argument("--result", required=True, type=Path) + parser.add_argument("--offline", action="store_true") + args = parser.parse_args() + print(json.dumps(run_workload(args.out, args.result, offline=args.offline), indent=2)) + + +if __name__ == "__main__": + main() diff --git a/src/miniverl/bridge/opd_runtime.py b/src/miniverl/bridge/opd_runtime.py index e02c7c0..41c97fa 100644 --- a/src/miniverl/bridge/opd_runtime.py +++ b/src/miniverl/bridge/opd_runtime.py @@ -44,10 +44,11 @@ class OPDSystemPlan(BaseModel): def _parameter_estimate(model_id: str) -> tuple[int | None, str]: - match = re.search(r"(? float: diff --git a/tests/unit/test_opd_reference_workload.py b/tests/unit/test_opd_reference_workload.py index 7c1d0b8..6cb1d5a 100644 --- a/tests/unit/test_opd_reference_workload.py +++ b/tests/unit/test_opd_reference_workload.py @@ -111,3 +111,52 @@ def test_second_family_smoke_is_compatibility_only_and_frozen() -> None: assert payload["scope"]["alignment_quality_evaluated"] is False assert payload["scope"]["full_recipe_supported"] is False assert payload["runtime"]["distributed_execution_tested"] is False + + +def test_smollm2_full_recipe_is_pinned_and_compiles_without_network() -> None: + from miniverl.bridge.opd_runtime import build_system_plan, compile_native_run_config + from miniverl.bridge.opd_v08 import load_verl_opd_v08_source + + source = Path("examples/verl-opd-v0.8-single-gpu-smollm2.yaml") + compiled = load_verl_opd_v08_source( + source, + accept_local_reinterpretations=True, + ) + native = compile_native_run_config(compiled, system_plan=build_system_plan(compiled)) + + assert compiled.executable is True + assert native.models.student.model_id == "HuggingFaceTB/SmolLM2-360M-Instruct" + assert native.models.student.revision == "a10cc1512eabd3dde888204e902eca88bddb4951" + assert native.models.teacher.model_id == "HuggingFaceTB/SmolLM2-1.7B-Instruct" + assert native.models.teacher.revision == "31b70e2e869a7173562077fd711b654946d38674" + assert native.train.rollouts_per_cycle == 4 + assert native.train.cycles == 8 + assert native.rollout.max_new_tokens_per_turn == 64 + assert native.loss.mode.value == "forward_kl_topk" + assert native.loss.top_k == 32 + + +def test_planner_understands_million_parameter_model_identities() -> None: + from miniverl.bridge.opd_runtime import _parameter_estimate + + assert _parameter_estimate("HuggingFaceTB/SmolLM2-360M-Instruct") == ( + 360_000_000, + "estimated_from_model_identity", + ) + assert _parameter_estimate("Qwen/Qwen3-0.6B") == ( + 600_000_000, + "estimated_from_model_identity", + ) + + +def test_smollm2_workload_driver_has_full_recipe_and_fail_closed_scope() -> None: + path = Path("scripts/run_smollm2_opd_reference_workload.py") + text = path.read_text(encoding="utf-8") + + assert "write_dataset(dataset)" in text + assert "_train_uninterrupted" in text + assert "_train_resumed" in text + assert "_equivalence(reference, resumed)" in text + assert "materialize_verl_bundle" in text + assert '"distributed_execution_tested": False' in text + assert '"task_quality_evaluated": False' in text From 6cf11cfacb455c872559dc1ca48d928594556b15 Mon Sep 17 00:00:00 2001 From: Daoyuan Li <94409450+DaoyuanLi2816@users.noreply.github.com> Date: Fri, 14 Aug 2026 00:40:38 -0700 Subject: [PATCH 2/3] Filter model materialization snapshots --- src/miniverl/bridge/materialize.py | 40 +++++++++++++++++++++++-- tests/unit/test_verl_opd_materialize.py | 13 +++++++- 2 files changed, 50 insertions(+), 3 deletions(-) diff --git a/src/miniverl/bridge/materialize.py b/src/miniverl/bridge/materialize.py index 0aca06b..93ef63d 100644 --- a/src/miniverl/bridge/materialize.py +++ b/src/miniverl/bridge/materialize.py @@ -8,6 +8,7 @@ from __future__ import annotations +import fnmatch import gc import hashlib import importlib.metadata @@ -239,7 +240,6 @@ def _copy_cached_snapshot_to_regular(source: Path, destination: Path) -> None: destination.mkdir(parents=True) for item in sorted(source.rglob("*")): relative = item.relative_to(source) - target = destination / relative try: attributes = getattr(item.lstat(), "st_file_attributes", 0) except OSError as exc: @@ -249,8 +249,10 @@ def _copy_cached_snapshot_to_regular(source: Path, destination: Path) -> None: if item.is_dir(): if item.is_symlink(): raise ConfigError(f"cached snapshot contains a linked directory: {relative}") - target.mkdir(parents=True, exist_ok=True) continue + if not _is_model_snapshot_file(relative): + continue + target = destination / relative try: resolved = item.resolve(strict=True) resolved.relative_to(repository_cache) @@ -262,6 +264,38 @@ def _copy_cached_snapshot_to_regular(source: Path, destination: Path) -> None: shutil.copy2(resolved, target) +_MODEL_SNAPSHOT_ALLOW_PATTERNS = ( + "config.json", + "generation_config.json", + "tokenizer.json", + "tokenizer_config.json", + "special_tokens_map.json", + "added_tokens.json", + "chat_template.jinja", + "vocab.json", + "vocab.txt", + "merges.txt", + "tokenizer.model", + "spiece.model", + "model.safetensors", + "model-*.safetensors", + "model.safetensors.index.json", + "pytorch_model.bin", + "pytorch_model-*.bin", + "pytorch_model.bin.index.json", + "LICENSE*", + "NOTICE*", + "COPYING*", +) + + +def _is_model_snapshot_file(relative: Path) -> bool: + """Keep only the top-level files needed for a local Transformers load.""" + return len(relative.parts) == 1 and any( + fnmatch.fnmatchcase(relative.name, pattern) for pattern in _MODEL_SNAPSHOT_ALLOW_PATTERNS + ) + + def _download_snapshot(*, model_id: str, revision: str, destination: Path, offline: bool) -> Path: try: from huggingface_hub import snapshot_download @@ -278,6 +312,7 @@ def _download_snapshot(*, model_id: str, revision: str, destination: Path, offli repo_id=model_id, revision=revision, local_files_only=True, + allow_patterns=list(_MODEL_SNAPSHOT_ALLOW_PATTERNS), ) ) except Exception: @@ -300,6 +335,7 @@ def _download_snapshot(*, model_id: str, revision: str, destination: Path, offli revision=revision, local_dir=destination, local_files_only=False, + allow_patterns=list(_MODEL_SNAPSHOT_ALLOW_PATTERNS), ) except Exception as exc: mode = "offline cache resolution" if offline else "snapshot download" diff --git a/tests/unit/test_verl_opd_materialize.py b/tests/unit/test_verl_opd_materialize.py index d7709f6..9e49666 100644 --- a/tests/unit/test_verl_opd_materialize.py +++ b/tests/unit/test_verl_opd_materialize.py @@ -331,7 +331,15 @@ def test_offline_download_dereferences_only_one_repository_cache( source = cache / "snapshots" / ("a" * 40) source.mkdir(parents=True) (source / "config.json").write_text("{}", encoding="utf-8") - monkeypatch.setattr(huggingface_hub, "snapshot_download", lambda **kwargs: str(source)) + (source / "onnx").mkdir() + (source / "onnx/model_q4.onnx").write_bytes(b"not part of a Transformers snapshot") + received: dict[str, object] = {} + + def fake_download(**kwargs: object) -> str: + received.update(kwargs) + return str(source) + + monkeypatch.setattr(huggingface_hub, "snapshot_download", fake_download) destination = tmp_path / "regular" resolved = _download_snapshot( @@ -344,6 +352,9 @@ def test_offline_download_dereferences_only_one_repository_cache( assert resolved == destination assert (resolved / "config.json").read_text(encoding="utf-8") == "{}" assert not (resolved / "config.json").is_symlink() + assert not (resolved / "onnx").exists() + assert "model.safetensors" in received["allow_patterns"] + assert "*.onnx" not in received["allow_patterns"] def test_snapshot_symlink_is_rejected_before_publication(tmp_path: Path) -> None: From ff0468a6d0d76d8c29ee84688ee0d718238351a5 Mon Sep 17 00:00:00 2001 From: Daoyuan Li <94409450+DaoyuanLi2816@users.noreply.github.com> Date: Fri, 14 Aug 2026 00:53:25 -0700 Subject: [PATCH 3/3] Publish measured SmolLM2 and WSL2 evidence --- PROJECT_STATE.md | 17 +- PYPI.md | 6 +- README.md | 6 +- README.zh-CN.md | 5 +- .../rtx4080-smollm2-opd-developer-v1.json | 180 ++++++++++++++++++ .../evidence/wsl2-rtx4080-smollm2-opd-v1.json | 71 +++++++ docs/for-verl-users.md | 10 +- docs/release-checklist.md | 6 +- docs/smollm2-opd-workload.md | 46 +++++ docs/verl-opd-reference-workload.md | 19 +- mkdocs.yml | 1 + tests/unit/test_opd_reference_workload.py | 37 ++++ tests/unit/test_packaging.py | 1 + 13 files changed, 375 insertions(+), 30 deletions(-) create mode 100644 benchmarks/results/rtx4080-smollm2-opd-developer-v1.json create mode 100644 docs/evidence/wsl2-rtx4080-smollm2-opd-v1.json create mode 100644 docs/smollm2-opd-workload.md diff --git a/PROJECT_STATE.md b/PROJECT_STATE.md index ce20edc..46507d0 100644 --- a/PROJECT_STATE.md +++ b/PROJECT_STATE.md @@ -4,7 +4,7 @@ Current maintainer handoff for **miniVERL** (`mini-verl` package, `miniverl` CLI). `release-state.yaml` is the canonical version source; this page indexes current product and evidence state rather than repeating release history. -Last updated: 2026-08-13. +Last updated: 2026-08-14. Canonical release state: stable `v0.9.1` (`6c0f3d818c10419e0bfba81f3ad1c5adf24eaf09`), development `0.10.0.dev0`. @@ -31,13 +31,12 @@ Executable compatibility claims are mutation-tested and recorded in unknown-size quantized roles require proof instead of receiving an executable plan. -Development `0.10.0.dev0` now has a closed typed profile registry and -torch-free compatibility introspection. New profile-scoped plans, caches, -checkpoints and exports bind an independent identity; the only registered -executable profile remains the measured direct-GKD path until PG-k1 conformance -and runtime work is complete. +Development `0.10.0.dev0` has a closed typed profile registry and torch-free +compatibility introspection. Profile-scoped plans, caches, checkpoints and +exports bind an independent identity. The direct-GKD and sampled-k1 vanilla +policy-loss profiles both have pinned conformance and measured RTX 4080 paths. -Arbitrary verl YAML, policy-gradient OPD, rewards, PPO/GRPO, Ray, FSDP, +Arbitrary verl YAML, other policy-gradient modes, rewards, PPO/GRPO, Ray, FSDP, Megatron, multi-GPU and distributed execution remain unsupported. The legacy environment/PPO artifact bridge is migration-only. @@ -46,7 +45,9 @@ environment/PPO artifact bridge is migration-only. | Evidence | Status | | --- | --- | | Qwen3-0.6B/1.7B developer workload | 32 prompts, 8 current-policy updates, 3.1914 GiB peak reserved on one RTX 4080; matched interruption/resume was byte-identical | -| SmolLM2-360M/1.7B | one pinned compatibility smoke; not a second measured recipe | +| Qwen3 sampled-k1 PG | 32 prompts, 8 updates, exact interruption/resume, 3.1914 GiB peak reserved; no quality comparison | +| SmolLM2-360M/1.7B direct GKD | 32 prompts, 8 updates, 1.4961 GiB peak reserved; exact resume, PEFT reload and materialized export passed | +| Ubuntu 26.04 WSL2 | plan, bounded probe, rollout, teacher scoring, one update and PEFT reload measured on the same RTX 4080 | | external alignment v1 | preregistered early stop: 0 selected checkpoints, teachers, continuation arms or final-test accesses | | distributed verl execution | not tested | diff --git a/PYPI.md b/PYPI.md index 5260151..6913715 100644 --- a/PYPI.md +++ b/PYPI.md @@ -163,8 +163,10 @@ and update times were 9.7200, 0.4864 and 2.3260 seconds. A matched 4-update interruption resumed to the same byte-identical trajectories, adapter and optimizer tensors. See the [data-bound figure and full record](https://github.com/DaoyuanLi2816/mini-verl/blob/main/docs/verl-opd-reference-workload.md); the original one-update [pip smoke](https://github.com/DaoyuanLi2816/mini-verl/blob/main/docs/opd-quickstart.md) remains preserved. -A separate pinned SmolLM2-360M/1.7B compatibility smoke completed one full -rollout/scoring/update cycle; it is not a second measured recipe. +A separate pinned SmolLM2-360M/1.7B recipe consumed 32 distinct prompts across +8 updates at 1.4961 GiB peak reserved VRAM. Interruption/resume was +byte-identical, PEFT reload passed, and the exact-snapshot scale-out bundle +materialized successfully. See the [full SmolLM2 systems record](https://github.com/DaoyuanLi2816/mini-verl/blob/main/docs/smollm2-opd-workload.md). This is deliberately a runtime and artifact proof. It is not a throughput benchmark, an alignment-quality endpoint, or evidence that OPD beats SFT, DPO diff --git a/README.md b/README.md index beb641a..be1e009 100644 --- a/README.md +++ b/README.md @@ -163,8 +163,10 @@ and update times were 9.7200, 0.4864 and 2.3260 seconds. A matched 4-update interruption resumed to the same byte-identical trajectories, adapter and optimizer tensors. See the [data-bound figure and full record](docs/verl-opd-reference-workload.md); the original one-update [pip smoke](docs/opd-quickstart.md) remains preserved. -A separate pinned SmolLM2-360M/1.7B compatibility smoke completed one full -rollout/scoring/update cycle; it is not a second measured recipe. +A separate pinned SmolLM2-360M/1.7B recipe consumed 32 distinct prompts across +8 updates at 1.4961 GiB peak reserved VRAM. Interruption/resume was +byte-identical, PEFT reload passed, and the exact-snapshot scale-out bundle +materialized successfully. See the [full SmolLM2 systems record](docs/smollm2-opd-workload.md). This is deliberately a runtime and artifact proof. It is not a throughput benchmark, an alignment-quality endpoint, or evidence that OPD beats SFT, DPO diff --git a/README.zh-CN.md b/README.zh-CN.md index b63d7ad..f8b5bb8 100644 --- a/README.zh-CN.md +++ b/README.zh-CN.md @@ -138,8 +138,9 @@ Qwen3-0.6B/1.7B developer workload 实际消费 **32 个不同 prompt**，respon 匹配的运行在第 4 次 update 后中断并恢复；最终 trajectory、adapter 与 optimizer tensor 均字节一致。详见[数据绑定图与完整记录](docs/verl-opd-reference-workload.md)；原始一次更新的 [pip smoke](docs/opd-quickstart.md)仍完整保留。 -另一个固定版本的 SmolLM2-360M/1.7B 兼容性 smoke 完成了一次完整的 -rollout/scoring/update 循环；它不是第二套完整实测 recipe。 +另一套固定版本的 SmolLM2-360M/1.7B recipe 消费 32 个不同 prompt 并完成 8 次 +update，peak reserved VRAM 为 1.4961 GiB；中断恢复保持字节一致，PEFT 重载与精确 +snapshot 的 scale-out 物化均通过。详见[完整 SmolLM2 系统记录](docs/smollm2-opd-workload.md)。 这只证明一个运行时与产物路径，不是吞吐 benchmark、对齐质量 endpoint，也不证明 OPD 优于 SFT、DPO 或 KD。其他 NVIDIA GPU 使用相同的 device-name-agnostic CUDA 路径，但 diff --git a/benchmarks/results/rtx4080-smollm2-opd-developer-v1.json b/benchmarks/results/rtx4080-smollm2-opd-developer-v1.json new file mode 100644 index 0000000..5f6cb3a --- /dev/null +++ b/benchmarks/results/rtx4080-smollm2-opd-developer-v1.json @@ -0,0 +1,180 @@ +{ + "artifacts": { + "checkpoint_hashes": { + "adapter.safetensors": "7d00e5e99ac19ae8c35055e6dee527b737c3af3504b846feef60a0e60d21856d", + "optimizer.safetensors": "1b3473085336cca8b41458c300df368979cb625520462a9152a4d49031b6dd12", + "state.json": "a8f459ce233bd59c011ed31f9b9b308d10572865a2227e0f6e77c04d2de56307" + }, + "standard_peft_adapter_sha256": "6132c9aa94b828deed70d03108805beece3392e50409395423a8ae092aac3e4f", + "standard_peft_load_verified": true, + "total_workload_bytes": 4214630242, + "trajectory_sha256": "f8137936991a17d1531352534eb6e5dd4428e2495a0f5de0b9c68fc21d63bb43" + }, + "hardware": { + "bitsandbytes": "0.50.0", + "cuda_runtime": "13.0", + "driver": "596.49", + "gpu": "NVIDIA GeForce RTX 4080", + "gpu_count": 1, + "peft": "0.18.0", + "torch": "2.13.0+cu130", + "transformers": "5.14.1", + "vram_gib": 15.992 + }, + "kind": "single_gpu_smollm2_direct_gkd_developer_workload", + "measured_at": "2026-08-14T07:46:06Z", + "measurements": { + "adapter_bytes": 6825692, + "batch_downshifts": { + "rollout_oom": 0, + "update_chunk_oom": 0 + }, + "cache_bytes": 81819, + "checkpoint_bytes": 9912101, + "cold_startup_seconds": 8.4132, + "distinct_prompts_consumed": 32, + "observed_rollout_physical_batch_sizes": [ + 4 + ], + "peak_allocated_gib": 1.4254, + "peak_reserved_gib": 1.4961, + "prompts_consumed": 32, + "steady_state_median_seconds": { + "actor_update": 1.127, + "rollout": 16.3067, + "teacher_scoring": 0.1941 + }, + "steady_state_median_throughput": { + "rollout_tokens_per_second": 13.7173, + "teacher_scored_positions_per_second": 1190.38, + "update_positions_per_second": 186.9733 + }, + "time_to_first_rollout_seconds": 21.6392, + "time_to_first_teacher_targets_seconds": 21.8723, + "time_to_first_update_seconds": 23.1082, + "total_run_bytes": 27178611 + }, + "miniverl_version": "0.10.0.dev0", + "models": { + "student": { + "adapter": null, + "attn_implementation": "sdpa", + "dtype": "auto", + "gradient_checkpointing": true, + "lora": { + "alpha": 16, + "bias": "none", + "dropout": 0.0, + "enabled": true, + "r": 8, + "target_modules": [ + "q_proj", + "v_proj" + ] + }, + "model_id": "HuggingFaceTB/SmolLM2-360M-Instruct", + "prepare_kbit_training": true, + "quantization": "nf4", + "revision": "a10cc1512eabd3dde888204e902eca88bddb4951", + "tokenizer_id": null, + "tokenizer_revision": null, + "toy": { + "hidden_size": 64, + "intermediate_size": 128, + "max_position_embeddings": 1024, + "num_heads": 4, + "num_layers": 2 + }, + "trust_remote_code": false + }, + "teacher": { + "adapter": null, + "attn_implementation": "sdpa", + "dtype": "bfloat16", + "mode": "standard", + "model_id": "HuggingFaceTB/SmolLM2-1.7B-Instruct", + "quantization": "nf4", + "revision": "31b70e2e869a7173562077fd711b654946d38674", + "tokenizer_id": null, + "tokenizer_revision": null, + "toy": { + "hidden_size": 64, + "intermediate_size": 128, + "max_position_embeddings": 1024, + "num_heads": 4, + "num_layers": 2 + }, + "toy_pretrain_lr": 0.003, + "toy_pretrain_steps": 0, + "toy_teacher_seed": 99, + "trust_remote_code": false + } + }, + "profile": "verl-opd-v0.8-single-gpu-v1", + "profile_identity": { + "digest": "cc30bae03b65fe1bb915990fbd0359bbff436332b1b44c4887bd77ae8e0e4c35", + "export_version": "verl-opd-export-v1", + "field_rule_digest": "8479344d1d033974b2bbc3a8212c0a01bcc54afbe697ded1987794b7a458ee97", + "loss_conformance_version": "forward-kl-topk-verl-v0.8-v1", + "native_compiler_version": "direct-gkd-native-v1", + "profile_name": "verl-opd-v0.8-single-gpu-v1", + "profile_schema_version": 1, + "upstream_commit": "7aed6b230776f963fa09509c10d9c3a767d1102c", + "upstream_repository": "https://github.com/verl-project/verl", + "upstream_tag": "v0.8.0" + }, + "recipe": { + "compiled_plan_sha256": "d3ff5ab77f2c57d89a2fb52a04a842cbc06b1c2ca5350a82313aadb6131d6067", + "input_parquet_sha256": "baa99ba8c59d4c30b2e3081346860e7e8b2665dd70fdbcd7952935545a0cd770", + "interrupt_after_updates": 4, + "logical_batch": 4, + "optimizer_updates": 8, + "prompt_limit": 128, + "prompts_available": 64, + "prompts_consumed": 32, + "response_limit": 64, + "teacher_target": "top_k_ids_log_probs_and_mass", + "top_k": 32 + }, + "resource_contract": { + "gpu_hour_limit": 3.0, + "peak_reserved_limit_gib": 14.5, + "peak_reserved_within_limit": true, + "total_gpu_workload_seconds": 301.27 + }, + "resume": { + "adapter_and_optimizer_byte_identical": true, + "excluded_run_identity_field": "resolved_config_digest", + "global_optimizer_steps": 8, + "interrupt_checkpoint_bytes": 9912101, + "resume_load_seconds": 3.1039, + "status": "exact_match", + "task_cursor": 32, + "training_state_fields_identical": true, + "trajectories_byte_identical": true + }, + "scaleout": { + "artifact_bundle_complete": true, + "bundle_bytes": 4160177169, + "distributed_execution_tested": false, + "launchable": true, + "model_data_load_smoke_passed": true, + "upstream_config_parse_passed": true + }, + "schema_version": 1, + "scientific_scope": { + "algorithm_comparison": false, + "alignment_quality_evaluated": false, + "runtime_correctness_only": true, + "task_quality_evaluated": false + }, + "source_commit": "6cf11cfacb455c872559dc1ca48d928594556b15", + "status": "maintainer_measured", + "verl": { + "commit": "7aed6b230776f963fa09509c10d9c3a767d1102c", + "distributed_execution_tested": false, + "repository": "https://github.com/verl-project/verl", + "tag": "v0.8.0" + }, + "workload_driver_sha256": "f249766c774e1b44e09ff323d22f548aa5ca9bfa782f54bfbf9ca8782ce59fcd" +} diff --git a/docs/evidence/wsl2-rtx4080-smollm2-opd-v1.json b/docs/evidence/wsl2-rtx4080-smollm2-opd-v1.json new file mode 100644 index 0000000..81736e3 --- /dev/null +++ b/docs/evidence/wsl2-rtx4080-smollm2-opd-v1.json @@ -0,0 +1,71 @@ +{ + "schema_version": 1, + "kind": "wsl2_single_gpu_opd_smoke", + "status": "maintainer_measured", + "measured_at": "2026-08-14T07:49:21Z", + "source_commit": "6cf11cfacb455c872559dc1ca48d928594556b15", + "miniverl_version": "0.10.0.dev0", + "profile": "verl-opd-v0.8-single-gpu-v1", + "profile_identity_digest": "cc30bae03b65fe1bb915990fbd0359bbff436332b1b44c4887bd77ae8e0e4c35", + "platform": { + "distribution": "Ubuntu 26.04 LTS", + "kernel": "6.18.33.2-microsoft-standard-WSL2", + "python": "3.12.13", + "gpu": "NVIDIA GeForce RTX 4080", + "vram_gib": 15.992, + "driver": "596.49", + "torch": "2.13.0+cu130", + "cuda_runtime": "13.0", + "transformers": "5.15.0", + "bitsandbytes": "0.50.1", + "peft": "0.20.0" + }, + "models": { + "student": { + "id": "HuggingFaceTB/SmolLM2-360M-Instruct", + "revision": "a10cc1512eabd3dde888204e902eca88bddb4951", + "quantization": "nf4" + }, + "teacher": { + "id": "HuggingFaceTB/SmolLM2-1.7B-Instruct", + "revision": "31b70e2e869a7173562077fd711b654946d38674", + "quantization": "nf4" + } + }, + "probe": { + "status": "measured", + "parameter_updates": 0, + "duration_seconds": 38.648, + "peak_reserved_gib": 1.0449 + }, + "execution": { + "plan_digest": "a7e941448a73562235a5b05ad8294dcfff5c508c1424adfbddf908523bb9075e", + "prompts": 1, + "response_limit": 64, + "generated_tokens": 42, + "optimizer_updates": 1, + "time_to_first_update_seconds": 9.5106, + "rollout_tokens_per_second": 12.84, + "teacher_scored_positions_per_second": 313.15, + "update_positions_per_second": 75.1401, + "peak_allocated_gib": 1.3808, + "peak_reserved_gib": 1.4043, + "peft_export_and_reload": "passed" + }, + "artifacts": { + "checkpoint_adapter_sha256": "0ad396b576ab89bba894407331bedff529272dbc80a10d5b73935f42f0d444de", + "checkpoint_optimizer_sha256": "b41f8da8946c3d6691bbf9266866a95a7fbd0e76a0d927a56f802749c4d4cd0e", + "trajectory_sha256": "0adbe928a45d5aba8c2c85234fcbba246b4c1bbd1c48397c4f6ea584d701d3e3", + "peft_adapter_sha256": "9429bc79db18ac773a65c7f6a776eef45f9275f7a7f5c783ccd10286d5127c55" + }, + "scope": { + "plan_completed": true, + "probe_completed": true, + "rollout_completed": true, + "teacher_scoring_completed": true, + "optimizer_update_completed": true, + "peft_reload_completed": true, + "task_quality_evaluated": false, + "distributed_execution_tested": false + } +} diff --git a/docs/for-verl-users.md b/docs/for-verl-users.md index 6c7cd6d..2e10257 100644 --- a/docs/for-verl-users.md +++ b/docs/for-verl-users.md @@ -1,6 +1,6 @@ # For verl users -miniVERL is a local runtime for one documented subset of verl v0.8 OPD. It +miniVERL is a local runtime for two documented subsets of verl v0.8 OPD. It keeps familiar field names and Parquet data, then compiles distributed resource intent into sequential phases on one CUDA GPU. It is an independent project; the mapping is explicit and does not imply endorsement or full compatibility. @@ -19,7 +19,8 @@ by plans, caches, checkpoints and exports. - A resolved YAML using the `verl-opd-v0.8-single-gpu-v1` field subset. - Reward-free verl-style Parquet prompts with structured chat messages. -- One actor, one teacher, `n=1`, forward top-k GKD and token-mean aggregation. +- One actor, one teacher, `n=1`, token-mean aggregation, and either direct + forward-top-k GKD or the sampled-k1 vanilla policy-loss profile. - Immutable Hugging Face revisions, PEFT adapters and tokenizer snapshots. - Familiar fields such as `actor_rollout_ref.model.path`, `distillation.teacher_models.teacher_model.model_path`, response bounds, @@ -27,8 +28,9 @@ by plans, caches, checkpoints and exports. What is not reusable: arbitrary Hydra composition inside miniVERL, shell launch scripts, resource pools, Ray actors, FSDP/Megatron checkpoints, PPO/GRPO, critics, -policy-gradient OPD, task-reward mixtures, multiple teachers and multimodal -workers. Unsupported semantics fail closed instead of falling back silently. +policy-gradient modes beyond the closed sampled-k1 profile, task-reward +mixtures, multiple teachers and multimodal workers. Unsupported semantics fail +closed instead of falling back silently. ## Command mapping diff --git a/docs/release-checklist.md b/docs/release-checklist.md index 9c4595d..0ab5fec 100644 --- a/docs/release-checklist.md +++ b/docs/release-checklist.md @@ -7,11 +7,11 @@ after the exact release commit and its remote checks are green. ## v0.10.0 development - [x] Begin from the verified v0.9.1 release and `0.10.0.dev0` canonical state. -- [ ] Add the versioned internal compatibility-profile registry and profile +- [x] Add the versioned internal compatibility-profile registry and profile identity binding without an unrestricted plugin loader. -- [ ] Add the pinned verl v0.8 policy-gradient k1 profile with scalar, gradient, +- [x] Add the pinned verl v0.8 policy-gradient k1 profile with scalar, gradient, freshness, resume and export conformance. -- [ ] Publish bounded RTX 4080 systems evidence for PG k1 and a full SmolLM2 +- [x] Publish bounded RTX 4080 systems evidence for PG k1 and a full SmolLM2 direct-GKD developer recipe; do not add a task-quality benchmark. - [ ] Add validated community hardware records and document the measured or explicitly unmeasured Linux/WSL state. diff --git a/docs/smollm2-opd-workload.md b/docs/smollm2-opd-workload.md new file mode 100644 index 0000000..e5f22be --- /dev/null +++ b/docs/smollm2-opd-workload.md @@ -0,0 +1,46 @@ +# SmolLM2 direct-GKD developer workload + +This is a maintainer-measured systems recipe, not a model-quality benchmark. +It uses the direct `forward_kl_topk` profile with pinned Apache-2.0 +SmolLM2-360M-Instruct actor and SmolLM2-1.7B-Instruct teacher snapshots. + +| Contract | Measured value | +| --- | --- | +| GPU | 1× RTX 4080, 15.992 GiB | +| Data | 64 distinct prompts available; 32 consumed | +| Bounds | 128 prompt tokens; 64 response tokens | +| Schedule | 8 current-policy updates; interrupt after update 4 | +| Objective | direct GKD `forward_kl_topk`, k=32, token mean | +| Peak allocated / reserved | 1.4254 / 1.4961 GiB | +| First update | 23.1082 s including 8.4132 s construction | +| Median rollout / score / update | 16.3067 / 0.1941 / 1.1270 s | +| Median throughput | 13.7173 rollout tok/s; 1190.38 scored positions/s; 186.9733 update positions/s | + +The interrupted execution resumed in 3.1039 seconds. Its trajectories, +adapter and optimizer tensors were byte-identical to the uninterrupted run; +all training-state fields matched except the intentionally run-specific +resolved-config digest. The standard PEFT adapter reloaded successfully. + +The export was then materialized with both exact base snapshots. The pinned +upstream config parse, Parquet check, PEFT load, tokenizer identity and +sequential CPU model-load/forward smoke passed, producing `launchable: true`. +That status describes a complete local artifact bundle; distributed verl +execution remains **not tested**. + +The machine-readable record is +[`rtx4080-smollm2-opd-developer-v1.json`](https://github.com/DaoyuanLi2816/mini-verl/blob/main/benchmarks/results/rtx4080-smollm2-opd-developer-v1.json), +and the reproducible driver is +[`run_smollm2_opd_reference_workload.py`](https://github.com/DaoyuanLi2816/mini-verl/blob/main/scripts/run_smollm2_opd_reference_workload.py). +The earlier one-update compatibility smoke remains immutable and is no longer +the strongest evidence for this pair. + +## WSL2 check + +The same physical RTX 4080 also completed a separate Ubuntu 26.04 WSL2 path: +plan, measured zero-update probe, one 64-token-bounded rollout, teacher +scoring, one update and standard PEFT export/reload. Peak reserved VRAM was +1.4043 GiB and time to first update was 9.5106 seconds. See the checksummed +[`wsl2-rtx4080-smollm2-opd-v1.json`](evidence/wsl2-rtx4080-smollm2-opd-v1.json). + +Neither record evaluates task quality, alignment, preference or safety, and +neither demonstrates distributed verl execution. diff --git a/docs/verl-opd-reference-workload.md b/docs/verl-opd-reference-workload.md index a618571..8b7d6ab 100644 --- a/docs/verl-opd-reference-workload.md +++ b/docs/verl-opd-reference-workload.md @@ -52,16 +52,17 @@ record is [`rtx4080-verl-opd-developer-v1.json`](https://github.com/DaoyuanLi281 and [`run_verl_opd_reference_workload.py`](https://github.com/DaoyuanLi2816/mini-verl/blob/main/scripts/run_verl_opd_reference_workload.py) reconstructs the dataset, plan, uninterrupted run and resumed run. -## Scope +## Second fully measured model pair + +Pinned Apache-2.0 SmolLM2-360M/1.7B snapshots now have a separate full systems +recipe: 32 distinct prompts, eight updates, exact interruption/resume, PEFT +reload and materialized scale-out validation. See the +[SmolLM2 developer workload](smollm2-opd-workload.md). The earlier one-update +compatibility smoke stays immutable as historical evidence. -A separate model-family compatibility smoke used pinned Apache-2.0 -SmolLM2-360M/1.7B snapshots. Their structural tokenizer identity matched, and -one 16-token rollout, teacher-score phase, optimizer update and standard PEFT -reload completed at 1.416 GiB peak reserved VRAM. The checksummed -[`rtx4080-smollm2-opd-family-smoke-v1.json`](https://github.com/DaoyuanLi2816/mini-verl/blob/main/benchmarks/results/rtx4080-smollm2-opd-family-smoke-v1.json) -is compatibility evidence only—not a second supported recipe or quality result. +## Scope No reward, task correctness, alignment, preference, safety or method-comparison endpoint was evaluated. This record does not show that OPD beats SFT, DPO or -KD, and it does not show that a distributed verl job ran. It validates one -Qwen3, one-GPU, forward-top-k compatibility profile. +KD, and it does not show that a distributed verl job ran. It validates two +pinned model pairs on the same one-GPU, forward-top-k profile. diff --git a/mkdocs.yml b/mkdocs.yml index cd72f22..54a7127 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -50,6 +50,7 @@ nav: - Current verl-style OPD runtime: verl-opd-runtime.md - Run verl-style OPD: opd-quickstart.md - RTX 4080 developer workload: verl-opd-reference-workload.md + - SmolLM2 developer workload: smollm2-opd-workload.md - RTX 4080 PG-k1 workload: verl-pg-k1-workload.md - Start on one GPU: single-gpu-guide.md - Align: diff --git a/tests/unit/test_opd_reference_workload.py b/tests/unit/test_opd_reference_workload.py index 6cb1d5a..4500d74 100644 --- a/tests/unit/test_opd_reference_workload.py +++ b/tests/unit/test_opd_reference_workload.py @@ -160,3 +160,40 @@ def test_smollm2_workload_driver_has_full_recipe_and_fail_closed_scope() -> None assert "materialize_verl_bundle" in text assert '"distributed_execution_tested": False' in text assert '"task_quality_evaluated": False' in text + + +def test_measured_smollm2_workload_is_full_scoped_and_frozen() -> None: + path = Path("benchmarks/results/rtx4080-smollm2-opd-developer-v1.json") + assert hashlib.sha256(path.read_bytes()).hexdigest() == ( + "6961c272db01048ab899821a682d030de3f17cdc673e30bc8fe36005f7dce3ef" + ) + payload = json.loads(path.read_text(encoding="utf-8")) + + assert payload["status"] == "maintainer_measured" + assert payload["recipe"]["prompts_consumed"] == 32 + assert payload["recipe"]["response_limit"] == 64 + assert payload["recipe"]["optimizer_updates"] == 8 + assert payload["resume"]["status"] == "exact_match" + assert payload["artifacts"]["standard_peft_load_verified"] is True + assert payload["scaleout"]["launchable"] is True + assert payload["scaleout"]["distributed_execution_tested"] is False + assert payload["measurements"]["peak_reserved_gib"] <= 14.5 + assert payload["scientific_scope"]["task_quality_evaluated"] is False + assert payload["scientific_scope"]["algorithm_comparison"] is False + + +def test_wsl2_smoke_is_measured_scoped_and_frozen() -> None: + path = Path("docs/evidence/wsl2-rtx4080-smollm2-opd-v1.json") + assert hashlib.sha256(path.read_bytes()).hexdigest() == ( + "1e88d0b8a02cae137fe539b82f632dfbb7973e7362c2e0a13f26de0fdd7965d2" + ) + payload = json.loads(path.read_text(encoding="utf-8")) + + assert payload["status"] == "maintainer_measured" + assert payload["platform"]["kernel"].endswith("microsoft-standard-WSL2") + assert payload["probe"]["parameter_updates"] == 0 + assert payload["execution"]["optimizer_updates"] == 1 + assert payload["execution"]["peft_export_and_reload"] == "passed" + assert payload["scope"]["teacher_scoring_completed"] is True + assert payload["scope"]["task_quality_evaluated"] is False + assert payload["scope"]["distributed_execution_tested"] is False diff --git a/tests/unit/test_packaging.py b/tests/unit/test_packaging.py index 18c6f54..362789a 100644 --- a/tests/unit/test_packaging.py +++ b/tests/unit/test_packaging.py @@ -299,6 +299,7 @@ def test_every_published_benchmark_result_validates_against_the_schema(): "rtx4080-verl-opd-runtime-v1.json", "rtx4080-verl-opd-developer-v1.json", "rtx4080-smollm2-opd-family-smoke-v1.json", + "rtx4080-smollm2-opd-developer-v1.json", } ) assert results, "benchmarks/results/ has no published result to validate"