From 33ddbf1d9f637c245176316a686c5681670bbaea Mon Sep 17 00:00:00 2001 From: Daoyuan Li <94409450+DaoyuanLi2816@users.noreply.github.com> Date: Wed, 12 Aug 2026 00:32:46 -0700 Subject: [PATCH 1/4] Add verl-style OPD plan and run workflow --- examples/verl-opd-v0.8-single-gpu.yaml | 5 +- pyproject.toml | 1 + src/miniverl/bridge/opd_runtime.py | 312 ++++++++++++++++++ src/miniverl/bridge/opd_v08.py | 37 +++ src/miniverl/cli.py | 258 ++++++++++++++- src/miniverl/config/models.py | 1 + .../resources/qwen3_0_6b_1_7b_opd.yaml | 81 +++++ src/miniverl/runtime/rollout.py | 9 +- src/miniverl/training/trainer.py | 20 ++ tests/cli/test_opd_product_cli.py | 65 ++++ tests/unit/test_opd_runtime_plan.py | 56 ++++ tests/unit/test_prompt_rollout_runtime.py | 32 ++ 12 files changed, 871 insertions(+), 6 deletions(-) create mode 100644 src/miniverl/bridge/opd_runtime.py create mode 100644 src/miniverl/resources/qwen3_0_6b_1_7b_opd.yaml create mode 100644 tests/cli/test_opd_product_cli.py create mode 100644 tests/unit/test_opd_runtime_plan.py diff --git a/examples/verl-opd-v0.8-single-gpu.yaml b/examples/verl-opd-v0.8-single-gpu.yaml index 43371de..fb14f6b 100644 --- a/examples/verl-opd-v0.8-single-gpu.yaml +++ b/examples/verl-opd-v0.8-single-gpu.yaml @@ -1,5 +1,6 @@ # Resolved fixture for the pinned verl-opd-v0.8-single-gpu-v1 compiler. -# This file compiles offline. Runtime execution is added in later v0.8 PRs. +# This file compiles offline and can be executed through `miniverl run` once +# the referenced Parquet files exist. data: train_files: [data/train.parquet] val_files: [data/val.parquet] @@ -76,6 +77,8 @@ trainer: n_gpus_per_node: 1 nnodes: 1 miniverl: + student_revision: c1899de289a04d12100db370d81485cdf75e47ca + teacher_revision: 70d244cc86ccca08cf5af4e1e306ecf908b1ad5e runtime: mode: auto memory: diff --git a/pyproject.toml b/pyproject.toml index 97aab6b..659c41e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -127,6 +127,7 @@ core-metadata-version = "2.4" "benchmarks/schema/alignment-external-result.schema.json" = "miniverl/evidence/data/alignment-external-v1/result.schema.json" "benchmarks/preregistration/alignment-external-v1.yaml" = "miniverl/evidence/data/alignment-external-v1/preregistration.yaml" "benchmarks/evidence/alignment-external-v1/jsonnav-selection-records.jsonl" = "miniverl/evidence/data/alignment-external-v1/task-evidence.jsonl" +"src/miniverl/resources/qwen3_0_6b_1_7b_opd.yaml" = "miniverl/resources/qwen3_0_6b_1_7b_opd.yaml" [tool.hatch.build.targets.sdist] core-metadata-version = "2.4" diff --git a/src/miniverl/bridge/opd_runtime.py b/src/miniverl/bridge/opd_runtime.py new file mode 100644 index 0000000..ba158a3 --- /dev/null +++ b/src/miniverl/bridge/opd_runtime.py @@ -0,0 +1,312 @@ +"""Local planning and native-runtime compilation for the pinned verl OPD profile.""" + +from __future__ import annotations + +import math +import re +from typing import Any, Literal + +from pydantic import BaseModel, ConfigDict + +from miniverl.bridge.opd_v08 import CompiledLocalExecutionPlan +from miniverl.config.models import RunConfig +from miniverl.errors import ConfigError + +__all__ = ["OPDSystemPlan", "build_system_plan", "compile_native_run_config"] + + +class OPDSystemPlan(BaseModel): + """A weight-free plan whose estimates are never labelled as measurements.""" + + model_config = ConfigDict(extra="forbid") + + schema_version: Literal[1] = 1 + profile: str + upstream: dict[str, str] + compiled_digest: str + executable: bool + field_classification_counts: dict[str, int] + unsupported_fields: list[str] + student: dict[str, Any] + teacher: dict[str, Any] + data: dict[str, Any] + loss: dict[str, Any] + local_execution: dict[str, Any] + memory: dict[str, Any] + batching: dict[str, int] + disk: dict[str, Any] + time_to_first_update: dict[str, Any] + + +def _parameter_estimate(model_id: str) -> tuple[int | None, str]: + match = re.search(r"(? float: + return round(value / (1024**3), 3) + + +def build_system_plan(compiled: CompiledLocalExecutionPlan) -> OPDSystemPlan: + """Build a deterministic plan without importing torch or loading weights.""" + source = compiled.source + student_params, student_basis = _parameter_estimate(source.actor_rollout_ref.model.path) + teacher_params, teacher_basis = _parameter_estimate( + source.distillation.teacher_models.teacher_model.model_path + ) + rank = source.actor_rollout_ref.model.lora_rank + # NF4 estimates include a conservative 0.65 byte/parameter allowance for + # quantized weights, scales and allocator overhead. LoRA/Adam is estimated + # separately and deliberately rounded up. + student_static = None if student_params is None else student_params * 0.65 + teacher_static = None if teacher_params is None else teacher_params * 0.65 + trainable_optimizer = ( + None if student_params is None else max(96 * rank * math.sqrt(student_params), 64 * 1024**2) + ) + max_tokens = source.data.max_prompt_length + source.data.max_response_length + rollout_batch = source.miniverl.batching.rollout_batch_size + token_buffers = max_tokens * rollout_batch * 64 * 1024 + target_bytes = ( + source.data.train_batch_size + * source.data.max_response_length + * source.distillation.distillation_loss.topk + * 12 + ) + known_static = sum( + value + for value in (student_static, teacher_static, trainable_optimizer) + if value is not None + ) + limit = source.miniverl.memory.vram_limit_gib + headroom = source.miniverl.memory.headroom_gib + requested = source.miniverl.runtime.mode + if requested == "auto": + if student_static is None or teacher_static is None: + strategy = "swap" + reason = "auto -> swap because one or more model-size estimates are unknown" + elif _gib(known_static + token_buffers + target_bytes) <= limit - headroom: + strategy = "dual_model_resident" + reason = "auto -> dual_model_resident from model metadata and configured headroom" + else: + strategy = "swap" + reason = "auto -> swap because estimated resident demand exceeds usable VRAM" + else: + strategy = requested + reason = f"runtime mode was set explicitly to {requested}" + if strategy == "shared_backbone" and ( + source.actor_rollout_ref.model.path + != source.distillation.teacher_models.teacher_model.model_path + ): + raise ConfigError( + "shared_backbone requires the student and teacher to use the same base model", + hint="use miniverl.runtime.mode=auto, dual_model_resident, or swap", + ) + classifications: dict[str, int] = {} + for item in compiled.compatibility: + classifications[item.classification] = classifications.get(item.classification, 0) + 1 + return OPDSystemPlan( + profile=compiled.profile, + upstream=compiled.upstream, + compiled_digest=compiled.compiled_digest, + executable=compiled.executable, + field_classification_counts=classifications, + unsupported_fields=[ + item.upstream_field + for item in compiled.compatibility + if item.classification == "unsupported" + ], + student={ + "model_id": source.actor_rollout_ref.model.path, + "revision": source.miniverl.student_revision, + "parameter_count": student_params, + "parameter_count_status": student_basis, + }, + teacher={ + "model_id": source.distillation.teacher_models.teacher_model.model_path, + "revision": source.miniverl.teacher_revision, + "parameter_count": teacher_params, + "parameter_count_status": teacher_basis, + "adapter": source.miniverl.teacher_adapter.model_dump(mode="json"), + }, + data={ + "train_files": source.data.train_files, + "val_files": source.data.val_files, + "row_counts": "unknown_until_scan", + "prompt_tokens_max": source.data.max_prompt_length, + "response_tokens_max": source.data.max_response_length, + }, + loss={ + "mode": source.distillation.distillation_loss.loss_mode, + "aggregation": source.actor_rollout_ref.actor.loss_agg_mode, + "top_k": source.distillation.distillation_loss.topk, + "task_rewards": False, + }, + local_execution={ + "strategy": strategy, + "reason": reason, + "roles": [ + "ActorRuntime", + "TeacherRuntime", + "ReferenceRuntime:not_used", + "RolloutRuntime", + "UpdateRuntime", + "ArtifactRuntime", + ], + "phases": [ + "actor_rollout", + "teacher_scoring", + "actor_forward_backward_update", + "checkpoint_evaluation", + ], + "distributed_execution": False, + }, + memory={ + "status": "estimated", + "static_weight_gib": { + "student": None if student_static is None else _gib(student_static), + "teacher": None if teacher_static is None else _gib(teacher_static), + }, + "trainable_and_optimizer_gib": ( + None if trainable_optimizer is None else _gib(trainable_optimizer) + ), + "kv_and_token_buffer_gib": _gib(token_buffers), + "teacher_target_gib": _gib(target_bytes), + "configured_vram_limit_gib": limit, + "configured_headroom_gib": headroom, + "measured_peak_reserved_gib": None, + }, + batching={ + "rollout": source.miniverl.batching.rollout_batch_size, + "teacher_score": source.miniverl.batching.teacher_score_batch_size, + "update_trajectories": source.miniverl.batching.update_trajectory_batch_size, + }, + disk={ + "status": "estimated", + "model_download_gib": _gib(known_static * 2), + "run_artifacts_gib": round(max(0.25, _gib(target_bytes) * 4), 3), + }, + time_to_first_update={"status": "unknown", "seconds": None}, + ) + + +def compile_native_run_config( + compiled: CompiledLocalExecutionPlan, + *, + system_plan: OPDSystemPlan | None = None, +) -> RunConfig: + """Translate the supported profile into an executable, validated RunConfig.""" + source = compiled.source + plan = system_plan or build_system_plan(compiled) + strategy = plan.local_execution["strategy"] + if strategy == "shared_backbone": + raise ConfigError( + "shared_backbone execution needs a materialized frozen teacher adapter", + hint="the packaged Qwen3 pair uses auto/dual_model_resident/swap", + ) + if not source.miniverl.student_revision or not source.miniverl.teacher_revision: + raise ConfigError( + "runnable OPD plans require immutable student and teacher revisions", + hint="set miniverl.student_revision and miniverl.teacher_revision", + ) + total_steps = source.trainer.total_training_steps + if total_steps is None: + total_steps = source.trainer.total_epochs + logical_batch = source.data.train_batch_size + memory_strategy = "resident" if strategy == "dual_model_resident" else strategy + payload = { + "schema_version": 1, + "run": { + "name": source.trainer.experiment_name, + "mode": "opd", + "seed": source.data.seed or 0, + "output_dir": "runs", + "deterministic": True, + "tags": [compiled.profile, "verl-v0.8.0", "pure-opd"], + }, + "models": { + "backend": "hf", + "runtime": "dual_model", + "device": "cuda", + "student": { + "model_id": source.actor_rollout_ref.model.path, + "revision": source.miniverl.student_revision, + "dtype": "auto", + "quantization": "nf4", + "gradient_checkpointing": source.actor_rollout_ref.model.enable_gradient_checkpointing, + "lora": { + "enabled": True, + "r": source.actor_rollout_ref.model.lora_rank, + "alpha": source.actor_rollout_ref.model.lora_alpha, + "target_modules": source.actor_rollout_ref.model.target_modules, + }, + }, + "teacher": { + "model_id": source.distillation.teacher_models.teacher_model.model_path, + "revision": source.miniverl.teacher_revision, + "dtype": "auto", + "quantization": "nf4", + "mode": "standard", + "toy_pretrain_steps": 0, + }, + }, + "source": { + "kind": "verl_parquet", + "train_files": source.data.train_files, + "val_files": source.data.val_files, + "prompt_key": source.data.prompt_key, + "allow_plain_string_prompts": False, + "use_task_rewards": False, + "max_prompt_length": source.data.max_prompt_length, + "max_response_length": source.data.max_response_length, + "truncation": source.data.truncation, + "shuffle": source.data.shuffle, + "seed": source.data.seed or 0, + }, + "rollout": { + "max_turns": 1, + "max_total_tokens": source.data.max_prompt_length + source.data.max_response_length, + "temperature": source.actor_rollout_ref.rollout.temperature, + "top_p": source.actor_rollout_ref.rollout.top_p, + "prompt_batch_size": source.miniverl.batching.rollout_batch_size, + "max_padded_tokens": source.actor_rollout_ref.rollout.max_num_batched_tokens + or source.actor_rollout_ref.actor.ppo_max_token_len_per_gpu, + }, + "selection": {"selector": "all_model_tokens"}, + "loss": { + "mode": "forward_kl_topk", + "aggregation": "token-mean", + "divergence": "forward_kl", + "temperature": 1.0, + "scale_by_temperature_squared": False, + "top_k": source.distillation.distillation_loss.topk, + "log_prob_min_clamp": source.distillation.distillation_loss.log_prob_min_clamp, + "loss_max_clamp": source.distillation.distillation_loss.loss_max_clamp, + "sampled_token_nll_weight": 0.0, + }, + "train": { + "cycles": total_steps, + "rollouts_per_cycle": logical_batch, + "gradient_accumulation_steps": logical_batch, + "trajectory_batch_size": source.miniverl.batching.update_trajectory_batch_size, + "learning_rate": source.actor_rollout_ref.actor.optim.lr, + "weight_decay": source.actor_rollout_ref.actor.optim.weight_decay, + "warmup_steps": source.actor_rollout_ref.actor.optim.lr_warmup_steps, + "save_every_cycles": max(source.trainer.save_freq, 0), + "eval_every_cycles": max(source.trainer.test_freq, 0), + }, + "memory": { + "strategy": memory_strategy, + "auto_swap_vram_headroom_gb": source.miniverl.memory.headroom_gib, + }, + "cache": { + "dtype": "float32", + "strict_policy_version": True, + "reuse_across_policy_versions": False, + "keep_cycles": 1, + }, + "eval": {"enabled": bool(source.data.val_files), "baseline_enabled": False}, + "report": {"enabled": True}, + } + return RunConfig.model_validate(payload) diff --git a/src/miniverl/bridge/opd_v08.py b/src/miniverl/bridge/opd_v08.py index d1f333c..747ab54 100644 --- a/src/miniverl/bridge/opd_v08.py +++ b/src/miniverl/bridge/opd_v08.py @@ -28,6 +28,7 @@ "VerlOPDV08Profile", "compile_verl_opd_v08", "load_verl_opd_v08", + "load_verl_opd_v08_source", "parse_overrides", ] @@ -239,6 +240,8 @@ class MiniVerlTeacherAdapterExtensions(_StrictModel): class MiniVerlLocalExtensions(_StrictModel): + student_revision: str | None = None + teacher_revision: str | None = None runtime: MiniVerlRuntimeExtensions = Field(default_factory=MiniVerlRuntimeExtensions) memory: MiniVerlMemoryExtensions = Field(default_factory=MiniVerlMemoryExtensions) batching: MiniVerlBatchingExtensions = Field(default_factory=MiniVerlBatchingExtensions) @@ -514,6 +517,12 @@ def _rule( ), "trainer.nnodes": _rule("placement.node_count", "locally_reinterpreted", "must be one", "high"), "miniverl.runtime.mode": _rule(None, "informational_only", "miniVERL-only local extension"), + "miniverl.student_revision": _rule( + "student.revision", "informational_only", "miniVERL-only immutable Hub revision" + ), + "miniverl.teacher_revision": _rule( + "teacher.revision", "informational_only", "miniVERL-only immutable Hub revision" + ), "miniverl.memory.vram_limit_gib": _rule( None, "informational_only", "miniVERL-only planner limit" ), @@ -843,3 +852,31 @@ def load_verl_opd_v08( overrides=overrides, require_executable=require_executable, ) + + +def load_verl_opd_v08_source( + source: str | Path, + *, + overrides: Sequence[str] = (), + require_executable: bool = True, +) -> CompiledLocalExecutionPlan: + """Load a path or the packaged Qwen3 quickstart profile.""" + source_text = str(source) + if source_text != "builtin:qwen3-0.6b-1.7b-opd": + return load_verl_opd_v08( + Path(source), overrides=overrides, require_executable=require_executable + ) + from importlib.resources import files + + resource = files("miniverl").joinpath("resources/qwen3_0_6b_1_7b_opd.yaml") + try: + payload = yaml.safe_load(resource.read_text(encoding="utf-8")) + except (OSError, UnicodeError, yaml.YAMLError) as exc: + raise ConfigError(f"cannot read packaged OPD profile: {exc}") from exc + if not isinstance(payload, Mapping): + raise ConfigError("packaged OPD profile must contain one mapping") + return compile_verl_opd_v08( + payload, + overrides=overrides, + require_executable=require_executable, + ) diff --git a/src/miniverl/cli.py b/src/miniverl/cli.py index 6503430..0429bf8 100644 --- a/src/miniverl/cli.py +++ b/src/miniverl/cli.py @@ -13,6 +13,7 @@ import hashlib import json import sys +import time from pathlib import Path from typing import Any, Literal, Optional @@ -55,6 +56,8 @@ help="Show and validate evidence packaged with the installed wheel.", no_args_is_help=True ) app.add_typer(evidence_app, name="evidence") +data_app = typer.Typer(help="Create and inspect portable prompt data.", no_args_is_help=True) +app.add_typer(data_app, name="data") console = Console() err_console = Console(stderr=True) @@ -142,6 +145,51 @@ def main( configure_logging(log_level) +@data_app.command("sample") +def data_sample_command( + out: Path = typer.Option(..., "--out", help="Output Parquet path."), + format_name: str = typer.Option( + "verl-parquet", "--format", help="Portable output format (verl-parquet only)." + ), + rows: int = typer.Option(4, "--rows", min=1, max=1024, help="Number of sample prompts."), +) -> None: + """Create a small reward-free verl-style Parquet prompt dataset.""" + if format_name != "verl-parquet": + _fail(ConfigError("--format must be verl-parquet")) + return + try: + import pyarrow as pa + import pyarrow.parquet as pq + except ModuleNotFoundError as exc: + from miniverl.errors import MissingDependencyError + + _fail(MissingDependencyError(exc.name or "pyarrow", "bridge", "Parquet sample data")) + return + prompts = [ + "Explain why exact provenance matters in one concise sentence.", + "Give one safe way to recover from a CUDA out-of-memory error.", + "What does token-mean loss aggregation mean?", + "State one limitation of a single-GPU training runtime.", + ] + records = [] + for index in range(rows): + records.append( + { + "prompt": [ + {"role": "system", "content": "Answer clearly and briefly."}, + {"role": "user", "content": prompts[index % len(prompts)]}, + ], + "data_source": "miniverl_quickstart", + "ability": "short_answer", + "extra_info": {"sample_index": index}, + } + ) + out.parent.mkdir(parents=True, exist_ok=True) + pq.write_table(pa.Table.from_pylist(records), out) + digest = hashlib.sha256(out.read_bytes()).hexdigest() + console.print(f"[green]wrote[/green] {_esc(out)} ({rows} rows, sha256 {digest})") + + # ---------------------------------------------------------------- doctor @@ -857,6 +905,210 @@ def align( console.print(f" card {_esc(Path(str(payload['run_dir'])) / 'alignment-card.md')}") +# ---------------------------------------------------------- verl-shaped plan/run + + +@app.command("plan") +def plan_command( + config: str = typer.Option(..., "--config", help="Resolved YAML path or builtin profile."), + profile: str = typer.Option( + "verl-opd-v0.8-single-gpu-v1", "--profile", help="Pinned compatibility profile." + ), + overrides: list[str] = typer.Option([], "--set", help="Repeatable dotted key=value override."), + as_json: bool = typer.Option(False, "--json", help="Emit machine-readable JSON."), + offline: bool = typer.Option(False, "--offline", help="Do not access the network."), + probe: bool = typer.Option( + False, "--probe", help="Load models for a bounded probe (not available in v0.8.0)." + ), +) -> None: + """Plan pinned single-GPU verl-style OPD without loading model weights.""" + del offline + try: + from miniverl.bridge.opd_runtime import build_system_plan + from miniverl.bridge.opd_v08 import ( + VERL_OPD_V08_PROFILE, + load_verl_opd_v08_source, + ) + + if profile != VERL_OPD_V08_PROFILE: + raise ConfigError( + f"unsupported OPD profile {profile!r}", hint=f"use --profile {VERL_OPD_V08_PROFILE}" + ) + if probe: + raise ConfigError( + "--probe is not implemented in v0.8.0", + hint="run without --probe for the weight-free estimate", + ) + compiled = load_verl_opd_v08_source(config, overrides=overrides) + plan = build_system_plan(compiled) + payload = plan.model_dump(mode="json") + except MiniVerlError as exc: + _fail(exc) + return + if as_json: + _emit_json(payload) + return + console.print(f"[green]executable: {str(plan.executable).lower()}[/green]") + console.print(f" profile {_esc(plan.profile)}") + console.print(f" verl {_esc(plan.upstream['tag'])} @ {_esc(plan.upstream['commit'][:12])}") + console.print(f" student {_esc(plan.student['model_id'])} @ {_esc(plan.student['revision'])}") + console.print(f" teacher {_esc(plan.teacher['model_id'])} @ {_esc(plan.teacher['revision'])}") + console.print( + f" loss {_esc(plan.loss['mode'])}, {_esc(plan.loss['aggregation'])}, " + f"top-k {_esc(plan.loss['top_k'])}" + ) + console.print( + f" placement {_esc(plan.local_execution['strategy'])}: " + f"{_esc(plan.local_execution['reason'])}" + ) + console.print(" memory estimates (not measurements)") + for key, value in plan.memory.items(): + console.print(f" {_esc(key)}: {_esc(value)}") + console.print(" time to first update: unknown (not measured by plan)") + console.print(f" plan sha256 {_esc(plan.compiled_digest)}") + + +@app.command("run") +def verl_run_command( + config: str = typer.Option(..., "--config", help="Resolved YAML path or builtin profile."), + profile: str = typer.Option( + "verl-opd-v0.8-single-gpu-v1", "--profile", help="Pinned compatibility profile." + ), + overrides: list[str] = typer.Option([], "--set", help="Repeatable dotted key=value override."), + output: Optional[Path] = typer.Option(None, "--output", help="Parent run directory."), + run_id: Optional[str] = typer.Option(None, "--run-id", help="Explicit run id."), + resume: Optional[Path] = typer.Option(None, "--resume", help="Resume an existing run."), + offline: bool = typer.Option(False, "--offline", help="Use cached model files only."), + dry_run: bool = typer.Option(False, "--dry-run", help="Compile native config only."), + as_json: bool = typer.Option(False, "--json", help="Emit machine-readable JSON."), +) -> None: + """Execute the pinned verl v0.8 pure-OPD subset on one local CUDA GPU.""" + try: + from miniverl.bridge.opd_runtime import build_system_plan, compile_native_run_config + from miniverl.bridge.opd_v08 import ( + VERL_OPD_V08_PROFILE, + load_verl_opd_v08_source, + ) + + if profile != VERL_OPD_V08_PROFILE: + raise ConfigError( + f"unsupported OPD profile {profile!r}", hint=f"use --profile {VERL_OPD_V08_PROFILE}" + ) + compiled = load_verl_opd_v08_source(config, overrides=overrides) + system_plan = build_system_plan(compiled) + native = compile_native_run_config(compiled, system_plan=system_plan) + if dry_run: + payload = { + "dry_run": True, + "compatibility": compiled.model_dump(mode="json"), + "system_plan": system_plan.model_dump(mode="json"), + "resolved_native_config": native.model_dump(mode="json"), + } + if as_json: + _emit_json(payload) + else: + console.print("[green]run dry run ok[/green]") + console.print(f" plan sha256 {_esc(compiled.compiled_digest)}") + console.print(" no model weights loaded") + return + _require_training_stack("miniverl run") + from miniverl.trainer import OPDTrainer + from miniverl.utils.runs import read_jsonl, write_json_atomic + + construction_started = time.perf_counter() + trainer_instance = OPDTrainer.from_config( + native, + output_dir=output, + run_id=run_id, + local_files_only=offline, + resume=resume, + ) + construction_seconds = time.perf_counter() - construction_started + with trainer_instance as trainer: + write_json_atomic( + trainer.paths.root / "verl-source-config.json", + compiled.source.model_dump(mode="json"), + ) + write_json_atomic( + trainer.paths.root / "verl-compatibility-report.json", + compiled.model_dump(mode="json"), + ) + write_json_atomic( + trainer.paths.root / "local-execution-plan.json", + system_plan.model_dump(mode="json"), + ) + result = trainer.train() + paths = trainer.paths + cycle_rows = [ + row for row in read_jsonl(paths.metrics) if row.get("phase") == "opd_cycle" + ] + first_cycle = cycle_rows[0] if cycle_rows else {} + final_metrics = result.final_metrics + checkpoint = paths.checkpoints / "final" + checkpoint_bytes = sum( + item.stat().st_size for item in checkpoint.rglob("*") if item.is_file() + ) + run_bytes = sum(item.stat().st_size for item in paths.root.rglob("*") if item.is_file()) + adapter_file = checkpoint / "adapter.safetensors" + measurements = { + "schema_version": 1, + "status": "measured", + "hardware": { + "device": trainer.plan.device, + "gpu_count": 1, + "distributed_execution": False, + }, + "construction_seconds": round(construction_seconds, 4), + "time_to_first_rollout_seconds": round( + construction_seconds + float(first_cycle.get("rollout_seconds") or 0.0), 4 + ), + "time_to_first_teacher_target_batch_seconds": round( + construction_seconds + + float(first_cycle.get("rollout_seconds") or 0.0) + + float(first_cycle.get("teacher_scoring_seconds") or 0.0), + 4, + ), + "time_to_first_optimizer_update_seconds": round( + construction_seconds + float(first_cycle.get("seconds") or 0.0), 4 + ), + "rollout_tokens_per_second": ( + round( + float((first_cycle.get("rollouts") or {}).get("generated_tokens") or 0) + / float(first_cycle.get("rollout_seconds") or 1.0), + 2, + ) + ), + "teacher_scored_positions_per_second": first_cycle.get( + "teacher_scored_positions_per_second" + ), + "update_selected_positions_per_second": final_metrics.get( + "train_selected_tokens_per_second" + ), + "peak_allocated_gib": (final_metrics.get("memory") or {}).get("peak_allocated_gib"), + "peak_reserved_gib": (final_metrics.get("memory") or {}).get("peak_reserved_gib"), + "checkpoint_bytes": checkpoint_bytes, + "run_artifacts_bytes_before_measurement_manifest": run_bytes, + "adapter_sha256": ( + hashlib.sha256(adapter_file.read_bytes()).hexdigest() + if adapter_file.is_file() + else None + ), + "runtime_correctness_only": True, + "alignment_quality_claim": False, + } + write_json_atomic(paths.root / "verl-reference-measurements.json", measurements) + except (MiniVerlError, ModuleNotFoundError, ValidationError) as exc: + _fail(exc) + return + payload = {**result.to_dict(), "run_dir": str(paths.root), "measurements": measurements} + if as_json: + _emit_json(payload) + else: + console.print(f"[bold green]OPD run complete[/bold green] {_esc(paths.root)}") + console.print(f" optimizer steps {_esc(result.global_step)}") + console.print(" distributed execution false") + + # ----------------------------------------------------------------- train @@ -1352,7 +1604,7 @@ def bridge_doctor_command( @bridge_app.command("compile-opd") def bridge_compile_opd_command( - config: Path = typer.Option(..., "--config", help="Resolved verl v0.8 OPD YAML."), + config: str = typer.Option(..., "--config", help="Resolved YAML path or builtin profile."), profile: str = typer.Option( "verl-opd-v0.8-single-gpu-v1", "--profile", @@ -1377,14 +1629,14 @@ def bridge_compile_opd_command( ) -> None: """Compile the pinned single-GPU verl v0.8 OPD config subset offline.""" try: - from miniverl.bridge.opd_v08 import VERL_OPD_V08_PROFILE, load_verl_opd_v08 + from miniverl.bridge.opd_v08 import VERL_OPD_V08_PROFILE, load_verl_opd_v08_source if profile != VERL_OPD_V08_PROFILE: raise ConfigError( f"unsupported OPD profile {profile!r}", hint=f"use --profile {VERL_OPD_V08_PROFILE}", ) - plan = load_verl_opd_v08( + plan = load_verl_opd_v08_source( config, overrides=overrides, require_executable=not inspect_unsupported, diff --git a/src/miniverl/config/models.py b/src/miniverl/config/models.py index dd77c31..3f924ad 100644 --- a/src/miniverl/config/models.py +++ b/src/miniverl/config/models.py @@ -542,6 +542,7 @@ class VerlParquetSourceConfig(_Base): allow_plain_string_prompts: bool = False use_task_rewards: bool = False max_prompt_length: int = Field(default=512, ge=1, le=131072) + max_response_length: int = Field(default=256, ge=1, le=131072) truncation: PromptTruncation = PromptTruncation.ERROR shuffle: bool = True seed: int = Field(default=0, ge=0) diff --git a/src/miniverl/resources/qwen3_0_6b_1_7b_opd.yaml b/src/miniverl/resources/qwen3_0_6b_1_7b_opd.yaml new file mode 100644 index 0000000..37640b7 --- /dev/null +++ b/src/miniverl/resources/qwen3_0_6b_1_7b_opd.yaml @@ -0,0 +1,81 @@ +data: + train_files: [data/opd-smoke.parquet] + val_files: [] + prompt_key: prompt + train_batch_size: 2 + max_prompt_length: 128 + max_response_length: 16 + filter_overlong_prompts: false + truncation: error + shuffle: false + seed: 20260812 +actor_rollout_ref: + model: + path: Qwen/Qwen3-0.6B + enable_gradient_checkpointing: true + lora_rank: 8 + lora_alpha: 16 + target_modules: [q_proj, k_proj, v_proj, o_proj] + lora_adapter_path: null + actor: + optim: {lr: 1e-5, weight_decay: 0.0, lr_warmup_steps: 0} + loss_agg_mode: token-mean + use_kl_loss: false + ppo_mini_batch_size: 2 + ppo_max_token_len_per_gpu: 512 + use_dynamic_bsz: true + rollout: + name: local_hf + n: 1 + temperature: 0.0 + top_p: 1.0 + tensor_model_parallel_size: 1 + gpu_memory_utilization: 0.5 + max_model_len: 144 + max_num_batched_tokens: 288 + max_num_seqs: 2 +algorithm: {use_kl_in_reward: false} +distillation: + enabled: true + teacher_key: data_source + n_gpus_per_node: 1 + nnodes: 1 + teacher_models: + teacher_model: + model_path: Qwen/Qwen3-1.7B + num_replicas: 1 + inference: + name: local_hf + dtype: bfloat16 + tensor_model_parallel_size: 1 + data_parallel_size: 1 + pipeline_model_parallel_size: 1 + gpu_memory_utilization: 0.5 + max_model_len: 144 + distillation_loss: + loss_mode: forward_kl_topk + topk: 32 + use_task_rewards: false + distillation_loss_coef: 1.0 + loss_max_clamp: null + log_prob_min_clamp: -10.0 + use_policy_gradient: false +trainer: + project_name: mini-verl + experiment_name: qwen3-opd-quickstart + save_freq: 1 + test_freq: -1 + total_epochs: 1 + total_training_steps: 1 + n_gpus_per_node: 1 + nnodes: 1 +miniverl: + student_revision: c1899de289a04d12100db370d81485cdf75e47ca + teacher_revision: 70d244cc86ccca08cf5af4e1e306ecf908b1ad5e + runtime: {mode: auto} + memory: {vram_limit_gib: 16.0, headroom_gib: 1.5} + batching: + rollout_batch_size: 2 + teacher_score_batch_size: 2 + update_trajectory_batch_size: 1 + teacher_adapter: {path: null, revision: null} diff --git a/src/miniverl/runtime/rollout.py b/src/miniverl/runtime/rollout.py index 5028750..5ef6b5d 100644 --- a/src/miniverl/runtime/rollout.py +++ b/src/miniverl/runtime/rollout.py @@ -78,12 +78,17 @@ def __init__( self.config = rollout_config self._closed = False + @property + def max_new_tokens(self) -> int: + """Independent response bound for the verl Parquet profile.""" + return min(self.config.max_new_tokens_per_turn, self.source_config.max_response_length) + def prepare_batch(self, inputs: list[RenderedPrompt]) -> PreparedPromptBatch: if self._closed: raise RuntimeError("prompt rollout runtime is closed") if not inputs: raise ValueError("prompt rollout batch cannot be empty") - max_new = self.config.max_new_tokens_per_turn + max_new = self.max_new_tokens for prompt in inputs: if len(prompt.token_ids) + max_new > self.config.max_total_tokens: raise ValueError( @@ -125,7 +130,7 @@ def _generate_group( try: output = self.backend.generate_batch( [batch.prompts[index].token_ids for index in indices], - max_new_tokens=self.config.max_new_tokens_per_turn, + max_new_tokens=self.max_new_tokens, temperature=self.config.temperature, top_p=self.config.top_p, top_k=self.config.top_k, diff --git a/src/miniverl/training/trainer.py b/src/miniverl/training/trainer.py index dfe9709..91a219a 100644 --- a/src/miniverl/training/trainer.py +++ b/src/miniverl/training/trainer.py @@ -2315,6 +2315,8 @@ def _run_cycle(self) -> list[dict[str, Any]]: cycle_started = time.perf_counter() rollout_policy_version = self.parameter_version self._last_selection_stats = [] + rollout_seconds = 0.0 + teacher_scoring_seconds = 0.0 if mode is TrainingMode.OFFLINE_KD and self._offline_samples is not None: samples = self._offline_batch_for_cycle() @@ -2355,6 +2357,7 @@ def _run_cycle(self) -> list[dict[str, Any]]: if mode is TrainingMode.SFT or self.teacher is None: samples = self._build_samples_ce_only(trajectories) else: + teacher_scoring_started = time.perf_counter() swap = self.plan.strategy is MemoryStrategy.SWAP student_state = None if swap: @@ -2370,6 +2373,7 @@ def _run_cycle(self) -> list[dict[str, Any]]: self.student.load_trainable_state_dict(student_state) if swap: samples = self._reload_targets_from_cache(samples) + teacher_scoring_seconds = time.perf_counter() - teacher_scoring_started self.events.emit( "rollouts_collected", @@ -2389,6 +2393,8 @@ def _run_cycle(self) -> list[dict[str, Any]]: ), generated_tokens=stats.generated_tokens, rollout_tokens_per_second=round(stats.generated_tokens / rollout_seconds, 2), + rollout_seconds=round(rollout_seconds, 4), + teacher_scoring_seconds=round(teacher_scoring_seconds, 4), ) if mode is TrainingMode.OFFLINE_KD: self._offline_samples = samples @@ -2430,6 +2436,10 @@ def _run_cycle(self) -> list[dict[str, Any]]: selection_stats = aggregate_selection_stats( self._last_selection_stats or [sample.selection.stats for sample in samples] ) + selected_for_rate = selection_stats.get("selected_model_tokens") + selected_count = ( + int(selected_for_rate) if isinstance(selected_for_rate, (int, float)) else 0 + ) cycle_metrics: dict[str, Any] = { "phase": f"{config.run.mode.value}_cycle", "cycle": self.cycle, @@ -2440,6 +2450,16 @@ def _run_cycle(self) -> list[dict[str, Any]]: "policy_version": self.policy_version, "rollout_policy_version": rollout_policy_version, "seconds": round(time.perf_counter() - cycle_started, 3), + "rollout_seconds": round(rollout_seconds, 4), + "teacher_scoring_seconds": round(teacher_scoring_seconds, 4), + "teacher_scored_positions_per_second": ( + round( + selected_count / teacher_scoring_seconds, + 2, + ) + if teacher_scoring_seconds > 0 + else None + ), "rollouts": stats.to_dict(), "selection": selection_stats, "memory": gpu.snapshot().to_dict(), diff --git a/tests/cli/test_opd_product_cli.py b/tests/cli/test_opd_product_cli.py new file mode 100644 index 0000000..b82754c --- /dev/null +++ b/tests/cli/test_opd_product_cli.py @@ -0,0 +1,65 @@ +from __future__ import annotations + +import json + +import pytest +from typer.testing import CliRunner + +from miniverl.cli import app + +runner = CliRunner() + + +def test_plan_builtin_is_a_no_network_compiler_smoke() -> None: + result = runner.invoke( + app, + [ + "plan", + "--profile", + "verl-opd-v0.8-single-gpu-v1", + "--config", + "builtin:qwen3-0.6b-1.7b-opd", + "--offline", + "--json", + ], + ) + assert result.exit_code == 0, result.output + payload = json.loads(result.stdout) + assert payload["executable"] is True + assert payload["memory"]["status"] == "estimated" + + +def test_run_dry_compiles_a_valid_native_recipe() -> None: + result = runner.invoke( + app, + [ + "run", + "--profile", + "verl-opd-v0.8-single-gpu-v1", + "--config", + "builtin:qwen3-0.6b-1.7b-opd", + "--dry-run", + "--offline", + "--json", + ], + ) + assert result.exit_code == 0, result.output + payload = json.loads(result.stdout) + assert payload["resolved_native_config"]["source"]["kind"] == "verl_parquet" + assert payload["resolved_native_config"]["loss"]["mode"] == "forward_kl_topk" + + +def test_data_sample_writes_portable_message_rows(tmp_path) -> None: + pytest.importorskip("pyarrow") + import pyarrow.parquet as pq + + target = tmp_path / "sample.parquet" + result = runner.invoke( + app, ["data", "sample", "--format", "verl-parquet", "--out", str(target)] + ) + assert result.exit_code == 0, result.output + rows = pq.read_table(target).to_pylist() + assert len(rows) == 4 + assert rows[0]["prompt"][1]["role"] == "user" + assert rows[0]["data_source"] == "miniverl_quickstart" + assert "reward_model" not in rows[0] diff --git a/tests/unit/test_opd_runtime_plan.py b/tests/unit/test_opd_runtime_plan.py new file mode 100644 index 0000000..23550a4 --- /dev/null +++ b/tests/unit/test_opd_runtime_plan.py @@ -0,0 +1,56 @@ +from __future__ import annotations + +from pathlib import Path + +import pytest + +from miniverl.bridge.opd_runtime import build_system_plan, compile_native_run_config +from miniverl.bridge.opd_v08 import compile_verl_opd_v08, load_verl_opd_v08_source +from miniverl.errors import ConfigError + + +def test_builtin_plan_is_weight_free_truthful_and_executable() -> None: + compiled = load_verl_opd_v08_source("builtin:qwen3-0.6b-1.7b-opd") + plan = build_system_plan(compiled) + + assert plan.executable is True + assert plan.upstream["commit"] == "7aed6b230776f963fa09509c10d9c3a767d1102c" + assert plan.student["revision"] == "c1899de289a04d12100db370d81485cdf75e47ca" + assert plan.teacher["revision"] == "70d244cc86ccca08cf5af4e1e306ecf908b1ad5e" + assert plan.memory["status"] == "estimated" + assert plan.memory["measured_peak_reserved_gib"] is None + assert plan.time_to_first_update == {"status": "unknown", "seconds": None} + assert plan.local_execution["distributed_execution"] is False + assert plan.unsupported_fields == [] + + +def test_native_compilation_preserves_pure_opd_semantics() -> None: + compiled = load_verl_opd_v08_source("builtin:qwen3-0.6b-1.7b-opd") + native = compile_native_run_config(compiled) + + assert native.source.train_files == ["data/opd-smoke.parquet"] + assert native.models.student.revision == "c1899de289a04d12100db370d81485cdf75e47ca" + assert native.models.teacher.revision == "70d244cc86ccca08cf5af4e1e306ecf908b1ad5e" + assert native.loss.mode.value == "forward_kl_topk" + assert native.loss.aggregation.value == "token-mean" + assert native.loss.sampled_token_nll_weight == 0.0 + assert native.train.cycles == 1 + assert native.eval.enabled is False + + +def test_shared_backbone_fails_for_distinct_model_pair() -> None: + compiled = load_verl_opd_v08_source( + "builtin:qwen3-0.6b-1.7b-opd", overrides=["miniverl.runtime.mode=shared_backbone"] + ) + with pytest.raises(ConfigError, match="same base model"): + build_system_plan(compiled) + + +def test_unknown_model_sizes_make_auto_fail_safe_to_swap(tmp_path: Path) -> None: + source = load_verl_opd_v08_source("builtin:qwen3-0.6b-1.7b-opd").source.model_dump(mode="json") + source["actor_rollout_ref"]["model"]["path"] = "owner/student" + source["distillation"]["teacher_models"]["teacher_model"]["model_path"] = "owner/teacher" + compiled = compile_verl_opd_v08(source) + plan = build_system_plan(compiled) + assert plan.local_execution["strategy"] == "swap" + assert "unknown" in plan.local_execution["reason"] diff --git a/tests/unit/test_prompt_rollout_runtime.py b/tests/unit/test_prompt_rollout_runtime.py index 997b3f3..45bd9db 100644 --- a/tests/unit/test_prompt_rollout_runtime.py +++ b/tests/unit/test_prompt_rollout_runtime.py @@ -100,6 +100,38 @@ def test_prompt_trajectories_never_select_prompt_or_padding_tokens() -> None: assert trajectory.metadata["response_token_count"] == len(output.token_ids) +def test_prompt_runtime_enforces_independent_response_bound() -> None: + tokenizer = ToyTokenizer() + backend = ToyBackend(tokenizer=tokenizer, model_id="toy", seed=7, trainable=False) + runtime = PromptDatasetRolloutRuntime( + backend=backend, + source_config=VerlParquetSourceConfig( + train_files=["unused.parquet"], + allow_plain_string_prompts=True, + max_response_length=3, + ), + rollout_config=RolloutConfig( + max_new_tokens_per_turn=20, + max_total_tokens=64, + temperature=0.0, + max_padded_tokens=64, + ), + ) + prompt_ids = tuple(tokenizer.encode("short")) + rendered = RenderedPrompt( + record=_record(0), + text="short", + token_ids=prompt_ids, + tokenizer_identity=tokenizer.identity, + rendered_prompt_digest="f" * 64, + prompt_token_count=len(prompt_ids), + truncation_decision="not_needed", + original_prompt_token_count=len(prompt_ids), + ) + generated = runtime.generate(runtime.prepare_batch([rendered]), policy_version=0, seed=1) + assert len(generated.outputs[0].token_ids) == 3 + + def test_physical_batches_respect_padded_token_budget_without_reordering() -> None: tokenizer = ToyTokenizer() backend = ToyBackend(tokenizer=tokenizer, model_id="toy", seed=7, trainable=False) From d441c458ab5e30a939352e64391b6f875b489c99 Mon Sep 17 00:00:00 2001 From: Daoyuan Li <94409450+DaoyuanLi2816@users.noreply.github.com> Date: Wed, 12 Aug 2026 00:33:37 -0700 Subject: [PATCH 2/4] Record resume timing separately --- src/miniverl/cli.py | 24 +++++++++++++++++++----- 1 file changed, 19 insertions(+), 5 deletions(-) diff --git a/src/miniverl/cli.py b/src/miniverl/cli.py index 0429bf8..954b1d7 100644 --- a/src/miniverl/cli.py +++ b/src/miniverl/cli.py @@ -1039,9 +1039,21 @@ def verl_run_command( ) result = trainer.train() paths = trainer.paths - cycle_rows = [ - row for row in read_jsonl(paths.metrics) if row.get("phase") == "opd_cycle" - ] + if resume is not None: + measurements = { + "schema_version": 1, + "status": "measured", + "resume_load_seconds": round(construction_seconds, 4), + "resume_from": str(resume), + "global_optimizer_step": result.global_step, + "distributed_execution": False, + } + write_json_atomic(paths.root / "verl-resume-measurements.json", measurements) + cycle_rows = [] + else: + cycle_rows = [ + row for row in read_jsonl(paths.metrics) if row.get("phase") == "opd_cycle" + ] first_cycle = cycle_rows[0] if cycle_rows else {} final_metrics = result.final_metrics checkpoint = paths.checkpoints / "final" @@ -1050,7 +1062,7 @@ def verl_run_command( ) run_bytes = sum(item.stat().st_size for item in paths.root.rglob("*") if item.is_file()) adapter_file = checkpoint / "adapter.safetensors" - measurements = { + fresh_measurements = { "schema_version": 1, "status": "measured", "hardware": { @@ -1096,7 +1108,9 @@ def verl_run_command( "runtime_correctness_only": True, "alignment_quality_claim": False, } - write_json_atomic(paths.root / "verl-reference-measurements.json", measurements) + if resume is None: + measurements = fresh_measurements + write_json_atomic(paths.root / "verl-reference-measurements.json", measurements) except (MiniVerlError, ModuleNotFoundError, ValidationError) as exc: _fail(exc) return From eeb257d219fe5a17a9b97b15e6bd3b3be8dd44cd Mon Sep 17 00:00:00 2001 From: Daoyuan Li <94409450+DaoyuanLi2816@users.noreply.github.com> Date: Wed, 12 Aug 2026 00:37:19 -0700 Subject: [PATCH 3/4] Publish measured RTX 4080 OPD quickstart --- PROJECT_STATE.md | 20 ++++++ .../results/rtx4080-verl-opd-runtime-v1.json | 70 +++++++++++++++++++ docs/opd-quickstart.md | 47 +++++++++++++ mkdocs.yml | 1 + src/miniverl/cli.py | 37 ++++++++++ tests/unit/test_opd_runtime_evidence.py | 31 ++++++++ 6 files changed, 206 insertions(+) create mode 100644 benchmarks/results/rtx4080-verl-opd-runtime-v1.json create mode 100644 docs/opd-quickstart.md create mode 100644 tests/unit/test_opd_runtime_evidence.py diff --git a/PROJECT_STATE.md b/PROJECT_STATE.md index b6de58b..c190fe9 100644 --- a/PROJECT_STATE.md +++ b/PROJECT_STATE.md @@ -64,6 +64,26 @@ response or implementation fails closed. The prompt-data integration exercises rollout, exact-state scoring, cache reload, one token-mean optimizer update and the emitted verl diagnostics. +## v0.8.0 single-GPU verl OPD pivot — PR D + +`miniverl plan` now compiles a path or the packaged +`builtin:qwen3-0.6b-1.7b-opd` profile without importing torch or allocating +weights. Estimated memory, disk and placement stay distinct from measured and +unknown values. Auto placement uses parsed model-scale metadata and configured +headroom, never a GPU-name allowlist; unknown sizes conservatively select swap. +`miniverl run` validates a generated native `RunConfig`, preserves the source, +field report and local plan in the run, and executes the supported prompt OPD +path. `miniverl data sample` makes reward-free verl-style message Parquet from +an installed wheel. + +One RTX 4080 runtime-conformance run at `d441c45` used pinned Qwen3-0.6B / +Qwen3-1.7B NF4 snapshots, two prompts, 16 response tokens each, top-k 32 and +one optimizer update. Peak reserved memory was 3.1758 GiB; first rollout, +teacher target batch and update completed at 10.8684 s, 11.0630 s and 12.0224 s +from construction start. The standard PEFT adapter load passed. This is a +systems result only; no alignment endpoint or method comparison ran. The +machine-readable record is `benchmarks/results/rtx4080-verl-opd-runtime-v1.json`. + ## v0.7.1 Product correction — RELEASE CANDIDATE Branch `v0.7.1-product-correction` starts from synchronized main diff --git a/benchmarks/results/rtx4080-verl-opd-runtime-v1.json b/benchmarks/results/rtx4080-verl-opd-runtime-v1.json new file mode 100644 index 0000000..0d57bef --- /dev/null +++ b/benchmarks/results/rtx4080-verl-opd-runtime-v1.json @@ -0,0 +1,70 @@ +{ + "schema_version": 1, + "kind": "single_gpu_runtime_conformance", + "status": "measured", + "measured_at": "2026-08-12T07:34:15Z", + "miniverl_version": "0.8.0.dev0", + "source_commit": "d441c458ab5e30a939352e64391b6f875b489c99", + "profile": "verl-opd-v0.8-single-gpu-v1", + "verl": { + "tag": "v0.8.0", + "commit": "7aed6b230776f963fa09509c10d9c3a767d1102c", + "distributed_execution_tested": false + }, + "hardware": { + "gpu": "NVIDIA GeForce RTX 4080", + "gpu_count": 1, + "vram_gib": 15.992, + "driver": "596.49", + "torch": "2.13.0+cu130" + }, + "models": { + "student": { + "id": "Qwen/Qwen3-0.6B", + "revision": "c1899de289a04d12100db370d81485cdf75e47ca", + "quantization": "nf4", + "adapter": "lora-r8-alpha16" + }, + "teacher": { + "id": "Qwen/Qwen3-1.7B", + "revision": "70d244cc86ccca08cf5af4e1e306ecf908b1ad5e", + "quantization": "nf4" + } + }, + "recipe": { + "runtime_strategy": "dual_model_resident", + "prompt_limit": 128, + "response_limit": 16, + "logical_batch": 2, + "rollout_physical_batch": 2, + "update_physical_batch": 1, + "top_k": 32, + "optimizer_updates": 1, + "compiled_plan_sha256": "a4e7a377e313022766c80098cadd67577743905c9c2a96862696b5afeeabfa8c", + "input_parquet_sha256": "a686aaa602a5a6cd3411de137190779e692806ddedd03f4d728e808539800c54" + }, + "measurements": { + "peak_allocated_gib": 2.1713, + "peak_reserved_gib": 3.1758, + "time_to_first_rollout_seconds": 10.8684, + "time_to_first_teacher_target_batch_seconds": 11.063, + "time_to_first_optimizer_update_seconds": 12.0224, + "rollout_tokens_per_second": 23.39, + "teacher_scored_positions_per_second": 164.44, + "update_selected_positions_per_second": 41.2463, + "checkpoint_bytes": 27649321, + "run_disk_bytes": 27757228, + "resume_load_seconds": 9.4933 + }, + "artifacts": { + "checkpoint_adapter_sha256": "8c20e2136593c7a37bd084df6324bdd8e937561e63af6c8eec05e5b543589667", + "standard_peft_adapter_sha256": "595046de169790870bc87d6a04f043ebab0ed94bee2ab7bc37db8cf30665ef58", + "standard_peft_manifest_sha256": "e6d0e7b0ad8b39221775ad0655c3fbb355baa0d659ca3c89927e46030890353c", + "standard_peft_load_verified": true + }, + "scientific_scope": { + "runtime_correctness_only": true, + "alignment_quality_evaluated": false, + "opd_beats_sft_dpo_or_kd_claimed": false + } +} diff --git a/docs/opd-quickstart.md b/docs/opd-quickstart.md new file mode 100644 index 0000000..6985b23 --- /dev/null +++ b/docs/opd-quickstart.md @@ -0,0 +1,47 @@ +# Run verl-style OPD locally + +miniVERL v0.8 supports one pinned, fail-closed subset of verl v0.8: one actor, +one teacher, one generation per prompt, reward-free direct GKD with +`forward_kl_topk`, token-mean aggregation and a LoRA/QLoRA student. It is local +single-GPU execution, not Ray/FSDP or distributed verl execution. + +Install a CUDA build of PyTorch that matches your machine first, then: + +```bash +python -m pip install "miniverl[train,cuda,bridge]" +miniverl data sample --format verl-parquet --out data/opd-smoke.parquet +miniverl plan --profile verl-opd-v0.8-single-gpu-v1 \ + --config builtin:qwen3-0.6b-1.7b-opd +miniverl run --profile verl-opd-v0.8-single-gpu-v1 \ + --config builtin:qwen3-0.6b-1.7b-opd +``` + +The built-in recipe downloads pinned Qwen3-0.6B and Qwen3-1.7B snapshots when +they are not cached. Allow roughly 6 GiB of download/cache space and 0.25 GiB +for run artifacts. The plan command itself is CPU-only and weight-free; use +`--offline` for a zero-network compiler smoke. + +## Measured reference + +| GPU | student / teacher | strategy | limits / top-k | peak reserved | first update | status | +| --- | --- | --- | --- | ---: | ---: | --- | +| RTX 4080 16 GiB | Qwen3-0.6B / Qwen3-1.7B, both NF4 | dual resident | 128 + 16 tokens / 32 | 3.176 GiB | 12.02 s | measured | +| 12 GiB CUDA GPU | same built-in recipe | planner-selected | same | — | — | not measured | +| 24 GiB CUDA GPU | same built-in recipe | planner-selected | same | — | — | not measured | + +The measured run completed one current-policy rollout/scoring/update cycle, +exported a loadable PEFT adapter, and used one RTX 4080. It demonstrates +runtime and artifact correctness only; it did not evaluate alignment quality. +The checksummed record is +[`rtx4080-verl-opd-runtime-v1.json`](https://github.com/DaoyuanLi2816/mini-verl/blob/main/benchmarks/results/rtx4080-verl-opd-runtime-v1.json). + +## What `plan` means + +`plan` reports estimates separately from measurements. Auto placement uses +model metadata plus the configured VRAM headroom; it never branches on a GPU +product name. Unknown model sizes conservatively select swap. `--probe` is +reserved but fails closed in v0.8.0 rather than loading weights unexpectedly. + +Unsupported settings—including policy-gradient OPD, task rewards, reference +KL, multiple teachers, multiple generations, multimodal inputs and every +distributed dimension—are rejected rather than reinterpreted. diff --git a/mkdocs.yml b/mkdocs.yml index 8d237dd..4f604b7 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -39,6 +39,7 @@ plugins: - search nav: - Home: index.md + - Run verl-style OPD: opd-quickstart.md - Start on one GPU: single-gpu-guide.md - Align: - "External Alignment Gate v1: early stop": alignment-external/alignment-external-v1.md diff --git a/src/miniverl/cli.py b/src/miniverl/cli.py index 954b1d7..0dc7ad4 100644 --- a/src/miniverl/cli.py +++ b/src/miniverl/cli.py @@ -997,6 +997,10 @@ def verl_run_command( compiled = load_verl_opd_v08_source(config, overrides=overrides) system_plan = build_system_plan(compiled) native = compile_native_run_config(compiled, system_plan=system_plan) + from miniverl.config.models import VerlParquetSourceConfig + + if not isinstance(native.source, VerlParquetSourceConfig): # pragma: no cover + raise ConfigError("verl OPD compiler produced a non-Parquet native source") if dry_run: payload = { "dry_run": True, @@ -1015,6 +1019,18 @@ def verl_run_command( from miniverl.trainer import OPDTrainer from miniverl.utils.runs import read_jsonl, write_json_atomic + if not as_json: + console.print("[bold]verl-style local OPD[/bold]") + console.print(f" data {_esc(', '.join(native.source.train_files))}") + console.print(f" actor {_esc(native.models.student.model_id)}") + console.print( + f" rollout local HF, n=1, max response " + f"{_esc(native.source.max_response_length)}" + ) + console.print(f" teacher {_esc(native.models.teacher.model_id)}") + console.print(" distill forward_kl_topk / token-mean / no reward") + console.print(f" trainer {_esc(native.train.cycles)} optimizer update(s)") + console.print(f" placement {_esc(system_plan.local_execution['strategy'])}") construction_started = time.perf_counter() trainer_instance = OPDTrainer.from_config( native, @@ -1111,6 +1127,27 @@ def verl_run_command( if resume is None: measurements = fresh_measurements write_json_atomic(paths.root / "verl-reference-measurements.json", measurements) + if resume is None: + from miniverl.models.adapter_io import export_adapter + + export_started = time.perf_counter() + adapter_manifest, adapter_manifest_path = export_adapter( + paths.root, + paths.checkpoints / "final", + paths.root / "final-peft-adapter", + local_files_only=offline, + ) + measurements["peft_export_seconds"] = round(time.perf_counter() - export_started, 4) + measurements["peft_adapter"] = { + "directory": "final-peft-adapter", + "manifest": adapter_manifest_path.name, + "checksums": adapter_manifest["checksums"], + "load_verified": True, + } + measurements["run_disk_bytes"] = sum( + item.stat().st_size for item in paths.root.rglob("*") if item.is_file() + ) + write_json_atomic(paths.root / "verl-reference-measurements.json", measurements) except (MiniVerlError, ModuleNotFoundError, ValidationError) as exc: _fail(exc) return diff --git a/tests/unit/test_opd_runtime_evidence.py b/tests/unit/test_opd_runtime_evidence.py new file mode 100644 index 0000000..f102b69 --- /dev/null +++ b/tests/unit/test_opd_runtime_evidence.py @@ -0,0 +1,31 @@ +from __future__ import annotations + +import json +from pathlib import Path + + +def test_rtx4080_runtime_record_is_scoped_and_complete() -> None: + path = Path("benchmarks/results/rtx4080-verl-opd-runtime-v1.json") + payload = json.loads(path.read_text(encoding="utf-8")) + + assert payload["kind"] == "single_gpu_runtime_conformance" + assert payload["status"] == "measured" + assert payload["hardware"]["gpu_count"] == 1 + assert payload["verl"]["commit"] == "7aed6b230776f963fa09509c10d9c3a767d1102c" + assert payload["verl"]["distributed_execution_tested"] is False + assert payload["measurements"]["peak_reserved_gib"] <= 14.5 + assert payload["recipe"]["response_limit"] == 16 + assert payload["recipe"]["optimizer_updates"] == 1 + assert payload["artifacts"]["standard_peft_load_verified"] is True + assert payload["scientific_scope"] == { + "runtime_correctness_only": True, + "alignment_quality_evaluated": False, + "opd_beats_sft_dpo_or_kd_claimed": False, + } + + +def test_unmeasured_hardware_rows_are_not_promoted_to_measurements() -> None: + text = Path("docs/opd-quickstart.md").read_text(encoding="utf-8") + assert "12 GiB CUDA GPU" in text and "not measured" in text + assert "24 GiB CUDA GPU" in text and "not measured" in text + assert "demonstrates\nruntime and artifact correctness only" in text From 5dc5473632001d5ab84dc848073cb84663a0edcb Mon Sep 17 00:00:00 2001 From: Daoyuan Li <94409450+DaoyuanLi2816@users.noreply.github.com> Date: Wed, 12 Aug 2026 00:49:45 -0700 Subject: [PATCH 4/4] Preserve legacy cycle metric contracts --- src/miniverl/training/trainer.py | 25 +++++++++++++++---------- tests/cli/test_cli.py | 3 +++ tests/unit/test_packaging.py | 6 ++++-- 3 files changed, 22 insertions(+), 12 deletions(-) diff --git a/src/miniverl/training/trainer.py b/src/miniverl/training/trainer.py index 91a219a..8dcf691 100644 --- a/src/miniverl/training/trainer.py +++ b/src/miniverl/training/trainer.py @@ -2450,21 +2450,26 @@ def _run_cycle(self) -> list[dict[str, Any]]: "policy_version": self.policy_version, "rollout_policy_version": rollout_policy_version, "seconds": round(time.perf_counter() - cycle_started, 3), - "rollout_seconds": round(rollout_seconds, 4), - "teacher_scoring_seconds": round(teacher_scoring_seconds, 4), - "teacher_scored_positions_per_second": ( - round( - selected_count / teacher_scoring_seconds, - 2, - ) - if teacher_scoring_seconds > 0 - else None - ), "rollouts": stats.to_dict(), "selection": selection_stats, "memory": gpu.snapshot().to_dict(), "ts": utc_now(), } + # These phase timings describe the portable prompt-data runtime added + # for the verl-style OPD path. Keep the established environment-backed + # metric contract byte-stable so resume comparisons remain exact. + if self.prompt_dataset is not None: + cycle_metrics.update( + { + "rollout_seconds": round(rollout_seconds, 4), + "teacher_scoring_seconds": round(teacher_scoring_seconds, 4), + "teacher_scored_positions_per_second": ( + round(selected_count / teacher_scoring_seconds, 2) + if teacher_scoring_seconds > 0 + else None + ), + } + ) if self._cache is not None: cycle_metrics["cache"] = self._cache.stats().model_dump(mode="json") self.metrics_log.write(cycle_metrics) diff --git a/tests/cli/test_cli.py b/tests/cli/test_cli.py index 6e1f74a..77254fc 100644 --- a/tests/cli/test_cli.py +++ b/tests/cli/test_cli.py @@ -39,6 +39,7 @@ #: Every documented command, spelled exactly as it is typed. EXPECTED_COMMANDS = { + "data sample", "align", "alignment-suite prepare", "alignment-suite validate", @@ -56,9 +57,11 @@ "export-verl", "prepare-offline-kd", "pilot", + "plan", "qualify-teacher", "inspect", "report", + "run", "export-adapter", "export-benchmark", "schema", diff --git a/tests/unit/test_packaging.py b/tests/unit/test_packaging.py index e34e845..14b46ac 100644 --- a/tests/unit/test_packaging.py +++ b/tests/unit/test_packaging.py @@ -278,8 +278,9 @@ def test_every_published_benchmark_result_validates_against_the_schema(): validator = jsonschema.Draft202012Validator(schema) # RecoveryBench's paired analysis, consumer-runtime, Alignment Lab and - # external-alignment early-stop artifacts have dedicated exact/schema - # contracts; none is a BenchmarkResult document. + # external-alignment early-stop and single-GPU runtime-conformance + # artifacts have dedicated exact/schema contracts; none is a + # BenchmarkResult document. results = sorted( path for path in (root / "benchmarks" / "results").glob("*.json") @@ -291,6 +292,7 @@ def test_every_published_benchmark_result_validates_against_the_schema(): "alignment-lab-v1.json", "alignment-lab-v1-state-supervision.json", "alignment-external-v1.json", + "rtx4080-verl-opd-runtime-v1.json", } ) assert results, "benchmarks/results/ has no published result to validate"