diff --git a/EVAL_HARNESS_README.md b/EVAL_HARNESS_README.md index bfb2f82..7fb10c1 100644 --- a/EVAL_HARNESS_README.md +++ b/EVAL_HARNESS_README.md @@ -2,6 +2,8 @@ A single Rust binary combining the Janus project harness with eval-harness behavior-regression testing. No external dependencies — uses the user's current model context for LLM evaluation. +> ⚗️ **Experimental / opt-in.** The eval-harness (`harness-cli eval …`, all 14 commands) is **not** part of the default build. It is gated behind the `eval` cargo feature and compiles only with `cargo build --release --features eval` (or `cargo install --features eval`). The default binary ships the 9 quality gates without this eval layer. Everything documented below assumes a build with `--features eval`. + --- ## Bắt đầu nhanh (cho người mới) diff --git a/README.md b/README.md index f2bf440..e56932d 100644 --- a/README.md +++ b/README.md @@ -45,7 +45,7 @@ all backed by one SQLite database. | | | |---|---| | 🛡️ **9 quality gates** | Pre‑flight → AC extraction → recall → matrix → evidence → review → push permission → Jira tier → reconciliation, scaled by risk lane. | -| 🎯 **14 eval commands** | `run · baseline · diff · status · promote · trend · accept · apply · ab · rebaseline · analyze · suggest · goal · quality`. | +| 🎯 **14 eval commands** _(experimental, opt-in)_ | `run · baseline · diff · status · promote · trend · accept · apply · ab · rebaseline · analyze · suggest · goal · quality`. Not in the default binary — build with `--features eval`. | | 🔬 **9 check kinds** | Shell, JSON‑path, file, transcript grep (±), LLM judge, prompt/response quality, context relevance. | | 🧭 **5 attribution classes** | When an eval regresses, Janus tells you *why*: skill, fixture, model, cross‑skill, or unknown drift. | | ⚡ **Single Rust binary** | 5.4 MB, **zero runtime dependencies** — no bash, no jq, no python. Cold start ≈ 10 ms. | @@ -80,6 +80,9 @@ bash ~/janus/scripts/install-harness.sh --yes # Build from source git clone https://github.com/hoainho/janus.git cd janus && cargo build --release + +# Include the experimental eval harness (opt-in — off by default) +cargo build --release --features eval ``` @@ -190,6 +193,11 @@ flowchart LR ## 🔬 Eval harness +> ⚗️ **Experimental / opt-in.** The eval harness is not compiled into the default +> binary. Build it with `cargo build --release --features eval` (or `cargo install +> --features eval`); without that flag the `harness-cli eval …` subcommand is absent. +> The 9 quality gates above ship in every build — only this eval layer is gated. + The eval harness answers one question on every push: **does this skill still do what it did at baseline?** Pick a mode, run the checks, and on a regression Janus attributes the likely cause instead of just turning red. diff --git a/crates/harness-cli/Cargo.toml b/crates/harness-cli/Cargo.toml index fb1eb17..32a9b7b 100644 --- a/crates/harness-cli/Cargo.toml +++ b/crates/harness-cli/Cargo.toml @@ -5,6 +5,11 @@ edition.workspace = true license.workspace = true repository.workspace = true +[features] +# Experimental, opt-in eval-harness (behavior-regression testing). Not part of +# the default build; enable with `--features eval`. +eval = [] + [dependencies] clap = { version = "4.6.1", features = ["derive"] } rusqlite = { version = "0.39.0", features = ["bundled"] } diff --git a/crates/harness-cli/src/interface.rs b/crates/harness-cli/src/interface.rs index 5e35aee..5adc572 100644 --- a/crates/harness-cli/src/interface.rs +++ b/crates/harness-cli/src/interface.rs @@ -1,4 +1,5 @@ use std::env; +#[cfg(feature = "eval")] use std::io::Write; use std::path::PathBuf; use std::str::FromStr; @@ -52,6 +53,7 @@ enum Command { GateLog(GateLogArgs), Query(QueryArgs), Export(ExportArgs), + #[cfg(feature = "eval")] /// Eval harness commands for behavior-regression testing. Eval(EvalArgs), } @@ -491,12 +493,14 @@ struct ExportStoryArgs { id: String, } +#[cfg(feature = "eval")] #[derive(Args, Debug)] struct EvalArgs { #[command(subcommand)] action: EvalAction, } +#[cfg(feature = "eval")] #[derive(Subcommand, Debug)] enum EvalAction { /// Run eval cases for a skill. @@ -529,6 +533,7 @@ enum EvalAction { Rebaseline(EvalRebaselineArgs), } +#[cfg(feature = "eval")] #[derive(Args, Debug)] struct EvalQualityArgs { #[arg(long)] @@ -537,6 +542,7 @@ struct EvalQualityArgs { case: Option, } +#[cfg(feature = "eval")] #[derive(Args, Debug)] struct EvalAnalyzeArgs { #[arg(long)] @@ -545,12 +551,14 @@ struct EvalAnalyzeArgs { days: i64, } +#[cfg(feature = "eval")] #[derive(Args, Debug)] struct EvalSuggestArgs { #[arg(long)] skill: String, } +#[cfg(feature = "eval")] #[derive(Args, Debug)] struct EvalGoalArgs { #[arg(long)] @@ -563,6 +571,7 @@ struct EvalGoalArgs { target: String, } +#[cfg(feature = "eval")] #[derive(Args, Debug)] struct EvalRunArgs { #[arg(long)] @@ -583,30 +592,35 @@ struct EvalRunArgs { eval_type: String, } +#[cfg(feature = "eval")] #[derive(Args, Debug)] struct EvalBaselineArgs { #[arg(long)] skill: String, } +#[cfg(feature = "eval")] #[derive(Args, Debug)] struct EvalDiffArgs { #[arg(long)] run_id: String, } +#[cfg(feature = "eval")] #[derive(Args, Debug)] struct EvalStatusArgs { #[arg(long)] skill: Option, } +#[cfg(feature = "eval")] #[derive(Args, Debug)] struct EvalPromoteArgs { #[arg(long)] skill: String, } +#[cfg(feature = "eval")] #[derive(Args, Debug)] struct EvalTrendArgs { #[arg(long)] @@ -615,6 +629,7 @@ struct EvalTrendArgs { last: usize, } +#[cfg(feature = "eval")] #[derive(Args, Debug)] struct EvalAcceptArgs { #[arg(long)] @@ -623,6 +638,7 @@ struct EvalAcceptArgs { case: String, } +#[cfg(feature = "eval")] #[derive(Args, Debug)] struct EvalApplyArgs { #[arg(long)] @@ -631,6 +647,7 @@ struct EvalApplyArgs { run_id: Option, } +#[cfg(feature = "eval")] #[derive(Args, Debug)] struct EvalAbArgs { #[arg(long)] @@ -639,6 +656,7 @@ struct EvalAbArgs { skill_b: String, } +#[cfg(feature = "eval")] #[derive(Args, Debug)] struct EvalRebaselineArgs { #[arg(long)] @@ -942,6 +960,7 @@ pub fn run(cli: Cli) -> Result<(), InterfaceError> { print_export_story_md(&service.query_export_story(&args.id)?) } }, + #[cfg(feature = "eval")] Command::Eval(args) => match args.action { EvalAction::Run(args) => { run_eval(&args)?; @@ -991,6 +1010,7 @@ pub fn run(cli: Cli) -> Result<(), InterfaceError> { Ok(()) } +#[cfg(feature = "eval")] fn run_eval(args: &EvalRunArgs) -> Result<(), InterfaceError> { let skills_root = resolve_skills_root(); let skill_dir = skills_root.join(&args.skill); @@ -1393,6 +1413,7 @@ fn run_eval(args: &EvalRunArgs) -> Result<(), InterfaceError> { Ok(()) } +#[cfg(feature = "eval")] fn get_state_dir() -> std::path::PathBuf { if let Ok(dir) = std::env::var("EVAL_STATE_DIR") { return std::path::PathBuf::from(dir); @@ -1405,6 +1426,7 @@ fn get_state_dir() -> std::path::PathBuf { std::path::PathBuf::from(".opencode/eval-harness") } +#[cfg(feature = "eval")] fn get_home_dir() -> Option { if let Ok(home) = std::env::var("HOME") { return Some(std::path::PathBuf::from(home)); @@ -1418,6 +1440,7 @@ fn get_home_dir() -> Option { None } +#[cfg(feature = "eval")] fn run_baseline(args: &EvalBaselineArgs) -> Result<(), InterfaceError> { let skills_root = resolve_skills_root(); let skill_dir = skills_root.join(&args.skill); @@ -1519,6 +1542,7 @@ fn run_baseline(args: &EvalBaselineArgs) -> Result<(), InterfaceError> { Ok(()) } +#[cfg(feature = "eval")] fn run_diff(_args: &EvalDiffArgs) -> Result<(), InterfaceError> { let state_dir = get_state_dir(); let db_path = state_dir.join("harness.db"); @@ -1604,6 +1628,7 @@ fn run_diff(_args: &EvalDiffArgs) -> Result<(), InterfaceError> { Ok(()) } +#[cfg(feature = "eval")] fn run_status(args: &EvalStatusArgs) -> Result<(), InterfaceError> { let state_dir = get_state_dir(); let db_path = state_dir.join("harness.db"); @@ -1658,6 +1683,7 @@ fn run_status(args: &EvalStatusArgs) -> Result<(), InterfaceError> { Ok(()) } +#[cfg(feature = "eval")] fn run_promote(args: &EvalPromoteArgs) -> Result<(), InterfaceError> { let state_dir = get_state_dir(); let db_path = state_dir.join("harness.db"); @@ -1713,6 +1739,7 @@ fn run_promote(args: &EvalPromoteArgs) -> Result<(), InterfaceError> { Ok(()) } +#[cfg(feature = "eval")] fn run_analyze(args: &EvalAnalyzeArgs) -> Result<(), InterfaceError> { let state_dir = get_state_dir(); let db_path = state_dir.join("harness.db"); @@ -1805,6 +1832,7 @@ fn run_analyze(args: &EvalAnalyzeArgs) -> Result<(), InterfaceError> { Ok(()) } +#[cfg(feature = "eval")] fn run_suggest(args: &EvalSuggestArgs) -> Result<(), InterfaceError> { let state_dir = get_state_dir(); let db_path = state_dir.join("harness.db"); @@ -1869,6 +1897,7 @@ fn run_suggest(args: &EvalSuggestArgs) -> Result<(), InterfaceError> { Ok(()) } +#[cfg(feature = "eval")] fn resolve_skills_root() -> std::path::PathBuf { if let Ok(root) = std::env::var("OPENCODE_SKILLS_ROOT") { return std::path::PathBuf::from(root); @@ -2677,6 +2706,7 @@ fn print_row(values: &[String], widths: &[usize]) { println!(); } +#[cfg(feature = "eval")] fn run_goal(args: &EvalGoalArgs) -> Result<(), InterfaceError> { let state_dir = get_state_dir(); let db_path = state_dir.join("harness.db"); @@ -2715,6 +2745,7 @@ fn run_goal(args: &EvalGoalArgs) -> Result<(), InterfaceError> { Ok(()) } +#[cfg(feature = "eval")] fn run_quality(args: &EvalQualityArgs) -> Result<(), InterfaceError> { let state_dir = get_state_dir(); let db_path = state_dir.join("harness.db"); @@ -2778,6 +2809,7 @@ fn run_quality(args: &EvalQualityArgs) -> Result<(), InterfaceError> { Ok(()) } +#[cfg(feature = "eval")] fn run_trend(args: &EvalTrendArgs) -> Result<(), InterfaceError> { let state_dir = get_state_dir(); let db_path = state_dir.join("harness.db"); @@ -2834,6 +2866,7 @@ fn run_trend(args: &EvalTrendArgs) -> Result<(), InterfaceError> { Ok(()) } +#[cfg(feature = "eval")] fn run_accept(args: &EvalAcceptArgs) -> Result<(), InterfaceError> { let state_dir = get_state_dir(); let db_path = state_dir.join("harness.db"); @@ -2903,6 +2936,7 @@ fn run_accept(args: &EvalAcceptArgs) -> Result<(), InterfaceError> { Ok(()) } +#[cfg(feature = "eval")] fn run_apply(args: &EvalApplyArgs) -> Result<(), InterfaceError> { let state_dir = get_state_dir(); let db_path = state_dir.join("harness.db"); @@ -2969,6 +3003,7 @@ fn run_apply(args: &EvalApplyArgs) -> Result<(), InterfaceError> { Ok(()) } +#[cfg(feature = "eval")] fn run_ab(args: &EvalAbArgs) -> Result<(), InterfaceError> { let state_dir = get_state_dir(); let db_path = state_dir.join("harness.db"); @@ -3069,6 +3104,7 @@ fn run_ab(args: &EvalAbArgs) -> Result<(), InterfaceError> { Ok(()) } +#[cfg(feature = "eval")] fn run_rebaseline(args: &EvalRebaselineArgs) -> Result<(), InterfaceError> { println!("[eval-harness] Rebaselining skill '{}'...", args.skill); run_baseline(&EvalBaselineArgs { diff --git a/crates/harness-cli/src/main.rs b/crates/harness-cli/src/main.rs index 6884951..c1d7a8b 100644 --- a/crates/harness-cli/src/main.rs +++ b/crates/harness-cli/src/main.rs @@ -1,5 +1,6 @@ mod application; mod domain; +#[cfg(feature = "eval")] mod eval; mod infrastructure; mod interface;