Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -16,3 +16,7 @@ scripts/__pycache__/
scripts/recall_health.json
.omc/
.omc/

# Python bytecode from capture hook.
__pycache__/
*.pyc
9 changes: 7 additions & 2 deletions crates/harness-cli/src/application.rs
Original file line number Diff line number Diff line change
@@ -1,8 +1,9 @@
use std::path::PathBuf;

use crate::domain::{
AuditResult, BacklogFilter, BacklogRecord, BoolFlag, ContextScoreResult, CsvList,
DecisionRecord, FrictionRecord, GateLogRecord, GcrRecord, HarnessStats, ImprovementProposal,
AuditResult, BacklogFilter, BacklogRecord, BoolFlag, ContextScoreResult, CoverageReport,
CsvList, DecisionRecord, FrictionRecord, GateLogRecord, GcrRecord, HarnessStats,
ImprovementProposal,
InputType, IntakeRecord, InterventionRecord, RiskLane, StoryExportRecord, StoryMatrixRecord,
StoryVerifyAllResult, StoryVerifyStatus, ToolArgSpec, ToolEntry, TraceRecord, TraceScoreResult,
};
Expand Down Expand Up @@ -328,6 +329,10 @@ impl HarnessService {
pub fn query_gcr(&self) -> crate::infrastructure::Result<Vec<GcrRecord>> {
self.repository.query_gcr()
}

pub fn query_coverage(&self) -> crate::infrastructure::Result<CoverageReport> {
self.repository.query_coverage()
}
}

#[derive(Debug, PartialEq, Eq)]
Expand Down
23 changes: 23 additions & 0 deletions crates/harness-cli/src/domain.rs
Original file line number Diff line number Diff line change
Expand Up @@ -1204,6 +1204,29 @@ pub struct GcrRecord {
pub rag: String,
}

/// Adoption + instrumentation coverage snapshot for `query coverage`.
///
/// Answers the question the raw stats can't: are sessions actually being
/// captured, and do the traces carry the token/duration telemetry that makes
/// GCR/ROI meaningful. Distinct from GCR, which scores *how well* a story
/// passed its gates — this scores *whether the durable layer is being fed at
/// all*.
#[derive(Debug, Clone)]
pub struct CoverageReport {
pub total_stories: i64,
pub stories_with_trace: i64,
pub verified_stories: i64,
pub total_traces: i64,
pub traces_with_token: i64,
pub traces_with_duration: i64,
pub total_tokens: i64,
}

/// Minimum number of stories before a green/yellow/red verdict is
/// statistically meaningful. Below this, GCR and coverage report `grey
/// (insufficient data)` instead of a falsely reassuring colour.
pub const MIN_STORIES_FOR_RAG: usize = 10;

/// Full story projection used by `export matrix` and `export story`.
/// Extends the proof columns from [`StoryMatrixRecord`] with the lane,
/// last verify result, T4 verdict, and T4 notes — all needed for
Expand Down
32 changes: 31 additions & 1 deletion crates/harness-cli/src/infrastructure.rs
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,8 @@ use crate::application::{
use crate::domain::{
compiled_tool_registry, normalize_token, score_context, score_trace, validate_tool_description,
AuditFinding, AuditResult, BacklogFilter, BacklogRecord, ContextScoreResult,
ContextScoreSource, DecisionRecord, FrictionRecord, GateLogRecord, GcrRecord, HarnessStats,
ContextScoreSource, CoverageReport, DecisionRecord, FrictionRecord, GateLogRecord, GcrRecord,
HarnessStats,
ImprovementProposal, IntakeRecord, InterventionRecord, RiskLane, StoryExportRecord,
StoryMatrixRecord, StoryVerifyAllItem, StoryVerifyAllResult, StoryVerifyStatus, ToolArgSpec,
ToolEntry, TraceRecord, TraceScoreResult, TraceScoreSource,
Expand Down Expand Up @@ -117,6 +118,8 @@ pub trait HarnessRepository {
fn query_export_story(&self, id: &str) -> Result<StoryExportRecord>;
/// Per-story gate-compliance rate, plus an overall-average summary row.
fn query_gcr(&self) -> Result<Vec<GcrRecord>>;
/// Adoption + instrumentation coverage snapshot.
fn query_coverage(&self) -> Result<CoverageReport>;
}

#[derive(Debug)]
Expand Down Expand Up @@ -1651,6 +1654,33 @@ ORDER BY gcr ASC;";

collect_rows(rows)
}

fn query_coverage(&self) -> Result<CoverageReport> {
let connection = self.open_existing()?;
let sql = "
SELECT
(SELECT COUNT(*) FROM story),
(SELECT COUNT(DISTINCT story_id) FROM trace WHERE story_id IS NOT NULL),
(SELECT COUNT(*) FROM story
WHERE unit_proof = 1 OR integration_proof = 1 OR e2e_proof = 1
OR evidence IS NOT NULL OR last_verified_result = 'pass'),
(SELECT COUNT(*) FROM trace),
(SELECT COUNT(token_estimate) FROM trace),
(SELECT COUNT(duration_seconds) FROM trace),
(SELECT CAST(COALESCE(SUM(token_estimate), 0) AS INTEGER) FROM trace);";
let report = connection.query_row(sql, [], |row| {
Ok(CoverageReport {
total_stories: row.get(0)?,
stories_with_trace: row.get(1)?,
verified_stories: row.get(2)?,
total_traces: row.get(3)?,
traces_with_token: row.get(4)?,
traces_with_duration: row.get(5)?,
total_tokens: row.get(6)?,
})
})?;
Ok(report)
}
}

impl From<HarnessContext> for SqliteHarnessRepository {
Expand Down
89 changes: 86 additions & 3 deletions crates/harness-cli/src/interface.rs
Original file line number Diff line number Diff line change
Expand Up @@ -15,8 +15,9 @@ use crate::application::{
use crate::domain::{
normalize_capability, parse_optional_integer, parse_tool_args, proof_display,
validate_responsibility, validate_tool_kind, BacklogFilter, BacklogRecord, BoolFlag,
ContextScoreResult, CsvList, DecisionRecord, FrictionRecord, GateLogRecord, GcrRecord,
HarnessStats, ImprovementProposal, InputType, IntakeRecord, InterventionRecord, RiskLane,
ContextScoreResult, CoverageReport, CsvList, DecisionRecord, FrictionRecord, GateLogRecord,
GcrRecord, HarnessStats, ImprovementProposal, InputType, IntakeRecord, InterventionRecord,
MIN_STORIES_FOR_RAG, RiskLane,
StoryExportRecord, StoryMatrixRecord, StoryVerifyAllResult, ToolEntry, TraceQualityTier,
TraceRecord, TraceScoreResult, RISK_LANE_HELP,
};
Expand Down Expand Up @@ -441,6 +442,8 @@ enum QueryView {
GateLog,
/// Per-story gate-compliance rate (GCR) from docs/p3-ac6-compliance-metric.md.
Gcr,
/// Adoption + instrumentation coverage snapshot (are sessions being captured?).
Coverage,
/// Run arbitrary SQL.
Sql { query: Vec<String> },
}
Expand Down Expand Up @@ -929,6 +932,7 @@ pub fn run(cli: Cli) -> Result<(), InterfaceError> {
QueryView::Stats => print_stats(&service.query_stats()?),
QueryView::GateLog => print_gate_log(&service.query_gate_log()?),
QueryView::Gcr => print_gcr(&service.query_gcr()?),
QueryView::Coverage => print_coverage(&service.query_coverage()?),
QueryView::Sql { query } => {
if query.is_empty() {
return Err(InterfaceError::EmptySql);
Expand Down Expand Up @@ -2553,7 +2557,13 @@ fn print_gcr(records: &[GcrRecord]) {
let total_expected: i64 = records.iter().map(|record| record.gates_expected).sum();
if total_expected > 0 {
let avg = total_recorded as f64 / total_expected as f64;
let avg_rag = if avg >= 0.85 {
// Data-sufficiency guard: below MIN_STORIES_FOR_RAG a colour is
// misleading (a single well-formed story reads "green" while the
// harness is effectively unused). Report grey instead.
let insufficient = records.len() < MIN_STORIES_FOR_RAG;
let avg_rag = if insufficient {
"grey (insufficient data)"
} else if avg >= 0.85 {
"green"
} else if avg >= 0.50 {
"yellow"
Expand All @@ -2568,6 +2578,79 @@ fn print_gcr(records: &[GcrRecord]) {
avg * 100.0,
avg_rag
);
if insufficient {
println!(
" ⚠ only {} story(ies) tracked (need ≥{} for a meaningful verdict).",
records.len(),
MIN_STORIES_FOR_RAG
);
}
}
}

fn print_coverage(report: &CoverageReport) {
fn pct(part: i64, whole: i64) -> f64 {
if whole == 0 {
0.0
} else {
part as f64 / whole as f64 * 100.0
}
}

let story_cov = pct(report.stories_with_trace, report.total_stories);
let verify_cov = pct(report.verified_stories, report.total_stories);
let token_cov = pct(report.traces_with_token, report.total_traces);
let dur_cov = pct(report.traces_with_duration, report.total_traces);

print_table(
&["metric", "value", "coverage"],
&[
vec![
"stories with ≥1 trace".to_owned(),
format!("{}/{}", report.stories_with_trace, report.total_stories),
format!("{story_cov:.1}%"),
],
vec![
"stories verified".to_owned(),
format!("{}/{}", report.verified_stories, report.total_stories),
format!("{verify_cov:.1}%"),
],
vec![
"traces with token estimate".to_owned(),
format!("{}/{}", report.traces_with_token, report.total_traces),
format!("{token_cov:.1}%"),
],
vec![
"traces with duration".to_owned(),
format!("{}/{}", report.traces_with_duration, report.total_traces),
format!("{dur_cov:.1}%"),
],
vec![
"total tokens captured".to_owned(),
format!("{}", report.total_tokens),
"—".to_owned(),
],
],
);

println!();
if (report.total_stories as usize) < MIN_STORIES_FOR_RAG {
println!(
"Adoption: grey (insufficient data) — {} story(ies), need ≥{}.",
report.total_stories, MIN_STORIES_FOR_RAG
);
} else {
let rag = if token_cov >= 90.0 && story_cov >= 80.0 {
"green"
} else if token_cov >= 50.0 {
"yellow"
} else {
"red"
};
println!(
"Adoption: {} — {:.0}% traces instrumented, {:.0}% stories traced.",
rag, token_cov, story_cov
);
}
}

Expand Down
Loading
Loading