From f7426b89ff0a2e1dfa1ca464495b6e5be073a06a Mon Sep 17 00:00:00 2001 From: Dor Kalev Date: Wed, 5 Aug 2026 14:55:17 +0300 Subject: [PATCH] fix(dashboard): render signed readiness summary Closes #16 --- website/SPEC.md | 4 +- website/app/src/main.rs | 138 +++++++++++++++++++++++++++++++++++--- website/app/src/render.rs | 10 ++- 3 files changed, 137 insertions(+), 15 deletions(-) diff --git a/website/SPEC.md b/website/SPEC.md index 0765917..00c0aa6 100644 --- a/website/SPEC.md +++ b/website/SPEC.md @@ -11,7 +11,7 @@ ## Purpose -One page. One number. A fixed gauge from 0 to 100% ("would you pass a SOC 2 examination today?") above the full 61-criterion auditor checklist **and the machinery section** — the agents, workflows, webhooks, scanners, and registers installed on the project, sourced from [`../procedures/PROCEDURES.md`](../procedures/PROCEDURES.md). The site **renders state; it never computes compliance** — Runbook 03 computes and writes; the site reads. This division keeps the site tiny and keeps the scoring logic in one auditable place (the runbook + criteria files). +One page. One number. A fixed gauge from 0 to 100% ("how completely does the current signed readiness snapshot satisfy its declared design, technical, observation, and operating dimensions?") above the full 61-criterion auditor checklist **and the machinery section** — the agents, workflows, webhooks, scanners, and registers installed on the project, sourced from [`../procedures/PROCEDURES.md`](../procedures/PROCEDURES.md). The gauge is the arithmetic mean of the signed report's design, technical, and operating percentages plus observation coverage (the percentage of checks with a known verdict); `unknown` therefore lowers the score. The site imports and renders the deterministic verifier's signed summary; it does not independently reinterpret criterion credit as the official score. Everything renders on one sheet — no tabs, no view flips (they contradicted the one-pager paradigm and were removed): @@ -107,7 +107,7 @@ Responsive: below 900px the sheet goes single-column (cards stack, header stacks 4. **The Criteria** — the 61-cell checkbox matrix (see above), mirroring [CHECKLIST.md](../CHECKLIST.md) content via hover. 5. **Footer** — last verify run time, count of `unknown` checks ("blind spots"), link to `/db`. -Honest-rendering rules: a stale verify run (>48h) banners the whole page ("state is stale — monitor may be dead"); `unknown` never displays as pass; the gauge is always shown with its computation date, never as a timeless fact. +Honest-rendering rules: a stale verify run (>48h) banners the whole page ("state is stale — monitor may be dead"); `unknown` never displays as pass and lowers observation coverage; the gauge is always shown with its computation date, never as a timeless fact. A perfect dashboard score means the declared machine-verifiable readiness checks all pass with no blind spots; it is not an auditor's opinion or a substitute for a Type II observation period. ## Seeding diff --git a/website/app/src/main.rs b/website/app/src/main.rs index a3211cc..1d78d07 100644 --- a/website/app/src/main.rs +++ b/website/app/src/main.rs @@ -40,6 +40,8 @@ CREATE TABLE IF NOT EXISTS gauge_history ( CREATE TABLE IF NOT EXISTS readiness_history ( ts TEXT PRIMARY KEY, design REAL NOT NULL, technical REAL NOT NULL, evidence REAL NOT NULL, operating REAL NOT NULL); +CREATE TABLE IF NOT EXISTS metadata ( + key TEXT PRIMARY KEY, value TEXT NOT NULL); CREATE TABLE IF NOT EXISTS procedures ( id TEXT PRIMARY KEY, name TEXT NOT NULL, category TEXT NOT NULL, criteria TEXT NOT NULL, install TEXT NOT NULL, detect TEXT NOT NULL, @@ -149,9 +151,30 @@ struct VerifyReport { schema_version: u8, #[serde(default)] observed_at: String, + #[serde(default)] + subject: VerifySubject, + #[serde(default)] + summary: VerifySummary, checks: Vec, } +#[derive(Deserialize, Default)] +struct VerifySubject { + #[serde(default)] + repository: String, +} + +#[derive(Deserialize, Default)] +struct VerifySummary { + #[serde(default)] + dimensions: HashMap, +} + +#[derive(Deserialize)] +struct VerifyDimension { + percent: f64, +} + #[derive(Deserialize)] struct VerifyCheck { id: String, @@ -237,6 +260,57 @@ fn readiness_metrics(conn: &Connection) -> (f64, f64, f64, f64) { (design, technical, evidence, operating) } +fn report_readiness( + report: &VerifyReport, + fallback: (f64, f64, f64, f64), +) -> (f64, f64, f64, f64) { + let dimension = |name: &str, default: f64| { + report + .summary + .dimensions + .get(name) + .map(|d| d.percent.clamp(0.0, 100.0)) + .unwrap_or(default) + }; + let observation_coverage = if report.checks.is_empty() { + fallback.2 + } else { + report + .checks + .iter() + .filter(|check| check.verdict != "unknown") + .count() as f64 + * 100.0 + / report.checks.len() as f64 + }; + ( + dimension("design", fallback.0), + dimension("technical", fallback.1), + observation_coverage, + dimension("operating", fallback.3), + ) +} + +fn readiness_gauge(readiness: (f64, f64, f64, f64)) -> f64 { + (readiness.0 + readiness.1 + readiness.2 + readiness.3) / 4.0 +} + +fn dashboard_subject(conn: &Connection) -> String { + std::env::var("SHADOW_ORG") + .ok() + .filter(|value| !value.trim().is_empty()) + .or_else(|| { + conn.query_row( + "SELECT value FROM metadata WHERE key='subject_repository'", + [], + |row| row.get::<_, String>(0), + ) + .ok() + .filter(|value| !value.trim().is_empty()) + }) + .unwrap_or_else(|| "unnamed subject".into()) +} + fn import_verify(db_path: &str, report_path: &str) { let report: VerifyReport = serde_json::from_str( &std::fs::read_to_string(report_path).expect("read deterministic verify report"), @@ -256,6 +330,14 @@ fn import_verify(db_path: &str, report_path: &str) { let mut procedure_state: HashMap = HashMap::new(); let mut cap_reason = None; + if !report.subject.repository.is_empty() { + conn.execute( + "INSERT OR REPLACE INTO metadata (key, value) VALUES ('subject_repository', ?1)", + [&report.subject.repository], + ) + .expect("record report subject"); + } + // A readiness report is a complete current snapshot. Clearing the render // cache prevents retired checks from surviving forever as ghost evidence. conn.execute("DELETE FROM checks", []).expect("clear old checks"); @@ -324,26 +406,21 @@ fn import_verify(db_path: &str, report_path: &str) { ) .expect("update procedure"); } - let gauge: f64 = conn - .query_row( - "SELECT COALESCE(SUM(weight * credit) * 100.0 / NULLIF(SUM(weight), 0), 0) FROM criteria WHERE in_scope=1", - [], - |r| r.get(0), - ) - .expect("compute gauge"); + let readiness = report_readiness(&report, readiness_metrics(&conn)); + let (design, technical, evidence, operating) = readiness; + let gauge = readiness_gauge(readiness); let cap = cap_reason.as_ref().map(|_| 79.0); conn.execute( "INSERT OR REPLACE INTO gauge_history (ts, gauge, cap, cap_reason) VALUES (?1,?2,?3,?4)", params![ts, gauge, cap, cap_reason], ) .expect("record gauge"); - let (design, technical, evidence, operating) = readiness_metrics(&conn); conn.execute( "INSERT OR REPLACE INTO readiness_history (ts, design, technical, evidence, operating) VALUES (?1,?2,?3,?4,?5)", params![ts, design, technical, evidence, operating], ) .expect("record readiness dimensions"); - println!("imported schema v{}: {check_count} checks; gauge {gauge:.1}%; design {design:.1}%; technical {technical:.1}%; evidence {evidence:.1}%; operating {operating:.1}%", report.schema_version); + println!("imported schema v{}: {check_count} checks; gauge {gauge:.1}%; design {design:.1}%; technical {technical:.1}%; observation coverage {evidence:.1}%; operating {operating:.1}%", report.schema_version); } // ---------- seeding: the markdown corpus is the source of truth ---------- @@ -843,6 +920,7 @@ async fn export_db(State(app): State>, headers: HeaderMap) -> impl Into fn serve(db_path: String, port: u16) { let conn = Connection::open(&db_path).expect("open db"); ensure_schema(&conn); + let org = dashboard_subject(&conn); let runner = match std::env::var("SHADOW_RUNNER") { Ok(cmd) if !cmd.is_empty() => Runner::Shell(cmd), _ => { @@ -858,7 +936,7 @@ fn serve(db_path: String, port: u16) { db: Mutex::new(conn), db_path, token: std::env::var("SHADOW_TOKEN").ok(), - org: std::env::var("SHADOW_ORG").unwrap_or_else(|_| "unnamed org".into()), + org, running: Mutex::new(std::collections::HashSet::new()), runner, criteria_dir: std::env::var("SHADOW_CRITERIA_DIR").unwrap_or_else(|_| "../../criteria".into()), @@ -886,7 +964,7 @@ fn serve(db_path: String, port: u16) { fn render_static(db_path: &str, out: &str) { let conn = Connection::open(db_path).expect("open db"); ensure_schema(&conn); - let org = std::env::var("SHADOW_ORG").unwrap_or_else(|_| "unnamed org".into()); + let org = dashboard_subject(&conn); let model = load_model(&conn, &org); std::fs::create_dir_all(format!("{out}/criteria")).expect("mkdir"); @@ -960,4 +1038,42 @@ mod tests { let conflict = CriterionEvidence { pass_operating: true, failed: true, ..CriterionEvidence::default() }; assert_eq!(status_for_evidence(&conflict), ("failing", 0.0)); } + + #[test] + fn signed_dimension_summary_drives_the_gauge() { + let report: VerifyReport = serde_json::from_str( + r#"{ + "summary":{"dimensions":{ + "design":{"percent":100.0}, + "technical":{"percent":90.0}, + "operating":{"percent":100.0} + }}, + "checks":[ + {"id":"design-control","verdict":"pass"}, + {"id":"technical-control","verdict":"fail"} + ] + }"#, + ) + .unwrap(); + + let readiness = report_readiness(&report, (0.0, 0.0, 0.0, 0.0)); + assert_eq!(readiness, (100.0, 90.0, 100.0, 100.0)); + assert_eq!(readiness_gauge(readiness), 97.5); + } + + #[test] + fn unknown_verdicts_reduce_observation_coverage() { + let report: VerifyReport = serde_json::from_str( + r#"{ + "summary":{"dimensions":{}}, + "checks":[ + {"id":"observed","verdict":"pass"}, + {"id":"blind-spot","verdict":"unknown"} + ] + }"#, + ) + .unwrap(); + + assert_eq!(report_readiness(&report, (1.0, 2.0, 3.0, 4.0)), (1.0, 2.0, 50.0, 4.0)); + } } diff --git a/website/app/src/render.rs b/website/app/src/render.rs index 1da9a6b..c264069 100644 --- a/website/app/src/render.rs +++ b/website/app/src/render.rs @@ -205,7 +205,7 @@ fn readiness_cards(r: &Readiness) -> String { let items = [ ("Design readiness", r.design, "documented controls"), ("Technical health", r.technical, "live automated checks"), - ("Evidence coverage", r.evidence, "criteria with current proof"), + ("Observation coverage", r.evidence, "checks with a known verdict"), ("Operating maturity", r.operating, "controls proven in operation"), ]; let mut s = String::from(r#"
"#); @@ -288,7 +288,13 @@ const MAP: &[MapItem] = &[ label: "Gates", note: "required checks — merging is impossible until all are green; the general controls over technology, deployed", crit: "CC8.1 · CC4.1 · CC5.1 · CC5.2", - pins: &["ci-tests", "review-bot", "compliance-audit-agent", "compliance-review-gate"], + pins: &["ci-tests", "compliance-audit-agent", "compliance-review-gate"], + }, + MapItem::Station { + label: "Optional advisory review", + note: "off by default; a deliberately invoked model review may supplement the deterministic gates, but never satisfies one", + crit: "CC8.1 · CC7.1", + pins: &["review-bot"], }, MapItem::Station { label: "Merge to main",