diff --git a/actions/shadow-ci/src/verify.rs b/actions/shadow-ci/src/verify.rs index defd4db..8cea924 100644 --- a/actions/shadow-ci/src/verify.rs +++ b/actions/shadow-ci/src/verify.rs @@ -728,11 +728,18 @@ fn summary(checks: &[Value]) -> Value { let mut dimensions: BTreeMap = BTreeMap::new(); let mut evidenced = BTreeSet::new(); let mut operating = BTreeSet::new(); + let mut observations = BTreeMap::from([ + ("pass", 0usize), + ("fail", 0usize), + ("unknown", 0usize), + ("not_applicable", 0usize), + ]); for check in checks { let dimension = check["dimension"].as_str().unwrap_or("unknown").to_string(); let entry = dimensions.entry(dimension.clone()).or_default(); match check["verdict"].as_str().unwrap_or("unknown") { "pass" => { + observations.entry("pass").and_modify(|count| *count += 1); entry.0 += 1; if let Some(criteria) = check["criteria"].as_array() { for criterion in criteria.iter().filter_map(Value::as_str) { @@ -743,9 +750,19 @@ fn summary(checks: &[Value]) -> Value { } } } - "fail" => entry.1 += 1, - "n/a" => {} - _ => entry.2 += 1, + "fail" => { + observations.entry("fail").and_modify(|count| *count += 1); + entry.1 += 1; + } + "n/a" => { + observations + .entry("not_applicable") + .and_modify(|count| *count += 1); + } + _ => { + observations.entry("unknown").and_modify(|count| *count += 1); + entry.2 += 1; + } } } let dimensions: BTreeMap = dimensions @@ -763,8 +780,18 @@ fn summary(checks: &[Value]) -> Value { ) }) .collect(); + let total = checks.len(); + let not_applicable = observations["not_applicable"]; json!({ "dimensions": dimensions, + "observations": { + "total": total, + "applicable": total - not_applicable, + "pass": observations["pass"], + "fail": observations["fail"], + "unknown": observations["unknown"], + "not_applicable": not_applicable + }, "criteria_with_evidence": evidenced.len(), "criteria_with_operating_evidence": operating.len() }) @@ -1238,9 +1265,24 @@ pub fn run_verify() -> Result { let unknowns = checks.iter().filter(|c| c["verdict"] == "unknown").count(); let report_summary = summary(&checks); let report = json!({ - "schema_version": 2, + "schema_version": 3, "subject": {"repository": repo, "gcp_projects": gcp_projects}, "observed_at": observed_at, + "assurance": { + "basis": "automated-point-in-time", + "auditor_opinion": false, + "soc2_type_1": "not-determined", + "soc2_type_2": "not-determined", + "note": "Type II requires an elapsed CPA examination period and auditor-selected operating samples." + }, + "provenance": { + "generator": "shadow-ci", + "repository": std::env::var("GITHUB_REPOSITORY").ok(), + "commit": std::env::var("GITHUB_SHA").ok(), + "run_id": std::env::var("GITHUB_RUN_ID").ok(), + "workflow": std::env::var("GITHUB_WORKFLOW").ok(), + "report_signature": "none" + }, "checks": checks, "summary": report_summary, "failures": failures, @@ -1329,6 +1371,10 @@ mod tests { let s = summary(&checks); assert_eq!(s["dimensions"]["design"]["percent"], 100.0); assert_eq!(s["dimensions"]["technical"]["percent"], 0.0); + assert_eq!(s["observations"]["total"], 2); + assert_eq!(s["observations"]["applicable"], 2); + assert_eq!(s["observations"]["pass"], 1); + assert_eq!(s["observations"]["fail"], 1); } #[test] diff --git a/website/SPEC.md b/website/SPEC.md index 00c0aa6..981b34d 100644 --- a/website/SPEC.md +++ b/website/SPEC.md @@ -11,7 +11,9 @@ ## Purpose -One page. One number. A fixed gauge from 0 to 100% ("how completely does the current signed readiness snapshot satisfy its declared design, technical, observation, and operating dimensions?") above the full 61-criterion auditor checklist **and the machinery section** — the agents, workflows, webhooks, scanners, and registers installed on the project, sourced from [`../procedures/PROCEDURES.md`](../procedures/PROCEDURES.md). The gauge is the arithmetic mean of the signed report's design, technical, and operating percentages plus observation coverage (the percentage of checks with a known verdict); `unknown` therefore lowers the score. The site imports and renders the deterministic verifier's signed summary; it does not independently reinterpret criterion credit as the official score. +One page with one primary internal metric above the full 61-criterion checklist **and the machinery section** — the agents, workflows, webhooks, scanners, and registers installed on the project, sourced from [`../procedures/PROCEDURES.md`](../procedures/PROCEDURES.md). The gauge is **weighted, in-scope criterion evidence maturity**: `Σ(weight × credit) / Σ(weight)`, where verified is 100% credit, implemented/design-only is 60%, and failing or not-started is 0%. It is not a probability of passing an examination. + +Automated observation results are rendered separately with an explicit denominator: pass, fail, unknown, and not-applicable. The page always says that it is an automated point-in-time assessment, not a SOC 2 report or CPA opinion; neither Type I nor Type II status is inferred. Type II requires an elapsed examination period and auditor-selected operating samples. Provenance names the repository, commit, workflow run, generator, and whether the JSON report itself has a cryptographic signature. Everything renders on one sheet — no tabs, no view flips (they contradicted the one-pager paradigm and were removed): @@ -63,9 +65,9 @@ CREATE TABLE attestations ( -- manual evidence for organizational criteria CREATE TABLE gauge_history ( -- one row per verify run ts TEXT PRIMARY KEY, - gauge REAL NOT NULL, - cap REAL, -- 79.0 when a hard gate is tripped, else NULL - cap_reason TEXT + gauge REAL NOT NULL, -- weighted in-scope criterion evidence maturity + cap REAL, -- retained for schema compatibility; currently NULL + cap_reason TEXT -- retained for schema compatibility ); CREATE TABLE procedures ( -- the machinery ledger, seeded from procedures/PROCEDURES.md @@ -82,7 +84,7 @@ CREATE TABLE procedures ( -- the machinery ledger, seeded from procedures/P ## Micro board (`/micro`) -The one-pager's dense sibling: one small box per criterion (ID + status glyph, status-colored, category headers, out-of-scope dimmed), the gauge in the corner. **Clicking a box runs that criterion's checks right now**: the click is a form POST (`/run/{id}`, zero JS) that spawns the verifier — the `claude` CLI if found on PATH (built-in single-criterion prompt: execute the criterion file's "Automated shadow checks" table, POST results to `/ingest`), or any command set in `SHADOW_RUNNER` (invoked via `sh -c` with `CRITERION`, `CRITERION_FILE`, `SHADOW_URL` env). While anything runs, boxes pulse ⟳ and the page polls via a meta-refresh. Two honesty rules: single-box runs never write a gauge entry (the official gauge moves only on the full verify), and with no verifier available the board renders read-only and says so. The site still never computes compliance — it triggers the agent that does. +The one-pager's dense sibling: one small box per criterion (ID + status glyph, status-colored, category headers, out-of-scope dimmed), with weighted criterion maturity in the corner. **Clicking a box runs that criterion's checks right now**: the click is a form POST (`/run/{id}`, zero JS) that spawns the verifier — the `claude` CLI if found on PATH (built-in single-criterion prompt: execute the criterion file's "Automated shadow checks" table, POST results to `/ingest`), or any command set in `SHADOW_RUNNER` (invoked via `sh -c` with `CRITERION`, `CRITERION_FILE`, `SHADOW_URL` env). While anything runs, boxes pulse ⟳ and the page polls via a meta-refresh. Two honesty rules: single-box runs never write a gauge entry (the primary metric moves only on the full verify), and with no verifier available the board renders read-only and says so. The site never computes or claims compliance; it records evidence state. ## Routes @@ -101,13 +103,13 @@ Responsive: below 900px the sheet goes single-column (cards stack, header stacks ## The page (top to bottom) -1. **Gauge** — fixed semicircular arc, 0–100%, needle at current gauge. Color bands: 0–49 red, 50–79 amber, 80–94 green, 95–100 deep green. If a hard-gate cap is active, the arc beyond the cap renders hatched with the cap reason underneath ("capped at 79% — org 2FA not enforced"). Below the needle: the trend sparkline from `gauge_history`. -2. **Category chips** — Security 33/33 in scope, Availability, Confidentiality, PI, Privacy — with per-category sub-scores; out-of-scope categories greyed with "not in scope". +1. **Criterion maturity gauge** — fixed semicircular arc, 0–100%, needle at weighted in-scope evidence maturity. The formula and credits are printed beside it. Below the needle: the comparable trend from `gauge_history`; a metric-version migration discards incompatible legacy history. +2. **Evidence summary and category chips** — verified/implemented/not-started/failing criterion counts; applicable automated pass/fail/unknown counts with n/a separated; Security, Availability, Confidentiality, PI, and Privacy weighted sub-scores; out-of-scope categories greyed with "not in scope". 3. **The Machinery** — the ten territory cards (see above). 4. **The Criteria** — the 61-cell checkbox matrix (see above), mirroring [CHECKLIST.md](../CHECKLIST.md) content via hover. 5. **Footer** — last verify run time, count of `unknown` checks ("blind spots"), link to `/db`. -Honest-rendering rules: a stale verify run (>48h) banners the whole page ("state is stale — monitor may be dead"); `unknown` never displays as pass and lowers observation coverage; the gauge is always shown with its computation date, never as a timeless fact. A perfect dashboard score means the declared machine-verifiable readiness checks all pass with no blind spots; it is not an auditor's opinion or a substitute for a Type II observation period. +Honest-rendering rules: a stale verify run (>48h) banners the whole page ("state is stale — monitor may be dead"); `unknown` never displays as pass and not-applicable never inflates the applicable denominator; the gauge is always shown with its computation date, formula, and denominator. A perfect maturity score only means every in-scope criterion received full credit under this internal evidence rubric. It is never an auditor's opinion, a prediction of examination outcome, or a substitute for either a Type I CPA evaluation or a Type II observation period. ## Seeding diff --git a/website/app/src/main.rs b/website/app/src/main.rs index 1d78d07..bbf5dc1 100644 --- a/website/app/src/main.rs +++ b/website/app/src/main.rs @@ -155,6 +155,8 @@ struct VerifyReport { subject: VerifySubject, #[serde(default)] summary: VerifySummary, + #[serde(default)] + provenance: VerifyProvenance, checks: Vec, } @@ -164,6 +166,20 @@ struct VerifySubject { repository: String, } +#[derive(Deserialize, Default)] +struct VerifyProvenance { + #[serde(default)] + commit: Option, + #[serde(default)] + run_id: Option, + #[serde(default)] + workflow: Option, + #[serde(default)] + generator: Option, + #[serde(default)] + report_signature: Option, +} + #[derive(Deserialize, Default)] struct VerifySummary { #[serde(default)] @@ -260,6 +276,16 @@ fn readiness_metrics(conn: &Connection) -> (f64, f64, f64, f64) { (design, technical, evidence, operating) } +fn criterion_maturity(conn: &Connection) -> f64 { + conn.query_row( + "SELECT COALESCE(SUM(weight * credit) * 100.0 / NULLIF(SUM(weight), 0), 0) + FROM criteria WHERE in_scope=1", + [], + |row| row.get(0), + ) + .unwrap_or(0.0) +} + fn report_readiness( report: &VerifyReport, fallback: (f64, f64, f64, f64), @@ -272,16 +298,21 @@ fn report_readiness( .map(|d| d.percent.clamp(0.0, 100.0)) .unwrap_or(default) }; - let observation_coverage = if report.checks.is_empty() { + let applicable = report + .checks + .iter() + .filter(|check| check.verdict != "n/a") + .count(); + let observation_coverage = if applicable == 0 { fallback.2 } else { report .checks .iter() - .filter(|check| check.verdict != "unknown") + .filter(|check| check.verdict == "pass" || check.verdict == "fail") .count() as f64 * 100.0 - / report.checks.len() as f64 + / applicable as f64 }; ( dimension("design", fallback.0), @@ -291,10 +322,6 @@ fn report_readiness( ) } -fn readiness_gauge(readiness: (f64, f64, f64, f64)) -> f64 { - (readiness.0 + readiness.1 + readiness.2 + readiness.3) / 4.0 -} - fn dashboard_subject(conn: &Connection) -> String { std::env::var("SHADOW_ORG") .ok() @@ -328,8 +355,6 @@ fn import_verify(db_path: &str, report_path: &str) { let ts = if report.observed_at.is_empty() { fallback_ts } else { report.observed_at.clone() }; let mut evidence_by_criterion: HashMap = HashMap::new(); let mut procedure_state: HashMap = HashMap::new(); - let mut cap_reason = None; - if !report.subject.repository.is_empty() { conn.execute( "INSERT OR REPLACE INTO metadata (key, value) VALUES ('subject_repository', ?1)", @@ -337,6 +362,49 @@ fn import_verify(db_path: &str, report_path: &str) { ) .expect("record report subject"); } + let observation_count = |verdict: &str| { + report.checks.iter().filter(|check| check.verdict == verdict).count() + }; + let pass = observation_count("pass"); + let fail = observation_count("fail"); + let unknown = observation_count("unknown"); + let not_applicable = observation_count("n/a"); + let metadata = [ + ("observations_total", check_count.to_string()), + ("observations_pass", pass.to_string()), + ("observations_fail", fail.to_string()), + ("observations_unknown", unknown.to_string()), + ("observations_na", not_applicable.to_string()), + ("report_schema_version", report.schema_version.to_string()), + ("assurance_basis", "automated-point-in-time".into()), + ("soc2_type_1_status", "not-determined".into()), + ("soc2_type_2_status", "not-determined".into()), + ]; + for (key, value) in metadata { + conn.execute( + "INSERT OR REPLACE INTO metadata (key, value) VALUES (?1, ?2)", + params![key, value], + ) + .expect("record report metadata"); + } + for (key, value) in [ + ("report_commit", report.provenance.commit.as_deref()), + ("report_run_id", report.provenance.run_id.as_deref()), + ("report_workflow", report.provenance.workflow.as_deref()), + ("report_generator", report.provenance.generator.as_deref()), + ("report_signature", report.provenance.report_signature.as_deref()), + ] { + if let Some(value) = value.filter(|value| !value.trim().is_empty()) { + conn.execute( + "INSERT OR REPLACE INTO metadata (key, value) VALUES (?1, ?2)", + params![key, value], + ) + .expect("record provenance metadata"); + } else { + conn.execute("DELETE FROM metadata WHERE key=?1", [key]) + .expect("clear absent provenance metadata"); + } + } // A readiness report is a complete current snapshot. Clearing the render // cache prevents retired checks from surviving forever as ghost evidence. @@ -345,11 +413,6 @@ fn import_verify(db_path: &str, report_path: &str) { .expect("reset criterion cache"); for check in &report.checks { - if (check.id == "github.org_2fa_required" || check.id.starts_with("github.branch_protection.")) - && check.verdict == "fail" - { - cap_reason = Some(check.id.clone()); - } let criteria = if check.criteria.is_empty() { criteria_for_legacy_verify_check(&check.id) } else { @@ -408,11 +471,26 @@ fn import_verify(db_path: &str, report_path: &str) { } let readiness = report_readiness(&report, readiness_metrics(&conn)); let (design, technical, evidence, operating) = readiness; - let gauge = readiness_gauge(readiness); - let cap = cap_reason.as_ref().map(|_| 79.0); + let gauge = criterion_maturity(&conn); + let metric_version = conn + .query_row( + "SELECT value FROM metadata WHERE key='gauge_metric_version'", + [], + |row| row.get::<_, String>(0), + ) + .ok(); + if metric_version.as_deref() != Some("criterion-maturity-v1") { + conn.execute("DELETE FROM gauge_history", []) + .expect("reset incompatible gauge history"); + } + conn.execute( + "INSERT OR REPLACE INTO metadata (key, value) VALUES ('gauge_metric_version', 'criterion-maturity-v1')", + [], + ) + .expect("record gauge metric version"); conn.execute( "INSERT OR REPLACE INTO gauge_history (ts, gauge, cap, cap_reason) VALUES (?1,?2,?3,?4)", - params![ts, gauge, cap, cap_reason], + params![ts, gauge, Option::::None, Option::::None], ) .expect("record gauge"); conn.execute( @@ -420,7 +498,7 @@ fn import_verify(db_path: &str, report_path: &str) { params![ts, design, technical, evidence, operating], ) .expect("record readiness dimensions"); - println!("imported schema v{}: {check_count} checks; gauge {gauge:.1}%; design {design:.1}%; technical {technical:.1}%; observation coverage {evidence:.1}%; operating {operating:.1}%", report.schema_version); + println!("imported schema v{}: {check_count} observations ({pass} pass, {fail} fail, {unknown} unknown, {not_applicable} n/a); in-scope criterion maturity {gauge:.1}%", report.schema_version); } // ---------- seeding: the markdown corpus is the source of truth ---------- @@ -658,6 +736,21 @@ async fn ingest( // ---------- read model ---------- +fn metadata_value(conn: &Connection, key: &str) -> Option { + conn.query_row( + "SELECT value FROM metadata WHERE key=?1", + [key], + |row| row.get::<_, String>(0), + ) + .ok() +} + +fn metadata_count(conn: &Connection, key: &str, fallback_sql: &str) -> i64 { + metadata_value(conn, key) + .and_then(|value| value.parse().ok()) + .unwrap_or_else(|| conn.query_row(fallback_sql, [], |row| row.get(0)).unwrap_or(0)) +} + fn load_model(conn: &Connection, org: &str) -> render::Model { let mut criteria = Vec::new(); { @@ -740,15 +833,7 @@ fn load_model(conn: &Connection, org: &str) -> render::Model { history.push(row.unwrap()); } } - let computed = { - let scoped: Vec<&render::Crit> = criteria.iter().filter(|c| c.in_scope).collect(); - let wsum: f64 = scoped.iter().map(|c| c.weight as f64).sum(); - if wsum > 0.0 { - scoped.iter().map(|c| c.weight as f64 * c.credit).sum::() / wsum * 100.0 - } else { - 0.0 - } - }; + let computed = criterion_maturity(conn); let gauge = match latest { Some((ts, g, cap, reason, hours)) => render::Gauge { value: cap.map_or(g, |c| g.min(c)), @@ -768,22 +853,40 @@ fn load_model(conn: &Connection, org: &str) -> render::Model { }, }; - let unknown_checks: i64 = conn - .query_row("SELECT COUNT(*) FROM checks WHERE verdict='unknown'", [], |r| r.get(0)) - .unwrap_or(0); - - let readiness = conn - .query_row( - "SELECT design, technical, evidence, operating FROM readiness_history ORDER BY ts DESC LIMIT 1", - [], - |r| Ok(render::Readiness { design: r.get(0)?, technical: r.get(1)?, evidence: r.get(2)?, operating: r.get(3)? }), - ) - .unwrap_or_else(|_| { - let (design, technical, evidence, operating) = readiness_metrics(conn); - render::Readiness { design, technical, evidence, operating } - }); + let observations = render::ObservationSummary { + total: metadata_count(conn, "observations_total", "SELECT COUNT(DISTINCT name) FROM checks"), + pass: metadata_count(conn, "observations_pass", "SELECT COUNT(DISTINCT CASE WHEN verdict='pass' THEN name END) FROM checks"), + fail: metadata_count(conn, "observations_fail", "SELECT COUNT(DISTINCT CASE WHEN verdict='fail' THEN name END) FROM checks"), + unknown: metadata_count(conn, "observations_unknown", "SELECT COUNT(DISTINCT CASE WHEN verdict='unknown' THEN name END) FROM checks"), + not_applicable: metadata_count(conn, "observations_na", "SELECT COUNT(DISTINCT CASE WHEN verdict='n/a' THEN name END) FROM checks"), + }; + let criterion_summary = render::CriterionSummary { + in_scope: criteria.iter().filter(|criterion| criterion.in_scope).count(), + verified: criteria.iter().filter(|criterion| criterion.in_scope && criterion.status == "verified").count(), + implemented: criteria.iter().filter(|criterion| criterion.in_scope && criterion.status == "implemented").count(), + failing: criteria.iter().filter(|criterion| criterion.in_scope && criterion.status == "failing").count(), + not_started: criteria.iter().filter(|criterion| criterion.in_scope && criterion.status == "not_started").count(), + }; + let provenance = render::Provenance { + repository: metadata_value(conn, "subject_repository").unwrap_or_else(|| org.to_string()), + commit: metadata_value(conn, "report_commit"), + run_id: metadata_value(conn, "report_run_id"), + workflow: metadata_value(conn, "report_workflow"), + generator: metadata_value(conn, "report_generator"), + report_signature: metadata_value(conn, "report_signature"), + }; + let unknown_checks = observations.unknown; - render::Model { org: org.to_string(), gauge, readiness, criteria, procedures, unknown_checks } + render::Model { + org: org.to_string(), + gauge, + observations, + criterion_summary, + provenance, + criteria, + procedures, + unknown_checks, + } } // ---------- routes ---------- @@ -841,7 +944,7 @@ async fn run_criterion(State(app): State>, Path(id): Path) -> i .status(), Runner::Claude => { let prompt = format!( - "You are the compliance shadow's single-criterion verifier. Criterion: {id}. Read {file} and execute each row of its 'Automated shadow checks' table (skip rows marked MANUAL) using gh / gcloud / file checks. Scope config: shadow/scope.json if present, else infer the org and repo from `gh repo view`. Then POST the results with curl to {url}/ingest as JSON: {{\"checks\":[{{\"criterion\":\"{id}\",\"name\":\"\",\"verdict\":\"pass|fail|unknown\",\"evidence\":\"\",\"last_run\":\"\"}}],\"criteria\":[{{\"id\":\"{id}\",\"status\":\"verified|implemented|in_progress|failing\",\"credit\":1.0}}]}}. Credit rules: all checks pass and evidence fresh = verified/1.0; controls exist, evidence partial = implemented/0.6; some pass = in_progress/0.25; failures = failing/0.0. unknown is never treated as pass. Do NOT write a gauge entry (single-criterion runs must not move the official gauge). Be quick; no commentary." + "You are the compliance shadow's single-criterion verifier. Criterion: {id}. Read {file} and execute each row of its 'Automated shadow checks' table (skip rows marked MANUAL) using gh / gcloud / file checks. Scope config: shadow/scope.json if present, else infer the org and repo from `gh repo view`. Then POST the results with curl to {url}/ingest as JSON: {{\"checks\":[{{\"criterion\":\"{id}\",\"name\":\"\",\"verdict\":\"pass|fail|unknown\",\"evidence\":\"\",\"last_run\":\"\"}}],\"criteria\":[{{\"id\":\"{id}\",\"status\":\"verified|implemented|in_progress|failing\",\"credit\":1.0}}]}}. Credit rules: all checks pass and evidence fresh = verified/1.0; controls exist, evidence partial = implemented/0.6; some pass = in_progress/0.25; failures = failing/0.0. unknown is never treated as pass. Do NOT write a gauge entry (single-criterion runs must not move the full-snapshot maturity metric). Be quick; no commentary." ); std::process::Command::new("claude") .args(["-p", &prompt, "--allowedTools", "Bash,Read,Glob,Grep", "--max-turns", "40"]) @@ -1040,7 +1143,7 @@ mod tests { } #[test] - fn signed_dimension_summary_drives_the_gauge() { + fn report_dimensions_are_diagnostic_not_the_primary_gauge() { let report: VerifyReport = serde_json::from_str( r#"{ "summary":{"dimensions":{ @@ -1058,7 +1161,22 @@ mod tests { let readiness = report_readiness(&report, (0.0, 0.0, 0.0, 0.0)); assert_eq!(readiness, (100.0, 90.0, 100.0, 100.0)); - assert_eq!(readiness_gauge(readiness), 97.5); + } + + #[test] + fn primary_gauge_is_weighted_in_scope_criterion_maturity() { + let conn = Connection::open_in_memory().unwrap(); + ensure_schema(&conn); + conn.execute( + "INSERT INTO criteria (id,family,category,text,weight,in_scope,status,credit) VALUES + ('CC1.1','CC1','security','one',3,1,'verified',1.0), + ('CC1.2','CC1','security','two',2,1,'implemented',0.6), + ('P1.1','P1','privacy','out',100,0,'verified',1.0)", + [], + ) + .unwrap(); + + assert!((criterion_maturity(&conn) - 84.0).abs() < f64::EPSILON); } #[test] @@ -1076,4 +1194,20 @@ mod tests { assert_eq!(report_readiness(&report, (1.0, 2.0, 3.0, 4.0)), (1.0, 2.0, 50.0, 4.0)); } + + #[test] + fn not_applicable_observations_are_excluded_from_coverage_denominator() { + let report: VerifyReport = serde_json::from_str( + r#"{ + "summary":{"dimensions":{}}, + "checks":[ + {"id":"observed","verdict":"pass"}, + {"id":"account-control","verdict":"n/a"} + ] + }"#, + ) + .unwrap(); + + assert_eq!(report_readiness(&report, (1.0, 2.0, 3.0, 4.0)), (1.0, 2.0, 100.0, 4.0)); + } } diff --git a/website/app/src/render.rs b/website/app/src/render.rs index c264069..eb6df3d 100644 --- a/website/app/src/render.rs +++ b/website/app/src/render.rs @@ -37,17 +37,37 @@ pub struct Gauge { pub struct Model { pub org: String, pub gauge: Gauge, - pub readiness: Readiness, + pub observations: ObservationSummary, + pub criterion_summary: CriterionSummary, + pub provenance: Provenance, pub criteria: Vec, pub procedures: Vec, pub unknown_checks: i64, } -pub struct Readiness { - pub design: f64, - pub technical: f64, - pub evidence: f64, - pub operating: f64, +pub struct ObservationSummary { + pub total: i64, + pub pass: i64, + pub fail: i64, + pub unknown: i64, + pub not_applicable: i64, +} + +pub struct CriterionSummary { + pub in_scope: usize, + pub verified: usize, + pub implemented: usize, + pub failing: usize, + pub not_started: usize, +} + +pub struct Provenance { + pub repository: String, + pub commit: Option, + pub run_id: Option, + pub workflow: Option, + pub generator: Option, + pub report_signature: Option, } pub struct CheckRow { @@ -126,7 +146,7 @@ fn arc(from: f64, to: f64, r: f64) -> String { fn gauge_svg(g: &Gauge) -> String { let mut s = String::new(); - s.push_str(r#""#); + s.push_str(r#""#); s.push_str(r##""##); // colour bands for (a, b, c) in [ @@ -201,20 +221,49 @@ fn category_chips(m: &Model) -> String { s } -fn readiness_cards(r: &Readiness) -> String { +fn evidence_cards(m: &Model) -> String { + let applicable = m.observations.total - m.observations.not_applicable; + let observed = m.observations.pass + m.observations.fail; let items = [ - ("Design readiness", r.design, "documented controls"), - ("Technical health", r.technical, "live automated checks"), - ("Observation coverage", r.evidence, "checks with a known verdict"), - ("Operating maturity", r.operating, "controls proven in operation"), + ( + format!("{} / {}", m.criterion_summary.verified, m.criterion_summary.in_scope), + "Criteria verified", + "operating or technical evidence attached", + ), + ( + m.criterion_summary.implemented.to_string(), + "Criteria implemented", + "design evidence only; not yet verified", + ), + ( + format!("{} + {}", m.criterion_summary.not_started, m.criterion_summary.failing), + "Not started + failing", + "unresolved in-scope criterion states", + ), + ( + format!("{} / {}", m.observations.pass, applicable.max(0)), + "Automated checks passed", + "applicable observations; not an audit sample", + ), ]; let mut s = String::from(r#"
"#); - for (label, value, note) in items { + for (value, label, note) in items { let _ = write!( s, - r#"
{value:.1}%{label}{note}
"# + r#"
{value}{label}{note}
"# ); } + let _ = write!( + s, + r#"
Observation denominator: {total} total = {applicable} applicable + {na} n/a. Applicable results: {pass} pass, {fail} fail, {unknown} unknown; {observed} have a pass/fail verdict.
"#, + total = m.observations.total, + applicable = applicable.max(0), + na = m.observations.not_applicable, + pass = m.observations.pass, + fail = m.observations.fail, + unknown = m.observations.unknown, + observed = observed, + ); s.push_str("
"); s } @@ -725,6 +774,7 @@ h1 .org{font-style:italic;font-weight:400;color:var(--faint)} .banner{margin:18px 0 0;padding:10px 16px;border:1.5px solid var(--amber);color:#7a5a0c;background:rgba(176,125,16,.07); font-family:"IBM Plex Mono",monospace;font-size:12px;letter-spacing:.06em;text-transform:uppercase} .banner.dead{border-color:var(--red);color:var(--red);background:rgba(158,43,37,.06)} +.banner.assurance{border-color:var(--ink);color:var(--ink);background:rgba(33,28,20,.035);text-transform:none;letter-spacing:.025em;line-height:1.55} .instrument{display:grid;grid-template-columns:minmax(320px,460px) 1fr;gap:44px;align-items:center;padding:36px 0 8px} .dialwrap{position:relative} .dial{width:100%;display:block} @@ -733,6 +783,7 @@ h1 .org{font-style:italic;font-weight:400;color:var(--faint)} @keyframes sweep{from{transform:rotate(0deg)}} .reading{text-align:center;margin-top:-8px} .reading .big{font-size:60px;font-weight:600;letter-spacing:-.02em;font-variation-settings:"opsz" 72} +.metric-name{font-family:"IBM Plex Mono",monospace;font-size:10px;color:var(--faint);letter-spacing:.1em;text-transform:uppercase;margin:-3px 0 6px} .reading .delta{font-family:"IBM Plex Mono",monospace;font-size:12px;color:var(--faint);letter-spacing:.08em} .reading .delta .up{color:var(--green)} .reading .delta .down{color:var(--red)} .stamp{position:absolute;top:8%;right:-2%;transform:rotate(-6deg);border:2.5px double var(--red);color:var(--red); @@ -745,6 +796,11 @@ h1 .org{font-style:italic;font-weight:400;color:var(--faint)} .rvalue{display:block;font-family:"IBM Plex Mono",monospace;font-size:20px;color:var(--deep)} .rlabel{display:block;font-size:14px;font-weight:600;margin-top:3px} .rnote{display:block;font-family:"IBM Plex Mono",monospace;font-size:8.5px;line-height:1.4;color:var(--faint);letter-spacing:.06em;text-transform:uppercase;margin-top:3px} +.observation-note{grid-column:1/-1;font-family:"IBM Plex Mono",monospace;font-size:9.5px;line-height:1.55;color:var(--faint);padding:2px 1px} +.formula{border-top:1px solid var(--ink);margin-top:22px;padding-top:12px;font-family:"IBM Plex Mono",monospace;font-size:10.5px;line-height:1.65;color:var(--faint)} +.provenance{display:grid;grid-template-columns:repeat(3,minmax(0,1fr));gap:7px 16px;margin-top:12px;padding:12px;border:1px solid var(--rule);font-family:"IBM Plex Mono",monospace;font-size:9.5px;line-height:1.5;color:var(--faint)} +.provenance span{overflow-wrap:anywhere}.provenance strong{color:var(--ink);text-transform:uppercase;letter-spacing:.06em}.provenance a{color:var(--deep)} +.integrity-note{font-family:"IBM Plex Mono",monospace;font-size:9px;line-height:1.5;color:var(--faint);margin-top:6px} .chip{display:grid;grid-template-columns:1fr auto;grid-template-rows:auto auto;padding:12px 4px;border-bottom:1px solid var(--rule)} .chip-name{font-size:19px;font-weight:600} .chip-score{font-family:"IBM Plex Mono",monospace;font-size:19px;grid-row:span 2;align-self:center} @@ -837,6 +893,7 @@ a.back{font-family:"IBM Plex Mono",monospace;font-size:11px;letter-spacing:.14em .hd-right{text-align:left;line-height:1.8} h1{font-size:30px} .instrument{grid-template-columns:1fr;gap:26px;padding:24px 0 4px} + .provenance{grid-template-columns:1fr} .reading .big{font-size:46px} .cards{column-count:1} .st{flex-wrap:wrap} @@ -862,7 +919,7 @@ fn head(title: &str) -> String { } pub fn index(m: &Model) -> String { - let mut s = head("Shadow Audit — SOC 2 readiness"); + let mut s = head("Shadow Audit — control evidence maturity"); s.push_str(r#"
"#); // header @@ -883,9 +940,10 @@ pub fn index(m: &Model) -> String { } _ => {} } + s.push_str(r#""#); // I. instrument - s.push_str(r#"

I. The Instrument would you pass an examination today?

"#); + s.push_str(r#"

I. Evidence Maturity internal control-evidence rubric · not probability of passing an audit

"#); s.push_str(&gauge_svg(&m.gauge)); if let (Some(cap), Some(reason)) = (m.gauge.cap, m.gauge.cap_reason.as_deref()) { let _ = write!(s, r#"
Capped {cap:.0}% — {}
"#, esc(reason)); @@ -905,15 +963,45 @@ pub fn index(m: &Model) -> String { }; let _ = write!( s, - r#"
{:.1}%
{delta}
{}
"#, + r#"
{:.1}%
weighted in-scope criterion maturity
{delta}
{}
"#, m.gauge.value, sparkline(&m.gauge.history) ); s.push_str("
"); - s.push_str(&readiness_cards(&m.readiness)); + s.push_str(&evidence_cards(m)); s.push_str(&category_chips(m)); s.push_str("
"); - s.push_str("
"); + s.push_str("
"); + s.push_str(r#"
Formula: Σ(in-scope criterion weight × credit) ÷ Σ(in-scope criterion weight). Verified = 100% credit; implemented/design-only = 60%; failing or not started = 0%. A passing observation supports a criterion but does not by itself establish audit readiness.
"#); + let commit = m.provenance.commit.as_deref().unwrap_or("not recorded"); + let run_id = m.provenance.run_id.as_deref().unwrap_or("not recorded"); + let workflow = m.provenance.workflow.as_deref().unwrap_or("not recorded"); + let generator = m.provenance.generator.as_deref().unwrap_or("shadow-ci"); + let signature = m.provenance.report_signature.as_deref().unwrap_or("none"); + let repo_url = format!("https://github.com/{}", m.provenance.repository); + let commit_value = if m.provenance.commit.is_some() { + let short_commit: String = commit.chars().take(12).collect(); + format!(r#"{}"#, esc(&repo_url), esc(commit), esc(&short_commit)) + } else { + esc(commit) + }; + let run_value = if m.provenance.run_id.is_some() { + format!(r#"{}"#, esc(&repo_url), esc(run_id), esc(run_id)) + } else { + esc(run_id) + }; + let _ = write!( + s, + r#"
Subject {repository}Commit {commit}Run {run}Workflow {workflow}Generator {generator}JSON signature {signature}
The GitHub artifact may have a platform-provided SHA-256 digest. The JSON report itself is not cryptographically signed unless a signature is explicitly listed above.
"#, + repo_url = esc(&repo_url), + repository = esc(&m.provenance.repository), + commit = commit_value, + run = run_value, + workflow = esc(workflow), + generator = esc(generator), + signature = esc(signature), + ); + s.push_str(""); // II + III s.push_str(&machinery_cards(m)); @@ -922,7 +1010,7 @@ pub fn index(m: &Model) -> String { // footer let _ = write!( s, - r#"
{} unknown checks (blind spots)state renders; the agent computes — export shadow.db
"#, + r#"
{} unknown automated observations (blind spots)evidence aid, not certification — export shadow.db
"#, m.unknown_checks ); s.push_str(""); @@ -942,7 +1030,7 @@ pub fn micro(m: &Model, running: &std::collections::HashSet, runner_ok: let _ = write!( s, r#"
Compliance Shadow — Micro Board

{:.1}% / {}

-
full working papers →
click a box to run its checks now
the official gauge moves on the next full verify
"#, +
full working papers →
weighted criterion maturity
not an audit opinion
"#, m.gauge.value, esc(&m.org) ); @@ -1174,4 +1262,51 @@ mod tests { .collect(); assert_eq!(defined, seen, "map pins must match PROCEDURES.md exactly"); } + + #[test] + fn primary_page_never_presents_maturity_as_an_audit_opinion() { + let model = Model { + org: "example/repo".into(), + gauge: Gauge { + value: 61.2, + cap: None, + cap_reason: None, + ts: Some("2026-08-06T00:00:00Z".into()), + history: vec![61.2], + stale_hours: Some(0.0), + }, + observations: ObservationSummary { + total: 40, + pass: 39, + fail: 0, + unknown: 0, + not_applicable: 1, + }, + criterion_summary: CriterionSummary { + in_scope: 36, + verified: 14, + implemented: 14, + failing: 0, + not_started: 8, + }, + provenance: Provenance { + repository: "example/repo".into(), + commit: Some("0123456789abcdef".into()), + run_id: Some("123".into()), + workflow: Some("verify".into()), + generator: Some("shadow-ci".into()), + report_signature: Some("none".into()), + }, + criteria: vec![], + procedures: vec![], + unknown_checks: 0, + }; + + let html = index(&model); + assert!(html.contains("not a SOC 2 report or CPA opinion")); + assert!(html.contains("weighted in-scope criterion maturity")); + assert!(html.contains("39 / 39")); + assert!(html.contains("1 n/a")); + assert!(!html.contains("would you pass an examination today")); + } }