Queue and live activity
Independent current activity projection; it does not establish that durable lifecycle history is available or complete.
@@ -207,7 +209,7 @@ dialog{border:0;padding:0;margin:0 0 0 auto;width:min(580px,94vw);height:100vh;m
var TERMINAL_GROUPS=[{id:"completed",label:"Completed",stages:["completed"]},{id:"attention",label:"Failed / cancelled",stages:["failed","cancelled"]}];
var OUTCOME_CONFIRM_MS=150000;
var TIDE_CREST_MS=3500,TIDE_RECEDING_MS=4550,TIDE_REVEAL_MS=6100,TIDE_END_MS=7200;
- var state={data:null,items:[],filter:"all",found:null,includeLegacyBatch:false,tideGeneration:null,consumedDirectTideGeneration:null,consumedLegacyTideGeneration:null,tideBusy:false,tideMode:null,tideSequence:0,realTidePending:false,brush:"patrol",previousRuns:{},confirmingOutcomes:{},previewSource:false,chatTimer:null,lastChatStage:null,chatSequence:0,masterTimer:null,masterSequence:0,masterPhase:"resting",masterPending:false,masterFingerprint:null,pendingItems:null,pendingTransitionKey:null,pendingTransitionKind:null,loadInFlight:false,loadQueued:false,healthRange:"6h",healthHistory:[],healthHistoryByRange:{},healthHistoryLoadedAt:{},healthHistoryLoading:{},egressByRange:{},egressLoadedAt:{},egressLoading:{},durableLifecycle:null,durableLifecycleLoading:false};
+ var state={data:null,items:[],filter:"all",found:null,includeLegacyBatch:false,tideGeneration:null,consumedDirectTideGeneration:null,consumedLegacyTideGeneration:null,tideBusy:false,tideMode:null,tideSequence:0,realTidePending:false,brush:"patrol",previousRuns:{},confirmingOutcomes:{},previewSource:false,chatTimer:null,lastChatStage:null,chatSequence:0,masterTimer:null,masterSequence:0,masterPhase:"resting",masterPending:false,masterFingerprint:null,pendingItems:null,pendingTransitionKey:null,pendingTransitionKind:null,loadInFlight:false,loadQueued:false,healthRange:"6h",healthHistory:[],healthHistoryByRange:{},healthHistoryContractByRange:{},healthHistoryLoadedAt:{},healthHistoryLoading:{},egressByRange:{},egressLoadedAt:{},egressLoading:{},durableLifecycle:null,durableLifecycleLoading:false};
var repoPalette=["#df552f","#2f827a","#b54130","#526b72","#8b659c","#b17a28"];
function esc(value){return String(value==null?"":value).replace(/[&<>\"]/g,function(ch){return {"&":"&","<":"<",">":">",'\"':"""}[ch];});}
var DURABLE_LANES=["pending","acknowledgement_pending","completed","superseded","requeued","terminal_attention"];
@@ -277,7 +279,10 @@ dialog{border:0;padding:0;margin:0 0 0 auto;width:min(580px,94vw);height:100vh;m
function bayHealthExactReview(value){var source=bayObject(value);if(!source||typeof source.collection_ok!=="boolean")return null;if(!source.collection_ok)return {collection_ok:false};var review=bayHealthLane(source.review,true),publication=bayHealthLane(source.publication,false);if(!review||!publication)return null;var result={collection_ok:true,review:review,publication:publication};if(source.handoff!==undefined){var handoff=bayObject(source.handoff),pending=handoff&&bayHealthCount(handoff.pending),dispatching=handoff&&bayHealthCount(handoff.dispatching),leased=handoff&&bayHealthCount(handoff.leased);if(!handoff||HANDOFF_STATUSES.indexOf(handoff.status)<0||pending===null||dispatching===null||leased===null)return null;result.handoff={status:handoff.status,pending:pending,dispatching:dispatching,leased:leased};}return result;}
function bayHealthStateWriter(value){var source=bayObject(value);if(!source||typeof source.collection_ok!=="boolean")return null;if(!source.collection_ok)return {collection_ok:false};if(typeof source.terminal_collection_ok!=="boolean"||BAY_HEALTH_WRITER_MODES.indexOf(source.mode)<0)return null;var result={collection_ok:true,terminal_collection_ok:source.terminal_collection_ok,mode:source.mode};["tracked_holding","tracked_waiting","tracked_releasing","accepted_operations_total","state_commits_total","materialized_items_total","contention_timeouts_total"].forEach(function(field){var parsed=bayHealthOptionalCount(source[field]);if(parsed===null)result.invalid=true;else if(parsed!==undefined)result[field]=parsed;});if(result.invalid)return null;var wait=bayHealthPercentiles(source.wait_ms),hold=bayHealthPercentiles(source.hold_ms);if(!wait||!hold)return null;var last=source.last_successful_materialization_at===null?null:strictBayTimestamp(source.last_successful_materialization_at);if(last===null&&source.last_successful_materialization_at!==null)return null;result.wait_ms=wait;result.hold_ms=hold;result.last_successful_materialization_at=last;return result;}
function bayHealthSample(value){var source=bayObject(value),at=source&&strictBayTimestamp(source.at);if(!source||!at)return null;var operationalFields=["status","collection_ok","queued","queued_over_30m","oldest_queued_minutes","running","running_over_150m","oldest_running_minutes"],hasOperational=operationalFields.some(function(field){return Object.prototype.hasOwnProperty.call(source,field);}),result={at:at};if(hasOperational){if(BAY_HEALTH_STATUSES.indexOf(source.status)<0||typeof source.collection_ok!=="boolean")return null;var counts={};for(var index=2;index
Awaiting handoff history.
';var width=430,height=46,left=27,right=3,top=3,bottom=3;var first=Date.parse(samples[0].at),last=Date.parse(samples[samples.length-1].at),span=Math.max(1,last-first);var values=valid.flatMap(function(sample){return [sample.pending,sample.dispatching,sample.leased].filter(function(value){return Number.isFinite(value);});});var maximum=bayNiceMaximum(Math.max.apply(null,values));var plotWidth=width-left-right,plotHeight=height-top-bottom;var pathFor=function(field){var previous=null;return samples.map(function(sample){var value=sample.valid?sample[field]:null;if(!Number.isFinite(value)){previous=null;return "";}var at=Date.parse(sample.at);var point={x:left+(at-first)/span*plotWidth,y:height-bottom-(value/maximum)*plotHeight};var connected=previous!==null&&at-previous<=12*60000;previous=at;return (connected?"L":"M")+point.x.toFixed(1)+" "+point.y.toFixed(1);}).filter(Boolean).join(" ");};return 'pendingdispatchingleased
Queue handoff
Handoff telemetry is unavailable in this snapshot.
Queue handoff
Pending'+pending+'
Dispatching'+dispatching+'
Leased'+leased+'
'+esc(copy+(ages.length?" · "+ages.join(" · "):""))+'
Exact review control board
Queue telemetry is unavailable in this snapshot.
Exact review control board
Queue telemetry is unavailable in this snapshot.
-
Exact Review
+Exact Review
History coverage unavailable.
@@ -1947,7 +1947,7 @@ const STATUS_CONTAINER_FIELDS = new Set([
"recent_durable_publication_events", "collection", "review", "publication", "handoff_health",
"phases", "pending", "dispatching", "leased", "pressure", "scheduled_feed", "bay_projection", "activity", "queue_stages", "live_stages", "stages",
"active_stages", "window", "direct", "batch", "counts", "buckets", "provenance",
- "backoff_reasons", "parked_reasons", "recovery_reasons", "errors"
+ "backoff_reasons", "parked_reasons", "recovery_reasons", "errors", "freshness"
]);
const STATUS_BOOLEAN_FIELDS = new Set([
"active_census_complete", "complete", "cursor_required", "is_codex_worker",
@@ -1956,7 +1956,7 @@ const STATUS_BOOLEAN_FIELDS = new Set([
]);
const STATUS_TEXT_FIELDS = new Set([
"conclusion", "mode", "outcome", "reason", "sample_kind", "severity", "source", "stage", "state",
- "status", "terminal_outcome", "work_kind", "errors"
+ "status", "terminal_outcome", "work_kind", "errors", "cache_state"
]);
const STATUS_TEXT_VALUES = new Set([
"active", "apply", "applying", "arriving", "all_clear", "amber", "assist", "automerge",
@@ -1971,10 +1971,11 @@ const STATUS_TEXT_VALUES = new Set([
"superseded", "fallback", "retryable", "permanent", "saturated", "malformed", "mixed",
"observed", "queue_empty", "claim_stalled", "dispatcher_blocked", "dispatcher_paused",
"claim_delayed", "handoff_current", "handoff_unknown", "capacity_unavailable", "capacity_available",
- "no_ready_backlog", "no_admissible_backlog", "dispatcher_inactive", "capacity_full_with_backlog"
+ "no_ready_backlog", "no_admissible_backlog", "dispatcher_inactive", "capacity_full_with_backlog",
+ "fresh", "miss"
]);
const STATUS_TIME_FIELDS = new Set([
- "at", "completed_at", "generated_at", "observed_at", "oldest_at", "oldest_pending_at",
+ "at", "client_checked_at", "completed_at", "generated_at", "observed_at", "oldest_at", "oldest_pending_at",
"oldest_ready_at", "oldest_backoff_at", "oldest_dispatching_at", "oldest_leased_at",
"next_attempt_at", "next_wake_at", "last_tide_at", "received_at", "since", "started_at",
"updated_at", "washed_at"
@@ -2001,7 +2002,7 @@ const STATUS_NUMBER_FIELDS = new Set([
"worker_budget", "worker_detail_fallbacks", "worker_detail_runs", "waiting", "window_minutes",
"window_hours", "wedged_rerun_runs", "zombie_queued_runs", "apply_ready_count", "attention_count",
"automerge_command_to_merge_ms", "average_duration_ms", "average_ms", "candidate_count",
- "completed_attempts", "duration_ms", "elapsed_ms", "error_count", "estimated_full_cycle_minutes",
+ "completed_attempts", "duration_ms", "elapsed_ms", "age_ms", "error_count", "estimated_full_cycle_minutes",
"failure_rate_percent", "generated_count", "longest_duration_ms", "maximum_age_ms", "median_ms",
"oldest_age_seconds", "oldest_dispatching_age_seconds", "oldest_leased_age_seconds",
"oldest_pending_age_seconds", "omitted_count", "ready_pending", "admissible_pending",
@@ -2199,7 +2200,63 @@ function dashboardStatusSnapshot(value) {
},
dashboard_health: source.dashboard_health || { conclusion: "needs_attention", severity: "amber" },
exact_review_queue: exactReviewQueue,
- recent_durable_publication_events: source.recent_durable_publication_events ?? null
+ recent_durable_publication_events: source.recent_durable_publication_events ?? null,
+ freshness: dashboardStatusFreshness(source)
+ };
+}
+function unavailableDashboardStatusFreshness(cacheState = "miss", maximumAgeMs = 60000) {
+ return { state: "unavailable", cache_state: cacheState, generated_at: null, age_ms: null, maximum_age_ms: maximumAgeMs };
+}
+function dashboardStatusFreshness(source) {
+ const freshness = source?.freshness && typeof source.freshness === "object" && !Array.isArray(source.freshness)
+ ? source.freshness
+ : null;
+ const cacheState = freshness?.cache_state === "fresh" || freshness?.cache_state === "stale"
+ ? freshness.cache_state
+ : "miss";
+ const maximumAgeMs = Number.isSafeInteger(freshness?.maximum_age_ms) &&
+ freshness.maximum_age_ms > 0 && freshness.maximum_age_ms <= 900000
+ ? freshness.maximum_age_ms
+ : 60000;
+ if (!freshness) return unavailableDashboardStatusFreshness(cacheState, maximumAgeMs);
+ if (
+ freshness.state === "unavailable" && freshness.generated_at === null &&
+ freshness.age_ms === null
+ ) return unavailableDashboardStatusFreshness(cacheState, maximumAgeMs);
+ const generatedAt = dashboardObservabilityTimestamp(freshness.generated_at);
+ const generatedMs = generatedAt ? Date.parse(generatedAt) : NaN;
+ const clientCheckedAt = freshness.client_checked_at === undefined
+ ? null
+ : dashboardObservabilityTimestamp(freshness.client_checked_at);
+ const clientCheckedMs = clientCheckedAt ? Date.parse(clientCheckedAt) : null;
+ const now = Date.now();
+ if (
+ !generatedAt || generatedAt !== source.generated_at || !Number.isFinite(now) ||
+ generatedMs > now + 60000 ||
+ (freshness.client_checked_at !== undefined && !clientCheckedAt) ||
+ (clientCheckedMs !== null && clientCheckedMs > now + 60000) ||
+ !Number.isSafeInteger(freshness.age_ms) || freshness.age_ms < 0 ||
+ (freshness.state !== "fresh" && freshness.state !== "stale")
+ ) return unavailableDashboardStatusFreshness(cacheState, maximumAgeMs);
+ const elapsedMs = clientCheckedMs === null
+ ? Math.max(0, now - generatedMs)
+ : Math.max(0, now - clientCheckedMs);
+ const ageMs = clientCheckedMs === null
+ ? Math.max(freshness.age_ms, elapsedMs)
+ : freshness.age_ms + elapsedMs;
+ if (!Number.isSafeInteger(ageMs) || ageMs > 1000000000000) {
+ return unavailableDashboardStatusFreshness(cacheState, maximumAgeMs);
+ }
+ const state = freshness.state === "stale" || cacheState === "stale" || ageMs > maximumAgeMs
+ ? "stale"
+ : "fresh";
+ return {
+ state,
+ cache_state: cacheState,
+ generated_at: generatedAt,
+ age_ms: ageMs,
+ maximum_age_ms: maximumAgeMs,
+ client_checked_at: new Date(now).toISOString()
};
}
let lastData = null;
@@ -2217,6 +2274,7 @@ let activeHealthRange = "6h";
let activeApplyRange = "24h";
let healthHistoryLoadedAt = 0;
let healthHistorySamples = [];
+let healthHistoryContract = unavailableDashboardHealthHistoryContract();
let applyObservabilityRequestGeneration = 0;
let lastApplyObservability = null;
let lastReviewCoverage = null;
@@ -2762,6 +2820,97 @@ function dashboardHealthHistorySample(value) {
}
return hasOperational || result.exact_review || result.state_writer ? result : null;
}
+function unavailableDashboardHealthHistoryContract() {
+ return {
+ coverage: { state: "unavailable", expected_slots: null, observed_slots: null, usable_slots: null, failed_slots: null, missing_slots: null, coverage_percent: null, largest_gap_slots: null, largest_gap_ms: null, window_started_at: null, window_ended_at: null },
+ freshness: { state: "unavailable", latest_sample_at: null, age_ms: null, maximum_age_ms: 720000 }
+ };
+}
+function exactUnavailableDashboardHealthHistoryContract(source) {
+ if (source.generated_at !== null) return null;
+ const expected = unavailableDashboardHealthHistoryContract();
+ const coverage = dashboardObservabilityObject(source.coverage);
+ const freshness = dashboardObservabilityObject(source.freshness);
+ if (!coverage || !freshness) return null;
+ const coverageFields = Object.keys(expected.coverage);
+ const freshnessFields = Object.keys(expected.freshness);
+ if (
+ Object.keys(coverage).length !== coverageFields.length ||
+ Object.keys(freshness).length !== freshnessFields.length ||
+ coverageFields.some((field) => coverage[field] !== expected.coverage[field]) ||
+ freshnessFields.some((field) => freshness[field] !== expected.freshness[field])
+ ) return null;
+ return expected;
+}
+function dashboardHealthHistoryContract(source, rangeMs) {
+ const coverage = dashboardObservabilityObject(source.coverage);
+ const freshness = dashboardObservabilityObject(source.freshness);
+ const generatedAt = dashboardObservabilityTimestamp(source.generated_at);
+ if (source.generated_at === null) return exactUnavailableDashboardHealthHistoryContract(source);
+ if (!coverage && !freshness && !generatedAt) return unavailableDashboardHealthHistoryContract();
+ const expected = dashboardObservabilityCount(coverage?.expected_slots);
+ const observed = dashboardObservabilityCount(coverage?.observed_slots);
+ const usable = dashboardObservabilityCount(coverage?.usable_slots);
+ const failed = dashboardObservabilityCount(coverage?.failed_slots);
+ const missing = dashboardObservabilityCount(coverage?.missing_slots);
+ const largestGap = dashboardObservabilityCount(coverage?.largest_gap_slots);
+ const largestGapMs = dashboardObservabilityCount(coverage?.largest_gap_ms);
+ const coveragePercent =
+ typeof coverage?.coverage_percent === "number" &&
+ Number.isFinite(coverage.coverage_percent) &&
+ coverage.coverage_percent >= 0 && coverage.coverage_percent <= 100
+ ? coverage.coverage_percent
+ : null;
+ const expectedCoveragePercent =
+ expected === 0 || usable === null ? null : Math.round((usable / expected) * 10_000) / 100;
+ const windowStartedAt = dashboardObservabilityTimestamp(coverage?.window_started_at);
+ const windowEndedAt = dashboardObservabilityTimestamp(coverage?.window_ended_at);
+ const windowStartedMs = windowStartedAt ? Date.parse(windowStartedAt) : NaN;
+ const windowEndedMs = windowEndedAt ? Date.parse(windowEndedAt) : NaN;
+ const expectedFromWindow =
+ Number.isFinite(windowStartedMs) && Number.isFinite(windowEndedMs)
+ ? Math.floor(windowEndedMs / DASHBOARD_HEALTH_HISTORY_SAMPLE_MS) -
+ Math.ceil(windowStartedMs / DASHBOARD_HEALTH_HISTORY_SAMPLE_MS) +
+ 1
+ : null;
+ const latestSampleAt = freshness?.latest_sample_at === null
+ ? null
+ : dashboardObservabilityTimestamp(freshness?.latest_sample_at);
+ const ageMs = freshness?.age_ms === null ? null : dashboardObservabilityCount(freshness?.age_ms);
+ const maximumAgeMs = dashboardObservabilityCount(freshness?.maximum_age_ms);
+ const generatedMs = generatedAt ? Date.parse(generatedAt) : NaN;
+ const latestSampleMs = latestSampleAt ? Date.parse(latestSampleAt) : null;
+ const expectedAgeMs = latestSampleMs === null || !Number.isFinite(generatedMs)
+ ? null
+ : Math.max(0, generatedMs - latestSampleMs);
+ const expectedFreshnessState = expectedAgeMs === null
+ ? "unavailable"
+ : expectedAgeMs <= 720000 ? "fresh" : "stale";
+ if (
+ !coverage || !freshness || !generatedAt ||
+ !["complete", "partial", "unavailable"].includes(coverage.state) ||
+ !["fresh", "stale", "unavailable"].includes(freshness.state) ||
+ windowEndedMs - windowStartedMs !== rangeMs || expected !== expectedFromWindow ||
+ observed === null || usable === null || failed === null || missing === null ||
+ usable + failed !== observed || observed + missing !== expected ||
+ largestGap === null || largestGapMs !== largestGap * DASHBOARD_HEALTH_HISTORY_SAMPLE_MS ||
+ coveragePercent === undefined || coveragePercent === null || coveragePercent !== expectedCoveragePercent ||
+ (coverage.state === "unavailable") !== (usable === 0) ||
+ (coverage.state === "complete") !== (usable === expected) ||
+ !windowStartedAt || !windowEndedAt ||
+ maximumAgeMs !== 720000 ||
+ (latestSampleAt === null) !== (freshness.latest_sample_at === null) ||
+ (ageMs === null) !== (freshness.age_ms === null) ||
+ (freshness.state === "unavailable") !== (latestSampleAt === null) ||
+ (latestSampleMs !== null && latestSampleMs > generatedMs) ||
+ ageMs !== expectedAgeMs || freshness.state !== expectedFreshnessState
+ ) return null;
+ return {
+ generated_at: generatedAt,
+ coverage: { state: coverage.state, expected_slots: expected, observed_slots: observed, usable_slots: usable, failed_slots: failed, missing_slots: missing, coverage_percent: coveragePercent, largest_gap_slots: largestGap, largest_gap_ms: largestGapMs, window_started_at: windowStartedAt, window_ended_at: windowEndedAt },
+ freshness: { state: freshness.state, latest_sample_at: latestSampleAt, age_ms: ageMs, maximum_age_ms: maximumAgeMs }
+ };
+}
function dashboardHealthHistorySnapshot(value, requestedRange) {
const source = dashboardObservabilityObject(value);
const rangeMs = DASHBOARD_HEALTH_HISTORY_RANGE_MS[requestedRange];
@@ -2789,10 +2938,15 @@ function dashboardHealthHistorySnapshot(value, requestedRange) {
slots.add(slot);
samples.push(sample);
}
+ const contract = dashboardHealthHistoryContract(source, rangeMs);
+ if (!contract || (source.generated_at === null && samples.length > 0)) return null;
return {
schema_version: 1,
range: requestedRange,
retention_days: DASHBOARD_HEALTH_HISTORY_RETENTION_DAYS,
+ generated_at: contract.generated_at || null,
+ coverage: contract.coverage,
+ freshness: contract.freshness,
samples
};
}
@@ -3228,11 +3382,19 @@ async function loadHealthHistory(range, force) {
if (requestedRange !== activeHealthRange) return;
if (!payload) throw new Error("invalid health history");
healthHistorySamples = payload.samples;
+ healthHistoryContract = { coverage: payload.coverage, freshness: payload.freshness };
healthHistoryLoadedAt = Date.now();
} catch {
if (requestedRange !== activeHealthRange) return;
healthHistorySamples = [];
+ healthHistoryContract = unavailableDashboardHealthHistoryContract();
}
+ const contractNode = document.getElementById("exact-review-history-contract");
+ const coverage = healthHistoryContract.coverage || {};
+ const freshness = healthHistoryContract.freshness || {};
+ if (contractNode) contractNode.textContent = coverage.state === "unavailable"
+ ? "History coverage unavailable."
+ : "History " + coverage.usable_slots + " / " + coverage.expected_slots + " usable slots · " + coverage.state + " · " + freshness.state + (coverage.failed_slots ? " · " + coverage.failed_slots + " failed polls" : "") + (coverage.largest_gap_slots ? " · largest gap " + coverage.largest_gap_slots + " slots" : "");
renderExactReviewLanes(lastData?.exact_review_queue);
renderStateWriter(lastData?.exact_review_queue);
}
@@ -3972,11 +4134,28 @@ function renderDashboard(data, note) {
fmt.format(workerCount) + " claw worker" + (workerCount === 1 ? "" : "s") + " sweeping " +
fmt.format(repoCount) + " " + (repoCount === 1 ? "repository" : "repositories");
document.getElementById("subtitle").textContent = "Identity-safe public status";
- document.getElementById("updated").textContent = "Updated " + since(data.generated_at) + (note ? " \u00b7 " + note : "");
+ const freshnessCopy = data.freshness?.state === "stale"
+ ? " · stale snapshot"
+ : data.freshness?.state === "unavailable"
+ ? " · freshness unavailable"
+ : "";
+ document.getElementById("updated").textContent = "Updated " + since(data.generated_at) + freshnessCopy + (note ? " \u00b7 " + note : "");
const fleet = data.fleet;
+ const attempts = typeof data.health?.attempts === "number" ? data.health.attempts : NaN;
+ const failedAttempts = typeof data.health?.failed_attempts === "number" ? data.health.failed_attempts : NaN;
+ const errorRate = typeof data.health?.error_rate_percent === "number" ? data.health.error_rate_percent : NaN;
+ const attemptsKnown = Number.isSafeInteger(attempts) && attempts > 0;
+ const failedAttemptsKnown = Number.isSafeInteger(failedAttempts) && failedAttempts >= 0 && attemptsKnown && failedAttempts <= attempts;
+ const expectedErrorRate = failedAttemptsKnown ? Math.round((failedAttempts / attempts) * 1000) / 10 : NaN;
+ const errorRateKnown = failedAttemptsKnown && errorRate === expectedErrorRate;
+ const errorRateAvailability = !attemptsKnown
+ ? "denominator unavailable"
+ : !failedAttemptsKnown
+ ? "numerator unavailable"
+ : "rate unavailable or inconsistent";
document.getElementById("metrics").innerHTML = [
metric("Codex Workers", fmt.format(fleet.active_codex_jobs), "Codex budget " + fleet.worker_budget, fleet.budget_used_percent, "var(--green)"),
- metric("Error Rate", (data.health?.error_rate_percent || 0) + "%", fmt.format(data.health?.failed_attempts || 0) + " failed / " + fmt.format(data.health?.attempts || 0) + " attempts", Math.min(100, data.health?.error_rate_percent || 0), data.health?.failed_attempts ? "var(--red)" : "var(--green)"),
+ metric("Error Rate", errorRateKnown ? errorRate + "%" : "n/a", (failedAttemptsKnown ? fmt.format(failedAttempts) : "n/a") + " failed / " + (attemptsKnown ? fmt.format(attempts) : "n/a") + " attempts" + (errorRateKnown ? "" : " · " + errorRateAvailability), errorRateKnown ? Math.min(100, errorRate) : 0, errorRateKnown && failedAttempts > 0 ? "var(--red)" : errorRateKnown ? "var(--green)" : "var(--muted)"),
metric("Recovery Rate", data.health?.recovery_rate_percent == null ? "n/a" : data.health.recovery_rate_percent + "%", fmt.format(data.health?.unresolved_failures || 0) + " unresolved", data.health?.recovery_rate_percent == null ? 100 : data.health.recovery_rate_percent, data.health?.unresolved_failures ? "var(--amber)" : "var(--green)"),
metric("Codex Capacity", fleet.budget_used_percent + "%", "Codex slot utilization", fleet.budget_used_percent, "var(--green)")
].join("");
diff --git a/dashboard/exact-review-lifecycle-telemetry.ts b/dashboard/exact-review-lifecycle-telemetry.ts
index 51568316a19..4718364276f 100644
--- a/dashboard/exact-review-lifecycle-telemetry.ts
+++ b/dashboard/exact-review-lifecycle-telemetry.ts
@@ -32,6 +32,9 @@ export const EXACT_REVIEW_LIFECYCLE_BAY_MAX_JOURNEY_MS = 24 * 60 * 60 * 1000;
const EXACT_REVIEW_LIFECYCLE_BAY_COVERAGE_RACE_MS = 60_000;
export const EXACT_REVIEW_LIFECYCLE_BAY_TIDE_THRESHOLD = 20;
export const EXACT_REVIEW_LIFECYCLE_BAY_SCAN_LIMIT = 10_000;
+const EXACT_REVIEW_LIFECYCLE_RECONCILIATION_CANDIDATE_LIMIT =
+ EXACT_REVIEW_LIFECYCLE_BAY_SCAN_LIMIT * 10;
+const EXACT_REVIEW_LIFECYCLE_RECONCILIATION_PAGE_SIZE = 512;
export const EXACT_REVIEW_LIFECYCLE_BAY_RECOVERY_BATCH_LIMIT = 256;
// The empty string is the valid durable scope for an explicitly empty public
@@ -189,6 +192,44 @@ export type ExactReviewBayLifecycleSnapshot = {
} | null;
};
+export type ExactReviewBayTelemetryReconciliation = {
+ version: 1;
+ source: "canonical-lifecycle-projection-v1";
+ generated_at: string;
+ scope: { repository_count: number };
+ collection:
+ | { state: "complete" }
+ | { state: "unknown"; reason: "unavailable" | "over_cap" | "mixed" };
+ window: {
+ started_at: string;
+ ended_at: string;
+ minutes: number;
+ event_limit: number;
+ candidate_scan_limit: number;
+ candidates_scanned: number;
+ } | null;
+ comparison: {
+ canonical_events: number;
+ aggregate_events: number;
+ missing_events: number;
+ unexpected_events: number;
+ mismatched_events: number;
+ event_sets_match: boolean;
+ public_snapshot_matches_aggregate: boolean;
+ canonical: BayReconciliationAggregate;
+ aggregate: BayReconciliationAggregate;
+ } | null;
+};
+
+type BayReconciliationAggregate = {
+ normal_direct: { average_ms: number | null; median_ms: number | null; samples: number };
+ including_legacy_batch: {
+ average_ms: number | null;
+ median_ms: number | null;
+ samples: number;
+ };
+};
+
const TERMINAL_CLASSES: LifecycleTerminalDisposition[] = [
"review_completed_routed",
"superseded",
@@ -1030,6 +1071,220 @@ export class ExactReviewLifecycleTelemetryStore {
}
}
+ /**
+ * Authenticated summary-only audit of the public timing aggregate. The comparison
+ * never returns lifecycle identities or rows: they are used only inside the Durable
+ * Object to recompute the current public window from the canonical projection.
+ */
+ reconcileBaySnapshot(
+ now = Date.now(),
+ allowedRepositories: ReadonlySet = new Set(),
+ ): ExactReviewBayTelemetryReconciliation {
+ const unknown = (
+ reason: "unavailable" | "over_cap" | "mixed",
+ ): ExactReviewBayTelemetryReconciliation => ({
+ version: 1,
+ source: "canonical-lifecycle-projection-v1",
+ generated_at: new Date(now).toISOString(),
+ scope: { repository_count: allowedRepositories.size },
+ collection: { state: "unknown", reason },
+ window: null,
+ comparison: null,
+ });
+ try {
+ this.ensureSchemaSync();
+ const repositoryFilter = bayRepositoryFilter(allowedRepositories, "canonical_target_key");
+ const aggregateRepositoryFilter = bayRepositoryFilter(
+ allowedRepositories,
+ "events.canonical_target_key",
+ );
+ if (!repositoryFilter || !aggregateRepositoryFilter || this.hasBayLifecyclePending()) {
+ return unknown("unavailable");
+ }
+ const scopeRow = this.tideScopeRowSync();
+ if (!scopeRow || scopeRow.scope !== repositoryFilter.scope) return unknown("unavailable");
+ const timingCutoff = Math.max(
+ now - EXACT_REVIEW_LIFECYCLE_BAY_TIMING_WINDOW_MS,
+ scopeRow.progress.coverageStartedAt,
+ );
+ const triggerCoverageStartedAt = scopeRow.triggerCoverageStartedAt;
+ // Every projection mutation refreshes updated_at. A projection whose current
+ // completion falls inside the timing window must therefore have been updated
+ // inside that window too. Apply that indexed, scalar bound before parsing any
+ // JSON so retained lifecycle history cannot turn this operator audit into an
+ // unbounded projection-table scan.
+ const canonicalEvents: BayLifecycleEvent[] = [];
+ let candidatesScanned = 0;
+ let cursor: {
+ updatedAt: number;
+ canonicalTargetKey: string;
+ fenceKey: string;
+ revision: number;
+ } | null = null;
+ while (true) {
+ const cursorWhere: string = cursor
+ ? `AND (
+ updated_at < ?
+ OR (updated_at = ? AND canonical_target_key > ?)
+ OR (updated_at = ? AND canonical_target_key = ? AND fence_key > ?)
+ OR (updated_at = ? AND canonical_target_key = ? AND fence_key = ? AND revision < ?)
+ )`
+ : "";
+ const cursorBindings: unknown[] = cursor
+ ? [
+ cursor.updatedAt,
+ cursor.updatedAt,
+ cursor.canonicalTargetKey,
+ cursor.updatedAt,
+ cursor.canonicalTargetKey,
+ cursor.fenceKey,
+ cursor.updatedAt,
+ cursor.canonicalTargetKey,
+ cursor.fenceKey,
+ cursor.revision,
+ ]
+ : [];
+ const projectionRows: Array> = Array.from(
+ this.storage.sql.exec(
+ `SELECT projection_json, updated_at, canonical_target_key, fence_key, revision
+ FROM ${EXACT_REVIEW_LIFECYCLE_PROJECTION_TABLE}
+ WHERE updated_at >= ?
+ ${cursorWhere}
+ ${repositoryFilter.where}
+ ORDER BY updated_at DESC, canonical_target_key, fence_key, revision DESC
+ LIMIT ?`,
+ timingCutoff,
+ ...cursorBindings,
+ ...repositoryFilter.bindings,
+ EXACT_REVIEW_LIFECYCLE_RECONCILIATION_PAGE_SIZE,
+ ),
+ );
+ if (projectionRows.length === 0) break;
+ candidatesScanned += projectionRows.length;
+ if (candidatesScanned > EXACT_REVIEW_LIFECYCLE_RECONCILIATION_CANDIDATE_LIMIT) {
+ return unknown("over_cap");
+ }
+ for (const row of projectionRows) {
+ const projection = projectionFromRow(String(row.projection_json || ""));
+ if (!projection) return unknown("mixed");
+ let event = bayLifecycleEvent(projection);
+ // A later requeue or other canonical non-timing terminal state can retain
+ // the earlier final-review receipt timestamp used by the bounded query.
+ // The aggregate retracts that event, so it is intentionally absent from
+ // both sides of the reconciliation rather than making the audit unknown.
+ if (!event) continue;
+ if (event.completed_at < timingCutoff || event.completed_at > now) continue;
+ if (triggerCoverageStartedAt !== null && event.triggered_at < triggerCoverageStartedAt) {
+ continue;
+ }
+ if (
+ event.legacy_batch_path &&
+ this.hasAcceptedDirectOutcomeSync(
+ projection.canonicalTargetKey,
+ projection.fenceKey,
+ projection.revision,
+ )
+ ) {
+ event = { ...event, legacy_batch_path: false };
+ }
+ canonicalEvents.push(event);
+ if (canonicalEvents.length > EXACT_REVIEW_LIFECYCLE_BAY_SCAN_LIMIT) {
+ return unknown("over_cap");
+ }
+ }
+ if (projectionRows.length < EXACT_REVIEW_LIFECYCLE_RECONCILIATION_PAGE_SIZE) break;
+ const last: Record = projectionRows.at(-1)!;
+ const updatedAt: number = Number(last.updated_at);
+ const canonicalTargetKey: string = String(last.canonical_target_key || "");
+ const fenceKey: string = String(last.fence_key || "");
+ const revision: number = Number(last.revision);
+ if (
+ !validTimestamp(updatedAt) ||
+ !validCanonicalTargetKey(canonicalTargetKey) ||
+ !fenceKey ||
+ !Number.isSafeInteger(revision) ||
+ revision < 1
+ ) {
+ return unknown("mixed");
+ }
+ cursor = { updatedAt, canonicalTargetKey, fenceKey, revision };
+ }
+ const aggregateRows = Array.from(
+ this.storage.sql.exec(
+ `SELECT events.event_id, events.canonical_target_key, events.outcome,
+ events.triggered_at, events.completed_at, events.legacy_batch_path
+ FROM ${EXACT_REVIEW_LIFECYCLE_BAY_EVENT_TABLE} AS events
+ WHERE events.completed_at >= ? AND events.completed_at <= ?
+ AND (? = 0 OR events.triggered_at >= ?)
+ ${aggregateRepositoryFilter.where}
+ ORDER BY events.completed_at, events.event_id LIMIT ?`,
+ timingCutoff,
+ now,
+ Number(triggerCoverageStartedAt !== null),
+ triggerCoverageStartedAt ?? 0,
+ ...aggregateRepositoryFilter.bindings,
+ EXACT_REVIEW_LIFECYCLE_BAY_SCAN_LIMIT + 1,
+ ),
+ );
+ if (aggregateRows.length > EXACT_REVIEW_LIFECYCLE_BAY_SCAN_LIMIT) {
+ return unknown("over_cap");
+ }
+ const aggregateEvents = aggregateRows.map(bayLifecycleEventFromTimingRow);
+ const canonicalById = new Map(canonicalEvents.map((event) => [event.event_id, event]));
+ const aggregateById = new Map(aggregateEvents.map((event) => [event.event_id, event]));
+ let missingEvents = 0;
+ let mismatchedEvents = 0;
+ for (const [eventId, expected] of canonicalById) {
+ const observed = aggregateById.get(eventId);
+ if (!observed) missingEvents += 1;
+ else if (!sameBayLifecycleEvent(expected, observed)) mismatchedEvents += 1;
+ }
+ let unexpectedEvents = 0;
+ for (const eventId of aggregateById.keys()) {
+ if (!canonicalById.has(eventId)) unexpectedEvents += 1;
+ }
+ const canonical = bayReconciliationAggregate(canonicalEvents);
+ const aggregate = bayReconciliationAggregate(aggregateEvents);
+ const publicSnapshot = this.baySnapshot(now, allowedRepositories);
+ const publicSnapshotMatchesAggregate =
+ publicSnapshot.collection.state === "complete" &&
+ publicSnapshot.timings !== null &&
+ sameBayTimingAggregate(publicSnapshot.timings.overall, aggregate.normal_direct) &&
+ sameBayTimingAggregate(
+ publicSnapshot.timings.including_legacy_batch.overall,
+ aggregate.including_legacy_batch,
+ );
+ return {
+ version: 1,
+ source: "canonical-lifecycle-projection-v1",
+ generated_at: new Date(now).toISOString(),
+ scope: { repository_count: allowedRepositories.size },
+ collection: { state: "complete" },
+ window: {
+ started_at: new Date(timingCutoff).toISOString(),
+ ended_at: new Date(now).toISOString(),
+ minutes: EXACT_REVIEW_LIFECYCLE_BAY_TIMING_WINDOW_MS / 60_000,
+ event_limit: EXACT_REVIEW_LIFECYCLE_BAY_SCAN_LIMIT,
+ candidate_scan_limit: EXACT_REVIEW_LIFECYCLE_RECONCILIATION_CANDIDATE_LIMIT,
+ candidates_scanned: candidatesScanned,
+ },
+ comparison: {
+ canonical_events: canonicalEvents.length,
+ aggregate_events: aggregateEvents.length,
+ missing_events: missingEvents,
+ unexpected_events: unexpectedEvents,
+ mismatched_events: mismatchedEvents,
+ event_sets_match: missingEvents === 0 && unexpectedEvents === 0 && mismatchedEvents === 0,
+ public_snapshot_matches_aggregate: publicSnapshotMatchesAggregate,
+ canonical,
+ aggregate,
+ },
+ };
+ } catch {
+ return unknown("unavailable");
+ }
+ }
+
recordDirectOutcome(input: DirectOutcomeInput) {
validateIdentity(input);
if (!DIRECT_OUTCOMES.has(input.outcome)) throw new Error("invalid direct telemetry outcome");
@@ -2094,6 +2349,41 @@ function bayTimingAggregate(durations: readonly number[]) {
};
}
+function bayReconciliationAggregate(
+ events: readonly BayLifecycleEvent[],
+): BayReconciliationAggregate {
+ const allDurations = events.map((event) => event.completed_at - event.triggered_at);
+ const directDurations = events
+ .filter((event) => !event.legacy_batch_path)
+ .map((event) => event.completed_at - event.triggered_at);
+ return {
+ normal_direct: bayTimingAggregate(directDurations),
+ including_legacy_batch: bayTimingAggregate(allDurations),
+ };
+}
+
+function sameBayTimingAggregate(
+ left: { average_ms: number | null; median_ms: number | null; samples: number | null },
+ right: { average_ms: number | null; median_ms: number | null; samples: number },
+) {
+ return (
+ left.average_ms === right.average_ms &&
+ left.median_ms === right.median_ms &&
+ left.samples === right.samples
+ );
+}
+
+function sameBayLifecycleEvent(left: BayLifecycleEvent, right: BayLifecycleEvent) {
+ return (
+ left.event_id === right.event_id &&
+ left.item_key === right.item_key &&
+ left.outcome === right.outcome &&
+ left.triggered_at === right.triggered_at &&
+ left.completed_at === right.completed_at &&
+ left.legacy_batch_path === right.legacy_batch_path
+ );
+}
+
function hadBayLifecycleTerminalEvent(projection: ExactReviewLifecycleProjection) {
return projection.terminalDispositions.some(
(terminal) => terminal.kind === "review_completed_routed" || terminal.kind === "failure",
diff --git a/dashboard/exact-review-queue.ts b/dashboard/exact-review-queue.ts
index 805ba9843f5..4c77447a356 100644
--- a/dashboard/exact-review-queue.ts
+++ b/dashboard/exact-review-queue.ts
@@ -876,6 +876,31 @@ export class ExactReviewQueue {
if (request.method === "POST" && url.pathname === "/lifecycle-audit/inventory") {
return this.readLifecycleAuditInventory(await request.json().catch(() => null));
}
+ if (request.method === "POST" && url.pathname === "/telemetry-reconciliation") {
+ await this.lifecycleProjectionReady.catch(() => undefined);
+ const publicRepositories = url.searchParams
+ .getAll("public_repo")
+ .map((value) => value.trim().toLowerCase());
+ const validPublicRepositories =
+ publicRepositories.length <= 32 &&
+ publicRepositories.every((value) => /^[a-z0-9_.-]+\/[a-z0-9_.-]+$/.test(value));
+ const publicRepositoryScope = validPublicRepositories ? new Set(publicRepositories) : null;
+ const recoveryPending = this.bayTelemetryRecoveryPendingSync();
+ return json({
+ exact_review_telemetry_reconciliation:
+ publicRepositoryScope && !recoveryPending
+ ? this.lifecycleTelemetryStore.reconcileBaySnapshot(Date.now(), publicRepositoryScope)
+ : {
+ version: 1,
+ source: "canonical-lifecycle-projection-v1",
+ generated_at: new Date().toISOString(),
+ scope: { repository_count: publicRepositoryScope?.size ?? 0 },
+ collection: { state: "unknown", reason: "unavailable" },
+ window: null,
+ comparison: null,
+ },
+ });
+ }
await this.ensureReady();
this.cleanupLegacyCompatibilitySync();
if (request.method === "POST" && url.pathname === "/github-egress-telemetry") {
diff --git a/dashboard/worker.ts b/dashboard/worker.ts
index fcb3997b0cf..6aa41aec619 100644
--- a/dashboard/worker.ts
+++ b/dashboard/worker.ts
@@ -1036,6 +1036,21 @@ export default {
request.method === "POST"
)
return authenticatedExactReviewOperatorRequest(request, env, "/lifecycle-audit/inventory");
+ if (
+ url.pathname === "/internal/exact-review/telemetry-reconciliation" &&
+ request.method === "POST"
+ ) {
+ const scope = new URLSearchParams();
+ for (const repository of verifiedPublicBayRepositories(env)) {
+ scope.append("public_repo", repository);
+ }
+ const query = scope.size ? `?${scope.toString()}` : "";
+ return authenticatedExactReviewOperatorRequest(
+ request,
+ env,
+ `/telemetry-reconciliation${query}`,
+ );
+ }
if (url.pathname === "/internal/exact-review/dead-letters/replay" && request.method === "POST")
return authenticatedExactReviewQueueRequest(request, env, "/dead-letters/replay");
if (
@@ -1292,16 +1307,123 @@ async function healthHistoryJson(request: Request, env: DashboardEnv) {
const samples = [...samplesBySlot.values()]
.sort((left, right) => Date.parse(left.at) - Date.parse(right.at))
.slice(-sampleLimit);
+ const contract = publicHealthHistoryContract(range, samples, now);
return cors(
json({
schema_version: 1,
range,
retention_days: HEALTH_HISTORY_RETENTION_DAYS,
+ generated_at: new Date(now).toISOString(),
+ coverage: contract.coverage,
+ freshness: contract.freshness,
samples,
}),
);
}
+export function publicHealthHistoryContract(range, samples, now = Date.now()) {
+ const rangeMs =
+ range === "6h"
+ ? 6 * 60 * 60 * 1000
+ : range === "7d"
+ ? 7 * 24 * 60 * 60 * 1000
+ : range === "24h"
+ ? 24 * 60 * 60 * 1000
+ : 0;
+ if (!rangeMs || !Number.isFinite(now)) {
+ return {
+ coverage: {
+ state: "unavailable",
+ expected_slots: null,
+ observed_slots: null,
+ usable_slots: null,
+ failed_slots: null,
+ missing_slots: null,
+ coverage_percent: null,
+ largest_gap_slots: null,
+ largest_gap_ms: null,
+ window_started_at: null,
+ window_ended_at: null,
+ },
+ freshness: {
+ state: "unavailable",
+ latest_sample_at: null,
+ age_ms: null,
+ maximum_age_ms: 12 * 60_000,
+ },
+ };
+ }
+ const windowStartedAt = now - rangeMs;
+ const firstExpectedSlot = Math.ceil(windowStartedAt / HEALTH_HISTORY_SAMPLE_MS);
+ const lastExpectedSlot = Math.floor(now / HEALTH_HISTORY_SAMPLE_MS);
+ const expectedSlots = Math.max(0, lastExpectedSlot - firstExpectedSlot + 1);
+ const observed = new Map();
+ for (const sample of Array.isArray(samples) ? samples : []) {
+ const at = Date.parse(String(sample?.at || ""));
+ if (!Number.isFinite(at)) continue;
+ const collectionOk = sample?.exact_review?.collection_ok;
+ if (typeof collectionOk !== "boolean") continue;
+ const slot = Math.floor(at / HEALTH_HISTORY_SAMPLE_MS);
+ if (slot < firstExpectedSlot || slot > lastExpectedSlot) continue;
+ const current = observed.get(slot);
+ if (!current || current.at < at) {
+ observed.set(slot, {
+ at,
+ usable: collectionOk,
+ });
+ }
+ }
+ const usable = new Set();
+ let latestSampleAt: number | null = null;
+ for (const [slot, sample] of observed) {
+ if (!sample.usable) continue;
+ usable.add(slot);
+ latestSampleAt = Math.max(latestSampleAt ?? sample.at, sample.at);
+ }
+ let largestGapSlots = 0;
+ let currentGapSlots = 0;
+ for (let slot = firstExpectedSlot; slot <= lastExpectedSlot; slot += 1) {
+ if (usable.has(slot)) currentGapSlots = 0;
+ else {
+ currentGapSlots += 1;
+ largestGapSlots = Math.max(largestGapSlots, currentGapSlots);
+ }
+ }
+ const observedSlots = observed.size;
+ const usableSlots = usable.size;
+ const failedSlots = observedSlots - usableSlots;
+ const missingSlots = Math.max(0, expectedSlots - observedSlots);
+ const freshnessMaximumAgeMs = 12 * 60_000;
+ const ageMs = latestSampleAt === null ? null : Math.max(0, now - latestSampleAt);
+ return {
+ coverage: {
+ state:
+ latestSampleAt === null
+ ? "unavailable"
+ : usableSlots === expectedSlots
+ ? "complete"
+ : "partial",
+ expected_slots: expectedSlots,
+ observed_slots: observedSlots,
+ usable_slots: usableSlots,
+ failed_slots: failedSlots,
+ missing_slots: missingSlots,
+ coverage_percent:
+ expectedSlots === 0 ? null : Math.round((usableSlots / expectedSlots) * 10_000) / 100,
+ largest_gap_slots: largestGapSlots,
+ largest_gap_ms: largestGapSlots * HEALTH_HISTORY_SAMPLE_MS,
+ window_started_at: new Date(windowStartedAt).toISOString(),
+ window_ended_at: new Date(now).toISOString(),
+ },
+ freshness: {
+ state: ageMs === null ? "unavailable" : ageMs <= freshnessMaximumAgeMs ? "fresh" : "stale",
+ latest_sample_at: latestSampleAt === null ? null : new Date(latestSampleAt).toISOString(),
+ age_ms: ageMs,
+ maximum_age_ms: freshnessMaximumAgeMs,
+ },
+ };
+}
+
function healthHistoryDates(fromMs: number, toMs: number) {
const dates = [];
const cursor = new Date(fromMs);
@@ -3023,17 +3145,60 @@ function statusSnapshotResponse(snapshot, cacheState, env) {
responseHeaders.set("content-type", "application/json; charset=utf-8");
responseHeaders.set("cache-control", "no-store");
responseHeaders.set("x-clawsweeper-cache", cacheState);
+ const projection = publicStatusProjection(snapshot, verifiedPublicBayRepositories(env));
+ const freshness = publicStatusFreshness(
+ projection.public_projection_complete === true ? snapshot : null,
+ cacheState,
+ numberFrom(env.CACHE_TTL_SECONDS, 60) * 1000,
+ );
return cors(
- new Response(
- JSON.stringify(publicStatusProjection(snapshot, verifiedPublicBayRepositories(env)), null, 2),
- {
- status: 200,
- headers: responseHeaders,
- },
- ),
+ new Response(JSON.stringify({ ...projection, freshness }, null, 2), {
+ status: 200,
+ headers: responseHeaders,
+ }),
);
}
+export function publicStatusFreshness(
+ snapshot,
+ cacheState,
+ maximumAgeMs = 60_000,
+ now = Date.now(),
+) {
+ const generatedAt = publicTimestamp(snapshot?.generated_at);
+ const boundedMaximumAgeMs =
+ Number.isSafeInteger(maximumAgeMs) && maximumAgeMs > 0
+ ? Math.min(maximumAgeMs, STALE_CACHE_TTL_SECONDS * 1000)
+ : 60_000;
+ if (!generatedAt || !Number.isFinite(now)) {
+ return {
+ state: "unavailable",
+ cache_state: cacheState === "stale" || cacheState === "fresh" ? cacheState : "miss",
+ generated_at: null,
+ age_ms: null,
+ maximum_age_ms: boundedMaximumAgeMs,
+ };
+ }
+ const generatedAtMs = Date.parse(generatedAt);
+ if (generatedAtMs > now) {
+ return {
+ state: "unavailable",
+ cache_state: cacheState === "stale" || cacheState === "fresh" ? cacheState : "miss",
+ generated_at: null,
+ age_ms: null,
+ maximum_age_ms: boundedMaximumAgeMs,
+ };
+ }
+ const ageMs = now - generatedAtMs;
+ return {
+ state: cacheState === "stale" || ageMs > boundedMaximumAgeMs ? "stale" : "fresh",
+ cache_state: cacheState === "stale" || cacheState === "fresh" ? cacheState : "miss",
+ generated_at: generatedAt,
+ age_ms: ageMs,
+ maximum_age_ms: boundedMaximumAgeMs,
+ };
+}
+
function refreshStatus(request, env) {
const key = [
new URL(request.url).origin,
diff --git a/docs/proof/csw-143-telemetry-accuracy-foundation/behavior-contract.md b/docs/proof/csw-143-telemetry-accuracy-foundation/behavior-contract.md
new file mode 100644
index 00000000000..1fc4ea19bc7
--- /dev/null
+++ b/docs/proof/csw-143-telemetry-accuracy-foundation/behavior-contract.md
@@ -0,0 +1,82 @@
+# CSW-143 telemetry accuracy foundation behavior contract
+
+## Claim
+
+An authenticated, read-only reconciliation path can compare the public exact-review
+telemetry aggregate with the canonical lifecycle projection without exposing lifecycle
+rows. Public Bay and Overview responses describe snapshot freshness and history coverage
+explicitly, use safe null states when observations are unavailable, and never render a
+zero-denominator error rate as zero percent. Bay keeps durable lifecycle availability
+separate from independent queue and live activity.
+
+## Exercised surface
+
+- The production dashboard Worker running locally through Wrangler in Docker-backed
+ Crabbox `local-container`.
+- The existing authenticated lifecycle audit boundary and canonical Durable Object
+ projection store.
+- Public `GET /api/status` and `GET /api/health-history` responses.
+- The rendered Overview and Bay observer-only pages.
+
+## Scenario and fixture
+
+Use synthetic local credentials and deterministic synthetic lifecycle and health-history
+records. Populate completed, incomplete, and empty observations through the same Worker
+and Durable Object routes used by production. Request the protected reconciliation with a
+valid operator signature and repeat it without valid authorization. Exercise a fresh
+complete public snapshot, a stale or partial snapshot, usable and failed exact-review
+history polls, an empty history, an Overview sample with zero attempts, and a lifecycle
+projection whose Bay telemetry materialization is pending.
+
+## Command and environment
+
+Run focused Node tests first. Then run the production Worker through Crabbox provider
+`local-container` with the repository's supported Node runtime and Wrangler, recording the
+candidate commit, container image, Crabbox run or lease identifier, and exact proof script
+and HTTP transcript. Use only loopback endpoints and synthetic secrets; do not deploy,
+read production Durable Objects, or mutate GitHub.
+
+## Observable result
+
+- The protected reconciliation returns summary counts and aggregate comparison only,
+ never repository names, item numbers, run identifiers, workflow details, or row payloads.
+- Invalid or absent authentication cannot read reconciliation results.
+- Public status includes a bounded freshness state and nullable age/source timestamps.
+- Public health history includes an explicit complete, partial, or unavailable coverage
+ state, denominator, observed/failed/missing counts, gap information, and nullable
+ freshness fields.
+- Empty or incomplete inputs remain `null`/unavailable rather than becoming a synthetic
+ zero or a false complete state.
+- Reconciliation remains unavailable while either durable lifecycle-to-telemetry recovery
+ source is pending, matching the public Bay aggregate boundary.
+- Overview renders an unavailable error rate for zero attempts.
+- Bay labels the durable lifecycle projection separately from independent queue and live
+ activity when lifecycle inventory is unavailable.
+
+## Artifact or trace
+
+Retain the focused test transcript and Crabbox timing JSON plus the sanitized HTTP/browser
+proof transcript under `.artifacts/csw-143-telemetry-accuracy-foundation/`. The PR body
+will identify exact files and reproduce the relevant observed values without publishing
+credentials or lifecycle rows.
+
+## Anti-cheat probes
+
+- Change one synthetic canonical projection without updating the telemetry event and
+ require the reconciliation result to report a mismatch.
+- Request reconciliation with no signature and with an invalid signature.
+- Supply zero history samples, one failed collection slot, and an overdue latest sample.
+- Supply freshness states and ages that contradict the generated/latest timestamps and
+ require both public dashboard parsers to reject the payload.
+- Supply `failed = 0` and `attempts = 0` and require `n/a`, not `0%`.
+- Make durable lifecycle inventory unavailable while retaining queue/live activity and
+ require both states to remain independently visible.
+- Leave a completed lifecycle projection pending telemetry materialization and require the
+ protected reconciliation to return an unavailable collection with no comparison.
+
+## Limits
+
+The proof establishes contract calculation, authentication, privacy shape, and local
+rendering against synthetic data. It does not deploy, validate every historical lifecycle
+row in production, establish causal effects for pull request 1280, persist lane-transition
+timestamps, publish per-lane timing, or redesign the dashboard hierarchy.
diff --git a/test/dashboard-worker-bay-records-routes.test.ts b/test/dashboard-worker-bay-records-routes.test.ts
index 19ae0afeb8a..03bbc35344d 100644
--- a/test/dashboard-worker-bay-records-routes.test.ts
+++ b/test/dashboard-worker-bay-records-routes.test.ts
@@ -39,6 +39,7 @@ import {
leasedExactReviewQueueItem,
leasedExactReviewPublicationItem,
} from "./dashboard-worker-harness.ts";
+import { publicHealthHistoryContract } from "../dashboard/worker.ts";
import {
EXACT_REVIEW_LIFECYCLE_BAY_EVENT_TABLE,
EXACT_REVIEW_LIFECYCLE_BAY_META_TABLE,
@@ -290,6 +291,225 @@ test("Bay lifecycle metrics include every durable ingress source and only final
);
});
+test("Bay telemetry reconciliation compares the public aggregate with canonical lifecycle facts without returning identities", () => {
+ const storage = new MemoryDurableStorage();
+ const lifecycle = new ExactReviewLifecycleProjectionStore(storage);
+ const telemetry = new ExactReviewLifecycleTelemetryStore(storage);
+ const startedAt = Date.now();
+ const now = startedAt + 5 * 60_000;
+ const publicScope = new Set(["openclaw/openclaw"]);
+ telemetry.syncBayRepositoryScope(publicScope, startedAt);
+ const identity = {
+ canonicalTargetKey: "openclaw/openclaw#14300",
+ fenceKey: "private-fence-csw-143",
+ revision: 1,
+ };
+ lifecycle.recordAdmission({
+ ...identity,
+ deliveryId: "private-delivery-csw-143",
+ sourceAction: "opened",
+ commandOriginated: false,
+ statusMarker: null,
+ statusCommentId: null,
+ triggeredAt: startedAt + 30_000,
+ observedAt: startedAt + 30_000,
+ });
+ lifecycle.recordGithubEffect({
+ ...identity,
+ commentId: 143_001,
+ digest: createHash("sha256").update("private-digest-csw-143").digest("hex"),
+ observedAt: startedAt + 210_000,
+ });
+ const completed = lifecycle.recordTerminalDisposition({
+ ...identity,
+ kind: "review_completed_routed",
+ observedAt: startedAt + 210_000,
+ });
+ assert.equal(telemetry.syncBayLifecycle(completed), true);
+
+ const matching = telemetry.reconcileBaySnapshot(now, publicScope);
+ assert.deepEqual(matching.collection, { state: "complete" });
+ assert.equal(matching.comparison?.event_sets_match, true);
+ assert.equal(matching.comparison?.public_snapshot_matches_aggregate, true);
+ assert.deepEqual(matching.comparison?.canonical.normal_direct, {
+ average_ms: 180_000,
+ median_ms: 180_000,
+ samples: 1,
+ });
+ const publicText = JSON.stringify(matching);
+ for (const privateValue of [
+ identity.canonicalTargetKey,
+ identity.fenceKey,
+ "private-delivery-csw-143",
+ "private-digest-csw-143",
+ ]) {
+ assert.doesNotMatch(
+ publicText,
+ new RegExp(privateValue.replace(/[.*+?^${}()|[\]\\]/g, "\\$&")),
+ );
+ }
+
+ storage.sql.exec(
+ `UPDATE ${EXACT_REVIEW_LIFECYCLE_BAY_EVENT_TABLE}
+ SET completed_at = completed_at + 1000
+ WHERE canonical_target_key = ?`,
+ identity.canonicalTargetKey,
+ );
+ const mismatched = telemetry.reconcileBaySnapshot(now, publicScope);
+ assert.equal(mismatched.comparison?.event_sets_match, false);
+ assert.equal(mismatched.comparison?.mismatched_events, 1);
+ assert.equal(mismatched.comparison?.missing_events, 0);
+ assert.equal(mismatched.comparison?.unexpected_events, 0);
+});
+
+test("Bay telemetry reconciliation fails closed while lifecycle telemetry recovery is pending", async () => {
+ const storage = new MemoryDurableStorage();
+ const queue = new ExactReviewQueue(
+ {
+ storage,
+ blockConcurrencyWhile: async (callback: () => Promise) => callback(),
+ },
+ { PUBLIC_BAY_REPOS: "openclaw/openclaw" },
+ );
+ const endpoint =
+ "https://clawsweeper-exact-review-queue/telemetry-reconciliation?public_repo=openclaw%2Fopenclaw";
+ const before = await queue.fetch(new Request(endpoint, { method: "POST", body: "{}" }));
+ assert.equal(before.status, 200);
+ assert.deepEqual((await before.json()).exact_review_telemetry_reconciliation.collection, {
+ state: "complete",
+ });
+
+ const lifecycle = new ExactReviewLifecycleProjectionStore(storage);
+ const now = Date.now();
+ const identity = {
+ canonicalTargetKey: "openclaw/openclaw#9144",
+ fenceKey: "openclaw/openclaw#9144@exact:1",
+ revision: 1,
+ };
+ lifecycle.recordAdmission({
+ ...identity,
+ deliveryId: "csw-143-pending-recovery",
+ sourceAction: "opened",
+ commandOriginated: false,
+ statusMarker: null,
+ statusCommentId: null,
+ triggeredAt: now - 1_000,
+ observedAt: now - 1_000,
+ });
+ recordBayFinalReceipt(lifecycle, identity, now);
+ lifecycle.recordTerminalDisposition({
+ ...identity,
+ kind: "review_completed_routed",
+ observedAt: now,
+ });
+ assert.equal(lifecycle.hasBayTelemetryPending(), true);
+
+ const response = await queue.fetch(new Request(endpoint, { method: "POST", body: "{}" }));
+ assert.equal(response.status, 200);
+ const reconciliation = (await response.json()).exact_review_telemetry_reconciliation;
+ assert.equal(reconciliation.version, 1);
+ assert.equal(reconciliation.source, "canonical-lifecycle-projection-v1");
+ assert.equal(typeof reconciliation.generated_at, "string");
+ assert.deepEqual(reconciliation.scope, { repository_count: 1 });
+ assert.deepEqual(reconciliation.collection, { state: "unknown", reason: "unavailable" });
+ assert.equal(reconciliation.window, null);
+ assert.equal(reconciliation.comparison, null);
+});
+
+test("Bay telemetry reconciliation pages recent lifecycle candidates without capping on active rows", () => {
+ const storage = new MemoryDurableStorage();
+ const lifecycle = new ExactReviewLifecycleProjectionStore(storage);
+ const telemetry = new ExactReviewLifecycleTelemetryStore(storage);
+ const startedAt = Date.now();
+ const now = startedAt + 5 * 60_000;
+ const scope = new Set(["openclaw/openclaw"]);
+ telemetry.syncBayRepositoryScope(scope, startedAt);
+ for (let index = 0; index < 600; index += 1) {
+ lifecycle.recordAdmission({
+ canonicalTargetKey: `openclaw/openclaw#${20_000 + index}`,
+ fenceKey: `openclaw/openclaw#${20_000 + index}@exact:1`,
+ revision: 1,
+ deliveryId: `csw-143-active:${index}`,
+ sourceAction: "opened",
+ commandOriginated: false,
+ statusMarker: null,
+ statusCommentId: null,
+ triggeredAt: startedAt + index,
+ observedAt: startedAt + index,
+ });
+ }
+ const completedIdentity = {
+ canonicalTargetKey: "openclaw/openclaw#20599",
+ fenceKey: "openclaw/openclaw#20599@exact:1",
+ revision: 1,
+ };
+ recordBayFinalReceipt(lifecycle, completedIdentity, startedAt + 120_000);
+ assert.equal(
+ telemetry.syncBayLifecycle(
+ lifecycle.recordTerminalDisposition({
+ ...completedIdentity,
+ kind: "review_completed_routed",
+ observedAt: startedAt + 120_000,
+ }),
+ ),
+ true,
+ );
+
+ const reconciliation = telemetry.reconcileBaySnapshot(now, scope);
+ assert.deepEqual(reconciliation.collection, { state: "complete" });
+ assert.equal(reconciliation.window?.candidates_scanned, 600);
+ assert.equal(reconciliation.comparison?.canonical_events, 1);
+ assert.equal(reconciliation.comparison?.aggregate_events, 1);
+ assert.equal(reconciliation.comparison?.event_sets_match, true);
+});
+
+test("Bay telemetry reconciliation ignores a completed review retracted by a later requeue", () => {
+ const storage = new MemoryDurableStorage();
+ const lifecycle = new ExactReviewLifecycleProjectionStore(storage);
+ const telemetry = new ExactReviewLifecycleTelemetryStore(storage);
+ const now = Date.now();
+ const scope = new Set(["openclaw/openclaw"]);
+ telemetry.syncBayRepositoryScope(scope, now - 60_000);
+ const identity = {
+ canonicalTargetKey: "openclaw/openclaw#9143",
+ fenceKey: "openclaw/openclaw#9143@exact:1",
+ revision: 1,
+ };
+ lifecycle.recordAdmission({
+ ...identity,
+ deliveryId: "csw-143-requeue",
+ sourceAction: "opened",
+ commandOriginated: false,
+ statusMarker: null,
+ statusCommentId: null,
+ triggeredAt: now - 30_000,
+ observedAt: now - 30_000,
+ });
+ recordBayFinalReceipt(lifecycle, identity, now - 1_000);
+ assert.equal(
+ telemetry.syncBayLifecycle(
+ lifecycle.recordTerminalDisposition({
+ ...identity,
+ kind: "review_completed_routed",
+ observedAt: now - 1_000,
+ }),
+ ),
+ true,
+ );
+ assert.equal(
+ telemetry.syncBayLifecycle(
+ lifecycle.recordTerminalDisposition({ ...identity, kind: "requeue", observedAt: now }),
+ ),
+ true,
+ );
+
+ const reconciliation = telemetry.reconcileBaySnapshot(now, scope);
+ assert.deepEqual(reconciliation.collection, { state: "complete" });
+ assert.equal(reconciliation.comparison?.canonical_events, 0);
+ assert.equal(reconciliation.comparison?.aggregate_events, 0);
+ assert.equal(reconciliation.comparison?.event_sets_match, true);
+});
+
test("Bay lifecycle excludes the retired batch path from normal review timing by default", () => {
const storage = new MemoryDurableStorage();
const lifecycle = new ExactReviewLifecycleProjectionStore(storage);
@@ -5884,14 +6104,14 @@ test("OpenClaw Bay is a public, indexable, hardened canonical route", async () =
assert.doesNotMatch(body, /function bayRecentPublicationEvents/);
assert.match(body, /id="durable-lifecycle-kanban"/);
assert.match(body, /Durable lifecycle Kanban/);
- assert.doesNotMatch(body, /Live activity/i);
- assert.doesNotMatch(body, /live-activity/);
+ assert.match(body, /Queue and live activity/i);
+ assert.match(body, /does not establish that durable lifecycle history is available or complete/i);
assert.doesNotMatch(body, /fetch\("\/api\/live-activity-bay"/);
assert.match(body, /function durableSnapshot/);
assert.match(body, /fetch\("\/api\/durable-lifecycle-bay"/);
assert.match(body, /durableLifecycleLoading/);
assert.match(body, /if\(state\.durableLifecycleLoading\)return/);
- assert.match(body, /bounded sample of verified public GitHub references/);
+ assert.match(body, /Canonical lifecycle projection only/);
assert.match(body, /Internal revisions and workflow details remain withheld/);
assert.match(body, /Empty complete lifecycle snapshot/);
assert.match(
@@ -6137,6 +6357,7 @@ test("OpenClaw Bay is a public, indexable, hardened canonical route", async () =
healthRange: "6h",
healthHistory: [] as unknown[],
healthHistoryByRange: {} as Record,
+ healthHistoryContractByRange: {} as Record,
healthHistoryLoadedAt: {} as Record,
healthHistoryLoading: {} as Record,
previewSource: false,
@@ -6208,10 +6429,7 @@ test("OpenClaw Bay is a public, indexable, hardened canonical route", async () =
assert.equal(bayHistoryRuntime.bayHandoffHistory()[0].dispatching, 1);
assert.equal(bayHistoryRuntime.bayStateWriterHistory()[0].pending, 2);
const projectedBayHistory = bayHistoryRuntime.bayHealthHistorySnapshot(validBayHistory, "6h");
- assert.equal(
- JSON.stringify(projectedBayHistory),
- JSON.stringify(bayHistoryRuntime.bayHealthHistorySnapshot(projectedBayHistory, "6h")),
- );
+ assert.equal(bayHistoryRuntime.bayHealthHistorySnapshot(projectedBayHistory, "6h"), null);
const bayMarker = "synthetic-bay-history-marker";
const bayMarkerUrl =
@@ -6227,6 +6445,42 @@ test("OpenClaw Bay is a public, indexable, hardened canonical route", async () =
},
})),
};
+ const projectedBayGeneratedAt = Date.now();
+ const contractedBayHistory = {
+ ...validBayHistory,
+ generated_at: new Date(projectedBayGeneratedAt).toISOString(),
+ ...publicHealthHistoryContract("6h", validBayHistory.samples, projectedBayGeneratedAt),
+ };
+ const unavailableBayHistory = {
+ ...validBayHistory,
+ generated_at: null,
+ coverage: {
+ state: "unavailable",
+ expected_slots: null,
+ observed_slots: null,
+ usable_slots: null,
+ failed_slots: null,
+ missing_slots: null,
+ coverage_percent: null,
+ largest_gap_slots: null,
+ largest_gap_ms: null,
+ window_started_at: null,
+ window_ended_at: null,
+ },
+ freshness: {
+ state: "unavailable",
+ latest_sample_at: null,
+ age_ms: null,
+ maximum_age_ms: 720_000,
+ },
+ samples: [],
+ };
+ assert.deepEqual(
+ JSON.parse(
+ JSON.stringify(bayHistoryRuntime.bayHealthHistorySnapshot(unavailableBayHistory, "6h")),
+ ),
+ unavailableBayHistory,
+ );
for (const malformed of [
{ samples: validBayHistory.samples },
{ ...validBayHistory, range: bayMarker },
@@ -6254,6 +6508,32 @@ test("OpenClaw Bay is a public, indexable, hardened canonical route", async () =
at: new Date(bayAt - index * 5 * 60_000).toISOString(),
})).reverse(),
},
+ {
+ ...contractedBayHistory,
+ freshness: {
+ ...contractedBayHistory.freshness,
+ state: contractedBayHistory.freshness.state === "fresh" ? "stale" : "fresh",
+ },
+ },
+ {
+ ...contractedBayHistory,
+ freshness: {
+ ...contractedBayHistory.freshness,
+ age_ms: Number(contractedBayHistory.freshness.age_ms) + 1,
+ },
+ },
+ { ...unavailableBayHistory, samples: validBayHistory.samples },
+ {
+ ...unavailableBayHistory,
+ coverage: { ...unavailableBayHistory.coverage, observed_slots: 0 },
+ },
+ {
+ ...unavailableBayHistory,
+ freshness: {
+ ...unavailableBayHistory.freshness,
+ latest_sample_at: new Date(bayAt).toISOString(),
+ },
+ },
]) {
assert.equal(bayHistoryRuntime.bayHealthHistorySnapshot(malformed, "6h"), null);
}
@@ -6335,6 +6615,8 @@ test("OpenClaw Bay is a public, indexable, hardened canonical route", async () =
assert.match(body, /Have you been in this lane long\?/);
assert.match(body, /The final journey time is still being verified\./);
assert.match(body, /verified final receipt/);
+ assert.match(body, //);
+ assert.doesNotMatch(body, /]+aria-labelledby="queue-live-activity-title"/);
assert.match(body, /chatSequence:0/);
assert.doesNotMatch(body, /Things are moving|30m end to end/);
const chatScript = [...body.matchAll(/