diff --git a/Makefile b/Makefile index 0b7b10d4..c16ff170 100644 --- a/Makefile +++ b/Makefile @@ -13,4 +13,4 @@ typecheck: cd frontend && bun run lint test: - cd frontend && bun test + cd frontend && bun test && bun run test:dev-port diff --git a/README.md b/README.md index 7c65c8a5..b21421f8 100644 --- a/README.md +++ b/README.md @@ -31,7 +31,8 @@ separate service layer — the Next.js API routes are the API layer. surfaced. - **Staging runs** (`/microcosm/staging`) — monitor pre-release US Microcosm build runs from the staging Hub repo: current stage, calibration loss progress, - final candidate diagnostics once uploaded, and candidate-vs-latest fit. + final candidate diagnostics once uploaded, candidate-vs-current-release fit, + and a hierarchical map of weighted target-error increases and reductions. Countries without a staging repository show an explicit unavailable state. - **Calibration target investigations** (`docs/ai/`) — tool-independent procedures, specialist review responsibilities, and a reusable checklist for identifying @@ -55,6 +56,20 @@ Published-release endpoints accept `country=us|uk|be` (default `us`). | `GET /api/microcosm/staging/run?id=` | One staging run's progress and uploaded candidate diagnostics | | `GET /api/microcosm/staging/target-diagnostics?id=&...` | Faceted diagnostics for a staging candidate once diagnostics exist | | `GET /api/microcosm/staging/compare?run=&release=latest` | Diff staging candidate against a published release | +| `GET /api/microcosm/staging/target-change-tree?run=&release=&mode=reported\|shared&...` | One hierarchy level of weighted target-error changes for a staging candidate and an explicit current release | + +The staging target-change route supports two comparison modes. `reported` uses +each release's actual target weights and complete target surface, including +added and removed targets. `shared` restricts the calculation to shared targets, +normalizes each release's weights over that shared set, averages the two shares +target by target, and applies the resulting pooled weights to both releases. + +Version and staging comparisons use the same normalized target matcher. It first +matches an exact period-normalized target name, then a unique Chronicle fact key, +then an exact structured source/statistic/measure/dimensions identity. A key must +identify one remaining target on each side; ambiguous keys are reported and left +unmatched. Comparison responses include the representation of each target, the +matching method, both release identifiers, and counts by matching method. ## Calibration target investigations @@ -76,12 +91,19 @@ combining their evidence into one report. ```bash make install # cd frontend && bun install -make dev # next dev (http://localhost:3000) +make dev # first available loopback port, starting at 3000 make typecheck # tsc --noEmit -make test # bun test (data-layer suite) +make test # frontend data and development-launcher tests make build # next build ``` +The development launcher prints the selected dashboard URL and records its port +in `frontend/.next/dev-port`. Set `PORT` to begin the search at a different +port; if that port is occupied on the IPv4 or IPv6 loopback address, the +launcher increments by one until it finds a port that is free on both network +families. Next.js binds to `127.0.0.1`, and the launcher prints that exact URL +to avoid hostname resolution selecting a different local process. + Run the Python Chronicle evaluation harness and its public numerical adapter gate manually when reviewing Chronicle or dependency updates: diff --git a/docs/spec-driven-countries.md b/docs/spec-driven-countries.md index f34d2394..3d97c258 100644 --- a/docs/spec-driven-countries.md +++ b/docs/spec-driven-countries.md @@ -302,6 +302,32 @@ variable parsing; structured dimensions, then known filter dimensions, then legacy metadata and name dimensions. This prevents one partially migrated row from changing unrelated legacy rows in the same file. +### Cross-release target matching + +The legacy and structured readers normalize each row independently. Candidate +validation and weighted target-error comparisons then use one collection-level +matcher with this order: + +1. Exact, non-empty `base_name`, which uses `target_name` when supplied and + otherwise removes the period suffix from `name`. +2. Exact, non-empty Chronicle `fact_key`. +3. For two structured rows, an exact tuple of source ID, variable ID, measure, + and raw dimension ID/value pairs sorted by dimension ID. + +At each step, a key is used only when it identifies exactly one still-unmatched +row in each release. Duplicate and many-to-one keys remain unmatched unless a +later identity uniquely resolves them. Chronicle semantic keys, source-record +IDs, display labels, benchmark values, and inferred identifiers are not used. +The current artifact contract does not publish an independent producer target +ID beyond `target_name`/`base_name`, so the dashboard does not invent one. + +Comparison rows include `comparison_id`, `match_kind`, both original target +names, and both per-row representations. Comparison summaries include collection +representations, counts matched by each method, and ambiguous key-group counts. +The pinned production artifact remains legacy-format; structured cross-release +matching is therefore covered with synthetic fixtures until a production pair +using the structured representation is available. + ### Producer follow-up Microcosm release producers must publish all of the following before the legacy diff --git a/frontend/app/api/microcosm/compare/route.ts b/frontend/app/api/microcosm/compare/route.ts index 1ca81e01..bc6fa558 100644 --- a/frontend/app/api/microcosm/compare/route.ts +++ b/frontend/app/api/microcosm/compare/route.ts @@ -8,7 +8,7 @@ import { scrub, } from "@/lib/microcosm/latest-artifact"; -export const revalidate = 300; +export const revalidate = 21_600; export const runtime = "nodejs"; export const maxDuration = 300; diff --git a/frontend/app/api/microcosm/releases/route.ts b/frontend/app/api/microcosm/releases/route.ts index cc2b0558..d2a861d5 100644 --- a/frontend/app/api/microcosm/releases/route.ts +++ b/frontend/app/api/microcosm/releases/route.ts @@ -7,7 +7,7 @@ import { scrub, } from "@/lib/microcosm/latest-artifact"; -export const revalidate = 300; +export const revalidate = 21_600; export async function GET(request: Request) { const country = parseCountry(new URL(request.url).searchParams.get("country")); diff --git a/frontend/app/api/microcosm/route.ts b/frontend/app/api/microcosm/route.ts index f59c0ee6..7f43ad0d 100644 --- a/frontend/app/api/microcosm/route.ts +++ b/frontend/app/api/microcosm/route.ts @@ -13,7 +13,7 @@ import { scrub, } from "@/lib/microcosm/latest-artifact"; -export const revalidate = 300; +export const revalidate = 21_600; export const runtime = "nodejs"; export const maxDuration = 300; diff --git a/frontend/app/api/microcosm/staging/target-change-tree/route.ts b/frontend/app/api/microcosm/staging/target-change-tree/route.ts new file mode 100644 index 00000000..02bbd2fa --- /dev/null +++ b/frontend/app/api/microcosm/staging/target-change-tree/route.ts @@ -0,0 +1,72 @@ +import { NextResponse } from "next/server"; + +import { calibrationTreeRequestState } from "@/lib/microcosm/calibration-tree-request"; +import { + classifyApiError, + parseCountry, + scrub, +} from "@/lib/microcosm/latest-artifact"; +import { loadStagingTargetChangeDataset } from "@/lib/microcosm/staging-artifact"; +import type { TargetChangeMode } from "@/lib/microcosm/target-change"; +import { buildTargetChangeTree } from "@/lib/microcosm/target-change-tree"; + +export const dynamic = "force-dynamic"; +export const revalidate = 0; +export const runtime = "nodejs"; +export const maxDuration = 300; + +export async function GET(request: Request) { + const params = new URL(request.url).searchParams; + const runId = params.get("run")?.trim(); + const releaseId = params.get("release")?.trim(); + const requestedMode = params.get("mode")?.trim() || "reported"; + if (!runId) { + return NextResponse.json( + { detail: "Provide a staging run id via ?run=." }, + { status: 400 }, + ); + } + if (!releaseId || releaseId === "latest") { + return NextResponse.json( + { detail: "Provide the resolved current release id via ?release=." }, + { status: 400 }, + ); + } + if (requestedMode !== "reported" && requestedMode !== "shared") { + return NextResponse.json( + { detail: "Comparison mode must be reported or shared." }, + { status: 400 }, + ); + } + const mode = requestedMode as TargetChangeMode; + const country = parseCountry(params.get("country")); + try { + const dataset = await loadStagingTargetChangeDataset( + runId, + releaseId, + country, + ); + if (!dataset) { + return NextResponse.json( + { + available: false, + reason: "This staging run has not uploaded calibration diagnostics yet.", + }, + { headers: { "Cache-Control": "no-store" } }, + ); + } + return NextResponse.json( + scrub( + buildTargetChangeTree( + dataset, + calibrationTreeRequestState(params), + mode, + ), + ), + { headers: { "Cache-Control": "no-store" } }, + ); + } catch (error) { + const { status, body } = classifyApiError(error); + return NextResponse.json(body, { status }); + } +} diff --git a/frontend/app/api/microcosm/target-diagnostics/route.ts b/frontend/app/api/microcosm/target-diagnostics/route.ts index e9be3698..befa44bc 100644 --- a/frontend/app/api/microcosm/target-diagnostics/route.ts +++ b/frontend/app/api/microcosm/target-diagnostics/route.ts @@ -9,7 +9,7 @@ import { } from "@/lib/microcosm/latest-artifact"; import { loadStagingTargetDiagnostics } from "@/lib/microcosm/staging-artifact"; -export const revalidate = 300; +export const revalidate = 21_600; export const runtime = "nodejs"; export const maxDuration = 300; diff --git a/frontend/app/api/microcosm/target-investigation/route.ts b/frontend/app/api/microcosm/target-investigation/route.ts index cfef1005..3d19e740 100644 --- a/frontend/app/api/microcosm/target-investigation/route.ts +++ b/frontend/app/api/microcosm/target-investigation/route.ts @@ -8,7 +8,7 @@ import { scrub, } from "@/lib/microcosm/latest-artifact"; -export const revalidate = 300; +export const revalidate = 21_600; export async function GET(request: Request) { try { diff --git a/frontend/app/api/microcosm/target-tree/route.ts b/frontend/app/api/microcosm/target-tree/route.ts index d527da19..0e314c9b 100644 --- a/frontend/app/api/microcosm/target-tree/route.ts +++ b/frontend/app/api/microcosm/target-tree/route.ts @@ -1,15 +1,10 @@ import { NextResponse } from "next/server"; -import { - FIT_BANDS, - createExplorerState, - type CalibrationStatus, - type FitBand, -} from "@/lib/microcosm/calibration-explorer"; import { buildCalibrationTree, type CalibrationTreeTarget, } from "@/lib/microcosm/calibration-tree"; +import { calibrationTreeRequestState } from "@/lib/microcosm/calibration-tree-request"; import { classifyApiError, loadRelease, @@ -17,53 +12,17 @@ import { scrub, } from "@/lib/microcosm/latest-artifact"; -export const revalidate = 300; +export const revalidate = 21_600; export const runtime = "nodejs"; export const maxDuration = 300; -const CALIBRATION_STATUSES = new Set([ - "included", - "skipped", -]); - -function requestState(params: URLSearchParams) { - const state = createExplorerState(); - state.breakdown = params.get("breakdown") === "geography" ? "geography" : "program"; - const source = params.get("source")?.trim(); - const program = params.get("program")?.trim(); - const geography = params.get("path_geography")?.trim(); - if (geography) state.path.geography = geography; - if (source && program) { - state.path.source = source; - state.path.program = program; - for (const [key, value] of params.entries()) { - if (key.startsWith("dim.") && value.trim()) { - state.path.dimensions.push({ key: key.slice(4), value: value.trim() }); - } - } - state.path.target = params.get("target")?.trim() || undefined; - } - state.filters.geographyLevels = params.getAll("geography_level"); - state.filters.geographies = params.getAll("geography"); - state.filters.fitBands = params - .getAll("fit_band") - .filter((value): value is FitBand => FIT_BANDS.includes(value as FitBand)); - state.filters.calibrationStatuses = params - .getAll("status") - .map((value) => (value === "not_materialized" ? "skipped" : value)) - .filter((value): value is CalibrationStatus => - CALIBRATION_STATUSES.has(value as CalibrationStatus), - ); - return state; -} - export async function GET(request: Request) { const params = new URL(request.url).searchParams; const release = params.get("release") ?? "latest"; const country = parseCountry(params.get("country")); try { const calibration = await loadRelease(release, revalidate, country); - const state = requestState(params); + const state = calibrationTreeRequestState(params); return NextResponse.json( scrub( buildCalibrationTree( diff --git a/frontend/app/api/microcosm/target-treemap/route.ts b/frontend/app/api/microcosm/target-treemap/route.ts index b6049b8f..db910387 100644 --- a/frontend/app/api/microcosm/target-treemap/route.ts +++ b/frontend/app/api/microcosm/target-treemap/route.ts @@ -8,7 +8,7 @@ import { scrub, } from "@/lib/microcosm/latest-artifact"; -export const revalidate = 300; +export const revalidate = 21_600; export const runtime = "nodejs"; export const maxDuration = 300; diff --git a/frontend/app/api/microcosm/variable/route.ts b/frontend/app/api/microcosm/variable/route.ts index eed9a34e..f5254e0a 100644 --- a/frontend/app/api/microcosm/variable/route.ts +++ b/frontend/app/api/microcosm/variable/route.ts @@ -4,6 +4,7 @@ import { promisify } from "node:util"; import { NextResponse } from "next/server"; +import { PUBLISHED_RELEASE_CACHE_SECONDS } from "@/lib/api/cache-policy"; import { MICROCOSM_HF_REPO, loadPointerReleaseId, @@ -116,7 +117,7 @@ export async function GET(request: Request) { try { const release = requestedRelease === "latest" - ? (await loadPointerReleaseId(300)).release_id + ? (await loadPointerReleaseId(PUBLISHED_RELEASE_CACHE_SECONDS)).release_id : requestedRelease; if (process.env.VERCEL === "1" && !process.env.PYTHON) { return NextResponse.redirect( diff --git a/frontend/app/providers.tsx b/frontend/app/providers.tsx index 584c451b..91fb9f0c 100644 --- a/frontend/app/providers.tsx +++ b/frontend/app/providers.tsx @@ -5,6 +5,7 @@ import { ReactQueryDevtools } from "@tanstack/react-query-devtools"; import { useState } from "react"; import { CountryProvider } from "@/components/layout/country-context"; +import { PUBLISHED_RELEASE_STALE_TIME_MS } from "@/lib/api/cache-policy"; export function Providers({ children }: { children: React.ReactNode }) { const [queryClient] = useState( @@ -12,7 +13,7 @@ export function Providers({ children }: { children: React.ReactNode }) { new QueryClient({ defaultOptions: { queries: { - staleTime: 5 * 60 * 1000, + staleTime: PUBLISHED_RELEASE_STALE_TIME_MS, refetchOnWindowFocus: false, retry: 2, }, diff --git a/frontend/components/microcosm/microcosm-overview-view.tsx b/frontend/components/microcosm/microcosm-overview-view.tsx index f5f3a757..1ef2b437 100644 --- a/frontend/components/microcosm/microcosm-overview-view.tsx +++ b/frontend/components/microcosm/microcosm-overview-view.tsx @@ -1,6 +1,6 @@ "use client"; -import { type ReactNode, useMemo, useState } from "react"; +import { useMemo, useState } from "react"; import { Button } from "@policyengine/ui-kit"; import { @@ -19,6 +19,7 @@ import { EmptyState } from "@/components/shared/empty-state"; import { fmt, fmtCompact } from "@/components/shared/format"; import { HelpHint } from "@/components/shared/help-hint"; import { LoadingBlock } from "@/components/shared/LoadingBlock"; +import { OverviewMetric } from "@/components/shared/overview-metric"; import { PageHeader } from "@/components/shared/page-header"; import { SectionCard } from "@/components/shared/section-card"; import { StatusPill } from "@/components/shared/status-pill"; @@ -59,22 +60,6 @@ function fmtLoss(value: number | null | undefined, kind: LossKind): string { return value.toExponential(3).replace("e+", "e"); } -function OverviewMetric({ label, value }: { label: ReactNode; value: string }) { - return ( -
-
- {label} -
-
- {value} -
-
- ); -} - export function MicrocosmOverviewView({ initialCountry = "us", initialRelease = "", diff --git a/frontend/components/microcosm/microcosm-staging-view.tsx b/frontend/components/microcosm/microcosm-staging-view.tsx index e0a774b2..297ac992 100644 --- a/frontend/components/microcosm/microcosm-staging-view.tsx +++ b/frontend/components/microcosm/microcosm-staging-view.tsx @@ -1,19 +1,26 @@ "use client"; -import { useEffect, useMemo, useState } from "react"; +import { useCallback, useEffect, useMemo, useRef, useState } from "react"; import { useCountry } from "@/components/layout/country-context"; +import { StagingTargetChangeMap } from "@/components/microcosm/staging-target-change-map"; import { EmptyState } from "@/components/shared/empty-state"; import { + differingPercentDigits, fmtUnitValue, fmt, fmtCompact, fmtMoney, fmtSignedMoney, - releaseLabel, + shortReleaseId, } from "@/components/shared/format"; +import { HelpHint } from "@/components/shared/help-hint"; import { KpiCard } from "@/components/shared/kpi-card"; import { LoadingBlock } from "@/components/shared/LoadingBlock"; +import { + overviewMetricLabelTypographyClassName, + overviewMetricValueClassName, +} from "@/components/shared/overview-metric"; import { PageHeader } from "@/components/shared/page-header"; import { SectionCard } from "@/components/shared/section-card"; import { StatusPill, type StatusTone } from "@/components/shared/status-pill"; @@ -21,10 +28,19 @@ import { useMicrocosmStagingCompare, useMicrocosmStagingRun, useMicrocosmStagingRuns, + type MicrocosmStagingRunResponse, type MicrocosmStagingRunSummary, type ReformValidationRow, } from "@/lib/api/hooks/use-microcosm"; import { countryRegistration, hasCapability } from "@/lib/microcosm/countries"; +import { + formatStagingCurrentStatus, + formatStagingStatus, +} from "@/lib/microcosm/staging-status"; +import { + formatWeightedTargetError, + targetChangeMapIdentity, +} from "@/lib/microcosm/target-change-visualization"; type LossKind = "normalized_target_loss" | "raw_optimizer_objective" | undefined; @@ -65,9 +81,30 @@ function timeLabel(value: string | null | undefined): string { }); } -// A "running" run that hasn't reported for two hours is dead in practice — -// builds emit events at least every stage, and stages run minutes, not hours. -const STALL_MS = 2 * 60 * 60 * 1000; +function timestampFromId(value: string | null | undefined): string | null { + const match = value?.match(/(\d{4})(\d{2})(\d{2})T(\d{2})(\d{2})(\d{2})Z$/); + if (!match) return null; + const [, year, month, day, hour, minute, second] = match; + return `${year}-${month}-${day}T${hour}:${minute}:${second}Z`; +} + +function runStartTime(run: MicrocosmStagingRunSummary): string | null | undefined { + return ( + run.started_at ?? + timestampFromId(run.candidate_release_id) ?? + timestampFromId(run.run_id) ?? + run.updated_at + ); +} + +function runLabel(run: MicrocosmStagingRunSummary): string { + return `${timeLabel(runStartTime(run))} · ${shortReleaseId( + run.candidate_release_id || run.run_id, + )}`; +} + +// Display a running or queued run as stalled after six hours without an update. +const STALL_MS = 6 * 60 * 60 * 1000; function effectiveStatus( status: string | null | undefined, @@ -79,6 +116,140 @@ function effectiveStatus( return status ?? null; } +const STATUS_INDICATOR_CLASS: Record = { + success: "swatch-pos", + warning: "swatch-warn", + danger: "swatch-neg", + info: "swatch-info", + neutral: "swatch-neutral", +}; + +function RunSelect({ + runs, + selected, + placeholder, + onSelect, +}: { + runs: MicrocosmStagingRunSummary[]; + selected: string; + placeholder: string; + onSelect: (runId: string) => void; +}) { + const [open, setOpen] = useState(false); + const rootRef = useRef(null); + const selectedRun = runs.find((run) => run.run_id === selected); + const selectedStatus = effectiveStatus( + selectedRun?.status, + selectedRun?.updated_at, + ); + const selectedTone = statusTone(selectedStatus); + const disabled = !runs.length; + + useEffect(() => { + if (!open) return; + const closeOnOutsideClick = (event: MouseEvent) => { + if (rootRef.current && !rootRef.current.contains(event.target as Node)) { + setOpen(false); + } + }; + const closeOnEscape = (event: KeyboardEvent) => { + if (event.key === "Escape") setOpen(false); + }; + document.addEventListener("mousedown", closeOnOutsideClick); + document.addEventListener("keydown", closeOnEscape); + return () => { + document.removeEventListener("mousedown", closeOnOutsideClick); + document.removeEventListener("keydown", closeOnEscape); + }; + }, [open]); + + return ( +
+ + + {open ? ( +
+
    + {runs.map((run) => { + const active = run.run_id === selected; + const shownStatus = effectiveStatus(run.status, run.updated_at); + const tone = statusTone(shownStatus); + return ( +
  • + +
    • + ); + })} +
+
+ ) : null} +
+ ); +} + function agoLabel(value: string | null | undefined): string { const t = value ? new Date(value).valueOf() : NaN; if (!Number.isFinite(t)) return ""; @@ -114,62 +285,6 @@ function durationLabel(ms: number | null): string { return `${Math.floor(s / 3600)}h ${Math.round((s % 3600) / 60)}m`; } -function RunList({ - runs, - selected, - onSelect, -}: { - runs: MicrocosmStagingRunSummary[]; - selected: string; - onSelect: (runId: string) => void; -}) { - if (!runs.length) { - return ( - - ); - } - return ( -
-
- {runs.map((run) => { - const active = run.run_id === selected; - return ( - - ); - })} -
-
- ); -} - function LossSparkline({ values }: { values: number[] }) { if (!values.length) return
No loss points yet.
; const finite = values.filter((value) => Number.isFinite(value)); @@ -190,12 +305,331 @@ function LossSparkline({ values }: { values: number[] }) { ); } +function RunInternalsPanel({ + runData, + lossValues, + status, + stage, + buildManifest, + artifacts, + open, + onOpenChange, + className = "", +}: { + runData: MicrocosmStagingRunResponse; + lossValues: number[]; + status: string | null; + stage: string | null; + buildManifest: Record | null; + artifacts: Record; + open: boolean; + onOpenChange: (open: boolean) => void; + className?: string; +}) { + return ( +
onOpenChange(event.currentTarget.open)} + className={`group overflow-hidden rounded-lg border border-border/80 bg-card shadow-[var(--elev-1)] ${className}`} + > + +
+
Run internals
+
+ Optimizer progress, stage timeline with logged numbers, build manifest (versions, + hashes, validation results), and uploaded artifacts. +
+
+ + ▾ + + +
+ +
+ +
+
+
+ Latest loss +
+
{fmt(lossValues.at(-1), { digits: 4 })}
+
+
+
+ Best loss +
+
+ {lossValues.length ? fmt(Math.min(...lossValues), { digits: 4 }) : "—"} +
+
+
+
Stage
+ {stage || status || "unknown"} +
+
+
+ + + {runData.calibration ? ( + +
+ + + + +
+ + ) : ( + + + + )} + + + {(runData.events ?? []).length ? ( +
+ + + + + + + + + + + {(runData.events ?? []).map((event, index, all) => { + const time = typeof event.time === "string" ? event.time : null; + const next = all[index + 1]; + const nextTime = next && typeof next.time === "string" ? next.time : null; + const duration = + time && nextTime + ? new Date(nextTime).valueOf() - new Date(time).valueOf() + : null; + const chips = detailChips(event.details); + const failed = event.status === "failed"; + return ( + + + + + + + ); + })} + +
StageStartedDurationDetail
+ + {String(event.stage ?? "—")} + + {failed && failed} + + {timeLabel(time)} + + {index === all.length - 1 && event.stage !== "complete" && !failed + ? status === "stalled" + ? "⚠ last event" + : "…" + : durationLabel(duration)} + +
+ {String(event.message ?? "—")} +
+ {chips.length > 0 && ( +
+ {chips.map(([key, value]) => ( + + {key}={value} + + ))} +
+ )} +
+
+ ) : ( + + )} + + + {buildManifest && ( + +
+ {(() => { + const code = (buildManifest.code ?? {}) as Record; + const runtime = (buildManifest.runtime ?? {}) as Record; + const gates = (buildManifest.gates ?? {}) as Record; + const dataset = (buildManifest.dataset ?? {}) as Record; + return ( + <> +
+ + {Object.entries(runtime) + .filter(([key]) => + ["python", "policyengine-us", "policyengine-core", "torch"].includes( + key, + ), + ) + .map(([key, value]) => ( +
+ {key} + {String(value)} +
+ ))} +
+ Dataset sha256 + + {String(dataset.sha256 ?? "—").slice(0, 12)}… + +
+
+ {Object.keys(gates).length > 0 && ( +
+
+ Validation results +
+
+ {Object.entries(gates).map(([name, result]) => { + const validation = (result ?? {}) as Record; + const passed = validation.passed === true; + const failures = Array.isArray(validation.failures) + ? validation.failures + : []; + return ( +
+ {name}{" "} + {passed ? "passed" : "failed"} + {failures.length > 0 && ( + + {failures.map((failure) => String(failure)).join("; ")} + + )} +
+ ); + })} +
+
+ )} + + ); + })()} +
+ + )} + + {Object.keys(artifacts).length > 0 && ( + + + + {Object.entries(artifacts).map(([name, metadata]) => ( + + + + + ))} + +
{name} + {metadata.staging_path ? ( + + {metadata.staging_path} + + ) : ( + (metadata.path ?? "—") + )} +
+ + )} +
+
+ ); +} + function ReformValidationTable({ rows }: { rows: ReformValidationRow[] }) { - const ordered = [...rows] - .filter((row) => row.microcosm_estimate != null || row.jct_score != null) - .sort((a, b) => Number(a.in_sample ?? false) - Number(b.in_sample ?? false)); + const ordered = rows.filter( + (row) => + !row.in_sample && + (row.microcosm_estimate != null || row.jct_score != null), + ); if (!ordered.length) { - return ; + return ; } return (
@@ -213,12 +647,7 @@ function ReformValidationTable({ rows }: { rows: ReformValidationRow[] }) { return ( -
- {row.name} - - {row.in_sample ? "in-sample" : "out-of-sample"} - -
+ {row.name}
{row.category || "Reform score"}
@@ -249,69 +678,91 @@ function ReformValidationTable({ rows }: { rows: ReformValidationRow[] }) { ); } -// Common-target fit stats for the candidate-vs-published verdict: computed on -// the SAME targets, since headline within-10% rates over different target sets -// (32k national-only vs 4k) are not comparable. +// Unweighted common-target statistics are computed on the same targets because +// rates over different target sets are not directly comparable. interface SideStats { n: number; within10: number; median: number | null; - mean: number | null; } +const VALIDATION_METHOD_HELP = { + targetWithin10: + "Measures the share of targets present in both releases whose absolute relative error is at most 10% of the benchmark value; higher is better. This metric is restricted to shared targets to ensure direct comparability.", + targetMedianAbsoluteError: + "Calculates the median absolute relative error across targets present in both releases. Lower is better.", + weightedTargetError: + "Calculates the importance-weighted mean of each release's scaled target errors after applying its target-loss cap. This is the same aggregate used by the Calibration map and the reported Target error change comparison; lower is better.", + reformMeanAbsoluteError: + "Calculates the mean absolute relative error between the candidate's estimated reform effects and the external benchmarks for out-of-sample reforms. Out-of-sample means that the reform includes values that are not used as calibration targets; lower is better.", + reformWithin10: + "Measures the share of scored out-of-sample reforms whose estimated effect is within 10% of the external benchmark. Out-of-sample means that the reform includes values that are not used as calibration targets; higher is better.", + targetCoverage: + "Counts the complete set of calibration targets available in each release, including targets that are not shared. The verdict reports how many targets the candidate adds and removes relative to the current release.", +} as const; + function sideStats(errors: number[]): SideStats { - if (!errors.length) return { n: 0, within10: 0, median: null, mean: null }; + if (!errors.length) return { n: 0, within10: 0, median: null }; const sorted = [...errors].sort((a, b) => a - b); const mid = Math.floor(sorted.length / 2); return { n: errors.length, within10: errors.filter((e) => e <= 0.1).length, median: sorted.length % 2 ? sorted[mid] : (sorted[mid - 1] + sorted[mid]) / 2, - mean: errors.reduce((s, e) => s + e, 0) / errors.length, }; } // One row of the validation scorecard: a metric on both sides plus a verdict. function ScoreRow({ label, - published, + about, + currentRelease, candidate, higherBetter, render = pct, }: { label: string; - published: number | null; + about: string; + currentRelease: number | null; candidate: number | null; higherBetter: boolean; render?: (v: number | null | undefined) => string; }) { const better = - published != null && candidate != null + currentRelease != null && candidate != null ? higherBetter - ? candidate > published + 1e-6 - : candidate < published - 1e-6 + ? candidate > currentRelease + 1e-6 + : candidate < currentRelease - 1e-6 : null; const worse = - published != null && candidate != null + currentRelease != null && candidate != null ? higherBetter - ? candidate < published - 1e-6 - : candidate > published + 1e-6 + ? candidate < currentRelease - 1e-6 + : candidate > currentRelease + 1e-6 : null; return ( - - {label} - - {render(published)} + + {label} + + {render(currentRelease)} - + {render(candidate)} - {better ? "candidate better" : worse ? "candidate worse" : published == null || candidate == null ? "—" : "tie"} + {better ? "candidate better" : worse ? "candidate worse" : currentRelease == null || candidate == null ? "—" : "tie"} + + + About {label}} + tooltip={about} + interaction="click" + underline={false} + /> ); @@ -340,14 +791,25 @@ function MicrocosmStagingRunsView() { const runs = runsData?.runs ?? []; const [selectedRun, setSelectedRun] = useState(""); const [targetSearch, setTargetSearch] = useState(""); + const [runInternalsOpen, setRunInternalsOpen] = useState(false); + + const resetRunVisualState = useCallback((runId: string) => { + setTargetSearch(""); + setRunInternalsOpen(false); + setSelectedRun(runId); + }, []); useEffect(() => { - if (!selectedRun && runs[0]) setSelectedRun(runs[0].run_id); - }, [runs, selectedRun]); + if (!selectedRun && runs[0]) resetRunVisualState(runs[0].run_id); + }, [resetRunVisualState, runs, selectedRun]); const { data: runData, isLoading: runLoading, error: runError } = useMicrocosmStagingRun(selectedRun); - const { data: compareData, isLoading: compareLoading } = useMicrocosmStagingCompare( + const { + data: compareData, + isLoading: compareLoading, + error: compareError, + } = useMicrocosmStagingCompare( runData?.has_calibration ? selectedRun : undefined, "latest", ); @@ -367,6 +829,10 @@ function MicrocosmStagingRunsView() { } return { a: sideStats(a), b: sideStats(b) }; }, [compareData]); + const medianAbsoluteErrorDigits = differingPercentDigits( + commonStats.a.median, + commonStats.b.median, + ); const calibrationEvents = runData?.calibration_progress?.events ?? []; const lossValues = useMemo( () => @@ -375,18 +841,29 @@ function MicrocosmStagingRunsView() { .filter((value): value is number => value != null), [calibrationEvents], ); - const lastCalibrationEvent = calibrationEvents.at(-1); const progress = runData?.progress ?? {}; const rawStatus = typeof progress.status === "string" ? progress.status : null; const stage = typeof progress.stage === "string" ? progress.stage : null; - const message = typeof progress.message === "string" ? progress.message : null; const updatedAt = typeof progress.updated_at === "string" ? progress.updated_at : null; const status = effectiveStatus(rawStatus, updatedAt); - const progressDetails = detailChips(progress.details); + const statusLabel = formatStagingStatus(status); + const lastUpdate = `${timeLabel(updatedAt)}${agoLabel(updatedAt) ? ` · ${agoLabel(updatedAt)}` : ""}`; + const currentStatus = formatStagingCurrentStatus(progress); const candidateReleaseId = runData?.candidate_release_id ?? selectedRun; const buildManifest = (runData?.build_manifest ?? null) as Record | null; const artifacts = ((runData?.run_manifest as Record | null)?.artifacts ?? {}) as Record; + const hasCandidateValidation = Boolean(compareData?.summary || runData?.reform_validation); + const targetComparisonPending = Boolean( + runData?.has_calibration && compareLoading && !compareData, + ); + const candidateValidationPending = targetComparisonPending && !hasCandidateValidation; + const showsCandidateValidation = hasCandidateValidation || candidateValidationPending; + const runSelectPlaceholder = runsLoading + ? "Loading runs…" + : runsError || runsData?.available === false + ? "Runs unavailable" + : "No staging runs"; return (
@@ -394,150 +871,179 @@ function MicrocosmStagingRunsView() { eyebrow="Microcosm · staging" title="Staging candidates" description="Monitor Microcosm build candidates before they are promoted to the published Hugging Face release channel." + actions={ + + } /> -
- - {runsLoading ? ( - +
+ {runsLoading && !runs.length ? ( + ) : runsError ? ( ) : runsData && runsData.available === false ? ( - ) : ( - - )} - - -
- {!selectedRun ? ( - - ) : runLoading ? ( - - ) : runError || !runData ? ( + ) : !runs.length ? ( + ) : !selectedRun ? ( +
+ +
+ ) : runLoading ? ( +
+ +
+ ) : runError || !runData ? ( +
+ +
) : ( <> -
- - - - -
- - {progressDetails.length > 0 && ( -
- Last stage detail: - {progressDetails.map(([k, v]) => ( - - {k}={v} - - ))} +
+ +
+
+ Candidate +
+
+ {candidateReleaseId} +
+
+ Status +
+
+ {statusLabel} +
+
+ Last update +
+
+ {lastUpdate} +
+
+ Current status +
+
+ {currentStatus} +
- )} + - {runData.has_calibration && compareLoading && !compareData && ( - + {candidateValidationPending && ( + + + )} - {(compareData?.summary || runData.reform_validation) && ( + {hasCandidateValidation && ( - - - - - - - +
+
Validation pointPublishedCandidateVerdict
+ + + + + + + - + + {targetComparisonPending && ( + + + + )} {compareData?.summary && ( <> fmt(value, { + pct: true, + digits: medianAbsoluteErrorDigits, + })} /> )} {runData.reform_validation && ( <> 0 ? (runData.reform_validation.summary?.out_of_sample_within_10pct ?? @@ -559,53 +1066,110 @@ function MicrocosmStagingRunsView() { )} {compareData?.summary && ( - - - + + - - + )} -
Validation pointCurrent releaseCandidateVerdictAbout
+ Loading target comparison… +
Coverage · target surface +
Coverage target surface {fmt(compareData.a.total_targets, { digits: 0 })} + {fmt(compareData.b.total_targets, { digits: 0 })} + +{fmt(compareData.summary.added, { digits: 0 })} new / - {fmt(compareData.summary.removed, { digits: 0 })} dropped + About Coverage target surface} + tooltip={VALIDATION_METHOD_HELP.targetCoverage} + interaction="click" + underline={false} + /> +
+ +
{compareData?.summary && ( -
- - {fmt(compareData.summary.improved, { digits: 0 })} targets improved - - {" · "} - - {fmt(compareData.summary.regressed, { digits: 0 })} regressed - {" "} - — search the breakdown below for any statistic. +
+
+ + {fmt(compareData.summary.improved, { digits: 0 })} targets improved + + {" · "} + + {fmt(compareData.summary.regressed, { digits: 0 })} regressed + +
)} )} - {(compareData?.rows ?? []).length > 0 && ( + {!showsCandidateValidation && ( + + )} +
+ +
+ {runData.has_calibration && ( + + {compareData?.summary ? ( + + ) : compareLoading ? ( + + ) : ( + + )} + + )} + + {runData.has_calibration && ( setTargetSearch(e.target.value)} - className="h-8 w-56 rounded-md border border-border bg-card px-2.5 text-sm focus:border-primary/60 focus:outline-none" - /> + compareData?.summary ? ( + setTargetSearch(e.target.value)} + className="h-8 w-56 rounded-md border border-border bg-card px-2.5 text-sm focus:border-primary/60 focus:outline-none" + /> + ) : undefined } padded={false} > - {(() => { + {compareData?.summary ? (() => { const q = targetSearch.trim().toLowerCase(); const usable = (compareData?.rows ?? []).filter( (row) => @@ -644,7 +1208,7 @@ function MicrocosmStagingRunsView() { Target - Published + Current release Candidate Δ @@ -708,330 +1272,64 @@ function MicrocosmStagingRunsView() {
); - })()} - - )} - - {runData.reform_validation ? ( - - - - ) : ( - - - - )} - -
- -
-
- Run internals -
-
- Optimizer progress, stage timeline with logged numbers, build manifest - (versions, hashes, gates), and uploaded artifacts. -
-
- - ▾ - - -
- -
- -
-
-
- Latest loss -
-
{fmt(lossValues.at(-1), { digits: 4 })}
-
-
-
- Best loss -
-
- {lossValues.length ? fmt(Math.min(...lossValues), { digits: 4 }) : "—"} -
-
-
-
- Stage -
- {stage || status || "unknown"} -
-
-
- - - {runData.calibration ? ( - -
- - - - -
- - ) : ( - - + )} )} - - - - {(runData.events ?? []).length ? ( -
- - - - - - - - - - - {(runData.events ?? []).map((event, index, all) => { - const time = typeof event.time === "string" ? event.time : null; - const next = all[index + 1]; - const nextTime = - next && typeof next.time === "string" ? next.time : null; - const duration = - time && nextTime - ? new Date(nextTime).valueOf() - new Date(time).valueOf() - : null; - const chips = detailChips(event.details); - const failed = event.status === "failed"; - return ( - - - - - - - ); - })} - -
StageStartedDurationDetail
- - {String(event.stage ?? "—")} - - {failed && ( - failed - )} - - {timeLabel(time)} - - {index === all.length - 1 && - event.stage !== "complete" && - !failed - ? status === "stalled" - ? "⚠ last event" - : "…" - : durationLabel(duration)} - -
- {String(event.message ?? "—")} -
- {chips.length > 0 && ( -
- {chips.map(([k, v]) => ( - - {k}={v} - - ))} -
- )} -
-
- ) : ( - - )} - - - {buildManifest && ( + {runData.reform_validation && ( + Cross-release external comparisons live in the{" "} + + PolicyEngine scorecard + + . + + } + padded={false} > -
- {(() => { - const code = (buildManifest.code ?? {}) as Record; - const runtime = (buildManifest.runtime ?? {}) as Record; - const gates = (buildManifest.gates ?? {}) as Record; - const dataset = (buildManifest.dataset ?? {}) as Record; - return ( - <> -
- - {Object.entries(runtime) - .filter(([k]) => - ["python", "policyengine-us", "policyengine-core", "torch"].includes(k), - ) - .map(([k, v]) => ( -
- {k} - {String(v)} -
- ))} -
- Dataset sha256 - - {String(dataset.sha256 ?? "—").slice(0, 12)}… - -
-
- {Object.keys(gates).length > 0 && ( -
-
- Gates -
-
- {Object.entries(gates).map(([name, result]) => { - const r = (result ?? {}) as Record; - const passed = r.passed === true; - const failures = Array.isArray(r.failures) ? r.failures : []; - return ( -
- {name}{" "} - {passed ? "passed" : "failed"} - {failures.length > 0 && ( - - {failures.map((f) => String(f)).join("; ")} - - )} -
- ); - })} -
-
- )} - - ); - })()} -
+ )} - {Object.keys(artifacts).length > 0 && ( - - - - {Object.entries(artifacts).map(([name, meta]) => ( - - - - - ))} - -
{name} - {meta.staging_path ? ( - - {meta.staging_path} - - ) : ( - (meta.path ?? "—") - )} -
- + {showsCandidateValidation && ( + )} -
-
+
)} -
); diff --git a/frontend/components/microcosm/staging-target-change-map.tsx b/frontend/components/microcosm/staging-target-change-map.tsx new file mode 100644 index 00000000..a6e8bee6 --- /dev/null +++ b/frontend/components/microcosm/staging-target-change-map.tsx @@ -0,0 +1,649 @@ +"use client"; + +import { + useEffect, + useReducer, + useRef, + useState, +} from "react"; + +import { + explorerBreadcrumbs, + explorerNodeLabel, + explorerUpLabel, +} from "@/components/microcosm/calibration-explorer-view"; +import { fmt } from "@/components/shared/format"; +import { HelpHint } from "@/components/shared/help-hint"; +import { LoadingBlock } from "@/components/shared/LoadingBlock"; +import { StatusPill } from "@/components/shared/status-pill"; +import { useMicrocosmStagingTargetChangeTree } from "@/lib/api/hooks/use-microcosm"; +import { + createExplorerState, + explorerReducer, +} from "@/lib/microcosm/calibration-explorer"; +import type { + CalibrationTreeGroup, + CalibrationTreeMetrics, +} from "@/lib/microcosm/calibration-tree"; +import { + condenseCalibrationTreemapByMetric, + expandedGroupsForNode, + type CalibrationTreemapGroup, + type CalibrationTreemapNode, +} from "@/lib/microcosm/calibration-treemap-layout"; +import type { + TargetChangeMode, + TargetChangeRow, +} from "@/lib/microcosm/target-change"; +import type { TargetChangeTreeResponse } from "@/lib/microcosm/target-change-tree"; +import { + targetMatchKindExplanation, + targetRepresentationPairLabel, +} from "@/lib/microcosm/target-matching-presentation"; +import { + formatTargetChange, + formatWeightedTargetError, + targetChangeDetailValues, + targetChangeDirectionAreas, + targetChangeGroupsForDirection, + targetChangeDirectionValue, + type TargetChangeDirection, + type TargetChangeDirectionData, +} from "@/lib/microcosm/target-change-visualization"; +import { squarify, type Placed, type Rect } from "@/lib/treemap/squarify"; + +const DIRECTION_GAP = 8; +const DIRECTION_HEADER_HEIGHT = 34; +const GROUP_GAP = 7; +const GROUP_HEADER_HEIGHT = 22; +const NODE_GAP = 3; + +const MODE_HELP: Record = { + reported: + "Uses each release's actual target weights and complete target surface. Added targets contribute candidate error, while removed targets subtract their current-release contribution.", + shared: + "Uses only targets present in both releases and applies the same pooled target weights to both. Each release still uses its own verified benchmark, scale, cap, and capped error.", +}; + +interface LaidGroup { + group: CalibrationTreemapGroup; + rect: Placed; + headerHeight: number; + nodes: Placed[]; +} + +interface LaidDirection { + direction: TargetChangeDirection; + label: string; + rect: Rect; + groups: LaidGroup[]; +} + +function metricFor(direction: TargetChangeDirection) { + return (metrics: CalibrationTreeMetrics) => + targetChangeDirectionValue(metrics, direction); +} + +function insetRect(rect: Rect, gap: number): Rect { + return { + x: rect.x + gap / 2, + y: rect.y + gap / 2, + w: Math.max(rect.w - gap, 0), + h: Math.max(rect.h - gap, 0), + }; +} + +function layoutGroups( + groups: CalibrationTreeGroup[], + direction: TargetChangeDirection, + rect: Rect, +): LaidGroup[] { + const metric = metricFor(direction); + const directionalGroups = targetChangeGroupsForDirection(groups, direction); + const condensed = condenseCalibrationTreemapByMetric( + directionalGroups, + metric, + rect.w, + rect.h, + ); + const placedGroups = squarify( + condensed + .map((group) => ({ value: metric(group.metrics), data: group })) + .filter((entry) => entry.value > 0), + rect, + ); + return placedGroups.map((placed) => { + const groupRect = insetRect(placed, GROUP_GAP); + const headerHeight = + !placed.data.synthetic && groupRect.h >= 58 && groupRect.w >= 90 + ? GROUP_HEADER_HEIGHT + : 0; + const inner = { + x: groupRect.x, + y: groupRect.y + headerHeight, + w: groupRect.w, + h: Math.max(groupRect.h - headerHeight, 0), + }; + const nodes = squarify( + placed.data.nodes + .map((node) => ({ value: metric(node.metrics), data: node })) + .filter((entry) => entry.value > 0), + inner, + ); + return { + group: placed.data, + rect: { ...placed, ...groupRect }, + headerHeight, + nodes, + }; + }); +} + +function layoutDirections( + data: TargetChangeTreeResponse, + width: number, + height: number, + expanded: { + label: string; + groups: CalibrationTreeGroup[]; + direction: TargetChangeDirection; + } | null, +): LaidDirection[] { + const displayedGroups = expanded?.groups ?? data.groups; + const areas: Array> = targetChangeDirectionAreas( + displayedGroups, + width, + height, + ).filter((area) => !expanded || area.data.direction === expanded.direction); + return areas.map((area) => { + const directionRect = insetRect(area, DIRECTION_GAP); + const contentRect = { + x: directionRect.x, + y: directionRect.y + DIRECTION_HEADER_HEIGHT, + w: directionRect.w, + h: Math.max(directionRect.h - DIRECTION_HEADER_HEIGHT, 0), + }; + return { + direction: area.data.direction, + label: area.data.label, + rect: directionRect, + groups: layoutGroups( + expanded?.groups ?? data.groups, + area.data.direction, + contentRect, + ), + }; + }); +} + +function isTreeResponse( + data: ReturnType["data"], +): data is TargetChangeTreeResponse { + return Boolean(data && "groups" in data); +} + +function Control({ + label, + value, + options, + onChange, +}: { + label: string; + value: T; + options: Array<{ value: T; label: string; tooltip?: string }>; + onChange: (value: T) => void; +}) { + return ( +
+ + {label} + +
+ {options.map((option) => ( +
+ + {option.tooltip ? ( + + About {option.label}} + tooltip={option.tooltip} + interaction="click" + underline={false} + inheritTypography + /> + + ) : null} +
+ ))} +
+
+ ); +} + +function targetStatus(target: TargetChangeRow) { + if (target.comparison_status === "added") { + return Added target; + } + if (target.comparison_status === "removed") { + return Removed target; + } + return Shared target; +} + +function nullableNumber(value: number | null | undefined): string { + return value == null ? "—" : fmt(value, { digits: 2 }); +} + +function nullablePercent(value: number | null | undefined): string { + return value == null ? "—" : formatWeightedTargetError(value); +} + +function TargetChangeDetail({ + target, + mode, + onClose, +}: { + target: TargetChangeRow; + mode: TargetChangeMode; + onClose: () => void; +}) { + const detail = targetChangeDetailValues(target, mode); + const rows = [ + { + label: "Target identifier", + current: target.current_name ?? "—", + candidate: target.candidate_name ?? "—", + }, + { + label: "Benchmark", + current: nullableNumber(target.current?.target), + candidate: nullableNumber(target.candidate?.target), + }, + { + label: "Final estimate", + current: nullableNumber(target.current?.finalEstimate), + candidate: nullableNumber(target.candidate?.finalEstimate), + }, + { + label: "Artifact target weight", + current: nullablePercent(detail.currentWeightShare), + candidate: nullablePercent(detail.candidateWeightShare), + }, + { + label: "Capped scaled error", + current: nullablePercent(target.current?.cappedError), + candidate: nullablePercent(target.candidate?.cappedError), + }, + { + label: mode === "reported" ? "Weighted error contribution" : "Pooled-weight contribution", + current: nullablePercent(detail.currentContribution), + candidate: nullablePercent(detail.candidateContribution), + }, + ]; + return ( +
+
+
+
+

+ {String(target.variable ?? target.name)} +

+ {targetStatus(target)} +
+

+ {targetRepresentationPairLabel( + target.current_representation, + target.candidate_representation, + )} +

+
+ +
+ {mode === "shared" && detail.comparisonWeight != null ? ( +
+ Shared comparison weight: {formatWeightedTargetError(detail.comparisonWeight)} on both sides. +
+ ) : null} + {target.match_kind ? ( +
+ {targetMatchKindExplanation(target.match_kind)} +
+ ) : null} +
+ + + + + + + + + + {rows.map((row) => ( + + + + + + ))} + +
ValueCurrent releaseCandidate
{row.label}{row.current}{row.candidate}
+
+
+ Selected-mode change + 0 + ? "font-semibold tabular-nums tone-neg" + : (detail.change ?? 0) < 0 + ? "font-semibold tabular-nums tone-pos" + : "font-semibold tabular-nums text-muted-foreground" + }> + {formatTargetChange(detail.change)} + +
+ {target.comparison_status === "removed" ? ( +

+ Removing this target lowers the reported aggregate mechanically. It does not show that the candidate fits this target better. +

+ ) : null} +
+ ); +} + +export function StagingTargetChangeMap({ + runId, + releaseId, +}: { + runId: string; + releaseId: string; +}) { + const [state, dispatch] = useReducer(explorerReducer, undefined, createExplorerState); + const [mode, setMode] = useState("reported"); + const [expanded, setExpanded] = useState<{ + label: string; + groups: CalibrationTreeGroup[]; + direction: TargetChangeDirection; + } | null>(null); + const containerRef = useRef(null); + const [size, setSize] = useState({ width: 960, height: 520 }); + const { data, isLoading, error } = useMicrocosmStagingTargetChangeTree({ + runId, + releaseId, + mode, + state, + }); + + useEffect(() => { + const element = containerRef.current; + if (!element) return; + const update = () => { + const rect = element.getBoundingClientRect(); + if (rect.width > 0 && rect.height > 0) { + setSize({ width: Math.round(rect.width), height: Math.round(rect.height) }); + } + }; + update(); + const observer = new ResizeObserver(update); + observer.observe(element); + return () => observer.disconnect(); + }, [data]); + + if (isLoading && !data) { + return ; + } + if (error) { + return ( +
+ {error instanceof Error ? error.message : "Target error change is unavailable."} +
+ ); + } + if (!data || !isTreeResponse(data)) { + return ( +
+ {data?.reason ?? "Target error change is unavailable."} +
+ ); + } + if (!data.available || !data.summary) { + return ( +
+ {data.reason ?? "Weighted target-error attribution is unavailable for this comparison."} +
+ ); + } + + const directions = layoutDirections(data, size.width, size.height, expanded); + const breadcrumbs = explorerBreadcrumbs(state); + const upLabel = expanded ? `Up to ${data.currentLevel.label.toLowerCase()}` : explorerUpLabel(state); + + return ( +
+
+
+ { + setExpanded(null); + dispatch({ type: "clear_target" }); + setMode(nextMode); + }} + /> + { + setExpanded(null); + dispatch({ type: "breakdown", breakdown }); + }} + /> +
+
+ +
+ {upLabel ? ( + + ) : null} + +
+ +
+ {directions.length === 0 ? ( +
+ No weighted target-error changes exceed the display tolerance at this level. +
+ ) : directions.map((direction) => { + const directionSign = direction.direction === "increase" ? 1 : -1; + const tone = direction.direction === "increase" ? "var(--neg)" : "var(--pos)"; + return ( +
+
+
+ {direction.label} +
+ {direction.groups.map(({ group, rect, headerHeight, nodes }) => ( +
+ {headerHeight > 0 ? ( +
+ {group.label} + + {formatTargetChange(directionSign * targetChangeDirectionValue(group.metrics, direction.direction), false)} + +
+ ) : null} + {nodes.map((placed) => { + const item = placed.data; + const tile = insetRect(placed, NODE_GAP); + if (tile.w < 2 || tile.h < 2) return null; + const itemLabel = item.kind === "grouped" + ? item.label + : explorerNodeLabel({ id: item.id, kind: item.kind, label: item.label }); + const value = targetChangeDirectionValue(item.metrics, direction.direction); + const showLabel = tile.w >= 44 && tile.h >= 24; + const showValue = tile.w >= 72 && tile.h >= 48; + const selected = item.kind === "target" && item.id === state.path.target; + return ( + + ); + })} +
+ ))} +
+ ); + })} +
+ +

+ Area shows each category's absolute net change in weighted target error at the current level. Each category appears once, on the side determined by its net change. Select a category to drill down or select a target to compare its values directly. +

+ + {data.selectedTarget ? ( + dispatch({ type: "clear_target" })} + /> + ) : null} +
+ ); +} diff --git a/frontend/components/shared/format.test.ts b/frontend/components/shared/format.test.ts index 1d9031ae..d53466bc 100644 --- a/frontend/components/shared/format.test.ts +++ b/frontend/components/shared/format.test.ts @@ -1,6 +1,22 @@ import { expect, test } from "bun:test"; -import { fmtUnitValue, releaseLabel } from "./format"; +import { + differingPercentDigits, + fmtUnitValue, + releaseLabel, + shortReleaseId, +} from "./format"; + +test("percent comparison precision increases until the rendered values differ", () => { + expect(differingPercentDigits(0.041234, 0.041236)).toBe(3); + expect(differingPercentDigits(0.04, 0.05)).toBe(1); +}); + +test("percent comparison precision stops at ten decimal places", () => { + expect(differingPercentDigits(0.04, 0.04)).toBe(10); + expect(differingPercentDigits(0.04, 0.04 + 1e-14)).toBe(10); + expect(differingPercentDigits(null, 0.04)).toBe(1); +}); test("percent-unit values render as percentages from decimal fractions", () => { expect(fmtUnitValue(0.134, "percent")).toBe("13.4%"); @@ -20,3 +36,10 @@ test("Belgium Chronicle release labels expose the distinguishing commit", () => releaseLabel("microcosm-be-2026-chronicle-3cef97b-20260823T134247Z"), ).toBe("2026-08-23 13:42Z · 3cef97b"); }); + +test("short release ids use six characters from the varying identifier", () => { + expect( + shortReleaseId("populace-us-2024-32fcf6b-480cc7c54024-20260702T134754Z"), + ).toBe("32fcf6"); + expect(shortReleaseId("rel-1")).toBe("rel"); +}); diff --git a/frontend/components/shared/format.ts b/frontend/components/shared/format.ts index 3976491a..379ab9c7 100644 --- a/frontend/components/shared/format.ts +++ b/frontend/components/shared/format.ts @@ -31,6 +31,13 @@ function releaseDateFromId(releaseId: string): string { return releaseId.match(/-(\d{8}(?:T\d{6}Z)?)$/)?.[1] ?? ""; } +// Release identifiers begin with a stable product/year prefix. Return the +// varying identifier segment so compact labels distinguish one release from +// another instead of all beginning with "popula". +export function shortReleaseId(releaseId: string, length = 6): string { + return releaseId.replace(/^populace-us-\d{4}-/, "").split("-")[0].slice(0, length); +} + // A readable label for a release: "2026-06-14 · f32c2e5". export function releaseLabel(releaseId: string, date?: string | null): string { // Deprecated upstream identifier: Microcosm release IDs still use the former @@ -60,6 +67,26 @@ export function fmt( return value.toFixed(opts.digits ?? 4); } +export function differingPercentDigits( + left: number | null | undefined, + right: number | null | undefined, + maxDigits = 10, +): number { + if ( + left == null || right == null || + !Number.isFinite(left) || !Number.isFinite(right) + ) { + return 1; + } + const limit = Math.max(1, Math.floor(maxDigits)); + for (let digits = 1; digits <= limit; digits += 1) { + if (fmt(left, { pct: true, digits }) !== fmt(right, { pct: true, digits })) { + return digits; + } + } + return limit; +} + export function fmtSigned( value: number | null | undefined, opts: { pct?: boolean; digits?: number } = {}, diff --git a/frontend/components/shared/help-hint.tsx b/frontend/components/shared/help-hint.tsx index 397af042..4ae9dc9a 100644 --- a/frontend/components/shared/help-hint.tsx +++ b/frontend/components/shared/help-hint.tsx @@ -1,6 +1,15 @@ "use client"; -import { type ReactNode, useEffect, useId, useRef, useState } from "react"; +import { + Popover, + PopoverContent, + PopoverTrigger, + Tooltip, + TooltipContent, + TooltipProvider, + TooltipTrigger, +} from "@policyengine/ui-kit"; +import type { ReactNode } from "react"; interface HelpHintProps { label: ReactNode; @@ -17,84 +26,54 @@ export function HelpHint({ underline = true, inheritTypography = false, }: HelpHintProps) { - const [open, setOpen] = useState(false); - const rootRef = useRef(null); - const tooltipId = useId(); - - useEffect(() => { - if (interaction !== "click" || !open) return; - - const onPointerDown = (event: MouseEvent) => { - if (rootRef.current && !rootRef.current.contains(event.target as Node)) { - setOpen(false); - } - }; - const onKeyDown = (event: KeyboardEvent) => { - if (event.key === "Escape") setOpen(false); - }; - - document.addEventListener("mousedown", onPointerDown); - document.addEventListener("keydown", onKeyDown); - return () => { - document.removeEventListener("mousedown", onPointerDown); - document.removeEventListener("keydown", onKeyDown); - }; - }, [interaction, open]); - const typography = inheritTypography ? "" : "normal-case tracking-normal"; const labelClass = underline ? "underline decoration-dotted underline-offset-2" : ""; + const triggerClassName = `inline-flex cursor-pointer items-center gap-1 text-inherit ${typography} ${ + inheritTypography ? "uppercase" : "" + }`; + const trigger = ( + + ); + const overlayClassName = + "z-[100] max-h-[var(--radix-popper-available-height)] w-[min(18rem,calc(100vw-1.5rem))] overflow-y-auto border-border bg-popover p-3 text-left text-xs font-normal normal-case leading-snug tracking-normal text-popover-foreground shadow-lg"; if (interaction === "click") { return ( - - - {open && ( - - {tooltip} - - )} - + {tooltip} + + ); } return ( - - {label} - - ? - - - {tooltip} - - + + + {trigger} + + {tooltip} + + + ); } diff --git a/frontend/components/shared/overview-metric.tsx b/frontend/components/shared/overview-metric.tsx new file mode 100644 index 00000000..b0aff2b5 --- /dev/null +++ b/frontend/components/shared/overview-metric.tsx @@ -0,0 +1,28 @@ +import type { ReactNode } from "react"; + +export const overviewMetricLabelTypographyClassName = + "text-[10px] font-semibold uppercase leading-tight tracking-[0.12em]"; + +export const overviewMetricLabelClassName = + `${overviewMetricLabelTypographyClassName} text-muted-foreground`; + +export const overviewMetricValueClassName = + "font-semibold tabular-nums text-foreground"; + +export function OverviewMetric({ label, value }: { label: ReactNode; value: string }) { + return ( +
+
+ {label} +
+
+ {value} +
+
+ ); +} diff --git a/frontend/lib/api/cache-policy.ts b/frontend/lib/api/cache-policy.ts new file mode 100644 index 00000000..81f1167f --- /dev/null +++ b/frontend/lib/api/cache-policy.ts @@ -0,0 +1,2 @@ +export const PUBLISHED_RELEASE_CACHE_SECONDS = 6 * 60 * 60; +export const PUBLISHED_RELEASE_STALE_TIME_MS = PUBLISHED_RELEASE_CACHE_SECONDS * 1000; diff --git a/frontend/lib/api/client.ts b/frontend/lib/api/client.ts index 36285db8..5f32df33 100644 --- a/frontend/lib/api/client.ts +++ b/frontend/lib/api/client.ts @@ -9,7 +9,9 @@ type ParamValue = string | number | boolean | undefined | null | (string | numbe function apiUrl(path: string): URL { if (EXPLICIT_API_BASE) return new URL(path, EXPLICIT_API_BASE); const origin = - typeof window === "undefined" ? "http://localhost:3000" : window.location.origin; + typeof window === "undefined" + ? `http://127.0.0.1:${process.env.PORT ?? "3000"}` + : window.location.origin; // Next.js API route handlers live under the app basePath; the native Python // function is routed back to root by a rewrite (see next.config.ts), so all // API calls resolve through `${origin}${BASE_PATH}/api`. diff --git a/frontend/lib/api/hooks/use-microcosm.ts b/frontend/lib/api/hooks/use-microcosm.ts index 77ff7e99..faae5e70 100644 --- a/frontend/lib/api/hooks/use-microcosm.ts +++ b/frontend/lib/api/hooks/use-microcosm.ts @@ -4,9 +4,12 @@ import { useCountry, type Country, } from "@/components/layout/country-context"; +import { PUBLISHED_RELEASE_STALE_TIME_MS } from "@/lib/api/cache-policy"; import { withBasePath } from "@/lib/base-path"; import type { ExplorerState } from "@/lib/microcosm/calibration-explorer"; import type { CalibrationTreeResponse } from "@/lib/microcosm/calibration-tree"; +import type { TargetChangeMode } from "@/lib/microcosm/target-change"; +import type { TargetChangeTreeApiResponse } from "@/lib/microcosm/target-change-tree"; import { hasCapability, type CountryCapability, @@ -81,6 +84,7 @@ export interface MicrocosmTargetRow { rank?: number; }[] | null; dimension_adapter?: "structured" | "legacy_filter" | "legacy_name" | null; + target_representation?: "legacy" | "structured" | null; variable_key?: string | null; // schema v2 published registry metadata (null on v1). source_citation?: string | null; @@ -377,6 +381,12 @@ export interface MicrocosmTargetDiagnostics { export interface MicrocosmComparisonRow { name: string; + comparison_id?: string; + match_kind?: "base_name" | "chronicle_fact_key" | "structured_identity" | null; + current_name?: string | null; + candidate_name?: string | null; + current_representation?: "legacy" | "structured" | null; + candidate_representation?: "legacy" | "structured" | null; target_label?: string | null; source?: string | null; variable_key?: string | null; @@ -432,6 +442,7 @@ export interface MicrocosmComparison { initial_loss: number | null; final_loss: number | null; loss_kind: "normalized_target_loss" | "raw_optimizer_objective"; + weighted_target_error: number | null; fraction_within_10pct: number | null; }; b: { @@ -441,6 +452,7 @@ export interface MicrocosmComparison { initial_loss: number | null; final_loss: number | null; loss_kind: "normalized_target_loss" | "raw_optimizer_objective"; + weighted_target_error: number | null; fraction_within_10pct: number | null; }; summary: { @@ -452,6 +464,20 @@ export interface MicrocosmComparison { unchanged: number; losses_comparable: boolean; loss_kind: "normalized_target_loss" | "raw_optimizer_objective" | "mixed"; + matching: { + current_representation: "legacy" | "structured" | "mixed" | "unknown"; + candidate_representation: "legacy" | "structured" | "mixed" | "unknown"; + matched_by: { + base_name: number; + chronicle_fact_key: number; + structured_identity: number; + }; + ambiguous_key_groups: { + base_name: number; + chronicle_fact_key: number; + structured_identity: number; + }; + }; }; variables: MicrocosmComparisonVariableRow[]; rows: MicrocosmComparisonRow[]; @@ -672,7 +698,6 @@ export function useMicrocosmStagingRun(runId?: string) { country, }), enabled: staging && Boolean(runId), - placeholderData: keepPreviousData, staleTime: 10 * 1000, refetchInterval: staging ? 30 * 1000 : false, }); @@ -688,7 +713,47 @@ export function useMicrocosmStagingCompare(runId?: string, release = "latest") { { run: runId, release, country }, ), enabled: hasCapability(country, "staging") && Boolean(runId), - placeholderData: keepPreviousData, + staleTime: 30 * 1000, + }); +} + +export function useMicrocosmStagingTargetChangeTree({ + runId, + releaseId, + mode, + state, +}: { + runId?: string; + releaseId?: string; + mode: TargetChangeMode; + state: ExplorerState; +}) { + const { country } = useCountry(); + return useQuery({ + queryKey: [ + "microcosm", + "staging", + "target-change-tree", + country, + runId, + releaseId, + mode, + state, + ], + queryFn: () => + apiGet( + "/microcosm/staging/target-change-tree", + { + ...explorerApiParams(state), + run: runId, + release: releaseId, + mode, + country, + }, + ), + enabled: + hasCapability(country, "staging") && + Boolean(runId && releaseId && releaseId !== "latest"), staleTime: 30 * 1000, }); } @@ -698,7 +763,7 @@ export function useMicrocosmReleases() { return useQuery({ queryKey: ["microcosm", "releases", country], queryFn: () => apiGet("/microcosm/releases", { country }), - staleTime: 5 * 60 * 1000, + staleTime: PUBLISHED_RELEASE_STALE_TIME_MS, }); } @@ -739,7 +804,7 @@ export function useMicrocosm(release?: string) { queryKey: ["microcosm", country, release ?? "latest"], queryFn: () => apiGet("/microcosm", { release: release || undefined, country }), - staleTime: 5 * 60 * 1000, + staleTime: PUBLISHED_RELEASE_STALE_TIME_MS, }); } @@ -794,7 +859,7 @@ export function useMicrocosmTargetTreemap(release?: string, breakdown?: "program breakdown: breakdown || undefined, country, }), - staleTime: 5 * 60 * 1000, + staleTime: PUBLISHED_RELEASE_STALE_TIME_MS, }); } @@ -846,7 +911,7 @@ export function microcosmCalibrationTreeQueryOptions( release: release || undefined, country, }), - staleTime: 5 * 60 * 1000, + staleTime: PUBLISHED_RELEASE_STALE_TIME_MS, }; } diff --git a/frontend/lib/microcosm/calibration-tree-request.ts b/frontend/lib/microcosm/calibration-tree-request.ts new file mode 100644 index 00000000..f3dbadb1 --- /dev/null +++ b/frontend/lib/microcosm/calibration-tree-request.ts @@ -0,0 +1,42 @@ +import { + FIT_BANDS, + createExplorerState, + type CalibrationStatus, + type FitBand, +} from "./calibration-explorer"; + +const CALIBRATION_STATUSES = new Set([ + "included", + "skipped", +]); + +export function calibrationTreeRequestState(params: URLSearchParams) { + const state = createExplorerState(); + state.breakdown = params.get("breakdown") === "geography" ? "geography" : "program"; + const source = params.get("source")?.trim(); + const program = params.get("program")?.trim(); + const geography = params.get("path_geography")?.trim(); + if (geography) state.path.geography = geography; + if (source && program) { + state.path.source = source; + state.path.program = program; + for (const [key, value] of params.entries()) { + if (key.startsWith("dim.") && value.trim()) { + state.path.dimensions.push({ key: key.slice(4), value: value.trim() }); + } + } + state.path.target = params.get("target")?.trim() || undefined; + } + state.filters.geographyLevels = params.getAll("geography_level"); + state.filters.geographies = params.getAll("geography"); + state.filters.fitBands = params + .getAll("fit_band") + .filter((value): value is FitBand => FIT_BANDS.includes(value as FitBand)); + state.filters.calibrationStatuses = params + .getAll("status") + .map((value) => (value === "not_materialized" ? "skipped" : value)) + .filter((value): value is CalibrationStatus => + CALIBRATION_STATUSES.has(value as CalibrationStatus), + ); + return state; +} diff --git a/frontend/lib/microcosm/calibration-tree.ts b/frontend/lib/microcosm/calibration-tree.ts index 80290aae..cc64c832 100644 --- a/frontend/lib/microcosm/calibration-tree.ts +++ b/frontend/lib/microcosm/calibration-tree.ts @@ -21,6 +21,7 @@ export interface CalibrationTreeDimension { export interface CalibrationTreeTarget { name?: string | null; base_name?: string | null; + comparison_id?: string | null; source?: string | null; source_label?: string | null; variable?: string | null; @@ -34,11 +35,24 @@ export interface CalibrationTreeTarget { target_loss_weight_share?: number | null; final_capped_scaled_error?: number | null; final_loss_contribution?: number | null; + target_change?: number | null; + comparison_status?: "shared" | "added" | "removed" | null; calibration_status?: CalibrationStatus | "not_materialized" | null; target_dimensions?: CalibrationTreeDimension[] | null; [key: string]: unknown; } +export interface CalibrationTreeChangeMetrics { + increasedError: number; + reducedError: number; + netChange: number; + changedTargets: number; + unchangedTargets: number; + sharedTargets: number; + addedTargets: number; + removedTargets: number; +} + export interface CalibrationTreeMetrics { nTargets: number; scored: number; @@ -48,6 +62,7 @@ export interface CalibrationTreeMetrics { weightedAverageCappedError: number | null; meanAbsRelativeError: number | null; medianAbsRelativeError: number | null; + change?: CalibrationTreeChangeMetrics; } export interface CalibrationTreeNode { @@ -89,6 +104,10 @@ export interface CalibrationTreeResponse { export type CalibrationTreeSizeMode = "targets" | "weight" | "loss"; +function finiteNumber(value: unknown): number | null { + return typeof value === "number" && Number.isFinite(value) ? value : null; +} + function finiteError(row: CalibrationTreeTarget): number | null { const value = row.abs_relative_error; return typeof value === "number" && Number.isFinite(value) ? Math.abs(value) : null; @@ -156,6 +175,23 @@ export function calibrationTreeMetrics( (sum, row) => sum + (finiteTargetLossWeightShare(row) ?? 0), 0, ); + const changes = rows + .map((row) => finiteNumber(row.target_change)) + .filter((value): value is number => value != null); + const increasedError = changes.reduce((sum, value) => sum + Math.max(value, 0), 0); + const reducedError = changes.reduce((sum, value) => sum + Math.max(-value, 0), 0); + const change = changes.length + ? { + increasedError, + reducedError, + netChange: changes.reduce((sum, value) => sum + value, 0), + changedTargets: changes.filter((value) => Math.abs(value) > 1e-12).length, + unchangedTargets: changes.filter((value) => Math.abs(value) <= 1e-12).length, + sharedTargets: rows.filter((row) => row.comparison_status === "shared").length, + addedTargets: rows.filter((row) => row.comparison_status === "added").length, + removedTargets: rows.filter((row) => row.comparison_status === "removed").length, + } + : undefined; return { nTargets: rows.length, scored: errors.length, @@ -169,6 +205,7 @@ export function calibrationTreeMetrics( ? errors.reduce((sum, error) => sum + error, 0) / errors.length : null, medianAbsRelativeError: median(errors), + change, }; } @@ -610,7 +647,7 @@ export function buildCalibrationTree( const visibleTargets = partition.targets.filter((row) => visibleRows.has(row)); const targetNodes = visibleTargets .map((row, index) => { - const id = String(row.name ?? row.base_name ?? `target-${index}`); + const id = String(row.comparison_id ?? row.name ?? row.base_name ?? `target-${index}`); return node( id, targetLabel(row), diff --git a/frontend/lib/microcosm/calibration-treemap-layout.ts b/frontend/lib/microcosm/calibration-treemap-layout.ts index 4be639eb..c97a8081 100644 --- a/frontend/lib/microcosm/calibration-treemap-layout.ts +++ b/frontend/lib/microcosm/calibration-treemap-layout.ts @@ -39,11 +39,15 @@ function metricValue( return metrics.loss; } +export type CalibrationTreemapMetricSelector = ( + metrics: CalibrationTreeMetrics, +) => number; + function effectiveMetricValues( items: Array<{ metrics: CalibrationTreeMetrics }>, - mode: CalibrationTreeSizeMode, + metric: CalibrationTreemapMetricSelector, ): number[] { - const values = items.map((item) => metricValue(item.metrics, mode)); + const values = items.map((item) => metric(item.metrics)); return values.some((value) => value > 0) ? values : items.map((item) => item.metrics.nTargets); @@ -75,6 +79,21 @@ export function aggregateCalibrationTreeMetrics( (sum, item) => sum + item.targetLossWeightShare, 0, ); + const changeMetrics = metrics + .map((item) => item.change) + .filter((item): item is NonNullable => item != null); + const change = changeMetrics.length + ? { + increasedError: changeMetrics.reduce((sum, item) => sum + item.increasedError, 0), + reducedError: changeMetrics.reduce((sum, item) => sum + item.reducedError, 0), + netChange: changeMetrics.reduce((sum, item) => sum + item.netChange, 0), + changedTargets: changeMetrics.reduce((sum, item) => sum + item.changedTargets, 0), + unchangedTargets: changeMetrics.reduce((sum, item) => sum + item.unchangedTargets, 0), + sharedTargets: changeMetrics.reduce((sum, item) => sum + item.sharedTargets, 0), + addedTargets: changeMetrics.reduce((sum, item) => sum + item.addedTargets, 0), + removedTargets: changeMetrics.reduce((sum, item) => sum + item.removedTargets, 0), + } + : undefined; return { nTargets: metrics.reduce((sum, item) => sum + item.nTargets, 0), scored, @@ -86,6 +105,7 @@ export function aggregateCalibrationTreeMetrics( : null, meanAbsRelativeError: weightedError(metrics, "meanAbsRelativeError"), medianAbsRelativeError: weightedError(metrics, "medianAbsRelativeError"), + change, }; } @@ -105,10 +125,10 @@ function groupedNode( function condenseNodes( group: CalibrationTreeGroup, - mode: CalibrationTreeSizeMode, + metric: CalibrationTreemapMetricSelector, projectedGroupArea: number, ): CalibrationTreemapNode[] { - const values = effectiveMetricValues(group.nodes, mode); + const values = effectiveMetricValues(group.nodes, metric); const total = values.reduce((sum, value) => sum + value, 0); if (total <= 0) return group.nodes; @@ -145,9 +165,23 @@ export function condenseCalibrationTreemap( mode: CalibrationTreeSizeMode, width: number, height: number, +): CalibrationTreemapGroup[] { + return condenseCalibrationTreemapByMetric( + groups, + (metrics) => metricValue(metrics, mode), + width, + height, + ); +} + +export function condenseCalibrationTreemapByMetric( + groups: CalibrationTreeGroup[], + metric: CalibrationTreemapMetricSelector, + width: number, + height: number, ): CalibrationTreemapGroup[] { const canvasArea = Math.max(width, 0) * Math.max(height, 0); - const groupValues = effectiveMetricValues(groups, mode); + const groupValues = effectiveMetricValues(groups, metric); const total = groupValues.reduce((sum, value) => sum + value, 0); if (total <= 0 || canvasArea <= 0) return groups; @@ -164,7 +198,7 @@ export function condenseCalibrationTreemap( const kept: CalibrationTreemapGroup[] = large.map(({ group, value }) => ({ ...group, - nodes: condenseNodes(group, mode, (value / total) * canvasArea), + nodes: condenseNodes(group, metric, (value / total) * canvasArea), })); if (small.length < 2) { @@ -174,7 +208,7 @@ export function condenseCalibrationTreemap( ...group, nodes: condenseNodes( group, - mode, + metric, (groupValues[groups.indexOf(group)] / total) * canvasArea, ), })), diff --git a/frontend/lib/microcosm/latest-artifact.test.ts b/frontend/lib/microcosm/latest-artifact.test.ts index 73acb2b0..6b82cd2a 100644 --- a/frontend/lib/microcosm/latest-artifact.test.ts +++ b/frontend/lib/microcosm/latest-artifact.test.ts @@ -260,6 +260,7 @@ test("structured dimensions shape rows and honor artifact value order", () => { target_representation: "structured", }); expect(cal.rows.every((row) => row.dimension_adapter === "structured")).toBe(true); + expect(cal.rows.every((row) => row.target_representation === "structured")).toBe(true); expect(cal.rows[0]).toMatchObject({ family: "novastat_agency/population", geography: "North", @@ -554,7 +555,7 @@ test("live-US-shaped schema 5 rows preserve the legacy dotted contract", () => { ).targets[0]; // JSON round-tripping matches the API boundary and locks every legacy field; - // the four new contract fields are strictly additive. + // the additional contract fields are strictly additive. expect(JSON.parse(JSON.stringify(responseRow))).toEqual({ name: "bea_nipa.cy2023.proprietors_income.a041rc.amount@2024", target: 100, @@ -599,6 +600,7 @@ test("live-US-shaped schema 5 rows preserve the legacy dotted contract", () => { }, ], dimension_adapter: "legacy_name", + target_representation: "legacy", variable_key: "bea_nipa / proprietors income · total", source_citation: sourceCitation, source_url: null, @@ -1248,6 +1250,111 @@ test("comparison matches on base_name across the @period boundary", () => { expect(Array.isArray(cmp.rows[0].target_dimensions)).toBe(true); }); +test("comparison exposes each release's weighted target-error aggregate", () => { + const current = { + ...SAMPLE, + final_loss: 0.91, + target_loss_attribution: { + ...SAMPLE.target_loss_attribution, + status: "reported" as const, + aggregate: 0.123, + }, + }; + const candidate = { + ...SAMPLE, + release_id: "weighted-candidate", + final_loss: 0.82, + target_loss_attribution: { + ...SAMPLE.target_loss_attribution, + status: "reported" as const, + aggregate: 0.087, + }, + }; + + const cmp = buildComparison(current, candidate); + + expect(cmp.a.weighted_target_error).toBe(0.123); + expect(cmp.b.weighted_target_error).toBe(0.087); + expect(cmp.a.weighted_target_error).not.toBe(cmp.a.final_loss); + expect(cmp.b.weighted_target_error).not.toBe(cmp.b.final_loss); +}); + +test("comparison matches renamed legacy and structured targets by Chronicle fact key", () => { + const current = calibration([ + { + name: "legacy.population.total@2024", + target_name: "legacy.population.total", + metadata: { ledger_fact_key: "agency.population.total" }, + target: 100, + initial_estimate: 90, + final_estimate: 95, + }, + ], "legacy-current"); + const candidate = calibration([ + { + name: "resident-population@2024", + source: { id: "agency", label: "Statistical agency" }, + variable: { id: "resident_population", measure: "count" }, + dimensions: {}, + metadata: { ledger_fact_key: "agency.population.total" }, + target: 100, + initial_estimate: 90, + final_estimate: 99, + }, + ], "structured-candidate"); + + const cmp = buildComparison(current, candidate); + expect(cmp.summary).toMatchObject({ + common: 1, + added: 0, + removed: 0, + improved: 1, + matching: { + current_representation: "legacy", + candidate_representation: "structured", + matched_by: { chronicle_fact_key: 1 }, + }, + }); + expect(cmp.rows[0]).toMatchObject({ + match_kind: "chronicle_fact_key", + current_name: "legacy.population.total@2024", + candidate_name: "resident-population@2024", + current_representation: "legacy", + candidate_representation: "structured", + }); +}); + +test("comparison preserves duplicate names and resolves them by unique fallback keys", () => { + const row = (name: string, targetName: string, factKey: string) => ({ + name, + target_name: targetName, + metadata: { ledger_fact_key: factKey }, + target: 100, + initial_estimate: 100, + final_estimate: 100, + }); + const current = calibration([ + row("same@2024", "same", "fact-a"), + row("same@2025", "same", "fact-b"), + ], "duplicate-current"); + const candidate = calibration([ + row("renamed-a@2026", "renamed-a", "fact-a"), + row("renamed-b@2026", "renamed-b", "fact-b"), + ], "duplicate-candidate"); + + const cmp = buildComparison(current, candidate); + expect(cmp.summary).toMatchObject({ + common: 2, + added: 0, + removed: 0, + matching: { + matched_by: { chronicle_fact_key: 2 }, + ambiguous_key_groups: { base_name: 0 }, + }, + }); + expect(new Set(cmp.rows.map((comparison) => comparison.comparison_id)).size).toBe(2); +}); + test("new target loss weighting metadata marks loss as normalized", () => { const normalized = buildCalibration( { @@ -2084,6 +2191,7 @@ test("mixed diagnostics dispatch complete legacy and structured rows independent source: "bea", variable: "amount", dimension_adapter: "legacy_name", + target_representation: "legacy", }); expect(cal.rows[1]).toMatchObject({ source: "artifact_agency", @@ -2093,6 +2201,7 @@ test("mixed diagnostics dispatch complete legacy and structured rows independent variable_label: "Resident population", measure: "count", dimension_adapter: "structured", + target_representation: "structured", }); }); diff --git a/frontend/lib/microcosm/latest-artifact.ts b/frontend/lib/microcosm/latest-artifact.ts index 6032263d..9133bd0c 100644 --- a/frontend/lib/microcosm/latest-artifact.ts +++ b/frontend/lib/microcosm/latest-artifact.ts @@ -36,6 +36,7 @@ import { type TargetRepresentation, } from "./target-representation"; import { readStructuredTarget } from "./structured-target-reader"; +import { matchTargetSurfaces } from "./target-surface-matcher"; // The registry is the registration point; these re-exports keep the server // modules and routes that import country helpers from here working. @@ -1127,6 +1128,7 @@ function enrichTargetRow( dims, target_dimensions: targetDimensions, dimension_adapter: dimensionAdapter, + target_representation: rowRepresentation, variable_key: variableKey, // v2 published metadata (null on v1). source_citation: @@ -2030,7 +2032,9 @@ export function hfResolveUrl(path: string, country: MicrocosmCountry = "us"): st } // A hung HF request would otherwise block the function for the whole route -// maxDuration and pin the shared in-flight cache promise; cap each fetch. +// maxDuration and pin the shared in-flight cache promise. The calibration +// diagnostics artifact is large enough that a cold authenticated download can +// exceed 20 seconds, so leave adequate room below the routes' 300-second limit. const MICROCOSM_RELEASE_FETCH_TIMEOUT_MS = 120_000; async function hfFetch(url: string, revalidate: number): Promise { @@ -2282,6 +2286,7 @@ function targetResponseRow(row: TargetRow): TargetRow { dims: row.dims, target_dimensions: row.target_dimensions, dimension_adapter: row.dimension_adapter, + target_representation: row.target_representation, variable_key: row.variable_key, source_citation: row.source_citation, source_url: row.source_url, @@ -2797,67 +2802,63 @@ function comparisonVariableRows(rows: TargetRow[]) { }); } -// Diff two releases' calibration by matching targets on name. Common targets -// get a fit delta (|b rel err| - |a rel err|; negative = b fits better); -// targets present in only one release are listed as added/removed. Losses -// across releases are NOT comparable when the surfaces differ — flagged. +// Diff two releases' calibration using normalized target identities. Shared +// targets get a fit delta (|b rel err| - |a rel err|; negative = b fits +// better); targets present in only one release are listed as added/removed. +// Losses across releases are not comparable when the surfaces differ. export function buildComparison(a: Calibration, b: Calibration) { - // Match on base_name (the period-stripped name) so v1 and v2 releases align — - // v2 appends an @ suffix the older convention lacks. - const key = (r: TargetRow) => String(r.base_name ?? r.name); - const aByName = new Map(a.rows.map((r) => [key(r), r])); - const bByName = new Map(b.rows.map((r) => [key(r), r])); - const names = new Set([...aByName.keys(), ...bByName.keys()]); - + const matched = matchTargetSurfaces(a, b); const common: TargetRow[] = []; - let added = 0; - let removed = 0; + const added = matched.matches.filter((match) => match.comparison_status === "added").length; + const removed = matched.matches.filter((match) => match.comparison_status === "removed").length; let improved = 0; let regressed = 0; - for (const name of names) { - const ar = aByName.get(name); - const br = bByName.get(name); - if (ar && br) { - const aAbs = absRel(ar); - const bAbs = absRel(br); - const delta = aAbs != null && bAbs != null ? bAbs - aAbs : null; - if (delta != null && delta < -1e-9) improved += 1; - else if (delta != null && delta > 1e-9) regressed += 1; - const aRelative = comparableRelative(ar); - const bRelative = comparableRelative(br); - const errorKind = aRelative != null && bRelative != null ? "relative" : "absolute"; - common.push({ - name, - target_label: [br.geography ?? ar.geography, br.breakdown ?? ar.breakdown] - .filter(Boolean) - .join(" · "), - source: br.source ?? ar.source, - variable_key: br.variable_key ?? ar.variable_key, - variable: br.variable ?? ar.variable, - measure: br.measure ?? ar.measure, - level: br.level ?? ar.level, - breakdown: br.breakdown ?? ar.breakdown, - dims: br.dims ?? ar.dims, - target_dimensions: br.target_dimensions ?? ar.target_dimensions, - geography: br.geography ?? ar.geography, - a_target: numberOrNull(ar.target), - b_target: numberOrNull(br.target), - a_final_estimate: ar.final_estimate ?? null, - b_final_estimate: br.final_estimate ?? null, - error_kind: errorKind, - a_error: errorKind === "relative" ? aRelative : absoluteMiss(ar), - b_error: errorKind === "relative" ? bRelative : absoluteMiss(br), - a_relative_error: aRelative, - b_relative_error: bRelative, - a_within_tolerance: ar.within_tolerance ?? null, - b_within_tolerance: br.within_tolerance ?? null, - abs_rel_delta: delta, - }); - } else if (ar) { - removed += 1; - } else { - added += 1; - } + for (const match of matched.matches) { + if (!match.current || !match.candidate) continue; + const ar = match.current; + const br = match.candidate; + const name = String(br.base_name ?? br.name ?? ar.base_name ?? ar.name ?? ""); + const aAbs = absRel(ar); + const bAbs = absRel(br); + const delta = aAbs != null && bAbs != null ? bAbs - aAbs : null; + if (delta != null && delta < -1e-9) improved += 1; + else if (delta != null && delta > 1e-9) regressed += 1; + const aRelative = comparableRelative(ar); + const bRelative = comparableRelative(br); + const errorKind = aRelative != null && bRelative != null ? "relative" : "absolute"; + common.push({ + comparison_id: match.comparison_id, + match_kind: match.match_kind, + current_name: match.current_name, + candidate_name: match.candidate_name, + current_representation: match.current_representation, + candidate_representation: match.candidate_representation, + name, + target_label: [br.geography ?? ar.geography, br.breakdown ?? ar.breakdown] + .filter(Boolean) + .join(" · "), + source: br.source ?? ar.source, + variable_key: br.variable_key ?? ar.variable_key, + variable: br.variable ?? ar.variable, + measure: br.measure ?? ar.measure, + level: br.level ?? ar.level, + breakdown: br.breakdown ?? ar.breakdown, + dims: br.dims ?? ar.dims, + target_dimensions: br.target_dimensions ?? ar.target_dimensions, + geography: br.geography ?? ar.geography, + a_target: numberOrNull(ar.target), + b_target: numberOrNull(br.target), + a_final_estimate: ar.final_estimate ?? null, + b_final_estimate: br.final_estimate ?? null, + error_kind: errorKind, + a_error: errorKind === "relative" ? aRelative : absoluteMiss(ar), + b_error: errorKind === "relative" ? bRelative : absoluteMiss(br), + a_relative_error: aRelative, + b_relative_error: bRelative, + a_within_tolerance: ar.within_tolerance ?? null, + b_within_tolerance: br.within_tolerance ?? null, + abs_rel_delta: delta, + }); } common.sort( (x, y) => @@ -2875,6 +2876,7 @@ export function buildComparison(a: Calibration, b: Calibration) { initial_loss: a.initial_loss, final_loss: a.final_loss, loss_kind: a.loss_kind, + weighted_target_error: a.target_loss_attribution.aggregate, fraction_within_10pct: a.fraction_within_10pct, }, b: { @@ -2884,6 +2886,7 @@ export function buildComparison(a: Calibration, b: Calibration) { initial_loss: b.initial_loss, final_loss: b.final_loss, loss_kind: b.loss_kind, + weighted_target_error: b.target_loss_attribution.aggregate, fraction_within_10pct: b.fraction_within_10pct, }, summary: { @@ -2895,6 +2898,7 @@ export function buildComparison(a: Calibration, b: Calibration) { unchanged: common.length - improved - regressed, losses_comparable: !surfacesDiffer && a.loss_kind === b.loss_kind, loss_kind: a.loss_kind === b.loss_kind ? a.loss_kind : "mixed", + matching: matched.matching, }, variables: comparisonVariableRows(common), rows: common, diff --git a/frontend/lib/microcosm/legacy-target-regression.test.ts b/frontend/lib/microcosm/legacy-target-regression.test.ts index c4d4fd3c..beb0352f 100644 --- a/frontend/lib/microcosm/legacy-target-regression.test.ts +++ b/frontend/lib/microcosm/legacy-target-regression.test.ts @@ -52,6 +52,7 @@ test("the pinned US release retains its legacy source and statistic grouping", ( ); expect(calibration.target_schema.target_representation).toBe("legacy"); + expect(calibration.rows.every((row) => row.target_representation === "legacy")).toBe(true); expect(calibration.rows).toHaveLength(5_659); expect(tree.groups).toHaveLength(15); expect(Object.values(nodeCounts).reduce((sum, count) => sum + count, 0)).toBe(53); diff --git a/frontend/lib/microcosm/reforms.test.ts b/frontend/lib/microcosm/reforms.test.ts index 452255bb..16d3c3bc 100644 --- a/frontend/lib/microcosm/reforms.test.ts +++ b/frontend/lib/microcosm/reforms.test.ts @@ -114,6 +114,49 @@ test("derives microcosm-vs-JCT error per reform", () => { expect(row.direction).toBe("over"); // microcosm less negative than JCT }); +test("normalizes legacy populace reform calculations", () => { + const v = buildReformValidation( + raw([ + { + id: "legacy", + name: "Legacy calculation", + in_sample: true, + jct: { score: 1000 }, + populace: { + budget_effect: 900, + window: "FY2024", + annual: { "2024": 900 }, + }, + }, + ]), + "rel-a", + ); + + expect(v.rows[0].microcosm_estimate).toBe(900); + expect(v.rows[0].microcosm_window).toBe("FY2024"); + expect(v.rows[0].microcosm_annual).toEqual({ "2024": 900 }); + expect(v.rows[0].relative_error).toBeCloseTo(-0.1, 6); + expect(v.summary.n_scored).toBe(1); +}); + +test("prefers current microcosm calculations over legacy populace values", () => { + const v = buildReformValidation( + raw([ + { + id: "both", + name: "Both calculation formats", + jct: { score: 1000 }, + microcosm: { budget_effect: 1100 }, + populace: { budget_effect: 900 }, + }, + ]), + "rel-a", + ); + + expect(v.rows[0].microcosm_estimate).toBe(1100); + expect(v.rows[0].relative_error).toBeCloseTo(0.1, 6); +}); + test("summary counts only scored reforms and averages |error|", () => { const unscored = { id: "x", name: "No microcosm estimate", in_sample: false, jct: { score: -500 } }; const v = buildReformValidation(raw([obbba, salt, unscored]), "rel-a"); diff --git a/frontend/lib/microcosm/reforms.ts b/frontend/lib/microcosm/reforms.ts index 45705996..9f5a1239 100644 --- a/frontend/lib/microcosm/reforms.ts +++ b/frontend/lib/microcosm/reforms.ts @@ -123,6 +123,9 @@ export interface ReformValidation { function enrichReform(raw: JsonObject): ReformValidationRow { const jct = asObject(raw.jct); const microcosm = asObject(raw.microcosm); + const calculation = Object.keys(microcosm).length + ? microcosm + : asObject(raw.populace); const jctFy2026 = numberOrNull(jct.score); const jctFy2027 = numberOrNull(jct.score_fy2027); // Benchmark defaults to JCT's first full fiscal year (FY2027). FY2026 is a @@ -130,7 +133,7 @@ function enrichReform(raw: JsonObject): ReformValidationRow { // overstates the gap against microcosm's calendar-year liability. In-sample // rows have no FY2027 figure, so they fall back to their annual (FY2026) one. const benchmark = jctFy2027 ?? jctFy2026; - const estimate = numberOrNull(microcosm.budget_effect); + const estimate = numberOrNull(calculation.budget_effect); const absError = benchmark != null && estimate != null ? estimate - benchmark : null; // A zero benchmark has no meaningful relative error — leave it unscored // rather than storing the raw dollar delta, which would otherwise be treated @@ -140,7 +143,7 @@ function enrichReform(raw: JsonObject): ReformValidationRow { ? (estimate - benchmark) / Math.abs(benchmark) : null; const absRel = relError == null ? null : Math.abs(relError); - const annual = asObject(microcosm.annual); + const annual = asObject(calculation.annual); const annualClean: Record = {}; for (const [k, v] of Object.entries(annual)) { const n = numberOrNull(v); @@ -163,7 +166,7 @@ function enrichReform(raw: JsonObject): ReformValidationRow { jct_source_url: stringOrNull(jct.source_url), jct_published: stringOrNull(jct.published), microcosm_estimate: estimate, - microcosm_window: stringOrNull(microcosm.window), + microcosm_window: stringOrNull(calculation.window), microcosm_annual: Object.keys(annualClean).length ? annualClean : null, abs_error: absError, relative_error: relError, diff --git a/frontend/lib/microcosm/staging-artifact.test.ts b/frontend/lib/microcosm/staging-artifact.test.ts index 85ce9501..7aa70530 100644 --- a/frontend/lib/microcosm/staging-artifact.test.ts +++ b/frontend/lib/microcosm/staging-artifact.test.ts @@ -5,6 +5,7 @@ import { loadStagingRun, loadStagingRuns, loadStagingTargetDiagnostics, + stagingTargetChangeCacheTtlSeconds, stagingUnavailableReason, } from "./staging-artifact"; @@ -21,6 +22,15 @@ test("names the country when staging is unavailable", () => { ); }); +test("target-change cache duration follows whether a run can still change", () => { + expect(stagingTargetChangeCacheTtlSeconds("passed")).toBe(21_600); + expect(stagingTargetChangeCacheTtlSeconds("published")).toBe(21_600); + expect(stagingTargetChangeCacheTtlSeconds("failed")).toBe(21_600); + expect(stagingTargetChangeCacheTtlSeconds("running")).toBe(30); + expect(stagingTargetChangeCacheTtlSeconds("stalled")).toBe(30); + expect(stagingTargetChangeCacheTtlSeconds(null)).toBe(30); +}); + test("Belgium staging loaders return an empty state before resolving artifacts", async () => { const unavailable = { available: false as const, diff --git a/frontend/lib/microcosm/staging-artifact.ts b/frontend/lib/microcosm/staging-artifact.ts index b53faddb..b7628f65 100644 --- a/frontend/lib/microcosm/staging-artifact.ts +++ b/frontend/lib/microcosm/staging-artifact.ts @@ -15,9 +15,23 @@ import { type ReformValidation, buildReformValidation, } from "@/lib/microcosm/reforms"; +import { + buildTargetChangeDataset, + type TargetChangeDataset, +} from "@/lib/microcosm/target-change"; type JsonObject = Record; +interface TargetChangeCacheEntry { + expiresAt: number; + promise: Promise; +} + +const TARGET_CHANGE_FINAL_CACHE_SECONDS = 21_600; +const TARGET_CHANGE_MUTABLE_CACHE_SECONDS = 30; +const TARGET_CHANGE_CACHE_LIMIT = 8; +const targetChangeCache = new Map(); + export const MICROCOSM_STAGING_HF_REPO_ENV = "POPULACE_STAGING_HF_REPO"; export const MICROCOSM_STAGING_HF_REVISION_ENV = "POPULACE_STAGING_HF_REVISION"; export const MICROCOSM_STAGING_HF_REPO = @@ -150,6 +164,12 @@ function stringValue(value: unknown): string | null { return typeof value === "string" && value.trim() ? value : null; } +export function stagingTargetChangeCacheTtlSeconds(status: unknown): number { + return ["passed", "published", "failed"].includes(String(status ?? "").trim()) + ? TARGET_CHANGE_FINAL_CACHE_SECONDS + : TARGET_CHANGE_MUTABLE_CACHE_SECONDS; +} + function summaryFromProgress(runId: string, progress: JsonObject | null): StagingRunSummary { const candidate = stringValue(progress?.candidate_release_id); return { @@ -315,6 +335,52 @@ export async function loadStagingCalibration( ); } +export async function loadStagingTargetChangeDataset( + runId: string, + releaseId: string, + country: MicrocosmCountry = "us", +): Promise { + if (stagingUnavailableReason(country)) return null; + assertSafeReleaseId(runId, "run"); + assertSafeReleaseId(releaseId, "release"); + const progress = await stagingJsonOrNull( + `runs/${runId}/progress.json`, + TARGET_CHANGE_MUTABLE_CACHE_SECONDS, + ); + const ttlSeconds = stagingTargetChangeCacheTtlSeconds(progress?.status); + const cacheKey = `${country}:${runId}:${releaseId}`; + const now = Date.now(); + for (const [key, entry] of targetChangeCache) { + if (entry.expiresAt <= now) targetChangeCache.delete(key); + } + const cached = targetChangeCache.get(cacheKey); + if (cached && cached.expiresAt > now) return cached.promise; + + const promise = Promise.all([ + loadRelease(releaseId, TARGET_CHANGE_FINAL_CACHE_SECONDS, country), + loadStagingCalibration(runId, ttlSeconds, country), + ]).then(([release, candidate]) => + candidate ? buildTargetChangeDataset(release, candidate) : null, + ); + targetChangeCache.set(cacheKey, { + expiresAt: now + ttlSeconds * 1000, + promise, + }); + while (targetChangeCache.size > TARGET_CHANGE_CACHE_LIMIT) { + const oldest = targetChangeCache.keys().next().value; + if (typeof oldest !== "string") break; + targetChangeCache.delete(oldest); + } + try { + const result = await promise; + if (!result) targetChangeCache.delete(cacheKey); + return result; + } catch (error) { + targetChangeCache.delete(cacheKey); + throw error; + } +} + export async function loadStagingRun( runId: string, revalidate: number, diff --git a/frontend/lib/microcosm/staging-status.test.ts b/frontend/lib/microcosm/staging-status.test.ts new file mode 100644 index 00000000..84b5c1a1 --- /dev/null +++ b/frontend/lib/microcosm/staging-status.test.ts @@ -0,0 +1,34 @@ +import { expect, test } from "bun:test"; + +import { formatStagingCurrentStatus, formatStagingStatus } from "./staging-status"; + +test("sentence-cases status values", () => { + expect(formatStagingStatus("running")).toBe("Running"); + expect(formatStagingStatus("waiting_for_input")).toBe("Waiting for input"); + expect(formatStagingStatus(null)).toBe("Unknown"); +}); + +test("formats ordinary progress stages as readable current statuses", () => { + expect( + formatStagingCurrentStatus({ + status: "running", + stage: "base_population_repair", + }), + ).toBe("Base population repair"); +}); + +test("formats calibration progress with the current and total epochs", () => { + expect( + formatStagingCurrentStatus({ + status: "running", + stage: "calibrating", + calibration: { epoch: 125, epochs: 6000 }, + }), + ).toBe("Calibration, epoch 125 of 6000"); +}); + +test("falls back safely when stage details are incomplete", () => { + expect(formatStagingCurrentStatus({ stage: "calibrating" })).toBe("Calibration"); + expect(formatStagingCurrentStatus({ status: "queued" })).toBe("Queued"); + expect(formatStagingCurrentStatus({})).toBe("Unknown"); +}); diff --git a/frontend/lib/microcosm/staging-status.ts b/frontend/lib/microcosm/staging-status.ts new file mode 100644 index 00000000..68eb9cd3 --- /dev/null +++ b/frontend/lib/microcosm/staging-status.ts @@ -0,0 +1,40 @@ +type JsonObject = Record; + +function objectValue(value: unknown): JsonObject { + return value != null && typeof value === "object" && !Array.isArray(value) + ? (value as JsonObject) + : {}; +} + +function stringValue(value: unknown): string | null { + return typeof value === "string" && value.trim() ? value.trim() : null; +} + +function finiteNumber(value: unknown): number | null { + return typeof value === "number" && Number.isFinite(value) ? value : null; +} + +function sentenceCaseIdentifier(value: string): string { + const words = value.replaceAll("_", " ").trim(); + return words ? `${words[0].toUpperCase()}${words.slice(1)}` : "Unknown"; +} + +export function formatStagingStatus(status: unknown): string { + const value = stringValue(status); + return value ? sentenceCaseIdentifier(value) : "Unknown"; +} + +export function formatStagingCurrentStatus(progress: JsonObject): string { + const stage = stringValue(progress.stage); + if (stage === "calibrating" || stage === "calibration") { + const calibration = objectValue(progress.calibration); + const epoch = finiteNumber(calibration.epoch); + const epochs = finiteNumber(calibration.epochs); + return epoch != null && epochs != null + ? `Calibration, epoch ${epoch} of ${epochs}` + : "Calibration"; + } + if (stage) return sentenceCaseIdentifier(stage); + + return formatStagingStatus(progress.status); +} diff --git a/frontend/lib/microcosm/target-change-tree.test.ts b/frontend/lib/microcosm/target-change-tree.test.ts new file mode 100644 index 00000000..47752597 --- /dev/null +++ b/frontend/lib/microcosm/target-change-tree.test.ts @@ -0,0 +1,167 @@ +import { describe, expect, test } from "bun:test"; + +import { + createExplorerState, + type ExplorerState, +} from "./calibration-explorer"; +import type { Calibration } from "./latest-artifact"; +import { buildTargetChangeDataset } from "./target-change"; +import { buildTargetChangeTree } from "./target-change-tree"; + +interface RowInput { + name: string; + contribution: number; + share: number; + error: number; + source?: string; + variable?: string; + geography?: string; +} + +function calibration(releaseId: string, rows: RowInput[]): Calibration { + const aggregate = rows.reduce((sum, row) => sum + row.contribution, 0); + return { + release_id: releaseId, + rows: rows.map((row) => ({ + name: `${row.name}@2024`, + base_name: row.name, + source: row.source ?? "tax", + variable: row.variable ?? "benefits", + variable_key: row.variable ?? "benefits", + geography: row.geography ?? "United States", + level: row.geography ? "state" : "national", + target: 100, + final_estimate: 100 + row.error * 100, + target_loss_weight: row.share * 100, + target_loss_weight_share: row.share, + target_loss_scale: 100, + final_capped_scaled_error: row.error, + final_loss_contribution: row.contribution, + })), + target_loss_attribution: { + status: "reported", + aggregate, + historical_final_loss: aggregate, + cap: 2, + basis_identifier: "sqrt:target", + basis_hash: "hash", + verification: null, + producer_warnings: [], + reason: null, + targets: [], + }, + } as unknown as Calibration; +} + +function fixture() { + return buildTargetChangeDataset( + calibration("current", [ + { name: "one", contribution: 0.1, share: 0.4, error: 0.25, geography: "CA" }, + { name: "two", contribution: 0.2, share: 0.4, error: 0.5, geography: "NY" }, + { name: "removed", contribution: 0.05, share: 0.2, error: 0.25, variable: "income", geography: "CA" }, + ]), + calibration("candidate", [ + { name: "one", contribution: 0.2, share: 0.5, error: 0.4, geography: "CA" }, + { name: "two", contribution: 0.1, share: 0.25, error: 0.4, geography: "NY" }, + { name: "added", contribution: 0.05, share: 0.25, error: 0.2, variable: "income", geography: "TX" }, + ]), + ); +} + +function programState(): ExplorerState { + const state = createExplorerState(); + state.path = { + source: "tax", + program: "benefits", + dimensions: [], + }; + return state; +} + +describe("target change hierarchy", () => { + test("keeps increases and reductions separate inside a net-zero category", () => { + const tree = buildTargetChangeTree(fixture(), createExplorerState(), "reported"); + const benefits = tree.groups + .flatMap((group) => group.nodes) + .find((node) => node.id === "benefits"); + expect(benefits?.metrics.change).toEqual(expect.objectContaining({ + increasedError: 0.1, + reducedError: 0.1, + netChange: 0, + changedTargets: 2, + })); + expect(tree.filteredMetrics.change?.increasedError).toBeCloseTo(0.15); + expect(tree.filteredMetrics.change?.reducedError).toBeCloseTo(0.15); + }); + + test("supports geography-first and program-first drill-down", () => { + const geographyState = createExplorerState(); + geographyState.breakdown = "geography"; + const geographyTree = buildTargetChangeTree(fixture(), geographyState, "reported"); + expect(geographyTree.groups[0].nodes.map((node) => node.id).sort()).toEqual([ + "CA", + "NY", + "TX", + ]); + + const programTree = buildTargetChangeTree(fixture(), programState(), "reported"); + expect(programTree.currentLevel.kind).toBe("geography"); + expect(programTree.groups[0].nodes.map((node) => node.id).sort()).toEqual([ + "CA", + "NY", + ]); + }); + + test("returns selected compact target detail at the target level", () => { + const dataset = fixture(); + const state = programState(); + state.path.geography = "CA"; + state.path.target = dataset.rows.find((row) => row.name === "one")?.comparison_id; + const tree = buildTargetChangeTree(dataset, state, "reported"); + expect(tree.currentLevel.kind).toBe("target"); + expect(tree.selectedTarget).toEqual(expect.objectContaining({ + name: "one", + comparison_status: "shared", + reported_change: 0.1, + })); + expect(tree.selectedTarget?.current).toEqual(expect.objectContaining({ + contribution: 0.1, + })); + expect(tree.selectedTarget?.candidate).toEqual(expect.objectContaining({ + contribution: 0.2, + })); + }); + + test("selects a shared target whose release identifiers differ", () => { + const current = calibration("current", [ + { name: "old-name", contribution: 0.1, share: 1, error: 0.1 }, + ]); + const candidate = calibration("candidate", [ + { name: "new-name", contribution: 0.2, share: 1, error: 0.2 }, + ]); + current.rows[0].chronicle = { fact_key: "agency.population.total" }; + candidate.rows[0].chronicle = { fact_key: "agency.population.total" }; + const dataset = buildTargetChangeDataset(current, candidate); + const state = programState(); + state.path.geography = "United States"; + state.path.target = dataset.rows[0].comparison_id; + + const tree = buildTargetChangeTree(dataset, state, "reported"); + expect(tree.selectedTarget).toMatchObject({ + current_name: "old-name@2024", + candidate_name: "new-name@2024", + match_kind: "chronicle_fact_key", + }); + expect(tree.groups.flatMap((group) => group.nodes)[0]?.id).toBe( + dataset.rows[0].comparison_id, + ); + }); + + test("shared mode excludes added and removed target leaves", () => { + const tree = buildTargetChangeTree(fixture(), createExplorerState(), "shared"); + expect(tree.summary?.comparisonTargets).toBe(2); + expect(tree.filteredMetrics.change?.addedTargets).toBe(0); + expect(tree.filteredMetrics.change?.removedTargets).toBe(0); + expect(tree.filteredMetrics.change?.sharedTargets).toBe(2); + }); +}); diff --git a/frontend/lib/microcosm/target-change-tree.ts b/frontend/lib/microcosm/target-change-tree.ts new file mode 100644 index 00000000..a41563ed --- /dev/null +++ b/frontend/lib/microcosm/target-change-tree.ts @@ -0,0 +1,82 @@ +import type { ExplorerState } from "./calibration-explorer"; +import { + buildCalibrationTree, + type CalibrationTreeResponse, + type CalibrationTreeTarget, +} from "./calibration-tree"; +import { + targetChangeForMode, + type TargetChangeAttributionSide, + type TargetChangeDataset, + type TargetChangeMethodology, + type TargetChangeMode, + type TargetChangeRow, + type TargetChangeSummary, +} from "./target-change"; +import type { TargetMatchingSummary } from "./target-surface-matcher"; + +export interface TargetChangeTreeResponse extends CalibrationTreeResponse { + available: boolean; + reason: string | null; + mode: TargetChangeMode; + current: TargetChangeAttributionSide; + candidate: TargetChangeAttributionSide; + methodology: TargetChangeMethodology; + matching: TargetMatchingSummary; + summary: TargetChangeSummary | null; + selectedTarget: TargetChangeRow | null; +} + +export interface TargetChangeTreeUnavailableResponse { + available: false; + reason: string; +} + +export type TargetChangeTreeApiResponse = + | TargetChangeTreeResponse + | TargetChangeTreeUnavailableResponse; + +function rowsForMode( + dataset: TargetChangeDataset, + mode: TargetChangeMode, +): CalibrationTreeTarget[] { + return dataset.rows.flatMap((row) => { + if (mode === "shared" && row.comparison_status !== "shared") return []; + const change = targetChangeForMode(row, mode); + return change == null ? [] : [{ ...row, target_change: change }]; + }); +} + +export function buildTargetChangeTree( + dataset: TargetChangeDataset, + state: ExplorerState, + mode: TargetChangeMode, +): TargetChangeTreeResponse { + const summary = dataset.summaries[mode]; + const reason = dataset.reason ?? dataset.modeReasons[mode]; + const comparisonRows = summary ? rowsForMode(dataset, mode) : []; + const tree = buildCalibrationTree( + comparisonRows, + state, + dataset.candidate.releaseId, + dataset.available, + ); + const selectedTarget = state.path.target + ? dataset.rows.find((row) => { + if (row.comparison_id !== state.path.target) return false; + return mode === "reported" || row.comparison_status === "shared"; + }) ?? null + : null; + return { + ...tree, + available: dataset.available && summary != null, + reason, + mode, + current: dataset.current, + candidate: dataset.candidate, + methodology: dataset.methodology, + matching: dataset.matching, + summary, + selectedTarget, + }; +} diff --git a/frontend/lib/microcosm/target-change-visualization.test.ts b/frontend/lib/microcosm/target-change-visualization.test.ts new file mode 100644 index 00000000..4033d962 --- /dev/null +++ b/frontend/lib/microcosm/target-change-visualization.test.ts @@ -0,0 +1,155 @@ +import { describe, expect, test } from "bun:test"; + +import type { + CalibrationTreeGroup, + CalibrationTreeMetrics, +} from "./calibration-tree"; +import type { TargetChangeRow } from "./target-change"; +import { + formatTargetChange, + targetChangeDetailValues, + targetChangeDirectionAreas, + targetChangeDirectionValue, + targetChangeGroupsForDirection, + targetChangeMapIdentity, +} from "./target-change-visualization"; + +function changeMetrics( + netChange: number, + increasedError: number, + reducedError: number, +): CalibrationTreeMetrics { + return { + nTargets: 1, + scored: 1, + within10Pct: 1, + loss: 0, + targetLossWeightShare: 0, + weightedAverageCappedError: null, + meanAbsRelativeError: null, + medianAbsRelativeError: null, + change: { + increasedError, + reducedError, + netChange, + changedTargets: 1, + unchangedTargets: 0, + sharedTargets: 1, + addedTargets: 0, + removedTargets: 0, + }, + }; +} + +function groupWithChanges(changes: number[]): CalibrationTreeGroup { + const nodes = changes.map((change, index) => ({ + id: `category-${index}`, + label: `Category ${index}`, + kind: "program" as const, + selection: { + kind: "program" as const, + source: "source", + value: `category-${index}`, + }, + metrics: changeMetrics( + change, + change > 0 ? change : 0, + change < 0 ? -change : 0, + ), + })); + return { + id: "source", + label: "Source", + nodes, + metrics: changeMetrics( + changes.reduce((sum, change) => sum + change, 0), + changes.reduce((sum, change) => sum + Math.max(change, 0), 0), + changes.reduce((sum, change) => sum + Math.max(-change, 0), 0), + ), + }; +} + +describe("target change visualization data", () => { + test("assigns a mixed-target category only to its net direction", () => { + const mixedCategory = changeMetrics(0.2, 0.3, 0.1); + expect(targetChangeDirectionValue(mixedCategory, "increase")).toBeCloseTo(0.2); + expect(targetChangeDirectionValue(mixedCategory, "reduction")).toBe(0); + }); + + test("keeps increases left and reductions right when reductions are larger", () => { + const areas = targetChangeDirectionAreas( + [groupWithChanges([0.1, -0.2])], + 400, + 200, + ); + const increase = areas.find((area) => area.data.direction === "increase"); + const reduction = areas.find((area) => area.data.direction === "reduction"); + expect(increase?.x).toBe(0); + expect(reduction?.x).toBeCloseTo(increase?.w ?? 0); + expect(increase?.h).toBe(200); + expect(reduction?.h).toBe(200); + expect((increase?.w ?? 0) * (increase?.h ?? 0)).toBeCloseTo(80_000 * (1 / 3)); + expect((reduction?.w ?? 0) * (reduction?.h ?? 0)).toBeCloseTo(80_000 * (2 / 3)); + }); + + test("places each category in only one directional group", () => { + const group = groupWithChanges([0.2, -0.1, 0]); + const increased = targetChangeGroupsForDirection([group], "increase"); + const reduced = targetChangeGroupsForDirection([group], "reduction"); + expect(increased[0].nodes.map((node) => node.id)).toEqual(["category-0"]); + expect(reduced[0].nodes.map((node) => node.id)).toEqual(["category-1"]); + expect(increased[0].metrics.change?.netChange).toBeCloseTo(0.2); + expect(reduced[0].metrics.change?.netChange).toBeCloseTo(-0.1); + }); + + test("omits an empty direction without assigning false visual area", () => { + const areas = targetChangeDirectionAreas( + [groupWithChanges([-0.1])], + 400, + 200, + ); + expect(areas).toHaveLength(1); + expect(areas[0].data.direction).toBe("reduction"); + expect(areas[0].w * areas[0].h).toBeCloseTo(80_000); + }); + + test("formats weighted error changes as percentage points", () => { + expect(formatTargetChange(0.004)).toBe("+0.40 pp"); + expect(formatTargetChange(-0.004)).toBe("−0.40 pp"); + expect(formatTargetChange(0)).toBe("±0.00 pp"); + expect(formatTargetChange(null)).toBe("—"); + }); + + test("changes visual identity when the run or resolved release changes", () => { + expect(targetChangeMapIdentity("run-a", "release-a")).not.toBe( + targetChangeMapIdentity("run-b", "release-a"), + ); + expect(targetChangeMapIdentity("run-a", "release-a")).not.toBe( + targetChangeMapIdentity("run-a", "release-b"), + ); + }); + + test("selects actual or pooled contributions for target detail", () => { + const target = { + reported_change: 0.02, + pooled_weight_share: 0.4, + shared_current_contribution: 0.04, + shared_candidate_contribution: 0.06, + shared_change: 0.02, + current: { weightShare: 0.3, contribution: 0.03 }, + candidate: { weightShare: 0.5, contribution: 0.05 }, + } as TargetChangeRow; + expect(targetChangeDetailValues(target, "reported")).toEqual(expect.objectContaining({ + comparisonWeight: null, + currentContribution: 0.03, + candidateContribution: 0.05, + change: 0.02, + })); + expect(targetChangeDetailValues(target, "shared")).toEqual(expect.objectContaining({ + comparisonWeight: 0.4, + currentContribution: 0.04, + candidateContribution: 0.06, + change: 0.02, + })); + }); +}); diff --git a/frontend/lib/microcosm/target-change-visualization.ts b/frontend/lib/microcosm/target-change-visualization.ts new file mode 100644 index 00000000..99bd2e20 --- /dev/null +++ b/frontend/lib/microcosm/target-change-visualization.ts @@ -0,0 +1,170 @@ +import type { Placed } from "@/lib/treemap/squarify"; +import type { + CalibrationTreeChangeMetrics, + CalibrationTreeGroup, +} from "./calibration-tree"; +import { aggregateCalibrationTreeMetrics } from "./calibration-treemap-layout"; +import type { + TargetChangeMode, + TargetChangeRow, +} from "./target-change"; + +export type TargetChangeDirection = "increase" | "reduction"; + +export function targetChangeMapIdentity(runId: string, releaseId: string): string { + return `${runId}:${releaseId}`; +} + +export interface TargetChangeDirectionData { + direction: TargetChangeDirection; + label: string; +} + +export function targetChangeDirectionValue( + metrics: { change?: CalibrationTreeChangeMetrics }, + direction: TargetChangeDirection, +): number { + const netChange = metrics.change?.netChange ?? 0; + if (Math.abs(netChange) <= 1e-12) return 0; + return direction === "increase" + ? Math.max(netChange, 0) + : Math.max(-netChange, 0); +} + +/** + * Retains only categories whose net change belongs to the requested direction. + * A source group may be present in both results, but each category node is + * present in no more than one result. + */ +export function targetChangeGroupsForDirection( + groups: CalibrationTreeGroup[], + direction: TargetChangeDirection, +): CalibrationTreeGroup[] { + return groups.flatMap((group) => { + const nodes = group.nodes.filter( + (node) => targetChangeDirectionValue(node.metrics, direction) > 0, + ); + if (!nodes.length) return []; + return [{ + ...group, + nodes, + metrics: aggregateCalibrationTreeMetrics(nodes), + }]; + }); +} + +export function targetChangeDirectionTotals( + groups: CalibrationTreeGroup[], +): Record { + return groups.reduce( + (totals, group) => { + for (const node of group.nodes) { + totals.increase += targetChangeDirectionValue(node.metrics, "increase"); + totals.reduction += targetChangeDirectionValue(node.metrics, "reduction"); + } + return totals; + }, + { increase: 0, reduction: 0 }, + ); +} + +export function targetChangeDirectionAreas( + groups: CalibrationTreeGroup[], + width: number, + height: number, +): Placed[] { + const totals = targetChangeDirectionTotals(groups); + const safeWidth = Math.max(width, 0); + const safeHeight = Math.max(height, 0); + const total = totals.increase + totals.reduction; + if (total <= 0 || safeWidth <= 0 || safeHeight <= 0) return []; + + if (totals.increase <= 0) { + return [{ + x: 0, + y: 0, + w: safeWidth, + h: safeHeight, + value: totals.reduction, + data: { direction: "reduction", label: "Reduced weighted target error" }, + }]; + } + if (totals.reduction <= 0) { + return [{ + x: 0, + y: 0, + w: safeWidth, + h: safeHeight, + value: totals.increase, + data: { direction: "increase", label: "Increased weighted target error" }, + }]; + } + + const increaseWidth = safeWidth * (totals.increase / total); + return [ + { + x: 0, + y: 0, + w: increaseWidth, + h: safeHeight, + value: totals.increase, + data: { direction: "increase", label: "Increased weighted target error" }, + }, + { + x: increaseWidth, + y: 0, + w: safeWidth - increaseWidth, + h: safeHeight, + value: totals.reduction, + data: { direction: "reduction", label: "Reduced weighted target error" }, + }, + ]; +} + +export function formatWeightedTargetError(value: number | null | undefined): string { + if (value == null || !Number.isFinite(value)) return "—"; + return `${(value * 100).toFixed(2)}%`; +} + +export function formatTargetChange( + value: number | null | undefined, + alwaysSign = true, +): string { + if (value == null || !Number.isFinite(value)) return "—"; + const percentagePoints = value * 100; + const sign = percentagePoints > 0 + ? "+" + : percentagePoints < 0 + ? "−" + : alwaysSign + ? "±" + : ""; + return `${sign}${Math.abs(percentagePoints).toFixed(2)} pp`; +} + +export interface TargetChangeDetailValues { + currentWeightShare: number | null; + candidateWeightShare: number | null; + comparisonWeight: number | null; + currentContribution: number | null; + candidateContribution: number | null; + change: number | null; +} + +export function targetChangeDetailValues( + target: TargetChangeRow, + mode: TargetChangeMode, +): TargetChangeDetailValues { + return { + currentWeightShare: target.current?.weightShare ?? null, + candidateWeightShare: target.candidate?.weightShare ?? null, + comparisonWeight: mode === "shared" ? target.pooled_weight_share : null, + currentContribution: mode === "reported" + ? target.current?.contribution ?? null + : target.shared_current_contribution, + candidateContribution: mode === "reported" + ? target.candidate?.contribution ?? null + : target.shared_candidate_contribution, + change: mode === "reported" ? target.reported_change : target.shared_change, + }; +} diff --git a/frontend/lib/microcosm/target-change.test.ts b/frontend/lib/microcosm/target-change.test.ts new file mode 100644 index 00000000..f020d18f --- /dev/null +++ b/frontend/lib/microcosm/target-change.test.ts @@ -0,0 +1,221 @@ +import { describe, expect, test } from "bun:test"; + +import type { Calibration } from "./latest-artifact"; +import { + buildTargetChangeDataset, + type TargetChangeSide, +} from "./target-change"; + +type RowInput = { + name: string; + contribution: number; + share: number; + error: number; + source?: string; + variable?: string; + geography?: string; +}; + +function calibration( + releaseId: string, + rows: RowInput[], + options: { + status?: Calibration["target_loss_attribution"]["status"]; + cap?: number; + basis?: string; + } = {}, +): Calibration { + const aggregate = rows.reduce((sum, row) => sum + row.contribution, 0); + const status = options.status ?? "reported"; + return { + release_id: releaseId, + rows: rows.map((row) => ({ + name: `${row.name}@2024`, + base_name: row.name, + source: row.source ?? "source", + variable: row.variable ?? row.name, + variable_key: row.variable ?? row.name, + geography: row.geography ?? "United States", + target: 100, + final_estimate: 100 + row.error * 100, + target_loss_weight: row.share * 100, + target_loss_weight_share: row.share, + target_loss_scale: 100, + final_capped_scaled_error: row.error, + final_loss_contribution: row.contribution, + })), + target_loss_attribution: { + status, + aggregate: status === "unavailable" ? null : aggregate, + historical_final_loss: aggregate, + cap: options.cap ?? 2, + basis_identifier: options.basis ?? "sqrt:target", + basis_hash: "hash", + verification: null, + producer_warnings: [], + reason: status === "unavailable" ? "fixture unavailable" : null, + targets: [], + }, + } as unknown as Calibration; +} + +function side(row: TargetChangeSide | null): TargetChangeSide { + if (!row) throw new Error("Expected target side"); + return row; +} + +describe("target change attribution", () => { + test("reported changes preserve weights, additions, removals, and the aggregate delta", () => { + const current = calibration("current", [ + { name: "shared", contribution: 0.1, share: 0.4, error: 0.25 }, + { name: "removed", contribution: 0.2, share: 0.6, error: 1 / 3 }, + ]); + const candidate = calibration("candidate", [ + { name: "shared", contribution: 0.15, share: 0.75, error: 0.2 }, + { name: "added", contribution: 0.05, share: 0.25, error: 0.2 }, + ]); + + const result = buildTargetChangeDataset(current, candidate); + expect(result.available).toBe(true); + expect(result.rows.find((row) => row.name === "shared")?.reported_change).toBeCloseTo(0.05); + expect(result.rows.find((row) => row.name === "added")?.reported_change).toBeCloseTo(0.05); + expect(result.rows.find((row) => row.name === "removed")?.reported_change).toBeCloseTo(-0.2); + expect(result.summaries.reported).toEqual(expect.objectContaining({ + shared: 1, + added: 1, + removed: 1, + })); + expect(result.summaries.reported?.currentScore).toBeCloseTo(0.3); + expect(result.summaries.reported?.candidateScore).toBeCloseTo(0.2); + expect(result.summaries.reported?.grossIncrease).toBeCloseTo(0.1); + expect(result.summaries.reported?.grossReduction).toBeCloseTo(0.2); + expect(result.summaries.reported?.netChange).toBeCloseTo(-0.1); + expect(result.summaries.reported?.reconciliationDifference).toBeCloseTo(0); + }); + + test("shared mode normalizes each side over shared targets and applies pooled weights", () => { + const current = calibration("current", [ + { name: "one", contribution: 0.08, share: 0.8, error: 0.1 }, + { name: "two", contribution: 0.02, share: 0.1, error: 0.2 }, + { name: "removed", contribution: 0.03, share: 0.1, error: 0.3 }, + ]); + const candidate = calibration("candidate", [ + { name: "one", contribution: 0.08, share: 0.2, error: 0.4 }, + { name: "two", contribution: 0.04, share: 0.4, error: 0.1 }, + { name: "added", contribution: 0.08, share: 0.4, error: 0.2 }, + ]); + + const result = buildTargetChangeDataset(current, candidate); + const one = result.rows.find((row) => row.name === "one"); + const two = result.rows.find((row) => row.name === "two"); + const expectedOne = ((0.8 / 0.9) + (0.2 / 0.6)) / 2; + expect(one?.pooled_weight_share).toBeCloseTo(expectedOne); + expect((one?.pooled_weight_share ?? 0) + (two?.pooled_weight_share ?? 0)).toBeCloseTo(1); + expect(one?.shared_change).toBeCloseTo(expectedOne * (0.4 - 0.1)); + expect(result.summaries.shared?.comparisonTargets).toBe(2); + expect(result.summaries.shared?.reconciliationDifference).toBeCloseTo(0); + }); + + test("shared rows use candidate metadata while removed rows keep current metadata", () => { + const current = calibration("current", [ + { name: "shared", contribution: 0.1, share: 0.5, error: 0.2, source: "old" }, + { name: "removed", contribution: 0.1, share: 0.5, error: 0.2, source: "old" }, + ]); + const candidate = calibration("candidate", [ + { name: "shared", contribution: 0.2, share: 1, error: 0.2, source: "new" }, + ]); + + const result = buildTargetChangeDataset(current, candidate); + expect(result.rows.find((row) => row.name === "shared")?.source).toBe("new"); + expect(result.rows.find((row) => row.name === "removed")?.source).toBe("old"); + }); + + test("uses cross-format Chronicle matches without changing contribution arithmetic", () => { + const current = calibration("current", [ + { name: "legacy-name", contribution: 0.1, share: 1, error: 0.1 }, + ]); + const candidate = calibration("candidate", [ + { name: "structured-name", contribution: 0.2, share: 1, error: 0.2 }, + ]); + current.rows[0].chronicle = { fact_key: "agency.population.total" }; + candidate.rows[0].chronicle = { fact_key: "agency.population.total" }; + candidate.rows[0].target_representation = "structured"; + candidate.rows[0].dimension_adapter = "structured"; + candidate.rows[0].dimensions = {}; + + const result = buildTargetChangeDataset(current, candidate); + expect(result.available).toBe(true); + expect(result.matching).toMatchObject({ + current_representation: "legacy", + candidate_representation: "structured", + matched_by: { chronicle_fact_key: 1 }, + }); + expect(result.rows[0]).toMatchObject({ + comparison_status: "shared", + match_kind: "chronicle_fact_key", + current_name: "legacy-name@2024", + candidate_name: "structured-name@2024", + reported_change: 0.1, + }); + expect(result.summaries.reported?.reconciliationDifference).toBeCloseTo(0); + }); + + test("fails closed when attribution is unavailable or a row is incomplete", () => { + const unavailable = buildTargetChangeDataset( + calibration("current", [], { status: "unavailable" }), + calibration("candidate", []), + ); + expect(unavailable.available).toBe(false); + expect(unavailable.rows).toEqual([]); + + const current = calibration("current", [ + { name: "shared", contribution: 0.1, share: 1, error: 0.1 }, + ]); + delete current.rows[0].final_loss_contribution; + const incomplete = buildTargetChangeDataset( + current, + calibration("candidate", [ + { name: "shared", contribution: 0.1, share: 1, error: 0.1 }, + ]), + ); + expect(incomplete.available).toBe(false); + expect(incomplete.reason).toContain("incomplete"); + }); + + test("preserves ambiguous duplicate rows as additions and removals", () => { + const current = calibration("current", [ + { name: "duplicate", contribution: 0.1, share: 0.5, error: 0.2 }, + { name: "duplicate", contribution: 0.2, share: 0.5, error: 0.4 }, + ]); + const candidate = calibration("candidate", [ + { name: "duplicate", contribution: 0.2, share: 1, error: 0.2 }, + ]); + const result = buildTargetChangeDataset(current, candidate); + expect(result.available).toBe(true); + expect(result.rows).toHaveLength(3); + expect(result.summaries.reported).toMatchObject({ + shared: 0, + added: 1, + removed: 2, + reconciliationDifference: 0, + }); + expect(result.matching.ambiguous_key_groups.base_name).toBe(1); + expect(result.modeReasons.shared).toContain("no shared targets"); + }); + + test("retains additive results while warning about methodology differences", () => { + const current = calibration("current", [ + { name: "shared", contribution: 0.1, share: 1, error: 0.1 }, + ], { cap: 1, basis: "old" }); + const candidate = calibration("candidate", [ + { name: "shared", contribution: 0.2, share: 1, error: 0.2 }, + ], { cap: 2, basis: "new" }); + + const result = buildTargetChangeDataset(current, candidate); + expect(result.available).toBe(true); + expect(result.methodology.comparable).toBe(false); + expect(result.methodology.warning).toContain("different"); + expect(side(result.rows[0].current).contribution).toBe(0.1); + expect(side(result.rows[0].candidate).contribution).toBe(0.2); + }); +}); diff --git a/frontend/lib/microcosm/target-change.ts b/frontend/lib/microcosm/target-change.ts new file mode 100644 index 00000000..e004aad2 --- /dev/null +++ b/frontend/lib/microcosm/target-change.ts @@ -0,0 +1,377 @@ +import type { Calibration } from "./latest-artifact"; +import { + matchTargetSurfaces, + type TargetMatchingSummary, + type TargetSurfaceStatus, +} from "./target-surface-matcher"; + +type TargetRow = Calibration["rows"][number]; + +export const TARGET_CHANGE_EPSILON = 1e-12; + +export type TargetChangeMode = "reported" | "shared"; +export type { TargetSurfaceStatus } from "./target-surface-matcher"; + +export interface TargetChangeSide { + target: number | null; + finalEstimate: number | null; + weight: number; + weightShare: number; + scale: number; + cappedError: number; + contribution: number; +} + +export interface TargetChangeRow extends Record { + name: string; + base_name: string; + comparison_id: string; + match_kind: "base_name" | "chronicle_fact_key" | "structured_identity" | null; + current_name: string | null; + candidate_name: string | null; + current_representation: "legacy" | "structured" | null; + candidate_representation: "legacy" | "structured" | null; + comparison_status: TargetSurfaceStatus; + current: TargetChangeSide | null; + candidate: TargetChangeSide | null; + reported_change: number; + pooled_weight_share: number | null; + shared_current_contribution: number | null; + shared_candidate_contribution: number | null; + shared_change: number | null; +} + +export interface TargetChangeAttributionSide { + releaseId: string; + status: Calibration["target_loss_attribution"]["status"]; + aggregate: number | null; + cap: number | null; + basisIdentifier: string | null; +} + +export interface TargetChangeMethodology { + comparable: boolean; + warning: string | null; +} + +export interface TargetChangeSummary { + mode: TargetChangeMode; + currentScore: number; + candidateScore: number; + netChange: number; + grossIncrease: number; + grossReduction: number; + reconciliationDifference: number; + comparisonTargets: number; + shared: number; + added: number; + removed: number; + changed: number; + unchanged: number; +} + +export interface TargetChangeDataset { + available: boolean; + reason: string | null; + current: TargetChangeAttributionSide; + candidate: TargetChangeAttributionSide; + methodology: TargetChangeMethodology; + matching: TargetMatchingSummary; + rows: TargetChangeRow[]; + summaries: Record; + modeReasons: Record; +} + +function finiteNumber(value: unknown): number | null { + return typeof value === "number" && Number.isFinite(value) ? value : null; +} + +function hierarchyFields(row: TargetRow): Record { + return { + source: row.source ?? null, + source_label: row.source_label ?? null, + variable: row.variable ?? null, + variable_key: row.variable_key ?? null, + measure: row.measure ?? null, + source_measure_id: row.source_measure_id ?? null, + level: row.level ?? null, + geography: row.geography ?? null, + family: row.family ?? null, + breakdown: row.breakdown ?? null, + dims: row.dims ?? null, + target_dimensions: row.target_dimensions ?? null, + calibration_status: row.calibration_status ?? null, + abs_relative_error: row.abs_relative_error ?? null, + }; +} + +function attributionSide(calibration: Calibration): TargetChangeAttributionSide { + const attribution = calibration.target_loss_attribution; + return { + releaseId: calibration.release_id, + status: attribution.status, + aggregate: finiteNumber(attribution.aggregate), + cap: finiteNumber(attribution.cap), + basisIdentifier: attribution.basis_identifier, + }; +} + +function targetSide(row: TargetRow | undefined): TargetChangeSide | null { + if (!row) return null; + const weight = finiteNumber(row.target_loss_weight); + const weightShare = finiteNumber(row.target_loss_weight_share); + const scale = finiteNumber(row.target_loss_scale); + const cappedError = finiteNumber(row.final_capped_scaled_error); + const contribution = finiteNumber(row.final_loss_contribution); + if ( + weight == null || weight < 0 || + weightShare == null || weightShare < 0 || + scale == null || scale <= 0 || + cappedError == null || cappedError < 0 || + contribution == null || contribution < 0 + ) { + return null; + } + return { + target: finiteNumber(row.target ?? row.value), + finalEstimate: finiteNumber(row.final_estimate ?? row.estimate), + weight, + weightShare, + scale, + cappedError, + contribution, + }; +} + +function close(left: number | null, right: number | null): boolean { + if (left == null || right == null) return left === right; + return Math.abs(left - right) <= TARGET_CHANGE_EPSILON; +} + +function methodology( + current: TargetChangeAttributionSide, + candidate: TargetChangeAttributionSide, +): TargetChangeMethodology { + const capMatches = close(current.cap, candidate.cap); + const basisMatches = current.basisIdentifier === candidate.basisIdentifier; + const comparable = capMatches && basisMatches; + return { + comparable, + warning: comparable + ? null + : "The current release and candidate use different target-loss caps or weighting methods. The comparison still uses each artifact's verified values, but it includes that methodology difference.", + }; +} + +function summarize( + rows: TargetChangeRow[], + mode: TargetChangeMode, + currentScore: number, + candidateScore: number, +): TargetChangeSummary { + const included = mode === "reported" + ? rows + : rows.filter((row) => row.comparison_status === "shared"); + const changeOf = (row: TargetChangeRow) => + mode === "reported" ? row.reported_change : row.shared_change ?? 0; + const rowTotal = included.reduce((sum, row) => sum + changeOf(row), 0); + const grossIncrease = included.reduce( + (sum, row) => sum + Math.max(changeOf(row), 0), + 0, + ); + const grossReduction = included.reduce( + (sum, row) => sum + Math.max(-changeOf(row), 0), + 0, + ); + const shared = rows.filter((row) => row.comparison_status === "shared").length; + const added = rows.filter((row) => row.comparison_status === "added").length; + const removed = rows.filter((row) => row.comparison_status === "removed").length; + const changed = included.filter( + (row) => Math.abs(changeOf(row)) > TARGET_CHANGE_EPSILON, + ).length; + return { + mode, + currentScore, + candidateScore, + netChange: candidateScore - currentScore, + grossIncrease, + grossReduction, + reconciliationDifference: rowTotal - (candidateScore - currentScore), + comparisonTargets: included.length, + shared, + added, + removed, + changed, + unchanged: included.length - changed, + }; +} + +function unavailableDataset( + current: TargetChangeAttributionSide, + candidate: TargetChangeAttributionSide, + matching: TargetMatchingSummary, + reason: string, +): TargetChangeDataset { + return { + available: false, + reason, + current, + candidate, + methodology: methodology(current, candidate), + matching, + rows: [], + summaries: { reported: null, shared: null }, + modeReasons: { reported: reason, shared: reason }, + }; +} + +export function targetChangeForMode( + row: TargetChangeRow, + mode: TargetChangeMode, +): number | null { + return mode === "reported" ? row.reported_change : row.shared_change; +} + +export function buildTargetChangeDataset( + currentCalibration: Calibration, + candidateCalibration: Calibration, +): TargetChangeDataset { + const current = attributionSide(currentCalibration); + const candidate = attributionSide(candidateCalibration); + const matched = matchTargetSurfaces(currentCalibration, candidateCalibration); + if (current.status === "unavailable" || current.aggregate == null) { + return unavailableDataset( + current, + candidate, + matched.matching, + "Weighted target-error attribution is unavailable for the current release.", + ); + } + if (candidate.status === "unavailable" || candidate.aggregate == null) { + return unavailableDataset( + current, + candidate, + matched.matching, + "Weighted target-error attribution is unavailable for the candidate.", + ); + } + + const rows: TargetChangeRow[] = []; + + for (const match of matched.matches) { + const currentRow = match.current ?? undefined; + const candidateRow = match.candidate ?? undefined; + const name = String( + candidateRow?.base_name ?? candidateRow?.name ?? + currentRow?.base_name ?? currentRow?.name ?? + match.comparison_id, + ); + const currentTarget = targetSide(currentRow); + const candidateTarget = targetSide(candidateRow); + if ((currentRow && !currentTarget) || (candidateRow && !candidateTarget)) { + return unavailableDataset( + current, + candidate, + matched.matching, + `Weighted target-error attribution is incomplete for target ${name}.`, + ); + } + const categoryRow = candidateRow ?? currentRow; + rows.push({ + ...(categoryRow ? hierarchyFields(categoryRow) : {}), + name, + base_name: name, + comparison_id: match.comparison_id, + match_kind: match.match_kind, + current_name: match.current_name, + candidate_name: match.candidate_name, + current_representation: match.current_representation, + candidate_representation: match.candidate_representation, + comparison_status: match.comparison_status, + current: currentTarget, + candidate: candidateTarget, + reported_change: + (candidateTarget?.contribution ?? 0) - + (currentTarget?.contribution ?? 0), + pooled_weight_share: null, + shared_current_contribution: null, + shared_candidate_contribution: null, + shared_change: null, + }); + } + + const sharedRows = rows.filter( + (row) => row.comparison_status === "shared" && row.current && row.candidate, + ); + const reportedSummary = summarize( + rows, + "reported", + current.aggregate, + candidate.aggregate, + ); + if (Math.abs(reportedSummary.reconciliationDifference) > TARGET_CHANGE_EPSILON) { + return unavailableDataset( + current, + candidate, + matched.matching, + "Per-target weighted error changes do not reconcile to the two attribution aggregates.", + ); + } + const currentSharedWeight = sharedRows.reduce( + (sum, row) => sum + (row.current?.weightShare ?? 0), + 0, + ); + const candidateSharedWeight = sharedRows.reduce( + (sum, row) => sum + (row.candidate?.weightShare ?? 0), + 0, + ); + let sharedReason: string | null = null; + let sharedSummary: TargetChangeSummary | null = null; + if (!sharedRows.length) { + sharedReason = "The current release and candidate have no shared targets."; + } else if (currentSharedWeight <= 0 || candidateSharedWeight <= 0) { + sharedReason = "Shared targets do not have positive total weight on both sides."; + } else { + for (const row of sharedRows) { + const pooledShare = ( + (row.current?.weightShare ?? 0) / currentSharedWeight + + (row.candidate?.weightShare ?? 0) / candidateSharedWeight + ) / 2; + const currentContribution = pooledShare * (row.current?.cappedError ?? 0); + const candidateContribution = pooledShare * (row.candidate?.cappedError ?? 0); + row.pooled_weight_share = pooledShare; + row.shared_current_contribution = currentContribution; + row.shared_candidate_contribution = candidateContribution; + row.shared_change = candidateContribution - currentContribution; + } + const currentSharedScore = sharedRows.reduce( + (sum, row) => sum + (row.shared_current_contribution ?? 0), + 0, + ); + const candidateSharedScore = sharedRows.reduce( + (sum, row) => sum + (row.shared_candidate_contribution ?? 0), + 0, + ); + sharedSummary = summarize( + rows, + "shared", + currentSharedScore, + candidateSharedScore, + ); + } + + return { + available: true, + reason: null, + current, + candidate, + methodology: methodology(current, candidate), + matching: matched.matching, + rows, + summaries: { + reported: reportedSummary, + shared: sharedSummary, + }, + modeReasons: { reported: null, shared: sharedReason }, + }; +} diff --git a/frontend/lib/microcosm/target-loss-attribution-manifest.ts b/frontend/lib/microcosm/target-loss-attribution-manifest.ts index 0c57915e..42afca77 100644 --- a/frontend/lib/microcosm/target-loss-attribution-manifest.ts +++ b/frontend/lib/microcosm/target-loss-attribution-manifest.ts @@ -58,10 +58,11 @@ const CONCEPT_BUDGET_WEIGHTING = "sqrt_value_concept_budget_weighted_mape_50_50_amount_count_target_scale_cap_100pct"; // This manifest is required because older builds do not expose target-importance -// weights in their published diagnostic files. It was audited against every US -// release eligible for the picker on 2026-08-18. These fingerprints cover -// ordered diagnostic row names, not downloaded files; runtime reconstruction -// must refuse a release whose target surface changes. +// weights in their diagnostic files. It was audited against every US release +// eligible for the picker on 2026-08-18 and includes explicitly verified staging +// candidates when needed. These fingerprints cover ordered diagnostic row names, +// not downloaded files; runtime reconstruction must refuse an artifact whose +// target surface changes. export const HISTORICAL_ATTRIBUTION_SUPPORT: readonly HistoricalAttributionSupport[] = [ { releaseId: "populace-us-2024-f32c2e5-20260614", @@ -213,6 +214,21 @@ export const HISTORICAL_ATTRIBUTION_SUPPORT: readonly HistoricalAttributionSuppo expectedStatus: "exact_reconstructed", tolerance: FLOATING_POINT_TOLERANCE, }, + { + releaseId: "populace-us-2024-f0af251-0ad74ed34493-20260619T181855Z", + buildId: "populace-us-2024-f0af251-0ad74ed34493-20260619T181855Z", + buildSha: "0ad74ed", + producerCommit: "0ad74ed344935e29d7c46703ce457ae6e8847b4d", + releaseFamily: "national", + diagnosticsSchema: 2, + targetCount: 4356, + orderedTargetNamesSha256: "0f562e9142dd33adb31339b457c6d5959575070b1a53c1dba5d0093b64664d60", + producerTargetSurfaceSha256: "67b491fe59f72e4622fd0d13c0f6a71e43e1c6ed74afae4032cb238515bd0269", + weightingIdentifier: HISTORICAL_WEIGHTING, + recipe: "newer_sqrt_value_50_50_v2", + expectedStatus: "exact_reconstructed", + tolerance: FLOATING_POINT_TOLERANCE, + }, { releaseId: "populace-us-2024-f0af251-703bd81a565c-20260620T201958Z", buildId: "populace-us-2024-f0af251-703bd81a565c-20260620T201958Z", diff --git a/frontend/lib/microcosm/target-loss-attribution.test.ts b/frontend/lib/microcosm/target-loss-attribution.test.ts index 9c31befd..fb3e8f7a 100644 --- a/frontend/lib/microcosm/target-loss-attribution.test.ts +++ b/frontend/lib/microcosm/target-loss-attribution.test.ts @@ -144,9 +144,9 @@ describe("schema-version-6 reported target-loss attribution", () => { }); describe("audited historical support manifest", () => { - test("classifies every picker release from all pinned evidence", () => { - expect(HISTORICAL_ATTRIBUTION_SUPPORT).toHaveLength(22); - expect(new Set(HISTORICAL_ATTRIBUTION_SUPPORT.map((entry) => entry.releaseId)).size).toBe(22); + test("classifies every pinned release and staging candidate from all evidence", () => { + expect(HISTORICAL_ATTRIBUTION_SUPPORT).toHaveLength(23); + expect(new Set(HISTORICAL_ATTRIBUTION_SUPPORT.map((entry) => entry.releaseId)).size).toBe(23); for (const entry of HISTORICAL_ATTRIBUTION_SUPPORT) { expect(classifyHistoricalAttributionEvidence(evidence(entry))).toEqual({ @@ -159,12 +159,35 @@ describe("audited historical support manifest", () => { HISTORICAL_ATTRIBUTION_SUPPORT.filter( (entry) => entry.expectedStatus === "exact_reconstructed", ), - ).toHaveLength(16); + ).toHaveLength(17); expect( HISTORICAL_ATTRIBUTION_SUPPORT.filter((entry) => entry.expectedStatus === "derived"), ).toHaveLength(6); }); + test("pins the completed schema-version-2 staging candidate", () => { + const entry = HISTORICAL_ATTRIBUTION_SUPPORT.find( + (candidate) => + candidate.releaseId === + "populace-us-2024-f0af251-0ad74ed34493-20260619T181855Z", + ); + + expect(entry).toMatchObject({ + buildSha: "0ad74ed", + diagnosticsSchema: 2, + targetCount: 4356, + producerTargetSurfaceSha256: + "67b491fe59f72e4622fd0d13c0f6a71e43e1c6ed74afae4032cb238515bd0269", + recipe: "newer_sqrt_value_50_50_v2", + expectedStatus: "exact_reconstructed", + }); + expect(classifyHistoricalAttributionEvidence(evidence(entry!))).toEqual({ + status: "exact_reconstructed", + recipe: "newer_sqrt_value_50_50_v2", + reason: null, + }); + }); + test("refuses a familiar weighting name on an unexpected target surface", () => { const entry = HISTORICAL_ATTRIBUTION_SUPPORT.find( (candidate) => candidate.expectedStatus === "exact_reconstructed", diff --git a/frontend/lib/microcosm/target-matching-presentation.test.ts b/frontend/lib/microcosm/target-matching-presentation.test.ts new file mode 100644 index 00000000..b6adbcd4 --- /dev/null +++ b/frontend/lib/microcosm/target-matching-presentation.test.ts @@ -0,0 +1,29 @@ +import { describe, expect, test } from "bun:test"; + +import { + targetMatchKindExplanation, + targetRepresentationPairLabel, +} from "./target-matching-presentation"; + +describe("target matching presentation", () => { + test("explains each supported match method literally", () => { + expect(targetMatchKindExplanation("base_name")).toBe( + "Matched by period-normalized target name.", + ); + expect(targetMatchKindExplanation("chronicle_fact_key")).toBe( + "Matched by exact Chronicle fact key.", + ); + expect(targetMatchKindExplanation("structured_identity")).toContain( + "structured source ID", + ); + expect(targetMatchKindExplanation(null)).toBe("Not matched across releases."); + }); + + test("formats row representation transitions", () => { + expect(targetRepresentationPairLabel("legacy", "structured")).toBe( + "Legacy format → Structured format", + ); + expect(targetRepresentationPairLabel(null, "structured")).toBe("Structured format"); + }); + +}); diff --git a/frontend/lib/microcosm/target-matching-presentation.ts b/frontend/lib/microcosm/target-matching-presentation.ts new file mode 100644 index 00000000..0fc728aa --- /dev/null +++ b/frontend/lib/microcosm/target-matching-presentation.ts @@ -0,0 +1,33 @@ +import type { TargetMatchKind } from "./target-surface-matcher"; +import type { + TargetRepresentation, + TargetRowRepresentation, +} from "./target-representation"; + +export function targetRepresentationLabel( + representation: TargetRepresentation | TargetRowRepresentation, +): string { + if (representation === "structured") return "Structured format"; + if (representation === "legacy") return "Legacy format"; + if (representation === "mixed") return "Mixed formats"; + return "Unknown format"; +} + +export function targetRepresentationPairLabel( + current: TargetRowRepresentation | null, + candidate: TargetRowRepresentation | null, +): string { + if (current && candidate) { + return `${targetRepresentationLabel(current)} → ${targetRepresentationLabel(candidate)}`; + } + return targetRepresentationLabel(candidate ?? current ?? "unknown"); +} + +export function targetMatchKindExplanation(kind: TargetMatchKind | null): string { + if (kind === "base_name") return "Matched by period-normalized target name."; + if (kind === "chronicle_fact_key") return "Matched by exact Chronicle fact key."; + if (kind === "structured_identity") { + return "Matched by structured source ID, statistic ID, measure, and raw dimensions."; + } + return "Not matched across releases."; +} diff --git a/frontend/lib/microcosm/target-surface-matcher.test.ts b/frontend/lib/microcosm/target-surface-matcher.test.ts new file mode 100644 index 00000000..7adf1b7c --- /dev/null +++ b/frontend/lib/microcosm/target-surface-matcher.test.ts @@ -0,0 +1,167 @@ +import { describe, expect, test } from "bun:test"; + +import type { Calibration } from "./latest-artifact"; +import { matchTargetSurfaces } from "./target-surface-matcher"; + +type Row = Calibration["rows"][number]; + +function calibration( + rows: Row[], + representation: "legacy" | "structured" | "mixed" = "legacy", +): Calibration { + return { + rows, + target_schema: { + diagnostics_schema_version: 7, + structured_dimensions: representation !== "legacy", + target_representation: representation, + }, + } as unknown as Calibration; +} + +function legacy( + name: string, + options: { baseName?: string; factKey?: string | null } = {}, +): Row { + return { + name, + base_name: options.baseName ?? name.replace(/@[^@]+$/, ""), + target_representation: "legacy", + dimension_adapter: "legacy_name", + chronicle: { fact_key: options.factKey ?? null }, + }; +} + +function structured( + name: string, + dimensions: Record, + options: { + baseName?: string; + factKey?: string | null; + source?: string; + variable?: string; + measure?: string; + } = {}, +): Row { + return { + name, + base_name: options.baseName ?? name.replace(/@[^@]+$/, ""), + target_representation: "structured", + dimension_adapter: "structured", + source: options.source ?? "agency", + variable: options.variable ?? "population", + measure: options.measure ?? "count", + dimensions, + chronicle: { fact_key: options.factKey ?? null }, + }; +} + +describe("target surface matching", () => { + test("matches exact period-normalized target names before other identities", () => { + const result = matchTargetSurfaces( + calibration([legacy("population@2024", { factKey: "old-fact" })]), + calibration([legacy("population@2025", { factKey: "new-fact" })]), + ); + + expect(result.matches).toHaveLength(1); + expect(result.matches[0]).toMatchObject({ + comparison_status: "shared", + match_kind: "base_name", + current_name: "population@2024", + candidate_name: "population@2025", + }); + expect(result.matching.matched_by).toEqual({ + base_name: 1, + chronicle_fact_key: 0, + structured_identity: 0, + }); + }); + + test("uses a unique Chronicle fact key to bridge legacy and structured names", () => { + const result = matchTargetSurfaces( + calibration([legacy("legacy-population", { factKey: "agency.population.total" })]), + calibration( + [structured("new-population", {}, { factKey: "agency.population.total" })], + "structured", + ), + ); + + expect(result.matches[0]).toMatchObject({ + match_kind: "chronicle_fact_key", + current_representation: "legacy", + candidate_representation: "structured", + }); + expect(result.matching).toMatchObject({ + current_representation: "legacy", + candidate_representation: "structured", + matched_by: { chronicle_fact_key: 1 }, + }); + }); + + test("canonicalizes structured dimension order without using labels", () => { + const current = structured("old-name", { region: "north", sex: "female" }); + current.source_label = "Old display label"; + const candidate = structured("new-name", { sex: "female", region: "north" }); + candidate.source_label = "New display label"; + const result = matchTargetSurfaces( + calibration([current], "structured"), + calibration([candidate], "structured"), + ); + + expect(result.matches[0].match_kind).toBe("structured_identity"); + expect(result.matching.matched_by.structured_identity).toBe(1); + }); + + test("lets unique fallback keys resolve a duplicated target name", () => { + const result = matchTargetSurfaces( + calibration([ + legacy("same@2024", { factKey: "fact-a" }), + legacy("same@2025", { factKey: "fact-b" }), + ]), + calibration([ + legacy("same@2026", { factKey: "fact-b" }), + legacy("same@2027", { factKey: "fact-a" }), + ]), + ); + + expect(result.matches.map((match) => match.match_kind)).toEqual([ + "chronicle_fact_key", + "chronicle_fact_key", + ]); + expect(result.matching.ambiguous_key_groups.base_name).toBe(1); + }); + + test("does not guess when a fallback key is many-to-one", () => { + const result = matchTargetSurfaces( + calibration([ + legacy("old-a", { factKey: "shared-fact" }), + legacy("old-b", { factKey: "shared-fact" }), + ]), + calibration([legacy("new", { factKey: "shared-fact" })]), + ); + + expect(result.matches.map((match) => match.comparison_status).sort()).toEqual([ + "added", + "removed", + "removed", + ]); + expect(result.matching.ambiguous_key_groups.chronicle_fact_key).toBe(1); + }); + + test("preserves every row once and generates deterministic unique identifiers", () => { + const current = calibration([ + legacy(""), + legacy("duplicate"), + legacy("duplicate"), + ]); + const candidate = calibration([legacy("candidate-only")]); + const first = matchTargetSurfaces(current, candidate); + const second = matchTargetSurfaces(current, candidate); + const ids = first.matches.map((match) => match.comparison_id); + + expect(first.matches).toHaveLength(4); + expect(new Set(ids).size).toBe(4); + expect(ids).toEqual(second.matches.map((match) => match.comparison_id)); + expect(first.matches.every((match) => match.match_kind == null)).toBe(true); + }); +}); diff --git a/frontend/lib/microcosm/target-surface-matcher.ts b/frontend/lib/microcosm/target-surface-matcher.ts new file mode 100644 index 00000000..95391dd2 --- /dev/null +++ b/frontend/lib/microcosm/target-surface-matcher.ts @@ -0,0 +1,243 @@ +import type { Calibration } from "./latest-artifact"; +import type { + TargetRepresentation, + TargetRowRepresentation, +} from "./target-representation"; + +type TargetRow = Calibration["rows"][number]; + +export type TargetMatchKind = + | "base_name" + | "chronicle_fact_key" + | "structured_identity"; +export type TargetSurfaceStatus = "shared" | "added" | "removed"; + +export interface TargetMatchingCounts { + base_name: number; + chronicle_fact_key: number; + structured_identity: number; +} + +export interface TargetMatchingSummary { + current_representation: TargetRepresentation; + candidate_representation: TargetRepresentation; + matched_by: TargetMatchingCounts; + ambiguous_key_groups: TargetMatchingCounts; +} + +export interface TargetSurfaceMatch { + comparison_id: string; + comparison_status: TargetSurfaceStatus; + match_kind: TargetMatchKind | null; + current: TargetRow | null; + candidate: TargetRow | null; + current_name: string | null; + candidate_name: string | null; + current_representation: TargetRowRepresentation | null; + candidate_representation: TargetRowRepresentation | null; +} + +export interface TargetSurfaceMatchResult { + matches: TargetSurfaceMatch[]; + matching: TargetMatchingSummary; +} + +interface IndexedTarget { + index: number; + row: TargetRow; +} + +interface MatchStrategy { + kind: TargetMatchKind; + key: (row: TargetRow) => string | null; +} + +function asObject(value: unknown): Record { + return value != null && typeof value === "object" && !Array.isArray(value) + ? value as Record + : {}; +} + +function nonEmptyString(value: unknown): string | null { + return typeof value === "string" && value.trim() ? value.trim() : null; +} + +function normalizedTargetName(row: TargetRow): string | null { + return nonEmptyString(row.base_name) ?? nonEmptyString(row.name); +} + +function originalTargetName(row: TargetRow): string | null { + return nonEmptyString(row.name) ?? nonEmptyString(row.base_name); +} + +function rowRepresentation(row: TargetRow): TargetRowRepresentation { + if (row.target_representation === "structured") return "structured"; + if (row.target_representation === "legacy") return "legacy"; + return row.dimension_adapter === "structured" ? "structured" : "legacy"; +} + +function collectionRepresentation(calibration: Calibration): TargetRepresentation { + const published = calibration.target_schema?.target_representation; + if ( + published === "legacy" || + published === "structured" || + published === "mixed" || + published === "unknown" + ) { + return published; + } + if (!calibration.rows.length) return "unknown"; + const representations = new Set(calibration.rows.map(rowRepresentation)); + if (representations.size > 1) return "mixed"; + return representations.has("structured") ? "structured" : "legacy"; +} + +function chronicleFactKey(row: TargetRow): string | null { + return nonEmptyString(asObject(row.chronicle).fact_key); +} + +function structuredIdentity(row: TargetRow): string | null { + if (rowRepresentation(row) !== "structured") return null; + const source = nonEmptyString(row.source); + const variable = nonEmptyString(row.variable); + if (!source || !variable) return null; + const dimensions = Object.entries(asObject(row.dimensions)) + .flatMap(([id, value]) => { + const rawValue = nonEmptyString(value); + return id && rawValue ? [[id, rawValue] as const] : []; + }) + .sort(([left], [right]) => left.localeCompare(right)); + return JSON.stringify({ + source, + variable, + measure: nonEmptyString(row.measure), + dimensions, + }); +} + +function groupsByKey( + rows: Map, + keyOf: MatchStrategy["key"], +): Map { + const groups = new Map(); + for (const indexed of rows.values()) { + const key = keyOf(indexed.row); + if (!key) continue; + const group = groups.get(key) ?? []; + group.push(indexed); + groups.set(key, group); + } + return groups; +} + +function encoded(value: string): string { + return encodeURIComponent(value); +} + +function sharedMatch( + kind: TargetMatchKind, + key: string, + current: IndexedTarget, + candidate: IndexedTarget, +): TargetSurfaceMatch { + return { + comparison_id: `shared:${kind}:${encoded(key)}`, + comparison_status: "shared", + match_kind: kind, + current: current.row, + candidate: candidate.row, + current_name: originalTargetName(current.row), + candidate_name: originalTargetName(candidate.row), + current_representation: rowRepresentation(current.row), + candidate_representation: rowRepresentation(candidate.row), + }; +} + +function sideOnlyMatch( + side: "current" | "candidate", + indexed: IndexedTarget, +): TargetSurfaceMatch { + const row = indexed.row; + const name = originalTargetName(row); + return { + comparison_id: `${side}:${indexed.index}:${encoded(normalizedTargetName(row) ?? name ?? "")}`, + comparison_status: side === "current" ? "removed" : "added", + match_kind: null, + current: side === "current" ? row : null, + candidate: side === "candidate" ? row : null, + current_name: side === "current" ? name : null, + candidate_name: side === "candidate" ? name : null, + current_representation: side === "current" ? rowRepresentation(row) : null, + candidate_representation: side === "candidate" ? rowRepresentation(row) : null, + }; +} + +const MATCH_STRATEGIES: MatchStrategy[] = [ + { kind: "base_name", key: normalizedTargetName }, + { kind: "chronicle_fact_key", key: chronicleFactKey }, + { kind: "structured_identity", key: structuredIdentity }, +]; + +function zeroCounts(): TargetMatchingCounts { + return { + base_name: 0, + chronicle_fact_key: 0, + structured_identity: 0, + }; +} + +export function matchTargetSurfaces( + currentCalibration: Calibration, + candidateCalibration: Calibration, +): TargetSurfaceMatchResult { + const current = new Map( + currentCalibration.rows.map((row, index) => [index, { index, row }]), + ); + const candidate = new Map( + candidateCalibration.rows.map((row, index) => [index, { index, row }]), + ); + const matchedBy = zeroCounts(); + const ambiguousKeyGroups = zeroCounts(); + const matches: TargetSurfaceMatch[] = []; + + for (const strategy of MATCH_STRATEGIES) { + const currentGroups = groupsByKey(current, strategy.key); + const candidateGroups = groupsByKey(candidate, strategy.key); + const keys = [...new Set([...currentGroups.keys(), ...candidateGroups.keys()])] + .sort((left, right) => left.localeCompare(right)); + for (const key of keys) { + const currentGroup = currentGroups.get(key) ?? []; + const candidateGroup = candidateGroups.get(key) ?? []; + if (!currentGroup.length || !candidateGroup.length) continue; + if (currentGroup.length !== 1 || candidateGroup.length !== 1) { + ambiguousKeyGroups[strategy.kind] += 1; + continue; + } + const currentTarget = currentGroup[0]; + const candidateTarget = candidateGroup[0]; + matches.push(sharedMatch(strategy.kind, key, currentTarget, candidateTarget)); + matchedBy[strategy.kind] += 1; + current.delete(currentTarget.index); + candidate.delete(candidateTarget.index); + } + } + + matches.push( + ...[...current.values()] + .sort((left, right) => left.index - right.index) + .map((row) => sideOnlyMatch("current", row)), + ...[...candidate.values()] + .sort((left, right) => left.index - right.index) + .map((row) => sideOnlyMatch("candidate", row)), + ); + + return { + matches, + matching: { + current_representation: collectionRepresentation(currentCalibration), + candidate_representation: collectionRepresentation(candidateCalibration), + matched_by: matchedBy, + ambiguous_key_groups: ambiguousKeyGroups, + }, + }; +} diff --git a/frontend/package.json b/frontend/package.json index e8a15c37..36c32a72 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -4,10 +4,11 @@ "private": true, "type": "module", "scripts": { - "dev": "next dev --turbopack", + "dev": "node scripts/dev.mjs", "build": "next build", "start": "next start", - "lint": "tsc --noEmit" + "lint": "tsc --noEmit", + "test:dev-port": "node --test scripts/available-port.node-test.mjs" }, "dependencies": { "@policyengine/ui-kit": "^0.9.0", diff --git a/frontend/scripts/available-port.mjs b/frontend/scripts/available-port.mjs new file mode 100644 index 00000000..bd979296 --- /dev/null +++ b/frontend/scripts/available-port.mjs @@ -0,0 +1,62 @@ +import { createServer } from "node:net"; + +const MIN_PORT = 1; +const MAX_PORT = 65_535; + +function assertPort(port, label) { + if (!Number.isInteger(port) || port < MIN_PORT || port > MAX_PORT) { + throw new RangeError(`${label} must be an integer between ${MIN_PORT} and ${MAX_PORT}`); + } +} + +function canListen(port, host, ipv6Only = false) { + return new Promise((resolve, reject) => { + const server = createServer(); + + server.once("error", (error) => { + if (error.code === "EADDRINUSE") { + resolve(false); + return; + } + if ( + ipv6Only && + (error.code === "EAFNOSUPPORT" || error.code === "EADDRNOTAVAIL") + ) { + resolve(true); + return; + } + reject(error); + }); + + server.listen({ port, host, exclusive: true, ipv6Only }, () => { + server.close((error) => { + if (error) { + reject(error); + return; + } + resolve(true); + }); + }); + }); +} + +export async function isPortAvailable(port) { + assertPort(port, "port"); + + if (!(await canListen(port, "127.0.0.1"))) return false; + return canListen(port, "::1", true); +} + +export async function findAvailablePort(startPort = 3000, maxPort = MAX_PORT) { + assertPort(startPort, "startPort"); + assertPort(maxPort, "maxPort"); + if (maxPort < startPort) { + throw new RangeError("maxPort must be greater than or equal to startPort"); + } + + for (let port = startPort; port <= maxPort; port += 1) { + if (await isPortAvailable(port)) return port; + } + + throw new Error(`No available port found from ${startPort} through ${maxPort}`); +} diff --git a/frontend/scripts/available-port.node-test.mjs b/frontend/scripts/available-port.node-test.mjs new file mode 100644 index 00000000..49e566e1 --- /dev/null +++ b/frontend/scripts/available-port.node-test.mjs @@ -0,0 +1,87 @@ +import assert from "node:assert/strict"; +import { afterEach, describe, test } from "node:test"; +import { createServer } from "node:net"; + +import { findAvailablePort, isPortAvailable } from "./available-port.mjs"; + +const openServers = []; + +afterEach(async () => { + await Promise.all( + openServers.splice(0).map( + (server) => + new Promise((resolve, reject) => { + server.close((error) => (error ? reject(error) : resolve())); + }), + ), + ); +}); + +async function occupyEphemeralIpv4Port() { + const server = createServer(); + await new Promise((resolve, reject) => { + server.once("error", reject); + server.listen({ port: 0, host: "127.0.0.1" }, resolve); + }); + openServers.push(server); + const address = server.address(); + if (!address || typeof address === "string") { + throw new Error("Expected the test server to have a TCP address"); + } + return address.port; +} + +async function occupyEphemeralIpv6Port() { + const server = createServer(); + await new Promise((resolve, reject) => { + server.once("error", reject); + server.listen({ port: 0, host: "::1", ipv6Only: true }, resolve); + }); + openServers.push(server); + const address = server.address(); + if (!address || typeof address === "string") { + throw new Error("Expected the test server to have a TCP address"); + } + return address.port; +} + +describe("development port selection", () => { + test("recognizes a port occupied only on IPv4", async () => { + const occupiedPort = await occupyEphemeralIpv4Port(); + + assert.equal(await isPortAvailable(occupiedPort), false); + }); + + test("recognizes a port occupied only on IPv6", async (context) => { + let occupiedPort; + try { + occupiedPort = await occupyEphemeralIpv6Port(); + } catch (error) { + if (error.code === "EAFNOSUPPORT" || error.code === "EADDRNOTAVAIL") { + context.skip("IPv6 loopback is not available on this host"); + return; + } + throw error; + } + + assert.equal(await isPortAvailable(occupiedPort), false); + }); + + test("increments until it finds a free port", async () => { + const occupiedPort = await occupyEphemeralIpv4Port(); + + const selectedPort = await findAvailablePort(occupiedPort); + + assert.ok(selectedPort > occupiedPort); + assert.equal(await isPortAvailable(selectedPort), true); + }); + + test("reports when the requested range is fully occupied", async () => { + const occupiedPort = await occupyEphemeralIpv4Port(); + + await assert.rejects( + findAvailablePort(occupiedPort, occupiedPort), + new Error(`No available port found from ${occupiedPort} through ${occupiedPort}`), + ); + }); +}); diff --git a/frontend/scripts/dev.mjs b/frontend/scripts/dev.mjs new file mode 100644 index 00000000..ded2ca42 --- /dev/null +++ b/frontend/scripts/dev.mjs @@ -0,0 +1,66 @@ +#!/usr/bin/env node + +import { spawn } from "node:child_process"; +import { mkdir, readFile, rm, writeFile } from "node:fs/promises"; +import process from "node:process"; + +import { findAvailablePort } from "./available-port.mjs"; + +const DEFAULT_START_PORT = 3000; +const APP_PATH = "/calibration/dashboard"; +const DEV_PORT_FILE = new URL("../.next/dev-port", import.meta.url); + +function startPort() { + const configured = process.env.PORT?.trim(); + if (!configured) return DEFAULT_START_PORT; + + const parsed = Number(configured); + if (!Number.isInteger(parsed) || parsed < 1 || parsed > 65_535) { + throw new Error("PORT must be an integer between 1 and 65535"); + } + return parsed; +} + +async function recordPort(port) { + await mkdir(new URL("../.next", import.meta.url), { recursive: true }); + await writeFile(DEV_PORT_FILE, `${port}\n`, "utf8"); +} + +async function removePortRecord(port) { + try { + const recordedPort = (await readFile(DEV_PORT_FILE, "utf8")).trim(); + if (recordedPort === String(port)) await rm(DEV_PORT_FILE); + } catch (error) { + if (error.code !== "ENOENT") throw error; + } +} + +const port = await findAvailablePort(startPort()); +await recordPort(port); + +const origin = `http://127.0.0.1:${port}`; +console.log(`Starting calibration dashboard at ${origin}${APP_PATH}`); + +const child = spawn( + "next", + ["dev", "--turbopack", "--hostname", "127.0.0.1", "--port", String(port)], + { + env: { ...process.env, PORT: String(port) }, + stdio: "inherit", + }, +); + +for (const signal of ["SIGINT", "SIGTERM"]) { + process.once(signal, () => child.kill(signal)); +} + +child.once("error", async (error) => { + await removePortRecord(port); + console.error(`Unable to start Next.js: ${error.message}`); + process.exitCode = 1; +}); + +child.once("exit", async (code) => { + await removePortRecord(port); + process.exitCode = code ?? 1; +});