diff --git a/LANE_E2_REPORT.md b/LANE_E2_REPORT.md new file mode 100644 index 00000000..9fc1cfa7 --- /dev/null +++ b/LANE_E2_REPORT.md @@ -0,0 +1,165 @@ +# Lane E2 report + +## Outcome + +The free-form `release_manifest.external_validations` surface is removed. The +Belgium overview now uses the same component, sections, and order as US/UK, with +only the already-shared country intro-copy table varying by country. Release +descriptions remain a generic, quiet provenance note. + +Belgium source IDs now come from the first segment of +`metadata.chronicle_record_ids`, labels come from one shared publisher map, and +`cell___` facets are handled by a country-independent +filter-pattern spec. There are no Belgium branches left in the target adapter. + +`README.md` contained no external-validation or EUROMOD-dashboard description, +so no README edit was needed for the removed feature. Cross-dataset comparison +remains confined to the evaluation-bundle path. + +## File-by-file changes + +- `frontend/components/layout/country-context.tsx`: exports the existing client + country predicate so the conformance TODO can exercise the still-closed client + country registry behavior. +- `frontend/components/microcosm/artifact-description-banner.tsx`: keeps release + descriptions generic while replacing the warning pill and warning border with + the ordinary provenance/notes visual weight. +- `frontend/components/microcosm/artifact-description-banner.test.ts`: verifies + generic description rendering, quiet styling, and the no-description null case. +- `frontend/components/microcosm/external-validations-panel.tsx`: deleted. +- `frontend/components/microcosm/microcosm-overview-view.tsx`: removes the + external-validations import and section; no country-specific section branch was + added. +- `frontend/lib/microcosm/external-validations.ts`: deleted. +- `frontend/lib/microcosm/external-validations.test.ts`: deleted. +- `frontend/lib/microcosm/fixtures/be-release/release_manifest.json`: removes the + complete `external_validations` payload. +- `frontend/lib/microcosm/fixtures/zz-release/calibration_diagnostics.json`: adds + four synthetic targets, including filter-coded facets and an unknown Chronicle + publisher prefix. +- `frontend/lib/microcosm/fixtures/zz-release/release_manifest.json`: adds the + synthetic release identity and generic description fixture. +- `frontend/lib/microcosm/latest-artifact.ts`: makes `COUNTRY_REPO` the typed + server registry (including national geography and synthetic `zz`); derives + publishers from Chronicle IDs; replaces Belgium publisher/name parsers with a + filter-pattern decomposition spec and artifact variable metadata; preserves + legacy US/UK dotted family grouping. +- `frontend/lib/microcosm/latest-artifact.test.ts`: updates Belgium expectations + to Chronicle-derived publishers and filter-derived facets, and adds a US dotted + family regression test. +- `frontend/lib/microcosm/source-label.ts`: delegates to the single shared source + authority labeler instead of maintaining a second map. +- `frontend/lib/microcosm/source-label.test.ts`: aligns the fallback-label test + with the shared humanizer. +- `frontend/lib/microcosm/staging-artifact.ts`: reads country geography from the + repository registry instead of another exhaustive country-name table. +- `frontend/lib/microcosm/third-country-conformance.test.ts`: adds four passing + `zz` data-shape assertions and four behavioral TODO assertions for current + schema/consumer gaps. +- `frontend/lib/slack.ts`: retains curated existing-country labels with a generic + fallback, so adding a registry country does not require a Slack table entry. +- `frontend/lib/source-labels.ts`: adds Eurostat to the one shared authority map; + the existing Statbel/ONSS/JRC/SFPD/NASA entries now serve every dashboard path. +- `frontend/lib/source-labels.test.ts`: verifies all required publisher labels and + the readable unknown-prefix fallback. +- `docs/spec-driven-countries.md`: records the current contract, remaining tables, + `zz` acceptance test, and schema additions that would eliminate those tables. +- `LANE_E2_REPORT.md`: this report. + +## Third-country conformance + +Passing assertions: + +1. One `COUNTRY_REPO` registration supplies parsing, repository URL, revision, + and national-geography behavior. +2. The fixture produces the common overview/highlights shape, including its + artifact description and two family groups. +3. Chronicle prefixes become source keys; an unseen prefix gets a readable label + and appears correctly in the treemap. +4. Filter-coded targets produce Region, Sex, and Age band facets and target + dimensions without a `zz` parser or label table. + +Behavioral TODOs (each callback contains an assertion verified to fail today): + +1. Client country parsing is still closed to a TypeScript country table. +2. Overview shaping does not propagate a typed artifact `presentation` block for + overview/target-browser copy. +3. Treemap shaping does not consume artifact `publisher_labels` overrides. +4. Target shaping does not consume top-level/target-level structured + `dimensions`; it still needs the legacy filter-pattern adapter. + +## Belgium special cases remaining from PR #163 + +- `country-context.tsx` and `nav-sidebar.tsx` still own the closed client country + union, selector order, and dataset labels (`us`/`uk`/`be`). +- `microcosm-overview-view.tsx` and `microcosm-targets-view.tsx` retain Belgium + entries in the same shared country-copy mechanisms already used for US/UK. + These are allowed copy variations, not section variations. +- `microcosm-staging-view.tsx` retains the UK/Belgium unavailable-reason table and + US-only staging behavior. +- `components/shared/format.ts` recognizes Belgium's Chronicle release-ID format + with an explicit `-be-` regex. +- `latest-artifact.ts` retains Belgium environment-variable names, repository, + revision, and national-geography registration. These are repository + configuration, not target-shaping branches. +- `slack.ts` retains the curated Belgium flag/label; unknown registered countries + now use the generic fallback. +- `README.md` still documents the supported `us|uk|be` query values and Belgium's + private repository/environment variables. +- PR #163's Belgium-specific fixture and regression tests remain as contract + coverage. Its source-attribution and staging API behavior is otherwise shared + with UK/non-US countries, while staging hooks remain capability-gated by the + existing US-only checks. + +The removed external-validation implementation/manifest field and the former +Belgium publisher, population-facet, family, and geography shaping branches do +not remain. + +## Gates + +Run from `frontend/`. + +### `bun test` + +```text +bun test v1.3.11 (af24e281) + + 205 pass + 4 todo + 0 fail + 800 expect() calls +Ran 209 tests across 27 files. [413.00ms] +``` + +### `bun run lint` + +```text +$ tsc --noEmit +``` + +### `IS_WEBPACK_TEST=1 bun run build` + +```text +├ ƒ /api/microcosm/variable +├ ○ /icon.svg +├ ○ /microcosm +├ ○ /microcosm/compare +├ ○ /microcosm/datasets +├ ƒ /microcosm/model-coverage +├ ○ /microcosm/pipeline +├ ○ /microcosm/staging +├ ƒ /microcosm/targets +└ ○ /microcosm/variables + + +○ (Static) prerendered as static content +ƒ (Dynamic) server-rendered on demand +``` + +The build used the unchanged `build` script with Next's webpack selector because +the literal Turbopack invocation attempted to fetch Google Fonts and the lane is +network-disabled. The green run compiled successfully, ran TypeScript, generated +all 20 static pages, and collected build traces. No workaround was committed and +`frontend/app/layout.tsx`, global CSS, and package scripts are untouched. + +LANE E2 DONE diff --git a/docs/spec-driven-countries.md b/docs/spec-driven-countries.md new file mode 100644 index 00000000..5b5a4590 --- /dev/null +++ b/docs/spec-driven-countries.md @@ -0,0 +1,64 @@ +# Make Microcosm countries spec-driven + +## Contract today + +The dashboard resolves a country's `latest.json`, then reads +`releases//calibration_diagnostics.json`. `build_manifest.json`, +`release_manifest.json`, and `demographics.json` are optional enrichments. The +diagnostics artifact owns fit values and target metadata; the build manifest owns +gates and compilation/loss metadata; the release manifest owns release identity, +role/default status, and the optional provenance `description`; demographics owns +geography coverage. A description may fill the existing provenance note, but an +artifact cannot add sections or choose components. + +Target sources are derived generically from the first dotted segment of +`metadata.chronicle_record_ids`, with one shared publisher-label map and a +humanized fallback for unknown prefixes. Filter-coded target facets may be +decoded by a country-agnostic pattern spec. Cross-dataset comparisons are not +part of this release contract: they are served only by `/microcosm/datasets` +from `cross_dataset.frontend_bundle.v1`. + +## Code-owned metadata that remains + +`COUNTRY_REPO` should be the only required registration point, but today adding a +country also touches closed country unions/parsers and selector lists, dataset and +national-geography labels, overview and target-browser copy, public/private link +behavior, and feature checks such as US-only staging and navigation. Publisher +display names remain in a shared TypeScript map. Target decomposition still has a +generic filter-pattern table plus legacy US name grammar (FIPS/state, filing +status, return type, income band, and qualifying-child rules). + +These tables are presentation or parsing metadata, not calibration logic. They +should not grow another country branch. + +## Acceptance test + +`frontend/lib/microcosm/third-country-conformance.test.ts` defines a synthetic +fourth country, `zz`, with four targets, filter-coded facets, a release +description, and Chronicle record IDs whose publisher prefix is unknown to the +label map. Registering its repository in `COUNTRY_REPO` must be the only country +specific code change. The same builders used by US/UK/BE must then produce: + +- the normal overview and targets response shapes and section order; +- the artifact description in the existing provenance-note slot; +- publisher/source values derived from Chronicle IDs, including a readable + unknown-prefix fallback; and +- filter-derived geography and target dimensions with stable labels and values. + +Assertions that expose a current contract gap are `test.todo` with a one-line +reason. The todos are the migration backlog; conformance is complete when all can +be enabled without adding `zz` conditionals or tables. + +## Schema additions that remove the tables + +| Current code-owned table or rule | Artifact addition | +| --- | --- | +| Country display name, national geography, repository visibility, and supported pages | A typed `country` block in `release_manifest` with `code`, `label`, national `geography_id`/`geography_label`, `repository_visibility`, and enumerated `capabilities`. Selector options should come from registered repositories; capabilities, not country codes, gate pages. | +| Overview authorities/examples and target-browser prompt | A narrowly typed `presentation` block in `release_manifest` for those existing text slots. It must not define arbitrary sections, component names, or payload renderers. | +| Shared publisher-label map | `release_manifest.publisher_labels`, keyed by the first Chronicle record-ID segment. Artifact labels override the generic humanizer; unknown prefixes remain valid. | +| Filter-pattern decomposition, region/sex/age value maps, and legacy US target-name parsing | A `dimensions` dictionary in `calibration_diagnostics` (label, semantic role, value labels, ordering) plus `targets[].dimensions` values. Geography dimensions also declare their level/id so no country geography fallback is needed. | +| Source/variable guesses from flat target names | Structured `targets[].source` and `targets[].variable` identifiers, with the publisher still traceable to `metadata.chronicle_record_ids`. | + +Schema readers must remain backward-compatible while published releases migrate. +After migration, name/filter parsing is a legacy adapter selected by artifact +schema version, never by country. diff --git a/frontend/components/layout/country-context.tsx b/frontend/components/layout/country-context.tsx index 1be64879..854bd3c3 100644 --- a/frontend/components/layout/country-context.tsx +++ b/frontend/components/layout/country-context.tsx @@ -12,7 +12,7 @@ export type Country = "us" | "uk" | "be"; const COUNTRIES = new Set(["us", "uk", "be"]); -function isCountry(value: string | null): value is Country { +export function isCountry(value: string | null): value is Country { return value != null && COUNTRIES.has(value as Country); } diff --git a/frontend/components/microcosm/artifact-description-banner.test.ts b/frontend/components/microcosm/artifact-description-banner.test.ts new file mode 100644 index 00000000..74ccc2c4 --- /dev/null +++ b/frontend/components/microcosm/artifact-description-banner.test.ts @@ -0,0 +1,27 @@ +import { expect, test } from "bun:test"; +import { createElement } from "react"; +import { renderToStaticMarkup } from "react-dom/server"; + +import { ArtifactDescriptionBanner } from "./artifact-description-banner"; + +test("renders artifact provenance as a quiet generic note", () => { + const markup = renderToStaticMarkup( + createElement(ArtifactDescriptionBanner, { + description: "Producer-authored provenance.", + }), + ); + + expect(markup).toContain('role="note"'); + expect(markup).toContain("Provenance"); + expect(markup).toContain("Producer-authored provenance."); + expect(markup).toContain("border-border/80"); + expect(markup).not.toContain("var(--warn)"); +}); + +test("renders nothing when the artifact has no description", () => { + expect( + renderToStaticMarkup( + createElement(ArtifactDescriptionBanner, { description: " " }), + ), + ).toBe(""); +}); diff --git a/frontend/components/microcosm/artifact-description-banner.tsx b/frontend/components/microcosm/artifact-description-banner.tsx index 9a191ca6..95b65e25 100644 --- a/frontend/components/microcosm/artifact-description-banner.tsx +++ b/frontend/components/microcosm/artifact-description-banner.tsx @@ -1,9 +1,5 @@ -import { StatusPill } from "@/components/shared/status-pill"; - -// Provenance banner: renders the release's own description verbatim whenever -// the artifact carries one (Belgium states its US-survey donor support pool -// and Chronicle-fact target rule here). Releases without a description render -// nothing. +// Provenance note: render the release's own description verbatim whenever the +// artifact carries one. Releases without a description render nothing. export function ArtifactDescriptionBanner({ description, }: { @@ -14,10 +10,12 @@ export function ArtifactDescriptionBanner({ return (
- Provenance -

{description}

+

+ Provenance +

+

{description}

); } diff --git a/frontend/components/microcosm/external-validations-panel.tsx b/frontend/components/microcosm/external-validations-panel.tsx deleted file mode 100644 index 31b89508..00000000 --- a/frontend/components/microcosm/external-validations-panel.tsx +++ /dev/null @@ -1,185 +0,0 @@ -import { EmptyState } from "@/components/shared/empty-state"; -import { fmt, fmtCompact } from "@/components/shared/format"; -import { SectionCard } from "@/components/shared/section-card"; -import { StatusPill, type StatusTone } from "@/components/shared/status-pill"; -import { - shapeExternalValidations, - type ExternalValidationColumnLedger, - type ExternalValidationKeyRow, - type ExternalValidationSurfaceRow, -} from "@/lib/microcosm/external-validations"; - -function ledgerTone(classification: string | null): StatusTone { - switch (classification?.toLowerCase()) { - case "matched": - return "success"; - case "gap": - return "danger"; - case "explained": - return "info"; - default: - return "neutral"; - } -} - -function euroValue(value: number | null): string { - if (value == null || !Number.isFinite(value)) return "—"; - if (Math.abs(value) >= 1_000) return fmtCompact(value); - return new Intl.NumberFormat("en", { maximumFractionDigits: 6 }).format(value); -} - -function LedgerChip({ label, value }: { label: string; value: number | null }) { - return ( -
-
- {label} -
-
- {fmt(value, { digits: 0 })} -
-
- ); -} - -function KeyRowsTable({ rows }: { rows: ExternalValidationKeyRow[] }) { - if (!rows.length) { - return ; - } - return ( -
- - - - - - - - - - - - {rows.map((row) => ( - - - - - - - - ))} - -
ColumnEUROMOD €Axiom €Δ €Classification
-
{row.column}
- {row.label ? ( -
{row.label}
- ) : null} - 		- {euroValue(row.euromodEur)} - - {euroValue(row.axiomEur)} - - {euroValue(row.deltaAxiomMinusEuromodEur)} - - {row.classification ? ( - - {row.classification} - - ) : ( - — - )} - {row.explanationClass ? ( -
- {row.explanationClass} -
- ) : null} -
-
- ); -} - -function ColumnLedger({ ledger }: { ledger: ExternalValidationColumnLedger }) { - return ( -
-
- - - - -
-

- {fmt(ledger.substantiveColumns, { digits: 0 })} substantive columns of{" "} - {fmt(ledger.totalColumns, { digits: 0 })} total · tolerance € - {ledger.toleranceEur == null ? "—" : euroValue(ledger.toleranceEur)} -

- -
- ); -} - -function ValidationSurfaceTable({ rows }: { rows: ExternalValidationSurfaceRow[] }) { - if (!rows.length) { - return ; - } - return ( -
- - - - - - - - - - - {rows.map((row) => ( - - - - - - - ))} - -
NameValueYearPublisher
-
{row.name}
- {row.concept ? ( -
{row.concept}
- ) : null} - 		- {fmtCompact(row.value)} - - {fmt(row.year, { digits: 0 })} - - {row.publisher ?? "—"} -
-
- ); -} - -export function ExternalValidationsPanel({ releaseManifest }: { releaseManifest: unknown }) { - const validations = shapeExternalValidations(releaseManifest); - if (!validations.length) return null; - - return ( - <> - {validations.map((validation) => ( - -
- {validation.columnLedger ? ( - - ) : null} - {validation.columnLedger && validation.validationSurface.length ? ( -
- ) : null} - -
- - ))} - - ); -} diff --git a/frontend/components/microcosm/microcosm-overview-view.tsx b/frontend/components/microcosm/microcosm-overview-view.tsx index 44d1ca4f..762df186 100644 --- a/frontend/components/microcosm/microcosm-overview-view.tsx +++ b/frontend/components/microcosm/microcosm-overview-view.tsx @@ -8,7 +8,6 @@ import { CalibrationExplorerMap, } from "@/components/microcosm/calibration-explorer-map"; import { ArtifactDescriptionBanner } from "@/components/microcosm/artifact-description-banner"; -import { ExternalValidationsPanel } from "@/components/microcosm/external-validations-panel"; import { WEIGHTED_TARGET_ERROR_HELP } from "@/components/microcosm/calibration-explorer-view"; import { useCountry, type Country } from "@/components/layout/country-context"; import { EmptyState } from "@/components/shared/empty-state"; @@ -258,8 +257,6 @@ export function MicrocosmOverviewView() {
- - { - test("shapes the real BE EUROMOD headline without losing producer labels or zeroes", () => { - const [validation] = shapeExternalValidations(releaseManifestFixture); - - expect(validation.key).toBe("euromod"); - expect(validation.comparator).toBe("EUROMOD BE_2025 (JRC), plain baseline run"); - expect(validation.description).toBe( - "Cross-engine column ledger: every EUROMOD BE_2025 output column computed on this population by both EUROMOD and the Axiom rules engine, classified matched / explained (named mechanism) / gap (named owner). " + - DISCLAIMER, - ); - expect(validation.columnLedger).toMatchObject({ - totalColumns: 558, - substantiveColumns: 182, - matched: 10, - explained: 143, - gap: 29, - unclassified: 0, - toleranceEur: 0.01, - }); - expect(validation.columnLedger?.keyRows).toEqual([ - { - column: "tin_s", - label: "income tax", - euromodEur: 68393536598.46828, - axiomEur: 68581762491.69119, - deltaAxiomMinusEuromodEur: 188225893.22291565, - classification: "GAP", - explanationClass: "unencoded_corpus_blocked", - }, - { - column: "ils_origy", - label: null, - euromodEur: 236840063794.08032, - axiomEur: 236840063794.08026, - deltaAxiomMinusEuromodEur: -0.00006103515625, - classification: "MATCHED", - explanationClass: null, - }, - { - column: "bsa_s", - label: "Income support", - euromodEur: 20866361321.659664, - axiomEur: 11063297817.463074, - deltaAxiomMinusEuromodEur: -9803063504.19659, - classification: "EXPLAINED", - explanationClass: "population_basis_construct", - }, - ]); - }); - - test("derives every validation-surface publisher from its first Chronicle id", () => { - const [validation] = shapeExternalValidations(releaseManifestFixture); - - expect(validation.validationSurface.map((row) => row.publisher)).toEqual([ - "onem_rva", - "jrc", - "eurostat", - "nbb", - ]); - expect(validation.validationSurface[0]).toEqual({ - name: "onem_complete_unemployment_recipients", - concept: "complete-unemployment monthly-average recipients", - value: 323029, - year: 2024, - role: "validation", - chronicleRecordIds: [ - "onem_rva.unemployment.cy2024.complete_unemployment.country.country_total.receives_unemployment_benefit", - ], - publisher: "onem_rva", - }); - }); - - test("iterates arbitrary validator names and preserves their strings verbatim", () => { - const [validation] = shapeExternalValidations({ - external_validations: { - independent_engine: { - comparator: " Published comparator label ", - description: "Producer-authored description / classification labels.", - column_ledger: { - matched: 0, - key_rows: [{ column: "x", classification: "Producer Label" }], - }, - }, - }, - }); - - expect(validation.key).toBe("independent_engine"); - expect(validation.comparator).toBe(" Published comparator label "); - expect(validation.description).toBe( - "Producer-authored description / classification labels.", - ); - expect(validation.columnLedger?.matched).toBe(0); - expect(validation.columnLedger?.keyRows[0].classification).toBe("Producer Label"); - }); - - test("degrades partial and malformed blocks to nullable fields and empty rows", () => { - expect(shapeExternalValidations(null)).toEqual([]); - expect(shapeExternalValidations({ external_validations: [] })).toEqual([]); - expect(chroniclePublisher([null, " jrc.fact.id "])).toBe("jrc"); - - expect( - shapeExternalValidations({ - external_validations: { - ignored: null, - partial: { - column_ledger: { - matched: Number.NaN, - unclassified: 0, - key_rows: [null, { column: "" }, { column: "kept" }], - }, - validation_surface: [42, { name: "" }, { name: "kept" }], - }, - }, - }), - ).toEqual([ - { - key: "partial", - comparator: null, - description: null, - columnLedger: { - totalColumns: null, - substantiveColumns: null, - matched: null, - explained: null, - gap: null, - unclassified: 0, - toleranceEur: null, - keyRows: [ - { - column: "kept", - label: null, - euromodEur: null, - axiomEur: null, - deltaAxiomMinusEuromodEur: null, - classification: null, - explanationClass: null, - }, - ], - }, - validationSurface: [ - { - name: "kept", - concept: null, - value: null, - year: null, - role: null, - chronicleRecordIds: [], - publisher: null, - }, - ], - }, - ]); - }); -}); - -test("trimmed BE fixture keeps representative targets and the exact demo disclaimer", () => { - expect(diagnosticsFixture.targets).toHaveLength(3); - expect(diagnosticsFixture.targets.map((row) => row.name)).toEqual([ - "statbel_population_be1_male_0_17@2026", - "jrc_national_income_tax_amount@2023", - "nasa_taxable_movable_income_analogue@2024", - ]); - expect(diagnosticsFixture.description).toBe(DISCLAIMER); - expect(diagnosticsFixture.targets.every((row) => row.metadata.disclaimer === DISCLAIMER)).toBe( - true, - ); -}); diff --git a/frontend/lib/microcosm/external-validations.ts b/frontend/lib/microcosm/external-validations.ts deleted file mode 100644 index 3a63665d..00000000 --- a/frontend/lib/microcosm/external-validations.ts +++ /dev/null @@ -1,144 +0,0 @@ -type JsonObject = Record; - -export interface ExternalValidationKeyRow { - column: string; - label: string | null; - euromodEur: number | null; - axiomEur: number | null; - deltaAxiomMinusEuromodEur: number | null; - classification: string | null; - explanationClass: string | null; -} - -export interface ExternalValidationColumnLedger { - totalColumns: number | null; - substantiveColumns: number | null; - matched: number | null; - explained: number | null; - gap: number | null; - unclassified: number | null; - toleranceEur: number | null; - keyRows: ExternalValidationKeyRow[]; -} - -export interface ExternalValidationSurfaceRow { - name: string; - concept: string | null; - value: number | null; - year: number | null; - role: string | null; - chronicleRecordIds: string[]; - publisher: string | null; -} - -export interface ExternalValidation { - key: string; - comparator: string | null; - description: string | null; - columnLedger: ExternalValidationColumnLedger | null; - validationSurface: ExternalValidationSurfaceRow[]; -} - -function objectOrNull(value: unknown): JsonObject | null { - return value && typeof value === "object" && !Array.isArray(value) - ? (value as JsonObject) - : null; -} - -function stringOrNull(value: unknown): string | null { - return typeof value === "string" && value.trim() ? value : null; -} - -function numberOrNull(value: unknown): number | null { - return typeof value === "number" && Number.isFinite(value) ? value : null; -} - -function strings(value: unknown): string[] { - return Array.isArray(value) - ? value.filter((entry): entry is string => typeof entry === "string" && Boolean(entry.trim())) - : []; -} - -/** Return the source prefix of the first Chronicle record id, if one is published. */ -export function chroniclePublisher(recordIds: unknown): string | null { - const first = strings(recordIds)[0]?.trim(); - if (!first) return null; - return first.split(".", 1)[0] || null; -} - -function shapeKeyRows(value: unknown): ExternalValidationKeyRow[] { - if (!Array.isArray(value)) return []; - return value.flatMap((entry) => { - const row = objectOrNull(entry); - const column = stringOrNull(row?.column); - if (!row || !column) return []; - return [{ - column, - label: stringOrNull(row.label), - euromodEur: numberOrNull(row.euromod_eur), - axiomEur: numberOrNull(row.axiom_eur), - deltaAxiomMinusEuromodEur: numberOrNull(row.delta_axiom_minus_euromod_eur), - classification: stringOrNull(row.classification), - explanationClass: stringOrNull(row.explanation_class), - }]; - }); -} - -function shapeColumnLedger(value: unknown): ExternalValidationColumnLedger | null { - const ledger = objectOrNull(value); - if (!ledger) return null; - return { - totalColumns: numberOrNull(ledger.total_columns), - substantiveColumns: numberOrNull(ledger.substantive_columns), - matched: numberOrNull(ledger.matched), - explained: numberOrNull(ledger.explained), - gap: numberOrNull(ledger.gap), - unclassified: numberOrNull(ledger.unclassified), - toleranceEur: numberOrNull(ledger.tolerance_eur), - keyRows: shapeKeyRows(ledger.key_rows), - }; -} - -function shapeValidationSurface(value: unknown): ExternalValidationSurfaceRow[] { - if (!Array.isArray(value)) return []; - return value.flatMap((entry) => { - const row = objectOrNull(entry); - const name = stringOrNull(row?.name); - if (!row || !name) return []; - const chronicleRecordIds = strings(row.chronicle_record_ids); - return [{ - name, - concept: stringOrNull(row.concept), - value: numberOrNull(row.value), - year: numberOrNull(row.year), - role: stringOrNull(row.role), - chronicleRecordIds, - publisher: chroniclePublisher(chronicleRecordIds), - }]; - }); -} - -/** - * Shape every external-validation block published by a release manifest. - * - * Validator keys and user-facing strings are left exactly as the producer - * supplied them. Unknown or partial blocks degrade to nullable headline fields - * and empty row arrays so a new country or validator cannot crash the summary. - */ -export function shapeExternalValidations(releaseManifest: unknown): ExternalValidation[] { - const manifest = objectOrNull(releaseManifest); - const validations = objectOrNull(manifest?.external_validations); - if (!validations) return []; - - return Object.entries(validations).flatMap(([key, value]) => { - const validation = objectOrNull(value); - if (!validation) return []; - return [{ - key, - comparator: stringOrNull(validation.comparator), - description: stringOrNull(validation.description), - columnLedger: shapeColumnLedger(validation.column_ledger), - validationSurface: shapeValidationSurface(validation.validation_surface), - }]; - }); -} diff --git a/frontend/lib/microcosm/fixtures/be-release/release_manifest.json b/frontend/lib/microcosm/fixtures/be-release/release_manifest.json index 315f7330..7fb4bacf 100644 --- a/frontend/lib/microcosm/fixtures/be-release/release_manifest.json +++ b/frontend/lib/microcosm/fixtures/be-release/release_manifest.json @@ -20,91 +20,5 @@ }, "default_datasets": { "national": "microcosm_be_v04_2026" - }, - "external_validations": { - "euromod": { - "comparator": "EUROMOD BE_2025 (JRC), plain baseline run", - "description": "Cross-engine column ledger: every EUROMOD BE_2025 output column computed on this population by both EUROMOD and the Axiom rules engine, classified matched / explained (named mechanism) / gap (named owner). Microcosm-BE: synthetic Belgian population calibrated to Belgian administrative and national-accounts targets (sums of Chronicle facts; surveys validation-only). Support records: US survey donor pool, reweighted; a Belgian donor pool is the planned upgrade. Cross-engine agreement is evidence about the encodings.", - "column_ledger": { - "total_columns": 558, - "substantive_columns": 182, - "matched": 10, - "explained": 143, - "gap": 29, - "unclassified": 0, - "tolerance_eur": 0.01, - "key_rows": [ - { - "column": "tin_s", - "label": "income tax", - "euromod_eur": 68393536598.46828, - "axiom_eur": 68581762491.69119, - "delta_axiom_minus_euromod_eur": 188225893.22291565, - "classification": "GAP", - "explanation_class": "unencoded_corpus_blocked" - }, - { - "column": "ils_origy", - "label": null, - "euromod_eur": 236840063794.08032, - "axiom_eur": 236840063794.08026, - "delta_axiom_minus_euromod_eur": -0.00006103515625, - "classification": "MATCHED", - "explanation_class": null - }, - { - "column": "bsa_s", - "label": "Income support", - "euromod_eur": 20866361321.659664, - "axiom_eur": 11063297817.463074, - "delta_axiom_minus_euromod_eur": -9803063504.19659, - "classification": "EXPLAINED", - "explanation_class": "population_basis_construct" - } - ] - }, - "validation_surface": [ - { - "name": "onem_complete_unemployment_recipients", - "concept": "complete-unemployment monthly-average recipients", - "value": 323029.0, - "year": 2024, - "role": "validation", - "chronicle_record_ids": [ - "onem_rva.unemployment.cy2024.complete_unemployment.country.country_total.receives_unemployment_benefit" - ] - }, - { - "name": "jrc_employment_income_amount", - "concept": "administrative employment income", - "value": 175998000000.0, - "year": 2023, - "role": "validation", - "chronicle_record_ids": [ - "jrc.euromod_be.cy2023.macrovalidation.external_amounts.a3_2_employment_income_external_2023.value" - ] - }, - { - "name": "nasa_b3g_paid", - "concept": "ESA national-accounts B3G PAID", - "value": 39700000000.0, - "year": 2024, - "role": "validation", - "chronicle_record_ids": [ - "eurostat.nasa_10_nf_tr.cy2024.household_non_financial_transactions.paid.be.belgium_household_mixed_income_gross.amount" - ] - }, - { - "name": "nbb_household_disposable_income_2024", - "concept": "household gross disposable income", - "value": 362214800000.0, - "year": 2024, - "role": "validation", - "chronicle_record_ids": [ - "nbb.national_accounts.cy2024.household_disposable_income.country.country_total.household_disposable_income" - ] - } - ] - } } } diff --git a/frontend/lib/microcosm/fixtures/zz-release/calibration_diagnostics.json b/frontend/lib/microcosm/fixtures/zz-release/calibration_diagnostics.json new file mode 100644 index 00000000..00780cd3 --- /dev/null +++ b/frontend/lib/microcosm/fixtures/zz-release/calibration_diagnostics.json @@ -0,0 +1,120 @@ +{ + "schema_version": 1, + "effective_sample_size": 812.5, + "final_loss": 0.0125, + "fraction_within_10pct": 0.75, + "fraction_within_tolerance": 0.75, + "initial_loss": 0.25, + "l0_lambda": 0.0, + "n_nonzero": 2400, + "n_records": 2500, + "options": { + "method": "adam", + "epochs": 500, + "seed": 7 + }, + "weight_entity": "household", + "skipped": [], + "targets": [ + { + "name": "fixture.population.north.female.0_17@2026", + "target_name": "fixture.population.north.female.0_17", + "period": 2026, + "entity": "person", + "measure": { + "kind": "column", + "name": "person" + }, + "filter": "cell_north_female_0_17", + "source": "ZZ official population table", + "metadata": { + "chronicle_record_ids": [ + "novastat_agency.population.cy2026.people.by_region_age_sex.north_female_0_17.people" + ], + "concept": "resident population", + "variable": "population", + "source_measure_id": "population_count" + }, + "target": 1000, + "initial_estimate": 850, + "final_estimate": 980, + "relative_error": -0.02, + "within_tolerance": true + }, + { + "name": "fixture.population.south.male.18_64@2026", + "target_name": "fixture.population.south.male.18_64", + "period": 2026, + "entity": "person", + "measure": { + "kind": "column", + "name": "person" + }, + "filter": "cell_south_male_18_64", + "source": "ZZ official population table", + "metadata": { + "chronicle_record_ids": [ + "novastat_agency.population.cy2026.people.by_region_age_sex.south_male_18_64.people" + ], + "concept": "resident population", + "variable": "population", + "source_measure_id": "population_count" + }, + "target": 4000, + "initial_estimate": 4600, + "final_estimate": 4200, + "relative_error": 0.05, + "within_tolerance": true + }, + { + "name": "fixture.population.north.male.65_plus@2026", + "target_name": "fixture.population.north.male.65_plus", + "period": 2026, + "entity": "person", + "measure": { + "kind": "column", + "name": "person" + }, + "filter": "cell_north_male_65_plus", + "source": "ZZ official population table", + "metadata": { + "chronicle_record_ids": [ + "novastat_agency.population.cy2026.people.by_region_age_sex.north_male_65_plus.people" + ], + "concept": "resident population", + "variable": "population", + "source_measure_id": "population_count" + }, + "target": 900, + "initial_estimate": 700, + "final_estimate": 1035, + "relative_error": 0.15, + "within_tolerance": false + }, + { + "name": "fixture_revenue_personal_income_tax@2025", + "target_name": "fixture_revenue_personal_income_tax", + "period": 2025, + "entity": "person", + "measure": { + "kind": "column", + "name": "income_tax" + }, + "filter": null, + "source": "ZZ official revenue table", + "metadata": { + "chronicle_record_ids": [ + "novastat_agency.revenue.cy2025.personal_income_tax.country.total.amount" + ], + "concept": "personal income tax", + "variable": "personal_income_tax", + "source_measure_id": "personal_income_tax_amount" + }, + "target": 5000000000, + "initial_estimate": 4250000000, + "final_estimate": 4950000000, + "relative_error": -0.01, + "within_tolerance": true + } + ] +} diff --git a/frontend/lib/microcosm/fixtures/zz-release/release_manifest.json b/frontend/lib/microcosm/fixtures/zz-release/release_manifest.json new file mode 100644 index 00000000..54f1f11c --- /dev/null +++ b/frontend/lib/microcosm/fixtures/zz-release/release_manifest.json @@ -0,0 +1,17 @@ +{ + "schema_version": 1, + "description": "Synthetic ZZ release whose dashboard presentation comes from release artifacts.", + "build": { + "schema_version": 1, + "build_id": "microcosm-zz-2026-conformance", + "built_at": "2026-08-23T18:00:00Z", + "population_year": 2026 + }, + "data_package": { + "name": "microcosm-zz", + "version": "1.0.0" + }, + "default_datasets": { + "national": "microcosm_zz_2026" + } +} diff --git a/frontend/lib/microcosm/latest-artifact.test.ts b/frontend/lib/microcosm/latest-artifact.test.ts index 6ef6675b..68df241d 100644 --- a/frontend/lib/microcosm/latest-artifact.test.ts +++ b/frontend/lib/microcosm/latest-artifact.test.ts @@ -105,7 +105,7 @@ test("loads trimmed Belgium diagnostics without optional US artifact fields", () level: "national", }); expect(cal.rows[2]).toMatchObject({ - source: "nasa", + source: "eurostat", variable: "taxable movable income analogue", geography: "Belgium", level: "national", @@ -118,6 +118,7 @@ test("derives Belgium population region, sex, and age-band browser facets", () = ...first, name: `${name}@2026`, target_name: name, + filter: `cell_${name.replace(/^statbel_population_/, "")}`, }); const cal = buildCalibration( { @@ -148,6 +149,39 @@ test("derives Belgium population region, sex, and age-band browser facets", () = ]); }); +test("keeps legacy US dotted target families when Chronicle publisher metadata is present", () => { + const cal = buildCalibration( + { + targets: [ + { + name: "irs.population.total@2024", + target_name: "irs.population.total", + period: 2024, + entity: "person", + measure: { kind: "column", name: "person" }, + source: "IRS SOI", + metadata: { + chronicle_record_ids: ["irs.population.cy2024.total"], + variable: "population", + }, + target: 100, + initial_estimate: 95, + final_estimate: 99, + relative_error: -0.01, + within_tolerance: true, + }, + ], + }, + "us-dotted-family", + ); + + expect(cal.rows[0]).toMatchObject({ + source: "irs", + variable: "population", + family: "irs.population.total", + }); +}); + // A v2-shaped target: AGI bracket × return type × filing status, with @period. function agiTarget(band: string, ret: string, filing: string, rel: number) { return { diff --git a/frontend/lib/microcosm/latest-artifact.ts b/frontend/lib/microcosm/latest-artifact.ts index cf7f885f..453b4f53 100644 --- a/frontend/lib/microcosm/latest-artifact.ts +++ b/frontend/lib/microcosm/latest-artifact.ts @@ -32,30 +32,45 @@ export const MICROCOSM_HF_REPO = process.env[MICROCOSM_HF_REPO_ENV] ?? "policyengine/populace-us"; export const MICROCOSM_HF_REVISION = process.env[MICROCOSM_HF_REVISION_ENV] ?? "main"; -// Microcosm ships one HF dataset per country. US is public; UK and Belgium are -// private and need an HF token on the server. -export type MicrocosmCountry = "us" | "uk" | "be"; +interface MicrocosmCountryRepository { + repo: string; + revision: string; + geography: string; +} -const COUNTRY_REPO: Record = { - us: { repo: MICROCOSM_HF_REPO, revision: MICROCOSM_HF_REVISION }, +// A repository entry is the server-side registration point for a country. Keep +// its national geography beside the repository so downstream shaping does not +// require a second exhaustive country table. +export const COUNTRY_REPO = { + us: { + repo: MICROCOSM_HF_REPO, + revision: MICROCOSM_HF_REVISION, + geography: "United States", + }, uk: { repo: process.env[MICROCOSM_UK_HF_REPO_ENV] ?? "policyengine/populace-uk-private", revision: process.env[MICROCOSM_UK_HF_REVISION_ENV] ?? "main", + geography: "United Kingdom", }, be: { repo: process.env[MICROCOSM_BE_HF_REPO_ENV] ?? "policyengine/populace-be-private", revision: process.env[MICROCOSM_BE_HF_REVISION_ENV] ?? "main", + geography: "Belgium", }, -}; + // Synthetic repository used only by the third-country conformance fixture. + zz: { + repo: "policyengine/microcosm-zz-fixture", + revision: "main", + geography: "Zedland", + }, +} satisfies Record; -const COUNTRY_GEOGRAPHY: Record = { - us: "United States", - uk: "United Kingdom", - be: "Belgium", -}; +export type MicrocosmCountry = keyof typeof COUNTRY_REPO; export function parseCountry(value: string | null | undefined): MicrocosmCountry { - return value === "uk" || value === "be" ? value : "us"; + return value != null && Object.hasOwn(COUNTRY_REPO, value) + ? (value as MicrocosmCountry) + : "us"; } // Release/run ids are interpolated into HuggingFace URLs that carry the @@ -95,6 +110,10 @@ export function microcosmRevision(country: MicrocosmCountry): string { return COUNTRY_REPO[country].revision; } +export function microcosmCountryGeography(country: MicrocosmCountry): string { + return COUNTRY_REPO[country].geography; +} + function hfAuthHeaders(): HeadersInit | undefined { const token = process.env.HF_TOKEN ?? process.env.HUGGINGFACE_TOKEN; return token ? { Authorization: `Bearer ${token}` } : undefined; @@ -576,81 +595,119 @@ function metadataDimensions(row: TargetRow): TargetBreakdownDimension[] | null { return dimensions; } -const BELGIUM_PUBLISHERS = ["statbel", "onss", "jrc", "sfpd", "nasa"] as const; -const BELGIUM_REGIONS: Readonly> = { - be1: "Brussels", - be2: "Flanders", - be3: "Wallonia", -}; -const BELGIUM_SEXES: Readonly> = { - male: "Male", - female: "Female", -}; -const BELGIUM_AGE_BANDS: Readonly> = { - "0_17": "0–17", - "18_64": "18–64", - "65_plus": "65+", -}; -const BELGIUM_POPULATION_RE = - /^statbel_population_(be[123])_(male|female)_(0_17|18_64|65_plus)$/; +interface FilterDimensionSpec { + capture: number; + label: string; + role?: "geography"; + valueLabels?: Readonly>; +} + +interface FilterDecompositionSpec { + pattern: RegExp; + geographyLevel: string; + dimensions: readonly FilterDimensionSpec[]; +} + +// Legacy artifacts encode some dimensions in a compiled filter name. The +// adapter is selected by the filter pattern, never by country; future artifacts +// should publish structured target dimensions instead. +const FILTER_DECOMPOSITION_SPECS: readonly FilterDecompositionSpec[] = [ + { + pattern: /^cell_([a-z][a-z0-9-]*)_(male|female)_(\d+_(?:\d+|plus))$/, + geographyLevel: "region", + dimensions: [ + { + capture: 1, + label: "Region", + role: "geography", + valueLabels: { + be1: "Brussels", + be2: "Flanders", + be3: "Wallonia", + }, + }, + { capture: 2, label: "Sex" }, + { capture: 3, label: "Age band" }, + ], + }, +]; -function belgiumPublisher(name: string): string | null { - return BELGIUM_PUBLISHERS.find((publisher) => name.startsWith(`${publisher}_`)) ?? null; +interface DecomposedTargetFilter { + geography: string; + level: string; + dimensions: TargetBreakdownDimension[]; +} + +function filterDimensionValue(spec: FilterDimensionSpec, rawValue: string): string { + const explicit = spec.valueLabels?.[rawValue]; + if (explicit) return explicit; + if (spec.label === "Age band") { + const range = /^(\d+)_(\d+)$/.exec(rawValue); + if (range) return `${range[1]}–${range[2]}`; + const openEnded = /^(\d+)_plus$/.exec(rawValue); + if (openEnded) return `${openEnded[1]}+`; + } + return titleCase(rawValue); +} + +function decomposeTargetFilter(value: unknown): DecomposedTargetFilter | null { + const filter = stringValue(value); + if (!filter) return null; + for (const spec of FILTER_DECOMPOSITION_SPECS) { + const match = spec.pattern.exec(filter); + if (!match) continue; + let geography = ""; + const dimensions: TargetBreakdownDimension[] = []; + for (const dimension of spec.dimensions) { + const rawValue = match[dimension.capture]; + if (!rawValue) continue; + const valueLabel = filterDimensionValue(dimension, rawValue); + if (dimension.role === "geography") { + geography = valueLabel; + continue; + } + dimensions.push({ + key: dimensionKey(dimension.label), + label: dimension.label, + value: valueLabel, + source_key: "filter", + raw_value: rawValue, + }); + } + if (geography) { + return { geography, level: spec.geographyLevel, dimensions }; + } + } + return null; } -function belgiumPopulationParts(name: string) { - const match = BELGIUM_POPULATION_RE.exec(name); - if (!match) return null; - return { - region: BELGIUM_REGIONS[match[1]], - sex: BELGIUM_SEXES[match[2]], - ageBand: BELGIUM_AGE_BANDS[match[3]], - rawSex: match[2], - rawAgeBand: match[3], - }; +function chroniclePublisherFromMetadata(metadata: JsonObject): string | null { + if (!Array.isArray(metadata.chronicle_record_ids)) return null; + const first = metadata.chronicle_record_ids.find( + (value): value is string => typeof value === "string" && Boolean(value.trim()), + ); + return first?.trim().split(".", 1)[0] || null; } -function parseBelgiumTarget(name: string): ParsedTarget | null { - const publisher = belgiumPublisher(name); - if (!publisher) return null; - const population = belgiumPopulationParts(name); - if (population) { - return { - geography: population.region, - level: "region", - source: publisher, - variable: "population", - breakdown: `${population.sex} · ${population.ageBand}`, - }; - } - return { - geography: COUNTRY_GEOGRAPHY.be, - level: "national", - source: publisher, - variable: readableToken(name.slice(publisher.length + 1)) ?? "", - breakdown: "", - }; -} +function artifactVariable( + name: string, + row: TargetRow, + decomposition: DecomposedTargetFilter | null, +): string | null { + const metadata = asObject(row.metadata); + const published = stringValue(metadata.variable) ?? stringValue(row.variable); + if (published) return readableToken(published); + if (!name.includes("_") || name.includes("/")) return null; -function belgiumTargetDimensions(name: string): TargetBreakdownDimension[] | null { - const population = belgiumPopulationParts(name); - if (!population) return null; - return [ - { - key: dimensionKey("Sex"), - label: "Sex", - value: population.sex, - source_key: "target_name", - raw_value: population.rawSex, - }, - { - key: dimensionKey("Age band"), - label: "Age band", - value: population.ageBand, - source_key: "target_name", - raw_value: population.rawAgeBand, - }, - ]; + let identifier = name; + const filter = decomposition ? stringValue(row.filter) : null; + const filterSuffix = filter?.replace(/^cell_/, ""); + if (filterSuffix && identifier.endsWith(`_${filterSuffix}`)) { + identifier = identifier.slice(0, -(filterSuffix.length + 1)); + } + const separator = identifier.indexOf("_"); + if (separator >= 0) identifier = identifier.slice(separator + 1); + return readableToken(identifier); } function parseDottedTarget( @@ -667,7 +724,7 @@ function parseDottedTarget( const geoId = stringValue(metadata.ledger_geography_id); const geography = geoLevel === "country" - ? COUNTRY_GEOGRAPHY[country] + ? microcosmCountryGeography(country) : geoLevel === "congressional_district" ? districtFromGeoId(geoId) ?? "" : stateFromGeoId(geoId) ?? stringValue(metadata.state) ?? ""; @@ -700,10 +757,6 @@ function parseDottedTarget( } function parseTarget(name: string, country: MicrocosmCountry): ParsedTarget { - if (country === "be") { - const belgium = parseBelgiumTarget(name); - if (belgium) return belgium; - } const parts = name.split("/"); const p0 = parts[0] ?? ""; const fips = /^US(\d{2})$/.exec(p0); @@ -738,7 +791,7 @@ function parseTarget(name: string, country: MicrocosmCountry): ParsedTarget { } if (p0 === "nation" || p0 === "national" || p0 === "us") { return { - geography: COUNTRY_GEOGRAPHY[country], level: "national", source: parts[1] ?? "", + geography: microcosmCountryGeography(country), level: "national", source: parts[1] ?? "", variable: parts[2] ?? "", breakdown: parts.slice(3).join(" · "), }; } @@ -920,10 +973,10 @@ function computeDimensions(rows: TargetRow[]): TargetDimension[] { function deriveFamily( name: string, - country: MicrocosmCountry, parsed: ParsedTarget, + usesArtifactFamily: boolean, ): string { - if (country === "be") { + if (usesArtifactFamily) { return [parsed.source, parsed.variable].filter(Boolean).join("/"); } const parts = name.split("/"); @@ -1052,14 +1105,30 @@ function enrichTargetRow( initialError == null || finalError == null ? null : Math.abs(initialError) - Math.abs(finalError); - const parsed = parseDottedTarget(baseName, row, country) ?? parseTarget(baseName, country); + const filterDecomposition = decomposeTargetFilter(row.filter); + const publisher = chroniclePublisherFromMetadata(metadata); + const parsedFromName = + parseDottedTarget(baseName, row, country) ?? parseTarget(baseName, country); + const parsed: ParsedTarget = { + ...parsedFromName, + geography: filterDecomposition?.geography ?? parsedFromName.geography, + level: filterDecomposition?.level ?? parsedFromName.level, + source: publisher ?? parsedFromName.source, + variable: + artifactVariable(baseName, row, filterDecomposition) ?? + parsedFromName.variable, + breakdown: filterDecomposition + ? filterDecomposition.dimensions.map((dimension) => dimension.value).join(" · ") + : parsedFromName.breakdown, + }; const hasGeography = Boolean(parsed.geography.trim()); - const geography = hasGeography ? parsed.geography : COUNTRY_GEOGRAPHY[country]; + const geography = hasGeography + ? parsed.geography + : microcosmCountryGeography(country); const level = hasGeography ? parsed.level : DEFAULT_GEOGRAPHY_LEVEL; const measureCol = asObject(row.measure); - const countryTargetDimensions = - country === "be" ? belgiumTargetDimensions(baseName) : null; - const metadataTargetDimensions = countryTargetDimensions ?? metadataDimensions(row); + const metadataTargetDimensions = + filterDecomposition?.dimensions ?? metadataDimensions(row); const targetDimensions = metadataTargetDimensions ?? splitBreakdown(parsed.breakdown).map((value, index) => ({ @@ -1081,11 +1150,18 @@ function enrichTargetRow( : measureFromMetadata(metadata); const variableKey = variableKeyOf(parsed) + (measure ? ` · ${measure}` : ""); + // Underscore identifiers and filter-decomposed targets use the structured + // publisher/variable identity supplied by the artifact. Dotted identifiers + // without filter dimensions retain their legacy family so US/UK releases do + // not regroup merely because they also carry Chronicle record IDs. + const usesArtifactFamily = + filterDecomposition != null || + (publisher != null && !baseName.includes("/") && !baseName.includes(".")); return { ...row, name: fullName, base_name: baseName, - family: deriveFamily(baseName, country, parsed), + family: deriveFamily(baseName, parsed, usesArtifactFamily), state: stateFromGeoId(stringValue(metadata.ledger_geography_id)) ?? deriveState(baseName), geography, level, diff --git a/frontend/lib/microcosm/source-label.test.ts b/frontend/lib/microcosm/source-label.test.ts index 8e595e8f..b465c8dc 100644 --- a/frontend/lib/microcosm/source-label.test.ts +++ b/frontend/lib/microcosm/source-label.test.ts @@ -12,7 +12,7 @@ describe("sourceLabel", () => { test("formats an unknown source without changing its identifier", () => { const source = "new_data_source"; - expect(sourceLabel(source)).toBe("NEW Data Source"); + expect(sourceLabel(source)).toBe("New Data Source"); expect(source).toBe("new_data_source"); }); }); diff --git a/frontend/lib/microcosm/source-label.ts b/frontend/lib/microcosm/source-label.ts index 06f4437a..9823a7d5 100644 --- a/frontend/lib/microcosm/source-label.ts +++ b/frontend/lib/microcosm/source-label.ts @@ -1,33 +1,5 @@ -// Canonical labels for the authorities behind calibration targets. Keep this -// shared between the legacy aggregate map and the drillable explorer so raw -// source IDs never leak into either UI. -const SOURCE_LABELS: Record = { - cbo: "CBO", - census_population: "Census population", - cms_aca: "CMS · ACA marketplace", - cms_medicaid: "CMS · Medicaid / CHIP", - cms_medicare: "CMS · Medicare", - hhs_acf_tanf: "HHS · TANF", - irs_soi: "IRS Statistics of Income", - jct: "JCT", - jrc: "JRC", - nasa: "NASA", - onss: "ONSS", - sfpd: "SFPD", - ssa: "SSA", - statbel: "Statbel", - state_income_tax: "State income tax", - usda_snap: "USDA · SNAP", -}; +import { sourceAuthorityLabel } from "@/lib/source-labels"; export function sourceLabel(source: string): string { - if (SOURCE_LABELS[source]) return SOURCE_LABELS[source]; - return source - .split("_") - .map((word) => - word.length <= 3 - ? word.toUpperCase() - : word[0].toUpperCase() + word.slice(1), - ) - .join(" "); + return sourceAuthorityLabel(source); } diff --git a/frontend/lib/microcosm/staging-artifact.ts b/frontend/lib/microcosm/staging-artifact.ts index b533d882..937a063b 100644 --- a/frontend/lib/microcosm/staging-artifact.ts +++ b/frontend/lib/microcosm/staging-artifact.ts @@ -8,6 +8,7 @@ import { latestMicrocosmCalibrationSummary, latestMicrocosmTargetDiagnosticsPage, loadRelease, + microcosmCountryGeography, } from "@/lib/microcosm/latest-artifact"; import { type ReformValidation, @@ -25,14 +26,10 @@ export const MICROCOSM_STAGING_HF_REPO = export const MICROCOSM_STAGING_HF_REVISION = process.env[MICROCOSM_STAGING_HF_REVISION_ENV] ?? "main"; -const COUNTRY_NAME: Record = { - us: "United States", - uk: "United Kingdom", - be: "Belgium", -}; - export function stagingUnavailableReason(country: MicrocosmCountry): string | null { - return country === "us" ? null : `${COUNTRY_NAME[country]} has no staging repository.`; + return country === "us" + ? null + : `${microcosmCountryGeography(country)} has no staging repository.`; } function unavailableStaging(country: MicrocosmCountry) { diff --git a/frontend/lib/microcosm/third-country-conformance.test.ts b/frontend/lib/microcosm/third-country-conformance.test.ts new file mode 100644 index 00000000..a0264c86 --- /dev/null +++ b/frontend/lib/microcosm/third-country-conformance.test.ts @@ -0,0 +1,252 @@ +import { describe, expect, test } from "bun:test"; + +import { isCountry as isClientCountry } from "@/components/layout/country-context"; +import { sourceAuthorityLabel } from "@/lib/source-labels"; + +import diagnosticsFixture from "./fixtures/zz-release/calibration_diagnostics.json"; +import releaseManifestFixture from "./fixtures/zz-release/release_manifest.json"; +import { + buildCalibration, + hfResolveUrl, + latestMicrocosmCalibrationHighlights, + latestMicrocosmCalibrationSummary, + latestMicrocosmTargetDiagnosticsPage, + microcosmRepo, + microcosmRevision, + microcosmTargetTreemap, + parseCountry, +} from "./latest-artifact"; + +const COUNTRY = "zz" as const; +const RELEASE_ID = "microcosm-zz-2026-conformance"; +const DESCRIPTION = + "Synthetic ZZ release whose dashboard presentation comes from release artifacts."; +const PUBLISHER = "novastat_agency"; + +// Registration-only contract: production presentation/parsing code must not +// gain a `zz` branch or table entry to make these data-shape assertions pass. +const calibration = buildCalibration( + diagnosticsFixture, + RELEASE_ID, + "2026-08-23T18:00:00Z", + {}, + releaseManifestFixture, + {}, + COUNTRY, +); + +describe("synthetic third-country conformance", () => { + test("one COUNTRY_REPO registration supplies repository and national geography behavior", () => { + expect(parseCountry(COUNTRY)).toBe(COUNTRY); + expect(microcosmRepo(COUNTRY)).toBe("policyengine/microcosm-zz-fixture"); + expect(microcosmRevision(COUNTRY)).toBe("main"); + expect(hfResolveUrl("latest.json", COUNTRY)).toBe( + "https://huggingface.co/datasets/policyengine/microcosm-zz-fixture/resolve/main/latest.json", + ); + + const nationalTarget = calibration.rows.find( + (row) => row.base_name === "fixture_revenue_personal_income_tax", + ); + expect(nationalTarget).toMatchObject({ + geography: "Zedland", + level: "national", + }); + }); + + test("release artifacts produce the common overview response shape", () => { + const overview = latestMicrocosmCalibrationSummary(calibration); + const highlights = latestMicrocosmCalibrationHighlights(calibration); + + expect(overview).toMatchObject({ + available: true, + description: DESCRIPTION, + diagnostics_status: "ok", + is_default: true, + release_id: RELEASE_ID, + total_targets: 4, + included_target_count: 4, + within_tolerance_count: 3, + fraction_within_10pct: 0.75, + }); + expect(overview.family_fit).toHaveLength(2); + expect(highlights.worst_bounded_relative_fit).toHaveLength(4); + expect(highlights.extreme_relative_outlier_count).toBe(0); + }); + + test("Chronicle publisher prefixes drive source keys and readable labels", () => { + expect(calibration.rows.every((row) => row.source === PUBLISHER)).toBe(true); + expect(sourceAuthorityLabel(PUBLISHER)).toBe("Novastat Agency"); + + const treemap = microcosmTargetTreemap( + calibration.rows, + calibration.release_id, + "program", + false, + ); + expect(treemap.groups).toEqual([ + expect.objectContaining({ + source: PUBLISHER, + label: "Novastat Agency", + n_targets: 4, + }), + ]); + }); + + test("filter-coded targets produce generic region, sex, and age-band facets", () => { + const variableKey = `${PUBLISHER} / population · count`; + const page = latestMicrocosmTargetDiagnosticsPage( + `http://x/api/microcosm/target-diagnostics?variable=${encodeURIComponent(variableKey)}`, + calibration, + ); + + expect(page).toMatchObject({ + available: true, + description: DESCRIPTION, + diagnostics_status: "ok", + release_id: RELEASE_ID, + sources: [PUBLISHER], + filtered_total: 3, + returned: 3, + }); + expect(page.dimensions).toEqual([ + { key: "geography", label: "Region", values: ["North", "South"] }, + { key: "bd_sex", label: "Sex", values: ["Female", "Male"] }, + { key: "bd_age_band", label: "Age band", values: ["0–17", "18–64", "65+"] }, + ]); + expect(page.targets).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + source: PUBLISHER, + source_citation: "ZZ official population table", + variable: "population", + geography: "North", + level: "region", + target_dimensions: expect.arrayContaining([ + expect.objectContaining({ + key: "bd_sex", + label: "Sex", + value: "Female", + source_key: "filter", + raw_value: "female", + }), + expect.objectContaining({ + key: "bd_age_band", + label: "Age band", + value: "0–17", + source_key: "filter", + raw_value: "0_17", + }), + ]), + }), + ]), + ); + }); + + test.todo( + "zz can enter the shared client views: the client country parser still uses a closed country table", + () => { + expect(isClientCountry(COUNTRY)).toBe(true); + }, + ); + test.todo( + "zz overview data can carry artifact-provided intro copy: the summary omits a typed presentation contract", + () => { + const presentation = { + overview_intro: "Zedland overview copy from the release artifact.", + targets_intro: "Zedland target-browser copy from the release artifact.", + }; + const futureCalibration = buildCalibration( + diagnosticsFixture, + RELEASE_ID, + "2026-08-23T18:00:00Z", + {}, + { ...releaseManifestFixture, presentation }, + {}, + COUNTRY, + ); + const overview = latestMicrocosmCalibrationSummary(futureCalibration) as { + presentation?: typeof presentation; + }; + + expect(overview.presentation).toEqual(presentation); + }, + ); + test.todo( + "zz can override a publisher display name: treemap shaping ignores release_manifest.publisher_labels", + () => { + const futureCalibration = buildCalibration( + diagnosticsFixture, + RELEASE_ID, + "2026-08-23T18:00:00Z", + {}, + { + ...releaseManifestFixture, + publisher_labels: { [PUBLISHER]: "Nova Statistics Agency" }, + }, + {}, + COUNTRY, + ); + const treemap = microcosmTargetTreemap( + futureCalibration.rows, + futureCalibration.release_id, + "program", + false, + ); + + expect(treemap.groups[0]?.label).toBe("Nova Statistics Agency"); + }, + ); + test.todo( + "zz can supply structured facets: target shaping ignores calibration_diagnostics dimensions blocks", + () => { + const facetValues = [ + { region: "north", sex: "female", age_band: "0_17" }, + { region: "south", sex: "male", age_band: "18_64" }, + { region: "north", sex: "male", age_band: "65_plus" }, + ]; + const futureDiagnostics = { + ...diagnosticsFixture, + dimensions: { + region: { + label: "Region", + role: "geography", + level: "region", + values: { north: "North", south: "South" }, + }, + sex: { + label: "Sex", + values: { female: "Female", male: "Male" }, + }, + age_band: { + label: "Age band", + values: { "0_17": "0–17", "18_64": "18–64", "65_plus": "65+" }, + }, + }, + targets: diagnosticsFixture.targets.map((target, index) => + index < facetValues.length + ? { ...target, filter: null, dimensions: facetValues[index] } + : target, + ), + }; + const futureCalibration = buildCalibration( + futureDiagnostics, + RELEASE_ID, + "2026-08-23T18:00:00Z", + {}, + releaseManifestFixture, + {}, + COUNTRY, + ); + const page = latestMicrocosmTargetDiagnosticsPage( + `http://x/api/microcosm/target-diagnostics?variable=${encodeURIComponent(`${PUBLISHER} / population · count`)}`, + futureCalibration, + ); + + expect(page.dimensions).toEqual([ + { key: "geography", label: "Region", values: ["North", "South"] }, + { key: "bd_sex", label: "Sex", values: ["Female", "Male"] }, + { key: "bd_age_band", label: "Age band", values: ["0–17", "18–64", "65+"] }, + ]); + }, + ); +}); diff --git a/frontend/lib/slack.ts b/frontend/lib/slack.ts index 5c9b3995..f0d97e2c 100644 --- a/frontend/lib/slack.ts +++ b/frontend/lib/slack.ts @@ -2,7 +2,7 @@ import type { MicrocosmCountry } from "@/lib/microcosm/latest-artifact"; const DASHBOARD_URL = "https://microcosm.institute/calibration/dashboard/microcosm"; -const COUNTRY_LABEL: Record = { +const COUNTRY_LABEL: Partial> = { us: "🇺🇸 US", uk: "🇬🇧 UK", be: "🇧🇪 BE", @@ -28,7 +28,7 @@ export async function postReleaseAlert(opts: { const webhookUrl = process.env[webhookEnv]; if (!webhookUrl) return false; - const label = COUNTRY_LABEL[opts.country]; + const label = COUNTRY_LABEL[opts.country] ?? opts.country.toUpperCase(); const context = [ opts.repo, opts.updatedAt ? `published ${opts.updatedAt}` : "", diff --git a/frontend/lib/source-labels.test.ts b/frontend/lib/source-labels.test.ts index 205264c4..a21c884f 100644 --- a/frontend/lib/source-labels.test.ts +++ b/frontend/lib/source-labels.test.ts @@ -18,3 +18,11 @@ test("uses curated authority names for Chronicle and calibration sources", () => test("uses the calibration-fit acronym-aware fallback for unknown sources", () => { expect(sourceAuthorityLabel("new_api_source")).toBe("New API Source"); }); + +test("shares publisher labels across country dashboards", () => { + expect( + ["statbel", "onss", "jrc", "sfpd", "nasa", "eurostat"].map( + sourceAuthorityLabel, + ), + ).toEqual(["Statbel", "ONSS", "JRC", "SFPD", "NASA", "Eurostat"]); +}); diff --git a/frontend/lib/source-labels.ts b/frontend/lib/source-labels.ts index fc4b1da6..52b20bb5 100644 --- a/frontend/lib/source-labels.ts +++ b/frontend/lib/source-labels.ts @@ -13,6 +13,7 @@ const SOURCE_LABELS: Record = { cms_medicaid: "CMS · Medicaid / CHIP", cms_medicare: "CMS · Medicare", cms_nhe: "CMS · National Health Expenditure Accounts", + eurostat: "Eurostat", federal_reserve: "Federal Reserve", hhs_acf_liheap: "HHS · LIHEAP", hhs_acf_tanf: "HHS · TANF",