diff --git a/docs/spec-driven-countries.md b/docs/spec-driven-countries.md index 553603b..6d438af 100644 --- a/docs/spec-driven-countries.md +++ b/docs/spec-driven-countries.md @@ -11,12 +11,14 @@ role/default status, and the optional provenance `description`; demographics own geography coverage. A description may fill the existing provenance note, but an artifact cannot add sections or choose components. -Target sources are derived generically from the first dotted segment of -`metadata.chronicle_record_ids`, with one shared publisher-label map and a -humanized fallback for unknown prefixes. Filter-coded target facets may be -decoded by a country-agnostic pattern spec. Cross-dataset comparisons are not -part of this release contract: they are served only by `/microcosm/datasets` -from `cross_dataset.frontend_bundle.v1`. +Target source IDs are derived generically from the first dotted segment of +`metadata.chronicle_record_ids`, with structured source IDs and legacy name +parsing as fallbacks. Artifact publisher labels take precedence over the shared +publisher-label map and humanized unknown prefixes. Filter-coded target facets +may be decoded by a country-agnostic pattern spec when a row does not publish +structured dimensions. Cross-dataset comparisons are not part of this release +contract: they are served only by `/microcosm/datasets` from +`cross_dataset.frontend_bundle.v1`. ## Code-owned metadata that remains @@ -24,10 +26,12 @@ from `cross_dataset.frontend_bundle.v1`. registration point: country unions and parsers, selector lists, dataset and national-geography labels, public/private link behavior, and page gating read the registration or the artifact `country` block. Overview and target-browser -copy still sit in legacy per-country tables with a generic fallback. Publisher -display names remain in a shared TypeScript map. Target decomposition still has a -generic filter-pattern table plus legacy US name grammar (FIPS/state, filing -status, return type, income band, and qualifying-child rules). +copy retains marked per-country tables only as fallbacks for releases without +artifact `presentation`. Publisher display names retain a shared TypeScript map +as the fallback for publisher keys absent from artifact `publisher_labels`. +Target decomposition retains a generic filter-pattern table plus legacy US name +grammar (FIPS/state, filing status, return type, income band, and +qualifying-child rules) for rows without structured dimensions. These tables are presentation or parsing metadata, not calibration logic. They should not grow another country branch. @@ -47,9 +51,10 @@ produce: unknown-prefix fallback; and - filter-derived geography and target dimensions with stable labels and values. -Assertions that expose a current contract gap are `test.todo` with a one-line -reason. The todos are the migration backlog; conformance is complete when all can -be enabled without adding `zz` conditionals or tables. +The conformance suite includes passing assertions for artifact presentation, +publisher labels, structured dimensions, and structured source and variable +identifiers. It has no remaining todos and requires no `zz` conditionals or +production tables. ## Schema additions that remove the tables @@ -61,7 +66,9 @@ be enabled without adding `zz` conditionals or tables. | Filter-pattern decomposition, region/sex/age value maps, and legacy US target-name parsing | A `dimensions` dictionary in `calibration_diagnostics` (label, semantic role, value labels, ordering) plus `targets[].dimensions` values. Geography dimensions also declare their level/id so no country geography fallback is needed. | | Source/variable guesses from flat target names | Structured `targets[].source` and `targets[].variable` identifiers, with the publisher still traceable to `metadata.chronicle_record_ids`. | -### Implemented: the `country` block +## Contract as implemented + +### Country `release_manifest.country` is read by `releaseCountry` in `frontend/lib/microcosm/latest-artifact.ts` and served as `country` on the @@ -101,5 +108,184 @@ country for parsers and builders but is never listed in selectors or the release-alert allowlist. Schema readers must remain backward-compatible while published releases migrate. -After migration, name/filter parsing is a legacy adapter selected by artifact -schema version, never by country. +Name/filter parsing is a legacy adapter selected from each row's artifact shape, +never by country. + +### Presentation + +`release_manifest.presentation` fills only the two existing overview and +target-browser introduction slots: + +```json +{ + "presentation": { + "overview_intro": "Release-owned overview introduction.", + "targets_intro": "Release-owned target-browser prompt." + } +} +``` + +`releasePresentation` accepts only these keys when their values are non-empty +strings. Values are trimmed and capped at 600 characters; unknown keys and +malformed values are dropped. The reader returns `null` when neither slot is +valid. The typed block is returned as `presentation` on the calibration summary +and target-diagnostics page. + +Each view resolves copy in this order: the artifact slot, the marked legacy +US/UK/BE copy, then generic copy. The overview's live-source attribution +sentence remains code-owned. The legacy tables can be deleted after all three +existing producers publish `presentation`. + +### Publisher labels + +`release_manifest.publisher_labels` maps the first Chronicle record-ID segment +to its display name: + +```json +{ + "publisher_labels": { + "novastat_agency": "Nova Statistics Agency" + } +} +``` + +The block must be a plain object. Keys must match +`^[a-z][a-z0-9_]*$` case-insensitively, and values must be non-empty strings; +values are trimmed and invalid entries are dropped. An absent or malformed +block becomes `{}`. Every enriched target row carries `source_label`, and the +field also appears on target responses and variable summaries. Treemap, +calibration-tree, target-browser, and target-detail presentations use the row +label when available. Label precedence is the manifest map, then a structured +row's `source.label`, then the shared authority humanizer. + +### Structured dimensions + +`calibration_diagnostics.json` may publish a dimension dictionary and a +dimension-value object on each target: + +```json +{ + "schema_version": 7, + "dimensions": { + "region": { + "label": "Region", + "role": "geography", + "level": "region", + "values": { + "north": "North", + "south": "South" + }, + "order": ["north", "south"] + }, + "sex": { + "label": "Sex", + "values": { + "female": "Female", + "male": "Male" + } + }, + "age_band": { + "label": "Age band" + } + }, + "targets": [ + { + "dimensions": { + "region": "north", + "sex": "female", + "age_band": "0_17" + } + } + ] +} +``` + +`diagnosticsDimensions` requires a plain-object dictionary and a non-empty +string `label` on each retained entry. It recognizes only `"geography"` as a +semantic `role` today. Optional `level` and string value labels are trimmed; +malformed optional entries are dropped without throwing. An explicit `order` +array overrides `values` key order. Unknown target dimension IDs remain valid: +their IDs and raw values are humanized. When an age-band value lacks an +artifact label, range values such as `0_17` and `65_plus` become `0–17` and +`65+`. + +A geography-role dimension sets `row.geography` and uses its declared `level` +or `"region"`. Other dimensions become `target_dimensions` with `key`, +`label`, `value`, `source_key`, `raw_value`, and an optional zero-based `rank`. +Facet values use rank order only when every displayed value has a rank; +otherwise the legacy facet sorter remains in force. Structured rows are also +excluded from whole-population estimate-scope inference. + +Adapter selection is per row and never by country: + +1. A plain-object `targets[].dimensions` selects `structured`, even when the + dictionary is absent or the row object is empty. +2. Otherwise, a filter matched by the shared filter-decomposition specs selects + `legacy_filter`. +3. Every other row selects `legacy_name`, preserving dotted/slash name parsing + and Chronicle metadata dimensions. + +Each target response records this choice as `dimension_adapter`. Calibration +summary and target-diagnostics responses include: + +```json +{ + "target_schema": { + "diagnostics_schema_version": 7, + "structured_dimensions": true + } +} +``` + +`structured_dimensions` reports whether the diagnostics published a plain +dimension dictionary; it does not choose every row's adapter. + +### Structured source and variable identifiers + +Targets accept their legacy strings or the following objects: + +```json +{ + "targets": [ + { + "source": { + "id": "novastat_agency", + "citation": "ZZ official population table", + "label": "Nova Statistics Agency", + "url": "https://stats.example/zz/pop" + }, + "variable": { + "id": "population", + "label": "Resident population", + "measure": "count" + } + } + ] +} +``` + +Publisher-key precedence is Chronicle metadata, then `source.id`, then legacy +name grammar. `source_citation` is the legacy source string or +`source.citation`; `source_url` is the structured URL or `null`. The publisher +label precedence is the manifest map, `source.label`, then the shared +humanizer. + +Variable-ID precedence is `variable.id`, `metadata.variable`, then the existing +artifact/name fallbacks. A structured ID remains an identifier; its display +name is separately returned as `variable_label`. Measure precedence is +`variable.measure`, then the existing first-dimension and Chronicle metadata +logic. Legacy string sources and variables retain their existing values, +families, citations, facets, and grouping. The new response fields are +additive: `source_label`, `source_url`, `variable_label`, and +`dimension_adapter`. + +### Producer follow-up + +Microcosm release producers must publish all of the following before the legacy +presentation and parsing adapters can be retired: + +- `release_manifest.country`; +- `release_manifest.presentation`; +- `release_manifest.publisher_labels`; +- `calibration_diagnostics.dimensions` and `targets[].dimensions`; and +- structured `targets[].source` and `targets[].variable` objects. diff --git a/frontend/components/microcosm/microcosm-overview-view.tsx b/frontend/components/microcosm/microcosm-overview-view.tsx index 3c11a14..87b3f6d 100644 --- a/frontend/components/microcosm/microcosm-overview-view.tsx +++ b/frontend/components/microcosm/microcosm-overview-view.tsx @@ -23,7 +23,7 @@ import { useMicrocosm, useMicrocosmReleases, } from "@/lib/api/hooks/use-microcosm"; -import type { MicrocosmCountry } from "@/lib/microcosm/countries"; +import { microcosmOverviewIntro } from "@/lib/microcosm/presentation"; import { microcosmPublicationUrl, microcosmSourceAttribution, @@ -41,26 +41,6 @@ function formatPublishedAt(value: string | null | undefined): string { }); } -// Legacy per-country intro copy for releases published before a typed -// `release_manifest.presentation` block; countries without an entry get the -// generic sentences below. -const COUNTRY_OVERVIEW_COPY: Partial< - Record -> = { - us: { - authorities: "the IRS, the Census Bureau, and CMS", - examples: "EITC statistics, population, and Medicaid enrollment", - }, - uk: { - authorities: "the ONS, OBR, and HMRC", - examples: "population by region and age, household types, and tax receipts", - }, - be: { - authorities: "Statbel, ONSS, JRC, and SFPD", - examples: "population by region, sex, and age band, tax receipts, and benefit totals", - }, -}; - type LossKind = "normalized_target_loss" | "raw_optimizer_objective" | undefined; function isNormalizedLoss(kind: LossKind): boolean { @@ -129,7 +109,7 @@ export function MicrocosmOverviewView() { cal.country?.repository_visibility, ); const publicationUrl = microcosmPublicationUrl(data.source_repo, data.release_id); - const overviewCopy = COUNTRY_OVERVIEW_COPY[country]; + const overviewIntro = microcosmOverviewIntro(country, cal.presentation); return (
@@ -139,14 +119,7 @@ export function MicrocosmOverviewView() { title="What the data is anchored to" description={ <> - Microcosm reweights survey microdata so it matches official statistics - from{" "} - {overviewCopy - ? `agencies like ${overviewCopy.authorities}` - : "national statistical agencies and administrative sources"} - . Each tile in the Calibration fit explorer below is a category we - calibrate to{overviewCopy ? `, including ${overviewCopy.examples}` : ""}. - Data is built live from{" "} + {overviewIntro} Data is built live from{" "} {sourceAttribution.href ? ( { expect(markup).toContain("Not available"); }); + test("prefers artifact labels and direct source URLs", () => { + const markup = render({ + ...TARGET, + source: "novastat_agency", + source_label: "Nova Statistics Agency", + source_url: "https://stats.example/zz/pop", + variable: "population", + variable_label: "Resident population", + }); + + expect(markup).toContain(">Resident population"); + expect(markup).toContain("Nova Statistics Agency"); + expect(markup).toContain("Source link { const markup = render({ ...TARGET, diff --git a/frontend/components/microcosm/microcosm-target-detail.tsx b/frontend/components/microcosm/microcosm-target-detail.tsx index 7863f03..9f0166b 100644 --- a/frontend/components/microcosm/microcosm-target-detail.tsx +++ b/frontend/components/microcosm/microcosm-target-detail.tsx @@ -47,6 +47,7 @@ function periodText(row: MicrocosmTargetRow): string { function measureText(row: MicrocosmTargetRow): string { return ( titleFromIdentifier(row.chronicle?.measure_concept) || + row.variable_label || canonicalLabel(row.variable as string) || titleFromIdentifier(row.chronicle?.layout_measure_id) || canonicalLabel(row.measure_name) @@ -427,11 +428,15 @@ export function MicrocosmTargetDetail({ ? row.abs_relative_error <= 0.1 : null; const chronicle = row.chronicle; - const chronicleEntryUrl = chronicleSourceEntryUrl( - row.source_citation, - chronicle?.layout_record_set_id, - ); - const sourceName = row.source ? sourceLabel(row.source) : "Source not specified"; + const officialSourceUrl = + row.source_url ?? + chronicleSourceEntryUrl( + row.source_citation, + chronicle?.layout_record_set_id, + ); + const sourceName = + row.source_label ?? + (row.source ? sourceLabel(row.source) : "Source not specified"); const measure = measureText(row) || titleFromIdentifier(row.name) || "Calibration target"; const rootRef = useRef(null); @@ -595,16 +600,18 @@ export function MicrocosmTargetDetail({ - View Chronicle entry ↗ + {row.source_url + ? "View official source ↗" + : "View Chronicle entry ↗"} ) : ( "Not available" diff --git a/frontend/components/microcosm/microcosm-targets-view.tsx b/frontend/components/microcosm/microcosm-targets-view.tsx index e625b2f..99edfec 100644 --- a/frontend/components/microcosm/microcosm-targets-view.tsx +++ b/frontend/components/microcosm/microcosm-targets-view.tsx @@ -13,7 +13,8 @@ import { SectionCard } from "@/components/shared/section-card"; import { ToolbarSelect } from "@/components/shared/toolbar-select"; import { MicrocosmTargetDetail } from "@/components/microcosm/microcosm-target-detail"; import { withBasePath } from "@/lib/base-path"; -import { hasCapability, type MicrocosmCountry } from "@/lib/microcosm/countries"; +import { hasCapability } from "@/lib/microcosm/countries"; +import { microcosmTargetsIntro } from "@/lib/microcosm/presentation"; import { sourceLabel } from "@/lib/microcosm/source-label"; import { releaseSelectOptions, @@ -27,19 +28,6 @@ import { const PAGE_SIZE = 50; -// Legacy per-country browse prompt for releases published before a typed -// `release_manifest.presentation` block; countries without an entry get the -// generic prompt. -const COUNTRY_BROWSE_COPY: Partial> = { - us: - "Pick a measure like EITC, population, or AGI and see how each breakdown is calibrated.", - uk: - "Pick a measure like population, household type, or tax receipts and see how each breakdown is calibrated.", - be: - "Pick a measure like population, income tax, or pension recipients and see how each breakdown is calibrated.", -}; -const GENERIC_BROWSE_COPY = "Pick a measure and see how each breakdown is calibrated."; - interface SortState { by: string; dir: "asc" | "desc"; @@ -62,7 +50,9 @@ interface VariableMeasureOption { interface VariableGroup { groupKey: string; source: string; + sourceLabel: string; variable: string; + variableLabel: string | null; level: string; options: VariableMeasureOption[]; defaultKey: string; @@ -103,6 +93,7 @@ function titleFromIdentifier(value: string | null | undefined): string { function measureTitle(row: MicrocosmTargetRow): string { return ( titleFromIdentifier(row.chronicle?.measure_concept) || + row.variable_label || humanizeName(row.variable as string) || titleFromIdentifier(row.chronicle?.layout_measure_id) || "—" @@ -190,7 +181,9 @@ const OVERVIEW_COLUMNS: Column[] = [ sortable: true, render: (row) => (
-
{row.source || "—"}
+
+ {row.source_label ?? row.source ?? "—"} +
{titleFromIdentifier(row.chronicle?.domain)}
@@ -293,7 +286,9 @@ function groupVariables(variables: MicrocosmVariableRow[]): VariableGroup[] { return { groupKey, source: first.source, + sourceLabel: first.source_label || sourceLabel(first.source), variable: first.variable, + variableLabel: first.variable_label ?? null, level: first.level, options, defaultKey: defaultOption.key, @@ -345,7 +340,12 @@ function VariableBrowser({ const q = query.trim().toLowerCase(); const filtered = q ? groups.filter((group) => - [group.variable, sourceLabel(group.source), group.source] + [ + group.variableLabel, + group.variable, + group.sourceLabel, + group.source, + ] .join(" ") .toLowerCase() .includes(q), @@ -360,6 +360,7 @@ function VariableBrowser({ return [...map.entries()] .map(([source, items]) => ({ source, + sourceLabel: items[0]?.sourceLabel ?? sourceLabel(source), items: [...items].sort((a, b) => b.nTargets - a.nTargets), total: items.reduce((sum, item) => sum + item.nTargets, 0), })) @@ -389,7 +390,9 @@ function VariableBrowser({ sections.map((section) => (
-

{sourceLabel(section.source)}

+

+ {section.sourceLabel} +

{fmt(section.items.length, { digits: 0 })} statistics · {fmt(section.total, { digits: 0 })} targets @@ -410,7 +413,7 @@ function VariableBrowser({ >
- {humanizeName(group.variable)} + {group.variableLabel ?? humanizeName(group.variable)} {fmt(group.nTargets, { digits: 0 })} @@ -1084,7 +1087,7 @@ export function MicrocosmTargetsView({ { ); }); + test("prefers artifact publisher labels for source groups", () => { + const tree = buildCalibrationTree( + [ + target("artifact-labelled", { + source: "novastat_agency", + source_label: "Nova Statistics Agency", + variable: "population", + }), + ], + state({ dimensions: [] }), + ); + + expect(tree.groups[0].label).toBe("Nova Statistics Agency"); + }); + test("renders geography first, then programs grouped by their source", () => { const overview = buildCalibrationTree( rows, diff --git a/frontend/lib/microcosm/calibration-tree.ts b/frontend/lib/microcosm/calibration-tree.ts index 203533c..59663e4 100644 --- a/frontend/lib/microcosm/calibration-tree.ts +++ b/frontend/lib/microcosm/calibration-tree.ts @@ -22,6 +22,7 @@ export interface CalibrationTreeTarget { name?: string | null; base_name?: string | null; source?: string | null; + source_label?: string | null; variable?: string | null; variable_key?: string | null; measure?: string | null; @@ -409,6 +410,9 @@ function programGroups(rows: CalibrationTreeTarget[]): CalibrationTreeGroup[] { ); return [...bySource.entries()] .map(([source, sourceRows]) => { + const artifactSourceLabel = sourceRows + .map((row) => row.source_label?.trim()) + .find((label): label is string => Boolean(label)); const byProgram = groupRows(sourceRows, programId); const nodes = sortNodes( [...byProgram.entries()].map(([program, programRows]) => @@ -423,7 +427,7 @@ function programGroups(rows: CalibrationTreeTarget[]): CalibrationTreeGroup[] { ); return { id: source, - label: sourceLabel(source), + label: artifactSourceLabel ?? sourceLabel(source), nodes, metrics: calibrationTreeMetrics(sourceRows), }; diff --git a/frontend/lib/microcosm/latest-artifact.test.ts b/frontend/lib/microcosm/latest-artifact.test.ts index f184a91..886e402 100644 --- a/frontend/lib/microcosm/latest-artifact.test.ts +++ b/frontend/lib/microcosm/latest-artifact.test.ts @@ -9,6 +9,7 @@ import beReleaseManifestFixture from "./fixtures/be-release/release_manifest.jso import { buildCalibration, buildComparison, + diagnosticsDimensions, MICROCOSM_HF_REPO_ENV, MICROCOSM_HF_REVISION_ENV, MICROCOSM_BE_HF_REPO_ENV, @@ -21,8 +22,11 @@ import { latestMicrocosmTargetDiagnosticsPage, microcosmRepo, microcosmRevision, + microcosmTargetTreemap, parseCountry, releaseCountry, + releasePresentation, + releasePublisherLabels, releasePublishedAtFromTree, releaseRole, type ArtifactCountry, @@ -99,6 +103,7 @@ test("loads trimmed Belgium diagnostics without optional US artifact fields", () family: "statbel/population", geography: "Brussels", level: "region", + dimension_adapter: "legacy_filter", breakdown: "Male · 0–17", target_dimensions: [ expect.objectContaining({ key: "bd_sex", label: "Sex", value: "Male" }), @@ -110,6 +115,7 @@ test("loads trimmed Belgium diagnostics without optional US artifact fields", () variable: "national income tax amount", geography: "Belgium", level: "national", + dimension_adapter: "legacy_name", }); expect(cal.rows[2]).toMatchObject({ source: "eurostat", @@ -156,6 +162,319 @@ test("derives Belgium population region, sex, and age-band browser facets", () = ]); }); +test("diagnosticsDimensions drops malformed entries and normalizes optional metadata", () => { + expect(diagnosticsDimensions({ dimensions: [] })).toEqual({}); + expect( + diagnosticsDimensions({ + dimensions: { + region: { + label: " Region ", + role: "geography", + level: " region ", + values: { north: " North ", south: " ", broken: 3 }, + order: [" south ", 3, "north", " "], + }, + sex: { label: "Sex", role: "category", values: { female: "Female" } }, + missing_label: { values: { value: "Value" } }, + scalar: "Region", + }, + }), + ).toEqual({ + region: { + label: "Region", + role: "geography", + level: "region", + values: { north: "North" }, + order: ["south", "north"], + }, + sex: { label: "Sex", values: { female: "Female" } }, + }); +}); + +test("structured dimensions shape rows and honor artifact value order", () => { + const target = ( + suffix: string, + dimensions: Record, + ) => ({ + name: `fixture.population.${suffix}@2026`, + target_name: `fixture.population.${suffix}`, + source: "ZZ official population table", + metadata: { + chronicle_record_ids: [`novastat_agency.population.${suffix}`], + variable: "population", + source_measure_id: "population_count", + }, + dimensions, + target: 100, + initial_estimate: 90, + final_estimate: 100, + }); + const cal = buildCalibration( + { + schema_version: 7, + dimensions: { + region: { + label: "Region", + role: "geography", + level: "region", + values: { north: "North", south: "South" }, + }, + sex: { + label: "Sex", + values: { female: "Female", male: "Male" }, + order: ["male", "female"], + }, + age_band: { + label: "Age band", + values: { "65_plus": "65+", "0_17": "0–17", "18_64": "18–64" }, + }, + }, + targets: [ + target("north_female_0_17", { + region: "north", + sex: "female", + age_band: "0_17", + settlement_type: "urban_core", + }), + target("south_male_65_plus", { + region: "south", + sex: "male", + age_band: "65_plus", + settlement_type: "urban_core", + }), + target("north_male_18_64", { + region: "north", + sex: "male", + age_band: "18_64", + settlement_type: "urban_core", + }), + ], + }, + "structured-dimensions", + ); + + expect(cal.target_schema).toEqual({ + diagnostics_schema_version: 7, + structured_dimensions: true, + }); + expect(cal.rows.every((row) => row.dimension_adapter === "structured")).toBe(true); + expect(cal.rows[0]).toMatchObject({ + family: "novastat_agency/population", + geography: "North", + level: "region", + breakdown: "Female · 0–17 · Urban Core", + target_dimensions: [ + { + key: "bd_sex", + label: "Sex", + value: "Female", + source_key: "sex", + raw_value: "female", + rank: 1, + }, + { + key: "bd_age_band", + label: "Age band", + value: "0–17", + source_key: "age_band", + raw_value: "0_17", + rank: 1, + }, + { + key: "bd_settlement_type", + label: "Settlement Type", + value: "Urban Core", + source_key: "settlement_type", + raw_value: "urban_core", + }, + ], + }); + const page = latestMicrocosmTargetDiagnosticsPage( + "http://x/api/microcosm/target-diagnostics?variable=novastat_agency%20%2F%20population%20%C2%B7%20count", + cal, + ); + expect(page.target_schema).toEqual(cal.target_schema); + expect(page.dimensions).toEqual([ + { key: "geography", label: "Region", values: ["North", "South"] }, + { key: "bd_sex", label: "Sex", values: ["Male", "Female"] }, + { key: "bd_age_band", label: "Age band", values: ["65+", "0–17", "18–64"] }, + ]); +}); + +test("structured facet ordering falls back when any displayed value lacks a rank", () => { + const target = (suffix: string, category: string) => ({ + name: `fixture.population.${suffix}@2026`, + source: "Citation", + metadata: { + chronicle_record_ids: ["novastat_agency.population.total"], + variable: "population", + source_measure_id: "population_count", + }, + dimensions: { category }, + target: 1, + initial_estimate: 1, + final_estimate: 1, + }); + const cal = buildCalibration( + { + dimensions: { + category: { + label: "Category", + values: { zeta: "Zeta" }, + order: ["zeta"], + }, + }, + targets: [target("zeta", "zeta"), target("beta", "beta")], + }, + "partial-dimension-order", + ); + const page = latestMicrocosmTargetDiagnosticsPage( + "http://x/api/microcosm/target-diagnostics?variable=novastat_agency%20%2F%20population%20%C2%B7%20count", + cal, + ); + + expect(page.dimensions).toEqual([ + { key: "bd_category", label: "Category", values: ["Beta", "Zeta"] }, + ]); +}); + +test("dimension adapters are selected per row and structured values beat legacy filters", () => { + const base = { + source: "ZZ official population table", + metadata: { + chronicle_record_ids: ["novastat_agency.population.total"], + variable: "population", + source_measure_id: "population_count", + }, + target: 100, + initial_estimate: 90, + final_estimate: 100, + }; + const cal = buildCalibration( + { + schema_version: 7, + dimensions: { + region: { + label: "Region", + role: "geography", + values: { north: "North", south: "South" }, + }, + sex: { label: "Sex" }, + age_band: { label: "Age band" }, + }, + targets: [ + { + ...base, + name: "fixture_population_north_female_0_17@2026", + filter: "cell_south_male_65_plus", + dimensions: { region: "north", sex: "female", age_band: "0_17" }, + }, + { + ...base, + name: "fixture_population_south_male_18_64@2026", + filter: "cell_south_male_18_64", + }, + { + ...base, + name: "novastat_agency.population.total@2026", + filter: null, + }, + ], + }, + "mixed-dimension-adapters", + ); + + expect(cal.rows.map((row) => row.dimension_adapter)).toEqual([ + "structured", + "legacy_filter", + "legacy_name", + ]); + expect(cal.rows[0]).toMatchObject({ + geography: "North", + level: "region", + breakdown: "Female · 0–17", + }); + expect(latestMicrocosmCalibrationSummary(cal).target_schema).toEqual( + cal.target_schema, + ); +}); + +test("structured rows do not require a dimensions dictionary, including empty objects", () => { + const base = { + source: "Citation", + metadata: { variable: "population", source_measure_id: "population_count" }, + filter: "cell_south_male_18_64", + target: 1, + initial_estimate: 1, + final_estimate: 1, + }; + const cal = buildCalibration( + { + targets: [ + { + ...base, + name: "fixture_population_age@2026", + dimensions: { age_band: "65_plus" }, + }, + { + ...base, + name: "fixture_population_total@2026", + dimensions: {}, + }, + ], + }, + "dictionary-free-dimensions", + ); + + expect(cal.target_schema.structured_dimensions).toBe(false); + expect(cal.rows[0]).toMatchObject({ + dimension_adapter: "structured", + geography: "United States", + breakdown: "65+", + target_dimensions: [ + expect.objectContaining({ + key: "bd_age_band", + label: "Age Band", + value: "65+", + }), + ], + }); + expect(cal.rows[1]).toMatchObject({ + dimension_adapter: "structured", + geography: "United States", + breakdown: "", + target_dimensions: [], + }); +}); + +test("structured dimensions prevent whole-population estimate-scope warnings", () => { + const target = (recordSet: string, category: string, targetValue: number) => ({ + name: `source.example.${category}.amount@2026`, + source: "Citation", + metadata: { + variable: "example", + source_measure_id: "example_amount", + ledger_layout_record_set_id: recordSet, + }, + dimensions: { category }, + target: targetValue, + initial_estimate: 100, + final_estimate: 80, + }); + const cal = buildCalibration( + { + dimensions: { category: { label: "Category" } }, + targets: [ + target("source.example.slice_a", "a", 10), + target("source.example.slice_b", "b", 20), + ], + }, + "structured-scope", + ); + + expect(cal.rows.every((row) => row.estimate_warning == null)).toBe(true); +}); + test("keeps legacy US dotted target families when Chronicle publisher metadata is present", () => { const cal = buildCalibration( { @@ -189,6 +508,137 @@ test("keeps legacy US dotted target families when Chronicle publisher metadata i }); }); +test("live-US-shaped schema 5 rows preserve the legacy dotted contract", () => { + const sourceCitation = + "Bureau of Economic Analysis, National Income and Product Accounts, Table 1.12"; + const cal = buildCalibration( + { + schema_version: 5, + targets: [ + { + name: "bea_nipa.cy2023.proprietors_income.a041rc.amount@2024", + filter: null, + source: sourceCitation, + metadata: { + chronicle_record_ids: [ + "bea_nipa.cy2023.proprietors_income.a041rc.amount", + ], + variable: "proprietors_income", + source_measure_id: "proprietors_income_amount", + ledger_geography_level: "country", + ledger_geography_id: "0100000US", + ledger_layout_groupby_dimension: "bea_nipa.line_code", + ledger_layout_groupby_value_id: "a041rc", + ledger_measure_unit: "usd", + }, + target: 100, + initial_estimate: 90, + final_estimate: 99, + relative_error: -0.01, + within_tolerance: true, + }, + ], + }, + "live-us-shaped", + ); + const responseRow = latestMicrocosmTargetDiagnosticsPage( + "http://x/api/microcosm/target-diagnostics", + cal, + ).targets[0]; + + // JSON round-tripping matches the API boundary and locks every legacy field; + // the four new contract fields are strictly additive. + expect(JSON.parse(JSON.stringify(responseRow))).toEqual({ + name: "bea_nipa.cy2023.proprietors_income.a041rc.amount@2024", + target: 100, + initial_estimate: 90, + final_estimate: 99, + relative_error: -0.01, + within_tolerance: true, + base_name: "bea_nipa.cy2023.proprietors_income.a041rc.amount", + family: "bea_nipa.cy2023.proprietors_income.a041rc.amount", + state: null, + geography: "United States", + level: "national", + source: "bea_nipa", + source_label: "BEA Nipa", + variable: "proprietors income", + variable_label: null, + measure: "total", + target_role: null, + source_measure_id: "proprietors_income_amount", + policyengine_variables: [], + policyengine_map_to: null, + policyengine_filter_variable: null, + materializer: null, + measure_mode: null, + error_kind: "relative", + initial_error: -0.1, + final_error: -0.01, + initial_miss: -10, + final_miss: -1, + abs_final_miss: 1, + absolute_improvement: 9, + abs_error: 0.01, + breakdown: "a041rc", + dims: ["a041rc"], + target_dimensions: [ + { + key: "bd_line_code", + label: "Line Code", + value: "a041rc", + source_key: "ledger_layout_groupby_value_id", + raw_value: "a041rc", + }, + ], + dimension_adapter: "legacy_name", + variable_key: "bea_nipa / proprietors income · total", + source_citation: sourceCitation, + source_url: null, + entity: null, + aggregation: null, + measure_name: null, + period: null, + chronicle: { + fact_key: null, + source_record_id: null, + semantic_fact_key: null, + aggregate_fact_key: null, + legacy_fact_key: null, + period_type: null, + source_period: null, + target_period: null, + geography_level: "country", + geography_id: "0100000US", + geography_vintage: null, + domain: null, + entity_name: null, + entity_role: null, + measure_concept: null, + source_concept: null, + concept_relation: null, + concept_authority: null, + measure_unit: "usd", + value_operation: null, + layout_record_set_id: null, + layout_groupby_dimension: "bea_nipa.line_code", + layout_groupby_value_id: "a041rc", + layout_measure_id: null, + dimension_set_key: null, + universe_constraint_set_key: null, + universe_constraint_count: null, + filters: [], + }, + calibration_status: "included", + calibration_status_label: "Included", + calibration_status_reason: null, + initial_relative_error: -0.1, + abs_relative_error: 0.01, + improvement: 0.09000000000000001, + direction: "under", + }); +}); + // A v2-shaped target: AGI bracket × return type × filing status, with @period. function agiTarget(band: string, ret: string, filing: string, rel: number) { return { @@ -1261,6 +1711,212 @@ test("releaseRole classifies national default vs non-default local-area", () => }); }); +test("releasePresentation keeps only trimmed, capped intro slots", () => { + const longIntro = ` ${"x".repeat(610)} `; + expect( + releasePresentation({ + presentation: { + overview_intro: " Artifact overview. ", + targets_intro: longIntro, + arbitrary_section: "Ignored", + }, + }), + ).toEqual({ + overview_intro: "Artifact overview.", + targets_intro: "x".repeat(600), + }); +}); + +test("releasePresentation returns null when no valid intro slot exists", () => { + expect(releasePresentation({})).toBeNull(); + expect(releasePresentation({ presentation: [] })).toBeNull(); + expect(releasePresentation({ presentation: "copy" })).toBeNull(); + expect( + releasePresentation({ + presentation: { + overview_intro: " ", + targets_intro: 12, + unknown: "Ignored", + }, + }), + ).toBeNull(); +}); + +test("presentation flows through calibration, summary, and target responses", () => { + const presentation = { + overview_intro: "Artifact overview.", + targets_intro: "Artifact target prompt.", + }; + const cal = buildCalibration( + { targets: [] }, + "presentation-release", + null, + {}, + { presentation }, + ); + + expect(cal.presentation).toEqual(presentation); + expect(latestMicrocosmCalibrationSummary(cal).presentation).toEqual(presentation); + expect( + latestMicrocosmTargetDiagnosticsPage( + "http://x/api/microcosm/target-diagnostics", + cal, + ).presentation, + ).toEqual(presentation); +}); + +test("releasePublisherLabels keeps valid keys and trimmed non-empty labels", () => { + expect(releasePublisherLabels({})).toEqual({}); + expect(releasePublisherLabels({ publisher_labels: [] })).toEqual({}); + expect(releasePublisherLabels({ publisher_labels: "labels" })).toEqual({}); + expect( + releasePublisherLabels({ + publisher_labels: { + novastat_agency: " Nova Statistics Agency ", + IRS2: "IRS second series", + "bad-key": "Dropped", + _bad: "Dropped", + blank: " ", + numeric: 12, + }, + }), + ).toEqual({ + novastat_agency: "Nova Statistics Agency", + IRS2: "IRS second series", + }); +}); + +test("publisher labels flow through rows, variables, target responses, and treemaps", () => { + const cal = buildCalibration( + { + targets: [ + { + name: "fixture_population@2026", + target_name: "fixture_population", + source: "ZZ official population table", + metadata: { + chronicle_record_ids: ["novastat_agency.population.cy2026.total"], + variable: "population", + source_measure_id: "population_count", + }, + target: 100, + initial_estimate: 90, + final_estimate: 100, + }, + ], + }, + "publisher-labels", + null, + {}, + { publisher_labels: { novastat_agency: "Nova Statistics Agency" } }, + ); + + expect(cal.publisher_labels).toEqual({ + novastat_agency: "Nova Statistics Agency", + }); + expect(cal.rows[0].source_label).toBe("Nova Statistics Agency"); + const page = latestMicrocosmTargetDiagnosticsPage( + "http://x/api/microcosm/target-diagnostics", + cal, + ); + expect(page.targets[0].source_label).toBe("Nova Statistics Agency"); + expect(page.variables[0].source_label).toBe("Nova Statistics Agency"); + expect(microcosmTargetTreemap(cal.rows, cal.release_id).groups[0].label).toBe( + "Nova Statistics Agency", + ); +}); + +test("publisher label lookup does not read inherited object properties", () => { + const cal = buildCalibration( + { + targets: [ + { + name: "constructor.population.total@2026", + source: "Citation", + metadata: { chronicle_record_ids: ["constructor.population.total"] }, + target: 1, + initial_estimate: 1, + final_estimate: 1, + }, + ], + }, + "publisher-prototype", + ); + + expect(cal.rows[0].source_label).toBe("Constructor"); +}); + +test("structured source and variable fields follow artifact precedence", () => { + const target = { + name: "legacy.publisher.population.total@2026", + source: { + id: "artifact_agency", + citation: "Official population table", + label: "Row agency label", + url: "https://stats.example/population", + }, + variable: { + id: "resident_population", + label: "Resident population", + measure: "mean", + }, + metadata: { + chronicle_record_ids: ["chronicle_agency.population.total"], + variable: "legacy_population", + source_measure_id: "legacy_population_count", + }, + target: 100, + initial_estimate: 90, + final_estimate: 100, + }; + const cal = buildCalibration( + { targets: [target] }, + "structured-identifiers", + null, + {}, + { publisher_labels: { chronicle_agency: "Manifest agency label" } }, + ); + + expect(cal.rows[0]).toMatchObject({ + source: "chronicle_agency", + source_label: "Manifest agency label", + source_citation: "Official population table", + source_url: "https://stats.example/population", + variable: "resident_population", + variable_label: "Resident population", + measure: "mean", + variable_key: "chronicle_agency / resident_population · mean", + }); + const response = latestMicrocosmTargetDiagnosticsPage( + "http://x/api/microcosm/target-diagnostics", + cal, + ); + expect(response.targets[0]).toMatchObject({ + source_url: "https://stats.example/population", + variable_label: "Resident population", + }); + expect(response.variables[0].variable_label).toBe("Resident population"); + + const withoutChronicle = buildCalibration( + { + targets: [ + { + ...target, + metadata: { + variable: "legacy_population", + source_measure_id: "legacy_population_count", + }, + }, + ], + }, + "structured-source-fallback", + ); + expect(withoutChronicle.rows[0]).toMatchObject({ + source: "artifact_agency", + source_label: "Row agency label", + }); +}); + const BE_COUNTRY_DEFAULTS: ArtifactCountry = { code: "be", label: "Belgium", diff --git a/frontend/lib/microcosm/latest-artifact.ts b/frontend/lib/microcosm/latest-artifact.ts index 4ac4cca..f211060 100644 --- a/frontend/lib/microcosm/latest-artifact.ts +++ b/frontend/lib/microcosm/latest-artifact.ts @@ -168,6 +168,12 @@ export function asObject(value: unknown): JsonObject { : {}; } +function isPlainObject(value: unknown): value is JsonObject { + if (!value || typeof value !== "object" || Array.isArray(value)) return false; + const prototype = Object.getPrototypeOf(value); + return prototype === Object.prototype || prototype === null; +} + export function scrub(value: unknown): unknown { if (Array.isArray(value)) return value.map(scrub); if (value && typeof value === "object") { @@ -233,6 +239,7 @@ interface TargetBreakdownDimension { value: string; source_key?: string; raw_value?: string; + rank?: number; } interface ChronicleFilter { @@ -614,6 +621,55 @@ interface FilterDecompositionSpec { dimensions: readonly FilterDimensionSpec[]; } +export interface DiagnosticsDimension { + label: string; + role?: "geography"; + level?: string; + values?: Record; + order?: string[]; +} + +export function diagnosticsDimensions( + diag: JsonObject, +): Record { + if (!isPlainObject(diag.dimensions)) return {}; + return Object.fromEntries( + Object.entries(diag.dimensions).flatMap(([id, rawDefinition]) => { + if (!id || !isPlainObject(rawDefinition)) return []; + const label = stringValue(rawDefinition.label)?.trim(); + if (!label) return []; + const role = rawDefinition.role === "geography" ? "geography" : undefined; + const level = stringValue(rawDefinition.level)?.trim(); + const values = isPlainObject(rawDefinition.values) + ? Object.fromEntries( + Object.entries(rawDefinition.values).flatMap(([rawValue, rawLabel]) => { + if (!rawValue || typeof rawLabel !== "string") return []; + const valueLabel = rawLabel.trim(); + return valueLabel ? [[rawValue, valueLabel]] : []; + }), + ) + : undefined; + const order = Array.isArray(rawDefinition.order) + ? rawDefinition.order.flatMap((rawValue) => { + if (typeof rawValue !== "string") return []; + const value = rawValue.trim(); + return value ? [value] : []; + }) + : undefined; + return [[ + id, + { + label, + ...(role ? { role } : {}), + ...(level ? { level } : {}), + ...(values ? { values } : {}), + ...(order ? { order } : {}), + }, + ]]; + }), + ); +} + // Legacy artifacts encode some dimensions in a compiled filter name. The // adapter is selected by the filter pattern, never by country; future artifacts // should publish structured target dimensions instead. @@ -644,10 +700,23 @@ interface DecomposedTargetFilter { dimensions: TargetBreakdownDimension[]; } -function filterDimensionValue(spec: FilterDimensionSpec, rawValue: string): string { - const explicit = spec.valueLabels?.[rawValue]; +interface DecomposedStructuredDimensions { + geography: string | null; + level: string | null; + dimensions: TargetBreakdownDimension[]; +} + +function dimensionValue( + label: string, + rawValue: string, + valueLabels?: Readonly>, +): string { + const explicit = + valueLabels && Object.hasOwn(valueLabels, rawValue) + ? valueLabels[rawValue] + : undefined; if (explicit) return explicit; - if (spec.label === "Age band") { + if (label.toLowerCase() === "age band") { const range = /^(\d+)_(\d+)$/.exec(rawValue); if (range) return `${range[1]}–${range[2]}`; const openEnded = /^(\d+)_plus$/.exec(rawValue); @@ -656,6 +725,43 @@ function filterDimensionValue(spec: FilterDimensionSpec, rawValue: string): stri return titleCase(rawValue); } +function filterDimensionValue(spec: FilterDimensionSpec, rawValue: string): string { + return dimensionValue(spec.label, rawValue, spec.valueLabels); +} + +function decomposeStructuredDimensions( + values: JsonObject, + definitions: Record, +): DecomposedStructuredDimensions { + let geography: string | null = null; + let level: string | null = null; + const dimensions: TargetBreakdownDimension[] = []; + for (const [id, raw] of Object.entries(values)) { + const rawValue = stringValue(raw)?.trim(); + if (!rawValue) continue; + const definition = definitions[id]; + const label = definition?.label ?? dimensionLabel(id); + const value = dimensionValue(label, rawValue, definition?.values); + const rankOrder = definition?.order ?? + (definition?.values ? Object.keys(definition.values) : undefined); + const rank = rankOrder?.indexOf(rawValue) ?? -1; + if (definition?.role === "geography") { + geography ??= value; + level ??= definition.level ?? "region"; + continue; + } + dimensions.push({ + key: dimensionKey(label), + label, + value, + source_key: id, + raw_value: rawValue, + ...(rank >= 0 ? { rank } : {}), + }); + } + return { geography, level, dimensions }; +} + function decomposeTargetFilter(value: unknown): DecomposedTargetFilter | null { const filter = stringValue(value); if (!filter) return null; @@ -695,13 +801,51 @@ function chroniclePublisherFromMetadata(metadata: JsonObject): string | null { return first?.trim().split(".", 1)[0] || null; } +interface StructuredTargetSource { + id: string | null; + citation: string | null; + label: string | null; + url: string | null; +} + +interface StructuredTargetVariable { + id: string; + label: string | null; + measure: string | null; +} + +function structuredTargetSource(value: unknown): StructuredTargetSource | null { + if (!isPlainObject(value)) return null; + return { + id: stringValue(value.id)?.trim() ?? null, + citation: stringValue(value.citation)?.trim() ?? null, + label: stringValue(value.label)?.trim() ?? null, + url: stringValue(value.url)?.trim() ?? null, + }; +} + +function structuredTargetVariable(value: unknown): StructuredTargetVariable | null { + if (!isPlainObject(value)) return null; + const id = stringValue(value.id)?.trim(); + if (!id) return null; + return { + id, + label: stringValue(value.label)?.trim() ?? null, + measure: stringValue(value.measure)?.trim() ?? null, + }; +} + function artifactVariable( name: string, row: TargetRow, decomposition: DecomposedTargetFilter | null, + structuredVariable: StructuredTargetVariable | null, ): string | null { const metadata = asObject(row.metadata); - const published = stringValue(metadata.variable) ?? stringValue(row.variable); + if (structuredVariable) return structuredVariable.id; + const published = + stringValue(metadata.variable) ?? + (typeof row.variable === "string" ? stringValue(row.variable) : null); if (published) return readableToken(published); if (!name.includes("_") || name.includes("/")) return null; @@ -972,7 +1116,34 @@ function computeDimensions(rows: TargetRow[]): TargetDimension[] { ]; if (values.length <= 1) continue; const label = candidate.label ?? classifyDimension(values); - facets.push({ key: candidate.key, label, values: sortFacetValues(label, values) }); + const ranks = new Map(); + let everyValueRanked = candidate.key !== "geography"; + for (const value of values) { + const matchingDimensions = rows.flatMap((row) => + ((row.target_dimensions as TargetBreakdownDimension[] | undefined) ?? []) + .filter((dimension) => + dimension.key === candidate.key && dimension.value === value, + ), + ); + if ( + !matchingDimensions.length || + matchingDimensions.some((dimension) => typeof dimension.rank !== "number") + ) { + everyValueRanked = false; + break; + } + ranks.set( + value, + Math.min(...matchingDimensions.map((dimension) => dimension.rank as number)), + ); + } + const sortedValues = everyValueRanked + ? [...values].sort( + (a, b) => + (ranks.get(a) ?? 0) - (ranks.get(b) ?? 0) || a.localeCompare(b), + ) + : sortFacetValues(label, values); + facets.push({ key: candidate.key, label, values: sortedValues }); } return facets; } @@ -1073,6 +1244,8 @@ function enrichTargetRow( skippedByName: Map, droppedTargetNames: Set, artifactCountry: ArtifactCountry, + publisherLabels: Record, + dimensionDefinitions: Record, ): TargetRow { const nationalGeography = artifactCountry.geography_label; const metadata = normalizeChronicleMetadata(rawRow.metadata); @@ -1112,29 +1285,61 @@ function enrichTargetRow( initialError == null || finalError == null ? null : Math.abs(initialError) - Math.abs(finalError); - const filterDecomposition = decomposeTargetFilter(row.filter); - const publisher = chroniclePublisherFromMetadata(metadata); + const publishedSource = structuredTargetSource(row.source); + const publishedVariable = structuredTargetVariable(row.variable); + const structuredDecomposition = isPlainObject(row.dimensions) + ? decomposeStructuredDimensions(row.dimensions, dimensionDefinitions) + : null; + const filterDecomposition = structuredDecomposition + ? null + : decomposeTargetFilter(row.filter); + const dimensionAdapter = structuredDecomposition + ? "structured" + : filterDecomposition + ? "legacy_filter" + : "legacy_name"; + const publisher = + chroniclePublisherFromMetadata(metadata) ?? publishedSource?.id ?? null; const parsedFromName = parseDottedTarget(baseName, row, nationalGeography) ?? parseTarget(baseName, nationalGeography); const parsed: ParsedTarget = { ...parsedFromName, - geography: filterDecomposition?.geography ?? parsedFromName.geography, - level: filterDecomposition?.level ?? parsedFromName.level, + geography: + structuredDecomposition?.geography ?? + filterDecomposition?.geography ?? + parsedFromName.geography, + level: + structuredDecomposition?.level ?? + filterDecomposition?.level ?? + parsedFromName.level, source: publisher ?? parsedFromName.source, variable: - artifactVariable(baseName, row, filterDecomposition) ?? + artifactVariable( + baseName, + row, + filterDecomposition, + publishedVariable, + ) ?? parsedFromName.variable, - breakdown: filterDecomposition - ? filterDecomposition.dimensions.map((dimension) => dimension.value).join(" · ") - : parsedFromName.breakdown, + breakdown: structuredDecomposition + ? structuredDecomposition.dimensions + .map((dimension) => dimension.value) + .join(" · ") + : filterDecomposition + ? filterDecomposition.dimensions + .map((dimension) => dimension.value) + .join(" · ") + : parsedFromName.breakdown, }; const hasGeography = Boolean(parsed.geography.trim()); const geography = hasGeography ? parsed.geography : nationalGeography; const level = hasGeography ? parsed.level : DEFAULT_GEOGRAPHY_LEVEL; const measureCol = asObject(row.measure); const metadataTargetDimensions = - filterDecomposition?.dimensions ?? metadataDimensions(row); + structuredDecomposition?.dimensions ?? + filterDecomposition?.dimensions ?? + metadataDimensions(row); const targetDimensions = metadataTargetDimensions ?? splitBreakdown(parsed.breakdown).map((value, index) => ({ @@ -1151,9 +1356,11 @@ function enrichTargetRow( // IRS variables publish both a total (dollar amount) and a count (number of // returns), so the measure is part of the variable's identity, not a // breakdown within it — fold it into variable_key so they're distinct things. - const measure = dims[0] && MEASURES.has(dims[0]) - ? dims[0] - : measureFromMetadata(metadata); + const measure = + publishedVariable?.measure ?? + (dims[0] && MEASURES.has(dims[0]) + ? dims[0] + : measureFromMetadata(metadata)); const variableKey = variableKeyOf(parsed) + (measure ? ` · ${measure}` : ""); // Underscore identifiers and filter-decomposed targets use the structured @@ -1161,6 +1368,9 @@ function enrichTargetRow( // without filter dimensions retain their legacy family so US/UK releases do // not regroup merely because they also carry Chronicle record IDs. const usesArtifactFamily = + publishedSource?.id != null || + publishedVariable != null || + structuredDecomposition != null || filterDecomposition != null || (publisher != null && !baseName.includes("/") && !baseName.includes(".")); return { @@ -1172,7 +1382,14 @@ function enrichTargetRow( geography, level, source: parsed.source, + source_label: + (Object.hasOwn(publisherLabels, parsed.source) + ? publisherLabels[parsed.source] + : undefined) ?? + publishedSource?.label ?? + sourceAuthorityLabel(parsed.source), variable: parsed.variable, + variable_label: publishedVariable?.label ?? null, measure, target_role: targetRole, source_measure_id: sourceMeasureId, @@ -1194,9 +1411,14 @@ function enrichTargetRow( breakdown, dims, target_dimensions: targetDimensions, + dimension_adapter: dimensionAdapter, variable_key: variableKey, // v2 published metadata (null on v1). - source_citation: typeof row.source === "string" ? (row.source as string) : null, + source_citation: + typeof row.source === "string" + ? (row.source as string) + : publishedSource?.citation ?? null, + source_url: publishedSource?.url ?? null, entity: typeof row.entity === "string" ? (row.entity as string) : null, aggregation: typeof row.aggregation === "string" ? (row.aggregation as string) : null, measure_name: typeof measureCol.name === "string" ? (measureCol.name as string) : null, @@ -1212,7 +1434,7 @@ function enrichTargetRow( } function estimateScopeKey(row: TargetRow): string | null { - if (row.filter != null) return null; + if (row.filter != null || row.dimensions != null) return null; const metadata = asObject(row.metadata); const recordSet = stringValue(metadata.ledger_layout_record_set_id); const initial = numberOrNull(row.initial_estimate); @@ -1483,7 +1705,11 @@ export function microcosmVariableSummary(rows: TargetRow[]) { return { variable_key, source: String(first.source ?? ""), + source_label: String( + first.source_label ?? sourceAuthorityLabel(String(first.source ?? "")), + ), variable: String(first.variable ?? ""), + variable_label: uniqueString("variable_label"), measure: first.measure ? String(first.measure) : null, level: String(first.level ?? ""), policyengine_variables: policyengineVariables, @@ -1689,9 +1915,16 @@ export function microcosmTargetTreemap( const scored = children.reduce((s, c) => s + c.scored, 0); const within_10pct = children.reduce((s, c) => s + c.within_10pct, 0); const loss = children.reduce((s, c) => s + c.loss, 0); + const rowSourceLabel = [...byVar.values()] + .flat() + .map((row) => stringValue(row.source_label)) + .find((label): label is string => label != null); return { source, - label: source === "geography" ? "Geography" : sourceAuthorityLabel(source), + label: + source === "geography" + ? "Geography" + : rowSourceLabel ?? sourceAuthorityLabel(source), n_targets, scored, within_10pct, @@ -1717,11 +1950,19 @@ export function microcosmTargetTreemap( } // --- the calibration source (one release) ----------------------------------- +export interface TargetSchema { + diagnostics_schema_version: number | null; + structured_dimensions: boolean; +} + export interface Calibration { source: "huggingface_live"; country: MicrocosmCountry; // Typed `release_manifest.country` merged over the registration. country_info: ArtifactCountry; + presentation: ArtifactPresentation | null; + publisher_labels: Record; + target_schema: TargetSchema; description: string | null; diagnostics_status: DiagnosticsStatus; release_id: string; @@ -1832,6 +2073,44 @@ function diagnosticsStatus(diag: JsonObject, rows: TargetRow[]): DiagnosticsStat return anyUsable ? "ok" : "incompatible"; } +export interface ArtifactPresentation { + overview_intro?: string; + targets_intro?: string; +} + +function presentationText(value: unknown): string | null { + if (typeof value !== "string") return null; + const trimmed = value.trim(); + return trimmed ? trimmed.slice(0, 600) : null; +} + +export function releasePresentation( + releaseManifest: JsonObject, +): ArtifactPresentation | null { + const block = asObject(releaseManifest.presentation); + const overviewIntro = presentationText(block.overview_intro); + const targetsIntro = presentationText(block.targets_intro); + if (!overviewIntro && !targetsIntro) return null; + return { + ...(overviewIntro ? { overview_intro: overviewIntro } : {}), + ...(targetsIntro ? { targets_intro: targetsIntro } : {}), + }; +} + +export function releasePublisherLabels( + releaseManifest: JsonObject, +): Record { + const block = releaseManifest.publisher_labels; + if (!isPlainObject(block)) return {}; + return Object.fromEntries( + Object.entries(block).flatMap(([key, value]) => { + if (!/^[a-z][a-z0-9_]*$/i.test(key) || typeof value !== "string") return []; + const label = value.trim(); + return label ? [[key, label]] : []; + }), + ); +} + export interface ReleaseRole { dataset_role: string | null; is_default: boolean; @@ -1940,8 +2219,20 @@ export function buildCalibration( const skippedByName = skippedTargetReasons(skipped); const dropped = new Set(droppedTargetNames); const artifactCountry = releaseCountry(releaseManifest, country); + const presentation = releasePresentation(releaseManifest); + const publisherLabels = releasePublisherLabels(releaseManifest); + const dimensionDefinitions = diagnosticsDimensions(diag); const enrichedRows = addEstimateScopeWarnings( - targets.map((row) => enrichTargetRow(row, skippedByName, dropped, artifactCountry)), + targets.map((row) => + enrichTargetRow( + row, + skippedByName, + dropped, + artifactCountry, + publisherLabels, + dimensionDefinitions, + ), + ), ); const role = releaseRole(releaseManifest); const normalizedAttribution = normalizeTargetLossAttribution({ @@ -1957,6 +2248,12 @@ export function buildCalibration( source: "huggingface_live", country, country_info: artifactCountry, + presentation, + publisher_labels: publisherLabels, + target_schema: { + diagnostics_schema_version: numberOrNull(diag.schema_version), + structured_dimensions: isPlainObject(diag.dimensions), + }, description: stringValue(diag.description) ?? stringValue(releaseManifest.description) ?? @@ -2231,7 +2528,9 @@ function targetResponseRow(row: TargetRow): TargetRow { geography: row.geography, level: row.level, source: row.source, + source_label: row.source_label, variable: row.variable, + variable_label: row.variable_label, measure: row.measure, target_role: row.target_role, source_measure_id: row.source_measure_id, @@ -2251,8 +2550,10 @@ function targetResponseRow(row: TargetRow): TargetRow { breakdown: row.breakdown, dims: row.dims, target_dimensions: row.target_dimensions, + dimension_adapter: row.dimension_adapter, variable_key: row.variable_key, source_citation: row.source_citation, + source_url: row.source_url, entity: row.entity, aggregation: row.aggregation, measure_name: row.measure_name, @@ -2440,6 +2741,8 @@ export function latestMicrocosmCalibrationSummary(cal: Calibration) { return { available: true, country: cal.country_info, + presentation: cal.presentation, + target_schema: cal.target_schema, description: cal.description, diagnostics_status: cal.diagnostics_status, ...releaseRole(cal.release_manifest), @@ -2620,6 +2923,8 @@ export function latestMicrocosmTargetDiagnosticsPage(requestUrl: string, cal: Ca return { available: true, country: cal.country_info, + presentation: cal.presentation, + target_schema: cal.target_schema, description: cal.description, diagnostics_status: cal.diagnostics_status, ...releaseRole(cal.release_manifest), diff --git a/frontend/lib/microcosm/presentation.test.ts b/frontend/lib/microcosm/presentation.test.ts new file mode 100644 index 0000000..31b402e --- /dev/null +++ b/frontend/lib/microcosm/presentation.test.ts @@ -0,0 +1,31 @@ +import { expect, test } from "bun:test"; + +import { microcosmOverviewIntro, microcosmTargetsIntro } from "./presentation"; + +test("artifact presentation takes precedence over legacy country copy", () => { + const presentation = { + overview_intro: "Artifact overview.", + targets_intro: "Artifact target prompt.", + }; + + expect(microcosmOverviewIntro("us", presentation)).toBe("Artifact overview."); + expect(microcosmTargetsIntro("us", presentation)).toBe("Artifact target prompt."); +}); + +test("legacy presentation copy remains the fallback for existing countries", () => { + expect(microcosmOverviewIntro("us")).toBe( + "Microcosm reweights survey microdata so it matches official statistics from agencies like the IRS, the Census Bureau, and CMS. Each tile in the Calibration fit explorer below is a category we calibrate to, including EITC statistics, population, and Medicaid enrollment.", + ); + expect(microcosmTargetsIntro("us")).toBe( + "Pick a measure like EITC, population, or AGI and see how each breakdown is calibrated.", + ); +}); + +test("unpublished countries receive the generic presentation copy", () => { + expect(microcosmOverviewIntro("zz")).toBe( + "Microcosm reweights survey microdata so it matches official statistics from national statistical agencies and administrative sources. Each tile in the Calibration fit explorer below is a category we calibrate to.", + ); + expect(microcosmTargetsIntro("zz")).toBe( + "Pick a measure and see how each breakdown is calibrated.", + ); +}); diff --git a/frontend/lib/microcosm/presentation.ts b/frontend/lib/microcosm/presentation.ts new file mode 100644 index 0000000..5e674f6 --- /dev/null +++ b/frontend/lib/microcosm/presentation.ts @@ -0,0 +1,62 @@ +import type { MicrocosmCountry } from "./countries"; + +export interface MicrocosmPresentationSlots { + overview_intro?: string; + targets_intro?: string; +} + +// Legacy copy for releases published before `release_manifest.presentation`; +// delete once US/UK/BE publish the block. +const LEGACY_OVERVIEW_COPY: Partial< + Record +> = { + us: { + authorities: "the IRS, the Census Bureau, and CMS", + examples: "EITC statistics, population, and Medicaid enrollment", + }, + uk: { + authorities: "the ONS, OBR, and HMRC", + examples: "population by region and age, household types, and tax receipts", + }, + be: { + authorities: "Statbel, ONSS, JRC, and SFPD", + examples: "population by region, sex, and age band, tax receipts, and benefit totals", + }, +}; + +const GENERIC_OVERVIEW_INTRO = + "Microcosm reweights survey microdata so it matches official statistics from national statistical agencies and administrative sources. Each tile in the Calibration fit explorer below is a category we calibrate to."; + +// Legacy copy for releases published before `release_manifest.presentation`; +// delete once US/UK/BE publish the block. +const LEGACY_TARGETS_COPY: Partial> = { + us: + "Pick a measure like EITC, population, or AGI and see how each breakdown is calibrated.", + uk: + "Pick a measure like population, household type, or tax receipts and see how each breakdown is calibrated.", + be: + "Pick a measure like population, income tax, or pension recipients and see how each breakdown is calibrated.", +}; + +const GENERIC_TARGETS_INTRO = + "Pick a measure and see how each breakdown is calibrated."; + +export function microcosmOverviewIntro( + country: MicrocosmCountry, + presentation?: MicrocosmPresentationSlots | null, +): string { + if (presentation?.overview_intro) return presentation.overview_intro; + const legacy = LEGACY_OVERVIEW_COPY[country]; + return legacy + ? `Microcosm reweights survey microdata so it matches official statistics from agencies like ${legacy.authorities}. Each tile in the Calibration fit explorer below is a category we calibrate to, including ${legacy.examples}.` + : GENERIC_OVERVIEW_INTRO; +} + +export function microcosmTargetsIntro( + country: MicrocosmCountry, + presentation?: MicrocosmPresentationSlots | null, +): string { + return presentation?.targets_intro ?? + LEGACY_TARGETS_COPY[country] ?? + GENERIC_TARGETS_INTRO; +} diff --git a/frontend/lib/microcosm/third-country-conformance.test.ts b/frontend/lib/microcosm/third-country-conformance.test.ts index c84b291..6acc972 100644 --- a/frontend/lib/microcosm/third-country-conformance.test.ts +++ b/frontend/lib/microcosm/third-country-conformance.test.ts @@ -204,7 +204,7 @@ describe("synthetic third-country conformance", () => { .map((item) => item.href), ).toEqual(["/microcosm", "/microcosm/targets", "/microcosm/compare"]); }); - test.todo( + test( "zz overview data can carry artifact-provided intro copy: the summary omits a typed presentation contract", () => { const presentation = { @@ -227,7 +227,7 @@ describe("synthetic third-country conformance", () => { expect(overview.presentation).toEqual(presentation); }, ); - test.todo( + test( "zz can override a publisher display name: treemap shaping ignores release_manifest.publisher_labels", () => { const futureCalibration = buildCalibration( @@ -252,7 +252,7 @@ describe("synthetic third-country conformance", () => { expect(treemap.groups[0]?.label).toBe("Nova Statistics Agency"); }, ); - test.todo( + test( "zz can supply structured facets: target shaping ignores calibration_diagnostics dimensions blocks", () => { const facetValues = [ @@ -305,4 +305,84 @@ describe("synthetic third-country conformance", () => { ]); }, ); + + test("structured source and variable identifiers replace name parsing", () => { + const structuredSource = { + id: "novastat_agency", + citation: "ZZ official population table", + url: "https://stats.example/zz/pop", + }; + const structuredVariable = { + id: "population", + label: "Resident population", + measure: "count", + }; + const futureDiagnostics = { + ...diagnosticsFixture, + dimensions: { + region: { + label: "Region", + role: "geography", + level: "region", + values: { north: "North", south: "South" }, + }, + sex: { label: "Sex", values: { female: "Female", male: "Male" } }, + age_band: { label: "Age band" }, + }, + targets: diagnosticsFixture.targets.map((target, index) => { + const metadata = Object.fromEntries( + Object.entries(target.metadata).filter(([key]) => key !== "variable"), + ); + return { + ...target, + ...(index === 0 + ? { + filter: null, + dimensions: { + region: "north", + sex: "female", + age_band: "0_17", + }, + } + : {}), + source: structuredSource, + variable: structuredVariable, + metadata, + }; + }), + }; + const futureCalibration = buildCalibration( + futureDiagnostics, + RELEASE_ID, + "2026-08-23T18:00:00Z", + {}, + releaseManifestFixture, + {}, + COUNTRY, + ); + + expect(futureCalibration.rows.map((row) => row.dimension_adapter)).toEqual([ + "structured", + "legacy_filter", + "legacy_filter", + "legacy_name", + ]); + const page = latestMicrocosmTargetDiagnosticsPage( + `http://x/api/microcosm/target-diagnostics?variable=${encodeURIComponent("novastat_agency / population · count")}`, + futureCalibration, + ); + const structuredRow = page.targets.find( + (row) => row.dimension_adapter === "structured", + ); + expect(structuredRow).toMatchObject({ + source: "novastat_agency", + source_citation: "ZZ official population table", + source_url: "https://stats.example/zz/pop", + variable: "population", + variable_label: "Resident population", + measure: "count", + variable_key: "novastat_agency / population · count", + dimension_adapter: "structured", + }); + }); }); diff --git a/frontend/lib/source-labels.test.ts b/frontend/lib/source-labels.test.ts index a21c884..2e2daa2 100644 --- a/frontend/lib/source-labels.test.ts +++ b/frontend/lib/source-labels.test.ts @@ -17,6 +17,7 @@ test("uses curated authority names for Chronicle and calibration sources", () => test("uses the calibration-fit acronym-aware fallback for unknown sources", () => { expect(sourceAuthorityLabel("new_api_source")).toBe("New API Source"); + expect(sourceAuthorityLabel("constructor")).toBe("Constructor"); }); test("shares publisher labels across country dashboards", () => { diff --git a/frontend/lib/source-labels.ts b/frontend/lib/source-labels.ts index 52b20bb..9397c71 100644 --- a/frontend/lib/source-labels.ts +++ b/frontend/lib/source-labels.ts @@ -58,7 +58,7 @@ const SOURCE_ACRONYMS = new Set([ ]); export function sourceAuthorityLabel(source: string): string { - if (SOURCE_LABELS[source]) return SOURCE_LABELS[source]; + if (Object.hasOwn(SOURCE_LABELS, source)) return SOURCE_LABELS[source]; return source .split("_") .map((word) =>