From ec7a91051cee47248ceface9cb0b5cf08b065a0b Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sat, 29 Aug 2026 21:07:02 -0400 Subject: [PATCH 1/2] Add NZ transport spec and verified Axiom Frame bridge --- README.md | 4 + .../343-nz-transport-scaffold.added.md | 1 + docs/country-expansion-playbook.md | 148 ++++++ docs/nz-take-up-engine-plan.md | 79 +++- .../microcosm/build/nz/country_package.json | 66 +++ .../microcosm/build/nz/export_contract.json | 148 ++++++ .../src/microcosm/build/nz/gates.json | 136 ++++++ .../microcosm/build/nz/geography_spine.json | 23 + .../microcosm/build/nz/release_contract.json | 54 +++ .../src/microcosm/build/nz/source_stages.json | 161 +++++++ .../src/microcosm/build/nz/spec/bundle.yaml | 16 + .../src/microcosm/build/nz/spec/catalogs.yaml | 331 +++++++++++++ .../microcosm/build/nz/spec/geography.yaml | 29 ++ .../src/microcosm/build/nz/spec/sources.yaml | 197 ++++++++ .../src/microcosm/build/nz/spec/spine.yaml | 10 + .../src/microcosm/build/nz/spec/vintages.yaml | 41 ++ .../microcosm/build/nz/target_references.json | 311 +++++++++++++ .../microcosm/build/spec_engine/resolver.py | 37 +- .../spec_engine/schema/catalogs.schema.json | 3 + .../tests/golden/nz_country_spec.json | 85 ++++ .../tests/test_country_spec.py | 434 +++++++++++++++++- .../tests/test_spec_engine_country_bundles.py | 109 +++++ .../tests/test_spec_engine_reemission.py | 2 +- .../tests/test_spec_only_country_packages.py | 42 +- .../src/microcosm/frame/adapters/axiom.py | 207 ++++++++- .../xx/policies}/axiom_toy_country.yaml | 0 .../tests/test_axiom_adapter.py | 152 +++++- .../tests/test_rules_engine_contract.py | 4 +- 28 files changed, 2744 insertions(+), 86 deletions(-) create mode 100644 changelog.d/343-nz-transport-scaffold.added.md create mode 100644 docs/country-expansion-playbook.md create mode 100644 packages/microcosm-build/src/microcosm/build/nz/country_package.json create mode 100644 packages/microcosm-build/src/microcosm/build/nz/export_contract.json create mode 100644 packages/microcosm-build/src/microcosm/build/nz/gates.json create mode 100644 packages/microcosm-build/src/microcosm/build/nz/geography_spine.json create mode 100644 packages/microcosm-build/src/microcosm/build/nz/release_contract.json create mode 100644 packages/microcosm-build/src/microcosm/build/nz/source_stages.json create mode 100644 packages/microcosm-build/src/microcosm/build/nz/spec/bundle.yaml create mode 100644 packages/microcosm-build/src/microcosm/build/nz/spec/catalogs.yaml create mode 100644 packages/microcosm-build/src/microcosm/build/nz/spec/geography.yaml create mode 100644 packages/microcosm-build/src/microcosm/build/nz/spec/sources.yaml create mode 100644 packages/microcosm-build/src/microcosm/build/nz/spec/spine.yaml create mode 100644 packages/microcosm-build/src/microcosm/build/nz/spec/vintages.yaml create mode 100644 packages/microcosm-build/src/microcosm/build/nz/target_references.json create mode 100644 packages/microcosm-build/tests/golden/nz_country_spec.json rename packages/microcosm-frame/tests/fixtures/{ => rulespec-xx/xx/policies}/axiom_toy_country.yaml (100%) diff --git a/README.md b/README.md index 964fb2cdb..08513ccf7 100644 --- a/README.md +++ b/README.md @@ -28,6 +28,10 @@ Incumbent comparisons and historical replacement benchmarks live outside this repo. The live Microcosm repo owns the library, build contracts, published population registry, and acceptance gates. +For new countries, follow the [country-expansion playbook](docs/country-expansion-playbook.md): +parallel facts, Axiom coverage, and external scores; then population calibration +and dashboard publication; then independently receipted score replication. + ## Development ```bash diff --git a/changelog.d/343-nz-transport-scaffold.added.md b/changelog.d/343-nz-transport-scaffold.added.md new file mode 100644 index 000000000..29cb2efba --- /dev/null +++ b/changelog.d/343-nz-transport-scaffold.added.md @@ -0,0 +1 @@ +Add a spec-only New Zealand transport package with pinned donor and WFF contracts, a closed family export surface, real-Axiom Frame smoke tests, a calibration-dashboard release contract, and a reusable country-expansion playbook. Extend the shared Axiom adapter with canonical roots, explicit policy periods, bounded exact Decimal conversion, and lossless Decimal HDF persistence. diff --git a/docs/country-expansion-playbook.md b/docs/country-expansion-playbook.md new file mode 100644 index 000000000..8c257423d --- /dev/null +++ b/docs/country-expansion-playbook.md @@ -0,0 +1,148 @@ +# Country expansion playbook + +Expand countries through three parallel evidence lanes, then join them at a +population-build gate. A country registry entry, a compiled rules module, a +calibrated population, and a published replication are different milestones. +Record each milestone separately. + +## Start three lanes in parallel + +| Lane | Owner and output | Completion receipt | +| --- | --- | --- | +| Official facts and calibration candidates | Chronicle stores source cells, observed facts, periods, geographies, units, universes, uncertainty, and provenance. Microcosm authors the later target bindings. | Immutable fact-package revision, source hashes, cell identities, coverage inventory, and explicit unavailable families. | +| Full Axiom coverage | Axiom maintains the jurisdiction's source and policy coverage frontier, RuleSpec, real-runtime tests, and the root-input/output ABI. | RuleSpec and engine commits, artifact hashes, coverage report, passing runtime fixtures, required inputs, permitted defaults, and formula-owned export exclusions. | +| External score registry | Scorecard records official Budget or fiscal-office scores independently of any replication. Prioritize reforms whose complete scored scope can be represented. | Official publication and table/cell, reform definition, baseline, score components, currency, sign, period, and an explicit replication-readiness assessment. | + +Chronicle is facts-only. Legacy Microcosm interfaces still use the name +`Ledger`, but that does not move target profiles or observed values into this +repository. Do not embed observed calibration values in country packages. +Official model projections and Budget reform scores remain validation evidence, +not targets selected to make the model reproduce its comparator. + +For each official score, separate legal entitlement from cash timing, accruals, +debt impairment, administration, behavioural assumptions, and interactions. A +reform is fully scoreable only when the whole published quantity is covered or +the registered comparison explicitly selects a separately published component. +Registering the external score does not imply that replication is complete. + +## Join at the population-input gate + +Choose a native survey, a public donor population, or an explicitly licensed +combination. Before binding a build, record: + +- Country, tax/fiscal periods, population universe, and geography vintages. +- Exact source revision, filename, SHA-256, byte size, and licence. No mutable + `main` or `latest` fallback may substitute for a pinned donor. +- Frame entities, stable identifiers, memberships, links, and typed weights. + Keep one explicit calibrated weight source and use `Frame.resolve_weights` + for another entity. Do not persist an ambiguous second weight vector. +- The rules engine's actual ABI: entity names, input dtypes and units, required + fields, legal/data bridges, allowed padding, output periods, and export + exclusions. Missing substantive inputs must fail closed. +- Cell-pinned Chronicle references and reviewed candidate-column bindings. + A table selector is an authoring contract, not an active target; it must not + silently fan out or choose the first matching fact. +- Separate source-period, currency, geography, family-definition, and + entitlement-to-payment bridges. Do not infer a joint distribution from + marginal agreement. + +Keep country folders under `packages/microcosm-build/src/microcosm/build//` +spec-only. Use `load_country_spec`, the shared spec compiler, and real shared +operators. A contract-only kernel proves that the compiler can express the +country; it is not an executable implementation. `country_stage_plan` must +refuse missing operators rather than synthesize a fallback. + +## Build, calibrate, and expose diagnostics + +Run the same logical pipeline at reduced scale first, then at release scale: + +1. Authenticate sources and construct the Frame. Preserve donor country and + support-stratum provenance. When transporting a donor, receipt the change + from source-population mass to destination-population mass. +2. Bind the real rules engine and prove its entity/input contract. Do not + replace a missing runtime stage with handwritten policy calculations. +3. Compile the target matrix from the pinned facts and reviewed bindings. + Reconcile totals and components; do not count both as independent evidence. +4. Calibrate household weights, then evaluate support, per-family fit, + reference coverage, effective sample size, weight concentration, input + coverage, and formula-owned export gates. Set country thresholds before + inspecting a candidate. An absent threshold is unfinished work, not a pass. +5. Emit release and build manifests, structured calibration diagnostics, + source coverage, demographics where supported, and held-out reform + validation. Persist failures with their evidence and owners. + +Fast mode reduces records, epochs, or draws; it must not replace rules, targets, +or gating logic. Compile/refusal tests are appropriate while shared operators +are missing. Label them as scaffold tests, not as a population calibration run. + +Recipient counts can constrain an explicitly modelled receipt/take-up layer. +They must not force full-entitlement outputs to match observed expenditure. +Hold out official Budget scores and independent model results from calibration. + +## Publish and connect the calibration dashboard + +The country release contract declares the public/private repository boundary, +required artifacts, immutable release identity, and staging destination. +Publication remains a separate authorized action after gates pass. + +The dashboard consumes a real `latest.json` pointer, the referenced +`release_manifest.json`, and `calibration_diagnostics.json`. Country labels, +geography, units, and source publishers must come from structured metadata, +not filename parsing or US-specific assumptions. The producer and the dashboard +registry must agree on the `release_manifest.json.country` block and supported +capabilities. A registered country with no valid release shows an unavailable +state; it must not display fabricated charts or another country's data. + +For NZ, the agreed block is: + +```json +{ + "code": "nz", + "label": "New Zealand", + "geography_id": null, + "geography_label": "New Zealand", + "repository_visibility": "public", + "capabilities": ["calibration", "targets", "compare"] +} +``` + +The producer contract uses `policyengine/populace-nz` for releases and +`policyengine/populace-nz-staging` for build telemetry. These identifiers alone +do not create repositories or publish a release. + +## Replicate scores and attach results + +After the population and rules legs pass their gates, execute baseline and +reform against the same immutable population. Record the engine, RuleSpec or +model version, data build, target snapshot, reform definition, periods, +accounting bridges, output units, and static/behavioural assumptions. + +Use the PolicyEngine interface for PolicyEngine replications. That interface +may run a real Axiom backend; a separate legacy country package is not a +prerequisite. Until the interface and input surface are wired, label direct +Axiom executions as Axiom and leave the PolicyEngine replication pending. +Never relabel an unexecuted or handwritten result as a PolicyEngine run. + +Attach the result to the existing external Scorecard claim with a run receipt, +the like-for-like amount, residual difference, and known exclusions. Report a +partial entitlement comparison as partial; do not fill missing operating-cost +components with zero. Keep independent official scores immutable when adding +replications. + +## Track country milestones + +| Milestone | Evidence required before marking complete | +| --- | --- | +| Registered | Country and external-score records exist; missing data is visible. | +| Compile-ready | Typed country package, exact ABI and source references, reproducible spec hash, and refusal tests pass. | +| Build-ready | Required source facts, population fields, shared operators, licences, and country thresholds are bound. | +| Calibrated | A real solve completed and the complete gate report and diagnostics exist. | +| Live | An immutable release and valid pointer load successfully in the deployed dashboard. | +| Replicated | Real engine runs cover the official score's stated scope and Scorecard links their receipts. | + +Belgium and NZ share the same Frame/Axiom seam but not their data-access +posture: Belgium's SILC package names a private artifact repository; NZ's first +transport package names a public US donor and public NZ aggregates. Neither +country may borrow another country's thresholds or claim that donor records +were observed locally. See the [NZ plan](nz-take-up-engine-plan.md) and +[NZ target inventory](nz-calibration-targets.md) for the country-specific gates. diff --git a/docs/nz-take-up-engine-plan.md b/docs/nz-take-up-engine-plan.md index 9fa900b8c..55b3a261d 100644 --- a/docs/nz-take-up-engine-plan.md +++ b/docs/nz-take-up-engine-plan.md @@ -1,6 +1,56 @@ -# microcosm-nz v1 plan: the New Zealand dollar take-up engine - -Working plan for [epic #343](https://github.com/PolicyEngine/microcosm/issues/343). Companion inventory: [nz-calibration-targets.md](./nz-calibration-targets.md). Rules: [rulespec-nz](https://github.com/TheAxiomFoundation/rulespec-nz) through the Frame `RulesEngine` protocol via the Axiom adapter (shared with the Belgium pilot under #259 — NZ adds coverage, not adapter code). +# New Zealand transport and take-up plan + +Working plan for [epic #343](https://github.com/PolicyEngine/microcosm/issues/343). Companion inventory: [nz-calibration-targets.md](./nz-calibration-targets.md). Rules: [rulespec-nz](https://github.com/TheAxiomFoundation/rulespec-nz) through the Frame `RulesEngine` protocol via the Axiom adapter shared with the Belgium pilot under #259. + +## Implementation status: 29 August 2026 + +The spec-only [NZ country package](../packages/microcosm-build/src/microcosm/build/nz/country_package.json) +now compiles through the shared country seam. It is a scaffold, not a calibrated +population or live release. Follow the [country-expansion playbook](country-expansion-playbook.md) +for the three parallel evidence lanes and the build/publication gates. + +The donor pin is the public US Build P release +`populace-us-2024-buildp-sparse-rmloss100-cae8640-20260728T011454Z`, file +`populace_us_2024.h5`; the package records its SHA-256 and byte size. The first +rules bridge is the WFF official-Budget reform contract at rulespec-nz commit +`3b663b3e6eb6408351154990be0c4b92d42c92da`. It requires 11 evidenced Family +inputs and permits 10 adapter-padding zeros outside the requested outputs' +dependency graphs. Automatic unit derivation is not yet bound by the adapter. + +The [closed export contract](../packages/microcosm-build/src/microcosm/build/nz/export_contract.json) +now has a real-runtime smoke: three synthetic families in two households, +household weights 2 and 7, and family weights resolved by the Frame as 2, 7, +and 7. The two upstream companion cases produce entitlement changes of +NZD 568.50 and NZD 438.50; Frame-weighted fixture totals are NZD 1,137 and +NZD 3,069.50. These are test-fixture results, not NZ population estimates. +The smoke verifies tax-year bounds, missing-input refusal, formula-owned +export exclusions, and exact Decimal-input HDF round trips. The generic +adapter now accepts canonical RuleSpec roots and `AxiomPeriod` bounds. +Its native float64 input boundary accepts Decimal values only when nine-place +rounding recovers them exactly; it does not claim unrestricted Decimal128 ABI +support. Treasury source activation remains pending immutable corpus +publication and protected validation. + +The release contract names `policyengine/populace-nz` and declares calibration, +target, and comparison dashboard capabilities. Build readiness still requires +cell-pinned Chronicle facts, period/currency/family bridges, shared transport +operators, and reviewed NZ calibration thresholds. The WFF reform checks are +the first bounded official-score validation slice; the intended take-up +research product below remains separate. + +Run the smoke against the pinned rules checkout and a real Axiom dense native +extension (engine commit `bb4b5684870547756078a62f1866a77c5b56f7f3`): + +```bash +POPULACE_RULESPEC_NZ=/absolute/path/rulespec-nz \ + uv run --no-sync pytest \ + packages/microcosm-build/tests/test_country_spec.py -k NewZealandAxiomTransport +``` + +Install `microcosm-build`, `microcosm-frame[axiom]`, the Axiom Python wrapper, +and its built native extension into the workspace environment first; +`--no-sync` retains the separately installed native build. The test reports +a skip, not a pass, when the real extension or rules checkout is absent. ## Product definition @@ -8,14 +58,18 @@ For each major NZ transfer, estimate **predicted entitlement dollars** (rules × ## Method (Belgium pilot recipe, recalibrated) -1. **Donor pool**: the populace-us support pool (57,240 households / 166,302 persons in the BE pilot vintage). Donor records are US support records and are never presented as NZ microdata; support-stratum labels ship with the artifact. +1. **Donor pool**: the immutable populace-us release pinned in the NZ country package. Historical BE pilot record counts do not describe this pin; the loader must inventory the authenticated artifact. Donor records are US support records and are never presented as NZ microdata; support-stratum labels ship with the artifact. 2. **Reweighting**: microcosm-calibrate against the v1 target set below. Two BE-pilot solver lessons apply verbatim: initialize design weights at the destination population scale (or `target_loss_cap` saturates and gradients vanish), and consume total rows, not component+total double counts, from multi-row target tables. 3. **Rules leg**: rulespec-nz composed modules via the Axiom adapter. Child support is excluded from composed outputs until the income-shares re-encode lands (rulespec-nz#74). 4. **Periods**: tax year 2026-27 primary (1 Apr 2026 rates are current across the encoded surface); PPL/rates-rebate use their 1 July steps. -## v1 target set +## Candidate calibration and validation inputs -Sixteen margins, all public, drawn from the inventory's ranked list (numbers refer to inventory rows): +The public inventory below contains both calibration candidates and held-out +validation inputs. Observed values and source cells live in Chronicle; the NZ +package contains unactivated authoring references until every selected cell has +a reviewed Frame binding and target-period basis. Recipient counts constrain a +receipt/take-up layer, not full-entitlement outputs. | Margin | Source | Kind | |---|---|---| @@ -29,14 +83,17 @@ Sixteen margins, all public, drawn from the inventory's ranked list (numbers ref | Main-benefit recipients by benefit × age group | MSD M1 | count | | AS recipients by W&I region | MSD M1d | count | | NZS/VP recipients | MSD O4 | count | -| WfF recipient families + expenditure by credit type | IRD I3 | count + $ | -| Benefit expenditure by program | MSD M2 (2024/25 schedule) | $ | -| Rent distribution by TA × bedrooms | MBIE H3 | $ | +| WfF recipient families by credit type | IRD I3 | receipt-state count | +| WfF and benefit entitlement/expenditure schedules | IRD I3 + MSD M2 | held-out dollar validation | +| Rent distribution by TA × bedrooms | MBIE H3 | input/validation, not a hard expenditure target | | Student loan borrowers | IRD I4 | count | -| Minimum/average earnings anchors | MBIE O1 + QES S12 | $ | +| Minimum/average earnings anchors | MBIE O1 + QES S12 | input/macro validation | | Ethnicity × age × region (equity cuts) | Census O5 | count | -Deliberate non-targets: Treasury DistributionExplorer values (full-entitlement modelled — that's the comparison output, not a calibration input); household projections (stale 2013-base); anything IDI-derived. +Deliberate non-targets: Treasury DistributionExplorer values and official Budget +reform scores (independent model comparisons); programme expenditure and +entitlement-dollar schedules (held-out take-up and accounting evidence); +household projections (stale 2013-base); anything IDI-derived. ## The joint-income problem diff --git a/packages/microcosm-build/src/microcosm/build/nz/country_package.json b/packages/microcosm-build/src/microcosm/build/nz/country_package.json new file mode 100644 index 000000000..cd7d499b5 --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/nz/country_package.json @@ -0,0 +1,66 @@ +{ + "schema_version": 1, + "country": "nz", + "resources": [ + { + "path": "spec/bundle.yaml", + "kind": "bundle", + "schema_id": "bundle.schema.json" + }, + { + "path": "spec/catalogs.yaml", + "kind": "catalogs", + "schema_id": "catalogs.schema.json" + }, + { + "path": "spec/geography.yaml", + "kind": "geography", + "schema_id": "geography.schema.json" + }, + { + "path": "spec/sources.yaml", + "kind": "sources", + "schema_id": "sources.schema.json" + }, + { + "path": "spec/spine.yaml", + "kind": "spine", + "schema_id": "spine.schema.json" + }, + { + "path": "spec/vintages.yaml", + "kind": "vintages", + "schema_id": "vintages.schema.json" + }, + { + "path": "export_contract.json", + "kind": "legacy_json", + "schema_id": "legacy_json" + }, + { + "path": "gates.json", + "kind": "legacy_json", + "schema_id": "legacy_json" + }, + { + "path": "geography_spine.json", + "kind": "legacy_json", + "schema_id": "legacy_json" + }, + { + "path": "release_contract.json", + "kind": "legacy_json", + "schema_id": "legacy_json" + }, + { + "path": "source_stages.json", + "kind": "legacy_json", + "schema_id": "legacy_json" + }, + { + "path": "target_references.json", + "kind": "legacy_json", + "schema_id": "legacy_json" + } + ] +} diff --git a/packages/microcosm-build/src/microcosm/build/nz/export_contract.json b/packages/microcosm-build/src/microcosm/build/nz/export_contract.json new file mode 100644 index 000000000..0ef92fbb4 --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/nz/export_contract.json @@ -0,0 +1,148 @@ +{ + "_description": "Closed Frame export contract for the NZ WFF official-Budget transport smoke. This is an input artifact contract, not a calibrated population or official operating-cost replication. Fields must already be evidenced and bound on the family table.", + "_source": { + "repository": "TheAxiomFoundation/rulespec-nz", + "commit": "3b663b3e6eb6408351154990be0c4b92d42c92da", + "path": "data/microsimulation/official-budget-reform-transport.json", + "sha256": "cd650c191a8a98faedf2bc45c80b6026eb06037b3357d9af37d25ee1dd460dfa", + "source_release": "pending_treasury_corpus_publication" + }, + "_schema": { + "entities": [ + "person", + "household", + "family" + ], + "engine_entity_names": { + "person": "Person", + "household": "Household", + "family": "Family" + }, + "structural_columns": { + "person": [ + "person_id", + "person_household_id", + "person_family_id" + ], + "household": [ + "household_id" + ], + "family": [ + "family_id", + "family_household_id" + ] + } + }, + "_period": { + "label": 2026, + "period_kind": "tax_year", + "start": "2026-04-01", + "end": "2027-03-31" + }, + "_required_target_inputs": [ + "family_tax_credit_eldest_dependent_child_care_units", + "family_tax_credit_subsequent_dependent_child_care_units", + "family_tax_credit_entitlement_days", + "wff_family_scheme_income_for_relationship_period", + "wff_family_credit_abatement_days", + "entitled_to_in_work_tax_credit", + "in_work_tax_credit_allowed_children_count", + "in_work_tax_credit_weekly_periods", + "child_tax_credit_for_entitlement_period", + "parental_tax_credit_for_entitlement_period", + "parental_tax_credit_additional_abatement" + ], + "_input_dtypes": { + "family_tax_credit_eldest_dependent_child_care_units": "decimal128(18,2)", + "family_tax_credit_subsequent_dependent_child_care_units": "decimal128(18,2)", + "family_tax_credit_entitlement_days": "int16", + "wff_family_scheme_income_for_relationship_period": "decimal128(18,2)", + "wff_family_credit_abatement_days": "int16", + "entitled_to_in_work_tax_credit": "bool", + "in_work_tax_credit_allowed_children_count": "int16", + "in_work_tax_credit_weekly_periods": "int16", + "child_tax_credit_for_entitlement_period": "decimal128(18,2)", + "parental_tax_credit_for_entitlement_period": "decimal128(18,2)", + "parental_tax_credit_additional_abatement": "decimal128(18,2)", + "best_start_abatement_days": "int64", + "best_start_child_care_fraction": "float64", + "best_start_entitlement_days": "int64", + "best_start_family_scheme_income_for_relationship_period": "float64", + "minimum_family_adjusted_income_tax_liability": "float64", + "minimum_family_amount_paid": "float64", + "minimum_family_amount_received": "float64", + "minimum_family_full_time_earner_weeks": "int64", + "minimum_family_scheme_income_attributable_to_full_time_weeks": "float64", + "minimum_family_tax_credit_weekly_periods": "int64" + }, + "_padding_defaults": { + "best_start_abatement_days": 0, + "best_start_child_care_fraction": 0, + "best_start_entitlement_days": 0, + "best_start_family_scheme_income_for_relationship_period": 0, + "minimum_family_adjusted_income_tax_liability": 0, + "minimum_family_amount_paid": 0, + "minimum_family_amount_received": 0, + "minimum_family_full_time_earner_weeks": 0, + "minimum_family_scheme_income_attributable_to_full_time_weeks": 0, + "minimum_family_tax_credit_weekly_periods": 0 + }, + "_decimal_boundary": "Store substantive decimal128(18,2) values as Decimal columns. The native dense ABI accepts float64, so materialization permits only values exactly recovered by its nine-decimal-place rounding and rejects the remainder; arbitrary Decimal128 support is not claimed. HDF export preserves Decimal objects exactly.", + "_weight_contract": { + "stored_weight_entity": "household", + "stored_weight_column": "household_weight", + "family_weight_column_stored": false, + "family_weight_resolution": "The frame must call resolve_weights('family'); each family must map to exactly one household and inherit that household's calibrated weight.", + "forbid_manual_person_weight_aggregation": true + }, + "required": [ + "person_id", + "person_household_id", + "person_family_id", + "household_id", + "household_weight", + "family_id", + "family_household_id", + "family_tax_credit_eldest_dependent_child_care_units", + "family_tax_credit_subsequent_dependent_child_care_units", + "family_tax_credit_entitlement_days", + "wff_family_scheme_income_for_relationship_period", + "wff_family_credit_abatement_days", + "entitled_to_in_work_tax_credit", + "in_work_tax_credit_allowed_children_count", + "in_work_tax_credit_weekly_periods", + "child_tax_credit_for_entitlement_period", + "parental_tax_credit_for_entitlement_period", + "parental_tax_credit_additional_abatement", + "best_start_abatement_days", + "best_start_child_care_fraction", + "best_start_entitlement_days", + "best_start_family_scheme_income_for_relationship_period", + "minimum_family_adjusted_income_tax_liability", + "minimum_family_amount_paid", + "minimum_family_amount_received", + "minimum_family_full_time_earner_weeks", + "minimum_family_scheme_income_attributable_to_full_time_weeks", + "minimum_family_tax_credit_weekly_periods" + ], + "forbidden": [ + "person_weight", + "family_weight" + ], + "optional": [ + "age", + "sex", + "region_code", + "support_stratum", + "donor_country_code" + ], + "formula_owned_excluded": [ + "budget_2025_wff_abatement_baseline_total_credit", + "budget_2025_wff_abatement_reform_total_credit", + "budget_2025_wff_abatement_entitlement_change", + "budget_2026_iwtc_baseline_total_credit", + "budget_2026_iwtc_reform_total_credit", + "budget_2026_iwtc_entitlement_change" + ], + "closed": true +} diff --git a/packages/microcosm-build/src/microcosm/build/nz/gates.json b/packages/microcosm-build/src/microcosm/build/nz/gates.json new file mode 100644 index 000000000..4659bea22 --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/nz/gates.json @@ -0,0 +1,136 @@ +{ + "version": 2, + "country": "nz", + "policy": "New Zealand is greenfield and has no incumbent population, so parity, export_surface, and target_surface are absent. Public NZ facts and Treasury oracles govern the build. Gates with not_applicable are explicit receipts for missing country policy or evidence, never borrowed Belgium/US thresholds; active identity, reference, support, weights, nonzero, formula-ownership, and nonnegative gates fail closed.", + "phases": [ + "terminal" + ], + "gates": [ + { + "id": "calibration_per_family_fit", + "gate": "per_family_fit", + "phase": "terminal", + "criticality": "release_blocking", + "not_applicable": "Chronicle has not yet supplied the complete cell-expanded NZ target inventory, covariance, or steward-reviewed per-family tolerances. Belgium and shared runner defaults may not silently govern New Zealand.", + "notes": "Activate with explicit NZ policy after the target harvest. Every hard family must report separately so an easy demographic family cannot hide a weak income or programme family." + }, + { + "id": "national_and_region_admin_aggregates", + "gate": "aggregate_admin", + "phase": "terminal", + "criticality": "release_blocking", + "not_applicable": "The complete ledger-nz fact rows, projection basis to tax year 2026-27, uncertainty, and reviewed fact-specific tolerances are not yet bound.", + "notes": "Activate at country and region levels only after every hard reference resolves. Consume table leaves or totals, never both as independent targets." + }, + { + "id": "calibration_reference_inventory", + "gate": "calibration_reference_coverage", + "phase": "terminal", + "criticality": "release_blocking", + "parameters": {}, + "evidence_absent_blocks": true, + "notes": "Every table-level authoring contract must be replaced by generated cell-pinned Chronicle references with direct, prepared, or engine-derived bindings; every activated reference must enter the matrix." + }, + { + "id": "target_profile_coverage", + "gate": "target_profile_coverage", + "phase": "terminal", + "criticality": "release_blocking", + "parameters": { + "required_families": [ + "demography", + "household_structure", + "housing_tenure", + "income", + "earnings", + "main_benefits", + "accommodation_supplement", + "superannuation", + "working_for_families", + "student_loans", + "equity" + ] + }, + "evidence_absent_blocks": true, + "notes": "The active national profile must retain every declared v0 calibration family. Treasury full-entitlement model outputs and official Budget reform scores are validation oracles, never calibration targets." + }, + { + "id": "release_input_coverage", + "gate": "release_input_coverage", + "phase": "terminal", + "criticality": "release_blocking", + "parameters": {}, + "evidence_absent_blocks": true, + "notes": "Authenticate the exact Build P donor artifact and close every declared Frame/Axiom input. A missing substantive WFF root input is a build failure, never an engine default." + }, + { + "id": "donor_support_bounds", + "gate": "support", + "phase": "terminal", + "criticality": "release_blocking", + "parameters": {}, + "evidence_absent_blocks": true, + "notes": "Every direct or derived calibration measure and every substantive Axiom input must carry a donor-support receipt." + }, + { + "id": "calibration_weights_audit", + "gate": "weights_audit", + "phase": "terminal", + "criticality": "release_blocking", + "parameters": {}, + "evidence_absent_blocks": true, + "notes": "Household weights must receipt their design-to-calibrated transition and destination-population mass change. Family scoring resolves those weights through Frame; it may not persist a second ambiguous vector." + }, + { + "id": "exported_nonzero", + "gate": "exported_nonzero", + "phase": "terminal", + "criticality": "release_blocking", + "parameters": {}, + "evidence_absent_blocks": true, + "notes": "No all-zero stored calibration layer, substantive Axiom input, geography layer, or provenance banner may ship." + }, + { + "id": "formula_owned_export", + "gate": "formula_owned_export", + "phase": "terminal", + "criticality": "release_blocking", + "parameters": {}, + "evidence_absent_blocks": true, + "notes": "No Axiom-derived rule output may persist as an input column and mask a baseline or reform calculation." + }, + { + "id": "nonnegative_columns", + "gate": "nonnegative_columns", + "phase": "terminal", + "criticality": "release_blocking", + "parameters": {}, + "evidence_absent_blocks": true, + "notes": "The source manifest is authoritative for physically non-negative weights, counts, days, periods, care units, and currency inputs." + }, + { + "id": "weight_effective_sample_size", + "gate": "weight_ess", + "phase": "terminal", + "criticality": "release_blocking", + "not_applicable": "No ex-ante steward-reviewed NZ effective-sample-size floor or shared country-neutral evaluator binding has been supplied.", + "notes": "Activate before a candidate release; never set the threshold from the candidate to make it pass." + }, + { + "id": "weight_max_to_median_ratio", + "gate": "weight_ratio", + "phase": "terminal", + "criticality": "release_blocking", + "not_applicable": "No ex-ante steward-reviewed NZ max-to-median weight threshold or shared country-neutral evaluator binding has been supplied.", + "notes": "Activate before a candidate release; do not copy an empirical US, UK, Belgium, or Armenia fence." + }, + { + "id": "macro_realism_nz", + "gate": "macro_realism", + "phase": "terminal", + "criticality": "release_blocking", + "not_applicable": "The Stats NZ national-accounts quantities, destination-unit candidate metrics, reviewed concept bridges, and low/high NZ bands are not yet packaged.", + "notes": "Activate broad income and expenditure backstops after evidence review. Treasury DistributionExplorer remains an external model comparator, not a macro target." + } + ] +} diff --git a/packages/microcosm-build/src/microcosm/build/nz/geography_spine.json b/packages/microcosm-build/src/microcosm/build/nz/geography_spine.json new file mode 100644 index 000000000..5e9d39b1f --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/nz/geography_spine.json @@ -0,0 +1,23 @@ +{ + "version": 1, + "country": "nz", + "policy": "One national New Zealand dataset carries Stats NZ region codes for calibration and filtering. Region is target-assigned on US donor support records, never represented as observed donor geography. Territorial-authority and SA2 assignment are deliberately out of scope for the first national build; future local layers remain columns on this same national dataset, not separate files.", + "geography_spine": { + "stage": "assign_nz_regions", + "method": "clone_assign_uniform", + "geography_level": "region", + "code_system": "nz_stats_region", + "code_column": "region_code", + "vintage": "2025", + "vintage_policy": "error", + "clones_per_record": 1, + "collision_avoidance": true, + "constrain_to_column": "", + "assignment_source": { + "survey": "Stats NZ subnational population estimates at 30 June 2025", + "source": "https://www.stats.govt.nz/information-releases/subnational-population-estimates-at-30-june-2025/", + "notes": "Assign one region at household grain from the harvested age-by-sex-by-region population cells, then broadcast it to family and person tables. Chronicle must supply the complete current region roster and cell-level facts before the operator can run." + }, + "notes": "One clone per record is the compile-safe national starting point. The shared clone-and-assign operator, exact 16-region roster, geography source artifact identity, and reviewed support diagnostics must land before execution." + } +} diff --git a/packages/microcosm-build/src/microcosm/build/nz/release_contract.json b/packages/microcosm-build/src/microcosm/build/nz/release_contract.json new file mode 100644 index 000000000..ca0ed43ce --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/nz/release_contract.json @@ -0,0 +1,54 @@ +{ + "version": 1, + "country": "nz", + "policy": "New Zealand v0 is a public synthetic transport artifact: its unit records derive only from the public Microcosm US support release and its calibration facts are public New Zealand aggregates. Every record and public presentation must retain the US-donor/support-stratum provenance banner and must never be described as observed New Zealand microdata. Releases use data_build_id plus immutable Hugging Face revision, never ordinal versions.", + "builder": "populace-nz", + "hf": { + "artifact_repo": "policyengine/populace-nz", + "private": false, + "staging_repo": "policyengine/populace-nz-staging" + }, + "release_manifest_country": { + "code": "nz", + "label": "New Zealand", + "geography_id": null, + "geography_label": "New Zealand", + "repository_visibility": "public", + "capabilities": [ + "calibration", + "targets", + "compare" + ] + }, + "dataset_filename_template": "populace_nz_{year}.h5", + "required_release_files": [ + "populace_nz_{year}.h5", + "populace_nz_{year}_calibration.npz", + "build_manifest.json", + "release_manifest.json", + "calibration_diagnostics.json", + "source_coverage.json", + "demographics.json", + "reform_validation.json" + ], + "boundary": { + "private": [], + "public": [ + "populace_nz_{year}.h5", + "populace_nz_{year}_calibration.npz", + "build_manifest.json", + "release_manifest.json", + "calibration_diagnostics.json", + "source_coverage.json", + "demographics.json", + "reform_validation.json", + "latest.json" + ] + }, + "licence": { + "name": "Microcosm US public synthetic release plus public NZ administrative aggregates", + "restricted": false, + "notes": "The donor artifact is the immutable public Build P release pinned in source_stages.json. Chronicle owns per-source licence evidence for each NZ aggregate. Any future HES CURF, IDI, or other restricted unit-record contribution requires a separate private-repository and destination-guard review before it enters this contract." + }, + "notes": "The target dataset period is tax year 2026-27, represented by 2026 in the current single-year kernel. The generic release assembler must copy release_manifest_country to release_manifest.json.country exactly. The dashboard registry reads policyengine/populace-nz/latest.json; the staging repository is for build telemetry. calibration_diagnostics.json is the published target-fit surface and reform_validation.json carries the Treasury official-score comparisons with explicit entitlement-to-operating-cost bridges. The two Hugging Face repositories and generic NZ release assembler do not yet exist, so this contract authorizes no publication by itself." +} diff --git a/packages/microcosm-build/src/microcosm/build/nz/source_stages.json b/packages/microcosm-build/src/microcosm/build/nz/source_stages.json new file mode 100644 index 000000000..879bb0941 --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/nz/source_stages.json @@ -0,0 +1,161 @@ +{ + "version": 1, + "country": "nz", + "policy": "New Zealand v0 transports the immutable public Microcosm US support artifact into a destination Frame. Donor records remain US support records, preserve their support-stratum provenance, and are never represented as observed New Zealand microdata. Only the Axiom inputs declared by the reviewed rulespec-nz transport contract may cross the rules boundary; missing substantive inputs fail closed and no country-specific Python runtime is permitted.", + "stages": [ + { + "stage": "load_populace_us_support_pool", + "survey": "Microcosm US Build P support population (US donor support records)", + "source": "https://huggingface.co/datasets/policyengine/populace-us", + "grain": "person", + "artifacts": [ + { + "kind": "public_microdata", + "format": "hdf5", + "vintage": "2024", + "locator": "populace_us_2024.h5", + "revision": "populace-us-2024-buildp-sparse-rmloss100-cae8640-20260728T011454Z", + "repository": { + "provider": "huggingface", + "owner": "policyengine", + "name_parts": [ + "populace-us" + ], + "repo_type": "dataset" + }, + "sha256": "48b9d479fb4fd1c3537f9383ce4697d130b6f618658409d74f6233c43b994c7e", + "size_bytes": 462915783, + "integrity_note": "The revision is the immutable release tag named by the public latest.json pointer on 2026-08-29. Execution must verify both SHA-256 and byte size before reading; main/latest fallback is forbidden." + } + ], + "operations": [ + { + "kind": "read_tables", + "tables": [ + "person", + "household" + ], + "weight": "household_weight" + } + ], + "outputs": [ + "person_id", + "household_id", + "person_household_id", + "household_weight", + "support_stratum", + "donor_country_code", + "age", + "sex" + ], + "nonnegative_outputs": [ + "household_weight", + "age" + ], + "notes": "This pins an authenticated donor artifact, not an executable NZ loader. The generic loader must preserve Frame entity ids, membership, typed household design weights, and support strata; set donor_country_code to US; refuse unreceipted aliases; and convert donor mass to a Chronicle-supplied NZ population total with a mass-conservation receipt. The typed operation vocabulary cannot represent that destination-mass bridge without inventing its still-missing target value, so execution remains blocked." + }, + { + "stage": "prepare_nz_wff_axiom_inputs", + "survey": "rulespec-nz Working for Families transport contract", + "source": "https://github.com/TheAxiomFoundation/rulespec-nz", + "grain": "family", + "artifacts": [ + { + "kind": "public_parameter_reference", + "role": "rules_engine_contract", + "resource": "rulespec-nz@3b663b3e6eb6408351154990be0c4b92d42c92da:data/microsimulation/official-budget-reform-transport.json", + "format": "rulespec_json" + }, + { + "kind": "public_parameter_reference", + "role": "rules_engine_module", + "resource": "rulespec-nz@3b663b3e6eb6408351154990be0c4b92d42c92da:nz/policies/budget/official_budget_reform_replication.yaml", + "format": "rulespec_yaml" + }, + { + "kind": "public_parameter_reference", + "role": "rules_engine_program", + "resource": "rulespec-nz@3b663b3e6eb6408351154990be0c4b92d42c92da:nz/programs/official_budget_reform_replication.yaml", + "format": "rulespec_yaml" + } + ], + "operations": [ + { + "kind": "materialize_rules_engine_predictors", + "predictors": [ + "family_tax_credit_eldest_dependent_child_care_units", + "family_tax_credit_subsequent_dependent_child_care_units", + "family_tax_credit_entitlement_days", + "wff_family_scheme_income_for_relationship_period", + "wff_family_credit_abatement_days", + "entitled_to_in_work_tax_credit", + "in_work_tax_credit_allowed_children_count", + "in_work_tax_credit_weekly_periods", + "child_tax_credit_for_entitlement_period", + "parental_tax_credit_for_entitlement_period", + "parental_tax_credit_additional_abatement", + "best_start_abatement_days", + "best_start_child_care_fraction", + "best_start_entitlement_days", + "best_start_family_scheme_income_for_relationship_period", + "minimum_family_adjusted_income_tax_liability", + "minimum_family_amount_paid", + "minimum_family_amount_received", + "minimum_family_full_time_earner_weeks", + "minimum_family_scheme_income_attributable_to_full_time_weeks", + "minimum_family_tax_credit_weekly_periods" + ], + "consumed_only": true + } + ], + "outputs": [ + "family_id", + "family_household_id", + "person_family_id", + "family_tax_credit_eldest_dependent_child_care_units", + "family_tax_credit_subsequent_dependent_child_care_units", + "family_tax_credit_entitlement_days", + "wff_family_scheme_income_for_relationship_period", + "wff_family_credit_abatement_days", + "entitled_to_in_work_tax_credit", + "in_work_tax_credit_allowed_children_count", + "in_work_tax_credit_weekly_periods", + "child_tax_credit_for_entitlement_period", + "parental_tax_credit_for_entitlement_period", + "parental_tax_credit_additional_abatement", + "best_start_abatement_days", + "best_start_child_care_fraction", + "best_start_entitlement_days", + "best_start_family_scheme_income_for_relationship_period", + "minimum_family_adjusted_income_tax_liability", + "minimum_family_amount_paid", + "minimum_family_amount_received", + "minimum_family_full_time_earner_weeks", + "minimum_family_scheme_income_attributable_to_full_time_weeks", + "minimum_family_tax_credit_weekly_periods" + ], + "nonnegative_outputs": [ + "family_tax_credit_eldest_dependent_child_care_units", + "family_tax_credit_subsequent_dependent_child_care_units", + "family_tax_credit_entitlement_days", + "wff_family_credit_abatement_days", + "in_work_tax_credit_allowed_children_count", + "in_work_tax_credit_weekly_periods", + "child_tax_credit_for_entitlement_period", + "parental_tax_credit_for_entitlement_period", + "parental_tax_credit_additional_abatement", + "best_start_abatement_days", + "best_start_child_care_fraction", + "best_start_entitlement_days", + "best_start_family_scheme_income_for_relationship_period", + "minimum_family_adjusted_income_tax_liability", + "minimum_family_amount_paid", + "minimum_family_amount_received", + "minimum_family_full_time_earner_weeks", + "minimum_family_scheme_income_attributable_to_full_time_weeks", + "minimum_family_tax_credit_weekly_periods" + ], + "notes": "The transport contract carries 21 inputs from the imported module; the current Family-root dense batch requests 17 of them. Eleven target-dependent inputs have no default and must be transported from reviewed donor or derived evidence. Only the ten Best Start and Minimum Family Tax Credit fields may be adapter-padding zeros because they lie outside both requested Budget reform outputs' dependency graphs. The generic binder must create family_id, family_household_id, and person_family_id; refuse ambiguous or cross-household families; and invoke the real AxiomEngine. Pinned receipts: contract SHA-256 cd650c191a8a98faedf2bc45c80b6026eb06037b3357d9af37d25ee1dd460dfa; module SHA-256 fbbc936a6a14b02b1301658c91075def93a5481184ed73fa3fd0cfb06608bf69; program SHA-256 c041310945a3d35e1392262162030497b43ddfca63f4fc4a90deb19fe9044868; Axiom Engine v0.2.2 commit bb4b5684870547756078a62f1866a77c5b56f7f3. Household is the sole explicit weighted entity; family accounting uses Frame.resolve_weights(\"family\") and never persists family_weight. Automatic NZ unit derivation remains blocked because the current upstream unit-aggregation CLI golden fixture fails and the existing Belgium adapter does not run that stage. Source activation remains pending immutable Treasury corpus publication and protected validation; a passing runtime smoke is not that source-release receipt. Decimal inputs cross the native float64 ABI only within the adapter's proven exact nine-place recovery boundary. The scaffold's input-preparation operation is a dependency contract, not an implemented family-input binder." + } + ] +} diff --git a/packages/microcosm-build/src/microcosm/build/nz/spec/bundle.yaml b/packages/microcosm-build/src/microcosm/build/nz/spec/bundle.yaml new file mode 100644 index 000000000..41a0f8ae0 --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/nz/spec/bundle.yaml @@ -0,0 +1,16 @@ +country: nz +identity_generation: 1 +seed_protocol: legacy-v1 +dataset_run: + target_period: 2026 +status: >- + New Zealand public-donor transport walking skeleton for tax year 2026-27. + The exact Microcosm US Build P donor is pinned, and the rules boundary names + the reviewed 21-input Axiom transport contract at rulespec-nz commit 3b663b3. + Execution still requires cell-expanded + Chronicle facts with a reviewed 2026 basis, shared donor/family/geography + operator bindings, the eleven evidenced Family input bridges, and country + calibration thresholds. The upstream CLI unit-aggregation fixture is not + green, so automatic unit derivation is not claimed. A tiny real-Axiom + Frame smoke passes, but Treasury source activation still needs immutable + corpus publication. No runtime or release may substitute stubs. diff --git a/packages/microcosm-build/src/microcosm/build/nz/spec/catalogs.yaml b/packages/microcosm-build/src/microcosm/build/nz/spec/catalogs.yaml new file mode 100644 index 000000000..1443599bd --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/nz/spec/catalogs.yaml @@ -0,0 +1,331 @@ +columns: +- key: household.household_id + contract: + entity: household + dtype: int64 + unit: count + definition_period: eternity + vintage: vintage:nz_target_2026 + nullable: false + domain: frame_identity + public_stability: internal +- key: household.household_weight + contract: + entity: household + dtype: float64 + unit: count + definition_period: year + vintage: vintage:nz_target_2026 + nullable: false + domain: frame_weight + public_stability: internal +- key: family.family_id + contract: + entity: family + dtype: int64 + unit: count + definition_period: eternity + vintage: vintage:nz_target_2026 + nullable: false + domain: frame_identity + public_stability: internal +- key: family.family_household_id + contract: + entity: family + dtype: int64 + unit: count + definition_period: eternity + vintage: vintage:nz_target_2026 + nullable: false + domain: frame_membership + public_stability: internal +- key: person.person_id + contract: + entity: person + dtype: int64 + unit: count + definition_period: eternity + vintage: vintage:nz_target_2026 + nullable: false + domain: frame_identity + public_stability: internal +- key: person.person_household_id + contract: + entity: person + dtype: int64 + unit: count + definition_period: eternity + vintage: vintage:nz_target_2026 + nullable: false + domain: frame_membership + public_stability: internal +- key: person.person_family_id + contract: + entity: person + dtype: int64 + unit: count + definition_period: eternity + vintage: vintage:nz_target_2026 + nullable: false + domain: frame_membership + public_stability: internal +- key: person.support_stratum + contract: + entity: person + dtype: category + unit: categorical + definition_period: eternity + vintage: vintage:nz_support_pool_2024_buildp + nullable: false + domain: support_provenance + public_stability: internal +- key: person.donor_country_code + contract: + entity: person + dtype: category + unit: categorical + definition_period: eternity + vintage: vintage:nz_support_pool_2024_buildp + nullable: false + domain: support_provenance + public_stability: internal +- key: person.age + contract: + entity: person + dtype: int64 + unit: count + definition_period: year + vintage: vintage:nz_target_2026 + nullable: false + domain: donor_demography + public_stability: internal +- key: person.sex + contract: + entity: person + dtype: category + unit: categorical + definition_period: year + vintage: vintage:nz_target_2026 + nullable: false + domain: donor_demography + public_stability: internal +- key: person.region_code + contract: + entity: person + dtype: category + unit: categorical + definition_period: year + vintage: vintage:nz_region_2025 + nullable: false + domain: target_assigned_geography + public_stability: internal +- key: family.family_tax_credit_eldest_dependent_child_care_units + contract: + entity: family + dtype: decimal128(18,2) + unit: count + definition_period: year + vintage: vintage:nz_target_2026 + nullable: false + domain: axiom_required_input + public_stability: internal +- key: family.family_tax_credit_subsequent_dependent_child_care_units + contract: + entity: family + dtype: decimal128(18,2) + unit: count + definition_period: year + vintage: vintage:nz_target_2026 + nullable: false + domain: axiom_required_input + public_stability: internal +- key: family.family_tax_credit_entitlement_days + contract: + entity: family + dtype: int16 + unit: count + definition_period: year + vintage: vintage:nz_target_2026 + nullable: false + domain: axiom_required_input + public_stability: internal +- key: family.wff_family_scheme_income_for_relationship_period + contract: + entity: family + dtype: decimal128(18,2) + unit: nzd + definition_period: year + vintage: vintage:nz_target_2026 + nullable: false + domain: axiom_required_input + public_stability: internal +- key: family.wff_family_credit_abatement_days + contract: + entity: family + dtype: int16 + unit: count + definition_period: year + vintage: vintage:nz_target_2026 + nullable: false + domain: axiom_required_input + public_stability: internal +- key: family.entitled_to_in_work_tax_credit + contract: + entity: family + dtype: bool + unit: boolean + definition_period: year + vintage: vintage:nz_target_2026 + nullable: false + domain: axiom_required_input + public_stability: internal +- key: family.in_work_tax_credit_allowed_children_count + contract: + entity: family + dtype: int16 + unit: count + definition_period: year + vintage: vintage:nz_target_2026 + nullable: false + domain: axiom_required_input + public_stability: internal +- key: family.in_work_tax_credit_weekly_periods + contract: + entity: family + dtype: int16 + unit: count + definition_period: year + vintage: vintage:nz_target_2026 + nullable: false + domain: axiom_required_input + public_stability: internal +- key: family.child_tax_credit_for_entitlement_period + contract: + entity: family + dtype: decimal128(18,2) + unit: nzd + definition_period: year + vintage: vintage:nz_target_2026 + nullable: false + domain: axiom_required_input + public_stability: internal +- key: family.parental_tax_credit_for_entitlement_period + contract: + entity: family + dtype: decimal128(18,2) + unit: nzd + definition_period: year + vintage: vintage:nz_target_2026 + nullable: false + domain: axiom_required_input + public_stability: internal +- key: family.parental_tax_credit_additional_abatement + contract: + entity: family + dtype: decimal128(18,2) + unit: nzd + definition_period: year + vintage: vintage:nz_target_2026 + nullable: false + domain: axiom_required_input + public_stability: internal +- key: family.best_start_abatement_days + contract: + entity: family + dtype: int64 + unit: count + definition_period: year + vintage: vintage:nz_target_2026 + nullable: false + domain: axiom_adapter_padding_zero + public_stability: internal +- key: family.best_start_child_care_fraction + contract: + entity: family + dtype: float64 + unit: ratio + definition_period: year + vintage: vintage:nz_target_2026 + nullable: false + domain: axiom_adapter_padding_zero + public_stability: internal +- key: family.best_start_entitlement_days + contract: + entity: family + dtype: int64 + unit: count + definition_period: year + vintage: vintage:nz_target_2026 + nullable: false + domain: axiom_adapter_padding_zero + public_stability: internal +- key: family.best_start_family_scheme_income_for_relationship_period + contract: + entity: family + dtype: float64 + unit: nzd + definition_period: year + vintage: vintage:nz_target_2026 + nullable: false + domain: axiom_adapter_padding_zero + public_stability: internal +- key: family.minimum_family_adjusted_income_tax_liability + contract: + entity: family + dtype: float64 + unit: nzd + definition_period: year + vintage: vintage:nz_target_2026 + nullable: false + domain: axiom_adapter_padding_zero + public_stability: internal +- key: family.minimum_family_amount_paid + contract: + entity: family + dtype: float64 + unit: nzd + definition_period: year + vintage: vintage:nz_target_2026 + nullable: false + domain: axiom_adapter_padding_zero + public_stability: internal +- key: family.minimum_family_amount_received + contract: + entity: family + dtype: float64 + unit: nzd + definition_period: year + vintage: vintage:nz_target_2026 + nullable: false + domain: axiom_adapter_padding_zero + public_stability: internal +- key: family.minimum_family_full_time_earner_weeks + contract: + entity: family + dtype: int64 + unit: count + definition_period: year + vintage: vintage:nz_target_2026 + nullable: false + domain: axiom_adapter_padding_zero + public_stability: internal +- key: family.minimum_family_scheme_income_attributable_to_full_time_weeks + contract: + entity: family + dtype: float64 + unit: nzd + definition_period: year + vintage: vintage:nz_target_2026 + nullable: false + domain: axiom_adapter_padding_zero + public_stability: internal +- key: family.minimum_family_tax_credit_weekly_periods + contract: + entity: family + dtype: int64 + unit: count + definition_period: year + vintage: vintage:nz_target_2026 + nullable: false + domain: axiom_adapter_padding_zero + public_stability: internal diff --git a/packages/microcosm-build/src/microcosm/build/nz/spec/geography.yaml b/packages/microcosm-build/src/microcosm/build/nz/spec/geography.yaml new file mode 100644 index 000000000..af173ae1b --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/nz/spec/geography.yaml @@ -0,0 +1,29 @@ +phase: legacy +status: >- + Contract-only expression of region assignment for the national NZ build. + The complete 2025 Stats NZ region roster and cell facts remain Chronicle + inputs, and the shared executable assignment operator is not yet bound. + Region is target-assigned on US donor records, never source-observed. +assignment: + anchor: region + order: before_gap_fill + kernels: + assign: kernel:assign_nz_regions + validate: kernel:nz_region_geography_gate + draw: + region: + universe: region_nationally + weight: stats_nz_population + derive: + - region_code + assertions: + - assigned_region_complete + - geography_vintage_exact + ladder_source: source:nz_region_assignment_contract + seed: stream:geography_legacy + default_seed: 0 + layer_vintages: + region: vintage:nz_region_2025 + validation: + - region_assignment_support + - vintage_refusal diff --git a/packages/microcosm-build/src/microcosm/build/nz/spec/sources.yaml b/packages/microcosm-build/src/microcosm/build/nz/spec/sources.yaml new file mode 100644 index 000000000..0d1412fd3 --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/nz/spec/sources.yaml @@ -0,0 +1,197 @@ +sources: +- id: nz_populace_us_support_contract + role: nz_populace_us_support_pool + sha256: 1b9e3267730774d39008f1ef7d73ff943abc43acc12adcd446e02d6621c6440b + byte_size: 8873 + loader: kernel:load_populace_us_support_pool + vintages: + - vintage:nz_support_pool_2024_buildp + vintage_authorities: + - id: nz_support_pool_2024_buildp + kind: survey_period + value: '2024' +- id: nz_wff_axiom_transport_contract + role: nz_wff_axiom_family_transport_contract + sha256: 1b9e3267730774d39008f1ef7d73ff943abc43acc12adcd446e02d6621c6440b + byte_size: 8873 + loader: kernel:prepare_nz_wff_axiom_inputs + vintages: + - vintage:nz_rulespec_wff_2026 + vintage_authorities: + - id: nz_rulespec_wff_2026 + kind: tax_period + value: '2026-04-01' +- id: nz_region_assignment_contract + role: nz_region_assignment_contract + sha256: fe248a59c11a2a7ee275ca9a32a58feb4c0b2bc6f873b446caf312bc16513e36 + byte_size: 1520 + loader: kernel:assign_nz_regions + vintages: + - vintage:nz_region_2025 + - vintage:nz_region_roster_2025 + vintage_authorities: + - id: nz_region_2025 + kind: geography_vintage + value: 2025_region + - id: nz_region_roster_2025 + kind: geography_vintage + value: 2025_region_roster +stage_manifest: + version: 1 + country: nz + policy: New Zealand v0 transports the immutable public Microcosm US support artifact into + a destination Frame. Donor records remain US support records, preserve their support-stratum + provenance, and are never represented as observed New Zealand microdata. Only the Axiom + inputs declared by the reviewed rulespec-nz transport contract may cross the rules boundary; + missing substantive inputs fail closed and no country-specific Python runtime is permitted. +stages: +- stage: load_populace_us_support_pool + survey: Microcosm US Build P support population (US donor support records) + source: https://huggingface.co/datasets/policyengine/populace-us + grain: person + artifacts: + - kind: public_microdata + format: hdf5 + vintage: '2024' + locator: populace_us_2024.h5 + revision: populace-us-2024-buildp-sparse-rmloss100-cae8640-20260728T011454Z + repository: + provider: huggingface + owner: policyengine + name_parts: + - populace-us + repo_type: dataset + sha256: 48b9d479fb4fd1c3537f9383ce4697d130b6f618658409d74f6233c43b994c7e + size_bytes: 462915783 + integrity_note: The revision is the immutable release tag named by the public latest.json + pointer on 2026-08-29. Execution must verify both SHA-256 and byte size before reading; + main/latest fallback is forbidden. + operations: + - kind: read_tables + tables: + - person + - household + weight: household_weight + outputs: + - person_id + - household_id + - person_household_id + - household_weight + - support_stratum + - donor_country_code + - age + - sex + nonnegative_outputs: + - household_weight + - age + notes: This pins an authenticated donor artifact, not an executable NZ loader. The generic + loader must preserve Frame entity ids, membership, typed household design weights, and + support strata; set donor_country_code to US; refuse unreceipted aliases; and convert + donor mass to a Chronicle-supplied NZ population total with a mass-conservation receipt. + The typed operation vocabulary cannot represent that destination-mass bridge without inventing + its still-missing target value, so execution remains blocked. +- stage: prepare_nz_wff_axiom_inputs + survey: rulespec-nz Working for Families transport contract + source: https://github.com/TheAxiomFoundation/rulespec-nz + grain: family + artifacts: + - kind: public_parameter_reference + role: rules_engine_contract + resource: rulespec-nz@3b663b3e6eb6408351154990be0c4b92d42c92da:data/microsimulation/official-budget-reform-transport.json + format: rulespec_json + - kind: public_parameter_reference + role: rules_engine_module + resource: rulespec-nz@3b663b3e6eb6408351154990be0c4b92d42c92da:nz/policies/budget/official_budget_reform_replication.yaml + format: rulespec_yaml + - kind: public_parameter_reference + role: rules_engine_program + resource: rulespec-nz@3b663b3e6eb6408351154990be0c4b92d42c92da:nz/programs/official_budget_reform_replication.yaml + format: rulespec_yaml + operations: + - kind: materialize_rules_engine_predictors + predictors: + - family_tax_credit_eldest_dependent_child_care_units + - family_tax_credit_subsequent_dependent_child_care_units + - family_tax_credit_entitlement_days + - wff_family_scheme_income_for_relationship_period + - wff_family_credit_abatement_days + - entitled_to_in_work_tax_credit + - in_work_tax_credit_allowed_children_count + - in_work_tax_credit_weekly_periods + - child_tax_credit_for_entitlement_period + - parental_tax_credit_for_entitlement_period + - parental_tax_credit_additional_abatement + - best_start_abatement_days + - best_start_child_care_fraction + - best_start_entitlement_days + - best_start_family_scheme_income_for_relationship_period + - minimum_family_adjusted_income_tax_liability + - minimum_family_amount_paid + - minimum_family_amount_received + - minimum_family_full_time_earner_weeks + - minimum_family_scheme_income_attributable_to_full_time_weeks + - minimum_family_tax_credit_weekly_periods + consumed_only: true + outputs: + - family_id + - family_household_id + - person_family_id + - family_tax_credit_eldest_dependent_child_care_units + - family_tax_credit_subsequent_dependent_child_care_units + - family_tax_credit_entitlement_days + - wff_family_scheme_income_for_relationship_period + - wff_family_credit_abatement_days + - entitled_to_in_work_tax_credit + - in_work_tax_credit_allowed_children_count + - in_work_tax_credit_weekly_periods + - child_tax_credit_for_entitlement_period + - parental_tax_credit_for_entitlement_period + - parental_tax_credit_additional_abatement + - best_start_abatement_days + - best_start_child_care_fraction + - best_start_entitlement_days + - best_start_family_scheme_income_for_relationship_period + - minimum_family_adjusted_income_tax_liability + - minimum_family_amount_paid + - minimum_family_amount_received + - minimum_family_full_time_earner_weeks + - minimum_family_scheme_income_attributable_to_full_time_weeks + - minimum_family_tax_credit_weekly_periods + nonnegative_outputs: + - family_tax_credit_eldest_dependent_child_care_units + - family_tax_credit_subsequent_dependent_child_care_units + - family_tax_credit_entitlement_days + - wff_family_credit_abatement_days + - in_work_tax_credit_allowed_children_count + - in_work_tax_credit_weekly_periods + - child_tax_credit_for_entitlement_period + - parental_tax_credit_for_entitlement_period + - parental_tax_credit_additional_abatement + - best_start_abatement_days + - best_start_child_care_fraction + - best_start_entitlement_days + - best_start_family_scheme_income_for_relationship_period + - minimum_family_adjusted_income_tax_liability + - minimum_family_amount_paid + - minimum_family_amount_received + - minimum_family_full_time_earner_weeks + - minimum_family_scheme_income_attributable_to_full_time_weeks + - minimum_family_tax_credit_weekly_periods + notes: 'The transport contract carries 21 inputs from the imported module; the current Family-root + dense batch requests 17 of them. Eleven target-dependent inputs have no default and must + be transported from reviewed donor or derived evidence. Only the ten Best Start and Minimum + Family Tax Credit fields may be adapter-padding zeros because they lie outside both requested + Budget reform outputs'' dependency graphs. The generic binder must create family_id, family_household_id, + and person_family_id; refuse ambiguous or cross-household families; and invoke the real + AxiomEngine. Pinned receipts: contract SHA-256 cd650c191a8a98faedf2bc45c80b6026eb06037b3357d9af37d25ee1dd460dfa; + module SHA-256 fbbc936a6a14b02b1301658c91075def93a5481184ed73fa3fd0cfb06608bf69; program + SHA-256 c041310945a3d35e1392262162030497b43ddfca63f4fc4a90deb19fe9044868; Axiom Engine + v0.2.2 commit bb4b5684870547756078a62f1866a77c5b56f7f3. Household is the sole explicit + weighted entity; family accounting uses Frame.resolve_weights("family") and never persists + family_weight. Automatic NZ unit derivation remains blocked because the current upstream + unit-aggregation CLI golden fixture fails and the existing Belgium adapter does not run + that stage. Source activation remains pending immutable Treasury corpus publication and + protected validation; a passing runtime smoke is not that source-release receipt. Decimal + inputs cross the native float64 ABI only within the adapter''s proven exact nine-place + recovery boundary. The scaffold''s input-preparation operation is a dependency contract, + not an implemented family-input binder.' diff --git a/packages/microcosm-build/src/microcosm/build/nz/spec/spine.yaml b/packages/microcosm-build/src/microcosm/build/nz/spec/spine.yaml new file mode 100644 index 000000000..9b3fca58b --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/nz/spec/spine.yaml @@ -0,0 +1,10 @@ +channels: +- id: populace_us_support + source: nz_populace_us_support_contract + observed_geography: country +assembly: + mass_anchor_channel: populace_us_support + shared_dtype_policy: canonical_string_storage +support_roles: +- id: populace_us_donor_base + kind: none diff --git a/packages/microcosm-build/src/microcosm/build/nz/spec/vintages.yaml b/packages/microcosm-build/src/microcosm/build/nz/spec/vintages.yaml new file mode 100644 index 000000000..ab2aa568d --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/nz/spec/vintages.yaml @@ -0,0 +1,41 @@ +records: +- id: nz_support_pool_2024_buildp + kind: survey_period_ref + authority_ref: + kind: source_record + source: source:nz_populace_us_support_contract + authority: nz_support_pool_2024_buildp + compatible_with: + - vintage:nz_target_2026 +- id: nz_target_2026 + kind: target_period_ref + authority_ref: + kind: dataset_run + pointer: /dataset_run/target_period + compatible_with: + - vintage:nz_support_pool_2024_buildp + - vintage:nz_rulespec_wff_2026 +- id: nz_region_2025 + kind: geography_vintage_ref + authority_ref: + kind: source_record + source: source:nz_region_assignment_contract + authority: nz_region_2025 + compatible_with: + - vintage:nz_region_roster_2025 +- id: nz_region_roster_2025 + kind: geography_vintage_ref + authority_ref: + kind: source_record + source: source:nz_region_assignment_contract + authority: nz_region_roster_2025 + compatible_with: + - vintage:nz_region_2025 +- id: nz_rulespec_wff_2026 + kind: tax_period_ref + authority_ref: + kind: source_record + source: source:nz_wff_axiom_transport_contract + authority: nz_rulespec_wff_2026 + compatible_with: + - vintage:nz_target_2026 diff --git a/packages/microcosm-build/src/microcosm/build/nz/target_references.json b/packages/microcosm-build/src/microcosm/build/nz/target_references.json new file mode 100644 index 000000000..42cc19921 --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/nz/target_references.json @@ -0,0 +1,311 @@ +{ + "country": "nz", + "description": "New Zealand v0 calibration authoring contracts. Observed values and source lineage live only in Chronicle. WFF and IWTC national counts have cell-pinned identities; the remaining series-level rows require expansion. Every row needs a reviewed Frame column and period bridge before activation; no table-level placeholder is itself a calibration target. Treasury DistributionExplorer full-entitlement outputs, official Budget reform scores, and administrative expenditure totals remain external validation evidence rather than solver targets.", + "allowed_value_operations": [ + "identity" + ], + "target_references": [ + { + "name": "stats_nz_population_by_age_sex_region", + "ledger_selector": { + "source_name": "stats_nz_population_estimates", + "source_measure_id": "estimated_resident_population_by_age_sex_region", + "geography_level": "region" + }, + "entity": "person", + "measure": "people", + "period": 2025, + "family": "demography", + "metadata": { + "activation_status": "requires_harvested_cell_references_and_2026_basis", + "criticality": "release_blocking", + "geography_vintage": "2025", + "measure_kind": "prepared_column", + "publisher": "Stats NZ", + "target_role": "calibration" + }, + "notes": "Primary age-band x sex x region population surface. Chronicle must supply one dimension- and region-pinned fact per selected cell, the complete region roster, uncertainty, and a reviewed bridge from the 30 June 2025 reference date to the 2026-27 build basis." + }, + { + "name": "stats_nz_population_by_single_year_age_sex", + "ledger_selector": { + "source_name": "stats_nz_population_estimates", + "source_measure_id": "estimated_resident_population_by_single_year_age_sex", + "geography_level": "country" + }, + "entity": "person", + "measure": "people", + "period": 2025, + "family": "demography", + "metadata": { + "activation_status": "requires_harvested_cell_references_and_2026_basis", + "criticality": "release_blocking", + "measure_kind": "prepared_column", + "publisher": "Stats NZ", + "target_role": "calibration" + }, + "notes": "National single-year age x sex cells constrain the age tail. The profile must prove reconciliation with the region surface rather than double-counting inconsistent totals." + }, + { + "name": "stats_nz_census_households_by_composition", + "ledger_selector": { + "source_name": "stats_nz_census_2023_households", + "source_measure_id": "households_by_composition", + "geography_level": "country" + }, + "entity": "household", + "measure": "households", + "period": 2023, + "family": "household_structure", + "metadata": { + "activation_status": "requires_harvested_cell_references_and_2026_basis", + "criticality": "release_blocking", + "measure_kind": "prepared_column", + "publisher": "Stats NZ", + "target_role": "calibration" + }, + "notes": "Household and family composition is the structural control for WFF family assembly. Chronicle must expand the published categories and supply a reviewed 2023-to-2026 basis before activation." + }, + { + "name": "stats_nz_census_households_by_tenure_region", + "ledger_selector": { + "source_name": "stats_nz_census_2023_households", + "source_measure_id": "households_by_tenure_region", + "geography_level": "region" + }, + "entity": "household", + "measure": "households", + "filter": "housing_tenure_cell_indicator", + "period": 2023, + "family": "housing_tenure", + "metadata": { + "activation_status": "requires_harvested_cell_references_and_prepared_indicators", + "criticality": "release_blocking", + "measure_kind": "prepared_column", + "publisher": "Stats NZ", + "target_role": "calibration" + }, + "notes": "Each tenure x region cell needs its own deterministic indicator binding. Market-rent amounts are validation/input evidence for Accommodation Supplement, not household-count calibration rows." + }, + { + "name": "ird_taxable_income_people_by_band", + "ledger_selector": { + "source_name": "ird_taxable_income_distribution", + "source_measure_id": "individuals_by_taxable_income_band", + "geography_level": "country" + }, + "entity": "person", + "measure": "people", + "filter": "taxable_income_band_indicator", + "period": 2024, + "family": "income", + "metadata": { + "activation_status": "requires_harvested_cell_references_and_2026_basis", + "criticality": "release_blocking", + "measure_kind": "prepared_column", + "publisher": "Inland Revenue", + "target_role": "calibration" + }, + "notes": "Administrative count backbone. Chronicle must emit one income-band fact and reviewed prepared indicator per cell; the target basis must be advanced explicitly to the build period." + }, + { + "name": "ird_taxable_income_total_by_band", + "ledger_selector": { + "source_name": "ird_taxable_income_distribution", + "source_measure_id": "total_taxable_income_by_band_nzd", + "geography_level": "country" + }, + "entity": "person", + "measure": "taxable_income_nzd", + "filter": "taxable_income_band_indicator", + "period": 2024, + "family": "income", + "metadata": { + "activation_status": "requires_harvested_cell_references_candidate_column_and_2026_basis", + "criticality": "release_blocking", + "measure_kind": "prepared_column", + "publisher": "Inland Revenue", + "target_role": "calibration" + }, + "notes": "Dollar companion to the count bands. Activation requires a destination-NZD taxable-income column with a reviewed donor bridge; US-dollar values may never be compared directly with NZD thresholds." + }, + { + "name": "ird_wage_salary_people_by_band", + "ledger_selector": { + "source_name": "ird_wage_salary_statistics", + "source_measure_id": "individuals_by_wage_salary_band", + "geography_level": "country" + }, + "entity": "person", + "measure": "people", + "filter": "wage_salary_band_indicator", + "period": 2024, + "family": "earnings", + "metadata": { + "activation_status": "requires_harvested_cell_references_candidate_column_and_2026_basis", + "criticality": "release_blocking", + "measure_kind": "prepared_column", + "publisher": "Inland Revenue", + "target_role": "calibration" + }, + "notes": "Each wage/salary band needs a destination-NZD indicator and source-period bridge. QES average earnings is a macro diagnostic, not an additional hard total over these cells." + }, + { + "name": "msd_main_benefit_recipients_by_program_age", + "ledger_selector": { + "source_name": "msd_quarterly_benefit_facts", + "source_measure_id": "main_benefit_recipients_by_program_age", + "geography_level": "country" + }, + "entity": "person", + "measure": "receives_main_benefit", + "period": "2026-03", + "family": "main_benefits", + "metadata": { + "activation_status": "requires_harvested_cell_references_and_take_up_assignment", + "criticality": "release_blocking", + "measure_kind": "prepared_column", + "publisher": "Ministry of Social Development", + "target_role": "calibration" + }, + "notes": "Count-only administrative cells constrain an explicit receipt/take-up layer. They must never be bound to Axiom full-entitlement outputs or used to force entitlement dollars to equal actual expenditure." + }, + { + "name": "msd_accommodation_supplement_recipients_by_region", + "ledger_selector": { + "source_name": "msd_quarterly_benefit_facts", + "source_measure_id": "accommodation_supplement_recipients_by_region", + "geography_level": "region" + }, + "entity": "family", + "measure": "receives_accommodation_supplement", + "period": "2026-03", + "family": "accommodation_supplement", + "metadata": { + "activation_status": "requires_harvested_cell_references_geography_bridge_and_take_up_assignment", + "criticality": "release_blocking", + "measure_kind": "prepared_column", + "publisher": "Ministry of Social Development", + "target_role": "calibration" + }, + "notes": "MSD Work and Income regions are not Stats NZ regions. Activation requires an explicit geography bridge and a family-level receipt assignment; the ~44 percent survey estimate remains held-out validation." + }, + { + "name": "msd_nz_super_veterans_pension_recipients", + "ledger_selector": { + "source_name": "msd_nz_super_veterans_pension_facts", + "source_measure_id": "nz_super_veterans_pension_recipients", + "geography_level": "country" + }, + "entity": "person", + "measure": "receives_nz_super_or_veterans_pension", + "period": "2025-06", + "family": "superannuation", + "metadata": { + "activation_status": "requires_harvested_fact_reference_and_take_up_assignment", + "criticality": "release_blocking", + "measure_kind": "prepared_column", + "publisher": "Ministry of Social Development", + "target_role": "calibration" + }, + "notes": "Recipient stock is a receipt-state target, not an entitlement-output target. Expenditure is validation evidence and stays outside the calibration manifest." + }, + { + "name": "ird_wff_recipient_families", + "ledger_selector": { + "source_name": "ird", + "source_measure_id": "wff_recipient_families", + "geography_level": "country" + }, + "entity": "family", + "measure": "receives_working_for_families", + "period": 2024, + "family": "working_for_families", + "metadata": { + "activation_status": "requires_receipt_assignment_and_2026_basis", + "chronicle_package_id": "ird-working-for-families-statistics-sept-2025", + "criticality": "release_blocking", + "measure_kind": "prepared_column", + "publisher": "Inland Revenue", + "target_role": "calibration", + "chronicle_package_path": "ird/working_for_families_statistics_sept_2025", + "source_period_kind": "tax_year", + "source_period_start": "2023-04-01", + "source_period_end": "2024-03-31" + }, + "notes": "Cell-pinned IRD administrative count of families with a claim and non-zero final entitlement, not every eligible family and not cash paid. Chronicle owns the September 2025 workbook facts. Activation requires a separate family receipt/claim state and an explicit bridge from tax year ending 2024 to the 2026-27 build basis. Total and average entitlement NZD remain held-out validation, never calibration values.", + "ledger_source_record_id": "ird_wff_statistics.ty2024.recipient_families.all.wff_recipient_families" + }, + { + "name": "ird_iwtc_recipient_families", + "ledger_selector": { + "source_name": "ird", + "source_measure_id": "iwtc_recipient_families", + "geography_level": "country" + }, + "entity": "family", + "measure": "receives_in_work_tax_credit", + "period": 2024, + "family": "working_for_families", + "metadata": { + "activation_status": "requires_receipt_assignment_and_2026_basis", + "chronicle_package_id": "ird-working-for-families-statistics-sept-2025", + "criticality": "release_blocking", + "measure_kind": "prepared_column", + "publisher": "Inland Revenue", + "target_role": "calibration", + "chronicle_package_path": "ird/working_for_families_statistics_sept_2025", + "source_period_kind": "tax_year", + "source_period_start": "2023-04-01", + "source_period_end": "2024-03-31" + }, + "notes": "Cell-pinned IRD administrative count of families with a claim and non-zero final entitlement, not every eligible family and not cash paid. Chronicle owns the September 2025 workbook facts. Activation requires a separate family receipt/claim state and an explicit bridge from tax year ending 2024 to the 2026-27 build basis. Total and average entitlement NZD remain held-out validation, never calibration values.", + "ledger_source_record_id": "ird_wff_statistics.ty2024.recipient_families.all.iwtc_recipient_families" + }, + { + "name": "ird_student_loan_borrowers", + "ledger_selector": { + "source_name": "ird_student_loan_statistics", + "source_measure_id": "nz_based_borrowers", + "geography_level": "country" + }, + "entity": "person", + "measure": "has_student_loan_balance", + "period": 2025, + "family": "student_loans", + "metadata": { + "activation_status": "requires_harvested_fact_reference_and_borrower_assignment", + "criticality": "release_blocking", + "measure_kind": "prepared_column", + "publisher": "Inland Revenue", + "target_role": "calibration" + }, + "notes": "Borrower counts require a reviewed person-level assignment and period definition. Balance dollars are a separate validation or imputation problem." + }, + { + "name": "stats_nz_census_ethnicity_by_age_region", + "ledger_selector": { + "source_name": "stats_nz_census_2023_individuals", + "source_measure_id": "people_by_ethnicity_age_region", + "geography_level": "region" + }, + "entity": "person", + "measure": "people", + "filter": "ethnicity_age_region_cell_indicator", + "period": 2023, + "family": "equity", + "metadata": { + "activation_status": "requires_harvested_cell_references_multi_response_policy_and_2026_basis", + "criticality": "release_blocking", + "measure_kind": "prepared_column", + "publisher": "Stats NZ", + "target_role": "calibration" + }, + "notes": "Ethnicity is multiple-response, not a partition. Chronicle and the target compiler must preserve the publisher's total-response-person convention and must not force mutually exclusive assignment." + } + ], + "target_profile": { + "schema_version": 1, + "hierarchy_reconciliations": [] + } +} diff --git a/packages/microcosm-build/src/microcosm/build/spec_engine/resolver.py b/packages/microcosm-build/src/microcosm/build/spec_engine/resolver.py index eba46efc8..b252a4789 100644 --- a/packages/microcosm-build/src/microcosm/build/spec_engine/resolver.py +++ b/packages/microcosm-build/src/microcosm/build/spec_engine/resolver.py @@ -62,8 +62,7 @@ def from_ids( overlap = sorted(implemented & contract_only) if overlap: raise ValueError( - "kernel ids cannot be both implemented and contract-only: " - f"{overlap!r}" + f"kernel ids cannot be both implemented and contract-only: {overlap!r}" ) digest = hashlib.sha256(canonical_json_bytes(sorted(implemented))).hexdigest() return cls(implemented | contract_only, implemented, digest) @@ -150,7 +149,7 @@ def contract_only_ids(self) -> frozenset[str]: } ) -# UK, BE, and AM use these reviewed ids to prove that their shared-core bundle +# UK, BE, AM, and NZ use these reviewed ids to prove that their shared-core bundle # shapes resolve through the same compiler. F0 does not yet bind them into the # generic executor: the Belgian and Armenian implementations are absent, and # the existing UK functions have no compiler-owned producer binding or @@ -160,12 +159,15 @@ def contract_only_ids(self) -> frozenset[str]: { "am_community_geography_gate", "assign_am_marz", + "assign_nz_regions", "assign_uk_geography_ladder", "be_commune_geography_gate", "clone_assign_communities", "clone_assign_communes", "load_populace_us_support_pool", "load_uk_national_frame", + "nz_region_geography_gate", + "prepare_nz_wff_axiom_inputs", "silc_load", "uk_geography_ladder_gate", } @@ -648,9 +650,7 @@ def _validate_imputation_structure( # RAW dependencies have one authority: each input's producing_stage. # Global edges and equality-only depends_on arrays are compiler outputs. - predecessors = { - node_id: set(input_predecessors[node_id]) for node_id in node_ids - } + predecessors = {node_id: set(input_predecessors[node_id]) for node_id in node_ids} successors = {node_id: set() for node_id in node_ids} for consumer, producers in predecessors.items(): for producer in producers: @@ -1063,9 +1063,7 @@ def _validate_imputation_structure( producer_family_links: dict[str, tuple[int, Mapping[str, object]]] = {} for family_index, family in [*primary_families, *late_families]: family_location = f"imputation/families/{family_index}" - stage = _identifier( - family.get("stage"), location=f"{family_location}/stage" - ) + stage = _identifier(family.get("stage"), location=f"{family_location}/stage") producer_field = ( "execution_contract" if stage == "primary_puf_qrf" else "runtime_name" ) @@ -1086,8 +1084,7 @@ def _validate_imputation_structure( orphan_primary_nodes = sorted( node_id for node_id, node in nodes_by_id.items() - if node.get("kind") == "primary_puf" - and node_id not in producer_family_links + if node.get("kind") == "primary_puf" and node_id not in producer_family_links ) if orphan_primary_nodes: raise SpecResolutionError( @@ -1131,8 +1128,7 @@ def _validate_imputation_structure( else "conflicting" ) raise SpecResolutionError( - f"{output_location}: {relation} authored producer output " - f"{key!r}" + f"{output_location}: {relation} authored producer output {key!r}" ) authored_outputs[key] = coverage_scope @@ -1148,12 +1144,8 @@ def _validate_imputation_structure( target_location = f"{family_location}/targets/{target_index}" target = _mapping(target_value, target_location) key = ( - _identifier( - target.get("entity"), location=f"{target_location}/entity" - ), - _identifier( - target.get("name"), location=f"{target_location}/name" - ), + _identifier(target.get("entity"), location=f"{target_location}/entity"), + _identifier(target.get("name"), location=f"{target_location}/name"), ) coverage_scope = _identifier( target.get("output_coverage_scope"), @@ -1166,8 +1158,7 @@ def _validate_imputation_structure( else "conflicting" ) raise SpecResolutionError( - f"{target_location}: {relation} expanded producer output " - f"{key!r}" + f"{target_location}: {relation} expanded producer output {key!r}" ) expanded_outputs[key] = coverage_scope if key in authored_outputs: @@ -1182,9 +1173,7 @@ def _validate_imputation_structure( ) # Materialize the deterministic compile order here so resolution fails # before an adapter can observe an ambiguous union. - compiled_output_keys = tuple( - sorted({**authored_outputs, **expanded_outputs}) - ) + compiled_output_keys = tuple(sorted({**authored_outputs, **expanded_outputs})) if len(compiled_output_keys) != len(authored_outputs) + len(expanded_outputs): raise SpecResolutionError( f"{node_location}/outputs: duplicate expanded producer output" diff --git a/packages/microcosm-build/src/microcosm/build/spec_engine/schema/catalogs.schema.json b/packages/microcosm-build/src/microcosm/build/spec_engine/schema/catalogs.schema.json index 520afe351..40af69f59 100644 --- a/packages/microcosm-build/src/microcosm/build/spec_engine/schema/catalogs.schema.json +++ b/packages/microcosm-build/src/microcosm/build/spec_engine/schema/catalogs.schema.json @@ -97,8 +97,10 @@ "enum": [ "float64", "float32", + "decimal128(18,2)", "int64", "int32", + "int16", "bool", "string", "category" @@ -109,6 +111,7 @@ { "enum": [ "usd", + "nzd", "count", "ratio", "boolean", diff --git a/packages/microcosm-build/tests/golden/nz_country_spec.json b/packages/microcosm-build/tests/golden/nz_country_spec.json new file mode 100644 index 000000000..c97eb6d32 --- /dev/null +++ b/packages/microcosm-build/tests/golden/nz_country_spec.json @@ -0,0 +1,85 @@ +{ + "country": "nz", + "fingerprint": "b8b1b7a745c50ddc548adc9843d7e217d19e7d774d31b631999ce9ee6a1901c7", + "gate_ids": [ + "calibration_per_family_fit", + "national_and_region_admin_aggregates", + "calibration_reference_inventory", + "target_profile_coverage", + "release_input_coverage", + "donor_support_bounds", + "calibration_weights_audit", + "exported_nonzero", + "formula_owned_export", + "nonnegative_columns", + "weight_effective_sample_size", + "weight_max_to_median_ratio", + "macro_realism_nz" + ], + "geography_spine_stage": "assign_nz_regions", + "release": { + "artifact_repo": "policyengine/populace-nz", + "builder": "populace-nz", + "dataset_filename_template": "populace_nz_{year}.h5", + "required_release_files": [ + "populace_nz_{year}.h5", + "populace_nz_{year}_calibration.npz", + "build_manifest.json", + "release_manifest.json", + "calibration_diagnostics.json", + "source_coverage.json", + "demographics.json", + "reform_validation.json" + ], + "staging_repo": "policyengine/populace-nz-staging" + }, + "resource_hashes": { + "country_package.json": "074e0571215a87c10d8933b3b401b02dc0d91171740cd412e43b88512a81fd22", + "export_contract.json": "5e8e4cf9a8aa3363958dfd816fdf412d7d51b3d27ba77caa7b087257107a4487", + "gates.json": "bca805b7aa3c0589ec6242ea8b99c013e8e84a654850ec2d81a3e3796d28eeb9", + "geography_spine.json": "fe248a59c11a2a7ee275ca9a32a58feb4c0b2bc6f873b446caf312bc16513e36", + "release_contract.json": "c0acc8122acf90e7a1d0bcb363ac95dd92928d9f521608525f729df10de421a2", + "source_stages.json": "1b9e3267730774d39008f1ef7d73ff943abc43acc12adcd446e02d6621c6440b", + "spec/bundle.yaml": "47526762837b4479f14156061ec33cf173e91bf5e747244d23a71e6060fc3f05", + "spec/catalogs.yaml": "8376a74620d3c49d34343704a4661064eae0b86c1bf601e50d67949ec06ecea4", + "spec/geography.yaml": "346ae6a19f1da015121e64498549e9b88daf77073629782443171b1d7099e287", + "spec/sources.yaml": "f1d0e3fb981d3e8306cd6c66421d270ae673ee9b11a0c9e928954b7a79ca0316", + "spec/spine.yaml": "b3e6a2832ff7671777cae113b7fc45f5e3492bb519526f229118d0851083533e", + "spec/vintages.yaml": "9050d7d05082c23eacfb418bfc5445af88d92263fad5bc7376e5d32fb53fa843", + "target_references.json": "8798f394f356d9c7d407d7bf4a985debdf48648bc924e3a8f1613e3a45c567d7" + }, + "resources": [ + "spec/bundle.yaml", + "spec/catalogs.yaml", + "spec/geography.yaml", + "spec/sources.yaml", + "spec/spine.yaml", + "spec/vintages.yaml", + "export_contract.json", + "gates.json", + "geography_spine.json", + "release_contract.json", + "source_stages.json", + "target_references.json" + ], + "stage_names": [ + "load_populace_us_support_pool", + "prepare_nz_wff_axiom_inputs" + ], + "target_reference_names": [ + "stats_nz_population_by_age_sex_region", + "stats_nz_population_by_single_year_age_sex", + "stats_nz_census_households_by_composition", + "stats_nz_census_households_by_tenure_region", + "ird_taxable_income_people_by_band", + "ird_taxable_income_total_by_band", + "ird_wage_salary_people_by_band", + "msd_main_benefit_recipients_by_program_age", + "msd_accommodation_supplement_recipients_by_region", + "msd_nz_super_veterans_pension_recipients", + "ird_wff_recipient_families", + "ird_iwtc_recipient_families", + "ird_student_loan_borrowers", + "stats_nz_census_ethnicity_by_age_region" + ] +} diff --git a/packages/microcosm-build/tests/test_country_spec.py b/packages/microcosm-build/tests/test_country_spec.py index 26a8cb76c..e365ef6c1 100644 --- a/packages/microcosm-build/tests/test_country_spec.py +++ b/packages/microcosm-build/tests/test_country_spec.py @@ -11,7 +11,9 @@ from __future__ import annotations import json +import os from dataclasses import FrozenInstanceError +from decimal import Decimal from pathlib import Path import pytest @@ -29,7 +31,7 @@ COUNTRY_PACKAGE_ROOT = Path(__file__).parents[1] / "src/microcosm/build" GOLDEN_ROOT = Path(__file__).parent / "golden" -GOLDEN_COUNTRIES = ("am", "be") +GOLDEN_COUNTRIES = ("am", "be", "nz") FORBIDDEN_TARGET_VALUE_KEYS = {"value", "values", "observed", "observed_value"} @@ -497,6 +499,436 @@ def test_fingerprint_is_stable_across_loads(self, spec) -> None: assert load_country_spec("be").fingerprint == spec.fingerprint +class TestNewZealandPackage: + @pytest.fixture(scope="class") + def spec(self): + return load_country_spec("nz") + + def test_loads_with_every_declared_resource(self, spec) -> None: + assert spec.country == "nz" + assert set(spec.resources) == { + "spec/bundle.yaml", + "spec/catalogs.yaml", + "spec/geography.yaml", + "spec/sources.yaml", + "spec/spine.yaml", + "spec/vintages.yaml", + "source_stages.json", + "geography_spine.json", + "target_references.json", + "gates.json", + "export_contract.json", + "release_contract.json", + } + assert set(spec.resource_hashes) == set(spec.resources) | { + "country_package.json" + } + + def test_donor_pin_and_us_support_provenance_are_explicit(self, spec) -> None: + stages = spec.sources.stage_map() + assert tuple(stages) == ( + "load_populace_us_support_pool", + "prepare_nz_wff_axiom_inputs", + ) + donor = stages["load_populace_us_support_pool"] + artifact = donor.artifacts[0] + assert artifact["revision"] == ( + "populace-us-2024-buildp-sparse-rmloss100-cae8640-20260728T011454Z" + ) + assert artifact["locator"] == "populace_us_2024.h5" + assert artifact["sha256"] == ( + "48b9d479fb4fd1c3537f9383ce4697d130b6f618658409d74f6233c43b994c7e" + ) + assert artifact["size_bytes"] == 462915783 + assert {"donor_country_code", "support_stratum"} <= set(donor.outputs) + assert "US donor support records" in donor.survey + assert "execution remains blocked" in donor.notes + + def test_axiom_contract_has_eleven_required_inputs_and_ten_padding_fields( + self, spec + ) -> None: + columns = {column.key: column for column in spec.resolved_spec.columns} + required = { + column.key.removeprefix("family."): column.dtype + for column in columns.values() + if column.domain == "axiom_required_input" + } + assert required == { + "family_tax_credit_eldest_dependent_child_care_units": "decimal128(18,2)", + "family_tax_credit_subsequent_dependent_child_care_units": "decimal128(18,2)", + "family_tax_credit_entitlement_days": "int16", + "wff_family_scheme_income_for_relationship_period": "decimal128(18,2)", + "wff_family_credit_abatement_days": "int16", + "entitled_to_in_work_tax_credit": "bool", + "in_work_tax_credit_allowed_children_count": "int16", + "in_work_tax_credit_weekly_periods": "int16", + "child_tax_credit_for_entitlement_period": "decimal128(18,2)", + "parental_tax_credit_for_entitlement_period": "decimal128(18,2)", + "parental_tax_credit_additional_abatement": "decimal128(18,2)", + } + padding = { + column.key.removeprefix("family.") + for column in columns.values() + if column.domain == "axiom_adapter_padding_zero" + } + assert len(padding) == 10 + assert all( + name.startswith(("best_start_", "minimum_family_")) for name in padding + ) + stage = spec.sources.stage_map()["prepare_nz_wff_axiom_inputs"] + predictors = stage.operations[0].parameters["predictors"] + assert set(predictors) == set(required) | padding + assert len(predictors) == 21 + assert {"family_id", "family_household_id", "person_family_id"} <= set( + stage.outputs + ) + assert all( + "3b663b3e6eb6408351154990be0c4b92d42c92da" in row["resource"] + for row in stage.artifacts + ) + assert 'Frame.resolve_weights("family")' in stage.notes + assert "family_weight" not in stage.outputs + assert ( + "wff_family_scheme_income_for_relationship_period" + not in stage.nonnegative_outputs + ) + assert "budget_2025_wff_abatement_entitlement_change" not in stage.outputs + assert "budget_2026_iwtc_entitlement_change" not in stage.outputs + income = columns["family.wff_family_scheme_income_for_relationship_period"] + assert income.unit == "nzd" + assert income.unit_waiver is None + + def test_geography_is_national_with_target_assigned_regions(self, spec) -> None: + spine = spec.geography_spine.geography_spine + assert spine.stage == "assign_nz_regions" + assert spine.geography_level == "region" + assert spine.code_system == "nz_stats_region" + assert spine.vintage == "2025" + assert spine.vintage_policy == "error" + assert spine.clones_per_record == 1 + assert spine.collision_avoidance is True + + def test_targets_are_unactivated_chronicle_references_not_budget_scores( + self, spec + ) -> None: + references = {reference.name: reference for reference in spec.target_references} + assert len(references) == 14 + assert all( + reference.metadata["activation_status"].startswith("requires_") + for reference in references.values() + ) + assert all( + reference.metadata["target_role"] == "calibration" + for reference in references.values() + ) + wff = references["ird_wff_recipient_families"] + assert wff.entity == "family" + assert wff.measure == "receives_working_for_families" + assert ( + wff.metadata["chronicle_package_id"] + == "ird-working-for-families-statistics-sept-2025" + ) + assert wff.ledger_selector["source_name"] == "ird" + assert wff.ledger_source_record_id == ( + "ird_wff_statistics.ty2024.recipient_families.all.wff_recipient_families" + ) + assert references["ird_iwtc_recipient_families"].ledger_source_record_id == ( + "ird_wff_statistics.ty2024.recipient_families.all.iwtc_recipient_families" + ) + assert not any( + token in name + for name in references + for token in ("budget_2025", "budget_2026", "expenditure", "treasury") + ) + raw = json.loads( + (COUNTRY_PACKAGE_ROOT / "nz/target_references.json").read_text( + encoding="utf-8" + ) + ) + assert FORBIDDEN_TARGET_VALUE_KEYS.isdisjoint( + _nested_mapping_keys(raw["target_references"]) + ) + + def test_release_gates_refuse_missing_inputs_and_preserve_external_oracles( + self, spec + ) -> None: + selected = {gate.gate for gate in spec.gates.gates} + assert {"parity", "export_surface", "target_surface"}.isdisjoint(selected) + active = { + gate.gate: gate for gate in spec.gates.gates if gate.not_applicable is None + } + for name in ( + "calibration_reference_coverage", + "target_profile_coverage", + "release_input_coverage", + "weights_audit", + "formula_owned_export", + ): + assert active[name].criticality == "release_blocking" + assert active[name].evidence_absent_blocks is True + assert ( + len(active["target_profile_coverage"].parameters["required_families"]) == 11 + ) + + def test_dashboard_contract_is_public_and_capability_scoped(self, spec) -> None: + contract = spec.release_contract + assert contract.artifact_repo == "policyengine/populace-nz" + assert contract.staging_repo == "policyengine/populace-nz-staging" + assert contract.artifact_repo_private is False + assert contract.licence_restricted is False + assert contract.dataset_filename_template == "populace_nz_{year}.h5" + assert contract.private_artifacts == () + assert set(contract.required_release_files) <= set(contract.public_artifacts) + assert {"calibration_diagnostics.json", "reform_validation.json"} <= set( + contract.required_release_files + ) + raw = json.loads( + (COUNTRY_PACKAGE_ROOT / "nz/release_contract.json").read_text( + encoding="utf-8" + ) + ) + assert raw["release_manifest_country"] == { + "code": "nz", + "label": "New Zealand", + "geography_id": None, + "geography_label": "New Zealand", + "repository_visibility": "public", + "capabilities": ["calibration", "targets", "compare"], + } + + def test_fingerprint_is_stable_across_loads(self, spec) -> None: + assert load_country_spec("nz").fingerprint == spec.fingerprint + + def test_closed_export_contract_matches_the_typed_input_catalog(self, spec) -> None: + from microcosm.frame import ExportContract + + path = COUNTRY_PACKAGE_ROOT / "nz/export_contract.json" + raw = json.loads(path.read_text()) + contract = ExportContract.from_path(path) + fields = { + column.key.removeprefix("family."): column.dtype + for column in spec.resolved_spec.columns + if column.domain in {"axiom_required_input", "axiom_adapter_padding_zero"} + } + assert contract.closed + assert set(fields) <= set(contract.required) + assert raw["_input_dtypes"] == fields + assert len(raw["_required_target_inputs"]) == 11 + assert len(raw["_padding_defaults"]) == 10 + assert set(raw["_required_target_inputs"]).isdisjoint(raw["_padding_defaults"]) + assert set(contract.forbidden) == {"person_weight", "family_weight"} + assert len(contract.formula_owned_excluded) == 6 + assert raw["_period"] == { + "label": 2026, + "period_kind": "tax_year", + "start": "2026-04-01", + "end": "2027-03-31", + } + assert raw["_source"]["source_release"] == "pending_treasury_corpus_publication" + + +@pytest.mark.skipif( + not os.environ.get("POPULACE_RULESPEC_NZ"), + reason="set POPULACE_RULESPEC_NZ to a pinned rulespec-nz checkout for real-runtime smoke", +) +class TestNewZealandAxiomTransport: + """Real Axiom execution on synthetic records; no population/calibration claim. + + The first two family cases and expected entitlement deltas come from + rulespec-nz's official_budget_reform_replication companion tests. The + third is a zero-entitlement control. Unequal household weights exercise + the actual Frame family-weight path rather than a manual person sum. + """ + + @pytest.fixture(scope="class") + def adapter(self): + pytest.importorskip("axiom_rules_engine") + from axiom_rules_engine.dense import NativeCompiledDenseProgram + + from microcosm.frame import ExportContract + from microcosm.frame.adapters.axiom import NZ_SCHEMA, AxiomEngine, AxiomPeriod + + if NativeCompiledDenseProgram is None: + pytest.skip("real Axiom dense native extension is not installed") + root = Path(os.environ["POPULACE_RULESPEC_NZ"]) + return AxiomEngine( + root / "nz/policies/budget/official_budget_reform_replication.yaml", + schema=NZ_SCHEMA, + rulespec_roots=[root], + periods={ + 2026: AxiomPeriod(start="2026-04-01", end="2027-03-31", kind="tax_year") + }, + contract=ExportContract.from_path( + COUNTRY_PACKAGE_ROOT / "nz/export_contract.json" + ), + arithmetic="decimal", + ) + + @pytest.fixture + def frame(self): + import numpy as np + import pandas as pd + + from microcosm.frame import Frame, WeightKind, Weights + from microcosm.frame.adapters.axiom import NZ_SCHEMA + + raw = json.loads((COUNTRY_PACKAGE_ROOT / "nz/export_contract.json").read_text()) + inputs = { + "family_tax_credit_eldest_dependent_child_care_units": [1, 1, 0], + "family_tax_credit_subsequent_dependent_child_care_units": [1, 0, 0], + "family_tax_credit_entitlement_days": [365, 365, 0], + "wff_family_scheme_income_for_relationship_period": [50000, 100000, 0], + "wff_family_credit_abatement_days": [365, 365, 0], + "entitled_to_in_work_tax_credit": [False, True, False], + "in_work_tax_credit_allowed_children_count": [0, 1, 0], + "in_work_tax_credit_weekly_periods": [0, 52, 0], + "child_tax_credit_for_entitlement_period": [0, 0, 0], + "parental_tax_credit_for_entitlement_period": [0, 0, 0], + "parental_tax_credit_additional_abatement": [0, 0, 0], + } + inputs.update( + {name: [value] * 3 for name, value in raw["_padding_defaults"].items()} + ) + family = pd.DataFrame( + {"family_id": [1, 2, 3], "family_household_id": [1, 2, 2]} + ) + for name, values in inputs.items(): + dtype = raw["_input_dtypes"][name] + family[name] = ( + pd.Series([Decimal(value) for value in values], dtype=object) + if dtype.startswith("decimal128") + else pd.Series(values, dtype=dtype) + ) + return Frame( + { + "person": pd.DataFrame( + { + "person_id": [1, 2, 3, 4, 5, 6], + "person_household_id": [1, 1, 1, 2, 2, 2], + "person_family_id": [1, 1, 1, 2, 2, 3], + } + ), + "household": pd.DataFrame({"household_id": [1, 2]}), + "family": family, + }, + NZ_SCHEMA, + {"household": Weights(values=np.array([2.0, 7.0]), kind=WeightKind.DESIGN)}, + ) + + def test_real_rules_and_nonuniform_family_weights(self, adapter, frame) -> None: + import numpy as np + + from microcosm.frame import Frame, wsum + + names = [ + "budget_2025_wff_abatement_entitlement_change", + "budget_2026_iwtc_entitlement_change", + ] + outputs = adapter.materialize(frame, names, period=2026) + np.testing.assert_array_equal(outputs[names[0]], [568.5, 0.0, 0.0]) + np.testing.assert_array_equal(outputs[names[1]], [0.0, 438.5, 0.0]) + np.testing.assert_array_equal( + frame.resolve_weights("family").values, [2.0, 7.0, 7.0] + ) + assert all(adapter.variable_metadata(name).entity == "family" for name in names) + tables = {entity: frame.table(entity) for entity in frame.entities} + tables["family"] = frame.table("family").assign(**outputs) + result_frame = Frame( + tables, frame.schema, {"household": frame.weights_for("household")} + ) + assert wsum(result_frame, names[0], entity="family") == 1137.0 + assert wsum(result_frame, names[1], entity="family") == 3069.5 + + def test_exact_inputs_round_trip_without_formula_or_family_weights( + self, adapter, frame, tmp_path + ) -> None: + pytest.importorskip("tables") + from microcosm.frame.adapters.axiom import AxiomEntityTableDataset + + path = tmp_path / "nz_runtime_fixture.h5" + adapter.write_dataset(frame, path, period=2026) + dataset = AxiomEntityTableDataset(file_path=path) + assert dataset.time_period == 2026 + assert dataset.household["household_weight"].tolist() == [2.0, 7.0] + assert dataset.family[ + "wff_family_scheme_income_for_relationship_period" + ].tolist() == [Decimal(50000), Decimal(100000), Decimal(0)] + columns = {name for table in dataset.tables.values() for name in table.columns} + assert {"person_weight", "family_weight"}.isdisjoint(columns) + assert set(adapter.export_contract().formula_owned_excluded).isdisjoint(columns) + + @pytest.mark.parametrize( + "field", + [ + "wff_family_scheme_income_for_relationship_period", + "child_tax_credit_for_entitlement_period", + "entitled_to_in_work_tax_credit", + ], + ) + def test_missing_substantive_inputs_never_default( + self, adapter, frame, field, tmp_path + ) -> None: + from microcosm.frame import Frame + + tables = {entity: frame.table(entity) for entity in frame.entities} + tables["family"] = frame.table("family").drop(columns=[field]) + incomplete = Frame( + tables, frame.schema, {"household": frame.weights_for("household")} + ) + with pytest.raises((ValueError, RuntimeError), match=field): + adapter.materialize( + incomplete, ["budget_2025_wff_abatement_entitlement_change"], 2026 + ) + path = tmp_path / "must_not_exist.h5" + with pytest.raises(ValueError, match=field): + adapter.write_dataset(incomplete, path, 2026) + assert not path.exists() + + def test_formula_owned_output_blocks_export(self, adapter, frame, tmp_path) -> None: + from microcosm.frame import Frame + + name = "budget_2026_iwtc_entitlement_change" + tables = {entity: frame.table(entity) for entity in frame.entities} + tables["family"] = frame.table("family").assign(**{name: [0.0, 0.0, 0.0]}) + poisoned = Frame( + tables, frame.schema, {"household": frame.weights_for("household")} + ) + path = tmp_path / "must_not_exist.h5" + with pytest.raises(ValueError, match=name): + adapter.write_dataset(poisoned, path, 2026) + assert not path.exists() + + def test_misplaced_family_link_cannot_bypass_nesting( + self, adapter, frame, tmp_path + ) -> None: + import numpy as np + + from microcosm.frame import Frame, WeightKind, Weights + + person = frame.table("person").copy() + person["family_household_id"] = person["person_household_id"] + # Attach a person from household 2 to family 1; equal weights cannot + # reveal the invalid cross-household membership by themselves. + person.loc[3, "person_family_id"] = 1 + misplaced = Frame( + { + "person": person, + "household": frame.table("household"), + "family": frame.table("family").drop(columns=["family_household_id"]), + }, + frame.schema, + {"household": Weights(values=np.array([2.0, 2.0]), kind=WeightKind.DESIGN)}, + ) + with pytest.raises(ValueError, match="family_household_id.*family.*person"): + adapter.materialize( + misplaced, ["budget_2025_wff_abatement_entitlement_change"], 2026 + ) + path = tmp_path / "wrong_owner.h5" + with pytest.raises(ValueError, match="family_household_id.*family.*person"): + adapter.write_dataset(misplaced, path, 2026) + assert not path.exists() + + class TestGoldenCountrySpecs: @pytest.mark.parametrize("country", GOLDEN_COUNTRIES) def test_loaded_spec_matches_the_golden_file_byte_for_byte( diff --git a/packages/microcosm-build/tests/test_spec_engine_country_bundles.py b/packages/microcosm-build/tests/test_spec_engine_country_bundles.py index 736925be2..6b1024495 100644 --- a/packages/microcosm-build/tests/test_spec_engine_country_bundles.py +++ b/packages/microcosm-build/tests/test_spec_engine_country_bundles.py @@ -26,6 +26,42 @@ "vintages", } AM_SPEC_SHA256 = "659b6baf5ebbd71fb7786ec4c4d49df565b2bddabeb868a9385ed226c56880f9" +NZ_SPEC_SHA256 = "7ca62ede964e4e0d5d9d700a136ff2be170b863de3f9fe9e2577394bac12f92f" +NZ_COLUMN_KEYS = { + "family.best_start_abatement_days", + "family.best_start_child_care_fraction", + "family.best_start_entitlement_days", + "family.best_start_family_scheme_income_for_relationship_period", + "family.child_tax_credit_for_entitlement_period", + "family.entitled_to_in_work_tax_credit", + "family.family_household_id", + "family.family_id", + "family.family_tax_credit_eldest_dependent_child_care_units", + "family.family_tax_credit_entitlement_days", + "family.family_tax_credit_subsequent_dependent_child_care_units", + "family.in_work_tax_credit_allowed_children_count", + "family.in_work_tax_credit_weekly_periods", + "family.minimum_family_adjusted_income_tax_liability", + "family.minimum_family_amount_paid", + "family.minimum_family_amount_received", + "family.minimum_family_full_time_earner_weeks", + "family.minimum_family_scheme_income_attributable_to_full_time_weeks", + "family.minimum_family_tax_credit_weekly_periods", + "family.parental_tax_credit_additional_abatement", + "family.parental_tax_credit_for_entitlement_period", + "family.wff_family_credit_abatement_days", + "family.wff_family_scheme_income_for_relationship_period", + "household.household_id", + "household.household_weight", + "person.age", + "person.donor_country_code", + "person.person_family_id", + "person.person_household_id", + "person.person_id", + "person.region_code", + "person.sex", + "person.support_stratum", +} @pytest.mark.parametrize( @@ -53,6 +89,12 @@ }, {"household", "person"}, ), + ( + "nz", + NZ_SPEC_SHA256, + NZ_COLUMN_KEYS, + {"family", "household", "person"}, + ), ( "uk", "cce1c98ea40364a398ae361f4d15790c925d8379d8b9b427076d61059c7d6715", @@ -96,6 +138,7 @@ def test_country_bundle_loads_once_and_compiles_through_the_shared_core( def test_country_bundles_exercise_distinct_support_and_geography_kinds() -> None: am = compile_spec(load_bundle("am")) be = compile_spec(load_bundle("be")) + nz = compile_spec(load_bundle("nz")) uk = compile_spec(load_bundle("uk")) assert am.resource("spine")["support_roles"] == [ @@ -104,6 +147,9 @@ def test_country_bundles_exercise_distinct_support_and_geography_kinds() -> None assert be.resource("spine")["support_roles"] == [ {"id": "silc_base", "kind": "none"} ] + assert nz.resource("spine")["support_roles"] == [ + {"id": "populace_us_donor_base", "kind": "none"} + ] assert uk.resource("spine")["support_roles"] == [ { "id": "spi_income_support", @@ -119,6 +165,10 @@ def test_country_bundles_exercise_distinct_support_and_geography_kinds() -> None "assign": "kernel:clone_assign_communes", "validate": "kernel:be_commune_geography_gate", } + assert nz.resource("geography")["assignment"]["kernels"] == { + "assign": "kernel:assign_nz_regions", + "validate": "kernel:nz_region_geography_gate", + } assert uk.resource("geography")["assignment"]["kernels"] == { "assign": "kernel:assign_uk_geography_ladder", "validate": "kernel:uk_geography_ladder_gate", @@ -129,12 +179,15 @@ def test_country_kernel_contract_ids_are_closed_in_the_compiler_registry() -> No country_contract_ids = { "am_community_geography_gate", "assign_am_marz", + "assign_nz_regions", "clone_assign_communities", "load_populace_us_support_pool", "silc_load", "clone_assign_communes", "be_commune_geography_gate", "load_uk_national_frame", + "nz_region_geography_gate", + "prepare_nz_wff_axiom_inputs", "assign_uk_geography_ladder", "uk_geography_ladder_gate", } @@ -198,6 +251,29 @@ def test_be_generation_zero_views_still_come_from_the_country_spec_seam() -> Non } +def test_nz_generation_zero_views_come_from_the_country_spec_seam() -> None: + spec = load_country_spec("nz") + + assert spec.sources is not None + assert tuple(spec.sources.stage_map()) == ( + "load_populace_us_support_pool", + "prepare_nz_wff_axiom_inputs", + ) + assert spec.geography_spine is not None + assert spec.geography_spine.geography_spine.stage == "assign_nz_regions" + assert spec.gates is not None + assert spec.release_contract is not None + assert not spec.release_contract.artifact_repo_private + assert {row.path for row in spec.resource_rows if row.kind == "legacy_json"} == { + "export_contract.json", + "gates.json", + "geography_spine.json", + "release_contract.json", + "source_stages.json", + "target_references.json", + } + + @pytest.mark.parametrize( ("before", "after", "field"), [ @@ -305,3 +381,36 @@ def test_am_smoke_build_refuses_without_harvests_and_stage_bindings() -> None: ), ): country_stage_plan(spec, {}) + + +def test_nz_smoke_build_refuses_without_evidenced_inputs_and_stage_bindings() -> None: + """A valid scaffold must not turn missing NZ runtime bindings into stubs.""" + + spec = load_country_spec("nz") + compiled = compile_spec(spec.resolved_spec) + assert compiled.spec_binding.country == "nz" + assert not compiled.producer_graph.present + assert compiled.nodes == () + + stages = spec.sources.stage_map() + assert { + artifact["kind"] + for artifact in stages["load_populace_us_support_pool"].artifacts + } == {"public_microdata"} + assert ( + len( + stages["prepare_nz_wff_axiom_inputs"].operations[0].parameters["predictors"] + ) + == 21 + ) + build_package = Path(__file__).parents[1] / "src/microcosm/build" + assert not (build_package / "nz_runtime").exists() + + with pytest.raises( + ValueError, + match=( + r"missing \['load_populace_us_support_pool', 'prepare_nz_wff_axiom_inputs', " + r"'assign_nz_regions'\].*There are no stubs or fallbacks" + ), + ): + country_stage_plan(spec, {}) diff --git a/packages/microcosm-build/tests/test_spec_engine_reemission.py b/packages/microcosm-build/tests/test_spec_engine_reemission.py index 5b5fada43..15c8ab34d 100644 --- a/packages/microcosm-build/tests/test_spec_engine_reemission.py +++ b/packages/microcosm-build/tests/test_spec_engine_reemission.py @@ -44,7 +44,7 @@ def _mutate_yaml(path: Path, mutation) -> None: path.write_bytes(canonical_yaml_bytes(value, source=str(path))) -@pytest.mark.parametrize("country", ["be", "uk", "us"]) +@pytest.mark.parametrize("country", ["am", "be", "nz", "uk", "us"]) def test_resolved_bundle_round_trip_is_lossless_deterministic_and_alias_free( tmp_path: Path, country: str, diff --git a/packages/microcosm-build/tests/test_spec_only_country_packages.py b/packages/microcosm-build/tests/test_spec_only_country_packages.py index 307db3274..c073bc7d7 100644 --- a/packages/microcosm-build/tests/test_spec_only_country_packages.py +++ b/packages/microcosm-build/tests/test_spec_only_country_packages.py @@ -131,31 +131,31 @@ def test__given_country_like_directories__then_each_has_a_manifest() -> None: assert offenders == [] -def test__given_greenfield_country_packages__then_armenia_uses_the_typed_schema() -> ( - None -): +def test__given_greenfield_country_packages__then_each_uses_the_typed_schema() -> None: # Given country_roots = {root.name: root for root in _country_package_roots()} # When - armenia = country_roots["am"] - manifest = json.loads( - (armenia / "country_package.json").read_text(encoding="utf-8") - ) - resource_kinds = {row["kind"] for row in manifest["resources"]} - - # Then - assert {"am", "be"} <= set(country_roots) - assert manifest["schema_version"] == 1 - assert resource_kinds == ALLOWED_COUNTRY_RESOURCE_KINDS & { - "bundle", - "catalogs", - "geography", - "legacy_json", - "sources", - "spine", - "vintages", - } + for country in ("am", "be", "nz"): + manifest = json.loads( + (country_roots[country] / "country_package.json").read_text( + encoding="utf-8" + ) + ) + resource_kinds = {row["kind"] for row in manifest["resources"]} + + # Then + assert manifest["schema_version"] == 1 + assert resource_kinds == ALLOWED_COUNTRY_RESOURCE_KINDS & { + "bundle", + "catalogs", + "geography", + "legacy_json", + "sources", + "spine", + "vintages", + } + assert {"am", "be", "nz"} <= set(country_roots) def test__given_country_specs__then_no_python_entrypoints_are_declared() -> None: diff --git a/packages/microcosm-frame/src/microcosm/frame/adapters/axiom.py b/packages/microcosm-frame/src/microcosm/frame/adapters/axiom.py index 623639f88..db7a441bb 100644 --- a/packages/microcosm-frame/src/microcosm/frame/adapters/axiom.py +++ b/packages/microcosm-frame/src/microcosm/frame/adapters/axiom.py @@ -51,6 +51,9 @@ """ from collections.abc import Mapping, Sequence +from dataclasses import dataclass +from datetime import date +from decimal import Decimal, localcontext from pathlib import Path from typing import Any @@ -62,7 +65,13 @@ from microcosm.frame.rules import ExportContract from microcosm.frame.schema import EntitySchema, VariableMetadata -__all__ = ["AxiomEngine", "AxiomEntityTableDataset", "BE_SCHEMA"] +__all__ = [ + "AxiomEngine", + "AxiomEntityTableDataset", + "AxiomPeriod", + "BE_SCHEMA", + "NZ_SCHEMA", +] #: The Belgian frame schema for the populace-be pilot: persons in households. #: Belgian PIT is individual with household-level elements (joint assessment, @@ -71,6 +80,10 @@ #: needs them, mapped via ``entity_names``. BE_SCHEMA = EntitySchema(group_entities=("household",)) +#: The NZ transport contract declares family units separately from households. +#: Only households carry explicit weights; families inherit through membership. +NZ_SCHEMA = EntitySchema(group_entities=("household", "family")) + #: Engine dtype vocabulary -> kernel dtype kind. ``judgment`` is tri-state #: (holds / not holds / undetermined) and materializes as int8 codes #: ``1 / -1 / 0``, so it reports as ``int``, not ``bool``. @@ -91,6 +104,38 @@ _WEIGHT_COLUMN_SUFFIX = "_weight" +@dataclass(frozen=True) +class AxiomPeriod: + """Explicit dense-execution dates for a non-calendar policy period. + + ``kind`` is the engine's period identifier (for example ``tax_year``), + not a display label. No fiscal-year convention is inferred from a year. + """ + + start: str + end: str + kind: str + + def __post_init__(self) -> None: + if not isinstance(self.kind, str) or not self.kind.strip(): + raise ValueError("Axiom period kind must be a non-empty string.") + try: + start = date.fromisoformat(self.start) + end = date.fromisoformat(self.end) + except (TypeError, ValueError) as exc: + raise ValueError( + "Axiom period dates must be valid ISO YYYY-MM-DD dates." + ) from exc + if start.isoformat() != self.start or end.isoformat() != self.end: + raise ValueError("Axiom period dates must use ISO YYYY-MM-DD format.") + if start > end: + raise ValueError("Axiom period start must not follow end.") + + def bounds(self) -> tuple[str, str, str]: + """Return the dense executor's ``(start, end, period_kind)`` tuple.""" + return self.start, self.end, self.kind + + class AxiomEngine: """RulesEngine adapter backed by the Axiom dense vectorized surface. @@ -111,6 +156,14 @@ class AxiomEngine: arithmetic: ``"decimal"`` (exact, canonical) or ``"f64"`` (faster, floating-point rounding) — which dense execution mode :meth:`materialize` uses. + rulespec_roots: Canonical RuleSpec repository roots used to resolve + imports. Paths are made absolute. Defaults to the nearest + ``rulespec-`` ancestor when present; otherwise the + module's parent (which the engine may reject as non-canonical). + periods: Optional year/period-label to explicit bounds mapping. If + supplied, an unmapped label fails instead of becoming a calendar + year. :meth:`materialize` also accepts an :class:`AxiomPeriod` + directly. The compiled dense programs (one per engine entity) and the module's variable metadata are loaded lazily and cached; constructing the adapter @@ -126,12 +179,40 @@ def __init__( defaults: Mapping[str, object] | None = None, entity_names: Mapping[str, str] | None = None, arithmetic: str = "decimal", + rulespec_roots: Sequence[str | Path] | None = None, + periods: Mapping[int | str, AxiomPeriod] | None = None, ) -> None: if arithmetic not in ("decimal", "f64"): raise ValueError( f"arithmetic must be 'decimal' or 'f64', got {arithmetic!r}." ) - self._module = Path(module) + self._module = Path(module).resolve() + self._rulespec_roots = ( + tuple(Path(root).resolve() for root in rulespec_roots) + if rulespec_roots is not None + else ( + next( + ( + parent + for parent in self._module.parents + if parent.name.startswith("rulespec-") + ), + self._module.parent, + ), + ) + ) + if not self._rulespec_roots: + raise ValueError("rulespec_roots must contain at least one canonical root.") + self._periods: dict[str, AxiomPeriod] | None = None + if periods is not None: + self._periods = {} + for label, bounds in periods.items(): + if not isinstance(bounds, AxiomPeriod): + raise TypeError("periods values must be AxiomPeriod instances.") + key = str(label) + if key in self._periods: + raise ValueError(f"Duplicate explicit Axiom period label {key!r}.") + self._periods[key] = bounds self._schema = schema self._contract = contract if contract is not None else ExportContract.empty() self._defaults = dict(defaults or {}) @@ -225,7 +306,7 @@ def materialize( self, bundle: Frame, variables: Sequence[str], - period: int | str, + period: int | str | AxiomPeriod, ) -> Mapping[str, np.ndarray]: """Compute ``variables`` for ``period`` over the bundle's tables. @@ -237,7 +318,8 @@ def materialize( bundle: A bundle whose entities match the adapter's schema. variables: Computed (derived) variable names. period: ``2025`` / ``"2025"`` for a calendar year, ``"2025-01"`` - for a month. + for a month, or explicit :class:`AxiomPeriod` bounds. Labels + use the constructor's ``periods`` mapping when supplied. Returns: One array per variable, row-aligned to the variable's entity @@ -254,7 +336,7 @@ def materialize( pilot slice declares none). """ self._require_schema(bundle) - start, end, period_kind = _period_bounds(period) + start, end, period_kind = self._materialization_period(period) by_entity: dict[str, list[str]] = {} for name in variables: @@ -294,6 +376,17 @@ def materialize( results[name] = values return results + def _materialization_period( + self, period: int | str | AxiomPeriod + ) -> tuple[str, str, str]: + if isinstance(period, AxiomPeriod): + return period.bounds() + if self._periods is not None: + if str(period) not in self._periods: + raise ValueError(f"No explicit Axiom period bounds for {period!r}.") + return self._periods[str(period)].bounds() + return _period_bounds(period) + # ------------------------------------------------------------------ # Export # ------------------------------------------------------------------ @@ -431,9 +524,15 @@ def _program(self, frame_entity: str, *, missing_ok: bool = False) -> Any: engine = self._import_engine() try: program = engine.CompiledDenseProgram.from_file( - self._module, entity=engine_entity + self._module, + rulespec_roots=self._rulespec_roots, + entity=engine_entity, ) - except ValueError: + except ValueError as exc: + # Only an absent entity is optional. Never disguise canonical + # root, import, or compilation errors as an empty program. + if "could not find derived outputs for entity" not in str(exc): + raise self._programs[frame_entity] = None if missing_ok: return None @@ -467,6 +566,33 @@ def _require_schema(self, bundle: Frame) -> None: f"{list(self._schema.entities)}; bundle has " f"{list(bundle.entities)}." ) + # Explicit group-to-group id columns assert nesting. Check them + # through the Frame's membership placement, independently of whether + # household weights happen to be equal. Weight agreement alone does + # not prove that a family belongs to one household. + person = bundle.table(self._schema.person_entity) + for group in self._schema.group_entities: + for parent in self._schema.group_entities: + if group == parent: + continue + column = f"{group}_{parent}_id" + try: + owner = bundle.column_entity(column) + except ValueError: + continue + if owner != group: + raise ValueError( + f"{column!r} must be on entity {group!r}, not {owner!r}." + ) + broadcast = bundle.place( + column, self._schema.person_entity, how="broadcast" + ).table(self._schema.person_entity)[column] + membership = person[self._schema.membership_column(parent)] + if not np.array_equal(broadcast.to_numpy(), membership.to_numpy()): + raise ValueError( + f"{column!r} disagrees with person membership; every " + f"{group!r} must be nested in its declared {parent!r}." + ) def _engine_tables(self, bundle: Frame) -> dict[str, pd.DataFrame]: """Copy the bundle's tables and materialize typed weights as columns. @@ -613,11 +739,25 @@ def save(self, file_path: str | Path) -> None: with pd.HDFStore(str(path)) as store: for name, table in self.tables.items(): if len(table) > 0: + # PyTables table format cannot store Decimal objects. + # Keep exact decimal inputs in fixed format; never coerce + # persisted money to float just to satisfy the writer. + decimal_columns = [ + column + for column in table.columns + if table[column].dtype.kind == "O" + and any(isinstance(value, Decimal) for value in table[column]) + ] + stored_table = table + if decimal_columns: + stored_table = table.astype( + {column: object for column in decimal_columns} + ) put_frame_table( store, name, - table, - preferred_format="table", + stored_table, + preferred_format="fixed" if decimal_columns else "table", data_columns=True, ) store.put( @@ -675,12 +815,15 @@ def _batch_from_table( The table's column dtypes are authoritative: bool columns become Bool engine columns (truthiness-context inputs require them), integers become Integer, floats become the numeric column of the active arithmetic. + Decimal columns cross the native float64 ABI only when its nine-place + rounding recovers every original value exactly; wider decimal values + are refused, not rounded silently. Inputs the table does not carry are omitted — the engine defaults declared-optional inputs and errors on required ones, naming the input. Raises: - ValueError: If a needed column's dtype is not bool/integer/float - (object/string columns cannot become dense columns). + ValueError: If a needed column is not bool/integer/float/Decimal, + or a Decimal cannot be recovered through the native boundary. """ batch: dict[str, np.ndarray] = {} for name in root_inputs: @@ -694,9 +837,49 @@ def _batch_from_table( batch[name] = column.to_numpy(dtype=np.int64) elif kind == "f": batch[name] = column.to_numpy(dtype=np.float64) + elif any(isinstance(value, Decimal) for value in column): + batch[name] = _checked_decimal_array(name, column) else: raise ValueError( f"Column {name!r} has dtype kind {kind!r}; dense inputs must " - "be bool, integer, or float columns." + "be bool, integer, float, or exact-boundary decimal columns." ) return batch + + +def _checked_decimal_array(name: str, column: pd.Series) -> np.ndarray: + """Prove recovery through native ``from_f64_retain(...).round_dp(9)``. + + The strict half-unit error bound avoids depending on the native tie + rounding rule. This is bounded conversion, not general Decimal128 ABI + support; the original Decimal values remain untouched in the Frame. + """ + values: list[float] = [] + for value in column: + valid = isinstance(value, Decimal) and value.is_finite() + if valid: + parts = value.as_tuple() + trailing_zeros = 0 + for digit in reversed(parts.digits): + if digit != 0: + break + trailing_zeros += 1 + # Inspect the tuple, not a context-rounded rescaling: extreme + # subnormals must not underflow to zero during validation. + valid = value.is_zero() or parts.exponent + trailing_zeros >= -9 + floating = float(value) + valid = valid and np.isfinite(floating) + if valid: + with localcontext() as context: + context.prec = max(50, len(parts.digits) + 10) + valid = abs(Decimal.from_float(floating) - value) < Decimal( + "0.0000000005" + ) + if not valid: + raise ValueError( + f"Column {name!r} contains a decimal value outside the exact " + "native nine-place boundary (finite Decimal, at most nine " + "places, float64 conversion error strictly below 5e-10 required)." + ) + values.append(floating) + return np.asarray(values, dtype=np.float64) diff --git a/packages/microcosm-frame/tests/fixtures/axiom_toy_country.yaml b/packages/microcosm-frame/tests/fixtures/rulespec-xx/xx/policies/axiom_toy_country.yaml similarity index 100% rename from packages/microcosm-frame/tests/fixtures/axiom_toy_country.yaml rename to packages/microcosm-frame/tests/fixtures/rulespec-xx/xx/policies/axiom_toy_country.yaml diff --git a/packages/microcosm-frame/tests/test_axiom_adapter.py b/packages/microcosm-frame/tests/test_axiom_adapter.py index 51379a32d..9e4fad580 100644 --- a/packages/microcosm-frame/tests/test_axiom_adapter.py +++ b/packages/microcosm-frame/tests/test_axiom_adapter.py @@ -15,6 +15,7 @@ import importlib.util import os +from decimal import Decimal from pathlib import Path import numpy as np @@ -31,8 +32,11 @@ ) from microcosm.frame.adapters.axiom import ( BE_SCHEMA, + NZ_SCHEMA, AxiomEngine, AxiomEntityTableDataset, + AxiomPeriod, + _batch_from_table, _period_bounds, ) @@ -54,7 +58,9 @@ reason="pytables (microcosm-frame[axiom]) is not installed", ) -FIXTURE_MODULE = Path(__file__).parent / "fixtures" / "axiom_toy_country.yaml" +FIXTURE_MODULE = ( + Path(__file__).parent / "fixtures/rulespec-xx/xx/policies/axiom_toy_country.yaml" +) RULESPEC_BE = os.environ.get("POPULACE_RULESPEC_BE") @@ -109,6 +115,25 @@ def test_engine_methods_describe_installation_when_missing(self) -> None: class TestConstruction: + def test_nz_schema_has_families_and_households(self) -> None: + assert NZ_SCHEMA.entities == ("person", "household", "family") + + def test_explicit_rulespec_roots_and_tax_year_bounds(self) -> None: + tax_year = AxiomPeriod(start="2026-04-01", end="2027-03-31", kind="tax_year") + adapter = AxiomEngine( + FIXTURE_MODULE, + rulespec_roots=(FIXTURE_MODULE.parent,), + periods={"2026": tax_year}, + ) + assert adapter._materialization_period(2026) == ( + "2026-04-01", + "2027-03-31", + "tax_year", + ) + assert adapter._materialization_period(tax_year) == tax_year.bounds() + with pytest.raises(ValueError, match="No explicit Axiom period"): + adapter._materialization_period(2025) + def test_rejects_unknown_arithmetic(self) -> None: with pytest.raises(ValueError, match="arithmetic"): AxiomEngine(FIXTURE_MODULE, arithmetic="float32") @@ -124,8 +149,62 @@ def test_default_entity_names_capitalize(self) -> None: adapter = AxiomEngine(FIXTURE_MODULE) assert adapter._entity_names == {"person": "Person", "household": "Household"} + def test_cross_household_family_is_refused_even_with_equal_weights(self) -> None: + frame = Frame( + { + "person": pd.DataFrame( + { + "person_id": [1, 2], + "person_household_id": [1, 2], + "person_family_id": [1, 1], + } + ), + "household": pd.DataFrame({"household_id": [1, 2]}), + "family": pd.DataFrame({"family_id": [1], "family_household_id": [1]}), + }, + NZ_SCHEMA, + {"household": Weights(values=np.array([2.0, 2.0]), kind=WeightKind.DESIGN)}, + ) + with pytest.raises(ValueError, match="family_household_id.*membership"): + AxiomEngine(FIXTURE_MODULE, schema=NZ_SCHEMA).materialize(frame, [], 2026) + + def test_group_parent_id_on_wrong_entity_blocks_materialization_and_export( + self, tmp_path + ) -> None: + frame = Frame( + { + "person": pd.DataFrame( + { + "person_id": [1, 2], + "person_household_id": [1, 2], + "person_family_id": [1, 1], + "family_household_id": [1, 2], + } + ), + "household": pd.DataFrame({"household_id": [1, 2]}), + "family": pd.DataFrame({"family_id": [1]}), + }, + NZ_SCHEMA, + {"household": Weights(values=np.array([2.0, 2.0]), kind=WeightKind.DESIGN)}, + ) + adapter = AxiomEngine(FIXTURE_MODULE, schema=NZ_SCHEMA) + with pytest.raises(ValueError, match="family_household_id.*family.*person"): + adapter.materialize(frame, [], 2026) + path = tmp_path / "wrong_owner.h5" + with pytest.raises(ValueError, match="family_household_id.*family.*person"): + adapter.write_dataset(frame, path, 2026) + assert not path.exists() + class TestPeriodBounds: + def test_explicit_period_refuses_invalid_or_reversed_dates(self) -> None: + with pytest.raises(ValueError, match="start.*end"): + AxiomPeriod(start="2027-03-31", end="2026-04-01", kind="tax_year") + with pytest.raises(ValueError): + AxiomPeriod(start="2026-02-30", end="2027-03-31", kind="tax_year") + with pytest.raises(ValueError, match="kind"): + AxiomPeriod(start="2026-04-01", end="2027-03-31", kind="") + def test_year_as_int_and_str(self) -> None: assert _period_bounds(2025) == ("2025-01-01", "2025-12-31", "calendar_year") assert _period_bounds("2025") == ("2025-01-01", "2025-12-31", "calendar_year") @@ -140,6 +219,45 @@ def test_rejects_other_shapes(self) -> None: _period_bounds("2025-13") +class TestDecimalBatch: + def test_decimal_cents_recover_exactly_through_the_native_nine_place_boundary( + self, + ) -> None: + table = pd.DataFrame( + { + "money": [Decimal("50000.01"), Decimal("100000.00")], + "days": pd.Series([365, 0], dtype="int16"), + "eligible": [True, False], + } + ) + batch = _batch_from_table(table, ("money", "days", "eligible")) + assert batch["money"].dtype == np.float64 + assert batch["days"].dtype == np.int64 + assert batch["eligible"].dtype == np.bool_ + np.testing.assert_array_equal(batch["money"], [50000.01, 100000.0]) + + @pytest.mark.parametrize( + "value", + [ + Decimal("123456789.01"), + Decimal("0.0000000001"), + Decimal("NaN"), + Decimal("1e-10000000"), + Decimal("1e10000000"), + ], + ) + def test_decimal_values_outside_the_exact_dense_boundary_are_refused( + self, value + ) -> None: + with pytest.raises(ValueError, match="money.*decimal"): + _batch_from_table(pd.DataFrame({"money": [value]}), ("money",)) + + def test_object_floats_do_not_impersonate_decimal_columns(self) -> None: + table = pd.DataFrame({"money": pd.Series([1.0], dtype=object)}) + with pytest.raises(ValueError, match="money"): + _batch_from_table(table, ("money",)) + + @needs_engine class TestVariableMetadata: @pytest.fixture(scope="class") @@ -188,9 +306,7 @@ def test_person_values_row_aligned_and_hand_computed(self, adapter) -> None: results = adapter.materialize(bundle, ["toy_income_tax"], period=2025) # 5,000 * 10% = 500; 10,000 * 10% = 1,000; # 10,000 * 10% + 10,000 * 25% = 3,500. - np.testing.assert_allclose( - results["toy_income_tax"], [500.0, 1_000.0, 3_500.0] - ) + np.testing.assert_allclose(results["toy_income_tax"], [500.0, 1_000.0, 3_500.0]) def test_bool_column_drives_the_exemption_predicate(self, adapter) -> None: bundle = _toy_bundle(exempt=(True, False, True)) @@ -204,9 +320,7 @@ def test_household_values_align_to_the_household_table(self, adapter) -> None: ) assert results["toy_income_tax"].shape == (bundle.n("person"),) assert results["toy_housing_allowance"].shape == (bundle.n("household"),) - np.testing.assert_allclose( - results["toy_housing_allowance"], [1_200.0, 0.0] - ) + np.testing.assert_allclose(results["toy_housing_allowance"], [1_200.0, 0.0]) def test_integer_column_feeds_count_inputs(self, adapter) -> None: bundle = _toy_bundle(children=(0, 2, 1)) @@ -217,9 +331,7 @@ def test_f64_arithmetic_matches_decimal(self) -> None: fast = AxiomEngine(FIXTURE_MODULE, arithmetic="f64") bundle = _toy_bundle() results = fast.materialize(bundle, ["toy_income_tax"], period=2025) - np.testing.assert_allclose( - results["toy_income_tax"], [500.0, 1_000.0, 3_500.0] - ) + np.testing.assert_allclose(results["toy_income_tax"], [500.0, 1_000.0, 3_500.0]) def test_wrong_bundle_entities_are_refused(self, adapter) -> None: person = pd.DataFrame( @@ -409,6 +521,19 @@ def test_formula_owned_exclusion_applies_under_a_closed_contract( class TestAxiomEntityTableDataset: + @needs_tables + def test_decimal_columns_round_trip_without_float_coercion(self, tmp_path) -> None: + values = [Decimal("50000.01"), Decimal("100000.00")] + tables = { + "person": pd.DataFrame({"person_id": [1, 2]}), + "family": pd.DataFrame({"family_id": [1, 2], "money": values}), + } + path = tmp_path / "decimal.h5" + AxiomEntityTableDataset(tables=tables, time_period=2026).save(path) + reloaded = AxiomEntityTableDataset(file_path=path) + assert reloaded.family["money"].tolist() == values + assert all(isinstance(value, Decimal) for value in reloaded.family["money"]) + def test_requires_exactly_one_construction_mode(self, tmp_path) -> None: with pytest.raises(ValueError, match="tables and time_period"): AxiomEntityTableDataset() @@ -463,11 +588,8 @@ class TestBelgianPilotSlice: @pytest.fixture(scope="class") def adapter(self) -> AxiomEngine: - module = ( - Path(RULESPEC_BE) - / "be/statutes/income_tax/individual/rate_scale.yaml" - ) - return AxiomEngine(module) + module = Path(RULESPEC_BE) / "be/statutes/income_tax/individual/rate_scale.yaml" + return AxiomEngine(module, rulespec_roots=[Path(RULESPEC_BE)]) def _be_bundle(self) -> Frame: person = pd.DataFrame( diff --git a/packages/microcosm-frame/tests/test_rules_engine_contract.py b/packages/microcosm-frame/tests/test_rules_engine_contract.py index 5e33340cb..7c2e64e72 100644 --- a/packages/microcosm-frame/tests/test_rules_engine_contract.py +++ b/packages/microcosm-frame/tests/test_rules_engine_contract.py @@ -47,7 +47,9 @@ else: _AXIOM_DENSE = False -_FIXTURE_MODULE = Path(__file__).parent / "fixtures" / "axiom_toy_country.yaml" +_FIXTURE_MODULE = ( + Path(__file__).parent / "fixtures/rulespec-xx/xx/policies/axiom_toy_country.yaml" +) @dataclass(frozen=True) From 05b956738aec976fe8d1231dfe13ca635ca3add9 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sat, 29 Aug 2026 21:09:34 -0400 Subject: [PATCH 2/2] Refuse missing required Axiom relationship columns before execution --- packages/microcosm-build/tests/test_country_spec.py | 1 + .../microcosm-frame/src/microcosm/frame/adapters/axiom.py | 5 +++++ 2 files changed, 6 insertions(+) diff --git a/packages/microcosm-build/tests/test_country_spec.py b/packages/microcosm-build/tests/test_country_spec.py index e365ef6c1..208249324 100644 --- a/packages/microcosm-build/tests/test_country_spec.py +++ b/packages/microcosm-build/tests/test_country_spec.py @@ -863,6 +863,7 @@ def test_exact_inputs_round_trip_without_formula_or_family_weights( "wff_family_scheme_income_for_relationship_period", "child_tax_credit_for_entitlement_period", "entitled_to_in_work_tax_credit", + "family_household_id", ], ) def test_missing_substantive_inputs_never_default( diff --git a/packages/microcosm-frame/src/microcosm/frame/adapters/axiom.py b/packages/microcosm-frame/src/microcosm/frame/adapters/axiom.py index db7a441bb..99fc89210 100644 --- a/packages/microcosm-frame/src/microcosm/frame/adapters/axiom.py +++ b/packages/microcosm-frame/src/microcosm/frame/adapters/axiom.py @@ -579,6 +579,11 @@ def _require_schema(self, bundle: Frame) -> None: try: owner = bundle.column_entity(column) except ValueError: + if column in self._contract.required: + raise ValueError( + f"Required relation column {column!r} is missing " + f"from entity {group!r}." + ) from None continue if owner != group: raise ValueError(