diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 000000000..324641f92 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,2 @@ +# Preserve exact unified-diff context markers in the archived measured experiment. +experiments/us-acs-code-memo-not-adopted-20260910.patch -whitespace diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 0c721babe..86d472a21 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -207,7 +207,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: ["3.13", "3.14"] + python-version: ["3.13", "3.14.4"] steps: - uses: actions/checkout@v4 - uses: astral-sh/setup-uv@v6 @@ -220,6 +220,70 @@ jobs: mapfile -t files < <(uv run --no-sync python tools/ci_test_groups.py --list shared-spec) /usr/bin/time -v uv run --no-sync pytest "${files[@]}" -p no:cacheprovider + spec-seed-diagnostics: + # Produce bounded diagnosis immediately, independently of the full suite. + # This remains required by ci-ok and never updates assertion goldens. + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: astral-sh/setup-uv@v6 + with: + python-version: "3.14.4" + - name: Sync workspace with US and UK engines + run: uv sync --all-packages --locked --extra us --extra uk + - name: Check diagnostic bootstrap and failure reporting + timeout-minutes: 2 + env: + PYTHONDONTWRITEBYTECODE: "1" + PYTEST_DISABLE_PLUGIN_AUTOLOAD: "1" + run: >- + uv run --no-sync pytest --noconftest -p no:cacheprovider -q + packages/microcosm-build/tests/test_spec_seed_identity_cpu_bootstrap.py + packages/microcosm-build/tests/test_spec_seed_identity_outer_failure.py + packages/microcosm-build/tests/test_spec_seed_identity_owned_temp.py + packages/microcosm-build/tests/test_spec_seed_identity_system_metadata.py + packages/microcosm-build/tests/test_spec_seed_identity_per_code_context.py + - name: Derive candidate spec and seed identities (diagnostic only) + id: spec_seed_diagnostics + timeout-minutes: 15 + env: + DIAG_WORKFLOW_SHA: ${{ github.workflow_sha }} + DIAG_PR_HEAD_SHA: ${{ github.event.pull_request.head.sha }} + DIAG_PR_BASE_SHA: ${{ github.event.pull_request.base.sha }} + DIAG_MERGE_SHA: ${{ github.event.pull_request.merge_commit_sha }} + PYTHONDONTWRITEBYTECODE: "1" + PYTEST_DISABLE_PLUGIN_AUTOLOAD: "1" + OMP_NUM_THREADS: "1" + OPENBLAS_NUM_THREADS: "1" + MKL_NUM_THREADS: "1" + VECLIB_MAXIMUM_THREADS: "1" + NUMEXPR_NUM_THREADS: "1" + BLIS_NUM_THREADS: "1" + POPULACE_FIT_N_JOBS: "1" + POPULACE_FIT_PREDICT_WORKERS: "1" + run: | + DIAG_CHECKOUT_SHA="$(git rev-parse --verify HEAD)" + DIAG_UV_VERSION="$(uv --version)" + export DIAG_CHECKOUT_SHA DIAG_UV_VERSION + uv run --no-sync python tools/spec_seed_identity_diagnostics.py \ + --output-dir "$RUNNER_TEMP/spec-seed-diagnostics" + - name: Retain bounded candidate identity evidence + if: ${{ always() && steps.spec_seed_diagnostics.outcome != 'skipped' }} + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + with: + name: spec-seed-diagnostics-${{ github.run_id }}-${{ github.run_attempt }}-py3.14.4 + path: | + ${{ runner.temp }}/spec-seed-diagnostics/candidate-digests.json + ${{ runner.temp }}/spec-seed-diagnostics/seed-protocol.json + ${{ runner.temp }}/spec-seed-diagnostics/seed-map.json + ${{ runner.temp }}/spec-seed-diagnostics/seed-bindings.json + ${{ runner.temp }}/spec-seed-diagnostics/environment-and-source.json + ${{ runner.temp }}/spec-seed-diagnostics/diagnostic-status.json + if-no-files-found: error + retention-days: 3 + compression-level: 0 + include-hidden-files: false + engine-us: needs: changes if: github.event_name == 'push' || needs.changes.outputs.us == 'true' || needs.changes.outputs.shared == 'true' @@ -336,6 +400,43 @@ jobs: PY - name: Spec identity section digests (cross-environment diffing aid) run: env -u PYTHONPATH /tmp/wheels-venv/bin/python -I tools/spec_envelope_digests.py be uk + - name: Add the engine-free US extra the source runtimes require + # microunit is microcosm-frame's `us` extra: the sanctioned tax-unit + # constructor, and the only optional dependency the US source runtimes + # need without a country engine. Every ACS/CREATE stage names it in + # us_runtime.graph_implementation.STAGE_DEPENDENCIES, so an installed + # constellation without it cannot hash those stages: the suite then + # fails closed (SOURCE_PREPARATION_REFUSED / CATALOGUE_ISSUANCE_REFUSED) + # instead of skipping, which is a false packaging verdict. + # + # This runs after the boundary check above, so that check still proves + # the base wheels import with no optional dependency installed at all. + # Requesting the extra against the built wheel — rather than naming + # `microunit` here — keeps the extra declared only in + # packages/microcosm-frame/pyproject.toml and makes this gate prove the + # wheel carries resolvable extra metadata. policyengine-us stays absent: + # it is declared by microcosm-build[us] and microcosm-frame[policyengine], + # and neither is requested. + run: | + frame_wheel="$(ls dist/microcosm_frame-*.whl)" + uv pip install --python /tmp/wheels-venv/bin/python \ + --constraint /tmp/wheel-constraints.txt \ + "microcosm-frame[us] @ file://$(realpath "$frame_wheel")" + env -u PYTHONPATH /tmp/wheels-venv/bin/python -I - <<'PY' + from importlib import metadata, util + + assert util.find_spec("microunit") is not None, ( + "microcosm-frame[us] must install microunit into the wheel venv" + ) + assert util.find_spec("policyengine_us") is None + try: + metadata.distribution("policyengine-us") + except metadata.PackageNotFoundError: + pass + else: + raise AssertionError("policyengine-us must be absent from the base wheel gate") + print("microunit", metadata.version("microunit")) + PY - name: Run the suite against the installed wheels run: | /usr/bin/time -v env -u PYTHONPATH /tmp/wheels-venv/bin/python -I -m pytest \ @@ -350,7 +451,7 @@ jobs: # dependency-skip lets a failed `changes` take the engine lanes down without # any required check going red: a lane skip is accepted only when # classification itself succeeded. - needs: [changes, lint, fast, engine-shared, engine-us, engine-uk, wheels] + needs: [changes, lint, fast, engine-shared, spec-seed-diagnostics, engine-us, engine-uk, wheels] if: always() runs-on: ubuntu-latest steps: @@ -365,6 +466,7 @@ jobs: require_success lint "${{ needs.lint.result }}" require_success fast "${{ needs.fast.result }}" require_success engine-shared "${{ needs['engine-shared'].result }}" + require_success spec-seed-diagnostics "${{ needs['spec-seed-diagnostics'].result }}" require_success wheels "${{ needs.wheels.result }}" for lane in "engine-us:${{ needs['engine-us'].result }}" "engine-uk:${{ needs['engine-uk'].result }}"; do name="${lane%%:*}"; result="${lane#*:}" diff --git a/CLAUDE.md b/CLAUDE.md index 68109e01d..d5e44c2a3 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -39,6 +39,12 @@ shard's real wheel, install into a clean uv-export-constrained venv, assert the wheel/import boundary and spec digests, and run the suite against installed wheels. +The shared engine lane also runs a bounded source/seed identity diagnostic on +Python 3.14.4 after its normal tests, including when those tests fail. Its six +JSON artifacts retain candidate digests and their complete canonical seed +records for review. They never replace the test assertions or certify coverage; +review the captured source and dependency identities before updating any pins. + New commits to a PR cancel older unfinished CI runs for that same PR. Each main-push run has a unique concurrency group, so all main-push runs remain independent and can finish validating their merged changes. diff --git a/PROGRESS.md b/PROGRESS.md index ae184206e..bdd76df6e 100644 --- a/PROGRESS.md +++ b/PROGRESS.md @@ -1,3 +1,60 @@ +# #893 reconciliation to main's amended graph interface (amendments 19 and 20) + +Lane: `microcosm-us-launch-verified-lanes-20260910` (the live integration +worktree for PR #893, branch `microcosm-us-launch-integration-20260909`). +Started 2026-09-12 at `069d5ed9a`. Everything below the `---` rule at the end +of this section is prior-lane history; see "Root journals are history, not +state" in `CLAUDE.md`. + +## State + +Merges done, four graph pieces re-applied, graph package green (518 passed), +fit green (191), spec check / groups verify / ruff clean. Build-side consumer +suites running at the time of this entry; final results in `out.md` §5 and +`experiments/893-reconciliation-amendments-19-20-20260912.md`. Nothing +pushed, no PR, no new branch, `uv.lock` untouched. + +## Done + +- Read `CLAUDE.md`, the charter's "Interface freeze" at `23ba24770`, PR + #893's body, the Amendment 19 lane's "Scope" list; diffed the graph package + against `23ba24770` per file; AST-scanned every non-graph consumer. +- `uv sync --all-packages --locked --extra us --extra uk` exit 0. Baseline + graph suite: 5 failed / 362 passed (the five the brief names). +- `3010b7788` merge `origin/main`: graph package, fit sources, lock and + charter resolved to main's bytes; branch-only `attachments.py`, + `availability.py`, `schema.py` and the two branch-only graph tests removed + in the merge, to return only where consumed. +- `051357909` merge `23ba24770` (amendment 20, merged from the branch head + because #912 was still on CI; the dispatcher re-runs `git merge origin/main` + after it lands — expected no-op for graph/fit/lock/charter). +- `dc621c14c` seed digests re-pinned (the branch's `acs_transfer` and + `housing_inputs` plus amendment 20's `fit.qrf` move them); coverage report + regenerated; `--check` exit 0. +- `cff8fbf32` calibrate/simulate H1 pins re-recorded (the branch's solver and + `Frame.__reduce__` changes move them); parity files 27 passed. +- Pieces re-applied on main's files, each with graph-level tests the branch + never had: `752ab840f` raw-byte codec (13 US consumers; 8 codec tests), + `db1b7821a` Frame-metadata store (3 named consumers; 274 passed across the + store/population/executor/manifest files), `d1019762b` execution states + (the post-clone geography gate's typed artifact; replaces the amendment-19 + gate refusal and its test; whole graph package 518 passed), `a16eeacf8` + population observer (4 consumers; 94 passed on the executor + B/F files). +- Dropped for lack of a consumer: lazy retention (`attachments.py`, layer + 08), `schema.py`, `keys._stream_file`, the `_write_node` refactor. +- Layer map computed from `git log --name-only` attribution plus an AST + import scan with hard / name-hard / soft link classes; written to `out.md`. + +## Next + +- Record the build-side consumer results in `out.md` §5, commit the + `experiments/` copy of the report, leave `out.md` uncommitted (it is + another lane's tracked report; see the memory note). +- For Max: piece C supersedes amendment 19's gate refusal (`out.md` §8.1); + the dropped lazy retention / `_stream_file` / `_write_node` pieces (§8.2–3). + +--- + # Amendment 19 — typed opaque artifacts on the graph interface Lane: `amend-typed-artifacts`, off `origin/main` at `3094bfe84`. Started @@ -960,3 +1017,65 @@ The still-earlier PolicyEngine-US 1.819.0 lock-bump lane merged into `origin/main` at `7b90bb18` on 2026-08-24; its final state remains at commit `05d254aa` and its detailed receipts remain in the historical section of `_LANE-NOTES.md`. + +## US launch integration staging — 2026-09-09 + +### State +Source-only staging in progress. Execution and source/data admission remain root-owned. + +### Done +Verified requested clean branch, base HEAD, main ancestry and preservation pins. + +### Next +Apply thirteen explicit source layers, commit each, then separately review isolated ordinary execution. Existing journal history above is retained. + +Layer 1: SAFE-ADDITIVE.patch applied; all declared postimages and preservation hashes verified. No tests executed. + +Layer 2: GRAPH-RESTORE.patch applied; all declared postimages and preservation hashes verified. No tests executed. + +Layer 3: ACCEPTED-SHARED-RESTORE.patch applied; all declared postimages and preservation hashes verified. No tests executed. + +Layer 4: PUF-SUPPORT-MERGE.patch applied; all declared postimages and preservation hashes verified. No tests executed. + +Layer 5: SOLVE-MERGE-PROPOSAL.patch applied; all declared postimages and preservation hashes verified. No tests executed. + +Layer 6: J-GRAPH-COMPATIBILITY.patch applied; all declared postimages and preservation hashes verified. No tests executed. + +Layer 7: F-CATALOGUE-OPTIMIZATION.patch applied; all declared postimages and preservation hashes verified. No tests executed. + +Layer 8: GRAPH-ATTACHMENT-METADATA.patch applied; all declared postimages and preservation hashes verified. No tests executed. + +Layer 9: F-JOINT-GEOGRAPHY-GATE.patch applied; all declared postimages and preservation hashes verified. No tests executed. + +Layer 10: SOURCE-CLOSURE.patch applied; all declared postimages and preservation hashes verified. No tests executed. + +Layer 11: PLACEMENT-ADDITIONS.patch applied; all declared postimages and preservation hashes verified. No tests executed. + +Layer 12: ORDINARY-CLOSURE.patch applied; all declared postimages and preservation hashes verified. No tests executed. + +Layer 13: INTEGRATION-REGRESSIONS.patch applied; all declared postimages and preservation hashes verified. No tests executed. + +### State +All thirteen source layers staged and committed; behavioral qualification pending. + +### Done +Per-layer pins and actual commit messages checked; store/current-main preservation retained. + +### Next +Root reviews exact ordinary guard/source/resource admissions before execution. Full65 findings and survey/SCF lanes remain separately owned. No remote action is authorized. + +### Integration source-resource closure — 2026-09-09 + +Root preflight found seven JSON source definitions declared by the ordinary guard but omitted by the staged patch delivery. Added the exact previously reviewed resource bytes from the accepted full65 source projection; no new resource admission or genuine payload. Preserved source staging and earlier evidence. + +## Layer14 full65 replay correction — 2026-09-09 + +State: exact accepted two-file correction staged; successor57 integration execution pending. +Done: verified clean84243 preimages, exact r2 postimages, fixed store/current-main sources and frozen36 evidence. +Next: root admits the separate exact-allowlist57-case guard and final source identities before execution; no new resources. + +## Source review publication — 2026-09-09 + +The user explicitly authorized pushing the current source work and creating PRs for Anthony to review. This supersedes earlier source-only local restrictions for source publication; it does not authorize a data release, merge or deployment. + +Integration controls passed 36 cases; the exact replay correction subsequently passed all 57 cases at dfa7f872cd3eba3c42adf5758cde8b3ca38f3d17. Source/control, model-declaration and resource hashes matched externally after both runs. A final formatting/import cleanup and explicit test-observer loop binding are included for CI; no release result is claimed. See docs/us-launch-review.md for current scope, evidence and related PRs. diff --git a/changelog.d/20260906-frame-metadata-storage.fixed.md b/changelog.d/20260906-frame-metadata-storage.fixed.md new file mode 100644 index 000000000..dfa6dd022 --- /dev/null +++ b/changelog.d/20260906-frame-metadata-storage.fixed.md @@ -0,0 +1 @@ +Preserve complete Frame metadata in graph content-store format v2, including nested source evidence, and refuse older frame caches that cannot restore it or reuse of one frame key with different metadata. diff --git a/changelog.d/893-age-count-fixture.internal.md b/changelog.d/893-age-count-fixture.internal.md new file mode 100644 index 000000000..1e691423f --- /dev/null +++ b/changelog.d/893-age-count-fixture.internal.md @@ -0,0 +1 @@ +Use the maintained Frame store API in the national age-count cold and fresh-process replay tests after removal of the transfer example module. diff --git a/changelog.d/893-asec-prepared-resources.fixed.md b/changelog.d/893-asec-prepared-resources.fixed.md new file mode 100644 index 000000000..95dc74d99 --- /dev/null +++ b/changelog.d/893-asec-prepared-resources.fixed.md @@ -0,0 +1 @@ +Restore three pinned ASEC resource files omitted during integration so prepared and composed source stages can compute their implementation manifests. Preserve the existing empty engine-default allowlist, source declarations, and reported-income contract. diff --git a/changelog.d/893-diagnostic-current-lock.fixed.md b/changelog.d/893-diagnostic-current-lock.fixed.md new file mode 100644 index 000000000..cff257d2e --- /dev/null +++ b/changelog.d/893-diagnostic-current-lock.fixed.md @@ -0,0 +1 @@ +Align the source/seed diagnostic with the lockfile already approved for the primary QRF worker after the UK openpyxl dependency declaration. diff --git a/changelog.d/893-diagnostic-refusal-context.internal.md b/changelog.d/893-diagnostic-refusal-context.internal.md new file mode 100644 index 000000000..c6e330a2f --- /dev/null +++ b/changelog.d/893-diagnostic-refusal-context.internal.md @@ -0,0 +1 @@ +Record bounded code-path context for refused CI spec diagnostics while preserving the existing read restrictions and incomplete result status. diff --git a/changelog.d/893-historical-zero-provenance.internal.md b/changelog.d/893-historical-zero-provenance.internal.md new file mode 100644 index 000000000..5c7d83751 --- /dev/null +++ b/changelog.d/893-historical-zero-provenance.internal.md @@ -0,0 +1 @@ +Keep the exact historical ASEC zero-origin citations in the dependency scan while continuing to reject other incumbent data-package references in that module. diff --git a/changelog.d/893-population-observer-isolation.fixed.md b/changelog.d/893-population-observer-isolation.fixed.md new file mode 100644 index 000000000..710af6485 --- /dev/null +++ b/changelog.d/893-population-observer-isolation.fixed.md @@ -0,0 +1 @@ +Give graph population observers detached snapshots so callback mutations and retained references cannot change subsequent calculations or cached populations. Preserve the complete frame, provenance, weights and mass accounting on both cold runs and cache hits. diff --git a/changelog.d/893-population-only-block-source.added.md b/changelog.d/893-population-only-block-source.added.md new file mode 100644 index 000000000..09da0ddba --- /dev/null +++ b/changelog.d/893-population-only-block-source.added.md @@ -0,0 +1 @@ +Normalize pinned population-only Census API responses into atomic block support. Reconcile every positive block against independent state totals and CD/PUMA mappings, retain exact source bytes and request descriptions, and keep publisher qualification separate from byte integrity. diff --git a/changelog.d/893-source-demographic-conditioning.added.md b/changelog.d/893-source-demographic-conditioning.added.md new file mode 100644 index 000000000..88bc7fb66 --- /dev/null +++ b/changelog.d/893-source-demographic-conditioning.added.md @@ -0,0 +1 @@ +Add optional source-qualified sex and state predictors to survey financial imputation. Preserve observed-year distinctions and default behavior, refuse unresolved source demographics, and verify complete population preservation through fitting, attachment and required replay. diff --git a/changelog.d/893-spec-identity-diagnostics.internal.md b/changelog.d/893-spec-identity-diagnostics.internal.md new file mode 100644 index 000000000..3eb49f5a0 --- /dev/null +++ b/changelog.d/893-spec-identity-diagnostics.internal.md @@ -0,0 +1 @@ +Capture bounded spec and seed identity evidence after shared CI tests so changed source fingerprints can be reviewed without weakening coverage assertions. diff --git a/changelog.d/893-survey-atomic-prefix.added.md b/changelog.d/893-survey-atomic-prefix.added.md new file mode 100644 index 000000000..30658e355 --- /dev/null +++ b/changelog.d/893-survey-atomic-prefix.added.md @@ -0,0 +1 @@ +Connect qualified survey geography to shared Census-block assignment before cloning and age calibration. Preserve raw sampling allocation separately, reconstruct geography in calibration budgets, and verify complete geography inheritance through cold execution and required replay. Add bounded source-byte normalization controls and record the remaining native-source and release checks. diff --git a/changelog.d/893-uk-atomic-area-and-lazy-exports.added.md b/changelog.d/893-uk-atomic-area-and-lazy-exports.added.md new file mode 100644 index 000000000..9614ce4df --- /dev/null +++ b/changelog.d/893-uk-atomic-area-and-lazy-exports.added.md @@ -0,0 +1 @@ +Add supplied-source UK atomic-area assignment keyed to stable post-clone household identity, and load UK runtime exports only when requested. diff --git a/changelog.d/893-worker-resource-trace.fixed.md b/changelog.d/893-worker-resource-trace.fixed.md new file mode 100644 index 000000000..a48093a90 --- /dev/null +++ b/changelog.d/893-worker-resource-trace.fixed.md @@ -0,0 +1 @@ +Update the worker resource identity regression to follow the actual lazy import closure, and verify that opened JSON resources affect identity while unlisted resources do not. diff --git a/changelog.d/common-frame-export-contract.added.md b/changelog.d/common-frame-export-contract.added.md new file mode 100644 index 000000000..ea5fadddb --- /dev/null +++ b/changelog.d/common-frame-export-contract.added.md @@ -0,0 +1 @@ +Add supplied-parent comparison for US full, pruned and local frame exports, preserving retained inputs and missingness through checkpoint readback. diff --git a/changelog.d/fiscal-measurement-predicate-and-math-identity.fixed.md b/changelog.d/fiscal-measurement-predicate-and-math-identity.fixed.md new file mode 100644 index 000000000..8f4a31c03 --- /dev/null +++ b/changelog.d/fiscal-measurement-predicate-and-math-identity.fixed.md @@ -0,0 +1 @@ +Bind fiscal measurement caches to target arithmetic and engine table materialization, refuse missing consumed predicate inputs, and count boolean terms arithmetically in shared sum expressions. diff --git a/changelog.d/puf55-canonical-donor.added.md b/changelog.d/puf55-canonical-donor.added.md new file mode 100644 index 000000000..10e06ae48 --- /dev/null +++ b/changelog.d/puf55-canonical-donor.added.md @@ -0,0 +1 @@ +Add a canonical59 artifact adapter for the explicit PUF55 survey Social Security profile, retaining return identity and weights and validating the total-carrier convention. Record the scoped passing native seven-node age-development run. diff --git a/changelog.d/puf55-checked-output.added.md b/changelog.d/puf55-checked-output.added.md new file mode 100644 index 000000000..c5fe0d86c --- /dev/null +++ b/changelog.d/puf55-checked-output.added.md @@ -0,0 +1 @@ +Retain PUF55 output handles for source, producer, store, and complete-population rechecks before downstream consumption, revoking a handle after a failed check without rerunning imputation. diff --git a/changelog.d/puf55-survey-host.added.md b/changelog.d/puf55-survey-host.added.md new file mode 100644 index 000000000..f0e410b85 --- /dev/null +++ b/changelog.d/puf55-survey-host.added.md @@ -0,0 +1 @@ +Add the PUF55 host over the combined ACS/ASEC support-clone population, with source-qualified Social Security routes, complete population preservation and required graph replay. diff --git a/changelog.d/puf55-survey-social-security.added.md b/changelog.d/puf55-survey-social-security.added.md new file mode 100644 index 000000000..ba625b57e --- /dev/null +++ b/changelog.d/puf55-survey-social-security.added.md @@ -0,0 +1 @@ +Add an explicit 55-output PUF profile that preserves survey Social Security components and requires a separate total as its ninth conditioning predictor. Validate canonical donor projection and whole-Population attachment/replay while retaining source identities, weights and Social Security unknownness. diff --git a/changelog.d/shared-atomic-geography.added.md b/changelog.d/shared-atomic-geography.added.md new file mode 100644 index 000000000..4eed5d1a9 --- /dev/null +++ b/changelog.d/shared-atomic-geography.added.md @@ -0,0 +1 @@ +Add shared graph operators for atomic-area assignment and geographic derivation. Country declarations choose observed constraints, stable draw identities, sampling stages and versioned mapping relations. Preserve observed locations, reject ambiguous support, and verify mapping consistency on cloned or pruned populations. Existing country builds retain their current geography paths pending native support integration. Normalize US block mappings, qualify observed ACS/ASEC geography against source receipts, and compose shared assignment before the combined-survey clone, with an inherited-mapping gate afterward. diff --git a/changelog.d/signed-income-reconciliation.added.md b/changelog.d/signed-income-reconciliation.added.md new file mode 100644 index 000000000..428496f2c --- /dev/null +++ b/changelog.d/signed-income-reconciliation.added.md @@ -0,0 +1,2 @@ +Add a deterministic weighted projection onto a signed total with explicit +nonnegative component bounds, adjustment diagnostics and floating-point checks. diff --git a/changelog.d/survey-social-security-source.added.md b/changelog.d/survey-social-security-source.added.md new file mode 100644 index 000000000..11d76e2dd --- /dev/null +++ b/changelog.d/survey-social-security-source.added.md @@ -0,0 +1 @@ +Add current survey Social Security source qualification with explicit reporting units, question-universe unknownness, allocation literals and component mapping judgments. diff --git a/changelog.d/us-atomic-financial-composition.added.md b/changelog.d/us-atomic-financial-composition.added.md new file mode 100644 index 000000000..a28385f6c --- /dev/null +++ b/changelog.d/us-atomic-financial-composition.added.md @@ -0,0 +1 @@ +Compose survey atomic geography and current financial imputation in a checked twenty-node graph, retaining geography through cloning and financial attachment. Add native Delaware population-only source normalization and complete support readback controls. diff --git a/changelog.d/us-current-acs-income-anchor-source.added.md b/changelog.d/us-current-acs-income-anchor-source.added.md new file mode 100644 index 000000000..d8d6fce32 --- /dev/null +++ b/changelog.d/us-current-acs-income-anchor-source.added.md @@ -0,0 +1,3 @@ +Qualify original ACS property and retirement income anchors through the retained +survey source owner, preserving literal missingness, allocation, age universe and +signed values while verifying raw identities and adjusted amount bits. diff --git a/changelog.d/us-current-survey-health-coverage.added.md b/changelog.d/us-current-survey-health-coverage.added.md new file mode 100644 index 000000000..5443947b8 --- /dev/null +++ b/changelog.d/us-current-survey-health-coverage.added.md @@ -0,0 +1,4 @@ +Add a source-qualified health coverage graph fragment for the common US survey +frame and its PUF clones. Preserve literal source and allocation codes, explicit +unknowns, and incoming amount fields; keep broader ACS concepts as documented +gaps instead of manufacturing narrower coverage flags. diff --git a/changelog.d/us-declared-fiscal-measurement.added.md b/changelog.d/us-declared-fiscal-measurement.added.md new file mode 100644 index 000000000..58d36e459 --- /dev/null +++ b/changelog.d/us-declared-fiscal-measurement.added.md @@ -0,0 +1 @@ +Add a candidate-only US graph measurement stage using declared target bindings, household-aligned sparse measures, explicit national/state/district support checks, and complete input-closure checks for optional model evaluation. diff --git a/changelog.d/us-fiscal-dense-calibration.added.md b/changelog.d/us-fiscal-dense-calibration.added.md new file mode 100644 index 000000000..497b5415f --- /dev/null +++ b/changelog.d/us-fiscal-dense-calibration.added.md @@ -0,0 +1 @@ +Add a numerical fiscal graph calibration stage using existing grouped Adam, explicit original weight and origin bounds, exact household alignment, and target and origin diagnostics without claiming source or release admission. diff --git a/changelog.d/us-launch-integration.fixed.md b/changelog.d/us-launch-integration.fixed.md new file mode 100644 index 000000000..170634a55 --- /dev/null +++ b/changelog.d/us-launch-integration.fixed.md @@ -0,0 +1 @@ +Preserve grouped calibration closing-state results alongside ordinary best-iterate selection, retain the combined US runtime facade and joint geography validation, and publish complete metadata for lazy graph snapshots. diff --git a/changelog.d/us-launch-store-grouped-validation.fixed.md b/changelog.d/us-launch-store-grouped-validation.fixed.md new file mode 100644 index 000000000..c341c76e2 --- /dev/null +++ b/changelog.d/us-launch-store-grouped-validation.fixed.md @@ -0,0 +1 @@ +Reject nonfinite stored JSON as a store-corruption error and normalize the deprecated calibration method alias before validating grouped constraints. diff --git a/changelog.d/us-national-atomic-support.added.md b/changelog.d/us-national-atomic-support.added.md new file mode 100644 index 000000000..085c1627a --- /dev/null +++ b/changelog.d/us-national-atomic-support.added.md @@ -0,0 +1,4 @@ +Add an opt-in national Census population-only source control covering the fifty +states and DC, with independent block/state reconciliation, complete CD119 and +PUMA joins, and full support serialization/readback. Record the accepted native +normalization separately from household assignment and dataset release checks. diff --git a/changelog.d/us-puf55-canonical-and-output-seals.added.md b/changelog.d/us-puf55-canonical-and-output-seals.added.md new file mode 100644 index 000000000..0e6679803 --- /dev/null +++ b/changelog.d/us-puf55-canonical-and-output-seals.added.md @@ -0,0 +1,4 @@ +Add a source-bound PUF55 canonical donor graph node with cold/replay controls, +stable live-code identity checks, and finalized-candidate seals for the shared +two-route numerical finalizer. Complete native survey attachment and release +validation remain separate build steps. diff --git a/changelog.d/us-puf55-survey-ss-measurement.added.md b/changelog.d/us-puf55-survey-ss-measurement.added.md new file mode 100644 index 000000000..ebe159455 --- /dev/null +++ b/changelog.d/us-puf55-survey-ss-measurement.added.md @@ -0,0 +1,5 @@ +Add an authenticated filer/joint-spouse Social Security report-sum measurement +for PUF55 conditioning, with an explicit eight-predictor fallback for incomplete +reports and preservation of all survey Social Security cells and masks. Record +31 passing invented controls and the remaining recipient integration and +measurement limitations. diff --git a/changelog.d/us-source-string-seal-efficiency.changed.md b/changelog.d/us-source-string-seal-efficiency.changed.md new file mode 100644 index 000000000..a0c33268e --- /dev/null +++ b/changelog.d/us-source-string-seal-efficiency.changed.md @@ -0,0 +1 @@ +Reuse bounded, column-local encodings for repeated source strings during ASEC frame hashing while preserving the exact digest byte stream and checking all values on every validation. diff --git a/changelog.d/us-survey-catalogue-digest.changed.md b/changelog.d/us-survey-catalogue-digest.changed.md new file mode 100644 index 000000000..96d1c8d13 --- /dev/null +++ b/changelog.d/us-survey-catalogue-digest.changed.md @@ -0,0 +1 @@ +Stream eligible ACS catalogue seals through bounded record JSON encoding while preserving canonical digest bytes, full-value fallback, and repeated source and owner validation. Bind the additional JSON provider calls into the existing preparation authority checks. diff --git a/changelog.d/us-survey-copy-and-identity.fixed.md b/changelog.d/us-survey-copy-and-identity.fixed.md new file mode 100644 index 000000000..76d9520fa --- /dev/null +++ b/changelog.d/us-survey-copy-and-identity.fixed.md @@ -0,0 +1 @@ +Keep frame metadata copying and pickling from changing producer identity on Python 3.14. Refresh ACS source pins after verified formatting-only changes, and reduce per-cell survey identity encoding overhead while preserving canonical bytes, mutation checks, and the current-wage projection. diff --git a/changelog.d/us-survey-current-amount-enrichment.added.md b/changelog.d/us-survey-current-amount-enrichment.added.md new file mode 100644 index 000000000..dab37b769 --- /dev/null +++ b/changelog.d/us-survey-current-amount-enrichment.added.md @@ -0,0 +1 @@ +Add a checked US survey enrichment host that preserves the PUF population while attaching source-qualified unemployment, medical costs and current health coverage. Keep reporting unknowns, original observations and source-to-clone provenance explicit. diff --git a/changelog.d/us-survey-diagnostic-options.fixed.md b/changelog.d/us-survey-diagnostic-options.fixed.md new file mode 100644 index 000000000..e8668b7e0 --- /dev/null +++ b/changelog.d/us-survey-diagnostic-options.fixed.md @@ -0,0 +1 @@ +Reconstruct the grouped calibration solver's closing-state selection options during survey diagnostic verification, preserving exact diagnostic comparison and late-mutation checks. diff --git a/changelog.d/us-survey-interest-conservation.fixed.md b/changelog.d/us-survey-interest-conservation.fixed.md new file mode 100644 index 000000000..b58d850b5 --- /dev/null +++ b/changelog.d/us-survey-interest-conservation.fixed.md @@ -0,0 +1 @@ +Retain the tax-exempt remainder when the survey financial graph splits observed ASEC or modeled ACS interest totals, preserving both parts on the original survey support before PUF enrichment. diff --git a/docs/current-survey-puf59-progress.md b/docs/current-survey-puf59-progress.md new file mode 100644 index 000000000..a4351354a --- /dev/null +++ b/docs/current-survey-puf59-progress.md @@ -0,0 +1,72 @@ +# PUF59 and current survey predictor implementation status + +The PUF59 profile fits and transfers 56 person outcomes and three tax-unit outcomes. Six detailed mortgage fields remain owned by the separate SCF producer. The historical full65 default remains available. PUF59 conditions on the source-specific PUF2015 filing-status class and capped return-size proxy, plus six monetary predictors. Its self-employment predictor includes ordinary and SSTB Schedule C income. Return-level Boolean outputs use an explicit incidence capacity of one; that is not a physical person count. + +The local PUF59/full65 regression run passed 128 ordinary tests. Those tests execute real weighted QRF fits, sequential target conditioning, draws, finalization, whole-Population preservation and required cache replay. They include excluded mortgage incumbent preservation and source-specific predictor checks. The run took 93.9 seconds and peaked at 622.7 MB RSS. These are invented mechanism fixtures, not held-out fit-quality or launch acceptance. + +Current survey financial completion passed 26 ordinary tests, including 15 ASEC demographic projection controls. The financial graph uses original ASEC design weights to fit interest, dividends and capital-gains outcomes for native ACS people. It preserves ACS observed wage/self-employment amounts and source-owned age-15 universe zeros. A draw is shared with its clone by source identity. The graph retains typed preparation, allocation and context evidence while separating the donor branch from survey allocation, avoiding a structural dependency cycle. Cold execution and required replay passed with the actual fitted model artifacts. The combined run took 121.4 seconds and peaked at 524.0 MB RSS. + +The source projection records the distinct survey observation, income and price periods. ASEC sex and state come from the retained original person and household sources, with allocation flags and unknownness preserved. Attaching those demographic projections to the graph is a separate pending integration step. The current financial model conditions on age and two earnings fields; sex/state conditioning and held-out fit quality remain explicit modeling work. + +Social Security handling remains a launch blocker. The historical PUF59 donor transports its total through the retirement component with modeled zero carrier columns for the other components. Those zeros do not establish absent recipient benefits. Its reconciliation code can treat unknown recipient components as zero and use equal component shares. The explicit `PUF55_SURVEY_SS` profile now excludes these four outcomes from PUF enrichment and conditions the remaining 55 outcomes on an additional Social Security total predictor. It preserves existing survey component fields, including unknownness. This profile is not yet wired into the native build. + +The new profile requires a known, finite, nonnegative `tax_unit.puf_conditioning_social_security_total` supplied by an upstream source owner. It does not infer that total from unknown components or move it between entity grains. All eight historical PUF59 predictors retain their order, with the total appended as predictor nine. The profile has a distinct train/apply phase and retains return-level incidence-capacity validation. Historical full65/PUF59 defaults and their 65/59 outcome contracts remain unchanged. + +Eleven new controls pass with real 55-target QRF fits, finalization, raw-draw validation and required cache replay. They check exact Social Security incumbent bit patterns, including signed zero and NaN, and refuse changes to the conditioning total after fitting. The run took 28.5 seconds and peaked at 631.6 MB RSS; receipt SHA256 is `8d5beacf15d8eb7bf568705d4e12bd71d810f3e748571b669a6ec44921ebf0cf`. All 128 historical profile cases also pass against the identical production module. An earlier combined run retained a failure in the new test's comparison of Boolean storage types; the corrected eleven-case run preserves exact values and missingness while treating the requested PUF Boolean output's nullable storage explicitly. Native recipient total qualification, beneficiary completion and whole-population PUF55 attachment remain separate pending checks. The native donor projection now passes as described below. + +The additive current-source qualifier now verifies original ASEC total, reason and allocation literals against the live survey preparation and retained original member. It also binds the ACS 2024 issuance and SSP price adjustment. Both surveys preserve under-15 question-universe unknownness. ASEC may record combined family payments on one person's report; a reason code is not a separately observed component amount or an assignment to individual beneficiaries. See [Social Security source semantics](survey-social-security-source.md) for the source contract, judgment boundaries and unresolved model work. + +The qualifier and numerical basis pass 52 controls: 50 numerical/literal cases and two actual maintained source issuances on invented original inputs. They include reordered source identities, ambiguous reasons, allocation flags, exact requalification, unknownness and no input mutation. The corrected run took 67.6 seconds and peaked at 428.9 MB RSS; the receipt SHA256 is `2b9d8e45cec11a6ebda2ccc01c5f12374e794ec0c006ba6525f082fd6454049b`. An earlier run's two source tests exposed an incorrect capture-budget argument; the corrected caller uses the source owner's existing decoded-body bound while retaining exact CSV byte and digest checks. No native Social Security completion or beneficiary allocation has been accepted. + +The combined current source also passes all **225 component integration controls**: 34 shared geography cases, 52 survey Social Security cases, 11 PUF55 cases and all 128 historical full65/PUF59 cases. The exact combined run completed in 205.6 seconds at 687.8 MB peak RSS with no skips or unexpected access refusals. Before/after/current source, model-source and code-resource checks all agree; the maintained checkout matches the tested source map. Receipt SHA256: `63ef0ad0602a66d40f0736490d4c1e29f57723ba938c58d317802b387f947a87`. This remains an invented-input component integration, not a native population certification. + +The consolidated source passes a 314-case selected integration and identity suite, including the 128 profile cases, 26 financial/demographic cases, two canonical/growth wrappers, five metadata regressions and 153 identity controls. A subsequent diagnostic-option reconstruction correction passes all 16 current seven-node age-development tests, including late-mutation refusals. All source, model-source and resource pins survived external postchecks for each run. See the [review guide](us-launch-review.md) for exact revisions, receipts and exclusions. + +The new canonical59-to-PUF55 donor adapter also passes the maintained invented-source construction and artifact wrapper on 64 and 2,048 returns. It verifies the supplied artifact identity and the exact Social Security carrier convention before projecting the grown total into predictor nine. All 55 retained outputs, source RECIDs and weights survive unchanged. Internally rehashed artifacts with altered carrier semantics refuse. The corrected wrapper passed in 9.4 seconds at 486.9 MB peak RSS; receipt SHA256: `db3e9d8aa91863973de5d3201efcbabe31cbc1067c8c3f7fd72be79d887ffc27`. The initial knownness-column mismatch is preserved as a failed control; the fix supplies the owner's exact nonstructural column roster. This adapter grants no source authority. A subsequent separately guarded native projection passes for all 207,692 returns, retaining every target, source RECID, weight and incidence capacity and appending the modeled Social Security total as predictor nine. It took 11.05 seconds at 1.43 GB peak RSS. The original canonical artifact remained unchanged; source/control/model/code checks agree, with no unexpected access refusals. Receipt SHA256: `ce09455947f0efbca16014a23baa19037ac6450876c639568af6584c81bfc45c`. The earlier attempt passed its value assertions but lacked collection accounting and was refused by the guard; that failure is preserved. This native projection ran in memory and produced metadata only. It did not fit a recipient model or attach results to the survey population. + +The genuine seven-node survey age-development run now passes at the unchanged 1/1000 sampling fraction. It prepares 1,584 households and 3,464 people (ACS: 1,529 households/3,324 people; ASEC: 55/140), then retains 3,168 household records and 6,928 person records after the support clone. Export/readback, final source-owner and target checks completed under the frozen `a9895d8a5` source; the final manifest was written only after those checks. All 473 source/control/runtime and 12 code-resource postchecks agree, with no unexpected access refusals. The run took 4,268.8 seconds and peaked at 8.26 GB RSS. Receipt SHA256: `2aa76b5daca7bb0df1ac85d4db23d6b5482073d31d437418d46442ef01b002ca`. This run does not include the subsequent Social Security, PUF55 or atomic-geography integration. + +This is accepted native age-development evidence, not a national or congressional-district release, native enrichment fit-quality result or country-engine result. Complete enrichment, geography, calibration and release-export acceptance remain pending. + +The PUF55 profile now also passes three invented whole-Population attachment +controls. They execute all 55 fitted outcomes through the actual graph, typed +artifact loader and materialized population verifier, followed by Frame storage +and required replay. Every non-owned field, Social Security bit pattern and +unknown cell, membership, design anchor and household weight remains intact. +Two negative cases reject changed Social Security values and unknown-to-zero +conversions even when predictor matrices remain identical. The run took 21.03 +seconds at 631.3 MB peak RSS; receipt SHA256: +`ad8519d867279ebce9ee78bf8f495f3799afb45ec1b3fcda1be649f8a129af30`. +All 482 source/control, 5,983 model-source and 12 code-resource postchecks agree, +with no skips or unexpected access refusals. A subsequent independent review +required a direct finalizer comparison, because the attachment and its verifier +share a column-construction helper. The strengthened three-case run also passes: +all 55 attached outputs agree with the direct finalizer after independent entity +ID alignment, on cold and required replay. It took 20.73 seconds at 630.5 MB; +receipt SHA256 is +`e3a4d548f0209cfacd7d9e3beddb63c1b46204a3168eb9869d2b2eafe9cf2291`. +All 486 source/control, 5,983 model-source and 12 code-resource postchecks agree. +Native recipient qualification, measurement comparability and actual survey +attachment remain pending. + +The observed-geography source qualifier passes 11 invented original-source +controls. It binds current ASEC state and ACS state/PUMA to literal source +household keys, preserves unknown locations, and keeps draw keys stable across +sampling fractions. The initial ten-case run correctly refused a smaller sample +with no positive ASEC support; its corrected fixture supplies positive support +before issuance. Review also added a receipt-digest check against mutation of +the detached ASEC projection. The passing run took 182.50 seconds at 430.3 MB; +receipt SHA256 is +`1900801b1bbcea89a3747f656410e500b1328519694c3351100562bba61cab08`. +The 484 source/control, 5,983 model-source and 12 code-resource postchecks agree, +with no skips or unexpected refusals. + +The observed-geography graph node separately passes 14 controls over the same +kind of invented original sources. Actual three-node execution and required +replay preserve inputs, weights and unknown geography. Declaration, source +receipt, typed-artifact and final-mutation cases refuse invalid output. The run +took 319.44 seconds at 439.1 MB; receipt SHA256 is +`28a6b5c58a258c7b3129a48054a3360647c788f8b76d781cc3bc8fc35a73f997`. +All 486 source/control, 5,983 model-source and 12 code-resource postchecks agree. +Full geography-before-clone composition, native block-source integration and +complete calibration remain separate work. diff --git a/docs/evidence/puf2015-target2024/GROWTH-RECIPE.json b/docs/evidence/puf2015-target2024/GROWTH-RECIPE.json new file mode 100644 index 000000000..79974b85d --- /dev/null +++ b/docs/evidence/puf2015-target2024/GROWTH-RECIPE.json @@ -0,0 +1,447 @@ +{ + "schema": "microcosm.us.puf2015_target2024_growth/1", + "input_money_year": 2015, + "output_money_year": 2024, + "output_profile": "puf59_scf_mortgage_v1", + "ordered_targets": [ + "employment_income_before_lsr", + "self_employment_income_before_lsr", + "taxable_interest_income", + "qualified_dividend_income", + "non_qualified_dividend_income", + "tax_exempt_interest_income", + "short_term_capital_gains", + "long_term_capital_gains_before_response", + "long_term_capital_gains_on_collectibles", + "non_sch_d_capital_gains", + "taxable_private_pension_income", + "taxable_ira_distributions", + "social_security_retirement", + "social_security_disability", + "social_security_dependents", + "social_security_survivors", + "alimony_income", + "alimony_expense", + "salt_refund_income", + "charitable_cash_donations", + "charitable_non_cash_donations", + "real_estate_taxes", + "home_mortgage_interest", + "investment_interest_expense", + "investment_income_elected_form_4952", + "student_loan_interest", + "educator_expense", + "qualified_tuition_expenses", + "casualty_loss", + "unreimbursed_business_employee_expenses", + "traditional_ira_contributions_desired", + "self_employed_pension_contributions_desired", + "rental_income", + "estate_income", + "farm_income", + "farm_operations_income", + "farm_rent_income", + "miscellaneous_income", + "partnership_income", + "s_corp_income", + "partnership_self_employment_net_earnings", + "estate_income_would_be_qualified", + "farm_operations_income_would_be_qualified", + "farm_rent_income_would_be_qualified", + "partnership_s_corp_income_would_be_qualified", + "rental_income_would_be_qualified", + "self_employment_income_would_be_qualified", + "sstb_self_employment_income_would_be_qualified", + "business_is_sstb", + "qualified_bdc_income", + "qualified_reit_and_ptp_income", + "sstb_self_employment_income_before_lsr", + "sstb_unadjusted_basis_qualified_property", + "sstb_w2_wages_from_qualified_business", + "unadjusted_basis_qualified_property", + "w2_wages_from_qualified_business", + "domestic_production_ald", + "unrecaptured_section_1250_gain", + "health_savings_account_ald" + ], + "money_fields": { + "employment_income_before_lsr": { + "positive_family": "awi", + "negative_family": "awi", + "positive_factor": "1.452153003110483604210764423019948801037", + "negative_factor": "1.452153003110483604210764423019948801037", + "assumption": "Average-worker wage index proxy; modeled QBI W2 amounts follow the same wage index." + }, + "w2_wages_from_qualified_business": { + "positive_family": "awi", + "negative_family": "awi", + "positive_factor": "1.452153003110483604210764423019948801037", + "negative_factor": "1.452153003110483604210764423019948801037", + "assumption": "Average-worker wage index proxy; modeled QBI W2 amounts follow the same wage index." + }, + "sstb_w2_wages_from_qualified_business": { + "positive_family": "awi", + "negative_family": "awi", + "positive_factor": "1.452153003110483604210764423019948801037", + "negative_factor": "1.452153003110483604210764423019948801037", + "assumption": "Average-worker wage index proxy; modeled QBI W2 amounts follow the same wage index." + }, + "self_employment_income_before_lsr": { + "positive_family": "business_profit", + "negative_family": "business_loss", + "positive_factor": "1.222517420922031096142334753675412822692", + "negative_factor": "1.718898116890934040328182671660914217539", + "assumption": "Both modeled Schedule C branches use the same sign-specific family factors, preserving the base/SSTB partition." + }, + "sstb_self_employment_income_before_lsr": { + "positive_family": "business_profit", + "negative_family": "business_loss", + "positive_factor": "1.222517420922031096142334753675412822692", + "negative_factor": "1.718898116890934040328182671660914217539", + "assumption": "Both modeled Schedule C branches use the same sign-specific family factors, preserving the base/SSTB partition." + }, + "taxable_interest_income": { + "positive_family": "taxable_interest", + "negative_family": "taxable_interest", + "positive_factor": "2.599750625636417399355437523686408804619", + "negative_factor": "2.599750625636417399355437523686408804619", + "assumption": "Reporting-return mean transport; 2023\u21922024 stable-real-amount bridge; no unchanged reporting incidence claim." + }, + "tax_exempt_interest_income": { + "positive_family": "tax_exempt_interest", + "negative_family": "tax_exempt_interest", + "positive_factor": "0.9295395824924337172788943063597248917385", + "negative_factor": "0.9295395824924337172788943063597248917385", + "assumption": "Reporting-return mean transport; 2023\u21922024 stable-real-amount bridge; no unchanged reporting incidence claim." + }, + "qualified_dividend_income": { + "positive_family": "qualified_dividends", + "negative_family": "qualified_dividends", + "positive_factor": "1.409891045424454325551259680558160347719", + "negative_factor": "1.409891045424454325551259680558160347719", + "assumption": "Reporting-return mean transport; 2023\u21922024 stable-real-amount bridge; no unchanged reporting incidence claim." + }, + "non_qualified_dividend_income": { + "positive_family": "nonqualified_dividends", + "negative_family": "nonqualified_dividends", + "positive_factor": "2.483795459689869824933006427197633652403", + "negative_factor": "2.483795459689869824933006427197633652403", + "assumption": "Reporting-return mean transport; 2023\u21922024 stable-real-amount bridge; no unchanged reporting incidence claim." + }, + "non_sch_d_capital_gains": { + "positive_family": "non_schedule_d_distributions", + "negative_family": "non_schedule_d_distributions", + "positive_factor": "1.120350061633613230084419890940102481805", + "negative_factor": "1.120350061633613230084419890940102481805", + "assumption": "Reporting-return mean transport; 2023\u21922024 stable-real-amount bridge; no unchanged reporting incidence claim." + }, + "taxable_private_pension_income": { + "positive_family": "taxable_pensions", + "negative_family": "taxable_pensions", + "positive_factor": "1.327588034973940867230285994253866499704", + "negative_factor": "1.327588034973940867230285994253866499704", + "assumption": "Reporting-return mean transport; 2023\u21922024 stable-real-amount bridge; no unchanged reporting incidence claim." + }, + "taxable_ira_distributions": { + "positive_family": "ira_distributions", + "negative_family": "ira_distributions", + "positive_factor": "1.510871073557979130794209158828995676924", + "negative_factor": "1.510871073557979130794209158828995676924", + "assumption": "Reporting-return mean transport; 2023\u21922024 stable-real-amount bridge; no unchanged reporting incidence claim." + }, + "alimony_income": { + "positive_family": "alimony_income", + "negative_family": "alimony_income", + "positive_factor": "1.542033706590010885068081742929926580179", + "negative_factor": "1.542033706590010885068081742929926580179", + "assumption": "Reporting-return mean transport; 2023\u21922024 stable-real-amount bridge; no unchanged reporting incidence claim." + }, + "salt_refund_income": { + "positive_family": "state_tax_refund", + "negative_family": "state_tax_refund", + "positive_factor": "0.9296526775720357466732616002244135476521", + "negative_factor": "0.9296526775720357466732616002244135476521", + "assumption": "Reporting-return mean transport; 2023\u21922024 stable-real-amount bridge; no unchanged reporting incidence claim." + }, + "short_term_capital_gains": { + "positive_family": "short_current_gain", + "negative_family": "short_current_loss", + "positive_factor": "2.042865587775988756504710527637911842775", + "negative_factor": "1.738139193325432816803529458382722438642", + "assumption": "Current-year gross gains/loss components per all return proxy for signed net PUF amount; carried losses are excluded." + }, + "long_term_capital_gains_before_response": { + "positive_family": "long_current_gain", + "negative_family": "long_current_loss", + "positive_factor": "1.283710381837894882785950440755768785471", + "negative_factor": "2.262384673681395979556694950087192296750", + "assumption": "Current-year gross long-term components proxy for signed net amounts and narrower collectibles/1250 subfamilies; carried losses are excluded." + }, + "long_term_capital_gains_on_collectibles": { + "positive_family": "long_current_gain", + "negative_family": "long_current_loss", + "positive_factor": "1.283710381837894882785950440755768785471", + "negative_factor": "2.262384673681395979556694950087192296750", + "assumption": "Current-year gross long-term components proxy for signed net amounts and narrower collectibles/1250 subfamilies; carried losses are excluded." + }, + "unrecaptured_section_1250_gain": { + "positive_family": "long_current_gain", + "negative_family": "long_current_loss", + "positive_factor": "1.283710381837894882785950440755768785471", + "negative_factor": "2.262384673681395979556694950087192296750", + "assumption": "Current-year gross long-term components proxy for signed net amounts and narrower collectibles/1250 subfamilies; carried losses are excluded." + }, + "social_security_retirement": { + "positive_family": "cola", + "negative_family": "cola", + "positive_factor": "1.285886314567344447728640", + "negative_factor": "1.285886314567344447728640", + "assumption": "Cash-year COLA transport of fixed entitlement; carrier component convention remains modeled upstream." + }, + "social_security_disability": { + "positive_family": "cola", + "negative_family": "cola", + "positive_factor": "1.285886314567344447728640", + "negative_factor": "1.285886314567344447728640", + "assumption": "Cash-year COLA transport of fixed entitlement; carrier component convention remains modeled upstream." + }, + "social_security_dependents": { + "positive_family": "cola", + "negative_family": "cola", + "positive_factor": "1.285886314567344447728640", + "negative_factor": "1.285886314567344447728640", + "assumption": "Cash-year COLA transport of fixed entitlement; carrier component convention remains modeled upstream." + }, + "social_security_survivors": { + "positive_family": "cola", + "negative_family": "cola", + "positive_factor": "1.285886314567344447728640", + "negative_factor": "1.285886314567344447728640", + "assumption": "Cash-year COLA transport of fixed entitlement; carrier component convention remains modeled upstream." + }, + "rental_income": { + "positive_family": "rental_royalty_profit", + "negative_family": "rental_royalty_loss", + "positive_factor": "1.476663829685885394438774259073237903799", + "negative_factor": "1.690793641559507564235995133214906035185", + "assumption": "Related rental/royalty reporting-return mean proxy; published total family includes farm rent and does not exactly equal E25850/E25860 component definitions." + }, + "estate_income": { + "positive_family": "estate_profit", + "negative_family": "estate_loss", + "positive_factor": "1.476331638358104936021886589313386353228", + "negative_factor": "1.582810027948011789708036602150526302352", + "assumption": "" + }, + "farm_operations_income": { + "positive_family": "farm_profit", + "negative_family": "farm_loss", + "positive_factor": "1.444720208215047581256304812994467826979", + "negative_factor": "1.728383897337776476443532509645504817056", + "assumption": "Schedule F family; elected farm income T27800 is a selected subfamily proxy, not general farm income." + }, + "farm_income": { + "positive_family": "farm_profit", + "negative_family": "farm_loss", + "positive_factor": "1.444720208215047581256304812994467826979", + "negative_factor": "1.728383897337776476443532509645504817056", + "assumption": "Schedule F family; elected farm income T27800 is a selected subfamily proxy, not general farm income." + }, + "farm_rent_income": { + "positive_family": "farm_rent_profit", + "negative_family": "farm_rent_loss", + "positive_factor": "1.629452810459537374541656575493532799084", + "negative_factor": "1.150378978298943195689490419607276484593", + "assumption": "" + }, + "miscellaneous_income": { + "positive_family": "other_property_gain", + "negative_family": "other_property_loss", + "positive_factor": "1.738242164922618767195190248113271033066", + "negative_factor": "1.411645744283618865226902643798237455207", + "assumption": "Canonical source E01200 is other property gains/losses, not Table1.4 other income." + }, + "partnership_income": { + "positive_family": "partnership_s_corp_profit", + "negative_family": "partnership_s_corp_loss", + "positive_factor": "1.595257264233857504616939193376066035754", + "negative_factor": "1.968942227400735507640732147170530268700", + "assumption": "Combined component dollars per all return; reporting counts overlap and are never summed. Active partnership earnings use this same explicitly related-family proxy." + }, + "s_corp_income": { + "positive_family": "partnership_s_corp_profit", + "negative_family": "partnership_s_corp_loss", + "positive_factor": "1.595257264233857504616939193376066035754", + "negative_factor": "1.968942227400735507640732147170530268700", + "assumption": "Combined component dollars per all return; reporting counts overlap and are never summed. Active partnership earnings use this same explicitly related-family proxy." + }, + "partnership_self_employment_net_earnings": { + "positive_family": "partnership_s_corp_profit", + "negative_family": "partnership_s_corp_loss", + "positive_factor": "1.595257264233857504616939193376066035754", + "negative_factor": "1.968942227400735507640732147170530268700", + "assumption": "Combined component dollars per all return; reporting counts overlap and are never summed. Active partnership earnings use this same explicitly related-family proxy." + }, + "qualified_bdc_income": { + "positive_family": "nonqualified_dividends", + "negative_family": "nonqualified_dividends", + "positive_factor": "2.483795459689869824933006427197633652403", + "negative_factor": "2.483795459689869824933006427197633652403", + "assumption": "Modeled BDC component follows its parent nonqualified dividend pool; no independent source observation." + }, + "qualified_reit_and_ptp_income": { + "positive_family": "cpi", + "negative_family": "cpi", + "positive_factor": "1.323487344789614247079323424058189918867", + "negative_factor": "1.323487344789614247079323424058189918867", + "assumption": "Explicit constant-real proxy for the combined modeled REIT/PTP output, which mixes nonqualified-dividend and pass-through components. No claim that this is a pure dividend pool." + }, + "alimony_expense": { + "positive_family": "cpi", + "negative_family": "cpi", + "positive_factor": "1.323487344789614247079323424058189918867", + "negative_factor": "1.323487344789614247079323424058189918867", + "assumption": "Explicit constant-real-amount proxy. Realized deductions/desires and UBIA are upstream models; this is not a statutory limit, current eligibility, loan turnover, interest-rate, or asset-price model." + }, + "charitable_cash_donations": { + "positive_family": "cpi", + "negative_family": "cpi", + "positive_factor": "1.323487344789614247079323424058189918867", + "negative_factor": "1.323487344789614247079323424058189918867", + "assumption": "Explicit constant-real-amount proxy. Realized deductions/desires and UBIA are upstream models; this is not a statutory limit, current eligibility, loan turnover, interest-rate, or asset-price model." + }, + "charitable_non_cash_donations": { + "positive_family": "cpi", + "negative_family": "cpi", + "positive_factor": "1.323487344789614247079323424058189918867", + "negative_factor": "1.323487344789614247079323424058189918867", + "assumption": "Explicit constant-real-amount proxy. Realized deductions/desires and UBIA are upstream models; this is not a statutory limit, current eligibility, loan turnover, interest-rate, or asset-price model." + }, + "real_estate_taxes": { + "positive_family": "cpi", + "negative_family": "cpi", + "positive_factor": "1.323487344789614247079323424058189918867", + "negative_factor": "1.323487344789614247079323424058189918867", + "assumption": "Explicit constant-real-amount proxy. Realized deductions/desires and UBIA are upstream models; this is not a statutory limit, current eligibility, loan turnover, interest-rate, or asset-price model." + }, + "home_mortgage_interest": { + "positive_family": "cpi", + "negative_family": "cpi", + "positive_factor": "1.323487344789614247079323424058189918867", + "negative_factor": "1.323487344789614247079323424058189918867", + "assumption": "Explicit constant-real-amount proxy. Realized deductions/desires and UBIA are upstream models; this is not a statutory limit, current eligibility, loan turnover, interest-rate, or asset-price model." + }, + "investment_interest_expense": { + "positive_family": "cpi", + "negative_family": "cpi", + "positive_factor": "1.323487344789614247079323424058189918867", + "negative_factor": "1.323487344789614247079323424058189918867", + "assumption": "Explicit constant-real-amount proxy. Realized deductions/desires and UBIA are upstream models; this is not a statutory limit, current eligibility, loan turnover, interest-rate, or asset-price model." + }, + "investment_income_elected_form_4952": { + "positive_family": "cpi", + "negative_family": "cpi", + "positive_factor": "1.323487344789614247079323424058189918867", + "negative_factor": "1.323487344789614247079323424058189918867", + "assumption": "Explicit constant-real-amount proxy. Realized deductions/desires and UBIA are upstream models; this is not a statutory limit, current eligibility, loan turnover, interest-rate, or asset-price model." + }, + "student_loan_interest": { + "positive_family": "cpi", + "negative_family": "cpi", + "positive_factor": "1.323487344789614247079323424058189918867", + "negative_factor": "1.323487344789614247079323424058189918867", + "assumption": "Explicit constant-real-amount proxy. Realized deductions/desires and UBIA are upstream models; this is not a statutory limit, current eligibility, loan turnover, interest-rate, or asset-price model." + }, + "educator_expense": { + "positive_family": "cpi", + "negative_family": "cpi", + "positive_factor": "1.323487344789614247079323424058189918867", + "negative_factor": "1.323487344789614247079323424058189918867", + "assumption": "Explicit constant-real-amount proxy. Realized deductions/desires and UBIA are upstream models; this is not a statutory limit, current eligibility, loan turnover, interest-rate, or asset-price model." + }, + "qualified_tuition_expenses": { + "positive_family": "cpi", + "negative_family": "cpi", + "positive_factor": "1.323487344789614247079323424058189918867", + "negative_factor": "1.323487344789614247079323424058189918867", + "assumption": "Explicit constant-real-amount proxy. Realized deductions/desires and UBIA are upstream models; this is not a statutory limit, current eligibility, loan turnover, interest-rate, or asset-price model." + }, + "casualty_loss": { + "positive_family": "cpi", + "negative_family": "cpi", + "positive_factor": "1.323487344789614247079323424058189918867", + "negative_factor": "1.323487344789614247079323424058189918867", + "assumption": "Explicit constant-real-amount proxy. Realized deductions/desires and UBIA are upstream models; this is not a statutory limit, current eligibility, loan turnover, interest-rate, or asset-price model." + }, + "unreimbursed_business_employee_expenses": { + "positive_family": "cpi", + "negative_family": "cpi", + "positive_factor": "1.323487344789614247079323424058189918867", + "negative_factor": "1.323487344789614247079323424058189918867", + "assumption": "Explicit constant-real-amount proxy. Realized deductions/desires and UBIA are upstream models; this is not a statutory limit, current eligibility, loan turnover, interest-rate, or asset-price model." + }, + "traditional_ira_contributions_desired": { + "positive_family": "cpi", + "negative_family": "cpi", + "positive_factor": "1.323487344789614247079323424058189918867", + "negative_factor": "1.323487344789614247079323424058189918867", + "assumption": "Explicit constant-real-amount proxy. Realized deductions/desires and UBIA are upstream models; this is not a statutory limit, current eligibility, loan turnover, interest-rate, or asset-price model." + }, + "self_employed_pension_contributions_desired": { + "positive_family": "cpi", + "negative_family": "cpi", + "positive_factor": "1.323487344789614247079323424058189918867", + "negative_factor": "1.323487344789614247079323424058189918867", + "assumption": "Explicit constant-real-amount proxy. Realized deductions/desires and UBIA are upstream models; this is not a statutory limit, current eligibility, loan turnover, interest-rate, or asset-price model." + }, + "domestic_production_ald": { + "positive_family": "cpi", + "negative_family": "cpi", + "positive_factor": "1.323487344789614247079323424058189918867", + "negative_factor": "1.323487344789614247079323424058189918867", + "assumption": "Explicit constant-real-amount proxy. Realized deductions/desires and UBIA are upstream models; this is not a statutory limit, current eligibility, loan turnover, interest-rate, or asset-price model." + }, + "health_savings_account_ald": { + "positive_family": "cpi", + "negative_family": "cpi", + "positive_factor": "1.323487344789614247079323424058189918867", + "negative_factor": "1.323487344789614247079323424058189918867", + "assumption": "Explicit constant-real-amount proxy. Realized deductions/desires and UBIA are upstream models; this is not a statutory limit, current eligibility, loan turnover, interest-rate, or asset-price model." + }, + "unadjusted_basis_qualified_property": { + "positive_family": "cpi", + "negative_family": "cpi", + "positive_factor": "1.323487344789614247079323424058189918867", + "negative_factor": "1.323487344789614247079323424058189918867", + "assumption": "Explicit constant-real-amount proxy. Realized deductions/desires and UBIA are upstream models; this is not a statutory limit, current eligibility, loan turnover, interest-rate, or asset-price model." + }, + "sstb_unadjusted_basis_qualified_property": { + "positive_family": "cpi", + "negative_family": "cpi", + "positive_factor": "1.323487344789614247079323424058189918867", + "negative_factor": "1.323487344789614247079323424058189918867", + "assumption": "Explicit constant-real-amount proxy. Realized deductions/desires and UBIA are upstream models; this is not a statutory limit, current eligibility, loan turnover, interest-rate, or asset-price model." + } + }, + "return_incidence_fields": [ + "business_is_sstb", + "estate_income_would_be_qualified", + "farm_operations_income_would_be_qualified", + "farm_rent_income_would_be_qualified", + "partnership_s_corp_income_would_be_qualified", + "rental_income_would_be_qualified", + "self_employment_income_would_be_qualified", + "sstb_self_employment_income_would_be_qualified" + ], + "source_pins": { + "NATIONAL-GROWTH-EXTRACT.json": "775d89d3fe7a3b1eb9a09c50f6077d41bf838cdb80d9e4aaa9f0ab80ada4e0c8", + "INDEX-VALUES.json": "221f3786a462fe105d3da15df3624de5ad3e3f921b4d0205e096067685fd59b3", + "extract_national.py": "b6c3a8e4fea52653c01d801d6efe9620cdba4a489d6d7ae8c95cf00ee6dfbe1b", + "ACQUISITION.json": "e9a2c34c2ca06a229b2a790ed2987f25d31996653490f342b4b54af37d5f098c" + }, + "sensitivity_cpi_factor": "1.323487344789614247079323424058189918867", + "provenance": "All growth applications are modeled distribution transport from observed national series; source observed/derived/modeled flags remain separate.", + "operation_order": "Decode source statistical2015 \u2192 source canonical transformations \u2192 QBI2015 model (all16 outputs) \u2192 this growth transform once \u2192 2024 canonical donor. Monetary recipient predictors remain 2024.", + "noninterference": "RECID, S006, raw FLPDYR/FLPDMO, raw money, count source fields and actual survey membership are separate inputs and are not accepted or mutated by this transform.", + "sensitivities": "cpi_only uses exactly the same source cohort, zeros, signs and return incidence; no new policy evaluation or calibration authority." +} diff --git a/docs/evidence/puf2015-target2024/INDEX-VALUES.json b/docs/evidence/puf2015-target2024/INDEX-VALUES.json new file mode 100644 index 000000000..5dd19e557 --- /dev/null +++ b/docs/evidence/puf2015-target2024/INDEX-VALUES.json @@ -0,0 +1,8 @@ +{ + "schema": "microcosm.public_index_value_transcript/1", + "retrieved_date": "2026-09-09", + "retrieval": "Official URLs successfully read with web retrieval; exact values transcribed and independently checkable. These are not hashes of original publisher HTML/PDF bytes. Direct SSA urllib acquisition returned HTTP403 before writing any document.", + "awi": {"url": "https://www.ssa.gov/oact/cola/AWI.html", "series": "National Average Wage Index", "2015": "48098.63", "2024": "69846.57", "measurement": "Wages subject to federal income tax and deferred compensation; average-worker index, used as an explicit proxy for PUF wage amount growth."}, + "cola": {"url": "https://www.ssa.gov/oact/cola/colaseries.html", "cash_timing_url": "https://www.ssa.gov/cola/", "december_payable_percent": {"2015": "0.0", "2016": "0.3", "2017": "2.0", "2018": "2.8", "2019": "1.6", "2020": "1.3", "2021": "5.9", "2022": "8.7", "2023": "3.2"}, "timing": "December benefits are received in January of the following year. Product covers January2016 through January2024 cash increases relative to calendar2015 cash benefits. This transports a fixed entitlement; it does not forecast the recipient mix or new awards."}, + "cpi": {"url": "https://www.bls.gov/cpi/tables/supplemental-files/historical-cpi-u-202412.pdf", "series": "CPI-U, US city average, all items, annual average, 1982-84=100", "table": "Historical CPI-U index averages, annual average column", "2015": "237.017", "2023": "304.702", "2024": "313.689", "pages_one_based": [4,5], "monthly_2024_check": ["308.417","310.326","312.332","313.548","314.069","314.175","314.540","314.796","315.301","315.664","315.493","315.605"]} +} diff --git a/docs/evidence/puf2015-target2024/NATIONAL-GROWTH-EXTRACT.json b/docs/evidence/puf2015-target2024/NATIONAL-GROWTH-EXTRACT.json new file mode 100644 index 000000000..e24f8d37d --- /dev/null +++ b/docs/evidence/puf2015-target2024/NATIONAL-GROWTH-EXTRACT.json @@ -0,0 +1,4439 @@ +{ + "schema": "microcosm.us.puf_observed_growth_sources/1", + "scope": "exact headers and national row only; no AGI rows used", + "tables": { + "15in14ar.xls": { + "url": "https://www.irs.gov/pub/irs-soi/15in14ar.xls", + "sha256": "9f41bee263f5a8bb971df39a9673b55136c94954131a95bfd83b73a22801ec89", + "sheet": "TBL14", + "year": 2015, + "title": "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, \nby Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "unit": "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "national_row": 9, + "all_returns": { + "cell": "B9", + "value": 150493263, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Number of returns", + "Number of returns", + "Number of returns", + "Number of returns", + "Number of returns" + ] + }, + "families": { + "taxable_interest": { + "count": { + "cell": "H9", + "value": 42636696, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Taxable interest", + "Taxable interest", + "Taxable interest", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "I9", + "value": 95881223, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Taxable interest", + "Taxable interest", + "Taxable interest", + "Amount", + "Amount" + ] + } + }, + "tax_exempt_interest": { + "count": { + "cell": "J9", + "value": 5827038, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Tax-exempt interest [1]", + "Tax-exempt interest [1]", + "Tax-exempt interest [1]", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "K9", + "value": 61871455, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Tax-exempt interest [1]", + "Tax-exempt interest [1]", + "Tax-exempt interest [1]", + "Amount", + "Amount" + ] + } + }, + "ordinary_dividends": { + "count": { + "cell": "L9", + "value": 27607044, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Ordinary dividends", + "Ordinary dividends", + "Ordinary dividends", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "M9", + "value": 260252720, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Ordinary dividends", + "Ordinary dividends", + "Ordinary dividends", + "Amount", + "Amount" + ] + } + }, + "qualified_dividends": { + "count": { + "cell": "N9", + "value": 25755976, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Qualified dividends [1]", + "Qualified dividends [1]", + "Qualified dividends [1]", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "O9", + "value": 203187788, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Qualified dividends [1]", + "Qualified dividends [1]", + "Qualified dividends [1]", + "Amount", + "Amount" + ] + } + }, + "state_tax_refund": { + "count": { + "cell": "P9", + "value": 20256512, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "State income tax refunds", + "State income tax refunds", + "State income tax refunds", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "Q9", + "value": 31110732, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "State income tax refunds", + "State income tax refunds", + "State income tax refunds", + "Amount", + "Amount" + ] + } + }, + "alimony_income": { + "count": { + "cell": "R9", + "value": 414420, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Alimony received", + "Alimony received", + "Alimony received", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "S9", + "value": 10077086, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Alimony received", + "Alimony received", + "Alimony received", + "Amount", + "Amount" + ] + } + }, + "business_profit": { + "count": { + "cell": "T9", + "value": 18791200, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Business or profession", + "Net income", + "Net income", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "U9", + "value": 391975736, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Business or profession", + "Net income", + "Net income", + "Amount", + "Amount" + ] + } + }, + "business_loss": { + "count": { + "cell": "V9", + "value": 5935725, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Business or profession", + "Net loss", + "Net loss", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "W9", + "value": 60161435, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Business or profession", + "Net loss", + "Net loss", + "Amount", + "Amount" + ] + } + }, + "non_schedule_d_distributions": { + "count": { + "cell": "X9", + "value": 4323250, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Capital gain distributions reported on Form 1040", + "Capital gain distributions reported on Form 1040", + "Capital gain distributions reported on Form 1040", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "Y9", + "value": 11563203, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Capital gain distributions reported on Form 1040", + "Capital gain distributions reported on Form 1040", + "Capital gain distributions reported on Form 1040", + "Amount", + "Amount" + ] + } + }, + "other_property_gain": { + "count": { + "cell": "AD9", + "value": 1063576, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of property other than capital assets", + "Sales of property other than capital assets", + "Net gain", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "AE9", + "value": 33036562, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of property other than capital assets", + "Sales of property other than capital assets", + "Net gain", + "Amount", + "Amount" + ] + } + }, + "other_property_loss": { + "count": { + "cell": "AF9", + "value": 1153117, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of property other than capital assets", + "Sales of property other than capital assets", + "Net loss", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "AG9", + "value": 21093508, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of property other than capital assets", + "Sales of property other than capital assets", + "Net loss", + "Amount", + "Amount" + ] + } + }, + "ira_distributions": { + "count": { + "cell": "AH9", + "value": 14159018, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Taxable Individual Retirement Arrangement (IRA) distributions", + "Taxable Individual Retirement Arrangement (IRA) distributions", + "Taxable Individual Retirement Arrangement (IRA) distributions", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "AI9", + "value": 253213041, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Taxable Individual Retirement Arrangement (IRA) distributions", + "Taxable Individual Retirement Arrangement (IRA) distributions", + "Taxable Individual Retirement Arrangement (IRA) distributions", + "Amount", + "Amount" + ] + } + }, + "taxable_pensions": { + "count": { + "cell": "AL9", + "value": 28199160, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Pensions and annuities", + "Pensions and annuities", + "Taxable", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "AM9", + "value": 689991999, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Pensions and annuities", + "Pensions and annuities", + "Taxable", + "Amount", + "Amount" + ] + } + }, + "rental_royalty_profit": { + "count": { + "cell": "AZ9", + "value": 6768234, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Total rental and royalty", + "Net income", + "Net income", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "BA9", + "value": 103058883, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Total rental and royalty", + "Net income", + "Net income", + "Amount", + "Amount" + ] + } + }, + "rental_royalty_loss": { + "count": { + "cell": "BB9", + "value": 4531666, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Total rental and royalty", + "Net loss", + "Net loss", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "BC9", + "value": 46245560, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Total rental and royalty", + "Net loss", + "Net loss", + "Amount", + "Amount" + ] + } + }, + "farm_rent_profit": { + "count": { + "cell": "AV9", + "value": 394273, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Farm rental", + "Net income", + "Net income", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "AW9", + "value": 5205330, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Farm rental", + "Net income", + "Net income", + "Amount", + "Amount" + ] + } + }, + "farm_rent_loss": { + "count": { + "cell": "AX9", + "value": 101041, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Farm rental", + "Net loss", + "Net loss", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "AY9", + "value": 710953, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Farm rental", + "Net loss", + "Net loss", + "Amount", + "Amount" + ] + } + }, + "estate_profit": { + "count": { + "cell": "BH9", + "value": 630256, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Estate and trust", + "Net income", + "Net income", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "BI9", + "value": 32453002, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Estate and trust", + "Net income", + "Net income", + "Amount", + "Amount" + ] + } + }, + "estate_loss": { + "count": { + "cell": "BJ9", + "value": 57802, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Estate and trust", + "Net loss", + "Net loss", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "BK9", + "value": 5033199, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Estate and trust", + "Net loss", + "Net loss", + "Amount", + "Amount" + ] + } + }, + "farm_profit": { + "count": { + "cell": "BL9", + "value": 520802, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Farm", + "Net income", + "Net income", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "BM9", + "value": 13533867, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Farm", + "Net income", + "Net income", + "Amount", + "Amount" + ] + } + }, + "farm_loss": { + "count": { + "cell": "BN9", + "value": 1278825, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Farm", + "Net loss", + "Net loss", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "BO9", + "value": 27497651, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Farm", + "Net loss", + "Net loss", + "Amount", + "Amount" + ] + } + }, + "other_income_profit": { + "count": { + "cell": "BX9", + "value": 6121770, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Other income", + "Net income", + "Net income", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "BY9", + "value": 46769290, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Other income", + "Net income", + "Net income", + "Amount", + "Amount" + ] + } + }, + "other_income_loss": { + "count": { + "cell": "BZ9", + "value": 332708, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Other income", + "Net loss", + "Net loss", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "CA9", + "value": 6693961, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Other income", + "Net loss", + "Net loss", + "Amount", + "Amount" + ] + } + }, + "partnership_s_corp_profit": { + "count": { + "cell": "BD9", + "value": 6044409, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Partnership and S corporation", + "Net income", + "Net income", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "BE9", + "value": 755622761, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Partnership and S corporation", + "Net income", + "Net income", + "Amount", + "Amount" + ] + } + }, + "partnership_s_corp_loss": { + "count": { + "cell": "BF9", + "value": 2699817, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Partnership and S corporation", + "Net loss", + "Net loss", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "BG9", + "value": 126618186, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Partnership and S corporation", + "Net loss", + "Net loss", + "Amount", + "Amount" + ] + } + } + } + }, + "23in14ar.xls": { + "url": "https://www.irs.gov/pub/irs-soi/23in14ar.xls", + "sha256": "b6c1f87fbb5533417e195f6938538e5de09b6a0825a6a54346bf9363a18d96af", + "sheet": "TBL14", + "year": 2023, + "title": "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, \nby Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "unit": "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "national_row": 9, + "all_returns": { + "cell": "B9", + "value": 160602107, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Number of returns", + "Number of returns", + "Number of returns", + "Number of returns", + "Number of returns" + ] + }, + "families": { + "taxable_interest": { + "count": { + "cell": "T9", + "value": 55260238, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Taxable interest", + "Taxable interest", + "Taxable interest", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "U9", + "value": 313812674, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Taxable interest", + "Taxable interest", + "Taxable interest", + "Amount", + "Amount" + ] + } + }, + "tax_exempt_interest": { + "count": { + "cell": "V9", + "value": 6893860, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Tax-exempt interest [2]", + "Tax-exempt interest [2]", + "Tax-exempt interest [2]", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "W9", + "value": 66091992, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Tax-exempt interest [2]", + "Tax-exempt interest [2]", + "Tax-exempt interest [2]", + "Amount", + "Amount" + ] + } + }, + "ordinary_dividends": { + "count": { + "cell": "X9", + "value": 33718460, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Ordinary dividends", + "Ordinary dividends", + "Ordinary dividends", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "Y9", + "value": 504224981, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Ordinary dividends", + "Ordinary dividends", + "Ordinary dividends", + "Amount", + "Amount" + ] + } + }, + "qualified_dividends": { + "count": { + "cell": "Z9", + "value": 31106344, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Qualified dividends [2]", + "Qualified dividends [2]", + "Qualified dividends [2]", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "AA9", + "value": 336070296, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Qualified dividends [2]", + "Qualified dividends [2]", + "Qualified dividends [2]", + "Amount", + "Amount" + ] + } + }, + "state_tax_refund": { + "count": { + "cell": "AB9", + "value": 3109096, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "State income tax refunds", + "State income tax refunds", + "State income tax refunds", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "AC9", + "value": 4311977, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "State income tax refunds", + "State income tax refunds", + "State income tax refunds", + "Amount", + "Amount" + ] + } + }, + "alimony_income": { + "count": { + "cell": "AD9", + "value": 183582, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Alimony received", + "Alimony received", + "Alimony received", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "AE9", + "value": 6686429, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Alimony received", + "Alimony received", + "Alimony received", + "Amount", + "Amount" + ] + } + }, + "business_profit": { + "count": { + "cell": "AF9", + "value": 21411713, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Business or profession", + "Net income", + "Net income", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "AG9", + "value": 530380019, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Business or profession", + "Net income", + "Net income", + "Amount", + "Amount" + ] + } + }, + "business_loss": { + "count": { + "cell": "AH9", + "value": 9067174, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Business or profession", + "Net loss", + "Net loss", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "AI9", + "value": 153441387, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Business or profession", + "Net loss", + "Net loss", + "Amount", + "Amount" + ] + } + }, + "non_schedule_d_distributions": { + "count": { + "cell": "AJ9", + "value": 3209131, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Capital gain distributions reported on Form 1040", + "Capital gain distributions reported on Form 1040", + "Capital gain distributions reported on Form 1040", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "AK9", + "value": 9340820, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Capital gain distributions reported on Form 1040", + "Capital gain distributions reported on Form 1040", + "Capital gain distributions reported on Form 1040", + "Amount", + "Amount" + ] + } + }, + "other_property_gain": { + "count": { + "cell": "AP9", + "value": 945122, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of property other than capital assets", + "Sales of property other than capital assets", + "Net gain", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "AQ9", + "value": 49567896, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of property other than capital assets", + "Sales of property other than capital assets", + "Net gain", + "Amount", + "Amount" + ] + } + }, + "other_property_loss": { + "count": { + "cell": "AR9", + "value": 775387, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of property other than capital assets", + "Sales of property other than capital assets", + "Net loss", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "AS9", + "value": 19448929, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of property other than capital assets", + "Sales of property other than capital assets", + "Net loss", + "Amount", + "Amount" + ] + } + }, + "ira_distributions": { + "count": { + "cell": "AT9", + "value": 16694154, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Taxable Individual Retirement Arrangement (IRA) distributions", + "Taxable Individual Retirement Arrangement (IRA) distributions", + "Taxable Individual Retirement Arrangement (IRA) distributions", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "AU9", + "value": 438147938, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Taxable Individual Retirement Arrangement (IRA) distributions", + "Taxable Individual Retirement Arrangement (IRA) distributions", + "Taxable Individual Retirement Arrangement (IRA) distributions", + "Amount", + "Amount" + ] + } + }, + "taxable_pensions": { + "count": { + "cell": "AX9", + "value": 29541284, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Pensions and annuities", + "Pensions and annuities", + "Taxable", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "AY9", + "value": 932130236, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Pensions and annuities", + "Pensions and annuities", + "Taxable", + "Amount", + "Amount" + ] + } + }, + "rental_royalty_profit": { + "count": { + "cell": "BL9", + "value": 6346357, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Total rental and royalty", + "Net income", + "Net income", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "BM9", + "value": 138609251, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Total rental and royalty", + "Net income", + "Net income", + "Amount", + "Amount" + ] + } + }, + "rental_royalty_loss": { + "count": { + "cell": "BN9", + "value": 3196282, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Total rental and royalty", + "Net loss", + "Net loss", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "BO9", + "value": 53570273, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Total rental and royalty", + "Net loss", + "Net loss", + "Amount", + "Amount" + ] + } + }, + "farm_rent_profit": { + "count": { + "cell": "BH9", + "value": 325953, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Farm rental", + "Net income", + "Net income", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "BI9", + "value": 6811206, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Farm rental", + "Net income", + "Net income", + "Amount", + "Amount" + ] + } + }, + "farm_rent_loss": { + "count": { + "cell": "BJ9", + "value": 67356, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Farm rental", + "Net loss", + "Net loss", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "BK9", + "value": 529586, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Farm rental", + "Net loss", + "Net loss", + "Amount", + "Amount" + ] + } + }, + "estate_profit": { + "count": { + "cell": "BX9", + "value": 648583, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Estate and trust", + "Net income", + "Net income", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "BY9", + "value": 47892046, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Estate and trust", + "Net income", + "Net income", + "Amount", + "Amount" + ] + } + }, + "estate_loss": { + "count": { + "cell": "BZ9", + "value": 36592, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Estate and trust", + "Net loss", + "Net loss", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "CA9", + "value": 4898828, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Estate and trust", + "Net loss", + "Net loss", + "Amount", + "Amount" + ] + } + }, + "farm_profit": { + "count": { + "cell": "CB9", + "value": 484162, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Farm", + "Net income", + "Net income", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "CC9", + "value": 17656301, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Farm", + "Net income", + "Net income", + "Amount", + "Amount" + ] + } + }, + "farm_loss": { + "count": { + "cell": "CD9", + "value": 1258869, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Farm", + "Net loss", + "Net loss", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "CE9", + "value": 45444491, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Farm", + "Net loss", + "Net loss", + "Amount", + "Amount" + ] + } + }, + "other_income_profit": { + "count": { + "cell": "CN9", + "value": 7468434, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Other income", + "Net income", + "Net income", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "CO9", + "value": 60816508, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Other income", + "Net income", + "Net income", + "Amount", + "Amount" + ] + } + }, + "other_income_loss": { + "count": { + "cell": "CP9", + "value": 318839, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Other income", + "Net loss", + "Net loss", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "CQ9", + "value": 12213628, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Other income", + "Net loss", + "Net loss", + "Amount", + "Amount" + ] + } + }, + "partnership_profit": { + "count": { + "cell": "BP9", + "value": 3266345, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Partnership", + "Net income", + "Net income", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "BQ9", + "value": 452077038, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Partnership", + "Net income", + "Net income", + "Amount", + "Amount" + ] + } + }, + "partnership_loss": { + "count": { + "cell": "BR9", + "value": 2103634, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Partnership", + "Net loss", + "Net loss", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "BS9", + "value": 168884100, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Partnership", + "Net loss", + "Net loss", + "Amount", + "Amount" + ] + } + }, + "s_corp_profit": { + "count": { + "cell": "BT9", + "value": 4138040, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "S corporation", + "Net income", + "Net income", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "BU9", + "value": 797450861, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "S corporation", + "Net income", + "Net income", + "Amount", + "Amount" + ] + } + }, + "s_corp_loss": { + "count": { + "cell": "BV9", + "value": 1415974, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "S corporation", + "Net loss", + "Net loss", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "BW9", + "value": 89543715, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "S corporation", + "Net loss", + "Net loss", + "Amount", + "Amount" + ] + } + } + } + }, + "15in14acg.xls": { + "url": "https://www.irs.gov/pub/irs-soi/15in14acg.xls", + "sha256": "7e128eb06e14371a15baa72060e91d74e4c55ba8596610ddd047c9ef168cbbd1", + "sheet": "Sheet1", + "year": 2015, + "title": "Table 1.4A Returns with Income or Loss from Sales of Capital Assets\nReported on Form 1040, Schedule D: Selected Items, by Size of\nAdjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "unit": "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "national_row": 10, + "families": { + "short_term_gain": { + "count": { + "cell": "F10", + "value": 3900677, + "headers": [ + "Table 1.4A Returns with Income or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Net short-term capital gain", + "Net short-term capital gain", + "Net short-term capital gain", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "G10", + "value": 38164189, + "headers": [ + "Table 1.4A Returns with Income or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Net short-term capital gain", + "Net short-term capital gain", + "Net short-term capital gain", + "Amount", + "Amount" + ] + } + }, + "short_term_loss": { + "count": { + "cell": "H10", + "value": 7820586, + "headers": [ + "Table 1.4A Returns with Income or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Net short-term capital loss", + "Net short-term capital loss", + "Net short-term capital loss", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "I10", + "value": 250592206, + "headers": [ + "Table 1.4A Returns with Income or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Net short-term capital loss", + "Net short-term capital loss", + "Net short-term capital loss", + "Amount", + "Amount" + ] + } + }, + "long_term_gain": { + "count": { + "cell": "BJ10", + "value": 11735160, + "headers": [ + "Table 1.4A Returns with Income or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Net long-term capital gain", + "Net long-term capital gain", + "Net long-term capital gain", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "BK10", + "value": 733313255, + "headers": [ + "Table 1.4A Returns with Income or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Net long-term capital gain", + "Net long-term capital gain", + "Net long-term capital gain", + "Amount", + "Amount" + ] + } + }, + "long_term_loss": { + "count": { + "cell": "BL10", + "value": 6375392, + "headers": [ + "Table 1.4A Returns with Income or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Net long-term capital loss", + "Net long-term capital loss", + "Net long-term capital loss", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "BM10", + "value": 312723716, + "headers": [ + "Table 1.4A Returns with Income or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Net long-term capital loss", + "Net long-term capital loss", + "Net long-term capital loss", + "Amount", + "Amount" + ] + } + }, + "short_assets_gain": { + "count": { + "cell": "J10", + "value": 3923409, + "headers": [ + "Table 1.4A Returns with Income or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Net short-term gain from sales of capital assets", + "Net short-term gain from sales of capital assets", + "Net short-term gain from sales of capital assets", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "K10", + "value": 22196394, + "headers": [ + "Table 1.4A Returns with Income or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Net short-term gain from sales of capital assets", + "Net short-term gain from sales of capital assets", + "Net short-term gain from sales of capital assets", + "Amount", + "Amount" + ] + } + }, + "short_assets_loss": { + "count": { + "cell": "L10", + "value": 6269981, + "headers": [ + "Table 1.4A Returns with Income or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Net short-term loss from sales of capital assets", + "Net short-term loss from sales of capital assets", + "Net short-term loss from sales of capital assets", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "M10", + "value": 68624843, + "headers": [ + "Table 1.4A Returns with Income or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Net short-term loss from sales of capital assets", + "Net short-term loss from sales of capital assets", + "Net short-term loss from sales of capital assets", + "Amount", + "Amount" + ] + } + }, + "short_other_gain": { + "count": { + "cell": "AZ10", + "value": 231076, + "headers": [ + "Table 1.4A Returns with Income or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Net short-term gain from other forms (2119, 4797, etc.)", + "Net short-term gain from other forms (2119, 4797, etc.)", + "Net short-term gain from other forms (2119, 4797, etc.)", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "BA10", + "value": 5948464, + "headers": [ + "Table 1.4A Returns with Income or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Net short-term gain from other forms (2119, 4797, etc.)", + "Net short-term gain from other forms (2119, 4797, etc.)", + "Net short-term gain from other forms (2119, 4797, etc.)", + "Amount", + "Amount" + ] + } + }, + "short_other_loss": { + "count": { + "cell": "BB10", + "value": 336867, + "headers": [ + "Table 1.4A Returns with Income or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Net short-term loss from other forms (4684, 6781, and 8824)", + "Net short-term loss from other forms (4684, 6781, and 8824)", + "Net short-term loss from other forms (4684, 6781, and 8824)", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "BC10", + "value": 1980907, + "headers": [ + "Table 1.4A Returns with Income or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Net short-term loss from other forms (4684, 6781, and 8824)", + "Net short-term loss from other forms (4684, 6781, and 8824)", + "Net short-term loss from other forms (4684, 6781, and 8824)", + "Amount", + "Amount" + ] + } + }, + "short_pass_through_gain": { + "count": { + "cell": "BD10", + "value": 483349, + "headers": [ + "Table 1.4A Returns with Income or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Net short-term gain from partnership/S corporation", + "Net short-term gain from partnership/S corporation", + "Net short-term gain from partnership/S corporation", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "BE10", + "value": 17194108, + "headers": [ + "Table 1.4A Returns with Income or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Net short-term gain from partnership/S corporation", + "Net short-term gain from partnership/S corporation", + "Net short-term gain from partnership/S corporation", + "Amount", + "Amount" + ] + } + }, + "short_pass_through_loss": { + "count": { + "cell": "BF10", + "value": 725964, + "headers": [ + "Table 1.4A Returns with Income or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Net short-term loss from partnership/S corporation", + "Net short-term loss from partnership/S corporation", + "Net short-term loss from partnership/S corporation", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "BG10", + "value": 16056081, + "headers": [ + "Table 1.4A Returns with Income or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Net short-term loss from partnership/S corporation", + "Net short-term loss from partnership/S corporation", + "Net short-term loss from partnership/S corporation", + "Amount", + "Amount" + ] + } + }, + "short_carryover": { + "count": { + "cell": "BH10", + "value": 2000156, + "headers": [ + "Table 1.4A Returns with Income or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Short-term loss carryover", + "Short-term loss carryover", + "Short-term loss carryover", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "BI10", + "value": 171105171, + "headers": [ + "Table 1.4A Returns with Income or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Short-term loss carryover", + "Short-term loss carryover", + "Short-term loss carryover", + "Amount", + "Amount" + ] + } + }, + "long_assets_gain": { + "count": { + "cell": "BN10", + "value": 8440065, + "headers": [ + "Table 1.4A Returns with Income or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Net long-term gain from sales of capital assets", + "Net long-term gain from sales of capital assets", + "Net long-term gain from sales of capital assets", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "BO10", + "value": 282493545, + "headers": [ + "Table 1.4A Returns with Income or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Net long-term gain from sales of capital assets", + "Net long-term gain from sales of capital assets", + "Net long-term gain from sales of capital assets", + "Amount", + "Amount" + ] + } + }, + "long_assets_loss": { + "count": { + "cell": "BP10", + "value": 4441176, + "headers": [ + "Table 1.4A Returns with Income or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Net long-term loss from sales of capital assets", + "Net long-term loss from sales of capital assets", + "Net long-term loss from sales of capital assets", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "BQ10", + "value": 62343837, + "headers": [ + "Table 1.4A Returns with Income or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Net long-term loss from sales of capital assets", + "Net long-term loss from sales of capital assets", + "Net long-term loss from sales of capital assets", + "Amount", + "Amount" + ] + } + }, + "long_other_gain": { + "count": { + "cell": "DD10", + "value": 2177243, + "headers": [ + "Table 1.4A Returns with Income or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Net long-term gain from other forms (2119, 4797, etc.)", + "Net long-term gain from other forms (2119, 4797, etc.)", + "Net long-term gain from other forms (2119, 4797, etc.)", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "DE10", + "value": 226671889, + "headers": [ + "Table 1.4A Returns with Income or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Net long-term gain from other forms (2119, 4797, etc.)", + "Net long-term gain from other forms (2119, 4797, etc.)", + "Net long-term gain from other forms (2119, 4797, etc.)", + "Amount", + "Amount" + ] + } + }, + "long_other_loss": { + "count": { + "cell": "DF10", + "value": 296456, + "headers": [ + "Table 1.4A Returns with Income or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Net long-term loss from other forms (4684, 6781, and 8824)", + "Net long-term loss from other forms (4684, 6781, and 8824)", + "Net long-term loss from other forms (4684, 6781, and 8824)", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "DG10", + "value": 2562723, + "headers": [ + "Table 1.4A Returns with Income or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Net long-term loss from other forms (4684, 6781, and 8824)", + "Net long-term loss from other forms (4684, 6781, and 8824)", + "Net long-term loss from other forms (4684, 6781, and 8824)", + "Amount", + "Amount" + ] + } + }, + "long_pass_through_gain": { + "count": { + "cell": "DH10", + "value": 1721361, + "headers": [ + "Table 1.4A Returns with Income or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Net long-term gain from partnership/S corporation", + "Net long-term gain from partnership/S corporation", + "Net long-term gain from partnership/S corporation", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "DI10", + "value": 216227585, + "headers": [ + "Table 1.4A Returns with Income or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Net long-term gain from partnership/S corporation", + "Net long-term gain from partnership/S corporation", + "Net long-term gain from partnership/S corporation", + "Amount", + "Amount" + ] + } + }, + "long_pass_through_loss": { + "count": { + "cell": "DJ10", + "value": 552552, + "headers": [ + "Table 1.4A Returns with Income or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Net long-term loss from partnership/S corporation", + "Net long-term loss from partnership/S corporation", + "Net long-term loss from partnership/S corporation", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "DK10", + "value": 10694019, + "headers": [ + "Table 1.4A Returns with Income or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Net long-term loss from partnership/S corporation", + "Net long-term loss from partnership/S corporation", + "Net long-term loss from partnership/S corporation", + "Amount", + "Amount" + ] + } + }, + "long_distributions": { + "count": { + "cell": "DL10", + "value": 9733033, + "headers": [ + "Table 1.4A Returns with Income or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Schedule D capital gain distributions", + "Schedule D capital gain distributions", + "Schedule D capital gain distributions", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "DM10", + "value": 62496866, + "headers": [ + "Table 1.4A Returns with Income or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Schedule D capital gain distributions", + "Schedule D capital gain distributions", + "Schedule D capital gain distributions", + "Amount", + "Amount" + ] + } + }, + "long_carryover": { + "count": { + "cell": "DN10", + "value": 4371225, + "headers": [ + "Table 1.4A Returns with Income or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Long-term loss carryover", + "Long-term loss carryover", + "Long-term loss carryover", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "DO10", + "value": 291699768, + "headers": [ + "Table 1.4A Returns with Income or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Long-term loss carryover", + "Long-term loss carryover", + "Long-term loss carryover", + "Amount", + "Amount" + ] + } + } + } + }, + "23in14acg.xls": { + "url": "https://www.irs.gov/pub/irs-soi/23in14acg.xls", + "sha256": "5b35c2f007f16503c0dcf6cc085d3a6b049ca523d99b7d7ae29afdbdd2d02e70", + "sheet": "Sheet1", + "year": 2023, + "title": "Table 1.4A. Returns with Gain or Loss from Sales of Capital Assets\nReported on Form 1040, Schedule D: Selected Items, by Size of\nAdjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "unit": "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "national_row": 10, + "families": { + "short_term_gain": { + "count": { + "cell": "F10", + "value": 6215519, + "headers": [ + "Table 1.4A. Returns with Gain or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Net short-term capital gain", + "Net short-term capital gain", + "Net short-term capital gain", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "G10", + "value": 70482448, + "headers": [ + "Table 1.4A. Returns with Gain or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Net short-term capital gain", + "Net short-term capital gain", + "Net short-term capital gain", + "Amount", + "Amount" + ] + } + }, + "short_term_loss": { + "count": { + "cell": "H10", + "value": 11370982, + "headers": [ + "Table 1.4A. Returns with Gain or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Net short-term capital loss", + "Net short-term capital loss", + "Net short-term capital loss", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "I10", + "value": 673910232, + "headers": [ + "Table 1.4A. Returns with Gain or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Net short-term capital loss", + "Net short-term capital loss", + "Net short-term capital loss", + "Amount", + "Amount" + ] + } + }, + "long_term_gain": { + "count": { + "cell": "BJ10", + "value": 11888516, + "headers": [ + "Table 1.4A. Returns with Gain or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Net long-term capital gain", + "Net long-term capital gain", + "Net long-term capital gain", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "BK10", + "value": 971279947, + "headers": [ + "Table 1.4A. Returns with Gain or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Net long-term capital gain", + "Net long-term capital gain", + "Net long-term capital gain", + "Amount", + "Amount" + ] + } + }, + "long_term_loss": { + "count": { + "cell": "BL10", + "value": 11230240, + "headers": [ + "Table 1.4A. Returns with Gain or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Net long-term capital loss", + "Net long-term capital loss", + "Net long-term capital loss", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "BM10", + "value": 450065138, + "headers": [ + "Table 1.4A. Returns with Gain or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Net long-term capital loss", + "Net long-term capital loss", + "Net long-term capital loss", + "Amount", + "Amount" + ] + } + }, + "short_assets_gain": { + "count": { + "cell": "J10", + "value": 7140149, + "headers": [ + "Table 1.4A. Returns with Gain or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Net short-term gain from sales of capital assets", + "Net short-term gain from sales of capital assets", + "Net short-term gain from sales of capital assets", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "K10", + "value": 58927145, + "headers": [ + "Table 1.4A. Returns with Gain or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Net short-term gain from sales of capital assets", + "Net short-term gain from sales of capital assets", + "Net short-term gain from sales of capital assets", + "Amount", + "Amount" + ] + } + }, + "short_assets_loss": { + "count": { + "cell": "L10", + "value": 8377604, + "headers": [ + "Table 1.4A. Returns with Gain or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Net short-term loss from sales of capital assets", + "Net short-term loss from sales of capital assets", + "Net short-term loss from sales of capital assets", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "M10", + "value": 135549749, + "headers": [ + "Table 1.4A. Returns with Gain or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Net short-term loss from sales of capital assets", + "Net short-term loss from sales of capital assets", + "Net short-term loss from sales of capital assets", + "Amount", + "Amount" + ] + } + }, + "short_other_gain": { + "count": { + "cell": "AZ10", + "value": 174491, + "headers": [ + "Table 1.4A. Returns with Gain or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Net short-term gain from other forms (2119, 4797, etc.)", + "Net short-term gain from other forms (2119, 4797, etc.)", + "Net short-term gain from other forms (2119, 4797, etc.)", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "BA10", + "value": 8619769, + "headers": [ + "Table 1.4A. Returns with Gain or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Net short-term gain from other forms (2119, 4797, etc.)", + "Net short-term gain from other forms (2119, 4797, etc.)", + "Net short-term gain from other forms (2119, 4797, etc.)", + "Amount", + "Amount" + ] + } + }, + "short_other_loss": { + "count": { + "cell": "BB10", + "value": 401474, + "headers": [ + "Table 1.4A. Returns with Gain or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Net short-term loss from other forms (4684, 6781, and 8824)", + "Net short-term loss from other forms (4684, 6781, and 8824)", + "Net short-term loss from other forms (4684, 6781, and 8824)", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "BC10", + "value": 3833687, + "headers": [ + "Table 1.4A. Returns with Gain or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Net short-term loss from other forms (4684, 6781, and 8824)", + "Net short-term loss from other forms (4684, 6781, and 8824)", + "Net short-term loss from other forms (4684, 6781, and 8824)", + "Amount", + "Amount" + ] + } + }, + "short_pass_through_gain": { + "count": { + "cell": "BD10", + "value": 495780, + "headers": [ + "Table 1.4A. Returns with Gain or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Net short-term gain from partnership/S corporation", + "Net short-term gain from partnership/S corporation", + "Net short-term gain from partnership/S corporation", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "BE10", + "value": 28464220, + "headers": [ + "Table 1.4A. Returns with Gain or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Net short-term gain from partnership/S corporation", + "Net short-term gain from partnership/S corporation", + "Net short-term gain from partnership/S corporation", + "Amount", + "Amount" + ] + } + }, + "short_pass_through_loss": { + "count": { + "cell": "BF10", + "value": 560758, + "headers": [ + "Table 1.4A. Returns with Gain or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Net short-term loss from partnership/S corporation", + "Net short-term loss from partnership/S corporation", + "Net short-term loss from partnership/S corporation", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "BG10", + "value": 16759596, + "headers": [ + "Table 1.4A. Returns with Gain or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Net short-term loss from partnership/S corporation", + "Net short-term loss from partnership/S corporation", + "Net short-term loss from partnership/S corporation", + "Amount", + "Amount" + ] + } + }, + "short_carryover": { + "count": { + "cell": "BH10", + "value": 5151589, + "headers": [ + "Table 1.4A. Returns with Gain or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Short-term loss carryover", + "Short-term loss carryover", + "Short-term loss carryover", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "BI10", + "value": 543302365, + "headers": [ + "Table 1.4A. Returns with Gain or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Short-term loss carryover", + "Short-term loss carryover", + "Short-term loss carryover", + "Amount", + "Amount" + ] + } + }, + "long_assets_gain": { + "count": { + "cell": "BN10", + "value": 8891406, + "headers": [ + "Table 1.4A. Returns with Gain or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Net long-term gain from sales of capital assets", + "Net long-term gain from sales of capital assets", + "Net long-term gain from sales of capital assets", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "BO10", + "value": 405281941, + "headers": [ + "Table 1.4A. Returns with Gain or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Net long-term gain from sales of capital assets", + "Net long-term gain from sales of capital assets", + "Net long-term gain from sales of capital assets", + "Amount", + "Amount" + ] + } + }, + "long_assets_loss": { + "count": { + "cell": "BP10", + "value": 9496262, + "headers": [ + "Table 1.4A. Returns with Gain or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Net long-term loss from sales of capital assets", + "Net long-term loss from sales of capital assets", + "Net long-term loss from sales of capital assets", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "BQ10", + "value": 149558927, + "headers": [ + "Table 1.4A. Returns with Gain or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Net long-term loss from sales of capital assets", + "Net long-term loss from sales of capital assets", + "Net long-term loss from sales of capital assets", + "Amount", + "Amount" + ] + } + }, + "long_other_gain": { + "count": { + "cell": "DD10", + "value": 1979380, + "headers": [ + "Table 1.4A. Returns with Gain or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Net long-term gain from other forms (2119, 4797, etc.)", + "Net long-term gain from other forms (2119, 4797, etc.)", + "Net long-term gain from other forms (2119, 4797, etc.)", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "DE10", + "value": 316779349, + "headers": [ + "Table 1.4A. Returns with Gain or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Net long-term gain from other forms (2119, 4797, etc.)", + "Net long-term gain from other forms (2119, 4797, etc.)", + "Net long-term gain from other forms (2119, 4797, etc.)", + "Amount", + "Amount" + ] + } + }, + "long_other_loss": { + "count": { + "cell": "DF10", + "value": 338620, + "headers": [ + "Table 1.4A. Returns with Gain or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Net long-term loss from other forms (4684, 6781, and 8824)", + "Net long-term loss from other forms (4684, 6781, and 8824)", + "Net long-term loss from other forms (4684, 6781, and 8824)", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "DG10", + "value": 5211519, + "headers": [ + "Table 1.4A. Returns with Gain or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Net long-term loss from other forms (4684, 6781, and 8824)", + "Net long-term loss from other forms (4684, 6781, and 8824)", + "Net long-term loss from other forms (4684, 6781, and 8824)", + "Amount", + "Amount" + ] + } + }, + "long_pass_through_gain": { + "count": { + "cell": "DH10", + "value": 1219800, + "headers": [ + "Table 1.4A. Returns with Gain or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Net long-term gain from partnership/S corporation", + "Net long-term gain from partnership/S corporation", + "Net long-term gain from partnership/S corporation", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "DI10", + "value": 282281744, + "headers": [ + "Table 1.4A. Returns with Gain or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Net long-term gain from partnership/S corporation", + "Net long-term gain from partnership/S corporation", + "Net long-term gain from partnership/S corporation", + "Amount", + "Amount" + ] + } + }, + "long_pass_through_loss": { + "count": { + "cell": "DJ10", + "value": 766861, + "headers": [ + "Table 1.4A. Returns with Gain or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Net long-term loss from partnership/S corporation", + "Net long-term loss from partnership/S corporation", + "Net long-term loss from partnership/S corporation", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "DK10", + "value": 22526709, + "headers": [ + "Table 1.4A. Returns with Gain or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Net long-term loss from partnership/S corporation", + "Net long-term loss from partnership/S corporation", + "Net long-term loss from partnership/S corporation", + "Amount", + "Amount" + ] + } + }, + "long_distributions": { + "count": { + "cell": "DL10", + "value": 9487999, + "headers": [ + "Table 1.4A. Returns with Gain or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Schedule D capital gain distributions", + "Schedule D capital gain distributions", + "Schedule D capital gain distributions", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "DM10", + "value": 44095015, + "headers": [ + "Table 1.4A. Returns with Gain or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Schedule D capital gain distributions", + "Schedule D capital gain distributions", + "Schedule D capital gain distributions", + "Amount", + "Amount" + ] + } + }, + "long_carryover": { + "count": { + "cell": "DN10", + "value": 5622923, + "headers": [ + "Table 1.4A. Returns with Gain or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Long-term loss carryover", + "Long-term loss carryover", + "Long-term loss carryover", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "DO10", + "value": 349944357, + "headers": [ + "Table 1.4A. Returns with Gain or Loss from Sales of Capital Assets Reported on Form 1040, Schedule D: Selected Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of capital assets reported on Form 1040, Schedule D", + "Long-term loss carryover", + "Long-term loss carryover", + "Long-term loss carryover", + "Amount", + "Amount" + ] + } + } + } + } + }, + "factors": { + "taxable_interest": { + "method": "amount_per_reporting_return", + "source": { + "count": { + "cell": "H9", + "value": 42636696, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Taxable interest", + "Taxable interest", + "Taxable interest", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "I9", + "value": 95881223, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Taxable interest", + "Taxable interest", + "Taxable interest", + "Amount", + "Amount" + ] + } + }, + "destination_2023": { + "count": { + "cell": "T9", + "value": 55260238, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Taxable interest", + "Taxable interest", + "Taxable interest", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "U9", + "value": 313812674, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Taxable interest", + "Taxable interest", + "Taxable interest", + "Amount", + "Amount" + ] + } + }, + "observed_2015_2023_factor": "2.525269343625908636956987730976528139607", + "modeled_2023_2024_factor": "1.029494391241278363778380187855675381192", + "factor_2015_2024": "2.599750625636417399355437523686408804619" + }, + "tax_exempt_interest": { + "method": "amount_per_reporting_return", + "source": { + "count": { + "cell": "J9", + "value": 5827038, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Tax-exempt interest [1]", + "Tax-exempt interest [1]", + "Tax-exempt interest [1]", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "K9", + "value": 61871455, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Tax-exempt interest [1]", + "Tax-exempt interest [1]", + "Tax-exempt interest [1]", + "Amount", + "Amount" + ] + } + }, + "destination_2023": { + "count": { + "cell": "V9", + "value": 6893860, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Tax-exempt interest [2]", + "Tax-exempt interest [2]", + "Tax-exempt interest [2]", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "W9", + "value": 66091992, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Tax-exempt interest [2]", + "Tax-exempt interest [2]", + "Tax-exempt interest [2]", + "Amount", + "Amount" + ] + } + }, + "observed_2015_2023_factor": "0.9029088360274333448808012169263853497016", + "modeled_2023_2024_factor": "1.029494391241278363778380187855675381192", + "factor_2015_2024": "0.9295395824924337172788943063597248917385" + }, + "ordinary_dividends": { + "method": "amount_per_reporting_return", + "source": { + "count": { + "cell": "L9", + "value": 27607044, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Ordinary dividends", + "Ordinary dividends", + "Ordinary dividends", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "M9", + "value": 260252720, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Ordinary dividends", + "Ordinary dividends", + "Ordinary dividends", + "Amount", + "Amount" + ] + } + }, + "destination_2023": { + "count": { + "cell": "X9", + "value": 33718460, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Ordinary dividends", + "Ordinary dividends", + "Ordinary dividends", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "Y9", + "value": 504224981, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Ordinary dividends", + "Ordinary dividends", + "Ordinary dividends", + "Amount", + "Amount" + ] + } + }, + "observed_2015_2023_factor": "1.586285145427430620786783837166923431342", + "modeled_2023_2024_factor": "1.029494391241278363778380187855675381192", + "factor_2015_2024": "1.633071660126895406016322292262784767590" + }, + "qualified_dividends": { + "method": "amount_per_reporting_return", + "source": { + "count": { + "cell": "N9", + "value": 25755976, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Qualified dividends [1]", + "Qualified dividends [1]", + "Qualified dividends [1]", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "O9", + "value": 203187788, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Qualified dividends [1]", + "Qualified dividends [1]", + "Qualified dividends [1]", + "Amount", + "Amount" + ] + } + }, + "destination_2023": { + "count": { + "cell": "Z9", + "value": 31106344, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Qualified dividends [2]", + "Qualified dividends [2]", + "Qualified dividends [2]", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "AA9", + "value": 336070296, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Qualified dividends [2]", + "Qualified dividends [2]", + "Qualified dividends [2]", + "Amount", + "Amount" + ] + } + }, + "observed_2015_2023_factor": "1.369498520263452278862567470282453558367", + "modeled_2023_2024_factor": "1.029494391241278363778380187855675381192", + "factor_2015_2024": "1.409891045424454325551259680558160347719" + }, + "state_tax_refund": { + "method": "amount_per_reporting_return", + "source": { + "count": { + "cell": "P9", + "value": 20256512, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "State income tax refunds", + "State income tax refunds", + "State income tax refunds", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "Q9", + "value": 31110732, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "State income tax refunds", + "State income tax refunds", + "State income tax refunds", + "Amount", + "Amount" + ] + } + }, + "destination_2023": { + "count": { + "cell": "AB9", + "value": 3109096, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "State income tax refunds", + "State income tax refunds", + "State income tax refunds", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "AC9", + "value": 4311977, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "State income tax refunds", + "State income tax refunds", + "State income tax refunds", + "Amount", + "Amount" + ] + } + }, + "observed_2015_2023_factor": "0.9030186910014518713848306957259555062394", + "modeled_2023_2024_factor": "1.029494391241278363778380187855675381192", + "factor_2015_2024": "0.9296526775720357466732616002244135476521" + }, + "alimony_income": { + "method": "amount_per_reporting_return", + "source": { + "count": { + "cell": "R9", + "value": 414420, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Alimony received", + "Alimony received", + "Alimony received", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "S9", + "value": 10077086, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Alimony received", + "Alimony received", + "Alimony received", + "Amount", + "Amount" + ] + } + }, + "destination_2023": { + "count": { + "cell": "AD9", + "value": 183582, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Alimony received", + "Alimony received", + "Alimony received", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "AE9", + "value": 6686429, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Alimony received", + "Alimony received", + "Alimony received", + "Amount", + "Amount" + ] + } + }, + "observed_2015_2023_factor": "1.497855374161636196047724476262267688168", + "modeled_2023_2024_factor": "1.029494391241278363778380187855675381192", + "factor_2015_2024": "1.542033706590010885068081742929926580179" + }, + "business_profit": { + "method": "amount_per_reporting_return", + "source": { + "count": { + "cell": "T9", + "value": 18791200, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Business or profession", + "Net income", + "Net income", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "U9", + "value": 391975736, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Business or profession", + "Net income", + "Net income", + "Amount", + "Amount" + ] + } + }, + "destination_2023": { + "count": { + "cell": "AF9", + "value": 21411713, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Business or profession", + "Net income", + "Net income", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "AG9", + "value": 530380019, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Business or profession", + "Net income", + "Net income", + "Amount", + "Amount" + ] + } + }, + "observed_2015_2023_factor": "1.187493036701270108472919624578501757792", + "modeled_2023_2024_factor": "1.029494391241278363778380187855675381192", + "factor_2015_2024": "1.222517420922031096142334753675412822692" + }, + "business_loss": { + "method": "amount_per_reporting_return", + "source": { + "count": { + "cell": "V9", + "value": 5935725, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Business or profession", + "Net loss", + "Net loss", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "W9", + "value": 60161435, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Business or profession", + "Net loss", + "Net loss", + "Amount", + "Amount" + ] + } + }, + "destination_2023": { + "count": { + "cell": "AH9", + "value": 9067174, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Business or profession", + "Net loss", + "Net loss", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "AI9", + "value": 153441387, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Business or profession", + "Net loss", + "Net loss", + "Amount", + "Amount" + ] + } + }, + "observed_2015_2023_factor": "1.669652726148833347538733957583542565766", + "modeled_2023_2024_factor": "1.029494391241278363778380187855675381192", + "factor_2015_2024": "1.718898116890934040328182671660914217539" + }, + "non_schedule_d_distributions": { + "method": "amount_per_reporting_return", + "source": { + "count": { + "cell": "X9", + "value": 4323250, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Capital gain distributions reported on Form 1040", + "Capital gain distributions reported on Form 1040", + "Capital gain distributions reported on Form 1040", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "Y9", + "value": 11563203, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Capital gain distributions reported on Form 1040", + "Capital gain distributions reported on Form 1040", + "Capital gain distributions reported on Form 1040", + "Amount", + "Amount" + ] + } + }, + "destination_2023": { + "count": { + "cell": "AJ9", + "value": 3209131, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Capital gain distributions reported on Form 1040", + "Capital gain distributions reported on Form 1040", + "Capital gain distributions reported on Form 1040", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "AK9", + "value": 9340820, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Capital gain distributions reported on Form 1040", + "Capital gain distributions reported on Form 1040", + "Capital gain distributions reported on Form 1040", + "Amount", + "Amount" + ] + } + }, + "observed_2015_2023_factor": "1.088252710423015210712466518141315463440", + "modeled_2023_2024_factor": "1.029494391241278363778380187855675381192", + "factor_2015_2024": "1.120350061633613230084419890940102481805" + }, + "other_property_gain": { + "method": "amount_per_reporting_return", + "source": { + "count": { + "cell": "AD9", + "value": 1063576, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of property other than capital assets", + "Sales of property other than capital assets", + "Net gain", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "AE9", + "value": 33036562, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of property other than capital assets", + "Sales of property other than capital assets", + "Net gain", + "Amount", + "Amount" + ] + } + }, + "destination_2023": { + "count": { + "cell": "AP9", + "value": 945122, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of property other than capital assets", + "Sales of property other than capital assets", + "Net gain", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "AQ9", + "value": 49567896, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of property other than capital assets", + "Sales of property other than capital assets", + "Net gain", + "Amount", + "Amount" + ] + } + }, + "observed_2015_2023_factor": "1.688442578911762234575993608257254511052", + "modeled_2023_2024_factor": "1.029494391241278363778380187855675381192", + "factor_2015_2024": "1.738242164922618767195190248113271033066" + }, + "other_property_loss": { + "method": "amount_per_reporting_return", + "source": { + "count": { + "cell": "AF9", + "value": 1153117, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of property other than capital assets", + "Sales of property other than capital assets", + "Net loss", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "AG9", + "value": 21093508, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of property other than capital assets", + "Sales of property other than capital assets", + "Net loss", + "Amount", + "Amount" + ] + } + }, + "destination_2023": { + "count": { + "cell": "AR9", + "value": 775387, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of property other than capital assets", + "Sales of property other than capital assets", + "Net loss", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "AS9", + "value": 19448929, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Sales of property other than capital assets", + "Sales of property other than capital assets", + "Net loss", + "Amount", + "Amount" + ] + } + }, + "observed_2015_2023_factor": "1.371202948062275806522918206792748706765", + "modeled_2023_2024_factor": "1.029494391241278363778380187855675381192", + "factor_2015_2024": "1.411645744283618865226902643798237455207" + }, + "ira_distributions": { + "method": "amount_per_reporting_return", + "source": { + "count": { + "cell": "AH9", + "value": 14159018, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Taxable Individual Retirement Arrangement (IRA) distributions", + "Taxable Individual Retirement Arrangement (IRA) distributions", + "Taxable Individual Retirement Arrangement (IRA) distributions", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "AI9", + "value": 253213041, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Taxable Individual Retirement Arrangement (IRA) distributions", + "Taxable Individual Retirement Arrangement (IRA) distributions", + "Taxable Individual Retirement Arrangement (IRA) distributions", + "Amount", + "Amount" + ] + } + }, + "destination_2023": { + "count": { + "cell": "AT9", + "value": 16694154, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Taxable Individual Retirement Arrangement (IRA) distributions", + "Taxable Individual Retirement Arrangement (IRA) distributions", + "Taxable Individual Retirement Arrangement (IRA) distributions", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "AU9", + "value": 438147938, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Taxable Individual Retirement Arrangement (IRA) distributions", + "Taxable Individual Retirement Arrangement (IRA) distributions", + "Taxable Individual Retirement Arrangement (IRA) distributions", + "Amount", + "Amount" + ] + } + }, + "observed_2015_2023_factor": "1.467585531705808482641269279807429143993", + "modeled_2023_2024_factor": "1.029494391241278363778380187855675381192", + "factor_2015_2024": "1.510871073557979130794209158828995676924" + }, + "taxable_pensions": { + "method": "amount_per_reporting_return", + "source": { + "count": { + "cell": "AL9", + "value": 28199160, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Pensions and annuities", + "Pensions and annuities", + "Taxable", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "AM9", + "value": 689991999, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Pensions and annuities", + "Pensions and annuities", + "Taxable", + "Amount", + "Amount" + ] + } + }, + "destination_2023": { + "count": { + "cell": "AX9", + "value": 29541284, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Pensions and annuities", + "Pensions and annuities", + "Taxable", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "AY9", + "value": 932130236, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Pensions and annuities", + "Pensions and annuities", + "Taxable", + "Amount", + "Amount" + ] + } + }, + "observed_2015_2023_factor": "1.289553441251142788324750319651443404751", + "modeled_2023_2024_factor": "1.029494391241278363778380187855675381192", + "factor_2015_2024": "1.327588034973940867230285994253866499704" + }, + "rental_royalty_profit": { + "method": "amount_per_reporting_return", + "source": { + "count": { + "cell": "AZ9", + "value": 6768234, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Total rental and royalty", + "Net income", + "Net income", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "BA9", + "value": 103058883, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Total rental and royalty", + "Net income", + "Net income", + "Amount", + "Amount" + ] + } + }, + "destination_2023": { + "count": { + "cell": "BL9", + "value": 6346357, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Total rental and royalty", + "Net income", + "Net income", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "BM9", + "value": 138609251, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Total rental and royalty", + "Net income", + "Net income", + "Amount", + "Amount" + ] + } + }, + "observed_2015_2023_factor": "1.434358304667835504133977902598222238470", + "modeled_2023_2024_factor": "1.029494391241278363778380187855675381192", + "factor_2015_2024": "1.476663829685885394438774259073237903799" + }, + "rental_royalty_loss": { + "method": "amount_per_reporting_return", + "source": { + "count": { + "cell": "BB9", + "value": 4531666, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Total rental and royalty", + "Net loss", + "Net loss", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "BC9", + "value": 46245560, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Total rental and royalty", + "Net loss", + "Net loss", + "Amount", + "Amount" + ] + } + }, + "destination_2023": { + "count": { + "cell": "BN9", + "value": 3196282, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Total rental and royalty", + "Net loss", + "Net loss", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "BO9", + "value": 53570273, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Total rental and royalty", + "Net loss", + "Net loss", + "Amount", + "Amount" + ] + } + }, + "observed_2015_2023_factor": "1.642353427026338423846026443645930519505", + "modeled_2023_2024_factor": "1.029494391241278363778380187855675381192", + "factor_2015_2024": "1.690793641559507564235995133214906035185" + }, + "farm_rent_profit": { + "method": "amount_per_reporting_return", + "source": { + "count": { + "cell": "AV9", + "value": 394273, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Farm rental", + "Net income", + "Net income", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "AW9", + "value": 5205330, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Farm rental", + "Net income", + "Net income", + "Amount", + "Amount" + ] + } + }, + "destination_2023": { + "count": { + "cell": "BH9", + "value": 325953, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Farm rental", + "Net income", + "Net income", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "BI9", + "value": 6811206, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Farm rental", + "Net income", + "Net income", + "Amount", + "Amount" + ] + } + }, + "observed_2015_2023_factor": "1.582769973612852083106490319603270854083", + "modeled_2023_2024_factor": "1.029494391241278363778380187855675381192", + "factor_2015_2024": "1.629452810459537374541656575493532799084" + }, + "farm_rent_loss": { + "method": "amount_per_reporting_return", + "source": { + "count": { + "cell": "AX9", + "value": 101041, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Farm rental", + "Net loss", + "Net loss", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "AY9", + "value": 710953, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Farm rental", + "Net loss", + "Net loss", + "Amount", + "Amount" + ] + } + }, + "destination_2023": { + "count": { + "cell": "BJ9", + "value": 67356, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Farm rental", + "Net loss", + "Net loss", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "BK9", + "value": 529586, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Farm rental", + "Net loss", + "Net loss", + "Amount", + "Amount" + ] + } + }, + "observed_2015_2023_factor": "1.117421316799902418041369349372073484912", + "modeled_2023_2024_factor": "1.029494391241278363778380187855675381192", + "factor_2015_2024": "1.150378978298943195689490419607276484593" + }, + "estate_profit": { + "method": "amount_per_reporting_return", + "source": { + "count": { + "cell": "BH9", + "value": 630256, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Estate and trust", + "Net income", + "Net income", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "BI9", + "value": 32453002, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Estate and trust", + "Net income", + "Net income", + "Amount", + "Amount" + ] + } + }, + "destination_2023": { + "count": { + "cell": "BX9", + "value": 648583, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Estate and trust", + "Net income", + "Net income", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "BY9", + "value": 47892046, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Estate and trust", + "Net income", + "Net income", + "Amount", + "Amount" + ] + } + }, + "observed_2015_2023_factor": "1.434035630420548027555129084975779987827", + "modeled_2023_2024_factor": "1.029494391241278363778380187855675381192", + "factor_2015_2024": "1.476331638358104936021886589313386353228" + }, + "estate_loss": { + "method": "amount_per_reporting_return", + "source": { + "count": { + "cell": "BJ9", + "value": 57802, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Estate and trust", + "Net loss", + "Net loss", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "BK9", + "value": 5033199, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Estate and trust", + "Net loss", + "Net loss", + "Amount", + "Amount" + ] + } + }, + "destination_2023": { + "count": { + "cell": "BZ9", + "value": 36592, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Estate and trust", + "Net loss", + "Net loss", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "CA9", + "value": 4898828, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Estate and trust", + "Net loss", + "Net loss", + "Amount", + "Amount" + ] + } + }, + "observed_2015_2023_factor": "1.537463478591264240530009559622650667952", + "modeled_2023_2024_factor": "1.029494391241278363778380187855675381192", + "factor_2015_2024": "1.582810027948011789708036602150526302352" + }, + "farm_profit": { + "method": "amount_per_reporting_return", + "source": { + "count": { + "cell": "BL9", + "value": 520802, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Farm", + "Net income", + "Net income", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "BM9", + "value": 13533867, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Farm", + "Net income", + "Net income", + "Amount", + "Amount" + ] + } + }, + "destination_2023": { + "count": { + "cell": "CB9", + "value": 484162, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Farm", + "Net income", + "Net income", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "CC9", + "value": 17656301, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Farm", + "Net income", + "Net income", + "Amount", + "Amount" + ] + } + }, + "observed_2015_2023_factor": "1.403329848619305835091312061082920777637", + "modeled_2023_2024_factor": "1.029494391241278363778380187855675381192", + "factor_2015_2024": "1.444720208215047581256304812994467826979" + }, + "farm_loss": { + "method": "amount_per_reporting_return", + "source": { + "count": { + "cell": "BN9", + "value": 1278825, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Farm", + "Net loss", + "Net loss", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "BO9", + "value": 27497651, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Farm", + "Net loss", + "Net loss", + "Amount", + "Amount" + ] + } + }, + "destination_2023": { + "count": { + "cell": "CD9", + "value": 1258869, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Farm", + "Net loss", + "Net loss", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "CE9", + "value": 45444491, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Farm", + "Net loss", + "Net loss", + "Amount", + "Amount" + ] + } + }, + "observed_2015_2023_factor": "1.678866744726831887395787683833365558775", + "modeled_2023_2024_factor": "1.029494391241278363778380187855675381192", + "factor_2015_2024": "1.728383897337776476443532509645504817056" + }, + "other_income_profit": { + "method": "amount_per_reporting_return", + "source": { + "count": { + "cell": "BX9", + "value": 6121770, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Other income", + "Net income", + "Net income", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "BY9", + "value": 46769290, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Other income", + "Net income", + "Net income", + "Amount", + "Amount" + ] + } + }, + "destination_2023": { + "count": { + "cell": "CN9", + "value": 7468434, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Other income", + "Net income", + "Net income", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "CO9", + "value": 60816508, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Other income", + "Net income", + "Net income", + "Amount", + "Amount" + ] + } + }, + "observed_2015_2023_factor": "1.065879634464584874641519579136196913601", + "modeled_2023_2024_factor": "1.029494391241278363778380187855675381192", + "factor_2015_2024": "1.097317105419594110775195552571543585636" + }, + "other_income_loss": { + "method": "amount_per_reporting_return", + "source": { + "count": { + "cell": "BZ9", + "value": 332708, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Other income", + "Net loss", + "Net loss", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "CA9", + "value": 6693961, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Other income", + "Net loss", + "Net loss", + "Amount", + "Amount" + ] + } + }, + "destination_2023": { + "count": { + "cell": "CP9", + "value": 318839, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Other income", + "Net loss", + "Net loss", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "CQ9", + "value": 12213628, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Other income", + "Net loss", + "Net loss", + "Amount", + "Amount" + ] + } + }, + "observed_2015_2023_factor": "1.903940252832176698123261573865488200400", + "modeled_2023_2024_factor": "1.029494391241278363778380187855675381192", + "factor_2015_2024": "1.960095811549227363973941096035769795063" + }, + "short_current_gain": { + "method": "current_year_gross_component_dollars_per_all_return", + "source_families": [ + "short_assets_gain", + "short_other_gain", + "short_pass_through_gain" + ], + "amounts_thousands_usd": [ + 45338966, + 96011134 + ], + "denominator_counts": [ + 150493263, + 160602107 + ], + "application_assumption": "Positive and negative net PUF flows follow corresponding gross current-year components; no carryover is introduced.", + "observed_2015_2023_factor": "1.984338725063739328074935070060878814103", + "modeled_2023_2024_factor": "1.029494391241278363778380187855675381192", + "factor_2015_2024": "2.042865587775988756504710527637911842775" + }, + "short_current_loss": { + "method": "current_year_gross_component_dollars_per_all_return", + "source_families": [ + "short_assets_loss", + "short_other_loss", + "short_pass_through_loss" + ], + "amounts_thousands_usd": [ + 86661831, + 156143032 + ], + "denominator_counts": [ + 150493263, + 160602107 + ], + "application_assumption": "Positive and negative net PUF flows follow corresponding gross current-year components; no carryover is introduced.", + "observed_2015_2023_factor": "1.688342557388515472795249540239320768338", + "modeled_2023_2024_factor": "1.029494391241278363778380187855675381192", + "factor_2015_2024": "1.738139193325432816803529458382722438642" + }, + "long_current_gain": { + "method": "current_year_gross_component_dollars_per_all_return", + "source_families": [ + "long_assets_gain", + "long_other_gain", + "long_pass_through_gain", + "long_distributions" + ], + "amounts_thousands_usd": [ + 787889885, + 1048438049 + ], + "denominator_counts": [ + 150493263, + 160602107 + ], + "application_assumption": "Positive and negative net PUF flows follow corresponding gross current-year components; no carryover is introduced.", + "observed_2015_2023_factor": "1.246932856321931105568396313543555115004", + "modeled_2023_2024_factor": "1.029494391241278363778380187855675381192", + "factor_2015_2024": "1.283710381837894882785950440755768785471" + }, + "long_current_loss": { + "method": "current_year_gross_component_dollars_per_all_return", + "source_families": [ + "long_assets_loss", + "long_other_loss", + "long_pass_through_loss" + ], + "amounts_thousands_usd": [ + 75600579, + 177297155 + ], + "denominator_counts": [ + 150493263, + 160602107 + ], + "application_assumption": "Positive and negative net PUF flows follow corresponding gross current-year components; no carryover is introduced.", + "observed_2015_2023_factor": "2.197568722014698372473641296575486125444", + "modeled_2023_2024_factor": "1.029494391241278363778380187855675381192", + "factor_2015_2024": "2.262384673681395979556694950087192296750" + }, + "partnership_s_corp_profit": { + "method": "combined_dollars_per_all_return", + "source": { + "count": { + "cell": "BD9", + "value": 6044409, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Partnership and S corporation", + "Net income", + "Net income", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "BE9", + "value": 755622761, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Partnership and S corporation", + "Net income", + "Net income", + "Amount", + "Amount" + ] + } + }, + "destination_2023_components": [ + { + "count": { + "cell": "BP9", + "value": 3266345, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Partnership", + "Net income", + "Net income", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "BQ9", + "value": 452077038, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Partnership", + "Net income", + "Net income", + "Amount", + "Amount" + ] + } + }, + { + "count": { + "cell": "BT9", + "value": 4138040, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "S corporation", + "Net income", + "Net income", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "BU9", + "value": 797450861, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "S corporation", + "Net income", + "Net income", + "Amount", + "Amount" + ] + } + } + ], + "denominator_counts": [ + 150493263, + 160602107 + ], + "observed_2015_2023_factor": "1.549554109090802831376907083449129785317", + "modeled_2023_2024_factor": "1.029494391241278363778380187855675381192", + "factor_2015_2024": "1.595257264233857504616939193376066035754" + }, + "partnership_s_corp_loss": { + "method": "combined_dollars_per_all_return", + "source": { + "count": { + "cell": "BF9", + "value": 2699817, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Partnership and S corporation", + "Net loss", + "Net loss", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "BG9", + "value": 126618186, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2015 (Filing Year 2016)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Partnership and S corporation", + "Net loss", + "Net loss", + "Amount", + "Amount" + ] + } + }, + "destination_2023_components": [ + { + "count": { + "cell": "BR9", + "value": 2103634, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Partnership", + "Net loss", + "Net loss", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "BS9", + "value": 168884100, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "Partnership", + "Net loss", + "Net loss", + "Amount", + "Amount" + ] + } + }, + { + "count": { + "cell": "BV9", + "value": 1415974, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "S corporation", + "Net loss", + "Net loss", + "Number of returns", + "Number of returns" + ] + }, + "amount_thousands_usd": { + "cell": "BW9", + "value": 89543715, + "headers": [ + "Table 1.4. All Returns: Sources of Income, Adjustments, and Tax Items, by Size of Adjusted Gross Income, Tax Year 2023 (Filing Year 2024)", + "(All figures are estimates based on samples\u2014money amounts are in thousands of dollars)", + "S corporation", + "Net loss", + "Net loss", + "Amount", + "Amount" + ] + } + } + ], + "denominator_counts": [ + 150493263, + 160602107 + ], + "observed_2015_2023_factor": "1.912533224223542778513579904641714927631", + "modeled_2023_2024_factor": "1.029494391241278363778380187855675381192", + "factor_2015_2024": "1.968942227400735507640732147170530268700" + }, + "nonqualified_dividends": { + "method": "residual_dollars_per_ordinary_dividend_return", + "observed_2015_2023_factor": "2.412636223005660751243227924409122950293", + "modeled_2023_2024_factor": "1.029494391241278363778380187855675381192", + "factor_2015_2024": "2.483795459689869824933006427197633652403", + "source_families": [ + "ordinary_dividends", + "qualified_dividends" + ] + } + }, + "modeled_bridge": { + "method": "constant real conditional amounts after 2023", + "cpi_2023": "304.702", + "cpi_2024": "313.689" + }, + "capital_carryover_reconciliation_thousands_usd": { + "short": { + "2015": { + "net_including_carried_loss": -212428017, + "carried_loss": 171105171, + "current_year_net_by_removing_carryover": -41322846, + "current_year_gross_component_gain": 45338966, + "current_year_gross_component_loss": 86661831, + "current_year_net_from_component_difference": -41322865, + "reported_less_component_net_residual": 19, + "reconciliation_status": "Both constructions retained; differences not attributed to rounding or unspecified items without documentary proof." + }, + "2023": { + "net_including_carried_loss": -603427784, + "carried_loss": 543302365, + "current_year_net_by_removing_carryover": -60125419, + "current_year_gross_component_gain": 96011134, + "current_year_gross_component_loss": 156143032, + "current_year_net_from_component_difference": -60131898, + "reported_less_component_net_residual": 6479, + "reconciliation_status": "Both constructions retained; differences not attributed to rounding or unspecified items without documentary proof." + } + }, + "long": { + "2015": { + "net_including_carried_loss": 420589539, + "carried_loss": 291699768, + "current_year_net_by_removing_carryover": 712289307, + "current_year_gross_component_gain": 787889885, + "current_year_gross_component_loss": 75600579, + "current_year_net_from_component_difference": 712289306, + "reported_less_component_net_residual": 1, + "reconciliation_status": "Both constructions retained; differences not attributed to rounding or unspecified items without documentary proof." + }, + "2023": { + "net_including_carried_loss": 521214809, + "carried_loss": 349944357, + "current_year_net_by_removing_carryover": 871159166, + "current_year_gross_component_gain": 1048438049, + "current_year_gross_component_loss": 177297155, + "current_year_net_from_component_difference": 871140894, + "reported_less_component_net_residual": 18272, + "reconciliation_status": "Both constructions retained; differences not attributed to rounding or unspecified items without documentary proof." + } + } + }, + "notes": [ + "Reporting-return means mix intensive-margin growth and changing composition. This is a distribution transport assumption, not panel growth.", + "Positive/loss magnitudes are distinct series. Source zeros and sign states are retained.", + "No total calibration or reporting-incidence claim; source design weights are unchanged." + ] +} diff --git a/docs/evidence/puf2015-target2024/PUBLIC-WORKBOOK-SOURCES.json b/docs/evidence/puf2015-target2024/PUBLIC-WORKBOOK-SOURCES.json new file mode 100644 index 000000000..23975da76 --- /dev/null +++ b/docs/evidence/puf2015-target2024/PUBLIC-WORKBOOK-SOURCES.json @@ -0,0 +1,34 @@ +[ + { + "url": "https://www.irs.gov/pub/irs-soi/15in14ar.xls", + "bytes": 83456, + "sha256": "9f41bee263f5a8bb971df39a9673b55136c94954131a95bfd83b73a22801ec89", + "format": "OLE xls", + "max_bytes": 4194304, + "filename": "15in14ar.xls" + }, + { + "url": "https://www.irs.gov/pub/irs-soi/23in14ar.xls", + "bytes": 115712, + "sha256": "b6c1f87fbb5533417e195f6938538e5de09b6a0825a6a54346bf9363a18d96af", + "format": "OLE xls", + "max_bytes": 4194304, + "filename": "23in14ar.xls" + }, + { + "url": "https://www.irs.gov/pub/irs-soi/15in14acg.xls", + "bytes": 64000, + "sha256": "7e128eb06e14371a15baa72060e91d74e4c55ba8596610ddd047c9ef168cbbd1", + "format": "OLE xls", + "max_bytes": 4194304, + "filename": "15in14acg.xls" + }, + { + "url": "https://www.irs.gov/pub/irs-soi/23in14acg.xls", + "bytes": 74752, + "sha256": "5b35c2f007f16503c0dcf6cc085d3a6b049ca523d99b7d7ae29afdbdd2d02e70", + "format": "OLE xls", + "max_bytes": 4194304, + "filename": "23in14acg.xls" + } +] diff --git a/docs/evidence/spec-engine/us-f0-coverage.json b/docs/evidence/spec-engine/us-f0-coverage.json index a759198bc..c7032335f 100644 --- a/docs/evidence/spec-engine/us-f0-coverage.json +++ b/docs/evidence/spec-engine/us-f0-coverage.json @@ -1656,13 +1656,13 @@ "compiler_ir.node_slices" ], "expected": { - "map_sha256": "20058e544f6034cee2e76d3b864cf86b6230c8dce042931dfe9247d8ac1329c4", - "protocol_sha256": "553d5e0bd5afa93146dfbf8dca684926ee773dbfc89a559434528af5b4eb1d97" + "map_sha256": "be2d24ad74bdc6339dfa073d01469074baee4e3a691eaa350451d36fa17d13dc", + "protocol_sha256": "7c67a5a315a615b152d7de106ae10b1828c166bb11c67a713c8a52d65ae9db47" }, "failures": [], "observed": { - "map_sha256": "20058e544f6034cee2e76d3b864cf86b6230c8dce042931dfe9247d8ac1329c4", - "protocol_sha256": "553d5e0bd5afa93146dfbf8dca684926ee773dbfc89a559434528af5b4eb1d97" + "map_sha256": "be2d24ad74bdc6339dfa073d01469074baee4e3a691eaa350451d36fa17d13dc", + "protocol_sha256": "7c67a5a315a615b152d7de106ae10b1828c166bb11c67a713c8a52d65ae9db47" }, "status": "covered" }, @@ -1677,7 +1677,7 @@ "compiler_ir.seed_stream_map" ], "expected": { - "implementation_sha256": "553d5e0bd5afa93146dfbf8dca684926ee773dbfc89a559434528af5b4eb1d97", + "implementation_sha256": "7c67a5a315a615b152d7de106ae10b1828c166bb11c67a713c8a52d65ae9db47", "protocol": "legacy-v1", "streams": [ "build_model", @@ -1698,7 +1698,7 @@ }, "failures": [], "observed": { - "implementation_sha256": "553d5e0bd5afa93146dfbf8dca684926ee773dbfc89a559434528af5b4eb1d97", + "implementation_sha256": "7c67a5a315a615b152d7de106ae10b1828c166bb11c67a713c8a52d65ae9db47", "protocol": "legacy-v1", "streams": [ "build_model", @@ -2599,7 +2599,7 @@ "country": "us", "schema_id": "country_spec", "schema_version": 1, - "spec_sha256": "1eeca53aa80da949a292fbd8cb0afefde95c68888ed35f3f477f3c962e6bc644" + "spec_sha256": "f03e4d08d0ba4e15da53aab86bd2fbf50286e19e248657ac859b760103af2405" } }, "report_schema_version": 3, @@ -2609,7 +2609,7 @@ "country": "us", "schema_id": "country_spec", "schema_version": 1, - "spec_sha256": "1eeca53aa80da949a292fbd8cb0afefde95c68888ed35f3f477f3c962e6bc644" + "spec_sha256": "f03e4d08d0ba4e15da53aab86bd2fbf50286e19e248657ac859b760103af2405" }, "status": "pass" } diff --git a/docs/geography-assignment.md b/docs/geography-assignment.md new file mode 100644 index 000000000..5793b0b1f --- /dev/null +++ b/docs/geography-assignment.md @@ -0,0 +1,261 @@ +# Geography assignment in the population graph + +Design decision, corrected 10 September 2026: finish constructing the full survey +multispine, including its initial support and PUF clones, then assign one small +census area to each resulting household. Derive larger geographies from that +area using identified, versioned mappings. Distinct clones may receive different +areas subject to their observed source constraints. The assigned location then +remains fixed through subsequent enrichment, calibration and analysis views. + +The earlier block-before-clone implementation and its passing controls remain +historical evidence. They do not demonstrate this corrected ordering. Geographic +assignment must use a stable post-clone identity, including a clone discriminator +where original-source identity alone is shared by multiple households. + +The corrected order now passes 70 controls on invented inputs, plus a separate +default-path compatibility check. The nine-node survey prefix expands the initial +clones before assignment; its nineteen-node financial and twelve-node age +extensions retain the assigned geography. Draw identity combines +`survey_geography_origin_key` with `household_support_clone_index`, so renumbering +households does not redraw their location. A typed validation artifact orders +financial enrichment after the geography gate. See the +[scoped acceptance](../experiments/us-postclone-geography-70-controls-20260910.json). +Native execution of this corrected order and population-quality acceptance +remain pending. The older results below retain their original source scope. + +## Declarative country capability + +Geography is a declarative country capability backed by shared graph operators. +A country declares its atomic area type, code system and vintage, source mapping +artifacts, observed-geography constraints and sampling-weight convention. The UK +declaration can select an area system by nation. Adding a country should not +require writing a country-specific assignment or derivation kernel. + +The shared assignment operator selects the atomic area using the declared +support and stable household identity. Shared lookup/join operations derive +larger geographies from identified mappings. If an input already has a qualified +atomic area, validate and retain it. Source acquisition and normalization may +still require publisher-specific adapters; they produce the common support and +mapping contracts, rather than owning a separate assignment algorithm. + +Sampling and derivation remain executable graph operations with typed inputs, +outputs and replay identity. A country configuration does not itself execute +them. Mapping edges retain their relation type, including exact nesting or an +explicit best-fit convention. Prefer existing graph lookup/join primitives where +they support these contracts; introduce only the shared behavior they lack. + +The existing country-spec geography declaration is a starting point, but its +legacy clone-and-assign contract and country-specific runtime references do not +yet implement this shared atomic-area contract. Preserve those compatibility +paths while the shared operators and country adapters receive their own checks. + +```mermaid +flowchart LR + S[Complete survey multispine including clones] --> A[Assign one small area per resulting household] + C[Observed survey geography constraints] --> A + L[Versioned small-area support] --> A + A --> D[Derive larger geographies] + M[Versioned geographic mappings] --> D + D --> E[Subsequent enrichment retains location] + E --> R[Rules and calibration] + R --> P[Pruned analysis file retains location] +``` + +## Country anchors + +| Country | Assigned household anchor | Derived geographies | +| --- | --- | --- | +| US | Census block, with census vintage | Block group, tract, county, state, PUMA and congressional district using the declared mapping/boundary vintages | +| England and Wales | 2021 Output Area | LSOA, MSOA, local authority, ward, constituency and region | +| Scotland | 2022 Output Area | Scottish statistical areas, council area, ward, constituency and region | +| Northern Ireland | 2021 Data Zone | Super Data Zone, local government district, constituency and nation | + +Northern Ireland's 2021 Data Zones replaced the 2011 Small Areas. Scottish +Output Areas use the 2022 census. These are different source systems; the graph +must retain the original area type and vintage rather than imply identical units. +See [NISRA census output geography](https://www.nisra.gov.uk/statistics/census-2021-results/census-output-geography) +and [Scotland's census general report](https://www.scotlandscensus.gov.uk/about/scotlands-census-2022-general-report/). + +Some larger boundaries do not nest exactly. England and Wales small-area estimates +use best-fit mappings for wards and parliamentary constituencies. The derived +edge must identify whether its mapping is exact, best-fit or another explicit +approximation, with its source and vintage. A mapping convention cannot be +presented as observed household location. See the +[ONS methodology](https://www.ons.gov.uk/peoplepopulationandcommunity/populationandmigration/populationestimates/methodologies/smallareapopulationestimatesqmi). + +For US congressional districts, the Census Bureau's CD119 block equivalency +file assigns whole original 2020 tabulation blocks to districts. Colorado's +block `080010096072000` crosses the enacted CD07/CD08 boundary but is assigned +to CD08 for tabulation. Represent this as an official tabulation mapping, +not exact spatial containment. The same source distinguishes original 2020 +blocks from subsequently adjusted geometry and includes undefined `ZZ` areas. +See the [Census Bureau's CD119 documentation](https://www.census.gov/geographies/mapping-files/2025/dec/rdo/119-congressional-district-bef.html). +By contrast, the same-vintage tract-to-PUMA relation is nested, as described in +the [Census PUMA guidance](https://www.census.gov/programs-surveys/geography/guidance/geo-areas/pumas.html). + +## Required behavior + +- Respect observed source constraints, such as ACS PUMA and FRS region. Record + inferred geographic detail as modeled. Preserve observed source geography + separately from the assigned location. +- Bind each draw to a stable household identity, seed and assignment definition. + Reordering rows or selecting an existing household must not redraw its location. + Changed source support or boundary mappings produce a new identified revision. +- Assign household members consistently with their resulting household. Initial + clones have distinct draw identities and may receive different anchors. + Materialized county, district and other fields must agree with that household's + selected anchor. Source geography remains a separate observed constraint. +- Show assignment and derivation as separate operations in the graph. Expose the + source constraints, support, sampling weights, mapping conventions and judgment + annotations in the inspector. +- Refuse missing, ambiguous or incompatible mappings unless an explicit, + source-supported resolution rule is part of the assignment definition. +- Verify inheritance and mapping consistency again on every pruned export. + Engine input profiles and geographic build invariants are separate contracts. + +## Existing implementation and next changes + +The shared implementation now lives in `microcosm.build.atomic_geography` and +`microcosm.build.graph_atomic_geography`. It provides four country-neutral nodes: +`geography.support_import@1`, `geography.assign_atomic@1`, +`geography.derive@1` and `geography.gate@1`. Countries supply declarations and +normalized, pinned support files. The graph interfaces and legacy country +operators are unchanged. Country configuration and native support integration +remain necessary before these nodes can assign locations in a release build. + +The support contract uses deterministic NPZ bytes with no object arrays. It has +one unique string code per atomic area; every mapping column identifies its +source, vintage and relation, and every integer sampling-weight column identifies +its source and basis. Nonnegative weights have a bounded exact total. A source +adapter must resolve any nonfunctional crosswalk before admission; the shared +lookup never multiplies households through a many-to-many join. + +Assignment conditions on all nonmissing observed constraints jointly. Declared +stages can use different weights—for example, household counts to select a +constituency followed by population counts to select its small area. Each draw +uses the existing `keyed_uniform` protocol with stable source identity, the +declared stream, system, stage, assignment definition and support digest. +Selection uses an integer inverse CDF. Reordering, subsetting or adding unrelated +households preserves draws. The numerical declaration remains `platform_bitwise`; +cross-platform equivalence has not been established. + +The version-1 declaration lists `identity`, `stream`, three assignment `outputs` +(`area`, `system`, `basis`), and `systems`. Each system supplies its atomic +identity and source, a selector, observed constraints, an optional observed-area +column, sampling stages and derived layers. Geographic codes use nullable +strings throughout, retaining leading zeroes. An engine that needs integer +storage must declare that conversion separately. A system without a particular +layer produces missing values for that layer, which the integrity gate checks. +The graph stores the canonical declaration in its normative `definition` +parameter and exposes the stream separately; no graph parameter grammar changed. + +The 34 invented-data controls include real graph execution, cold and warm reuse, +source-change invalidation, exact lookup metadata, independent sampling-boundary +checks, three-system routing, observed-area retention, row/subset stability and +clone/prune validation. All passed under the isolated control runner on +9 September 2026. These checks establish shared operator behavior; they do not +admit a Census/ONS source or certify any population's geographic fit. The gate +checks mapping integrity, while the executor and build own structural lineage +and population-quality acceptance. The country's chosen identity still needs +verification across actual sampling rungs. + +The US development composed graph attaches geography after harmonization, but +`us_runtime/graph_geography.py` selects a joint tract/congressional-district cell +and emits PUMA, county and district. It does not assign a Census block. Its prior +national source and joint-support checks therefore do not establish block-first +acceptance. The separate atomic survey graph now provides that block assignment +and derivation connection before enrichment. Its population-only national block +support and source review are recorded below. + +The UK already has an area-based ladder in `uk_runtime/geography_ladder.py`. +Its sampler selects a constituency within FRS region using household counts, +then an OA within constituency using population; one selected ladder row supplies +the other geographies. It currently consumes a shared seeded random stream, so +stable household-keyed assignment under reordering and subsets still needs work. +The graph should expose the final OA/Data Zone assignment and the mapping +derivations explicitly, including that sampling convention. + +The Northern Ireland source builder currently infers Data Zone constituencies +using the modal active-postcode constituency. Review the official +[NISRA constituency aggregation and lookup resources](https://www.nisra.gov.uk/publications/census-2021-output-geography-information-papers) +before choosing or replacing that approximation. Source existence is not +acceptance of a particular lookup or its application to the population. + +The US block adapter in `us_runtime/atomic_block_support.py` now preserves each +supplied block, leading zeroes, population weight and the source-labeled +tract-to-PUMA and block-to-district mappings. Its declaration uses the shared +operators, preserves observed state/PUMA separately, and labels CD119 as +`official_tabulation`. Its sampling proxy is 2020 persons; this does not imply +household counts or support for subsequent construction in unpopulated 2020 +blocks. All 13 invented tests passed, including missing/inconsistent mapping +refusals and subset-stable assignment. Receipt SHA256: +`404deb7666b3ac9b8a654181228c545ab95c131f30a1fbe65a2e15d5a08a4c4c`. +These initial adapter tests do not establish native geographic fit. Subsequent +national support and native pilot results are recorded below. + +The earlier `us_runtime/graph_atomic_survey_clone.py` declared this sequence: +shared import, assignment, derivation and integrity gate, then the existing +combined-survey support clone and an inherited-mapping gate. The graph compiler +makes the clone depend on every member of its base version, including the +pre-clone gate. The post-clone gate never draws another location. An actual +executor control passes cold execution and required replay: four invented +households become eight, every location column is inherited, all entity rows +are copied, and each household weight is split equally across the pair. +Subset mapping verification also passes. Receipt SHA256: +`1bb75a5ee53edcbdd199a6d0163cbe75c32e54c80513cb3eb8d13021c3c2b1c3`. +A separate source-only Codex review found no actionable defects. + +The current survey path now qualifies observed state and ACS PUMA from the +retained original-source preparation. Eleven source controls and fourteen graph +controls passed, covering missing geography, source changes, replay and stable +household identity across sampling rungs. These controls use invented originals +through the actual source issuers. Their receipts are recorded in +`experiments/us-survey-geography-source-controls-20260909.json` and +`experiments/us-survey-geography-graph-controls-20260909.json`. + +The earlier `us_runtime/graph_atomic_survey_population.py` connected that projection +to block assignment, geographic derivation, the integrity gate and the combined +survey clone. It retains the raw allocation separately from the enriched +pre-clone population, even though both share the allocation version identifier. +The complete ten-node prefix passed five controls: fresh execution, required +replay, full geography inheritance, support-byte mismatch and late mutation +refusals. The test population contains six survey households before cloning and +twelve afterward. The exact tested revisions and remaining checks are recorded +in `experiments/us-atomic-survey-population-controls-20260909.json`. + +The earlier calibration budget and age runner consumed this optional prefix +and independently reconstructed it from the raw allocation. Eight budget +controls passed, as did the complete thirteen-node age graph through fresh +execution and required replay. A separate compatibility control verifies that +the existing predictor and PUF-host qualifiers still accept the default survey +prefix. See the corresponding `us-atomic-budget-semantic-8`, +`us-atomic-age-v2-1`, and `us-budget-predictor-compatibility-1` records in +`experiments/`. + +Replay can canonicalize the hidden backing values of missing numeric cells. +Persistent budget identities therefore bind logical Frame values, population +version, complete ownership, weight kinds, mass ledger and exact design-weight +bytes. They exclude only the helper receipt's physical population stamp. +Same-object mutation checks retain physical identity. The initial physical-stamp +replay failure remains recorded; the corrected budget controls pass. + +The byte-source adapter passed thirty-six invented-source controls through the +maintained PL, CD and PUMA parsers. This does not authorize acquisition of native +PL archives, which also contain housing segments. Native support preparation +will instead request only Census `P1_001N` block populations and independent +state totals, then join CD119 and tract-to-PUMA mappings. Delaware is the first +source control; national survey assignment requires complete admitted support +for the fifty states and DC. No native acquisition is accepted by these tests. + +`us_runtime/atomic_block_api_sources.py` implements that population-only response +adapter. All thirty-five invented controls pass, including independent state +totals, exact block preservation, missing mappings, malformed response fields, +byte limits and late source-record mutation. Request descriptors preserve the +repeated Census `in` parameters and omit credentials. The adapter does not infer +the response's origin from its contents; acquisition and publisher qualification +remain separate. Evidence: +`experiments/us-atomic-block-api-sources-35-controls-20260909.json`. + +Publisher-qualified native block support, national and district calibration, +and pruned export verification remain necessary for a release. Source integrity +checks and successful invented-data runs do not establish geographic fit. diff --git a/docs/puf2015-canonical59-and-growth.md b/docs/puf2015-canonical59-and-growth.md new file mode 100644 index 000000000..36733c8fe --- /dev/null +++ b/docs/puf2015-canonical59-and-growth.md @@ -0,0 +1,74 @@ +# PUF 2015 canonical donor and 2024 transport + +These source owners turn the statistical 2015 PUF into an explicit return-level donor with 59 canonical outputs. They preserve raw fields, source status and documentary decisions separately from modeled values. The six detailed mortgage balance, interest and origination-year outputs belong to the downstream SCF stage. + +```mermaid +flowchart LR + A[Authenticated PUF typed source] --> B[Observed and derived return fields] + B --> C[Eleven explicit baseline models] + C --> D[QBI model in 2015 money] + D --> E[Source-specific 2024 monetary transport] + E --> F[Canonical donor with 59 outputs typed donor] + F --> G[Eight-feature PUF59 donor interface] +``` + +The source codec retains all ordinary returns, including those missing the demographic supplement. Disclosure aggregates retain their source lexemes but are excluded from individual donors. Neither a demographic gap nor zero reported exemptions means that a source return is dropped. + +## Count measurements + +PUF 2015 MARS 2 combines joint and qualifying widow(er) returns. The matching predictor maps the survey's separate widow category into that combined class. It is named `puf_2015_filing_status_code`; generic model filing status is unchanged. + +The corresponding `puf_2015_capped_return_size` equals one plus an extra unit for MARS 2 plus the sum of the four reported dependent-category counts. The source owner validates their disclosure caps by filing class. On the survey side, the matching measurement caps the reported dependent total before adding the filing-class units. The result is a statistical predictor, not a count of physical people or co-residents. Actual survey membership remains unchanged. + +The separate `puf_person_incidence_capacity` is one for return-level modeled zero/one QBI incidence. It is used only to validate the canonical outcome representation and is excluded from the eight fitted predictors. + +## Observed fields and explicit models + +The decoder emits 33 directly mapped or algebraically derived baseline fields. Eleven additional baseline outputs explicitly model source gaps: a normalized interest split, a component-neutral Social Security carrier, desired contributions proxied by realized deductions, incomplete tuition and employee-expense proxies, and active-partnership earnings. These transformations retain their assumptions in the receipt. + +The interest model uses the existing [IRS 2015 Table 2.1 workbook](https://www.irs.gov/pub/irs-soi/15in21id.xls). It normalizes the four published component amounts within the source AGI band. Mortgage interest, points and mortgage insurance share the available home-interest leaf; the investment component alone enters investment expense. Complementary rounding preserves the source total exactly. This does not create a mortgage balance, origination year or observed loan structure. + +The QBI owner reproduces the archived data model's assumptions with RECID-keyed random draws, retaining a reusable full-cohort employee calibration. Its 15 added leaves and Schedule C replacement are modeled inputs. The statutory engine owns deductions and limits. Ordinary and SSTB Schedule C branches are summed for the donor's total self-employment predictor, matching the survey measurement. + +## Source-specific monetary transport + +The publisher's statistical 2015 convention is the input basis. Raw filing-year/month fields remain intact; the transform does not apply another row-year CPI adjustment. + +Wages and modeled W2 amounts use the [SSA average wage index](https://www.ssa.gov/oact/cola/AWI.html). Social Security carriers use [SSA COLAs](https://www.ssa.gov/oact/cola/colaseries.html) received from January 2016 through January 2024. This transports fixed entitlements and does not predict changes in recipient composition. + +Other income families use the observed 2015 and 2023 national amount/count cells of [IRS Table 1.4 and 1.4A](https://www.irs.gov/statistics/soi-tax-stats-individual-statistical-tables-by-size-of-adjusted-gross-income), with positive and loss magnitudes kept separate. The 2023→2024 step uses the [BLS annual CPI-U series](https://www.bls.gov/cpi/tables/supplemental-files/historical-cpi-u-202412.pdf), explicitly assuming stable real conditional amounts for that year. Expenses, UBIA and selected mixed model outputs use an explicitly named CPI proxy. Every monetary field has a declared rule; incidence fields are unchanged. + +Partnership and S-corporation reporting counts overlap, so their dollar components are summed with the common all-return denominator. Dividend residuals use the ordinary-dividend reporting universe and do not subtract overlapping recipient counts. + +The PUF short- and long-term capital fields remove the effect of carried losses. Published net-loss means that include carryovers cannot be substituted directly. The model uses current-year gross gain/loss components per all return as a declared proxy for signed net flows. The evidence retains both the carryover-adjusted reported net and component-difference net, including their unresolved residual. It does not attribute that difference to rounding without evidence. + +The observed series are source facts. Their application to individual donor amounts, the related-family substitutions, and the last-year bridge are modeling decisions. They do not establish forecast accuracy or current reporting incidence. A same-cohort CPI-only sensitivity is available. Source design weights are divided from S006 hundredths once and are never grown. + +## Validation status + +The ordinary verification executes real source decoding, the QBI model, growth, the deterministic typed envelope and the PUF59 donor interface at 64 and 2,048 invented records. It checks all 59 outputs, sign/zero preservation, physical type and knownness refusal, exact interest conservation, SSTB/base identities, source immutability, stable full-fit subset replay, artifact byte/value replay and malformed-input refusal. + +The 59 invented controls passed. The separately reviewed genuine construction then produced all 59 canonical outcomes for 207,692 ordinary returns in 23.07 seconds. The 64-return subset reproduced the full-cohort outputs when it reused the full QBI employee fit, and the typed donor passed byte/value replay through the canonical donor interface. + +Independent review subsequently found that the version 1 envelope could attach +an old descriptive receipt to altered, domain-valid values. The version 2 codec +now checks the row-ordered identity/weight/source-feature digest and all 59 output +values against that receipt on encode and decode. It also pins the actual +interest-band facts, verifies the embedded growth recipe, requires canonical +JSON and physical integer row metadata, and makes CPI-sensitivity acceptance +explicit. The corrected source passed 80 invented controls, including the 21 +new integrity cases. Formatting followed that run; the composed integration +checkout still needs its own execution checks. + +The original genuine donor and evidence remain on version 1. A version 2 genuine +construction or repacking requires separate verification, and the new decoder +refuses old magic. Serialized receipt consistency does not authenticate an +issuer, establish execution or confer release status. + +This result verifies source donor construction. Social Security component reconciliation, downstream SCF mortgage semantics, conditional-fit quality, recipient placement and calibration remain open release requirements. It does not certify a calibrated population or policy-analysis file. + +## Evidence bundled with the source + +The [national extract](evidence/puf2015-target2024/NATIONAL-GROWTH-EXTRACT.json) records the exact public workbook cells, units, hashes, family calculations and unresolved capital residuals. The [index transcript](evidence/puf2015-target2024/INDEX-VALUES.json) identifies the SSA and BLS values and retrieval sources; it is a transcription of retrieved values, not a hash of the original web pages. The [growth recipe](evidence/puf2015-target2024/GROWTH-RECIPE.json) assigns each monetary field a factor and interpretation. + +[Public workbook locators](evidence/puf2015-target2024/PUBLIC-WORKBOOK-SOURCES.json) provide the four official URLs and exact sizes and hashes without local cache paths. The recipe also preserves hashes of the original local acquisition receipt and extraction script. Those two local files are outside this publication bundle; their recorded hashes do not imply that a reader received them. diff --git a/docs/puf55-survey-ss-measurement-decision.md b/docs/puf55-survey-ss-measurement-decision.md new file mode 100644 index 000000000..fc12e6b85 --- /dev/null +++ b/docs/puf55-survey-ss-measurement-decision.md @@ -0,0 +1,107 @@ +# PUF55 survey Social Security conditioning decision + +On 2026-09-09 the build owner adopted +`social_security_filer_joint_spouse_report_sum_proxy` for PUF matching. This +supersedes the pending measurement judgment in +[survey-social-security-source.md](survey-social-security-source.md). The +decision authorizes this conditioning measurement and its explicit fallback; +it does not assert donor/recipient equivalence, allocate beneficiaries, complete +Social Security components, approve a fit, or admit a release. + +For each modeled return, include exactly one `HEAD` and, only when the actual +`filing_status_input` is `JOINT`, exactly one `SPOUSE`. Sum their available +nonnegative source report totals. Preserve combined family and child reports +without splitting, deduplicating, or subtracting amounts. Exclude dependents +from this return feature without changing any person's observed or unknown +Social Security cells or component masks. + +The public helper requires the actual authenticated survey preparation and +qualifies current source reports. It uses the exact modeled return roles +retained by that preparation. It does **not** rerun tax-unit construction using +current money. That limitation is accepted for this conditioning stage; later +production integration must qualify the chosen role/current-money relationship +explicitly. A caller-made role table or decoded receipt grants no authority. +The PUF disclosure-capped return-size predictor is not a person count, and the +coarsened PUF filing class 2 cannot identify an actual spouse: surviving-spouse +returns also map to that class. + +## Availability and refusal + +An available report-sum means every included reporter has an in-universe, +finite source amount. It does not mean those reporters legally own all the +reported benefits. Known zero reports are valid. Unknown components do not +invalidate a known total. Family/child ambiguity remains an explicit limitation +and does not itself trigger missingness. + +An unavailable included report, including an under-15 filer outside the source +question universe, leaves the proxy unknown and selects the separately named +`PUF55_SURVEY_SS_NO_TOTAL` profile. This profile has the same 55 outputs and +eight predictors. The existing `PUF55_SURVEY_SS` retains nine predictors. +Neither produces the four Social Security components. There is no partial sum, +dependent-zero convention, component-sum fallback, silent row deletion, or +implicit change to the 59/65-output profiles. + +Missing/invalid roles, duplicate or orphan membership, conflicting source +values, nonnumeric amounts, negative/nonfinite observations, and known amounts +outside the reporting universe refuse. They cannot route to eight predictors: +the first two PUF predictors still require valid roles. A source qualifier's own +refusals remain refusals; the fallback does not override them. Graph integration +must bind both disjoint recipient routes, account for each recipient exactly +once, authenticate inherited clone membership, and execute the corresponding +complete fit/apply/attachment chain. This numerical helper does none of that +graph admission or fitting. + +## Measurement limits + +[IRS Publication 915 (2015), pages 2–6](https://www.irs.gov/pub/irs-prior/p915--2015.pdf#page=2) +assigns benefits to the legal beneficiary, including a child whose parent +receives the check; joint filers combine spouses' benefits. The return amount +uses benefits net of repayments and includes the Social Security equivalent +part of tier-1 Railroad Retirement, excluding other pension portions. Form +1040EZ omits benefits, and some recipients need not file. These rules motivate +the selected return members but do not establish survey report ownership or +PUF editing conventions. [SSA's representative-payee guidance](https://www.ssa.gov/payee/faqrep.htm) +likewise distinguishes the person managing a payment from its beneficiary. + +The [2025 ASEC dictionary, PDF pages 48–49](https://www2.census.gov/programs-surveys/cps/datasets/2025/march/asec2025_ddl_pub_full.pdf#page=48) +permits combined family payments and identifies payments on behalf of children. +Its source total covers calendar 2024 for a 2025 interview. The +[ACS 2024 questionnaire, question 43](https://www2.census.gov/programs-surveys/acs/methodology/questionnaires/2024/quest24.pdf#page=18) +uses the prior twelve months, permits jointly received income entirely on one +person's record, and combines Social Security with Railroad Retirement. +[ACS SSP metadata, PDF page 45](https://www2.census.gov/programs-surveys/acs/tech_docs/pums/data_dict/PUMS_Data_Dictionary_2024.pdf#page=45) +defines the age-15 universe, rounded/topcoded amounts and ADJINC price +adjustment. The helper preserves those distinct observation windows; it does +not calendarize ACS or estimate repayments/Railroad adjustments. + +The ninth donor predictor remains the existing transported 2015 `E02400` +carrier under its declared 2024 money-growth convention. Carrier zeros do not +prove absent benefits. Exact 2015 publisher editing/omission rules remain +unverified; the [IRS PUF page](https://www.irs.gov/statistics/soi-tax-stats-individual-public-use-microdata-files) +reports the 2012–2015 files unavailable. Matching is therefore conditional on +this documented proxy assumption, not a newly authenticated observed 2024 +return total. Using eight predictors does not remove donor coverage concerns +for nonfilers or resolve assumptions in the other donor outcomes. + +## Diagnostics required before relying on a fit + +Family payments on selected reporters may overstate their own benefits; +payments reported elsewhere may understate them. An all-member sum includes +dependent reports and cannot solve this attribution problem. Topcoding, +allocation, return editing and transported donor money can distort zeros and +tails. Growth of fixed 2015 amounts does not predict 2024 cohort composition. + +Report route counts, known zero/positive distributions and tails by survey, +actual filing status, age, child-report reasons, allocation provenance, and +multiple-return household status. Preserve original-household holdouts across +clones. Compare design-weighted held-out performance and recipient overlap for +eight versus nine predictors; donor-only validation cannot establish cross-source +measurement validity. An adult in-universe all-member report sum may be a +labelled sensitivity diagnostic, with under-15 unknownness retained. It is not +the default feature. Benefit-component modeling and release checks remain +separate work. + +This change provides source-bound numerical values, explicit route metadata, +and an isolated graph profile declaration. Actual source guards, route-aware +graph integration, donor projection for the fallback, fits and diagnostics are +subsequent verification steps, not results claimed by this decision. diff --git a/docs/survey-social-security-source.md b/docs/survey-social-security-source.md new file mode 100644 index 000000000..ea9cfa1ce --- /dev/null +++ b/docs/survey-social-security-source.md @@ -0,0 +1,124 @@ +# Survey Social Security source semantics + +The source-report Social Security total precedes PUF enrichment. The current +qualifier verifies the original observations and preserves their limits; it +does not fit a completion model, assign individual beneficiaries, or certify a +population. The explicit `PUF55_SURVEY_SS` profile has 55 outputs and uses an +upstream-qualified tax-unit total as its ninth conditioning input. Historical +full65 and PUF59 behavior is unchanged. The new profile's native source and +whole-population attachment integration remain pending. + +## Source measurements + +The [2025 ASEC dictionary](https://www2.census.gov/programs-surveys/cps/datasets/2025/march/asec2025_ddl_pub_full.pdf) +describes `SS_YN` and `SS_VAL` on PDF page 49. The question covers people aged +15 or older and permits combined family payments on a person's report. A +positive report is therefore not necessarily one individual beneficiary's +payment. The qualifier preserves the source report's grain. + +The retained ASEC original member supplies `PERIDNUM`, household/line/age +coordinates, `SS_VAL`, `SS_YN`, `RESNSS1`, `RESNSS2`, `RESNSSA`, `I_SSVAL` +and `I_SSYN`. Source keys, the complete current cohort, raw member size/hash, +live source issuance and exact current money values must all agree. Source +year and income year are 2024; interview year is 2025. Carried reason fields +are not a substitute for these original literals. + +The [2024 ACS dictionary](https://www2.census.gov/programs-surveys/acs/tech_docs/pums/data_dict/PUMS_Data_Dictionary_2024.pdf) +defines combined `SSP`, its age-15 question universe, and allocation flag +`FSSP`. The qualifier binds the retained 2024 ACS issuance and verifies +`SSP * (ADJINC / 1000000)` against the prepared amount. This remains a rolling +income window expressed in the survey's price basis. `FSSP` is not in the +current native projection, so publisher allocation provenance stays explicitly +unresolved; no observed-value claim is inferred from a missing flag. + +## Mapping judgments and unknownness + +The reason mapping is an explicit judgment at source-report grain. Retirement +and disability map to their corresponding component. Widowed and surviving +child reasons map to survivors; spouse and dependent child reasons map to +dependents. A single resolved component receives the report total. Repeated +reasons in the same component do not duplicate the amount. + +Distinct component reasons, missing reasons, code 7's combined child categories +and code 8's residual category leave the component amounts unknown. The +published reasons restrict possible components where they can; there is no +reason priority, age fallback or equal-share default. Separately observed +component amounts and individual beneficiary assignments are not claimed. + +An ASEC under-15 `SS_VAL=0` is a not-in-universe sentinel. Both ASEC and ACS +under-15 beneficiary totals remain unknown in the projection, with the raw +ASEC zero retained in the separate literal table. An in-universe zero remains +a known report total of zero. Contradictory universe/recipiency combinations +and invalid nonempty codes refuse; blank literals remain distinguishable +from invalid codes. + +An eventual component model may normalize nonnegative scores within allowed +components, preserving the source total and already resolved source basis. +It must supply positive allowed mass. The numerical helper does not authenticate +such a model; the graph must bind real fit/apply artifacts. A reporting-unit +convention, allocation to individual beneficiaries where necessary, explicit +income periods, design-weighted training, native-to-clone inheritance and +held-out checks remain separate required work. + +## Conditioning measurement judgment, 2026-09-09 + +The recipient measurement for predictor nine remains undecided. Passing the +numerical `PUF55_SURVEY_SS` mechanism does not establish donor/recipient +comparability or authenticate a beneficiary total. + +[IRS Publication 915 (2015), pages 3 and 5–6](https://www.irs.gov/pub/irs-prior/p915--2015.pdf#page=3) +combines spouses' benefits on a joint return and assigns benefits to the person +legally entitled to receive them. A child's payment remains the child's even +when the check names the parent. Form 1040 line 20a reports net benefits after +repayments, including equivalent tier-1 railroad benefits; Form 1040EZ does not +report benefits. Thus a sum over all modeled tax-unit members is a different +measurement from the filer/joint-spouse return amount. + +Restricting survey records to the filer and joint-return spouse would still +produce a **report-sum proxy**, not an observed beneficiary total. The +[ASEC dictionary, PDF pages 48–49](https://www2.census.gov/programs-surveys/cps/datasets/2025/march/asec2025_ddl_pub_full.pdf#page=48) +permits combined family payments and includes reasons for payments on behalf +of children. The [ACS 2024 questionnaire, question 43, page 18](https://www2.census.gov/programs-surveys/acs/methodology/questionnaires/2024/quest24.pdf#page=18) +also permits jointly received income to appear entirely on one person's +record, with no receipt recorded for the other. Its income category includes +Social Security or Railroad Retirement. A parent may therefore report child +payments, or another household member may report an included person's income. +Multiple tax units within one household make reporter membership insufficient +to establish benefit ownership; household totals must not be duplicated across +returns or redistributed without a separate supported model. + +If adopted, a measurement such as +`social_security_filer_joint_spouse_report_sum_proxy` would need an explicit +upstream contract before supplying `puf_conditioning_social_security_total`. +That contract would bind actual return roles, source reporting records, +knownness and income periods; the coarsened PUF filing-class predictor alone +cannot establish joint-return membership. ASEC calendar-year income, ACS's +rolling window, transported donor money, repayment treatment and railroad +coverage remain comparability questions. Component completion is separate. + +Excluding a dependent from a parent's return feature does not assign that +dependent zero benefits. Both surveys' under-15 amounts remain unknown. If an +included filer has no in-universe report, an empty or missing report set cannot +silently become a known zero. Family-payment ambiguity also survives an +age-universe restriction. + +The exact 2015 `E02400` definition and publisher editing/omission conventions +remain unauthenticated by this lookup. The [live IRS PUF page](https://www.irs.gov/statistics/soi-tax-stats-individual-public-use-microdata-files), +checked on 2026-09-09, says the 2012–2015 files are unavailable. The verified +Form 1040 rules do not independently establish how every PUF record encodes +them. This record adopts no proxy, allocates no beneficiaries and grants no +dataset or release acceptance. + +## Verification scope + +The 52-case control run executes 50 numerical/literal tests and two complete +maintained source issuances from invented original CSV/checkpoint fixtures. +It verifies source-identity joins despite reordered rows, repeat qualification, +defensive returned data, exact source totals, unknown components and unchanged +input frames. Source/control files, installed model source and the 12 admitted +code resources were unchanged before, after and in the external postcheck. + +The run completed in 67.6 seconds with 428.9 MB peak RSS. Receipt SHA256: +`2b9d8e45cec11a6ebda2ccc01c5f12374e794ec0c006ba6525f082fd6454049b`. +These controls are not native-source execution, a fitted component model, +native PUF55 integration, policy-engine validation or release acceptance. diff --git a/docs/us-current-survey-amount-successor.md b/docs/us-current-survey-amount-successor.md new file mode 100644 index 000000000..250bab1ad --- /dev/null +++ b/docs/us-current-survey-amount-successor.md @@ -0,0 +1,196 @@ +# Current survey amount successor + +Implementation plan and source inventory, 12 September 2026. This is a local +successor to the retained survey PUF55 run, not an amendment to the frozen d35 +native pilot. The branch starts at reviewed integration 99543c3. + +## Immediate scope + +| Native ASEC field | Canonical person input | Reporting evidence | +| --- | --- | --- | +| UC_VAL | unemployment_compensation | UC_YN receipt code; amount question universe UC_YN = 1 | +| PHIP_VAL | health_insurance_premiums_without_medicare_part_b | All persons | +| PMED_VAL | other_medical_expenses | All persons | +| POTC_VAL | over_the_counter_health_expenses | All persons | + +These four names are absent from both PUF tax-detail ownership rosters. They +already have qualified current-money domains and direct amount mappings in +`cps_carried_current.py`. The complete legacy carry helper is not reusable: +it also writes existing PUF leaves and a legacy Social Security decomposition. +The modern `survey_social_security` source owner remains the authority for +Social Security; no age-62 zero fallback or prior wages is introduced here. + +The current ASEC source is the authenticated original 2025 person member, +income year and price basis 2024. ACS observations use the already qualified +2024 source predictors, whose earnings refer to the rolling prior 12 months. +That temporal harmonization is a documented model choice, not equivalent +observation windows. No new donor dataset is needed. + +## Amount and reporting status are different + +The [2025 CPS ASEC dictionary](https://www2.census.gov/programs-surveys/cps/datasets/2025/march/asec2025_ddl_pub_full.pdf), +PDF page 50, defines UC_YN codes 0 (NIU), 1 (yes), and 2 (no), with the receipt +question asked of persons aged 15 or older. UC_VAL is asked of recipients; +its zero code conflates none with NIU. The maintained domain metadata pins +this dictionary to SHA256 +`5cb80973326ef8b625fbaae70d80b0c641ce5d2b3911abd2fb4427abd5908a6f`. +The three health amounts have all-persons universes (PDF page 83). + +A small bounded UC_YN projection reads the same exact authenticated original +member. Native person and household/line coordinates must join the original +current-money owner exactly; carried columns and detached JSON do not confer +source authority. The projection retains the original amount, amount validity, +source classification, zero_origin, literal receipt code and reporting status. +Today's upstream preparation requires complete qualified current money before +issuing its handle. Literal missing-amount controls test the join's unknown +semantics; they do not claim that a missing amount passes this unchanged parent +prerequisite. Reporting unknownness can still occur with numerically valid amounts. + +UC attachment and donor policy: + +- A valid positive amount with an in-universe yes code is known receipt. +- A zero amount with an in-universe no code is known nonreceipt. +- Yes with zero is an ambiguous amount, not known nonreceipt. It remains in + the source projection but is excluded from fit and canonical attachment. +- NIU, blank/unrecognized codes, missing amounts, and contradictory amount/ + receipt pairs remain unresolved and are excluded from fit and attachment. +- Out-of-universe age does not infer a zero. ACS persons outside the receipt + question universe also retain unknown UC rather than receiving an adult draw. + +The health fields retain qualified valid all-persons amounts, including genuine +zero-dollar codes. No noanswer cell is converted to zero. Allocation and source +status remain evidence; known source values are not claimed to be unallocated. +PHIP_VAL is deliberately retained as the declared premium source, rather than +silently substituting PHIP_VAL2, whose treatment of inconsistent reported zero +premiums differs. The [Census health insurance user note](https://www.census.gov/programs-surveys/cps/technical-documentation/user-notes/health-insurance-user-notes/User-Notes-Selected-CPS-ASEC.html) +describes that distinction. Person-level conditional fitting preserves modeled +cross-component dependence; it does not certify household or insurance-unit +correlation, which still needs empirical validation. + +## One family, explicit graph operations + +The family has one parameterized source projection and attachment mechanism. +Each selected field declares its source transformation, donor eligibility, +recipient matrix, QRF fit and draw as graph nodes/artifacts. UC has its own +reporting mask. The health group uses an ordered PHIP → PMED → POTC conditional +chain, retaining observed health-component dependence through prior-target +conditioning instead of fitting the three amounts independently. This avoids +filling unknown UC to make a joint matrix finite. Fits use the existing regime-gated QRF implementation and +current source predictor qualification; donor weights remain original design +weights before allocation or cloning. No PUF value becomes an ASEC donor. + +Draws are keyed to the original ACS source person, then deliberately shared by +its original and PUF support clones. This preserves source observations and +paired support interpretation; clone labels are checked during attachment. +Source amount values and reporting status are copied to both ASEC clones. +Every existing parent column, owner, axis, membership, atomic geography, +weight, design anchor and mass ledger must remain identical. + +The parent is the exact `SurveyPuf55Run` checked by +`check_survey_puf55_run`. The original handle is retained and checked before +consumption and after source/store/export I/O. A checked JSON payload or copied +dataclass cannot issue authority. The extension copies kernel/source-codec +registries privately, preserves all prefix declarations/keys/artifacts, and +verifies actual observed populations on cold and required replay. Failed checks +revoke the original handle. The checked-output branch supplies this additive +API; the frozen native packet is left untouched. + +## Verification + +First run cheap invented controls: all UC code/amount/universe combinations, +health zero and unknown behavior, literal bounds and source-coordinate joins, +clone pairing, missing/duplicate source IDs, invalid source/clone channels, +ownership collisions and raw/status/zero-origin mutation. + +Then an actual invented authenticated-source fixture runs the family through +real graph cold and required replay, QRF train/apply, coverage, complete Frame +store/export/readback, parent retention and unchanged prefix seals. Rejection +controls mutate source projection, parent handle/population, declared graph, +artifact bytes, fit/draw lineage and output attachment independently. Acceptance +requires closed receipts and independent review. No native run, engine outcome, +held-out quality, calibration or release certification is claimed by these tests. + +## Remaining input inventory + +The accepted invented PUF baseline has 161 required names, of which 100 are +absent. Its 61 present names are not necessarily complete: some PUF leaves are +populated only on PUF clones. Counts are inventory for that fixture, not a native +coverage measurement. The next stage reduces absent names while reporting +unresolved cells separately. + +| Input family | Existing source producer or route | Integration requirement | +| --- | --- | --- | +| UC and three health amounts | authenticated current-money owner; direct `cps_carried_current` maps | This stage | +| Social Security components | `current_social_security_source`, `survey_social_security` | Preserve source reporting basis and unknown components; never legacy age split | +| Health coverage | Authenticated `current_survey_health_source` and explicit `graph_current_survey_health` fragment | This stage; seven narrow ACS concepts remain unknown where the survey cannot distinguish them | +| Disability, student status, children, veteran status | `eligibility_inputs` | Qualify literals and reporting universes before reuse | +| Parent relationships | `eligibility_inputs._own_children_in_household` resolves PEPAR1/PEPAR2 then drops links | Issue 884: retain parent_1_id/parent_2_id with household/line and clone joins; unresolved and absent differ; consumer issue 9404 is separate | +| Marital roles | `relationship_inputs` | Current source observations and clone preservation | +| Hours, work history, occupation, wage/tips | `hours_worked`, `org_wages`, `weeks_unemployed`, `sipp_tips` | Source/QRF qualification and explicit ownership; prior wages excluded | +| Child support, disability income, workers compensation | respective maintained runtime modules | Current source amount/universe qualification, then existing fit contracts | +| Housing, energy, care, education | `housing`, `energy_subsidy`, `adultcare`, education producers | Household/SPM source and clone-keyed joins; reconcile PUF tuition ownership | +| Financial wealth, vehicles, loans | SCF/SIPP producer modules | Actual donor authentication, existing fit dependencies and held-out quality | +| Retirement distributions/contributions | maintained distribution/contribution modules | Reconcile already PUF-owned taxable IRA/pension and self-employed pension leaves | +| Pregnancy, SSI disability, voluntary filing | maintained producer modules | Resolve source authority and distinct behavioral assumptions | +| Take-up | `take_up_contract` | Unsourced rates remain unresolved; no convenience constant fills | +| Mortgage details, marketplace benchmark ratio, remaining retirement details | Not yet mapped in this bounded inventory | Identify actual producer/owner or make an explicit source/model decision | + +The inventory used tracked files and `rg --no-ignore` within explicit source +directories because ordinary searches can skip the tracked build package. +Existing function names establish implementation leads; they do not prove that +those producers are integrated into the current graph or current native file. + + +## Fixed country composition + +`run_us_survey_enrichment` retains the original checked `SurveyPuf55Run` and +assembles the amount family and independently source-qualified health fragment. +Both attach to `survey_puf55.receiving`; the health CREATE and final attachment +explicitly consume `survey_amounts.attach/attachment`. Health adds nine coverage +inputs and their literal/status/knownness evidence after all four monetary leaves. +The host preserves the complete parent Frame, owners, design anchors and mass ledger, +compares every unchanged prefix version against the original parent manifest, and +requalifies both original-source projections before issuing its output. There is +one private issued-handle registry and one public checked country output, not a +family-level authority or executor. The health fragment's descriptive values do +not grant access to native sources. + +The combined invented fixture includes the exact source ACS HINS/flags and ASEC +NOW/flags before authenticating source owners. It runs the original financial and +PUF stages, both enrichment fragments, a required replay, input-coverage diagnosis +and complete Frame store/readback. No native build, model engine execution, +calibration, or release approval is implied by these checks. + + +## Invented acceptance, 12 September 2026 + +The final production source passed the 271-node combined graph: 245 validated +prefix hits followed by 26 newly executed enrichment nodes, then a required +replay with all 271 hits. Complete parent columns, owners, design anchors, +geography and the mass ledger were preserved. The full resulting Frame was +stored and read back with exact physical comparison. The fixture has 18 people; +required-name coverage improved from 61 to 74 of the 161-input profile, leaving +87 missing names. Presence does not imply complete values or statistical quality. + +All 18 people have the three health-cost values and an ESI observation. UC has +10 known and eight unresolved values. The narrow Medicaid-at-interview input has +eight known and ten unresolved values, preserving the ACS semantic gap. These +are counts from invented records, not estimates or native-data diagnostics. + +Acceptance combines five passing ordered tests from the third full attempt and +one corrected, focused final parent-revocation test. The third attempt retained +its failed result because that last test named a nonexistent generic income +column; it stopped before mutating anything. The final test uses the actual +`employment_income_before_lsr` input and verifies permanent parent and child +revocation after mutation, including refusal after restoring the original value. +It rebuilds a real parent and one child, without repeating the already-passed +required replay or readback. The focused attempt passed with all 857 Python +source pins stable, 624.872 seconds wall time and 885,669,888 bytes peak RSS. + +The separate 32-case cheap suite covers UC reporting classifications, literal +missingness, source/clone identity, ownership collisions and source-origin +serialization when `person_id` names both an index and a column. The serialization +binds the original int64 axis and its name separately from ordered column values. +Independent review covers the frozen UC source, health fragment, combined host +and this serialization correction. Preserved failed attempts remain part of the +local acceptance record; no native execution or release certificate is implied. diff --git a/docs/us-input-coverage-diagnostic.md b/docs/us-input-coverage-diagnostic.md new file mode 100644 index 000000000..1304d2fd0 --- /dev/null +++ b/docs/us-input-coverage-diagnostic.md @@ -0,0 +1,3 @@ +# US population input-coverage diagnostic + +[`diagnose_us_input_coverage`](../packages/microcosm-build/src/microcosm/build/us_runtime/population_input_coverage.py) takes a `Population`, its `CompiledGraph` and attached `RunManifest`, and returns a typed `PopulationInputCoverage` report. Its default `USInputProfile.NATIONAL_CD` describes 161 national/congressional-district input names; [`required_us_inputs()`](../packages/microcosm-build/src/microcosm/build/us_runtime/input_coverage_profile.py) keeps the historical 163-name default. The national/CD profile omits only engine `block_geoid` and `tract_geoid`, while separately reporting assigned `census_block_geoid`; neither profile adds `employment_income_last_year`. The diagnostic checks complete declared current ownership and replayed storage, then reports actual entity grains, source-channel/clone groups, null and invalid values, typed weights, and masked-writer versus carried rows. Missing grain and applicability remain unresolved. It verifies consistency of supplied graph objects; source ancestry and source requalification remain the host's responsibility. [Twenty-five frozen controls](../experiments/us-input-coverage25-20260910.json) passed, including actual cold/required replay and late-mutation refusals, with 515 physical source/owned checks and zero model/resource inputs. This is scoped evidence, not maintained-CI validation: four eager calibration imports differ from the frozen run and retain their newer maintained versions. The report establishes neither complete input coverage, statistical signal, applicability, native build acceptance nor release eligibility. diff --git a/docs/us-launch-review.md b/docs/us-launch-review.md new file mode 100644 index 000000000..8914d6bea --- /dev/null +++ b/docs/us-launch-review.md @@ -0,0 +1,454 @@ +# US graph build: review guide + +This branch consolidates the US graph build work for review. The intended file +supports national and 119th congressional district analysis from one population. +It is still a development build: complete enrichment, fiscal calibration and +release verification have not passed together. + +## Architecture and review order + +The intended construction order is ACS + CPS ASEC → complete survey multispine +including its initial support/PUF clones → household Census block assignment → +derived larger geographies → donor enrichment → rules evaluation and calibration +→ verified full and pruned exports. Each resulting household retains its assigned +location through the later stages. The PUF +operator acts on the combined survey frame. A separate ASEC–PUF base is not the +intended architecture. + +National and local analysis share one fully constructed, enriched Frame. The +analysis branch begins after harmonization, cloning, atomic geography assignment +and imputation. Calibration changes weights; L0 controls sparsity. An export can +select a geographic scope and remove zero-weight households, but must preserve +every retained record's input values, knownness, entity relationships and assigned +geography. There is no separate national or local enrichment operation. A source +or mapping correction produces a new common parent Frame for both views. + +Release verification must bind each full or pruned export to that parent's +identity and its calibration target/weight specification, compare retained +inputs by stable entity ID, and reject changed inputs, dangling entity links or +regenerated geography. Weight and scope changes are explicit exceptions; record +values are not. This is the required contract, not a claim that all of these +checks have already passed on the release candidate. The separate legacy +ACS-local hours omission in [#765](https://github.com/PolicyEngine/microcosm/issues/765) +illustrates the drift this common construction path removes. + +Pre-calibration source quality and post-calibration export integrity are distinct +checks. Joint calibration may change an origin's contribution or set its weights +to zero. A pruned export must not be required to make every survey origin +independently resemble a national population, or to rerun imputation to satisfy +such a requirement. + +The supplied-parent export comparison now passes 36 focused checks, including +actual checkpoint write/readback, complete household selection, unchanged input +values and missingness, signed and unsigned stable IDs, and full/pruned/local +weight and scope handling. A household with zero original weight can receive +positive calibrated weight without mutating its parent or other entity weights. +The comparison binds the full ordered weight vector and specification, including +rows outside the selected scope. It deliberately reports calibration ancestry +and release eligibility as unverified: the owning build must establish those +and seal its inputs across export I/O. See the +[scoped export acceptance](../experiments/us-common-frame-export36-20260910.json). + +The earlier ten-node survey prefix assigns a block before cloning, and its +thirteen-node age-calibration extension passes cold execution and required +replay on invented originals. A twenty-node extension now carries atomic +geography through current financial imputation and required replay. Native +Delaware and complete national block support pass source normalization; +national/CD acceptance of the enriched population remains pending. The earlier joint tract/district +operator is a separate path. See +[Geography assignment](geography-assignment.md) for the country contracts, +implementation and source boundaries. + +The 10 September sequencing correction now passes 70 inventoried controls on +invented inputs, across seven separately guarded runs, plus one separate default +compatibility case. The nine-node prefix completes its initial clones before +block assignment; the nineteen-node financial extension and twelve-node age +extension retain that geography through cold execution and required replay. +Assignment uses the qualified original-source key and stable clone discriminator, +so changing row order or numeric household coordinates does not redraw the same +clone. Observed state/PUMA constraints, input knownness, entity links, complete +population ownership and weights remain checked. Different clones may draw +different blocks; an occasional shared block is valid. + +The financial donor FILTER requires the exact typed geography-validation artifact +and retains original ASEC DESIGN weights. The shared gate emits this artifact +only when requested, preserving its default API for UK consumers. Two test-only +corrections repair the CREATE source declaration and isolate the obsolete-order +budget counterexample from unrelated serialized storage differences; production +matches the independently reviewed candidate. See the +[scoped postclone acceptance](../experiments/us-postclone-geography-70-controls-20260910.json). + +The earlier ten/twenty-node native runs retain their original order. Native +postclone execution, the new-order PUF host, held-out model quality, national/local +fiscal calibration and full release acceptance remain pending. The age control +uses invented targets and does not establish that complete calibration path. + +| Area | Main source entry points | What to review | +| --- | --- | --- | +| Graph execution and storage | `packages/microcosm-graph/src/microcosm/graph/{decl,executor,manifest,store,codecs,schema}.py` | Typed inputs and outputs, ownership, artifact ancestry, cache identity and nullable round trips | +| Survey source preparation | `packages/microcosm-build/src/microcosm/build/us_runtime/{acs_population_catalogue,asec_population_catalogue,survey_population_preparation}.py` | Source identity, entity links, source universes and current income transformations | +| Combined survey and clone | `us_runtime/{graph_composed_population,graph_survey_population,graph_combined_clone}.py` | ACS and ASEC composition before the clone; native versus detail channels | +| Enrichment | `us_runtime/{full_puf_enrichment,graph_full_puf_enrichment,graph_current_survey_puf_transfer}.py` | Target ordering, conditioning, observed-value preservation and complete replay | +| Conditional models | `packages/microcosm-fit/src/microcosm/fit/{qrf_target,graph_legacy_train,graph_legacy_apply_matrix}.py` | Reusable model artifacts, deterministic draws and target regimes | +| Geography | `atomic_geography.py`; `us_runtime/{atomic_block_support,atomic_block_api_sources,survey_atomic_geography,graph_atomic_survey_population}.py` | Atomic block assignment after complete initial cloning, observed-source constraints, versioned mappings, stable clone identities and the optional typed validation prerequisite | +| Survey mass and calibration | `us_runtime/{survey_origin_budget,graph_survey_budget,graph_survey_calibration}.py`; `packages/microcosm-calibrate/src/microcosm/calibrate/{group_bounds,solve}.py` | Original survey mass, grouped bounds, fixed support and existing ungrouped solver behavior | +| Compatibility | `packages/microcosm-build/src/microcosm/build/{frame_checkpoint,us_runtime/__init__}.py` | Current-main APIs, checkpoint metadata and existing country consumers | + +Paths beginning `us_runtime/` are relative to +`packages/microcosm-build/src/microcosm/build/`. +The branch contains a substantial consolidation; review each area and its tests +before deciding how to land it. Shared model work overlaps with +[#873](https://github.com/PolicyEngine/microcosm/pull/873). Anthony considers his +runtime proposal [#885](https://github.com/PolicyEngine/microcosm/pull/885) +superseded by this direction; it is reference material, not a prerequisite to +retain or merge separately. This does not imply that its GitHub PR is closed or +that this draft has received his approval. + +## Parallel review and dependencies + +Completion of this integration draft is not a prerequisite for all UK work. +Depend on identified shared changes where necessary, with independently reviewed +PRs and compatible interfaces. Keep the US native build, UK candidate repair, +source/target reconciliation and shared runtime work moving in parallel. + +| Reviewer | Review focus in this draft | Work that can continue independently | +| --- | --- | --- | +| Anthony | Shared graph/Frame and calibration compatibility; target hierarchy and metadata flow into diagnostics/dashboard, including #855 | Target/dashboard changes follow #900; UK staging #896 proceeds independently of full UK graph conversion and native US PUF integration | +| Maria | Register each existing UK pipeline stage as a graph node using this branch's shared machinery; verify UK behavior and coverage | #900 lands before graph conversion; UK imputation, support/fit diagnosis and candidate validation continue; graph-node registration stacks on #893, with stage reordering in a later separate PR | +| US integration owner | Resolve overlap, maintain exact compatibility evidence, complete the US population and release checks, and extract shared changes agreed in review | US-specific source preparation, imputation, quality diagnostics and progressive local builds | + +Coordination updated 10 September: Maria is deliberately stacking her UK +graph-node registration on #893. That specific dependent work should pin the +parent revision and coordinate shared-interface changes here. Her source/target +repairs and Anthony's staging work do not have to wait for the whole draft. +PR #900 merged at 14:54 UTC and is incorporated here from main at +`6f7571e1ab8c516154289bd8ebf75cb2646f03b5`. The combination preserves both parent +histories and does not rerun or recertify the frozen US controls. +Anthony's hierarchy/diagnostics [#855](https://github.com/PolicyEngine/microcosm/pull/855) +and its dashboard consumer follow the target reconciliation in +[#900](https://github.com/PolicyEngine/microcosm/pull/900). Registering UK graph +nodes and changing their order are separate review steps. The US integration +work must not build a competing full UK graph; its shared atomic-area adapter +is an input to Maria's owned integration. + +Record shared dependencies as named changes with pinned revisions. Extract a +smaller shared prerequisite where useful, while preserving compatibility with +Maria's intentional stack. Proposed sequencing does not certify or merge any PR. + +The shared post-clone geography interface now has a tested additive validation +artifact contract. The default gate API remains unchanged; UK registration can +adopt the optional typed prerequisite independently of the US stage ordering. +Existing target/source repairs can proceed on their reviewed base. Passing this +draft's invented controls does not certify Maria's UK candidate, and the final +US release need not gate her independent source and calibration work. No +whole-PR merge is implied by this review map. + +## Verification and review scope + +The first native twenty-node survey, block-geography and financial cold pilot +now passes at 1/1000: 1,584 source households and 3,464 people become 3,168 +household records and 6,928 person records after cloning. Ten financial nodes ran +cold over the stored ten-node prefix, which retains the block assigned before +cloning. The composition retains location inheritance and attaches seven financial +fields while preserving the other population fields. It took 110.12 minutes and +11.45 GB peak memory. The final receipt and independent source/resource +postcheck pass. This is the original frozen 491-source implementation; newer +cache and budget changes are separate revisions. Required native cached replay +now also passes: all twenty final nodes and ten prefix nodes were cache hits, +the four non-manifest exports matched the cold bytes, and the portable manifest +retained the same content and stored-artifact identities. Replay took 74.17 +minutes and 11.31 GB peak memory. Independent postchecks verified the recorded +result and exact code/resource identities without reopening native inputs or +exported payloads. PUF augmentation, model quality and calibration remain pending. +See the [scoped replay result](../experiments/us-native-atomic-financial-required-replay-20260910.json) and +[scoped native cold result](../experiments/us-native-atomic-financial-cold-20260910.json). + +The previously published head `a85c9cbb79081a980e7d3afe8916c80153de5bf8` +passed 36 integration controls and 57 enrichment/replay controls on invented +inputs. The 36 cover eight solver, eleven snapshot/storage, nine geography and +eight facade cases. The 57 include cold execution, required replay, reconstruction +through a fresh Frame store, inherited tax-unit values and malformed ancestry +refusals. Source/control, model-declaration and resource pins matched before, +after and in external postchecks; the tax-benefit engine was not executed. + +Those receipts are identified by SHA-256 +`ad8ab6d8761db3d21343bc1fc60023c0dfa305e1367622630a5de6c68892d62a` +and `85a192f8c5c851648875d66b90bea6b8a4ee11e86e67d1875c3ca34a076ee014`. +These checks do not certify subsequent source additions or the entire repository. + +CI exposed test-helper collection failures and a pool/registry circular import. +This revision adds the test helper directory to pytest's path and moves the +registry import after the pool functions it validates. Two fresh-process import +regressions are assigned to the US-engine CI lane; their runtime result remains +pending. No import-order result is inferred from an AST check. + +The corrected checkpoint test selection passed all 58 invented controls against +the current integration checkpoint source, including nullable integer widths, +missing-value masks, deterministic round trips and malformed v4 refusals. It +took 5.36 seconds wall time and 348.5 MB peak RSS. All 45 source/control files, +three provider pins and two code resources matched in the external postcheck. +Receipt: `f8009ec6d07d076eff6a90ec97061d538756293547a21758ab2bc38fa5f7aab8`. + +Fable's earlier replay review closed seven findings. A separate review of the +published consolidation identified the checkpoint test gap and additional +grouped-bound, metadata-store and facade coverage work. The portable grouped-bound, +fixed-support and metadata-store selection now passes all 117 tests with no skips. +The fixes reject nonfinite stored JSON as `StoreCorrupt` and normalize the deprecated +`apg` alias before grouped-mode validation. Source/control, resource and provider +bytes match before, after and in an external postcheck. Receipt: +`3674af9d610417897539da6eae766726d0f582ae6dd3bc2c3e550fefa45bab68`. +The historical platform fixture and complete facade import coverage remain separate. +Environment-specific prepatch byte fixtures do not establish portable CI byte +parity. Existing v1 graph stores remain preserved; v2 execution uses a new store +rather than silently promoting old frames. + +Private runtime records and generated population artifacts stay outside this +source PR. Receipt identifiers are audit references, not reproducibility inputs. + +The composed 156-case test run against source commit `8157300` at documentation +head `78d84f9` passed 152 cases and failed four current-survey cases with +`PRODUCER_CHANGED`. Its execution guard also recorded unexpected test-bytecode +and installed-model cache-directory probes. Source/control, model-source and +resource bytes remained unchanged. This is a failed integration run, not a +replacement for the earlier passing component evidence. Its receipt is +`67ff10fef45c58d2d6ea15c72b4311a4daa901dd2aae263fbb863d38c13bc7c8`. +The successor now passes all 314 selected invented integration and identity +checks with no skips or unexpected refusals. This includes the original 156 +cases, five metadata-copy/mutation regressions and 153 encoder controls; seven +synthetic-issuance controls remain explicitly outside this selection. Shallow +copy, deepcopy and pickle previously added a class cache entry captured by +Python 3.14 annotation closures. An explicit immutable-metadata representation +avoids that mutation without weakening producer checks. Two stale ACS pins were +updated only after verifying AST equality with their accepted source revisions. +All 476 source/control, 5,983 model-source and 12 resource entries matched before, +after and in the external postcheck. The run took 253.5 seconds and 738.7 MB peak +RSS. Receipt: +`d940913f918af4455dab2b9e680d55f9efc7bb922f72ff3d8fe26b0530b070f7`. + +A separate 16-case current seven-node age-development run initially passed 13 +cases and failed three on exact diagnostic reconstruction. The checker omitted +two solver-option fields added by the consolidated solver. Reconstructing the +grouped solver's closing-state selection and empty selection receipt fixes that +mismatch while retaining exact whole-document comparison. All 16 tests now pass, +including actual seven-node execution and late target/successor mutation checks, +in 170.3 seconds at 541.9 MB peak RSS. All 469 source/control, 5,983 model-source +and 12 resource entries matched before/after/current checks. Receipt: +`5849dd42e914e4d8304a5d290ff451da45910e7ac9a61e76fff36468949ffe3a`. +Broader CI's missing helpers and stale schema, seed and runtime classification +expectations remain separate from these selected passing suites. + +## Component evidence and remaining work + +Separate component work has verified the national Census joint-geography source +artifact: 2,462 PUMAs, 87,841 joint cells, all 50 states plus DC and 436 districts +including DC. PUF source ingestion retained 207,692 ordinary records and excluded +four disclosure records. A corrected SCF wage-code interpretation passed source +preparation and model-mechanism checks. Those results belong to their respective +component revisions; they do not establish a calibrated result on this branch. + +The corrected canonical-donor codec v2 was rebuilt from the same accepted typed +PUF source. All 207,692 returns and 59 outputs passed selected-cohort replay and +byte/value serialization checks in 24.37 seconds, using 1.95 GB peak RSS. The 481 +source/control files and four resources matched their before/after/current pins. +Receipt: `6b334ca8df6c6ba721b60f84d28c43151a1e1b54e097616a3c0fa208daeb8cba`. +This run constructs the donor; it does not fit or place it onto survey recipients. + +The identity encoder passed two 40-case preflights and four 40-case +timing runs in both execution orders. The 36 tiny invented cases retain exact +identity bytes and mutation checks. Nine of the 72 case/trial comparisons were +slower, including the small Python-string Frame in both orders; larger numeric +fixtures improved substantially. These timings do not establish genuine-build +speed. It is now integrated and passes the current 314-case selection, with the +newer current-wage projection preserved byte for byte. That projection was absent +from the benchmark's accepted baseline. + +The following work remains open: + +The [US and UK release path](us-uk-release-path.md) puts these implementation +items in the complete population-quality, calibration and consumer-release +sequence. Passing component controls is one step in that sequence. + +1. Extend the accepted native twenty-node atomic and financial cold/replay + pilot to the complete survey scale. Its + invented cold/required replay, complete retained + fields and geography, default three-feature versus explicit five-feature + conditioning, and both final mutation refusals pass on invented inputs. + Two test-code mistakes in the initial four-case run were corrected and + rerun separately; the passing mutation cases were preserved. See the + [composition acceptance](../experiments/us-atomic-financial-composition-acceptance-20260909.json). + Native fit quality and admission of the financial result into subsequent + calibration remain unassessed. +2. Integrate the adopted Social Security conditioning measurement and qualify + recipient return roles before native PUF55 fitting. Thirty-one invented + controls pass the authenticated filer/joint-spouse report-sum proxy, source + preservation, missingness and refusal checks. An incomplete selected report + uses a separately declared eight-predictor route; a known total uses nine. + The helper uses the preparation's retained modeled roles, whose relationship + to current-money tax-unit construction still needs explicit qualification. + See the [measurement decision](puf55-survey-ss-measurement-decision.md) and + [scoped control result](../experiments/us-puf55-survey-ss-measurement-31-controls-20260909.json). + Route-aware graph fitting and attachment remain pending. The genuine + 207,692-return canonical donor has already passed its 55-output projection; + that does not establish recipient matching. Complete survey + beneficiary/component modeling, and supply SCF loan inputs. Separate component runs passed + 128 profile/compatibility controls and 26 current-survey predictor controls; + see [the implementation status](current-survey-puf59-progress.md) and + [donor construction and growth](puf2015-canonical59-and-growth.md). + The genuine 207,692-return donor construction does not establish survey fit + quality, complete enrichment or a releasable population. +3. Verify every applicable model input on each source/clone channel. The proposed + national/CD profile retains 161 required model inputs, with state and county + but without requiring the engine to consume block and tract. Independently, + the build must retain one assigned Census block and derive its larger + geographies. Prior wages remain excluded. A source operator's existence is + not complete cell coverage. +4. Retain the earlier seven-node age-development result as historical evidence; + the twenty-node cold/replay pilot in item 1 is the newer native milestone. + The seven-node run passed at 1/1000: 1,584 source households, + 3,168 records after the support clone, final export/readback and owner/target + verification, 71.1 minutes and 8.26 GB peak RSS. Its source is frozen at + `a9895d8a5`, before the new Social Security, PUF55 and atomic-geography work. + The prior two-hour failure and stale-derived-attachment failure remain failed + historical runs. [The scoped acceptance record](../experiments/us-survey-age-development-20260909.json) + does not certify enrichment, full national/CD calibration or a release. +5. Run real model evaluation, national/CD calibration, holdout checks and complete + export/replay verification, followed by a dashboard bound to that exact file. + Check every pruned analysis file separately. + +These are active workstreams. No release, merge or deployment is implied by this +draft, and the source changes do not relax the outstanding acceptance checks. + +The two-route numerical finalization layer now passes 46 invented controls, +including 110 actual target fits, 220 applications and comparisons against the +maintained whole-cohort finalizer for both routes together and either route +alone. The tests also refuse changed donor row order, weights and late raw or +donor mutations. The v2 numerical receipt seals the complete finalized candidate +before return and refuses a change during receipt encoding. This verifies the +numerical layer; typed model ancestry and complete survey attachment remain +separate graph checks. +The first canonical donor CREATE attempt failed during kernel construction +because its live-code checker followed a circular function closure. The +[failed observation](../experiments/us-puf55-canonical-create-failed-20260910.json) +is preserved. The cycle correction passed its ten new marker controls, but +the [39-case follow-up](../experiments/us-puf55-canonical-cycle39-failed-20260910.json) +still refused before CREATE because the code-state snapshot changed during +initialization. The changing member was the serialized Python code object; +the corrected marker retains the actual code object and its immutable fields. +After correcting three obsolete test accesses to the public PopulationView API, +all 47 canonical CREATE/converter controls pass, including cold execution, +required replay and teardown. The corrected numerical and canonical sources are +adopted locally. These checks use invented donors; the native full PUF host +remains pending. See the +[46-control numerical evidence](../experiments/us-puf55-numerical-output-seal-46-controls-20260910.json) +and [47-control canonical evidence](../experiments/us-puf55-canonical-create-47-controls-20260910.json). + +All 149 selected calibration controls now pass on the combined US/UK solver, +including grouped bounds, fixed zero support, informed gates, budget search, +ordinary best-iterate behavior and strict diagnostic payloads. The run took +4.57 wall seconds at 394 MB peak memory; independent checks confirm the exact +34 source files, copied fixtures and all 149 unique test results. The preceding +run stopped before tests because two still-denied import probes exceeded their +reporting ceilings; the fresh invocation changed only those reporting ceilings +and run identities. No data access was added. This is invented-data solver +verification, separate from native national/CD calibration; see the +[calibration evidence](../experiments/us-calibration-consolidation-149-20260910.json). + +Recipient qualification now covers all 28 distinct original public controls: +27 passed in the original run and the five-case corrected detached-output run +passed, with four overlapping controls. The original run remains failed because +one NaN mutation test did not change its input bits; its correction flips a bit +in the actual float64 storage. This is a compound result across two source +snapshots, not a single passing 28-case run. Exact original-source issuers and +the 22-node cold/required graph are included. See the +[recipient evidence](../experiments/us-puf55-public-recipient-controls-20260910.json). + +The population-only Census API adapter passes thirty-five invented controls. +The native Delaware source control preserves all 15,317 populated blocks and +reconciles them to the independent Census state total of 989,948. It counts +4,881 zero-population blocks, checks CD119/PUMA joins and exactly reads back its +79,769-byte support artifact. The run took 10.94 seconds and 1.735 GB peak RSS. +The first attempt correctly refused an incomplete ZIP-member declaration; +a separate metadata probe established the six-member archive roster, and the +corrected check still decompresses only `NationalCD119.txt`. The +[accepted source control](../experiments/us-atomic-native-de-corrected-1-control-20260909.json) +preserves the original failed evidence. This is not national population or +release acceptance. Delaware-only support must not enter the national survey +graph. + +The subsequent population-only national acquisition and normalization now pass +for all fifty states and DC. The 104 exact sources produce 5,769,942 populated +blocks, independently reconciling every state to a total of 331,449,281 people. +Every block retains its identity and population, with complete CD119/PUMA joins. +The full support artifact is 28,862,508 bytes and passes exact serialization and +readback. Normalization took 82.97 seconds and 6.814 GB peak RSS. Its SHA-256 is +`5edc0e77471ba31d550a1eed416d5b46ada0a35425718eb87cfabe4d66fe4960`; +see the [national source control](../experiments/us-atomic-native-national-1-control-20260909.json). +Postchecks reauthenticated code and compared the 104 native pins from bounded +receipts without reopening source bodies. National support normalization and +household assignment in the small native financial pilot now pass. Evaluating +geographic fit and calibrating the enriched survey remain separate acceptance +steps. + +The financial successor's unchanged positive control now passes with both +original fixture teardowns and final code/resource checks: 647.19 CPU seconds, +651.21 wall seconds and 570.9 MB peak RSS. This separately reviewed invocation +used a 1,200 CPU-second budget; the earlier 600-second attempt remains recorded +as incomplete. No memo or profiler was used. The [positive control evidence](../experiments/us-financial-successor-positive-20260910.json) +checks financial replay and a subsequent weight-only change. The [seven remaining refusal and callback controls](../experiments/us-financial-successor-remaining-controls-20260910.json) +now also pass with their original shared fixture and teardowns: 798.61 CPU seconds, +802.89 wall seconds and 563.6 MB peak RSS. All eight original admission controls +are accepted on the same source snapshot. A subsequent source review found an +additional detached-view callback gap in the budget and weight-only views. +The correction is adopted after four targeted controls passed in 543 wall +seconds, including nine callback branches and independent source/resource checks. +The original eight and corrected four are separate source-version checks; see +the [correction evidence](../experiments/us-budget-detached-view-correction-20260910.json). +The native pilots exclude this overlay, and native calibration remains separate. + +A separate, bounded profile of the unchanged invented cold/required financial +fixture passes its test, original teardown and all final code/resource checks: +303.90 CPU seconds, 308.26 wall seconds and 594.3 MB peak RSS. This measures the +fixture rather than the additional financial and weight-only admission work. +See the [profile evidence](../experiments/us-financial-fixture-profile-20260910.json). +Loaded ACS code verification accounts for 180.20 cumulative seconds inside the +298.15-second profiled fixture. Its nested function checker runs 2,009,108 times +and uses 91.05 self seconds; complete Frame identity uses 7.31 cumulative seconds. +Cumulative timings overlap and profiling adds overhead. The invocation-local +code comparison memo now passes all 18 focused invented controls, including +changed source, aliases, globals, closures, mutable constants and the historical +catalogue caller. Source and resource checks pass before, after and in the +external postcheck. See the [control evidence](../experiments/us-acs-loaded-code-controls-20260910.json). +The paired full-fixture run also passed correctness and final checks, but took +319.79 wall seconds versus 308.26 for the baseline. The memo added eligibility +work and demonstrated no speed improvement, so its implementation was not +adopted. The exact experimental patch and [paired comparison](../experiments/us-acs-code-memo-comparison-20260910.json) +remain recorded. A separate [one-call diagnostic](../experiments/us-acs-producer-profile-20260910.json) +now passes without constructing a fixture: the original producer takes 0.656 +profiled wall seconds. Its 157 compiler calls take 0.238 seconds, including +0.122 seconds inside 103 AST parses; those overlapping times must not be added. +A bounded bytecode compilation cache now passes 25 invented controls and final +code/resource checks, with fresh source reads and every loaded-function check +retained. No native-data speedup is established. The compilation cache has separately passed +the complete invented fixture and is adopted; the native financial cold and +required-replay pilots passed with their original frozen code. The earlier +startup refusal remains preserved and confers no data acceptance. + +Current CI separately reports stale source-attested spec/seed fingerprints. +The worker resource identity test now follows the actual lazy import closure +and includes an invented opened-JSON regression; its new CI result is pending. +Neither the component passes nor this profile establishes a green consolidation. + +## Related reviews + +- [Microcosm → Orrery exporter, #888](https://github.com/PolicyEngine/microcosm/pull/888): + already published separately. It preserves the supplied schema metadata and + exact large integers; graph visibility does not infer domain verdicts. +- [Orrery review index, #6](https://github.com/TheAxiomFoundation/orrery/pull/6) + and [search improvements, #4](https://github.com/TheAxiomFoundation/orrery/pull/4). +- [Dynamics graph integration, #420](https://github.com/PolicyEngine/microcosm-dynamics/pull/420): + owned by the separate retirement-model task and tested on synthetic inputs. +- [Candidate methods and paper updates, site #72](https://github.com/PolicyEngine/microcosm.institute/pull/72): + public component evidence and remaining release checks. + +The accepted local shared viewer remains a separate consumer. There is no package +upgrade or competing generic graph shell in this consolidation. diff --git a/docs/us-uk-release-path.md b/docs/us-uk-release-path.md new file mode 100644 index 000000000..a9a5a51a9 --- /dev/null +++ b/docs/us-uk-release-path.md @@ -0,0 +1,277 @@ +# US and UK release path + +Planning snapshot, 12 September 2026. This is a release work plan, not a +certification of either country. Component tests, historical candidates and +current-candidate acceptance are distinct evidence. + +## Products and common construction + +The US product must support national and congressional-district analysis. The +UK product must support national, constituency and local-authority analysis. +Each country should have one fully constructed population from which full and +smaller analysis files are derived. Record the survey, monetary, policy and +boundary reference periods explicitly; do not infer them from a filename. + +After constructing and harmonizing the complete multispine, including its +initial clones, assign one atomic area to each resulting household and derive +larger geographies from versioned mappings. Distinct clones may receive different +areas, constrained by observed source geography and keyed by stable clone +identity. Subsequent enrichment retains the assigned location. Once construction +and enrichment are complete, +calibration changes household weights; scope selection and pruning retain the +original values, missingness and relationships for every retained household. + +For a shared set of weights and consistent measurement definitions, additive +local totals must reconcile to national totals. Separately refitted compact +views need declared approximation tolerances and their own diagnostics. A +common source population alone does not guarantee identical estimates. + +## Release sequence + +| Stage | US work | UK work | Exit evidence | +| --- | --- | --- | --- | +| Source and period coverage | ACS and ASEC survey multispine; PUF tax detail; SCF and other required auxiliary sources; declared aging | Raw FRS 2024–25; SPI/HMRC, WAS, LCFS/ETB and other required stages; declared uprating | Source register, licensed-input boundaries, source definitions, missingness, units and periods; every applicable input has a producer | +| Complete population | Harmonized relationships and tax/benefit units; geography; financial and PUF enrichment; remaining benefit, disability, housing and asset inputs | Integrated FRS stages and current relationship/claimant definitions; income, wealth, consumption and benefit inputs | Small real end-to-end build; source-channel coverage; plausible distributions and joints; no silent engine-default substitutions | +| Geographic support | Block assignment after initial multispine cloning; versioned county/PUMA/CD derivation; enough household types in each district | Shared atomic-area assignment after full spine cloning, and derivation for each nation; adequate constituency/LA support | Observed geography respected; distinct stable clone draws; subsequent location invariance; support and fit measured at advertised levels | +| Model and calibration | Locked US model; national/state/CD target matrix from identified facts | Locked UK model and companion input changes; national/local matrix with corrected UC and other target definitions | Baseline engine evaluation, calibrated population, target-level errors, support/weight diagnostics and explicit outstanding failures | +| Independent quality | Held-out distributions and geographic cells; tax/benefit totals and representative reforms; comparison with the incumbent | Current release battery, held-out areas and complete incumbent comparison; resolve remaining support and target-fit failures | Candidate-specific reports; explained regressions; thresholds agreed before choosing a candidate | +| Scale and compact views | Increase local build size after smaller checks; derive and refit the intended full/sparse family | Choose a size that meets local support and quality needs; complete compact/exact-count release evidence | Measured time, memory and cache replay; retained-input invariance; full/compact error and performance comparisons | +| Release and consumption | Exact-file dashboard, methods/papers, immutable package and real PolicyEngine loading | Exact-file dashboard and certification; immutable cut, loader/pointer integration and PolicyEngine-UK adoption | Fresh export/readback, reproducible manifest and checksums, consumer calculation smoke test, reviewed release and rollback path | + +## Current position + +| Stage | US demonstrated result | UK demonstrated result | +| --- | --- | --- | +| Population construction | Small real ACS/ASEC composition, clone and financial enrichment pass; maintained two-route PUF and subsequent amount/health hosts pass scoped invented controls; native PUF and remaining inputs pending | Real FRS 2024–25 spine and earlier whole-spine parity; latest integrated stage set needs fresh coverage and verification | +| Geography | Small native post-clone block assignment and financial extension passed on 11 September; strict required replay passed on 12 September; complete CD support/fit pending | Existing OA-ladder assignment exercised in real candidates; shared post-clone adapter passes 52 scoped checks with lazy imports, while native graph integration remains pending | +| Calibration | Declared national/state/CD measurement and dense grouped calibration stages pass on invented inputs; fully enriched native solve pending | Real 55,000-household build/calibration experiments complete but fail area-support and target-fit gates | +| Independent quality | Complete-candidate engine outcomes, holdouts and reform validation pending | Prior comparisons and diagnostics exist; the recorded 55k candidates skipped holdout evaluation and do not establish release quality | +| Full/compact files | Small prefix replay and 36 supplied-parent full/pruned/local export checks pass; complete enriched release family pending | Size-selection implementation and measured candidates exist; current accepted full/compact local family pending | +| Delivery | No accepted new-architecture release or default consumer adoption | Assembly implementation and historical assembled cuts exist; current certification and default promotion pending | + +The 11 September native pilot passed the corrected survey → full initial +support clone → atomic block assignment → financial-enrichment order. At a +1/1000 source sample, its 1,584 source households and 3,464 people become +3,168 households and 6,928 people after cloning. The nineteen-node graph fits +and attaches seven financial fields. Its closed receipt is tied to frozen +source `2ca11c85a`, and reports 5,362 seconds elapsed and 9.31 GB peak RSS. +This supersedes the earlier assignment-before-cloning milestone for ordering; +it does not establish full PUF enrichment or national/CD calibration. + +On 12 September, a strict required replay of that frozen source passed with a +fresh output directory and a read-only retained store. It verified the source +bytes again, reused all nine prefix and nineteen financial-graph nodes, and +reproduced the four non-manifest exports byte for byte. The portable manifest +also matched after excluding only operational timing and cache-hit fields. +The closed run took 3,981 seconds and peaked at 10.99 GB RSS; source, resource, +native-input, owned-file and thread-control checks remained unchanged, with no +unexpected access refusals. See the +[replay record](../experiments/us-financial19-required-replay-20260912.md). +Meanwhile the maintained graph's observer-mutation defect has been fixed and +independently reviewed. The shared graph consolidation merged in +[PR #913](https://github.com/PolicyEngine/microcosm/pull/913) at +`a9cc63e737fd4619a304117dc2ec7dcd86d97901`; the release branch has incorporated +that main revision without changing its existing graph implementation. Those +software changes still need their +own native candidate verification; this replay certifies only its frozen +financial stage, not the newer source or a release. + +The maintained PUF55 host is now adopted. Its 31 small controls and two actual +full-host controls pass on invented inputs: both conditioning routes, 110 fits +and 110 applications, cold execution, required replay, whole-population +preservation and numerical finalizer equality. The full-host run took 742 +seconds and 0.74 GB peak RSS. Those 64 invented donors do not establish the +runtime or quality of the full native donor. A separate current-source native +packet uses a fresh writable store and the actual retained financial run. +See [the PUF host record](../experiments/us-puf55-host-adoption-20260912.md). + +That native packet's second attempt closed with an assertion failure after +about 82 minutes, during its financial-stage acceptance and before any PUF +execution. The source and resource pins remained unchanged and no unexpected +access refusal was reported. Source review identified an invalid harness +assumption: the atomic host first materializes CREATE and ALLOCATION, so those +two nodes legitimately hit when its expanded graph runs in the same store. +Requiring every prefix node to miss rejects that valid execution. The refusal +record does not identify its precise assertion frame, so this diagnosis does +not establish completed native financial acceptance. The corrected v3 harness +passes two actual invented financial cold/replay controls and thirteen bounded +source and diagnostic controls. After independent review of its exact packet, +one fresh native run started with unchanged calculation source, native inputs +and resource limits. Its result is pending; no retry pass or native PUF result +is claimed. + +The maintained financial graph now retains an eighth field: the tax-exempt +interest remainder of its existing observed or modeled total. The split remains +a modeling assumption. Twenty-five financial/source controls and three full +invented PUF replay, finalizer and retained-output controls pass; the PUF stage +preserves the original-channel remainder. The historical native financial +receipt above still covers seven fields. See +[the interest correction](../experiments/us-survey-interest-conservation-20260912.md). +A separate reviewed source-hashing optimization preserves exact digest bytes +and passes thirty source tests. Its repeated-string benchmark improvement is +not a measured native build improvement; see +[the scoped benchmark](../experiments/us-source-string-seal-20260912.md). + +The new fiscal measurement stage passes all 66 interpreter/stage checks on the +combined integration, including an actual US-engine dividend calculation. It +uses declared target bindings, household-aligned CSR measurements and explicit +national/state/CD masks. Missing consumed predicates refuse, indicator sums +are numeric, and local calculation helpers participate in cache identity. +Specialized unsupported bindings refuse; this is not full fiscal-registry +coverage. + +The dense grouped calibration stage is now integrated. Independent review of +its numerical adapter found no actionable defects, and all 167 selected +calibration tests passed. The combined integration also passed all 42 tests +for the measurement and dense stages together. The adapter uses the existing +grouped Adam solver, keeps initially zero weights fixed, enforces group bounds +and refuses a result exceeding the declared original row caps. It returns +calibrated weights and schema-8 diagnostics. Numeric bound arrays do not prove +their source: the complete-population host must authenticate the original +sampling references and retained parent before and after use. Native fit, +full-size memory and calibration acceptance remain unestablished. + +Complete-population ancestry remains an explicit requirement. The PUF host now +retains its actual checked result for downstream consumption through +`check_survey_puf55_run`. Constructing a dataclass or decoding its receipt cannot +issue that authority. Consumers must recheck the original run before use and +after their last relevant I/O. The corrected checker passes 25 controls on +actual invented cold/required execution, including extra manifest attachments, +source/store corruption and late output mutation. Independent review approved +the source and mechanical test colocation; 45 cases share one expensive fixture. +See [the retained-output evidence](../experiments/us-puf55-checked-output-20260912.md). + +The fixed post-PUF country host now assembles source-qualified unemployment, +three health-cost amounts and nine current health-coverage inputs. It retains +the checked PUF owner and attaches both fragments to the same receiving +population, preserving existing columns, geography, design anchors and mass +ledger. ASEC observations and unresolved values are retained on both clones; +modeled ACS draws are deliberately shared within each original person's clone +pair. Reporting universes remain explicit; no prior-wage or age-only zero +fallback is introduced. The narrower ACS coverage concepts remain nullable +where the source does not establish equivalence. + +The accepted invented fixture reports 74 present input names out of 161, up +from 61, leaving 87 missing. It executed 26 new enrichment nodes over a validated +245-node prefix, then required all 271 replay hits and complete Frame readback. +Its six logical controls passed across five tests in the main attempt and a +corrected focused parent-revocation follow-up, not one all-green full run. +Independent review approved that scoped evidence. After integration with the +eighth interest field and merged shared graph, 80 cheap source, attachment and +health-fragment controls pass. Declaration checks still give 271 nodes: the +eighth interest field adds an output to the existing financial attachment, +not a new fit or graph node. This integration did not repeat the full combined +host or measure fresh native coverage. See +[the combined integration record](../experiments/us-survey-enrichment-integration-20260912.md) +and [source/model decisions](us-current-survey-amount-successor.md). + +Actual graph tracing also confirms that the legacy full ASEC carry and ACS +gap-fill are not executed by this host. Remaining pension, retirement-account, +property, farm and other-income inputs therefore need source-qualified producers +and appropriate ACS completion. The ACS property-income aggregate should +condition a joint decomposition before the dependent PUF fits; separately adding +rent afterward would not conserve that observed anchor. Source measurement +bridges and overlapping income categories must be resolved explicitly before +claiming reconciliation. The broader ACS retirement total is not merely pension +plus regular-IRA income. + +Two reviewed building blocks for that completion are adopted in the release +branch. The [ACS anchor qualifier](../experiments/us-current-acs-income-anchors-20260912.md) +preserves original INTP/RETP amounts, adjustment factors, original ages and +allocation flags through the retained source owner; its 35 invented-source +controls pass. The [signed reconciliation helper](../experiments/signed-income-reconciliation-20260912.md) +projects joint component draws onto a caller-qualified total while allowing +declared signed components and preserving loss offsets; its 45 numeric controls +pass. These components are not yet invoked by the financial or PUF host. +The donor measurement bridge, scales, named graph reconciliation operation and +dependent model replay remain to be implemented and verified. + +The remaining critical sequence is complete PUF/input integration, a small +real full build, model and calibration evaluation, then progressive scale +and release verification. The existing age-development runner is a separate +prefix demonstration, not calibration of the nineteen-node financial result. See +[the US review guide](us-launch-review.md) for scoped acceptance records. + +UK work already includes raw-source enrichment, national/local calibration, +size experiments and release assembly. Its recorded P50/P95b 55,000-household +candidates fail area-support and target-fit checks; the skipped holdout does +not establish an incumbent-comparison pass. Historical dense assembly also +does not satisfy today's release battery. See +[the size evidence](../experiments/355-uk-dataset-size-receipts.md) and +[the current dense release runbook](uk-dense-release-assembly-runbook-762.md). + +The current UK route clones for geographic support and then uses the existing +OA ladder. It has not adopted the new shared atomic geography graph keyed to +stable post-clone household identity. Its existing placement after cloning is +consistent with the user's 10 September ordering correction; the source, +identity and shared-operator migration remain separate work. Maria's +[PR #900](https://github.com/PolicyEngine/microcosm/pull/900) updates Chronicle +household targets and the Northern Ireland lookup; that source work does not +by itself migrate the assignment owner. Anthony's +[PR #896](https://github.com/PolicyEngine/microcosm/pull/896) provides staging +telemetry and synthetic smoke evidence, not a calibrated release. + +The shared UK adapter now has accepted supplied-array tests for England/Wales +Output Areas, Scottish Output Areas and Northern Ireland Data Zones. These +checks cover source conventions, observed-region constraints, stable post-clone +identity and row-order/subset invariance. Ordinary UK imports also preserve the +public export contract without eagerly loading unrelated stages or country +engines. All 52 checks passed with unchanged source bytes and no unexpected +access attempts; see the [scoped acceptance record](../experiments/uk-atomic-area-lazy52-20260910.json). +No native UK area source, whole-spine build or calibration result is certified +by those checks. Maria can use these shared primitives in the existing UK graph +integration rather than starting a second country runtime. + +UK release coverage must be re-established on the chosen integrated build and +locked model. The declared 145-input contract's older candidate evidence must +not certify the current stage set. The +[migration epic](https://github.com/PolicyEngine/microcosm/issues/665) tracks +source and calibration work; +[exact-count release parity](https://github.com/PolicyEngine/microcosm/issues/898) +and [default promotion](https://github.com/PolicyEngine/microcosm/issues/823) +remain separate consumer-release requirements. Existing open issues are leads +for candidate-specific verification, not proof every historical defect remains. + +## Team meeting decisions + +This document is the shared US/UK release-path reference. The country review +guides and run receipts provide supporting detail; update the sequence and +ownership here when they change. + +The 10 September coordination sequence is: + +- Maria's #900 source/target reconciliation merged on 10 September at 14:54 UTC; + Anthony's #855 target hierarchy/diagnostics and corresponding dashboard + consumer follow it. Its merge clears that dependency, not all release gates. +- Maria owns registering every existing UK stage as a graph node, stacking on + #893. Changes to UK stage order follow in a separate PR. The shared atomic-area + adapter work supports that integration and does not create a second UK graph. +- Anthony's UK staging #896 can proceed independently. His earlier runtime + proposal #885 is superseded in intent, not another required implementation. +- US construction and verification continue alongside these changes. Complete + graph coverage in both countries is required for the broader advertised + launch; passing tests for individual stages does not establish that coverage. + +The latest UK full rebuild and evaluation were reported as underway at the +meeting. Its new report must replace historical quality evidence only after the +actual candidate and results are available; the approximate four-hour runtime +reported for that build is not a US or release ETA. + +1. Confirm the release years, advertised geographies and full/compact products. +2. Assign one accountable owner to each country candidate and one owner to each + shared dependency: runtime, calibration, diagnostics and release consumers. +3. Review every row above for the latest actual candidate. Name the missing + artifact and next measurable result, rather than reporting code completion + as population acceptance. +4. Agree calibration, holdout, incumbent and representative-reform criteria, + including which differences need explanation and which block release. +5. Set dates for the next complete small candidate, the full quality review and + the production-sized candidate using measured runs rather than linear timing + extrapolation. + +Country data work, target/source reconciliation, independent validation, and +dashboard/consumer integration can run in parallel. Each final dashboard and +release verdict must refer to the same candidate bytes. Cross-sectional release +does not require retirement dynamics, every possible forecast year, or a +complete Chronicle/Orrery redesign. Keep those workstreams active without +making them prerequisites for the first accepted cross-section. diff --git a/docs/validation-cost-and-borrow-boundaries.md b/docs/validation-cost-and-borrow-boundaries.md new file mode 100644 index 000000000..f88c2cf4c --- /dev/null +++ b/docs/validation-cost-and-borrow-boundaries.md @@ -0,0 +1,43 @@ +# Validation cost and borrow boundaries + +The US graph must detect changed sources, substituted artifacts and changes to retained Populations without repeating the same expensive work at every nested call. This note records the current decision and the proposals still under review. It does not change dataset release requirements. + +## Adopted: reuse source compilation + +The ACS native coverage owner now keeps a bounded process-local cache of compiled code. Its key includes exact source bytes, filename, mode, flags, inheritance setting, effective optimization level and compiler identity. Every source read, AST check and loaded-function, global, alias and closure check remains fresh. Cached code is neither source authority nor a portable receipt. + +The change passes 25 focused controls and the original invented financial fixture, including cold execution, required replay and final checks. One paired profile took 216.82 wall seconds versus 308.26 seconds before the change, a 29.66% reduction. CPU time fell from 303.90 to 214.69 seconds. These instrumented observations establish an improvement for that fixture; they do not establish native-data performance. See [the experiment](../experiments/us-acs-compilation-cache-adoption-20260910.json) for exact source, test and verification identities. + +The native pilot already in progress retains its original frozen source snapshot. + +## Next: remove duplicate work within a check + +A source review confirmed nine direct Frame storage traversals in a healthy `verify_acs_native_coverage` call: three calls to `_verify_frame`, each performing three storage hashes. Some repetitions occur across supplemental axis and resolved-weight checks, so collapsing all nine into three is not automatically equivalent under the existing mutation checks. + +The next candidate will share the first storage calculation with the supplemental fingerprint calculation, while preserving the independent final prepared-Frame storage pass and all three outer verification points. That would reduce nine direct traversals to six. It needs a regression that mutates a cell during the supplemental check, unchanged receipt and fingerprint comparisons, and a paired measurement before adoption. + +## Keep separate: code, files and retained objects + +Three different facts currently appear together in nested validation: + +- Loaded-code identity: the interpreter's functions, code, globals, aliases and closures. +- Agreement with files: source bytes, archive snapshots and stored graph artifacts still match their recorded identities. +- Retained-object identity: the actual Frame, Population, weights and ancestry still match their issued state. + +An independent Fable 5.1 review recommended making these checks explicit and performing expensive file verification primarily at public entry and emission boundaries. That is a proposed contract change, not an adopted optimization. Moving file checks would change when an altered archive is detected. It requires a documented public-entry contract and tests at every relevant callback, file borrow and emission boundary before implementation. + +Similarly, an AST cache would hold mutable objects, unlike compiled code. It needs its own ownership design and evidence; the compilation-cache result does not establish its safety or benefit. + +## Composition review + +The same review raised kernel-registry sharing, registration order and recipient ordering. These require owner-level checks before being treated as bugs. For example, each financial run currently constructs a fresh survey prefix internally; it does not accept an existing prefix for a second extension. The PUF raw-route merger separately derives and verifies the complete receiving tax-unit axis before restoring row order. Neither observation alone proves every composition boundary correct. + +The budget returned-view correction is now adopted. Detached documents and grouped bounds are constructed after the last borrowed I/O, then checked against the issued payload and retained owners. Four targeted controls passed, including nine callback branches; the post-run verification rechecked 501 source/owned files, 5,983 model-code files and 12 resources. The earlier eight financial-successor controls remain evidence for their original source revision. These are separate component runs, not twelve tests of one integrated revision. See [the correction experiment](../experiments/us-budget-detached-view-correction-20260910.json). + +A second Fable source review found no confirmed correctness or ownership bug in the public PUF recipient interface. It recommended explicit cold-cache reuse assertions, exact comparisons with the original issuers, and clearer documentation of the original-source reads required by the recipient kernels. The frozen two-positive pilot remains unchanged; those additions belong to subsequent controls. + +The immediate implementation priorities remain actual public PUF recipient and graph checks, authenticated donor/model binding, one whole-cohort PUF finalization and attachment, then native calibration and release verification. Performance work proceeds alongside those tasks. + +## Review scope + +Fable 5.1 completed a read-only source review on September 10, 2026, using the saved Hivesight login. Its requested scratch-cache read was denied and that limitation was disclosed in its result. It supplied architectural advice, not runtime approval or a release verdict. The recommendations above are distinguished from independently verified source findings and adopted changes. diff --git a/experiments/893-reconciliation-amendments-19-20-20260912.md b/experiments/893-reconciliation-amendments-19-20-20260912.md new file mode 100644 index 000000000..f1627eb2b --- /dev/null +++ b/experiments/893-reconciliation-amendments-19-20-20260912.md @@ -0,0 +1,722 @@ +# #893 reconciliation to main's amended graph interface — lane report + +> **12 September Codex continuation:** this is the historical layer proposal +> at `80fef2217`, not an approved extraction plan. The final battery +> `bt0nx8fcc` was killed during its first pytest group: CI-group verification +> and coverage check exited zero, but no final consumer result was produced. +> Independent review found that G4 exposes mutable admitted populations to +> observers and that the layer order omits hard packaged-resource dependencies. +> Fix and review both before extracting or landing layers. PR912 is also held +> for fixes from Astra's executed review; the branch-head assumption below is +> superseded until its corrected head lands. +> +> Resource repair `a4b6fe88c` restores all three omitted ASEC JSON contracts +> from their exact historical blobs, matching unchanged loader pins. Eight +> targeted tests and wheel-content verification pass. These three resources, +> `acs_2024_housing_universe.json`, and current-money domains/consumers/price-basis +> resources must accompany their first PR-1 loaders or an earlier prerequisite; +> Python import analysis alone cannot establish extraction order. No engine +> defaults were admitted and no source/engine identity was refreshed. + +> **Later 12 September correction:** `6f66fa545` closes the demonstrated G4 +> mutation defect by handing observers detached snapshots, including nested +> table values, metadata, schema, weights and ledgers. Independent review is +> clean; 524 graph tests, five focused cases and all four existing synthetic +> consumers passed, including financial and calibration teardown checks. An +> earlier source-edit-during-test failure remains preserved and is excluded. +> Entry 7.4 below is historical: the current observer need not be trusted to +> leave its snapshot unmodified. Retaining snapshots still costs memory. +> +> `cb27bdb0b` makes the offline explanation display executor-owned unreached +> and gate-exception states independently from cache hits. Sixteen graph +> explanation/actual-gate cases and two free-form diagnostic controls passed. +> Its new `explain → availability` import is recorded in the implementation +> inventory; availability was already bound as a whole module in all ten +> using scopes. No scope, dependency, resource, or unbound-use exemption was +> added. All eight prepared-resource/full-manifest tests passed afterward. +> +> Corrected PR912 head `5e33d6971` is undergoing exact-head CI and Fable review. +> Graph-only extraction is being prepared against that revision, incorporating +> G1–G4 and numbered charter entries. Neither that extraction nor the remaining +> country layers are declared accepted by this historical report. + +> **Subsequent 12 September integration checks:** corrected PR912 head +> `5e33d6971` received Fable approval; its merge remains conditional on CI. +> The local integration merged that exact revision in `9e61d0d04` and +> recomputed the combined source/seed identities. All 734 graph and fitting +> tests passed. The six-file spec/build sweep collected 248 tests: 247 passed +> and one failed because its invented PUMA fixture still used schema 1 and +> omitted the required joint tract/CD population support. The fixture now +> supplies consistent schema-2 support and explicit invented provenance; that +> single test passed when rerun. The original failed sweep and intermediate +> incomplete fixture repair are preserved. This is separate-run evidence, +> not a claim that the complete 248-test sweep was rerun successfully. +> +> Shared graph consolidation is now public as PR913, temporarily based on +> PR912, with independent Fable review in progress. Its runtime matches this +> integration's graph implementation. Native strict replay, PUF55 host +> acceptance and complete national/CD release acceptance remain separate +> work and are not established by these software checks. + + +Date: 2026-09-12. Worktree `~/PolicyEngine/_worktrees/microcosm-us-launch-verified-lanes-20260910`, +branch `microcosm-us-launch-verified-lanes-20260910` (PR #893's branch is +`microcosm-us-launch-integration-20260909`). Started at `069d5ed9a`. No push, no +new branch, no stash, `uv.lock` untouched. Every commit below is on the +checked-out branch. + +## 1. Outcome + +- `origin/main` (`0c3f4f651`, amendment 19) and + `origin/amend-keyed-seed-and-uniform-draws` (`23ba24770`, amendment 20 / PR + #912) are merged. **#912 was merged from its branch head**, so once it lands + the dispatcher should re-run `git merge origin/main`; for the graph and fit + packages, `docs/graph-interface.lock` and `docs/graph-acceptance.md` that is + expected to be a no-op (this tree already carries `23ba24770`'s bytes for all + of them, verified with `git diff --quiet 23ba24770 HEAD -- `). +- Every frozen or amended file equals `23ba24770` byte for byte: `decl.py`, + `kernel.py`, `keys.py`, `serialize.py`, `view.py`, `artifact_edges.py`, + `randomness.py`, `population.py`, `fit/qrf.py`, the lock, the charter, + `test_graph_kernel_contract.py`, and every `test_acceptance_*` file. The + interface-lock test passes (it is part of the 518-test graph run below). +- Four not-yet-amended graph pieces the US runtime consumes were re-applied + as separate named commits on top of main's files; four pieces with no + consumer were dropped (section 3). +- The five graph tests the brief named are green; the whole graph package is + green (518 passed); the fit package is green (191 passed); the spec-engine + check, `ci_test_groups --verify` and ruff are clean. Build-side results are + in section 5. +- The layer map (section 6) partitions the branch's remaining 407-file diff + against `origin/main` into the eleven staged layers, the later closure and + post-staging work, and the graph pieces, with per-layer file lists, test + files, import dependencies, and a proposed landing order. Section 7 has a + draft charter amendment entry per kept graph piece. Section 8 lists what + needs Max. + +Lane commits (newest first): + +``` +07566eb17 Re-pin the US implementation inventory for the reconciled graph package +c704a1be6 Pin that unreached propagates through a structural node and its version +2e1857a4b Bring the #893 reconciliation journal up to the piece commits +a16eeacf8 Re-apply the private population observer on run_graph +d1019762b Re-apply executor execution states: gate exceptions leave their consumers unreached +db1b7821a Re-apply complete Frame metadata storage and non-finite JSON refusal in the store +752ab840f Re-apply the raw-byte source codec on main's graph package +cff8fbf32 Re-pin the calibrate and simulate H1 parity cases the branch's kernels move +dc621c14c Re-pin the seed protocol and compiled seed map digests on the merged tree +051357909 Merge origin/amend-keyed-seed-and-uniform-draws (23ba24770, amendment 20) +3010b7788 Merge origin/main (0c3f4f651, amendment 19) into the #893 integration branch +3ad1b1aae Open the #893 reconciliation lane journal +``` + +## 2. The two merges and the per-file resolution + +Baseline before any change (`uv run --no-sync pytest packages/microcosm-graph/tests`): +5 failed, 362 passed, exit 1 — exactly the five the brief names +(`test_acceptance_b_ownership::test_b2_executor_enforces_ownership`, +`test_acceptance_h_parity::test_h1_kernel_parity`, +`test_graph_executor::test_fit_qrf_tolerance_source_hash_pin_is_current`, +`test_graph_kernel_contract::test_context_numerics_default_empty_and_carry_scopes`, +`test_graph_serialize::test_generated_parity_graphs_bind_real_kernels_and_direct_bytes`). + +### Merge 1: `git merge origin/main` (commit `3010b7788`) + +Git reported 8 content conflicts plus one in `PROGRESS.md`; `store.py`, +`__init__.py` and `PROGRESS.md` auto-merged. Resolution: + +| Path | Resolution | +| --- | --- | +| `packages/microcosm-graph/src/microcosm/graph/{artifact_edges,decl,executor,kernel,keys,manifest,serialize,view}.py` (conflicts) | main's bytes (`git checkout origin/main -- packages/microcosm-graph`) | +| `packages/microcosm-graph/src/microcosm/graph/{store,__init__}.py` (auto-merged) | main's bytes, same checkout; the branch's hunks return as pieces A and B | +| `packages/microcosm-graph/src/microcosm/graph/{attachments,availability,schema}.py` (branch-only) | removed in the merge (`git rm`); `availability.py` returns as piece C, the other two are dropped | +| `packages/microcosm-graph/src/microcosm/graph/randomness.py` (branch-only until amendment 20) | kept as the branch had it; replaced by `23ba24770`'s in merge 2 | +| `packages/microcosm-graph/tests/*` | main's bytes; the branch's copies were strict subsets of main's (numstat showed only removals), so this also dropped nothing of the branch's; `test_frame_metadata_store.py` and `test_lazy_snapshot_metadata_integration.py` (branch-only) removed here, the first returns with piece B, the second is dropped | +| `packages/microcosm-graph/tests/fixtures/parity/kernels/fit.qrf/pins.json` | main's (consistent with fit/qrf.py being main's in this merge) | +| `packages/microcosm-fit/src/microcosm/fit/qrf.py` (auto-merged to the branch's) | main's bytes; the branch's `predict_from_uniforms` returns with amendment 20 in merge 2 (the branch's qrf.py was byte-identical to `23ba24770`'s) | +| `packages/microcosm-fit/src/microcosm/fit/{_graph_legacy_apply,_graph_legacy_qrf,graph_legacy_apply,graph_legacy_apply_matrix,graph_legacy_qrf,graph_legacy_train,model_input,qrf_target}.py` (branch-only) | kept: not amended content, consumed by the US runtime (layer 01) | +| `docs/graph-interface.lock`, `docs/graph-acceptance.md` | main's bytes | +| `PROGRESS.md` | both sides kept: this lane's section, then main's Amendment 19 and #907 sections, then the branch's history | +| everything else | ordinary auto-merge | + +### Merge 2: `git merge 23ba24770` (commit `051357909`) + +Six conflicts: + +| Path | Resolution | +| --- | --- | +| `packages/microcosm-graph/src/microcosm/graph/randomness.py` (add/add) | `23ba24770`'s (adds the signed-zero normalisation the branch lacked) | +| `packages/microcosm-fit/tests/test_qrf_stateless.py` (add/add) | `23ba24770`'s (182 lines; the branch's 159-line copy was a strict subset) | +| `docs/evidence/spec-engine/us-f0-coverage.json` | `23ba24770`'s, then recomputed (section 4) | +| `packages/microcosm-build/src/microcosm/build/spec_engine/inventory_coverage.py` (seed digests) | `23ba24770`'s, then recomputed (section 4) | +| `packages/microcosm-build/tests/test_spec_engine_country_bundles.py` (AM/BE/UK digests) | `23ba24770`'s; see section 5 for the merged-tree check | +| `packages/microcosm-build/tests/test_us_multispine_pool_tool.py` (pool tool `spec_sha256`) | `23ba24770`'s; see section 5 | + +After merge 2 the graph package, fit sources, lock and charter were +byte-identical to `23ba24770` except the eight branch-only fit modules above; +`docs/graph-interface.lock` matched `shasum -a 256` of the two frozen files. + +## 3. The not-yet-amended graph pieces + +Consumers were found by an AST scan of every `.py` under +`packages/microcosm-{build,calibrate,fit,frame,data}` and `tools/` for names +imported from `microcosm.graph.*` that `23ba24770`'s graph package does not +define, plus greps for the keyword-level uses (`_population_observer=`, +`population_retention=`, `emit_validation_artifact=`). + +### Kept (each re-applied on main's file as its own commit, with tests) + +| Piece | Commit | Consumers outside the graph package | Graph-level tests | +| --- | --- | --- | --- | +| A. `codecs.py` raw-byte mode: `SourceBytesCodec`, `register_bytes`/`load_bytes`, `raw-bytes-v1`, `RAW_BYTES_MAX_BYTES`, `load_source_bytes` (+ two `__init__` exports) | `752ab840f` | 13 modules: `graph_atomic_geography`, `graph_geography`, `graph_sources`, `graph_atomic_survey_population`, `graph_puf55_canonical_donor`, `puf_raw_source`, `puf_monetary_source`, `puf_monetary_agi_projection`, `puf_full_source_graph`, `atomic_block_sources`, `atomic_block_api_sources`, `survey_atomic_geography`, `puf55_survey_recipients`; 8 US test files | the branch had none; 4 added to `test_graph_codecs.py` (8 passed) | +| B. `store.py` Frame-metadata storage (`microcosm-graph-frame-v2`, `_encode/_decode_frame_metadata`, `frame_metadata_sha256`, `_put(validate_existing=…)`) and the non-finite JSON decode hooks | `db1b7821a` | `survey_population_replay.same_replayed_frame`, `population_input_coverage`, `graph_full_puf_enrichment` (all call `_encode_frame_metadata`), and every US stage relying on `Frame.metadata` surviving a store round trip | `test_frame_metadata_store.py` (the branch's file, 19 collected tests) | +| C. `availability.py` + executor/manifest execution states: `gate_exception`, `unreached`, `blocked_by`, cache-record schema 3, manifest schema 4, refusal of kernel-authored execution metadata; replaces amendment 19's refusal of a gate declaring a typed output | `d1019762b` | `graph_atomic_survey_clone` (`atomic_geography_nodes(..., emit_validation_artifact=True)` makes the geography gate declare a typed output, which main's executor refuses outright), its consumers `survey_age_calibration`, `graph_current_survey_predictors`, `survey_origin_budget`; `population_input_coverage` (`execution_state` on every producer receipt); build test `test_atomic_geography.py` parametrises the declaration | the branch had none; main's `test_a_gate_kernel_may_not_declare_a_typed_artifact_output` replaced by 6 tests in `test_graph_executor.py` (five in `d1019762b`, the structural-propagation one in `c704a1be6`) | +| D. `run_graph(_population_observer=…)` | `a16eeacf8` | `survey_age_calibration`, `graph_atomic_survey_financial`, `graph_survey_population`, `graph_atomic_survey_population`; 8 US test files | the branch had none; 1 added | + +What was deliberately not re-applied inside those files: the branch's +docstring deletions on the amended surface; its `_opaque_artifact_key` +inline hash (main's `graph_keys.opaque_artifact_key` is the same +derivation); its `KernelContext` argument order (frozen file); its eager +artifact payload reads on cache hits (main authenticates edges without +reading, `9018c4420`); its `StoreMiss`-on-hit for a missing declared artifact +(main's `5c4a8efde` decision, `StoreCorrupt`); the removal of main's #907 +datetime64/timedelta64 object-leaf refusal in `store.py`; and main's +`_validate_typed_ancestry` gate-ancestry shape check and the +`NodeRejectedError`→`ValueError` conversion (`402d9a631`, `7d45c32e5`), which +the branch's older copy lacked. + +### Dropped (no consumer left) + +| Piece | Why dropped | +| --- | --- | +| `attachments.py`, `_PopulationRetention`, `_LazyPopulations`, `run_graph(population_retention="lazy")`, `PopulationView.__slots__ = ("__weakref__",)` (staged as layer 08 GRAPH-ATTACHMENT-METADATA) | no call site outside the graph package passes `population_retention`; its only consumer was the branch's own `test_lazy_snapshot_metadata_integration.py`, dropped with it. It is a memory optimisation for long graphs (the native pilot's 11.5 GB peak is in the PR body); if it is wanted it is a self-contained later PR on top of piece B, which it imports. | +| `schema.py` (`graph_schema`, 316 lines) | imported by nothing in the repository (the `graph_schema_version` hits in the spec bundle are unrelated). | +| `keys.py` `_stream_file` chunked source hashing | no consumer and no test; it changes no identity (same digest sequence, it only bounds memory and refuses a file that changes while hashed). Main's `keys.py` is kept byte-identical. Candidate for a small standalone PR if large-source memory matters. | +| the `_write_node` per-coordinate memory refactor and the `del series` | no consumer; behaviour-preserving; main's amended `_write_node` kept. | + +## 4. Identities re-pinned on the merged tree, and why + +- **Seed protocol and compiled seed map digests** (`inventory_coverage.py` + `EXPECTED_HASHES`, `docs/evidence/spec-engine/us-f0-coverage.json`; + commit `dc621c14c`). Both hash the exact source bytes of the attested kernel + modules in `spec_engine/seeds.py` (no `microcosm.graph` module is on that + list, so the graph pieces do not move them). The merged tree carries the + branch's `acs_transfer` and `housing_inputs` changes and, through amendment + 20, the same `fit.qrf` bytes the branch already carried, so the actual + digests are the values the branch pinned in `1734b9e90` + (`a775cccd…` / `c15bff65…`), not `23ba24770`'s (computed on a tree without + the two US kernel changes). Computed directly (`spec.seed_protocol.implementation_sha256`, + `sha256_json(compiled.seed_stream_map.to_wire())`), then the report was + regenerated with `tools/spec_engine_coverage.py` and `--check` exits 0. +- **H1 parity pins for `calibrate` and `simulate`** (commit `cff8fbf32`). + Six graph tests were red on the merged tree, all on these two cases. + `CalibrateKernel.implementation_hash` binds `microcosm.calibrate`'s solve and + diagnostics modules, which the branch's grouped-bound solver path changes + (layer 05); `SimulateRulesKernel.implementation_hash` binds + `microcosm.frame.bundle`, to which the branch adds `Frame.__reduce__` (layer + 01). Re-recorded with `tools/graph_parity_repin.py calibrate` and + `… simulate`, which keep every pinned platform (both cases pin only + `arm64/darwin/py3.14`) and leave `direct.csv` untouched. The `fit.qrf` pin + came from `23ba24770` and needed no move. +- **The US implementation inventory** (`us_runtime/graph_implementation_inventory.json`; + commit `07566eb17`). This reviewed dependency fence is checked by every US + source stage before it runs (`graph_implementation.implementation_manifest`), + and its refusal surfaces as `PREPARATION_ISSUANCE_REFUSED` (the preparation + wraps every exception; the true cause was recovered from the exception's + `__context__`). Three entries no longer held: the `microcosm.graph` roster + listed the dropped `attachments.py` and `schema.py` (39 survey tests red on + that alone); the contracts for `executor.py`, `manifest.py`, `keys.py` and + `population.py` reflected the branch's versions rather than main's plus the + kept pieces, and `attachments.py` was a listed stage module in ten stages; + and `congressional_district_vintage.py`'s contract predated main's + `05ff48f05` (merged into the branch on 2026-09-11), which added the + `microcosm.calibrate.geography_constants` import — an inherited drift the + branch already carried at `069d5ed9a`, verified by recomputing the + pre-merge tree's contracts against its own inventory. Recomputed with the + module's own `_dependency_contract` / `_covered_imports`; the import + classification set is the exact union again. Seven of the ten stages now + build their manifest; three cannot (section 8.5). +- **Country-bundle digests and the pool tool's `spec_sha256`**: resolved to + `23ba24770`'s values in merge 2; their merged-tree check is in section 5. + +## 5. Commands, exit codes, counts + +All from the worktree root with the locked engine environment +(`uv sync --all-packages --locked --extra us --extra uk` → exit 0 before any +change). Tests ran with `uv run --no-sync pytest … -p no:cacheprovider`. + +| Stage | Command | Result | +| --- | --- | --- | +| baseline (`069d5ed9a`) | `pytest packages/microcosm-graph/tests` | 5 failed, 362 passed, exit 1 | +| baseline | `python tools/ci_test_groups.py --verify` | exit 0 | +| after both merges (`051357909`) | `pytest packages/microcosm-graph/tests` | 6 failed, 485 passed, exit 1 (calibrate/simulate parity pins; section 4) | +| after both merges | `pytest packages/microcosm-fit/tests` | 191 passed, exit 0 | +| after both merges | `python tools/spec_engine_coverage.py --check` | exit 1 (`seed protocol content digest differs`, `compiled seed map digest differs`) | +| after both merges | `python tools/ci_test_groups.py --verify` | `verification=ok`, exit 0 | +| after both merges | `ruff check .` | All checks passed, exit 0 | +| seed re-pin (`dc621c14c`) | `python tools/spec_engine_coverage.py` then `--check` | exit 0 / exit 0 | +| parity re-pin (`cff8fbf32`) | `python tools/graph_parity_repin.py calibrate`; `… simulate` | exit 0 / exit 0 | +| parity re-pin | `pytest test_acceptance_h_parity.py test_graph_parity_pins.py test_graph_parity_repin.py test_graph_serialize.py` | 27 passed, exit 0 | +| piece A (`752ab840f`) | `pytest packages/microcosm-graph/tests/test_graph_codecs.py` | 8 passed, exit 0 | +| piece B (`db1b7821a`) | `pytest test_frame_metadata_store.py test_graph_store.py test_graph_population.py test_graph_executor.py test_graph_manifest.py` | 274 passed, exit 0 | +| piece C (`d1019762b`) | `pytest packages/microcosm-graph/tests` (whole package, includes the interface-lock test) | **518 passed, exit 0** | +| piece C | `ruff check .`; `ruff format --check packages/microcosm-graph` | exit 0 / exit 0 | +| piece D (`a16eeacf8`) | `pytest test_graph_executor.py test_acceptance_b_ownership.py test_acceptance_f_gates.py` | 94 passed, exit 0 | +| structural test (`c704a1be6`) | `pytest packages/microcosm-graph/tests/test_graph_executor.py` | 82 passed, exit 0 | +| inventory re-pin (`07566eb17`) | `pytest test_us_survey_origin_budget.py::test_actual_initial_budget_and_zero test_us_survey_age_calibration_run.py::test_actual_seven_node_calibration_preserves_full_population` (two of the 39 formerly refused) | 2 passed, exit 0 | +| final tree | `python tools/ci_test_groups.py --verify` | exit 0 | +| final tree | `python tools/spec_engine_coverage.py --check` | exit 0 | + +Build-side and shared consumers, final tree (`07566eb17`). A first battery at `a16eeacf8` found 39 survey tests refused (`PREPARATION_ISSUANCE_REFUSED`) — all traced to the implementation inventory above, re-pinned in `07566eb17` — and one non-existent file name in my own raw-bytes group list (`test_us_puf_raw_source.py`, removed); the battery was then rerun in full: + +_(Historical draft marker resolved: task `bt0nx8fcc` was killed during availability consumers. No final battery counts or exit code exist; later groups did not start.)_ + +## 6. Layer map: landing #893 as sequential PRs + +Basis: `git diff --name-status origin/main HEAD` = 407 files (340 added, 67 +modified, 0 deleted), each attributed to the first branch commit that touched +it (`git log --name-only --no-merges 3094bfe84..HEAD`), with the eleven +`Stage integration layer NN` commits mapped to their layer names, the later +staging commits to layers 12–14, the sixty post-staging commits of 10–12 +September grouped by subject (P1–P6), and the graph pieces of section 3 as +their own layer. Import dependencies come from an AST scan of every changed +`.py` file resolved against that attribution (`microcosm.*` imports only). +The scripts and their JSON output are in this lane's scratchpad +(`layers.py`, `layers2.py`, `layers.json`, `per-file-deps.json`). + +Two corrections to the staged layer names as they sit in git history: + +- Layer 08 GRAPH-ATTACHMENT-METADATA is empty in the map: its only file, + `attachments.py`, is dropped (section 3). +- Layer 09 F-JOINT-GEOGRAPHY-GATE has no file of its own: its 27-line change + lives in `graph_geography.py`, which layer 01 created, so it lands with + layer 01. +- Nineteen files differ from `origin/main` only because `origin/main` does not + yet carry amendment 20 (`kernel.py`, `randomness.py`, `fit/qrf.py`, + `test_graph_kernel_contract.py`, `test_graph_parity_*`, + `test_graph_randomness.py`, `tools/graph_parity_repin.py`, the three + `pins.json`, the lock, the charter, the amendment-20 journal/receipts/ + changelog fragment) or were re-pinned by this lane. They vanish from the diff + when #912 lands and are not #893 content; they are listed under + "already on main's way" below and belong to no PR. + +### Dependency facts the order rests on + +Three kinds of link were measured, strongest first: + +1. **Hard**: a file imports a module that does not exist on `origin/main` + (a file the branch adds). These force order. +2. **Name-hard**: a file imports, from a module main already has, a *name* + that main's version of that module does not define (checked by parsing + `git show origin/main:`). Only three survive after discarding + submodule imports that merely look like names: `GroupedUpperBounds` + (layer 03's two-line re-export in `calibrate/__init__.py` of layer 01's + `group_bounds.py`; imported by layer 12's and 13's tests), + `decode_us_puma_ladder` (layer 06's `puma_ladder.py`; imported by layer + 13's `test_us_geography_integration.py`), and the graph pieces' names + (`load_source_bytes`, `load_raw_bytes`, `RAW_BYTES_MAX_BYTES`, + `_encode_frame_metadata`, `execution_state`). +3. **Soft**: a file imports a module main already has that the branch also + modifies (`frame_checkpoint`, `asec_checkpoint`, `operator_boundary`, + `us_runtime/__init__`, `calibrate/__init__`, `solve.py`, `bundle.py`, + `variable_labels.py`, and amendment 20's `qrf.py`/`kernel.py`). These do + not force order by themselves; each PR's own test run is what proves the + modified behaviour is or is not needed. + +Hard links between layers (source and tests): + +| Layer | hard on | of which tests only | +| --- | --- | --- | +| 01 SAFE-ADDITIVE | nothing | — | +| 03 ACCEPTED-SHARED-RESTORE | 01 (`group_bounds`, `reported_coverage_source`, `_person_signal_summary`, `operator_column_contracts`, `table_identity`), P3 (`asec_raw_stage_v4`) | P3 link is `test_us_asec_checkpoint.py` only | +| 04, 06 | nothing hard (soft on 01/03) | — | +| 05 SOLVE-MERGE-PROPOSAL | 01 (`group_bounds`) | — | +| 07 F-CATALOGUE-OPTIMIZATION | 01 (`survey_population_preparation`) | all three files are tests | +| 10 SOURCE-CLOSURE | G1 (soft) | — | +| 11 PLACEMENT-ADDITIONS | 01 (the eight fit modules, `full_puf_enrichment`, `graph_sources`, `survey_population_replay`), G1, G2 | — | +| 12 ORDINARY-CLOSURE | 01 (fit modules), 03 (`GroupedUpperBounds`) | all files are tests/fixtures | +| 13 INTEGRATION-REGRESSIONS | 01 (`graph_geography`), 03 (`GroupedUpperBounds`), 06 (`decode_us_puma_ladder`) | all three are tests | +| P1 POST-CLONE-ATOMIC-GEOGRAPHY | 01 (`randomness` = amendment 20, fit modules, `graph_combined_clone`, `graph_sources`, `graph_survey_population`, `survey_*`), G1, G3 (the gate's declared artifact), P2 | P2 link is two tests (`test_us_current_survey_predictor_demographics.py`, `test_us_survey_origin_budget_predictor_compatibility.py`) | +| P2 PUF55-TWO-ROUTE-AND-CANONICAL-DONOR | 01 (`model_input`, `_graph_legacy_qrf`, `graph_legacy_*`, and eleven test-side modules), 11 (`graph_full_puf_enrichment`, two tests), P1 (`graph_atomic_survey_financial`, one test), G1 | 11 and P1 links are tests only | +| P3 SURVEY-POPULATION-CATALOGUE-AND-BUDGET | 01 (`reported_coverage_source` from `asec_raw_stage_v4`; six test-side modules), G2, G3 (`population_input_coverage`), P2 (`asec_demographic_source`) | P2 link is `test_us_asec_demographic_source.py` only | +| P4 | nothing hard (soft on 03) | — | +| P5 | nothing hard (soft on 01/03/04/06) | — | +| P6 UK-ADAPTER-DOCS-AND-RECORDS | P1 (`atomic_geography`) | — | + +Two cycles exist and both are test-only: P1↔P2 (three tests) and 03↔P3 +(one test). Every source-level dependency is acyclic. + +### Landing order + +1. **PR-G1 raw-byte codec** — piece A (`codecs.py`, `__init__.py`, + `test_graph_codecs.py`). No dependency. Charter entry 7.1. +2. **PR-G2 Frame metadata storage** — piece B (`store.py`, + `test_frame_metadata_store.py`, + `changelog.d/20260906-frame-metadata-storage.fixed.md`). No dependency. + Charter entry 7.2. +3. **PR-G3 execution states** — piece C (`availability.py`, `executor.py`, + `manifest.py`, `test_graph_executor.py`). No code dependency, but it + supersedes an amendment-19 ruling: **needs Max's decision (section 8) + before it is opened.** Charter entry 7.3. +4. **PR-G4 population observer** — piece D (`executor.py`, + `test_graph_executor.py`). Stacks on G3 only because both edit + `executor.py`; G3+G4 can be one PR. Charter entry 7.4. +5. **PR-1 SAFE-ADDITIVE** — all of layer 01: 101 non-test files (100 modules + plus `graph_implementation_inventory.json`; `randomness.py` is excluded, it + is amendment 20) and its 26 test files, plus layer 03's two-line + `calibrate/__init__.py` re-export of `GroupedUpperBounds` (it belongs with + `group_bounds.py`) and layer 09's 27 lines (already inside + `graph_geography.py`), and `graph_implementation_inventory.json`, which + pins the `microcosm.graph` roster and module contracts and therefore has + to be re-pinned against whatever main's graph package is when PR-1 opens + (`07566eb17` shows the procedure). Hard on G1, G2 and G4 only (`_population_observer` + in `survey_age_calibration` and `graph_survey_population`). Its 21 soft + links are listed in the scratchpad `pr1-refined.json`; the PR's gate is + its own 26 test files on main + G1–G4 + PR-1. +6. **PR-2 US runtime restore** — layers 03 (without the `__init__` re-export + moved to PR-1, and with `asec_raw_stage_v4.py` moved in from P3 so + `test_us_asec_checkpoint.py` runs), 04 (`puf_support.py`), 05 + (`calibrate/solve.py`), 06 (`us_runtime/__init__.py`, `puma_ladder*.py`, + `tools/build_us_puma_ladder_artifact.py`, six tests), 12's calibrate + tests and fixture, 13's three regressions and `docs/us-launch-review.md`, + 14 (`dfa7f872c`), and the two re-pins that these files move: the + calibrate H1 pin (`solve.py`) and the seed digests (`acs_transfer.py`, + `housing_inputs.py`). 03/04/05/06 touch disjoint files and could be four + PRs (05 first, since only it is hard on 01); they are one here because + their tests import across all four. +7. **PR-3 catalogue, source closure, placement** — layers 07 (three digest + tests + fragment), 10 (`graph_acs_housing_universe.py`, eight JSON source + definitions), 11 (`graph_full_puf_enrichment.py` + test), and 12's fit + tests (`test_graph_legacy_*`, `test_qrf_target.py`, + `legacy-slice-apply-v1.json`). Hard on PR-1 and G1/G2. +8. **PR-4 calibration diagnostics** — P4 (`variable_labels.py`, its test, + fragments, experiments). Soft on PR-2. +9. **PR-5 survey population, catalogue and budget** — P3 minus + `asec_raw_stage_v4.py`: `common_frame_export_contract`, + `input_coverage_profile`, `population_input_coverage`, + `frame/bundle.py`'s `__reduce__` (carries the simulate H1 re-pin), ten + tests including `test_us_spine_blindness.py`; + `test_us_asec_demographic_source.py` rides with PR-7 instead. Hard on + PR-1, G2, G3. +10. **PR-6 post-clone atomic geography** — P1 (`atomic_geography.py`, + `graph_atomic_geography.py`, `atomic_block_*`, `graph_atomic_survey_*`, + `survey_atomic_geography.py`, `current_survey_geography.py`, + `tools/ci_test_groups.py`, fourteen tests); its two P2-importing tests + ride with PR-7. Hard on PR-1, G1, G3. +11. **PR-7 PUF55 two-route and canonical donor** — P2 (nineteen modules, + twenty-one tests plus the three deferred from PR-5/PR-6, PUF docs and + growth evidence, `pyproject.toml`'s `pythonpath` line). Hard on PR-1, + PR-3 (11), PR-5, PR-6. +12. **PR-8 CI, seed-identity diagnostics and spine blindness** — P5 + (`.github/workflows/test.yml`, `tools/spec_seed_identity_diagnostics.py`, + the seven `test_spec_seed_identity_*` files, `CLAUDE.md`, + `test_spec_engine_loader.py`, `test_us_puf_support.py`, + `test_us_stacked_spine.py`, the AM/BE/UK bundle digests). Soft on PR-2 + and PR-5; last because the diagnostic's roster names modules from every + earlier PR. +13. **PR-9 UK adapter, docs and records** — P6 (`uk_runtime/atomic_area_support.py`, + `atomic_household_identity.py`, two tests, `.gitattributes`, records). + Hard on PR-6. + +Every `experiments/` record and `changelog.d/` fragment rides with the PR +whose code it describes (listed per layer below). The "already on main's +way" files (amendment 20, listed at the top of this section) belong to no PR. + +### Per-layer file lists + +#### 00 JOURNAL (1 non-test files, 0 test files) + +- `./`: `PROGRESS.md` + +#### 01 SAFE-ADDITIVE (101 non-test files, 26 test files) + +- `packages/microcosm-build/src/microcosm/build/`: `survey_allocation.py`, `survey_domain_sample.py`, `table_identity.py` +- `packages/microcosm-build/src/microcosm/build/cd_benchmark/`: `__init__.py`, `canonical.py`, `origin.py`, `protocol.py`, `reasons.py` +- `packages/microcosm-build/src/microcosm/build/us_runtime/`: `_asec_current_money_codec.py`, `_person_signal_summary.py`, `acs_housing_universe.py`, `acs_housing_universe_source.py`, `acs_native_coverage_binding.py`, `acs_person_coverage_authentication.py`, `acs_person_coverage_columns.py`, `acs_population_catalogue.py`, `asec_2024_native_population.py`, `asec_coverage_authentication.py`, `asec_current_money.py`, `asec_current_money_graph_resources.py`, `asec_current_money_resources.py`, `asec_current_money_selection.py`, `asec_current_money_source.py`, `asec_current_money_units.py`, `asec_engine_evaluation.py`, `asec_household_coverage_fields.py`, `asec_household_observations.py`, `asec_housing_status.py`, `asec_housing_status_source.py`, `asec_housing_universe.py`, `asec_housing_universe_source.py`, `asec_income_observations.py`, `asec_original_household_weights.py`, `asec_person_coverage_source.py`, `asec_person_income_source.py`, `asec_population_catalogue.py`, `asec_prepared_source.py`, `asec_student_controls.py`, `cd_reference.py`, `cd_reference_sources.py`, `cps_carried_current.py`, `demographic_calibration_graph.py`, `full_puf_enrichment.py`, `graph_asec_income.py`, `graph_asec_prepared.py`, `graph_combined_clone.py`, `graph_composed_asec_binding.py`, `graph_composed_asec_measures.py`, `graph_composed_contracts.py`, `graph_composed_population.py`, `graph_context.py`, `graph_current_survey_puf_transfer.py`, `graph_geography.py`, `graph_housing_universe.py`, `graph_implementation.py`, `graph_implementation_inventory.json`, `graph_national_age_counts.py`, `graph_native_household_origin.py`, `graph_native_origin_implementation.py`, `graph_puf_detail_transfer.py`, `graph_puf_diagnostic_consumer.py`, `graph_sources.py`, `graph_survey_age_artifact.py`, `graph_survey_budget.py`, `graph_survey_calibration.py`, `graph_survey_population.py`, `national_age_activation.py`, `native_household_origin.py`, `operator_column_contracts.py`, `puf_detail_transfer.py`, `puf_diagnostic_consumer.py`, `puf_growth.py`, `puf_growth_graph.py`, `puf_monetary_agi_projection.py`, `puf_monetary_source.py`, `puf_price_baseline.py`, `puf_raw_source.py`, `reported_coverage_source.py`, `source_csv_builtin.py`, `survey_age_activation.py`, `survey_age_calibration.py`, `survey_age_sources.py`, `survey_calibration_diagnostics.py`, `survey_catalogue_selection.py`, `survey_observed_age.py`, `survey_origin_budget.py`, `survey_population_domains.py`, `survey_population_preparation.py`, `survey_population_replay.py` +- `packages/microcosm-calibrate/src/microcosm/calibrate/`: `group_bounds.py` +- `packages/microcosm-fit/src/microcosm/fit/`: `_graph_legacy_apply.py`, `_graph_legacy_qrf.py`, `graph_legacy_apply.py`, `graph_legacy_apply_matrix.py`, `graph_legacy_qrf.py`, `graph_legacy_train.py`, `model_input.py`, `qrf_target.py` +- `packages/microcosm-frame/src/microcosm/frame/adapters/`: `_policyengine_us_source_index.py`, `policyengine_us.py` +- `packages/microcosm-graph/src/microcosm/graph/`: `randomness.py` +- tests: `test_us_acs_housing_source.py`, `test_us_acs_person_coverage_authentication.py`, `test_us_acs_person_coverage_columns.py`, `test_us_acs_population_catalogue.py`, `test_us_asec_2024_native_population.py`, `test_us_asec_coverage_authentication.py`, `test_us_asec_current_money_source.py`, `test_us_asec_person_income_source.py`, `test_us_current_survey_puf_host.py`, `test_us_current_survey_puf_transfer.py`, `test_us_full_puf_enrichment.py`, `test_us_graph_survey_population.py`, `test_us_national_age_counts.py`, `test_us_puf_detail_transfer.py`, `test_us_puf_price_baseline.py`, `test_us_survey_age_activation.py`, `test_us_survey_age_artifact.py`, `test_us_survey_age_calibration_run.py`, `test_us_survey_age_development.py`, `test_us_survey_age_sources.py`, `test_us_survey_calibration.py`, `test_us_survey_observed_age.py`, `test_us_survey_origin_budget.py`, `test_us_survey_population_preparation.py`, `test_us_survey_population_replay.py`, `test_policyengine_us_ownership_index.py` +- source imports from: 02 GRAPH-RESTORE, 03 ACCEPTED-SHARED-RESTORE, 05 SOLVE-MERGE-PROPOSAL, 06 J-GRAPH-COMPATIBILITY, G RAW-BYTES-CODEC (graph piece), G STORE-FRAME-METADATA (graph piece), P3 SURVEY-POPULATION-CATALOGUE-AND-BUDGET, P4 CALIBRATION-SOLVER-AND-DIAGNOSTICS +- tests import from: 02 GRAPH-RESTORE, 03 ACCEPTED-SHARED-RESTORE, 04 PUF-SUPPORT-MERGE, 06 J-GRAPH-COMPATIBILITY, G RAW-BYTES-CODEC (graph piece), G STORE-FRAME-METADATA (graph piece) + +#### 02 GRAPH-RESTORE (1 non-test files, 0 test files) + +- `packages/microcosm-graph/src/microcosm/graph/`: `kernel.py` + +#### 03 ACCEPTED-SHARED-RESTORE (20 non-test files, 2 test files) + +- `packages/microcosm-build/src/microcosm/build/`: `frame_checkpoint.py` +- `packages/microcosm-build/src/microcosm/build/us_runtime/`: `acs_inputs.py`, `acs_pums.py`, `acs_transfer.py`, `asec_checkpoint.py`, `congressional_district_vintage.py`, `cps_carried.py`, `eligibility_inputs.py`, `hours_worked.py`, `housing_inputs.py`, `multispine_pool.py`, `operator_boundary.py`, `prior_year_income.py`, `puf_capital_gains_tail.py`, `qbi_inputs.py`, `relationship_inputs.py`, `spine_assembly.py`, `stacked_spine.py` +- `packages/microcosm-calibrate/src/microcosm/calibrate/`: `__init__.py` +- `packages/microcosm-fit/src/microcosm/fit/`: `qrf.py` +- tests: `test_us_acs_pums.py`, `test_us_asec_checkpoint.py` +- source imports from: 01 SAFE-ADDITIVE, 04 PUF-SUPPORT-MERGE, 05 SOLVE-MERGE-PROPOSAL, 06 J-GRAPH-COMPATIBILITY, P4 CALIBRATION-SOLVER-AND-DIAGNOSTICS +- tests import from: 01 SAFE-ADDITIVE, 06 J-GRAPH-COMPATIBILITY, P3 SURVEY-POPULATION-CATALOGUE-AND-BUDGET + +#### 04 PUF-SUPPORT-MERGE (1 non-test files, 0 test files) + +- `packages/microcosm-build/src/microcosm/build/us_runtime/`: `puf_support.py` +- source imports from: 01 SAFE-ADDITIVE, 03 ACCEPTED-SHARED-RESTORE + +#### 05 SOLVE-MERGE-PROPOSAL (1 non-test files, 0 test files) + +- `packages/microcosm-calibrate/src/microcosm/calibrate/`: `solve.py` +- source imports from: 01 SAFE-ADDITIVE + +#### 06 J-GRAPH-COMPATIBILITY (4 non-test files, 6 test files) + +- `packages/microcosm-build/src/microcosm/build/us_runtime/`: `__init__.py`, `puma_ladder.py`, `puma_ladder_sources.py` +- `tools/`: `build_us_puma_ladder_artifact.py` +- tests: `test_us_acs_multispine.py`, `test_us_acs_multispine_legacy_builder.py`, `test_us_base_pool.py`, `test_us_multispine_pool_tool.py`, `test_us_puma_ladder.py`, `test_us_puma_ladder_sources.py` +- tests import from: 03 ACCEPTED-SHARED-RESTORE, 04 PUF-SUPPORT-MERGE + +#### 07 F-CATALOGUE-OPTIMIZATION (1 non-test files, 3 test files) + +- `changelog.d/`: `us-survey-catalogue-digest.changed.md` +- tests: `test_us_survey_catalogue_digest.py`, `test_us_survey_catalogue_digest_authority.py`, `test_us_survey_catalogue_digest_fast_path.py` +- tests import from: 01 SAFE-ADDITIVE, 06 J-GRAPH-COMPATIBILITY + +#### 10 SOURCE-CLOSURE (9 non-test files, 0 test files) + +- `packages/microcosm-build/src/microcosm/build/us_runtime/`: `acs_2024_housing_universe.json`, `asec_current_money_consumers_v1.json`, `asec_current_money_domains_v1.json`, `asec_current_money_price_basis_v1.json`, `graph_acs_housing_universe.py`, `native_origin_graph_inventory.json`, `puf_2015_monetary_agi_source_projection.json`, `puf_2015_monetary_source_projection.json`, `puf_2015_raw_source_definition.json` +- source imports from: G RAW-BYTES-CODEC (graph piece) + +#### 11 PLACEMENT-ADDITIONS (1 non-test files, 1 test files) + +- `packages/microcosm-build/src/microcosm/build/us_runtime/`: `graph_full_puf_enrichment.py` +- tests: `test_us_graph_full_puf_enrichment.py` +- source imports from: 01 SAFE-ADDITIVE, 02 GRAPH-RESTORE, 03 ACCEPTED-SHARED-RESTORE, G RAW-BYTES-CODEC (graph piece), G STORE-FRAME-METADATA (graph piece) +- tests import from: 01 SAFE-ADDITIVE, 04 PUF-SUPPORT-MERGE, 06 J-GRAPH-COMPATIBILITY, G RAW-BYTES-CODEC (graph piece) + +#### 12 ORDINARY-CLOSURE (0 non-test files, 8 test files) + +- tests: `legacy_prepatch.json`, `test_group_bounds.py`, `test_grouped_fixed_support.py`, `legacy-slice-apply-v1.json`, `test_graph_legacy_matrix.py`, `test_graph_legacy_qrf.py`, `test_qrf_stateless.py`, `test_qrf_target.py` +- tests import from: 01 SAFE-ADDITIVE, 03 ACCEPTED-SHARED-RESTORE, 05 SOLVE-MERGE-PROPOSAL, G RAW-BYTES-CODEC (graph piece) + +#### 13 INTEGRATION-REGRESSIONS (2 non-test files, 3 test files) + +- `changelog.d/`: `us-launch-integration.fixed.md` +- `docs/`: `us-launch-review.md` +- tests: `test_us_geography_integration.py`, `test_us_runtime_facade_union.py`, `test_solve_us_launch_integration.py` +- tests import from: 01 SAFE-ADDITIVE, 02 GRAPH-RESTORE, 03 ACCEPTED-SHARED-RESTORE, 05 SOLVE-MERGE-PROPOSAL, 06 J-GRAPH-COMPATIBILITY, G RAW-BYTES-CODEC (graph piece) + +#### Already on main's way (amendment 20 / this lane's re-pins; not #893 content) (10 non-test files, 9 test files) + +- `./`: `PROGRESS-amendment-20-keyed-draws.md` +- `changelog.d/`: `amend-keyed-seed-and-uniform-draws.added.md` +- `docs/`: `graph-acceptance.md`, `graph-interface.lock` +- `experiments/`: `amendment-20-keyed-draws-receipts.md`, `us-native-atomic-financial-required-replay-20260910.json`, `us-puf-donor-boundaries-20260910.json`, `us-puf55-public-recipient-controls-20260910.json`, `us-puf55-two-route-numerical-20260910.json` +- `tools/`: `graph_parity_repin.py` +- tests: `test_us_puf55_model_boundaries.py`, `test_kernels.py`, `pins.json`, `pins.json`, `pins.json`, `test_graph_kernel_contract.py`, `test_graph_parity_pins.py`, `test_graph_parity_repin.py`, `test_graph_randomness.py` +- source imports from: G RAW-BYTES-CODEC (graph piece) +- tests import from: 01 SAFE-ADDITIVE, 03 ACCEPTED-SHARED-RESTORE, 06 J-GRAPH-COMPATIBILITY, G RAW-BYTES-CODEC (graph piece) + +#### G RAW-BYTES-CODEC (graph piece) (2 non-test files, 1 test files) + +- `packages/microcosm-graph/src/microcosm/graph/`: `__init__.py`, `codecs.py` +- tests: `test_graph_codecs.py` +- tests import from: G STORE-FRAME-METADATA (graph piece) + +#### G STORE-FRAME-METADATA (graph piece) (2 non-test files, 1 test files) + +- `changelog.d/`: `20260906-frame-metadata-storage.fixed.md` +- `packages/microcosm-graph/src/microcosm/graph/`: `store.py` +- tests: `test_frame_metadata_store.py` + +#### P1 POST-CLONE-ATOMIC-GEOGRAPHY (38 non-test files, 16 test files) + +- `changelog.d/`: `893-population-only-block-source.added.md`, `893-source-demographic-conditioning.added.md`, `893-survey-atomic-prefix.added.md`, `shared-atomic-geography.added.md`, `us-atomic-financial-composition.added.md`, `us-national-atomic-support.added.md` +- `experiments/`: `us-atomic-age-v2-1-controls-20260909.json`, `us-atomic-block-adapter-20260909.json`, `us-atomic-block-api-sources-35-controls-20260909.json`, `us-atomic-block-sources-36-controls-20260909.json`, `us-atomic-budget-semantic-8-controls-20260909.json`, `us-atomic-clone-graph-20260909.json`, `us-atomic-financial-4-controls-20260909.json`, `us-atomic-financial-composition-acceptance-20260909.json`, `us-atomic-financial-corrected-2-controls-20260909.json`, `us-atomic-national-acquisition-20260909.json`, `us-atomic-native-de-1-control-20260909.json`, `us-atomic-native-de-corrected-1-control-20260909.json`, `us-atomic-native-national-1-control-20260909.json`, `us-atomic-survey-population-controls-20260909.json`, `us-budget-predictor-compatibility-1-controls-20260909.json`, `us-financial-demographics-3-controls-20260909.json`, `us-postclone-geography-70-controls-20260910.json`, `us-puf55-native-donor-20260909.json`, `us-survey-geography-graph-controls-20260909.json`, `us-survey-geography-source-controls-20260909.json` +- `packages/microcosm-build/src/microcosm/build/`: `atomic_geography.py`, `graph_atomic_geography.py` +- `packages/microcosm-build/src/microcosm/build/us_runtime/`: `atomic_block_api_sources.py`, `atomic_block_sources.py`, `atomic_block_support.py`, `current_survey_geography.py`, `graph_atomic_survey_clone.py`, `graph_atomic_survey_financial.py`, `graph_atomic_survey_population.py`, `graph_current_survey_geography.py`, `survey_atomic_geography.py` +- `tools/`: `ci_test_groups.py` +- tests: `test_atomic_geography.py`, `test_us_atomic_block_api_sources.py`, `test_us_atomic_block_api_sources_native_de.py`, `test_us_atomic_block_api_sources_native_national.py`, `test_us_atomic_block_sources.py`, `test_us_atomic_block_support.py`, `test_us_atomic_survey_clone_graph.py`, `test_us_current_survey_geography.py`, `test_us_current_survey_predictor_demographics.py`, `test_us_graph_atomic_survey_financial.py`, `test_us_graph_atomic_survey_population.py`, `test_us_graph_current_survey_geography.py`, `test_us_postclone_geography_identity.py`, `test_us_survey_age_calibration_atomic_geography.py`, `test_us_survey_origin_budget_atomic_geography.py`, `test_us_survey_origin_budget_predictor_compatibility.py` +- source imports from: 01 SAFE-ADDITIVE, 02 GRAPH-RESTORE, G RAW-BYTES-CODEC (graph piece) +- tests import from: 01 SAFE-ADDITIVE, 03 ACCEPTED-SHARED-RESTORE, 04 PUF-SUPPORT-MERGE, 06 J-GRAPH-COMPATIBILITY, G RAW-BYTES-CODEC (graph piece), P2 PUF55-TWO-ROUTE-AND-CANONICAL-DONOR + +#### P2 PUF55-TWO-ROUTE-AND-CANONICAL-DONOR (51 non-test files, 21 test files) + +- `./`: `pyproject.toml` +- `changelog.d/`: `893-age-count-fixture.internal.md`, `893-diagnostic-refusal-context.internal.md`, `893-historical-zero-provenance.internal.md`, `puf55-canonical-donor.added.md`, `puf55-survey-social-security.added.md`, `survey-social-security-source.added.md`, `us-puf55-canonical-and-output-seals.added.md`, `us-puf55-survey-ss-measurement.added.md` +- `docs/`: `current-survey-puf59-progress.md`, `geography-assignment.md`, `puf2015-canonical59-and-growth.md`, `puf55-survey-ss-measurement-decision.md`, `survey-social-security-source.md`, `us-uk-release-path.md` +- `docs/evidence/puf2015-target2024/`: `GROWTH-RECIPE.json`, `INDEX-VALUES.json`, `NATIONAL-GROWTH-EXTRACT.json`, `PUBLIC-WORKBOOK-SOURCES.json` +- `experiments/`: `us-acs-compilation-cache-adoption-20260910.json`, `us-budget-detached-view-correction-20260910.json`, `us-financial-fixture-profile-20260910.json`, `us-financial-successor-positive-1-limit-stop-20260909.json`, `us-financial-successor-positive-20260910.json`, `us-financial-successor-remaining-controls-20260910.json`, `us-puf55-canonical-create-47-controls-20260910.json`, `us-puf55-numerical-output-seal-46-controls-20260910.json`, `us-puf55-population-controls-20260909.json`, `us-puf55-public-recipient-positives-20260910.json`, `us-puf55-survey-ss-measurement-31-controls-20260909.json`, `us-puf55-two-route-values-controls-20260910.json`, `us-survey-age-development-20260909.json` +- `packages/microcosm-build/src/microcosm/build/us_runtime/`: `asec_demographic_source.py`, `current_asec_demographics.py`, `current_social_security_source.py`, `current_survey_predictors.py`, `graph_current_survey_predictors.py`, `graph_puf55_canonical_donor.py`, `graph_puf55_survey_recipients.py`, `puf55_canonical_donor.py`, `puf55_route_finalization.py`, `puf55_survey_recipients.py`, `puf55_survey_ss_measurement.py`, `puf59_canonical.py`, `puf59_canonical_artifact.py`, `puf_full_source.py`, `puf_full_source_graph.py`, `puf_qbi_model.py`, `puf_target2024_growth.py`, `survey_financial_successor.py`, `survey_social_security.py` +- tests: `test_frame_checkpoint.py`, `test_spec_seed_diagnostic_refusal_context.py`, `test_us_acs_source_compile_cache.py`, `test_us_current_asec_demographics.py`, `test_us_current_survey_predictors.py`, `test_us_full_puf_output_profiles.py`, `test_us_graph_puf55_canonical_donor.py`, `test_us_graph_puf55_survey_recipients.py`, `test_us_graph_puf55_survey_ss.py`, `test_us_plan.py`, `test_us_puf55_route_finalization.py`, `test_us_puf55_route_numerical_finalization.py`, `test_us_puf55_survey_recipients.py`, `test_us_puf55_survey_ss_measurement.py`, `test_us_puf55_survey_ss_profile.py`, `test_us_puf59_canonical.py`, `test_us_puf_target2024_growth.py`, `test_us_runtime_import_order.py`, `test_us_survey_financial_successor.py`, `test_us_survey_origin_budget_detached_views.py`, `test_us_survey_social_security.py` +- source imports from: 01 SAFE-ADDITIVE, 03 ACCEPTED-SHARED-RESTORE, G RAW-BYTES-CODEC (graph piece) +- tests import from: 01 SAFE-ADDITIVE, 02 GRAPH-RESTORE, 03 ACCEPTED-SHARED-RESTORE, 04 PUF-SUPPORT-MERGE, 06 J-GRAPH-COMPATIBILITY, 11 PLACEMENT-ADDITIONS, G RAW-BYTES-CODEC (graph piece), P1 POST-CLONE-ATOMIC-GEOGRAPHY, P3 SURVEY-POPULATION-CATALOGUE-AND-BUDGET + +#### P3 SURVEY-POPULATION-CATALOGUE-AND-BUDGET (16 non-test files, 10 test files) + +- `changelog.d/`: `common-frame-export-contract.added.md`, `us-survey-copy-and-identity.fixed.md` +- `docs/`: `us-input-coverage-diagnostic.md` +- `experiments/`: `us-acs-code-memo-comparison-20260910.json`, `us-acs-code-memo-not-adopted-20260910.patch`, `us-acs-loaded-code-controls-20260910.json`, `us-common-frame-export36-20260910.json`, `us-input-coverage25-20260910.json`, `us-postclone-budget-replay-fence4-20260910.json`, `us-survey-catalogue-memo20-20260910.json`, `us-survey-numeric-diagnostics27-20260910.json` +- `packages/microcosm-build/src/microcosm/build/us_runtime/`: `asec_raw_stage_v4.py`, `common_frame_export_contract.py`, `input_coverage_profile.py`, `population_input_coverage.py` +- `packages/microcosm-frame/src/microcosm/frame/`: `bundle.py` +- tests: `test_us_acs_transfer.py`, `test_us_asec_catalogue_records_memo.py`, `test_us_asec_demographic_source.py`, `test_us_asec_household_coverage_fields.py`, `test_us_common_frame_export_contract.py`, `test_us_population_input_coverage.py`, `test_us_spine_blindness.py`, `test_us_survey_catalogue_immutable_memo.py`, `test_us_survey_frame_identity_encoding.py`, `test_us_survey_origin_budget_replay_producer.py` +- source imports from: 01 SAFE-ADDITIVE, 03 ACCEPTED-SHARED-RESTORE, 06 J-GRAPH-COMPATIBILITY, G RAW-BYTES-CODEC (graph piece), G STORE-FRAME-METADATA (graph piece) +- tests import from: 01 SAFE-ADDITIVE, 03 ACCEPTED-SHARED-RESTORE, 04 PUF-SUPPORT-MERGE, 06 J-GRAPH-COMPATIBILITY, G RAW-BYTES-CODEC (graph piece), P2 PUF55-TWO-ROUTE-AND-CANONICAL-DONOR + +#### P4 CALIBRATION-SOLVER-AND-DIAGNOSTICS (6 non-test files, 1 test files) + +- `changelog.d/`: `us-launch-store-grouped-validation.fixed.md`, `us-survey-diagnostic-options.fixed.md` +- `experiments/`: `us-calibration-consolidation-149-20260910.json`, `us-puf55-canonical-create-failed-20260910.json`, `us-puf55-canonical-cycle39-failed-20260910.json` +- `packages/microcosm-calibrate/src/microcosm/calibrate/`: `variable_labels.py` +- tests: `test_variable_labels.py` +- tests import from: 03 ACCEPTED-SHARED-RESTORE + +#### P5 CI-SEED-IDENTITY-AND-SPINE-BLINDNESS (10 non-test files, 10 test files) + +- `./`: `CLAUDE.md` +- `.github/workflows/`: `test.yml` +- `changelog.d/`: `893-diagnostic-current-lock.fixed.md`, `893-spec-identity-diagnostics.internal.md`, `893-worker-resource-trace.fixed.md` +- `docs/evidence/spec-engine/`: `us-f0-coverage.json` +- `experiments/`: `spec-seed-current-lock-20260910.json`, `us-acs-producer-profile-20260910.json` +- `packages/microcosm-build/src/microcosm/build/spec_engine/`: `inventory_coverage.py` +- `tools/`: `spec_seed_identity_diagnostics.py` +- tests: `test_spec_engine_country_bundles.py`, `test_spec_engine_loader.py`, `test_spec_seed_identity_cpu_bootstrap.py`, `test_spec_seed_identity_engine_parameter_files.py`, `test_spec_seed_identity_outer_failure.py`, `test_spec_seed_identity_owned_temp.py`, `test_spec_seed_identity_per_code_context.py`, `test_spec_seed_identity_system_metadata.py`, `test_us_puf_support.py`, `test_us_stacked_spine.py` +- tests import from: 01 SAFE-ADDITIVE, 03 ACCEPTED-SHARED-RESTORE, 04 PUF-SUPPORT-MERGE, 06 J-GRAPH-COMPATIBILITY + +#### P6 UK-ADAPTER-DOCS-AND-RECORDS (10 non-test files, 2 test files) + +- `./`: `.gitattributes` +- `changelog.d/`: `893-uk-atomic-area-and-lazy-exports.added.md` +- `docs/`: `validation-cost-and-borrow-boundaries.md` +- `experiments/`: `uk-atomic-area-lazy52-20260910.json`, `us-financial-default-2-controls-20260909.json`, `us-native-atomic-financial-cold-20260910.json`, `us-native-postclone-financial-incomplete-20260910.json` +- `packages/microcosm-build/src/microcosm/build/uk_runtime/`: `__init__.py`, `atomic_area_support.py`, `atomic_household_identity.py` +- tests: `test_uk_atomic_area_support.py`, `test_uk_runtime_lazy_exports.py` +- source imports from: P1 POST-CLONE-ATOMIC-GEOGRAPHY +- tests import from: P1 POST-CLONE-ATOMIC-GEOGRAPHY + +**Layer-01 modules with no cross-layer import at all (53; `randomness.py` among them is amendment 20, not #893 content):** `__init__.py`, `canonical.py`, `origin.py`, `protocol.py`, `reasons.py`, `survey_allocation.py`, `survey_domain_sample.py`, `table_identity.py`, `_asec_current_money_codec.py`, `_person_signal_summary.py`, `acs_housing_universe.py`, `acs_native_coverage_binding.py`, `acs_person_coverage_authentication.py`, `acs_person_coverage_columns.py`, `acs_population_catalogue.py`, `asec_coverage_authentication.py`, `asec_current_money.py`, `asec_current_money_graph_resources.py`, `asec_current_money_resources.py`, `asec_current_money_units.py`, `asec_household_coverage_fields.py`, `asec_housing_status.py`, `asec_housing_status_source.py`, `asec_housing_universe.py`, `asec_housing_universe_source.py`, `asec_income_observations.py`, `asec_original_household_weights.py`, `asec_person_coverage_source.py`, `asec_population_catalogue.py`, `asec_prepared_source.py`, `asec_student_controls.py`, `cd_reference.py`, `cd_reference_sources.py`, `cps_carried_current.py`, `graph_implementation.py`, `graph_implementation_inventory.json`, `graph_native_origin_implementation.py`, `native_household_origin.py`, `operator_column_contracts.py`, `puf_detail_transfer.py`, `puf_diagnostic_consumer.py`, `puf_growth.py`, `puf_price_baseline.py`, `reported_coverage_source.py`, `source_csv_builtin.py`, `survey_age_activation.py`, `survey_age_sources.py`, `survey_catalogue_selection.py`, `survey_observed_age.py`, `survey_population_domains.py`, `group_bounds.py`, `_policyengine_us_source_index.py`, `randomness.py` + + +**Layer-01 modules with a cross-layer import (48; 27 of them only on the graph pieces G1/G2, the other 21 soft on modified modules of 03/05/06/P3/P4 — see `pr1-refined.json`):** `acs_housing_universe_source.py`, `asec_2024_native_population.py`, `asec_current_money_selection.py`, `asec_current_money_source.py`, `asec_engine_evaluation.py`, `asec_household_observations.py`, `asec_person_income_source.py`, `demographic_calibration_graph.py`, `full_puf_enrichment.py`, `graph_asec_income.py`, `graph_asec_prepared.py`, `graph_combined_clone.py`, `graph_composed_asec_binding.py`, `graph_composed_asec_measures.py`, `graph_composed_contracts.py`, `graph_composed_population.py`, `graph_context.py`, `graph_current_survey_puf_transfer.py`, `graph_geography.py`, `graph_housing_universe.py`, `graph_national_age_counts.py`, `graph_native_household_origin.py`, `graph_puf_detail_transfer.py`, `graph_puf_diagnostic_consumer.py`, `graph_sources.py`, `graph_survey_age_artifact.py`, `graph_survey_budget.py`, `graph_survey_calibration.py`, `graph_survey_population.py`, `national_age_activation.py`, `puf_growth_graph.py`, `puf_monetary_agi_projection.py`, `puf_monetary_source.py`, `puf_raw_source.py`, `survey_age_calibration.py`, `survey_calibration_diagnostics.py`, `survey_origin_budget.py`, `survey_population_preparation.py`, `survey_population_replay.py`, `_graph_legacy_apply.py`, `_graph_legacy_qrf.py`, `graph_legacy_apply.py`, `graph_legacy_apply_matrix.py`, `graph_legacy_qrf.py`, `graph_legacy_train.py`, `model_input.py`, `qrf_target.py`, `policyengine_us.py` + +**Layer-01 tests (all ride with PR-1; 26):** `test_us_acs_housing_source.py`, `test_us_acs_person_coverage_authentication.py`, `test_us_acs_person_coverage_columns.py`, `test_us_acs_population_catalogue.py`, `test_us_asec_2024_native_population.py`, `test_us_asec_coverage_authentication.py`, `test_us_asec_current_money_source.py`, `test_us_asec_person_income_source.py`, `test_us_current_survey_puf_host.py`, `test_us_current_survey_puf_transfer.py`, `test_us_full_puf_enrichment.py`, `test_us_graph_survey_population.py`, `test_us_national_age_counts.py`, `test_us_puf_detail_transfer.py`, `test_us_puf_price_baseline.py`, `test_us_survey_age_activation.py`, `test_us_survey_age_artifact.py`, `test_us_survey_age_calibration_run.py`, `test_us_survey_age_development.py`, `test_us_survey_age_sources.py`, `test_us_survey_calibration.py`, `test_us_survey_observed_age.py`, `test_us_survey_origin_budget.py`, `test_us_survey_population_preparation.py`, `test_us_survey_population_replay.py`, `test_policyengine_us_ownership_index.py` + +## 7. Draft charter amendment entries (one per kept graph piece) + +Written in the style of `docs/graph-acceptance.md` "Interface freeze". None of +the four touches `decl.py` or `kernel.py`, so none re-records the lock; they +are offered as numbered entries because each changes what a receipt, a store +object or an executor outcome means, which the charter records. + +**7.1 Raw-byte sources.** A source codec is registered in exactly one of two +modes and a name belongs to at most one: *Frame mode* (`SourceCodec`, +`SourceCodecRegistry.register` / `load`) decodes a source into a population +`Frame`, and *raw-byte mode* (`SourceBytesCodec`, `register_bytes` / +`load_bytes`, `load_source_bytes`) decodes it into immutable `bytes` that the +consuming kernel alone interprets. Neither mode can be loaded through the +other: the mismatch is a `TypeError` naming the mode the codec has, so no +caller receives bytes where it declared a Frame. The shipped `raw-bytes-v1` +codec reads one regular file, bounded at `RAW_BYTES_MAX_BYTES` (64 MiB) and +opened non-blocking so a directory, FIFO, device or socket is refused from +the descriptor's own mode. Identity, the pre-run content key and the post-run +mutation check stay in the executor (E1, E3 unchanged; C3's "declared +predecessors only" is untouched because a source is not a node). Why: a +lookup NPZ or a crosswalk CSV that an import kernel turns into a typed +`ArtifactOutput` (amendment 19) is not a population, and the only way to read +it before this was a Frame codec registered as a pretence. Raised by the US +launch integration branch (thirteen consumers); no key moves, because +`SourceRef.codec` was already normative and `raw-bytes-v1` is a new name. + +**7.2 Frames keep their metadata across the store.** A frame object is +written as `microcosm-graph-frame-v2`: `Frame.metadata` is encoded +losslessly (mappings, tuples, frozensets, `float64` bit patterns, scalars) +into the frame manifest, its SHA-256 is recorded in the object header, and +`_read_frame` refuses a mismatch as `StoreCorruptError`. A v1 object is +`StoreUnavailableError` on load rather than a Frame with its metadata silently +dropped, and writing a frame under a key that already holds one with different +metadata is `StoreCorruptError` (E1: a key identifies exactly one content). +The JSON decode boundary refuses `NaN`/`Infinity` and overflowing numerals, +because every store JSON is written with `allow_nan=False`, so a corrupt +manifest is `StoreCorruptError` at the boundary rather than a `TypeError` +from the canonical re-encoder (E2). Why: the US survey stages carry source +provenance in `Frame.metadata` and replay verifiers compare it +(`same_replayed_frame`); until now a population that went through the store +came back without it. No node key moves: the frame key is the node's, and +metadata was never part of a key. Raised by the US launch integration +branch (`8d9f62c40`); acceptance items E1 and E2. + +**7.3 Execution states: a gate exception leaves its consumers unreached.** +Amendment 19 refused a gate kernel that declares a typed artifact output +because it had no regime for an output a node was unable to produce. This +entry supplies the regime and lifts the refusal. The executor owns an +execution state (`"microcosm.graph.execution.v1"`) that a kernel may not +author: a gate that declares typed outputs and raises still becomes a `fail` +verdict and the run continues (amendment 7, now literally true for every +node shape), and its receipt records `gate_exception` with the outputs it +could not produce; a node whose predecessor is `unreached`, or whose declared +byte input its producer recorded as unavailable, is itself recorded +`unreached` with its blockers named by node key — it runs no kernel and +invents no column, frame, weight or byte (F1: nothing is fabricated); a +release behind such an edge derives its tier from the same ancestry and stays +`evidence` (F2). Cache records that carry a state use schema 3; a cached +unreached record is a hit only while the same inputs are unavailable for the +same reason and is `StoreCorruptError` once they exist (E3); `require` +preflight derives blockers from the cached parents. A run manifest carrying +any state serializes at schema 4 and authenticates every blocker on load +(F5, E4). No node key moves: states live in receipts and records, never in a +key. Why: the US post-clone geography gate emits a typed validation artifact +that the financial stages consume, so a geography failure must leave those +stages unreached rather than abort the run or launder the gate into a compute +node. Raised by the US launch integration branch; supersedes the interim +ruling recorded in amendment 19. + +**7.4 A private population observer.** `run_graph` accepts +`_population_observer`, a callable handed each node's admitted population +(design anchors included) after the result is applied and before the node is +persisted, on cold execution and on cache hits alike; it must not mutate the +population, an exception it raises refuses the run, and an unreached node +has no population to observe. It is an integration seam for verifiers (the +US survey stages assert design-weight provenance through it), not a kernel +capability: it enters no key, receipt or record (A1, B4 unchanged). Raised by +the US launch integration branch. + +## 8. Decisions needed from Max + +1. **Piece C supersedes an amendment-19 ruling.** The Amendment 19 lane + recorded "the gate-artifact-output refusal is the one interim ruling this + lane made on his behalf". This branch's US runtime needs the opposite (the + post-clone geography gate declares its validation artifact and three + stages consume it), so piece C re-applies the branch's execution-state + regime and replaces main's test that pins the refusal with five tests of + the new regime. Both the frozen files and every `test_acceptance_*` file + are untouched, but the executor's behaviour on that one node shape now + differs from what the charter's amendment 19 text says. Landing PR-G3 + requires adopting entry 7.3 (or an edited form) in the charter; until then + this branch's executor and the charter disagree on that sentence. +2. **Lazy population retention** (`attachments.py`, layer 08) is dropped for + lack of a consumer. If the native pilot's memory ceiling is a reason to + keep it, it is a self-contained PR on top of PR-G2. +3. **`_stream_file` and the `_write_node` refactor** are dropped for lack of a + consumer. Both are memory-only; say if either should come back as its own + small PR. +4. **The report's home.** `out.md` at the repo root is a tracked file holding + the F1 Sol gate round-1 report (2026-09-04). This report is written there + for the dispatcher but is **not committed** there; the committed copy is + `experiments/893-reconciliation-amendments-19-20-20260912.md`. +5. **Three US stages name a resource that does not exist anywhere.** + `asec_prepared_v3`, `composed_asec_binding_v1` and `composed_population_v1` + list `us_runtime/asec_current_money_engine_defaults_v1.json` in their + resources; the file is on neither this branch (at `069d5ed9a` or now) nor + `origin/main` (it appears only in two commits of another branch, + `b0e7f54b2` / `b6cfc84b0`). `implementation_manifest` for those stages has + therefore been failing since before this lane. Not invented here; the + owner of layer 10 SOURCE-CLOSURE has to admit the file or drop the entry, + and the inventory's "scope review" rule applies to `07566eb17` as well. +6. **Two build tests fail locally for reasons outside this lane** if they are + run: `test_release_target_parity.py`'s two `_feed_or_skip` tests need a 131 + MB feed outside the repository (already recorded by the Amendment 19 lane). + They were not part of this lane's runs. + +## 9. Where things are + +- Journal: `PROGRESS.md` (top section, cumulative file). +- This report: `out.md` (uncommitted, the `-o` path) and + `experiments/893-reconciliation-amendments-19-20-20260912.md` (committed). +- Scratch evidence (session-local, not committed): the AST consumer scan, + the layer scripts and JSON, and every gate log named in section 5. diff --git a/experiments/signed-income-reconciliation-20260912.md b/experiments/signed-income-reconciliation-20260912.md new file mode 100644 index 000000000..4d5b6453e --- /dev/null +++ b/experiments/signed-income-reconciliation-20260912.md @@ -0,0 +1,85 @@ +# Deterministic reconciliation to a signed income total + +`microcosm.fit.signed_reconciliation.reconcile_signed_total` projects a joint +draw onto a caller-qualified total. It is a separate numeric operation for a +future graph node. No existing model, source qualifier, financial attachment, +PUF host or native candidate invokes it in this change. + +For draw `q`, anchor `A`, strictly positive scales `s`, and a declared set `B` of +nonnegative components, it solves the strictly convex problem + +```text +minimize sum_j ((z_j - q_j) / s_j)^2 +subject to sum_j z_j = A + z_j >= 0 for j in B +``` + +At least one component must remain unrestricted, making every finite signed +anchor feasible in exact arithmetic. Component names, the bound mask and scales +are explicit parameters. Ordinary interest and retirement-account interest can +therefore be separate components; this operator assigns neither a source meaning +nor tax treatment to either. It introduces no nonzero-presence restriction. + +Define `w_j = (s_j / max(s))^2`. Uniform scale normalization preserves the +minimizer. Given an active bound set `C`, the equality solution for the remaining +free coordinates `F` is + +```text +t = (A - sum_{j in F} q_j) / sum_{j in F} w_j +z_j = q_j + t*w_j for j in F +z_j = 0 for j in C +``` + +Start with every coordinate free. Fix any bounded coordinate with a negative +candidate to zero, then solve again. Removing negative candidates can only lower +`t`, so a fixed coordinate cannot need releasing later. An unrestricted +coordinate always remains free. The algorithm therefore terminates after at +most `len(B) + 1` solves, without a general optimizer or random draws. The final +free-coordinate stationarity and active-bound dual feasibility are checked. + +The signed anchor is never clamped or used as a divisor. A zero total does not +force its components to zero. For example, `[8, 4, -3]` with unit scales and a +zero anchor becomes `[5, 1, -6]`. Positive income and offsetting property loss +remain possible for negative, zero or positive totals. An exactly feasible draw +is preserved, including its float64 signed-zero bits. + +Draws have shape `(..., k)` and anchors have exactly the batch shape `(...)`. +Scales and the boolean bound mask each have shape `(k,)`; broadcasting an anchor +across records is deliberately not implicit. Empty batches are supported. +Inputs are finite real numeric values converted into detached float64 arrays. +The result preserves the raw draw, anchor, component roster and scales, together +with projected values, adjustment vectors, signed sum residuals, active bounds, +normalized shifts, objectives and KKT diagnostics. These arrays are descriptive, +mutable values, not an authority or immutable execution receipt. + +Callers must supply finite nonnegative `atol` and `rtol`. The sum check uses +`abs(residual) <= atol + rtol*abs(A)`. Component stationarity and dual checks use +the same form with the maximum absolute draw, result and proposed adjustment. +The sum uses `math.fsum`; no residual is patched into a final component and no +display rounding changes the optimum. Nonfinite input, intermediate or objective +overflow, a normalized scale weight lost to underflow, or failure of the stated +tolerances causes refusal. Floating-point refusal is possible even when an +exact-arithmetic solution exists. + +The 45-case guarded suite passed with zero failures/errors/skips in 4.470 seconds +wall time, 2.899 CPU seconds and 405,143,552 bytes peak RSS. This includes 160 +randomized small problems compared against an independent oracle that enumerates +bound faces, solves dense block KKT systems and selects the minimum objective. +It also verifies negative and zero-net totals, loss offsets, unequal scales, +scale-rescaling invariance, active bounds, exact feasible identity, detached +storage, alternate component rosters, multidimensional and empty batches, invalid +inputs and unstable arithmetic. All 990 source-plus-owned hashes and 15 allowed +resource hashes were unchanged; no child or unexpected refusal occurred. The +normal guard-induced dateutil zoneinfo warning occurred. + +Production source SHA256: +`d56c21959121cc7500f06cd747c74bea51561deba537ce2c58266d5966ae84b2`. +Test SHA256: `b5ee23c6ff4f633434c96a9e17c8b82b254435757b6eb24061b1835f57bdaa77`. +The local `codex-signed-reconciliation-20260912/numeric-v2/numeric-v2.json` receipt +SHA256 is `2f071f5116489c1febb4f594c97781c9b8d88d68cf125e94aec446d80e7c3321`. +The earlier guard preflight failure is preserved; it executed no numeric tests. + +This numerical acceptance does not establish ACS/ASEC measurement equivalence, +resolve the property or retirement donor bridge, approve scale choices, fit a +model, or certify a release. Source qualification, donor definitions, a named +reconciliation node and dependent model replay remain separate work. diff --git a/experiments/spec-seed-current-lock-20260910.json b/experiments/spec-seed-current-lock-20260910.json new file mode 100644 index 000000000..0005829fa --- /dev/null +++ b/experiments/spec-seed-current-lock-20260910.json @@ -0,0 +1,22 @@ +{ + "approved_lock_sha256": "751d5ef5d25406bbae1798667f0e29890d4aad933d323c12912c7d45d8809bb9", + "before_lock_sha256": "4ef1ef6eb39b65ebc47c2c00bafa44e7c872544b1b0146dd8493bcfe45b2da1b", + "ci_job": "https://github.com/PolicyEngine/microcosm/actions/runs/34506624007/job/102970618835", + "data_or_release_acceptance": false, + "diagnostic_rerun": "pending CI on corrected source", + "diagnostic_sha256": "18468f5319324169896942230ca3584188c9840fff03e913aaf16e8360adf4ef", + "local_subject_execution": false, + "matches_existing_worker_approval": true, + "observed_refusal": { + "code": "LOCK", + "completed": false, + "coverage_pass": false, + "phase": "environment" + }, + "only_lock_delta": "Declare existing openpyxl package in microcosm-build[uk] dependency and requires-dist; no new resolved package or version change", + "package_count_before_after": 125, + "resolved_package_versions_unchanged": true, + "scope": "Source-only audit of current-lock diagnostic correction", + "source_change": "One literal, retaining exact lock equality; no allowance for arbitrary locks", + "upstream_worker_approval_commit": "0d978f20bf4d1d899b707fe7d33ab084c153ee4f" +} diff --git a/experiments/uk-atomic-area-lazy52-20260910.json b/experiments/uk-atomic-area-lazy52-20260910.json new file mode 100644 index 000000000..076f3a32f --- /dev/null +++ b/experiments/uk-atomic-area-lazy52-20260910.json @@ -0,0 +1,37 @@ +{ + "before_after_current_frozen_and_maintained_source_equal": true, + "country_distribution_paths_dereferenced_by_postchecker": false, + "cpu_seconds": 2.335378, + "guard_policy_passed": true, + "model_resources": 0, + "model_source_files": 0, + "native_payloads_read": 0, + "physical_postcheck": "pass", + "physical_postcheck_sha256": "dc9ca4ad0c4cd77f40a4a0affb12487975d88a6e9d0e7ff2a8e41011e4430071", + "receipt_sha256": "b6abf03c647e79415f593a7f8a2cfc933b6ac1a47cf92ba552f2e0cf38d3a838", + "release_acceptance": false, + "resources": 0, + "reviewer": "root", + "scope": "Invented three-system UK atomic-area adapter and ordinary lazy-package behavior; no native UK population or calibration acceptance", + "source_count": 64, + "source_files": { + "packages/microcosm-build/src/microcosm/build/uk_runtime/__init__.py": "accb5208bfc22f0d2905dad12ad390b416f424506e61c417d169215fe4bad78d", + "packages/microcosm-build/src/microcosm/build/uk_runtime/atomic_area_support.py": "b2006451f4afcbc2c5192eb421dd7c94d11262889c0a10aa2d1040f1fa28ec8e", + "packages/microcosm-build/src/microcosm/build/uk_runtime/atomic_household_identity.py": "8d2164ae90eb521ee838936c6ca17e1ce08f43650889ea14afc9bcb82301a325", + "packages/microcosm-build/src/microcosm/build/uk_runtime/content_identity.py": "589002ba67e1c669f40b13968f16ee439937bfbd7e36d6ad4ac9ee195af2b3e5", + "packages/microcosm-build/src/microcosm/build/uk_runtime/rowwise_geography.py": "5b7287cd228fa7f5385cc19e6ce73184749b3c8b655953440d7273f86f3eabb3", + "packages/microcosm-build/tests/test_uk_atomic_area_support.py": "2109e447bf9446b6c4047bc742f0d9c1ed93f89b926fb9be335ae9567292759a", + "packages/microcosm-build/tests/test_uk_runtime_lazy_exports.py": "207f5ba4b21737902e42ed8f72f3251425f1aa68959309d3f2ff0345f0f4296b" + }, + "source_map_sha256": "e7c59f91af92643378cefffb2a59be700d1940d3b3dc5aadfb4a4bf0df8e9eff", + "source_plus_owned": 73, + "suite_accepted": true, + "supplied_geography_conventions": [ + "EW OA 2021", + "Scotland OA 2022", + "Northern Ireland Data Zone 2021" + ], + "tests_passed": 52, + "wall_seconds": 3.199442916957196, + "xml_sha256": "34a5afffba8a3f7cc8933e306da19e76c8b597c9d31cffbadfeda902a1edbf8d" +} diff --git a/experiments/us-acs-code-memo-comparison-20260910.json b/experiments/us-acs-code-memo-comparison-20260910.json new file mode 100644 index 000000000..1cdec2fc8 --- /dev/null +++ b/experiments/us-acs-code-memo-comparison-20260910.json @@ -0,0 +1,217 @@ +{ + "baseline": { + "cpu_seconds": 303.897467, + "peak_rss_bytes": 594296832, + "wall_seconds": 308.2577062920318 + }, + "baseline_receipt_sha256": "784d46a3a5ed3994f1018f6d05900122abdc6f63a0f18e318002fedfe7059af9", + "correctness_controls": 18, + "cpu_ratio": 1.0405705290067455, + "date": "2026-09-10", + "decision": "not_adopted", + "experiment_patch_sha256": "bc2b245b712256578ad6587c7a242af38e4244eaeacc268b589b2ed0c38ae557", + "guard_sha256": "8a2cf87bd0a00c1de1d8e4263764394551efed8e1ecb3d3297eeb5b1ab0eaf5e", + "implementation_restored_to_sha256": "db7aaae06961cc1ed9ce13b28dac74d8fd1041b9cd4c32294d2b11105fbef7af", + "native_speedup_established": false, + "optimized": { + "cpu_seconds": 316.226748, + "peak_rss_bytes": 601653248, + "wall_seconds": 319.78662233403884 + }, + "optimized_receipt_sha256": "49016ede5317b1e025e6d476364398b8233e3c589b7b80ef78781d154ab9f0d0", + "optimized_source_sha256": "4dc64558ed453f1c06557ef005e42193278496a9c31b374475417c407c037573", + "profile_and_original_teardown_passed": true, + "reason": "No measured speed improvement in the same instrumented fixture; added eligibility work exceeded the comparison savings.", + "release_acceptance": false, + "scope": "one paired invented cold/required fixture run, identical cProfile policy and dimensions", + "successor_controls_executed": false, + "timing_limitations": [ + "cProfile overhead changes elapsed time and can trigger the unchanged limits", + "partial snapshots omit unfinished active-call time until profiler flush", + "only exact frozen-source coordinates are reported; external time remains within cumulative callers", + "cumulative function times overlap and must not be summed" + ], + "top_optimized_records": [ + { + "calls": 1, + "cumulative_seconds": 309.704728666, + "file": "packages/microcosm-build/tests/test_us_graph_atomic_survey_financial.py", + "function": "known_financial_run", + "line": 19, + "recursive_calls": 0, + "self_seconds": 0.0052826240000000005 + }, + { + "calls": 2, + "cumulative_seconds": 308.7401165, + "file": "packages/microcosm-build/src/microcosm/build/us_runtime/graph_atomic_survey_financial.py", + "function": "run_atomic_survey_financial", + "line": 538, + "recursive_calls": 0, + "self_seconds": 0.005962359 + }, + { + "calls": 110, + "cumulative_seconds": 224.98760203900002, + "file": "packages/microcosm-build/src/microcosm/build/us_runtime/survey_population_preparation.py", + "function": "_validate", + "line": 1071, + "recursive_calls": 0, + "self_seconds": 0.00429048 + }, + { + "calls": 108, + "cumulative_seconds": 220.871511795, + "file": "packages/microcosm-build/src/microcosm/build/us_runtime/survey_population_preparation.py", + "function": "_checked", + "line": 1091, + "recursive_calls": 0, + "self_seconds": 0.000492867 + }, + { + "calls": 652, + "cumulative_seconds": 195.60535725600002, + "file": "packages/microcosm-build/src/microcosm/build/us_runtime/acs_native_coverage_binding.py", + "function": "_producer", + "line": 223, + "recursive_calls": 0, + "self_seconds": 4.97942924 + }, + { + "calls": 14, + "cumulative_seconds": 169.824184456, + "file": "packages/microcosm-build/src/microcosm/build/us_runtime/survey_atomic_geography.py", + "function": "reconstruct_atomic_survey_geography", + "line": 307, + "recursive_calls": 0, + "self_seconds": 0.006830769 + }, + { + "calls": 38464, + "cumulative_seconds": 168.34809192, + "file": "packages/microcosm-build/src/microcosm/build/us_runtime/acs_native_coverage_binding.py", + "function": "_live_code", + "line": 137, + "recursive_calls": 0, + "self_seconds": 8.731628122 + }, + { + "calls": 8, + "cumulative_seconds": 139.864694334, + "file": "packages/microcosm-build/src/microcosm/build/us_runtime/current_survey_predictors.py", + "function": "qualify_current_survey_predictors", + "line": 304, + "recursive_calls": 0, + "self_seconds": 0.0066878350000000005 + }, + { + "calls": 2, + "cumulative_seconds": 119.01038833400001, + "file": "packages/microcosm-build/src/microcosm/build/us_runtime/graph_atomic_survey_population.py", + "function": "run_atomic_survey_population", + "line": 213, + "recursive_calls": 0, + "self_seconds": 0.001211868 + }, + { + "calls": 6, + "cumulative_seconds": 116.761678708, + "file": "packages/microcosm-graph/src/microcosm/graph/executor.py", + "function": "run_graph", + "line": 2158, + "recursive_calls": 0, + "self_seconds": 0.007467228 + }, + { + "calls": 15, + "cumulative_seconds": 115.40236200000001, + "file": "packages/microcosm-build/src/microcosm/build/us_runtime/current_survey_geography.py", + "function": "qualify_current_survey_geography", + "line": 220, + "recursive_calls": 0, + "self_seconds": 0.0036097990000000003 + }, + { + "calls": 2010412, + "cumulative_seconds": 114.621920672, + "file": "packages/microcosm-build/src/microcosm/build/us_runtime/acs_native_coverage_binding.py", + "function": "check", + "line": 176, + "recursive_calls": 51772, + "self_seconds": 86.630530445 + }, + { + "calls": 348, + "cumulative_seconds": 111.58561966500001, + "file": "packages/microcosm-build/src/microcosm/build/us_runtime/acs_population_catalogue.py", + "function": "_producer", + "line": 102, + "recursive_calls": 0, + "self_seconds": 0.135475169 + }, + { + "calls": 150, + "cumulative_seconds": 98.955415883, + "file": "packages/microcosm-build/src/microcosm/build/us_runtime/acs_native_coverage_binding.py", + "function": "verify_acs_native_coverage", + "line": 490, + "recursive_calls": 0, + "self_seconds": 0.010436341 + }, + { + "calls": 8, + "cumulative_seconds": 98.24177225000001, + "file": "packages/microcosm-build/src/microcosm/build/us_runtime/survey_origin_budget.py", + "function": "_initial", + "line": 355, + "recursive_calls": 0, + "self_seconds": 0.0019372500000000002 + }, + { + "calls": 116, + "cumulative_seconds": 75.394407754, + "file": "packages/microcosm-build/src/microcosm/build/us_runtime/acs_population_catalogue.py", + "function": "_checked", + "line": 406, + "recursive_calls": 0, + "self_seconds": 0.003720891 + }, + { + "calls": 110, + "cumulative_seconds": 71.55709778800001, + "file": "packages/microcosm-build/src/microcosm/build/us_runtime/acs_population_catalogue.py", + "function": "verify_acs_source_catalogue", + "line": 430, + "recursive_calls": 0, + "self_seconds": 0.000156618 + }, + { + "calls": 4, + "cumulative_seconds": 70.568924458, + "file": "packages/microcosm-build/src/microcosm/build/us_runtime/graph_current_survey_predictors.py", + "function": "_qualified", + "line": 269, + "recursive_calls": 0, + "self_seconds": 0.0008037900000000001 + }, + { + "calls": 23, + "cumulative_seconds": 52.046634627, + "file": "packages/microcosm-build/src/microcosm/build/us_runtime/current_asec_demographics.py", + "function": "qualify_current_asec_demographics", + "line": 194, + "recursive_calls": 0, + "self_seconds": 0.003941233000000001 + }, + { + "calls": 2, + "cumulative_seconds": 42.917792458, + "file": "packages/microcosm-build/src/microcosm/build/us_runtime/graph_survey_population.py", + "function": "run_authenticated_survey_population", + "line": 815, + "recursive_calls": 0, + "self_seconds": 0.0007081860000000001 + } + ], + "wall_ratio": 1.037400252472796 +} diff --git a/experiments/us-acs-code-memo-not-adopted-20260910.patch b/experiments/us-acs-code-memo-not-adopted-20260910.patch new file mode 100644 index 000000000..246a86e19 --- /dev/null +++ b/experiments/us-acs-code-memo-not-adopted-20260910.patch @@ -0,0 +1,497 @@ +--- a/packages/microcosm-build/src/microcosm/build/us_runtime/acs_native_coverage_binding.py ++++ b/packages/microcosm-build/src/microcosm/build/us_runtime/acs_native_coverage_binding.py +@@ -48,13 +48,103 @@ + raise ACSNativeCoverageBindingError(code) + + +-def _live_code(module, compiled): ++def _immutable_code_constants(value, immutable_codes, depth=0): ++ """Recognize exact immutable compiler values without invoking custom equality. ++ ++ Runtime-created CodeType constants need not be compiler values. Unfamiliar ++ values and deeply nested trees take the original uncached equality path; ++ eligibility is never a new source-acceptance requirement. Positive code ++ entries retain the object itself so integer identity keys cannot be reused. ++ """ ++ if depth > 64: ++ return False ++ kind = type(value) ++ if any( ++ kind is scalar ++ for scalar in ( ++ type(None), ++ bool, ++ int, ++ float, ++ complex, ++ str, ++ bytes, ++ type(Ellipsis), ++ ) ++ ): ++ return True ++ if kind is tuple or kind is frozenset: ++ return all( ++ _immutable_code_constants(item, immutable_codes, depth + 1) ++ for item in value ++ ) ++ if kind is not CodeType: ++ return False ++ if immutable_codes.get(id(value)) is value: ++ return True ++ # Check all stored public CodeType fields, not merely co_consts: runtime ++ # constructors may admit scalar subclasses with custom comparison methods. ++ if not all( ++ type(getattr(value, name)) is int ++ for name in ( ++ "co_argcount", ++ "co_posonlyargcount", ++ "co_kwonlyargcount", ++ "co_nlocals", ++ "co_stacksize", ++ "co_flags", ++ "co_firstlineno", ++ ) ++ ) or not all( ++ type(getattr(value, name)) is str ++ for name in ("co_filename", "co_name", "co_qualname") ++ ): ++ return False ++ if not all( ++ type(getattr(value, name)) is bytes ++ for name in ("co_code", "co_linetable", "co_exceptiontable") ++ ): ++ return False ++ for name in ("co_names", "co_varnames", "co_freevars", "co_cellvars"): ++ items = getattr(value, name) ++ if type(items) is not tuple or any(type(item) is not str for item in items): ++ return False ++ if type(value.co_consts) is not tuple or not _immutable_code_constants( ++ value.co_consts, immutable_codes, depth + 1 ++ ): ++ return False ++ immutable_codes[id(value)] = value ++ return True ++ ++ ++def _same_loaded_code(live, reference, code_pairs, immutable_codes): ++ """Memoize equality only for strongly retained, immutable code pairs.""" ++ if type(live) is not CodeType or type(reference) is not CodeType: ++ return live == reference ++ key = (id(live), id(reference)) ++ retained = code_pairs.get(key) ++ if retained is not None and retained[0] is live and retained[1] is reference: ++ return True ++ eligible = _immutable_code_constants( ++ live, immutable_codes ++ ) and _immutable_code_constants(reference, immutable_codes) ++ matches = live == reference ++ if matches and eligible: ++ code_pairs[key] = (live, reference) ++ return matches ++ ++ ++def _live_code(module, compiled, code_pairs=None, immutable_codes=None): + """Check loaded Python implementations against their current source bytes. + + Covers module functions, imported function aliases and source class methods; + generated dataclass methods have no source code object. This is a drift + check in a trusted process, not a sandbox against arbitrary Python execution. + """ ++ if code_pairs is None: ++ code_pairs = {} ++ if immutable_codes is None: ++ immutable_codes = {} + path = getattr(module, "__file__", None) + if path is None: + return +@@ -102,15 +192,19 @@ + + visit(compile(Path(path).read_bytes(), path, "exec", dont_inherit=True)) + compiled[path] = codes ++ live = function.__code__ ++ _require(function.__globals__ is vars(origin), "LOADED_PRODUCER") + _require( +- function.__globals__ is vars(origin) +- and function.__code__ == compiled[path].get(function.__code__.co_qualname), ++ _same_loaded_code( ++ live, compiled[path].get(live.co_qualname), code_pairs, immutable_codes ++ ), + "LOADED_PRODUCER", + ) + + for cell in function.__closure__ or (): + if isinstance(cell.cell_contents, FunctionType): + check(cell.cell_contents) ++ _require(function.__code__ is live, "LOADED_PRODUCER") + + for value in vars(module).values(): + if isinstance(value, FunctionType): +@@ -147,10 +241,12 @@ + name = package + "." + relative.removesuffix(".py").replace("/", ".") + names.add(name.removesuffix(".__init__")) + compiled = {} ++ code_pairs = {} ++ immutable_codes = {} + for name in sorted(names): + module = sys.modules.get(name) + if module is not None: +- _live_code(module, compiled) ++ _live_code(module, compiled, code_pairs, immutable_codes) + return { + "preparation": manifest, + "coverage": coverage._producer(), +--- /dev/null ++++ b/packages/microcosm-build/tests/test_us_acs_loaded_code_validation.py +@@ -0,0 +1,351 @@ ++"""Invented Python source controls for invocation-local ACS code equality memo.""" ++ ++from __future__ import annotations ++ ++import sys ++from pathlib import Path ++from types import CodeType, FunctionType, ModuleType ++ ++import pytest ++ ++from microcosm.build.us_runtime import acs_native_coverage_binding as owner ++ ++ ++def _module(tmp_path, monkeypatch, name, source): ++ path = tmp_path / (name + ".py") ++ path.write_text(source, encoding="utf-8") ++ module = ModuleType(name) ++ module.__file__ = str(path) ++ module.__package__ = "" ++ monkeypatch.setitem(sys.modules, name, module) ++ exec(compile(source, str(path), "exec", dont_inherit=True), vars(module)) ++ return module ++ ++ ++def _code(source="def f():\n return 7\n"): ++ compiled = compile(source, "invented-code.py", "exec", dont_inherit=True) ++ return next(value for value in compiled.co_consts if type(value) is CodeType) ++ ++ ++def _state(): ++ return {}, {}, {} ++ ++ ++def _refuses(): ++ return pytest.raises(owner.ACSNativeCoverageBindingError, match="^LOADED_PRODUCER$") ++ ++ ++def _cell(value): ++ return (lambda: value).__closure__[0] ++ ++ ++def test_repeated_exact_pair_skips_only_second_deep_comparison(): ++ live, reference = _code(), _code() ++ pairs, immutable = {}, {} ++ eligibility_calls = [] ++ eligibility_code = owner._immutable_code_constants.__code__ ++ ++ def profile(frame, event, _arg): ++ if event == "call" and frame.f_code is eligibility_code: ++ eligibility_calls.append(1) ++ ++ previous = sys.getprofile() ++ try: ++ sys.setprofile(profile) ++ assert owner._same_loaded_code(live, reference, pairs, immutable) ++ first_calls = len(eligibility_calls) ++ assert first_calls > 0 ++ assert owner._same_loaded_code(live, reference, pairs, immutable) ++ assert len(eligibility_calls) == first_calls ++ finally: ++ sys.setprofile(previous) ++ assert list(pairs) == [(id(live), id(reference))] ++ assert pairs[(id(live), id(reference))][0] is live ++ assert pairs[(id(live), id(reference))][1] is reference ++ assert immutable[id(live)] is live ++ assert immutable[id(reference)] is reference ++ ++ ++def test_wrong_strong_reference_does_not_authorize_an_identity_key(): ++ live, reference, changed = _code(), _code(), _code("def f():\n return 8\n") ++ pairs = {(id(changed), id(reference)): (live, reference)} ++ assert not owner._same_loaded_code(changed, reference, pairs, {}) ++ ++ ++def test_historical_two_argument_caller_keeps_fresh_validation(tmp_path, monkeypatch): ++ module = _module( ++ tmp_path, monkeypatch, "memo_legacy_call", "def f():\n return 7\n" ++ ) ++ compiled = {} ++ owner._live_code(module, compiled) ++ owner._live_code(module, compiled) ++ monkeypatch.setattr(module.f, "__code__", _code("def f():\n return 8\n")) ++ with _refuses(): ++ owner._live_code(module, compiled) ++ ++ ++def test_import_alias_is_rechecked_after_pair_hit(tmp_path, monkeypatch): ++ origin = _module(tmp_path, monkeypatch, "memo_origin", "def f():\n return 7\n") ++ alias = _module(tmp_path, monkeypatch, "memo_alias", "from memo_origin import f\n") ++ state = _state() ++ owner._live_code(origin, *state) ++ owner._live_code(alias, *state) ++ alias.f = FunctionType(origin.f.__code__, vars(origin)) ++ with _refuses(): ++ owner._live_code(alias, *state) ++ ++ ++def test_equal_code_with_non_origin_globals_refuses_after_pair_hit( ++ tmp_path, monkeypatch ++): ++ module = _module( ++ tmp_path, monkeypatch, "memo_globals", "def f():\n return 7\nalias = f\n" ++ ) ++ state = _state() ++ owner._live_code(module, *state) ++ module.alias = FunctionType(module.f.__code__, {"__name__": module.__name__}) ++ with _refuses(): ++ owner._live_code(module, *state) ++ ++ ++def test_replaced_code_refuses_after_pair_hit(tmp_path, monkeypatch): ++ module = _module(tmp_path, monkeypatch, "memo_replaced", "def f():\n return 7\n") ++ state = _state() ++ owner._live_code(module, *state) ++ monkeypatch.setattr(module.f, "__code__", _code("def f():\n return 8\n")) ++ with _refuses(): ++ owner._live_code(module, *state) ++ ++ ++_CLOSURE_SOURCE = """def factory(): ++ def inner(): ++ return 7 ++ def wrapped(): ++ return inner() ++ return wrapped ++wrapped = factory() ++""" ++ ++ ++def test_replaced_closure_function_is_rechecked_after_pair_hit(tmp_path, monkeypatch): ++ module = _module(tmp_path, monkeypatch, "memo_closure", _CLOSURE_SOURCE) ++ state = _state() ++ owner._live_code(module, *state) ++ cell = module.wrapped.__closure__[0] ++ original = cell.cell_contents ++ bad = FunctionType(original.__code__, {"__name__": module.__name__}) ++ monkeypatch.setattr(cell, "cell_contents", bad) ++ with _refuses(): ++ owner._live_code(module, *state) ++ ++ ++def test_distinct_closure_with_same_outer_code_is_not_memoized_as_valid( ++ tmp_path, monkeypatch ++): ++ module = _module(tmp_path, monkeypatch, "memo_second_closure", _CLOSURE_SOURCE) ++ state = _state() ++ owner._live_code(module, *state) ++ original = module.wrapped.__closure__[0].cell_contents ++ bad = FunctionType(original.__code__, {"__name__": module.__name__}) ++ module.second = FunctionType( ++ module.wrapped.__code__, vars(module), closure=(_cell(bad),) ++ ) ++ with _refuses(): ++ owner._live_code(module, *state) ++ ++ ++def test_source_read_and_replacement_remain_fresh_within_shared_invocation_state( ++ tmp_path, monkeypatch ++): ++ source = "def f():\n return 7\n" ++ module = _module(tmp_path, monkeypatch, "memo_source", source) ++ path = Path(module.__file__) ++ state = _state() ++ reads = [] ++ reader = Path.read_bytes.__code__ ++ ++ def profile(frame, event, _arg): ++ if ( ++ event == "call" ++ and frame.f_code is reader ++ and frame.f_locals["self"] == path ++ ): ++ reads.append(1) ++ ++ previous = sys.getprofile() ++ try: ++ sys.setprofile(profile) ++ owner._live_code(module, *state) ++ owner._live_code(module, *state) ++ finally: ++ sys.setprofile(previous) ++ # One AST read on each scan, plus the existing first code-compilation read. ++ assert len(reads) == 3 ++ changed = "def f():\n return 8\n" ++ fired = [] ++ ++ def change_after_scan(frame, event, _arg): ++ if ( ++ event == "return" ++ and frame.f_code is owner._live_code.__code__ ++ and frame.f_locals["module"] is module ++ and not fired ++ ): ++ fired.append(1) ++ path.write_text(changed, encoding="utf-8") ++ exec(compile(changed, str(path), "exec", dont_inherit=True), vars(module)) ++ ++ try: ++ sys.setprofile(change_after_scan) ++ owner._live_code(module, *state) ++ with _refuses(): ++ owner._live_code(module, *state) ++ finally: ++ sys.setprofile(previous) ++ assert fired == [1] ++ # A distinct producer invocation recompiles the new current source bytes. ++ owner._live_code(module, *_state()) ++ ++ ++def test_code_swap_on_pair_return_refuses_before_function_check_returns( ++ tmp_path, monkeypatch ++): ++ module = _module(tmp_path, monkeypatch, "memo_return", "def f():\n return 7\n") ++ state = _state() ++ owner._live_code(module, *state) ++ original = module.f.__code__ ++ changed = _code("def f():\n return 8\n") ++ target = owner._same_loaded_code.__code__ ++ fired = [] ++ ++ def profile(frame, event, _arg): ++ if ( ++ event == "return" ++ and frame.f_code is target ++ and frame.f_locals["live"] is original ++ and not fired ++ ): ++ fired.append(1) ++ module.f.__code__ = changed ++ ++ previous = sys.getprofile() ++ try: ++ sys.setprofile(profile) ++ with _refuses(): ++ owner._live_code(module, *state) ++ finally: ++ sys.setprofile(previous) ++ module.f.__code__ = original ++ assert fired == [1] ++ ++ ++@pytest.mark.parametrize( ++ "container", [lambda x: x, lambda x: (x,), lambda x: frozenset((x,))] ++) ++def test_mutable_custom_constants_never_gain_pair_or_eligibility_authority(container): ++ class MutableEqual: ++ def __init__(self): ++ self.equal = True ++ ++ def __eq__(self, _other): ++ return self.equal ++ ++ __hash__ = object.__hash__ ++ ++ value = MutableEqual() ++ reference = _code() ++ live = reference.replace(co_consts=(None, container(value))) ++ pairs, immutable = {}, {} ++ for equal in (True, False, True): ++ value.equal = equal ++ # CPython may reject custom constant types before calling __eq__. ++ # Preserve the original decision; do not assume initial acceptance. ++ expected = live == reference ++ assert owner._same_loaded_code(live, reference, pairs, immutable) is expected ++ assert not pairs ++ assert id(live) not in immutable ++ ++ ++def test_nested_code_with_mutable_constant_remains_uncached(): ++ mutable = [] ++ inner = _code().replace(co_consts=(None, mutable)) ++ outer = _code().replace(co_consts=(None, inner)) ++ reference = _code().replace(co_consts=(None, _code())) ++ pairs, immutable = {}, {} ++ for content in ((), (7,), (8,)): ++ mutable[:] = content ++ assert owner._same_loaded_code(outer, reference, pairs, immutable) is ( ++ outer == reference ++ ) ++ assert not pairs ++ assert id(inner) not in immutable ++ assert id(outer) not in immutable ++ ++ ++def test_constant_eligibility_never_calls_custom_metaclass_equality(): ++ class CustomType(type): ++ def __eq__(cls, _other): ++ raise AssertionError("constant eligibility called custom type equality") ++ ++ class CustomValue(metaclass=CustomType): ++ pass ++ ++ live = _code().replace(co_consts=(None, CustomValue())) ++ immutable = {} ++ assert owner._immutable_code_constants(live, immutable) is False ++ assert id(live) not in immutable ++ ++ ++def test_scalar_subclasses_and_deep_values_fall_back_without_new_refusal(): ++ class CustomInt(int): ++ pass ++ ++ deep = 7 ++ for _ in range(70): ++ deep = (deep,) ++ for value in (CustomInt(7), deep): ++ live = _code().replace(co_consts=(None, value)) ++ reference = live.replace() ++ pairs, immutable = {}, {} ++ assert owner._same_loaded_code(live, reference, pairs, immutable) is ( ++ live == reference ++ ) ++ assert not pairs ++ ++ ++def test_compiler_constant_shapes_and_nested_functions_are_cacheable(): ++ source = """def f(x): ++ def inner(): ++ return (None, True, 7, 1.25, 2j, 'text', b'bytes', ...) ++ return x in {1, 2, 3}, inner ++""" ++ live, reference = _code(source), _code(source) ++ pairs, immutable = {}, {} ++ assert owner._same_loaded_code(live, reference, pairs, immutable) ++ assert pairs[(id(live), id(reference))] == (live, reference) ++ assert all(type(code) is CodeType for code in immutable.values()) ++ ++ ++def test_source_class_methods_properties_and_dataclass_escape_keep_existing_behavior( ++ tmp_path, monkeypatch ++): ++ source = """class Example: ++ @classmethod ++ def named(cls): ++ return cls.__name__ ++ @staticmethod ++ def value(): ++ return 7 ++ @property ++ def amount(self): ++ return 7 ++""" ++ module = _module(tmp_path, monkeypatch, "memo_methods", source) ++ state = _state() ++ owner._live_code(module, *state) ++ owner._live_code(module, *state) ++ # Match only the pre-existing generated-code escape; it is not broadened. ++ module.generated = FunctionType( ++ _code().replace(co_filename=""), vars(module) ++ ) ++ owner._live_code(module, *state) diff --git a/experiments/us-acs-compilation-cache-adoption-20260910.json b/experiments/us-acs-compilation-cache-adoption-20260910.json new file mode 100644 index 000000000..157e4596a --- /dev/null +++ b/experiments/us-acs-compilation-cache-adoption-20260910.json @@ -0,0 +1,75 @@ +{ + "control_postcheck_sha256": "58f10a538ad6db8efa30a4029a6eefaa4b0451c1bbb325e21d13a4bd2cc39cd7", + "control_receipt_sha256": "53b578332356b5e5700e2d1a6c40b2b1ccab47440aa886e53db9f19746c8c074", + "control_tests": 25, + "date": "2026-09-10", + "decision": "Adopt bounded process-local reuse of compiled bytecode keyed by complete compilation inputs. Retain every fresh source read, AST check, and loaded function, alias, global and closure check.", + "limitations": [ + "One paired cProfile observation, not a generalized benchmark or a native-data speedup.", + "Cache bounds retained source keys, not total process memory.", + "Compiler warning and audit events occur on cache misses; fresh AST parsing still occurs on every loaded-code check.", + "Cache shares the existing trusted-process boundary; it does not authenticate coherent forgery of private process state.", + "The already running native pilot retains its frozen original implementation." + ], + "native_acceptance": false, + "paired_fixture": { + "comparison": { + "baseline_cpu_seconds": 303.897467, + "baseline_postcheck_sha256": "146aea7545077b933f921fc53a15340b5818f478408199c12d101c0c920477de", + "baseline_profile_sha256": "badec8594bb1421483e0090f273f79f7560adc0b13b68434f089442c7c8d4bbb", + "baseline_wall_seconds": 308.2577062920318, + "cpu_reduction_fraction": 0.2935343617062791, + "same_original_fixture_scope": true, + "scope": "one_paired_profile_observation_not_generalized_benchmark", + "sole_production_delta_owner_sha256": "534042fdd4f2af5a238b556882d3b75c5495d9909e240bb07201ec48bd2bb4da", + "wall_reduction_fraction": 0.2966248653600234 + }, + "cpu_seconds": 214.693118, + "declared_invented_dimensions": { + "atomic_support_blocks": 4, + "clone_households": 12, + "clone_persons": 18, + "counts_are_declared_from_frozen_source_not_native_measurements": true, + "demographic_predictor_count": 5, + "financial_output_count": 7, + "fixture_function": "known_financial_run", + "fixture_source": "packages/microcosm-build/tests/test_us_graph_atomic_survey_financial.py", + "graph_nodes_per_run": 20, + "n_estimators": 2, + "origin": "invented_original_source_fixture", + "runs": [ + "cold_auto", + "required_replay" + ], + "source_households": 6, + "source_persons": 9, + "state_fips_count": 2 + }, + "guard_sha256": "a665ab886e7feeebadffc630175734434a0667cd6b5a0fe812c4cd2cb4f42ca2", + "junit_sha256": "3b17eabe18fb9d8e6d87cb7f84b5eae00cc331086ab4d90239456c00263f2639", + "peak_rss_bytes": 581271552, + "profile_sha256": "0348d21a7ed9edc1dd2152595b13fcc91d51e6347619d80b78b76b4c1a96e5a9", + "receipt_sha256": "ed856c409f4923e49379044a003f652ec3fcdfa29789be52dd8fee33802f01d2", + "test_counts": { + "errors": 0, + "failures": 0, + "skipped": 0, + "tests": 1 + }, + "verified_current": { + "code_resources": 12, + "model_source": 5983, + "source_and_owned": 500 + }, + "wall_seconds": 216.82080566696823 + }, + "paired_postcheck_sha256": "fc2c232eee9904077e578ead9ea6e68f4d9d44c7ac9930d44cb1f08c6b67eba8", + "postcheck_correction": "The first checker incorrectly excluded the inherited root pyproject.toml source entry. The preserved original refused before physical verification; the corrected checker accepts that exact entry and passed. No fixture rerun was needed.", + "protocol": "microcosm.acs-source-compilation-cache-adoption.v1", + "release_acceptance": false, + "source_files": { + "packages/microcosm-build/src/microcosm/build/us_runtime/acs_native_coverage_binding.py": "534042fdd4f2af5a238b556882d3b75c5495d9909e240bb07201ec48bd2bb4da", + "packages/microcosm-build/tests/test_us_acs_source_compile_cache.py": "3ee1adffa8a1420d6b95f0f6b8cc966f5193c6221420aed9b8dbf83b543d9dd6" + }, + "source_peer_review_sha256": "e5de7a750a3e33a985009a7e330f182a97c7c833bbc0a09e204125028203c244" +} diff --git a/experiments/us-acs-loaded-code-controls-20260910.json b/experiments/us-acs-loaded-code-controls-20260910.json new file mode 100644 index 000000000..5fec133ec --- /dev/null +++ b/experiments/us-acs-loaded-code-controls-20260910.json @@ -0,0 +1,33 @@ +{ + "accepted": true, + "acs_source_sha256": "4dc64558ed453f1c06557ef005e42193278496a9c31b374475417c407c037573", + "comparison_policy": "Memo only successful immutable CodeType identity pairs within one producer invocation; retain source, alias, globals and closure checks. Historical two-argument catalogue caller preserved.", + "cpu_seconds": 3.121793, + "date": "2026-09-10", + "financial_successor_acceptance": false, + "guard_sha256": "66d02d798933d7e2a677775c4037df9e16e4f5e6f45cd42363b78234daf56448", + "implementation_adopted": false, + "native_acceptance": false, + "peak_rss_bytes": 353026048, + "performance_gain_established": false, + "receipt_sha256": "da8cde54cb9c5b3ed909ed88b7ab12d6ab824148462593be798dfc86048b4bf1", + "release_acceptance": false, + "runtime": "Python 3.14.4; one numerical thread; no country engine, network, child process or native data access.", + "scope": "invented_acs_code_equality_controls", + "source_map_sha256": "3e5545835bf269a165738a6f59cdd9235171044120113bdd5568a030e59cdc66", + "subsequent_comparison": "us-acs-code-memo-comparison-20260910.json", + "test_sha256": "6d2e43acc5aea4533a99d12adfb94db00629c0f01f25b725d427bd47c7f1aa1f", + "tests": { + "errors": 0, + "failures": 0, + "skipped": 0, + "tests": 18 + }, + "unexpected_refusals": {}, + "verified_current": { + "code_resources": 12, + "model_source": 5983, + "source_and_owned": 500 + }, + "wall_seconds": 4.655691749998368 +} diff --git a/experiments/us-acs-producer-profile-20260910.json b/experiments/us-acs-producer-profile-20260910.json new file mode 100644 index 000000000..8c76e9386 --- /dev/null +++ b/experiments/us-acs-producer-profile-20260910.json @@ -0,0 +1,354 @@ +{ + "accepted_diagnostic": true, + "cprofile_options": { + "builtins": true, + "subcalls": false + }, + "current_reverified_files": { + "code_resources": 12, + "model_source": 5983, + "source_and_owned": 500 + }, + "date": "2026-09-10", + "decision": "Source-only experiment in bounded bytecode compilation reuse; preserve fresh reads and all loaded-function checks. No optimization adopted or native speedup established.", + "fixture_constructed": false, + "graph_executed": false, + "guard_sha256": "426cd0370e3e94d4d4e022fdcf954b89b373af7992da88e2d21a37e8670ec2a4", + "native_inputs_used": false, + "postcheck_sha256": "3426339877891cc0659118f788487fe54e7659094dcd454e193284cf6a6e60e7", + "producer_completed": true, + "producer_invocations_requested": 1, + "profile_sha256": "6b4508e6e96226746381a9862997c22eebb9880a328d50b24b9c19f9fd0e78a1", + "profiled_elapsed_cpu_seconds": 0.6406650000000003, + "profiled_elapsed_wall_seconds": 0.6561586250318214, + "protocol": "microcosm.acs-original-producer-profile.v1", + "receipt_sha256": "a956298f6bbe11438256ac5ad7183098c89ed889c457666602c68c32d4bc5e12", + "records": [ + { + "calls": 1, + "cumulative_seconds": 0.6560935, + "file": "packages/microcosm-build/src/microcosm/build/us_runtime/acs_native_coverage_binding.py", + "function": "_producer", + "kind": "frozen_source", + "line": 129, + "recursive_calls": 0, + "self_seconds": 0.005839213 + }, + { + "calls": 1, + "cumulative_seconds": 0.42354929100000005, + "file": "packages/microcosm-build/src/microcosm/build/us_runtime/graph_implementation.py", + "function": "implementation_manifest", + "kind": "frozen_source", + "line": 411, + "recursive_calls": 0, + "self_seconds": 0.000314576 + }, + { + "calls": 59, + "cumulative_seconds": 0.36231329, + "file": "packages/microcosm-build/src/microcosm/build/us_runtime/graph_implementation.py", + "function": "_dependency_contract", + "kind": "frozen_source", + "line": 303, + "recursive_calls": 0, + "self_seconds": 0.005696675 + }, + { + "calls": 59, + "cumulative_seconds": 0.355838413, + "file": "packages/microcosm-build/src/microcosm/build/us_runtime/graph_implementation.py", + "function": "_dependency_details", + "kind": "frozen_source", + "line": 202, + "recursive_calls": 0, + "self_seconds": 0.023903589000000003 + }, + { + "calls": 44, + "cumulative_seconds": 0.212273085, + "file": "packages/microcosm-build/src/microcosm/build/us_runtime/acs_native_coverage_binding.py", + "function": "_live_code", + "kind": "frozen_source", + "line": 51, + "recursive_calls": 0, + "self_seconds": 0.021575394 + }, + { + "calls": 2586, + "cumulative_seconds": 0.128561793, + "file": "packages/microcosm-build/src/microcosm/build/us_runtime/acs_native_coverage_binding.py", + "function": "check", + "kind": "frozen_source", + "line": 86, + "recursive_calls": 66, + "self_seconds": 0.0032278470000000003 + }, + { + "calls": 979, + "cumulative_seconds": 0.103954326, + "file": "packages/microcosm-build/src/microcosm/build/us_runtime/graph_implementation.py", + "function": "visit_FunctionDef", + "kind": "frozen_source", + "line": 243, + "recursive_calls": 344, + "self_seconds": 0.000322374 + }, + { + "calls": 9482, + "cumulative_seconds": 0.052384176000000005, + "file": "packages/microcosm-build/src/microcosm/build/us_runtime/graph_implementation.py", + "function": "visit_Call", + "kind": "frozen_source", + "line": 260, + "recursive_calls": 2364, + "self_seconds": 0.002310004 + }, + { + "calls": 3, + "cumulative_seconds": 0.016132043000000002, + "file": "packages/microcosm-build/src/microcosm/build/us_runtime/graph_implementation.py", + "function": "_validate_package_roster", + "kind": "frozen_source", + "line": 332, + "recursive_calls": 0, + "self_seconds": 7.2842e-05 + }, + { + "calls": 1, + "cumulative_seconds": 0.014024958, + "file": "packages/microcosm-build/src/microcosm/build/us_runtime/acs_person_coverage_authentication.py", + "function": "_producer", + "kind": "frozen_source", + "line": 124, + "recursive_calls": 0, + "self_seconds": 4.2163e-05 + }, + { + "calls": 36445, + "cumulative_seconds": 0.008270251000000001, + "file": "packages/microcosm-build/src/microcosm/build/us_runtime/graph_implementation.py", + "function": "visit_Name", + "kind": "frozen_source", + "line": 252, + "recursive_calls": 0, + "self_seconds": 0.005725035000000001 + }, + { + "calls": 16, + "cumulative_seconds": 0.005612417000000001, + "file": "packages/microcosm-build/src/microcosm/build/us_runtime/acs_person_coverage_authentication.py", + "function": "_sha", + "kind": "frozen_source", + "line": 65, + "recursive_calls": 0, + "self_seconds": 8.211e-06 + }, + { + "calls": 1, + "cumulative_seconds": 0.004030375, + "file": "packages/microcosm-build/src/microcosm/build/us_runtime/graph_implementation.py", + "function": "_package_roots", + "kind": "frozen_source", + "line": 180, + "recursive_calls": 0, + "self_seconds": 9.082e-06 + }, + { + "calls": 59, + "cumulative_seconds": 0.0036707920000000004, + "file": "packages/microcosm-build/src/microcosm/build/us_runtime/graph_implementation.py", + "function": "_covered_imports", + "kind": "frozen_source", + "line": 286, + "recursive_calls": 0, + "self_seconds": 0.001535637 + }, + { + "calls": 3277, + "cumulative_seconds": 0.002018209, + "file": "packages/microcosm-build/src/microcosm/build/us_runtime/acs_native_coverage_binding.py", + "function": "visit", + "kind": "frozen_source", + "line": 97, + "recursive_calls": 3223, + "self_seconds": 0.0012540350000000001 + }, + { + "calls": 1451, + "cumulative_seconds": 0.001515904, + "file": "packages/microcosm-build/src/microcosm/build/us_runtime/graph_implementation.py", + "function": "", + "kind": "frozen_source", + "line": 229, + "recursive_calls": 0, + "self_seconds": 0.000852642 + }, + { + "calls": 181, + "cumulative_seconds": 0.000619031, + "file": "packages/microcosm-build/src/microcosm/build/us_runtime/graph_implementation.py", + "function": "_digest", + "kind": "frozen_source", + "line": 176, + "recursive_calls": 0, + "self_seconds": 6.2668e-05 + }, + { + "calls": 118, + "cumulative_seconds": 0.000576835, + "file": "packages/microcosm-build/src/microcosm/build/us_runtime/graph_implementation.py", + "function": "_canonical", + "kind": "frozen_source", + "line": 170, + "recursive_calls": 0, + "self_seconds": 7.5121e-05 + }, + { + "calls": 555, + "cumulative_seconds": 0.00047688000000000006, + "file": "packages/microcosm-build/src/microcosm/build/us_runtime/graph_implementation.py", + "function": "", + "kind": "frozen_source", + "line": 289, + "recursive_calls": 0, + "self_seconds": 0.00047688000000000006 + }, + { + "calls": 1, + "cumulative_seconds": 0.000265083, + "file": "packages/microcosm-build/src/microcosm/build/us_runtime/graph_implementation.py", + "function": "_inventory", + "kind": "frozen_source", + "line": 312, + "recursive_calls": 0, + "self_seconds": 4.0999000000000005e-05 + }, + { + "calls": 1, + "cumulative_seconds": 0.000205333, + "file": "packages/microcosm-build/src/microcosm/build/us_runtime/graph_implementation.py", + "function": "_projections", + "kind": "frozen_source", + "line": 352, + "recursive_calls": 0, + "self_seconds": 4.0830000000000005e-06 + }, + { + "calls": 76, + "cumulative_seconds": 0.00019908300000000002, + "file": "packages/microcosm-build/src/microcosm/build/us_runtime/graph_implementation.py", + "function": "columns", + "kind": "frozen_source", + "line": 355, + "recursive_calls": 74, + "self_seconds": 4.3078e-05 + }, + { + "calls": 3811, + "cumulative_seconds": 0.00012806000000000002, + "file": "packages/microcosm-build/src/microcosm/build/us_runtime/acs_native_coverage_binding.py", + "function": "_require", + "kind": "frozen_source", + "line": 46, + "recursive_calls": 0, + "self_seconds": 0.00012806000000000002 + }, + { + "calls": 555, + "cumulative_seconds": 8.0924e-05, + "file": "packages/microcosm-build/src/microcosm/build/us_runtime/graph_implementation.py", + "function": "", + "kind": "frozen_source", + "line": 290, + "recursive_calls": 0, + "self_seconds": 8.0924e-05 + }, + { + "calls": 303, + "cumulative_seconds": 4.1642000000000006e-05, + "file": "packages/microcosm-build/src/microcosm/build/us_runtime/graph_implementation.py", + "function": "", + "kind": "frozen_source", + "line": 358, + "recursive_calls": 0, + "self_seconds": 3.0757000000000004e-05 + }, + { + "calls": 240, + "cumulative_seconds": 3.2792e-05, + "file": "packages/microcosm-build/src/microcosm/build/us_runtime/graph_implementation.py", + "function": "", + "kind": "frozen_source", + "line": 296, + "recursive_calls": 0, + "self_seconds": 3.2792e-05 + }, + { + "calls": 1, + "cumulative_seconds": 8.400000000000001e-08, + "file": "packages/microcosm-build/src/microcosm/build/us_runtime/acs_person_coverage_authentication.py", + "function": "_require", + "kind": "frozen_source", + "line": 60, + "recursive_calls": 0, + "self_seconds": 8.400000000000001e-08 + }, + { + "calls": 157, + "cumulative_seconds": 0.237641084, + "function": "builtins.compile", + "kind": "builtin", + "recursive_calls": 0, + "self_seconds": 0.237641084 + }, + { + "calls": 1, + "cumulative_seconds": 2.92e-07, + "function": "time.perf_counter", + "kind": "builtin", + "recursive_calls": 0, + "self_seconds": 2.92e-07 + }, + { + "calls": 1, + "cumulative_seconds": 3e-06, + "function": "time.process_time", + "kind": "builtin", + "recursive_calls": 0, + "self_seconds": 3e-06 + }, + { + "calls": 103, + "cumulative_seconds": 0.12155241700000001, + "function": "ast.parse", + "kind": "stdlib", + "recursive_calls": 0, + "self_seconds": 4.1292000000000004e-05 + } + ], + "scope": "one_original_producer_invocation_after_normal_helper_imports", + "source_or_population_authority_issued": false, + "successor_or_release_acceptance": false, + "test_counts": { + "errors": 0, + "failures": 0, + "skipped": 0, + "tests": 1 + }, + "timing_limitations": [ + "normal imports are outside profiling and no fixture is requested", + "one producer call has no within-process repetition or warmup", + "cumulative times overlap and must not be summed", + "ast.parse wraps compile and their cumulative times overlap", + "cProfile adds overhead; this diagnostic cannot establish a speedup", + "source rows cover only the three declared owner/manifest modules", + "other external time remains within cumulative callers", + "compiled-reference visitation and deep code equality remain unmodified" + ], + "whole_guard": { + "cpu_seconds": 4.779278, + "peak_rss_bytes": 481591296, + "wall_seconds": 6.85519720899174 + } +} diff --git a/experiments/us-atomic-age-v2-1-controls-20260909.json b/experiments/us-atomic-age-v2-1-controls-20260909.json new file mode 100644 index 000000000..c7fb9cfa7 --- /dev/null +++ b/experiments/us-atomic-age-v2-1-controls-20260909.json @@ -0,0 +1,34 @@ +{ + "code_resources": 12, + "configured_resource_limits": { + "cpu_seconds": 600, + "wall_seconds": 900 + }, + "cpu_seconds": 601.979524, + "frozen_before_after_current_equal": true, + "model_source_files": 5983, + "native_payloads_read": 0, + "peak_rss_bytes": 506183680, + "publisher_provenance_established": false, + "receipt_sha256": "fc77f8ead3a5b8f2f611aa8d1d20d4c19ab6b315322be17241c0b9ec5c12bbdf", + "release_eligible": false, + "resource_observation": "The configured limits were unchanged. Reported cumulative CPU was 601.979524 seconds; the guarded process returned zero after all checks.", + "result": 0, + "scope": "Actual thirteen-node cold and required replay age calibration over invented original sources and block support. Invented data only; no native or publisher admission, no release.", + "source_and_owned_files": 492, + "test_counts": { + "errors": 0, + "failures": 0, + "skipped": 0, + "tests": 1 + }, + "tested_source_changes": { + "packages/microcosm-build/src/microcosm/build/us_runtime/survey_age_calibration.py": "a10b9f3552c84563d845675c772130073b925b5835d6730ffb9afc0e93170777", + "packages/microcosm-build/src/microcosm/build/us_runtime/survey_origin_budget.py": "c0667ad329af2f58901b0fd2f333489ad2366b83e2bac8ded95b51088d9e5d5e", + "packages/microcosm-build/tests/test_us_survey_age_calibration_atomic_geography.py": "25a414517974d345541bcec6152b61151b09ccc476a5350a36015c4b33dcd70e", + "packages/microcosm-build/tests/test_us_survey_origin_budget_atomic_geography.py": "d130335f5fb8d66e3e032984c326e1077b0a889162da8d97324f85866095905d", + "packages/microcosm-build/tests/test_us_survey_origin_budget_predictor_compatibility.py": "d81a82b9aedecd26d62831e93355c233093dc8fc3adec1e702d8c7f138fd8bbd" + }, + "unexpected_refusals": {}, + "wall_seconds": 613.9018378330511 +} diff --git a/experiments/us-atomic-block-adapter-20260909.json b/experiments/us-atomic-block-adapter-20260909.json new file mode 100644 index 000000000..1980d44db --- /dev/null +++ b/experiments/us-atomic-block-adapter-20260909.json @@ -0,0 +1,25 @@ +{ + "before_after_current_frozen_and_maintained_source_equal": true, + "before_after_current_model_and_code_resources_equal": true, + "code_resources": 12, + "cpu_seconds": 3.078341, + "district_relation": "official_tabulation", + "model_source_files": 5983, + "native_payloads_read": 0, + "peak_rss_bytes": 353140736, + "receipt_sha256": "404deb7666b3ac9b8a654181228c545ab95c131f30a1fbe65a2e15d5a08a4c4c", + "release_eligible": false, + "result": 0, + "sampling_basis": "2020_census_persons", + "scope": "Invented US block normalization and declaration through shared assignment, derivation and validation; native support and graph integration remain pending.", + "source_and_owned_files": 479, + "source_sha256": "8d776a60e5ace340b0de745461b89679653522bb672b95ab43bdfabda0f602ee", + "tests": { + "errors": 0, + "failures": 0, + "skipped": 0, + "tests": 13 + }, + "unexpected_refusals": {}, + "wall_seconds": 4.607667459000368 +} diff --git a/experiments/us-atomic-block-api-sources-35-controls-20260909.json b/experiments/us-atomic-block-api-sources-35-controls-20260909.json new file mode 100644 index 000000000..6053026d2 --- /dev/null +++ b/experiments/us-atomic-block-api-sources-35-controls-20260909.json @@ -0,0 +1,26 @@ +{ + "code_resources": 12, + "cpu_seconds": 2.923694, + "frozen_before_after_current_equal": true, + "model_source_files": 5983, + "native_payloads_read": 0, + "peak_rss_bytes": 349110272, + "publisher_provenance_established": false, + "receipt_sha256": "e76a4373281f38f98d794c89265cedbaf50744e63c60f5256dbbdb7905a3a586", + "release_eligible": false, + "result": 0, + "scope": "35 invented population-only Census API source controls through actual CD/PUMA parsers and atomic normalization. Exact response header, row/code types, source hashes, state/block totals, retained byte snapshots and mapping completeness; no queries, credentials, native inputs, publisher admission or release.", + "source_and_owned_files": 496, + "test_counts": { + "errors": 0, + "failures": 0, + "skipped": 0, + "tests": 35 + }, + "tested_source_changes": { + "packages/microcosm-build/src/microcosm/build/us_runtime/atomic_block_api_sources.py": "4bb9f58364e75846d4cbb19c3e0254468089481e28caacb67ae17eb87d802235", + "packages/microcosm-build/tests/test_us_atomic_block_api_sources.py": "69821ca1b81c47647d4a6a7faaf0bf9279889ef409368ed20b84779613ab5241" + }, + "unexpected_refusals": {}, + "wall_seconds": 4.598902583005838 +} diff --git a/experiments/us-atomic-block-sources-36-controls-20260909.json b/experiments/us-atomic-block-sources-36-controls-20260909.json new file mode 100644 index 000000000..52c25620e --- /dev/null +++ b/experiments/us-atomic-block-sources-36-controls-20260909.json @@ -0,0 +1,26 @@ +{ + "code_resources": 12, + "cpu_seconds": 2.88845, + "frozen_before_after_current_equal": true, + "model_source_files": 5983, + "native_payloads_read": 0, + "peak_rss_bytes": 350126080, + "publisher_provenance_established": false, + "receipt_sha256": "a64003994a3f3823d06a22cbf7376ae8bcb3714d1a25110782cc72e83bfcf1cd", + "release_eligible": false, + "result": 0, + "scope": "36 invented source-byte adapter controls through actual PL/CD/PUMA parsers and atomic normalization: exact source/ZIP/CRC/bounds/roster/population/mapping checks, retained byte snapshots, output codec cap. Whole native PL acquisition remains excluded; no native/publisher/release acceptance.", + "source_and_owned_files": 494, + "test_counts": { + "errors": 0, + "failures": 0, + "skipped": 0, + "tests": 36 + }, + "tested_source_changes": { + "packages/microcosm-build/src/microcosm/build/us_runtime/atomic_block_sources.py": "b5be0b06085f0037206836a16625bb4be5dfe6dc6a12659f207e871aea53340b", + "packages/microcosm-build/tests/test_us_atomic_block_sources.py": "5b9a9fc74b14e354ec63b237b33736434a7b66f89135e798d222c32cc7ad788c" + }, + "unexpected_refusals": {}, + "wall_seconds": 3.594141000008676 +} diff --git a/experiments/us-atomic-budget-semantic-8-controls-20260909.json b/experiments/us-atomic-budget-semantic-8-controls-20260909.json new file mode 100644 index 000000000..0d13a0f9e --- /dev/null +++ b/experiments/us-atomic-budget-semantic-8-controls-20260909.json @@ -0,0 +1,29 @@ +{ + "code_resources": 12, + "cpu_seconds": 313.261743, + "frozen_before_after_current_equal": true, + "model_source_files": 5983, + "native_payloads_read": 0, + "peak_rss_bytes": 447545344, + "publisher_provenance_established": false, + "receipt_sha256": "76a132f01c33bd71af65eb0c17bb2187d9b15dfefc6e7e61ad5f935ea14f59bb", + "release_eligible": false, + "result": 0, + "scope": "Corrected semantic budget replay identity and FIFO/owner/mutation refusals. Invented data only; no native or publisher admission, no release.", + "source_and_owned_files": 492, + "test_counts": { + "errors": 0, + "failures": 0, + "skipped": 0, + "tests": 8 + }, + "tested_source_changes": { + "packages/microcosm-build/src/microcosm/build/us_runtime/survey_age_calibration.py": "a10b9f3552c84563d845675c772130073b925b5835d6730ffb9afc0e93170777", + "packages/microcosm-build/src/microcosm/build/us_runtime/survey_origin_budget.py": "c0667ad329af2f58901b0fd2f333489ad2366b83e2bac8ded95b51088d9e5d5e", + "packages/microcosm-build/tests/test_us_survey_age_calibration_atomic_geography.py": "25a414517974d345541bcec6152b61151b09ccc476a5350a36015c4b33dcd70e", + "packages/microcosm-build/tests/test_us_survey_origin_budget_atomic_geography.py": "d130335f5fb8d66e3e032984c326e1077b0a889162da8d97324f85866095905d", + "packages/microcosm-build/tests/test_us_survey_origin_budget_predictor_compatibility.py": "d81a82b9aedecd26d62831e93355c233093dc8fc3adec1e702d8c7f138fd8bbd" + }, + "unexpected_refusals": {}, + "wall_seconds": 321.5954028330161 +} diff --git a/experiments/us-atomic-clone-graph-20260909.json b/experiments/us-atomic-clone-graph-20260909.json new file mode 100644 index 000000000..a30a95cfe --- /dev/null +++ b/experiments/us-atomic-clone-graph-20260909.json @@ -0,0 +1,33 @@ +{ + "all_location_columns_inherited": "passed", + "before_after_current_frozen_and_maintained_source_equal": true, + "before_after_current_model_and_code_resources_equal": true, + "code_resources": 12, + "cpu_seconds": 4.555835, + "household_weight_halves": "passed", + "households_after": 8, + "households_before": 4, + "model_source_files": 5983, + "native_payloads_read": 0, + "peak_rss_bytes": 364150784, + "persons_after": 12, + "persons_before": 6, + "preclone_gate_order": "passed", + "production_source_sha256": "ef9b562b355abc450f3d2934ab9be04e49387d03de4b39e37fa09b1c0c1905c8", + "receipt_sha256": "1bb75a5ee53edcbdd199a6d0163cbe75c32e54c80513cb3eb8d13021c3c2b1c3", + "release_eligible": false, + "required_replay": "passed", + "result": 0, + "scope": "Invented combined-survey actual graph: atomic assignment and derivation before support clone, inherited mapping gate and required replay. Native source and runner integration remain pending.", + "source_and_owned_files": 481, + "source_only_peer_review": "Codex peer found no actionable findings in adapter, composition and tests; no independent runtime or native verification.", + "subset_mapping_check": "passed", + "tests": { + "errors": 0, + "failures": 0, + "skipped": 0, + "tests": 1 + }, + "unexpected_refusals": {}, + "wall_seconds": 6.214872374985134 +} diff --git a/experiments/us-atomic-financial-4-controls-20260909.json b/experiments/us-atomic-financial-4-controls-20260909.json new file mode 100644 index 000000000..508aeb133 --- /dev/null +++ b/experiments/us-atomic-financial-4-controls-20260909.json @@ -0,0 +1,29 @@ +{ + "code_resources": 12, + "control_accepted": false, + "cpu_seconds": 467.05899700000003, + "frozen_before_after_current_equal": true, + "model_source_files": 5983, + "native_payloads_read": 0, + "peak_rss_bytes": 565133312, + "publisher_provenance_established": false, + "receipt_sha256": "ae0397762505db7059865abf28e739b534026416cd39b690ca50dc27310cb34b", + "release_eligible": false, + "result": 1, + "scope": "Four invented atomic-plus-financial controls: twenty-node cold/required replay with five qualified features, default three-feature compatibility, detached money mutation refusal at final support return, and final geography mutation refusal. No fit quality, native admission, PUF recipient admission, budget successor, or release acceptance.", + "source_and_owned_files": 497, + "test_counts": { + "errors": 0, + "failures": 2, + "skipped": 0, + "tests": 4 + }, + "tested_source_changes": { + "packages/microcosm-build/src/microcosm/build/us_runtime/current_survey_predictors.py": "0e32704bd82e32382cd03936b8cb6faada32afc10e9256c78c9a4b2f9b3372c8", + "packages/microcosm-build/src/microcosm/build/us_runtime/graph_atomic_survey_financial.py": "af365e9135f2fffa9cdcaed643be9a082cb2d382255c9d93fdd4e7cd4cfe8a77", + "packages/microcosm-build/src/microcosm/build/us_runtime/graph_current_survey_predictors.py": "fd643fcaca956ea4830e2022ad737c7321dbc3e59057230c56c31f303e8f5c27", + "packages/microcosm-build/tests/test_us_graph_atomic_survey_financial.py": "8d0fc394f7f59bf64f387cb137819c1789bfdf310f1573a4b41d4bd3ea3cfcdb" + }, + "unexpected_refusals": {}, + "wall_seconds": 472.7063618339598 +} diff --git a/experiments/us-atomic-financial-composition-acceptance-20260909.json b/experiments/us-atomic-financial-composition-acceptance-20260909.json new file mode 100644 index 000000000..d6f21bf54 --- /dev/null +++ b/experiments/us-atomic-financial-composition-acceptance-20260909.json @@ -0,0 +1,42 @@ +{ + "calibration_successor_accepted": false, + "completed_controls": [ + "cold and required replay across all20 actual graph nodes", + "complete retained fields, metadata, IDs, weights, owners and clone inheritance", + "explicit five-feature conditioning and default three-feature compatibility", + "refuse detached money mutation during final support return", + "refuse materialized geography mutation during final owner return" + ], + "fit_quality_accepted": false, + "graph_nodes": 20, + "native_payloads_read": 0, + "native_survey_replay_accepted": false, + "negative_controls_repeated": false, + "population_scope": "invented original survey inputs; native survey replay pending", + "puf_attachment_accepted": false, + "release_eligible": false, + "runs": [ + { + "experiment": "us-atomic-financial-4-controls-20260909.json", + "failed_test_code_defects": [ + "nonexistent assertion helper name", + "incorrect expected refusal string" + ], + "receipt_sha256": "ae0397762505db7059865abf28e739b534026416cd39b690ca50dc27310cb34b", + "result": "two negative mutation controls passed; two test-code defects failed" + }, + { + "experiment": "us-atomic-financial-corrected-2-controls-20260909.json", + "receipt_sha256": "adc535f414c62ab01240f98d0a50f498f4b345531157f1b67c6be23c6deaa0b5", + "result": "two corrected tests passed with the same implementation and full cold/required fixture" + } + ], + "status": "invented_composition_controls_passed", + "test_only_corrections": true, + "tested_source_changes": { + "packages/microcosm-build/src/microcosm/build/us_runtime/current_survey_predictors.py": "0e32704bd82e32382cd03936b8cb6faada32afc10e9256c78c9a4b2f9b3372c8", + "packages/microcosm-build/src/microcosm/build/us_runtime/graph_atomic_survey_financial.py": "af365e9135f2fffa9cdcaed643be9a082cb2d382255c9d93fdd4e7cd4cfe8a77", + "packages/microcosm-build/src/microcosm/build/us_runtime/graph_current_survey_predictors.py": "fd643fcaca956ea4830e2022ad737c7321dbc3e59057230c56c31f303e8f5c27", + "packages/microcosm-build/tests/test_us_graph_atomic_survey_financial.py": "1b441049e527f24f339e2c2fed289ae8ff434015d7bc0b025cebda87f32d52d0" + } +} diff --git a/experiments/us-atomic-financial-corrected-2-controls-20260909.json b/experiments/us-atomic-financial-corrected-2-controls-20260909.json new file mode 100644 index 000000000..042978bd3 --- /dev/null +++ b/experiments/us-atomic-financial-corrected-2-controls-20260909.json @@ -0,0 +1,24 @@ +{ + "code_resources": 12, + "control_accepted": true, + "cpu_seconds": 248.723104, + "frozen_before_after_current_equal": true, + "model_source_files": 5983, + "native_payloads_read": 0, + "peak_rss_bytes": 552976384, + "publisher_provenance_established": false, + "receipt_sha256": "adc535f414c62ab01240f98d0a50f498f4b345531157f1b67c6be23c6deaa0b5", + "release_eligible": false, + "result": 0, + "scope": "Identical unexecuted-v2 source closure; select only the two previously failing corrected tests, retaining their actual cold/required replay fixture. Passed original negative controls are not rerun. No ongoing successor implementation, fit quality, native admission, or release acceptance.", + "source_and_owned_files": 497, + "test_counts": { + "errors": 0, + "failures": 0, + "skipped": 0, + "tests": 2 + }, + "tested_source_changes": {}, + "unexpected_refusals": {}, + "wall_seconds": 252.2364875409985 +} diff --git a/experiments/us-atomic-national-acquisition-20260909.json b/experiments/us-atomic-national-acquisition-20260909.json new file mode 100644 index 000000000..42485f90e --- /dev/null +++ b/experiments/us-atomic-national-acquisition-20260909.json @@ -0,0 +1,598 @@ +{ + "acquired_at_utc": "2026-09-09T23:45:56.994286+00:00", + "acquisition_receipt_sha256": "6e36758e59aa8536d35d25881caed122111da59e200a634c5a4b8d7baa6485d5", + "automatic_retry": false, + "cpu_seconds": 16.783967, + "de_metadata_sha256": "6fd38afe7e3b33995b13e14bc112827936a7871fd58b66a61e5b9b825350bb5d", + "guard_sha256": "1c9bf5810b787f19043cb744f1155d823b0f80703038c2a0f861fdcdf8276a18", + "native_inputs_reopened_for_postcheck": false, + "native_normalization": false, + "peak_rss_acceptance_bytes": 4294967296, + "peak_rss_bytes": 390889472, + "project_imports": false, + "release_eligible": false, + "result": 0, + "rss_hard_os_enforcement": false, + "scope": "104 once-only verified-HTTPS requests: P1_001N block populations and independent state totals for50states+DC, CD119 mapping archive and2020tract-to-PUMA mapping. Acquisition only; no normalization, survey construction or calibration.", + "script_and_roster_before_after_equal": true, + "script_sha256": "eccecc8f8927dd52b2f704cb1cc22c3c3fb10a59f7c1ae01f301edee240c8894", + "source_admission_issued": false, + "source_count": 104, + "sources": [ + { + "id": "pl2020_p1_state01_blocks", + "sha256": "2abb5d0be868baaa5d0b6c05490d832941084da51f1457090a10d9ff9657606d", + "size_bytes": 6427012 + }, + { + "id": "pl2020_p1_state01_total", + "sha256": "3a6d19e9e904b80afffb901a71bdda9cb5101bca1889dfc8efc50c8c39688ef5", + "size_bytes": 39 + }, + { + "id": "pl2020_p1_state02_blocks", + "sha256": "8534edc8dd05f33c878f33b7eb50ec359a3aff95198dbb91f95dc854fac10574", + "size_bytes": 982680 + }, + { + "id": "pl2020_p1_state02_total", + "sha256": "ba9387573141f95e32d1440192e5a14efe431ab1e54ae7bbf0fe2433adf61d43", + "size_bytes": 38 + }, + { + "id": "pl2020_p1_state04_blocks", + "sha256": "81ea2d71faef8aaebb95bf986a87c905b09bfd30036ca5d096da67e91fd7c8cf", + "size_bytes": 5395609 + }, + { + "id": "pl2020_p1_state04_total", + "sha256": "b7ba8c426c875025aa324794719d978c3167e3a49e332e4af9b8fdf83b7a231c", + "size_bytes": 39 + }, + { + "id": "pl2020_p1_state05_blocks", + "sha256": "627457173db87dd14a2cfdb5464ac69941c306229423431962f5e2a23b4e522e", + "size_bytes": 4704516 + }, + { + "id": "pl2020_p1_state05_total", + "sha256": "b67fed8d924bca67ea48d091b4677828288b5d4b76cf57e1ce5777e7b55af6d4", + "size_bytes": 39 + }, + { + "id": "pl2020_p1_state06_blocks", + "sha256": "af233c9cf00396a0e61735430268f09df3301695d4790ec577c58b3df2e30957", + "size_bytes": 18139193 + }, + { + "id": "pl2020_p1_state06_total", + "sha256": "6bcb55b367bdde18380c87a526c18ebc7bd929c6100951942c7df517ddba02c2", + "size_bytes": 40 + }, + { + "id": "pl2020_p1_state08_blocks", + "sha256": "148ad5135f22c8410ee3baadc40c9f22cef8960423a05d4a4d9f940e44969ae9", + "size_bytes": 4866694 + }, + { + "id": "pl2020_p1_state08_total", + "sha256": "d730fcb0310c63c7ab6d6cd8c59474790be28953a3e791e1086f5d77feca9764", + "size_bytes": 39 + }, + { + "id": "pl2020_p1_state09_blocks", + "sha256": "ef40acb552ad9447cba791ccc0a24aed8f572891eb81ae4561ee0d61de836c54", + "size_bytes": 1747484 + }, + { + "id": "pl2020_p1_state09_total", + "sha256": "96a3d6fd5abfd6b32b44653adbcce8f1fe1e133a5cd474718d0d7aad7ec305fd", + "size_bytes": 39 + }, + { + "id": "pl2020_p1_state10_blocks", + "sha256": "1a7a314a84aa0049adb0865d0bfd7466d808b36460f67eef6f3143a331fe6408", + "size_bytes": 702646 + }, + { + "id": "pl2020_p1_state10_total", + "sha256": "ac40e616243c2ea14060dd460009e0a7f15755324a17387adf8768e35787fb79", + "size_bytes": 38 + }, + { + "id": "pl2020_p1_state11_blocks", + "sha256": "d23c7ae418bc35270dc6eec54c014c92cd370aab7ff92de8168ff5c182db84a1", + "size_bytes": 210965 + }, + { + "id": "pl2020_p1_state11_total", + "sha256": "fd791728a6b47cb82daf1fdaa156cf3a166815510f8057341e44bc8b9c36f61b", + "size_bytes": 38 + }, + { + "id": "pl2020_p1_state12_blocks", + "sha256": "b8c5633d903a040396c7f4d386aacb72e7c698253851fde6a115720bc90ed5e0", + "size_bytes": 13566697 + }, + { + "id": "pl2020_p1_state12_total", + "sha256": "41b9bcced940cfe83c8d27aea2c3ff69e0c48b3e7dbd88ded2fb6c1cd5b8bff2", + "size_bytes": 40 + }, + { + "id": "pl2020_p1_state13_blocks", + "sha256": "d2ee481c091a4c316fbef61bc62d069a884fbd52d658cd8a43de8fd4a1c2a90f", + "size_bytes": 8068936 + }, + { + "id": "pl2020_p1_state13_total", + "sha256": "a6bf2b3f49f8393f3ff7d1762b99d927d949d4c18e65c2995488ee88e4b7b8f5", + "size_bytes": 40 + }, + { + "id": "pl2020_p1_state15_blocks", + "sha256": "12360bccbca022409819393314c4f22d960d5d7f8edc302f3b9feefc73a011b4", + "size_bytes": 514556 + }, + { + "id": "pl2020_p1_state15_total", + "sha256": "7ed5585fbe97d5dab88b0ab10de66f30e90fa4a14b0e8989c02d32c2c4d631fd", + "size_bytes": 39 + }, + { + "id": "pl2020_p1_state16_blocks", + "sha256": "74ee9f18bddd2d67b90d93d808944edac3d410cc95a761a34980e3ce55f151a9", + "size_bytes": 2821287 + }, + { + "id": "pl2020_p1_state16_total", + "sha256": "8f1db7ed1b345ba86cdce141ab9d645c2cff91b88e0191fe557089fe0ceb4fbe", + "size_bytes": 39 + }, + { + "id": "pl2020_p1_state17_blocks", + "sha256": "6b0eed0c5ec1bcc326cffecc00c98d77a389fd3aa01b2163a99ca174088861ca", + "size_bytes": 12820271 + }, + { + "id": "pl2020_p1_state17_total", + "sha256": "c02690f4ec95f7f8a9f3bd3a994d5dee9877ad3c2d2e07de9eeb2ebcb9bd0757", + "size_bytes": 40 + }, + { + "id": "pl2020_p1_state18_blocks", + "sha256": "3c91aeb1afc98189affebfd1a5d6884cc03755e413e46b906a4b0d47240c9255", + "size_bytes": 7095958 + }, + { + "id": "pl2020_p1_state18_total", + "sha256": "572d8013dcc0fb51697478648a8c1e9ddf0c8ef0c2aa583a4eb44e5c81c2ddcc", + "size_bytes": 39 + }, + { + "id": "pl2020_p1_state19_blocks", + "sha256": "98f4efd90409774596b5bc623078d75f579d781ca3bc392b586a24fa67e638a1", + "size_bytes": 6033554 + }, + { + "id": "pl2020_p1_state19_total", + "sha256": "5cd722954a80d81e07f29fe65056cd0f40967c163935eb450320d4e73a5d4169", + "size_bytes": 39 + }, + { + "id": "pl2020_p1_state20_blocks", + "sha256": "35e21c30e14e0c4bfdb3645076cdc7a89ef4708c17373047526875be9cb0ab41", + "size_bytes": 5931398 + }, + { + "id": "pl2020_p1_state20_total", + "sha256": "aaaa97cb26fbc4db8318b32549da0a4f64a3fc4ca316ef9fcc5b2da6bb3b1c77", + "size_bytes": 39 + }, + { + "id": "pl2020_p1_state21_blocks", + "sha256": "975c9821955f9472f2222f85e089ce6ce9caff44a18bed9edd797ca13bda49c8", + "size_bytes": 4591236 + }, + { + "id": "pl2020_p1_state21_total", + "sha256": "c65f5deb2c5dacbcd8c1e84e4caf7a41458399f3a388425fbc5a7af69dba0bb9", + "size_bytes": 39 + }, + { + "id": "pl2020_p1_state22_blocks", + "sha256": "9bdacd2c2ccfa5bac93f5df47ec1e63fe39643c15633e7d33f464ad7a171d869", + "size_bytes": 4942488 + }, + { + "id": "pl2020_p1_state22_total", + "sha256": "db924ac57fd4177be3508d795ce912f3b55454f8eb6331fcc297252c2bb20a19", + "size_bytes": 39 + }, + { + "id": "pl2020_p1_state23_blocks", + "sha256": "eb26cdc0e114b07154096f592447df295d7fcda6dd890ebfe6e012d7d7bb40ca", + "size_bytes": 1629179 + }, + { + "id": "pl2020_p1_state23_total", + "sha256": "2a6bc352f6304d0ceb8850333a47378fa2a324a1163850d56931b424b8281089", + "size_bytes": 39 + }, + { + "id": "pl2020_p1_state24_blocks", + "sha256": "2d181f3e6bf3c96c8c2f458024a699be8e4c8b2bb80b6433ee95c69efe655b29", + "size_bytes": 2925422 + }, + { + "id": "pl2020_p1_state24_total", + "sha256": "b9de28b8819798a841c36ac13d35cdc75573c8e47ec232d195817239ab0bff55", + "size_bytes": 39 + }, + { + "id": "pl2020_p1_state25_blocks", + "sha256": "ca73c4d84aaf50e22ca9b25c4fdc54ca314167473a5ba1c9ee31fac1881396e3", + "size_bytes": 3749394 + }, + { + "id": "pl2020_p1_state25_total", + "sha256": "db5b65758be7c88e4bcafd4413f2813fa32de96307aa50a450f28bdc14e3bb39", + "size_bytes": 39 + }, + { + "id": "pl2020_p1_state26_blocks", + "sha256": "732595037f8d58aee0d01a3b68e71a18bc8a7144c81433d1dfc805a1a83cfe8d", + "size_bytes": 8845352 + }, + { + "id": "pl2020_p1_state26_total", + "sha256": "29a349d67a3c6219b9675e6cc5e5bd9a0bf3511e4b48005a6c9a9215be033069", + "size_bytes": 40 + }, + { + "id": "pl2020_p1_state27_blocks", + "sha256": "8910dd3e06035629bbef028441acb5d29f29ea555381df98076a05289bf29803", + "size_bytes": 6863252 + }, + { + "id": "pl2020_p1_state27_total", + "sha256": "13b45f3487425184637b01256644d5f86599d6a445b1a4f1d9fb86ef3790cef9", + "size_bytes": 39 + }, + { + "id": "pl2020_p1_state28_blocks", + "sha256": "33350d9a37d539dd4724dc7f4b969de0fd1dbfcb5207cae093b8ed73e0a67d68", + "size_bytes": 3878667 + }, + { + "id": "pl2020_p1_state28_total", + "sha256": "5cabdefebb5363751a244a9b12720ff43e9ac3a6fef7ba8c1ee7f79e7adaf745", + "size_bytes": 39 + }, + { + "id": "pl2020_p1_state29_blocks", + "sha256": "0e9b1a932c3cfb283a80a0e831192482ee9db6f01d7aad83de67a16e5c4075d5", + "size_bytes": 8749056 + }, + { + "id": "pl2020_p1_state29_total", + "sha256": "b75c227a323978f36e25e2efab8ec11b20579b06d6393da203cb36d533743a6d", + "size_bytes": 39 + }, + { + "id": "pl2020_p1_state30_blocks", + "sha256": "b9a81e34cc8c228fef203728f4134f96b7a8acfc1b14395e2de251d7ab0e3964", + "size_bytes": 3033388 + }, + { + "id": "pl2020_p1_state30_total", + "sha256": "caa4444d14b67700b3db4a5653412bd633c995df0d279473441b885a40102968", + "size_bytes": 39 + }, + { + "id": "pl2020_p1_state31_blocks", + "sha256": "bebe7ca15c979b170cfc3e6f49e27cdc50f5639de2f9a1955e60187849404e56", + "size_bytes": 4095921 + }, + { + "id": "pl2020_p1_state31_total", + "sha256": "30c8aae0e54d78eeb7857f0954be04c95c3a2278a030e78317f2a361fe40c683", + "size_bytes": 39 + }, + { + "id": "pl2020_p1_state32_blocks", + "sha256": "f34926bc3ada9c07a7ace640a787495814b9893c8351b34a9a0fe9bcf91acd75", + "size_bytes": 1991100 + }, + { + "id": "pl2020_p1_state32_total", + "sha256": "3f3957d218a7eb94ae360e03f6b5924d95ab0e8cb80b685d0fc74ff3de3349a7", + "size_bytes": 39 + }, + { + "id": "pl2020_p1_state33_blocks", + "sha256": "11e83119f4924fdbc1c0dcf6f96113c84ba4173c507290578835dfd2c7f97dfa", + "size_bytes": 1110353 + }, + { + "id": "pl2020_p1_state33_total", + "sha256": "b2c3dae17fcbdccaa8eddf6f3e5fdb742d4b0ec017084610958d5182c0f70ce7", + "size_bytes": 39 + }, + { + "id": "pl2020_p1_state34_blocks", + "sha256": "280a8f28f13545bbff7d432e2356cdaeaedd9e0ea026f7f05f4d9e83dd28886e", + "size_bytes": 4823492 + }, + { + "id": "pl2020_p1_state34_total", + "sha256": "517cdd27711beea019fa1efc18c6017690cd1d44617aa446a52677e94281d0cc", + "size_bytes": 39 + }, + { + "id": "pl2020_p1_state35_blocks", + "sha256": "a012da889bc274fb518c2a258c9b1394f43c24e6d58fb3661151565cc5ce8757", + "size_bytes": 3690294 + }, + { + "id": "pl2020_p1_state35_total", + "sha256": "9632c8400a3052e17d23dd71427ccf66e8c062b15cb58df2cb949ecb7a287e9e", + "size_bytes": 39 + }, + { + "id": "pl2020_p1_state36_blocks", + "sha256": "7a47a92501f986ebca42a27a180fe897b4afb2c37cd463360179a0b32500ab2d", + "size_bytes": 10067666 + }, + { + "id": "pl2020_p1_state36_total", + "sha256": "e34eac8cee690ba81ef7ad607cee5e06697834056165979faf2148627d5aeb5f", + "size_bytes": 40 + }, + { + "id": "pl2020_p1_state37_blocks", + "sha256": "3a721726b3ee7a8426411e5ba5267b0c0f88d4123916e8ed40d71e0a601eb797", + "size_bytes": 8216489 + }, + { + "id": "pl2020_p1_state37_total", + "sha256": "bf0b0c6a90da12c51a71942982ec8a10b8e1dece22601462dd0391c2dc758ba2", + "size_bytes": 40 + }, + { + "id": "pl2020_p1_state38_blocks", + "sha256": "6f8985d46a8d8e97fce2f563f20442f7c6e73a0503cddf58ff59746e829e2b68", + "size_bytes": 2892476 + }, + { + "id": "pl2020_p1_state38_total", + "sha256": "3616ffbe27406d4e881587912b59c4494d2a0aa71132a25a6f8108bae23ccbb5", + "size_bytes": 38 + }, + { + "id": "pl2020_p1_state39_blocks", + "sha256": "aeb3eff352866ece982d0a31cfaf3ad63877bd8e2598bede9d883a0a168b450c", + "size_bytes": 9606700 + }, + { + "id": "pl2020_p1_state39_total", + "sha256": "3a93ca174df19c6f3377abf023177def2d8ff585b2182d56a9abd75b2e3454e3", + "size_bytes": 40 + }, + { + "id": "pl2020_p1_state40_blocks", + "sha256": "2078a38b5338f474099f7e1fe2ba0dba3cab95b1644db5abd6c52ca4a8a57703", + "size_bytes": 6212394 + }, + { + "id": "pl2020_p1_state40_total", + "sha256": "bd8b724c531ca5d6605aa1752c70c00a2666755070a18ef73ac0a3423d2812a5", + "size_bytes": 39 + }, + { + "id": "pl2020_p1_state41_blocks", + "sha256": "3cafb0c96a666767ea005ac1f08ca3171379934352c4d2ef285fd6ce44196ccd", + "size_bytes": 4521307 + }, + { + "id": "pl2020_p1_state41_total", + "sha256": "fb8d53cee89c2a14485cf42cf49b4723576407db7d550897609c1f5038b20e67", + "size_bytes": 39 + }, + { + "id": "pl2020_p1_state42_blocks", + "sha256": "34840439707bedac6bf76a35863287aa24263000ff2e1dca21ecdf0030b262f6", + "size_bytes": 11713716 + }, + { + "id": "pl2020_p1_state42_total", + "sha256": "b07244d4d81570c0c8bf49dc3653c382e8e3e74a2d798fb32c85e5808fd789a7", + "size_bytes": 40 + }, + { + "id": "pl2020_p1_state44_blocks", + "sha256": "8f8eb7ee08020dd7e3a49dab4d95f461acd5d29572fe8bf5b55f200ff84e7231", + "size_bytes": 894186 + }, + { + "id": "pl2020_p1_state44_total", + "sha256": "ec6f0565a6ebceccf1bf1a665f46a8105852b352762416b350f9630f802a8223", + "size_bytes": 39 + }, + { + "id": "pl2020_p1_state45_blocks", + "sha256": "a3daf2da71c547c02d8c260b009be331cd218c84896d602fafcdbc12acd59c3f", + "size_bytes": 5086274 + }, + { + "id": "pl2020_p1_state45_total", + "sha256": "3d15445808b86c1f785d6a1668f69eae773fbc75a462d3ead77c99bcc769ba9c", + "size_bytes": 39 + }, + { + "id": "pl2020_p1_state46_blocks", + "sha256": "1700e07604a6fb79d5cfb34fcb6bfe3d810429ecbca4e88d3012b8b2070a516d", + "size_bytes": 2447858 + }, + { + "id": "pl2020_p1_state46_total", + "sha256": "e5b885739fae3a34f28cc11f739435e8cd39926ebeb0a2ef1a18ae3ff035becd", + "size_bytes": 38 + }, + { + "id": "pl2020_p1_state47_blocks", + "sha256": "d8522733ea6f5971d5f340d86f71f7174b54f502ddd84a79d703fbd72d343eb2", + "size_bytes": 6232909 + }, + { + "id": "pl2020_p1_state47_total", + "sha256": "32de25caad8e71a6bb74f5639f5e9eae1c95836f61743275c60b76f716d0c790", + "size_bytes": 39 + }, + { + "id": "pl2020_p1_state48_blocks", + "sha256": "2e997c530f51c17b595226374c50eb1ed249d2d6945e37dde9a1ceed8a161c6d", + "size_bytes": 23175159 + }, + { + "id": "pl2020_p1_state48_total", + "sha256": "469d7f5efd70e8918a3c9744d7c7742e190e54b52d5b331faf648907f5882b0b", + "size_bytes": 40 + }, + { + "id": "pl2020_p1_state49_blocks", + "sha256": "73207e9605fda4b336c404455c9a369ed64e39e7ecebd623b633835a9664e495", + "size_bytes": 2471351 + }, + { + "id": "pl2020_p1_state49_total", + "sha256": "825dab7318dcc582b32eb2723473be42d111a00d32f47d88ae93beade83e2fae", + "size_bytes": 39 + }, + { + "id": "pl2020_p1_state50_blocks", + "sha256": "7e34e58509cec16138d786dda2e34a45370fc1386108591182313f306b252daa", + "size_bytes": 850760 + }, + { + "id": "pl2020_p1_state50_total", + "sha256": "c5ccb3d18ec85abae8ca3958cd4cca738812021cee35dd34be09150c07aa3c3d", + "size_bytes": 38 + }, + { + "id": "pl2020_p1_state51_blocks", + "sha256": "0d979951968f230cd5b7235f703e9acfabcafbeed99c4c018e34c0f88ba0ca52", + "size_bytes": 5678471 + }, + { + "id": "pl2020_p1_state51_total", + "sha256": "87ce18c978a4361494de558f74c03ce956079f6cfa8fb0401c8e77e5bf234c97", + "size_bytes": 39 + }, + { + "id": "pl2020_p1_state53_blocks", + "sha256": "0e3ec5282f8d894a504625cc9a366ada08430b60317fb5d529a982b0e2787a64", + "size_bytes": 5490219 + }, + { + "id": "pl2020_p1_state53_total", + "sha256": "ef5ad0f000c5505f923e4c91a5516b02138669c8b08b7a26c547cc856a7891e4", + "size_bytes": 39 + }, + { + "id": "pl2020_p1_state54_blocks", + "sha256": "28f1b09668784b1d8a064bd1433a1446e9697d8da2988c611bbc267ebee4456f", + "size_bytes": 2507211 + }, + { + "id": "pl2020_p1_state54_total", + "sha256": "b1cefc08915ac5c9bd05f47636d5ce6307b8ecb43c3c32502de2ec6d8b144c13", + "size_bytes": 39 + }, + { + "id": "pl2020_p1_state55_blocks", + "sha256": "fc491abf3fcd1a1fd3807ecde1d95b68bc2cab4e0a235a1b51f095ab5a2afcfa", + "size_bytes": 7025682 + }, + { + "id": "pl2020_p1_state55_total", + "sha256": "1c7bbd06d50dfe00b2e4461d2b92e98fac585f4b7b8c2425782bc822984fc5e9", + "size_bytes": 39 + }, + { + "id": "pl2020_p1_state56_blocks", + "sha256": "55aef5b66c91b47bda087b490e44a12067eabe7634b444351172e59ae126e66a", + "size_bytes": 1843303 + }, + { + "id": "pl2020_p1_state56_total", + "sha256": "0684454d90b7e6430f6f55991791113091593c8dab5577ff8d6dcc2d1fa2e10d", + "size_bytes": 38 + }, + { + "id": "census_cd119_bef", + "sha256": "1433feb5178dc7b4188ee30f5f7f715851f4400740b8fe1ce606a876c6294bd6", + "size_bytes": 22959130 + }, + { + "id": "census_tract2020_puma2020", + "sha256": "5262f460b2c8dbe86b00549e916317b1791fdc595c9af72ab18f6f6a67040531", + "size_bytes": 1709076 + } + ], + "state_fips": [ + "01", + "02", + "04", + "05", + "06", + "08", + "09", + "10", + "11", + "12", + "13", + "15", + "16", + "17", + "18", + "19", + "20", + "21", + "22", + "23", + "24", + "25", + "26", + "27", + "28", + "29", + "30", + "31", + "32", + "33", + "34", + "35", + "36", + "37", + "38", + "39", + "40", + "41", + "42", + "44", + "45", + "46", + "47", + "48", + "49", + "50", + "51", + "53", + "54", + "55", + "56" + ], + "transferred_bytes": 306552379, + "wall_seconds": 328.79918500001077 +} diff --git a/experiments/us-atomic-native-de-1-control-20260909.json b/experiments/us-atomic-native-de-1-control-20260909.json new file mode 100644 index 000000000..84d9bb62e --- /dev/null +++ b/experiments/us-atomic-native-de-1-control-20260909.json @@ -0,0 +1,42 @@ +{ + "archive_members": [ + "01_AL_CD119.txt", + "13_GA_CD119.txt", + "22_LA_CD119.txt", + "36_NY_CD119.txt", + "37_NC_CD119.txt", + "NationalCD119.txt" + ], + "automatic_retry": false, + "code_resources_rechecked": 12, + "corrected_control_executed": false, + "cpu_seconds": 3.002104, + "diagnosis": "The acquired CD archive contains six members; the test incorrectly declared only NationalCD119.txt. The exact-roster validation correctly refused it.", + "diagnostic_receipt_sha256": "54405497257274709b458baad3d0a8b0f4fd2727406d794d42f962d12cd851ca", + "failure_class": "ValueError", + "frozen_before_after_equal": true, + "frozen_code_current_equal": true, + "metadata_probe_member_contents_opened": 0, + "model_source_files": 5983, + "native_inputs_reopened_in_postcheck": false, + "native_resource_pins_receipt_only": 4, + "peak_rss_bytes": 372555776, + "receipt_sha256": "063d36af8b4a8352bb573396b39a921c21c8703d484e32542f24680702428df0", + "release_eligible": false, + "reserved_housing_targets_read": false, + "result": 1, + "scope": "One native DE original P1 population/state-total plus CD119/PUMA source normalization control. Exact four input pins, block preservation and state reconciliation, full support readback. No national coverage, survey Frame, source admission or release acceptance.", + "selected_member": "NationalCD119.txt", + "selected_member_size_bytes": 163499110, + "source_and_owned_files": 497, + "source_control_accepted": false, + "status": "failed_pending_diagnosis", + "test_counts": { + "errors": 0, + "failures": 1, + "skipped": 0, + "tests": 1 + }, + "unexpected_refusals": {}, + "wall_seconds": 4.633959750004578 +} diff --git a/experiments/us-atomic-native-de-corrected-1-control-20260909.json b/experiments/us-atomic-native-de-corrected-1-control-20260909.json new file mode 100644 index 000000000..1fea35f10 --- /dev/null +++ b/experiments/us-atomic-native-de-corrected-1-control-20260909.json @@ -0,0 +1,175 @@ +{ + "code_resources_rechecked": 12, + "cpu_seconds": 9.85925, + "frozen_before_after_equal": true, + "frozen_code_current_equal": true, + "model_source_files": 5983, + "national_coverage": false, + "native_inputs_reopened_in_postcheck": false, + "native_resource_pins_receipt_only": 4, + "normalization": { + "exact_support_readback": true, + "national_coverage": false, + "protocol": "microcosm.us.native-de-atomic-api-control.v1", + "release_eligible": false, + "source_admission_issued": false, + "source_receipt": { + "exact_input_hashes_verified": true, + "limits": { + "max_line_bytes": 65536, + "max_member_bytes": 536870912, + "max_response_rows": 250000, + "max_source_bytes": 67108864, + "max_total_geography_bytes": 805306368, + "max_zip_members": 6 + }, + "protocol": "microcosm.us.atomic-block-api-sources.v1", + "publisher_provenance_established": false, + "reconciliation": { + "output_state_population": { + "10": 989948 + }, + "populated_blocks": 15317, + "population_total": 989948, + "selected_geography_bytes": 165910870, + "state_population": { + "10": 989948 + } + }, + "release_eligible": false, + "request_origin_verified": false, + "source_admission_issued": false, + "source_ids": { + "district": "census-2025-cd119-NationalCD119.txt", + "population": "census-2020-dec-pl-api-P1_001N", + "puma": "census-2020-Census-Tract-to-2020-PUMA" + }, + "sources": { + "district": { + "assigned_blocks": 8174799, + "crc_checked_members": [ + "NationalCD119.txt" + ], + "delegate_normalization": "98_to_00", + "delegate_records": 47999, + "district_relation": "official_tabulation", + "selected_member": "NationalCD119.txt", + "selected_member_sha256": "3c5afab5b61e2d7151206f977981702815692df6f3680e7624a6c3f93808fddf", + "selected_member_size_bytes": 163499110, + "sha256": "1433feb5178dc7b4188ee30f5f7f715851f4400740b8fe1ce606a876c6294bd6", + "size_bytes": 22959130, + "source_id": "census-2025-cd119-NationalCD119.txt", + "source_records": 8174955, + "unassigned_blocks": 156, + "unselected_member_contents_read": false, + "zip_members": [ + "01_AL_CD119.txt", + "13_GA_CD119.txt", + "22_LA_CD119.txt", + "36_NY_CD119.txt", + "37_NC_CD119.txt", + "NationalCD119.txt" + ] + }, + "population": [ + { + "block_rows": 20198, + "blocks": { + "request": { + "endpoint": "https://api.census.gov/data/2020/dec/pl", + "parameters": [ + [ + "get", + "P1_001N" + ], + [ + "for", + "block:*" + ], + [ + "in", + "state:10" + ], + [ + "in", + "county:*" + ], + [ + "in", + "tract:*" + ] + ] + }, + "sha256": "1a7a314a84aa0049adb0865d0bfd7466d808b36460f67eef6f3143a331fe6408", + "size_bytes": 702646 + }, + "populated_blocks": 15317, + "source_id": "census-2020-dec-pl-api-P1_001N", + "state_fips": "10", + "state_population": 989948, + "state_total": { + "request": { + "endpoint": "https://api.census.gov/data/2020/dec/pl", + "parameters": [ + [ + "get", + "P1_001N" + ], + [ + "for", + "state:10" + ] + ] + }, + "sha256": "ac40e616243c2ea14060dd460009e0a7f15755324a17387adf8768e35787fb79", + "size_bytes": 38 + }, + "zero_population_blocks": 4881 + } + ], + "puma": { + "selected_state_tract_mappings": 262, + "sha256": "5262f460b2c8dbe86b00549e916317b1791fdc595c9af72ab18f6f6a67040531", + "size_bytes": 1709076, + "source_id": "census-2020-Census-Tract-to-2020-PUMA" + } + }, + "state_fips": [ + "10" + ], + "support_sha256": "a0ef3f86be7c76306527ec2cecc70455a9d5b51af0a1c441d30fc532057db84d" + }, + "source_receipt_sha256": "b0783c16860e1333b271876a48f0b82d16c543813c93904c3eb460aecfc1111b", + "state_fips": [ + "10" + ], + "status": "source_control_passed", + "support_filename": "de-atomic-support.npz", + "support_sha256": "a0ef3f86be7c76306527ec2cecc70455a9d5b51af0a1c441d30fc532057db84d", + "support_size_bytes": 79769, + "survey_frame_created": false + }, + "normalization_metadata_sha256": "6fd38afe7e3b33995b13e14bc112827936a7871fd58b66a61e5b9b825350bb5d", + "peak_rss_bytes": 1734787072, + "receipt_sha256": "ea260a22f8701839aa0ac9beab98d82e17738cf9359570a0317e26338697d26e", + "release_eligible": false, + "result": 0, + "scope": "Correct only the native DE test archive roster to the six metadata-observed names; selected decompression/CRC remains solely NationalCD119.txt. No adapter, input, other source or guard behavior change. Failed original run preserved.", + "source_and_owned_files": 497, + "source_control_accepted": true, + "status": "DE_source_control_passed", + "test_counts": { + "errors": 0, + "failures": 0, + "skipped": 0, + "tests": 1 + }, + "tested_source_changes": { + "packages/microcosm-build/tests/test_us_atomic_block_api_sources_native_de.py": { + "after": "4b918e4d28044045faad6618dd931077b3b07086929c34aa31cff049fdf096a2", + "before": "e22a550f4e95592a6bd596f32ffd772d6f43117f284ce0eebe992b0de42ed7ab" + } + }, + "unexpected_refusals": {}, + "wall_seconds": 10.939260708983056 +} diff --git a/experiments/us-atomic-native-national-1-control-20260909.json b/experiments/us-atomic-native-national-1-control-20260909.json new file mode 100644 index 000000000..822ea0e7c --- /dev/null +++ b/experiments/us-atomic-native-national-1-control-20260909.json @@ -0,0 +1,3434 @@ +{ + "code_resources_rechecked": 12, + "cpu_seconds": 81.174622, + "frozen_before_after_equal": true, + "frozen_code_current_equal": true, + "model_source_files": 5983, + "native_inputs_reopened_in_postcheck": false, + "native_resource_pins_receipt_only": 104, + "normalization": { + "exact_support_readback": true, + "mapping_oracle": "separate maintained-parser pass; complete selected joins", + "national_coverage": true, + "population_oracle": { + "01": { + "block_rows": 185976, + "ordered_positive_block_population_sha256": "db629f189d2658bec941146c499464bec1d7278d83da0b07ac8ed338d5d993ab", + "populated_blocks": 128366, + "state_population": 5024279, + "zero_population_blocks": 57610 + }, + "02": { + "block_rows": 28568, + "ordered_positive_block_population_sha256": "c82ab4e84dfee99b16fc0397e3bf57d0ef5c87e1e37e5fc766879c690d0f4caf", + "populated_blocks": 11765, + "state_population": 733391, + "zero_population_blocks": 16803 + }, + "04": { + "block_rows": 155444, + "ordered_positive_block_population_sha256": "57059741545abff19005434c387543790c8cb047bd8453c17aada74bda370c46", + "populated_blocks": 107938, + "state_population": 7151502, + "zero_population_blocks": 47506 + }, + "05": { + "block_rows": 136422, + "ordered_positive_block_population_sha256": "af7158b0ab1d4e4baf46766a0953bfe74f52036272af13c4cfe6b65ef572c5ed", + "populated_blocks": 88121, + "state_population": 3011524, + "zero_population_blocks": 48301 + }, + "06": { + "block_rows": 519723, + "ordered_positive_block_population_sha256": "fc244613d2faab0aa55e0064fff67ee2df2cda32d8e1cfef0f934a6e80e41a68", + "populated_blocks": 377591, + "state_population": 39538223, + "zero_population_blocks": 142132 + }, + "08": { + "block_rows": 140345, + "ordered_positive_block_population_sha256": "d69d69546b402b60bd4eb98fbb4003417ddc3d26c442e6051437628dff7708a1", + "populated_blocks": 99899, + "state_population": 5773714, + "zero_population_blocks": 40446 + }, + "09": { + "block_rows": 49926, + "ordered_positive_block_population_sha256": "5be4f0b5fba158152132db2671dbb607f0c8233c1dfc532d5beb99d51965ee0d", + "populated_blocks": 42008, + "state_population": 3605944, + "zero_population_blocks": 7918 + }, + "10": { + "block_rows": 20198, + "ordered_positive_block_population_sha256": "f32d84817caa2c3f3ea776da426dbee30f049c82b9335af41699ec4bee00e235", + "populated_blocks": 15317, + "state_population": 989948, + "zero_population_blocks": 4881 + }, + "11": { + "block_rows": 6012, + "ordered_positive_block_population_sha256": "c0d37648dff6f3ba16d524e2e375417669565acab25027d64ec03b05c89c603d", + "populated_blocks": 4500, + "state_population": 689545, + "zero_population_blocks": 1512 + }, + "12": { + "block_rows": 390066, + "ordered_positive_block_population_sha256": "683fb2c4854b4c7ede8815bfc5620c3cf26a5d9149eab4bbd25fd4eedb5eb00e", + "populated_blocks": 289652, + "state_population": 21538187, + "zero_population_blocks": 100414 + }, + "13": { + "block_rows": 232717, + "ordered_positive_block_population_sha256": "dbc9e610217cd9e5de12fb3825addb9508ae368c1dc4a88b98801960ff91cdb0", + "populated_blocks": 165333, + "state_population": 10711908, + "zero_population_blocks": 67384 + }, + "15": { + "block_rows": 14732, + "ordered_positive_block_population_sha256": "245ccc4831fdcb75b1d2d5202f0d38c2ac9c1fab91e4916a61191729d4fbc88f", + "populated_blocks": 10169, + "state_population": 1455271, + "zero_population_blocks": 4563 + }, + "16": { + "block_rows": 81879, + "ordered_positive_block_population_sha256": "47cc73fe9b89c0db1c66621484f12b2a426f7332c289c28a0c30915ab7135dc8", + "populated_blocks": 46134, + "state_population": 1839106, + "zero_population_blocks": 35745 + }, + "17": { + "block_rows": 369978, + "ordered_positive_block_population_sha256": "bf7e305eae384c4338298d4b2dad71cfa48a5a0dc2858dda558b92bc26cbceaa", + "populated_blocks": 278166, + "state_population": 12812508, + "zero_population_blocks": 91812 + }, + "18": { + "block_rows": 204568, + "ordered_positive_block_population_sha256": "a85fae03b48ad4703a5ed7f39bb81eca4974df835d7c9e99590486fbaf11e163", + "populated_blocks": 162739, + "state_population": 6785528, + "zero_population_blocks": 41829 + }, + "19": { + "block_rows": 175199, + "ordered_positive_block_population_sha256": "f89589b4fd957d0643af40784431bd793d8ebfa05ceed97b3921689f424ee938", + "populated_blocks": 124175, + "state_population": 3190369, + "zero_population_blocks": 51024 + }, + "20": { + "block_rows": 172529, + "ordered_positive_block_population_sha256": "75877b15ff41b689be4774c7ddd9294701b97b1762fd4a935d10fce1cfb9c2e4", + "populated_blocks": 109149, + "state_population": 2937880, + "zero_population_blocks": 63380 + }, + "21": { + "block_rows": 132662, + "ordered_positive_block_population_sha256": "c7b86e0bef4455178c4da1b99b16e615454602062db58de9b9e8bf5a389c9954", + "populated_blocks": 91572, + "state_population": 4505836, + "zero_population_blocks": 41090 + }, + "22": { + "block_rows": 142874, + "ordered_positive_block_population_sha256": "fa3122a5aee3bfb4f78dba6ad64d582c1515ee045298e8f2359870ec8d884457", + "populated_blocks": 92180, + "state_population": 4657757, + "zero_population_blocks": 50694 + }, + "23": { + "block_rows": 47138, + "ordered_positive_block_population_sha256": "c870139aa6e05735c7f1ff1e05399995dc54a722dc2df99198059e7f89324f85", + "populated_blocks": 30548, + "state_population": 1362359, + "zero_population_blocks": 16590 + }, + "24": { + "block_rows": 83827, + "ordered_positive_block_population_sha256": "519b855f8fa876aed559a5c4b57458301e9aa1323bd7ae5575b916055abe00f4", + "populated_blocks": 65274, + "state_population": 6177224, + "zero_population_blocks": 18553 + }, + "25": { + "block_rows": 107278, + "ordered_positive_block_population_sha256": "aa920cf2bee71bf21a2ef68d76b04eb417c2c72fbcf468523fd032956917ac41", + "populated_blocks": 88207, + "state_population": 7029917, + "zero_population_blocks": 19071 + }, + "26": { + "block_rows": 254730, + "ordered_positive_block_population_sha256": "8e57ac1ca0ab6cafc9385f23b658a0eafd4f4e01f55468d42da327355d95a445", + "populated_blocks": 198244, + "state_population": 10077331, + "zero_population_blocks": 56486 + }, + "27": { + "block_rows": 198705, + "ordered_positive_block_population_sha256": "e616047478baea1a265c120762ebc1ad234531a1c1459e8a97263dfa5645b225", + "populated_blocks": 137689, + "state_population": 5706494, + "zero_population_blocks": 61016 + }, + "28": { + "block_rows": 112241, + "ordered_positive_block_population_sha256": "ab34034369ef3795f7a2a8112de09a48789c5921102f64681e8f772fe5f24fe8", + "populated_blocks": 77426, + "state_population": 2961279, + "zero_population_blocks": 34815 + }, + "29": { + "block_rows": 253632, + "ordered_positive_block_population_sha256": "ec441928aa5f5ef297b96c37db8f81132a0ac50399eada71edff432e0d922b03", + "populated_blocks": 169004, + "state_population": 6154913, + "zero_population_blocks": 84628 + }, + "30": { + "block_rows": 88417, + "ordered_positive_block_population_sha256": "b09a76edbea2619f38d2ad70a31108726f24abd23b17099a0ff010f07b48b43d", + "populated_blocks": 43717, + "state_population": 1084225, + "zero_population_blocks": 44700 + }, + "31": { + "block_rows": 119103, + "ordered_positive_block_population_sha256": "ab1afedaab5dd2392593270ce5fd7536d54bfe4cf99b34d53ce7a66aad96830f", + "populated_blocks": 79779, + "state_population": 1961504, + "zero_population_blocks": 39324 + }, + "32": { + "block_rows": 57409, + "ordered_positive_block_population_sha256": "760c7b9e590052d373316c9800545a20a71495b42f0d4b7a20b55b2fbf6f12f9", + "populated_blocks": 35249, + "state_population": 3104614, + "zero_population_blocks": 22160 + }, + "33": { + "block_rows": 31948, + "ordered_positive_block_population_sha256": "d88a28f2f35cf1b8ef64fd5d42600e416aad959de76e06de9ac76dd4e86a21a8", + "populated_blocks": 25317, + "state_population": 1377529, + "zero_population_blocks": 6631 + }, + "34": { + "block_rows": 137972, + "ordered_positive_block_population_sha256": "d924c85d9ae80d624ed2902e925a9c7dabb4a996680a081fbe237fcd9a1746d3", + "populated_blocks": 113212, + "state_population": 9288994, + "zero_population_blocks": 24760 + }, + "35": { + "block_rows": 107215, + "ordered_positive_block_population_sha256": "3905ad0b0d16d2c86502cfa4797024fd6d2f895efe0280379bb7963836ccab82", + "populated_blocks": 56605, + "state_population": 2117522, + "zero_population_blocks": 50610 + }, + "36": { + "block_rows": 288819, + "ordered_positive_block_population_sha256": "65a2be271d6a207e1f3e7cabedd6778d19d909ba0e37f5232bb010ad8a30b2f7", + "populated_blocks": 230339, + "state_population": 20201249, + "zero_population_blocks": 58480 + }, + "37": { + "block_rows": 236638, + "ordered_positive_block_population_sha256": "0c135c4919f7b904a298fbb183620e452e02ba9fca6644e27c2e8cf65e40ed50", + "populated_blocks": 174988, + "state_population": 10439388, + "zero_population_blocks": 61650 + }, + "38": { + "block_rows": 84566, + "ordered_positive_block_population_sha256": "fb4b3241fa5f1d81ed332353f90b0130b2383dd17e840a11badc1c911d155597", + "populated_blocks": 40198, + "state_population": 779094, + "zero_population_blocks": 44368 + }, + "39": { + "block_rows": 276428, + "ordered_positive_block_population_sha256": "a3e40aaf54aaaa347573881c39a06eed40a1e7d02b1e57a4a70fdfdaafb6b7fb", + "populated_blocks": 219669, + "state_population": 11799448, + "zero_population_blocks": 56759 + }, + "40": { + "block_rows": 180154, + "ordered_positive_block_population_sha256": "10240ff44682b047e63e5da0e4e8a5c9f6750d80b9a7f176ad397500f06492c1", + "populated_blocks": 121826, + "state_population": 3959353, + "zero_population_blocks": 58328 + }, + "41": { + "block_rows": 130807, + "ordered_positive_block_population_sha256": "01b6c50dbc1320f96741aa159354a3e42b318ee7e3bda21dac3d5f122e6799b0", + "populated_blocks": 79081, + "state_population": 4237256, + "zero_population_blocks": 51726 + }, + "42": { + "block_rows": 336985, + "ordered_positive_block_population_sha256": "70bb990a03502d1c2359e19a5f469727933661dcabfdcb9082c7c9f5ba34ee07", + "populated_blocks": 275023, + "state_population": 13002700, + "zero_population_blocks": 61962 + }, + "44": { + "block_rows": 25649, + "ordered_positive_block_population_sha256": "2777acda8ed305d3fcad00ec03bcc9d5b16d295a82fc8250caa73a0acf4184cd", + "populated_blocks": 21382, + "state_population": 1097379, + "zero_population_blocks": 4267 + }, + "45": { + "block_rows": 146844, + "ordered_positive_block_population_sha256": "d1a102de62516378b036f30c3642a17d3f2a4040b8a76e380b2f01f8b33d8a4f", + "populated_blocks": 105469, + "state_population": 5118425, + "zero_population_blocks": 41375 + }, + "46": { + "block_rows": 71383, + "ordered_positive_block_population_sha256": "8c372d4e82aff59d8701ecd211e24224885561645bedb7de74db55ca0c60205c", + "populated_blocks": 40766, + "state_population": 886667, + "zero_population_blocks": 30617 + }, + "47": { + "block_rows": 179717, + "ordered_positive_block_population_sha256": "f3804b33c433afd24a7b433c6524b0d124754d289cc2d589798fde5987794357", + "populated_blocks": 133846, + "state_population": 6910840, + "zero_population_blocks": 45871 + }, + "48": { + "block_rows": 668757, + "ordered_positive_block_population_sha256": "83903c280fd7fb16e9cc5a5ce7266753a0445118c8a3151bfadc86e27b891b3d", + "populated_blocks": 449085, + "state_population": 29145505, + "zero_population_blocks": 219672 + }, + "49": { + "block_rows": 71207, + "ordered_positive_block_population_sha256": "d1ea8f0e3bfa6225218b0f3bd0c022453fa3038a196ebef8a17dadb1be0ca575", + "populated_blocks": 46792, + "state_population": 3271616, + "zero_population_blocks": 24415 + }, + "50": { + "block_rows": 24611, + "ordered_positive_block_population_sha256": "3df326b12b20dbf6af14229fae0dc2e5733b4eabbe7a142515bebf0455881a93", + "populated_blocks": 17976, + "state_population": 643077, + "zero_population_blocks": 6635 + }, + "51": { + "block_rows": 163491, + "ordered_positive_block_population_sha256": "756a961291abef27225b26548e1fc67a58725285de9bbf4d93b04a618d7731bb", + "populated_blocks": 117361, + "state_population": 8631393, + "zero_population_blocks": 46130 + }, + "53": { + "block_rows": 158093, + "ordered_positive_block_population_sha256": "11bbff165b4c5004c526e0420dff07601a637a2d9b145288c86ffc476dcd6f5c", + "populated_blocks": 111441, + "state_population": 7705281, + "zero_population_blocks": 46652 + }, + "54": { + "block_rows": 72558, + "ordered_positive_block_population_sha256": "3fac3d7a2c520bcfb0e3ddf04776e2e147d157d58b8b8dd04cebb91ec89a8a6e", + "populated_blocks": 51850, + "state_population": 1793716, + "zero_population_blocks": 20708 + }, + "55": { + "block_rows": 203059, + "ordered_positive_block_population_sha256": "16f7d47ca52eced522acd712209a41215fd99ddb43f5a2232974b5ef6bc510de", + "populated_blocks": 144948, + "state_population": 5893718, + "zero_population_blocks": 58111 + }, + "56": { + "block_rows": 53769, + "ordered_positive_block_population_sha256": "33eac054e9aaf0d266460d4c4df67192a5ac7c993658ece720b398f868f45648", + "populated_blocks": 22857, + "state_population": 576851, + "zero_population_blocks": 30912 + } + }, + "protocol": "microcosm.us.native-national-atomic-api-control.v1", + "release_eligible": false, + "source_admission_issued": false, + "source_count": 104, + "source_read_limits": { + "mapping_verification_passes": 1, + "selected_cd_member_bytes_per_pass": 163499110, + "selected_geography_bytes_per_adapter_pass": 1237240934, + "unique_acquired_bytes": 1073741824 + }, + "source_receipt": { + "exact_input_hashes_verified": true, + "limits": { + "max_line_bytes": 65536, + "max_member_bytes": 163499110, + "max_response_rows": 2000000, + "max_source_bytes": 67108864, + "max_total_geography_bytes": 1237240934, + "max_zip_members": 6 + }, + "protocol": "microcosm.us.atomic-block-api-sources.v1", + "publisher_provenance_established": false, + "reconciliation": { + "output_state_population": { + "01": 5024279, + "02": 733391, + "04": 7151502, + "05": 3011524, + "06": 39538223, + "08": 5773714, + "09": 3605944, + "10": 989948, + "11": 689545, + "12": 21538187, + "13": 10711908, + "15": 1455271, + "16": 1839106, + "17": 12812508, + "18": 6785528, + "19": 3190369, + "20": 2937880, + "21": 4505836, + "22": 4657757, + "23": 1362359, + "24": 6177224, + "25": 7029917, + "26": 10077331, + "27": 5706494, + "28": 2961279, + "29": 6154913, + "30": 1084225, + "31": 1961504, + "32": 3104614, + "33": 1377529, + "34": 9288994, + "35": 2117522, + "36": 20201249, + "37": 10439388, + "38": 779094, + "39": 11799448, + "40": 3959353, + "41": 4237256, + "42": 13002700, + "44": 1097379, + "45": 5118425, + "46": 886667, + "47": 6910840, + "48": 29145505, + "49": 3271616, + "50": 643077, + "51": 8631393, + "53": 7705281, + "54": 1793716, + "55": 5893718, + "56": 576851 + }, + "populated_blocks": 5769942, + "population_total": 331449281, + "selected_geography_bytes": 447092359, + "state_population": { + "01": 5024279, + "02": 733391, + "04": 7151502, + "05": 3011524, + "06": 39538223, + "08": 5773714, + "09": 3605944, + "10": 989948, + "11": 689545, + "12": 21538187, + "13": 10711908, + "15": 1455271, + "16": 1839106, + "17": 12812508, + "18": 6785528, + "19": 3190369, + "20": 2937880, + "21": 4505836, + "22": 4657757, + "23": 1362359, + "24": 6177224, + "25": 7029917, + "26": 10077331, + "27": 5706494, + "28": 2961279, + "29": 6154913, + "30": 1084225, + "31": 1961504, + "32": 3104614, + "33": 1377529, + "34": 9288994, + "35": 2117522, + "36": 20201249, + "37": 10439388, + "38": 779094, + "39": 11799448, + "40": 3959353, + "41": 4237256, + "42": 13002700, + "44": 1097379, + "45": 5118425, + "46": 886667, + "47": 6910840, + "48": 29145505, + "49": 3271616, + "50": 643077, + "51": 8631393, + "53": 7705281, + "54": 1793716, + "55": 5893718, + "56": 576851 + } + }, + "release_eligible": false, + "request_origin_verified": false, + "source_admission_issued": false, + "source_ids": { + "district": "census-2025-cd119-NationalCD119.txt", + "population": "census-2020-dec-pl-api-P1_001N", + "puma": "census-2020-Census-Tract-to-2020-PUMA" + }, + "sources": { + "district": { + "assigned_blocks": 8174799, + "crc_checked_members": [ + "NationalCD119.txt" + ], + "delegate_normalization": "98_to_00", + "delegate_records": 47999, + "district_relation": "official_tabulation", + "selected_member": "NationalCD119.txt", + "selected_member_sha256": "3c5afab5b61e2d7151206f977981702815692df6f3680e7624a6c3f93808fddf", + "selected_member_size_bytes": 163499110, + "sha256": "1433feb5178dc7b4188ee30f5f7f715851f4400740b8fe1ce606a876c6294bd6", + "size_bytes": 22959130, + "source_id": "census-2025-cd119-NationalCD119.txt", + "source_records": 8174955, + "unassigned_blocks": 156, + "unselected_member_contents_read": false, + "zip_members": [ + "01_AL_CD119.txt", + "13_GA_CD119.txt", + "22_LA_CD119.txt", + "36_NY_CD119.txt", + "37_NC_CD119.txt", + "NationalCD119.txt" + ] + }, + "population": [ + { + "block_rows": 185976, + "blocks": { + "request": { + "endpoint": "https://api.census.gov/data/2020/dec/pl", + "parameters": [ + [ + "get", + "P1_001N" + ], + [ + "for", + "block:*" + ], + [ + "in", + "state:01" + ], + [ + "in", + "county:*" + ], + [ + "in", + "tract:*" + ] + ] + }, + "sha256": "2abb5d0be868baaa5d0b6c05490d832941084da51f1457090a10d9ff9657606d", + "size_bytes": 6427012 + }, + "populated_blocks": 128366, + "source_id": "census-2020-dec-pl-api-P1_001N", + "state_fips": "01", + "state_population": 5024279, + "state_total": { + "request": { + "endpoint": "https://api.census.gov/data/2020/dec/pl", + "parameters": [ + [ + "get", + "P1_001N" + ], + [ + "for", + "state:01" + ] + ] + }, + "sha256": "3a6d19e9e904b80afffb901a71bdda9cb5101bca1889dfc8efc50c8c39688ef5", + "size_bytes": 39 + }, + "zero_population_blocks": 57610 + }, + { + "block_rows": 28568, + "blocks": { + "request": { + "endpoint": "https://api.census.gov/data/2020/dec/pl", + "parameters": [ + [ + "get", + "P1_001N" + ], + [ + "for", + "block:*" + ], + [ + "in", + "state:02" + ], + [ + "in", + "county:*" + ], + [ + "in", + "tract:*" + ] + ] + }, + "sha256": "8534edc8dd05f33c878f33b7eb50ec359a3aff95198dbb91f95dc854fac10574", + "size_bytes": 982680 + }, + "populated_blocks": 11765, + "source_id": "census-2020-dec-pl-api-P1_001N", + "state_fips": "02", + "state_population": 733391, + "state_total": { + "request": { + "endpoint": "https://api.census.gov/data/2020/dec/pl", + "parameters": [ + [ + "get", + "P1_001N" + ], + [ + "for", + "state:02" + ] + ] + }, + "sha256": "ba9387573141f95e32d1440192e5a14efe431ab1e54ae7bbf0fe2433adf61d43", + "size_bytes": 38 + }, + "zero_population_blocks": 16803 + }, + { + "block_rows": 155444, + "blocks": { + "request": { + "endpoint": "https://api.census.gov/data/2020/dec/pl", + "parameters": [ + [ + "get", + "P1_001N" + ], + [ + "for", + "block:*" + ], + [ + "in", + "state:04" + ], + [ + "in", + "county:*" + ], + [ + "in", + "tract:*" + ] + ] + }, + "sha256": "81ea2d71faef8aaebb95bf986a87c905b09bfd30036ca5d096da67e91fd7c8cf", + "size_bytes": 5395609 + }, + "populated_blocks": 107938, + "source_id": "census-2020-dec-pl-api-P1_001N", + "state_fips": "04", + "state_population": 7151502, + "state_total": { + "request": { + "endpoint": "https://api.census.gov/data/2020/dec/pl", + "parameters": [ + [ + "get", + "P1_001N" + ], + [ + "for", + "state:04" + ] + ] + }, + "sha256": "b7ba8c426c875025aa324794719d978c3167e3a49e332e4af9b8fdf83b7a231c", + "size_bytes": 39 + }, + "zero_population_blocks": 47506 + }, + { + "block_rows": 136422, + "blocks": { + "request": { + "endpoint": "https://api.census.gov/data/2020/dec/pl", + "parameters": [ + [ + "get", + "P1_001N" + ], + [ + "for", + "block:*" + ], + [ + "in", + "state:05" + ], + [ + "in", + "county:*" + ], + [ + "in", + "tract:*" + ] + ] + }, + "sha256": "627457173db87dd14a2cfdb5464ac69941c306229423431962f5e2a23b4e522e", + "size_bytes": 4704516 + }, + "populated_blocks": 88121, + "source_id": "census-2020-dec-pl-api-P1_001N", + "state_fips": "05", + "state_population": 3011524, + "state_total": { + "request": { + "endpoint": "https://api.census.gov/data/2020/dec/pl", + "parameters": [ + [ + "get", + "P1_001N" + ], + [ + "for", + "state:05" + ] + ] + }, + "sha256": "b67fed8d924bca67ea48d091b4677828288b5d4b76cf57e1ce5777e7b55af6d4", + "size_bytes": 39 + }, + "zero_population_blocks": 48301 + }, + { + "block_rows": 519723, + "blocks": { + "request": { + "endpoint": "https://api.census.gov/data/2020/dec/pl", + "parameters": [ + [ + "get", + "P1_001N" + ], + [ + "for", + "block:*" + ], + [ + "in", + "state:06" + ], + [ + "in", + "county:*" + ], + [ + "in", + "tract:*" + ] + ] + }, + "sha256": "af233c9cf00396a0e61735430268f09df3301695d4790ec577c58b3df2e30957", + "size_bytes": 18139193 + }, + "populated_blocks": 377591, + "source_id": "census-2020-dec-pl-api-P1_001N", + "state_fips": "06", + "state_population": 39538223, + "state_total": { + "request": { + "endpoint": "https://api.census.gov/data/2020/dec/pl", + "parameters": [ + [ + "get", + "P1_001N" + ], + [ + "for", + "state:06" + ] + ] + }, + "sha256": "6bcb55b367bdde18380c87a526c18ebc7bd929c6100951942c7df517ddba02c2", + "size_bytes": 40 + }, + "zero_population_blocks": 142132 + }, + { + "block_rows": 140345, + "blocks": { + "request": { + "endpoint": "https://api.census.gov/data/2020/dec/pl", + "parameters": [ + [ + "get", + "P1_001N" + ], + [ + "for", + "block:*" + ], + [ + "in", + "state:08" + ], + [ + "in", + "county:*" + ], + [ + "in", + "tract:*" + ] + ] + }, + "sha256": "148ad5135f22c8410ee3baadc40c9f22cef8960423a05d4a4d9f940e44969ae9", + "size_bytes": 4866694 + }, + "populated_blocks": 99899, + "source_id": "census-2020-dec-pl-api-P1_001N", + "state_fips": "08", + "state_population": 5773714, + "state_total": { + "request": { + "endpoint": "https://api.census.gov/data/2020/dec/pl", + "parameters": [ + [ + "get", + "P1_001N" + ], + [ + "for", + "state:08" + ] + ] + }, + "sha256": "d730fcb0310c63c7ab6d6cd8c59474790be28953a3e791e1086f5d77feca9764", + "size_bytes": 39 + }, + "zero_population_blocks": 40446 + }, + { + "block_rows": 49926, + "blocks": { + "request": { + "endpoint": "https://api.census.gov/data/2020/dec/pl", + "parameters": [ + [ + "get", + "P1_001N" + ], + [ + "for", + "block:*" + ], + [ + "in", + "state:09" + ], + [ + "in", + "county:*" + ], + [ + "in", + "tract:*" + ] + ] + }, + "sha256": "ef40acb552ad9447cba791ccc0a24aed8f572891eb81ae4561ee0d61de836c54", + "size_bytes": 1747484 + }, + "populated_blocks": 42008, + "source_id": "census-2020-dec-pl-api-P1_001N", + "state_fips": "09", + "state_population": 3605944, + "state_total": { + "request": { + "endpoint": "https://api.census.gov/data/2020/dec/pl", + "parameters": [ + [ + "get", + "P1_001N" + ], + [ + "for", + "state:09" + ] + ] + }, + "sha256": "96a3d6fd5abfd6b32b44653adbcce8f1fe1e133a5cd474718d0d7aad7ec305fd", + "size_bytes": 39 + }, + "zero_population_blocks": 7918 + }, + { + "block_rows": 20198, + "blocks": { + "request": { + "endpoint": "https://api.census.gov/data/2020/dec/pl", + "parameters": [ + [ + "get", + "P1_001N" + ], + [ + "for", + "block:*" + ], + [ + "in", + "state:10" + ], + [ + "in", + "county:*" + ], + [ + "in", + "tract:*" + ] + ] + }, + "sha256": "1a7a314a84aa0049adb0865d0bfd7466d808b36460f67eef6f3143a331fe6408", + "size_bytes": 702646 + }, + "populated_blocks": 15317, + "source_id": "census-2020-dec-pl-api-P1_001N", + "state_fips": "10", + "state_population": 989948, + "state_total": { + "request": { + "endpoint": "https://api.census.gov/data/2020/dec/pl", + "parameters": [ + [ + "get", + "P1_001N" + ], + [ + "for", + "state:10" + ] + ] + }, + "sha256": "ac40e616243c2ea14060dd460009e0a7f15755324a17387adf8768e35787fb79", + "size_bytes": 38 + }, + "zero_population_blocks": 4881 + }, + { + "block_rows": 6012, + "blocks": { + "request": { + "endpoint": "https://api.census.gov/data/2020/dec/pl", + "parameters": [ + [ + "get", + "P1_001N" + ], + [ + "for", + "block:*" + ], + [ + "in", + "state:11" + ], + [ + "in", + "county:*" + ], + [ + "in", + "tract:*" + ] + ] + }, + "sha256": "d23c7ae418bc35270dc6eec54c014c92cd370aab7ff92de8168ff5c182db84a1", + "size_bytes": 210965 + }, + "populated_blocks": 4500, + "source_id": "census-2020-dec-pl-api-P1_001N", + "state_fips": "11", + "state_population": 689545, + "state_total": { + "request": { + "endpoint": "https://api.census.gov/data/2020/dec/pl", + "parameters": [ + [ + "get", + "P1_001N" + ], + [ + "for", + "state:11" + ] + ] + }, + "sha256": "fd791728a6b47cb82daf1fdaa156cf3a166815510f8057341e44bc8b9c36f61b", + "size_bytes": 38 + }, + "zero_population_blocks": 1512 + }, + { + "block_rows": 390066, + "blocks": { + "request": { + "endpoint": "https://api.census.gov/data/2020/dec/pl", + "parameters": [ + [ + "get", + "P1_001N" + ], + [ + "for", + "block:*" + ], + [ + "in", + "state:12" + ], + [ + "in", + "county:*" + ], + [ + "in", + "tract:*" + ] + ] + }, + "sha256": "b8c5633d903a040396c7f4d386aacb72e7c698253851fde6a115720bc90ed5e0", + "size_bytes": 13566697 + }, + "populated_blocks": 289652, + "source_id": "census-2020-dec-pl-api-P1_001N", + "state_fips": "12", + "state_population": 21538187, + "state_total": { + "request": { + "endpoint": "https://api.census.gov/data/2020/dec/pl", + "parameters": [ + [ + "get", + "P1_001N" + ], + [ + "for", + "state:12" + ] + ] + }, + "sha256": "41b9bcced940cfe83c8d27aea2c3ff69e0c48b3e7dbd88ded2fb6c1cd5b8bff2", + "size_bytes": 40 + }, + "zero_population_blocks": 100414 + }, + { + "block_rows": 232717, + "blocks": { + "request": { + "endpoint": "https://api.census.gov/data/2020/dec/pl", + "parameters": [ + [ + "get", + "P1_001N" + ], + [ + "for", + "block:*" + ], + [ + "in", + "state:13" + ], + [ + "in", + "county:*" + ], + [ + "in", + "tract:*" + ] + ] + }, + "sha256": "d2ee481c091a4c316fbef61bc62d069a884fbd52d658cd8a43de8fd4a1c2a90f", + "size_bytes": 8068936 + }, + "populated_blocks": 165333, + "source_id": "census-2020-dec-pl-api-P1_001N", + "state_fips": "13", + "state_population": 10711908, + "state_total": { + "request": { + "endpoint": "https://api.census.gov/data/2020/dec/pl", + "parameters": [ + [ + "get", + "P1_001N" + ], + [ + "for", + "state:13" + ] + ] + }, + "sha256": "a6bf2b3f49f8393f3ff7d1762b99d927d949d4c18e65c2995488ee88e4b7b8f5", + "size_bytes": 40 + }, + "zero_population_blocks": 67384 + }, + { + "block_rows": 14732, + "blocks": { + "request": { + "endpoint": "https://api.census.gov/data/2020/dec/pl", + "parameters": [ + [ + "get", + "P1_001N" + ], + [ + "for", + "block:*" + ], + [ + "in", + "state:15" + ], + [ + "in", + "county:*" + ], + [ + "in", + "tract:*" + ] + ] + }, + "sha256": "12360bccbca022409819393314c4f22d960d5d7f8edc302f3b9feefc73a011b4", + "size_bytes": 514556 + }, + "populated_blocks": 10169, + "source_id": "census-2020-dec-pl-api-P1_001N", + "state_fips": "15", + "state_population": 1455271, + "state_total": { + "request": { + "endpoint": "https://api.census.gov/data/2020/dec/pl", + "parameters": [ + [ + "get", + "P1_001N" + ], + [ + "for", + "state:15" + ] + ] + }, + "sha256": "7ed5585fbe97d5dab88b0ab10de66f30e90fa4a14b0e8989c02d32c2c4d631fd", + "size_bytes": 39 + }, + "zero_population_blocks": 4563 + }, + { + "block_rows": 81879, + "blocks": { + "request": { + "endpoint": "https://api.census.gov/data/2020/dec/pl", + "parameters": [ + [ + "get", + "P1_001N" + ], + [ + "for", + "block:*" + ], + [ + "in", + "state:16" + ], + [ + "in", + "county:*" + ], + [ + "in", + "tract:*" + ] + ] + }, + "sha256": "74ee9f18bddd2d67b90d93d808944edac3d410cc95a761a34980e3ce55f151a9", + "size_bytes": 2821287 + }, + "populated_blocks": 46134, + "source_id": "census-2020-dec-pl-api-P1_001N", + "state_fips": "16", + "state_population": 1839106, + "state_total": { + "request": { + "endpoint": "https://api.census.gov/data/2020/dec/pl", + "parameters": [ + [ + "get", + "P1_001N" + ], + [ + "for", + "state:16" + ] + ] + }, + "sha256": "8f1db7ed1b345ba86cdce141ab9d645c2cff91b88e0191fe557089fe0ceb4fbe", + "size_bytes": 39 + }, + "zero_population_blocks": 35745 + }, + { + "block_rows": 369978, + "blocks": { + "request": { + "endpoint": "https://api.census.gov/data/2020/dec/pl", + "parameters": [ + [ + "get", + "P1_001N" + ], + [ + "for", + "block:*" + ], + [ + "in", + "state:17" + ], + [ + "in", + "county:*" + ], + [ + "in", + "tract:*" + ] + ] + }, + "sha256": "6b0eed0c5ec1bcc326cffecc00c98d77a389fd3aa01b2163a99ca174088861ca", + "size_bytes": 12820271 + }, + "populated_blocks": 278166, + "source_id": "census-2020-dec-pl-api-P1_001N", + "state_fips": "17", + "state_population": 12812508, + "state_total": { + "request": { + "endpoint": "https://api.census.gov/data/2020/dec/pl", + "parameters": [ + [ + "get", + "P1_001N" + ], + [ + "for", + "state:17" + ] + ] + }, + "sha256": "c02690f4ec95f7f8a9f3bd3a994d5dee9877ad3c2d2e07de9eeb2ebcb9bd0757", + "size_bytes": 40 + }, + "zero_population_blocks": 91812 + }, + { + "block_rows": 204568, + "blocks": { + "request": { + "endpoint": "https://api.census.gov/data/2020/dec/pl", + "parameters": [ + [ + "get", + "P1_001N" + ], + [ + "for", + "block:*" + ], + [ + "in", + "state:18" + ], + [ + "in", + "county:*" + ], + [ + "in", + "tract:*" + ] + ] + }, + "sha256": "3c91aeb1afc98189affebfd1a5d6884cc03755e413e46b906a4b0d47240c9255", + "size_bytes": 7095958 + }, + "populated_blocks": 162739, + "source_id": "census-2020-dec-pl-api-P1_001N", + "state_fips": "18", + "state_population": 6785528, + "state_total": { + "request": { + "endpoint": "https://api.census.gov/data/2020/dec/pl", + "parameters": [ + [ + "get", + "P1_001N" + ], + [ + "for", + "state:18" + ] + ] + }, + "sha256": "572d8013dcc0fb51697478648a8c1e9ddf0c8ef0c2aa583a4eb44e5c81c2ddcc", + "size_bytes": 39 + }, + "zero_population_blocks": 41829 + }, + { + "block_rows": 175199, + "blocks": { + "request": { + "endpoint": "https://api.census.gov/data/2020/dec/pl", + "parameters": [ + [ + "get", + "P1_001N" + ], + [ + "for", + "block:*" + ], + [ + "in", + "state:19" + ], + [ + "in", + "county:*" + ], + [ + "in", + "tract:*" + ] + ] + }, + "sha256": "98f4efd90409774596b5bc623078d75f579d781ca3bc392b586a24fa67e638a1", + "size_bytes": 6033554 + }, + "populated_blocks": 124175, + "source_id": "census-2020-dec-pl-api-P1_001N", + "state_fips": "19", + "state_population": 3190369, + "state_total": { + "request": { + "endpoint": "https://api.census.gov/data/2020/dec/pl", + "parameters": [ + [ + "get", + "P1_001N" + ], + [ + "for", + "state:19" + ] + ] + }, + "sha256": "5cd722954a80d81e07f29fe65056cd0f40967c163935eb450320d4e73a5d4169", + "size_bytes": 39 + }, + "zero_population_blocks": 51024 + }, + { + "block_rows": 172529, + "blocks": { + "request": { + "endpoint": "https://api.census.gov/data/2020/dec/pl", + "parameters": [ + [ + "get", + "P1_001N" + ], + [ + "for", + "block:*" + ], + [ + "in", + "state:20" + ], + [ + "in", + "county:*" + ], + [ + "in", + "tract:*" + ] + ] + }, + "sha256": "35e21c30e14e0c4bfdb3645076cdc7a89ef4708c17373047526875be9cb0ab41", + "size_bytes": 5931398 + }, + "populated_blocks": 109149, + "source_id": "census-2020-dec-pl-api-P1_001N", + "state_fips": "20", + "state_population": 2937880, + "state_total": { + "request": { + "endpoint": "https://api.census.gov/data/2020/dec/pl", + "parameters": [ + [ + "get", + "P1_001N" + ], + [ + "for", + "state:20" + ] + ] + }, + "sha256": "aaaa97cb26fbc4db8318b32549da0a4f64a3fc4ca316ef9fcc5b2da6bb3b1c77", + "size_bytes": 39 + }, + "zero_population_blocks": 63380 + }, + { + "block_rows": 132662, + "blocks": { + "request": { + "endpoint": "https://api.census.gov/data/2020/dec/pl", + "parameters": [ + [ + "get", + "P1_001N" + ], + [ + "for", + "block:*" + ], + [ + "in", + "state:21" + ], + [ + "in", + "county:*" + ], + [ + "in", + "tract:*" + ] + ] + }, + "sha256": "975c9821955f9472f2222f85e089ce6ce9caff44a18bed9edd797ca13bda49c8", + "size_bytes": 4591236 + }, + "populated_blocks": 91572, + "source_id": "census-2020-dec-pl-api-P1_001N", + "state_fips": "21", + "state_population": 4505836, + "state_total": { + "request": { + "endpoint": "https://api.census.gov/data/2020/dec/pl", + "parameters": [ + [ + "get", + "P1_001N" + ], + [ + "for", + "state:21" + ] + ] + }, + "sha256": "c65f5deb2c5dacbcd8c1e84e4caf7a41458399f3a388425fbc5a7af69dba0bb9", + "size_bytes": 39 + }, + "zero_population_blocks": 41090 + }, + { + "block_rows": 142874, + "blocks": { + "request": { + "endpoint": "https://api.census.gov/data/2020/dec/pl", + "parameters": [ + [ + "get", + "P1_001N" + ], + [ + "for", + "block:*" + ], + [ + "in", + "state:22" + ], + [ + "in", + "county:*" + ], + [ + "in", + "tract:*" + ] + ] + }, + "sha256": "9bdacd2c2ccfa5bac93f5df47ec1e63fe39643c15633e7d33f464ad7a171d869", + "size_bytes": 4942488 + }, + "populated_blocks": 92180, + "source_id": "census-2020-dec-pl-api-P1_001N", + "state_fips": "22", + "state_population": 4657757, + "state_total": { + "request": { + "endpoint": "https://api.census.gov/data/2020/dec/pl", + "parameters": [ + [ + "get", + "P1_001N" + ], + [ + "for", + "state:22" + ] + ] + }, + "sha256": "db924ac57fd4177be3508d795ce912f3b55454f8eb6331fcc297252c2bb20a19", + "size_bytes": 39 + }, + "zero_population_blocks": 50694 + }, + { + "block_rows": 47138, + "blocks": { + "request": { + "endpoint": "https://api.census.gov/data/2020/dec/pl", + "parameters": [ + [ + "get", + "P1_001N" + ], + [ + "for", + "block:*" + ], + [ + "in", + "state:23" + ], + [ + "in", + "county:*" + ], + [ + "in", + "tract:*" + ] + ] + }, + "sha256": "eb26cdc0e114b07154096f592447df295d7fcda6dd890ebfe6e012d7d7bb40ca", + "size_bytes": 1629179 + }, + "populated_blocks": 30548, + "source_id": "census-2020-dec-pl-api-P1_001N", + "state_fips": "23", + "state_population": 1362359, + "state_total": { + "request": { + "endpoint": "https://api.census.gov/data/2020/dec/pl", + "parameters": [ + [ + "get", + "P1_001N" + ], + [ + "for", + "state:23" + ] + ] + }, + "sha256": "2a6bc352f6304d0ceb8850333a47378fa2a324a1163850d56931b424b8281089", + "size_bytes": 39 + }, + "zero_population_blocks": 16590 + }, + { + "block_rows": 83827, + "blocks": { + "request": { + "endpoint": "https://api.census.gov/data/2020/dec/pl", + "parameters": [ + [ + "get", + "P1_001N" + ], + [ + "for", + "block:*" + ], + [ + "in", + "state:24" + ], + [ + "in", + "county:*" + ], + [ + "in", + "tract:*" + ] + ] + }, + "sha256": "2d181f3e6bf3c96c8c2f458024a699be8e4c8b2bb80b6433ee95c69efe655b29", + "size_bytes": 2925422 + }, + "populated_blocks": 65274, + "source_id": "census-2020-dec-pl-api-P1_001N", + "state_fips": "24", + "state_population": 6177224, + "state_total": { + "request": { + "endpoint": "https://api.census.gov/data/2020/dec/pl", + "parameters": [ + [ + "get", + "P1_001N" + ], + [ + "for", + "state:24" + ] + ] + }, + "sha256": "b9de28b8819798a841c36ac13d35cdc75573c8e47ec232d195817239ab0bff55", + "size_bytes": 39 + }, + "zero_population_blocks": 18553 + }, + { + "block_rows": 107278, + "blocks": { + "request": { + "endpoint": "https://api.census.gov/data/2020/dec/pl", + "parameters": [ + [ + "get", + "P1_001N" + ], + [ + "for", + "block:*" + ], + [ + "in", + "state:25" + ], + [ + "in", + "county:*" + ], + [ + "in", + "tract:*" + ] + ] + }, + "sha256": "ca73c4d84aaf50e22ca9b25c4fdc54ca314167473a5ba1c9ee31fac1881396e3", + "size_bytes": 3749394 + }, + "populated_blocks": 88207, + "source_id": "census-2020-dec-pl-api-P1_001N", + "state_fips": "25", + "state_population": 7029917, + "state_total": { + "request": { + "endpoint": "https://api.census.gov/data/2020/dec/pl", + "parameters": [ + [ + "get", + "P1_001N" + ], + [ + "for", + "state:25" + ] + ] + }, + "sha256": "db5b65758be7c88e4bcafd4413f2813fa32de96307aa50a450f28bdc14e3bb39", + "size_bytes": 39 + }, + "zero_population_blocks": 19071 + }, + { + "block_rows": 254730, + "blocks": { + "request": { + "endpoint": "https://api.census.gov/data/2020/dec/pl", + "parameters": [ + [ + "get", + "P1_001N" + ], + [ + "for", + "block:*" + ], + [ + "in", + "state:26" + ], + [ + "in", + "county:*" + ], + [ + "in", + "tract:*" + ] + ] + }, + "sha256": "732595037f8d58aee0d01a3b68e71a18bc8a7144c81433d1dfc805a1a83cfe8d", + "size_bytes": 8845352 + }, + "populated_blocks": 198244, + "source_id": "census-2020-dec-pl-api-P1_001N", + "state_fips": "26", + "state_population": 10077331, + "state_total": { + "request": { + "endpoint": "https://api.census.gov/data/2020/dec/pl", + "parameters": [ + [ + "get", + "P1_001N" + ], + [ + "for", + "state:26" + ] + ] + }, + "sha256": "29a349d67a3c6219b9675e6cc5e5bd9a0bf3511e4b48005a6c9a9215be033069", + "size_bytes": 40 + }, + "zero_population_blocks": 56486 + }, + { + "block_rows": 198705, + "blocks": { + "request": { + "endpoint": "https://api.census.gov/data/2020/dec/pl", + "parameters": [ + [ + "get", + "P1_001N" + ], + [ + "for", + "block:*" + ], + [ + "in", + "state:27" + ], + [ + "in", + "county:*" + ], + [ + "in", + "tract:*" + ] + ] + }, + "sha256": "8910dd3e06035629bbef028441acb5d29f29ea555381df98076a05289bf29803", + "size_bytes": 6863252 + }, + "populated_blocks": 137689, + "source_id": "census-2020-dec-pl-api-P1_001N", + "state_fips": "27", + "state_population": 5706494, + "state_total": { + "request": { + "endpoint": "https://api.census.gov/data/2020/dec/pl", + "parameters": [ + [ + "get", + "P1_001N" + ], + [ + "for", + "state:27" + ] + ] + }, + "sha256": "13b45f3487425184637b01256644d5f86599d6a445b1a4f1d9fb86ef3790cef9", + "size_bytes": 39 + }, + "zero_population_blocks": 61016 + }, + { + "block_rows": 112241, + "blocks": { + "request": { + "endpoint": "https://api.census.gov/data/2020/dec/pl", + "parameters": [ + [ + "get", + "P1_001N" + ], + [ + "for", + "block:*" + ], + [ + "in", + "state:28" + ], + [ + "in", + "county:*" + ], + [ + "in", + "tract:*" + ] + ] + }, + "sha256": "33350d9a37d539dd4724dc7f4b969de0fd1dbfcb5207cae093b8ed73e0a67d68", + "size_bytes": 3878667 + }, + "populated_blocks": 77426, + "source_id": "census-2020-dec-pl-api-P1_001N", + "state_fips": "28", + "state_population": 2961279, + "state_total": { + "request": { + "endpoint": "https://api.census.gov/data/2020/dec/pl", + "parameters": [ + [ + "get", + "P1_001N" + ], + [ + "for", + "state:28" + ] + ] + }, + "sha256": "5cabdefebb5363751a244a9b12720ff43e9ac3a6fef7ba8c1ee7f79e7adaf745", + "size_bytes": 39 + }, + "zero_population_blocks": 34815 + }, + { + "block_rows": 253632, + "blocks": { + "request": { + "endpoint": "https://api.census.gov/data/2020/dec/pl", + "parameters": [ + [ + "get", + "P1_001N" + ], + [ + "for", + "block:*" + ], + [ + "in", + "state:29" + ], + [ + "in", + "county:*" + ], + [ + "in", + "tract:*" + ] + ] + }, + "sha256": "0e9b1a932c3cfb283a80a0e831192482ee9db6f01d7aad83de67a16e5c4075d5", + "size_bytes": 8749056 + }, + "populated_blocks": 169004, + "source_id": "census-2020-dec-pl-api-P1_001N", + "state_fips": "29", + "state_population": 6154913, + "state_total": { + "request": { + "endpoint": "https://api.census.gov/data/2020/dec/pl", + "parameters": [ + [ + "get", + "P1_001N" + ], + [ + "for", + "state:29" + ] + ] + }, + "sha256": "b75c227a323978f36e25e2efab8ec11b20579b06d6393da203cb36d533743a6d", + "size_bytes": 39 + }, + "zero_population_blocks": 84628 + }, + { + "block_rows": 88417, + "blocks": { + "request": { + "endpoint": "https://api.census.gov/data/2020/dec/pl", + "parameters": [ + [ + "get", + "P1_001N" + ], + [ + "for", + "block:*" + ], + [ + "in", + "state:30" + ], + [ + "in", + "county:*" + ], + [ + "in", + "tract:*" + ] + ] + }, + "sha256": "b9a81e34cc8c228fef203728f4134f96b7a8acfc1b14395e2de251d7ab0e3964", + "size_bytes": 3033388 + }, + "populated_blocks": 43717, + "source_id": "census-2020-dec-pl-api-P1_001N", + "state_fips": "30", + "state_population": 1084225, + "state_total": { + "request": { + "endpoint": "https://api.census.gov/data/2020/dec/pl", + "parameters": [ + [ + "get", + "P1_001N" + ], + [ + "for", + "state:30" + ] + ] + }, + "sha256": "caa4444d14b67700b3db4a5653412bd633c995df0d279473441b885a40102968", + "size_bytes": 39 + }, + "zero_population_blocks": 44700 + }, + { + "block_rows": 119103, + "blocks": { + "request": { + "endpoint": "https://api.census.gov/data/2020/dec/pl", + "parameters": [ + [ + "get", + "P1_001N" + ], + [ + "for", + "block:*" + ], + [ + "in", + "state:31" + ], + [ + "in", + "county:*" + ], + [ + "in", + "tract:*" + ] + ] + }, + "sha256": "bebe7ca15c979b170cfc3e6f49e27cdc50f5639de2f9a1955e60187849404e56", + "size_bytes": 4095921 + }, + "populated_blocks": 79779, + "source_id": "census-2020-dec-pl-api-P1_001N", + "state_fips": "31", + "state_population": 1961504, + "state_total": { + "request": { + "endpoint": "https://api.census.gov/data/2020/dec/pl", + "parameters": [ + [ + "get", + "P1_001N" + ], + [ + "for", + "state:31" + ] + ] + }, + "sha256": "30c8aae0e54d78eeb7857f0954be04c95c3a2278a030e78317f2a361fe40c683", + "size_bytes": 39 + }, + "zero_population_blocks": 39324 + }, + { + "block_rows": 57409, + "blocks": { + "request": { + "endpoint": "https://api.census.gov/data/2020/dec/pl", + "parameters": [ + [ + "get", + "P1_001N" + ], + [ + "for", + "block:*" + ], + [ + "in", + "state:32" + ], + [ + "in", + "county:*" + ], + [ + "in", + "tract:*" + ] + ] + }, + "sha256": "f34926bc3ada9c07a7ace640a787495814b9893c8351b34a9a0fe9bcf91acd75", + "size_bytes": 1991100 + }, + "populated_blocks": 35249, + "source_id": "census-2020-dec-pl-api-P1_001N", + "state_fips": "32", + "state_population": 3104614, + "state_total": { + "request": { + "endpoint": "https://api.census.gov/data/2020/dec/pl", + "parameters": [ + [ + "get", + "P1_001N" + ], + [ + "for", + "state:32" + ] + ] + }, + "sha256": "3f3957d218a7eb94ae360e03f6b5924d95ab0e8cb80b685d0fc74ff3de3349a7", + "size_bytes": 39 + }, + "zero_population_blocks": 22160 + }, + { + "block_rows": 31948, + "blocks": { + "request": { + "endpoint": "https://api.census.gov/data/2020/dec/pl", + "parameters": [ + [ + "get", + "P1_001N" + ], + [ + "for", + "block:*" + ], + [ + "in", + "state:33" + ], + [ + "in", + "county:*" + ], + [ + "in", + "tract:*" + ] + ] + }, + "sha256": "11e83119f4924fdbc1c0dcf6f96113c84ba4173c507290578835dfd2c7f97dfa", + "size_bytes": 1110353 + }, + "populated_blocks": 25317, + "source_id": "census-2020-dec-pl-api-P1_001N", + "state_fips": "33", + "state_population": 1377529, + "state_total": { + "request": { + "endpoint": "https://api.census.gov/data/2020/dec/pl", + "parameters": [ + [ + "get", + "P1_001N" + ], + [ + "for", + "state:33" + ] + ] + }, + "sha256": "b2c3dae17fcbdccaa8eddf6f3e5fdb742d4b0ec017084610958d5182c0f70ce7", + "size_bytes": 39 + }, + "zero_population_blocks": 6631 + }, + { + "block_rows": 137972, + "blocks": { + "request": { + "endpoint": "https://api.census.gov/data/2020/dec/pl", + "parameters": [ + [ + "get", + "P1_001N" + ], + [ + "for", + "block:*" + ], + [ + "in", + "state:34" + ], + [ + "in", + "county:*" + ], + [ + "in", + "tract:*" + ] + ] + }, + "sha256": "280a8f28f13545bbff7d432e2356cdaeaedd9e0ea026f7f05f4d9e83dd28886e", + "size_bytes": 4823492 + }, + "populated_blocks": 113212, + "source_id": "census-2020-dec-pl-api-P1_001N", + "state_fips": "34", + "state_population": 9288994, + "state_total": { + "request": { + "endpoint": "https://api.census.gov/data/2020/dec/pl", + "parameters": [ + [ + "get", + "P1_001N" + ], + [ + "for", + "state:34" + ] + ] + }, + "sha256": "517cdd27711beea019fa1efc18c6017690cd1d44617aa446a52677e94281d0cc", + "size_bytes": 39 + }, + "zero_population_blocks": 24760 + }, + { + "block_rows": 107215, + "blocks": { + "request": { + "endpoint": "https://api.census.gov/data/2020/dec/pl", + "parameters": [ + [ + "get", + "P1_001N" + ], + [ + "for", + "block:*" + ], + [ + "in", + "state:35" + ], + [ + "in", + "county:*" + ], + [ + "in", + "tract:*" + ] + ] + }, + "sha256": "a012da889bc274fb518c2a258c9b1394f43c24e6d58fb3661151565cc5ce8757", + "size_bytes": 3690294 + }, + "populated_blocks": 56605, + "source_id": "census-2020-dec-pl-api-P1_001N", + "state_fips": "35", + "state_population": 2117522, + "state_total": { + "request": { + "endpoint": "https://api.census.gov/data/2020/dec/pl", + "parameters": [ + [ + "get", + "P1_001N" + ], + [ + "for", + "state:35" + ] + ] + }, + "sha256": "9632c8400a3052e17d23dd71427ccf66e8c062b15cb58df2cb949ecb7a287e9e", + "size_bytes": 39 + }, + "zero_population_blocks": 50610 + }, + { + "block_rows": 288819, + "blocks": { + "request": { + "endpoint": "https://api.census.gov/data/2020/dec/pl", + "parameters": [ + [ + "get", + "P1_001N" + ], + [ + "for", + "block:*" + ], + [ + "in", + "state:36" + ], + [ + "in", + "county:*" + ], + [ + "in", + "tract:*" + ] + ] + }, + "sha256": "7a47a92501f986ebca42a27a180fe897b4afb2c37cd463360179a0b32500ab2d", + "size_bytes": 10067666 + }, + "populated_blocks": 230339, + "source_id": "census-2020-dec-pl-api-P1_001N", + "state_fips": "36", + "state_population": 20201249, + "state_total": { + "request": { + "endpoint": "https://api.census.gov/data/2020/dec/pl", + "parameters": [ + [ + "get", + "P1_001N" + ], + [ + "for", + "state:36" + ] + ] + }, + "sha256": "e34eac8cee690ba81ef7ad607cee5e06697834056165979faf2148627d5aeb5f", + "size_bytes": 40 + }, + "zero_population_blocks": 58480 + }, + { + "block_rows": 236638, + "blocks": { + "request": { + "endpoint": "https://api.census.gov/data/2020/dec/pl", + "parameters": [ + [ + "get", + "P1_001N" + ], + [ + "for", + "block:*" + ], + [ + "in", + "state:37" + ], + [ + "in", + "county:*" + ], + [ + "in", + "tract:*" + ] + ] + }, + "sha256": "3a721726b3ee7a8426411e5ba5267b0c0f88d4123916e8ed40d71e0a601eb797", + "size_bytes": 8216489 + }, + "populated_blocks": 174988, + "source_id": "census-2020-dec-pl-api-P1_001N", + "state_fips": "37", + "state_population": 10439388, + "state_total": { + "request": { + "endpoint": "https://api.census.gov/data/2020/dec/pl", + "parameters": [ + [ + "get", + "P1_001N" + ], + [ + "for", + "state:37" + ] + ] + }, + "sha256": "bf0b0c6a90da12c51a71942982ec8a10b8e1dece22601462dd0391c2dc758ba2", + "size_bytes": 40 + }, + "zero_population_blocks": 61650 + }, + { + "block_rows": 84566, + "blocks": { + "request": { + "endpoint": "https://api.census.gov/data/2020/dec/pl", + "parameters": [ + [ + "get", + "P1_001N" + ], + [ + "for", + "block:*" + ], + [ + "in", + "state:38" + ], + [ + "in", + "county:*" + ], + [ + "in", + "tract:*" + ] + ] + }, + "sha256": "6f8985d46a8d8e97fce2f563f20442f7c6e73a0503cddf58ff59746e829e2b68", + "size_bytes": 2892476 + }, + "populated_blocks": 40198, + "source_id": "census-2020-dec-pl-api-P1_001N", + "state_fips": "38", + "state_population": 779094, + "state_total": { + "request": { + "endpoint": "https://api.census.gov/data/2020/dec/pl", + "parameters": [ + [ + "get", + "P1_001N" + ], + [ + "for", + "state:38" + ] + ] + }, + "sha256": "3616ffbe27406d4e881587912b59c4494d2a0aa71132a25a6f8108bae23ccbb5", + "size_bytes": 38 + }, + "zero_population_blocks": 44368 + }, + { + "block_rows": 276428, + "blocks": { + "request": { + "endpoint": "https://api.census.gov/data/2020/dec/pl", + "parameters": [ + [ + "get", + "P1_001N" + ], + [ + "for", + "block:*" + ], + [ + "in", + "state:39" + ], + [ + "in", + "county:*" + ], + [ + "in", + "tract:*" + ] + ] + }, + "sha256": "aeb3eff352866ece982d0a31cfaf3ad63877bd8e2598bede9d883a0a168b450c", + "size_bytes": 9606700 + }, + "populated_blocks": 219669, + "source_id": "census-2020-dec-pl-api-P1_001N", + "state_fips": "39", + "state_population": 11799448, + "state_total": { + "request": { + "endpoint": "https://api.census.gov/data/2020/dec/pl", + "parameters": [ + [ + "get", + "P1_001N" + ], + [ + "for", + "state:39" + ] + ] + }, + "sha256": "3a93ca174df19c6f3377abf023177def2d8ff585b2182d56a9abd75b2e3454e3", + "size_bytes": 40 + }, + "zero_population_blocks": 56759 + }, + { + "block_rows": 180154, + "blocks": { + "request": { + "endpoint": "https://api.census.gov/data/2020/dec/pl", + "parameters": [ + [ + "get", + "P1_001N" + ], + [ + "for", + "block:*" + ], + [ + "in", + "state:40" + ], + [ + "in", + "county:*" + ], + [ + "in", + "tract:*" + ] + ] + }, + "sha256": "2078a38b5338f474099f7e1fe2ba0dba3cab95b1644db5abd6c52ca4a8a57703", + "size_bytes": 6212394 + }, + "populated_blocks": 121826, + "source_id": "census-2020-dec-pl-api-P1_001N", + "state_fips": "40", + "state_population": 3959353, + "state_total": { + "request": { + "endpoint": "https://api.census.gov/data/2020/dec/pl", + "parameters": [ + [ + "get", + "P1_001N" + ], + [ + "for", + "state:40" + ] + ] + }, + "sha256": "bd8b724c531ca5d6605aa1752c70c00a2666755070a18ef73ac0a3423d2812a5", + "size_bytes": 39 + }, + "zero_population_blocks": 58328 + }, + { + "block_rows": 130807, + "blocks": { + "request": { + "endpoint": "https://api.census.gov/data/2020/dec/pl", + "parameters": [ + [ + "get", + "P1_001N" + ], + [ + "for", + "block:*" + ], + [ + "in", + "state:41" + ], + [ + "in", + "county:*" + ], + [ + "in", + "tract:*" + ] + ] + }, + "sha256": "3cafb0c96a666767ea005ac1f08ca3171379934352c4d2ef285fd6ce44196ccd", + "size_bytes": 4521307 + }, + "populated_blocks": 79081, + "source_id": "census-2020-dec-pl-api-P1_001N", + "state_fips": "41", + "state_population": 4237256, + "state_total": { + "request": { + "endpoint": "https://api.census.gov/data/2020/dec/pl", + "parameters": [ + [ + "get", + "P1_001N" + ], + [ + "for", + "state:41" + ] + ] + }, + "sha256": "fb8d53cee89c2a14485cf42cf49b4723576407db7d550897609c1f5038b20e67", + "size_bytes": 39 + }, + "zero_population_blocks": 51726 + }, + { + "block_rows": 336985, + "blocks": { + "request": { + "endpoint": "https://api.census.gov/data/2020/dec/pl", + "parameters": [ + [ + "get", + "P1_001N" + ], + [ + "for", + "block:*" + ], + [ + "in", + "state:42" + ], + [ + "in", + "county:*" + ], + [ + "in", + "tract:*" + ] + ] + }, + "sha256": "34840439707bedac6bf76a35863287aa24263000ff2e1dca21ecdf0030b262f6", + "size_bytes": 11713716 + }, + "populated_blocks": 275023, + "source_id": "census-2020-dec-pl-api-P1_001N", + "state_fips": "42", + "state_population": 13002700, + "state_total": { + "request": { + "endpoint": "https://api.census.gov/data/2020/dec/pl", + "parameters": [ + [ + "get", + "P1_001N" + ], + [ + "for", + "state:42" + ] + ] + }, + "sha256": "b07244d4d81570c0c8bf49dc3653c382e8e3e74a2d798fb32c85e5808fd789a7", + "size_bytes": 40 + }, + "zero_population_blocks": 61962 + }, + { + "block_rows": 25649, + "blocks": { + "request": { + "endpoint": "https://api.census.gov/data/2020/dec/pl", + "parameters": [ + [ + "get", + "P1_001N" + ], + [ + "for", + "block:*" + ], + [ + "in", + "state:44" + ], + [ + "in", + "county:*" + ], + [ + "in", + "tract:*" + ] + ] + }, + "sha256": "8f8eb7ee08020dd7e3a49dab4d95f461acd5d29572fe8bf5b55f200ff84e7231", + "size_bytes": 894186 + }, + "populated_blocks": 21382, + "source_id": "census-2020-dec-pl-api-P1_001N", + "state_fips": "44", + "state_population": 1097379, + "state_total": { + "request": { + "endpoint": "https://api.census.gov/data/2020/dec/pl", + "parameters": [ + [ + "get", + "P1_001N" + ], + [ + "for", + "state:44" + ] + ] + }, + "sha256": "ec6f0565a6ebceccf1bf1a665f46a8105852b352762416b350f9630f802a8223", + "size_bytes": 39 + }, + "zero_population_blocks": 4267 + }, + { + "block_rows": 146844, + "blocks": { + "request": { + "endpoint": "https://api.census.gov/data/2020/dec/pl", + "parameters": [ + [ + "get", + "P1_001N" + ], + [ + "for", + "block:*" + ], + [ + "in", + "state:45" + ], + [ + "in", + "county:*" + ], + [ + "in", + "tract:*" + ] + ] + }, + "sha256": "a3daf2da71c547c02d8c260b009be331cd218c84896d602fafcdbc12acd59c3f", + "size_bytes": 5086274 + }, + "populated_blocks": 105469, + "source_id": "census-2020-dec-pl-api-P1_001N", + "state_fips": "45", + "state_population": 5118425, + "state_total": { + "request": { + "endpoint": "https://api.census.gov/data/2020/dec/pl", + "parameters": [ + [ + "get", + "P1_001N" + ], + [ + "for", + "state:45" + ] + ] + }, + "sha256": "3d15445808b86c1f785d6a1668f69eae773fbc75a462d3ead77c99bcc769ba9c", + "size_bytes": 39 + }, + "zero_population_blocks": 41375 + }, + { + "block_rows": 71383, + "blocks": { + "request": { + "endpoint": "https://api.census.gov/data/2020/dec/pl", + "parameters": [ + [ + "get", + "P1_001N" + ], + [ + "for", + "block:*" + ], + [ + "in", + "state:46" + ], + [ + "in", + "county:*" + ], + [ + "in", + "tract:*" + ] + ] + }, + "sha256": "1700e07604a6fb79d5cfb34fcb6bfe3d810429ecbca4e88d3012b8b2070a516d", + "size_bytes": 2447858 + }, + "populated_blocks": 40766, + "source_id": "census-2020-dec-pl-api-P1_001N", + "state_fips": "46", + "state_population": 886667, + "state_total": { + "request": { + "endpoint": "https://api.census.gov/data/2020/dec/pl", + "parameters": [ + [ + "get", + "P1_001N" + ], + [ + "for", + "state:46" + ] + ] + }, + "sha256": "e5b885739fae3a34f28cc11f739435e8cd39926ebeb0a2ef1a18ae3ff035becd", + "size_bytes": 38 + }, + "zero_population_blocks": 30617 + }, + { + "block_rows": 179717, + "blocks": { + "request": { + "endpoint": "https://api.census.gov/data/2020/dec/pl", + "parameters": [ + [ + "get", + "P1_001N" + ], + [ + "for", + "block:*" + ], + [ + "in", + "state:47" + ], + [ + "in", + "county:*" + ], + [ + "in", + "tract:*" + ] + ] + }, + "sha256": "d8522733ea6f5971d5f340d86f71f7174b54f502ddd84a79d703fbd72d343eb2", + "size_bytes": 6232909 + }, + "populated_blocks": 133846, + "source_id": "census-2020-dec-pl-api-P1_001N", + "state_fips": "47", + "state_population": 6910840, + "state_total": { + "request": { + "endpoint": "https://api.census.gov/data/2020/dec/pl", + "parameters": [ + [ + "get", + "P1_001N" + ], + [ + "for", + "state:47" + ] + ] + }, + "sha256": "32de25caad8e71a6bb74f5639f5e9eae1c95836f61743275c60b76f716d0c790", + "size_bytes": 39 + }, + "zero_population_blocks": 45871 + }, + { + "block_rows": 668757, + "blocks": { + "request": { + "endpoint": "https://api.census.gov/data/2020/dec/pl", + "parameters": [ + [ + "get", + "P1_001N" + ], + [ + "for", + "block:*" + ], + [ + "in", + "state:48" + ], + [ + "in", + "county:*" + ], + [ + "in", + "tract:*" + ] + ] + }, + "sha256": "2e997c530f51c17b595226374c50eb1ed249d2d6945e37dde9a1ceed8a161c6d", + "size_bytes": 23175159 + }, + "populated_blocks": 449085, + "source_id": "census-2020-dec-pl-api-P1_001N", + "state_fips": "48", + "state_population": 29145505, + "state_total": { + "request": { + "endpoint": "https://api.census.gov/data/2020/dec/pl", + "parameters": [ + [ + "get", + "P1_001N" + ], + [ + "for", + "state:48" + ] + ] + }, + "sha256": "469d7f5efd70e8918a3c9744d7c7742e190e54b52d5b331faf648907f5882b0b", + "size_bytes": 40 + }, + "zero_population_blocks": 219672 + }, + { + "block_rows": 71207, + "blocks": { + "request": { + "endpoint": "https://api.census.gov/data/2020/dec/pl", + "parameters": [ + [ + "get", + "P1_001N" + ], + [ + "for", + "block:*" + ], + [ + "in", + "state:49" + ], + [ + "in", + "county:*" + ], + [ + "in", + "tract:*" + ] + ] + }, + "sha256": "73207e9605fda4b336c404455c9a369ed64e39e7ecebd623b633835a9664e495", + "size_bytes": 2471351 + }, + "populated_blocks": 46792, + "source_id": "census-2020-dec-pl-api-P1_001N", + "state_fips": "49", + "state_population": 3271616, + "state_total": { + "request": { + "endpoint": "https://api.census.gov/data/2020/dec/pl", + "parameters": [ + [ + "get", + "P1_001N" + ], + [ + "for", + "state:49" + ] + ] + }, + "sha256": "825dab7318dcc582b32eb2723473be42d111a00d32f47d88ae93beade83e2fae", + "size_bytes": 39 + }, + "zero_population_blocks": 24415 + }, + { + "block_rows": 24611, + "blocks": { + "request": { + "endpoint": "https://api.census.gov/data/2020/dec/pl", + "parameters": [ + [ + "get", + "P1_001N" + ], + [ + "for", + "block:*" + ], + [ + "in", + "state:50" + ], + [ + "in", + "county:*" + ], + [ + "in", + "tract:*" + ] + ] + }, + "sha256": "7e34e58509cec16138d786dda2e34a45370fc1386108591182313f306b252daa", + "size_bytes": 850760 + }, + "populated_blocks": 17976, + "source_id": "census-2020-dec-pl-api-P1_001N", + "state_fips": "50", + "state_population": 643077, + "state_total": { + "request": { + "endpoint": "https://api.census.gov/data/2020/dec/pl", + "parameters": [ + [ + "get", + "P1_001N" + ], + [ + "for", + "state:50" + ] + ] + }, + "sha256": "c5ccb3d18ec85abae8ca3958cd4cca738812021cee35dd34be09150c07aa3c3d", + "size_bytes": 38 + }, + "zero_population_blocks": 6635 + }, + { + "block_rows": 163491, + "blocks": { + "request": { + "endpoint": "https://api.census.gov/data/2020/dec/pl", + "parameters": [ + [ + "get", + "P1_001N" + ], + [ + "for", + "block:*" + ], + [ + "in", + "state:51" + ], + [ + "in", + "county:*" + ], + [ + "in", + "tract:*" + ] + ] + }, + "sha256": "0d979951968f230cd5b7235f703e9acfabcafbeed99c4c018e34c0f88ba0ca52", + "size_bytes": 5678471 + }, + "populated_blocks": 117361, + "source_id": "census-2020-dec-pl-api-P1_001N", + "state_fips": "51", + "state_population": 8631393, + "state_total": { + "request": { + "endpoint": "https://api.census.gov/data/2020/dec/pl", + "parameters": [ + [ + "get", + "P1_001N" + ], + [ + "for", + "state:51" + ] + ] + }, + "sha256": "87ce18c978a4361494de558f74c03ce956079f6cfa8fb0401c8e77e5bf234c97", + "size_bytes": 39 + }, + "zero_population_blocks": 46130 + }, + { + "block_rows": 158093, + "blocks": { + "request": { + "endpoint": "https://api.census.gov/data/2020/dec/pl", + "parameters": [ + [ + "get", + "P1_001N" + ], + [ + "for", + "block:*" + ], + [ + "in", + "state:53" + ], + [ + "in", + "county:*" + ], + [ + "in", + "tract:*" + ] + ] + }, + "sha256": "0e3ec5282f8d894a504625cc9a366ada08430b60317fb5d529a982b0e2787a64", + "size_bytes": 5490219 + }, + "populated_blocks": 111441, + "source_id": "census-2020-dec-pl-api-P1_001N", + "state_fips": "53", + "state_population": 7705281, + "state_total": { + "request": { + "endpoint": "https://api.census.gov/data/2020/dec/pl", + "parameters": [ + [ + "get", + "P1_001N" + ], + [ + "for", + "state:53" + ] + ] + }, + "sha256": "ef5ad0f000c5505f923e4c91a5516b02138669c8b08b7a26c547cc856a7891e4", + "size_bytes": 39 + }, + "zero_population_blocks": 46652 + }, + { + "block_rows": 72558, + "blocks": { + "request": { + "endpoint": "https://api.census.gov/data/2020/dec/pl", + "parameters": [ + [ + "get", + "P1_001N" + ], + [ + "for", + "block:*" + ], + [ + "in", + "state:54" + ], + [ + "in", + "county:*" + ], + [ + "in", + "tract:*" + ] + ] + }, + "sha256": "28f1b09668784b1d8a064bd1433a1446e9697d8da2988c611bbc267ebee4456f", + "size_bytes": 2507211 + }, + "populated_blocks": 51850, + "source_id": "census-2020-dec-pl-api-P1_001N", + "state_fips": "54", + "state_population": 1793716, + "state_total": { + "request": { + "endpoint": "https://api.census.gov/data/2020/dec/pl", + "parameters": [ + [ + "get", + "P1_001N" + ], + [ + "for", + "state:54" + ] + ] + }, + "sha256": "b1cefc08915ac5c9bd05f47636d5ce6307b8ecb43c3c32502de2ec6d8b144c13", + "size_bytes": 39 + }, + "zero_population_blocks": 20708 + }, + { + "block_rows": 203059, + "blocks": { + "request": { + "endpoint": "https://api.census.gov/data/2020/dec/pl", + "parameters": [ + [ + "get", + "P1_001N" + ], + [ + "for", + "block:*" + ], + [ + "in", + "state:55" + ], + [ + "in", + "county:*" + ], + [ + "in", + "tract:*" + ] + ] + }, + "sha256": "fc491abf3fcd1a1fd3807ecde1d95b68bc2cab4e0a235a1b51f095ab5a2afcfa", + "size_bytes": 7025682 + }, + "populated_blocks": 144948, + "source_id": "census-2020-dec-pl-api-P1_001N", + "state_fips": "55", + "state_population": 5893718, + "state_total": { + "request": { + "endpoint": "https://api.census.gov/data/2020/dec/pl", + "parameters": [ + [ + "get", + "P1_001N" + ], + [ + "for", + "state:55" + ] + ] + }, + "sha256": "1c7bbd06d50dfe00b2e4461d2b92e98fac585f4b7b8c2425782bc822984fc5e9", + "size_bytes": 39 + }, + "zero_population_blocks": 58111 + }, + { + "block_rows": 53769, + "blocks": { + "request": { + "endpoint": "https://api.census.gov/data/2020/dec/pl", + "parameters": [ + [ + "get", + "P1_001N" + ], + [ + "for", + "block:*" + ], + [ + "in", + "state:56" + ], + [ + "in", + "county:*" + ], + [ + "in", + "tract:*" + ] + ] + }, + "sha256": "55aef5b66c91b47bda087b490e44a12067eabe7634b444351172e59ae126e66a", + "size_bytes": 1843303 + }, + "populated_blocks": 22857, + "source_id": "census-2020-dec-pl-api-P1_001N", + "state_fips": "56", + "state_population": 576851, + "state_total": { + "request": { + "endpoint": "https://api.census.gov/data/2020/dec/pl", + "parameters": [ + [ + "get", + "P1_001N" + ], + [ + "for", + "state:56" + ] + ] + }, + "sha256": "0684454d90b7e6430f6f55991791113091593c8dab5577ff8d6dcc2d1fa2e10d", + "size_bytes": 38 + }, + "zero_population_blocks": 30912 + } + ], + "puma": { + "selected_state_tract_mappings": 84414, + "sha256": "5262f460b2c8dbe86b00549e916317b1791fdc595c9af72ab18f6f6a67040531", + "size_bytes": 1709076, + "source_id": "census-2020-Census-Tract-to-2020-PUMA" + } + }, + "state_fips": [ + "01", + "02", + "04", + "05", + "06", + "08", + "09", + "10", + "11", + "12", + "13", + "15", + "16", + "17", + "18", + "19", + "20", + "21", + "22", + "23", + "24", + "25", + "26", + "27", + "28", + "29", + "30", + "31", + "32", + "33", + "34", + "35", + "36", + "37", + "38", + "39", + "40", + "41", + "42", + "44", + "45", + "46", + "47", + "48", + "49", + "50", + "51", + "53", + "54", + "55", + "56" + ], + "support_sha256": "5edc0e77471ba31d550a1eed416d5b46ada0a35425718eb87cfabe4d66fe4960" + }, + "source_receipt_sha256": "9adaa4365fc4624dfcb80d456783d5a54806e672384e17b4e84ad08d10bde74b", + "state_fips": [ + "01", + "02", + "04", + "05", + "06", + "08", + "09", + "10", + "11", + "12", + "13", + "15", + "16", + "17", + "18", + "19", + "20", + "21", + "22", + "23", + "24", + "25", + "26", + "27", + "28", + "29", + "30", + "31", + "32", + "33", + "34", + "35", + "36", + "37", + "38", + "39", + "40", + "41", + "42", + "44", + "45", + "46", + "47", + "48", + "49", + "50", + "51", + "53", + "54", + "55", + "56" + ], + "status": "source_control_passed", + "support_filename": "national-atomic-support.npz", + "support_sha256": "5edc0e77471ba31d550a1eed416d5b46ada0a35425718eb87cfabe4d66fe4960", + "support_size_bytes": 28862508, + "survey_frame_created": false + }, + "normalization_report_sha256": "1da2a391b58956de6541ac8e541d1b67f5d560b601da86caca3baca1859b8c45", + "peak_rss_bytes": 6813728768, + "receipt_sha256": "6c65dee6d9317e522a2076f690341889f867a28652ba18ddd6120bc6c2a80ad0", + "release_eligible": false, + "result": 0, + "scope": "One ordinary-acceptance national source-normalization case after separate root review and approval;104 population-only/geography pins, exact51 states, no survey Frame/source admission/release eligibility. Existing failed and accepted DE runs preserved.", + "source_and_owned_files": 498, + "source_control_accepted": true, + "test_counts": { + "errors": 0, + "failures": 0, + "skipped": 0, + "tests": 1 + }, + "unexpected_refusals": {}, + "wall_seconds": 82.97476216696668 +} diff --git a/experiments/us-atomic-survey-population-controls-20260909.json b/experiments/us-atomic-survey-population-controls-20260909.json new file mode 100644 index 000000000..9a162052a --- /dev/null +++ b/experiments/us-atomic-survey-population-controls-20260909.json @@ -0,0 +1,31 @@ +{ + "before_after_current_frozen_source_equal": true, + "code_resources": 12, + "cpu_seconds": 306.843367, + "model_source_files": 5983, + "native_geography_acceptance": false, + "native_payloads_read": 0, + "peak_rss_bytes": 440320000, + "pending_followups": [ + "Nonblocking support-file open correction and its FIFO control are tested separately.", + "Geography-aware calibration budget and age-runner integration are tested separately.", + "Publisher-qualified native block support and a native calibrated build remain pending." + ], + "publisher_provenance_established": false, + "receipt_sha256": "025bc482224c63c06020acd537c603235b0c612dc805d3b86deccc683624993e", + "release_eligible": false, + "result": 0, + "scope": "Complete survey atomic-geography wrapper on invented original source files: cold and required replay, raw/enriched allocation separation, geography-before-clone ordering, full Frame/owner/weight/design inheritance and late mutation refusals.", + "source_and_owned_files": 489, + "test_source_sha256": "762c386d5333514ef1cd12c1573c2e97970573ae519f6faedff8f9e736f094b9", + "tested_reconstruction_source_sha256": "6c707c55e9c6a1b67241f86807700856a6438e98428b8f5704f22ae175e40f89", + "tests": { + "errors": 0, + "failures": 0, + "skipped": 0, + "tests": 5 + }, + "unexpected_refusals": {}, + "wall_seconds": 315.1505964579992, + "wrapper_source_sha256": "794789a80298b9d239d707ba4cae605b61265846ad6dbc39330d0e1a28c2d1fc" +} diff --git a/experiments/us-budget-detached-view-correction-20260910.json b/experiments/us-budget-detached-view-correction-20260910.json new file mode 100644 index 000000000..894069e3b --- /dev/null +++ b/experiments/us-budget-detached-view-correction-20260910.json @@ -0,0 +1,39 @@ +{ + "adopted": true, + "behavior": "Move detached checked-view construction after last borrowed I/O; seal fresh document and complete maintained GroupedUpperBounds state, including stable-ID buckets, canonical digest and immutable storage, against issued payload and pure owner state.", + "budget_owner_sha256": "e1faffa1b133878b3607e888fe0c1fe0503f23d46be52eeddc054cf6c28a1c2c", + "callback_branches": { + "new_detached_values": 7, + "old_population_mutations": 2 + }, + "cpu_seconds": 540.759706, + "guard_sha256": "8c0b29efd7dc6c272b94463c996a9a6d35bb81c16857046bc75f95d11fb69087", + "historical_original_eight": { + "budget_owner_sha256": "80418528db45266e7322a8984cc63f9414f6fa3c084119cd9d0f558f4a0e0260", + "positive_postcheck_sha256": "8698b43174b17faa1bcc4123ae1b7c7c1ea65bba94f5d41f207c259dc3c951e8", + "remaining_postcheck_sha256": "423a6cd3ae1537bfe372b7d735ad3a907e6fcc6837257e6c65d538db18e13486" + }, + "native_acceptance": false, + "new_test_sha256": "4a8078e93d95cd6c807218fe021128614ce0567490ef216b7c83555a25b65a0e", + "peak_rss_bytes": 452624384, + "performance_claim": false, + "public_recipient_pilot_uses_original_frozen_budget": true, + "release_acceptance": false, + "root_postcheck_sha256": "2f94929c903c86633c199d3570625dbfda29fe57a84dab9b52502b25fe1236ed", + "scope": "Original eight financial-successor controls and four corrected budget-view controls are separate versioned runs; this is not whole-tree or native certification.", + "source_candidate_peer_sha256": "5f4ef9c2ecb9b19803b3439ec26b0bfdd92b19acc09e1579a6cc216240d35801", + "source_map_sha256": "9f5858b3bd7aceb9e9ee5ca0a2b5b4033ab9356a845364e9b07a11373de75b5f", + "status": "adopted_after_exact_component_verification", + "test_counts": { + "errors": 0, + "failures": 0, + "skipped": 0, + "tests": 4 + }, + "verified_current": { + "code_resources": 12, + "model_source": 5983, + "source_and_owned": 501 + }, + "wall_seconds": 543.4429369170102 +} diff --git a/experiments/us-budget-predictor-compatibility-1-controls-20260909.json b/experiments/us-budget-predictor-compatibility-1-controls-20260909.json new file mode 100644 index 000000000..2c4af43e8 --- /dev/null +++ b/experiments/us-budget-predictor-compatibility-1-controls-20260909.json @@ -0,0 +1,29 @@ +{ + "code_resources": 12, + "cpu_seconds": 35.759673, + "frozen_before_after_current_equal": true, + "model_source_files": 5983, + "native_payloads_read": 0, + "peak_rss_bytes": 519110656, + "publisher_provenance_established": false, + "receipt_sha256": "de102101abef2cb6238092df3cf2d8c87e885efec91349d130031919a5eabd4b", + "release_eligible": false, + "result": 0, + "scope": "Historical pair-return compatibility through both actual existing current-survey financial qualifiers; no model fitting. Invented data only; no native or publisher admission, no release.", + "source_and_owned_files": 492, + "test_counts": { + "errors": 0, + "failures": 0, + "skipped": 0, + "tests": 1 + }, + "tested_source_changes": { + "packages/microcosm-build/src/microcosm/build/us_runtime/survey_age_calibration.py": "a10b9f3552c84563d845675c772130073b925b5835d6730ffb9afc0e93170777", + "packages/microcosm-build/src/microcosm/build/us_runtime/survey_origin_budget.py": "c0667ad329af2f58901b0fd2f333489ad2366b83e2bac8ded95b51088d9e5d5e", + "packages/microcosm-build/tests/test_us_survey_age_calibration_atomic_geography.py": "25a414517974d345541bcec6152b61151b09ccc476a5350a36015c4b33dcd70e", + "packages/microcosm-build/tests/test_us_survey_origin_budget_atomic_geography.py": "d130335f5fb8d66e3e032984c326e1077b0a889162da8d97324f85866095905d", + "packages/microcosm-build/tests/test_us_survey_origin_budget_predictor_compatibility.py": "d81a82b9aedecd26d62831e93355c233093dc8fc3adec1e702d8c7f138fd8bbd" + }, + "unexpected_refusals": {}, + "wall_seconds": 38.938349917007145 +} diff --git a/experiments/us-calibration-consolidation-149-20260910.json b/experiments/us-calibration-consolidation-149-20260910.json new file mode 100644 index 000000000..166e89640 --- /dev/null +++ b/experiments/us-calibration-consolidation-149-20260910.json @@ -0,0 +1,50 @@ +{ + "cases": 149, + "checks": { + "copied_test_helper_fixture_files": 10, + "exact_case_and_junit_rosters": true, + "junit_selector_order": true, + "junit_unique_names": 149, + "owned_before_after_current_match": true, + "owned_files": 20, + "source_before_after_current_match": true, + "source_files": 34, + "terminal_pins_final_match": true + }, + "coverage": [ + "US grouped upper bounds, fixed zero support and unsupported combinations", + "UK informed gates and budget search", + "ordinary best iterate plus same-runtime frozen optimizer controls", + "dense/CSR combined behavior and strict diagnostic payload/rebuild" + ], + "cpu_seconds": 3.9523219999999997, + "current_34_source_subset_matches": true, + "date": "2026-09-10", + "excluded_control": "cross-runtime saved golden requiring18 interop threads; same-runtime frozen optimizer comparisons still executed", + "failed_predecessor_receipt_sha256": "240b17fdb30b8dfa0cddb3f58343fc4a1ba0bc14e44b852cd188cb1bd7a6703b", + "full_checkout_certified": false, + "guard_sha256": "c4218acbd403c1ff4b05f3cbb4d47654e1a1b51cbcf7f6b8b045c9252248aa80", + "junit_sha256": "57ba56887978b71aa86152f36f83484471a216829f4b7cca9b20c15ea1af9bee", + "native_calibration_accepted": false, + "peak_rss_bytes": 394067968, + "postchecker_sha256": "ee5bd078009c7842d7f9c19adf5452bc2c1c8c97a44c4a890c6fd4e7ee924bff", + "predecessor_failure": "collection stopped before tests: two still-denied stdlib ZIP probe counts exceeded their reporting ceilings", + "receipt_sha256": "242fa8cb76d5e6f1fa4589319bf6764c4f63620b3cda2e9e2a15f69549f362c3", + "release_acceptance": false, + "root_postcheck_sha256": "dca84da504acf53c65d68f7076ebe1ecf12a8d64a6479c40255c164717f3d10d", + "schema_version": "microcosm.scoped-calibration-control.v1", + "scope": "exact invented calibration149 source/runtime evidence only; no native, release or unrelated consolidated-checkout acceptance", + "selectors": 68, + "source_files": 34, + "source_map_sha256": "7de4ee1007343d72a3618cd95850700cb28bf87cd89405d956a0f92d37634850", + "status": "accepted_invented_calibration149", + "successor_change": "fresh once-only run; two reporting ceilings changed2 to3; exact same source, selectors, permissions and600CPU/900wall/one-thread limits", + "test_counts": { + "errors": 0, + "failures": 0, + "skipped": 0, + "tests": 149 + }, + "unexpected_refusals": {}, + "wall_seconds": 4.565558874979615 +} diff --git a/experiments/us-common-frame-export36-20260910.json b/experiments/us-common-frame-export36-20260910.json new file mode 100644 index 000000000..10c567b06 --- /dev/null +++ b/experiments/us-common-frame-export36-20260910.json @@ -0,0 +1,67 @@ +{ + "adoption_formatting": { + "full_ast_unchanged": true, + "test_source_after_sha256": "d6502d76a36796081e8c937c9b80caf74806558ed89f5466c6699f89e7b46c3c", + "test_source_before_sha256": "7d5184303390508ff745f3d6f4f7d8bdd4cbb10f2e50a574adda38e5bf71c9c0", + "tool": "ruff format" + }, + "component": "US supplied-parent full/pruned/local export comparison", + "fable_source_review": { + "blocking_findings": 0, + "model": "Fable 5.1", + "record_note": "files identifies adopted bytes; tested_files identifies exact executed bytes. Full AST equality after Ruff formatting is recorded above.", + "reviewed_commit": "b591280674db6d606c7ab356cabffbb6ef582e2d", + "scope": "Six-file source review; no runtime or independent hash recomputation by the peer", + "sha256": "361eb8abccc7b227c7c2aef2e1704e807076b6481ce73e501839adf358143416", + "verdict": "clean" + }, + "files": { + "packages/microcosm-build/src/microcosm/build/us_runtime/common_frame_export_contract.py": "206159abfad714563fa3434209092a05f8e847d9ffef99a913373331f2ecbb8c", + "packages/microcosm-build/tests/test_us_common_frame_export_contract.py": "d6502d76a36796081e8c937c9b80caf74806558ed89f5466c6699f89e7b46c3c" + }, + "guard_sha256": "f8d8184e582a443ccae781342df2b8c0accf4c87c552ccaed843f1bad0586bd1", + "independent_review": { + "correctness_defects": 0, + "coverage_note": "Both new prune-flag cases select the same retained rows; existing full/pruned/local cases exercise differing selections.", + "sha256": "8ef45f1d1fd5b3790e9d15d42d4dcb98651c1f66e5badb059134df623c91d17b" + }, + "limits": { + "cpu_seconds": 600, + "numeric_threads": 1, + "wall_seconds": 900 + }, + "physical_postcheck_sha256": "486eb981a71ca13806ed2524cc7a7b9469751ff47d2496e85b566044b7afc0a8", + "prior_attempts": "Preserved original all-zero fixture failure, unexecuted34 packet, and two36 wrapper failures. Final36v3 corrected invocation serialization and passed all checks.", + "scope": "Invented frames and real frame-checkpoint write/readback. No native candidate, actual calibration ancestry or release is certified.", + "source_closure_parity": { + "new_files": 2, + "pyproject_difference": "Comment describing UK NISRA openpyxl use; dependency configuration unchanged", + "pyproject_toml_semantics_equal": true, + "unchanged_python_files": 57 + }, + "source_map_sha256": "d6bb7e1404569ec27c5231b37d92397399fca7f63b85744ff393dc630454215e", + "source_revision_before_adoption": "162869d4d350b06c263b2fbdbeea93b7f1b05050", + "tested_files": { + "packages/microcosm-build/src/microcosm/build/us_runtime/common_frame_export_contract.py": "206159abfad714563fa3434209092a05f8e847d9ffef99a913373331f2ecbb8c", + "packages/microcosm-build/tests/test_us_common_frame_export_contract.py": "7d5184303390508ff745f3d6f4f7d8bdd4cbb10f2e50a574adda38e5bf71c9c0" + }, + "validation": { + "actual_graph_calibration_ancestry_verified": false, + "cpu_seconds": 1.769343, + "model_sources": 0, + "owned_files": 11, + "peak_rss_bytes": 324517888, + "physical_postcheck": "pass", + "receipt_sha256": "2b6de385041b6bb5d59ff34a2672d215de85bf0d058e6853f05a7afaf05b84f9", + "release_eligible": false, + "resources": 0, + "reviewer": "root", + "source_files": 60, + "suite_accepted": true, + "tests_failed": 0, + "tests_passed": 36, + "unexpected_refusals": {}, + "wall_seconds": 2.5767143329721875, + "xml_sha256": "3ab7cc10509aad988dd767ce39425138e9c8a67b171fcb433387057c2c1d28cb" + } +} diff --git a/experiments/us-current-acs-income-anchors-20260912.md b/experiments/us-current-acs-income-anchors-20260912.md new file mode 100644 index 000000000..17fc09e1d --- /dev/null +++ b/experiments/us-current-acs-income-anchors-20260912.md @@ -0,0 +1,67 @@ +# Qualifying observed ACS income anchors + +`qualify_current_acs_income_anchors` reads the original ACS person archive through +the retained survey preparation owner. It returns the selected ACS people's +original `INTP`, `RETP`, `ADJINC`, `AGEP`, `FINTP`, and `FRETP` literals, joined by +the original `SERIALNO` and normalized `SPORDER`. Household serial numbers are +checked through the native person's household link. The selected native person +axis and the preparation's stacked person axis remain explicit. + +These are broad survey anchors. They are not separately observed tax inputs. + +| Anchor | Source meaning | Qualification | +| --- | --- | --- | +| `INTP` | Interest, dividends, net rental, royalty, estate and trust income over the preceding 12 months | Preserve signed values, published zero, blank and malformed literals separately. | +| `RETP` | Broad retirement, survivor and disability income over the preceding 12 months, excluding Social Security | Preserve the nonnegative source amount without choosing a pension or retirement-account decomposition. | +| `ADJINC` | Factor for expressing income in the release's dollar year | Match the mapper's multiplication and division order and every retained nonmissing float64 bit. This adjustment does not convert the rolling reference period into a calendar year. | +| `AGEP` | Original source age | Use the original age for the age-15 income universe, before any downstream top-code mapping. | +| `FINTP`, `FRETP` | Allocation flags | Keep allocation separate from amount validity. A valid allocated amount remains known and explicitly labeled. | + +The [2024 ACS questionnaire, question 43](https://www2.census.gov/programs-surveys/acs/methodology/questionnaires/2024/quest24.pdf), +[2024 PUMS dictionary](https://www2.census.gov/programs-surveys/acs/tech_docs/pums/data_dict/PUMS_Data_Dictionary_2024.pdf) +(printed pages 38, 43, 127 and 129), and +[2024 subject definitions](https://www2.census.gov/programs-surveys/acs/tech_docs/subject_definitions/2024_ACSSubjectDefinitions.pdf) +establish these meanings. The released INTP domain contains zero, -10,000 through +-4, and 4 through 999,999; RETP contains zero and 4 through 999,999. Those disclosure +bounds describe released records, not bounds on latent income. + +The output distinguishes observed, missing, malformed, outside-domain, invalid +adjustment, and outside-universe records. It supplies no analytical zero for an +under-15 person or for an adult with a blank amount. The original literal survives +every classification, including missing or unrecognized allocation flags. Numeric +coercion reproduces the existing source mapper only for the retained-storage +comparison; it never supplies observation knownness. + +The qualifier exhausts a privately captured archive, checks its recorded hash and +complete source row count, and compares selected raw fields and adjusted anchors +to the retained native frame. After capture cleanup, it rechecks the actual +preparation and catalogue owners. A final physical seal also detects mutations +that table JSON's decimal precision cannot represent. Returned values, projection +bytes and evidence are descriptive: they cannot replace the original preparation +or grant source authority. A consuming graph host must retain that owner and the +physical value seal, check them immediately before consumption, and requalify +after its last relevant I/O before returning a successor. + +The 35-case invented-source suite passed with no failures, errors or skips in +56.261 seconds wall time (55.239 CPU seconds; 487,407,616 bytes peak RSS). It covers +the real preparation issuer, household and group-quarter records, selected-row +reordering and ID changes, signed and missing amounts, allocation states, exact +adjustment bits, a copied preparation, source mutation after capture I/O, and a +returned amount mutation hidden by JSON precision. The guarded run used 982 +source files and 15 allowed resources, with all 990 source-plus-owned hashes and +all resource hashes unchanged, no children, no unexpected refusals, and Torch +threads fixed at 1/1. The normal guard-induced dateutil zoneinfo warning occurred. + +The source SHA256 is +`94cb447fe209e8b801cd6080151a93c80bfb06bb736b068affb44d9d595c50a2`; +the test SHA256 is +`9bffafce6cdf04d63940e15052306037e0bee31049996ec561d46d8cb9e1a2bc`. +The local `codex-acs-income-anchor-20260912/source-v4/source-v4.json` receipt SHA256 +is `9154b624cf98def81bcf84fb814faf72d4a866d29243eacbeef12a6607c737b6`. +Earlier failed and superseded fixture evidence remains preserved. + +This change does not fit or reconcile components, add an enrichment host, choose +between ASEC interest definitions, or resolve the retirement donor bridge. It +does not execute native microdata, a country engine or a model. Those steps need +their own source and graph acceptance before this qualification can support a +release candidate. diff --git a/experiments/us-current-health-coverage-20260912.md b/experiments/us-current-health-coverage-20260912.md new file mode 100644 index 000000000..586f8a8af --- /dev/null +++ b/experiments/us-current-health-coverage-20260912.md @@ -0,0 +1,108 @@ +# Current health coverage on the survey multispine + +This lane adds source-qualified interview-date coverage to the common ACS+ASEC +frame and its PUF clones. It does not estimate eligibility, annual receipt, +insurance costs, or a common observation date. The current ASEC arm is income +year 2024 / survey year 2025; ACS coverage was observed during 2024. Older ASEC +income-year vintages are outside this first attachment. + +## Source definitions and mapping decisions + +The [2025 ASEC dictionary](https://www2.census.gov/programs-surveys/cps/techdocs/cpsmar25.pdf) +defines these current recodes for all persons, with 1=yes and 2=no. Corresponding +`I_NOW_*` flags distinguish reported, hot-deck, logical, and whole-unit imputation. +The [2024 ACS questionnaire, item 16](https://www2.census.gov/programs-surveys/acs/methodology/questionnaires/2024/quest24.pdf) +asks all respondents about current coverage. The [ACS dictionary](https://www2.census.gov/programs-surveys/acs/tech_docs/pums/data_dict/PUMS_Data_Dictionary_2024.pdf) +defines HINS1–7 (printed pages 37–38) and allocation/coverage-edit flags (126). + +| NATIONAL_CD input | ASEC source | ACS exact source in this first stage | +| --- | --- | --- | +| `has_esi` | `NOW_GRP` | `HINS1` | +| `has_marketplace_health_coverage_at_interview` | `NOW_MRK` | Gap: `HINS2` combines direct-purchase types | +| `has_non_marketplace_direct_purchase_health_coverage_at_interview` | `NOW_NONM` | Same gap | +| `has_medicaid_health_coverage_at_interview` | `NOW_CAID` | Gap: `HINS4` includes CHIP/other means-tested coverage | +| `has_other_means_tested_health_coverage_at_interview` | `NOW_OTHMT` | Same gap | +| `has_tricare_health_coverage_at_interview` | `NOW_MIL` | Gap: `HINS5` includes other military coverage | +| `has_champva_health_coverage_at_interview` | `NOW_CHAMPVA` | No separate ACS item | +| `has_va_health_coverage_at_interview` | `NOW_VACARE` | Keep `HINS6` distinct pending CHAMPVA recode reconciliation | +| `has_indian_health_service_coverage_at_interview` | `NOW_IHSFLG` | `HINS7` | + +The existing `cps_carried._fill_health_coverage_inputs` supplies the field roster +and most ASEC mappings, but its absent-column-to-zero helper is unsuitable here. +Its Medicaid mapping uses `NOW_MCAID`, the broader Medicaid/CHIP/other-means-tested +aggregate, rather than `NOW_CAID`. Both names and their distinction also appear +in the [2023](https://www2.census.gov/programs-surveys/cps/techdocs/cpsmar23.pdf) and +[2024](https://www2.census.gov/programs-surveys/cps/techdocs/cpsmar24.pdf) dictionaries. +The old pipeline remains untouched; this stage refuses ownership collisions. + +This narrower mapping is supported by the pinned consumer source, rather than +inferred from the input name alone. PolicyEngine-US 1.819.0 declares the Medicaid +input as person-level current Medicaid coverage, declares other means-tested +coverage separately, and uses the Medicaid input alongside `receives_medicaid` +for California Medi-Cal continuity. The checked files are +`variables/household/expense/health/has_medicaid_health_coverage_at_interview.py` +(SHA256 `0500c4e86120b540e6e2f09fc108123758d8f9863c0e4e4bf94b178dd50e09b8`) +and `variables/gov/states/ca/chhs/is_ca_medicaid_immigration_status_eligible.py` +(`6902d01792b8d0902c2693edb78f5ba2554a34f95d4073bd4e85e4ad69ce41a5`). +Only source text was inspected; this is not an engine execution or a blanket +reinterpretation of earlier datasets. Both `NOW_CAID` and `NOW_MCAID`, with their +allocation flags, remain in the graph. A later CMS comparison must declare +whether its target covers Medicaid alone or Medicaid plus CHIP. + +The [ACS subject definitions, health insurance](https://www2.census.gov/programs-surveys/acs/tech_docs/subject_definitions/2024_ACSSubjectDefinitions.pdf) +describe source editing and the 2019 change to current VA enrollment wording. +The [Census programming guide](https://www.census.gov/topics/health/health-insurance/guidance/programming-code/acs-estimates.html) +still groups CHAMPVA with HINS6. Consequently this stage does not assume that +HINS6 is interchangeable with the narrower ASEC VACARE recode. Raw HINS2–6 stay +inspectable; no missing concept becomes a false flag. `HIMRKS` measures subsidized +direct purchase, not all Marketplace enrollment, and cannot repair that split. + +## Graph and authority + +Borrow the original checked `SurveyPuf55Run` and its retained survey preparation. +Capture the exact original ASEC PERSON member and ACS person archive under the +existing native owners' pins. Match source household/person coordinates to the +preparation's original person ledger; never join by row position. Record the +source codes, allocation literals, status, knownness, source year and survey year. + +The fragment exposes a source projection artifact, a CREATE source population, +literal code columns, and nine named per-field recodes. CREATE requires the +existing `survey_population_source` reference from the original source registry; +it introduces no additional source file or issuer. A post-PUF attachment follows the retained +original-person link to both clones, verifies native channel/ID and clone roster, +and adds nullable booleans plus provenance. This stage has no fitted model. +Future disaggregation models need separate declarations and assessment. + +One fixed country enrichment host assembles the amount and health fragments over +the original checked PUF handle. The health fragment receives the explicit +receiving version and the amount attachment's typed artifact edge. Its ordinary +attachment preserves all incoming columns, including amount unknowns. It does +not return a separate whole-population branch or issue a public health handle. + +Portable values do not issue authority. The owning consumer must requalify the +retained parent immediately before consumption and after its last relevant I/O, +then seal the complete successor before returning it. The fragment checks its +retained physical values around each kernel callback; callback success itself +grants no source authority. The host owns source and kernel registries, typed +artifact/store bindings, replay checks, and complete successor verification. + +## Bounded verification + +The 31-case suite passes over invented records. It exercises the real source +catalogues, preparation issuers, original-member qualifier and requalification; +literal yes/no/allocation and unknown handling; exact source-coordinate joins; +clone identity and collision refusal; all prior amount-column preservation; +and a 13-node actual graph cold run plus required replay. Independent complete +population reconstruction matches every observed health-node output. The graph +fixture's qualified value is explicitly descriptive; it does not claim a new +source admission. A separate test obtains the qualified value through the real +issuer path using privately pinned invented source bytes. + +The source-v3 guarded run closed with result 0: 20.61 seconds wall, 19.95 seconds +CPU and 490.9 MB peak RSS. Earlier failed fixtures remain preserved, including +the origin-coordinate type mismatch exposed by the real qualification test. +No native microdata, country engine, public upload or model benchmark was used. +This verifies the fragment, not a launchable population. Actual-source replay, +the combined host's final retained-parent fence and release coverage checks +remain its consumer's responsibilities; the seven unresolved ACS concepts stay +explicit gaps. diff --git a/experiments/us-financial-default-2-controls-20260909.json b/experiments/us-financial-default-2-controls-20260909.json new file mode 100644 index 000000000..1d3d92fef --- /dev/null +++ b/experiments/us-financial-default-2-controls-20260909.json @@ -0,0 +1,27 @@ +{ + "code_resources": 12, + "cpu_seconds": 93.988173, + "frozen_before_after_current_equal": true, + "model_source_files": 5983, + "native_payloads_read": 0, + "peak_rss_bytes": 526712832, + "publisher_provenance_established": false, + "receipt_sha256": "d2775bd9e1c1e14dabaa9b7162070310fdd00adc8ea996084932692bce8f9dc9", + "release_eligible": false, + "result": 0, + "scope": "Two existing default financial predictor cold/required replay cases, with the accepted optional demographic implementation left disabled. Invented original sources only; no native/quality/release acceptance.", + "source_and_owned_files": 495, + "test_counts": { + "errors": 0, + "failures": 0, + "skipped": 0, + "tests": 2 + }, + "tested_source_changes": { + "packages/microcosm-build/src/microcosm/build/us_runtime/current_survey_predictors.py": "91c1a167dc6f188721e226623f3f92d3b0c57a4bf55af5f880f146cbe7c0596a", + "packages/microcosm-build/src/microcosm/build/us_runtime/graph_current_survey_predictors.py": "1c49e18612176337ab95170bb862578262aa702926148647db34e88154b24e15", + "packages/microcosm-build/tests/test_us_current_survey_predictor_demographics.py": "99d8062331165a72ee3dbad4812676cb240080de066102ccde7a2512f570e302" + }, + "unexpected_refusals": {}, + "wall_seconds": 98.86093041597633 +} diff --git a/experiments/us-financial-demographics-3-controls-20260909.json b/experiments/us-financial-demographics-3-controls-20260909.json new file mode 100644 index 000000000..fdfd17581 --- /dev/null +++ b/experiments/us-financial-demographics-3-controls-20260909.json @@ -0,0 +1,27 @@ +{ + "code_resources": 12, + "cpu_seconds": 111.759989, + "frozen_before_after_current_equal": true, + "model_source_files": 5983, + "native_payloads_read": 0, + "peak_rss_bytes": 528891904, + "publisher_provenance_established": false, + "receipt_sha256": "37b9c1ceeefe38028772d8e6de64d10728c5fbcc5c6339549bb3b07c1513a6c0", + "release_eligible": false, + "result": 0, + "scope": "Three invented financial-demographic qualification, graph cold/required replay, unknown/default compatibility, and late detached-projection mutation controls. No fit quality, native admission, or release acceptance.", + "source_and_owned_files": 495, + "test_counts": { + "errors": 0, + "failures": 0, + "skipped": 0, + "tests": 3 + }, + "tested_source_changes": { + "packages/microcosm-build/src/microcosm/build/us_runtime/current_survey_predictors.py": "91c1a167dc6f188721e226623f3f92d3b0c57a4bf55af5f880f146cbe7c0596a", + "packages/microcosm-build/src/microcosm/build/us_runtime/graph_current_survey_predictors.py": "1c49e18612176337ab95170bb862578262aa702926148647db34e88154b24e15", + "packages/microcosm-build/tests/test_us_current_survey_predictor_demographics.py": "99d8062331165a72ee3dbad4812676cb240080de066102ccde7a2512f570e302" + }, + "unexpected_refusals": {}, + "wall_seconds": 114.70848075003596 +} diff --git a/experiments/us-financial-fixture-profile-20260910.json b/experiments/us-financial-fixture-profile-20260910.json new file mode 100644 index 000000000..964ba4c2e --- /dev/null +++ b/experiments/us-financial-fixture-profile-20260910.json @@ -0,0 +1,248 @@ +{ + "accepted": true, + "approval_sha256": "6e53647ed6bb1af53c120b2c8f105f3517d6264476bb767547ae2dda1cf69bc7", + "cpu_seconds": 303.897467, + "declared_invented_dimensions": { + "atomic_support_blocks": 4, + "clone_households": 12, + "clone_persons": 18, + "counts_are_declared_from_frozen_source_not_native_measurements": true, + "demographic_predictor_count": 5, + "financial_output_count": 7, + "fixture_function": "known_financial_run", + "fixture_source": "packages/microcosm-build/tests/test_us_graph_atomic_survey_financial.py", + "graph_nodes_per_run": 20, + "n_estimators": 2, + "origin": "invented_original_source_fixture", + "runs": [ + "cold_auto", + "required_replay" + ], + "source_households": 6, + "source_persons": 9, + "state_fips_count": 2 + }, + "fixture_acquisition_completed": true, + "fixture_original_teardown_passed": true, + "guard_sha256": "e7972505720ee6ede8d3859e28e2c9171d4f568d477d90670371ca17a36167d1", + "native_acceptance": false, + "native_reads_stats_hashes": false, + "next_step": "Review and test per-producer code-comparison memoization without removing live globals, aliases, closure checks, source reads or final I/O seals. No optimization or larger run accepted by this diagnostic.", + "partial_snapshot_needed": false, + "peak_rss_bytes": 594296832, + "peer_source_review_sha256": "6b4614e182df7fa519a1079a6c5d99ded55bbdc45f021b46908e041c8895f4af", + "profile_scope": "fixture_acquisition_only", + "profile_sha256": "badec8594bb1421483e0090f273f79f7560adc0b13b68434f089442c7c8d4bbb", + "profiled_implementation_sha256": { + "packages/microcosm-build/src/microcosm/build/us_runtime/acs_native_coverage_binding.py": "db7aaae06961cc1ed9ce13b28dac74d8fd1041b9cd4c32294d2b11105fbef7af", + "packages/microcosm-build/src/microcosm/build/us_runtime/graph_atomic_survey_financial.py": "06776801268115604cd3e544381a3e4ce81588876fd10bb9a8374bab03ba7f53" + }, + "profiled_seconds": 298.15339916600004, + "protocol": "microcosm.us.financial-fixture-profile-evidence.v1", + "receipt_sha256": "784d46a3a5ed3994f1018f6d05900122abdc6f63a0f18e318002fedfe7059af9", + "release_acceptance": false, + "runtime_result": 0, + "scope": "invented_financial_fixture_profile", + "successor_acceptance": false, + "test_counts": { + "errors": 0, + "failures": 0, + "skipped": 0, + "tests": 1 + }, + "timing_limitations": [ + "cProfile overhead changes elapsed time and can trigger the unchanged limits", + "partial snapshots omit unfinished active-call time until profiler flush", + "only exact frozen-source coordinates are reported; external time remains within cumulative callers", + "cumulative function times overlap and must not be summed" + ], + "top_functions": [ + { + "calls": 1, + "cumulative_seconds": 298.15339916600004, + "file": "packages/microcosm-build/tests/test_us_graph_atomic_survey_financial.py", + "function": "known_financial_run", + "line": 19, + "recursive_calls": 0, + "self_seconds": 0.004364749 + }, + { + "calls": 2, + "cumulative_seconds": 297.206106625, + "file": "packages/microcosm-build/src/microcosm/build/us_runtime/graph_atomic_survey_financial.py", + "function": "run_atomic_survey_financial", + "line": 538, + "recursive_calls": 0, + "self_seconds": 0.0062629930000000006 + }, + { + "calls": 110, + "cumulative_seconds": 212.956586461, + "file": "packages/microcosm-build/src/microcosm/build/us_runtime/survey_population_preparation.py", + "function": "_validate", + "line": 1071, + "recursive_calls": 0, + "self_seconds": 0.004810231 + }, + { + "calls": 108, + "cumulative_seconds": 209.131314091, + "file": "packages/microcosm-build/src/microcosm/build/us_runtime/survey_population_preparation.py", + "function": "_checked", + "line": 1091, + "recursive_calls": 0, + "self_seconds": 0.0005602070000000001 + }, + { + "calls": 652, + "cumulative_seconds": 180.19579690400002, + "file": "packages/microcosm-build/src/microcosm/build/us_runtime/acs_native_coverage_binding.py", + "function": "_producer", + "line": 129, + "recursive_calls": 0, + "self_seconds": 5.073519417 + }, + { + "calls": 14, + "cumulative_seconds": 161.371121044, + "file": "packages/microcosm-build/src/microcosm/build/us_runtime/survey_atomic_geography.py", + "function": "reconstruct_atomic_survey_geography", + "line": 307, + "recursive_calls": 0, + "self_seconds": 0.007554145000000001 + }, + { + "calls": 38464, + "cumulative_seconds": 151.50668402300002, + "file": "packages/microcosm-build/src/microcosm/build/us_runtime/acs_native_coverage_binding.py", + "function": "_live_code", + "line": 51, + "recursive_calls": 0, + "self_seconds": 9.209266879000001 + }, + { + "calls": 8, + "cumulative_seconds": 133.111351877, + "file": "packages/microcosm-build/src/microcosm/build/us_runtime/current_survey_predictors.py", + "function": "qualify_current_survey_predictors", + "line": 304, + "recursive_calls": 0, + "self_seconds": 0.006935434000000001 + }, + { + "calls": 2, + "cumulative_seconds": 115.295601876, + "file": "packages/microcosm-build/src/microcosm/build/us_runtime/graph_atomic_survey_population.py", + "function": "run_atomic_survey_population", + "line": 213, + "recursive_calls": 0, + "self_seconds": 0.001364704 + }, + { + "calls": 6, + "cumulative_seconds": 112.51926329300001, + "file": "packages/microcosm-graph/src/microcosm/graph/executor.py", + "function": "run_graph", + "line": 2158, + "recursive_calls": 0, + "self_seconds": 0.008178516 + }, + { + "calls": 15, + "cumulative_seconds": 109.250808543, + "file": "packages/microcosm-build/src/microcosm/build/us_runtime/current_survey_geography.py", + "function": "qualify_current_survey_geography", + "line": 220, + "recursive_calls": 0, + "self_seconds": 0.0037998010000000002 + }, + { + "calls": 348, + "cumulative_seconds": 103.018557294, + "file": "packages/microcosm-build/src/microcosm/build/us_runtime/acs_population_catalogue.py", + "function": "_producer", + "line": 102, + "recursive_calls": 0, + "self_seconds": 0.14776667400000001 + }, + { + "calls": 2009108, + "cumulative_seconds": 95.553654135, + "file": "packages/microcosm-build/src/microcosm/build/us_runtime/acs_native_coverage_binding.py", + "function": "check", + "line": 86, + "recursive_calls": 51772, + "self_seconds": 91.045168105 + }, + { + "calls": 8, + "cumulative_seconds": 93.21181225100001, + "file": "packages/microcosm-build/src/microcosm/build/us_runtime/survey_origin_budget.py", + "function": "_initial", + "line": 355, + "recursive_calls": 0, + "self_seconds": 0.0020371490000000003 + }, + { + "calls": 150, + "cumulative_seconds": 92.18578382800001, + "file": "packages/microcosm-build/src/microcosm/build/us_runtime/acs_native_coverage_binding.py", + "function": "verify_acs_native_coverage", + "line": 394, + "recursive_calls": 0, + "self_seconds": 0.011203179 + }, + { + "calls": 116, + "cumulative_seconds": 69.695211884, + "file": "packages/microcosm-build/src/microcosm/build/us_runtime/acs_population_catalogue.py", + "function": "_checked", + "line": 406, + "recursive_calls": 0, + "self_seconds": 0.0037891110000000004 + }, + { + "calls": 4, + "cumulative_seconds": 66.760753874, + "file": "packages/microcosm-build/src/microcosm/build/us_runtime/graph_current_survey_predictors.py", + "function": "_qualified", + "line": 269, + "recursive_calls": 0, + "self_seconds": 0.000840784 + }, + { + "calls": 110, + "cumulative_seconds": 66.121036298, + "file": "packages/microcosm-build/src/microcosm/build/us_runtime/acs_population_catalogue.py", + "function": "verify_acs_source_catalogue", + "line": 430, + "recursive_calls": 0, + "self_seconds": 0.00016753900000000002 + }, + { + "calls": 23, + "cumulative_seconds": 49.949269957000006, + "file": "packages/microcosm-build/src/microcosm/build/us_runtime/current_asec_demographics.py", + "function": "qualify_current_asec_demographics", + "line": 194, + "recursive_calls": 0, + "self_seconds": 0.0047751980000000005 + }, + { + "calls": 2, + "cumulative_seconds": 41.477013541000005, + "file": "packages/microcosm-build/src/microcosm/build/us_runtime/graph_survey_population.py", + "function": "run_authenticated_survey_population", + "line": 815, + "recursive_calls": 0, + "self_seconds": 0.0007954430000000001 + } + ], + "unexpected_refusals": {}, + "verified_after_run": { + "code_resource_files": 12, + "model_source_files": 5983, + "source_and_owned_files": 500 + }, + "wall_seconds": 308.2577062920318 +} diff --git a/experiments/us-financial-successor-positive-1-limit-stop-20260909.json b/experiments/us-financial-successor-positive-1-limit-stop-20260909.json new file mode 100644 index 000000000..c7b05b458 --- /dev/null +++ b/experiments/us-financial-successor-positive-1-limit-stop-20260909.json @@ -0,0 +1,34 @@ +{ + "all_expected_source_code_and_model_pins_match_after_stop": true, + "automatic_retry": false, + "code_resources": 12, + "completed_test_count": null, + "configured_cpu_seconds": 600, + "configured_wall_seconds": 900, + "control_accepted": false, + "expected_test_count": 1, + "final_runtime_receipt_written": false, + "guard_sha256": "d65bd3f82a64791a0720dab30a2938456a90757b6c9aa6cfa4a30662edab1b6b", + "in_process_final_verification_completed": false, + "model_source_files": 5983, + "model_source_map_sha256": "018d739504f5919c65104ad44a46c1df045d0bf71002622e66e8b68c3192b035", + "native_inputs_read": false, + "native_threads": 1, + "observed_cpu_seconds": null, + "observed_peak_rss_bytes": null, + "other_selections_executed": false, + "process_exit_code": 152, + "release_eligible": false, + "resource_map_sha256": "c6a557ab9d9231457a7c9f0d1c4a19d2ea354bcdeb7063812ddc88a3b3cf6ef4", + "result": "incomplete_resource_limit_stop", + "scope": "Positive invented financial successor cold/required graph, original budget, admitted financial ancestry, weight-only transition and unchanged default path. CPU-limit stop before final receipt; no pass or new successor/native acceptance. Next action is separately reviewed bounded profiling, not a budget increase.", + "source_files": 491, + "source_map_sha256": "44d9306d0964a4b72490a07895a080b129b83c251866c2aeac06c74deb4ca762", + "termination_signal": "SIGXCPU", + "tested_source_changes": { + "packages/microcosm-build/src/microcosm/build/us_runtime/graph_atomic_survey_financial.py": "06776801268115604cd3e544381a3e4ce81588876fd10bb9a8374bab03ba7f53", + "packages/microcosm-build/src/microcosm/build/us_runtime/survey_financial_successor.py": "613bf949a73ed4d0a6c9f92f01fb48a64b426db2825d719ccf023e938c5c18d7", + "packages/microcosm-build/src/microcosm/build/us_runtime/survey_origin_budget.py": "80418528db45266e7322a8984cc63f9414f6fa3c084119cd9d0f558f4a0e0260", + "packages/microcosm-build/tests/test_us_survey_financial_successor.py": "a42e7dadf3b77f6cf976db9c904243e787f8704bbc6d1a82bd820246f3c2e980" + } +} diff --git a/experiments/us-financial-successor-positive-20260910.json b/experiments/us-financial-successor-positive-20260910.json new file mode 100644 index 000000000..fa1a8d651 --- /dev/null +++ b/experiments/us-financial-successor-positive-20260910.json @@ -0,0 +1,41 @@ +{ + "all_successor_controls_passed": false, + "approval_sha256": "91efa2d737f38d6d6b11890241a1a9b5dbb1c6840da3a49f58966101e3f967ef", + "cpu_seconds": 647.185773, + "fixture_original_teardowns_passed": true, + "guard_sha256": "a9566eb1783d71dd1fc3c0468e2f1af763fbd5ed218f0a87d2cbca3ae1d4011b", + "interpretation": "The original financial replay and financial-to-weight-only successor behavior passes; remaining refusal/callback cases and native calibration admission are pending.", + "junit_sha256": "2af08e54f972192d299281c992dd4306dde99399699c4684037ad6bf60804d9a", + "limits": { + "cpu_seconds": 1200, + "numerical_threads": 1, + "wall_seconds": 1800 + }, + "memo_or_profiler": false, + "model_source_map_sha256": "018d739504f5919c65104ad44a46c1df045d0bf71002622e66e8b68c3192b035", + "native_acceptance": false, + "original_acs_owner_sha256": "db7aaae06961cc1ed9ce13b28dac74d8fd1041b9cd4c32294d2b11105fbef7af", + "peak_rss_bytes": 570884096, + "postcheck_sha256": "8698b43174b17faa1bcc4123ae1b7c7c1ea65bba94f5d41f207c259dc3c951e8", + "receipt_sha256": "96819afd6805e5cbcd6dfb2271ec1d8c4c3b72775135904a9efe1d97cbb79791", + "release_acceptance": false, + "remaining_controls": 7, + "resource_map_sha256": "088a55cd36f83e4b466947a216b0315a983d5888c603103e575f0cc7ae2ece38", + "schema_version": 1, + "scope": "one_invented_financial_successor_positive_control", + "source_map_sha256": "44d9306d0964a4b72490a07895a080b129b83c251866c2aeac06c74deb4ca762", + "status": "one_positive_control_accepted", + "test_counts": { + "errors": 0, + "failures": 0, + "skipped": 0, + "tests": 1 + }, + "unexpected_refusals": {}, + "verified_current": { + "code_resources": 12, + "model_source": 5983, + "source_and_owned": 499 + }, + "wall_seconds": 651.2133005840005 +} diff --git a/experiments/us-financial-successor-remaining-controls-20260910.json b/experiments/us-financial-successor-remaining-controls-20260910.json new file mode 100644 index 000000000..3180e591b --- /dev/null +++ b/experiments/us-financial-successor-remaining-controls-20260910.json @@ -0,0 +1,61 @@ +{ + "all_successor_controls_passed": true, + "approval_sha256": "9e4ed7c7924b130212a4580fa4ee4298bf7ea33d2ae3e5f2e9798226fb07baee", + "cpu_seconds": 798.6069409999999, + "guard_sha256": "51b3a5adad3c14e11a2f69b8566702abe5aecfffbc1c315dd6f08c412c5e671e", + "interpretation": "Together with the accepted positive control, all eight original financial successor cases pass. Native artifact replay, calibration admission in the completed graph and release verification remain separate.", + "junit_sha256": "6cffe5cac9baca685a7c2c0b032580edf83059bd653d4695b08aab75172d7a8c", + "limits": { + "cpu_seconds": 2400, + "native_threads": 1, + "qrf_fit_n_jobs": "1", + "qrf_predict_workers": "1", + "rss_monitor": false, + "rss_reporting": "final peak only", + "wall_seconds": 3600 + }, + "memo_or_profiler": false, + "model_source_map_sha256": "018d739504f5919c65104ad44a46c1df045d0bf71002622e66e8b68c3192b035", + "native_acceptance": false, + "original_fixture_and_teardowns": true, + "peak_rss_bytes": 563593216, + "peer_source_review_sha256": "12a541b3bcf620f0d931815f40f6b9fffac36e59064222c071c8128b9bdbab7d", + "positive_evidence": "us-financial-successor-positive-20260910.json", + "postcheck_sha256": "423a6cd3ae1537bfe372b7d735ad3a907e6fcc6837257e6c65d538db18e13486", + "receipt_sha256": "e394516086ca0f4c773764027ad2d7270a8a9d82d038873f0640cd49e3db483d", + "release_acceptance": false, + "resource_map_sha256": "63de09c280f733c91d07e674bf753c3017b7a375901623a2200a8538f835a4a2", + "root_source_review_sha256": "db0d7f3424cd85bcbc58c690d375d0ef58c557e8d4fefad3dddc666101143c12", + "schema_version": 1, + "scope": "seven_unchanged_invented_financial_successor_refusal_and_callback_controls", + "selected_test_cases": [ + "test_us_survey_financial_successor.py::test_unissued_copy_and_wrong_original_budget_refuse", + "test_us_survey_financial_successor.py::test_arbitrary_nonweight_changes_still_refuse", + "test_us_survey_financial_successor.py::test_rehashed_graph_artifact_cannot_replace_issued_draw", + "test_us_survey_financial_successor.py::test_final_budget_support_borrow_cannot_mutate_financial_run", + "test_us_survey_financial_successor.py::test_final_support_borrow_cannot_forge_returned_view", + "test_us_survey_financial_successor.py::test_financial_borrow_cannot_relax_detached_budget_bounds", + "test_us_survey_financial_successor.py::test_run_borrow_cannot_forge_original_budget_digest" + ], + "source_count": 491, + "source_map_sha256": "44d9306d0964a4b72490a07895a080b129b83c251866c2aeac06c74deb4ca762", + "status": "seven_remaining_controls_accepted", + "subsequent_source_review": { + "covered_by_these_eight_controls": false, + "finding": "Budget and weight-only checked views were constructed before the last support I/O, exposing returned detached values to callbacks.", + "status": "additional_gap_under_correction" + }, + "test_counts": { + "errors": 0, + "failures": 0, + "skipped": 0, + "tests": 7 + }, + "unexpected_refusals": {}, + "verified_current": { + "code_resources": 12, + "model_source": 5983, + "source_and_owned": 499 + }, + "wall_seconds": 802.8923444160027 +} diff --git a/experiments/us-financial19-required-replay-20260912.md b/experiments/us-financial19-required-replay-20260912.md new file mode 100644 index 000000000..f0865222b --- /dev/null +++ b/experiments/us-financial19-required-replay-20260912.md @@ -0,0 +1,60 @@ +# Native financial graph required replay — 12 September 2026 + +The frozen nineteen-node financial graph passed a separate required-cache +replay on real ACS/ASEC inputs. This accepts replay of the earlier financial +pilot, not PUF enrichment, a calibration, a current-source population or release. + +## Exact scope + +- Source: `2ca11c85a0f710b8fa495e363243b940f7eabae2`. +- One collected test passed; outer guard result `0`. +- Request: source fraction `1/1000`, survey/geography seed `20260908`, two + trees, demographic conditioning enabled, `resume="require"`. +- All nine prefix nodes and all nineteen financial-graph nodes were cache hits. +- The graph, preparation, projection and recipient-matrix exports matched the + cold run byte for byte. Portable manifest content matched after removing only + start/end times and per-node timing/cache-hit fields. The complete artifact + report also matched, excluding its cache-hit and manifest-file identity fields. +- Source, resources, native inputs, owned files, baseline exports and thread + controls remained unchanged. There were no unexpected access refusals. +- Elapsed: `3981.09802937496` seconds; CPU: `3944.4073700000004` seconds; + peak RSS: `10988191744` bytes. Declared limits were 7,200 CPU seconds, + 10,800 elapsed seconds and 32 GiB peak RSS, with one numerical worker. + +The source population remains 1,584 households / 3,464 people, expanded to +3,168 households / 6,928 people before block assignment. The final stage carries +seven financial fields. The same frozen source produced the 11 September cold +pilot; no broader coverage follows from replay. + +## Evidence identities + +| Evidence | SHA256 | +| --- | --- | +| Closed replay guard receipt | `71d6981518acad0ad0fd199f906c01ccfa867796aaf053cf6fa5f5a3df1193ae` | +| Final replay manifest | `c8a1b7715bae740c183aa592fca718849e35edf6722328c1de3d83f0cacafa77` | +| Cold baseline manifest | `0d745678802d8dae16750f57ebe779644237e1b2416f1cb78edf4c59d2163f0e` | +| Reviewed guard | `5a2bcd05665644a86ff2f3fc35a2e7e751d759412380dce07193f9b0c6f3c670` | +| Manifest content key, both runs | `134ab4ea39b434066e48258be345999425a6d23f3b5016680f457fd357be9edf` | + +Retained local evidence is in +`PolicyEngine/_recovered/pilot-runs/native19-required-20260912/run`: +`us-native-postclone-financial19-required-20260912.json`, +`native-financial-manifest.json`, and the reviewed harness and helper. +The coordinator's metadata postcheck is +`PolicyEngine/_recovered/scratch-backup/893/codex-takeover-20260912/native-required-replay-postcheck.json`. +It rechecked receipt consistency, exact owned-file hashes, reported bounds, +node hits and export identities without reading native data or generated Frame +payloads again. The actual payload comparison and final source/output checks ran +inside the reviewed native harness before its successful guard closure. + +The guard's expected optional dependency probes remain recorded separately from +unexpected refusals. The run enabled no engine, network, child processes or +automatic retry. Original cold and failed-attempt evidence remains retained. + +## Next acceptance + +The maintained integration includes later runtime repairs and the restored +PUF55 host. Its native candidate needs a fresh cold execution, required replay, +complete input coverage and full Frame export/readback on that identified source. +Calibration and complete-population acceptance follow separately. This replay +does not authorize reuse of the serialized receipt as a live population issuer. diff --git a/experiments/us-input-coverage25-20260910.json b/experiments/us-input-coverage25-20260910.json new file mode 100644 index 000000000..6ff56ed08 --- /dev/null +++ b/experiments/us-input-coverage25-20260910.json @@ -0,0 +1,125 @@ +{ + "api": { + "assigned_block_diagnostic_retained": "census_block_geoid", + "diagnostic_module": "packages/microcosm-build/src/microcosm/build/us_runtime/population_input_coverage.py", + "employment_income_last_year_in_required_profiles": false, + "existing_release_gate_or_manifest_modified": false, + "historical_required_us_inputs_default_count": 163, + "national_cd_engine_omissions": [ + "block_geoid", + "tract_geoid" + ], + "national_congressional_district_diagnostic_default_count": 161, + "profile_module": "packages/microcosm-build/src/microcosm/build/us_runtime/input_coverage_profile.py", + "report_protocol": "microcosm.us.input_coverage_diagnostic.v1", + "signature": "diagnose_us_input_coverage(population: Population, *, compiled: CompiledGraph, manifest: RunManifest, profile: USInputProfile = USInputProfile.NATIONAL_CD) -> PopulationInputCoverage" + }, + "code": [ + { + "after_sha256": "88a8456ad02846d8f41af7ead743c86caa8d82b9b2c18d196feee663acb02a33", + "before_sha256": null, + "path": "packages/microcosm-build/src/microcosm/build/us_runtime/input_coverage_profile.py" + }, + { + "after_sha256": "34455450437b5e4b7ce0377522d2bf2d815ac0a9d6e554b25c25d690e196e318", + "before_sha256": null, + "path": "packages/microcosm-build/src/microcosm/build/us_runtime/population_input_coverage.py" + }, + { + "after_sha256": "7271ca0197d1396dc42656f3220207b0ba96927c1cd11233543f72fcf57ce94d", + "before_sha256": null, + "path": "packages/microcosm-build/tests/test_us_population_input_coverage.py" + } + ], + "description": "Descriptive input coverage on a supplied actual graph Population and attached RunManifest; complete ownership, storage, origin/clone and writer-mask checks.", + "frozen_evidence": { + "cpu_seconds": 6.143905999999999, + "guard_sha256": "0fb8e3f9f25ec47d89d6a3f61c8483caa86ccdee83958f7e801a8be579c4a559", + "models": 0, + "peak_rss_bytes": 332939264, + "physical_postcheck_sha256": "017ccd3d1d97819c1d6332436449e0ba961a1873ce6962547be07743fe097fc7", + "receipt_sha256": "8fed087e9f44d87d47a2c3071653255c959cbfee53ca1166fd22994025ca8d43", + "resources": 0, + "selectors": { + "test_actual_masked_coverage_separates_origin_clone_grain_and_unknownness": 1, + "test_closed_profiles_match_manifest_without_changing_historical_default": 1, + "test_complete_current_ownership_refuses_wrong_carrier_or_writer": 5, + "test_declared_absence_is_not_an_applicability_exemption": 1, + "test_duplicate_grains_are_rejected_by_the_actual_frame_contract": 1, + "test_inconsistent_population_manifest_or_compiler_refuses": 5, + "test_last_manifest_borrow_mutation_refuses": 1, + "test_nonfinite_values_and_missing_grains_remain_unresolved": 1, + "test_population_version_must_be_an_actual_structural_version": 2, + "test_profile_omissions_do_not_omit_assigned_block_diagnostic": 1, + "test_required_replay_preserves_coverage_and_manifest_identity": 1, + "test_source_role_and_membership_are_checked_on_actual_rows": 3, + "test_unclaimed_physical_column_is_not_accepted_as_a_structural_carrier": 1, + "test_zero_weight_origin_and_zero_observations_remain_known": 1 + }, + "source_and_owned_physical_checks": 515, + "source_count": 507, + "source_map_sha256": "55af74ead417eeab1b0e977da24e91ed5de61e52cf5953cd060797f58bc40038", + "source_peer_proof_sha256": "e1b8ae0cadd118ca7d4cfd85af2f4fd707462d87ace779c94277a0ee87fbc4a3", + "source_peer_review_sha256": "5a3177a15b478da0a8dcda6030397483892b60ae4cfd7481aa10a4c889ad59e9", + "tests": { + "errors": 0, + "failures": 0, + "skipped": 0, + "tests": 25 + }, + "unexpected_refusals": {}, + "wall_seconds": 6.524979708017781, + "xml_sha256": "e5ea4cc97ceddbd3ad9153b35ab14dd2e7eb3ff13e0d878972f85e9a3e72ade8" + }, + "limitations": { + "actual_source_ancestry_verified": false, + "applicability_complete": false, + "calibration_modules_rolled_back": false, + "complete_population_input_coverage_proven": false, + "host_integration_added": false, + "native_build_or_model_evaluation_verified": false, + "release_eligible": false, + "statistical_signal_verified": false, + "supplied_graph_consistency_only": true + }, + "maintained_integration_execution_verified": false, + "maintained_source_comparison": { + "baseline_eager_modules_compared": 59, + "comparison_is_not_recomputed_current_import_closure": true, + "different_calibration_imports_preserved": [ + { + "comparison": "different_preserve_maintained", + "frozen_sha256": "397f02916de1535de3bd0c34fdd71a2e685fe37f1a61ff4d389a8343f9642d4b", + "maintained_sha256": "716018cfbb136c631ce56724e58048c94202d6d41b64a56c9ba901cc586fd253", + "module": "microcosm.calibrate", + "path": "packages/microcosm-calibrate/src/microcosm/calibrate/__init__.py" + }, + { + "comparison": "different_preserve_maintained", + "frozen_sha256": "c5084a0dd45617922f8efca08560f243195eb4456430c1d87258e6240f379ab4", + "maintained_sha256": "8a400c46e4c94773dc0d83122f173c8c3bd405694d411989b10f792597b94196", + "module": "microcosm.calibrate.exact_k", + "path": "packages/microcosm-calibrate/src/microcosm/calibrate/exact_k.py" + }, + { + "comparison": "different_preserve_maintained", + "frozen_sha256": "379e4adbb3ed7cdd13f7c7c548da45163f2a50330c874ee51095d38243d392e6", + "maintained_sha256": "14608a5ee02132f06eaad4a59de3b051c8b8e1d07bd1768f4eb57f237173cef7", + "module": "microcosm.calibrate.gates", + "path": "packages/microcosm-calibrate/src/microcosm/calibrate/gates.py" + }, + { + "comparison": "different_preserve_maintained", + "frozen_sha256": "c04df2f61544d4ba83ad467365333a69528ccf9b78913d449539603cd848b6cc", + "maintained_sha256": "925a1b6bae1c714de93df3ee44e26afeed7f3912377b93942f3a5173281d851a", + "module": "microcosm.calibrate.solve", + "path": "packages/microcosm-calibrate/src/microcosm/calibrate/solve.py" + } + ], + "identical": 53, + "maintained_CI_pending": true, + "new_owners": 2 + }, + "proposed_adoption_base": "151166377036a6f4703ed8effdcb328c7699e416", + "status": "25_frozen_scoped_controls_accepted" +} diff --git a/experiments/us-native-atomic-financial-cold-20260910.json b/experiments/us-native-atomic-financial-cold-20260910.json new file mode 100644 index 000000000..6a6d1d586 --- /dev/null +++ b/experiments/us-native-atomic-financial-cold-20260910.json @@ -0,0 +1,104 @@ +{ + "age_calibration_acceptance": false, + "approval_sha256": "837bf6c06fbe186f615080cd844ce0e92eecbad387e800f6d7cc4bfa02b673ae", + "artifact_identity_verification": "pinned manifest metadata and internal producer/key/size relations only; no graph store, export or snapshot reopen", + "collected_tests": 1, + "compiled_nodes": 20, + "composed_prefix_store_hits": 10, + "cpu_seconds": 6591.938158, + "cross_process_source_authority_granted": false, + "current_checkout_certified": false, + "date": "2026-09-10", + "entity_rows": { + "clone": { + "household": 3168, + "person": 6928 + }, + "create": { + "household": 1584, + "person": 3464 + }, + "financial": { + "household": 3168, + "person": 6928 + } + }, + "features": [ + "survey_predictor_age", + "survey_predictor_employment_income", + "survey_predictor_self_employment_income", + "survey_predictor_is_female", + "survey_predictor_state_fips" + ], + "financial_cold_nodes": 10, + "financial_successor_acceptance": false, + "guard_before_after_identity_count": 509, + "guard_sha256": "5bffe33e4c60055070bb9ab9a51ad2e87bfbf86fcd843e1da0d6407d53debda1", + "head_coordinate": "e0718150d062f0fb34702d1443c701f0e1ac59e9", + "implementation_overlays": { + "acs_fingerprint_sharing": false, + "budget_detached_view_correction": false, + "compilation_cache": false + }, + "limits": { + "automatic_retry": false, + "children": 0, + "cpu_seconds": 7200, + "engine": false, + "network": false, + "numeric_threads": 1, + "peak_rss_bytes": 34359738368, + "wall_seconds": 10800 + }, + "manifest_metadata_sha256": "2168b79ff60b481636b4dacf504ff70f4f1934b92359d86cb95ce1ae78987b7a", + "model_quality_established": false, + "model_source_map_sha256": "018d739504f5919c65104ad44a46c1df045d0bf71002622e66e8b68c3192b035", + "native_identity_verification": "config/staging/manifest/guard before-after metadata; no postcheck payload read, hash or stat", + "native_input_bytes": 3735332980, + "native_input_count": 10, + "owned_columns": [ + "employment_income_before_lsr", + "self_employment_income_before_lsr", + "taxable_interest_income", + "qualified_dividend_income", + "non_qualified_dividend_income", + "short_term_capital_gains", + "long_term_capital_gains_before_response" + ], + "peak_rss_bytes": 11452743680, + "postcheck_script_sha256": "ae9b4dacefbfb06c7571c81a55a87f50c080b9e3c27e4c3b39f8e4f878a95d1e", + "puf_enrichment_acceptance": false, + "receipt_sha256": "aea67b38a7a9af17a87b3bdce9e31c8033bdf2d14f7f01191cc7ec12d0d43eae", + "release_acceptance": false, + "remaining": [ + "Required cached replay on this exact native source and graph", + "Actual two-route PUF55 fitting and single finalization/attachment", + "Model-input coverage and fit-quality checks", + "National and congressional-district calibration/holdouts", + "Full and pruned export verification with exact-file dashboard" + ], + "required_replay_run": false, + "resource_map_sha256": "62a6329a8d84ad96410581f4e5043c4b62fb648a7aa7737a2ee8eed61a80a7bc", + "root_postcheck_sha256": "50eabf02b476f2fdc55e15812e374fd76cf6002feb7fd60f10810e4d2122ac8c", + "runtime_result": 0, + "schema_version": "microcosm.scoped-native-control.v1", + "scope": "one_selected_native_survey_atomic_financial_cold_development_pilot", + "source_count": 491, + "source_map_sha256": "44d9306d0964a4b72490a07895a080b129b83c251866c2aeac06c74deb4ca762", + "status": "cold_development_pilot_accepted", + "targets": [ + "survey_current_INT_VAL", + "survey_current_DIV_VAL", + "survey_current_CAP_VAL" + ], + "test_outcomes": [ + "passed" + ], + "unexpected_refusals": {}, + "verified_current": { + "code_resources": 12, + "model_source": 5983, + "source_and_safe_owned": 505 + }, + "wall_seconds": 6607.312281332968 +} diff --git a/experiments/us-native-atomic-financial-required-replay-20260910.json b/experiments/us-native-atomic-financial-required-replay-20260910.json new file mode 100644 index 000000000..60ddc0bf6 --- /dev/null +++ b/experiments/us-native-atomic-financial-required-replay-20260910.json @@ -0,0 +1,137 @@ +{ + "age_calibration_acceptance": false, + "approval_sha256": "3dbd7c5f0e939e803d292ece30eaeee2ea88b9b9adabd8534d22d8841fa3d343", + "artifact_identity_verification": "pinned manifest metadata and internal producer/key/size relations only; no graph store, export or snapshot reopen", + "cold_exports_unchanged": true, + "cold_root_postcheck_sha256": "50eabf02b476f2fdc55e15812e374fd76cf6002feb7fd60f10810e4d2122ac8c", + "collected_tests": 1, + "compiled_nodes": 20, + "composed_prefix_store_hits": 10, + "cpu_seconds": 4442.82255, + "cross_process_source_authority_granted": false, + "current_checkout_certified": false, + "date": "2026-09-10", + "entity_rows": { + "clone": { + "household": 3168, + "person": 6928 + }, + "create": { + "household": 1584, + "person": 3464 + }, + "financial": { + "household": 3168, + "person": 6928 + } + }, + "features": [ + "survey_predictor_age", + "survey_predictor_employment_income", + "survey_predictor_self_employment_income", + "survey_predictor_is_female", + "survey_predictor_state_fips" + ], + "final_manifest_store_hits": 20, + "financial_cold_nodes": 0, + "financial_successor_acceptance": false, + "guard_before_after_identity_count": 520, + "guard_sha256": "49c8971db347181c888e17b69d33efed987bf32ba3d4e93b2c06c283b4460b4c", + "head_coordinate": "e0718150d062f0fb34702d1443c701f0e1ac59e9", + "implementation_overlays": { + "acs_fingerprint_sharing": false, + "budget_detached_view_correction": false, + "compilation_cache": false + }, + "limits": { + "automatic_retry": false, + "children": 0, + "cpu_seconds": 7200, + "engine": false, + "network": false, + "numeric_threads": 1, + "peak_rss_bytes": 34359738368, + "wall_seconds": 10800 + }, + "manifest_metadata_sha256": "2d7816bf1260b72008a5c774a27534412fd2e4f72cc23effe457a2429dd76ff3", + "model_quality_established": false, + "model_source_map_sha256": "018d739504f5919c65104ad44a46c1df045d0bf71002622e66e8b68c3192b035", + "native_identity_verification": "config/staging/manifest/guard before-after metadata; no postcheck payload read, hash or stat", + "native_input_bytes": 3735332980, + "native_input_count": 10, + "owned_columns": [ + "employment_income_before_lsr", + "self_employment_income_before_lsr", + "taxable_interest_income", + "qualified_dividend_income", + "non_qualified_dividend_income", + "short_term_capital_gains", + "long_term_capital_gains_before_response" + ], + "peak_rss_bytes": 11312398336, + "postcheck_script_sha256": "a6514601c1a7e9c28e65ecad0e363822a67e373f96fa9421a3b204afefc87b63", + "puf_enrichment_acceptance": false, + "receipt_sha256": "7669f91957b1610e56d3284e013f2c2c4a113b610322975c2cb8c92274a80650", + "release_acceptance": false, + "remaining": [ + "Actual two-route PUF55 fitting and single finalization/attachment", + "Model-input coverage and fit-quality checks", + "National and congressional-district calibration/holdouts", + "Full and pruned export verification with exact-file dashboard" + ], + "request": { + "demographic_conditioning": true, + "fraction": [ + 1, + 1000 + ], + "geography_seed": 20260908, + "n_estimators": 2, + "resume": "require", + "seed": 20260908, + "selection_request": { + "declaration": "experiments/us-survey-allocation-declaration-v1-20260907.md", + "fraction": [ + 1, + 1000 + ], + "protocol": "microcosm.us.survey-population-request.v1", + "seed": 20260908 + }, + "selection_request_bytes": 168, + "selection_request_sha256": "33f8ec1279a7bac6a27302bf6ea230e704cbcabe02d6f85cdc9b22edaec1f745" + }, + "required_replay_comparison": { + "actual_stored_attach_reconstruction_required": true, + "all_ten_prefix_nodes_store_hits": true, + "all_twenty_final_nodes_store_hits": true, + "cold_manifest_key": "c9d9e2e4d2c04a294ece9b2325929b4344732a7e833a20c58d25eb9e95104567", + "four_non_manifest_export_bytes_equal": true, + "portable_manifest_content_equal": true, + "stored_create_clone_and_terminal_column_identities_equal": true + }, + "required_replay_run": true, + "resource_map_sha256": "62a6329a8d84ad96410581f4e5043c4b62fb648a7aa7737a2ee8eed61a80a7bc", + "root_postcheck_sha256": "596f29b8fd8aa3d4e59e3df80f863d33c443b526de191d7b2c6a77ac82b62d3c", + "runtime_result": 0, + "schema_version": "microcosm.scoped-native-control.v1", + "scope": "one_selected_native_survey_atomic_financial_required_replay", + "source_count": 491, + "source_map_sha256": "44d9306d0964a4b72490a07895a080b129b83c251866c2aeac06c74deb4ca762", + "status": "required_replay_development_pilot_accepted", + "targets": [ + "survey_current_INT_VAL", + "survey_current_DIV_VAL", + "survey_current_CAP_VAL" + ], + "test_outcomes": [ + "passed" + ], + "unexpected_refusals": {}, + "verified_current": { + "code_resources": 12, + "model_source": 5983, + "source_and_safe_owned": 516 + }, + "wall_seconds": 4450.337948458036 +} diff --git a/experiments/us-native-postclone-financial-incomplete-20260910.json b/experiments/us-native-postclone-financial-incomplete-20260910.json new file mode 100644 index 000000000..aeecacf9c --- /dev/null +++ b/experiments/us-native-postclone-financial-incomplete-20260910.json @@ -0,0 +1,22 @@ +{ + "automatic_retry": false, + "cpu_limit_seconds": 7200, + "exact_exit_signal_available": false, + "final_guard_receipt_present": false, + "guard_sha256": "f46e5dc061cbb50f7941891e2db8f7159ebe034f736280279b330605a40ad07b", + "interpretation": "The process is absent and its last sampled CPU exceeded the reviewed limit. Final verification did not complete; stored intermediate artifacts are not an accepted output. Retained progress samples identify repeated source/catalogue checks, but do not measure an isolated performance cause.", + "last_observed_cpu_seconds": 7216.063893, + "last_observed_wall_seconds": 7315.9110773339635, + "native_acceptance": false, + "next_step": "Review and test a semantics-preserving verification optimization before proposing another native run.", + "peak_rss_bytes": 11515363328, + "postcheck_native_payloads_reopened": false, + "published_financial_manifest_present": false, + "release_acceptance": false, + "resource_progress_sha256": "eaa7f4dc175f055d7b3e6997f9c47b2a9a3573e041412338a487ccba06fc7d7c", + "root_incomplete_observation_sha256": "b4c704e61cec4f00cc4ce0340cb8504eb592e4773a10b473c5e9cbcf497eb2b7", + "schema_version": "microcosm.scoped-native-control.v1", + "scope": "One cold 1/1000 native postclone nineteen-node financial pilot", + "status": "incomplete_without_final_guard_receipt", + "wall_limit_seconds": 10800 +} diff --git a/experiments/us-postclone-budget-replay-fence4-20260910.json b/experiments/us-postclone-budget-replay-fence4-20260910.json new file mode 100644 index 000000000..517507d08 --- /dev/null +++ b/experiments/us-postclone-budget-replay-fence4-20260910.json @@ -0,0 +1,52 @@ +{ + "code_adopted_on_top_of": "1cebe3547543bf2917a7af7362bfbb22a05bf964", + "code_changes": [ + { + "after_sha256": "4010b6518b295ec7ad18d65e8a4af0e5d0eaa09fddce60e46f70a197e4d79706", + "before_sha256": "b3a5596f3862c998a079e5d5368418df8169d3ddd262921632f587dbaab9c3fd", + "path": "packages/microcosm-build/src/microcosm/build/us_runtime/survey_origin_budget.py" + }, + { + "after_sha256": "281be16a58d8d72e7ee00a3eb4e62632b7a210f9c3b914b3187e48c775927d16", + "before_sha256": "91b3719210ca11e342b1bb0798ca615457b890fbe26f6be4b69c6c6aacff843f", + "path": "packages/microcosm-build/tests/test_us_survey_origin_budget_atomic_geography.py" + }, + { + "after_sha256": "0f79edc1ab92e16998b41baf9f1735501ef875fddc92a049237699d982ed70a1", + "before_sha256": null, + "path": "packages/microcosm-build/tests/test_us_survey_origin_budget_replay_producer.py" + } + ], + "description": "Include the actual geography replay dependency in the budget producer fence and strengthen the preclone block counterexample while retaining valid postclone owners.", + "earlier_70_control_evidence_unchanged": true, + "guard_sha256": "7424ca92c6aca7e3ad2e76cbb24232c65e2bda6af6af23c8542a5c28c5b8f0ea", + "native_dataset_accepted": false, + "physical_postcheck_counts": { + "models": 0, + "resources": 12, + "source_and_owned": 513 + }, + "physical_postcheck_sha256": "301c7b2812f030fbd329641ae4ec90c32e91f7117fec2eb2b9387e17a6b3ddc9", + "receipt_sha256": "24d783cac659a3a76a166259d61ecdf83a4bfb95ee85da0101f5415ba9412fa2", + "release_eligible": false, + "resource_use": { + "cpu_seconds": 156.80812, + "peak_rss_bytes": 431046656, + "wall_seconds": 161.48273254197557 + }, + "source_map_sha256": "315a018c3a3521a172b3de48bfffd6abaa38a72d37a02e246ef0009b17809fbe", + "source_review_sha256": "60e949960540f69e234ab132789993d95dfa2451ba5a755cb39af4bb0bcab245", + "status": "four_scoped_controls_accepted", + "tests": { + "errors": 0, + "failed": 0, + "passed": 4, + "selectors": [ + "test_budget_producer_refuses_replaced_replay_dependency (3 parametrizations)", + "test_atomic_budget_refuses_preclone_assignment_candidate" + ], + "skipped": 0 + }, + "unexpected_refusals": {}, + "xml_sha256": "41334743e72be57619c50839304d10e48d67ade99e7c74710baa4a5d8335679e" +} diff --git a/experiments/us-postclone-geography-70-controls-20260910.json b/experiments/us-postclone-geography-70-controls-20260910.json new file mode 100644 index 000000000..1abe29a64 --- /dev/null +++ b/experiments/us-postclone-geography-70-controls-20260910.json @@ -0,0 +1,197 @@ +{ + "adopted_production_sha256": { + "packages/microcosm-build/src/microcosm/build/atomic_geography.py": "ed20cdec84eebe46fda5ae213522b6b6699812f04cc6699411c83f27df0ac82a", + "packages/microcosm-build/src/microcosm/build/graph_atomic_geography.py": "9972f125566c4b697f8f862d236f6ae8cd3eece43fddb5d82df46552b0723496", + "packages/microcosm-build/src/microcosm/build/us_runtime/atomic_block_support.py": "c70cfb2cf8f7915666846e563fd3d729fcb814244fc4bc3b3292d064fd0fdd40", + "packages/microcosm-build/src/microcosm/build/us_runtime/current_survey_predictors.py": "ecb883827ca47c1fb1f2bf3ff25328706d0cc5c187d7914749b3ede4bd3ced31", + "packages/microcosm-build/src/microcosm/build/us_runtime/graph_atomic_survey_clone.py": "9aef169897cb0f91d1414d607c5025e8c9d80bcc84c6eef269a27570aa8c12b0", + "packages/microcosm-build/src/microcosm/build/us_runtime/graph_atomic_survey_financial.py": "d407086f05311e721e8961707e9471f5abffd74cef100eee2620cc75294492fc", + "packages/microcosm-build/src/microcosm/build/us_runtime/graph_atomic_survey_population.py": "e8a92fc299d6a388a0ef9ebf26d842460aefda9370006a4b1d929c78983cba0e", + "packages/microcosm-build/src/microcosm/build/us_runtime/graph_current_survey_predictors.py": "291285c35b19ff9dfccffc37ac2e768a72021db4196f3f662d86d26626790253", + "packages/microcosm-build/src/microcosm/build/us_runtime/survey_age_calibration.py": "f7169ef0631a25acb4ca2fea0b49e105a5ca97d8fe85da2ecb4a5bdd5669cbeb", + "packages/microcosm-build/src/microcosm/build/us_runtime/survey_atomic_geography.py": "d4e80f94ba79c4bad4b47d18fcb5daad9469d45aeb81dea64fba101554080f91", + "packages/microcosm-build/src/microcosm/build/us_runtime/survey_origin_budget.py": "b3a5596f3862c998a079e5d5368418df8169d3ddd262921632f587dbaab9c3fd" + }, + "adopted_test_sha256": { + "packages/microcosm-build/tests/test_atomic_geography.py": "bc66ae7461a57ac21afd1bdeb59a11549a5b74c6b81274544de0c9423579f5c9", + "packages/microcosm-build/tests/test_us_atomic_survey_clone_graph.py": "9bfce65ca47f74a325e652684b7c47fcf4fae1561a465e8f87e7518786bd4efa", + "packages/microcosm-build/tests/test_us_graph_atomic_survey_financial.py": "59adfd2af5225ad7d08aba2a55872546b9f252f61baea5f2647863b0b13a8c03", + "packages/microcosm-build/tests/test_us_graph_atomic_survey_population.py": "c2474f4943a2ccb0d89b77f74e1fa991b293bc977139da2c4f28b8626aed0c37", + "packages/microcosm-build/tests/test_us_postclone_geography_identity.py": "763d15b4d543b1e5b294d20a55169138d1a5d54bac15c11a9e9dc6fa93376c14", + "packages/microcosm-build/tests/test_us_survey_age_calibration_atomic_geography.py": "8dca207eb22b0709358d97c723c2367820e586f443462fd9039372145ed827ab", + "packages/microcosm-build/tests/test_us_survey_origin_budget_atomic_geography.py": "91b3719210ca11e342b1bb0798ca615457b890fbe26f6be4b69c6c6aacff843f" + }, + "adoption_exclusions": [ + "PUF nineteen-node predecessor bridge", + "optional ACS source cache/fingerprint overlays", + "split control collectors", + "native guard or payload changes" + ], + "assignment_identity": [ + "survey_geography_origin_key", + "household_support_clone_index" + ], + "compatibility": { + "default_no_geography_case_passed_separately": true, + "default_shared_gate_api_preserved": true, + "optional_typed_gate_artifact": "microcosm.geography.validation_result@1" + }, + "controls": { + "all_original_selected_function_and_fixture_asts_match_adopted_tests": true, + "candidate_cases": 70, + "candidate_split": [ + 45, + 8, + 5, + 5, + 1, + 4, + 2 + ], + "separate_default_compatibility_cases": 1, + "single_combined_suite_run": false + }, + "graph_nodes": { + "age_extension_total": 12, + "financial_extension_total": 19, + "survey_prefix": 9 + }, + "historical_native_note": "Accepted twenty-node cold/replay evidence assigned geography before cloning and remains separately versioned; it does not certify this new ordering.", + "model_quality_accepted": false, + "national_or_local_fiscal_calibration_accepted": false, + "native_full_puf_enrichment_accepted": false, + "native_postclone_survey_accepted": false, + "new_order_puf_host_accepted": false, + "original_candidate_source_map_sha256": "dd1a782dceb020fcf8d9ff5ba94fb32035aa01e8d670ea3529a42a01b26ce9da", + "release_eligible": false, + "runs": [ + { + "cases": 45, + "cpu_seconds": 3.744692, + "included_in_candidate70": true, + "label": "us-postclone-atomic-geography45-v2", + "model_sources": 0, + "physical_source_and_owned": 510, + "postcheck_sha256": "45efe59214d52901a9712304e8ca6f66175c13caaecba3ff773d7780bc2a5ffa", + "receipt_sha256": "f5d1390d9b5183d7f23a2e2492a2d24e06df44bb66cec6e33bc100e74659cfe3", + "resources": 12, + "source_count": 502, + "source_map_sha256": "a294308425861522235f68d025c3c5becf7c2922708ed70543ab54f6a5a15ba4", + "wall_seconds": 4.635587666998617, + "xml_sha256": "8fd2783fc1917170f7d6abcb7ee0c7154a794dcddf326b2c154d5d7b612a2f82" + }, + { + "cases": 8, + "cpu_seconds": 235.803439, + "included_in_candidate70": true, + "label": "us-postclone-prefix-budget8-v3", + "model_sources": 0, + "physical_source_and_owned": 511, + "postcheck_sha256": "85695bedda8003c9278d2d2ac4fc8d4d6e659898c3f3f5e8d8efcae4b08d74ae", + "receipt_sha256": "3b3fe93a0fe65ba6fd63889afdf99a8f45ba4a37c11aa5e243557fa05352b07c", + "resources": 12, + "source_count": 503, + "source_map_sha256": "71812aeb4d1600223f90882d8652405876c05379acbe0828fbee41b24680d7a1", + "wall_seconds": 237.32238883303944, + "xml_sha256": "caffc1d0eefa51cb0a71338324a213224e67cac50692d1f62d6d61d3cdb328bf" + }, + { + "cases": 5, + "cpu_seconds": 345.624984, + "included_in_candidate70": true, + "label": "us-postclone-financial5", + "model_sources": 0, + "physical_source_and_owned": 511, + "postcheck_sha256": "7e993eccb0085b7dfce203a50d43f868eb822068ab62b50f344f69ce8f478396", + "receipt_sha256": "e3d9e95210185976de73982f994b51386bf4aa1243cc10b4f8db893760e55a90", + "resources": 12, + "source_count": 503, + "source_map_sha256": "02ae2194338378ac652a4ee6c2049b68dd78a813962b665b010c947b5cab0a4b", + "wall_seconds": 352.15383283403935, + "xml_sha256": "224590dab0d79f68b30a8e50ef9e86e9144869d90521cd66c0689d2b21167ff5" + }, + { + "cases": 5, + "cpu_seconds": 203.630023, + "included_in_candidate70": true, + "label": "us-postclone-budget5", + "model_sources": 0, + "physical_source_and_owned": 511, + "postcheck_sha256": "9056592f58e29a73ba8b9ac69314e78602015be112261212911ca19a517e1df0", + "receipt_sha256": "0d577da19dd0086c8d5c47a7fc91101ad708ac51a22c62d8a487f1684c3f1b47", + "resources": 12, + "source_count": 503, + "source_map_sha256": "d316393b1aadf99fefadc342dcd0a88575491665971c8a3908c07b14d7f81f05", + "wall_seconds": 206.07668850000482, + "xml_sha256": "6a189b65471483178fc3518bc68692cf1c88678ae214db5d4407e178394722a1" + }, + { + "cases": 1, + "cpu_seconds": 549.278067, + "included_in_candidate70": true, + "label": "us-postclone-age1", + "model_sources": 0, + "physical_source_and_owned": 511, + "postcheck_sha256": "f2c6533c11be9b325e9a507e8356a1c516ca4b20a3f4aecc7721458a8480f9ff", + "receipt_sha256": "e3a513ba6483688fb30b6bd7199b690278f32a02bc49101c78385539447fd699", + "resources": 12, + "source_count": 503, + "source_map_sha256": "b8e4b18994687ea33bead77003ccaffc28d72de7b79d3a34316faa1c7f945b4f", + "wall_seconds": 552.7509316250216, + "xml_sha256": "621b6dff98084c6748eb4a57b3515ef98828ae634bda3362c1504a87c4aba937" + }, + { + "cases": 4, + "cpu_seconds": 438.682175, + "included_in_candidate70": true, + "label": "us-postclone-prefix-late4", + "model_sources": 0, + "physical_source_and_owned": 511, + "postcheck_sha256": "f25adff98c4645dabb503cd155e2652c9988342fca1be108d8b5aea8ee77f90e", + "receipt_sha256": "973fad9735b877c2867bb37af9c60025e10ae7b91216d846b056b5164e39ee33", + "resources": 12, + "source_count": 503, + "source_map_sha256": "b6b9faabf3080fce9cb3c9d1ab9521ba69da615c12d0c2328d75a24f30c976ec", + "wall_seconds": 459.0505850840127, + "xml_sha256": "c0034d745588c5d3821fb73d7086ada5e44c334b2a3e0b52e4ae4116bbfff13d" + }, + { + "cases": 2, + "cpu_seconds": 458.346596, + "included_in_candidate70": true, + "label": "us-postclone-financial-late2", + "model_sources": 0, + "physical_source_and_owned": 511, + "postcheck_sha256": "2ab808471fc7943f6a3e6a810480ef9a1dddc267a908f0df03fc11311915c37c", + "receipt_sha256": "306c8e250421248faf282a9ac89742a820684b2ba2f6072189426c8301bd5464", + "resources": 12, + "source_count": 503, + "source_map_sha256": "e62aac60cb960e323cbf127067b3c27aa22daf73cdc815e20b55683984ed3ce8", + "wall_seconds": 478.2829617500538, + "xml_sha256": "e303d11996dbe2289c612d5fc102f55da5724f3f259d2cd5de9610686c2d7bb4" + }, + { + "cases": 1, + "cpu_seconds": 28.545443, + "included_in_candidate70": false, + "label": "us-postclone-default1", + "model_sources": 0, + "physical_source_and_owned": 511, + "postcheck_sha256": "157961870ad76565d8c3634a457e4ef8d5ac2245abd75e6b7dc789288cc13fae", + "receipt_sha256": "9c96d01404c8a48fac7b991c2c7d7742d586bdd2b6b754a694e31c451d1bd5ad", + "resources": 12, + "source_count": 503, + "source_map_sha256": "38d7f20d01526ff275c74a8fcd92404b1a572cadade7f896d602f974d275fe2d", + "wall_seconds": 29.92554637498688, + "xml_sha256": "d8da781a858400f3670031566d774d7148b1333cecd6feee543901e68383ec8f" + } + ], + "scope": "Complete initial survey clones before constrained block assignment; current financial and limited age-calibration interfaces on invented source/target fixtures", + "source_adoption_base_head": "94b7c2a5d73c1512efcc41425224bfbb7a04cbb7", + "source_peer_review_sha256": "08c53c53cbd63481279c7c003f536404ec0b2163151b152ff39a6a72b6a3f713", + "status": "invented_postclone_controls_accepted", + "test_only_corrections": { + "packages/microcosm-build/tests/test_atomic_geography.py": "bc66ae7461a57ac21afd1bdeb59a11549a5b74c6b81274544de0c9423579f5c9", + "packages/microcosm-build/tests/test_us_survey_origin_budget_atomic_geography.py": "91b3719210ca11e342b1bb0798ca615457b890fbe26f6be4b69c6c6aacff843f" + } +} diff --git a/experiments/us-puf-donor-boundaries-20260910.json b/experiments/us-puf-donor-boundaries-20260910.json new file mode 100644 index 000000000..802668e0a --- /dev/null +++ b/experiments/us-puf-donor-boundaries-20260910.json @@ -0,0 +1,120 @@ +{ + "calibration_acceptance": false, + "cold_and_required_replay_checked": true, + "controls": [ + { + "cpu_seconds": 3.567905, + "guard_sha256": "348bc2816d499e592fcf90515ff79175fdabf535bb51bd23a47a15f355265f3a", + "junit_sha256": "29cc6d7aac55b4f6a64cf4940ef2613c5880fa42a2e9c4eb1f5b113ea02d9219", + "model_source_map_sha256": "018d739504f5919c65104ad44a46c1df045d0bf71002622e66e8b68c3192b035", + "name": "donor_validation", + "peak_rss_bytes": 439549952, + "receipt_sha256": "99a0efcb676ba96f5e485f204428120ce75e0f43b50f685c0eeace9d972c36ff", + "resource_map_sha256": "d58f2bc53f56cd36305f8c6f2855c9fa001153b2f51cd41c2cf430c0ea3d2683", + "root_postcheck_sha256": "e8c4a24705541e82b3fe834496a0bd32afd53627507bd75a055090f76d95b518", + "selected_test_cases": [ + "test_existing_profiles_share_exact_selected_donor_check[PufOutputProfile.FULL65]", + "test_existing_profiles_share_exact_selected_donor_check[PufOutputProfile.PUF59]", + "test_existing_profiles_share_exact_selected_donor_check[PufOutputProfile.PUF55_SURVEY_SS]", + "test_existing_profiles_share_exact_selected_donor_check[PufOutputProfile.PUF55_SURVEY_SS_NO_TOTAL]", + "test_complete_historical_donor_domains_are_retained[missing]", + "test_complete_historical_donor_domains_are_retained[numeric_string]", + "test_complete_historical_donor_domains_are_retained[unknown]", + "test_complete_historical_donor_domains_are_retained[nonfinite]", + "test_complete_historical_donor_domains_are_retained[integer_range]", + "test_complete_historical_donor_domains_are_retained[incidence_capacity]", + "test_complete_historical_donor_domains_are_retained[boolean_count]", + "test_complete_historical_donor_domains_are_retained[year]", + "test_donor_rejection_still_precedes_recipient_access", + "test_selected_values_are_detached_from_auxiliary_source_table", + "test_excluded_ss_outputs_are_not_accepted_as_extra_donor_columns", + "test_eight_route_selection_keeps_targets_recids_weights_and_six_money_values" + ], + "source_count": 498, + "source_map_sha256": "e2f500ab2055e34899d663f9792a2f97b1157ab087409072567f985d3874b853", + "test_counts": { + "errors": 0, + "failures": 0, + "skipped": 0, + "tests": 16 + }, + "verified_current": { + "code_resources": 12, + "model_source": 5983, + "source_and_owned": 506 + }, + "wall_seconds": 5.826573541969992 + }, + { + "cpu_seconds": 15.144223, + "guard_sha256": "b91941ff88c7dd446d650a68ee1ac015e9e39450f51e179f181b2cd75277a8a7", + "junit_sha256": "5cca9cf41e93c7b01bdc0beb5a42dc48b5ebd58e5cacde92e0f2b0435c4c5cce", + "model_source_map_sha256": "018d739504f5919c65104ad44a46c1df045d0bf71002622e66e8b68c3192b035", + "name": "full65_chain", + "peak_rss_bytes": 625786880, + "receipt_sha256": "29d3799c065ccc5277bca30a643dd615bf9f425835bf552abd63300d13a65525", + "resource_map_sha256": "9fadfa1d4f77bf1510d04895bada88d7558a5652b09b6d9c22b9b2739ca9370f", + "root_postcheck_sha256": "8c5654c6bdceb7fd91624644a06808202fa1b2dc6ad94e925d11a1bf4519871b", + "selected_test_cases": [ + "test_real_full_65_target_graph_draw_chain_finalization_and_replay" + ], + "source_count": 498, + "source_map_sha256": "e2f500ab2055e34899d663f9792a2f97b1157ab087409072567f985d3874b853", + "test_counts": { + "errors": 0, + "failures": 0, + "skipped": 0, + "tests": 1 + }, + "verified_current": { + "code_resources": 12, + "model_source": 5983, + "source_and_owned": 506 + }, + "wall_seconds": 16.31964554201113 + }, + { + "cpu_seconds": 13.678566, + "guard_sha256": "09e9e78fba275090101d569a037bc15b2ac9f6ed13bd656fe3b42f949909e5fc", + "junit_sha256": "eea60075e8c7a64720d52b9afc63f1779184072c5b41623b7ce41f2cfee741bd", + "model_source_map_sha256": "018d739504f5919c65104ad44a46c1df045d0bf71002622e66e8b68c3192b035", + "name": "survey_ss_puf55_chain", + "peak_rss_bytes": 628178944, + "receipt_sha256": "2546a4531c7c4099066cc346cd42115175972f35a8699614e1670ec4a8bcfb29", + "resource_map_sha256": "d8aef904d7ceb5067c0d4f39ccab0837ad241bd96eac34f7c0aecaa6fde54848", + "root_postcheck_sha256": "3a0180f061f10d873ae38de5e583bc7d8f1ef3c6262fea4665d2194b1d947f04", + "selected_test_cases": [ + "test_real_puf55_chain_preserves_survey_ss_and_replays" + ], + "source_count": 498, + "source_map_sha256": "e2f500ab2055e34899d663f9792a2f97b1157ab087409072567f985d3874b853", + "test_counts": { + "errors": 0, + "failures": 0, + "skipped": 0, + "tests": 1 + }, + "verified_current": { + "code_resources": 12, + "model_source": 5983, + "source_and_owned": 506 + }, + "wall_seconds": 14.389938707987312 + } + ], + "current_checkout_runtime_certification": false, + "full65_nodes": 133, + "native_execution": false, + "owner_sha256": "e0c9d34cd1815d323495e01ec5f94133c1b7da8da37d3fc714041ea38be677cc", + "release_acceptance": false, + "same_complete_fitted_model_donor_check_preserved": true, + "same_ordinary_donor_validation_preserved": true, + "scope": "invented_donor_validation_and_actual_full65_puf55_chain_regressions", + "scope_note": "Validation is on the pinned 498-file snapshot. Source adoption does not certify the later combined checkout or the prospective two-route host. The checks exercise actual graph fitting, raw draws, finalization, and required cache replay over invented donors; no native PUF or survey transfer is claimed.", + "selected_tests": 18, + "source_authority_issued": false, + "status": "frozen_donor_helper_extraction_accepted", + "survey_ss_puf55_nodes": 113, + "test_sha256": "8d84c4d379cfd75ad36ca232fcadcdf0e0bf899ad8c9b8f242319d0f3c8a8db0", + "two_route_host_execution": false +} diff --git a/experiments/us-puf55-canonical-create-47-controls-20260910.json b/experiments/us-puf55-canonical-create-47-controls-20260910.json new file mode 100644 index 000000000..0942a6a1b --- /dev/null +++ b/experiments/us-puf55-canonical-create-47-controls-20260910.json @@ -0,0 +1,52 @@ +{ + "cpu_seconds": 11.761269, + "current_checkout_runtime_certification": false, + "failed_predecessor_accepted": false, + "guard_sha256": "3dd78388b0426c92183db97aaa48516903b61ef9f4f646f35b9301c56662aa7a", + "junit_sha256": "0e6a0a3d4f77363c7ccd1705afd75e8082219ca4a24cfc6d36db383b77e2daab", + "model_source_map_sha256": "018d739504f5919c65104ad44a46c1df045d0bf71002622e66e8b68c3192b035", + "native_access": false, + "observed_outcomes": { + "error": 0, + "failed": 0, + "passed": 47, + "skipped": 0 + }, + "peak_rss_bytes": 501907456, + "postcheck_script_sha256": "f1c8d68d9c4edd92391317683f308f599f6dbc771a4ef9c538acf7d6e4b679e0", + "preceding_stable47_failed_postcheck_sha256": "cb2e3a344b4fa2e7bef7b1e60cf085b8a583567e54c7c2c940345639bc9081ce", + "preserved_failed_predecessors": [ + "us-puf55-canonical-create-failed-20260910.json", + "us-puf55-canonical-cycle39-failed-20260910.json" + ], + "receipt_sha256": "8f7ac625296fbb22dcd976918b4718af14cbaf31288e528583d1fb49ad6e95c9", + "release_acceptance": false, + "resource_map_sha256": "57927f284c647abf03ea1e8370ddeb6035b4e68ff93af1e4f745a5027307e7ef", + "root_postcheck_sha256": "5617f7949e261d42975cb088671f83bda71c37d424e190a209766df5e4ce20ec", + "scope": "47_invented_canonical_donor_and_converter_controls", + "scope_note": "The exact frozen canonical donor CREATE and projection controls pass cold execution and required replay. Source adopted locally. Native donor, complete survey attachment, calibration and release acceptance remain separate.", + "source_adoption": true, + "source_files": { + "packages/microcosm-build/src/microcosm/build/us_runtime/graph_puf55_canonical_donor.py": "a04c8dd2d89b4d3e5c563707587545299bd218aadf567ee7e772b9bbc5755311", + "packages/microcosm-build/src/microcosm/build/us_runtime/puf55_canonical_donor.py": "6a8d07e38f5f0bff2619a7eb53fd0f34a10eed798f79de44c82e7740db923066", + "packages/microcosm-build/tests/test_us_graph_puf55_canonical_donor.py": "675bd57728a6ef56e79a0a24c79713fbb98e454be385fae607528d57048ce76d", + "packages/microcosm-build/tests/test_us_puf59_canonical.py": "85a6d41e716e0d3f1794851dd6cc6007603834a22982f31058fa5bf6da31df6d" + }, + "source_graph_control_execution": "actual canonical CREATE cold and required replay with all 47 selected assertions and original fixture teardown passed", + "source_map_sha256": "dd16d99e9d0bbc3b1bf3aacddcd1159887758c2a8183bcee0333715270bcd9bb", + "status": "frozen_canonical_create_and_converter_controls_accepted", + "survey_attachment_host_execution": false, + "test_counts": { + "errors": 0, + "failures": 0, + "skipped": 0, + "tests": 47 + }, + "unexpected_refusals": {}, + "verified_current": { + "code_resources": 12, + "model_source": 5983, + "source_and_owned": 509 + }, + "wall_seconds": 14.356884333014023 +} diff --git a/experiments/us-puf55-canonical-create-failed-20260910.json b/experiments/us-puf55-canonical-create-failed-20260910.json new file mode 100644 index 000000000..e2d8251d6 --- /dev/null +++ b/experiments/us-puf55-canonical-create-failed-20260910.json @@ -0,0 +1,32 @@ +{ + "corrected_source_pending": true, + "cpu_seconds": 7.677668, + "date": "2026-09-10", + "failed_scope_accepted": false, + "failure_code": "PUF55_CANONICAL_SOURCE_LIVE_DEPTH", + "failure_stage": "kernel construction, while traversing cyclic function closure/mapping references", + "guard_sha256": "b50c65325ec901aada10368562ba481f3253f8d4fbdfba7c38fb6a5089de7612", + "independent_physical_postcheck": false, + "junit_sha256": "c9f6146ec4ada2ea277311ff8f9609207585acc7c12233ef6f9437f273a82e4d", + "native_input_access": false, + "passed_cases_reported": 1, + "peak_rss_bytes": 500629504, + "production_adopted": false, + "receipt_sha256": "fa34a78b323eda9873b24a0b11a8d7f84dc3696cd9e0eab0cfb8e409a1152da7", + "release_acceptance": false, + "result": 1, + "schema_version": "microcosm.failed-control-observation.v1", + "scope": "invented canonical donor CREATE and converter controls; no native input, current-checkout or release acceptance", + "source_before_after_equal_reported": true, + "source_count": 501, + "source_map_sha256": "6fdb0549fe8331db32450f3161a2d8cf1e00096c9245728e84360a08050277da", + "status": "failed_before_canonical_create", + "test_counts": { + "errors": 24, + "failures": 4, + "skipped": 0, + "tests": 29 + }, + "unexpected_refusals": {}, + "wall_seconds": 10.18145758396713 +} diff --git a/experiments/us-puf55-canonical-cycle39-failed-20260910.json b/experiments/us-puf55-canonical-cycle39-failed-20260910.json new file mode 100644 index 000000000..8f4142131 --- /dev/null +++ b/experiments/us-puf55-canonical-cycle39-failed-20260910.json @@ -0,0 +1,34 @@ +{ + "automatic_retry": false, + "cpu_seconds": 8.717074, + "date": "2026-09-10", + "exact_changed_module_or_member_identified": false, + "failed_scope_accepted": false, + "failure_code": "PUF55_CANONICAL_SOURCE_LIVE_STATE_CHANGED", + "failure_stage": "first executor implementation_hash after source fixture initializes, before CREATE execution", + "guard_sha256": "367e4de4c1587082d36dd4ae1f09847e7e8db3ef312ba33f5e867addf1119318", + "independent_physical_postcheck": false, + "junit_sha256": "fc00a99888c91d1d8fb6cf80ad381236a2c25d8b64cd0a95fa3d02a8951645d3", + "native_input_access": false, + "passed_cases_reported": 14, + "peak_rss_bytes": 507789312, + "predecessor_receipt_sha256": "fa34a78b323eda9873b24a0b11a8d7f84dc3696cd9e0eab0cfb8e409a1152da7", + "production_adopted": false, + "receipt_sha256": "ec5dfec245cc5cf1a29aebb9afe81f0768dcd79cefa9c28e2610af5c1c79c229", + "release_acceptance": false, + "result": 1, + "schema_version": "microcosm.failed-control-observation.v1", + "scope": "invented canonical donor CREATE and converter controls; no native input, current-checkout or release acceptance", + "source_before_after_equal_reported": true, + "source_count": 501, + "source_map_sha256": "027ab7d21adf3f74cac798e759eed27bd59d20a1b769839549e557d8c61ba0a8", + "status": "failed_before_canonical_create", + "test_counts": { + "errors": 24, + "failures": 1, + "skipped": 0, + "tests": 39 + }, + "unexpected_refusals": {}, + "wall_seconds": 11.895262666977942 +} diff --git a/experiments/us-puf55-checked-output-20260912.md b/experiments/us-puf55-checked-output-20260912.md new file mode 100644 index 000000000..7ce57a7f2 --- /dev/null +++ b/experiments/us-puf55-checked-output-20260912.md @@ -0,0 +1,83 @@ +# Retained PUF55 output verification + +On September 12, 2026, the PUF55 host gained an in-process checked run handle +for downstream graph stages. The actual `run_survey_puf55` execution retains +the handle only after its existing final source, artifact, numerical, and +complete-population checks. Constructing or copying `SurveyPuf55Run` does not +issue a handle. + +`check_survey_puf55_run(run)` and `run.checked_view()` return a descriptive +`CheckedSurveyPuf55Run(payload, digest, population)`. Downstream owners must +retain the original run, check it immediately before consumption, and recheck +it after their last relevant I/O before returning or exporting a successor. +A failed public check revokes that handle; restoring fields cannot reissue it. +The payload and view cannot authorize a reconstructed or copied run. + +## Retained checks + +The checker reuses the attachment boundary's real financial/source owners, +source and producer keys, donor resource declaration, source codecs, kernel +registry, recipient qualification, and receiving support. It reads declared +artifacts through the existing producer/type/store verifier, borrows the +boundary again, then checks the output without further external reads. + +The output checks preserve original object bindings, the compiled graph, +manifest JSON, exact attached population and ledger roster, heterogeneous +manifest contents, and the complete final and independently reconstructed +populations. Existing physical seals include values and missing backing +storage, axes, schema, memberships, weights and kinds, design anchors, strata, +owners, metadata, mass log, and ledger. The retained state holds final output +and expected populations, rather than all 245 observed node snapshots. + +Rechecking does not fit, execute a graph, decode model pickles, or reconstruct +the canonical donor. It verifies the current ancestry and identities of the +artifact bytes that actual execution already checked numerically. It still +performs source, implementation, and store checks; consumers should use it at +stage boundaries rather than inside calibration iterations. + +## Verification and independent review + +- Three public-constructor/API controls first failed against the absent API, + then passed against the implementation. The original red evidence remains. +- The first complete invented-source run passed 23 controls. Independent + review then reproduced a gap: adding an extra attached Frame or ledger key + did not change portable manifest JSON or the old expected-version seals. + The checker now requires the exact attachment roster, with separate + population and ledger regression cases. +- The corrected source passed all 25 controls through an actual invented + financial19 to PUF245 cold execution and required replay. It used 901.07 + CPU seconds, 910.28 wall seconds, and 740,917,248 bytes peak RSS. All 985 + source/owned-file and 15 resource hashes remained unchanged; the guard + recorded no unexpected refusals or child processes and one numeric thread. +- The 12 new test functions were then moved beside the existing host tests + without changing their decorated ASTs, any existing function/class AST, or + production source. Independent review approved the move. Collection found + 45 cases, including the 25 new cases, sharing one `composed` fixture + definition. The three cheap checks passed again. The combined 45-case suite + was collected, not rerun as one full suite. +- Ruff and the CI test inventory verifier pass. The source repair and test + move each received independent review with no remaining actionable finding. + +The first post-move guard recorded the expected 45-case collection and three +passing tests but returned nonzero because two pytest sessions doubled an +already-denied optional locale metadata probe from 12 to 24. Its evidence +remains. A new owned run adjusted only that denied-probe reporting ceiling; +it closed green without allowing the read or changing source or tests. + +Frozen production SHA256: +`5ba34ce40e19cb2bf22f9c9976576f9dc19e92b19a5a5cc1763569df7a40b796`. +Final colocated test module SHA256: +`bf8b53d229a61565337e2ee9766900158b2bde4edc3042c60c52fe09ecdc531f`. +Corrected 25-case receipt SHA256: +`e47f4408387fcfab6564f66ad9479c9c1cf32615862b2268ee854ba7781a4553`. +Post-move collection/cheap receipt SHA256: +`016a5defba66ce8d36372e890a000295d82f4825d1e516d477185eaccfc70633`. + +## Scope + +This is invented-source software acceptance. It grants no native source or +population admission, calibration acceptance, release eligibility, or +publication authorization. Graph node and artifact contracts are unchanged. +The attachment kernel hashes the host module, so this additive change moves +PUF producer identities on its own source revision. The separately frozen +d35 native pilot packet and its evidence remain unchanged. diff --git a/experiments/us-puf55-host-adoption-20260912.md b/experiments/us-puf55-host-adoption-20260912.md new file mode 100644 index 000000000..37a15fe78 --- /dev/null +++ b/experiments/us-puf55-host-adoption-20260912.md @@ -0,0 +1,101 @@ +# PUF55 host restoration on the reconciled graph + +This change restores the existing PUF55 composition over an issued nineteen-node +post-clone financial run. The shared ACS/ASEC survey population is cloned before +atomic geography is assigned. Recipient measurement follows financial completion; +a common canonical PUF donor supplies two disjoint conditioning routes, followed +by one placement into the full survey population. The original survey channel, +source identities, memberships, Social Security fields and knownness, geography, +design anchors and household weights are retained. + +`run_survey_puf55` accepts the actual financial run and original PUF source paths. +It requires the financial issuer, source keys, producer implementations and typed +model ancestry. Its explicit `resume="require"` call must hit every graph node. +There are 25 fixed-prefix/finalization nodes plus 110 per nonempty route: 245 +nodes for the two-route fixture. Each route fits and applies 55 targets. The +known survey report total adds a ninth predictor; unknown totals use eight +predictors without a zero fallback. The four Social Security components remain +owned by the survey completion stage. Prior wages are excluded. + +The restoration consists of two new production modules, the existing canonical +donor fixture's opt-in weighted-tail support, the full host controls and the tiny +heterogeneous-manifest controls. It incorporates the later corrected mutation +control, which changes the retained receiving DataFrame and proves its complete +population seal changed. All earlier fixture helpers and newer tests remain. +Ruff formatting/import order is the only difference from the preserved source +candidate's executable AST. No existing production module or implementation +inventory JSON was replaced. The new kernels' implementation hashes bind both +new host modules and their consumed canonical, recipient and numerical owners. + +## Current-source validation + +The source base is `9e61d0d04f5930aa72aef2faf83b2a9c7f2b4284`, including the +reconciled graph and keyed-draw corrections. Tests run against a frozen inventory +of 976 source/configuration files and 15 exact code resources, with no native +input, country engine, network or child process access. Numeric worker counts +are one. Runtime stores contain newly constructed invented inputs only. + +The 28 heterogeneous-manifest controls and three canonical fixture controls pass +together: 31 tests, zero failures/errors/skips, 5.97 wall seconds, 5.61 CPU seconds +and 437 MB peak RSS. The guard's source/owned/resource before-and-after checks +pass. The preceding run also passed its tests but failed its reporting gate +because optional pytest-owned current-symlink attempts were newly refused. That +failed record is preserved. A fresh run kept the symlinks refused and classified +only explicitly bounded attempts within its owned directory as expected. + +The two full-host positive controls pass: two tests, zero failures/errors/skips, +741.87 wall seconds, 734.55 CPU seconds and 739 MB peak RSS. They share one actual +financial fixture, use 64 invented PUF donors and two trees, and execute the +245-node cold extension plus required replay. The direct whole-cohort oracle +checks all outputs and non-owned storage. The finite limits were 1,800 CPU +seconds, 2,700 wall seconds and 8 GiB measured peak RSS. The guard exits zero, +records no unexpected refusals, and confirms all 984 source/owned hashes and +15 resource hashes stayed unchanged. This validates the restored composition on +the current frozen source, not the entire twenty-case host test file. + +An owned diagnostic hook successfully ran the existing NATIONAL_CD input-coverage +operation on the actual attached population within the fixture's checked +lifetime, bracketed by the normal purity and financial-owner checks. The fixture +contains 12 households and 18 people spanning ACS/ASEC and original/clone channels. +Of 161 required input names, 61 are present and 100 are absent; no ambiguous grain +or block-storage issue is reported, and all 12 household block values are known. +The absent roster includes survey demographics and disability, health coverage, +assets, take-up and several survey-owned income names. These include the four +canonical Social Security component names, which this PUF stage deliberately +does not write; the synthetic roster does not establish native input coverage. Prior wages remain excluded. +Known/unknown and writer/carried counts are recorded by source/clone. Applicability, +source ancestry, statistical signal and release eligibility remain unverified. + +The closed host receipt is SHA256 +`d2288ee0e321297875509ee962faef8ee886399a717cad47356921dada8be698`; +the descriptive synthetic coverage is +`d405c7d559a2f2141d83c076d8cef8bbe33f30ba403a7eaedf76cff1161cb7a6`. +Both live under the durable local adoption packet's `run-host2-v2` directory; +`ACCEPTANCE-SUMMARY.json` provides a bounded index without producer payloads. +An independent source review found no actionable defect in the five-file +restoration and confirmed the preserved source pins and executable AST parity. + +CI test-file partition verification passes with 487 tracked files. The new host +file enters fast `rest` and engine `us-am:build`; the heterogeneous-manifest file +enters fast `rest` and engine `us-p`. The canonical fixture file retains its +existing `us-am:build` placement. The classifier and country spec/seed pins are +unchanged. Ruff and staged whitespace checks pass. + +## Native and release work remains separate + +A new native harness must retain the actual financial run in process and provide +a writable store for new PUF artifacts. The retained historical financial replay +store remains read-only. Genuine PUF source validation, conditional-fit quality, +complete applicable input coverage, national/CD calibration, holdouts, pruning +and exact release export/readback are not established by these synthetic tests. +The host emits a development result with release eligibility false. + +`SurveyPuf55Run` is a public frozen dataclass, not a retained issuer. The runner +checks source/artifact ancestry and the complete output immediately before +return, but supplies no post-return `check_run` or issuance registry. The genuine +financial handle retained inside it authenticates the upstream financial run, +not subsequent mutations or copies of the final PUF population. Receipt booleans +cannot grant that authority. A downstream calibration bridge must consume the +actual checked output within an explicitly fenced host operation or implement +and test a retained output check; it cannot infer portable ownership from this +dataclass or receipt. This restoration adds no such admission mechanism. diff --git a/experiments/us-puf55-native-donor-20260909.json b/experiments/us-puf55-native-donor-20260909.json new file mode 100644 index 000000000..68dc87e6d --- /dev/null +++ b/experiments/us-puf55-native-donor-20260909.json @@ -0,0 +1,40 @@ +{ + "code_resources": 12, + "cpu_seconds": 8.784642, + "expected_test_file_counts": { + "test_native_puf55_projection.py": 1 + }, + "first_attempt": "Value checks passed, but removing the diagnostic plugin also removed collection accounting. Guard refused the run. The replacement retained exact collection accounting while suppressing private failure context; neither values nor access permissions changed.", + "input_artifact_sha256": "28cd44bd59218c1e77f14fa35c510ab550d2035527e3c139e7d19875c12ac11c", + "model_source_files": 5983, + "native_input_hash_verified_inside_guard_before_and_after": true, + "native_payload_reopened_by_postcheck": false, + "native_resources": 1, + "original_canonical_artifact_unchanged": true, + "outputs": 55, + "peak_rss_bytes": 1431109632, + "predictors": 9, + "qrf_fitted": false, + "receipt_sha256": "ce09455947f0efbca16014a23baa19037ac6450876c639568af6584c81bfc45c", + "release_eligible": false, + "report_sha256": "7ade9f442fe25f02e013b39afa5bf8077e89a75b43b4c6c7d0d2dc123f052766", + "result": 0, + "rows": 207692, + "scope": "Native canonical59 artifact projected to the survey-SS PUF55 donor; no recipient fit or whole-Population attachment.", + "source_and_code_before_after_current_equal": true, + "source_authority_granted": false, + "source_commit": "74814151a89e9ffc0ace982846152ef87623e73d", + "source_owned_files": 478, + "test_counts": { + "errors": 0, + "failures": 0, + "skipped": 0, + "tests": 1 + }, + "test_file_counts": { + "test_native_puf55_projection.py": 1 + }, + "unexpected_refusals": {}, + "wall_seconds": 11.048004040989326, + "whole_population_attachment": false +} diff --git a/experiments/us-puf55-numerical-output-seal-46-controls-20260910.json b/experiments/us-puf55-numerical-output-seal-46-controls-20260910.json new file mode 100644 index 000000000..17567c038 --- /dev/null +++ b/experiments/us-puf55-numerical-output-seal-46-controls-20260910.json @@ -0,0 +1,115 @@ +{ + "actual_model_chain_verified": true, + "approval_sha256": "359a56ef088584a149b04a6a4bfa5e047c97f6e90db368376cbcf67d1564756d", + "bridge_owner_sha256": "ca49788ebd67ea807b072cbfbf4817fd7051f967f77098d0e2f8586576add13f", + "cpu_seconds": 18.093259, + "current_checkout_runtime_certification": false, + "financial_fixture_execution": false, + "fixture_apply_target_calls": 220, + "fixture_donors_per_model": 4, + "fixture_fit_target_calls": 110, + "fixture_trees": 2, + "full65_and_puf55_chain_acceptance": false, + "graph_host_execution": false, + "guard_sha256": "5bc1ac74e18b6379d51cf34e55ada8afb17fbb11d06428ebaefd81b4a92baab6", + "host_receiving_value_seal_still_required": true, + "junit_sha256": "aa2825f0279ed6cdbec8b08cad1eebbc287d2bd6d813a538a68b26885c76f481", + "model_cache_metadata_scans": 1506, + "model_source_map_sha256": "018d739504f5919c65104ad44a46c1df045d0bf71002622e66e8b68c3192b035", + "native_acceptance": false, + "native_access": false, + "new_numerical_cases": 28, + "numerical_test_sha256": "7552d83574e000b5e5279e8155711f0d4435125a7e2fe95571ef8b4f9189db26", + "one_whole_cohort_finalizer_per_call": true, + "peak_rss_bytes": 680542208, + "peer_target_review_sha256": "a616d764c903fd90383f7e2b6147bd9f065ff4164e0b98f4ce315f8b646d7b42", + "postcheck_script_sha256": "c0a6e6f5ff02a9086488702b04a5d6c475c903108e6c933c654941141aa0bd59", + "preserved_failed_predecessor": { + "accepted": false, + "failed": 1, + "passed": 45, + "reason": "The new test addressed the source alias employment_income instead of finalized employment_income_before_lsr. It failed before applying its intended mutation.", + "receipt_sha256": "26bf35628a60abacbf765725ebc7cab3a0ed0afd844b1d71e9a153fa97ca6ef7", + "root_failed_postcheck_sha256": "709621c49cd300e83c96fe8100d5257c462b86e3b96b0dd29a4585cbd94c025b", + "tests": 46, + "unexpected_denied_traceback_labels": 22 + }, + "production_adopted": true, + "project_imports_or_runtime_reexecution": false, + "puf_fit_and_numerical_finalization": true, + "receipt_sha256": "b9e6349583a1959e918e491c1267c28535919fc2ef699d1608e9ea52b951f020", + "release_acceptance": false, + "resource_map_sha256": "8add0e99307b3cfe6ba7fb5cc4f68f711d5bf7f82ccde8213bd52ed04ab4ee80", + "root_postcheck_sha256": "e85d557093ac8b53ea4c16eac883b9dae2598889ca35cb56f8db0f9b0f270098", + "root_review_sha256": "73ae7e8ea10b317de40d2a11419aaaf15437b48cd689eb27a1ce6d4473cfc325", + "scope": "46_invented_two_route_numerical_and_raw_merge_controls", + "scope_note": "All 46 invented numerical/raw controls pass on the frozen source. The v2 numerical receipt seals the complete finalized candidate before return. The added test changes an owned finalized value during receipt encoding and requires refusal. Source is adopted locally; native donor/source admission, graph host ancestry, complete enrichment, calibration and release remain separate.", + "selected_test_cases": [ + "test_two_real_chains_finalize_once_on_the_complete_original_cohort", + "test_one_nonempty_route_uses_same_whole_cohort_finalizer[0]", + "test_one_nonempty_route_uses_same_whole_cohort_finalizer[1]", + "test_donor_parity_refuses_before_any_model_decode[recid_order]", + "test_donor_parity_refuses_before_any_model_decode[target]", + "test_donor_parity_refuses_before_any_model_decode[weight]", + "test_donor_parity_refuses_before_any_model_decode[capacity]", + "test_donor_parity_refuses_before_any_model_decode[shared_money]", + "test_donor_parity_refuses_before_any_model_decode[shared_status]", + "test_route_state_and_actual_model_consumption_are_bound[phase]", + "test_route_state_and_actual_model_consumption_are_bound[model_seed]", + "test_route_state_and_actual_model_consumption_are_bound[matrix]", + "test_route_state_and_actual_model_consumption_are_bound[producer]", + "test_route_state_and_actual_model_consumption_are_bound[raw_history]", + "test_route_state_and_actual_model_consumption_are_bound[last_model]", + "test_route_state_and_actual_model_consumption_are_bound[ninth_donor_value]", + "test_route_state_and_actual_model_consumption_are_bound[both_donor_targets]", + "test_route_state_and_actual_model_consumption_are_bound[both_recid_order]", + "test_route_state_and_actual_model_consumption_are_bound[both_weights]", + "test_empty_cohort_and_empty_route_roster_refuse_before_models[no_routes]", + "test_empty_cohort_and_empty_route_roster_refuse_before_models[no_clone_one]", + "test_last_model_return_cannot_change_a_retained_donor[target]", + "test_last_model_return_cannot_change_a_retained_donor[capacity]", + "test_model_donor_frame_matches_existing_single_profile_preparation", + "test_model_frame_constructor_return_cannot_coerce_a_donor_column", + "test_finalizer_return_cannot_change_retained_raw_or_donor[raw]", + "test_finalizer_return_cannot_change_retained_raw_or_donor[donor]", + "test_receipt_encoding_cannot_change_the_already_sealed_candidate", + "test_merge_restores_complete_receiving_order_and_preserves_source_values[False]", + "test_merge_restores_complete_receiving_order_and_preserves_source_values[True]", + "test_single_nonempty_route_omits_other_route[PufOutputProfile.PUF55_SURVEY_SS]", + "test_single_nonempty_route_omits_other_route[PufOutputProfile.PUF55_SURVEY_SS_NO_TOTAL]", + "test_refuses_inexact_recipient_partition[overlap]", + "test_refuses_inexact_recipient_partition[missing]", + "test_refuses_inexact_recipient_partition[extra]", + "test_refuses_inexact_recipient_partition[clone_zero]", + "test_refuses_changed_matrix_route_or_storage[matrix]", + "test_refuses_changed_matrix_route_or_storage[order]", + "test_refuses_changed_matrix_route_or_storage[mutable_bytes]", + "test_refuses_changed_matrix_route_or_storage[profile]", + "test_complete_chain_history_is_checked_before_merge[seed]", + "test_complete_chain_history_is_checked_before_merge[raw]", + "test_complete_chain_history_is_checked_before_merge[model_history]", + "test_complete_chain_history_is_checked_before_merge[target_history]", + "test_changed_detached_decoder_values_refuse", + "test_later_decoder_cannot_change_an_earlier_route_table" + ], + "source_adoption": true, + "source_and_typed_producer_authentication_still_required": true, + "source_count": 500, + "source_map_sha256": "67bdcfc92a36563de2b866b6d6b3bca88c127e90854977c681f56d61b6988e20", + "status": "frozen_numerical_output_seal_controls_accepted", + "test_counts": { + "errors": 0, + "failures": 0, + "skipped": 0, + "tests": 46 + }, + "unchanged_raw_cases": 18, + "unexpected_refusals": {}, + "verification_passed": true, + "verified_current": { + "code_resources": 12, + "model_source": 5983, + "source_and_owned": 508 + }, + "wall_seconds": 21.00948112498736 +} diff --git a/experiments/us-puf55-population-controls-20260909.json b/experiments/us-puf55-population-controls-20260909.json new file mode 100644 index 000000000..dc8cb5d14 --- /dev/null +++ b/experiments/us-puf55-population-controls-20260909.json @@ -0,0 +1,29 @@ +{ + "before_after_current_frozen_and_maintained_source_equal": true, + "before_after_current_model_and_code_resources_equal": true, + "code_resources": 12, + "cpu_seconds": 18.064019, + "independent_selected_value_oracle": "all 55 direct finalizer outputs compared by entity id", + "model_source_files": 5983, + "native_payloads_read": 0, + "peak_rss_bytes": 630538240, + "predictor_count": 9, + "previous_receipt_sha256": "ad8519d867279ebce9ee78bf8f495f3799afb45ec1b3fcda1be649f8a129af30", + "receipt_sha256": "e3a4d548f0209cfacd7d9e3beddb63c1b46204a3168eb9869d2b2eafe9cf2291", + "release_eligible": false, + "required_replay": "passed", + "result": 0, + "scope": "Invented PUF55 whole-Population attachment/replay and independent direct-finalizer oracle; no native recipient qualification or release acceptance.", + "source_and_owned_files": 486, + "ss_incumbent_bits_and_unknownness_preserved": true, + "tampered_ss_population_refusals": 2, + "target_count": 55, + "test_counts": { + "errors": 0, + "failures": 0, + "skipped": 0, + "tests": 3 + }, + "unexpected_refusals": {}, + "wall_seconds": 20.73042149998946 +} diff --git a/experiments/us-puf55-public-recipient-controls-20260910.json b/experiments/us-puf55-public-recipient-controls-20260910.json new file mode 100644 index 000000000..a4866c0ce --- /dev/null +++ b/experiments/us-puf55-public-recipient-controls-20260910.json @@ -0,0 +1,85 @@ +{ + "actual_original_source_issuer_comparisons": true, + "actual_twenty_two_node_cold_and_required_replay": true, + "adopted_source_hashes": { + "packages/microcosm-build/src/microcosm/build/us_runtime/graph_puf55_survey_recipients.py": "888caefb31861bc5892d607ed3598c1ea86eb78ecf1e06035cf3f4f004c8cbd4", + "packages/microcosm-build/tests/test_us_graph_puf55_survey_recipients.py": "b50b99d259c21ac71c4e07f6438465b6614f15ee937d496bac5a45720216246b", + "packages/microcosm-build/tests/test_us_puf55_survey_recipients.py": "fd7ff35d8e752ca7350c7022897f5afa806732ba7f1526f3ea2a1f26bf0a40b6" + }, + "calibration_acceptance": false, + "changed_source_files": [ + "packages/microcosm-build/tests/test_us_puf55_survey_recipients.py" + ], + "corrected_successor": { + "actual_model_chain_verified": false, + "actual_nan_storage_mutation_verified_by_test": true, + "approval_sha256": "fcc818febc58926eb12333217276daf3c41cfd665419b3583ef7d92422412a6a", + "cpu_seconds": 390.301149, + "financial_fixture_fit_and_graph_execution": true, + "full65_and_puf55_chain_acceptance": false, + "guard_sha256": "92b3cda8a663b073b55c5370a6a41d2a9143b30eb6a0c268ad7d9c561d1bd4c4", + "junit_sha256": "5fdb73added770d03cc56847234579f4dc4b7ac9cb11640337782d07d3966bd4", + "model_source_map_sha256": "018d739504f5919c65104ad44a46c1df045d0bf71002622e66e8b68c3192b035", + "native_acceptance": false, + "native_access": false, + "peak_rss_bytes": 539738112, + "postcheck_script_sha256": "f6d472f7e29dda2184df6b1819656c2ecf0679352b6517a44681d966b1fea90c", + "production_adopted": false, + "production_changes": false, + "project_imports_or_runtime_reexecution": false, + "puf_fit_or_finalization": false, + "receipt_sha256": "79d339ff7eb1fe9f9370d3f89976351612a8334bcce46ac91c4e3f549a05ba3d", + "recipient_test_sha256": "fd7ff35d8e752ca7350c7022897f5afa806732ba7f1526f3ea2a1f26bf0a40b6", + "release_acceptance": false, + "resource_map_sha256": "7200349a765cd20316304c73632608851b1320293f797c7c3da7e7daf2229742", + "root_review_sha256": "bb8d8adbea2e8579a39b69b32b774a0acaccf6b59d48d243f91164724fcc734a", + "scope": "five_invented_detached_recipient_output_mutation_controls", + "selected_test_cases": [ + "test_final_run_check_cannot_mutate_detached_recipient_outputs[person]", + "test_final_run_check_cannot_mutate_detached_recipient_outputs[matrix]", + "test_final_run_check_cannot_mutate_detached_recipient_outputs[columns_name]", + "test_final_run_check_cannot_mutate_detached_recipient_outputs[nan_bits]", + "test_final_run_check_cannot_mutate_detached_recipient_outputs[mutable_matrix]" + ], + "source_count": 499, + "source_map_sha256": "5d19ee15bcff9bd26458258bcff86fa18f03856afd93df32f214b599ab120695", + "test_counts": { + "errors": 0, + "failures": 0, + "skipped": 0, + "tests": 5 + }, + "unexpected_refusals": {}, + "verification_passed": true, + "verified_current": { + "code_resources": 12, + "model_source": 5983, + "source_and_owned": 507 + }, + "wall_seconds": 392.8034432080458 + }, + "current_checkout_runtime_certification": false, + "distinct_passing_controls": 28, + "duplicated_passing_cases": 4, + "historical_run": { + "failed": 1, + "failed_case": "test_final_run_check_cannot_mutate_detached_recipient_outputs[nan_bits]", + "junit_sha256": "256569c21fa52f78f4e8e3e809b81fd738564aca32f02b2876e0d625168fe1ed", + "passed": 27, + "receipt_sha256": "c174e4051b711f3beee1a1aff20f8d7aced657b094cd4831ceec5b82fd224451", + "root_evidence_verification_sha256": "d70f8cb7edc2ab2d026fe5ca282e54d45576536d5cb48eb84d92a125ed2bae39", + "source_map_sha256": "9a4888a721f53a4203ed188b81642bed5b423616991ab4c191081afdd1f43af4", + "suite_accepted": false, + "tests": 28 + }, + "native_acceptance": false, + "new_nan_setup_asserts_actual_uint64_storage_change": true, + "passing_executions": 32, + "production_changed_between_runs": false, + "puf_fit_or_finalization": false, + "recipient_nodes_declare_original_survey_and_atomic_support_sources": true, + "release_acceptance": false, + "retained_source_files": 498, + "scope_note": "28 distinct public controls are covered by 27 passing cases in the preserved failed run plus five passing test-only successor cases, four overlapping. This is not a single all-green 28-case invocation. The only source change between these frozen runs makes the NaN test mutation observable; all production and original fixtures are identical. These runs use actual issuers and graphs over invented survey sources.", + "status": "public_recipient_controls_covered_across_preserved_run_and_test_only_successor" +} diff --git a/experiments/us-puf55-public-recipient-positives-20260910.json b/experiments/us-puf55-public-recipient-positives-20260910.json new file mode 100644 index 000000000..51cf6e145 --- /dev/null +++ b/experiments/us-puf55-public-recipient-positives-20260910.json @@ -0,0 +1,48 @@ +{ + "accepted_compilation_cache_overlay_sha256": "534042fdd4f2af5a238b556882d3b75c5495d9909e240bb07201ec48bd2bb4da", + "actual_issued_recipient_positive": true, + "actual_twenty_two_node_cold_required_replay": true, + "cpu_seconds": 331.268576, + "fable_strengthened_issuer_comparisons_included": false, + "frozen_budget_owner_sha256": "80418528db45266e7322a8984cc63f9414f6fa3c084119cd9d0f558f4a0e0260", + "guard_sha256": "45e7faf460d74210613e2898c4471d226a2b8d2e9f241d4c6945fcb7ec7cdb09", + "head": "640c1db49e6a70d47d58cea194e44df26c0762e5", + "junit_case_seconds": { + "test_actual_issued_financial_run_retains_source_reports_and_current_money": "104.759", + "test_actual_twenty_two_node_cold_required_replay_keeps_complete_population": "223.518" + }, + "junit_sha256": "de2befa8fe7f3878a681f4de9e4c81ad467d51fa7d75c33494258b0fc8c46d60", + "junit_timing_scope": "pytest per-case durations include assigned fixture setup/teardown; not isolated issuer or graph compute timings", + "model_source_map_sha256": "018d739504f5919c65104ad44a46c1df045d0bf71002622e66e8b68c3192b035", + "native_acceptance": false, + "original_fixture_teardowns_passed": true, + "original_twenty_node_cold_hit_assertion_present": false, + "peak_rss_bytes": 541048832, + "receipt_sha256": "ae02970da176ad92efacd831de8cf78e18ea1d96a811c124f2d268e6c5da71e4", + "release_acceptance": false, + "remaining_public_cases_executed": 0, + "remaining_public_controls": 24, + "resource_map_sha256": "c7e4193c952bd0d28d81db4371b51b51462870326fe59d5d26c13e4617f3e5b7", + "root_postcheck_sha256": "465b347efcc12e5f717a846207f2db1703b3c31648abefbe355d4274c0c61eab", + "scope": "2_invented_actual_public_puf55_recipient_issuer_and_graph_controls", + "scope_note": "Actual issuers and graph over invented survey originals; no native donor fitting, whole-cohort PUF finalization or enriched calibration.", + "selected_test_cases": [ + "test_us_puf55_recipients_public_positive.py::test_actual_issued_financial_run_retains_source_reports_and_current_money", + "test_us_puf55_recipients_public_positive.py::test_actual_twenty_two_node_cold_required_replay_keeps_complete_population" + ], + "source_count": 498, + "source_map_sha256": "504e045e6551895fabf8bc377fdfce1c69da9e70aa17989d2b771d5b66176c4b", + "status": "two_original_public_positives_accepted", + "test_counts": { + "errors": 0, + "failures": 0, + "skipped": 0, + "tests": 2 + }, + "verified_current": { + "code_resources": 12, + "model_source": 5983, + "source_and_owned": 506 + }, + "wall_seconds": 333.96333820803557 +} diff --git a/experiments/us-puf55-survey-ss-measurement-31-controls-20260909.json b/experiments/us-puf55-survey-ss-measurement-31-controls-20260909.json new file mode 100644 index 000000000..581263a6f --- /dev/null +++ b/experiments/us-puf55-survey-ss-measurement-31-controls-20260909.json @@ -0,0 +1,28 @@ +{ + "code_resources": 12, + "control_accepted": true, + "cpu_seconds": 38.894051, + "frozen_before_after_current_equal": true, + "model_source_files": 5983, + "native_payloads_read": 0, + "peak_rss_bytes": 517455872, + "publisher_provenance_established": false, + "receipt_sha256": "ac932822b7693129e3be97923bbc6798f8883f9fad0365b76b671117f7770f8e", + "release_eligible": false, + "result": 0, + "scope": "31 invented filer/joint-spouse Social Security report-sum measurement, missingness/refusal, authenticated retained-source callback controls, and isolated55-output/eight-predictor declarations. No native source reads, route-aware graph fitting, donor transport acceptance, beneficiary allocation, source admission or release acceptance.", + "source_and_owned_files": 498, + "test_counts": { + "errors": 0, + "failures": 0, + "skipped": 0, + "tests": 31 + }, + "tested_source_changes": { + "packages/microcosm-build/src/microcosm/build/us_runtime/full_puf_enrichment.py": "f76bdc251811286e93f03d3aa9f5caac8548b60595739b5b17efce18d38e715e", + "packages/microcosm-build/src/microcosm/build/us_runtime/puf55_survey_ss_measurement.py": "06c4441f6397a69154f39ae905986a0a7928859cd38d7896fd1d8910ed5504cd", + "packages/microcosm-build/tests/test_us_puf55_survey_ss_measurement.py": "6e7b63e2ac1f7bc2cd0cd6902a89facb91685f2c1f5c8b0d184a7e89143b3dce" + }, + "unexpected_refusals": {}, + "wall_seconds": 41.406911208992824 +} diff --git a/experiments/us-puf55-two-route-numerical-20260910.json b/experiments/us-puf55-two-route-numerical-20260910.json new file mode 100644 index 000000000..3c21f20a6 --- /dev/null +++ b/experiments/us-puf55-two-route-numerical-20260910.json @@ -0,0 +1,105 @@ +{ + "actual_model_chain_verified": true, + "approval_sha256": "f6e923a32416426702e4f1cb92ac91fbf7d07e6534285bd5116475f28a134918", + "bridge_owner_sha256": "ce1d0229965e3e8d2938ff74c205a0b43fcd9f997b795e6441ea9a80be7dfe7d", + "cpu_seconds": 15.591476, + "current_checkout_runtime_certification": false, + "fable_source_review_sha256": "5e9831d555950cda0151d19555aa5157fda944782eb071428166d999484866d7", + "financial_fixture_execution": false, + "fixture_apply_target_calls": 220, + "fixture_donors_per_model": 4, + "fixture_fit_target_calls": 110, + "fixture_trees": 2, + "full65_and_puf55_chain_acceptance": false, + "graph_host_execution": false, + "guard_sha256": "ab00e84bc478f48e2343f2e4da05b8e8777336fc07bc0141c5e7157f00a34f08", + "host_receiving_value_seal_still_required": true, + "junit_sha256": "7242c609dd524533c94848021c843cd9bd6638b9c252be173fa7d28d581b3762", + "model_cache_metadata_scans": 1506, + "model_source_map_sha256": "018d739504f5919c65104ad44a46c1df045d0bf71002622e66e8b68c3192b035", + "native_acceptance": false, + "native_access": false, + "new_numerical_cases": 27, + "numerical_test_sha256": "e3d98a911967e44009485b62dd7405a3628e087f2534d3a614e63e6e8aab1267", + "one_whole_cohort_finalizer_per_call": true, + "peak_rss_bytes": 680591360, + "postcheck_script_sha256": "eb3a753fd91d611fb9185fb73d5a3fb84f83c8e535fcf151a2d7c14f4958f1cd", + "production_adopted": true, + "project_imports_or_runtime_reexecution": false, + "puf_fit_and_numerical_finalization": true, + "receipt_sha256": "4dbc0e70ac4802e6d0a288d0f5ad2c0cab8f081659d1f6d8df3caae6597ae352", + "release_acceptance": false, + "resource_map_sha256": "0742c4e0c6ae9f202d0be2b801fef252bcdec716a45516f7033ab18c89eeda4d", + "root_postcheck_sha256": "7134b14292ba950bb133fd73a347bbccc61df6442f27fb83069e4889fad84519", + "root_review_sha256": "98e95235902ffca638061bf89ca30ebe1efe99d8cc49e56d4ce88ee0ffca2cca", + "scope": "45_invented_two_route_numerical_and_raw_merge_controls", + "scope_note": "Two real invented 55-target chains, exact merged whole-cohort finalization, single-route oracle equality, donor/model binding refusals, and post-finalizer mutation checks. The original18 raw-merger cases remain unchanged. This does not execute the canonical-source CREATE or the full receiving graph host; that host must authenticate producer ancestry before trusted model decoding and seal receiving table values across model/finalizer I/O.", + "selected_test_cases": [ + "test_two_real_chains_finalize_once_on_the_complete_original_cohort", + "test_one_nonempty_route_uses_same_whole_cohort_finalizer[0]", + "test_one_nonempty_route_uses_same_whole_cohort_finalizer[1]", + "test_donor_parity_refuses_before_any_model_decode[recid_order]", + "test_donor_parity_refuses_before_any_model_decode[target]", + "test_donor_parity_refuses_before_any_model_decode[weight]", + "test_donor_parity_refuses_before_any_model_decode[capacity]", + "test_donor_parity_refuses_before_any_model_decode[shared_money]", + "test_donor_parity_refuses_before_any_model_decode[shared_status]", + "test_route_state_and_actual_model_consumption_are_bound[phase]", + "test_route_state_and_actual_model_consumption_are_bound[model_seed]", + "test_route_state_and_actual_model_consumption_are_bound[matrix]", + "test_route_state_and_actual_model_consumption_are_bound[producer]", + "test_route_state_and_actual_model_consumption_are_bound[raw_history]", + "test_route_state_and_actual_model_consumption_are_bound[last_model]", + "test_route_state_and_actual_model_consumption_are_bound[ninth_donor_value]", + "test_route_state_and_actual_model_consumption_are_bound[both_donor_targets]", + "test_route_state_and_actual_model_consumption_are_bound[both_recid_order]", + "test_route_state_and_actual_model_consumption_are_bound[both_weights]", + "test_empty_cohort_and_empty_route_roster_refuse_before_models[no_routes]", + "test_empty_cohort_and_empty_route_roster_refuse_before_models[no_clone_one]", + "test_last_model_return_cannot_change_a_retained_donor[target]", + "test_last_model_return_cannot_change_a_retained_donor[capacity]", + "test_model_donor_frame_matches_existing_single_profile_preparation", + "test_model_frame_constructor_return_cannot_coerce_a_donor_column", + "test_finalizer_return_cannot_change_retained_raw_or_donor[raw]", + "test_finalizer_return_cannot_change_retained_raw_or_donor[donor]", + "test_merge_restores_complete_receiving_order_and_preserves_source_values[False]", + "test_merge_restores_complete_receiving_order_and_preserves_source_values[True]", + "test_single_nonempty_route_omits_other_route[PufOutputProfile.PUF55_SURVEY_SS]", + "test_single_nonempty_route_omits_other_route[PufOutputProfile.PUF55_SURVEY_SS_NO_TOTAL]", + "test_refuses_inexact_recipient_partition[overlap]", + "test_refuses_inexact_recipient_partition[missing]", + "test_refuses_inexact_recipient_partition[extra]", + "test_refuses_inexact_recipient_partition[clone_zero]", + "test_refuses_changed_matrix_route_or_storage[matrix]", + "test_refuses_changed_matrix_route_or_storage[order]", + "test_refuses_changed_matrix_route_or_storage[mutable_bytes]", + "test_refuses_changed_matrix_route_or_storage[profile]", + "test_complete_chain_history_is_checked_before_merge[seed]", + "test_complete_chain_history_is_checked_before_merge[raw]", + "test_complete_chain_history_is_checked_before_merge[model_history]", + "test_complete_chain_history_is_checked_before_merge[target_history]", + "test_changed_detached_decoder_values_refuse", + "test_later_decoder_cannot_change_an_earlier_route_table" + ], + "source_adoption": true, + "source_and_typed_producer_authentication_still_required": true, + "source_count": 500, + "source_map_sha256": "4bfe1790225e49820547aedd02f84203fdfd8658bfb881ac76258ffcbc3f5d2b", + "status": "frozen_two_route_numerical_bridge_accepted", + "test_correction_diff_sha256": "43f79383df5805133baaa38802d8ed61fd2317dc547fa679a81e086d63fe6782", + "test_counts": { + "errors": 0, + "failures": 0, + "skipped": 0, + "tests": 45 + }, + "unchanged_raw_cases": 18, + "unexpected_refusals": {}, + "verification_passed": true, + "verified_current": { + "code_resources": 12, + "model_source": 5983, + "source_and_owned": 508 + }, + "wall_seconds": 16.730265708989464 +} diff --git a/experiments/us-puf55-two-route-values-controls-20260910.json b/experiments/us-puf55-two-route-values-controls-20260910.json new file mode 100644 index 000000000..5db3a77bc --- /dev/null +++ b/experiments/us-puf55-two-route-values-controls-20260910.json @@ -0,0 +1,38 @@ +{ + "attachment_performed": false, + "cpu_seconds": 3.8935099999999996, + "date": "2026-09-10", + "finalization_performed": false, + "fitted_model_or_donor_authority_verified": false, + "guard_sha256": "bde8f7ce2742b45a750e7705be6fc899b7f04f3d913d933d69594ff6724ca6e9", + "junit_sha256": "1619f4b57c5a39df909ee03f7c54d8dd05f634bdda68fe59d12185d012bf4431", + "native_acceptance": false, + "native_access": false, + "next_step": "Bind actual recipient and donor/model owners, execute both real routes, merge raw targets, finalize the whole clone cohort once, and attach once.", + "peak_rss_bytes": 438517760, + "peer_source_review_sha256": "df628b7d237decebbf252f0c51cc215d2801fa67a355068e05a0a0ae21ee70b1", + "postcheck_sha256": "15732db6c91ba14f8c9c012a0f4251c37e6cd1793287958ce0d7f92739e588c4", + "protocol": "microcosm.puf55-two-route-values-controls.v1", + "raw_merge_only": true, + "receipt_sha256": "d52648c33cf48d786a6ee7d4a29838a03e936700db8de05cb4ae0a5b476cac1e", + "release_acceptance": false, + "scope": "18_invented_puf55_raw_route_merge_controls", + "source_files": { + "packages/microcosm-build/src/microcosm/build/us_runtime/puf55_route_finalization.py": "5ce739283cd80d37c17a3063dc0f8585f303411c21b2f5d87cc92ffb6345f9cd", + "packages/microcosm-build/tests/test_us_puf55_route_finalization.py": "f3638ceb41953af57f50c2a43b58eac7fb9dcc4f959832d4921da9ac236041d9" + }, + "source_or_population_admission": false, + "test_counts": { + "errors": 0, + "failures": 0, + "skipped": 0, + "tests": 18 + }, + "typed_chain_metadata_synthetic": true, + "verified_current": { + "code_resources": 12, + "model_source": 5983, + "source_and_owned": 505 + }, + "wall_seconds": 4.931140874978155 +} diff --git a/experiments/us-source-string-seal-20260912.json b/experiments/us-source-string-seal-20260912.json new file mode 100644 index 000000000..c2ea1e348 --- /dev/null +++ b/experiments/us-source-string-seal-20260912.json @@ -0,0 +1,96 @@ +{ + "scope": "invented columns, exact old byte-stream parity; no native build timing claim", + "baseline_commit": "d5cbe60b2c6402648565f5139dbf7d94be209def", + "before_source_sha256": "b1c0ba9856714ce2131f73af6daa7731265a4fcc50c696d095a28f80a9d7cd03", + "after_source_sha256": "9e6ecca118942f7a76a2ef66de22abc082d3e19ca87bca67ca01477958c5b7e2", + "cases": [ + { + "case": "low_cardinality_python_strings", + "rows": 500000, + "dtype": ")>", + "exact_digest_equal": true, + "sha256": "44146e967c4e15ce95279827bcd34fed25b12731c223a3b2ffb16b50026df371", + "before_cpu_seconds": 0.15536000000000016, + "after_cpu_seconds": 0.0252460000000001, + "speedup": 6.1538461538461355, + "all_cpu_samples": { + "before": [ + 0.158949, + 0.1550100000000001, + 0.15536000000000016 + ], + "after": [ + 0.025262000000000118, + 0.024386999999999937, + 0.0252460000000001 + ] + } + }, + { + "case": "low_cardinality_arrow_strings", + "rows": 500000, + "dtype": ")>", + "exact_digest_equal": true, + "sha256": "03a54e7514b1419afbc6e092f3cd308e3c760be4f30c318e08ab85badc5f0241", + "before_cpu_seconds": 0.15287399999999995, + "after_cpu_seconds": 0.02783200000000008, + "speedup": 5.492742167289434, + "all_cpu_samples": { + "before": [ + 0.1512340000000001, + 0.1538870000000001, + 0.15287399999999995 + ], + "after": [ + 0.028616999999999893, + 0.02783200000000008, + 0.027589000000000086 + ] + } + }, + { + "case": "unique_source_keys", + "rows": 100000, + "dtype": ")>", + "exact_digest_equal": true, + "sha256": "3fa0e7051459014e6069c01acde5e7afc8de133e769a52e0a520fe75821a6d0b", + "before_cpu_seconds": 0.031774999999999665, + "after_cpu_seconds": 0.035517999999999716, + "speedup": 0.8946168140097955, + "all_cpu_samples": { + "before": [ + 0.03200500000000028, + 0.030060000000000198, + 0.031774999999999665 + ], + "after": [ + 0.035517999999999716, + 0.035597999999999796, + 0.03467199999999959 + ] + } + }, + { + "case": "numeric_control", + "rows": 500000, + "dtype": "dtype('int64')", + "exact_digest_equal": true, + "sha256": "d8577194aeb21cddc40a14a2e0a1f2f13dee18b0b9c26e83842214f3b98bb2ba", + "before_cpu_seconds": 0.0013450000000001516, + "after_cpu_seconds": 0.0013260000000001604, + "speedup": 1.0143288084464472, + "all_cpu_samples": { + "before": [ + 0.0014279999999997628, + 0.0013450000000001516, + 0.0013090000000000046 + ], + "after": [ + 0.0013320000000001109, + 0.0013260000000001604, + 0.0012980000000002434 + ] + } + } + ] +} diff --git a/experiments/us-source-string-seal-20260912.md b/experiments/us-source-string-seal-20260912.md new file mode 100644 index 000000000..2309362a3 --- /dev/null +++ b/experiments/us-source-string-seal-20260912.md @@ -0,0 +1,31 @@ +# Repeated source-string hashing + +The ASEC source validator now reuses the exact length-prefixed encoding of a +repeated string within one column's digest calculation. It still visits every +value and computes a new digest on every validation. The cache holds at most +1,024 entries and 256 KiB of encoded payload, then stops admitting new entries. +Only exact Python strings use it; other scalar families retain the original +codec. Dtype metadata, null sentinels, row order and all digest bytes retain +their prior meaning. + +Thirty invented-source tests pass, including eleven added byte-stream parity +and later-mutation controls. They cover Python and Arrow strings, distinct +Unicode encodings, nulls, more than 1,024 distinct values, strings exceeding the +payload budget, and the unchanged bytes, boolean, integer and floating object +paths. The suite took 1.860 seconds, with no failures, errors or skips. + +The [benchmark receipt](us-source-string-seal-20260912.json) compares the exact +pre-change function from `d5cbe60b2` with the new implementation. Each row reports +the median of three process-CPU measurements on invented columns: + +| Column | Rows | Before | After | Ratio | +| --- | ---: | ---: | ---: | ---: | +| Repeated Python strings | 500,000 | 0.15536 s | 0.02525 s | 6.15× faster | +| Repeated Arrow strings | 500,000 | 0.15287 s | 0.02783 s | 5.49× faster | +| Unique source keys | 100,000 | 0.03178 s | 0.03552 s | 12% slower | +| Integer control | 500,000 | 0.00135 s | 0.00133 s | Approximately unchanged | + +All four complete digests match exactly. These measurements do not establish +native build speed or quality. Source-file identity changes because executable +code changed; an old graph receipt is not acceptance of the new implementation. +The active native PUF pilot retains its original frozen source. diff --git a/experiments/us-survey-age-development-20260909.json b/experiments/us-survey-age-development-20260909.json new file mode 100644 index 000000000..54df4b623 --- /dev/null +++ b/experiments/us-survey-age-development-20260909.json @@ -0,0 +1,54 @@ +{ + "artifact_readback_and_final_owner_target_verification": "passed inside the unchanged reviewed harness; final manifest exists only after final verification", + "code_resource_pins_current_equal": true, + "code_resources": 12, + "compiled_order": [ + "survey_population.create", + "survey_population.allocate", + "combined_survey_puf_support_clone", + "combined_survey_puf_support_clone.owned", + "survey.age_count_matrix", + "survey.sampling_budget", + "survey.age_calibration" + ], + "cpu_seconds": 4115.737018, + "cross_process_source_authority_granted": false, + "entity_rows": { + "current": { + "household": 3168, + "person": 6928 + }, + "prepared": { + "household": 1584, + "person": 3464 + }, + "previous": { + "household": 3168, + "person": 6928 + } + }, + "export_count": 5, + "manifest_sha256": "2bdb68de84635c6cc2e0c25555887693304792e25d40052cd77996774cb7f669", + "native_counts": { + "acs": { + "households": 1529, + "persons": 3324 + }, + "asec": { + "households": 55, + "persons": 140 + } + }, + "native_payloads_read_by_postcheck": 0, + "owned_before_after_current_equal": true, + "peak_rss_bytes": 8260485120, + "receipt_sha256": "2aa76b5daca7bb0df1ac85d4db23d6b5482073d31d437418d46442ef01b002ca", + "release_eligible": false, + "result": 0, + "scope": "genuine seven-node ACS+ASEC age-development artifact acceptance; not full enrichment or release", + "source_before_after_current_equal": true, + "source_control_runtime_files": 473, + "source_head": "a9895d8a50886470f6dfe4b2b9d20c352c9871cf", + "unexpected_refusals": {}, + "wall_seconds": 4268.8438986669935 +} diff --git a/experiments/us-survey-catalogue-memo20-20260910.json b/experiments/us-survey-catalogue-memo20-20260910.json new file mode 100644 index 000000000..ff6b93661 --- /dev/null +++ b/experiments/us-survey-catalogue-memo20-20260910.json @@ -0,0 +1,32 @@ +{ + "code": [ + { + "maintained_sha256": "508d18bac53690f8841e3eb2770bc2aa6079b9180ad0a31c65c5566f3304a206", + "path": "packages/microcosm-build/src/microcosm/build/us_runtime/survey_population_preparation.py" + }, + { + "maintained_sha256": "8a9742e108347916937317a0901a462c315d688b32795758972ff9554b07d32f", + "path": "packages/microcosm-build/tests/test_us_survey_catalogue_immutable_memo.py" + } + ], + "evidence": { + "cpu_seconds": 18.188797, + "model_sources": 0, + "peak_rss_bytes": 405815296, + "physical_postcheck_sha256": "fcd8dcdb6b706f65888b1e33add9c042302d51358597973db14901e0c70f17b8", + "receipt_sha256": "ee5279f9678d035b0d6ede176704b4d04196d1eea351adfba56510dba973f972", + "resources": 12, + "source_peer_review_sha256": "c5e0e1188fdac1e84929f46d8e13f5e42e33b286f86a121ddd16aa38222c7e1d", + "source_plus_owned": 514, + "tests_failed": 0, + "tests_passed": 20, + "unexpected_refusals": {}, + "wall_seconds": 18.8203088750015, + "xml_sha256": "04dec52f2e8ca14815b8368a7ac2023d2f08f752c94367f9a6d7b2189402077f" + }, + "maintained_full_CI_accepted": false, + "native_build_accepted": false, + "release_eligible": false, + "scope": "Reuse a sealed immutable ACS catalogue value within its issued preparation. All source and producer checks, complete Frame seals and final checks remain; replacing roots or mutable contents takes the original hashing path. Native performance remains to be measured.", + "status": "scoped_controls_accepted" +} diff --git a/experiments/us-survey-enrichment-integration-20260912.md b/experiments/us-survey-enrichment-integration-20260912.md new file mode 100644 index 000000000..3fef6cc29 --- /dev/null +++ b/experiments/us-survey-enrichment-integration-20260912.md @@ -0,0 +1,90 @@ +# Survey amount and health integration + +The release branch now includes the source-qualified health fragment from +`7cb1384be82ae4bd559577c5ebe142bf708e8570` and the fixed post-PUF enrichment host +from `99c35b0865e6f6cfe86ede15a10da91360a5e66b`. Their local integration commits +are `f92418a859e576ab7fd40919d3e9d75f41bd1ee4` and +`091261c0e979d0ee191ddbaf08a069b2554d7165`. Both cherry-picks were conflict-free. + +The preceding main merge incorporated shared graph PR #913, main revision +`a9cc63e737fd4619a304117dc2ec7dcd86d97901`, without changing the release branch's +existing source tree. The checked PUF API, eighth financial output, source-string +hashing optimization, ACS anchor qualification and signed reconciliation helper +are preserved. None of those building blocks is interpreted as native acceptance +of this integrated candidate. + +## Accepted combined stage + +The host retains the original checked PUF run and original survey source owners. +It adds four amounts and nine coverage fields to `survey_puf55.receiving`, with +health explicitly dependent on the amount attachment. Source reporting-universe, +knownness and allocation information remain distinct from numeric values. +The [source/model specification](../docs/us-current-survey-amount-successor.md) +records UC classification, PHIP_VAL rather than PHIP_VAL2, the conditional +health-cost chain, deliberate ACS clone-sharing and narrower coverage gaps. + +On the accepted prior implementation, the 18-person invented fixture executed +26 new nodes over a validated 245-node prefix. Required replay hit all 271 nodes; +complete Frame readback preserved parent columns, owners, geography, design +anchors and mass ledger. The diagnostic profile contained 74 of 161 input names, +leaving 87 missing. UC had ten known and eight unresolved values; all 18 had the +three medical-cost amounts and ESI, while narrower Medicaid coverage had eight +known and ten unresolved values. These are invented counts, not population +estimates, native coverage or evidence of statistical quality. + +Acceptance combines the first five passing tests from the main v3 attempt and +the corrected final parent-revocation test from a focused v4 attempt. The v3 +result remains failed: its last test named a nonexistent income column and +stopped before mutation. The corrected test uses the actual +`employment_income_before_lsr` column and proves permanent parent and child +revocation, including after restoring the original value. It passed without +repeating the already-passed required replay and Frame readback. There is no +claim of one all-green full v3 run. Independent source and evidence review +approved this six-control coverage at unchanged production bytes. + +| Preserved local record | SHA256 | +| --- | --- | +| Combined v3 receipt | `19495b1cf3e6aad2f60771c8dce557ccbdc6a78b98c1a1ad453229276ac7e4a4` | +| Combined v3 aggregate acceptance | `07ac957851b19546a5ea1d2123975b9fa616dc8fe94a75e171c93e2379a094da` | +| Focused v4 parent-revocation receipt | `42bbeeae05b01a737d3c50cf3b5569be37cab43d71378fd8e78872965697170c` | + +The v3 acceptance took 1,388.313 seconds with 917,979,136 bytes peak RSS; the +focused v4 control took 624.872 seconds with 885,669,888 bytes peak RSS. Both +retained 857 Python source pins and Torch thread settings of 1/1. These wrappers +are not the strict native pilot audit guard. The source-reviewed execution path +used invented fixtures; receipt flags alone do not prove native admission or +independent network/resource restrictions. Failed attempts remain preserved. + +## Checks after integration + +At integrated source `091261c0e979d0ee191ddbaf08a069b2554d7165`, 80 tests passed +with zero failures, errors or skips in 27.149 seconds wall time; peak RSS was +503,201,792 bytes. The selection includes all 32 amount controls, all 31 health +source/fragment controls, the actual invented UC source-qualification test and +the cheap financial/source controls, including the conserving interest +attachment. The health fragment executes cold and required replay. The expensive +financial/PUF/combined fixture was not selected again. + +All 477 Python files under the shard source directories retained their hashes; +Torch threads remained 1/1. CPU and wall limits were applied. This is bounded +invented integration evidence, not the native admission guard. + +| Integrated record | SHA256 | +| --- | --- | +| Receipt | `b2a764144afe952bba7a0f274cbf4f71ad6f0e46174743610127df7d42e30ba9` | +| JUnit result | `201fff9ac69b9259a9a7c11a291904dac97f082a92364521b622fc80d6805074` | + +Actual declaration functions, invoked with invented descriptive values, confirm +three financial model targets, eight financial output fields and ten financial +extension nodes. Each PUF route retains 55 fits and 55 applications. The amount +fragment has 14 nodes and health has 12; together they compile with the explicit +parent and attachment dependencies. The complete host therefore still declares +271 nodes. The added tax-exempt-interest output changes the existing attachment +and its source identity, not the target chain or node count. No issued parent, +native source or full graph execution is established by this declaration probe. + +Native PUF execution, original-channel tax-detail completion, remaining inputs, +full-candidate coverage, model evaluation, calibration and release verification +remain separate work. The ACS anchor qualifier and signed projection helper are +adopted source/numeric building blocks; the current enrichment host does not yet +invoke them to reconcile property or retirement components. diff --git a/experiments/us-survey-geography-graph-controls-20260909.json b/experiments/us-survey-geography-graph-controls-20260909.json new file mode 100644 index 000000000..6952299e9 --- /dev/null +++ b/experiments/us-survey-geography-graph-controls-20260909.json @@ -0,0 +1,21 @@ +{ + "before_after_current_frozen_and_maintained_source_equal": true, + "before_after_current_model_and_code_resources_equal": true, + "code_resources": 12, + "cpu_seconds": 313.767356, + "model_source_files": 5983, + "peak_rss_bytes": 439091200, + "receipt_sha256": "28a6b5c58a258c7b3129a48054a3360647c788f8b76d781cc3bc8fc35a73f997", + "release_eligible": false, + "result": 0, + "scope": "Observed-geography graph node over actual invented original-source issuers: cold/replay/unknownness/binding/mutation controls. No native atomic assignment or release acceptance.", + "source_and_owned_files": 486, + "test_counts": { + "errors": 0, + "failures": 0, + "skipped": 0, + "tests": 14 + }, + "unexpected_refusals": {}, + "wall_seconds": 319.43621275003534 +} diff --git a/experiments/us-survey-geography-source-controls-20260909.json b/experiments/us-survey-geography-source-controls-20260909.json new file mode 100644 index 000000000..f34e40267 --- /dev/null +++ b/experiments/us-survey-geography-source-controls-20260909.json @@ -0,0 +1,21 @@ +{ + "before_after_and_frozen_source_control_equal": true, + "before_after_current_model_and_code_resources_equal": true, + "code_resources": 12, + "cpu_seconds": 178.432832, + "model_source_files": 5983, + "peak_rss_bytes": 430325760, + "receipt_sha256": "1900801b1bbcea89a3747f656410e500b1328519694c3351100562bba61cab08", + "release_eligible": false, + "result": 0, + "scope": "Invented source qualification only; no native atomic assignment, engine, data or release acceptance.", + "source_and_control_files": 484, + "test_counts": { + "errors": 0, + "failures": 0, + "skipped": 0, + "tests": 11 + }, + "unexpected_refusals": {}, + "wall_seconds": 182.49943441699725 +} diff --git a/experiments/us-survey-interest-conservation-20260912.md b/experiments/us-survey-interest-conservation-20260912.md new file mode 100644 index 000000000..97bd273a0 --- /dev/null +++ b/experiments/us-survey-interest-conservation-20260912.md @@ -0,0 +1,32 @@ +# Conserving survey interest before PUF enrichment + +The financial graph now retains the tax-exempt remainder of the same interest +total used for its taxable component. Previously it attached only the taxable +part, leaving the original survey support's exempt component unresolved after +the clone1-only PUF attachment. + +The new eighth financial output is `INT_VAL - taxable_interest_income`, computed +before source-key alignment to both clones. ASEC uses its qualified current +amount; ACS uses the existing modeled interest total. The original taxable +calculation and legacy seven-leaf helper are unchanged. The graph's judgment +metadata identifies the split as an assumption, not separately observed +taxable and exempt amounts. It does not establish that the modeled ACS interest +total reconciles to the broader observed ACS property-income aggregate. + +Two new numerical controls first reproduced the missing column. After repair, +all 25 predictor and atomic-financial tests passed in 294.225 seconds, including +nonconstant and zero-regime invented source runs, source-key permutation, +conservation and cold/required graph replay. + +Three full PUF controls then passed in 617.283 seconds: the actual two-route +cold/required extension, independent whole-cohort finalizer comparison and +non-owned storage preservation, and retained checked-parent validation without +re-execution. The original survey complement remains identical through the +PUF attachment; clone1 retains its existing PUF ownership. All tests use +invented sources. No failure, error or skip occurred in either green run. + +Independent source review found no actionable defect. Ruff and CI test inventory +verification pass. The executable source SHA256 is +`0df64e5f7768b326f43d7b0ac927ccb24c89de8e3d21556a9139628a369b5358`. +These checks do not certify a native candidate, fiscal calibration or release. +The running native PUF pilot remains on its earlier frozen source. diff --git a/experiments/us-survey-numeric-diagnostics27-20260910.json b/experiments/us-survey-numeric-diagnostics27-20260910.json new file mode 100644 index 000000000..c74a4409e --- /dev/null +++ b/experiments/us-survey-numeric-diagnostics27-20260910.json @@ -0,0 +1,38 @@ +{ + "code": [ + { + "maintained_sha256": "22ff28977582c3b903ecf5d52b791a9f864a6a3c3df3b64541571e9199876b81", + "path": "packages/microcosm-build/src/microcosm/build/us_runtime/survey_calibration_diagnostics.py" + }, + { + "maintained_sha256": "4d72fa2260af1178c889819295f39f3c2a0b79c37152eaf14e83cb7483b907aa", + "path": "packages/microcosm-build/tests/test_us_survey_calibration.py" + } + ], + "evidence": { + "cpu_seconds": 3.194314, + "model_sources": 0, + "peak_rss_bytes": 414924800, + "physical_postcheck_sha256": "e8a6f475e8bb8564c919ccfcbfce28182abd5b57e61029d0dab40404bae45d31", + "receipt_sha256": "50c1a09d4e71cffb00a375a703cb04623521a612478c34b2ac558c907bd4188b", + "resources": 0, + "source_peer_review_sha256": "4bfde0c677cecde4f2912a0a524c937292d33ee40ba35c10d5d29d34fa4c7b71", + "source_plus_owned": 512, + "tests_failed": 0, + "tests_passed": 27, + "unexpected_refusals": {}, + "wall_seconds": 3.6059712909627706, + "xml_sha256": "e8dc6d33a24fcf4c0a226216cb6be574b599c73f4b16cb666bb9db1724d35ba1" + }, + "maintained_full_CI_accepted": false, + "native_build_accepted": false, + "release_eligible": false, + "scope": "Independently reconstruct four closed solver options when verifying calibration diagnostics: no supplied gate initialization, nonzero-count budget basis, no feasible-draw upper bound and no budget search. Real grouped Adam over five invented households and four altered-option refusals pass against maintained calibration source. Complete age graph and native calibration remain separate.", + "status": "scoped_controls_accepted", + "test_formatting": { + "accepted_sha256": "e8a5fd5eea56ac357c5e919de0206d7ea44ff77b5dbf3809bea214b5d0e01109", + "change": "Ruff parenthesized one import; no AST change.", + "complete_AST_equal": true, + "maintained_sha256": "4d72fa2260af1178c889819295f39f3c2a0b79c37152eaf14e83cb7483b907aa" + } +} diff --git a/packages/microcosm-build/src/microcosm/build/atomic_geography.py b/packages/microcosm-build/src/microcosm/build/atomic_geography.py new file mode 100644 index 000000000..dc419490f --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/atomic_geography.py @@ -0,0 +1,591 @@ +"""Country-neutral atomic geography support, assignment and functional lookups. + +Publisher adapters supply one row per atomic area. Country declarations choose +constraints, sampling stages and output names. Observed geography is never +overwritten. This module does not acquire sources or certify their authority. +""" + +from __future__ import annotations + +import bisect +import hashlib +import json +from collections.abc import Mapping +from dataclasses import dataclass +from io import BytesIO +from types import MappingProxyType +from zipfile import ZIP_DEFLATED, ZipFile, ZipInfo + +import numpy as np +import pandas as pd + +from microcosm.graph.canonical import canonical_json +from microcosm.graph.randomness import keyed_uniform + +RELATIONS = frozenset({"exact", "best_fit", "official_tabulation", "inferred_modal"}) +_U53 = 2**53 + + +def _require(condition: bool, message: str) -> None: + if not condition: + raise ValueError("Atomic geography: " + message) + + +def _text(value: object) -> bool: + return isinstance(value, str) and bool(value) and value.strip() == value + + +def _unique_json(pairs): + result = {} + for key, value in pairs: + _require(key not in result, "duplicate metadata key") + result[key] = value + return result + + +@dataclass(frozen=True) +class AtomicSupport: + """Validated, immutable arrays decoded from a pinned support artifact.""" + + metadata: Mapping + arrays: Mapping[str, np.ndarray] + sha256: str + + +def encode_atomic_support(metadata: Mapping, arrays: Mapping[str, np.ndarray]) -> bytes: + """Create deterministic NPZ bytes; validate through the same decoder as readers.""" + _require("metadata_json" not in arrays, "reserved metadata array") + output = BytesIO() + members = {**arrays, "metadata_json": np.asarray(canonical_json(metadata).decode())} + with ZipFile(output, "w", compression=ZIP_DEFLATED) as archive: + for name, array in sorted(members.items()): + _require(_text(name) and name.isidentifier(), "invalid array name") + member = BytesIO() + np.lib.format.write_array(member, np.asarray(array), allow_pickle=False) + info = ZipInfo(name + ".npy", date_time=(1980, 1, 1, 0, 0, 0)) + info.compress_type = ZIP_DEFLATED + archive.writestr(info, member.getvalue()) + payload = output.getvalue() + decode_atomic_support(payload) + return payload + + +def decode_atomic_support(payload: bytes) -> AtomicSupport: + """Refuse ambiguous mappings, object arrays, untyped columns and invalid weights.""" + _require(type(payload) is bytes, "support requires immutable bytes") + with np.load(BytesIO(payload), allow_pickle=False) as archive: + _require(len(set(archive.files)) == len(archive.files), "duplicate array") + meta_array = archive["metadata_json"] + _require( + meta_array.shape == () and meta_array.dtype.kind == "U", "metadata encoding" + ) + metadata = json.loads(str(meta_array.item()), object_pairs_hook=_unique_json) + _require( + isinstance(metadata, dict) + and set(metadata) + == {"version", "system", "level", "code_system", "vintage", "columns"} + and type(metadata["version"]) is int + and metadata["version"] == 1, + "metadata schema", + ) + _require( + all( + _text(metadata[k]) + for k in ("system", "level", "code_system", "vintage") + ), + "area identity", + ) + columns = metadata["columns"] + _require( + isinstance(columns, dict) and "area" in columns and bool(columns), + "column metadata", + ) + _require(set(archive.files) == {"metadata_json", *columns}, "undeclared array") + arrays = {} + for name, description in columns.items(): + _require( + _text(name) and name.isidentifier() and name != "metadata_json", + "column name", + ) + _require(isinstance(description, dict), "column description") + array = archive[name] + _require( + array.ndim == 1 and len(array) > 0, "nonempty one-dimensional support" + ) + kind = description.get("kind") + if kind == "code": + _require( + set(description) == {"kind", "source", "vintage", "relation"}, + "code metadata", + ) + _require( + array.dtype.kind == "U" and np.all(np.char.str_len(array) > 0), + "nonempty string codes", + ) + _require(np.all(np.char.strip(array) == array), "whitespace in codes") + _require( + _text(description["vintage"]) + and description["relation"] in RELATIONS, + "mapping relation or vintage", + ) + else: + _require( + kind == "weight" + and set(description) == {"kind", "source", "basis"}, + "weight metadata", + ) + _require( + array.dtype.kind in "iu" and np.all(array >= 0), + "nonnegative integer weights", + ) + _require(_text(description["basis"]), "sampling basis") + # Python accumulation cannot wrap before the bound is checked. + total = sum(int(value) for value in array) + _require(0 < total <= _U53, "weight total outside exact sampling range") + array = array.astype(np.int64) + _require(_text(description["source"]), "source identity") + # Bytes-backed arrays cannot be made writable by changing flags. + arrays[name] = np.frombuffer(array.tobytes(), dtype=array.dtype) + area = arrays["area"] + _require(columns["area"]["kind"] == "code", "atomic area must be a code") + _require( + columns["area"]["vintage"] == metadata["vintage"], "atomic vintage mismatch" + ) + _require( + columns["area"]["relation"] == "exact", "atomic identity must be exact" + ) + _require( + all(len(a) == len(area) for a in arrays.values()), "array lengths differ" + ) + _require(len(np.unique(area)) == len(area), "atomic codes must be unique") + _require( + any(c["kind"] == "weight" for c in columns.values()), + "missing sampling weights", + ) + frozen = { + **metadata, + "columns": MappingProxyType( + {k: MappingProxyType(v) for k, v in columns.items()} + ), + } + return AtomicSupport( + MappingProxyType(frozen), + MappingProxyType(arrays), + hashlib.sha256(payload).hexdigest(), + ) + + +def validate_assignment_spec(spec: Mapping) -> dict: + """Normalize a graph parameter mapping without accepting undeclared options. + + All geography code columns use strings, preserving leading zeroes. A later + engine adapter may declare a separate storage conversion when needed. + """ + value = json.loads(canonical_json(spec)) + _require( + set(value) == {"version", "identity", "stream", "outputs", "systems"} + and value["version"] == 1 + and type(value["version"]) is int, + "assignment schema", + ) + identity, outputs, systems = value["identity"], value["outputs"], value["systems"] + _require( + isinstance(identity, list) + and bool(identity) + and all(_text(c) for c in identity) + and len(set(identity)) == len(identity), + "stable identity columns", + ) + _require( + isinstance(outputs, dict) + and set(outputs) == {"area", "system", "basis"} + and all(_text(c) for c in outputs.values()) + and len(set(outputs.values())) == 3, + "assignment outputs", + ) + keyed_uniform(stream=tuple(value["stream"]), keys=[]) + _require(isinstance(systems, list) and bool(systems), "area systems") + ids, layers = set(), {} + for system in systems: + _require( + isinstance(system, dict) + and set(system) + == { + "id", + "level", + "code_system", + "vintage", + "source", + "selector", + "constraints", + "observed_area", + "stages", + "layers", + }, + "system schema", + ) + _require( + all( + _text(system[k]) + for k in ("id", "level", "code_system", "vintage", "source") + ), + "system identity", + ) + _require(system["id"] not in ids, "duplicate system") + ids.add(system["id"]) + selector = system["selector"] + _require( + isinstance(selector, dict) + and all( + _text(c) + and isinstance(v, list) + and bool(v) + and all(_text(x) for x in v) + and len(set(v)) == len(v) + for c, v in selector.items() + ), + "system selector", + ) + _require( + system["observed_area"] is None or _text(system["observed_area"]), + "observed atomic column", + ) + constraints = system["constraints"] + _require(isinstance(constraints, list), "constraints") + seen = set() + for c in constraints: + _require( + isinstance(c, dict) + and set(c) == {"input", "support", "required"} + and _text(c["input"]) + and _text(c["support"]) + and type(c["required"]) is bool, + "constraint schema", + ) + _require( + c["support"] not in seen and c["support"] != "area", + "duplicate or atomic constraint", + ) + seen.add(c["support"]) + stages = system["stages"] + _require(isinstance(stages, list) and bool(stages), "sampling stages") + _require( + all( + isinstance(s, dict) + and set(s) == {"level", "weight"} + and _text(s["level"]) + and _text(s["weight"]) + for s in stages + ), + "stage schema", + ) + _require( + stages[-1]["level"] == "area" + and len({s["level"] for s in stages}) == len(stages), + "stages must end at the unique atomic level", + ) + _require(isinstance(system["layers"], list), "layers") + local_layers = set() + for layer in system["layers"]: + _require( + isinstance(layer, dict) + and set(layer) == {"input", "output", "vintage", "relation", "source"} + and all(_text(x) for x in layer.values()) + and layer["relation"] in RELATIONS, + "layer schema", + ) + name = layer["output"] + _require( + name not in outputs.values() and name not in local_layers, + "duplicate layer output", + ) + local_layers.add(name) + layers.setdefault(name, []).append(layer) + required = set(identity) + for system in systems: + required.update(system["selector"]) + required.update(c["input"] for c in system["constraints"]) + if system["observed_area"]: + required.add(system["observed_area"]) + _require( + not required.intersection({*outputs.values(), *layers}), + "observed or identity columns would be overwritten", + ) + return value + + +def _bind(spec: Mapping, supports: Mapping[str, AtomicSupport]): + spec = validate_assignment_spec(spec) + _require( + set(supports) == {s["id"] for s in spec["systems"]}, "support systems differ" + ) + for system in spec["systems"]: + support = supports[system["id"]] + _require( + all( + support.metadata[k] == system[v] + for k, v in ( + ("system", "id"), + ("level", "level"), + ("code_system", "code_system"), + ("vintage", "vintage"), + ) + ), + "support identity differs from declaration", + ) + columns = support.metadata["columns"] + for c in system["constraints"]: + _require( + c["support"] in columns and columns[c["support"]]["kind"] == "code", + "unknown constraint", + ) + for s in system["stages"]: + _require( + s["level"] in columns + and columns[s["level"]]["kind"] == "code" + and s["weight"] in columns + and columns[s["weight"]]["kind"] == "weight", + "unknown stage column", + ) + for layer in system["layers"]: + description = columns.get(layer["input"], {}) + _require( + description.get("kind") == "code" + and all( + description[k] == layer[k] + for k in ("source", "vintage", "relation") + ), + "layer source, relation or vintage differs", + ) + return spec + + +def _route(households: pd.DataFrame, spec: Mapping) -> dict[str, np.ndarray]: + masks, claimed = {}, np.zeros(len(households), dtype=np.int64) + for system in spec["systems"]: + mask = np.ones(len(households), dtype=bool) + for name, accepted in system["selector"].items(): + mask &= households[name].isin(accepted).to_numpy() + masks[system["id"]] = np.flatnonzero(mask) + claimed += mask + _require(np.all(claimed == 1), "every household must match exactly one area system") + return masks + + +def _constraints(row, system) -> tuple: + values = [] + for c in system["constraints"]: + value = row[c["input"]] + if pd.isna(value): + _require(not c["required"], "missing required observed geography") + else: + _require(_text(value), "observed codes require nonempty strings") + values.append((c["support"], value)) + return tuple(sorted(values)) + + +class _SupportIndex: + """Index each observed-column pattern once, rather than scanning per household.""" + + def __init__(self, support): + self.support = support + self.groupings = {} + self.area_index = pd.Index(support.arrays["area"]) + self.draw_cells = {} + + def rows(self, constraints): + columns = tuple(k for k, _ in constraints) + if columns not in self.groupings: + if columns: + table = pd.DataFrame({c: self.support.arrays[c] for c in columns}) + # Always group by a list: normalize pandas' single-column key below. + groups = table.groupby(list(columns), sort=False, observed=True).indices + self.groupings[columns] = { + (k,) if len(columns) == 1 and not isinstance(k, tuple) else k: v + for k, v in groups.items() + } + else: + self.groupings[columns] = {(): np.arange(len(self.area_index))} + key = tuple(v for _, v in constraints) + _require( + key in self.groupings[columns], + "observed constraints have no common support", + ) + return self.groupings[columns][key] + + def pick(self, constraints, stage, u): + key = (constraints, stage["level"], stage["weight"]) + if key not in self.draw_cells: + rows = self.rows(constraints) + levels, inverse = np.unique( + self.support.arrays[stage["level"]][rows], return_inverse=True + ) + weights = np.zeros(len(levels), dtype=np.int64) + np.add.at(weights, inverse, self.support.arrays[stage["weight"]][rows]) + cumulative = np.cumsum(weights).tolist() + _require(cumulative[-1] > 0, "constraint cell has zero sampling mass") + self.draw_cells[key] = levels, cumulative + levels, cumulative = self.draw_cells[key] + # Integer inverse CDF: no floating-point rounding across a cell boundary. + threshold = (int(u * _U53) * cumulative[-1]) // _U53 + return str(levels[bisect.bisect_right(cumulative, threshold)]) + + +def assign_atomic( + households: pd.DataFrame, spec: Mapping, supports: Mapping[str, AtomicSupport] +) -> pd.DataFrame: + """Select atomic areas without changing row order, ids or observed geography.""" + spec = _bind(spec, supports) + _require( + not set(spec["outputs"].values()).intersection(households.columns), + "assignment output already exists", + ) + _require( + not households[spec["identity"]].isna().any().any() + and not households.duplicated(spec["identity"]).any(), + "null or duplicate draw identity", + ) + routing = _route(households, spec) + result = pd.DataFrame(index=households.index) + for name in spec["outputs"].values(): + result[name] = pd.array([pd.NA] * len(households), dtype="string") + definition = hashlib.sha256(canonical_json(spec)).hexdigest() + for system in spec["systems"]: + positions = routing[system["id"]] + support = supports[system["id"]] + index = _SupportIndex(support) + identities = list( + households.iloc[positions][spec["identity"]].itertuples( + index=False, name=None + ) + ) + draws = [ + keyed_uniform( + stream=tuple(spec["stream"]), + keys=[ + ( + *identity, + system["id"], + stage["level"], + definition, + support.sha256, + ) + for identity in identities + ], + ) + for stage in system["stages"] + ] + for local, position in enumerate(positions): + row = households.iloc[position] + constraints = _constraints(row, system) + area = row[system["observed_area"]] if system["observed_area"] else None + observed = area is not None and not pd.isna(area) + if observed: + _require(_text(area), "invalid observed atomic code") + match = index.area_index.get_indexer([area])[0] + _require( + match >= 0 + and all( + support.arrays[c][match] == value for c, value in constraints + ), + "observed atomic code is unsupported or conflicts", + ) + else: + for stage, uniforms in zip(system["stages"], draws, strict=True): + area = index.pick(constraints, stage, uniforms[local]) + constraints = tuple( + sorted({**dict(constraints), stage["level"]: area}.items()) + ) + for key, value in ( + ("area", area), + ("system", system["id"]), + ("basis", "observed" if observed else "assigned"), + ): + result.iloc[position, result.columns.get_loc(spec["outputs"][key])] = ( + value + ) + return result + + +def derive_geography( + households: pd.DataFrame, spec: Mapping, supports: Mapping[str, AtomicSupport] +) -> pd.DataFrame: + """Functional area lookups, including explicitly identified non-nesting conventions.""" + spec = _bind(spec, supports) + routing = _route(households, spec) + result = pd.DataFrame(index=households.index) + for name in sorted( + {layer["output"] for s in spec["systems"] for layer in s["layers"]} + ): + result[name] = pd.array([pd.NA] * len(households), dtype="string") + for system in spec["systems"]: + positions = routing[system["id"]] + rows = households.iloc[positions] + support = supports[system["id"]] + assigned_system = rows[spec["outputs"]["system"]] + _require( + assigned_system.notna().all() and assigned_system.eq(system["id"]).all(), + "assigned system disagrees with routing", + ) + locations = pd.Index(support.arrays["area"]).get_indexer( + rows[spec["outputs"]["area"]] + ) + _require(np.all(locations >= 0), "missing atomic mapping") + for layer in system["layers"]: + result.iloc[positions, result.columns.get_loc(layer["output"])] = ( + support.arrays[layer["input"]][locations] + ) + return result + + +def validate_geography( + households: pd.DataFrame, spec: Mapping, supports: Mapping[str, AtomicSupport] +) -> dict: + """Recheck constraints and functional mappings on a full or pruned population. + + Draws are not repeated. Initial support clones receive their own assignment + after expansion; subsequent views retain it. Structural lineage verification + remains the executor's responsibility. This gate is not a population-quality + certificate. + """ + spec = _bind(spec, supports) + expected = derive_geography(households, spec, supports) + for name in expected: + _require( + households[name].astype("string").equals(expected[name]), + "derived geography differs from atomic mapping", + ) + routing = _route(households, spec) + for system in spec["systems"]: + support = supports[system["id"]] + rows = households.iloc[routing[system["id"]]] + positions = pd.Index(support.arrays["area"]).get_indexer( + rows[spec["outputs"]["area"]] + ) + for (_, row), position in zip(rows.iterrows(), positions, strict=True): + _require( + all( + support.arrays[c][position] == v + for c, v in _constraints(row, system) + ), + "assigned area violates observed geography", + ) + original = row[system["observed_area"]] if system["observed_area"] else None + observed = original is not None and not pd.isna(original) + _require( + row[spec["outputs"]["basis"]] + == ("observed" if observed else "assigned"), + "assignment basis differs", + ) + if observed: + _require( + row[spec["outputs"]["area"]] == original, + "observed atomic area changed", + ) + return { + "outcome": "pass", + "scope": "atomic_geography_mapping_integrity", + "households": len(households), + "support_sha256": {k: v.sha256 for k, v in sorted(supports.items())}, + "definition_sha256": hashlib.sha256(canonical_json(spec)).hexdigest(), + } diff --git a/packages/microcosm-build/src/microcosm/build/cd_benchmark/__init__.py b/packages/microcosm-build/src/microcosm/build/cd_benchmark/__init__.py new file mode 100644 index 000000000..2812fd3a5 --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/cd_benchmark/__init__.py @@ -0,0 +1,74 @@ +"""The US congressional-district published-benchmark contract, as executable rules. + +This package implements protocol ``us-cd-published-benchmark-v4`` (SHA-256 +``f7574a2a00734ce64ceb9c101ebc52469a176d37ee32b02c140d3ea2e6c568ea``) under an +approval whose scope is **invented-fixture implementation only**. Candidate +scoring and production ledger registration are explicitly not approved, and +nothing here performs either: no candidate, population, ACS or Census artifact +is read, no ledger file is created or appended, and no release, promotion or +publication decision is produced. + +The approved document itself ships as a packaged resource beside +:mod:`~microcosm.build.cd_benchmark.protocol` and is hash-verified on load. Every +threshold, vocabulary and payload roster is compiled out of those bytes rather +than retyped, so the code cannot drift from the contract without a test failing. +Paths named inside the document are locators recording what was inspected; this +package never opens them. + +Import cost: the modules below are pure and depend only on the standard library +and this package. The two source adapters the contract pins are reached through +:mod:`~microcosm.build.cd_benchmark.universe`, which defers those imports to call +time, so importing this package does not load ``microcosm.build.us_runtime`` and +does not load PolicyEngine. + +Layout: + +* :mod:`~microcosm.build.cd_benchmark.canonical` — canonical JSON and digests. +* :mod:`~microcosm.build.cd_benchmark.reasons` — the closed vocabularies. +* :mod:`~microcosm.build.cd_benchmark.protocol` — compile the approved document. +* :mod:`~microcosm.build.cd_benchmark.metrics` — point and categorical checks. +* :mod:`~microcosm.build.cd_benchmark.reference` — reference validity and precision. +* :mod:`~microcosm.build.cd_benchmark.support` — origin support and concentration. +* :mod:`~microcosm.build.cd_benchmark.origin` — typed original-source origin keys. +* :mod:`~microcosm.build.cd_benchmark.approval` — injected approval and registration. +* :mod:`~microcosm.build.cd_benchmark.ledger` — in-memory ledger records and states. +* :mod:`~microcosm.build.cd_benchmark.binding` — candidate binding slots. +* :mod:`~microcosm.build.cd_benchmark.universe` — the pinned source adapters. +* :mod:`~microcosm.build.cd_benchmark.evaluation` — the six axes and one reduction. +""" + +from __future__ import annotations + +from microcosm.build.cd_benchmark.protocol import ( + APPROVED_PROTOCOL_ID, + APPROVED_PROTOCOL_SHA256, + CompiledProtocol, + ProtocolError, + canonical_protocol, + compile_protocol, +) +from microcosm.build.cd_benchmark.reasons import ( + AXES, + BenchmarkVerdict, + ExposureStatus, + MappingStatus, + PrecisionStatus, + Reason, + SupportVerdict, +) + +__all__ = [ + "APPROVED_PROTOCOL_ID", + "APPROVED_PROTOCOL_SHA256", + "AXES", + "BenchmarkVerdict", + "CompiledProtocol", + "ExposureStatus", + "MappingStatus", + "PrecisionStatus", + "ProtocolError", + "Reason", + "SupportVerdict", + "canonical_protocol", + "compile_protocol", +] diff --git a/packages/microcosm-build/src/microcosm/build/cd_benchmark/canonical.py b/packages/microcosm-build/src/microcosm/build/cd_benchmark/canonical.py new file mode 100644 index 000000000..15108b494 --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/cd_benchmark/canonical.py @@ -0,0 +1,115 @@ +"""Canonical JSON bytes and digests, exactly as the protocol defines them. + +``/identity/encoding`` and ``/exposure_ledger/format`` require UTF-8 canonical +JSON with sorted keys, compact separators, no BOM, no nonfinite values and no +trailing LF inside the hashed bytes, and they additionally reject duplicate +object keys. The shared aggregate-JSON reader in +:mod:`microcosm.build.candidate_quality` enforces the first set but accepts +duplicate keys (``json.loads`` keeps the last one), so this module supplies the +stricter loader the contract names rather than relaxing the contract to match an +existing reader. The protocol also fixes ``ensure_ascii=false``, which that +reader leaves at the escaping default; for ASCII payloads the two encoders agree +byte for byte apart from its trailing newline, and a contract test pins that. +""" + +from __future__ import annotations + +import hashlib +import json +import math +import re +from typing import Any + +__all__ = [ + "CanonicalJsonError", + "SHA256_HEX", + "canonical_bytes", + "digest", + "is_sha256_hex", + "strict_load", +] + +SHA256_HEX = re.compile(r"[0-9a-f]{64}\Z") + + +class CanonicalJsonError(ValueError): + """Bytes are not the canonical JSON the protocol requires.""" + + +def is_sha256_hex(value: object) -> bool: + """Return whether ``value`` is a lowercase 64-character hex digest.""" + return isinstance(value, str) and SHA256_HEX.fullmatch(value) is not None + + +def _pairs(items: list[tuple[str, Any]]) -> dict[str, Any]: + result: dict[str, Any] = {} + for key, value in items: + if key in result: + raise CanonicalJsonError(f"Duplicate JSON key {key!r}.") + result[key] = value + return result + + +def _nonfinite(token: str) -> None: + raise CanonicalJsonError(f"Nonfinite JSON value {token}.") + + +def strict_load(data: bytes) -> Any: + """Parse canonical JSON bytes, refusing BOM, duplicate keys and nonfinite. + + Raises: + CanonicalJsonError: On any of those, or on invalid UTF-8/JSON. + """ + if not isinstance(data, bytes): + raise CanonicalJsonError("Canonical JSON input must be bytes.") + if data.startswith(b"\xef\xbb\xbf"): + raise CanonicalJsonError("Canonical JSON must not begin with a BOM.") + try: + text = data.decode("utf-8") + value = json.loads(text, object_pairs_hook=_pairs, parse_constant=_nonfinite) + except CanonicalJsonError: + raise + except (UnicodeError, ValueError, RecursionError) as error: + raise CanonicalJsonError(f"Invalid canonical JSON: {error}") from None + _assert_finite(value) + return value + + +def _assert_finite(value: object) -> None: + if isinstance(value, bool): + return + if isinstance(value, float | int): + try: + finite = math.isfinite(value) + except OverflowError: + finite = False + if not finite: + raise CanonicalJsonError("JSON numbers must be finite.") + elif isinstance(value, dict): + for child in value.values(): + _assert_finite(child) + elif isinstance(value, list): + for child in value: + _assert_finite(child) + + +def canonical_bytes(value: object) -> bytes: + """Serialize to the protocol's canonical UTF-8 JSON, with no trailing LF.""" + try: + text = json.dumps( + value, + sort_keys=True, + separators=(",", ":"), + ensure_ascii=False, + allow_nan=False, + ) + except (TypeError, ValueError, RecursionError) as error: + raise CanonicalJsonError(f"Value is not canonical JSON: {error}") from None + return text.encode("utf-8") + + +def digest(data: bytes) -> str: + """Return the lowercase hex SHA-256 of exact bytes.""" + if not isinstance(data, bytes): + raise CanonicalJsonError("Digest input must be bytes.") + return hashlib.sha256(data).hexdigest() diff --git a/packages/microcosm-build/src/microcosm/build/cd_benchmark/origin.py b/packages/microcosm-build/src/microcosm/build/cd_benchmark/origin.py new file mode 100644 index 000000000..2912a5e32 --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/cd_benchmark/origin.py @@ -0,0 +1,186 @@ +"""Typed original-source household origin keys, exactly as ``v4`` encodes them. + +``/source_household_origin/key_encoding`` fixes the encoding: +``SHA256(UTF8(domain) + one NUL byte + canonical payload bytes)``, lowercase +hex, where the payload is a canonical UTF-8 JSON array of four ordered typed +pairs. Each pair is a two-element array whose first element is the literal type +name. Integers are JSON integers, never booleans, and raw strings receive no +normalization, trimming or coercion. + +The two arms differ in their third and fourth components, and this module names +them per arm rather than generically, so a survey vintage cannot be filed as an +income cohort year: + +* ACS — ``arm=ACS_PUMS``, authenticated original household *archive* member + identity, **survey vintage**, and the **raw ``SERIALNO`` string** with its + leading and lexical characters preserved. +* ASEC — ``arm=ASEC``, authenticated original household member identity, + **income cohort year**, and the **native integer ``H_SEQ``**. + +Two limits carried from the document. A generated ``source_household_id`` or a +normalized ``household_id`` is positional and cannot replace this identity +(``/source_household_origin/ACS/not_equivalent``). Cohort or vintage +distinguishes statistical source records; it does not prove different real +households across years, and an origin key does not establish independent +households across surveys or independent imputation donors. + +Physical staging paths, recompressed archive hashes and clone or cache +filenames are not identities: the authenticated source manifest supplies +``canonical_member_id`` and the original member bytes hash, and aliases must +resolve to that same member before hashing. Typed components stay private in +authenticated lineage; public diagnostics use counts and digests only. +""" + +from __future__ import annotations + +import hashlib +from dataclasses import dataclass +from enum import StrEnum + +from microcosm.build.cd_benchmark.canonical import canonical_bytes, is_sha256_hex + +__all__ = [ + "AcsHouseholdOrigin", + "AsecHouseholdOrigin", + "HouseholdOrigin", + "ORIGIN_DOMAIN", + "OriginKeyError", + "SourceArm", + "SourceMember", + "origin_key", + "origin_payload", + "origin_payload_bytes", +] + +#: ``/source_household_origin/key_encoding/domain``. +ORIGIN_DOMAIN = "microcosm.statistical_source_household.v1" + + +class OriginKeyError(ValueError): + """The typed origin components are not what the encoding accepts.""" + + +class SourceArm(StrEnum): + """The two arms with a registered origin definition. + + ``/source_household_origin/other_arms``: any other arm needs its own + registered, authenticated source-specific definition; there is deliberately + no default to a synthetic row id here. + """ + + ACS_PUMS = "ACS_PUMS" + ASEC = "ASEC" + + +def _identifier(value: object, label: str) -> str: + if not isinstance(value, str) or not value: + raise OriginKeyError(f"{label} must be a nonempty string.") + return value + + +def _integer(value: object, label: str) -> int: + if isinstance(value, bool) or not isinstance(value, int): + raise OriginKeyError(f"{label} must be a JSON integer, not a boolean.") + return value + + +@dataclass(frozen=True) +class SourceMember: + """Canonical original member identity from the authenticated manifest.""" + + canonical_member_id: str + member_sha256: str + + def __post_init__(self) -> None: + _identifier(self.canonical_member_id, "canonical_member_id") + if not is_sha256_hex(self.member_sha256): + raise OriginKeyError( + "member_sha256 must be a lowercase 64-character hex digest." + ) + + def as_payload(self) -> dict[str, str]: + """The member pair's object value, with lexicographically sorted keys.""" + return { + "canonical_member_id": self.canonical_member_id, + "member_sha256": self.member_sha256, + } + + +@dataclass(frozen=True) +class AcsHouseholdOrigin: + """ACS PUMS origin: archive member identity, survey vintage, raw SERIALNO.""" + + member: SourceMember + survey_vintage: int + raw_serialno: str + + arm = SourceArm.ACS_PUMS + + def __post_init__(self) -> None: + if not isinstance(self.member, SourceMember): + raise OriginKeyError("member must be an authenticated SourceMember.") + _integer(self.survey_vintage, "survey_vintage") + # No normalization, trimming or coercion: an integer SERIALNO would be a + # coercion of the published string, and leading characters are meaning. + if not isinstance(self.raw_serialno, str) or not self.raw_serialno: + raise OriginKeyError("raw_serialno must be the nonempty published string.") + + +@dataclass(frozen=True) +class AsecHouseholdOrigin: + """ASEC origin: member identity, income cohort year, native integer H_SEQ.""" + + member: SourceMember + income_cohort_year: int + native_h_seq: int + + arm = SourceArm.ASEC + + def __post_init__(self) -> None: + if not isinstance(self.member, SourceMember): + raise OriginKeyError("member must be an authenticated SourceMember.") + _integer(self.income_cohort_year, "income_cohort_year") + # The shipped ASEC adapter's NATIVE_KEY invariant is H_SEQ > 0; a zero or + # negative sequence is not a native household key. + if _integer(self.native_h_seq, "native_h_seq") <= 0: + raise OriginKeyError("native_h_seq must be a positive native key.") + + +HouseholdOrigin = AcsHouseholdOrigin | AsecHouseholdOrigin + + +def origin_payload(origin: HouseholdOrigin) -> list[list[object]]: + """Return the four ordered typed pairs for ``origin``. + + Raises: + OriginKeyError: If ``origin`` is not a registered arm's origin. + """ + if isinstance(origin, AcsHouseholdOrigin): + return [ + ["string", str(SourceArm.ACS_PUMS)], + ["member", origin.member.as_payload()], + ["integer", origin.survey_vintage], + ["string", origin.raw_serialno], + ] + if isinstance(origin, AsecHouseholdOrigin): + return [ + ["string", str(SourceArm.ASEC)], + ["member", origin.member.as_payload()], + ["integer", origin.income_cohort_year], + ["integer", origin.native_h_seq], + ] + raise OriginKeyError( + "Only registered ACS and ASEC origins have a defined encoding; another " + "arm requires its own registered source-specific origin definition." + ) + + +def origin_payload_bytes(origin: HouseholdOrigin) -> bytes: + """Canonical payload bytes: sorted object keys, compact, no BOM or LF.""" + return canonical_bytes(origin_payload(origin)) + + +def origin_key(origin: HouseholdOrigin) -> str: + """Return the lowercase hex origin key for one original source household.""" + payload = origin_payload_bytes(origin) + return hashlib.sha256(ORIGIN_DOMAIN.encode("utf-8") + b"\x00" + payload).hexdigest() diff --git a/packages/microcosm-build/src/microcosm/build/cd_benchmark/protocol.py b/packages/microcosm-build/src/microcosm/build/cd_benchmark/protocol.py new file mode 100644 index 000000000..18c24024c --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/cd_benchmark/protocol.py @@ -0,0 +1,871 @@ +"""Compile the approved canonical CD published-benchmark protocol document. + +The exact approved bytes ship as a packaged resource beside this module. Nothing +here reads a path named inside the document: ``/identity/legacy_design_parents``, +``/reference_authority`` and the code locators are *locators* recording what was +inspected when the contract was written, never inputs this module opens. + +Compilation is total and closed. Every threshold is carried as an exact +:class:`~decimal.Decimal` parsed from the document's decimal string, never as a +float; every vocabulary is compared against the document rather than assumed; +unknown block shapes refuse compilation. :func:`compile_protocol` will compile +any candidate document so that a successor can be inspected, but only bytes +whose SHA-256 equals :data:`APPROVED_PROTOCOL_SHA256` compile to a protocol with +``canonical`` set, and approval is resolved from that digest through an injected +authority (:mod:`microcosm.build.cd_benchmark.approval`) — never from a field +inside the document and never from a caller's label. +""" + +from __future__ import annotations + +import re +from collections.abc import Mapping +from dataclasses import dataclass +from decimal import Decimal, InvalidOperation +from enum import StrEnum +from functools import lru_cache +from importlib import resources +from types import MappingProxyType +from typing import Any + +from microcosm.build.cd_benchmark.canonical import ( + CanonicalJsonError, + canonical_bytes, + digest, + is_sha256_hex, + strict_load, +) +from microcosm.build.cd_benchmark.reasons import ExposureStatus, Reason + +__all__ = [ + "APPROVED_PROTOCOL_BYTES", + "APPROVED_PROTOCOL_ID", + "APPROVED_PROTOCOL_SHA256", + "CompiledProtocol", + "DistributionBlock", + "PartitionRules", + "PointBlock", + "PrecisionKind", + "ProtocolError", + "ReferenceBin", + "RESERVED_FAMILIES", + "ScopeRules", + "SupportBounds", + "assert_vocabulary_closed", + "assign_bin", + "canonical_protocol", + "compile_protocol", + "load_canonical_bytes", +] + +#: Digest and byte count of the approved canonical document, verified against +#: the review directory's plan, its independent response and the peer review. +APPROVED_PROTOCOL_SHA256 = ( + "f7574a2a00734ce64ceb9c101ebc52469a176d37ee32b02c140d3ea2e6c568ea" +) +APPROVED_PROTOCOL_BYTES = 86626 +APPROVED_PROTOCOL_ID = "us-cd-published-benchmark-v4" +_RESOURCE = "us-cd-published-benchmark-v4.json" + +#: ``/exposure_ledger/reservation``: these table families and every derivative +#: stay out of fitting, tuning and selection. +RESERVED_FAMILIES = frozenset({"B19001", "B25003"}) + +_FAMILY_ROOT = re.compile(r"([A-Z][0-9]{4,5})(?:_|\Z)") + + +class ProtocolError(ValueError): + """The document is not a compilable CD published-benchmark contract.""" + + +class PrecisionKind(StrEnum): + """Which ``/reference_status/share_precision`` rule a distribution uses.""" + + PUBLISHED_PERCENTAGE_MOE = "published_percentage_moe" + SUBSET_SHARE_APPROXIMATION = "subset_share_approximation" + + +def _object(value: object, label: str) -> dict[str, Any]: + if not isinstance(value, dict): + raise ProtocolError(f"{label} must be a JSON object.") + return value + + +def _text(value: object, label: str) -> str: + if not isinstance(value, str) or not value: + raise ProtocolError(f"{label} must be a nonempty string.") + return value + + +def _flag(value: object, label: str) -> bool: + if not isinstance(value, bool): + raise ProtocolError(f"{label} must be a JSON boolean.") + return value + + +def _decimal(value: object, label: str) -> Decimal: + """Parse an exact decimal-string threshold; floats are refused.""" + if not isinstance(value, str): + raise ProtocolError(f"{label} must be an exact decimal string.") + try: + parsed = Decimal(value) + except InvalidOperation: + raise ProtocolError(f"{label} is not a decimal: {value!r}.") from None + if not parsed.is_finite(): + raise ProtocolError(f"{label} must be finite.") + return parsed + + +def _integer(value: object, label: str) -> int: + if isinstance(value, bool) or not isinstance(value, int): + raise ProtocolError(f"{label} must be a JSON integer.") + return value + + +def _list(container: Mapping[str, Any], key: str, label: str) -> list[Any]: + """Read a required roster, refusing an absent key and a non-list value. + + Compilation is total: an absent section leaks no ``KeyError`` and a string + in a roster's place is never iterated per character into field names. + """ + if key not in container: + raise ProtocolError(f"{label} is required and absent.") + value = container[key] + if not isinstance(value, list): + raise ProtocolError(f"{label} must be a JSON array.") + return value + + +def _bound(value: object, label: str) -> Decimal | None: + """A bin edge: an exact integer, or ``None`` for an open side.""" + if value is None: + return None + return Decimal(_integer(value, label)) + + +@dataclass(frozen=True) +class ReferenceBin: + """One published bin, or the protocol-constructed ``unclassified`` residual. + + ``lower_inclusive``/``upper_exclusive`` follow the document's boundary rule: + lower inclusive, upper exclusive, ``None`` meaning minus/plus infinity. The + residual bin carries no publisher variable and a reference share of exactly + zero; it is a protocol construct, not a publisher cell. + """ + + id: str + publisher_variable: str | None + publisher_label: str | None + lower_inclusive: Decimal | None + upper_exclusive: Decimal | None + reference_share: Decimal | None + unclassified: bool + authority: str | None + + +@dataclass(frozen=True) +class PointBlock: + """A mandatory scalar block compared by relative discrepancy.""" + + id: str + family: str + dataset: str + unit: str + universe: str + mandatory: bool + rule: str + variables: tuple[str, ...] + point_relative_bound: Decimal + reference_relative_moe_ceiling: Decimal + controlled_reference: str | None + exposure_families: frozenset[str] + count_once: bool + duplicate_reference_aliases: tuple[str, ...] + + @property + def reserved(self) -> bool: + """Whether any exposure family is a reserved holdout family.""" + return bool(self.exposure_families & RESERVED_FAMILIES) + + +@dataclass(frozen=True) +class DistributionBlock: + """A mandatory categorical block compared by per-bin share and total variation. + + ``independent_votes`` is ``False`` where the document says so (tenure's owner + and renter bins) and ``None`` where it says nothing. Undeclared is not a + licence to treat bins as independent votes: ``/claim`` withholds that + licence for overlapping cells, and ``/status_contract/no_rescue`` forbids + rescuing a block with them. + """ + + id: str + family: str + dataset: str + unit: str + mandatory: bool + denominator_variable: str + bins: tuple[ReferenceBin, ...] + max_each_bin_discrepancy_pp: Decimal + max_total_variation: Decimal + reference_moe_ceiling_pp: Decimal + precision_kind: PrecisionKind + precision_variables: tuple[str, ...] + exposure_families: frozenset[str] + independent_votes: bool | None + open_upper: bool + boundary_rule: str | None + additional_bounds: MappingProxyType + complement_only_when: str | None + + @property + def reserved(self) -> bool: + """Whether any exposure family is a reserved holdout family.""" + return bool(self.exposure_families & RESERVED_FAMILIES) + + @property + def unclassified_bin(self) -> ReferenceBin: + """The protocol-constructed residual bin.""" + return next(item for item in self.bins if item.unclassified) + + @property + def known_bins(self) -> tuple[ReferenceBin, ...]: + """Published bins, excluding the residual.""" + return tuple(item for item in self.bins if not item.unclassified) + + +@dataclass(frozen=True) +class SupportBounds: + """``/support`` floors and ceilings, as exact decimals and integers.""" + + minimum_positive_source_household_origins: int + minimum_positive_origins_per_positive_reference_bin: int + minimum_origin_cluster_concentration_ess: Decimal + maximum_origin_denominator_share: Decimal + maximum_origin_positive_bin_share: Decimal + nonnegative_finite_weights_required: bool + required_for_every_district_and_relevant_denominator: bool + + +@dataclass(frozen=True) +class PartitionRules: + """``/categorical_partition`` closure and residual-mass rules.""" + + maximum_unclassified_fraction: Decimal + comparison: str + renormalize_known_bins: bool + unknown_bin_id: str + applies_to: frozenset[str] + + +@dataclass(frozen=True) +class ScopeRules: + """``/scope`` — the predeclared geographic and product scope.""" + + country: str + area: str + congress: int + district_count: int + acs_year: int + acs_product: str + puerto_rico: str + allowed_missing_mandatory_block_fraction: Decimal + allowed_missing_mandatory_district_fraction: Decimal + dc_canonical_id: str + dc_published_id: str + published_and_canonical_geography_ids_retained: bool + + +@dataclass(frozen=True) +class CompiledProtocol: + """A compiled contract document plus the digest of the bytes it came from.""" + + sha256: str + size_bytes: int + protocol_id: str + schema: str + status: str + parent_sha256: str + hash_scope: str + point_blocks: tuple[PointBlock, ...] + distribution_blocks: tuple[DistributionBlock, ...] + partition: PartitionRules + support: SupportBounds + scope: ScopeRules + reason_codes: frozenset[str] + exposure_status_values: frozenset[str] + independent_axes: tuple[str, ...] + scope_reduction_order: tuple[str, ...] + scoring_permitted_in_document: bool + approval_installed_in_document: bool + reference_inventory_sha256: str + block_definition_hashes: Mapping[str, str] + pinned_code_sha256: Mapping[str, str] + ledger_event_types: frozenset[str] + ledger_record_fields: frozenset[str] + ledger_snapshot_fields: tuple[str, ...] + attempt_payload_fields: tuple[tuple[str, tuple[str, ...] | None], ...] + completion_payload_fields: tuple[tuple[str, tuple[str, ...] | None], ...] + registration_payload_fields: tuple[str, ...] + quality_required_fields: tuple[str, ...] + amendment_change_fields: tuple[str, ...] + required_binding_slots: tuple[str, ...] + + @property + def canonical(self) -> bool: + """Whether these bytes are the exact approved canonical document.""" + return ( + self.sha256 == APPROVED_PROTOCOL_SHA256 + and self.size_bytes == APPROVED_PROTOCOL_BYTES + ) + + @property + def blocks(self) -> tuple[PointBlock | DistributionBlock, ...]: + """Every mandatory block, point blocks first, in document order.""" + return (*self.point_blocks, *self.distribution_blocks) + + def block(self, block_id: str) -> PointBlock | DistributionBlock: + """Return the block with ``block_id``. + + Raises: + ProtocolError: If no block carries that id. + """ + for item in self.blocks: + if item.id == block_id: + return item + raise ProtocolError(f"Unknown block id {block_id!r}.") + + +def _families(block: dict[str, Any]) -> frozenset[str]: + """Reference table-family roots whose exposure this block consumes.""" + declared = block.get("exposure_families") + if declared is not None: + if not isinstance(declared, list) or not declared: + raise ProtocolError("exposure_families must be a nonempty list.") + return frozenset(_text(item, "exposure family") for item in declared) + match = _FAMILY_ROOT.match(_text(block.get("family"), "block family")) + if match is None: + raise ProtocolError( + f"Block family {block.get('family')!r} names no published table root " + "and declares no exposure_families." + ) + return frozenset({match.group(1)}) + + +def _bin(item: object) -> ReferenceBin: + data = _object(item, "distribution bin") + identifier = _text(data.get("id"), "bin id") + unclassified = "authority" in data + if unclassified: + if data.get("publisher_variable") is not None: + raise ProtocolError("The residual bin cannot name a publisher variable.") + share = _decimal(data.get("reference_share"), "residual reference_share") + if share != 0: + raise ProtocolError("The residual bin's reference share must be zero.") + return ReferenceBin( + id=identifier, + publisher_variable=None, + publisher_label=None, + lower_inclusive=None, + upper_exclusive=None, + reference_share=share, + unclassified=True, + authority=_text(data.get("authority"), "residual bin authority"), + ) + variable = ( + data["publisher_variable"] if "publisher_variable" in data else identifier + ) + return ReferenceBin( + id=identifier, + publisher_variable=_text(variable, "bin publisher variable"), + publisher_label=data.get("publisher_label"), + lower_inclusive=_bound(data.get("lower_inclusive"), "bin lower_inclusive"), + upper_exclusive=_bound(data.get("upper_exclusive"), "bin upper_exclusive"), + reference_share=None, + unclassified=False, + authority=None, + ) + + +def _point_block(block: dict[str, Any]) -> PointBlock: + ceilings = [ + key + for key in ( + "ordinary_reference_relative_moe_ceiling", + "reference_relative_moe_ceiling", + ) + if key in block + ] + if len(ceilings) != 1: + raise ProtocolError( + "A point block declares exactly one relative reference MOE ceiling." + ) + variables = block.get("variables") + if not isinstance(variables, list) or not variables: + raise ProtocolError("A point block declares its reference variables.") + aliases = block.get("duplicate_reference_aliases", []) + if not isinstance(aliases, list): + raise ProtocolError("duplicate_reference_aliases must be a list.") + return PointBlock( + id=_text(block.get("id"), "block id"), + family=_text(block.get("family"), "block family"), + dataset=_text(block.get("dataset"), "block dataset"), + unit=_text(block.get("unit"), "block unit"), + universe=_text(block.get("universe"), "block universe"), + mandatory=_flag(block.get("mandatory"), "block mandatory"), + rule=_text(block.get("rule"), "block rule"), + variables=tuple(_text(item, "block variable") for item in variables), + point_relative_bound=_decimal( + block.get("point_relative_bound"), "point_relative_bound" + ), + reference_relative_moe_ceiling=_decimal(block[ceilings[0]], ceilings[0]), + controlled_reference=block.get("controlled_reference"), + exposure_families=_families(block), + count_once=bool(block.get("count_once", False)), + duplicate_reference_aliases=tuple( + _text(item, "duplicate reference alias") for item in aliases + ), + ) + + +_PRECISION_CEILINGS = { + "reference_published_moe_ceiling_pp": PrecisionKind.PUBLISHED_PERCENTAGE_MOE, + "reference_approximate_moe_ceiling_pp": PrecisionKind.SUBSET_SHARE_APPROXIMATION, +} + + +def _distribution_block(block: dict[str, Any]) -> DistributionBlock: + present = [key for key in _PRECISION_CEILINGS if key in block] + if len(present) != 1: + raise ProtocolError( + "A distribution block declares exactly one reference precision ceiling." + ) + bins = tuple(_bin(item) for item in block.get("bins", [])) + if len(bins) < 2 or sum(item.unclassified for item in bins) != 1: + raise ProtocolError("A distribution declares its bins plus one residual bin.") + if not bins[-1].unclassified: + raise ProtocolError("The residual bin is declared last.") + identifiers = [item.id for item in bins] + if len(set(identifiers)) != len(identifiers): + raise ProtocolError("Duplicate bin ids.") + extra = { + key: _decimal(value, key) + for key, value in block.items() + if key.startswith("max_") + and key.endswith("_pp") + and key not in {"max_each_bin_discrepancy_pp"} + } + if set(extra) - {"max_renter_discrepancy_pp"}: + raise ProtocolError("Unsupported additional distribution bound.") + if "max_renter_discrepancy_pp" in extra and "renter" not in identifiers: + raise ProtocolError("The renter bound requires a renter bin.") + precision_variables = block.get("precision_variables", []) + if not isinstance(precision_variables, list): + raise ProtocolError("precision_variables must be a JSON array.") + return DistributionBlock( + id=_text(block.get("id"), "block id"), + family=_text(block.get("family"), "block family"), + dataset=_text(block.get("dataset"), "block dataset"), + unit=_text(block.get("unit"), "block unit"), + mandatory=_flag(block.get("mandatory"), "block mandatory"), + denominator_variable=_text(block.get("denominator"), "block denominator"), + bins=bins, + max_each_bin_discrepancy_pp=_decimal( + block.get("max_each_bin_discrepancy_pp"), "max_each_bin_discrepancy_pp" + ), + max_total_variation=_decimal( + block.get("max_total_variation"), "max_total_variation" + ), + reference_moe_ceiling_pp=_decimal(block[present[0]], present[0]), + precision_kind=_PRECISION_CEILINGS[present[0]], + precision_variables=tuple( + _text(item, "precision variable") for item in precision_variables + ), + exposure_families=_families(block), + independent_votes=block.get("owner_and_renter_independent_votes"), + open_upper=bool(block.get("open_upper_age", False)), + boundary_rule=block.get("boundary_rule"), + additional_bounds=MappingProxyType(extra), + complement_only_when=block.get("owner_is_renter_complement_only_when"), + ) + + +def compile_protocol(raw: bytes) -> CompiledProtocol: + """Compile canonical protocol bytes into typed, exact rules. + + Args: + raw: The exact document bytes. Their SHA-256 is the protocol identity; + the document carries no self-hash member (``/identity/hash_scope``). + + Raises: + ProtocolError: On any missing, mistyped or unknown-shaped section. + """ + try: + document = _object(strict_load(raw), "protocol document") + except CanonicalJsonError as error: + raise ProtocolError(str(error)) from None + identity = _object(document.get("identity"), "/identity") + if identity.get("algorithm") != "sha256": + raise ProtocolError("Only sha256 protocol identity is supported.") + parent = _text(identity.get("protocol_parent_sha256"), "protocol_parent_sha256") + if not is_sha256_hex(parent): + raise ProtocolError("protocol_parent_sha256 must be a lowercase digest.") + blocks = document.get("blocks") + if not isinstance(blocks, list) or not blocks: + raise ProtocolError("/blocks must be a nonempty list.") + point, distribution = [], [] + for item in blocks: + block = _object(item, "block") + if "bins" in block: + distribution.append(_distribution_block(block)) + else: + point.append(_point_block(block)) + identifiers = [item.id for item in (*point, *distribution)] + if len(set(identifiers)) != len(identifiers): + raise ProtocolError("Duplicate block ids.") + + partition_doc = _object( + document.get("categorical_partition"), "/categorical_partition" + ) + unknown_bin = _object(partition_doc.get("unknown_bin"), "unknown_bin") + partition = PartitionRules( + maximum_unclassified_fraction=_decimal( + partition_doc.get("maximum_unclassified_fraction_of_known_denominator"), + "maximum_unclassified_fraction_of_known_denominator", + ), + comparison=_text( + partition_doc.get("maximum_unclassified_fraction_comparison"), + "maximum_unclassified_fraction_comparison", + ), + renormalize_known_bins=_flag( + partition_doc.get("renormalize_known_bins"), "renormalize_known_bins" + ), + unknown_bin_id=_text(unknown_bin.get("id"), "unknown bin id"), + applies_to=frozenset( + _text(item, "partition applies_to") + for item in _list( + partition_doc, "applies_to", "/categorical_partition/applies_to" + ) + ), + ) + support_doc = _object(document.get("support"), "/support") + support = SupportBounds( + minimum_positive_source_household_origins=_integer( + support_doc.get("minimum_positive_source_household_origins"), + "minimum_positive_source_household_origins", + ), + minimum_positive_origins_per_positive_reference_bin=_integer( + support_doc.get("minimum_positive_origins_per_positive_reference_bin"), + "minimum_positive_origins_per_positive_reference_bin", + ), + minimum_origin_cluster_concentration_ess=_decimal( + support_doc.get("minimum_origin_cluster_concentration_ESS"), + "minimum_origin_cluster_concentration_ESS", + ), + maximum_origin_denominator_share=_decimal( + support_doc.get("maximum_origin_denominator_share"), + "maximum_origin_denominator_share", + ), + maximum_origin_positive_bin_share=_decimal( + support_doc.get("maximum_origin_positive_bin_share"), + "maximum_origin_positive_bin_share", + ), + nonnegative_finite_weights_required=_flag( + support_doc.get("nonnegative_finite_weights_required"), + "nonnegative_finite_weights_required", + ), + required_for_every_district_and_relevant_denominator=_flag( + support_doc.get("required_for_every_district_and_relevant_denominator"), + "required_for_every_district_and_relevant_denominator", + ), + ) + scope_doc = _object(document.get("scope"), "/scope") + alias = _object(scope_doc.get("DC_identity_alias"), "DC_identity_alias") + scope = ScopeRules( + country=_text(scope_doc.get("country"), "scope country"), + area=_text(scope_doc.get("area"), "scope area"), + congress=_integer(scope_doc.get("congress"), "scope congress"), + district_count=_integer(scope_doc.get("district_count"), "district_count"), + acs_year=_integer(scope_doc.get("acs_year"), "acs_year"), + acs_product=_text(scope_doc.get("acs_product"), "acs_product"), + puerto_rico=_text(scope_doc.get("puerto_rico"), "puerto_rico"), + allowed_missing_mandatory_block_fraction=_decimal( + scope_doc.get("allowed_missing_mandatory_block_fraction"), + "allowed_missing_mandatory_block_fraction", + ), + allowed_missing_mandatory_district_fraction=_decimal( + scope_doc.get("allowed_missing_mandatory_district_fraction"), + "allowed_missing_mandatory_district_fraction", + ), + dc_canonical_id=_text(alias.get("canonical"), "DC canonical id"), + dc_published_id=_text(alias.get("published"), "DC published id"), + published_and_canonical_geography_ids_retained=_flag( + scope_doc.get("published_and_canonical_geography_ids_retained"), + "published_and_canonical_geography_ids_retained", + ), + ) + status_doc = _object(document.get("status_contract"), "/status_contract") + ledger_doc = _object(document.get("exposure_ledger"), "/exposure_ledger") + approval_doc = _object(document.get("approval"), "/approval") + return CompiledProtocol( + sha256=digest(raw), + size_bytes=len(raw), + protocol_id=_text(document.get("protocol_id"), "/protocol_id"), + schema=_text(document.get("schema"), "/schema"), + status=_text(document.get("status"), "/status"), + parent_sha256=parent, + hash_scope=_text(identity.get("hash_scope"), "/identity/hash_scope"), + point_blocks=tuple(point), + distribution_blocks=tuple(distribution), + partition=partition, + support=support, + scope=scope, + reason_codes=frozenset(_object(status_doc.get("reason_codes"), "reason_codes")), + exposure_status_values=frozenset( + _object(ledger_doc.get("exposure_status_values"), "exposure_status_values") + ), + independent_axes=tuple( + _text(item, "independent axis") + for item in _list( + status_doc, + "independent_axes_required", + "/status_contract/independent_axes_required", + ) + ), + scope_reduction_order=tuple( + _text(item, "scope reduction step") + for item in _list( + status_doc, + "scope_reduction_order", + "/status_contract/scope_reduction_order", + ) + ), + scoring_permitted_in_document=_flag( + approval_doc.get("scoring_permitted"), "/approval/scoring_permitted" + ), + approval_installed_in_document=_flag( + approval_doc.get("installed"), "/approval/installed" + ), + reference_inventory_sha256=_inventory_digest(document, scope_doc), + block_definition_hashes=MappingProxyType( + { + _text(_object(item, "block").get("id"), "block id"): digest( + canonical_bytes(item) + ) + for item in blocks + } + ), + pinned_code_sha256=MappingProxyType(_pinned_code(document)), + ledger_event_types=frozenset( + _object(ledger_doc.get("other_event_types"), "other_event_types") + ), + ledger_record_fields=frozenset( + _object(ledger_doc.get("record_fields"), "record_fields") + ), + ledger_snapshot_fields=tuple( + _text(item, "snapshot field") + for item in _list( + ledger_doc, "snapshot_fields", "/exposure_ledger/snapshot_fields" + ) + ), + attempt_payload_fields=_payload_fields( + _list( + ledger_doc, + "evaluation_attempt_payload", + "/exposure_ledger/evaluation_attempt_payload", + ) + ), + completion_payload_fields=_payload_fields( + _list( + ledger_doc, + "evaluation_completion_payload", + "/exposure_ledger/evaluation_completion_payload", + ) + ), + registration_payload_fields=tuple( + _text(item, "registration payload field") + for item in _list( + ledger_doc, + "protocol_registration_payload", + "/exposure_ledger/protocol_registration_payload", + ) + ), + quality_required_fields=tuple( + _text(item, "quality required field") + for item in _list( + ledger_doc, + "quality_required_fields", + "/exposure_ledger/quality_required_fields", + ) + ), + amendment_change_fields=tuple( + _text(item, "amendment change field") + for item in _list( + _object(document.get("protocol_amendments"), "/protocol_amendments"), + "required_change_fields", + "/protocol_amendments/required_change_fields", + ) + ), + required_binding_slots=tuple( + _text(item, "binding slot") + for item in _list( + _object( + document.get("candidate_binding_requirements"), + "/candidate_binding_requirements", + ), + "must_authenticate_before_scoring", + "/candidate_binding_requirements/must_authenticate_before_scoring", + ) + ), + ) + + +def _payload_fields( + declared: object, +) -> tuple[tuple[str, tuple[str, ...] | None], ...]: + """Split a declared payload roster into names and any pinned values. + + The document writes a constrained field as ``name=value`` and an + alternative as ``a_or_b``: ``attempt_state=started`` and + ``terminal_state=completed_or_failed``. Everything else is an unconstrained + required field name. + """ + if not isinstance(declared, list) or not declared: + raise ProtocolError("A ledger payload roster must be a nonempty list.") + fields: list[tuple[str, tuple[str, ...] | None]] = [] + for item in declared: + entry = _text(item, "ledger payload field") + name, separator, value = entry.partition("=") + if not separator: + fields.append((name, None)) + continue + fields.append((name, tuple(value.split("_or_")))) + names = [name for name, _ in fields] + if len(set(names)) != len(names): + raise ProtocolError("Duplicate ledger payload field names.") + return tuple(fields) + + +def _inventory_digest(document: dict[str, Any], scope_doc: dict[str, Any]) -> str: + """The reference inventory digest, required to agree in both places.""" + authority = _object(document.get("reference_authority"), "/reference_authority") + inventory = _object(authority.get("inventory"), "reference inventory") + required = _object(scope_doc.get("required_district_set"), "required_district_set") + primary = _text(inventory.get("sha256"), "reference inventory sha256") + secondary = _text(required.get("inventory_sha256"), "district inventory sha256") + if not is_sha256_hex(primary) or primary != secondary: + raise ProtocolError( + "The reference inventory digest must be a lowercase digest and must " + "agree between /reference_authority and /scope/required_district_set." + ) + return primary + + +def _pinned_code(document: dict[str, Any]) -> dict[str, str]: + """Source-file digests the contract inspected, keyed by file name. + + These are locators recording what was read when the contract was written. + Nothing in this package opens the paths beside them; the digests exist so a + binding can be checked against the adapter it was reviewed against, and a + changed adapter is a scope question rather than an automatic pin refresh. + """ + sections = ( + _object( + _object(document.get("reference_authority"), "/reference_authority").get( + "HU_adapter_clarification" + ), + "HU_adapter_clarification", + ), + _object( + _object(document.get("candidate_semantics"), "/candidate_semantics").get( + "definition_authority" + ), + "definition_authority", + ), + ) + pins: dict[str, str] = {} + for section in sections: + for item in section.get("code", []): + entry = _object(item, "pinned code entry") + name = _text(entry.get("path"), "pinned code path").rsplit("/", 1)[-1] + value = _text(entry.get("sha256"), "pinned code sha256") + if not is_sha256_hex(value): + raise ProtocolError(f"Pinned code digest for {name} is not a digest.") + if pins.setdefault(name, value) != value: + raise ProtocolError(f"Conflicting pinned code digests for {name}.") + if not pins: + raise ProtocolError("The document pins no inspected source code.") + return pins + + +def load_canonical_bytes() -> bytes: + """Read the packaged approved document and verify its digest and size. + + Raises: + ProtocolError: If the packaged bytes are not the approved document. + """ + raw = resources.files(__package__).joinpath(_RESOURCE).read_bytes() + if len(raw) != APPROVED_PROTOCOL_BYTES or digest(raw) != APPROVED_PROTOCOL_SHA256: + raise ProtocolError("Packaged protocol resource is not the approved document.") + return raw + + +@lru_cache(maxsize=1) +def canonical_protocol() -> CompiledProtocol: + """Return the compiled approved canonical protocol. + + Raises: + ProtocolError: If the packaged document is not the approved bytes, does + not identify itself as the approved protocol, or has drifted from + the closed reason/exposure vocabularies this package implements. + """ + compiled = compile_protocol(load_canonical_bytes()) + if not compiled.canonical or compiled.protocol_id != APPROVED_PROTOCOL_ID: + raise ProtocolError("Packaged protocol is not the approved canonical contract.") + assert_vocabulary_closed(compiled) + return compiled + + +def assert_vocabulary_closed(compiled: CompiledProtocol) -> None: + """Check this package's closed vocabularies equal the document's. + + Raises: + ProtocolError: On any code this package would emit that the document + does not define, or any document code this package cannot emit. + """ + for name, implemented, declared in ( + ("reason", {item.value for item in Reason}, set(compiled.reason_codes)), + ( + "exposure status", + {item.value for item in ExposureStatus}, + set(compiled.exposure_status_values), + ), + ): + if implemented != declared: + raise ProtocolError( + f"Closed {name} vocabulary differs from the document: " + f"only in code {sorted(implemented - declared)}, " + f"only in document {sorted(declared - implemented)}." + ) + + +def assign_bin(block: DistributionBlock, value: Decimal) -> str | None: + """Return the bin id for ``value`` under the declared boundary rule. + + Lower bounds are inclusive and upper bounds exclusive; a ``None`` lower or + upper bound is minus or plus infinity. No rounding is applied — a monetary + amount of ``9999.5`` stays below a ``10000`` cutpoint. Returns ``None`` when + no published bin covers the value, which the caller may treat as + unclassified mass only under known membership and an approved mapping. + """ + if not isinstance(value, Decimal) or not value.is_finite(): + raise ProtocolError("Bin assignment needs a finite Decimal value.") + for item in block.known_bins: + if item.lower_inclusive is None and item.upper_exclusive is None: + raise ProtocolError(f"Bin {item.id!r} declares no numeric boundary.") + if item.lower_inclusive is not None and value < item.lower_inclusive: + continue + if item.upper_exclusive is not None and value >= item.upper_exclusive: + continue + return item.id + return None diff --git a/packages/microcosm-build/src/microcosm/build/cd_benchmark/reasons.py b/packages/microcosm-build/src/microcosm/build/cd_benchmark/reasons.py new file mode 100644 index 000000000..855d938b4 --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/cd_benchmark/reasons.py @@ -0,0 +1,135 @@ +"""Closed reason, exposure and status vocabularies for the CD benchmark protocol. + +Every member here is checked against the canonical protocol document by +:func:`microcosm.build.cd_benchmark.protocol.assert_vocabulary_closed`, so the +code's vocabulary is verified equal to the approved document's rather than +merely resembling it. Per ``/status_contract/reason_encoding`` the primary +reason namespace is closed: unknown codes refuse validation, several +independently established reasons may coexist, and block/bin/metric identifiers +and raw source detail are separate bounded evidence, never a replacement code. +""" + +from __future__ import annotations + +from collections.abc import Iterable +from enum import StrEnum + +__all__ = [ + "AXES", + "BenchmarkVerdict", + "ExposureStatus", + "MappingStatus", + "PrecisionStatus", + "Reason", + "SupportVerdict", + "dedupe", +] + + +class Reason(StrEnum): + """The 21 primary reason codes at ``/status_contract/reason_codes``.""" + + CANDIDATE_IDENTITY_UNAVAILABLE = "candidate_identity_unavailable" + DENOMINATOR_UNKNOWN = "denominator_unknown" + EXPOSURE_HISTORY_UNKNOWN = "exposure_history_unknown" + GQ_WEIGHT_BRIDGE_MISSING = "gq_weight_bridge_missing" + IDENTITY_MISMATCH = "identity_mismatch" + INVALID_REDUCER_INPUT = "invalid_reducer_input" + LEDGER_BUSY = "ledger_busy" + LEDGER_INTEGRITY_UNKNOWN = "ledger_integrity_unknown" + MAPPING_UNAPPROVED = "mapping_unapproved" + PROTOCOL_UNAPPROVED = "protocol_unapproved" + PUBLISHED_BENCHMARK_NONCONFORMITY = "published_benchmark_nonconformity" + REFERENCE_INVALID = "reference_invalid" + REFERENCE_PRECISION_INADEQUATE = "reference_precision_inadequate" + REQUIRED_CELL_OR_DISTRICT_MISSING = "required_cell_or_district_missing" + SOURCE_DOMAIN_REFUSED = "source_domain_refused" + SUPPORT_BOUND_EXCEEDED = "support_bound_exceeded" + UNCLASSIFIED_FRACTION_EXCEEDED = "unclassified_fraction_exceeded" + UNRESOLVED_SOURCE_LINEAGE = "unresolved_source_lineage" + VACANCY_BRIDGE_MISSING = "vacancy_bridge_missing" + ZERO_CANDIDATE_DENOMINATOR = "zero_candidate_denominator" + ZERO_REFERENCE_DENOMINATOR = "zero_reference_denominator" + + +class ExposureStatus(StrEnum): + """The four values at ``/exposure_ledger/exposure_status_values``.""" + + EXPOSURE_HISTORY_UNKNOWN = "exposure_history_unknown" + FRESH_SAME_SOURCE_HOLDOUT_ELIGIBLE = "fresh_same_source_holdout_eligible" + NOT_RESERVED_CONFORMITY = "not_reserved_conformity" + POST_EXPOSURE_CONFORMITY = "post_exposure_conformity" + + +class BenchmarkVerdict(StrEnum): + """Scope-level outcomes named by ``/status_contract``. + + There is deliberately no "release", "certified" or "passing" member: a + conforming verdict is operational published-benchmark conformity for the + stated scope and nothing else (``/claim``). + """ + + BENCHMARK_CONFORMING = "benchmark_conforming" + BENCHMARK_NONCONFORMING = "benchmark_nonconforming" + SCOPE_INCOMPLETE = "scope_incomplete" + + +class PrecisionStatus(StrEnum): + """The reference-precision axis, independent of the metric axis. + + ``CONTROLLED_UNKNOWN`` is the ``/reference_status/controlled_estimate`` + class: the point comparison stays valid, no variance is claimed, and no + adequacy is asserted. It is neither an adequacy verdict nor a failure. + """ + + ADEQUATE = "adequate" + INADEQUATE = "inadequate" + CONTROLLED_UNKNOWN = "controlled_unknown" + UNAVAILABLE = "unavailable" + + +class MappingStatus(StrEnum): + """Whether an authenticated producer bound the required mapping slots.""" + + APPROVED = "approved" + UNAPPROVED = "unapproved" + + +class SupportVerdict(StrEnum): + """Origin-support outcome, independent of the benchmark metric. + + ``NOT_APPLICABLE`` is the ``/support/reference_zero_unknown_bin_carrier_floor`` + case: the carrier floor does not apply to a bin whose published reference + share is zero, so mass and origin diagnostics are retained without a bound. + """ + + WITHIN_BOUNDS = "within_bounds" + BOUND_EXCEEDED = "bound_exceeded" + UNDETERMINED = "undetermined" + NOT_APPLICABLE = "not_applicable" + + +#: ``/status_contract/independent_axes_required`` — reported separately and +#: never collapsed into one scalar. +AXES = ( + "benchmark_metric_status", + "reference_precision_status", + "mapping_status", + "support_status", + "exposure_status", + "scope_completeness", +) + + +def dedupe(items: Iterable[Reason]) -> tuple[Reason, ...]: + """Deduplicate reasons, preserving first-seen order. + + Raises: + TypeError: If any item is outside the closed :class:`Reason` namespace. + """ + seen: dict[Reason, None] = {} + for item in items: + if not isinstance(item, Reason): + raise TypeError(f"Unknown primary reason code {item!r}.") + seen.setdefault(item) + return tuple(seen) diff --git a/packages/microcosm-build/src/microcosm/build/frame_checkpoint.py b/packages/microcosm-build/src/microcosm/build/frame_checkpoint.py index ac26729c6..0e4cf984a 100644 --- a/packages/microcosm-build/src/microcosm/build/frame_checkpoint.py +++ b/packages/microcosm-build/src/microcosm/build/frame_checkpoint.py @@ -41,10 +41,15 @@ "write_frame_checkpoint", ] -FRAME_CHECKPOINT_SCHEMA_VERSION = 3 +FRAME_CHECKPOINT_SCHEMA_VERSION = 4 +_NULLABLE_BOOLEAN_SCHEMA_VERSION = 3 _LEGACY_FRAME_CHECKPOINT_SCHEMA_VERSION = 2 _SUPPORTED_FRAME_CHECKPOINT_SCHEMA_VERSIONS = frozenset( - {_LEGACY_FRAME_CHECKPOINT_SCHEMA_VERSION, FRAME_CHECKPOINT_SCHEMA_VERSION} + { + _LEGACY_FRAME_CHECKPOINT_SCHEMA_VERSION, + _NULLABLE_BOOLEAN_SCHEMA_VERSION, + FRAME_CHECKPOINT_SCHEMA_VERSION, + } ) _ARTIFACT_KIND = "populace_frame_checkpoint" @@ -56,6 +61,20 @@ _ENCODING_TIMEDELTA = "timedelta64" _ENCODING_OBJECT = "object_scalars_v1" _ENCODING_NULLABLE_BOOLEAN = "nullable_boolean_v1" +_ENCODING_NULLABLE_INTEGER = "nullable_integer_v1" +_NULLABLE_INTEGER_DTYPES = { + str(dtype): dtype + for dtype in ( + pd.Int8Dtype(), + pd.Int16Dtype(), + pd.Int32Dtype(), + pd.Int64Dtype(), + pd.UInt8Dtype(), + pd.UInt16Dtype(), + pd.UInt32Dtype(), + pd.UInt64Dtype(), + ) +} _TAG_NONE = 0 _TAG_PD_NA = 1 @@ -346,25 +365,21 @@ def _checkpoint_metadata( ) strata_spec = _series_spec(frame.strata, label="strata") - uses_nullable_boolean = any( - column.get("encoding") == _ENCODING_NULLABLE_BOOLEAN - for table in table_specs - for column in table["columns"] - ) or any( - table["index"].get("encoding") == _ENCODING_NULLABLE_BOOLEAN + encodings = { + spec.get("encoding") for table in table_specs - ) - uses_nullable_boolean = uses_nullable_boolean or ( - strata_spec.get("encoding") == _ENCODING_NULLABLE_BOOLEAN - ) + for spec in [*table["columns"], table["index"]] + } | {strata_spec.get("encoding")} + if _ENCODING_NULLABLE_INTEGER in encodings: + version = FRAME_CHECKPOINT_SCHEMA_VERSION + elif _ENCODING_NULLABLE_BOOLEAN in encodings: + version = _NULLABLE_BOOLEAN_SCHEMA_VERSION + else: + version = _LEGACY_FRAME_CHECKPOINT_SCHEMA_VERSION schema = frame.schema return { "artifact_kind": _ARTIFACT_KIND, - "schema_version": ( - FRAME_CHECKPOINT_SCHEMA_VERSION - if uses_nullable_boolean - else _LEGACY_FRAME_CHECKPOINT_SCHEMA_VERSION - ), + "schema_version": version, "schema": { "person_entity": schema.person_entity, "group_entities": list(schema.group_entities), @@ -420,6 +435,15 @@ def _series_spec(series: pd.Series, *, label: str) -> dict[str, object]: "encoding": _ENCODING_NULLABLE_BOOLEAN, "has_null_mask": bool(series.isna().any()), } + if ( + str(dtype) in _NULLABLE_INTEGER_DTYPES + and dtype == (_NULLABLE_INTEGER_DTYPES[str(dtype)]) + ): + return { + "dtype": str(dtype), + "encoding": _ENCODING_NULLABLE_INTEGER, + "has_null_mask": bool(series.isna().any()), + } if isinstance(dtype, pd.api.extensions.ExtensionDtype) and not isinstance( dtype, pd.StringDtype ): @@ -561,6 +585,19 @@ def _write_series(group: Any, series: pd.Series, spec: Mapping[str, object]) -> null_mask.astype(np.uint8, copy=False), ) return + if encoding == _ENCODING_NULLABLE_INTEGER: + dtype = _NULLABLE_INTEGER_DTYPES[str(spec["dtype"])] + null_mask = series.isna().to_numpy(dtype=np.bool_, copy=False) + # Keep exact integer width and observed values, including UInt64 IDs. + # Hidden extension-array storage is not part of the logical payload. + values = series.to_numpy(dtype=dtype.numpy_dtype, na_value=0, copy=True) + values[null_mask] = 0 + _write_numpy_dataset(group, "values", values) + if spec.get("has_null_mask") is True: + _write_numpy_dataset( + group, "null_mask", null_mask.astype(np.uint8, copy=False) + ) + return raise RuntimeError(f"Unknown checkpoint series encoding {encoding!r}.") @@ -641,6 +678,55 @@ def _read_series( pd.arrays.BooleanArray(values, mask, copy=False), copy=False, ) + elif encoding == _ENCODING_NULLABLE_INTEGER: + integer_dtype = _NULLABLE_INTEGER_DTYPES.get(dtype) + if integer_dtype is None: + raise ValueError( + f"Frame checkpoint {path} {label!r} nullable integer encoding " + f"has unsupported declared dtype {dtype!r}." + ) + has_null_mask = spec.get("has_null_mask") + if type(has_null_mask) is not bool: + raise ValueError( + f"Frame checkpoint {path} {label!r} nullable integer " + "has_null_mask must be a boolean." + ) + values = _read_numpy_dataset(group, "values", path) + if values.ndim != 1 or values.dtype != integer_dtype.numpy_dtype: + raise ValueError( + f"Frame checkpoint {path} {label!r} nullable integer values " + f"must be a one-dimensional {integer_dtype.numpy_dtype} array." + ) + if has_null_mask: + if "null_mask" not in group: + raise ValueError( + f"Frame checkpoint {path} {label!r} is missing its null mask." + ) + null_mask = _read_numpy_dataset(group, "null_mask", path) + if ( + null_mask.ndim != 1 + or null_mask.dtype != np.dtype(np.uint8) + or len(null_mask) != len(values) + or ((null_mask != 0) & (null_mask != 1)).any() + or not null_mask.any() + ): + raise ValueError( + f"Frame checkpoint {path} {label!r} null mask must be a " + "one-dimensional uint8 0/1 array aligned to its values." + ) + mask = null_mask.astype(np.bool_, copy=False) + if (values[mask] != 0).any(): + raise ValueError( + f"Frame checkpoint {path} {label!r} null mask covers " + "noncanonical nonzero integer storage." + ) + else: + if "null_mask" in group: + raise ValueError( + f"Frame checkpoint {path} {label!r} has an unexpected null mask." + ) + mask = np.zeros(len(values), dtype=np.bool_) + series = pd.Series(pd.arrays.IntegerArray(values, mask, copy=False), copy=False) else: raise ValueError( f"Frame checkpoint {path} has unknown encoding {encoding!r} for {label!r}." @@ -838,39 +924,62 @@ def _read_metadata(root: Any, path: Path) -> dict[str, Any]: f"Frame checkpoint {path} schema version is {version!r}; expected " f"one of {sorted(_SUPPORTED_FRAME_CHECKPOINT_SCHEMA_VERSIONS)}." ) - nullable_spec_count = 0 + boolean_spec_count = 0 + integer_spec_count = 0 for label, spec in _checkpoint_series_specs(metadata): dtype = spec.get("dtype") encoding = spec.get("encoding") - declares_nullable = dtype == "boolean" - uses_nullable_encoding = encoding == _ENCODING_NULLABLE_BOOLEAN + declares_boolean = dtype == "boolean" + uses_boolean_encoding = encoding == _ENCODING_NULLABLE_BOOLEAN + declares_integer = isinstance(dtype, str) and dtype in _NULLABLE_INTEGER_DTYPES + uses_integer_encoding = encoding == _ENCODING_NULLABLE_INTEGER + if version < FRAME_CHECKPOINT_SCHEMA_VERSION and ( + declares_integer or uses_integer_encoding + ): + raise ValueError( + f"Frame checkpoint {path} schema version {version} cannot carry " + f"nullable integer spec {label!r}." + ) if version == _LEGACY_FRAME_CHECKPOINT_SCHEMA_VERSION and ( - declares_nullable or uses_nullable_encoding + declares_boolean or uses_boolean_encoding ): raise ValueError( f"Frame checkpoint {path} schema version 2 cannot carry nullable " f"boolean spec {label!r}." ) - if version == FRAME_CHECKPOINT_SCHEMA_VERSION and ( - declares_nullable != uses_nullable_encoding + if version >= _NULLABLE_BOOLEAN_SCHEMA_VERSION and ( + declares_boolean != uses_boolean_encoding ): raise ValueError( - f"Frame checkpoint {path} schema version 3 nullable boolean spec " - f"{label!r} must pair declared dtype 'boolean' with encoding " + f"Frame checkpoint {path} schema version {version} nullable boolean " + f"spec {label!r} must pair declared dtype 'boolean' with encoding " f"{_ENCODING_NULLABLE_BOOLEAN!r}." ) - if uses_nullable_encoding: - nullable_spec_count += 1 + if declares_integer != uses_integer_encoding: + raise ValueError( + f"Frame checkpoint {path} schema version {version} nullable integer " + f"spec {label!r} must pair a supported pandas integer dtype " + f"with encoding {_ENCODING_NULLABLE_INTEGER!r}." + ) + if uses_boolean_encoding or uses_integer_encoding: + family = "boolean" if uses_boolean_encoding else "integer" if type(spec.get("has_null_mask")) is not bool: raise ValueError( - f"Frame checkpoint {path} {label!r} nullable boolean " + f"Frame checkpoint {path} {label!r} nullable {family} " "has_null_mask must be a boolean." ) - if version == FRAME_CHECKPOINT_SCHEMA_VERSION and nullable_spec_count == 0: + boolean_spec_count += uses_boolean_encoding + integer_spec_count += uses_integer_encoding + if version == _NULLABLE_BOOLEAN_SCHEMA_VERSION and boolean_spec_count == 0: raise ValueError( f"Frame checkpoint {path} schema version 3 requires at least one " "nullable boolean spec." ) + if version == FRAME_CHECKPOINT_SCHEMA_VERSION and integer_spec_count == 0: + raise ValueError( + f"Frame checkpoint {path} schema version 4 requires at least one " + "nullable integer spec." + ) return metadata diff --git a/packages/microcosm-build/src/microcosm/build/graph_atomic_geography.py b/packages/microcosm-build/src/microcosm/build/graph_atomic_geography.py new file mode 100644 index 000000000..a245992a8 --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/graph_atomic_geography.py @@ -0,0 +1,281 @@ +"""Typed graph nodes for the shared atomic-area geography contract. + +Countries supply declarations and normalized source bytes. Import, assignment, +derivation and the integrity gate remain independently inspectable operations. +These nodes do not change weights, rows, memberships or observed columns. +""" + +from __future__ import annotations + +import json +from collections.abc import Mapping, Sequence + +import pandas as pd + +from microcosm.build import atomic_geography as geography +from microcosm.graph import ( + ArtifactInput, + ArtifactOutput, + ArtifactType, + Capabilities, + Determinism, + KernelBase, + KernelContext, + KernelRegistry, + KernelResult, + Node, + Numeric, + Owned, + SeedSource, + Slice, + source_hash, +) +from microcosm.graph.canonical import canonical_json +from microcosm.graph.codecs import load_source_bytes +from microcosm.graph.kernel import KernelRole +from microcosm.graph.randomness import keyed_uniform + +ATOMIC_SUPPORT_TYPE = ArtifactType("microcosm.geography.atomic_support_npz", 1) +ATOMIC_GEOGRAPHY_VALIDATION_TYPE = ArtifactType( + "microcosm.geography.validation_result", 1 +) +_DEPENDENCIES = ("numpy", "pandas") + + +class _GeographyKernel(KernelBase): + def implementation_hash(self) -> str: + return source_hash( + type(self), + geography, + canonical_json, + keyed_uniform, + load_source_bytes, + dependencies=_DEPENDENCIES, + ) + + +class AtomicSupportImportKernel(_GeographyKernel): + ref = "geography.support_import@1" + capabilities = Capabilities( + determinism=Determinism.DETERMINISTIC, + numeric=Numeric.PLATFORM_BITWISE, + dependencies=_DEPENDENCIES, + ) + + def run(self, context: KernelContext) -> KernelResult: + if set(context.params) != {"source", "system"} or set(context.sources) != { + context.params["source"] + }: + raise ValueError( + "Atomic geography import requires its sole declared source." + ) + payload = load_source_bytes( + "raw-bytes-v1", context.sources[context.params["source"]] + ) + support = geography.decode_atomic_support(payload) + if support.metadata["system"] != context.params["system"]: + raise ValueError("Atomic geography source belongs to a different system.") + return KernelResult( + artifacts={"support": payload}, + receipt={ + "support_sha256": support.sha256, + "areas": len(support.arrays["area"]), + "metadata": support.metadata, + }, + ) + + +def _inputs(context): + if set(context.params) != {"definition", "stream"}: + raise ValueError( + "Atomic geography requires its canonical definition and stream." + ) + spec = geography.validate_assignment_spec(json.loads(context.params["definition"])) + if ( + canonical_json(spec).decode() != context.params["definition"] + or tuple(spec["stream"]) != context.params["stream"] + ): + raise ValueError("Atomic geography definition or stream differs.") + if set(context.artifacts) != {s["id"] for s in spec["systems"]}: + raise ValueError( + "Atomic geography requires exactly its declared support artifacts." + ) + supports = {} + for name, value in context.artifacts.items(): + if value.type != ATOMIC_SUPPORT_TYPE: + raise ValueError("Atomic geography artifact type differs.") + supports[name] = geography.decode_atomic_support(value.payload) + return context.tables["household"], spec, supports + + +def _result(context, frame, receipt): + if {(o.entity, o.column, o.dtype) for o in context.node.outputs} != { + ("household", c, "string") for c in frame + }: + raise ValueError("Atomic geography owned outputs differ from the declaration.") + index = pd.Index(context.tables["household"]["household_id"], name="household_id") + return KernelResult( + columns={ + ("household", c): pd.Series(frame[c].array, index=index) for c in frame + }, + receipt=receipt, + ) + + +class AtomicAssignKernel(_GeographyKernel): + ref = "geography.assign_atomic@1" + capabilities = Capabilities( + determinism=Determinism.SEEDED, + seed_source=SeedSource.KEYED, + numeric=Numeric.PLATFORM_BITWISE, + dependencies=_DEPENDENCIES, + ) + + def run(self, context: KernelContext) -> KernelResult: + households, spec, supports = _inputs(context) + output = geography.assign_atomic(households, spec, supports) + return _result( + context, + output, + { + "scope": "atomic_area_assignment", + "households": len(output), + "support_sha256": {k: v.sha256 for k, v in supports.items()}, + }, + ) + + +class AtomicDeriveKernel(_GeographyKernel): + ref = "geography.derive@1" + capabilities = Capabilities( + determinism=Determinism.DETERMINISTIC, + numeric=Numeric.PLATFORM_BITWISE, + dependencies=_DEPENDENCIES, + ) + + def run(self, context: KernelContext) -> KernelResult: + households, spec, supports = _inputs(context) + output = geography.derive_geography(households, spec, supports) + return _result( + context, + output, + { + "scope": "atomic_area_functional_lookup", + "layers": {s["id"]: s["layers"] for s in spec["systems"]}, + }, + ) + + +class AtomicGeographyGateKernel(_GeographyKernel): + ref = "geography.gate@1" + capabilities = Capabilities( + determinism=Determinism.DETERMINISTIC, + role=KernelRole.GATE, + numeric=Numeric.PLATFORM_BITWISE, + dependencies=_DEPENDENCIES, + ) + + def run(self, context: KernelContext) -> KernelResult: + households, spec, supports = _inputs(context) + outputs = context.node.artifact_outputs + expected = (ArtifactOutput("validation", ATOMIC_GEOGRAPHY_VALIDATION_TYPE),) + if outputs not in ((), expected): + raise ValueError( + "Atomic geography validation artifact declaration differs." + ) + receipt = geography.validate_geography(households, spec, supports) + return KernelResult( + receipt=receipt, + artifacts={"validation": canonical_json(receipt)} if outputs else {}, + ) + + +def atomic_geography_nodes( + spec: Mapping, + columns: Sequence[Owned], + *, + base: str, + prefix: str = "geography", + emit_validation_artifact: bool = False, +) -> tuple[Node, ...]: + """Append shared nodes to an existing population, refusing column rewrites. + + SourceRef declarations use each system's ``source`` and ``raw-bytes-v1``. + ``identity`` must be stable across the country's sampling rungs; this builder + cannot establish that property merely from a column name. + + The optional typed gate result orders downstream consumers. Its bytes alone + do not authenticate their receiving population or source ancestry. + """ + if type(emit_validation_artifact) is not bool: + raise ValueError("Atomic geography validation artifact flag must be boolean.") + spec = geography.validate_assignment_spec(spec) + inventory = {(o.entity, o.column): o for o in columns} + if len(inventory) != len(columns): + raise ValueError("Atomic geography input inventory repeats columns.") + inputs = set(spec["identity"]) + for system in spec["systems"]: + inputs.update(system["selector"]) + inputs.update(c["input"] for c in system["constraints"]) + if system["observed_area"]: + inputs.add(system["observed_area"]) + assignment = tuple(spec["outputs"].values()) + layers = tuple( + sorted({layer["output"] for s in spec["systems"] for layer in s["layers"]}) + ) + if any(("household", c) not in inventory for c in inputs): + raise ValueError("Atomic geography is missing a declared household input.") + if any(("household", c) in inventory for c in (*assignment, *layers)): + raise ValueError("Atomic geography refuses to overwrite existing columns.") + imports, artifacts = [], [] + for i, system in enumerate(spec["systems"]): + name = f"{prefix}.support.{i}" + imports.append( + Node( + id=name, + kernel=AtomicSupportImportKernel.ref, + population=base, + sources=(system["source"],), + params={"source": system["source"], "system": system["id"]}, + artifact_outputs=(ArtifactOutput("support", ATOMIC_SUPPORT_TYPE),), + ) + ) + artifacts.append( + ArtifactInput(system["id"], name, "support", ATOMIC_SUPPORT_TYPE) + ) + + def node(suffix, kernel, read, write=()): + return Node( + id=f"{prefix}.{suffix}", + kernel=kernel.ref, + population=base, + inputs=(Slice("household", tuple(sorted(read))),), + outputs=tuple(Owned("household", c, "string") for c in write), + params={ + "definition": canonical_json(spec).decode(), + "stream": tuple(spec["stream"]), + }, + artifact_inputs=tuple(artifacts), + artifact_outputs=( + (ArtifactOutput("validation", ATOMIC_GEOGRAPHY_VALIDATION_TYPE),) + if suffix == "gate" and emit_validation_artifact + else () + ), + ) + + return ( + *imports, + node("assign", AtomicAssignKernel, inputs, assignment), + node("derive", AtomicDeriveKernel, inputs | set(assignment), layers), + node("gate", AtomicGeographyGateKernel, inputs | set(assignment) | set(layers)), + ) + + +def register_atomic_geography_kernels(registry: KernelRegistry) -> None: + for kernel in ( + AtomicSupportImportKernel(), + AtomicAssignKernel(), + AtomicDeriveKernel(), + AtomicGeographyGateKernel(), + ): + registry.register(kernel) diff --git a/packages/microcosm-build/src/microcosm/build/spec_engine/inventory_coverage.py b/packages/microcosm-build/src/microcosm/build/spec_engine/inventory_coverage.py index ca3c0357b..9a32bb9cd 100644 --- a/packages/microcosm-build/src/microcosm/build/spec_engine/inventory_coverage.py +++ b/packages/microcosm-build/src/microcosm/build/spec_engine/inventory_coverage.py @@ -359,8 +359,8 @@ "late_schedule": "e59c019d3d454eac99ac0ac209b6c5b6faaf9bdfcaeee18c36a25be19bf7da2f", "ownership": "5f64f0aac49e2313177564f71876bffc8c81b3ded4df701e70930e60e9c98356", "primary_tuples": "987b501c695e31f45521c4a178528f75ab3df22c09bc407b182213b2de99ee57", - "seed_map": "20058e544f6034cee2e76d3b864cf86b6230c8dce042931dfe9247d8ac1329c4", - "seed_protocol": "553d5e0bd5afa93146dfbf8dca684926ee773dbfc89a559434528af5b4eb1d97", + "seed_map": "be2d24ad74bdc6339dfa073d01469074baee4e3a691eaa350451d36fa17d13dc", + "seed_protocol": "7c67a5a315a615b152d7de106ae10b1828c166bb11c67a713c8a52d65ae9db47", "source_manifest": "cd5ba8924d64da5425ee14cca82a774e3f4b2bb5aabe06df291cc3cc457287a9", "take_up": "fa186daea0f8dd641cc470e41d1a2953f887d45282ec990201298f47bedf8d4d", "tail": "ac92829c88a1a4fb6460d61190918d5d99c6c377fc8dd8f62f02b332d09bf59c", diff --git a/packages/microcosm-build/src/microcosm/build/survey_allocation.py b/packages/microcosm-build/src/microcosm/build/survey_allocation.py new file mode 100644 index 000000000..417689e5a --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/survey_allocation.py @@ -0,0 +1,202 @@ +"""Domain allocation of supplied design weights, before support cloning. + +This numerical primitive performs no source classification, subsampling, frame +mutation or graph activation. Callers must supply an aligned household axis and +reviewed domain declarations. A row can be a physical housing unit or a cluster +of people; explicit unit counts keep its statistical grain out of its weight. +""" + +from __future__ import annotations + +from collections.abc import Mapping, Sequence +from dataclasses import dataclass +from fractions import Fraction +from numbers import Integral +from types import MappingProxyType + +import numpy as np + +from microcosm.frame import WeightKind, Weights + + +class SurveyAllocationError(ValueError): + """A supplied allocation contract cannot represent its household axis.""" + + +def _label(value: object, field: str) -> str: + if not isinstance(value, str) or not value.strip(): + raise SurveyAllocationError(f"{field} must be a nonempty string") + return value + + +@dataclass(frozen=True) +class SurveyDomain: + """One declared population/period and its prespecified source shares. + + Shares must be exact Fractions summing to one. Unit and basis labels are + declarations, not evidence of survey equivalence or source authentication. + """ + + key: str + statistical_unit: str + period: str + basis: str + source_shares: Mapping[str, Fraction] + + def __post_init__(self) -> None: + for field in ("key", "statistical_unit", "period", "basis"): + _label(getattr(self, field), field) + if not isinstance(self.source_shares, Mapping) or not self.source_shares: + raise SurveyAllocationError("source_shares must be a nonempty mapping") + shares = {} + for source, share in self.source_shares.items(): + _label(source, "source channel") + if not isinstance(share, Fraction) or not 0 <= share <= 1: + raise SurveyAllocationError( + "shares must be Fractions between zero and one" + ) + shares[source] = share + if sum(shares.values(), Fraction()) != 1: + raise SurveyAllocationError("domain shares must sum exactly to one") + object.__setattr__( + self, "source_shares", MappingProxyType(dict(sorted(shares.items()))) + ) + + +@dataclass(frozen=True) +class DomainSourceEstimate: + """Separate estimates in one declared unit, period and source domain.""" + + domain: str + source: str + statistical_unit: str + period: str + basis: str + source_share: Fraction + rows: int + design_estimate: float + allocated_estimate: float + + +@dataclass(frozen=True) +class SurveyAllocation: + """Output weights aligned to input row IDs; estimates never pool units.""" + + row_ids: tuple[int, ...] + weights: Weights + estimates: tuple[DomainSourceEstimate, ...] + + +def _axis(values: Sequence, size: int, field: str) -> tuple: + if isinstance(values, (str, bytes)): + raise SurveyAllocationError(f"{field} must be a row-aligned sequence") + result = tuple(values) + if len(result) != size: + raise SurveyAllocationError(f"{field} must match the household weight axis") + return result + + +def allocate_domain_weights( + design_weights: Weights, + *, + row_ids: Sequence[int], + source_channels: Sequence[str], + domain_keys: Sequence[str | None], + unit_counts: Sequence[int], + domains: Sequence[SurveyDomain], +) -> SurveyAllocation: + """Multiply each household weight by its declared domain/source share. + + ``unit_counts`` is mandatory: one for a housing unit or individual GQ + placeholder, the number of covered people for a person-counting cluster. + It affects accounting, never the household weight multiplier. No source + total is normalized to another source. Unknown domains and positive-share + cells lacking positive design support refuse. Known exclusions must be + selected explicitly before this operation; rows are never dropped here. + + The declarations do not authenticate their inputs. A production graph + adapter must bind IDs, source membership, coverage and period assumptions + to the actual population and use a new allocation context/receipt version. + """ + if ( + not isinstance(design_weights, Weights) + or design_weights.kind is not WeightKind.DESIGN + ): + raise SurveyAllocationError("allocation requires DESIGN weights") + size = len(design_weights.values) + ids = _axis(row_ids, size, "row_ids") + if any( + isinstance(x, bool) or not isinstance(x, Integral) or not 0 < x <= 2**63 - 1 + for x in ids + ): + raise SurveyAllocationError( + "row_ids must be positive int64 household identities" + ) + if len(set(ids)) != size: + raise SurveyAllocationError("household identities must be unique") + sources = _axis(source_channels, size, "source_channels") + keys = _axis(domain_keys, size, "domain_keys") + counts = _axis(unit_counts, size, "unit_counts") + if any( + isinstance(x, bool) or not isinstance(x, Integral) or not 0 < x <= 2**53 + for x in counts + ): + raise SurveyAllocationError( + "unit_counts must be positive exactly representable integers" + ) + declared = {} + for domain in domains: + if not isinstance(domain, SurveyDomain) or domain.key in declared: + raise SurveyAllocationError( + "domains must be unique SurveyDomain declarations" + ) + declared[domain.key] = domain + cells: dict[tuple[str, str], list[int]] = {} + for position, (key, source) in enumerate(zip(keys, sources, strict=True)): + _label(source, "source channel") + if not isinstance(key, str) or key not in declared: + raise SurveyAllocationError("unknown or undeclared coverage domain") + if source not in declared[key].source_shares: + raise SurveyAllocationError( + "source is not declared for its coverage domain" + ) + cells.setdefault((key, source), []).append(position) + incoming = design_weights.values + allocated = np.zeros(size, dtype=np.float64) + unit_array = np.asarray(counts, dtype=np.float64) + estimates = [] + for key, domain in sorted(declared.items()): + for source, share in domain.source_shares.items(): + positions = np.asarray(cells.get((key, source), ()), dtype=np.int64) + values = incoming[positions] + if share > 0 and not np.any(values > 0): + raise SurveyAllocationError( + "positive-share domain/source lacks sampled support" + ) + with np.errstate(over="ignore", under="ignore", invalid="ignore"): + output = values * float(share) + design_total = float(np.sum(values * unit_array[positions])) + allocated_total = float(np.sum(output * unit_array[positions])) + if not np.isfinite(design_total) or not np.isfinite(allocated_total): + raise SurveyAllocationError("domain estimate overflow") + if share > 0 and np.any((values > 0) & (output == 0)): + raise SurveyAllocationError("positive allocated weight underflow") + allocated[positions] = output + estimates.append( + DomainSourceEstimate( + key, + source, + domain.statistical_unit, + domain.period, + domain.basis, + share, + len(positions), + design_total, + allocated_total, + ) + ) + return SurveyAllocation( + tuple(int(x) for x in ids), + Weights(allocated, WeightKind.IMPORTANCE), + tuple(estimates), + ) diff --git a/packages/microcosm-build/src/microcosm/build/survey_domain_sample.py b/packages/microcosm-build/src/microcosm/build/survey_domain_sample.py new file mode 100644 index 000000000..5d28fd57e --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/survey_domain_sample.py @@ -0,0 +1,369 @@ +"""Whole-household domain sampling composed with survey-share allocation. + +Coverage columns are supplied declarations bound to this Frame's household +axis. This module does not authenticate their survey meaning, align periods, +classify sources, or authorize a production graph. It never normalizes a +sampled domain estimate to another estimate or to a mixed-unit weight sum. +""" + +from __future__ import annotations + +import hashlib +import json +from collections.abc import Sequence +from dataclasses import dataclass +from fractions import Fraction +from numbers import Integral + +import numpy as np + +from microcosm.build.survey_allocation import ( + DomainSourceEstimate, + SurveyAllocationError, + SurveyDomain, + allocate_domain_weights, +) +from microcosm.frame import Frame, MassChange, WeightKind, Weights + +PROTOCOL = "microcosm.survey-domain-sample-allocation.v1" + + +def _json(value: object) -> bytes: + return json.dumps( + value, sort_keys=True, separators=(",", ":"), allow_nan=False + ).encode() + + +def _id_digest(ids: Sequence[int]) -> str: + return hashlib.sha256(np.asarray(ids, dtype=" tuple[DomainHouseholdSelection, ...]: + """Use the same per-cell draw for native catalogues and full Frames. + + ``cells`` contains (domain, source) pairs, including declared empty cells. + IDs must be uniformly positive int64 integers or nonempty literal strings. + They are unique within each source, and sort numerically or lexically, + respectively. No ID conversion, source-share exclusion, classification, + weighting, or claim of full-source completeness occurs here. A source owner + must establish the complete eligible axis before calling this function. + + Each populated cell retains max(1, floor(fraction * N)) rows. Empty cells + have no inclusion probability; full selection consumes no random draw. + The existing protocol and PCG64 seed construction remain unchanged. + """ + if not isinstance(fraction, Fraction) or not 0 < fraction <= 1: + raise SurveyAllocationError("fraction must be a Fraction in (0, 1]") + if type(seed) is not int or not 0 <= seed < 2**64: + raise SurveyAllocationError("seed must be an unsigned 64-bit integer") + axes = (row_ids, source_channels, domain_keys, cells) + if any(isinstance(axis, (str, bytes)) for axis in axes): + raise SurveyAllocationError("selection inputs must be row sequences") + ids, sources, domains, declared = (tuple(axis) for axis in axes) + if len(ids) != len(sources) or len(ids) != len(domains): + raise SurveyAllocationError("selection axes must have equal lengths") + integer_ids = all( + not isinstance(value, bool) + and isinstance(value, Integral) + and 0 < value <= 2**63 - 1 + for value in ids + ) + string_ids = all(type(value) is str and bool(value.strip()) for value in ids) + if not integer_ids and not string_ids: + raise SurveyAllocationError( + "selection IDs must be uniformly positive int64 or literal strings" + ) + if any( + not isinstance(value, str) or not value.strip() + for value in (*sources, *domains) + ): + raise SurveyAllocationError("selection source/domain labels must be nonempty") + if any( + type(cell) is not tuple + or len(cell) != 2 + or any(not isinstance(label, str) or not label.strip() for label in cell) + for cell in declared + ): + raise SurveyAllocationError("selection cells require (domain, source) pairs") + if len(set(declared)) != len(declared): + raise SurveyAllocationError("selection cells must be unique") + if len(set(zip(sources, ids, strict=True))) != len(ids): + raise SurveyAllocationError("selection identities must be unique within source") + groups: dict[tuple[str, str], list[int]] = {cell: [] for cell in declared} + for position, cell in enumerate(zip(domains, sources, strict=True)): + if cell not in groups: + raise SurveyAllocationError("selection row has an undeclared cell") + groups[cell].append(position) + result = [] + for (domain, source), group in sorted(groups.items()): + positions = np.asarray(sorted(group, key=ids.__getitem__), dtype=np.int64) + eligible = len(positions) + if not eligible: + chosen, probability = (), None + else: + count = max(1, fraction.numerator * eligible // fraction.denominator) + probability = Fraction(count, eligible) + if count == eligible: + chosen = positions + else: + material = _json([PROTOCOL, seed, source, domain]) + cell_seed = int.from_bytes(hashlib.sha256(material).digest(), "big") + rng = np.random.Generator(np.random.PCG64(cell_seed)) + chosen = np.sort(rng.choice(positions, size=count, replace=False)) + result.append( + DomainHouseholdSelection( + domain, source, eligible, tuple(int(p) for p in chosen), probability + ) + ) + return tuple(result) + + +@dataclass(frozen=True) +class SampledDomainCell: + """One source/domain estimate in its declared unit, period and basis.""" + + full: DomainSourceEstimate + selected_households: int + inclusion_probability: Fraction | None + selected_household_ids_sha256: str + sampled_allocated_estimate: float + + +@dataclass(frozen=True) +class DomainSampleAllocation: + """A sampled IMPORTANCE Frame and separate, immutable domain accounting.""" + + frame: Frame + cells: tuple[SampledDomainCell, ...] + fraction: Fraction + seed: int + input_axis_sha256: str + domain_declarations_sha256: str + numpy_version: str + coverage_columns: tuple[str, str, str] + + def receipt(self) -> dict[str, object]: + """Describe the executed arithmetic without claiming source admission.""" + return { + "protocol": PROTOCOL, + "coverage_status": "declared_frame_columns_only", + "release_eligible": False, + "coverage_columns": dict( + zip( + ("source", "domain", "unit_count"), + self.coverage_columns, + strict=True, + ) + ), + "fraction": [self.fraction.numerator, self.fraction.denominator], + "seed": self.seed, + "rng": {"family": "PCG64", "numpy_version": self.numpy_version}, + "input_axis_sha256": self.input_axis_sha256, + "domain_declarations_sha256": self.domain_declarations_sha256, + "cells": [ + { + "domain": cell.full.domain, + "source": cell.full.source, + "statistical_unit": cell.full.statistical_unit, + "period": cell.full.period, + "basis": cell.full.basis, + "source_share": [ + cell.full.source_share.numerator, + cell.full.source_share.denominator, + ], + "eligible_households": cell.full.rows, + "selected_households": cell.selected_households, + "inclusion_probability": ( + None + if cell.inclusion_probability is None + else [ + cell.inclusion_probability.numerator, + cell.inclusion_probability.denominator, + ] + ), + "selected_household_ids_sha256": cell.selected_household_ids_sha256, + "full_design_estimate": cell.full.design_estimate, + "full_allocated_estimate": cell.full.allocated_estimate, + "sampled_allocated_estimate": cell.sampled_allocated_estimate, + } + for cell in self.cells + ], + } + + +def sample_and_allocate_domains( + frame: Frame, + *, + source_column: str, + domain_column: str, + unit_count_column: str, + domains: Sequence[SurveyDomain], + fraction: Fraction, + seed: int, +) -> DomainSampleAllocation: + """Sample each declared source/domain cell, then compose its multipliers. + + Each populated cell retains ``max(1, floor(fraction * N))`` households by + simple random sampling without replacement. Its output household weight + is ``source_design_weight * domain_source_share * N / n``. Unit counts + affect estimates only. Small domains therefore remain represented, with + their own actual inclusion probability recorded; no totals are forced to + match. Full selection does not draw. + + PCG64 streams are separated by protocol, source and domain. Sorted household + IDs make selection independent of row order and unrelated cells. The + returned Frame preserves complete selected lineages through Frame.select. + It never labels the sampling/allocation result DESIGN. + + A positive-share cell whose random sample has no positive weight refuses. + Callers must not search seeds to evade that refusal. Coverage exclusions and + period harmonization must be resolved explicitly before this operation. + """ + if not isinstance(frame, Frame): + raise TypeError("domain sampling requires a Frame") + frame.revalidate() + if frame.weighted_entities != ("household",): + raise SurveyAllocationError("domain sampling requires household-only weights") + if not isinstance(fraction, Fraction) or not 0 < fraction <= 1: + raise SurveyAllocationError("fraction must be a Fraction in (0, 1]") + if type(seed) is not int or not 0 <= seed < 2**64: + raise SurveyAllocationError("seed must be an unsigned 64-bit integer") + names = (source_column, domain_column, unit_count_column) + if any(not isinstance(name, str) or not name for name in names): + raise SurveyAllocationError("coverage column names must be nonempty strings") + if len(set(names)) != 3 or "household_id" in names: + raise SurveyAllocationError("coverage columns must be distinct from identities") + household = frame.table("household") + if not set(names).issubset(household): + raise SurveyAllocationError("household coverage columns are missing") + ids = household["household_id"].to_numpy() + sources = tuple(household[source_column]) + keys = tuple(household[domain_column]) + counts = tuple(household[unit_count_column]) + domains = tuple(domains) + weights = frame.weights_for("household") + full = allocate_domain_weights( + weights, + row_ids=ids, + source_channels=sources, + domain_keys=keys, + unit_counts=counts, + domains=domains, + ) + axis = hashlib.sha256() + for row, source, key, count, weight in zip( + ids, sources, keys, counts, weights.values, strict=True + ): + axis.update(_json([int(row), source, key, int(count), float(weight)])) + axis.update(b"\n") + declarations = [ + { + "key": domain.key, + "unit": domain.statistical_unit, + "period": domain.period, + "basis": domain.basis, + "shares": { + source: [share.numerator, share.denominator] + for source, share in domain.source_shares.items() + }, + } + for domain in sorted(domains, key=lambda domain: domain.key) + ] + selection = select_domain_households( + row_ids=ids, + source_channels=sources, + domain_keys=keys, + cells=tuple((estimate.domain, estimate.source) for estimate in full.estimates), + fraction=fraction, + seed=seed, + ) + selected_positions = [] + output_by_id = {} + cells = [] + for estimate, cell in zip(full.estimates, selection, strict=True): + chosen = np.asarray(cell.positions, dtype=np.int64) + selected = len(chosen) + probability = cell.inclusion_probability + if selected: + values = full.weights.values[chosen] + if estimate.source_share > 0 and not np.any(values > 0): + raise SurveyAllocationError( + "positive-share domain/source lacks sampled positive weight" + ) + with np.errstate(over="ignore", invalid="ignore"): + output = values * float(1 / probability) + sample_estimate = float( + np.sum(output * np.asarray([counts[p] for p in chosen])) + ) + if not np.isfinite(output).all() or not np.isfinite(sample_estimate): + raise SurveyAllocationError( + "sampled domain weight or estimate overflow" + ) + selected_positions.extend(chosen.tolist()) + output_by_id.update(zip(ids[chosen].tolist(), output.tolist(), strict=True)) + selected_ids = np.sort(ids[chosen]) + else: + selected, probability, sample_estimate = 0, None, 0.0 + selected_ids = () + cells.append( + SampledDomainCell( + estimate, + selected, + probability, + _id_digest(selected_ids), + sample_estimate, + ) + ) + selected_ids = ids[np.asarray(selected_positions, dtype=np.int64)] + if len(selected_ids) == len(ids): + sampled = frame + else: + membership = frame.schema.membership_column("household") + sampled = frame.select(frame.person[membership].isin(selected_ids).to_numpy()) + realized = sampled.table("household")["household_id"].to_numpy() + if set(realized) != set(selected_ids): + raise SurveyAllocationError("whole-household sample differs from selection") + result = sampled.with_weights( + "household", + Weights( + np.asarray([output_by_id[int(row)] for row in realized]), + WeightKind.IMPORTANCE, + ), + mass=MassChange( + factor=None, + reason=f"{PROTOCOL}: domain source shares and inverse household inclusion", + ), + ) + return DomainSampleAllocation( + result, + tuple(cells), + fraction, + seed, + axis.hexdigest(), + hashlib.sha256(_json(declarations)).hexdigest(), + np.__version__, + names, + ) diff --git a/packages/microcosm-build/src/microcosm/build/table_identity.py b/packages/microcosm-build/src/microcosm/build/table_identity.py new file mode 100644 index 000000000..7dcc9725f --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/table_identity.py @@ -0,0 +1,365 @@ +"""Native table identity without constructing a population Frame. + +The canonical_scalar_v1 values codec is factored unchanged from the US late +producer runtime. Its domain labels and optional string normalization remain +stable for existing receipts. native_table_identity additionally binds a closed +native dtype/schema descriptor, including pandas string storage and NA kind. +""" + +from __future__ import annotations + +import hashlib +import json +import struct +from collections.abc import Sequence +from typing import Any + +import numpy as np +import pandas as pd + +from microcosm.build.serialization_dtypes import canonicalize_table_string_dtypes + +TABLE_VALUES_DIGEST_CODEC = "canonical_scalar_v1" +_TABLE_DIGEST_CHUNK_ROWS = 65_536 + + +def _digest_part( + digest, + *, + domain: str, + payload: bytes | bytearray | memoryview | np.ndarray, +) -> None: + """Append one length-framed, domain-separated byte field to a digest.""" + + domain_bytes = domain.encode("utf-8") + payload_view = memoryview(payload) + if payload_view.format != "B" or payload_view.ndim != 1: + payload_view = payload_view.cast("B") + digest.update(struct.pack(" np.ndarray: + """Return a contiguous, explicitly little-endian numeric byte source.""" + + array = np.asarray(values) + array = array.astype(array.dtype.newbyteorder("<"), copy=False) + return np.ascontiguousarray(array) + + +def _scalar_bytes(value: object) -> bytes: + """Encode one supported object scalar without lossy intermediary hashes.""" + + missing = pd.isna(value) + if isinstance(missing, (bool, np.bool_)) and bool(missing): + return b"null" + if isinstance(value, (bool, np.bool_)): + return b"bool\x01" if bool(value) else b"bool\x00" + if isinstance(value, (int, np.integer)): + return b"integer\x00" + str(int(value)).encode("ascii") + if isinstance(value, (float, np.floating)): + if isinstance(value, np.floating) and value.dtype.itemsize > 8: + raise TypeError( + "US late-producer content digest does not support object " + f"floating scalar {value.dtype!s}." + ) + return b"float64\x00" + struct.pack(" 16: + raise TypeError( + "US late-producer content digest does not support object " + f"complex scalar {value.dtype!s}." + ) + numeric = complex(value) + return b"complex128\x00" + struct.pack(" None: + """Stream framed string or object scalars in bounded-memory chunks.""" + + for chunk_index, start in enumerate( + range(0, len(values), _TABLE_DIGEST_CHUNK_ROWS) + ): + stop = min(start + _TABLE_DIGEST_CHUNK_ROWS, len(values)) + lengths = np.zeros(stop - start, dtype=" None: + """Hash one ordered logical Series with explicit dtype and null domains.""" + + dtype = series.dtype + missing = series.isna().to_numpy(dtype=bool) + _digest_part( + digest, + domain=f"{domain}/dtype", + payload=str(dtype).encode("utf-8"), + ) + _digest_part( + digest, + domain=f"{domain}/row_count", + payload=struct.pack(" str: + """Hash ordered table scalars directly with typed, null-aware framing.""" + + values = ( + canonicalize_table_string_dtypes( + table, + boundary="late-producer content digest", + table_name="declared_surface", + ) + if normalize_strings + else table + ) + if isinstance(values.index, pd.MultiIndex): + index_levels = [ + pd.Series(values.index.get_level_values(level), copy=False) + for level in range(values.index.nlevels) + ] + else: + index_levels = [pd.Series(values.index, copy=False)] + header = { + "codec": TABLE_VALUES_DIGEST_CODEC, + "columns": [str(column) for column in values.columns], + "dtypes": [ + str(values.iloc[:, index].dtype) for index in range(values.shape[1]) + ], + "index_type": type(values.index).__name__, + "index_dtype": str(values.index.dtype), + "index_level_dtypes": [str(level.dtype) for level in index_levels], + "index_names": [ + None if name is None else str(name) for name in values.index.names + ], + } + digest = hashlib.sha256() + _digest_part( + digest, + domain="late_table_digest_codec", + payload=TABLE_VALUES_DIGEST_CODEC.encode("ascii"), + ) + _digest_part( + digest, + domain="late_table_header_json", + payload=json.dumps( + header, + sort_keys=True, + separators=(",", ":"), + allow_nan=False, + ).encode("utf-8"), + ) + for level_index, level in enumerate(index_levels): + _digest_series_values( + digest, + level, + domain=f"index_level/{level_index}", + ) + for column_index in range(values.shape[1]): + _digest_series_values( + digest, + values.iloc[:, column_index], + domain=f"column/{column_index}", + ) + return digest.hexdigest() + + +def _native_dtype(dtype: object) -> dict[str, str]: + if isinstance(dtype, pd.StringDtype): + return { + "kind": "pandas_string", + "storage": dtype.storage, + "na_value": "pd.NA" if dtype.na_value is pd.NA else "nan", + } + if isinstance(dtype, np.dtype) and dtype.kind in "biufO": + return {"kind": "numpy", "dtype": dtype.str} + if isinstance( + dtype, + ( + pd.BooleanDtype, + pd.Int8Dtype, + pd.Int16Dtype, + pd.Int32Dtype, + pd.Int64Dtype, + pd.UInt8Dtype, + pd.UInt16Dtype, + pd.UInt32Dtype, + pd.UInt64Dtype, + pd.Float32Dtype, + pd.Float64Dtype, + ), + ): + return {"kind": "pandas_nullable", "dtype": str(dtype)} + raise TypeError(f"Unsupported native table identity dtype: {dtype!r}.") + + +def native_table_identity(table: pd.DataFrame) -> dict[str, Any]: + """Bind native values, order, index and a closed exact dtype schema. + + This narrow identity supports numeric, boolean and string donor tables, + including nullable numeric columns. Categorical/temporal/custom extension + dtypes and MultiIndex axes require a separately declared contract. Nulls + retain their positions; individual floating NaN payload bits are canonical. + No normalization or coercion is performed on the caller's table. + """ + if not isinstance(table, pd.DataFrame): + raise TypeError("Native table identity requires a pandas DataFrame.") + if isinstance(table.index, pd.MultiIndex) or isinstance( + table.columns, pd.MultiIndex + ): + raise TypeError("Native table identity does not support MultiIndex axes.") + if table.columns.has_duplicates or not all( + isinstance(x, str) and x for x in table.columns + ): + raise TypeError( + "Native table identity requires unique nonempty string columns." + ) + for axis in (table.index, table.columns): + if axis.name is not None and not isinstance(axis.name, str): + raise TypeError( + "Native table identity requires string or absent axis names." + ) + series = [ + pd.Series(table.index), + *(table.iloc[:, i] for i in range(table.shape[1])), + ] + for values in series: + if values.dtype == object and not all( + isinstance(x, str) for x in values.dropna() + ): + raise TypeError("Native object dtype may contain only strings or nulls.") + identity = { + "schema": "microcosm.native_table_identity.v1", + "values_sha256": table_values_sha256(table, normalize_strings=False), + "index": { + "type": type(table.index).__name__, + "name": table.index.name, + "dtype": _native_dtype(table.index.dtype), + }, + "column_axis": { + "type": type(table.columns).__name__, + "name": table.columns.name, + "dtype": _native_dtype(table.columns.dtype), + }, + "columns": [ + {"name": name, "dtype": _native_dtype(table[name].dtype)} + for name in table.columns + ], + } + identity["sha256"] = hashlib.sha256( + json.dumps( + identity, sort_keys=True, separators=(",", ":"), allow_nan=False + ).encode() + ).hexdigest() + return identity diff --git a/packages/microcosm-build/src/microcosm/build/target_materialization.py b/packages/microcosm-build/src/microcosm/build/target_materialization.py index 2629e4507..bc5f899e6 100644 --- a/packages/microcosm-build/src/microcosm/build/target_materialization.py +++ b/packages/microcosm-build/src/microcosm/build/target_materialization.py @@ -716,6 +716,10 @@ def _expression(adapter: Any, entity: str, expression: str) -> np.ndarray: total: np.ndarray | None = None for part in parts: values = _column(adapter, entity, part) + # NumPy adds bool arrays with logical-OR semantics. A sum of indicator + # terms counts their True values, so widen only boolean operands. + if values.dtype.kind == "b": + values = values.astype(np.int64) total = values.copy() if total is None else total + values if total is None: raise ValueError(f"unsupported value_expression {expression!r}") diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/__init__.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/__init__.py index 187310e40..05a835787 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/__init__.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/__init__.py @@ -1,552 +1,564 @@ -"""UK build helpers for Microcosm-owned local-geography artifacts.""" +"""UK build helpers, resolved from their defining modules on explicit access. -from microcosm.build.uk_runtime.age_tail import ( - UK_AGE_TAIL_BAND_POPULATIONS_RESOURCE, - UK_AGE_TAIL_BANDS, - UK_AGE_TAIL_DECLARED_SEEDS, - UK_AGE_TOP_CODE, - UKAgeTailStageTransform, - disaggregate_uk_age_top_code, - load_uk_age_tail_band_populations, -) -from microcosm.build.uk_runtime.battery_bindings import ( - UK_GATE_REGISTRY, - UKGateBinding, -) -from microcosm.build.uk_runtime.calibration_run import ( - load_bound_spine_sidecar, - runtime_provenance, - spine_provenance_from_sidecar, -) -from microcosm.build.uk_runtime.cgt_calibration import ( - UK_CGT_ANNUAL_EXEMPT_AMOUNTS, - UK_CGT_GAINS_AMOUNT_COLUMN, - UK_CGT_SOURCE_COLUMN, - UK_CGT_TAXPAYER_COUNT_COLUMN, - UKCGTTargetMaterialization, - materialize_uk_cgt_calibration_frame, - uk_cgt_annual_exempt_amount, -) -from microcosm.build.uk_runtime.content_identity import ( - uk_frame_content_identity, -) -from microcosm.build.uk_runtime.diagnostics import ( - UK_DIAGNOSTICS_SCHEMA_VERSION, - UK_TARGET_GEOGRAPHY_LEVELS, - uk_calibration_diagnostics_payload, - uk_fit_by_family, - uk_support_limited_misses, - uk_weakest_areas_by_fit, - uk_weakest_families, - uk_weight_summary, - uk_zero_weight_strata, - write_uk_calibration_diagnostics, -) -from microcosm.build.uk_runtime.firm_generation import ( - EMPLOYMENT_BANDS, - HMRC_BAND_COLUMNS, - INPUT_FILES, - VAT_LIABILITY_BANDS, - VINTAGES, - AxiomVATRuleEvaluator, - UKFirmCalibrationResult, - UKFirmGenerationConfig, - UKFirmGenerationResult, - UKFirmSourceData, - UKFirmTargetLayout, - UKFirmValidationReport, - UKFirmVATRuleEvaluator, - assign_employment, - assign_vat_flags, - build_firm_target_matrix, - employment_band_name, - generate_base_firms, - generate_input_values, - generate_uk_firm_population, - hmrc_band_name, - map_to_hmrc_band_indices, - optimize_firm_weights, - read_uk_firm_source_data, - solve_firm_weights, - target_diagnostics, - uk_firm_source_data_from_frames, - uk_firm_source_data_from_ledger_facts, - validate_uk_firm_population, - write_uk_firm_population, -) -from microcosm.build.uk_runtime.fiscal_targets import ( - UK_CGT_REQUIRED_COLUMNS, - UK_CGT_TARGET_COVERAGE_REQUIREMENTS, - UK_CGT_TARGET_SPECS, - UK_FISCAL_TARGET_REGISTRY, -) -from microcosm.build.uk_runtime.frs_council_tax import ( - FRS_COUNCIL_TAX_OUTPUT_COLUMNS, - UKFRSCouncilTaxStageTransform, - add_frs_council_tax, - derive_council_tax, -) -from microcosm.build.uk_runtime.frs_disability import ( - FRS_DISABILITY_OUTPUT_COLUMNS, - UK_INTERNAL_DISABILITY_REPORTED_COLUMNS, - UKDWPDisabilityCategoryRates, - UKDWPDisabilityFlagRates, - UKFRSDisabilityStageTransform, - add_frs_disability, - derive_frs_disability, - uk_dwp_disability_category_rates, - uk_dwp_disability_flag_rates, -) -from microcosm.build.uk_runtime.frs_education import ( - EDUCQUAL_MAP, - FRS_EDUCATION_OUTPUT_COLUMNS, - UKFRSEducationStageTransform, - add_frs_education, - derive_current_education, - derive_frs_education, -) -from microcosm.build.uk_runtime.frs_education_grants import ( - FRS_EDUCATION_GRANT_OUTPUT_COLUMNS, - FRS_EDUCATION_GRANT_REWRITES, - UK_EDUCATION_GRANT_CAPACITY_PREDICTORS, - UKDSAPolicy, - UKFRSEducationGrantSplitStageTransform, - add_frs_education_grant_split, - allocate_reported_education_grants, - disabled_students_allowance_capacity, - uk_dsa_policy, -) -from microcosm.build.uk_runtime.frs_employment import ( - FRS_EMPLOYMENT_OUTPUT_COLUMNS, - UKFRSEmploymentStageTransform, - add_frs_employment, - derive_frs_employment, -) -from microcosm.build.uk_runtime.frs_hmrc_leaves import ( - FRS_HMRC_INCPBEN_COLUMN, - FRS_HMRC_OSSBEN_IDENTIFIABLE_SUBSET_COLUMN, - FRS_HMRC_PAY_COLUMN, - FRS_HMRC_RETAINED_LEAF_COLUMNS, - FRS_HMRC_RETAINED_LEAVES_STAGE_NAME, - FRS_HMRC_SRP_REGULAR_CODE5_COLUMN, - FRS_HMRC_UBISJA_COLUMN, - UKFRSHMRCRetainedLeavesResult, - UKFRSHMRCRetainedLeavesStageTransform, - retain_uk_frs_hmrc_leaves, -) -from microcosm.build.uk_runtime.frs_legacy_proxies import ( - FRS_LEGACY_PROXY_OUTPUT_COLUMNS, - UK_LEGACY_PROXY_PREDICTORS, - UKFRSLegacyProxiesStageTransform, - UKLegacyJSAPolicy, - derive_frs_legacy_proxies, - uk_legacy_jsa_policy, -) -from microcosm.build.uk_runtime.frs_release import ( - UK_YEAR_RULES, - UKFRSRelease, - load_uk_frs_release, - resolve_uk_year_rule, -) -from microcosm.build.uk_runtime.geography_ladder import ( - GEOGRAPHY_LADDER_ARTIFACT_SHA256_ATTR, - GEOGRAPHY_LADDER_VINTAGES_ATTR, - UK_ENGLAND_WALES_REGION_CODES, - UK_GEOGRAPHY_LADDER_COLUMNS, - UK_LONDON_REGION_CODE, - UK_OA_LADDER_DERIVED_LAYERS, - UK_OA_LADDER_KIND, - UK_OA_LADDER_SCHEMA_VERSION, - UkOaLadder, - assign_uk_geography_ladder, - expected_uk_ladder_area_support, - load_uk_oa_ladder, - uk_geography_ladder_assignment_summary, - uk_geography_ladder_gate, - uk_region_mix, -) -from microcosm.build.uk_runtime.geography_sources import ( - ENGLAND_LAD_REGION_URL, - ENGLAND_WALES_OA2021_COUNT, - EW_OA_CONSTITUENCY_URL, - EW_OA_HIERARCHY_URL, - EW_OA_HOUSEHOLDS_URL, - EW_OA_LAD23_URL, - EW_OA_POPULATION_URL, - EW_OA_WARD_URL, - LAD23_ITL_URL, - NI_DZ2021_COUNT, - NI_DZ_GEOJSON_ZIP_URL, - NI_DZ_HOUSEHOLDS_CSV_URL, - NI_DZ_LOOKUP_SHEET, - NI_DZ_PARLCON24_LOOKUP_XLSX_URL, - NI_DZ_POPULATION_CSV_URL, - NI_PARLCON24_COUNT, - SCOTLAND_CENSUS_INDEX_ZIP_URL, - SCOTLAND_OA2022_COUNT, - SCOTLAND_OA_CONSTITUENCY_URL, - SCOTLAND_OA_DZ_IZ_URL, - SCOTLAND_OA_LAU_ITL_URL, - SCOTLAND_OA_POPULATION_URL, - build_complete_uk_geography_crosswalk, - build_england_wales_crosswalk, - build_great_britain_crosswalk, - build_northern_ireland_crosswalk, - build_official_uk_geography_crosswalk, - build_scotland_crosswalk, - load_england_lad_region_lookup, - load_england_wales_oa_constituencies, - load_england_wales_oa_hierarchy, - load_england_wales_oa_households, - load_england_wales_oa_population, - load_england_wales_oa_ward_lookup, - load_ew_oa_lad23_lookup, - load_lad_itl_lookup, - load_ni_dz_hierarchy, - load_ni_dz_households, - load_ni_dz_parlcon24_lookup, - load_ni_dz_population, - load_ni_dz_ward_lookup, - load_scotland_oa_constituencies, - load_scotland_oa_dz_iz_lookup, - load_scotland_oa_households, - load_scotland_oa_lau_lookup, - load_scotland_oa_population, - load_scotland_oa_ward_lookup, - update_england_wales_lad_codes, - write_geography_crosswalk, -) -from microcosm.build.uk_runtime.hmrc_calibration import ( - DEFAULT_HMRC_CALIBRATION_EPOCHS, - DEFAULT_HMRC_CALIBRATION_LEARNING_RATE, - DEFAULT_HMRC_MAX_ABS_RELATIVE_ERROR, - DEFAULT_HMRC_MAX_WEIGHT_RATIO, - HMRC_ASSESSABLE_INCOME_COLUMN, - HMRC_TAXABLE_SAVINGS_INTEREST_COLUMN, - HMRC_TAXPAYER_COLUMN, - UKHMRCIncomeCalibration, - UKHMRCTargetMaterialization, - calibrate_uk_hmrc_income, - materialize_uk_hmrc_calibration_frame, -) -from microcosm.build.uk_runtime.hmrc_income import ( - HMRC_SPI_BUILD_PERIOD, - HMRC_SPI_COLLATED_ODS_URL, - HMRC_SPI_INCOME_COMPONENTS, - HMRC_SPI_PUBLICATION_URL, - HMRC_SPI_SOURCE_VINTAGE, - HMRC_SPI_TARGET_RECORD_COUNT, - HMRCIncomeBandTargetRecord, - HMRCIncomeSourceProvenance, - HMRCIncomeTargetSet, - materialize_hmrc_spi_income_band_targets, - verify_hmrc_spi_collated_ods, -) -from microcosm.build.uk_runtime.hmrc_replay import ( - CANONICAL_HMRC_FACT_FENCES, - FULL_FRS_TI_BAND_FENCE_ID, - HMRCFactFence, - HMRCReplayDiagnosticAggregate, - HMRCReplayFact, - HMRCReplayReport, - build_conservative_hmrc_replay_report, - classify_hmrc_replay_targets, - write_hmrc_replay_report, -) -from microcosm.build.uk_runtime.hmrc_source_contract import ( - HMRC_DISTRIBUTIONAL_INPUTS, - UK_HMRC_INCOME_SOURCE_STAGES_RESOURCE, - assert_uk_hmrc_income_source_contract_current, -) -from microcosm.build.uk_runtime.ladder_targets import ( - constituency_household_targets, - ladder_target_provenance, - ladder_vs_chronicle_household_dispersion, - local_authority_household_targets, -) -from microcosm.build.uk_runtime.ledger_targets import ( - UK_CENSUS_HOUSEHOLDS_TARGET_ID, - UK_CROSS_GRAIN_BRIDGES, - UK_CROSS_GRAIN_GRAIN_PRECEDENCE, - UK_CROSS_GRAIN_RULE, - UKFrameTargetAdapter, - UKLedgerTargetCompilation, - apply_uk_cross_grain_reconciliation, - compile_uk_local_target_registry, - compile_uk_target_registry, - load_uk_local_area_crosswalk, - materialize_uk_ledger_targets, - uk_census_household_uprating, - uk_ledger_households_total, - uk_local_target_surface, -) -from microcosm.build.uk_runtime.local_doctrine import ( - UK_LOCAL_CLONE_COUNT, - UK_LOCAL_MAX_WEIGHT_RATIO, - UK_LOCAL_SOLVE_DOCTRINE, - UK_LOCAL_SOLVE_EPOCHS, - UK_LOCAL_TARGET_LOSS_CAP, - UK_LOCAL_TARGET_WEIGHT_RULE, - UKLocalSolveDoctrine, - uk_local_doctrine_with_overrides, - uk_local_target_loss_weights, -) -from microcosm.build.uk_runtime.local_geography import ( - align_area_targets, -) -from microcosm.build.uk_runtime.local_rowwise import ( - UK_LOCAL_BINDING_ADJUDICATION_REGISTER_RESOURCE, - UK_LOCAL_HOLDOUT_FOLDS, - UK_LOCAL_HOLDOUT_SEED, - UKRowwiseDoctrineSolve, - UKRowwiseLocalMatrix, - UKRowwiseNationalRows, - build_uk_rowwise_local_matrix, - build_uk_rowwise_local_surface_matrix, - past_cap_census, - require_adjudicated_uk_local_binding, - rotated_uk_local_holdout, - rowwise_area_support_summary, - rowwise_calibration_mass_reason, - solve_uk_rowwise_weights_under_doctrine, - uk_area_support_summary, - uk_ladder_area_support_summary, -) -from microcosm.build.uk_runtime.local_target_census import ( - CENSUS_KIND, - CENSUS_RESOURCE, - CENSUS_SCHEMA_VERSION, - METRIC_STATUS_BOUND_IN_CODE, - SOURCE_STATUS_DOCUMENTED_UNPINNED, - assert_uk_local_target_census_current, - build_uk_local_target_census, - committed_uk_local_target_census_path, - load_uk_local_target_census, - write_uk_local_target_census, -) -from microcosm.build.uk_runtime.local_targets import ( - AGE_BANDS, - AREA_TYPE_TO_LEDGER_GEOGRAPHY_LEVEL, - AREA_TYPES, - COUNTRY_TO_REGION, - INCOME_VARIABLES, - LA_EXTRA_METRICS, - area_groups_from_codes, - compute_household_metrics, - metric_names, - metric_names_from_target_profile, - metric_tables_by_area_group, -) -from microcosm.build.uk_runtime.national_calibration import ( - CalibrationFrameAdapter, - drop_injected_measure_inputs, - inject_measure_inputs, - prepare_uk_target_frame, -) -from microcosm.build.uk_runtime.national_doctrine import ( - UK_NATIONAL_L0_LAMBDA, - UK_NATIONAL_LEARNING_RATE, - UK_NATIONAL_MASS_RULE, - UK_NATIONAL_MAX_WEIGHT_RATIO, - UK_NATIONAL_SEED, - UK_NATIONAL_SOLVE_DOCTRINE, - UK_NATIONAL_SOLVE_EPOCHS, - UK_NATIONAL_TARGET_LOSS_CAP, - UK_NATIONAL_TARGET_WEIGHT_RULE, - UKNationalSolveDoctrine, - uk_doctrine_with_overrides, - uk_national_target_loss_weights, -) -from microcosm.build.uk_runtime.national_frame import ( - UK_NATIONAL_SCHEMA, - UKNationalStage, - UKStagingProvenance, - load_uk_national_frame, - uk_household_weight_kind, - uk_national_frame, - uk_time_period, - validate_uk_national_frame, - write_uk_national_frame, -) -from microcosm.build.uk_runtime.national_sampling import ( - sample_uk_spine_frame, - uk_spine_source_family_units, -) -from microcosm.build.uk_runtime.oa_ladder_sources import ( - LADDER_OA_COLUMNS, - assemble_uk_oa_ladder, - concat_uk_ladder_frames, - join_uk_oa_ladder_layers, -) -from microcosm.build.uk_runtime.parity_reference import ( - EFRS_PARITY_KNOWN_GAPS_RESOURCE, - EFRS_PARITY_REFERENCE_RESOURCE, - EfrsParityKnownGap, - EfrsParityReference, - EfrsParitySource, - load_efrs_parity_known_gaps, - load_efrs_parity_reference, -) -from microcosm.build.uk_runtime.release_identity import ( - UK_DENSE_RELEASE_ID, - UK_RELEASE_TIER_CPS_TRANSFER, - UK_RELEASE_TIER_FRS, - UK_RELEASE_TIERS, - UKReleaseIdentity, - apply_uk_release_identity, - format_uk_release_id, - validate_uk_release_tier, -) -from microcosm.build.uk_runtime.release_input_coverage import ( - RESTORED_REFERENCE_EFRS_REQUIRED_INPUTS, - UK_LOADER_INPUT_ALIASES, - UK_RELEASE_INPUT_COVERAGE_RESOURCE, - PolicyEngineUKCoverageEngine, - UKEffectiveMassCoveragePolicy, - UKReleaseInputColumn, - UKReleaseInputCoverageManifest, - assert_uk_release_input_coverage_build_stages, - assert_uk_release_input_coverage_manifest_current, - load_uk_release_input_coverage_manifest, - uk_release_input_coverage_gate, - uk_release_input_coverage_required_columns, - uk_release_input_coverage_reviewed_exclusions, -) -from microcosm.build.uk_runtime.rowwise_dataset import ( - ARTIFACT_CLONE_INDEX_COLUMN, - BENUNIT_ID_COLUMNS, - HOUSEHOLD_ID_COLUMNS, - MASS_CONSERVATION_RELATIVE_TOLERANCE, - PERSON_ID_COLUMNS, - POOL_SOURCE_LINEAGE_COLUMN, - UK_SINGLE_YEAR_TABLES, - UKLadderRowwiseDatasetResult, - UKRowwiseDatasetResult, - apply_uk_source_lineage_modulus, - clone_uk_dataset_tables_with_ladder_geography, - clone_uk_dataset_tables_with_rowwise_geography, - clone_uk_dataset_with_ladder_geography, - clone_uk_dataset_with_rowwise_geography, - ladder_clone_index_column, - load_uk_rowwise_dataset, - read_uk_single_year_weight_metadata, - validate_uk_ladder_rowwise_dataset_tables, - validate_uk_rowwise_dataset_tables, - write_uk_rowwise_dataset, -) -from microcosm.build.uk_runtime.rowwise_geography import ( - AREA_TYPE_TO_CROSSWALK_COLUMN, - CROSSWALK_COLUMNS, - FRS_REGION_TO_COUNTRY, - FRS_REGION_TO_REGION_CODE, - ROWWISE_GEOGRAPHY_COLUMNS, - RowwiseGeographyAssignment, - assign_household_geography, - clone_entity_frame, - expected_uk_rowwise_area_support, - geography_coverage_summary, - id_multiplier_for_values, - prepare_geography_crosswalk, - validate_geography_coverage, -) -from microcosm.build.uk_runtime.size_evaluation import ( - PRE_REGISTERED_OUTCOMES_V1, - RowwiseRun, - area_support_tables, - dense_reference_deltas, - fit_tables, - footprint, - frozen_vs_recomputed, - gate_table, - load_run, - paired_targets, - run_acceptance, - summarize, - weight_tables, -) -from microcosm.build.uk_runtime.spi_income import ( - SPI_DONOR_DOI, - SPI_DONOR_FILENAME, - SPI_DONOR_SHA256, - SPI_DONOR_SIZE_BYTES, - SPI_DONOR_UKDS_STUDY, - SPI_DONOR_VINTAGE, - SPI_STAGE2_REVIEWED_ABSENT_OUTPUTS, - UKSPIIncomeImputationResult, - assert_frs_hmrc_auxiliary_crosswalk_available, - derive_hmrc_income_auxiliaries, - impute_uk_spi_income_support, - verify_spi_donor_identity, -) -from microcosm.build.uk_runtime.spi_spine import ( - EMPLOYER_PENSION_CONTRIBUTIONS_COLUMN, - UK_FRS_HMRC_SPINE_LEAF_OUTPUT_COLUMNS, - UK_FRS_HMRC_SPINE_LEAVES_STAGE_NAME, - UK_HMRC_SPI_INCOME_SPINE_STAGE_NAME, - UK_HMRC_SPI_SPINE_REPLAY_REPORT_KIND, - UK_SPI_INCOME_SPINE_NONNEGATIVE_OUTPUT_COLUMNS, - UK_SPI_INCOME_SPINE_OUTPUT_COLUMNS, - UK_SPI_INCOME_SPINE_REWRITE_COLUMNS, - UK_SPI_SUPPORT_CHANNEL_OUTPUT_COLUMNS, - UKFRSHMRCSpineLeavesResult, - UKFRSHMRCSpineLeavesStageTransform, - UKSPIIncomeSpineResult, - UKSPIIncomeSpineStageTransform, - UKSPISupportChannelStageTransform, -) -from microcosm.build.uk_runtime.spi_support import ( - BASE_FRS_SUPPORT_CHANNEL, - DEFAULT_SPI_PRIOR_MASS_SHARE, - DEFAULT_SPI_SUPPORT_HOUSEHOLDS, - FRS_ONLY_SPI_FILL_INCOME_PREDICTOR_COLUMNS, - FRS_ONLY_SPI_FILL_PERSON_COLUMNS, - FRS_ONLY_SPI_FILL_PREDICTOR_COLUMNS, - HOUSEHOLD_IS_SPI_SYNTHETIC_COLUMN, - SPI_INCOME_COMPONENT_COLUMNS, - SPI_INCOME_IMPUTATION_COLUMNS, - SPI_PRIOR_MASS_CHANGE_REASON, - SPI_REPLACEMENT_STRATA_COLUMNS, - SPI_SYNTHETIC_SUPPORT_CHANNEL, - UK_SPI_SUPPORT_STAGE_NAME, - UKSPISupportResult, - build_uk_spi_support_channel, - create_uk_spi_support_tables, - fill_support_channel_from_source, - replace_uk_spi_support_tables, - support_channel_column, - support_clone_index_column, - support_source_id_column, -) -from microcosm.build.uk_runtime.stage_checkpoints import ( - UK_FRAME_METADATA_KEY, - load_uk_stage_checkpoint, - load_uk_stage_predecessor, - uk_stage_metadata, -) -from microcosm.build.uk_runtime.terminal_gates import ( - UK_DEFAULT_ZERO_WEIGHT_STRATA, - UK_MAX_TARGET_ABS_RELATIVE_ERROR, - UKZeroWeightStratumDeclaration, - uk_degenerate_release_surface_gate, - uk_export_surface_gate, - uk_target_fit_gate, - uk_target_surface_gate, - uk_weight_ess_gate, - uk_weight_ratio_gate, - uk_zero_weight_strata_gate, -) -from microcosm.build.uk_runtime.weighted_integrity import ( - UKInputMassParityPolicy, - UKInputMassReference, - UKQRFTailConcentrationPolicy, - load_uk_input_mass_reference, - load_uk_local_area_support_exclusion_register, - load_uk_reviewed_exclusion_register, - uk_input_mass_parity_gate, - uk_input_mass_totals, - uk_qrf_tail_concentration_columns, - uk_qrf_tail_concentration_gate, -) +Direct helper imports do not initialize unrelated calibration or source loaders. +The public export roster is unchanged; requesting every export with ``import *`` +still imports every defining module. Use ``dir`` or ``__all__`` for discovery: +``vars`` contains only exports already resolved in this process. +""" + +from importlib import import_module as _import_module +from types import MappingProxyType as _MappingProxyType +from typing import TYPE_CHECKING as _TYPE_CHECKING +from typing import Any as _Any + +if _TYPE_CHECKING: + from microcosm.build.uk_runtime.age_tail import ( + UK_AGE_TAIL_BAND_POPULATIONS_RESOURCE, + UK_AGE_TAIL_BANDS, + UK_AGE_TAIL_DECLARED_SEEDS, + UK_AGE_TOP_CODE, + UKAgeTailStageTransform, + disaggregate_uk_age_top_code, + load_uk_age_tail_band_populations, + ) + from microcosm.build.uk_runtime.battery_bindings import ( + UK_GATE_REGISTRY, + UKGateBinding, + ) + from microcosm.build.uk_runtime.calibration_run import ( + load_bound_spine_sidecar, + runtime_provenance, + spine_provenance_from_sidecar, + ) + from microcosm.build.uk_runtime.cgt_calibration import ( + UK_CGT_ANNUAL_EXEMPT_AMOUNTS, + UK_CGT_GAINS_AMOUNT_COLUMN, + UK_CGT_SOURCE_COLUMN, + UK_CGT_TAXPAYER_COUNT_COLUMN, + UKCGTTargetMaterialization, + materialize_uk_cgt_calibration_frame, + uk_cgt_annual_exempt_amount, + ) + from microcosm.build.uk_runtime.content_identity import ( + uk_frame_content_identity, + ) + from microcosm.build.uk_runtime.diagnostics import ( + UK_DIAGNOSTICS_SCHEMA_VERSION, + UK_TARGET_GEOGRAPHY_LEVELS, + uk_calibration_diagnostics_payload, + uk_fit_by_family, + uk_support_limited_misses, + uk_weakest_areas_by_fit, + uk_weakest_families, + uk_weight_summary, + uk_zero_weight_strata, + write_uk_calibration_diagnostics, + ) + from microcosm.build.uk_runtime.firm_generation import ( + EMPLOYMENT_BANDS, + HMRC_BAND_COLUMNS, + INPUT_FILES, + VAT_LIABILITY_BANDS, + VINTAGES, + AxiomVATRuleEvaluator, + UKFirmCalibrationResult, + UKFirmGenerationConfig, + UKFirmGenerationResult, + UKFirmSourceData, + UKFirmTargetLayout, + UKFirmValidationReport, + UKFirmVATRuleEvaluator, + assign_employment, + assign_vat_flags, + build_firm_target_matrix, + employment_band_name, + generate_base_firms, + generate_input_values, + generate_uk_firm_population, + hmrc_band_name, + map_to_hmrc_band_indices, + optimize_firm_weights, + read_uk_firm_source_data, + solve_firm_weights, + target_diagnostics, + uk_firm_source_data_from_frames, + uk_firm_source_data_from_ledger_facts, + validate_uk_firm_population, + write_uk_firm_population, + ) + from microcosm.build.uk_runtime.fiscal_targets import ( + UK_CGT_REQUIRED_COLUMNS, + UK_CGT_TARGET_COVERAGE_REQUIREMENTS, + UK_CGT_TARGET_SPECS, + UK_FISCAL_TARGET_REGISTRY, + ) + from microcosm.build.uk_runtime.frs_council_tax import ( + FRS_COUNCIL_TAX_OUTPUT_COLUMNS, + UKFRSCouncilTaxStageTransform, + add_frs_council_tax, + derive_council_tax, + ) + from microcosm.build.uk_runtime.frs_disability import ( + FRS_DISABILITY_OUTPUT_COLUMNS, + UK_INTERNAL_DISABILITY_REPORTED_COLUMNS, + UKDWPDisabilityCategoryRates, + UKDWPDisabilityFlagRates, + UKFRSDisabilityStageTransform, + add_frs_disability, + derive_frs_disability, + uk_dwp_disability_category_rates, + uk_dwp_disability_flag_rates, + ) + from microcosm.build.uk_runtime.frs_education import ( + EDUCQUAL_MAP, + FRS_EDUCATION_OUTPUT_COLUMNS, + UKFRSEducationStageTransform, + add_frs_education, + derive_current_education, + derive_frs_education, + ) + from microcosm.build.uk_runtime.frs_education_grants import ( + FRS_EDUCATION_GRANT_OUTPUT_COLUMNS, + FRS_EDUCATION_GRANT_REWRITES, + UK_EDUCATION_GRANT_CAPACITY_PREDICTORS, + UKDSAPolicy, + UKFRSEducationGrantSplitStageTransform, + add_frs_education_grant_split, + allocate_reported_education_grants, + disabled_students_allowance_capacity, + uk_dsa_policy, + ) + from microcosm.build.uk_runtime.frs_employment import ( + FRS_EMPLOYMENT_OUTPUT_COLUMNS, + UKFRSEmploymentStageTransform, + add_frs_employment, + derive_frs_employment, + ) + from microcosm.build.uk_runtime.frs_hmrc_leaves import ( + FRS_HMRC_INCPBEN_COLUMN, + FRS_HMRC_OSSBEN_IDENTIFIABLE_SUBSET_COLUMN, + FRS_HMRC_PAY_COLUMN, + FRS_HMRC_RETAINED_LEAF_COLUMNS, + FRS_HMRC_RETAINED_LEAVES_STAGE_NAME, + FRS_HMRC_SRP_REGULAR_CODE5_COLUMN, + FRS_HMRC_UBISJA_COLUMN, + UKFRSHMRCRetainedLeavesResult, + UKFRSHMRCRetainedLeavesStageTransform, + retain_uk_frs_hmrc_leaves, + ) + from microcosm.build.uk_runtime.frs_legacy_proxies import ( + FRS_LEGACY_PROXY_OUTPUT_COLUMNS, + UK_LEGACY_PROXY_PREDICTORS, + UKFRSLegacyProxiesStageTransform, + UKLegacyJSAPolicy, + derive_frs_legacy_proxies, + uk_legacy_jsa_policy, + ) + from microcosm.build.uk_runtime.frs_release import ( + UK_YEAR_RULES, + UKFRSRelease, + load_uk_frs_release, + resolve_uk_year_rule, + ) + from microcosm.build.uk_runtime.geography_ladder import ( + GEOGRAPHY_LADDER_ARTIFACT_SHA256_ATTR, + GEOGRAPHY_LADDER_VINTAGES_ATTR, + UK_ENGLAND_WALES_REGION_CODES, + UK_GEOGRAPHY_LADDER_COLUMNS, + UK_LONDON_REGION_CODE, + UK_OA_LADDER_DERIVED_LAYERS, + UK_OA_LADDER_KIND, + UK_OA_LADDER_SCHEMA_VERSION, + UkOaLadder, + assign_uk_geography_ladder, + expected_uk_ladder_area_support, + load_uk_oa_ladder, + uk_geography_ladder_assignment_summary, + uk_geography_ladder_gate, + uk_region_mix, + ) + from microcosm.build.uk_runtime.geography_sources import ( + ENGLAND_LAD_REGION_URL, + ENGLAND_WALES_OA2021_COUNT, + EW_OA_CONSTITUENCY_URL, + EW_OA_HIERARCHY_URL, + EW_OA_HOUSEHOLDS_URL, + EW_OA_LAD23_URL, + EW_OA_POPULATION_URL, + EW_OA_WARD_URL, + LAD23_ITL_URL, + NI_DZ2021_COUNT, + NI_DZ_GEOJSON_ZIP_URL, + NI_DZ_HOUSEHOLDS_CSV_URL, + NI_DZ_LOOKUP_SHEET, + NI_DZ_PARLCON24_LOOKUP_XLSX_URL, + NI_DZ_POPULATION_CSV_URL, + NI_PARLCON24_COUNT, + SCOTLAND_CENSUS_INDEX_ZIP_URL, + SCOTLAND_OA2022_COUNT, + SCOTLAND_OA_CONSTITUENCY_URL, + SCOTLAND_OA_DZ_IZ_URL, + SCOTLAND_OA_LAU_ITL_URL, + SCOTLAND_OA_POPULATION_URL, + build_complete_uk_geography_crosswalk, + build_england_wales_crosswalk, + build_great_britain_crosswalk, + build_northern_ireland_crosswalk, + build_official_uk_geography_crosswalk, + build_scotland_crosswalk, + load_england_lad_region_lookup, + load_england_wales_oa_constituencies, + load_england_wales_oa_hierarchy, + load_england_wales_oa_households, + load_england_wales_oa_population, + load_england_wales_oa_ward_lookup, + load_ew_oa_lad23_lookup, + load_lad_itl_lookup, + load_ni_dz_hierarchy, + load_ni_dz_households, + load_ni_dz_parlcon24_lookup, + load_ni_dz_population, + load_ni_dz_ward_lookup, + load_scotland_oa_constituencies, + load_scotland_oa_dz_iz_lookup, + load_scotland_oa_households, + load_scotland_oa_lau_lookup, + load_scotland_oa_population, + load_scotland_oa_ward_lookup, + update_england_wales_lad_codes, + write_geography_crosswalk, + ) + from microcosm.build.uk_runtime.hmrc_calibration import ( + DEFAULT_HMRC_CALIBRATION_EPOCHS, + DEFAULT_HMRC_CALIBRATION_LEARNING_RATE, + DEFAULT_HMRC_MAX_ABS_RELATIVE_ERROR, + DEFAULT_HMRC_MAX_WEIGHT_RATIO, + HMRC_ASSESSABLE_INCOME_COLUMN, + HMRC_TAXABLE_SAVINGS_INTEREST_COLUMN, + HMRC_TAXPAYER_COLUMN, + UKHMRCIncomeCalibration, + UKHMRCTargetMaterialization, + calibrate_uk_hmrc_income, + materialize_uk_hmrc_calibration_frame, + ) + from microcosm.build.uk_runtime.hmrc_income import ( + HMRC_SPI_BUILD_PERIOD, + HMRC_SPI_COLLATED_ODS_URL, + HMRC_SPI_INCOME_COMPONENTS, + HMRC_SPI_PUBLICATION_URL, + HMRC_SPI_SOURCE_VINTAGE, + HMRC_SPI_TARGET_RECORD_COUNT, + HMRCIncomeBandTargetRecord, + HMRCIncomeSourceProvenance, + HMRCIncomeTargetSet, + materialize_hmrc_spi_income_band_targets, + verify_hmrc_spi_collated_ods, + ) + from microcosm.build.uk_runtime.hmrc_replay import ( + CANONICAL_HMRC_FACT_FENCES, + FULL_FRS_TI_BAND_FENCE_ID, + HMRCFactFence, + HMRCReplayDiagnosticAggregate, + HMRCReplayFact, + HMRCReplayReport, + build_conservative_hmrc_replay_report, + classify_hmrc_replay_targets, + write_hmrc_replay_report, + ) + from microcosm.build.uk_runtime.hmrc_source_contract import ( + HMRC_DISTRIBUTIONAL_INPUTS, + UK_HMRC_INCOME_SOURCE_STAGES_RESOURCE, + assert_uk_hmrc_income_source_contract_current, + ) + from microcosm.build.uk_runtime.ladder_targets import ( + constituency_household_targets, + ladder_target_provenance, + ladder_vs_chronicle_household_dispersion, + local_authority_household_targets, + ) + from microcosm.build.uk_runtime.ledger_targets import ( + UK_CENSUS_HOUSEHOLDS_TARGET_ID, + UK_CROSS_GRAIN_BRIDGES, + UK_CROSS_GRAIN_GRAIN_PRECEDENCE, + UK_CROSS_GRAIN_RULE, + UKFrameTargetAdapter, + UKLedgerTargetCompilation, + apply_uk_cross_grain_reconciliation, + compile_uk_local_target_registry, + compile_uk_target_registry, + load_uk_local_area_crosswalk, + materialize_uk_ledger_targets, + uk_census_household_uprating, + uk_ledger_households_total, + uk_local_target_surface, + ) + from microcosm.build.uk_runtime.local_doctrine import ( + UK_LOCAL_CLONE_COUNT, + UK_LOCAL_MAX_WEIGHT_RATIO, + UK_LOCAL_SOLVE_DOCTRINE, + UK_LOCAL_SOLVE_EPOCHS, + UK_LOCAL_TARGET_LOSS_CAP, + UK_LOCAL_TARGET_WEIGHT_RULE, + UKLocalSolveDoctrine, + uk_local_doctrine_with_overrides, + uk_local_target_loss_weights, + ) + from microcosm.build.uk_runtime.local_geography import ( + align_area_targets, + ) + from microcosm.build.uk_runtime.local_rowwise import ( + UK_LOCAL_BINDING_ADJUDICATION_REGISTER_RESOURCE, + UK_LOCAL_HOLDOUT_FOLDS, + UK_LOCAL_HOLDOUT_SEED, + UKRowwiseDoctrineSolve, + UKRowwiseLocalMatrix, + UKRowwiseNationalRows, + build_uk_rowwise_local_matrix, + build_uk_rowwise_local_surface_matrix, + past_cap_census, + require_adjudicated_uk_local_binding, + rotated_uk_local_holdout, + rowwise_area_support_summary, + rowwise_calibration_mass_reason, + solve_uk_rowwise_weights_under_doctrine, + uk_area_support_summary, + uk_ladder_area_support_summary, + ) + from microcosm.build.uk_runtime.local_target_census import ( + CENSUS_KIND, + CENSUS_RESOURCE, + CENSUS_SCHEMA_VERSION, + METRIC_STATUS_BOUND_IN_CODE, + SOURCE_STATUS_DOCUMENTED_UNPINNED, + assert_uk_local_target_census_current, + build_uk_local_target_census, + committed_uk_local_target_census_path, + load_uk_local_target_census, + write_uk_local_target_census, + ) + from microcosm.build.uk_runtime.local_targets import ( + AGE_BANDS, + AREA_TYPE_TO_LEDGER_GEOGRAPHY_LEVEL, + AREA_TYPES, + COUNTRY_TO_REGION, + INCOME_VARIABLES, + LA_EXTRA_METRICS, + area_groups_from_codes, + compute_household_metrics, + metric_names, + metric_names_from_target_profile, + metric_tables_by_area_group, + ) + from microcosm.build.uk_runtime.national_calibration import ( + CalibrationFrameAdapter, + drop_injected_measure_inputs, + inject_measure_inputs, + prepare_uk_target_frame, + ) + from microcosm.build.uk_runtime.national_doctrine import ( + UK_NATIONAL_L0_LAMBDA, + UK_NATIONAL_LEARNING_RATE, + UK_NATIONAL_MASS_RULE, + UK_NATIONAL_MAX_WEIGHT_RATIO, + UK_NATIONAL_SEED, + UK_NATIONAL_SOLVE_DOCTRINE, + UK_NATIONAL_SOLVE_EPOCHS, + UK_NATIONAL_TARGET_LOSS_CAP, + UK_NATIONAL_TARGET_WEIGHT_RULE, + UKNationalSolveDoctrine, + uk_doctrine_with_overrides, + uk_national_target_loss_weights, + ) + from microcosm.build.uk_runtime.national_frame import ( + UK_NATIONAL_SCHEMA, + UKNationalStage, + UKStagingProvenance, + load_uk_national_frame, + uk_household_weight_kind, + uk_national_frame, + uk_time_period, + validate_uk_national_frame, + write_uk_national_frame, + ) + from microcosm.build.uk_runtime.national_sampling import ( + sample_uk_spine_frame, + uk_spine_source_family_units, + ) + from microcosm.build.uk_runtime.oa_ladder_sources import ( + LADDER_OA_COLUMNS, + assemble_uk_oa_ladder, + concat_uk_ladder_frames, + join_uk_oa_ladder_layers, + ) + from microcosm.build.uk_runtime.parity_reference import ( + EFRS_PARITY_KNOWN_GAPS_RESOURCE, + EFRS_PARITY_REFERENCE_RESOURCE, + EfrsParityKnownGap, + EfrsParityReference, + EfrsParitySource, + load_efrs_parity_known_gaps, + load_efrs_parity_reference, + ) + from microcosm.build.uk_runtime.release_identity import ( + UK_DENSE_RELEASE_ID, + UK_RELEASE_TIER_CPS_TRANSFER, + UK_RELEASE_TIER_FRS, + UK_RELEASE_TIERS, + UKReleaseIdentity, + apply_uk_release_identity, + format_uk_release_id, + validate_uk_release_tier, + ) + from microcosm.build.uk_runtime.release_input_coverage import ( + RESTORED_REFERENCE_EFRS_REQUIRED_INPUTS, + UK_LOADER_INPUT_ALIASES, + UK_RELEASE_INPUT_COVERAGE_RESOURCE, + PolicyEngineUKCoverageEngine, + UKEffectiveMassCoveragePolicy, + UKReleaseInputColumn, + UKReleaseInputCoverageManifest, + assert_uk_release_input_coverage_build_stages, + assert_uk_release_input_coverage_manifest_current, + load_uk_release_input_coverage_manifest, + uk_release_input_coverage_gate, + uk_release_input_coverage_required_columns, + uk_release_input_coverage_reviewed_exclusions, + ) + from microcosm.build.uk_runtime.rowwise_dataset import ( + ARTIFACT_CLONE_INDEX_COLUMN, + BENUNIT_ID_COLUMNS, + HOUSEHOLD_ID_COLUMNS, + MASS_CONSERVATION_RELATIVE_TOLERANCE, + PERSON_ID_COLUMNS, + POOL_SOURCE_LINEAGE_COLUMN, + UK_SINGLE_YEAR_TABLES, + UKLadderRowwiseDatasetResult, + UKRowwiseDatasetResult, + apply_uk_source_lineage_modulus, + clone_uk_dataset_tables_with_ladder_geography, + clone_uk_dataset_tables_with_rowwise_geography, + clone_uk_dataset_with_ladder_geography, + clone_uk_dataset_with_rowwise_geography, + ladder_clone_index_column, + load_uk_rowwise_dataset, + read_uk_single_year_weight_metadata, + validate_uk_ladder_rowwise_dataset_tables, + validate_uk_rowwise_dataset_tables, + write_uk_rowwise_dataset, + ) + from microcosm.build.uk_runtime.rowwise_geography import ( + AREA_TYPE_TO_CROSSWALK_COLUMN, + CROSSWALK_COLUMNS, + FRS_REGION_TO_COUNTRY, + FRS_REGION_TO_REGION_CODE, + ROWWISE_GEOGRAPHY_COLUMNS, + RowwiseGeographyAssignment, + assign_household_geography, + clone_entity_frame, + expected_uk_rowwise_area_support, + geography_coverage_summary, + id_multiplier_for_values, + prepare_geography_crosswalk, + validate_geography_coverage, + ) + from microcosm.build.uk_runtime.size_evaluation import ( + PRE_REGISTERED_OUTCOMES_V1, + RowwiseRun, + area_support_tables, + dense_reference_deltas, + fit_tables, + footprint, + frozen_vs_recomputed, + gate_table, + load_run, + paired_targets, + run_acceptance, + summarize, + weight_tables, + ) + from microcosm.build.uk_runtime.spi_income import ( + SPI_DONOR_DOI, + SPI_DONOR_FILENAME, + SPI_DONOR_SHA256, + SPI_DONOR_SIZE_BYTES, + SPI_DONOR_UKDS_STUDY, + SPI_DONOR_VINTAGE, + SPI_STAGE2_REVIEWED_ABSENT_OUTPUTS, + UKSPIIncomeImputationResult, + assert_frs_hmrc_auxiliary_crosswalk_available, + derive_hmrc_income_auxiliaries, + impute_uk_spi_income_support, + verify_spi_donor_identity, + ) + from microcosm.build.uk_runtime.spi_spine import ( + EMPLOYER_PENSION_CONTRIBUTIONS_COLUMN, + UK_FRS_HMRC_SPINE_LEAF_OUTPUT_COLUMNS, + UK_FRS_HMRC_SPINE_LEAVES_STAGE_NAME, + UK_HMRC_SPI_INCOME_SPINE_STAGE_NAME, + UK_HMRC_SPI_SPINE_REPLAY_REPORT_KIND, + UK_SPI_INCOME_SPINE_NONNEGATIVE_OUTPUT_COLUMNS, + UK_SPI_INCOME_SPINE_OUTPUT_COLUMNS, + UK_SPI_INCOME_SPINE_REWRITE_COLUMNS, + UK_SPI_SUPPORT_CHANNEL_OUTPUT_COLUMNS, + UKFRSHMRCSpineLeavesResult, + UKFRSHMRCSpineLeavesStageTransform, + UKSPIIncomeSpineResult, + UKSPIIncomeSpineStageTransform, + UKSPISupportChannelStageTransform, + ) + from microcosm.build.uk_runtime.spi_support import ( + BASE_FRS_SUPPORT_CHANNEL, + DEFAULT_SPI_PRIOR_MASS_SHARE, + DEFAULT_SPI_SUPPORT_HOUSEHOLDS, + FRS_ONLY_SPI_FILL_INCOME_PREDICTOR_COLUMNS, + FRS_ONLY_SPI_FILL_PERSON_COLUMNS, + FRS_ONLY_SPI_FILL_PREDICTOR_COLUMNS, + HOUSEHOLD_IS_SPI_SYNTHETIC_COLUMN, + SPI_INCOME_COMPONENT_COLUMNS, + SPI_INCOME_IMPUTATION_COLUMNS, + SPI_PRIOR_MASS_CHANGE_REASON, + SPI_REPLACEMENT_STRATA_COLUMNS, + SPI_SYNTHETIC_SUPPORT_CHANNEL, + UK_SPI_SUPPORT_STAGE_NAME, + UKSPISupportResult, + build_uk_spi_support_channel, + create_uk_spi_support_tables, + fill_support_channel_from_source, + replace_uk_spi_support_tables, + support_channel_column, + support_clone_index_column, + support_source_id_column, + ) + from microcosm.build.uk_runtime.stage_checkpoints import ( + UK_FRAME_METADATA_KEY, + load_uk_stage_checkpoint, + load_uk_stage_predecessor, + uk_stage_metadata, + ) + from microcosm.build.uk_runtime.terminal_gates import ( + UK_DEFAULT_ZERO_WEIGHT_STRATA, + UK_MAX_TARGET_ABS_RELATIVE_ERROR, + UKZeroWeightStratumDeclaration, + uk_degenerate_release_surface_gate, + uk_export_surface_gate, + uk_target_fit_gate, + uk_target_surface_gate, + uk_weight_ess_gate, + uk_weight_ratio_gate, + uk_zero_weight_strata_gate, + ) + from microcosm.build.uk_runtime.weighted_integrity import ( + UKInputMassParityPolicy, + UKInputMassReference, + UKQRFTailConcentrationPolicy, + load_uk_input_mass_reference, + load_uk_local_area_support_exclusion_register, + load_uk_reviewed_exclusion_register, + uk_input_mass_parity_gate, + uk_input_mass_totals, + uk_qrf_tail_concentration_columns, + uk_qrf_tail_concentration_gate, + ) __all__ = [ "PRE_REGISTERED_OUTCOMES_V1", @@ -1006,3 +1018,1793 @@ "uk_weight_ratio_gate", "uk_zero_weight_strata_gate", ] + + +_EXPORTS = _MappingProxyType( + { + "UK_AGE_TAIL_BAND_POPULATIONS_RESOURCE": ( + "microcosm.build.uk_runtime.age_tail", + "UK_AGE_TAIL_BAND_POPULATIONS_RESOURCE", + ), + "UK_AGE_TAIL_BANDS": ( + "microcosm.build.uk_runtime.age_tail", + "UK_AGE_TAIL_BANDS", + ), + "UK_AGE_TAIL_DECLARED_SEEDS": ( + "microcosm.build.uk_runtime.age_tail", + "UK_AGE_TAIL_DECLARED_SEEDS", + ), + "UK_AGE_TOP_CODE": ("microcosm.build.uk_runtime.age_tail", "UK_AGE_TOP_CODE"), + "UKAgeTailStageTransform": ( + "microcosm.build.uk_runtime.age_tail", + "UKAgeTailStageTransform", + ), + "disaggregate_uk_age_top_code": ( + "microcosm.build.uk_runtime.age_tail", + "disaggregate_uk_age_top_code", + ), + "load_uk_age_tail_band_populations": ( + "microcosm.build.uk_runtime.age_tail", + "load_uk_age_tail_band_populations", + ), + "UK_GATE_REGISTRY": ( + "microcosm.build.uk_runtime.battery_bindings", + "UK_GATE_REGISTRY", + ), + "UKGateBinding": ( + "microcosm.build.uk_runtime.battery_bindings", + "UKGateBinding", + ), + "load_bound_spine_sidecar": ( + "microcosm.build.uk_runtime.calibration_run", + "load_bound_spine_sidecar", + ), + "runtime_provenance": ( + "microcosm.build.uk_runtime.calibration_run", + "runtime_provenance", + ), + "spine_provenance_from_sidecar": ( + "microcosm.build.uk_runtime.calibration_run", + "spine_provenance_from_sidecar", + ), + "UK_CGT_ANNUAL_EXEMPT_AMOUNTS": ( + "microcosm.build.uk_runtime.cgt_calibration", + "UK_CGT_ANNUAL_EXEMPT_AMOUNTS", + ), + "UK_CGT_GAINS_AMOUNT_COLUMN": ( + "microcosm.build.uk_runtime.cgt_calibration", + "UK_CGT_GAINS_AMOUNT_COLUMN", + ), + "UK_CGT_SOURCE_COLUMN": ( + "microcosm.build.uk_runtime.cgt_calibration", + "UK_CGT_SOURCE_COLUMN", + ), + "UK_CGT_TAXPAYER_COUNT_COLUMN": ( + "microcosm.build.uk_runtime.cgt_calibration", + "UK_CGT_TAXPAYER_COUNT_COLUMN", + ), + "UKCGTTargetMaterialization": ( + "microcosm.build.uk_runtime.cgt_calibration", + "UKCGTTargetMaterialization", + ), + "materialize_uk_cgt_calibration_frame": ( + "microcosm.build.uk_runtime.cgt_calibration", + "materialize_uk_cgt_calibration_frame", + ), + "uk_cgt_annual_exempt_amount": ( + "microcosm.build.uk_runtime.cgt_calibration", + "uk_cgt_annual_exempt_amount", + ), + "uk_frame_content_identity": ( + "microcosm.build.uk_runtime.content_identity", + "uk_frame_content_identity", + ), + "UK_DIAGNOSTICS_SCHEMA_VERSION": ( + "microcosm.build.uk_runtime.diagnostics", + "UK_DIAGNOSTICS_SCHEMA_VERSION", + ), + "UK_TARGET_GEOGRAPHY_LEVELS": ( + "microcosm.build.uk_runtime.diagnostics", + "UK_TARGET_GEOGRAPHY_LEVELS", + ), + "uk_calibration_diagnostics_payload": ( + "microcosm.build.uk_runtime.diagnostics", + "uk_calibration_diagnostics_payload", + ), + "uk_fit_by_family": ( + "microcosm.build.uk_runtime.diagnostics", + "uk_fit_by_family", + ), + "uk_support_limited_misses": ( + "microcosm.build.uk_runtime.diagnostics", + "uk_support_limited_misses", + ), + "uk_weakest_areas_by_fit": ( + "microcosm.build.uk_runtime.diagnostics", + "uk_weakest_areas_by_fit", + ), + "uk_weakest_families": ( + "microcosm.build.uk_runtime.diagnostics", + "uk_weakest_families", + ), + "uk_weight_summary": ( + "microcosm.build.uk_runtime.diagnostics", + "uk_weight_summary", + ), + "uk_zero_weight_strata": ( + "microcosm.build.uk_runtime.diagnostics", + "uk_zero_weight_strata", + ), + "write_uk_calibration_diagnostics": ( + "microcosm.build.uk_runtime.diagnostics", + "write_uk_calibration_diagnostics", + ), + "EMPLOYMENT_BANDS": ( + "microcosm.build.uk_runtime.firm_generation", + "EMPLOYMENT_BANDS", + ), + "HMRC_BAND_COLUMNS": ( + "microcosm.build.uk_runtime.firm_generation", + "HMRC_BAND_COLUMNS", + ), + "INPUT_FILES": ("microcosm.build.uk_runtime.firm_generation", "INPUT_FILES"), + "VAT_LIABILITY_BANDS": ( + "microcosm.build.uk_runtime.firm_generation", + "VAT_LIABILITY_BANDS", + ), + "VINTAGES": ("microcosm.build.uk_runtime.firm_generation", "VINTAGES"), + "AxiomVATRuleEvaluator": ( + "microcosm.build.uk_runtime.firm_generation", + "AxiomVATRuleEvaluator", + ), + "UKFirmCalibrationResult": ( + "microcosm.build.uk_runtime.firm_generation", + "UKFirmCalibrationResult", + ), + "UKFirmGenerationConfig": ( + "microcosm.build.uk_runtime.firm_generation", + "UKFirmGenerationConfig", + ), + "UKFirmGenerationResult": ( + "microcosm.build.uk_runtime.firm_generation", + "UKFirmGenerationResult", + ), + "UKFirmSourceData": ( + "microcosm.build.uk_runtime.firm_generation", + "UKFirmSourceData", + ), + "UKFirmTargetLayout": ( + "microcosm.build.uk_runtime.firm_generation", + "UKFirmTargetLayout", + ), + "UKFirmValidationReport": ( + "microcosm.build.uk_runtime.firm_generation", + "UKFirmValidationReport", + ), + "UKFirmVATRuleEvaluator": ( + "microcosm.build.uk_runtime.firm_generation", + "UKFirmVATRuleEvaluator", + ), + "assign_employment": ( + "microcosm.build.uk_runtime.firm_generation", + "assign_employment", + ), + "assign_vat_flags": ( + "microcosm.build.uk_runtime.firm_generation", + "assign_vat_flags", + ), + "build_firm_target_matrix": ( + "microcosm.build.uk_runtime.firm_generation", + "build_firm_target_matrix", + ), + "employment_band_name": ( + "microcosm.build.uk_runtime.firm_generation", + "employment_band_name", + ), + "generate_base_firms": ( + "microcosm.build.uk_runtime.firm_generation", + "generate_base_firms", + ), + "generate_input_values": ( + "microcosm.build.uk_runtime.firm_generation", + "generate_input_values", + ), + "generate_uk_firm_population": ( + "microcosm.build.uk_runtime.firm_generation", + "generate_uk_firm_population", + ), + "hmrc_band_name": ( + "microcosm.build.uk_runtime.firm_generation", + "hmrc_band_name", + ), + "map_to_hmrc_band_indices": ( + "microcosm.build.uk_runtime.firm_generation", + "map_to_hmrc_band_indices", + ), + "optimize_firm_weights": ( + "microcosm.build.uk_runtime.firm_generation", + "optimize_firm_weights", + ), + "read_uk_firm_source_data": ( + "microcosm.build.uk_runtime.firm_generation", + "read_uk_firm_source_data", + ), + "solve_firm_weights": ( + "microcosm.build.uk_runtime.firm_generation", + "solve_firm_weights", + ), + "target_diagnostics": ( + "microcosm.build.uk_runtime.firm_generation", + "target_diagnostics", + ), + "uk_firm_source_data_from_frames": ( + "microcosm.build.uk_runtime.firm_generation", + "uk_firm_source_data_from_frames", + ), + "uk_firm_source_data_from_ledger_facts": ( + "microcosm.build.uk_runtime.firm_generation", + "uk_firm_source_data_from_ledger_facts", + ), + "validate_uk_firm_population": ( + "microcosm.build.uk_runtime.firm_generation", + "validate_uk_firm_population", + ), + "write_uk_firm_population": ( + "microcosm.build.uk_runtime.firm_generation", + "write_uk_firm_population", + ), + "UK_CGT_REQUIRED_COLUMNS": ( + "microcosm.build.uk_runtime.fiscal_targets", + "UK_CGT_REQUIRED_COLUMNS", + ), + "UK_CGT_TARGET_COVERAGE_REQUIREMENTS": ( + "microcosm.build.uk_runtime.fiscal_targets", + "UK_CGT_TARGET_COVERAGE_REQUIREMENTS", + ), + "UK_CGT_TARGET_SPECS": ( + "microcosm.build.uk_runtime.fiscal_targets", + "UK_CGT_TARGET_SPECS", + ), + "UK_FISCAL_TARGET_REGISTRY": ( + "microcosm.build.uk_runtime.fiscal_targets", + "UK_FISCAL_TARGET_REGISTRY", + ), + "FRS_COUNCIL_TAX_OUTPUT_COLUMNS": ( + "microcosm.build.uk_runtime.frs_council_tax", + "FRS_COUNCIL_TAX_OUTPUT_COLUMNS", + ), + "UKFRSCouncilTaxStageTransform": ( + "microcosm.build.uk_runtime.frs_council_tax", + "UKFRSCouncilTaxStageTransform", + ), + "add_frs_council_tax": ( + "microcosm.build.uk_runtime.frs_council_tax", + "add_frs_council_tax", + ), + "derive_council_tax": ( + "microcosm.build.uk_runtime.frs_council_tax", + "derive_council_tax", + ), + "FRS_DISABILITY_OUTPUT_COLUMNS": ( + "microcosm.build.uk_runtime.frs_disability", + "FRS_DISABILITY_OUTPUT_COLUMNS", + ), + "UK_INTERNAL_DISABILITY_REPORTED_COLUMNS": ( + "microcosm.build.uk_runtime.frs_disability", + "UK_INTERNAL_DISABILITY_REPORTED_COLUMNS", + ), + "UKDWPDisabilityCategoryRates": ( + "microcosm.build.uk_runtime.frs_disability", + "UKDWPDisabilityCategoryRates", + ), + "UKDWPDisabilityFlagRates": ( + "microcosm.build.uk_runtime.frs_disability", + "UKDWPDisabilityFlagRates", + ), + "UKFRSDisabilityStageTransform": ( + "microcosm.build.uk_runtime.frs_disability", + "UKFRSDisabilityStageTransform", + ), + "add_frs_disability": ( + "microcosm.build.uk_runtime.frs_disability", + "add_frs_disability", + ), + "derive_frs_disability": ( + "microcosm.build.uk_runtime.frs_disability", + "derive_frs_disability", + ), + "uk_dwp_disability_category_rates": ( + "microcosm.build.uk_runtime.frs_disability", + "uk_dwp_disability_category_rates", + ), + "uk_dwp_disability_flag_rates": ( + "microcosm.build.uk_runtime.frs_disability", + "uk_dwp_disability_flag_rates", + ), + "EDUCQUAL_MAP": ("microcosm.build.uk_runtime.frs_education", "EDUCQUAL_MAP"), + "FRS_EDUCATION_OUTPUT_COLUMNS": ( + "microcosm.build.uk_runtime.frs_education", + "FRS_EDUCATION_OUTPUT_COLUMNS", + ), + "UKFRSEducationStageTransform": ( + "microcosm.build.uk_runtime.frs_education", + "UKFRSEducationStageTransform", + ), + "add_frs_education": ( + "microcosm.build.uk_runtime.frs_education", + "add_frs_education", + ), + "derive_current_education": ( + "microcosm.build.uk_runtime.frs_education", + "derive_current_education", + ), + "derive_frs_education": ( + "microcosm.build.uk_runtime.frs_education", + "derive_frs_education", + ), + "FRS_EDUCATION_GRANT_OUTPUT_COLUMNS": ( + "microcosm.build.uk_runtime.frs_education_grants", + "FRS_EDUCATION_GRANT_OUTPUT_COLUMNS", + ), + "FRS_EDUCATION_GRANT_REWRITES": ( + "microcosm.build.uk_runtime.frs_education_grants", + "FRS_EDUCATION_GRANT_REWRITES", + ), + "UK_EDUCATION_GRANT_CAPACITY_PREDICTORS": ( + "microcosm.build.uk_runtime.frs_education_grants", + "UK_EDUCATION_GRANT_CAPACITY_PREDICTORS", + ), + "UKDSAPolicy": ( + "microcosm.build.uk_runtime.frs_education_grants", + "UKDSAPolicy", + ), + "UKFRSEducationGrantSplitStageTransform": ( + "microcosm.build.uk_runtime.frs_education_grants", + "UKFRSEducationGrantSplitStageTransform", + ), + "add_frs_education_grant_split": ( + "microcosm.build.uk_runtime.frs_education_grants", + "add_frs_education_grant_split", + ), + "allocate_reported_education_grants": ( + "microcosm.build.uk_runtime.frs_education_grants", + "allocate_reported_education_grants", + ), + "disabled_students_allowance_capacity": ( + "microcosm.build.uk_runtime.frs_education_grants", + "disabled_students_allowance_capacity", + ), + "uk_dsa_policy": ( + "microcosm.build.uk_runtime.frs_education_grants", + "uk_dsa_policy", + ), + "FRS_EMPLOYMENT_OUTPUT_COLUMNS": ( + "microcosm.build.uk_runtime.frs_employment", + "FRS_EMPLOYMENT_OUTPUT_COLUMNS", + ), + "UKFRSEmploymentStageTransform": ( + "microcosm.build.uk_runtime.frs_employment", + "UKFRSEmploymentStageTransform", + ), + "add_frs_employment": ( + "microcosm.build.uk_runtime.frs_employment", + "add_frs_employment", + ), + "derive_frs_employment": ( + "microcosm.build.uk_runtime.frs_employment", + "derive_frs_employment", + ), + "FRS_HMRC_INCPBEN_COLUMN": ( + "microcosm.build.uk_runtime.frs_hmrc_leaves", + "FRS_HMRC_INCPBEN_COLUMN", + ), + "FRS_HMRC_OSSBEN_IDENTIFIABLE_SUBSET_COLUMN": ( + "microcosm.build.uk_runtime.frs_hmrc_leaves", + "FRS_HMRC_OSSBEN_IDENTIFIABLE_SUBSET_COLUMN", + ), + "FRS_HMRC_PAY_COLUMN": ( + "microcosm.build.uk_runtime.frs_hmrc_leaves", + "FRS_HMRC_PAY_COLUMN", + ), + "FRS_HMRC_RETAINED_LEAF_COLUMNS": ( + "microcosm.build.uk_runtime.frs_hmrc_leaves", + "FRS_HMRC_RETAINED_LEAF_COLUMNS", + ), + "FRS_HMRC_RETAINED_LEAVES_STAGE_NAME": ( + "microcosm.build.uk_runtime.frs_hmrc_leaves", + "FRS_HMRC_RETAINED_LEAVES_STAGE_NAME", + ), + "FRS_HMRC_SRP_REGULAR_CODE5_COLUMN": ( + "microcosm.build.uk_runtime.frs_hmrc_leaves", + "FRS_HMRC_SRP_REGULAR_CODE5_COLUMN", + ), + "FRS_HMRC_UBISJA_COLUMN": ( + "microcosm.build.uk_runtime.frs_hmrc_leaves", + "FRS_HMRC_UBISJA_COLUMN", + ), + "UKFRSHMRCRetainedLeavesResult": ( + "microcosm.build.uk_runtime.frs_hmrc_leaves", + "UKFRSHMRCRetainedLeavesResult", + ), + "UKFRSHMRCRetainedLeavesStageTransform": ( + "microcosm.build.uk_runtime.frs_hmrc_leaves", + "UKFRSHMRCRetainedLeavesStageTransform", + ), + "retain_uk_frs_hmrc_leaves": ( + "microcosm.build.uk_runtime.frs_hmrc_leaves", + "retain_uk_frs_hmrc_leaves", + ), + "FRS_LEGACY_PROXY_OUTPUT_COLUMNS": ( + "microcosm.build.uk_runtime.frs_legacy_proxies", + "FRS_LEGACY_PROXY_OUTPUT_COLUMNS", + ), + "UK_LEGACY_PROXY_PREDICTORS": ( + "microcosm.build.uk_runtime.frs_legacy_proxies", + "UK_LEGACY_PROXY_PREDICTORS", + ), + "UKFRSLegacyProxiesStageTransform": ( + "microcosm.build.uk_runtime.frs_legacy_proxies", + "UKFRSLegacyProxiesStageTransform", + ), + "UKLegacyJSAPolicy": ( + "microcosm.build.uk_runtime.frs_legacy_proxies", + "UKLegacyJSAPolicy", + ), + "derive_frs_legacy_proxies": ( + "microcosm.build.uk_runtime.frs_legacy_proxies", + "derive_frs_legacy_proxies", + ), + "uk_legacy_jsa_policy": ( + "microcosm.build.uk_runtime.frs_legacy_proxies", + "uk_legacy_jsa_policy", + ), + "UK_YEAR_RULES": ("microcosm.build.uk_runtime.frs_release", "UK_YEAR_RULES"), + "UKFRSRelease": ("microcosm.build.uk_runtime.frs_release", "UKFRSRelease"), + "load_uk_frs_release": ( + "microcosm.build.uk_runtime.frs_release", + "load_uk_frs_release", + ), + "resolve_uk_year_rule": ( + "microcosm.build.uk_runtime.frs_release", + "resolve_uk_year_rule", + ), + "GEOGRAPHY_LADDER_ARTIFACT_SHA256_ATTR": ( + "microcosm.build.uk_runtime.geography_ladder", + "GEOGRAPHY_LADDER_ARTIFACT_SHA256_ATTR", + ), + "GEOGRAPHY_LADDER_VINTAGES_ATTR": ( + "microcosm.build.uk_runtime.geography_ladder", + "GEOGRAPHY_LADDER_VINTAGES_ATTR", + ), + "UK_ENGLAND_WALES_REGION_CODES": ( + "microcosm.build.uk_runtime.geography_ladder", + "UK_ENGLAND_WALES_REGION_CODES", + ), + "UK_GEOGRAPHY_LADDER_COLUMNS": ( + "microcosm.build.uk_runtime.geography_ladder", + "UK_GEOGRAPHY_LADDER_COLUMNS", + ), + "UK_LONDON_REGION_CODE": ( + "microcosm.build.uk_runtime.geography_ladder", + "UK_LONDON_REGION_CODE", + ), + "UK_OA_LADDER_DERIVED_LAYERS": ( + "microcosm.build.uk_runtime.geography_ladder", + "UK_OA_LADDER_DERIVED_LAYERS", + ), + "UK_OA_LADDER_KIND": ( + "microcosm.build.uk_runtime.geography_ladder", + "UK_OA_LADDER_KIND", + ), + "UK_OA_LADDER_SCHEMA_VERSION": ( + "microcosm.build.uk_runtime.geography_ladder", + "UK_OA_LADDER_SCHEMA_VERSION", + ), + "UkOaLadder": ("microcosm.build.uk_runtime.geography_ladder", "UkOaLadder"), + "assign_uk_geography_ladder": ( + "microcosm.build.uk_runtime.geography_ladder", + "assign_uk_geography_ladder", + ), + "expected_uk_ladder_area_support": ( + "microcosm.build.uk_runtime.geography_ladder", + "expected_uk_ladder_area_support", + ), + "load_uk_oa_ladder": ( + "microcosm.build.uk_runtime.geography_ladder", + "load_uk_oa_ladder", + ), + "uk_geography_ladder_assignment_summary": ( + "microcosm.build.uk_runtime.geography_ladder", + "uk_geography_ladder_assignment_summary", + ), + "uk_geography_ladder_gate": ( + "microcosm.build.uk_runtime.geography_ladder", + "uk_geography_ladder_gate", + ), + "uk_region_mix": ( + "microcosm.build.uk_runtime.geography_ladder", + "uk_region_mix", + ), + "ENGLAND_LAD_REGION_URL": ( + "microcosm.build.uk_runtime.geography_sources", + "ENGLAND_LAD_REGION_URL", + ), + "ENGLAND_WALES_OA2021_COUNT": ( + "microcosm.build.uk_runtime.geography_sources", + "ENGLAND_WALES_OA2021_COUNT", + ), + "EW_OA_CONSTITUENCY_URL": ( + "microcosm.build.uk_runtime.geography_sources", + "EW_OA_CONSTITUENCY_URL", + ), + "EW_OA_HIERARCHY_URL": ( + "microcosm.build.uk_runtime.geography_sources", + "EW_OA_HIERARCHY_URL", + ), + "EW_OA_HOUSEHOLDS_URL": ( + "microcosm.build.uk_runtime.geography_sources", + "EW_OA_HOUSEHOLDS_URL", + ), + "EW_OA_LAD23_URL": ( + "microcosm.build.uk_runtime.geography_sources", + "EW_OA_LAD23_URL", + ), + "EW_OA_POPULATION_URL": ( + "microcosm.build.uk_runtime.geography_sources", + "EW_OA_POPULATION_URL", + ), + "EW_OA_WARD_URL": ( + "microcosm.build.uk_runtime.geography_sources", + "EW_OA_WARD_URL", + ), + "LAD23_ITL_URL": ( + "microcosm.build.uk_runtime.geography_sources", + "LAD23_ITL_URL", + ), + "NI_DZ2021_COUNT": ( + "microcosm.build.uk_runtime.geography_sources", + "NI_DZ2021_COUNT", + ), + "NI_DZ_GEOJSON_ZIP_URL": ( + "microcosm.build.uk_runtime.geography_sources", + "NI_DZ_GEOJSON_ZIP_URL", + ), + "NI_DZ_HOUSEHOLDS_CSV_URL": ( + "microcosm.build.uk_runtime.geography_sources", + "NI_DZ_HOUSEHOLDS_CSV_URL", + ), + "NI_DZ_LOOKUP_SHEET": ( + "microcosm.build.uk_runtime.geography_sources", + "NI_DZ_LOOKUP_SHEET", + ), + "NI_DZ_PARLCON24_LOOKUP_XLSX_URL": ( + "microcosm.build.uk_runtime.geography_sources", + "NI_DZ_PARLCON24_LOOKUP_XLSX_URL", + ), + "NI_DZ_POPULATION_CSV_URL": ( + "microcosm.build.uk_runtime.geography_sources", + "NI_DZ_POPULATION_CSV_URL", + ), + "NI_PARLCON24_COUNT": ( + "microcosm.build.uk_runtime.geography_sources", + "NI_PARLCON24_COUNT", + ), + "SCOTLAND_CENSUS_INDEX_ZIP_URL": ( + "microcosm.build.uk_runtime.geography_sources", + "SCOTLAND_CENSUS_INDEX_ZIP_URL", + ), + "SCOTLAND_OA2022_COUNT": ( + "microcosm.build.uk_runtime.geography_sources", + "SCOTLAND_OA2022_COUNT", + ), + "SCOTLAND_OA_CONSTITUENCY_URL": ( + "microcosm.build.uk_runtime.geography_sources", + "SCOTLAND_OA_CONSTITUENCY_URL", + ), + "SCOTLAND_OA_DZ_IZ_URL": ( + "microcosm.build.uk_runtime.geography_sources", + "SCOTLAND_OA_DZ_IZ_URL", + ), + "SCOTLAND_OA_LAU_ITL_URL": ( + "microcosm.build.uk_runtime.geography_sources", + "SCOTLAND_OA_LAU_ITL_URL", + ), + "SCOTLAND_OA_POPULATION_URL": ( + "microcosm.build.uk_runtime.geography_sources", + "SCOTLAND_OA_POPULATION_URL", + ), + "build_complete_uk_geography_crosswalk": ( + "microcosm.build.uk_runtime.geography_sources", + "build_complete_uk_geography_crosswalk", + ), + "build_england_wales_crosswalk": ( + "microcosm.build.uk_runtime.geography_sources", + "build_england_wales_crosswalk", + ), + "build_great_britain_crosswalk": ( + "microcosm.build.uk_runtime.geography_sources", + "build_great_britain_crosswalk", + ), + "build_northern_ireland_crosswalk": ( + "microcosm.build.uk_runtime.geography_sources", + "build_northern_ireland_crosswalk", + ), + "build_official_uk_geography_crosswalk": ( + "microcosm.build.uk_runtime.geography_sources", + "build_official_uk_geography_crosswalk", + ), + "build_scotland_crosswalk": ( + "microcosm.build.uk_runtime.geography_sources", + "build_scotland_crosswalk", + ), + "load_england_lad_region_lookup": ( + "microcosm.build.uk_runtime.geography_sources", + "load_england_lad_region_lookup", + ), + "load_england_wales_oa_constituencies": ( + "microcosm.build.uk_runtime.geography_sources", + "load_england_wales_oa_constituencies", + ), + "load_england_wales_oa_hierarchy": ( + "microcosm.build.uk_runtime.geography_sources", + "load_england_wales_oa_hierarchy", + ), + "load_england_wales_oa_households": ( + "microcosm.build.uk_runtime.geography_sources", + "load_england_wales_oa_households", + ), + "load_england_wales_oa_population": ( + "microcosm.build.uk_runtime.geography_sources", + "load_england_wales_oa_population", + ), + "load_england_wales_oa_ward_lookup": ( + "microcosm.build.uk_runtime.geography_sources", + "load_england_wales_oa_ward_lookup", + ), + "load_ew_oa_lad23_lookup": ( + "microcosm.build.uk_runtime.geography_sources", + "load_ew_oa_lad23_lookup", + ), + "load_lad_itl_lookup": ( + "microcosm.build.uk_runtime.geography_sources", + "load_lad_itl_lookup", + ), + "load_ni_dz_hierarchy": ( + "microcosm.build.uk_runtime.geography_sources", + "load_ni_dz_hierarchy", + ), + "load_ni_dz_households": ( + "microcosm.build.uk_runtime.geography_sources", + "load_ni_dz_households", + ), + "load_ni_dz_parlcon24_lookup": ( + "microcosm.build.uk_runtime.geography_sources", + "load_ni_dz_parlcon24_lookup", + ), + "load_ni_dz_population": ( + "microcosm.build.uk_runtime.geography_sources", + "load_ni_dz_population", + ), + "load_ni_dz_ward_lookup": ( + "microcosm.build.uk_runtime.geography_sources", + "load_ni_dz_ward_lookup", + ), + "load_scotland_oa_constituencies": ( + "microcosm.build.uk_runtime.geography_sources", + "load_scotland_oa_constituencies", + ), + "load_scotland_oa_dz_iz_lookup": ( + "microcosm.build.uk_runtime.geography_sources", + "load_scotland_oa_dz_iz_lookup", + ), + "load_scotland_oa_households": ( + "microcosm.build.uk_runtime.geography_sources", + "load_scotland_oa_households", + ), + "load_scotland_oa_lau_lookup": ( + "microcosm.build.uk_runtime.geography_sources", + "load_scotland_oa_lau_lookup", + ), + "load_scotland_oa_population": ( + "microcosm.build.uk_runtime.geography_sources", + "load_scotland_oa_population", + ), + "load_scotland_oa_ward_lookup": ( + "microcosm.build.uk_runtime.geography_sources", + "load_scotland_oa_ward_lookup", + ), + "update_england_wales_lad_codes": ( + "microcosm.build.uk_runtime.geography_sources", + "update_england_wales_lad_codes", + ), + "write_geography_crosswalk": ( + "microcosm.build.uk_runtime.geography_sources", + "write_geography_crosswalk", + ), + "DEFAULT_HMRC_CALIBRATION_EPOCHS": ( + "microcosm.build.uk_runtime.hmrc_calibration", + "DEFAULT_HMRC_CALIBRATION_EPOCHS", + ), + "DEFAULT_HMRC_CALIBRATION_LEARNING_RATE": ( + "microcosm.build.uk_runtime.hmrc_calibration", + "DEFAULT_HMRC_CALIBRATION_LEARNING_RATE", + ), + "DEFAULT_HMRC_MAX_ABS_RELATIVE_ERROR": ( + "microcosm.build.uk_runtime.hmrc_calibration", + "DEFAULT_HMRC_MAX_ABS_RELATIVE_ERROR", + ), + "DEFAULT_HMRC_MAX_WEIGHT_RATIO": ( + "microcosm.build.uk_runtime.hmrc_calibration", + "DEFAULT_HMRC_MAX_WEIGHT_RATIO", + ), + "HMRC_ASSESSABLE_INCOME_COLUMN": ( + "microcosm.build.uk_runtime.hmrc_calibration", + "HMRC_ASSESSABLE_INCOME_COLUMN", + ), + "HMRC_TAXABLE_SAVINGS_INTEREST_COLUMN": ( + "microcosm.build.uk_runtime.hmrc_calibration", + "HMRC_TAXABLE_SAVINGS_INTEREST_COLUMN", + ), + "HMRC_TAXPAYER_COLUMN": ( + "microcosm.build.uk_runtime.hmrc_calibration", + "HMRC_TAXPAYER_COLUMN", + ), + "UKHMRCIncomeCalibration": ( + "microcosm.build.uk_runtime.hmrc_calibration", + "UKHMRCIncomeCalibration", + ), + "UKHMRCTargetMaterialization": ( + "microcosm.build.uk_runtime.hmrc_calibration", + "UKHMRCTargetMaterialization", + ), + "calibrate_uk_hmrc_income": ( + "microcosm.build.uk_runtime.hmrc_calibration", + "calibrate_uk_hmrc_income", + ), + "materialize_uk_hmrc_calibration_frame": ( + "microcosm.build.uk_runtime.hmrc_calibration", + "materialize_uk_hmrc_calibration_frame", + ), + "HMRC_SPI_BUILD_PERIOD": ( + "microcosm.build.uk_runtime.hmrc_income", + "HMRC_SPI_BUILD_PERIOD", + ), + "HMRC_SPI_COLLATED_ODS_URL": ( + "microcosm.build.uk_runtime.hmrc_income", + "HMRC_SPI_COLLATED_ODS_URL", + ), + "HMRC_SPI_INCOME_COMPONENTS": ( + "microcosm.build.uk_runtime.hmrc_income", + "HMRC_SPI_INCOME_COMPONENTS", + ), + "HMRC_SPI_PUBLICATION_URL": ( + "microcosm.build.uk_runtime.hmrc_income", + "HMRC_SPI_PUBLICATION_URL", + ), + "HMRC_SPI_SOURCE_VINTAGE": ( + "microcosm.build.uk_runtime.hmrc_income", + "HMRC_SPI_SOURCE_VINTAGE", + ), + "HMRC_SPI_TARGET_RECORD_COUNT": ( + "microcosm.build.uk_runtime.hmrc_income", + "HMRC_SPI_TARGET_RECORD_COUNT", + ), + "HMRCIncomeBandTargetRecord": ( + "microcosm.build.uk_runtime.hmrc_income", + "HMRCIncomeBandTargetRecord", + ), + "HMRCIncomeSourceProvenance": ( + "microcosm.build.uk_runtime.hmrc_income", + "HMRCIncomeSourceProvenance", + ), + "HMRCIncomeTargetSet": ( + "microcosm.build.uk_runtime.hmrc_income", + "HMRCIncomeTargetSet", + ), + "materialize_hmrc_spi_income_band_targets": ( + "microcosm.build.uk_runtime.hmrc_income", + "materialize_hmrc_spi_income_band_targets", + ), + "verify_hmrc_spi_collated_ods": ( + "microcosm.build.uk_runtime.hmrc_income", + "verify_hmrc_spi_collated_ods", + ), + "CANONICAL_HMRC_FACT_FENCES": ( + "microcosm.build.uk_runtime.hmrc_replay", + "CANONICAL_HMRC_FACT_FENCES", + ), + "FULL_FRS_TI_BAND_FENCE_ID": ( + "microcosm.build.uk_runtime.hmrc_replay", + "FULL_FRS_TI_BAND_FENCE_ID", + ), + "HMRCFactFence": ("microcosm.build.uk_runtime.hmrc_replay", "HMRCFactFence"), + "HMRCReplayDiagnosticAggregate": ( + "microcosm.build.uk_runtime.hmrc_replay", + "HMRCReplayDiagnosticAggregate", + ), + "HMRCReplayFact": ("microcosm.build.uk_runtime.hmrc_replay", "HMRCReplayFact"), + "HMRCReplayReport": ( + "microcosm.build.uk_runtime.hmrc_replay", + "HMRCReplayReport", + ), + "build_conservative_hmrc_replay_report": ( + "microcosm.build.uk_runtime.hmrc_replay", + "build_conservative_hmrc_replay_report", + ), + "classify_hmrc_replay_targets": ( + "microcosm.build.uk_runtime.hmrc_replay", + "classify_hmrc_replay_targets", + ), + "write_hmrc_replay_report": ( + "microcosm.build.uk_runtime.hmrc_replay", + "write_hmrc_replay_report", + ), + "HMRC_DISTRIBUTIONAL_INPUTS": ( + "microcosm.build.uk_runtime.hmrc_source_contract", + "HMRC_DISTRIBUTIONAL_INPUTS", + ), + "UK_HMRC_INCOME_SOURCE_STAGES_RESOURCE": ( + "microcosm.build.uk_runtime.hmrc_source_contract", + "UK_HMRC_INCOME_SOURCE_STAGES_RESOURCE", + ), + "assert_uk_hmrc_income_source_contract_current": ( + "microcosm.build.uk_runtime.hmrc_source_contract", + "assert_uk_hmrc_income_source_contract_current", + ), + "constituency_household_targets": ( + "microcosm.build.uk_runtime.ladder_targets", + "constituency_household_targets", + ), + "ladder_target_provenance": ( + "microcosm.build.uk_runtime.ladder_targets", + "ladder_target_provenance", + ), + "ladder_vs_chronicle_household_dispersion": ( + "microcosm.build.uk_runtime.ladder_targets", + "ladder_vs_chronicle_household_dispersion", + ), + "local_authority_household_targets": ( + "microcosm.build.uk_runtime.ladder_targets", + "local_authority_household_targets", + ), + "UK_CENSUS_HOUSEHOLDS_TARGET_ID": ( + "microcosm.build.uk_runtime.ledger_targets", + "UK_CENSUS_HOUSEHOLDS_TARGET_ID", + ), + "UK_CROSS_GRAIN_BRIDGES": ( + "microcosm.build.uk_runtime.ledger_targets", + "UK_CROSS_GRAIN_BRIDGES", + ), + "UK_CROSS_GRAIN_GRAIN_PRECEDENCE": ( + "microcosm.build.uk_runtime.ledger_targets", + "UK_CROSS_GRAIN_GRAIN_PRECEDENCE", + ), + "UK_CROSS_GRAIN_RULE": ( + "microcosm.build.uk_runtime.ledger_targets", + "UK_CROSS_GRAIN_RULE", + ), + "UKFrameTargetAdapter": ( + "microcosm.build.uk_runtime.ledger_targets", + "UKFrameTargetAdapter", + ), + "UKLedgerTargetCompilation": ( + "microcosm.build.uk_runtime.ledger_targets", + "UKLedgerTargetCompilation", + ), + "apply_uk_cross_grain_reconciliation": ( + "microcosm.build.uk_runtime.ledger_targets", + "apply_uk_cross_grain_reconciliation", + ), + "compile_uk_local_target_registry": ( + "microcosm.build.uk_runtime.ledger_targets", + "compile_uk_local_target_registry", + ), + "compile_uk_target_registry": ( + "microcosm.build.uk_runtime.ledger_targets", + "compile_uk_target_registry", + ), + "load_uk_local_area_crosswalk": ( + "microcosm.build.uk_runtime.ledger_targets", + "load_uk_local_area_crosswalk", + ), + "materialize_uk_ledger_targets": ( + "microcosm.build.uk_runtime.ledger_targets", + "materialize_uk_ledger_targets", + ), + "uk_census_household_uprating": ( + "microcosm.build.uk_runtime.ledger_targets", + "uk_census_household_uprating", + ), + "uk_ledger_households_total": ( + "microcosm.build.uk_runtime.ledger_targets", + "uk_ledger_households_total", + ), + "uk_local_target_surface": ( + "microcosm.build.uk_runtime.ledger_targets", + "uk_local_target_surface", + ), + "UK_LOCAL_CLONE_COUNT": ( + "microcosm.build.uk_runtime.local_doctrine", + "UK_LOCAL_CLONE_COUNT", + ), + "UK_LOCAL_MAX_WEIGHT_RATIO": ( + "microcosm.build.uk_runtime.local_doctrine", + "UK_LOCAL_MAX_WEIGHT_RATIO", + ), + "UK_LOCAL_SOLVE_DOCTRINE": ( + "microcosm.build.uk_runtime.local_doctrine", + "UK_LOCAL_SOLVE_DOCTRINE", + ), + "UK_LOCAL_SOLVE_EPOCHS": ( + "microcosm.build.uk_runtime.local_doctrine", + "UK_LOCAL_SOLVE_EPOCHS", + ), + "UK_LOCAL_TARGET_LOSS_CAP": ( + "microcosm.build.uk_runtime.local_doctrine", + "UK_LOCAL_TARGET_LOSS_CAP", + ), + "UK_LOCAL_TARGET_WEIGHT_RULE": ( + "microcosm.build.uk_runtime.local_doctrine", + "UK_LOCAL_TARGET_WEIGHT_RULE", + ), + "UKLocalSolveDoctrine": ( + "microcosm.build.uk_runtime.local_doctrine", + "UKLocalSolveDoctrine", + ), + "uk_local_doctrine_with_overrides": ( + "microcosm.build.uk_runtime.local_doctrine", + "uk_local_doctrine_with_overrides", + ), + "uk_local_target_loss_weights": ( + "microcosm.build.uk_runtime.local_doctrine", + "uk_local_target_loss_weights", + ), + "align_area_targets": ( + "microcosm.build.uk_runtime.local_geography", + "align_area_targets", + ), + "UK_LOCAL_BINDING_ADJUDICATION_REGISTER_RESOURCE": ( + "microcosm.build.uk_runtime.local_rowwise", + "UK_LOCAL_BINDING_ADJUDICATION_REGISTER_RESOURCE", + ), + "UK_LOCAL_HOLDOUT_FOLDS": ( + "microcosm.build.uk_runtime.local_rowwise", + "UK_LOCAL_HOLDOUT_FOLDS", + ), + "UK_LOCAL_HOLDOUT_SEED": ( + "microcosm.build.uk_runtime.local_rowwise", + "UK_LOCAL_HOLDOUT_SEED", + ), + "UKRowwiseDoctrineSolve": ( + "microcosm.build.uk_runtime.local_rowwise", + "UKRowwiseDoctrineSolve", + ), + "UKRowwiseLocalMatrix": ( + "microcosm.build.uk_runtime.local_rowwise", + "UKRowwiseLocalMatrix", + ), + "UKRowwiseNationalRows": ( + "microcosm.build.uk_runtime.local_rowwise", + "UKRowwiseNationalRows", + ), + "build_uk_rowwise_local_matrix": ( + "microcosm.build.uk_runtime.local_rowwise", + "build_uk_rowwise_local_matrix", + ), + "build_uk_rowwise_local_surface_matrix": ( + "microcosm.build.uk_runtime.local_rowwise", + "build_uk_rowwise_local_surface_matrix", + ), + "past_cap_census": ( + "microcosm.build.uk_runtime.local_rowwise", + "past_cap_census", + ), + "require_adjudicated_uk_local_binding": ( + "microcosm.build.uk_runtime.local_rowwise", + "require_adjudicated_uk_local_binding", + ), + "rotated_uk_local_holdout": ( + "microcosm.build.uk_runtime.local_rowwise", + "rotated_uk_local_holdout", + ), + "rowwise_area_support_summary": ( + "microcosm.build.uk_runtime.local_rowwise", + "rowwise_area_support_summary", + ), + "rowwise_calibration_mass_reason": ( + "microcosm.build.uk_runtime.local_rowwise", + "rowwise_calibration_mass_reason", + ), + "solve_uk_rowwise_weights_under_doctrine": ( + "microcosm.build.uk_runtime.local_rowwise", + "solve_uk_rowwise_weights_under_doctrine", + ), + "uk_area_support_summary": ( + "microcosm.build.uk_runtime.local_rowwise", + "uk_area_support_summary", + ), + "uk_ladder_area_support_summary": ( + "microcosm.build.uk_runtime.local_rowwise", + "uk_ladder_area_support_summary", + ), + "CENSUS_KIND": ( + "microcosm.build.uk_runtime.local_target_census", + "CENSUS_KIND", + ), + "CENSUS_RESOURCE": ( + "microcosm.build.uk_runtime.local_target_census", + "CENSUS_RESOURCE", + ), + "CENSUS_SCHEMA_VERSION": ( + "microcosm.build.uk_runtime.local_target_census", + "CENSUS_SCHEMA_VERSION", + ), + "METRIC_STATUS_BOUND_IN_CODE": ( + "microcosm.build.uk_runtime.local_target_census", + "METRIC_STATUS_BOUND_IN_CODE", + ), + "SOURCE_STATUS_DOCUMENTED_UNPINNED": ( + "microcosm.build.uk_runtime.local_target_census", + "SOURCE_STATUS_DOCUMENTED_UNPINNED", + ), + "assert_uk_local_target_census_current": ( + "microcosm.build.uk_runtime.local_target_census", + "assert_uk_local_target_census_current", + ), + "build_uk_local_target_census": ( + "microcosm.build.uk_runtime.local_target_census", + "build_uk_local_target_census", + ), + "committed_uk_local_target_census_path": ( + "microcosm.build.uk_runtime.local_target_census", + "committed_uk_local_target_census_path", + ), + "load_uk_local_target_census": ( + "microcosm.build.uk_runtime.local_target_census", + "load_uk_local_target_census", + ), + "write_uk_local_target_census": ( + "microcosm.build.uk_runtime.local_target_census", + "write_uk_local_target_census", + ), + "AGE_BANDS": ("microcosm.build.uk_runtime.local_targets", "AGE_BANDS"), + "AREA_TYPE_TO_LEDGER_GEOGRAPHY_LEVEL": ( + "microcosm.build.uk_runtime.local_targets", + "AREA_TYPE_TO_LEDGER_GEOGRAPHY_LEVEL", + ), + "AREA_TYPES": ("microcosm.build.uk_runtime.local_targets", "AREA_TYPES"), + "COUNTRY_TO_REGION": ( + "microcosm.build.uk_runtime.local_targets", + "COUNTRY_TO_REGION", + ), + "INCOME_VARIABLES": ( + "microcosm.build.uk_runtime.local_targets", + "INCOME_VARIABLES", + ), + "LA_EXTRA_METRICS": ( + "microcosm.build.uk_runtime.local_targets", + "LA_EXTRA_METRICS", + ), + "area_groups_from_codes": ( + "microcosm.build.uk_runtime.local_targets", + "area_groups_from_codes", + ), + "compute_household_metrics": ( + "microcosm.build.uk_runtime.local_targets", + "compute_household_metrics", + ), + "metric_names": ("microcosm.build.uk_runtime.local_targets", "metric_names"), + "metric_names_from_target_profile": ( + "microcosm.build.uk_runtime.local_targets", + "metric_names_from_target_profile", + ), + "metric_tables_by_area_group": ( + "microcosm.build.uk_runtime.local_targets", + "metric_tables_by_area_group", + ), + "CalibrationFrameAdapter": ( + "microcosm.build.uk_runtime.national_calibration", + "CalibrationFrameAdapter", + ), + "drop_injected_measure_inputs": ( + "microcosm.build.uk_runtime.national_calibration", + "drop_injected_measure_inputs", + ), + "inject_measure_inputs": ( + "microcosm.build.uk_runtime.national_calibration", + "inject_measure_inputs", + ), + "prepare_uk_target_frame": ( + "microcosm.build.uk_runtime.national_calibration", + "prepare_uk_target_frame", + ), + "UK_NATIONAL_L0_LAMBDA": ( + "microcosm.build.uk_runtime.national_doctrine", + "UK_NATIONAL_L0_LAMBDA", + ), + "UK_NATIONAL_LEARNING_RATE": ( + "microcosm.build.uk_runtime.national_doctrine", + "UK_NATIONAL_LEARNING_RATE", + ), + "UK_NATIONAL_MASS_RULE": ( + "microcosm.build.uk_runtime.national_doctrine", + "UK_NATIONAL_MASS_RULE", + ), + "UK_NATIONAL_MAX_WEIGHT_RATIO": ( + "microcosm.build.uk_runtime.national_doctrine", + "UK_NATIONAL_MAX_WEIGHT_RATIO", + ), + "UK_NATIONAL_SEED": ( + "microcosm.build.uk_runtime.national_doctrine", + "UK_NATIONAL_SEED", + ), + "UK_NATIONAL_SOLVE_DOCTRINE": ( + "microcosm.build.uk_runtime.national_doctrine", + "UK_NATIONAL_SOLVE_DOCTRINE", + ), + "UK_NATIONAL_SOLVE_EPOCHS": ( + "microcosm.build.uk_runtime.national_doctrine", + "UK_NATIONAL_SOLVE_EPOCHS", + ), + "UK_NATIONAL_TARGET_LOSS_CAP": ( + "microcosm.build.uk_runtime.national_doctrine", + "UK_NATIONAL_TARGET_LOSS_CAP", + ), + "UK_NATIONAL_TARGET_WEIGHT_RULE": ( + "microcosm.build.uk_runtime.national_doctrine", + "UK_NATIONAL_TARGET_WEIGHT_RULE", + ), + "UKNationalSolveDoctrine": ( + "microcosm.build.uk_runtime.national_doctrine", + "UKNationalSolveDoctrine", + ), + "uk_doctrine_with_overrides": ( + "microcosm.build.uk_runtime.national_doctrine", + "uk_doctrine_with_overrides", + ), + "uk_national_target_loss_weights": ( + "microcosm.build.uk_runtime.national_doctrine", + "uk_national_target_loss_weights", + ), + "UK_NATIONAL_SCHEMA": ( + "microcosm.build.uk_runtime.national_frame", + "UK_NATIONAL_SCHEMA", + ), + "UKNationalStage": ( + "microcosm.build.uk_runtime.national_frame", + "UKNationalStage", + ), + "UKStagingProvenance": ( + "microcosm.build.uk_runtime.national_frame", + "UKStagingProvenance", + ), + "load_uk_national_frame": ( + "microcosm.build.uk_runtime.national_frame", + "load_uk_national_frame", + ), + "uk_household_weight_kind": ( + "microcosm.build.uk_runtime.national_frame", + "uk_household_weight_kind", + ), + "uk_national_frame": ( + "microcosm.build.uk_runtime.national_frame", + "uk_national_frame", + ), + "uk_time_period": ( + "microcosm.build.uk_runtime.national_frame", + "uk_time_period", + ), + "validate_uk_national_frame": ( + "microcosm.build.uk_runtime.national_frame", + "validate_uk_national_frame", + ), + "write_uk_national_frame": ( + "microcosm.build.uk_runtime.national_frame", + "write_uk_national_frame", + ), + "sample_uk_spine_frame": ( + "microcosm.build.uk_runtime.national_sampling", + "sample_uk_spine_frame", + ), + "uk_spine_source_family_units": ( + "microcosm.build.uk_runtime.national_sampling", + "uk_spine_source_family_units", + ), + "LADDER_OA_COLUMNS": ( + "microcosm.build.uk_runtime.oa_ladder_sources", + "LADDER_OA_COLUMNS", + ), + "assemble_uk_oa_ladder": ( + "microcosm.build.uk_runtime.oa_ladder_sources", + "assemble_uk_oa_ladder", + ), + "concat_uk_ladder_frames": ( + "microcosm.build.uk_runtime.oa_ladder_sources", + "concat_uk_ladder_frames", + ), + "join_uk_oa_ladder_layers": ( + "microcosm.build.uk_runtime.oa_ladder_sources", + "join_uk_oa_ladder_layers", + ), + "EFRS_PARITY_KNOWN_GAPS_RESOURCE": ( + "microcosm.build.uk_runtime.parity_reference", + "EFRS_PARITY_KNOWN_GAPS_RESOURCE", + ), + "EFRS_PARITY_REFERENCE_RESOURCE": ( + "microcosm.build.uk_runtime.parity_reference", + "EFRS_PARITY_REFERENCE_RESOURCE", + ), + "EfrsParityKnownGap": ( + "microcosm.build.uk_runtime.parity_reference", + "EfrsParityKnownGap", + ), + "EfrsParityReference": ( + "microcosm.build.uk_runtime.parity_reference", + "EfrsParityReference", + ), + "EfrsParitySource": ( + "microcosm.build.uk_runtime.parity_reference", + "EfrsParitySource", + ), + "load_efrs_parity_known_gaps": ( + "microcosm.build.uk_runtime.parity_reference", + "load_efrs_parity_known_gaps", + ), + "load_efrs_parity_reference": ( + "microcosm.build.uk_runtime.parity_reference", + "load_efrs_parity_reference", + ), + "UK_DENSE_RELEASE_ID": ( + "microcosm.build.uk_runtime.release_identity", + "UK_DENSE_RELEASE_ID", + ), + "UK_RELEASE_TIER_CPS_TRANSFER": ( + "microcosm.build.uk_runtime.release_identity", + "UK_RELEASE_TIER_CPS_TRANSFER", + ), + "UK_RELEASE_TIER_FRS": ( + "microcosm.build.uk_runtime.release_identity", + "UK_RELEASE_TIER_FRS", + ), + "UK_RELEASE_TIERS": ( + "microcosm.build.uk_runtime.release_identity", + "UK_RELEASE_TIERS", + ), + "UKReleaseIdentity": ( + "microcosm.build.uk_runtime.release_identity", + "UKReleaseIdentity", + ), + "apply_uk_release_identity": ( + "microcosm.build.uk_runtime.release_identity", + "apply_uk_release_identity", + ), + "format_uk_release_id": ( + "microcosm.build.uk_runtime.release_identity", + "format_uk_release_id", + ), + "validate_uk_release_tier": ( + "microcosm.build.uk_runtime.release_identity", + "validate_uk_release_tier", + ), + "RESTORED_REFERENCE_EFRS_REQUIRED_INPUTS": ( + "microcosm.build.uk_runtime.release_input_coverage", + "RESTORED_REFERENCE_EFRS_REQUIRED_INPUTS", + ), + "UK_LOADER_INPUT_ALIASES": ( + "microcosm.build.uk_runtime.release_input_coverage", + "UK_LOADER_INPUT_ALIASES", + ), + "UK_RELEASE_INPUT_COVERAGE_RESOURCE": ( + "microcosm.build.uk_runtime.release_input_coverage", + "UK_RELEASE_INPUT_COVERAGE_RESOURCE", + ), + "PolicyEngineUKCoverageEngine": ( + "microcosm.build.uk_runtime.release_input_coverage", + "PolicyEngineUKCoverageEngine", + ), + "UKEffectiveMassCoveragePolicy": ( + "microcosm.build.uk_runtime.release_input_coverage", + "UKEffectiveMassCoveragePolicy", + ), + "UKReleaseInputColumn": ( + "microcosm.build.uk_runtime.release_input_coverage", + "UKReleaseInputColumn", + ), + "UKReleaseInputCoverageManifest": ( + "microcosm.build.uk_runtime.release_input_coverage", + "UKReleaseInputCoverageManifest", + ), + "assert_uk_release_input_coverage_build_stages": ( + "microcosm.build.uk_runtime.release_input_coverage", + "assert_uk_release_input_coverage_build_stages", + ), + "assert_uk_release_input_coverage_manifest_current": ( + "microcosm.build.uk_runtime.release_input_coverage", + "assert_uk_release_input_coverage_manifest_current", + ), + "load_uk_release_input_coverage_manifest": ( + "microcosm.build.uk_runtime.release_input_coverage", + "load_uk_release_input_coverage_manifest", + ), + "uk_release_input_coverage_gate": ( + "microcosm.build.uk_runtime.release_input_coverage", + "uk_release_input_coverage_gate", + ), + "uk_release_input_coverage_required_columns": ( + "microcosm.build.uk_runtime.release_input_coverage", + "uk_release_input_coverage_required_columns", + ), + "uk_release_input_coverage_reviewed_exclusions": ( + "microcosm.build.uk_runtime.release_input_coverage", + "uk_release_input_coverage_reviewed_exclusions", + ), + "ARTIFACT_CLONE_INDEX_COLUMN": ( + "microcosm.build.uk_runtime.rowwise_dataset", + "ARTIFACT_CLONE_INDEX_COLUMN", + ), + "BENUNIT_ID_COLUMNS": ( + "microcosm.build.uk_runtime.rowwise_dataset", + "BENUNIT_ID_COLUMNS", + ), + "HOUSEHOLD_ID_COLUMNS": ( + "microcosm.build.uk_runtime.rowwise_dataset", + "HOUSEHOLD_ID_COLUMNS", + ), + "MASS_CONSERVATION_RELATIVE_TOLERANCE": ( + "microcosm.build.uk_runtime.rowwise_dataset", + "MASS_CONSERVATION_RELATIVE_TOLERANCE", + ), + "PERSON_ID_COLUMNS": ( + "microcosm.build.uk_runtime.rowwise_dataset", + "PERSON_ID_COLUMNS", + ), + "POOL_SOURCE_LINEAGE_COLUMN": ( + "microcosm.build.uk_runtime.rowwise_dataset", + "POOL_SOURCE_LINEAGE_COLUMN", + ), + "UK_SINGLE_YEAR_TABLES": ( + "microcosm.build.uk_runtime.rowwise_dataset", + "UK_SINGLE_YEAR_TABLES", + ), + "UKLadderRowwiseDatasetResult": ( + "microcosm.build.uk_runtime.rowwise_dataset", + "UKLadderRowwiseDatasetResult", + ), + "UKRowwiseDatasetResult": ( + "microcosm.build.uk_runtime.rowwise_dataset", + "UKRowwiseDatasetResult", + ), + "apply_uk_source_lineage_modulus": ( + "microcosm.build.uk_runtime.rowwise_dataset", + "apply_uk_source_lineage_modulus", + ), + "clone_uk_dataset_tables_with_ladder_geography": ( + "microcosm.build.uk_runtime.rowwise_dataset", + "clone_uk_dataset_tables_with_ladder_geography", + ), + "clone_uk_dataset_tables_with_rowwise_geography": ( + "microcosm.build.uk_runtime.rowwise_dataset", + "clone_uk_dataset_tables_with_rowwise_geography", + ), + "clone_uk_dataset_with_ladder_geography": ( + "microcosm.build.uk_runtime.rowwise_dataset", + "clone_uk_dataset_with_ladder_geography", + ), + "clone_uk_dataset_with_rowwise_geography": ( + "microcosm.build.uk_runtime.rowwise_dataset", + "clone_uk_dataset_with_rowwise_geography", + ), + "ladder_clone_index_column": ( + "microcosm.build.uk_runtime.rowwise_dataset", + "ladder_clone_index_column", + ), + "load_uk_rowwise_dataset": ( + "microcosm.build.uk_runtime.rowwise_dataset", + "load_uk_rowwise_dataset", + ), + "read_uk_single_year_weight_metadata": ( + "microcosm.build.uk_runtime.rowwise_dataset", + "read_uk_single_year_weight_metadata", + ), + "validate_uk_ladder_rowwise_dataset_tables": ( + "microcosm.build.uk_runtime.rowwise_dataset", + "validate_uk_ladder_rowwise_dataset_tables", + ), + "validate_uk_rowwise_dataset_tables": ( + "microcosm.build.uk_runtime.rowwise_dataset", + "validate_uk_rowwise_dataset_tables", + ), + "write_uk_rowwise_dataset": ( + "microcosm.build.uk_runtime.rowwise_dataset", + "write_uk_rowwise_dataset", + ), + "AREA_TYPE_TO_CROSSWALK_COLUMN": ( + "microcosm.build.uk_runtime.rowwise_geography", + "AREA_TYPE_TO_CROSSWALK_COLUMN", + ), + "CROSSWALK_COLUMNS": ( + "microcosm.build.uk_runtime.rowwise_geography", + "CROSSWALK_COLUMNS", + ), + "FRS_REGION_TO_COUNTRY": ( + "microcosm.build.uk_runtime.rowwise_geography", + "FRS_REGION_TO_COUNTRY", + ), + "FRS_REGION_TO_REGION_CODE": ( + "microcosm.build.uk_runtime.rowwise_geography", + "FRS_REGION_TO_REGION_CODE", + ), + "ROWWISE_GEOGRAPHY_COLUMNS": ( + "microcosm.build.uk_runtime.rowwise_geography", + "ROWWISE_GEOGRAPHY_COLUMNS", + ), + "RowwiseGeographyAssignment": ( + "microcosm.build.uk_runtime.rowwise_geography", + "RowwiseGeographyAssignment", + ), + "assign_household_geography": ( + "microcosm.build.uk_runtime.rowwise_geography", + "assign_household_geography", + ), + "clone_entity_frame": ( + "microcosm.build.uk_runtime.rowwise_geography", + "clone_entity_frame", + ), + "expected_uk_rowwise_area_support": ( + "microcosm.build.uk_runtime.rowwise_geography", + "expected_uk_rowwise_area_support", + ), + "geography_coverage_summary": ( + "microcosm.build.uk_runtime.rowwise_geography", + "geography_coverage_summary", + ), + "id_multiplier_for_values": ( + "microcosm.build.uk_runtime.rowwise_geography", + "id_multiplier_for_values", + ), + "prepare_geography_crosswalk": ( + "microcosm.build.uk_runtime.rowwise_geography", + "prepare_geography_crosswalk", + ), + "validate_geography_coverage": ( + "microcosm.build.uk_runtime.rowwise_geography", + "validate_geography_coverage", + ), + "PRE_REGISTERED_OUTCOMES_V1": ( + "microcosm.build.uk_runtime.size_evaluation", + "PRE_REGISTERED_OUTCOMES_V1", + ), + "RowwiseRun": ("microcosm.build.uk_runtime.size_evaluation", "RowwiseRun"), + "area_support_tables": ( + "microcosm.build.uk_runtime.size_evaluation", + "area_support_tables", + ), + "dense_reference_deltas": ( + "microcosm.build.uk_runtime.size_evaluation", + "dense_reference_deltas", + ), + "fit_tables": ("microcosm.build.uk_runtime.size_evaluation", "fit_tables"), + "footprint": ("microcosm.build.uk_runtime.size_evaluation", "footprint"), + "frozen_vs_recomputed": ( + "microcosm.build.uk_runtime.size_evaluation", + "frozen_vs_recomputed", + ), + "gate_table": ("microcosm.build.uk_runtime.size_evaluation", "gate_table"), + "load_run": ("microcosm.build.uk_runtime.size_evaluation", "load_run"), + "paired_targets": ( + "microcosm.build.uk_runtime.size_evaluation", + "paired_targets", + ), + "run_acceptance": ( + "microcosm.build.uk_runtime.size_evaluation", + "run_acceptance", + ), + "summarize": ("microcosm.build.uk_runtime.size_evaluation", "summarize"), + "weight_tables": ( + "microcosm.build.uk_runtime.size_evaluation", + "weight_tables", + ), + "SPI_DONOR_DOI": ("microcosm.build.uk_runtime.spi_income", "SPI_DONOR_DOI"), + "SPI_DONOR_FILENAME": ( + "microcosm.build.uk_runtime.spi_income", + "SPI_DONOR_FILENAME", + ), + "SPI_DONOR_SHA256": ( + "microcosm.build.uk_runtime.spi_income", + "SPI_DONOR_SHA256", + ), + "SPI_DONOR_SIZE_BYTES": ( + "microcosm.build.uk_runtime.spi_income", + "SPI_DONOR_SIZE_BYTES", + ), + "SPI_DONOR_UKDS_STUDY": ( + "microcosm.build.uk_runtime.spi_income", + "SPI_DONOR_UKDS_STUDY", + ), + "SPI_DONOR_VINTAGE": ( + "microcosm.build.uk_runtime.spi_income", + "SPI_DONOR_VINTAGE", + ), + "SPI_STAGE2_REVIEWED_ABSENT_OUTPUTS": ( + "microcosm.build.uk_runtime.spi_income", + "SPI_STAGE2_REVIEWED_ABSENT_OUTPUTS", + ), + "UKSPIIncomeImputationResult": ( + "microcosm.build.uk_runtime.spi_income", + "UKSPIIncomeImputationResult", + ), + "assert_frs_hmrc_auxiliary_crosswalk_available": ( + "microcosm.build.uk_runtime.spi_income", + "assert_frs_hmrc_auxiliary_crosswalk_available", + ), + "derive_hmrc_income_auxiliaries": ( + "microcosm.build.uk_runtime.spi_income", + "derive_hmrc_income_auxiliaries", + ), + "impute_uk_spi_income_support": ( + "microcosm.build.uk_runtime.spi_income", + "impute_uk_spi_income_support", + ), + "verify_spi_donor_identity": ( + "microcosm.build.uk_runtime.spi_income", + "verify_spi_donor_identity", + ), + "EMPLOYER_PENSION_CONTRIBUTIONS_COLUMN": ( + "microcosm.build.uk_runtime.spi_spine", + "EMPLOYER_PENSION_CONTRIBUTIONS_COLUMN", + ), + "UK_FRS_HMRC_SPINE_LEAF_OUTPUT_COLUMNS": ( + "microcosm.build.uk_runtime.spi_spine", + "UK_FRS_HMRC_SPINE_LEAF_OUTPUT_COLUMNS", + ), + "UK_FRS_HMRC_SPINE_LEAVES_STAGE_NAME": ( + "microcosm.build.uk_runtime.spi_spine", + "UK_FRS_HMRC_SPINE_LEAVES_STAGE_NAME", + ), + "UK_HMRC_SPI_INCOME_SPINE_STAGE_NAME": ( + "microcosm.build.uk_runtime.spi_spine", + "UK_HMRC_SPI_INCOME_SPINE_STAGE_NAME", + ), + "UK_HMRC_SPI_SPINE_REPLAY_REPORT_KIND": ( + "microcosm.build.uk_runtime.spi_spine", + "UK_HMRC_SPI_SPINE_REPLAY_REPORT_KIND", + ), + "UK_SPI_INCOME_SPINE_NONNEGATIVE_OUTPUT_COLUMNS": ( + "microcosm.build.uk_runtime.spi_spine", + "UK_SPI_INCOME_SPINE_NONNEGATIVE_OUTPUT_COLUMNS", + ), + "UK_SPI_INCOME_SPINE_OUTPUT_COLUMNS": ( + "microcosm.build.uk_runtime.spi_spine", + "UK_SPI_INCOME_SPINE_OUTPUT_COLUMNS", + ), + "UK_SPI_INCOME_SPINE_REWRITE_COLUMNS": ( + "microcosm.build.uk_runtime.spi_spine", + "UK_SPI_INCOME_SPINE_REWRITE_COLUMNS", + ), + "UK_SPI_SUPPORT_CHANNEL_OUTPUT_COLUMNS": ( + "microcosm.build.uk_runtime.spi_spine", + "UK_SPI_SUPPORT_CHANNEL_OUTPUT_COLUMNS", + ), + "UKFRSHMRCSpineLeavesResult": ( + "microcosm.build.uk_runtime.spi_spine", + "UKFRSHMRCSpineLeavesResult", + ), + "UKFRSHMRCSpineLeavesStageTransform": ( + "microcosm.build.uk_runtime.spi_spine", + "UKFRSHMRCSpineLeavesStageTransform", + ), + "UKSPIIncomeSpineResult": ( + "microcosm.build.uk_runtime.spi_spine", + "UKSPIIncomeSpineResult", + ), + "UKSPIIncomeSpineStageTransform": ( + "microcosm.build.uk_runtime.spi_spine", + "UKSPIIncomeSpineStageTransform", + ), + "UKSPISupportChannelStageTransform": ( + "microcosm.build.uk_runtime.spi_spine", + "UKSPISupportChannelStageTransform", + ), + "BASE_FRS_SUPPORT_CHANNEL": ( + "microcosm.build.uk_runtime.spi_support", + "BASE_FRS_SUPPORT_CHANNEL", + ), + "DEFAULT_SPI_PRIOR_MASS_SHARE": ( + "microcosm.build.uk_runtime.spi_support", + "DEFAULT_SPI_PRIOR_MASS_SHARE", + ), + "DEFAULT_SPI_SUPPORT_HOUSEHOLDS": ( + "microcosm.build.uk_runtime.spi_support", + "DEFAULT_SPI_SUPPORT_HOUSEHOLDS", + ), + "FRS_ONLY_SPI_FILL_INCOME_PREDICTOR_COLUMNS": ( + "microcosm.build.uk_runtime.spi_support", + "FRS_ONLY_SPI_FILL_INCOME_PREDICTOR_COLUMNS", + ), + "FRS_ONLY_SPI_FILL_PERSON_COLUMNS": ( + "microcosm.build.uk_runtime.spi_support", + "FRS_ONLY_SPI_FILL_PERSON_COLUMNS", + ), + "FRS_ONLY_SPI_FILL_PREDICTOR_COLUMNS": ( + "microcosm.build.uk_runtime.spi_support", + "FRS_ONLY_SPI_FILL_PREDICTOR_COLUMNS", + ), + "HOUSEHOLD_IS_SPI_SYNTHETIC_COLUMN": ( + "microcosm.build.uk_runtime.spi_support", + "HOUSEHOLD_IS_SPI_SYNTHETIC_COLUMN", + ), + "SPI_INCOME_COMPONENT_COLUMNS": ( + "microcosm.build.uk_runtime.spi_support", + "SPI_INCOME_COMPONENT_COLUMNS", + ), + "SPI_INCOME_IMPUTATION_COLUMNS": ( + "microcosm.build.uk_runtime.spi_support", + "SPI_INCOME_IMPUTATION_COLUMNS", + ), + "SPI_PRIOR_MASS_CHANGE_REASON": ( + "microcosm.build.uk_runtime.spi_support", + "SPI_PRIOR_MASS_CHANGE_REASON", + ), + "SPI_REPLACEMENT_STRATA_COLUMNS": ( + "microcosm.build.uk_runtime.spi_support", + "SPI_REPLACEMENT_STRATA_COLUMNS", + ), + "SPI_SYNTHETIC_SUPPORT_CHANNEL": ( + "microcosm.build.uk_runtime.spi_support", + "SPI_SYNTHETIC_SUPPORT_CHANNEL", + ), + "UK_SPI_SUPPORT_STAGE_NAME": ( + "microcosm.build.uk_runtime.spi_support", + "UK_SPI_SUPPORT_STAGE_NAME", + ), + "UKSPISupportResult": ( + "microcosm.build.uk_runtime.spi_support", + "UKSPISupportResult", + ), + "build_uk_spi_support_channel": ( + "microcosm.build.uk_runtime.spi_support", + "build_uk_spi_support_channel", + ), + "create_uk_spi_support_tables": ( + "microcosm.build.uk_runtime.spi_support", + "create_uk_spi_support_tables", + ), + "fill_support_channel_from_source": ( + "microcosm.build.uk_runtime.spi_support", + "fill_support_channel_from_source", + ), + "replace_uk_spi_support_tables": ( + "microcosm.build.uk_runtime.spi_support", + "replace_uk_spi_support_tables", + ), + "support_channel_column": ( + "microcosm.build.uk_runtime.spi_support", + "support_channel_column", + ), + "support_clone_index_column": ( + "microcosm.build.uk_runtime.spi_support", + "support_clone_index_column", + ), + "support_source_id_column": ( + "microcosm.build.uk_runtime.spi_support", + "support_source_id_column", + ), + "UK_FRAME_METADATA_KEY": ( + "microcosm.build.uk_runtime.stage_checkpoints", + "UK_FRAME_METADATA_KEY", + ), + "load_uk_stage_checkpoint": ( + "microcosm.build.uk_runtime.stage_checkpoints", + "load_uk_stage_checkpoint", + ), + "load_uk_stage_predecessor": ( + "microcosm.build.uk_runtime.stage_checkpoints", + "load_uk_stage_predecessor", + ), + "uk_stage_metadata": ( + "microcosm.build.uk_runtime.stage_checkpoints", + "uk_stage_metadata", + ), + "UK_DEFAULT_ZERO_WEIGHT_STRATA": ( + "microcosm.build.uk_runtime.terminal_gates", + "UK_DEFAULT_ZERO_WEIGHT_STRATA", + ), + "UK_MAX_TARGET_ABS_RELATIVE_ERROR": ( + "microcosm.build.uk_runtime.terminal_gates", + "UK_MAX_TARGET_ABS_RELATIVE_ERROR", + ), + "UKZeroWeightStratumDeclaration": ( + "microcosm.build.uk_runtime.terminal_gates", + "UKZeroWeightStratumDeclaration", + ), + "uk_degenerate_release_surface_gate": ( + "microcosm.build.uk_runtime.terminal_gates", + "uk_degenerate_release_surface_gate", + ), + "uk_export_surface_gate": ( + "microcosm.build.uk_runtime.terminal_gates", + "uk_export_surface_gate", + ), + "uk_target_fit_gate": ( + "microcosm.build.uk_runtime.terminal_gates", + "uk_target_fit_gate", + ), + "uk_target_surface_gate": ( + "microcosm.build.uk_runtime.terminal_gates", + "uk_target_surface_gate", + ), + "uk_weight_ess_gate": ( + "microcosm.build.uk_runtime.terminal_gates", + "uk_weight_ess_gate", + ), + "uk_weight_ratio_gate": ( + "microcosm.build.uk_runtime.terminal_gates", + "uk_weight_ratio_gate", + ), + "uk_zero_weight_strata_gate": ( + "microcosm.build.uk_runtime.terminal_gates", + "uk_zero_weight_strata_gate", + ), + "UKInputMassParityPolicy": ( + "microcosm.build.uk_runtime.weighted_integrity", + "UKInputMassParityPolicy", + ), + "UKInputMassReference": ( + "microcosm.build.uk_runtime.weighted_integrity", + "UKInputMassReference", + ), + "UKQRFTailConcentrationPolicy": ( + "microcosm.build.uk_runtime.weighted_integrity", + "UKQRFTailConcentrationPolicy", + ), + "load_uk_input_mass_reference": ( + "microcosm.build.uk_runtime.weighted_integrity", + "load_uk_input_mass_reference", + ), + "load_uk_local_area_support_exclusion_register": ( + "microcosm.build.uk_runtime.weighted_integrity", + "load_uk_local_area_support_exclusion_register", + ), + "load_uk_reviewed_exclusion_register": ( + "microcosm.build.uk_runtime.weighted_integrity", + "load_uk_reviewed_exclusion_register", + ), + "uk_input_mass_parity_gate": ( + "microcosm.build.uk_runtime.weighted_integrity", + "uk_input_mass_parity_gate", + ), + "uk_input_mass_totals": ( + "microcosm.build.uk_runtime.weighted_integrity", + "uk_input_mass_totals", + ), + "uk_qrf_tail_concentration_columns": ( + "microcosm.build.uk_runtime.weighted_integrity", + "uk_qrf_tail_concentration_columns", + ), + "uk_qrf_tail_concentration_gate": ( + "microcosm.build.uk_runtime.weighted_integrity", + "uk_qrf_tail_concentration_gate", + ), + } +) + +# Reload clears public aliases, retaining the real defining modules and submodules. +for _export_name in __all__: + globals().pop(_export_name, None) +del _export_name + + +def __getattr__(name: str) -> _Any: + """Resolve and cache the exact public object, preserving import failures.""" + try: + module_name, attribute = _EXPORTS[name] + except KeyError: + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") from None + value = getattr(_import_module(module_name), attribute) + globals()[name] = value + return value + + +def __dir__() -> list[str]: + """Discover declared exports without importing their defining modules.""" + return sorted(set(globals()) | set(__all__)) diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/atomic_area_support.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/atomic_area_support.py new file mode 100644 index 000000000..a6fedc9d6 --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/atomic_area_support.py @@ -0,0 +1,388 @@ +"""Convert supplied UK ladder arrays for shared atomic geography operators. + +This module does not acquire sources, admit native data, calibrate, or assign a +location. Source identities and mapping classifications are explicit caller +claims; publisher authentication belongs to the host. Initial spine/support +clones must exist before the resulting declaration is executed. +""" + +from __future__ import annotations + +import hashlib +import re +from collections.abc import Mapping +from numbers import Integral + +import numpy as np + +from microcosm.build.atomic_geography import ( + RELATIONS, + decode_atomic_support, + encode_atomic_support, + validate_assignment_spec, +) +from microcosm.build.uk_runtime.rowwise_geography import FRS_REGION_TO_REGION_CODE +from microcosm.graph.canonical import canonical_json + +SYSTEMS = ( + "uk_ew_output_area_2021", + "uk_scotland_output_area_2022", + "uk_ni_data_zone_2021", +) +SOURCES = {system: system + "_support" for system in SYSTEMS} +IDENTITY_COLUMN = "geography_household_key" +_INPUT_COLUMNS = ( + "oa_code", + "population", + "households", + "constituency_code", + "region_code", + "lsoa_code", + "msoa_code", + "local_authority_code", + "ward_code", + "itl3_code", +) +_COUNT_COLUMNS = frozenset({"population", "households"}) +_CODE = re.compile(r"[EWSN][0-9]{8}") +_WARD = re.compile(r"[EWSN][0-9]{2}[0-9RS][0-9]{5}") +_ITL = re.compile(r"TL[C-N][0-9A-Z]{2}") +_LIMIT = 2**53 + + +def _require(condition: bool, message: str) -> None: + if not condition: + raise ValueError("UK atomic support: " + message) + + +def _text(value: object) -> bool: + return type(value) is str and bool(value) and value.strip() == value + + +def _profile(system: str) -> tuple[str, str, tuple[str, ...], str]: + _require(system in SYSTEMS, "unknown area system") + if system == SYSTEMS[0]: + regions = tuple( + name + for name in FRS_REGION_TO_REGION_CODE + if name not in {"SCOTLAND", "NORTHERN_IRELAND"} + ) + return "output_area", "2021_census", regions, "EW" + if system == SYSTEMS[1]: + return "output_area", "2022_census", ("SCOTLAND",), "S" + return "data_zone", "2021_census", ("NORTHERN_IRELAND",), "N" + + +def _metadata(descriptions: Mapping, *, vintage: str, system: str) -> dict: + _require(type(descriptions) is dict, "column metadata must be a plain mapping") + _require(set(descriptions) == set(_INPUT_COLUMNS), "exact column metadata required") + result = {} + for column in _INPUT_COLUMNS: + item = descriptions[column] + _require(type(item) is dict, "plain column description required") + expected = ( + {"kind", "source", "basis"} + if column in _COUNT_COLUMNS + else {"kind", "source", "vintage", "relation"} + ) + _require(set(item) == expected, "column description keys differ") + _require( + all(_text(value) for value in item.values()), "nonempty metadata required" + ) + if column in _COUNT_COLUMNS: + _require(item["kind"] == "weight", "count metadata must describe weight") + else: + _require( + item["kind"] == "code" and item["relation"] in RELATIONS, + "code mapping classification", + ) + result[column] = dict(item) + _require( + result["oa_code"]["vintage"] == vintage + and result["oa_code"]["relation"] == "exact", + "atomic identity/vintage differs", + ) + if system == SYSTEMS[2]: + _require( + result["constituency_code"]["relation"] != "inferred_modal", + "NI postcode-modal mapping is not admitted by this adapter", + ) + return result + + +def _counts(array: np.ndarray, *, positive: bool) -> np.ndarray: + _require( + array.dtype.kind in "iu" or array.dtype == np.dtype("float64"), "count dtype" + ) + if array.dtype.kind == "f": + _require(np.isfinite(array).all(), "non-finite count") + _require(np.equal(array, np.floor(array)).all(), "non-integral count") + _require( + np.all(array > 0 if positive else array >= 0) and np.all(array <= _LIMIT), + "count range", + ) + integers = [int(value) for value in array] + _require(0 < sum(integers) <= _LIMIT, "count total outside exact sampling range") + converted = np.asarray(integers, dtype=np.int64) + _require( + np.array_equal(converted.astype(array.dtype), array), + "count conversion is not exact", + ) + return converted + + +def assemble_uk_atomic_area_support( + *, + system: str, + arrays: Mapping[str, np.ndarray], + column_metadata: Mapping, +) -> bytes: + """Normalize one complete nation system from supplied legacy ladder columns. + + Rows are sorted by atomic code, never dropped. Counts remain exact; old + float64 count arrays are accepted only when integral and exactly representable. + Each declared FRS region must have positive household sampling mass. The + caller supplies source/vintage/relation evidence; syntax is not authority. + NI legacy ward_code means district electoral area, and lsoa_code aliases DZ. + """ + level, vintage, regions, prefixes = _profile(system) + descriptions = _metadata(column_metadata, vintage=vintage, system=system) + _require( + type(arrays) is dict and set(arrays) == set(_INPUT_COLUMNS), + "exact arrays required", + ) + values = {} + for column in _INPUT_COLUMNS: + array = arrays[column] + _require( + type(array) is np.ndarray and array.ndim == 1 and len(array) > 0, + "nonempty one-dimensional arrays required", + ) + if column in _COUNT_COLUMNS: + values[column] = _counts(array, positive=column == "population") + continue + _require(array.dtype.kind == "U", "codes must be Unicode arrays") + _require( + np.all(np.char.str_len(array) > 0) + and np.all(np.char.strip(array) == array), + "blank or padded code", + ) + pattern = ( + _ITL if column == "itl3_code" else _WARD if column == "ward_code" else _CODE + ) + _require( + all(pattern.fullmatch(str(value)) for value in array), "invalid code syntax" + ) + if column != "itl3_code": + _require( + all(str(value)[0] in prefixes for value in array), "foreign nation code" + ) + values[column] = array.copy() + count = len(values["oa_code"]) + _require( + all(len(array) == count for array in values.values()), "array lengths differ" + ) + _require(len(np.unique(values["oa_code"])) == count, "duplicate atomic code") + nations = [str(value)[0] for value in values["oa_code"]] + for column in _INPUT_COLUMNS: + if column not in {*_COUNT_COLUMNS, "itl3_code"}: + _require( + [str(value)[0] for value in values[column]] == nations, + "mapping crosses a nation boundary", + ) + if system == SYSTEMS[2]: + _require( + np.array_equal(values["lsoa_code"], values["oa_code"]), + "NI legacy lsoa_code must alias its atomic Data Zone", + ) + region_names = {FRS_REGION_TO_REGION_CODE[name]: name for name in regions} + _require( + set(values["region_code"].tolist()) == set(region_names), + "incomplete or foreign FRS region coverage", + ) + for code in region_names: + _require( + sum(int(v) for v in values["households"][values["region_code"] == code]) + > 0, + "region has no household sampling mass", + ) + order = np.argsort(values["oa_code"], kind="stable") + values = {column: array[order] for column, array in values.items()} + normalized = {"area": values["oa_code"]} + normalized.update({k: v for k, v in values.items() if k != "oa_code"}) + columns = {"area": descriptions["oa_code"]} + columns.update({k: v for k, v in descriptions.items() if k != "oa_code"}) + normalized["frs_region"] = np.asarray( + [region_names[code] for code in values["region_code"]], dtype="U" + ) + region_rule = canonical_json( + { + "mapping": dict(FRS_REGION_TO_REGION_CODE), + "input": descriptions["region_code"], + } + ) + columns["frs_region"] = { + "kind": "code", + "source": "sha256:" + hashlib.sha256(region_rule).hexdigest(), + "vintage": "frs_region_enum_v1", + "relation": "exact", + } + for width in (3, 4): + column = "itl1_code" if width == 3 else "itl2_code" + normalized[column] = np.asarray([code[:width] for code in values["itl3_code"]]) + columns[column] = { + **descriptions["itl3_code"], + "source": descriptions["itl3_code"]["source"] + f"#prefix-{width}", + "relation": "exact", + } + return encode_atomic_support( + { + "version": 1, + "system": system, + "level": level, + "code_system": "uk_gss", + "vintage": vintage, + "columns": columns, + }, + normalized, + ) + + +def _layers(system: str) -> tuple[tuple[str, str], ...]: + shared = tuple( + (column, column) + for column in _INPUT_COLUMNS + if column not in {"oa_code", *_COUNT_COLUMNS} + ) + aliases = (("area", "oa_code"),) + if system == SYSTEMS[0]: + native = (("area", "output_area_code"),) + elif system == SYSTEMS[1]: + native = ( + ("area", "output_area_code"), + ("lsoa_code", "data_zone_code"), + ("msoa_code", "intermediate_zone_code"), + ) + else: + native = ( + ("area", "data_zone_code"), + ("msoa_code", "super_data_zone_code"), + ("ward_code", "district_electoral_area_code"), + ) + return ( + *shared, + *aliases, + *native, + ("itl1_code", "itl1_code"), + ("itl2_code", "itl2_code"), + ) + + +def uk_atomic_assignment_definition( + supports: Mapping[str, bytes], + *, + seed: int, +) -> dict: + """Declare shared assignment after the complete initial spine/support roster. + + Source names are fixed by system, not file paths. The host must bind admitted + bytes to those SourceRefs and authenticate the supplied full-spine identity. + This function neither creates a graph barrier nor admits a native artifact. + """ + _require( + type(supports) is dict and set(supports) == set(SYSTEMS), + "three systems required", + ) + _require( + isinstance(seed, Integral) + and not isinstance(seed, (bool, np.bool_)) + and 0 <= seed < 2**32, + "seed must be a bounded integer", + ) + systems = [] + for system in SYSTEMS: + _require(type(supports[system]) is bytes, "immutable support bytes required") + support = decode_atomic_support(supports[system]) + level, vintage, regions, _ = _profile(system) + _require( + all( + support.metadata[k] == value + for k, value in ( + ("system", system), + ("level", level), + ("code_system", "uk_gss"), + ("vintage", vintage), + ) + ), + "support system identity differs", + ) + expected_columns = { + "area", + *(_INPUT_COLUMNS[1:]), + "frs_region", + "itl1_code", + "itl2_code", + } + _require(set(support.arrays) == expected_columns, "support columns differ") + # The declaration accepts this adapter's normalized form. This closes + # forged derived aliases/region normalization without claiming source + # authority from a self-consistent payload. + restored_arrays = { + column: support.arrays["area" if column == "oa_code" else column] + for column in _INPUT_COLUMNS + } + restored_metadata = { + column: dict( + support.metadata["columns"]["area" if column == "oa_code" else column] + ) + for column in _INPUT_COLUMNS + } + _require( + assemble_uk_atomic_area_support( + system=system, arrays=restored_arrays, column_metadata=restored_metadata + ) + == supports[system], + "support is not the canonical UK adapter output", + ) + systems.append( + { + "id": system, + "level": level, + "code_system": "uk_gss", + "vintage": vintage, + "source": SOURCES[system], + "selector": {"region": list(regions)}, + "constraints": [ + {"input": "region", "support": "frs_region", "required": True} + ], + "observed_area": None, + "stages": [ + {"level": "constituency_code", "weight": "households"}, + {"level": "area", "weight": "population"}, + ], + "layers": [ + { + "input": column, + "output": output, + **{ + k: support.metadata["columns"][column][k] + for k in ("source", "vintage", "relation") + }, + } + for column, output in _layers(system) + ], + } + ) + return validate_assignment_spec( + { + "version": 1, + "identity": [IDENTITY_COLUMN], + "stream": ["sha256-u53-v1", "uk-post-clone-atomic-area-v1", 0, int(seed)], + "outputs": { + "area": "atomic_area_code", + "system": "atomic_area_system", + "basis": "atomic_area_basis", + }, + "systems": systems, + } + ) diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/atomic_household_identity.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/atomic_household_identity.py new file mode 100644 index 000000000..1d8435b8b --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/atomic_household_identity.py @@ -0,0 +1,85 @@ +"""Explicit post-clone household keys for UK atomic assignment. + +The host supplies original-source identity and ordered structural branches. +This helper neither infers ancestry from financial values/ID offsets nor +authenticates the supplied lineage. Source/EXPAND receipts remain authoritative. +""" + +from __future__ import annotations + +from numbers import Integral + +import numpy as np + +from microcosm.graph.canonical import canonical_json + +_BRANCHES = ( + "spi_support_channel", + "cgt_incidence_clone", + "cgt_band_donors", + "geographic_support", +) + + +def _require(condition: bool, message: str) -> None: + if not condition: + raise ValueError("UK geography identity: " + message) + + +def household_draw_key( + *, + source: str, + source_vintage: str, + source_household_id: int, + clone_path: tuple[tuple[str, int], ...], +) -> str: + """Encode an original-source ID and explicit ordered clone branch path. + + Every full-spine household needs a distinct key. Branch ordinals are + supplied lineage identities, never sample row counters. Original branches + should use their declared ordinal consistently; growing a pool must not + change an existing path. Calibration K, weights, income, and final offset + IDs are deliberately not inputs. Exact IDs above 2**53 remain distinct. + """ + _require( + all( + type(value) is str + and value + and value.strip() == value + and len(value) <= 256 + for value in (source, source_vintage) + ), + "qualified source and vintage required", + ) + _require( + isinstance(source_household_id, Integral) + and not isinstance(source_household_id, (bool, np.bool_)) + and 0 < source_household_id < 2**63, + "original source household ID must be an exact positive int64", + ) + _require(type(clone_path) is tuple, "explicit immutable clone path required") + path = [] + previous = -1 + for branch in clone_path: + _require( + type(branch) is tuple and len(branch) == 2, + "named branch/ordinal pair required", + ) + name, ordinal = branch + _require(type(name) is str and name in _BRANCHES, "unknown structural branch") + position = _BRANCHES.index(name) + _require(position > previous, "clone branches must be unique and ordered") + _require( + isinstance(ordinal, Integral) + and not isinstance(ordinal, (bool, np.bool_)) + and 0 <= ordinal < 2**32, + "clone ordinal must be a bounded exact integer", + ) + path.append([name, str(int(ordinal))]) + previous = position + return ( + "uk-household-v1:" + + canonical_json( + [source, source_vintage, str(int(source_household_id)), path] + ).decode() + ) diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/__init__.py b/packages/microcosm-build/src/microcosm/build/us_runtime/__init__.py index 9437b99e8..60b2d5d28 100644 --- a/packages/microcosm-build/src/microcosm/build/us_runtime/__init__.py +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/__init__.py @@ -24,7 +24,9 @@ from collections.abc import Callable, Mapping from dataclasses import dataclass, field +from importlib import import_module from importlib.resources import files +from threading import RLock from microcosm.build.plan import DonorSpec, Stage, StagePlan from microcosm.build.source_manifest import ( @@ -35,1061 +37,12 @@ load_source_manifest, load_support_spine_manifest, ) -from microcosm.build.us_runtime.adult_care import ( - US_ADULT_CARE_CHILD_QUALIFYING_AGE_LIMIT, - US_ADULT_CARE_EARNED_INCOME_SOURCES, - US_ADULT_CARE_OUTPUT_COLUMNS, - US_ADULT_CARE_REQUIRED_SOURCE_COLUMNS, - US_ADULT_CARE_STAGE_NAME, - derive_us_adult_care_from_manifest, - us_adult_care_signal_gate, - us_adult_care_stage_spec, - us_adult_care_summary, - with_us_adult_care_inputs, -) -from microcosm.build.us_runtime.alimony import ( - ALIMONY_ASEC_ARCHIVED_DERIVATION_URL, - ALIMONY_PUF_ARCHIVED_DERIVATION_URL, - STRIKE_BENEFITS_ASEC_ARCHIVED_DERIVATION_URL, - US_ALIMONY_NONCONSTANT_PERSON_COLUMNS, - US_ALIMONY_OUTPUT_COLUMNS, - US_ALIMONY_STAGE_NAME, - US_ASEC_OTHER_INCOME_OUTPUT_COLUMNS, - derive_us_alimony_from_asec, - derive_us_alimony_from_puf, - us_alimony_signal_gate, - us_alimony_stage_spec, - us_alimony_summary, -) -from microcosm.build.us_runtime.asec_checkpoint import ( - ASEC_RAW_STAGE_ARTIFACT_KIND, - ASEC_RAW_STAGE_CHECKPOINT_FILENAME, - ASEC_RAW_STAGE_OPERATOR_STATUS, - ASEC_RAW_STAGE_SCHEMA_VERSION, - ASEC_RAW_STAGE_STAGE, - load_asec_pre_clone_checkpoint, - load_asec_raw_stage_checkpoint, -) -from microcosm.build.us_runtime.asec_pool import ( - AsecSource, - build_pooled_asec_unit_frame, - load_asec_h5_tables, - pool_asec_sources, -) -from microcosm.build.us_runtime.capital_gain_details import ( - CAPITAL_GAIN_DETAILS_ARCHIVED_DERIVATION_URL, - CAPITAL_GAIN_DETAILS_ARCHIVED_EXPORT_URL, - CAPITAL_GAIN_DETAILS_ARCHIVED_IMPUTATION_URL, - CAPITAL_GAIN_DETAILS_ARCHIVED_PERSON_ALLOCATION_URL, - CAPITAL_GAIN_DETAILS_ARCHIVED_PUF_ARTIFACT_URL, - US_CAPITAL_GAIN_DETAILS_NONCONSTANT_PERSON_COLUMNS, - US_CAPITAL_GAIN_DETAILS_NONCONSTANT_TAX_UNIT_COLUMNS, - US_CAPITAL_GAIN_DETAILS_OUTPUT_COLUMNS, - US_CAPITAL_GAIN_DETAILS_STAGE_NAME, - derive_us_capital_gain_details_from_puf, - us_capital_gain_details_signal_gate, - us_capital_gain_details_stage_spec, - us_capital_gain_details_summary, -) -from microcosm.build.us_runtime.casualty_losses import ( - US_CASUALTY_LOSS_NONCONSTANT_PERSON_COLUMNS, - US_CASUALTY_LOSS_OUTPUT_COLUMNS, - US_CASUALTY_LOSS_STAGE_NAME, - derive_us_casualty_loss_from_puf, - us_casualty_loss_signal_gate, - us_casualty_loss_stage_spec, - us_casualty_loss_summary, -) -from microcosm.build.us_runtime.child_support import ( - CHILD_SUPPORT_ARCHIVED_PUF_IMPUTATION_URL, - CHILD_SUPPORT_ARCHIVED_PUF_OUTPUTS_URL, - CHILD_SUPPORT_EXPENSE_ARCHIVED_DERIVATION_URL, - CHILD_SUPPORT_RECEIVED_ARCHIVED_DERIVATION_URL, - US_CHILD_SUPPORT_NONCONSTANT_PERSON_COLUMNS, - US_CHILD_SUPPORT_OUTPUT_COLUMNS, - US_CHILD_SUPPORT_REQUIRED_SOURCE_COLUMNS, - US_CHILD_SUPPORT_STAGE_NAME, - derive_us_child_support_from_asec, - derive_us_child_support_from_manifest, - impute_us_child_support_to_puf_support_from_manifest, - us_child_support_signal_gate, - us_child_support_stage_spec, - us_child_support_summary, - with_us_child_support_inputs, -) -from microcosm.build.us_runtime.childcare import ( - US_CHILDCARE_OUTPUT_COLUMNS, - US_CHILDCARE_REQUIRED_SOURCE_COLUMNS, - US_CHILDCARE_STAGE_NAME, - derive_us_childcare_from_manifest, - impute_us_childcare_to_puf_support_from_manifest, - us_childcare_signal_gate, - us_childcare_stage_spec, - us_childcare_summary, - with_us_childcare_inputs, -) -from microcosm.build.us_runtime.congressional_district_geography import ( - CONGRESSIONAL_DISTRICT_GEOID_COLUMN, - SOI_CONGRESSIONAL_DISTRICT_RECORD_SET_ID, - assign_congressional_districts_to_households, - congressional_district_assignment_summary, - congressional_district_distribution_from_ledger_facts, - with_household_congressional_districts, -) -from microcosm.build.us_runtime.congressional_district_vintage import ( - CONGRESSIONAL_DISTRICT_VINTAGE_CROSSWALK_SHA256_ATTR, - CONGRESSIONAL_DISTRICT_VINTAGE_TARGET_ATTR, - CURRENT_CONGRESSIONAL_DISTRICT_PREFIX, - CURRENT_CONGRESSIONAL_DISTRICT_VINTAGE, - DEFAULT_CONGRESSIONAL_DISTRICT_VINTAGE_CROSSWALK_RESOURCE, - SOURCE_CONGRESSIONAL_DISTRICT_PREFIX, - default_congressional_district_vintage_crosswalk_path, - load_congressional_district_vintage_crosswalk, - load_default_congressional_district_vintage_crosswalk, - translate_congressional_district_facts_to_current_vintage, -) -from microcosm.build.us_runtime.congressional_district_vintage_crosswalk import ( - CROSSWALK_BASIS_BLOCK_POPULATION, - build_cd_vintage_crosswalk_rows, - normalize_district_code, - parse_baf_cd_layer, - parse_national_cd_bef_districts, -) -from microcosm.build.us_runtime.cps_carried import ( - CPS_CARRIED_FORMULA_OWNED_COLUMNS, - CPS_CARRIED_PERSON_INPUTS, - CPS_CARRIED_SPM_UNIT_INPUTS, - CPS_REPORTED_TANF_AMOUNT_RAW_COLUMN, - CPS_REPORTED_TANF_TYPE_RAW_COLUMN, - CPS_REPORTED_WIC_RAW_COLUMN, - US_REPORTED_COVERAGE_PERSON_INPUTS, - US_REPORTED_COVERAGE_VINTAGE_GATE_MIN_ROWS, - WIC_CARRIER_ADJUDICATION_URL, - derive_us_cps_carried_inputs, - reported_tanf_enrollment_by_spm_unit, - reported_wic_receipt_carrier, -) -from microcosm.build.us_runtime.demographics import ( - AGE_BANDS, - DEMOGRAPHICS_SCHEMA_VERSION, - AgeBand, - compute_age_distribution, - demographics_payload, - write_demographics, -) -from microcosm.build.us_runtime.disability_benefits import ( - DISABILITY_BENEFITS_ARCHIVED_DERIVATION_URL, - DISABILITY_BENEFITS_ARCHIVED_PUF_IMPUTATION_URL, - DISABILITY_BENEFITS_ARCHIVED_PUF_OUTPUTS_URL, - DISABILITY_BENEFITS_ARCHIVED_SOURCE_COLUMNS_URL, - US_DISABILITY_BENEFITS_NONCONSTANT_PERSON_COLUMNS, - US_DISABILITY_BENEFITS_OUTPUT_COLUMNS, - US_DISABILITY_BENEFITS_REQUIRED_SOURCE_COLUMNS, - US_DISABILITY_BENEFITS_STAGE_NAME, - derive_us_disability_benefits_from_asec, - derive_us_disability_benefits_from_manifest, - impute_us_disability_benefits_to_puf_support_from_manifest, - us_disability_benefits_signal_gate, - us_disability_benefits_stage_spec, - us_disability_benefits_summary, - with_us_disability_benefits, -) -from microcosm.build.us_runtime.domestic_production import ( - DOMESTIC_PRODUCTION_ALD_ARCHIVED_DERIVATION_URL, - DOMESTIC_PRODUCTION_ALD_ARCHIVED_EXPORT_URL, - DOMESTIC_PRODUCTION_ALD_ARCHIVED_IMPUTATION_URL, - DOMESTIC_PRODUCTION_ALD_ARCHIVED_PUF_ARTIFACT_URL, - US_DOMESTIC_PRODUCTION_ALD_NONCONSTANT_TAX_UNIT_COLUMNS, - US_DOMESTIC_PRODUCTION_ALD_OUTPUT_COLUMNS, - US_DOMESTIC_PRODUCTION_ALD_STAGE_NAME, - derive_us_domestic_production_ald_from_puf, - us_domestic_production_ald_signal_gate, - us_domestic_production_ald_stage_spec, - us_domestic_production_ald_summary, -) -from microcosm.build.us_runtime.education_assistance_source import ( - ASEC_EDUCATION_ASSISTANCE_ARCHIVES, - ASEC_EDUCATION_ASSISTANCE_INCOME_YEARS, - fetch_asec_education_assistance_source, - fill_asec_education_assistance_source, - load_asec_education_assistance_sources, -) -from microcosm.build.us_runtime.education_inputs import ( - US_AOTC_ELIGIBILITY_OUTPUT_COLUMNS, - US_EDUCATION_INPUTS_NONCONSTANT_PERSON_COLUMNS, - US_EDUCATION_INPUTS_OUTPUT_COLUMNS, - US_EDUCATION_INPUTS_OWNED_OUTPUT_COLUMNS, - US_EDUCATION_INPUTS_REQUIRED_SOURCE_COLUMNS, - US_EDUCATION_INPUTS_STAGE_NAME, - derive_us_education_inputs_from_manifest, - us_education_inputs_signal_gate, - us_education_inputs_stage_spec, - us_education_inputs_summary, - with_us_education_inputs, -) -from microcosm.build.us_runtime.educator_expenses import ( - EDUCATOR_EXPENSE_ARCHIVED_ALLOCATION_URL, - EDUCATOR_EXPENSE_ARCHIVED_DERIVATION_URL, - EDUCATOR_EXPENSE_ARCHIVED_EXPORT_URL, - EDUCATOR_EXPENSE_ARCHIVED_PUF_IMPUTATION_URL, - US_EDUCATOR_EXPENSE_NONCONSTANT_PERSON_COLUMNS, - US_EDUCATOR_EXPENSE_OUTPUT_COLUMNS, - US_EDUCATOR_EXPENSE_STAGE_NAME, - derive_us_educator_expense_from_puf, - us_educator_expense_signal_gate, - us_educator_expense_stage_spec, - us_educator_expense_summary, -) -from microcosm.build.us_runtime.eligibility_inputs import ( - US_ELIGIBILITY_INPUTS_NONCONSTANT_PERSON_COLUMNS, - US_ELIGIBILITY_INPUTS_OUTPUT_COLUMNS, - US_ELIGIBILITY_INPUTS_REQUIRED_SOURCE_COLUMNS, - US_ELIGIBILITY_INPUTS_STAGE_NAME, - derive_us_eligibility_inputs_from_manifest, - us_eligibility_inputs_signal_gate, - us_eligibility_inputs_stage_spec, - us_eligibility_inputs_summary, - with_us_eligibility_inputs, -) -from microcosm.build.us_runtime.energy_subsidy import ( - ENERGY_SUBSIDY_ARCHIVED_CPS_DERIVATION_URL, - ENERGY_SUBSIDY_ARCHIVED_PUF_IMPUTATION_URL, - US_ENERGY_SUBSIDY_OUTPUT_COLUMNS, - US_ENERGY_SUBSIDY_REQUIRED_SOURCE_COLUMNS, - US_ENERGY_SUBSIDY_STAGE_NAME, - derive_us_energy_subsidy_from_manifest, - impute_us_energy_subsidy_to_puf_support_from_manifest, - us_energy_subsidy_signal_gate, - us_energy_subsidy_stage_spec, - us_energy_subsidy_summary, - with_us_energy_subsidy_input, -) -from microcosm.build.us_runtime.farm_business_income import ( - FARM_BUSINESS_INCOME_ARCHIVED_CPS_FARM_INCOME_URL, - FARM_BUSINESS_INCOME_ARCHIVED_DERIVATION_URL, - FARM_BUSINESS_INCOME_ARCHIVED_EXPORT_URL, - FARM_BUSINESS_INCOME_ARCHIVED_IMPUTATION_URL, - FARM_BUSINESS_INCOME_ARCHIVED_OVERRIDE_URL, - FARM_BUSINESS_INCOME_ARCHIVED_PUF_ARTIFACT_URL, - US_FARM_BUSINESS_INCOME_NONCONSTANT_PERSON_COLUMNS, - US_FARM_BUSINESS_INCOME_OUTPUT_COLUMNS, - US_FARM_BUSINESS_INCOME_STAGE_NAME, - derive_us_farm_business_income_from_puf, - us_farm_business_income_signal_gate, - us_farm_business_income_stage_spec, - us_farm_business_income_summary, -) -from microcosm.build.us_runtime.fiscal_targets import ( - SOI_VARIABLE_MAP, - US_FISCAL_LEDGER_PARITY_REGISTRY, - US_FISCAL_LEDGER_PARITY_REPORT, - US_FISCAL_MACRO_REALISM_BANDS, - US_FISCAL_TARGET_COVERAGE_REQUIREMENTS, - US_FISCAL_TARGET_LEDGER_REFERENCES, - US_FISCAL_TARGET_REFERENCES, - US_FISCAL_TARGET_REGISTRY, - US_FISCAL_TARGET_SPECS, - US_FISCAL_TARGET_SUPPORT_EXCLUSIONS, - US_JCT_TAX_EXPENDITURE_REFORMS, - US_JCT_TAX_EXPENDITURE_TARGET_REFERENCES, - US_JCT_TAX_EXPENDITURE_TARGET_SPECS, - US_SOI_FISCAL_TARGET_REFERENCES, - US_SOI_FISCAL_TARGET_SPECS, - US_STATE_INCOME_TAX_TARGET_REFERENCES, - US_STATE_INCOME_TAX_TARGET_SPECS, - SimpleTaxExpenditureReform, - compile_us_fiscal_target_registry, -) -from microcosm.build.us_runtime.form_4952 import ( - FORM_4952_ARCHIVED_DERIVATION_URL, - FORM_4952_ARCHIVED_EXPORT_URL, - FORM_4952_ARCHIVED_IMPUTATION_URL, - FORM_4952_ARCHIVED_PERSON_ALLOCATION_URL, - FORM_4952_ARCHIVED_PUF_ARTIFACT_URL, - US_FORM_4952_NONCONSTANT_PERSON_COLUMNS, - US_FORM_4952_OUTPUT_COLUMNS, - US_FORM_4952_STAGE_NAME, - derive_us_form_4952_election_from_puf, - us_form_4952_election_signal_gate, - us_form_4952_election_stage_spec, - us_form_4952_election_summary, -) -from microcosm.build.us_runtime.geography_ladder import ( - GEOGRAPHY_LADDER_ARTIFACT_SHA256_ATTR, - GEOGRAPHY_LADDER_VINTAGES_ATTR, - US_BLOCK_LADDER_DERIVED_LAYERS, - US_BLOCK_LADDER_KIND, - US_BLOCK_LADDER_SCHEMA_VERSION, - US_GEOGRAPHY_LADDER_COLUMNS, - US_NYC_COUNTY_FIPS, - UsBlockLadder, - assign_us_geography_ladder, - load_us_block_ladder, - us_geography_ladder_assignment_summary, - us_geography_ladder_gate, - with_household_us_geography_ladder, -) -from microcosm.build.us_runtime.hours_worked import ( - US_HOURS_WORKED_NONCONSTANT_PERSON_COLUMNS, - US_HOURS_WORKED_OUTPUT_COLUMNS, - US_HOURS_WORKED_POOL_EXCLUDED_COLUMNS, - US_HOURS_WORKED_POOL_OUTPUT_COLUMNS, - US_HOURS_WORKED_REQUIRED_SOURCE_COLUMNS, - US_HOURS_WORKED_STAGE_NAME, - derive_us_hours_worked_from_manifest, - us_hours_worked_signal_gate, - us_hours_worked_stage_spec, - us_hours_worked_summary, - with_us_hours_worked_inputs, -) -from microcosm.build.us_runtime.housing_inputs import ( - ACS_2022_RENT_ARTIFACT_SHA256, - HOUSING_INPUTS_ARCHIVED_ACS_DERIVATION_URL, - HOUSING_INPUTS_ARCHIVED_CPS_RENT_URL, - HOUSING_INPUTS_ARCHIVED_CPS_SPM_URL, - HOUSING_INPUTS_ARCHIVED_PUF_IMPUTATION_URL, - HOUSING_TAKE_UP_ARCHIVED_DERIVATION_URL, - HOUSING_TAKE_UP_ARCHIVED_HUD_ETL_URL, - HOUSING_TAKE_UP_ARCHIVED_PARAMETER_URL, - US_HOUSING_HOUSEHOLD_OUTPUT_COLUMNS, - US_HOUSING_INPUTS_OUTPUT_COLUMNS, - US_HOUSING_INPUTS_STAGE_NAME, - US_HOUSING_NONCONSTANT_HOUSEHOLD_COLUMNS, - US_HOUSING_NONCONSTANT_PERSON_COLUMNS, - US_HOUSING_NONCONSTANT_SPM_UNIT_COLUMNS, - US_HOUSING_PERSON_OUTPUT_COLUMNS, - US_HOUSING_REQUIRED_HOUSEHOLD_SOURCE_COLUMNS, - US_HOUSING_REQUIRED_PERSON_SOURCE_COLUMNS, - US_HOUSING_SPM_UNIT_OUTPUT_COLUMNS, - derive_us_housing_inputs, - impute_us_housing_assistance_to_puf_support, - impute_us_pre_subsidy_rent, - load_acs_2022_rent_donor, - us_housing_inputs_signal_gate, - us_housing_inputs_stage_spec, - us_housing_inputs_summary, - with_us_housing_inputs, -) -from microcosm.build.us_runtime.immigration import ( - IMMIGRATION_STATUS_VALUES, - SSN_CARD_TYPE_VALUES, - US_IMMIGRATION_NONCONSTANT_PERSON_COLUMNS, - US_IMMIGRATION_OUTPUT_COLUMNS, - US_IMMIGRATION_REQUIRED_SOURCE_COLUMNS, - US_IMMIGRATION_STAGE_NAME, - UndocumentedControls, - derive_us_immigration_status_from_manifest, - us_immigration_composition_gate, - us_immigration_composition_summary, - us_immigration_stage_spec, - with_us_immigration_inputs, -) -from microcosm.build.us_runtime.input_mass import ( - us_input_mass_totals, -) -from microcosm.build.us_runtime.medicaid_take_up import ( - US_MEDICAID_ENROLLMENT_SUBSTITUTIONS, - US_MEDICAID_ENROLLMENT_TARGET_ROLE, - US_MEDICAID_ENROLLMENT_TARGET_TABLE, - US_MEDICAID_ENROLLMENT_TOLERANCE, - US_MEDICAID_TAKE_UP_ANCHOR, - US_MEDICAID_TAKE_UP_STAGE, - US_MEDICAID_TAKE_UP_VARIABLE, - MedicaidEnrollmentSubstitution, - apply_us_medicaid_enrollment_substitutions, - us_medicaid_source_person_table, - us_medicaid_take_up_diagnostics, - us_medicaid_take_up_gate, - with_us_medicaid_take_up, - write_us_medicaid_take_up_diagnostics, -) -from microcosm.build.us_runtime.medicare_take_up import ( - MEDICARE_TAKE_UP_ARCHIVED_CLONE_URL, - MEDICARE_TAKE_UP_ARCHIVED_DERIVATION_URL, - MEDICARE_TAKE_UP_ARCHIVED_EXPORT_URL, - MEDICARE_TAKE_UP_ARCHIVED_SOURCE_COLUMNS_URL, - US_MEDICARE_TAKE_UP_NONCONSTANT_PERSON_COLUMNS, - US_MEDICARE_TAKE_UP_OUTPUT_COLUMNS, - US_MEDICARE_TAKE_UP_REQUIRED_SOURCE_COLUMNS, - US_MEDICARE_TAKE_UP_STAGE_NAME, - derive_us_medicare_take_up_from_manifest, - us_medicare_take_up_signal_gate, - us_medicare_take_up_stage_spec, - us_medicare_take_up_summary, - with_us_medicare_take_up_input, -) -from microcosm.build.us_runtime.misc_itemized import ( - US_MISC_ITEMIZED_NONCONSTANT_PERSON_COLUMNS, - US_MISC_ITEMIZED_OUTPUT_COLUMNS, - US_MISC_ITEMIZED_STAGE_NAME, - derive_us_misc_itemized_from_puf, - us_misc_itemized_signal_gate, - us_misc_itemized_stage_spec, - us_misc_itemized_summary, -) -from microcosm.build.us_runtime.nonzero_shares import ( - nonzero_share, - us_nonzero_shares, -) -from microcosm.build.us_runtime.operator_boundary import ( - FORMULA_OWNED_SOURCE_COLUMNS, - PRE_ASSEMBLY_OPERATOR_OUTPUT_FAMILIES, - assert_operator_free_source_frame, -) -from microcosm.build.us_runtime.org_wages import ( - BLS_STATE_UNION_REPRESENTATION_RATE_2024, - FLSA_EXECUTIVE_ADMINISTRATIVE_PROFESSIONAL_OCCUPATION_CODES, - FLSA_OVERTIME_OCCUPATION_CODES, - ORG_2024_DONOR_CONTENT_SHA256, - ORG_2024_DONOR_FILENAME, - ORG_PREDICTORS, - US_ORG_WAGES_NONCONSTANT_PERSON_COLUMNS, - US_ORG_WAGES_OUTPUT_COLUMNS, - US_ORG_WAGES_REQUIRED_SOURCE_COLUMNS, - US_ORG_WAGES_STAGE_NAME, - derive_flsa_overtime_premium, - derive_us_org_occupation_inputs, - fetch_org_2024_donor, - impute_us_org_wages, - load_org_2024_donor, - us_org_wages_signal_gate, - us_org_wages_stage_spec, - us_org_wages_summary, - with_us_org_wages_inputs, -) -from microcosm.build.us_runtime.other_health_insurance import ( - OTHER_HEALTH_INSURANCE_ARCHIVED_DERIVATION_URL, - OTHER_HEALTH_INSURANCE_ARCHIVED_PUF_IMPUTATION_URL, - OTHER_HEALTH_INSURANCE_ARCHIVED_PUF_OUTPUTS_URL, - OTHER_HEALTH_INSURANCE_ARCHIVED_PUF_PREDICTORS_URL, - OTHER_HEALTH_INSURANCE_ARCHIVED_PUF_SPLICE_URL, - US_OTHER_HEALTH_INSURANCE_MODELED_PREMIUM_VARIABLES, - US_OTHER_HEALTH_INSURANCE_NONCONSTANT_PERSON_COLUMNS, - US_OTHER_HEALTH_INSURANCE_OUTPUT_COLUMNS, - US_OTHER_HEALTH_INSURANCE_REQUIRED_SOURCE_COLUMNS, - US_OTHER_HEALTH_INSURANCE_STAGE_NAME, - US_OTHER_HEALTH_INSURANCE_STAGE_OUTPUT_COLUMNS, - US_SE_HEALTH_ATTRIBUTION_OUTPUT_COLUMNS, - US_SE_HEALTH_MEDICARE_AGE_THRESHOLD, - US_SE_HEALTH_SELF_EMPLOYMENT_INCOME_SOURCES, - attribute_us_se_health_premiums, - attribute_us_se_health_premiums_from_manifest, - derive_us_other_health_insurance_from_asec, - derive_us_other_health_insurance_from_manifest, - impute_us_other_health_insurance_to_puf_support_from_manifest, - us_other_health_insurance_signal_gate, - us_other_health_insurance_stage_spec, - us_other_health_insurance_summary, - with_us_other_health_insurance_inputs, -) -from microcosm.build.us_runtime.parity_reference import ( - ECPS_PARITY_KNOWN_GAPS_RESOURCE, - ECPS_PARITY_REFERENCE_RESOURCE, - EcpsParityReference, - EcpsParitySource, - ParityKnownGap, - load_ecps_parity_known_gaps, - load_ecps_parity_reference, -) -from microcosm.build.us_runtime.pregnancy import ( - US_PREGNANCY_NONCONSTANT_PERSON_COLUMNS, - US_PREGNANCY_OUTPUT_COLUMN, - US_PREGNANCY_REQUIRED_SOURCE_COLUMNS, - US_PREGNANCY_STAGE_NAME, - derive_us_pregnancy_from_manifest, - us_pregnancy_signal_gate, - us_pregnancy_stage_spec, - us_pregnancy_summary, - with_us_pregnancy_inputs, -) -from microcosm.build.us_runtime.prior_year_income import ( - PRIOR_YEAR_INCOME_ARCHIVED_DERIVATION_URL, - PRIOR_YEAR_INCOME_ARCHIVED_FINALIZER_URL, - PRIOR_YEAR_INCOME_ARCHIVED_FORMULA_OUTPUT_URL, - PRIOR_YEAR_INCOME_ARCHIVED_PUF_IMPUTATION_URL, - PRIOR_YEAR_INCOME_ARCHIVED_PUF_OUTPUTS_URL, - PRIOR_YEAR_INCOME_ARCHIVED_PUF_SPLICE_URL, - US_PRIOR_YEAR_INCOME_FORMULA_OWNED_OUTPUT_COLUMNS, - US_PRIOR_YEAR_INCOME_NONCONSTANT_PERSON_COLUMNS, - US_PRIOR_YEAR_INCOME_OUTPUT_COLUMNS, - US_PRIOR_YEAR_INCOME_PERSISTED_OUTPUT_COLUMNS, - US_PRIOR_YEAR_INCOME_REQUIRED_SOURCE_COLUMNS, - US_PRIOR_YEAR_INCOME_STAGE_NAME, - derive_us_prior_year_income_from_manifest, - impute_us_prior_year_income_to_puf_support_from_manifest, - us_prior_year_income_signal_gate, - us_prior_year_income_source_reconciliation_gate, - us_prior_year_income_stage_spec, - us_prior_year_income_summary, - with_us_prior_year_income_inputs, -) -from microcosm.build.us_runtime.public_assistance_type_source import ( - ASEC_PUBLIC_ASSISTANCE_TYPE_AUDIT_PINS, - ASEC_PUBLIC_ASSISTANCE_TYPE_INCOME_YEARS, - PAW_TYPE_TANF_CODES, - PAW_TYPE_VALID_CODES, - fill_asec_public_assistance_type_source, - load_asec_public_assistance_type_sources, -) -from microcosm.build.us_runtime.puf_capital_gains_tail import ( - PUF_CAPITAL_GAINS_TAIL_APPLIED_COLUMN, - PUF_CAPITAL_GAINS_TAIL_DONOR_AGI_BAND_COLUMN, - PUF_CAPITAL_GAINS_TAIL_DONOR_FILING_STATUS_COLUMN, - PUF_CAPITAL_GAINS_TAIL_DONOR_SOURCE_ID_COLUMN, - PUF_CAPITAL_GAINS_TAIL_DONOR_SYNTHETIC_COLUMN, - PUF_CAPITAL_GAINS_TAIL_MANIFEST_SCHEMA_VERSION, - PUF_CAPITAL_GAINS_TAIL_PERSON_COLUMNS, - PUF_CAPITAL_GAINS_TAIL_POSITIVE_MASS_FIVE_X_TARGET, - PUF_CAPITAL_GAINS_TAIL_QUANTILE, - PUF_CAPITAL_GAINS_TAIL_STAGE_NAME, - PUF_CAPITAL_GAINS_TAIL_SUPPORT_CHANNEL, - PUF_CAPITAL_GAINS_TAIL_SUPPORT_CONTRACT_VERSION, - PUF_CAPITAL_GAINS_TAIL_TAX_UNIT_COLUMNS, - PUF_CAPITAL_GAINS_TAIL_TRANSFER_WEIGHT_COLUMN, - assert_puf_capital_gains_tail_survives_selection, - puf_capital_gains_tail_concentration_gate, - puf_capital_gains_tail_support_contract_identity, - puf_capital_gains_tail_terminal_support_receipt, - select_puf_capital_gains_tail_donors, - transfer_puf_capital_gains_tail, - validate_puf_capital_gains_tail_manifest, - validate_puf_capital_gains_tail_terminal_support_receipt, - write_puf_capital_gains_tail_manifest, -) -from microcosm.build.us_runtime.puf_donor_io import load_puf_tax_unit_donor -from microcosm.build.us_runtime.puf_e01000_reconciliation import ( - PUF_E01000_RECONCILIATION_SCHEMA_VERSION, - build_puf_e01000_reconciliation_basis, - finalize_puf_e01000_reconciliation, - puf_capital_gains_joint_metrics, - puf_processed_capital_gains_stage, - puf_raw_e01000_stage, -) -from microcosm.build.us_runtime.puf_interest_components import ( - US_PUF_E19200_AGI_BANDS, - US_PUF_E19200_ALL_RETURNS_COMPONENTS, - PufE19200AgiBand, - PufE19200InterestComponents, - split_us_puf_e19200_by_agi_band, -) -from microcosm.build.us_runtime.puf_source_agi import ( - PUF_AGGREGATE_DISAGGREGATION_SEED, - PUF_AGGREGATE_RECIDS, - PUF_SOURCE_YEAR, - PUF_SOURCE_YEAR_AGI_REQUIRED_COLUMNS, - PUF_SYNTHETIC_RECID_START, - source_year_puf_adjusted_gross_income, -) -from microcosm.build.us_runtime.puf_support import ( - BASE_ASEC_SUPPORT_CHANNEL, - PUF_DONOR_SOURCE_ADJUSTED_GROSS_INCOME_COLUMN, - PUF_TAX_DETAIL_CLONE_INDEX, - PUF_TAX_DETAIL_DEFAULT_PERSON_OUTPUTS, - PUF_TAX_DETAIL_DEFAULT_TAX_UNIT_OUTPUTS, - PUF_TAX_DETAIL_SUPPORT_CHANNEL, - US_PUF_DONOR_MORTGAGE_OUTLIER_CEILING, - US_PUF_SUPPORT_FIT_NAME, - US_PUF_SUPPORT_STAGE_NAME, - PufTaxDetailChainInputs, - clone_us_frame_for_puf_support, - finalize_us_puf_tax_detail_predictions, - has_assembled_support_metadata, - has_support_role_metadata, - impute_us_puf_tax_detail_support, - prepare_us_puf_tax_detail_chain_inputs, - puf_tax_detail_clone_mask, - puf_tax_unit_donor_from_arrays, - spine_source_id_column, - support_channel_column, - support_clone_index_column, - support_role_series, - support_source_id_column, -) -from microcosm.build.us_runtime.puma_ladder import ( - PUMA_LADDER_ARTIFACT_SHA256_ATTR, - PUMA_LADDER_VINTAGES_ATTR, - US_PUMA_LADDER_COLUMNS, - US_PUMA_LADDER_DERIVED_LAYERS, - US_PUMA_LADDER_KIND, - US_PUMA_LADDER_SCHEMA_VERSION, - US_PUMA_LADDER_TRACT_COLUMN, - UsPumaLadder, - assign_us_puma_ladder, - load_us_puma_ladder, - us_puma_ladder_assignment_summary, - us_puma_ladder_gate, - with_household_us_puma_ladder, -) -from microcosm.build.us_runtime.puma_ladder_sources import ( - assemble_us_puma_ladder, - parse_tract_to_puma_relationship, -) -from microcosm.build.us_runtime.qbi_inputs import ( - QBI_ARCHIVED_ASSUMPTIONS_URL, - QBI_ARCHIVED_CLONE_URL, - QBI_ARCHIVED_DERIVATION_URL, - QBI_ARCHIVED_EXPORT_URL, - QBI_ARCHIVED_IMPUTATION_URL, - QBI_ARCHIVED_PUF_ARTIFACT_URL, - QBI_ARCHIVED_SIMULATION_URL, - US_QBI_BOOLEAN_OUTPUT_COLUMNS, - US_QBI_NONCONSTANT_PERSON_COLUMNS, - US_QBI_NONNEGATIVE_OUTPUT_COLUMNS, - US_QBI_OUTPUT_COLUMNS, - US_QBI_STAGE_NAME, - us_qbi_inputs_signal_gate, - us_qbi_inputs_stage_spec, - us_qbi_inputs_summary, - with_us_qbi_input_reconciliation, -) -from microcosm.build.us_runtime.reform_coverage_smoke import ( - us_reform_coverage_smoke_gate, -) -from microcosm.build.us_runtime.reform_validation import ( - REFORM_VALIDATION_SCHEMA_VERSION, - ReformValidationSpec, - in_sample_reform_specs, - load_default_reform_specs, - out_of_sample_reform_specs, - reform_validation_payload, - write_reform_validation, -) -from microcosm.build.us_runtime.register_consistency import ( - us_register_consistency_gate, - us_register_contradictions, -) -from microcosm.build.us_runtime.relationship_inputs import ( - US_RELATIONSHIP_INPUTS_NONCONSTANT_PERSON_COLUMNS, - US_RELATIONSHIP_INPUTS_OUTPUT_COLUMNS, - US_RELATIONSHIP_INPUTS_REQUIRED_SOURCE_COLUMNS, - US_RELATIONSHIP_INPUTS_STAGE_NAME, - derive_us_relationship_inputs_from_manifest, - us_relationship_inputs_signal_gate, - us_relationship_inputs_stage_spec, - us_relationship_inputs_summary, - with_us_relationship_inputs, -) -from microcosm.build.us_runtime.release_input_coverage import ( - POST_REFERENCE_ECPS_REQUIRED_INPUTS, - SSI_COUNTABLE_RESOURCE_ASSETS, - US_CGD_ROUTE_REQUIRED_INPUTS, - US_RELEASE_INPUT_COVERAGE_RESOURCE, - ReformCoverageProbe, - ReleaseInputColumn, - ReleaseInputCoverageManifest, - assert_release_input_coverage_manifest_current, - load_release_input_coverage_manifest, - us_release_input_coverage_gate, - us_release_input_coverage_required_columns, - us_release_input_coverage_reviewed_exclusions, - us_release_reform_coverage_probes, -) -from microcosm.build.us_runtime.release_target_parity import ( - RED_LINE_COMPILED_FAMILIES, - US_TARGET_PARITY_FEED_FAMILIES_RESOURCE, - US_TARGET_PARITY_MANIFEST_RESOURCE, - TargetFamily, - TargetFence, - TargetParityManifest, - assert_target_parity_manifest_current, - load_target_parity_feed_families, - load_target_parity_manifest, - registry_target_family_ids, - us_release_target_parity_compiled_families, - us_release_target_parity_gate, - us_release_target_parity_reviewed_exclusions, - us_target_family_id, -) -from microcosm.build.us_runtime.retirement_contributions import ( - US_RETIREMENT_CONTRIBUTION_NONCONSTANT_PERSON_COLUMNS, - US_RETIREMENT_CONTRIBUTION_OUTPUT_COLUMNS, - US_RETIREMENT_CONTRIBUTION_REQUIRED_SOURCE_COLUMNS, - US_RETIREMENT_CONTRIBUTION_STAGE_NAME, - derive_us_retirement_contributions_from_manifest, - impute_us_retirement_contributions_to_puf_support_from_manifest, - us_retirement_contributions_signal_gate, - us_retirement_contributions_stage_spec, - us_retirement_contributions_summary, - with_us_retirement_contribution_inputs, -) -from microcosm.build.us_runtime.retirement_distributions import ( - RETIREMENT_DISTRIBUTIONS_ARCHIVED_DERIVATION_URL, - RETIREMENT_DISTRIBUTIONS_ARCHIVED_PARAMETERS_URL, - US_RETIREMENT_DISTRIBUTION_NONCONSTANT_PERSON_COLUMNS, - US_RETIREMENT_DISTRIBUTION_OUTPUT_COLUMNS, - US_RETIREMENT_DISTRIBUTION_REQUIRED_SOURCE_COLUMNS, - US_RETIREMENT_DISTRIBUTION_STAGE_NAME, - derive_us_retirement_distributions_from_manifest, - impute_us_retirement_distributions_to_puf_support_from_manifest, - us_retirement_distributions_signal_gate, - us_retirement_distributions_stage_spec, - us_retirement_distributions_summary, - with_us_retirement_distribution_inputs, -) -from microcosm.build.us_runtime.salt_refund_income import ( - SALT_REFUND_ARCHIVED_DERIVATION_URL, - SALT_REFUND_ARCHIVED_EXPORT_URL, - SALT_REFUND_ARCHIVED_IMPUTATION_URL, - SALT_REFUND_ARCHIVED_PERSON_ALLOCATION_URL, - SALT_REFUND_ARCHIVED_PUF_ARTIFACT_URL, - US_SALT_REFUND_NONCONSTANT_PERSON_COLUMNS, - US_SALT_REFUND_OUTPUT_COLUMNS, - US_SALT_REFUND_STAGE_NAME, - derive_us_salt_refund_income_from_puf, - us_salt_refund_income_signal_gate, - us_salt_refund_income_stage_spec, - us_salt_refund_income_summary, -) -from microcosm.build.us_runtime.scf_auto_loans import ( - QUALIFIED_AUTO_LOAN_ANNUAL_ISSUANCE_TARGET, - SCF_2022_FULL_EXTRACT_MEMBER, - SCF_2022_FULL_EXTRACT_MEMBER_SHA256, - SCF_2022_FULL_EXTRACT_URL, - SCF_2022_FULL_EXTRACT_ZIP_SHA256, - SCF_AUTO_LOAN_AMOUNT_COLUMNS, - SCF_AUTO_LOAN_RATE_COLUMNS, - US_SCF_AUTO_LOAN_NONCONSTANT_HOUSEHOLD_COLUMNS, - US_SCF_AUTO_LOAN_OUTPUT_COLUMNS, - fetch_scf_2022_full_extract, - impute_us_scf_auto_loans, - load_scf_2022_auto_loan_donor, - qualified_auto_loan_interest_proxy, - us_scf_auto_loans_signal_gate, - us_scf_auto_loans_stage_spec, - us_scf_auto_loans_summary, - with_us_scf_auto_loan_inputs, -) -from microcosm.build.us_runtime.scf_wealth import ( - FINANCIAL_ASSET_BLEND_AUDIT_KEY, - FINANCIAL_ASSET_SOURCE_SCF_PROBABILITY, - SCF_FINANCIAL_ASSET_TARGET_COMPONENTS, - SCF_NET_WORTH_TARGET_COMPONENTS, - SCF_WEALTH_PREDICTORS, - US_SCF_FINANCIAL_ASSET_OUTPUT_COLUMNS, - US_SCF_NET_WORTH_OUTPUT_COLUMNS, - US_SCF_WEALTH_NONCONSTANT_HOUSEHOLD_COLUMNS, - US_SCF_WEALTH_NONCONSTANT_PERSON_COLUMNS, - US_SCF_WEALTH_STAGE_NAME, - fetch_scf_2022_summary_extract, - financial_asset_source_is_scf, - impute_us_scf_financial_assets, - impute_us_scf_net_worth, - impute_us_sipp_scf_financial_assets, - load_scf_2022_financial_asset_donor, - us_scf_wealth_signal_gate, - us_scf_wealth_stage_spec, - us_scf_wealth_summary, - with_us_scf_wealth_inputs, -) -from microcosm.build.us_runtime.sipp_financial_assets import ( - SIPP_2023_FINANCIAL_ASSET_DONOR_REPOSITORY_ID_PARTS, - SIPP_2023_FINANCIAL_ASSET_DONOR_REPOSITORY_TYPE, - SIPP_2023_FINANCIAL_ASSET_DONOR_REVISION, - SIPP_2023_FINANCIAL_ASSET_DONOR_SHA256, - SIPP_2023_FINANCIAL_ASSET_DONOR_SIZE_BYTES, - SIPP_2023_FINANCIAL_ASSET_DONOR_URL, - SIPP_FINANCIAL_ASSET_DONOR_WEIGHT_COLUMN, - SIPP_FINANCIAL_ASSET_MODEL_PREDICTORS, - SIPP_FINANCIAL_ASSET_SOURCE_COLUMNS, - SIPP_FINANCIAL_ASSET_TARGET_ALLOCATION_COLUMNS, - SIPP_FINANCIAL_ASSET_TARGET_SOURCE_COLUMNS, - fetch_sipp_2023_financial_asset_donor, - impute_us_sipp_financial_assets, - load_sipp_2023_financial_asset_donor, -) -from microcosm.build.us_runtime.sipp_head_start import ( - HEAD_START_SIPP_DICTIONARY_URL, - SIPP_2023_HEAD_START_DONOR_REVISION, - SIPP_2023_HEAD_START_DONOR_SHA256, - SIPP_2023_HEAD_START_DONOR_SIZE_BYTES, - SIPP_2023_HEAD_START_DONOR_URL, - SIPP_HEAD_START_FIT_PARAMETERS, - SIPP_HEAD_START_MODEL_PREDICTORS, - SIPP_HEAD_START_READ_PARAMETERS, - SIPP_HEAD_START_SOURCE_COLUMNS, - US_SIPP_HEAD_START_NONCONSTANT_PERSON_COLUMNS, - US_SIPP_HEAD_START_OUTPUT_COLUMNS, - US_SIPP_HEAD_START_REQUIRED_SOURCE_COLUMNS, - US_SIPP_HEAD_START_STAGE_NAME, - fetch_sipp_2023_head_start_donor, - impute_us_sipp_head_start, - load_sipp_2023_head_start_donor, - us_sipp_head_start_signal_gate, - us_sipp_head_start_stage_spec, - us_sipp_head_start_summary, - with_us_sipp_head_start_input, -) -from microcosm.build.us_runtime.sipp_tips import ( - CENSUS_OCCUPATION_CODE_TO_TTOC, - SIPP_2023_TIP_DONOR_REVISION, - SIPP_2023_TIP_DONOR_SHA256, - SIPP_2023_TIP_DONOR_URL, - SIPP_TIP_OUTPUT_COLUMNS, - SIPP_TIP_PREDICTORS, - US_SIPP_TIPS_NONCONSTANT_PERSON_COLUMNS, - US_SIPP_TIPS_OUTPUT_COLUMNS, - US_SIPP_TIPS_REQUIRED_SOURCE_COLUMNS, - US_SIPP_TIPS_STAGE_NAME, - derive_treasury_tipped_occupation_code, - fetch_sipp_2023_tip_donor, - impute_us_sipp_tips, - load_sipp_2023_tip_donor, - us_sipp_tips_signal_gate, - us_sipp_tips_stage_spec, - us_sipp_tips_summary, - with_us_sipp_tip_inputs, -) -from microcosm.build.us_runtime.sipp_vehicles import ( - SIPP_2023_VEHICLE_DONOR_REVISION, - SIPP_2023_VEHICLE_DONOR_SHA256, - SIPP_2023_VEHICLE_DONOR_SIZE_BYTES, - SIPP_2023_VEHICLE_DONOR_URL, - US_SIPP_VEHICLE_NONCONSTANT_HOUSEHOLD_COLUMNS, - US_SIPP_VEHICLE_OUTPUT_COLUMNS, - fetch_sipp_2023_vehicle_donor, - load_sipp_2023_vehicle_donor, - us_sipp_vehicles_signal_gate, - us_sipp_vehicles_stage_spec, - us_sipp_vehicles_summary, - with_us_sipp_vehicle_inputs, -) -from microcosm.build.us_runtime.snap_discretionary_exemption import ( - US_SNAP_DISCRETIONARY_EXEMPTION_NONCONSTANT_PERSON_COLUMNS, - US_SNAP_DISCRETIONARY_EXEMPTION_OUTPUT_COLUMN, - US_SNAP_DISCRETIONARY_EXEMPTION_REQUIRED_SOURCE_COLUMNS, - US_SNAP_DISCRETIONARY_EXEMPTION_STAGE_NAME, - derive_us_snap_discretionary_exemption_from_manifest, - us_snap_discretionary_exemption_signal_gate, - us_snap_discretionary_exemption_stage_spec, - us_snap_discretionary_exemption_summary, - with_us_snap_discretionary_exemption_inputs, -) -from microcosm.build.us_runtime.snap_state_take_up import ( - US_SNAP_CASELOAD_TOLERANCE, - US_SNAP_HOUSEHOLDS_TARGET_TABLE, - US_SNAP_STATE_TAKE_UP_ANCHOR, - US_SNAP_STATE_TAKE_UP_STAGE, - us_snap_state_take_up_diagnostics, - us_snap_state_take_up_gate, - with_us_snap_state_take_up, - write_us_snap_state_take_up_diagnostics, -) -from microcosm.build.us_runtime.snap_take_up import ( - US_SNAP_TAKE_UP_OUTPUT_COLUMN, - US_SNAP_TAKE_UP_RAW_COLUMN, - US_SNAP_TAKE_UP_STAGE_NAME, - derive_us_snap_take_up_from_manifest, - us_snap_take_up_signal_gate, - us_snap_take_up_stage_spec, - us_snap_take_up_summary, - with_us_snap_take_up_inputs, -) -from microcosm.build.us_runtime.source_coverage import ( - CHRONICLE_US_SOURCE_COVERAGE_CONTRACT_COMMIT, - LEDGER_US_SOURCE_COVERAGE_CONTRACT_COMMIT, - US_SOURCE_COVERAGE, - hard_target_package_aliases, - source_gap_family_ids, - us_source_coverage_diagnostics, - us_source_coverage_gate, - validation_only_family_ids, - write_us_source_coverage_diagnostics, -) -from microcosm.build.us_runtime.source_runtime import ( - disaggregate_us_puf_aggregate_records_from_manifest, - us_source_operation_handlers, -) -from microcosm.build.us_runtime.spine_agreement import ( - DEFAULT_CATEGORICAL_TOTAL_VARIATION_TOLERANCE, - DEFAULT_INCIDENCE_RATIO_BOUNDS, - DEFAULT_QUANTILE_ENVELOPE_TOLERANCE, - DEFAULT_SPINE_AGREEMENT_QUANTILES, - US_SPINE_AGREEMENT_REGISTRY, - SpineAgreementSpec, - default_spine_agreement_registry, - normalize_transfer_family_name, - spine_agreement_gate, - validate_spine_agreement_registry, -) -from microcosm.build.us_runtime.spine_assembly import assemble_spines -from microcosm.build.us_runtime.ssi_disability_criteria import ( - SIPP_2023_SSI_DISABILITY_DONOR_REVISION, - SIPP_2023_SSI_DISABILITY_DONOR_SHA256, - SIPP_2023_SSI_DISABILITY_DONOR_SIZE_BYTES, - SIPP_2023_SSI_DISABILITY_DONOR_URL, - SIPP_SSI_DISABILITY_DIFFICULTY_PREDICTORS, - SIPP_SSI_DISABILITY_FIT_PARAMETERS, - SIPP_SSI_DISABILITY_MODEL_PREDICTORS, - SIPP_SSI_DISABILITY_READ_PARAMETERS, - SIPP_SSI_DISABILITY_SOURCE_COLUMNS, - SSI_DISABILITY_ARCHIVED_CPS_URL, - SSI_DISABILITY_ARCHIVED_EXTENDED_CPS_URL, - SSI_DISABILITY_ARCHIVED_SIPP_URL, - SSI_DISABILITY_ARCHIVED_SOURCE_IMPUTE_URL, - SSI_DISABILITY_SIPP_DICTIONARY_URL, - US_SSI_DISABILITY_CRITERIA_NONCONSTANT_PERSON_COLUMNS, - US_SSI_DISABILITY_CRITERIA_OUTPUT_COLUMNS, - US_SSI_DISABILITY_CRITERIA_STAGE_NAME, - fetch_sipp_2023_ssi_disability_donor, - impute_us_ssi_disability_criteria, - load_sipp_2023_ssi_disability_donor, - us_ssi_disability_criteria_signal_gate, - us_ssi_disability_criteria_stage_spec, - us_ssi_disability_criteria_summary, - with_us_ssi_disability_criteria, -) -from microcosm.build.us_runtime.ssi_take_up import ( - SSI_TAKE_UP_ARCHIVED_DERIVATION_URL, - SSI_TAKE_UP_ARCHIVED_EXPORT_URL, - SSI_TAKE_UP_ARCHIVED_RANDOMNESS_URL, - SSI_TAKE_UP_ARCHIVED_REPORTER_URL, - SSI_TAKE_UP_ARCHIVED_TARGETS_URL, - SSI_TAKE_UP_SSA_SOURCE_URL, - US_SSI_TAKE_UP_ANCHOR, - US_SSI_TAKE_UP_BAND_DELIVERY_RELATIVE_TOLERANCE, - US_SSI_TAKE_UP_ENFORCED_BAND_KEYS, - US_SSI_TAKE_UP_NONCONSTANT_PERSON_COLUMNS, - US_SSI_TAKE_UP_OUTPUT_COLUMNS, - US_SSI_TAKE_UP_PHASE_ASSIGNMENT, - US_SSI_TAKE_UP_PHASE_RELEASE_FINAL, - US_SSI_TAKE_UP_PRIOR_BASIS_CURRENT_FRAME, - US_SSI_TAKE_UP_PRIOR_BASIS_RELEASE_ARTIFACT, - US_SSI_TAKE_UP_REQUIRED_SOURCE_COLUMNS, - US_SSI_TAKE_UP_STAGE_NAME, - US_SSI_TAKE_UP_TARGET_TABLE_NAME, - SSITakeUpBandPriorBasis, - SSITakeUpPriorBasis, - ssi_take_up_prior_basis_from_artifact, - ssi_take_up_prior_basis_from_diagnostics, - us_ssi_take_up_delivery_gate, - us_ssi_take_up_diagnostics, - us_ssi_take_up_gate, - us_ssi_take_up_reporter_source_ids, - with_us_ssi_take_up, - write_us_ssi_take_up_diagnostics, -) -from microcosm.build.us_runtime.support_provenance import ( - us_reported_coverage_vintage_signal_gate, -) -from microcosm.build.us_runtime.take_up import ( - US_TAKE_UP_SHARE_BAND, - SeededTakeUpResult, - us_take_up_participation_diagnostics, - us_take_up_signal_gate, - us_take_up_summary, - with_us_take_up_inputs, - write_us_take_up_participation_diagnostics, -) -from microcosm.build.us_runtime.take_up_contract import ( - TakeUpContract, - TakeUpProgram, - assert_take_up_contract_current, - assert_take_up_treatments_consistent, - count_calibrated_take_up_programs, - load_take_up_contract, - seeded_take_up_programs, -) -from microcosm.build.us_runtime.validation_input_coverage import ( - US_VALIDATION_PROVISION_INPUT_LEAVES, - ValidationInputLeaf, - assert_validation_leaf_registry_current, - us_source_stage_outputs, - us_validation_input_coverage_gate, -) -from microcosm.build.us_runtime.voluntary_filing import ( - SIPP_2023_VOLUNTARY_FILING_DONOR_REVISION, - SIPP_2023_VOLUNTARY_FILING_DONOR_SHA256, - SIPP_2023_VOLUNTARY_FILING_DONOR_SIZE_BYTES, - SIPP_2023_VOLUNTARY_FILING_DONOR_URL, - SIPP_VOLUNTARY_FILING_MODEL_PREDICTORS, - SIPP_VOLUNTARY_FILING_SOURCE_COLUMNS, - US_VOLUNTARY_FILING_NONCONSTANT_TAX_UNIT_COLUMNS, - US_VOLUNTARY_FILING_OUTPUT_COLUMNS, - US_VOLUNTARY_FILING_STAGE_NAME, - VOLUNTARY_FILING_ARCHIVED_DERIVATION_URL, - VOLUNTARY_FILING_ARCHIVED_PARAMETERS_URL, - VOLUNTARY_FILING_SIPP_DICTIONARY_URL, - fetch_sipp_2023_voluntary_filing_donor, - impute_us_voluntary_filing, - load_sipp_2023_voluntary_filing_donor, - us_voluntary_filing_signal_gate, - us_voluntary_filing_stage_spec, - us_voluntary_filing_summary, - with_us_voluntary_filing_input, -) -from microcosm.build.us_runtime.weeks_unemployed import ( - ASEC_2023_WEEKS_UNEMPLOYED_MEMBER, - ASEC_2023_WEEKS_UNEMPLOYED_MEMBER_CRC32, - ASEC_2023_WEEKS_UNEMPLOYED_MEMBER_SHA256, - ASEC_2023_WEEKS_UNEMPLOYED_MEMBER_SIZE_BYTES, - ASEC_2023_WEEKS_UNEMPLOYED_RAW_ROWS, - ASEC_2023_WEEKS_UNEMPLOYED_SOURCE_COLUMNS, - ASEC_2023_WEEKS_UNEMPLOYED_SOURCE_SHA256, - ASEC_2023_WEEKS_UNEMPLOYED_SOURCE_YEAR, - ASEC_2023_WEEKS_UNEMPLOYED_UNIQUE_KEYS, - ASEC_2023_WEEKS_UNEMPLOYED_WEIGHTED_SOURCE_SHARE, - ASEC_2023_WEEKS_UNEMPLOYED_WEIGHTED_WEEKS, - ASEC_2023_WEEKS_UNEMPLOYED_ZIP_SHA256, - ASEC_2023_WEEKS_UNEMPLOYED_ZIP_SIZE_BYTES, - ASEC_2023_WEEKS_UNEMPLOYED_ZIP_URL, - US_WEEKS_UNEMPLOYED_NONCONSTANT_PERSON_COLUMNS, - US_WEEKS_UNEMPLOYED_OUTPUT_COLUMNS, - US_WEEKS_UNEMPLOYED_REQUIRED_SOURCE_COLUMNS, - US_WEEKS_UNEMPLOYED_STAGE_NAME, - WEEKS_UNEMPLOYED_ARCHIVED_DERIVATION_URL, - WEEKS_UNEMPLOYED_ARCHIVED_PUF_IMPUTATION_URL, - WEEKS_UNEMPLOYED_ARCHIVED_SOURCE_URL, - WEEKS_UNEMPLOYED_DERIVE_PARAMETERS, - WEEKS_UNEMPLOYED_PUF_IMPUTATION_PARAMETERS, - WEEKS_UNEMPLOYED_PUF_PREDICTORS, - WEEKS_UNEMPLOYED_READ_PARAMETERS, - derive_us_weeks_unemployed_from_manifest, - fetch_asec_2023_weeks_unemployed_source, - fill_asec_2022_weeks_unemployed_source, - impute_us_weeks_unemployed_to_puf_support_from_manifest, - load_asec_2023_weeks_unemployed_source, - us_weeks_unemployed_signal_gate, - us_weeks_unemployed_stage_spec, - us_weeks_unemployed_summary, - with_us_weeks_unemployed, -) -from microcosm.build.us_runtime.wic_claim import ( - US_WIC_CLAIM_NONCONSTANT_PERSON_COLUMNS, - US_WIC_CLAIM_OUTPUT_COLUMNS, - US_WIC_CLAIM_REQUIRED_SOURCE_COLUMNS, - US_WIC_CLAIM_STAGE_NAME, - WIC_CLAIM_ARCHIVED_DERIVATION_URL, - WIC_CLAIM_ARCHIVED_PARAMETERS_URL, - WIC_CLAIM_ARCHIVED_RANDOMNESS_URL, - WIC_CLAIM_FNS_SOURCE_URL, - derive_us_wic_claim_from_manifest, - us_wic_claim_signal_gate, - us_wic_claim_stage_spec, - us_wic_claim_summary, - with_us_wic_claim_input, -) -from microcosm.build.us_runtime.workers_compensation import ( - US_WORKERS_COMPENSATION_NONCONSTANT_PERSON_COLUMNS, - US_WORKERS_COMPENSATION_OUTPUT_COLUMNS, - US_WORKERS_COMPENSATION_REQUIRED_SOURCE_COLUMNS, - US_WORKERS_COMPENSATION_STAGE_NAME, - WORKERS_COMPENSATION_ARCHIVED_DERIVATION_URL, - WORKERS_COMPENSATION_ARCHIVED_PUF_IMPUTATION_URL, - WORKERS_COMPENSATION_ARCHIVED_PUF_OUTPUTS_URL, - WORKERS_COMPENSATION_ARCHIVED_SOURCE_COLUMNS_URL, - derive_us_workers_compensation_from_asec, - derive_us_workers_compensation_from_manifest, - impute_us_workers_compensation_to_puf_support_from_manifest, - us_workers_compensation_signal_gate, - us_workers_compensation_stage_spec, - us_workers_compensation_summary, - with_us_workers_compensation, -) from microcosm.frame import Frame __all__ = [ "ASEC_RAW_STAGE_ARTIFACT_KIND", "ASEC_RAW_STAGE_CHECKPOINT_FILENAME", + "ASEC_RAW_STAGE_COVERAGE_SCHEMA_VERSION", "ASEC_RAW_STAGE_OPERATOR_STATUS", "ASEC_RAW_STAGE_SCHEMA_VERSION", "ASEC_RAW_STAGE_STAGE", @@ -1875,6 +828,7 @@ "load_default_congressional_district_vintage_crosswalk", "load_asec_pre_clone_checkpoint", "load_asec_raw_stage_checkpoint", + "load_asec_raw_stage_checkpoint_v4", "load_puf_tax_unit_donor", "normalize_district_code", "parse_baf_cd_layer", @@ -1990,9 +944,1067 @@ "parse_tract_to_puma_relationship", "us_puma_ladder_assignment_summary", "us_puma_ladder_gate", + "us_puma_ladder_joint_support_gate", "with_household_us_puma_ladder", ] +# Explicit compatibility targets: no discovery or source parsing at runtime. +_EXPORT_MODULES = { + "microcosm.build.us_runtime.adult_care": ( + "US_ADULT_CARE_CHILD_QUALIFYING_AGE_LIMIT", + "US_ADULT_CARE_EARNED_INCOME_SOURCES", + "US_ADULT_CARE_OUTPUT_COLUMNS", + "US_ADULT_CARE_REQUIRED_SOURCE_COLUMNS", + "US_ADULT_CARE_STAGE_NAME", + "derive_us_adult_care_from_manifest", + "us_adult_care_signal_gate", + "us_adult_care_stage_spec", + "us_adult_care_summary", + "with_us_adult_care_inputs", + ), + "microcosm.build.us_runtime.alimony": ( + "ALIMONY_ASEC_ARCHIVED_DERIVATION_URL", + "ALIMONY_PUF_ARCHIVED_DERIVATION_URL", + "STRIKE_BENEFITS_ASEC_ARCHIVED_DERIVATION_URL", + "US_ALIMONY_NONCONSTANT_PERSON_COLUMNS", + "US_ALIMONY_OUTPUT_COLUMNS", + "US_ALIMONY_STAGE_NAME", + "US_ASEC_OTHER_INCOME_OUTPUT_COLUMNS", + "derive_us_alimony_from_asec", + "derive_us_alimony_from_puf", + "us_alimony_signal_gate", + "us_alimony_stage_spec", + "us_alimony_summary", + ), + "microcosm.build.us_runtime.asec_checkpoint": ( + "ASEC_RAW_STAGE_ARTIFACT_KIND", + "ASEC_RAW_STAGE_CHECKPOINT_FILENAME", + "ASEC_RAW_STAGE_COVERAGE_SCHEMA_VERSION", + "ASEC_RAW_STAGE_OPERATOR_STATUS", + "ASEC_RAW_STAGE_SCHEMA_VERSION", + "ASEC_RAW_STAGE_STAGE", + "load_asec_pre_clone_checkpoint", + "load_asec_raw_stage_checkpoint", + "load_asec_raw_stage_checkpoint_v4", + ), + "microcosm.build.us_runtime.asec_pool": ( + "AsecSource", + "build_pooled_asec_unit_frame", + "load_asec_h5_tables", + "pool_asec_sources", + ), + "microcosm.build.us_runtime.capital_gain_details": ( + "CAPITAL_GAIN_DETAILS_ARCHIVED_DERIVATION_URL", + "CAPITAL_GAIN_DETAILS_ARCHIVED_EXPORT_URL", + "CAPITAL_GAIN_DETAILS_ARCHIVED_IMPUTATION_URL", + "CAPITAL_GAIN_DETAILS_ARCHIVED_PERSON_ALLOCATION_URL", + "CAPITAL_GAIN_DETAILS_ARCHIVED_PUF_ARTIFACT_URL", + "US_CAPITAL_GAIN_DETAILS_NONCONSTANT_PERSON_COLUMNS", + "US_CAPITAL_GAIN_DETAILS_NONCONSTANT_TAX_UNIT_COLUMNS", + "US_CAPITAL_GAIN_DETAILS_OUTPUT_COLUMNS", + "US_CAPITAL_GAIN_DETAILS_STAGE_NAME", + "derive_us_capital_gain_details_from_puf", + "us_capital_gain_details_signal_gate", + "us_capital_gain_details_stage_spec", + "us_capital_gain_details_summary", + ), + "microcosm.build.us_runtime.casualty_losses": ( + "US_CASUALTY_LOSS_NONCONSTANT_PERSON_COLUMNS", + "US_CASUALTY_LOSS_OUTPUT_COLUMNS", + "US_CASUALTY_LOSS_STAGE_NAME", + "derive_us_casualty_loss_from_puf", + "us_casualty_loss_signal_gate", + "us_casualty_loss_stage_spec", + "us_casualty_loss_summary", + ), + "microcosm.build.us_runtime.child_support": ( + "CHILD_SUPPORT_ARCHIVED_PUF_IMPUTATION_URL", + "CHILD_SUPPORT_ARCHIVED_PUF_OUTPUTS_URL", + "CHILD_SUPPORT_EXPENSE_ARCHIVED_DERIVATION_URL", + "CHILD_SUPPORT_RECEIVED_ARCHIVED_DERIVATION_URL", + "US_CHILD_SUPPORT_NONCONSTANT_PERSON_COLUMNS", + "US_CHILD_SUPPORT_OUTPUT_COLUMNS", + "US_CHILD_SUPPORT_REQUIRED_SOURCE_COLUMNS", + "US_CHILD_SUPPORT_STAGE_NAME", + "derive_us_child_support_from_asec", + "derive_us_child_support_from_manifest", + "impute_us_child_support_to_puf_support_from_manifest", + "us_child_support_signal_gate", + "us_child_support_stage_spec", + "us_child_support_summary", + "with_us_child_support_inputs", + ), + "microcosm.build.us_runtime.childcare": ( + "US_CHILDCARE_OUTPUT_COLUMNS", + "US_CHILDCARE_REQUIRED_SOURCE_COLUMNS", + "US_CHILDCARE_STAGE_NAME", + "derive_us_childcare_from_manifest", + "impute_us_childcare_to_puf_support_from_manifest", + "us_childcare_signal_gate", + "us_childcare_stage_spec", + "us_childcare_summary", + "with_us_childcare_inputs", + ), + "microcosm.build.us_runtime.congressional_district_geography": ( + "CONGRESSIONAL_DISTRICT_GEOID_COLUMN", + "SOI_CONGRESSIONAL_DISTRICT_RECORD_SET_ID", + "assign_congressional_districts_to_households", + "congressional_district_assignment_summary", + "congressional_district_distribution_from_ledger_facts", + "with_household_congressional_districts", + ), + "microcosm.build.us_runtime.congressional_district_vintage": ( + "CONGRESSIONAL_DISTRICT_VINTAGE_CROSSWALK_SHA256_ATTR", + "CONGRESSIONAL_DISTRICT_VINTAGE_TARGET_ATTR", + "CURRENT_CONGRESSIONAL_DISTRICT_PREFIX", + "CURRENT_CONGRESSIONAL_DISTRICT_VINTAGE", + "DEFAULT_CONGRESSIONAL_DISTRICT_VINTAGE_CROSSWALK_RESOURCE", + "SOURCE_CONGRESSIONAL_DISTRICT_PREFIX", + "default_congressional_district_vintage_crosswalk_path", + "load_congressional_district_vintage_crosswalk", + "load_default_congressional_district_vintage_crosswalk", + "translate_congressional_district_facts_to_current_vintage", + ), + "microcosm.build.us_runtime.congressional_district_vintage_crosswalk": ( + "CROSSWALK_BASIS_BLOCK_POPULATION", + "build_cd_vintage_crosswalk_rows", + "normalize_district_code", + "parse_baf_cd_layer", + "parse_national_cd_bef_districts", + ), + "microcosm.build.us_runtime.cps_carried": ( + "CPS_CARRIED_FORMULA_OWNED_COLUMNS", + "CPS_CARRIED_PERSON_INPUTS", + "CPS_CARRIED_SPM_UNIT_INPUTS", + "CPS_REPORTED_TANF_AMOUNT_RAW_COLUMN", + "CPS_REPORTED_TANF_TYPE_RAW_COLUMN", + "CPS_REPORTED_WIC_RAW_COLUMN", + "US_REPORTED_COVERAGE_PERSON_INPUTS", + "US_REPORTED_COVERAGE_VINTAGE_GATE_MIN_ROWS", + "WIC_CARRIER_ADJUDICATION_URL", + "derive_us_cps_carried_inputs", + "reported_tanf_enrollment_by_spm_unit", + "reported_wic_receipt_carrier", + ), + "microcosm.build.us_runtime.demographics": ( + "AGE_BANDS", + "DEMOGRAPHICS_SCHEMA_VERSION", + "AgeBand", + "compute_age_distribution", + "demographics_payload", + "write_demographics", + ), + "microcosm.build.us_runtime.disability_benefits": ( + "DISABILITY_BENEFITS_ARCHIVED_DERIVATION_URL", + "DISABILITY_BENEFITS_ARCHIVED_PUF_IMPUTATION_URL", + "DISABILITY_BENEFITS_ARCHIVED_PUF_OUTPUTS_URL", + "DISABILITY_BENEFITS_ARCHIVED_SOURCE_COLUMNS_URL", + "US_DISABILITY_BENEFITS_NONCONSTANT_PERSON_COLUMNS", + "US_DISABILITY_BENEFITS_OUTPUT_COLUMNS", + "US_DISABILITY_BENEFITS_REQUIRED_SOURCE_COLUMNS", + "US_DISABILITY_BENEFITS_STAGE_NAME", + "derive_us_disability_benefits_from_asec", + "derive_us_disability_benefits_from_manifest", + "impute_us_disability_benefits_to_puf_support_from_manifest", + "us_disability_benefits_signal_gate", + "us_disability_benefits_stage_spec", + "us_disability_benefits_summary", + "with_us_disability_benefits", + ), + "microcosm.build.us_runtime.domestic_production": ( + "DOMESTIC_PRODUCTION_ALD_ARCHIVED_DERIVATION_URL", + "DOMESTIC_PRODUCTION_ALD_ARCHIVED_EXPORT_URL", + "DOMESTIC_PRODUCTION_ALD_ARCHIVED_IMPUTATION_URL", + "DOMESTIC_PRODUCTION_ALD_ARCHIVED_PUF_ARTIFACT_URL", + "US_DOMESTIC_PRODUCTION_ALD_NONCONSTANT_TAX_UNIT_COLUMNS", + "US_DOMESTIC_PRODUCTION_ALD_OUTPUT_COLUMNS", + "US_DOMESTIC_PRODUCTION_ALD_STAGE_NAME", + "derive_us_domestic_production_ald_from_puf", + "us_domestic_production_ald_signal_gate", + "us_domestic_production_ald_stage_spec", + "us_domestic_production_ald_summary", + ), + "microcosm.build.us_runtime.education_assistance_source": ( + "ASEC_EDUCATION_ASSISTANCE_ARCHIVES", + "ASEC_EDUCATION_ASSISTANCE_INCOME_YEARS", + "fetch_asec_education_assistance_source", + "fill_asec_education_assistance_source", + "load_asec_education_assistance_sources", + ), + "microcosm.build.us_runtime.education_inputs": ( + "US_AOTC_ELIGIBILITY_OUTPUT_COLUMNS", + "US_EDUCATION_INPUTS_NONCONSTANT_PERSON_COLUMNS", + "US_EDUCATION_INPUTS_OUTPUT_COLUMNS", + "US_EDUCATION_INPUTS_OWNED_OUTPUT_COLUMNS", + "US_EDUCATION_INPUTS_REQUIRED_SOURCE_COLUMNS", + "US_EDUCATION_INPUTS_STAGE_NAME", + "derive_us_education_inputs_from_manifest", + "us_education_inputs_signal_gate", + "us_education_inputs_stage_spec", + "us_education_inputs_summary", + "with_us_education_inputs", + ), + "microcosm.build.us_runtime.educator_expenses": ( + "EDUCATOR_EXPENSE_ARCHIVED_ALLOCATION_URL", + "EDUCATOR_EXPENSE_ARCHIVED_DERIVATION_URL", + "EDUCATOR_EXPENSE_ARCHIVED_EXPORT_URL", + "EDUCATOR_EXPENSE_ARCHIVED_PUF_IMPUTATION_URL", + "US_EDUCATOR_EXPENSE_NONCONSTANT_PERSON_COLUMNS", + "US_EDUCATOR_EXPENSE_OUTPUT_COLUMNS", + "US_EDUCATOR_EXPENSE_STAGE_NAME", + "derive_us_educator_expense_from_puf", + "us_educator_expense_signal_gate", + "us_educator_expense_stage_spec", + "us_educator_expense_summary", + ), + "microcosm.build.us_runtime.eligibility_inputs": ( + "US_ELIGIBILITY_INPUTS_NONCONSTANT_PERSON_COLUMNS", + "US_ELIGIBILITY_INPUTS_OUTPUT_COLUMNS", + "US_ELIGIBILITY_INPUTS_REQUIRED_SOURCE_COLUMNS", + "US_ELIGIBILITY_INPUTS_STAGE_NAME", + "derive_us_eligibility_inputs_from_manifest", + "us_eligibility_inputs_signal_gate", + "us_eligibility_inputs_stage_spec", + "us_eligibility_inputs_summary", + "with_us_eligibility_inputs", + ), + "microcosm.build.us_runtime.energy_subsidy": ( + "ENERGY_SUBSIDY_ARCHIVED_CPS_DERIVATION_URL", + "ENERGY_SUBSIDY_ARCHIVED_PUF_IMPUTATION_URL", + "US_ENERGY_SUBSIDY_OUTPUT_COLUMNS", + "US_ENERGY_SUBSIDY_REQUIRED_SOURCE_COLUMNS", + "US_ENERGY_SUBSIDY_STAGE_NAME", + "derive_us_energy_subsidy_from_manifest", + "impute_us_energy_subsidy_to_puf_support_from_manifest", + "us_energy_subsidy_signal_gate", + "us_energy_subsidy_stage_spec", + "us_energy_subsidy_summary", + "with_us_energy_subsidy_input", + ), + "microcosm.build.us_runtime.farm_business_income": ( + "FARM_BUSINESS_INCOME_ARCHIVED_CPS_FARM_INCOME_URL", + "FARM_BUSINESS_INCOME_ARCHIVED_DERIVATION_URL", + "FARM_BUSINESS_INCOME_ARCHIVED_EXPORT_URL", + "FARM_BUSINESS_INCOME_ARCHIVED_IMPUTATION_URL", + "FARM_BUSINESS_INCOME_ARCHIVED_OVERRIDE_URL", + "FARM_BUSINESS_INCOME_ARCHIVED_PUF_ARTIFACT_URL", + "US_FARM_BUSINESS_INCOME_NONCONSTANT_PERSON_COLUMNS", + "US_FARM_BUSINESS_INCOME_OUTPUT_COLUMNS", + "US_FARM_BUSINESS_INCOME_STAGE_NAME", + "derive_us_farm_business_income_from_puf", + "us_farm_business_income_signal_gate", + "us_farm_business_income_stage_spec", + "us_farm_business_income_summary", + ), + "microcosm.build.us_runtime.fiscal_targets": ( + "SOI_VARIABLE_MAP", + "US_FISCAL_LEDGER_PARITY_REGISTRY", + "US_FISCAL_LEDGER_PARITY_REPORT", + "US_FISCAL_MACRO_REALISM_BANDS", + "US_FISCAL_TARGET_COVERAGE_REQUIREMENTS", + "US_FISCAL_TARGET_LEDGER_REFERENCES", + "US_FISCAL_TARGET_REFERENCES", + "US_FISCAL_TARGET_REGISTRY", + "US_FISCAL_TARGET_SPECS", + "US_FISCAL_TARGET_SUPPORT_EXCLUSIONS", + "US_JCT_TAX_EXPENDITURE_REFORMS", + "US_JCT_TAX_EXPENDITURE_TARGET_REFERENCES", + "US_JCT_TAX_EXPENDITURE_TARGET_SPECS", + "US_SOI_FISCAL_TARGET_REFERENCES", + "US_SOI_FISCAL_TARGET_SPECS", + "US_STATE_INCOME_TAX_TARGET_REFERENCES", + "US_STATE_INCOME_TAX_TARGET_SPECS", + "SimpleTaxExpenditureReform", + "compile_us_fiscal_target_registry", + ), + "microcosm.build.us_runtime.form_4952": ( + "FORM_4952_ARCHIVED_DERIVATION_URL", + "FORM_4952_ARCHIVED_EXPORT_URL", + "FORM_4952_ARCHIVED_IMPUTATION_URL", + "FORM_4952_ARCHIVED_PERSON_ALLOCATION_URL", + "FORM_4952_ARCHIVED_PUF_ARTIFACT_URL", + "US_FORM_4952_NONCONSTANT_PERSON_COLUMNS", + "US_FORM_4952_OUTPUT_COLUMNS", + "US_FORM_4952_STAGE_NAME", + "derive_us_form_4952_election_from_puf", + "us_form_4952_election_signal_gate", + "us_form_4952_election_stage_spec", + "us_form_4952_election_summary", + ), + "microcosm.build.us_runtime.geography_ladder": ( + "GEOGRAPHY_LADDER_ARTIFACT_SHA256_ATTR", + "GEOGRAPHY_LADDER_VINTAGES_ATTR", + "US_BLOCK_LADDER_DERIVED_LAYERS", + "US_BLOCK_LADDER_KIND", + "US_BLOCK_LADDER_SCHEMA_VERSION", + "US_GEOGRAPHY_LADDER_COLUMNS", + "US_NYC_COUNTY_FIPS", + "UsBlockLadder", + "assign_us_geography_ladder", + "load_us_block_ladder", + "us_geography_ladder_assignment_summary", + "us_geography_ladder_gate", + "with_household_us_geography_ladder", + ), + "microcosm.build.us_runtime.hours_worked": ( + "US_HOURS_WORKED_NONCONSTANT_PERSON_COLUMNS", + "US_HOURS_WORKED_OUTPUT_COLUMNS", + "US_HOURS_WORKED_POOL_EXCLUDED_COLUMNS", + "US_HOURS_WORKED_POOL_OUTPUT_COLUMNS", + "US_HOURS_WORKED_REQUIRED_SOURCE_COLUMNS", + "US_HOURS_WORKED_STAGE_NAME", + "derive_us_hours_worked_from_manifest", + "us_hours_worked_signal_gate", + "us_hours_worked_stage_spec", + "us_hours_worked_summary", + "with_us_hours_worked_inputs", + ), + "microcosm.build.us_runtime.housing_inputs": ( + "ACS_2022_RENT_ARTIFACT_SHA256", + "HOUSING_INPUTS_ARCHIVED_ACS_DERIVATION_URL", + "HOUSING_INPUTS_ARCHIVED_CPS_RENT_URL", + "HOUSING_INPUTS_ARCHIVED_CPS_SPM_URL", + "HOUSING_INPUTS_ARCHIVED_PUF_IMPUTATION_URL", + "HOUSING_TAKE_UP_ARCHIVED_DERIVATION_URL", + "HOUSING_TAKE_UP_ARCHIVED_HUD_ETL_URL", + "HOUSING_TAKE_UP_ARCHIVED_PARAMETER_URL", + "US_HOUSING_HOUSEHOLD_OUTPUT_COLUMNS", + "US_HOUSING_INPUTS_OUTPUT_COLUMNS", + "US_HOUSING_INPUTS_STAGE_NAME", + "US_HOUSING_NONCONSTANT_HOUSEHOLD_COLUMNS", + "US_HOUSING_NONCONSTANT_PERSON_COLUMNS", + "US_HOUSING_NONCONSTANT_SPM_UNIT_COLUMNS", + "US_HOUSING_PERSON_OUTPUT_COLUMNS", + "US_HOUSING_REQUIRED_HOUSEHOLD_SOURCE_COLUMNS", + "US_HOUSING_REQUIRED_PERSON_SOURCE_COLUMNS", + "US_HOUSING_SPM_UNIT_OUTPUT_COLUMNS", + "derive_us_housing_inputs", + "impute_us_housing_assistance_to_puf_support", + "impute_us_pre_subsidy_rent", + "load_acs_2022_rent_donor", + "us_housing_inputs_signal_gate", + "us_housing_inputs_stage_spec", + "us_housing_inputs_summary", + "with_us_housing_inputs", + ), + "microcosm.build.us_runtime.immigration": ( + "IMMIGRATION_STATUS_VALUES", + "SSN_CARD_TYPE_VALUES", + "US_IMMIGRATION_NONCONSTANT_PERSON_COLUMNS", + "US_IMMIGRATION_OUTPUT_COLUMNS", + "US_IMMIGRATION_REQUIRED_SOURCE_COLUMNS", + "US_IMMIGRATION_STAGE_NAME", + "UndocumentedControls", + "derive_us_immigration_status_from_manifest", + "us_immigration_composition_gate", + "us_immigration_composition_summary", + "us_immigration_stage_spec", + "with_us_immigration_inputs", + ), + "microcosm.build.us_runtime.input_mass": ("us_input_mass_totals",), + "microcosm.build.us_runtime.medicaid_take_up": ( + "US_MEDICAID_ENROLLMENT_SUBSTITUTIONS", + "US_MEDICAID_ENROLLMENT_TARGET_ROLE", + "US_MEDICAID_ENROLLMENT_TARGET_TABLE", + "US_MEDICAID_ENROLLMENT_TOLERANCE", + "US_MEDICAID_TAKE_UP_ANCHOR", + "US_MEDICAID_TAKE_UP_STAGE", + "US_MEDICAID_TAKE_UP_VARIABLE", + "MedicaidEnrollmentSubstitution", + "apply_us_medicaid_enrollment_substitutions", + "us_medicaid_source_person_table", + "us_medicaid_take_up_diagnostics", + "us_medicaid_take_up_gate", + "with_us_medicaid_take_up", + "write_us_medicaid_take_up_diagnostics", + ), + "microcosm.build.us_runtime.medicare_take_up": ( + "MEDICARE_TAKE_UP_ARCHIVED_CLONE_URL", + "MEDICARE_TAKE_UP_ARCHIVED_DERIVATION_URL", + "MEDICARE_TAKE_UP_ARCHIVED_EXPORT_URL", + "MEDICARE_TAKE_UP_ARCHIVED_SOURCE_COLUMNS_URL", + "US_MEDICARE_TAKE_UP_NONCONSTANT_PERSON_COLUMNS", + "US_MEDICARE_TAKE_UP_OUTPUT_COLUMNS", + "US_MEDICARE_TAKE_UP_REQUIRED_SOURCE_COLUMNS", + "US_MEDICARE_TAKE_UP_STAGE_NAME", + "derive_us_medicare_take_up_from_manifest", + "us_medicare_take_up_signal_gate", + "us_medicare_take_up_stage_spec", + "us_medicare_take_up_summary", + "with_us_medicare_take_up_input", + ), + "microcosm.build.us_runtime.misc_itemized": ( + "US_MISC_ITEMIZED_NONCONSTANT_PERSON_COLUMNS", + "US_MISC_ITEMIZED_OUTPUT_COLUMNS", + "US_MISC_ITEMIZED_STAGE_NAME", + "derive_us_misc_itemized_from_puf", + "us_misc_itemized_signal_gate", + "us_misc_itemized_stage_spec", + "us_misc_itemized_summary", + ), + "microcosm.build.us_runtime.nonzero_shares": ("nonzero_share", "us_nonzero_shares"), + "microcosm.build.us_runtime.operator_boundary": ( + "FORMULA_OWNED_SOURCE_COLUMNS", + "PRE_ASSEMBLY_OPERATOR_OUTPUT_FAMILIES", + "assert_operator_free_source_frame", + ), + "microcosm.build.us_runtime.org_wages": ( + "BLS_STATE_UNION_REPRESENTATION_RATE_2024", + "FLSA_EXECUTIVE_ADMINISTRATIVE_PROFESSIONAL_OCCUPATION_CODES", + "FLSA_OVERTIME_OCCUPATION_CODES", + "ORG_2024_DONOR_CONTENT_SHA256", + "ORG_2024_DONOR_FILENAME", + "ORG_PREDICTORS", + "US_ORG_WAGES_NONCONSTANT_PERSON_COLUMNS", + "US_ORG_WAGES_OUTPUT_COLUMNS", + "US_ORG_WAGES_REQUIRED_SOURCE_COLUMNS", + "US_ORG_WAGES_STAGE_NAME", + "derive_flsa_overtime_premium", + "derive_us_org_occupation_inputs", + "fetch_org_2024_donor", + "impute_us_org_wages", + "load_org_2024_donor", + "us_org_wages_signal_gate", + "us_org_wages_stage_spec", + "us_org_wages_summary", + "with_us_org_wages_inputs", + ), + "microcosm.build.us_runtime.other_health_insurance": ( + "OTHER_HEALTH_INSURANCE_ARCHIVED_DERIVATION_URL", + "OTHER_HEALTH_INSURANCE_ARCHIVED_PUF_IMPUTATION_URL", + "OTHER_HEALTH_INSURANCE_ARCHIVED_PUF_OUTPUTS_URL", + "OTHER_HEALTH_INSURANCE_ARCHIVED_PUF_PREDICTORS_URL", + "OTHER_HEALTH_INSURANCE_ARCHIVED_PUF_SPLICE_URL", + "US_OTHER_HEALTH_INSURANCE_MODELED_PREMIUM_VARIABLES", + "US_OTHER_HEALTH_INSURANCE_NONCONSTANT_PERSON_COLUMNS", + "US_OTHER_HEALTH_INSURANCE_OUTPUT_COLUMNS", + "US_OTHER_HEALTH_INSURANCE_REQUIRED_SOURCE_COLUMNS", + "US_OTHER_HEALTH_INSURANCE_STAGE_NAME", + "US_OTHER_HEALTH_INSURANCE_STAGE_OUTPUT_COLUMNS", + "US_SE_HEALTH_ATTRIBUTION_OUTPUT_COLUMNS", + "US_SE_HEALTH_MEDICARE_AGE_THRESHOLD", + "US_SE_HEALTH_SELF_EMPLOYMENT_INCOME_SOURCES", + "attribute_us_se_health_premiums", + "attribute_us_se_health_premiums_from_manifest", + "derive_us_other_health_insurance_from_asec", + "derive_us_other_health_insurance_from_manifest", + "impute_us_other_health_insurance_to_puf_support_from_manifest", + "us_other_health_insurance_signal_gate", + "us_other_health_insurance_stage_spec", + "us_other_health_insurance_summary", + "with_us_other_health_insurance_inputs", + ), + "microcosm.build.us_runtime.parity_reference": ( + "ECPS_PARITY_KNOWN_GAPS_RESOURCE", + "ECPS_PARITY_REFERENCE_RESOURCE", + "EcpsParityReference", + "EcpsParitySource", + "ParityKnownGap", + "load_ecps_parity_known_gaps", + "load_ecps_parity_reference", + ), + "microcosm.build.us_runtime.pregnancy": ( + "US_PREGNANCY_NONCONSTANT_PERSON_COLUMNS", + "US_PREGNANCY_OUTPUT_COLUMN", + "US_PREGNANCY_REQUIRED_SOURCE_COLUMNS", + "US_PREGNANCY_STAGE_NAME", + "derive_us_pregnancy_from_manifest", + "us_pregnancy_signal_gate", + "us_pregnancy_stage_spec", + "us_pregnancy_summary", + "with_us_pregnancy_inputs", + ), + "microcosm.build.us_runtime.prior_year_income": ( + "PRIOR_YEAR_INCOME_ARCHIVED_DERIVATION_URL", + "PRIOR_YEAR_INCOME_ARCHIVED_FINALIZER_URL", + "PRIOR_YEAR_INCOME_ARCHIVED_FORMULA_OUTPUT_URL", + "PRIOR_YEAR_INCOME_ARCHIVED_PUF_IMPUTATION_URL", + "PRIOR_YEAR_INCOME_ARCHIVED_PUF_OUTPUTS_URL", + "PRIOR_YEAR_INCOME_ARCHIVED_PUF_SPLICE_URL", + "US_PRIOR_YEAR_INCOME_FORMULA_OWNED_OUTPUT_COLUMNS", + "US_PRIOR_YEAR_INCOME_NONCONSTANT_PERSON_COLUMNS", + "US_PRIOR_YEAR_INCOME_OUTPUT_COLUMNS", + "US_PRIOR_YEAR_INCOME_PERSISTED_OUTPUT_COLUMNS", + "US_PRIOR_YEAR_INCOME_REQUIRED_SOURCE_COLUMNS", + "US_PRIOR_YEAR_INCOME_STAGE_NAME", + "derive_us_prior_year_income_from_manifest", + "impute_us_prior_year_income_to_puf_support_from_manifest", + "us_prior_year_income_signal_gate", + "us_prior_year_income_source_reconciliation_gate", + "us_prior_year_income_stage_spec", + "us_prior_year_income_summary", + "with_us_prior_year_income_inputs", + ), + "microcosm.build.us_runtime.public_assistance_type_source": ( + "ASEC_PUBLIC_ASSISTANCE_TYPE_AUDIT_PINS", + "ASEC_PUBLIC_ASSISTANCE_TYPE_INCOME_YEARS", + "PAW_TYPE_TANF_CODES", + "PAW_TYPE_VALID_CODES", + "fill_asec_public_assistance_type_source", + "load_asec_public_assistance_type_sources", + ), + "microcosm.build.us_runtime.operator_column_contracts": ( + "PUF_CAPITAL_GAINS_TAIL_APPLIED_COLUMN", + "PUF_CAPITAL_GAINS_TAIL_DONOR_AGI_BAND_COLUMN", + "PUF_CAPITAL_GAINS_TAIL_DONOR_FILING_STATUS_COLUMN", + "PUF_CAPITAL_GAINS_TAIL_DONOR_SOURCE_ID_COLUMN", + "PUF_CAPITAL_GAINS_TAIL_DONOR_SYNTHETIC_COLUMN", + "PUF_CAPITAL_GAINS_TAIL_PERSON_COLUMNS", + "PUF_CAPITAL_GAINS_TAIL_TAX_UNIT_COLUMNS", + "PUF_CAPITAL_GAINS_TAIL_TRANSFER_WEIGHT_COLUMN", + "PUF_TAX_DETAIL_DEFAULT_PERSON_OUTPUTS", + "PUF_TAX_DETAIL_DEFAULT_TAX_UNIT_OUTPUTS", + "US_PUF_SUPPORT_STAGE_NAME", + "US_QBI_BOOLEAN_OUTPUT_COLUMNS", + "US_QBI_NONNEGATIVE_OUTPUT_COLUMNS", + "US_QBI_OUTPUT_COLUMNS", + ), + "microcosm.build.us_runtime.puf_capital_gains_tail": ( + "PUF_CAPITAL_GAINS_TAIL_MANIFEST_SCHEMA_VERSION", + "PUF_CAPITAL_GAINS_TAIL_POSITIVE_MASS_FIVE_X_TARGET", + "PUF_CAPITAL_GAINS_TAIL_QUANTILE", + "PUF_CAPITAL_GAINS_TAIL_STAGE_NAME", + "PUF_CAPITAL_GAINS_TAIL_SUPPORT_CHANNEL", + "PUF_CAPITAL_GAINS_TAIL_SUPPORT_CONTRACT_VERSION", + "assert_puf_capital_gains_tail_survives_selection", + "puf_capital_gains_tail_concentration_gate", + "puf_capital_gains_tail_support_contract_identity", + "puf_capital_gains_tail_terminal_support_receipt", + "select_puf_capital_gains_tail_donors", + "transfer_puf_capital_gains_tail", + "validate_puf_capital_gains_tail_manifest", + "validate_puf_capital_gains_tail_terminal_support_receipt", + "write_puf_capital_gains_tail_manifest", + ), + "microcosm.build.us_runtime.puf_donor_io": ("load_puf_tax_unit_donor",), + "microcosm.build.us_runtime.puf_e01000_reconciliation": ( + "PUF_E01000_RECONCILIATION_SCHEMA_VERSION", + "build_puf_e01000_reconciliation_basis", + "finalize_puf_e01000_reconciliation", + "puf_capital_gains_joint_metrics", + "puf_processed_capital_gains_stage", + "puf_raw_e01000_stage", + ), + "microcosm.build.us_runtime.puf_interest_components": ( + "US_PUF_E19200_AGI_BANDS", + "US_PUF_E19200_ALL_RETURNS_COMPONENTS", + "PufE19200AgiBand", + "PufE19200InterestComponents", + "split_us_puf_e19200_by_agi_band", + ), + "microcosm.build.us_runtime.puf_source_agi": ( + "PUF_AGGREGATE_DISAGGREGATION_SEED", + "PUF_AGGREGATE_RECIDS", + "PUF_SOURCE_YEAR", + "PUF_SOURCE_YEAR_AGI_REQUIRED_COLUMNS", + "PUF_SYNTHETIC_RECID_START", + "source_year_puf_adjusted_gross_income", + ), + "microcosm.build.us_runtime.puf_support": ( + "BASE_ASEC_SUPPORT_CHANNEL", + "PUF_DONOR_SOURCE_ADJUSTED_GROSS_INCOME_COLUMN", + "PUF_TAX_DETAIL_CLONE_INDEX", + "PUF_TAX_DETAIL_SUPPORT_CHANNEL", + "US_PUF_DONOR_MORTGAGE_OUTLIER_CEILING", + "US_PUF_SUPPORT_FIT_NAME", + "PufTaxDetailChainInputs", + "clone_us_frame_for_puf_support", + "finalize_us_puf_tax_detail_predictions", + "has_assembled_support_metadata", + "has_support_role_metadata", + "impute_us_puf_tax_detail_support", + "prepare_us_puf_tax_detail_chain_inputs", + "puf_tax_detail_clone_mask", + "puf_tax_unit_donor_from_arrays", + "spine_source_id_column", + "support_channel_column", + "support_clone_index_column", + "support_role_series", + "support_source_id_column", + ), + "microcosm.build.us_runtime.puma_ladder": ( + "PUMA_LADDER_ARTIFACT_SHA256_ATTR", + "PUMA_LADDER_VINTAGES_ATTR", + "US_PUMA_LADDER_COLUMNS", + "US_PUMA_LADDER_DERIVED_LAYERS", + "US_PUMA_LADDER_KIND", + "US_PUMA_LADDER_SCHEMA_VERSION", + "US_PUMA_LADDER_TRACT_COLUMN", + "UsPumaLadder", + "assign_us_puma_ladder", + "load_us_puma_ladder", + "us_puma_ladder_assignment_summary", + "us_puma_ladder_gate", + "us_puma_ladder_joint_support_gate", + "with_household_us_puma_ladder", + ), + "microcosm.build.us_runtime.puma_ladder_sources": ( + "assemble_us_puma_ladder", + "parse_tract_to_puma_relationship", + ), + "microcosm.build.us_runtime.qbi_inputs": ( + "QBI_ARCHIVED_ASSUMPTIONS_URL", + "QBI_ARCHIVED_CLONE_URL", + "QBI_ARCHIVED_DERIVATION_URL", + "QBI_ARCHIVED_EXPORT_URL", + "QBI_ARCHIVED_IMPUTATION_URL", + "QBI_ARCHIVED_PUF_ARTIFACT_URL", + "QBI_ARCHIVED_SIMULATION_URL", + "US_QBI_NONCONSTANT_PERSON_COLUMNS", + "US_QBI_STAGE_NAME", + "us_qbi_inputs_signal_gate", + "us_qbi_inputs_stage_spec", + "us_qbi_inputs_summary", + "with_us_qbi_input_reconciliation", + ), + "microcosm.build.us_runtime.reform_coverage_smoke": ( + "us_reform_coverage_smoke_gate", + ), + "microcosm.build.us_runtime.reform_validation": ( + "REFORM_VALIDATION_SCHEMA_VERSION", + "ReformValidationSpec", + "in_sample_reform_specs", + "load_default_reform_specs", + "out_of_sample_reform_specs", + "reform_validation_payload", + "write_reform_validation", + ), + "microcosm.build.us_runtime.register_consistency": ( + "us_register_consistency_gate", + "us_register_contradictions", + ), + "microcosm.build.us_runtime.relationship_inputs": ( + "US_RELATIONSHIP_INPUTS_NONCONSTANT_PERSON_COLUMNS", + "US_RELATIONSHIP_INPUTS_OUTPUT_COLUMNS", + "US_RELATIONSHIP_INPUTS_REQUIRED_SOURCE_COLUMNS", + "US_RELATIONSHIP_INPUTS_STAGE_NAME", + "derive_us_relationship_inputs_from_manifest", + "us_relationship_inputs_signal_gate", + "us_relationship_inputs_stage_spec", + "us_relationship_inputs_summary", + "with_us_relationship_inputs", + ), + "microcosm.build.us_runtime.release_input_coverage": ( + "POST_REFERENCE_ECPS_REQUIRED_INPUTS", + "SSI_COUNTABLE_RESOURCE_ASSETS", + "US_CGD_ROUTE_REQUIRED_INPUTS", + "US_RELEASE_INPUT_COVERAGE_RESOURCE", + "ReformCoverageProbe", + "ReleaseInputColumn", + "ReleaseInputCoverageManifest", + "assert_release_input_coverage_manifest_current", + "load_release_input_coverage_manifest", + "us_release_input_coverage_gate", + "us_release_input_coverage_required_columns", + "us_release_input_coverage_reviewed_exclusions", + "us_release_reform_coverage_probes", + ), + "microcosm.build.us_runtime.release_target_parity": ( + "RED_LINE_COMPILED_FAMILIES", + "US_TARGET_PARITY_FEED_FAMILIES_RESOURCE", + "US_TARGET_PARITY_MANIFEST_RESOURCE", + "TargetFamily", + "TargetFence", + "TargetParityManifest", + "assert_target_parity_manifest_current", + "load_target_parity_feed_families", + "load_target_parity_manifest", + "registry_target_family_ids", + "us_release_target_parity_compiled_families", + "us_release_target_parity_gate", + "us_release_target_parity_reviewed_exclusions", + "us_target_family_id", + ), + "microcosm.build.us_runtime.retirement_contributions": ( + "US_RETIREMENT_CONTRIBUTION_NONCONSTANT_PERSON_COLUMNS", + "US_RETIREMENT_CONTRIBUTION_OUTPUT_COLUMNS", + "US_RETIREMENT_CONTRIBUTION_REQUIRED_SOURCE_COLUMNS", + "US_RETIREMENT_CONTRIBUTION_STAGE_NAME", + "derive_us_retirement_contributions_from_manifest", + "impute_us_retirement_contributions_to_puf_support_from_manifest", + "us_retirement_contributions_signal_gate", + "us_retirement_contributions_stage_spec", + "us_retirement_contributions_summary", + "with_us_retirement_contribution_inputs", + ), + "microcosm.build.us_runtime.retirement_distributions": ( + "RETIREMENT_DISTRIBUTIONS_ARCHIVED_DERIVATION_URL", + "RETIREMENT_DISTRIBUTIONS_ARCHIVED_PARAMETERS_URL", + "US_RETIREMENT_DISTRIBUTION_NONCONSTANT_PERSON_COLUMNS", + "US_RETIREMENT_DISTRIBUTION_OUTPUT_COLUMNS", + "US_RETIREMENT_DISTRIBUTION_REQUIRED_SOURCE_COLUMNS", + "US_RETIREMENT_DISTRIBUTION_STAGE_NAME", + "derive_us_retirement_distributions_from_manifest", + "impute_us_retirement_distributions_to_puf_support_from_manifest", + "us_retirement_distributions_signal_gate", + "us_retirement_distributions_stage_spec", + "us_retirement_distributions_summary", + "with_us_retirement_distribution_inputs", + ), + "microcosm.build.us_runtime.salt_refund_income": ( + "SALT_REFUND_ARCHIVED_DERIVATION_URL", + "SALT_REFUND_ARCHIVED_EXPORT_URL", + "SALT_REFUND_ARCHIVED_IMPUTATION_URL", + "SALT_REFUND_ARCHIVED_PERSON_ALLOCATION_URL", + "SALT_REFUND_ARCHIVED_PUF_ARTIFACT_URL", + "US_SALT_REFUND_NONCONSTANT_PERSON_COLUMNS", + "US_SALT_REFUND_OUTPUT_COLUMNS", + "US_SALT_REFUND_STAGE_NAME", + "derive_us_salt_refund_income_from_puf", + "us_salt_refund_income_signal_gate", + "us_salt_refund_income_stage_spec", + "us_salt_refund_income_summary", + ), + "microcosm.build.us_runtime.scf_auto_loans": ( + "QUALIFIED_AUTO_LOAN_ANNUAL_ISSUANCE_TARGET", + "SCF_2022_FULL_EXTRACT_MEMBER", + "SCF_2022_FULL_EXTRACT_MEMBER_SHA256", + "SCF_2022_FULL_EXTRACT_URL", + "SCF_2022_FULL_EXTRACT_ZIP_SHA256", + "SCF_AUTO_LOAN_AMOUNT_COLUMNS", + "SCF_AUTO_LOAN_RATE_COLUMNS", + "US_SCF_AUTO_LOAN_NONCONSTANT_HOUSEHOLD_COLUMNS", + "US_SCF_AUTO_LOAN_OUTPUT_COLUMNS", + "fetch_scf_2022_full_extract", + "impute_us_scf_auto_loans", + "load_scf_2022_auto_loan_donor", + "qualified_auto_loan_interest_proxy", + "us_scf_auto_loans_signal_gate", + "us_scf_auto_loans_stage_spec", + "us_scf_auto_loans_summary", + "with_us_scf_auto_loan_inputs", + ), + "microcosm.build.us_runtime.scf_wealth": ( + "FINANCIAL_ASSET_BLEND_AUDIT_KEY", + "FINANCIAL_ASSET_SOURCE_SCF_PROBABILITY", + "SCF_FINANCIAL_ASSET_TARGET_COMPONENTS", + "SCF_NET_WORTH_TARGET_COMPONENTS", + "SCF_WEALTH_PREDICTORS", + "US_SCF_FINANCIAL_ASSET_OUTPUT_COLUMNS", + "US_SCF_NET_WORTH_OUTPUT_COLUMNS", + "US_SCF_WEALTH_NONCONSTANT_HOUSEHOLD_COLUMNS", + "US_SCF_WEALTH_NONCONSTANT_PERSON_COLUMNS", + "US_SCF_WEALTH_STAGE_NAME", + "fetch_scf_2022_summary_extract", + "financial_asset_source_is_scf", + "impute_us_scf_financial_assets", + "impute_us_scf_net_worth", + "impute_us_sipp_scf_financial_assets", + "load_scf_2022_financial_asset_donor", + "us_scf_wealth_signal_gate", + "us_scf_wealth_stage_spec", + "us_scf_wealth_summary", + "with_us_scf_wealth_inputs", + ), + "microcosm.build.us_runtime.sipp_financial_assets": ( + "SIPP_2023_FINANCIAL_ASSET_DONOR_REPOSITORY_ID_PARTS", + "SIPP_2023_FINANCIAL_ASSET_DONOR_REPOSITORY_TYPE", + "SIPP_2023_FINANCIAL_ASSET_DONOR_REVISION", + "SIPP_2023_FINANCIAL_ASSET_DONOR_SHA256", + "SIPP_2023_FINANCIAL_ASSET_DONOR_SIZE_BYTES", + "SIPP_2023_FINANCIAL_ASSET_DONOR_URL", + "SIPP_FINANCIAL_ASSET_DONOR_WEIGHT_COLUMN", + "SIPP_FINANCIAL_ASSET_MODEL_PREDICTORS", + "SIPP_FINANCIAL_ASSET_SOURCE_COLUMNS", + "SIPP_FINANCIAL_ASSET_TARGET_ALLOCATION_COLUMNS", + "SIPP_FINANCIAL_ASSET_TARGET_SOURCE_COLUMNS", + "fetch_sipp_2023_financial_asset_donor", + "impute_us_sipp_financial_assets", + "load_sipp_2023_financial_asset_donor", + ), + "microcosm.build.us_runtime.sipp_head_start": ( + "HEAD_START_SIPP_DICTIONARY_URL", + "SIPP_2023_HEAD_START_DONOR_REVISION", + "SIPP_2023_HEAD_START_DONOR_SHA256", + "SIPP_2023_HEAD_START_DONOR_SIZE_BYTES", + "SIPP_2023_HEAD_START_DONOR_URL", + "SIPP_HEAD_START_FIT_PARAMETERS", + "SIPP_HEAD_START_MODEL_PREDICTORS", + "SIPP_HEAD_START_READ_PARAMETERS", + "SIPP_HEAD_START_SOURCE_COLUMNS", + "US_SIPP_HEAD_START_NONCONSTANT_PERSON_COLUMNS", + "US_SIPP_HEAD_START_OUTPUT_COLUMNS", + "US_SIPP_HEAD_START_REQUIRED_SOURCE_COLUMNS", + "US_SIPP_HEAD_START_STAGE_NAME", + "fetch_sipp_2023_head_start_donor", + "impute_us_sipp_head_start", + "load_sipp_2023_head_start_donor", + "us_sipp_head_start_signal_gate", + "us_sipp_head_start_stage_spec", + "us_sipp_head_start_summary", + "with_us_sipp_head_start_input", + ), + "microcosm.build.us_runtime.sipp_tips": ( + "CENSUS_OCCUPATION_CODE_TO_TTOC", + "SIPP_2023_TIP_DONOR_REVISION", + "SIPP_2023_TIP_DONOR_SHA256", + "SIPP_2023_TIP_DONOR_URL", + "SIPP_TIP_OUTPUT_COLUMNS", + "SIPP_TIP_PREDICTORS", + "US_SIPP_TIPS_NONCONSTANT_PERSON_COLUMNS", + "US_SIPP_TIPS_OUTPUT_COLUMNS", + "US_SIPP_TIPS_REQUIRED_SOURCE_COLUMNS", + "US_SIPP_TIPS_STAGE_NAME", + "derive_treasury_tipped_occupation_code", + "fetch_sipp_2023_tip_donor", + "impute_us_sipp_tips", + "load_sipp_2023_tip_donor", + "us_sipp_tips_signal_gate", + "us_sipp_tips_stage_spec", + "us_sipp_tips_summary", + "with_us_sipp_tip_inputs", + ), + "microcosm.build.us_runtime.sipp_vehicles": ( + "SIPP_2023_VEHICLE_DONOR_REVISION", + "SIPP_2023_VEHICLE_DONOR_SHA256", + "SIPP_2023_VEHICLE_DONOR_SIZE_BYTES", + "SIPP_2023_VEHICLE_DONOR_URL", + "US_SIPP_VEHICLE_NONCONSTANT_HOUSEHOLD_COLUMNS", + "US_SIPP_VEHICLE_OUTPUT_COLUMNS", + "fetch_sipp_2023_vehicle_donor", + "load_sipp_2023_vehicle_donor", + "us_sipp_vehicles_signal_gate", + "us_sipp_vehicles_stage_spec", + "us_sipp_vehicles_summary", + "with_us_sipp_vehicle_inputs", + ), + "microcosm.build.us_runtime.snap_discretionary_exemption": ( + "US_SNAP_DISCRETIONARY_EXEMPTION_NONCONSTANT_PERSON_COLUMNS", + "US_SNAP_DISCRETIONARY_EXEMPTION_OUTPUT_COLUMN", + "US_SNAP_DISCRETIONARY_EXEMPTION_REQUIRED_SOURCE_COLUMNS", + "US_SNAP_DISCRETIONARY_EXEMPTION_STAGE_NAME", + "derive_us_snap_discretionary_exemption_from_manifest", + "us_snap_discretionary_exemption_signal_gate", + "us_snap_discretionary_exemption_stage_spec", + "us_snap_discretionary_exemption_summary", + "with_us_snap_discretionary_exemption_inputs", + ), + "microcosm.build.us_runtime.snap_state_take_up": ( + "US_SNAP_CASELOAD_TOLERANCE", + "US_SNAP_HOUSEHOLDS_TARGET_TABLE", + "US_SNAP_STATE_TAKE_UP_ANCHOR", + "US_SNAP_STATE_TAKE_UP_STAGE", + "us_snap_state_take_up_diagnostics", + "us_snap_state_take_up_gate", + "with_us_snap_state_take_up", + "write_us_snap_state_take_up_diagnostics", + ), + "microcosm.build.us_runtime.snap_take_up": ( + "US_SNAP_TAKE_UP_OUTPUT_COLUMN", + "US_SNAP_TAKE_UP_RAW_COLUMN", + "US_SNAP_TAKE_UP_STAGE_NAME", + "derive_us_snap_take_up_from_manifest", + "us_snap_take_up_signal_gate", + "us_snap_take_up_stage_spec", + "us_snap_take_up_summary", + "with_us_snap_take_up_inputs", + ), + "microcosm.build.us_runtime.source_coverage": ( + "CHRONICLE_US_SOURCE_COVERAGE_CONTRACT_COMMIT", + "LEDGER_US_SOURCE_COVERAGE_CONTRACT_COMMIT", + "US_SOURCE_COVERAGE", + "hard_target_package_aliases", + "source_gap_family_ids", + "us_source_coverage_diagnostics", + "us_source_coverage_gate", + "validation_only_family_ids", + "write_us_source_coverage_diagnostics", + ), + "microcosm.build.us_runtime.source_runtime": ( + "disaggregate_us_puf_aggregate_records_from_manifest", + "us_source_operation_handlers", + ), + "microcosm.build.us_runtime.spine_agreement": ( + "DEFAULT_CATEGORICAL_TOTAL_VARIATION_TOLERANCE", + "DEFAULT_INCIDENCE_RATIO_BOUNDS", + "DEFAULT_QUANTILE_ENVELOPE_TOLERANCE", + "DEFAULT_SPINE_AGREEMENT_QUANTILES", + "US_SPINE_AGREEMENT_REGISTRY", + "SpineAgreementSpec", + "default_spine_agreement_registry", + "normalize_transfer_family_name", + "spine_agreement_gate", + "validate_spine_agreement_registry", + ), + "microcosm.build.us_runtime.spine_assembly": ("assemble_spines",), + "microcosm.build.us_runtime.ssi_disability_criteria": ( + "SIPP_2023_SSI_DISABILITY_DONOR_REVISION", + "SIPP_2023_SSI_DISABILITY_DONOR_SHA256", + "SIPP_2023_SSI_DISABILITY_DONOR_SIZE_BYTES", + "SIPP_2023_SSI_DISABILITY_DONOR_URL", + "SIPP_SSI_DISABILITY_DIFFICULTY_PREDICTORS", + "SIPP_SSI_DISABILITY_FIT_PARAMETERS", + "SIPP_SSI_DISABILITY_MODEL_PREDICTORS", + "SIPP_SSI_DISABILITY_READ_PARAMETERS", + "SIPP_SSI_DISABILITY_SOURCE_COLUMNS", + "SSI_DISABILITY_ARCHIVED_CPS_URL", + "SSI_DISABILITY_ARCHIVED_EXTENDED_CPS_URL", + "SSI_DISABILITY_ARCHIVED_SIPP_URL", + "SSI_DISABILITY_ARCHIVED_SOURCE_IMPUTE_URL", + "SSI_DISABILITY_SIPP_DICTIONARY_URL", + "US_SSI_DISABILITY_CRITERIA_NONCONSTANT_PERSON_COLUMNS", + "US_SSI_DISABILITY_CRITERIA_OUTPUT_COLUMNS", + "US_SSI_DISABILITY_CRITERIA_STAGE_NAME", + "fetch_sipp_2023_ssi_disability_donor", + "impute_us_ssi_disability_criteria", + "load_sipp_2023_ssi_disability_donor", + "us_ssi_disability_criteria_signal_gate", + "us_ssi_disability_criteria_stage_spec", + "us_ssi_disability_criteria_summary", + "with_us_ssi_disability_criteria", + ), + "microcosm.build.us_runtime.ssi_take_up": ( + "SSI_TAKE_UP_ARCHIVED_DERIVATION_URL", + "SSI_TAKE_UP_ARCHIVED_EXPORT_URL", + "SSI_TAKE_UP_ARCHIVED_RANDOMNESS_URL", + "SSI_TAKE_UP_ARCHIVED_REPORTER_URL", + "SSI_TAKE_UP_ARCHIVED_TARGETS_URL", + "SSI_TAKE_UP_SSA_SOURCE_URL", + "US_SSI_TAKE_UP_ANCHOR", + "US_SSI_TAKE_UP_BAND_DELIVERY_RELATIVE_TOLERANCE", + "US_SSI_TAKE_UP_ENFORCED_BAND_KEYS", + "US_SSI_TAKE_UP_NONCONSTANT_PERSON_COLUMNS", + "US_SSI_TAKE_UP_OUTPUT_COLUMNS", + "US_SSI_TAKE_UP_PHASE_ASSIGNMENT", + "US_SSI_TAKE_UP_PHASE_RELEASE_FINAL", + "US_SSI_TAKE_UP_PRIOR_BASIS_CURRENT_FRAME", + "US_SSI_TAKE_UP_PRIOR_BASIS_RELEASE_ARTIFACT", + "US_SSI_TAKE_UP_REQUIRED_SOURCE_COLUMNS", + "US_SSI_TAKE_UP_STAGE_NAME", + "US_SSI_TAKE_UP_TARGET_TABLE_NAME", + "SSITakeUpBandPriorBasis", + "SSITakeUpPriorBasis", + "ssi_take_up_prior_basis_from_artifact", + "ssi_take_up_prior_basis_from_diagnostics", + "us_ssi_take_up_delivery_gate", + "us_ssi_take_up_diagnostics", + "us_ssi_take_up_gate", + "us_ssi_take_up_reporter_source_ids", + "with_us_ssi_take_up", + "write_us_ssi_take_up_diagnostics", + ), + "microcosm.build.us_runtime.support_provenance": ( + "us_reported_coverage_vintage_signal_gate", + ), + "microcosm.build.us_runtime.take_up": ( + "US_TAKE_UP_SHARE_BAND", + "SeededTakeUpResult", + "us_take_up_participation_diagnostics", + "us_take_up_signal_gate", + "us_take_up_summary", + "with_us_take_up_inputs", + "write_us_take_up_participation_diagnostics", + ), + "microcosm.build.us_runtime.take_up_contract": ( + "TakeUpContract", + "TakeUpProgram", + "assert_take_up_contract_current", + "assert_take_up_treatments_consistent", + "count_calibrated_take_up_programs", + "load_take_up_contract", + "seeded_take_up_programs", + ), + "microcosm.build.us_runtime.validation_input_coverage": ( + "US_VALIDATION_PROVISION_INPUT_LEAVES", + "ValidationInputLeaf", + "assert_validation_leaf_registry_current", + "us_source_stage_outputs", + "us_validation_input_coverage_gate", + ), + "microcosm.build.us_runtime.voluntary_filing": ( + "SIPP_2023_VOLUNTARY_FILING_DONOR_REVISION", + "SIPP_2023_VOLUNTARY_FILING_DONOR_SHA256", + "SIPP_2023_VOLUNTARY_FILING_DONOR_SIZE_BYTES", + "SIPP_2023_VOLUNTARY_FILING_DONOR_URL", + "SIPP_VOLUNTARY_FILING_MODEL_PREDICTORS", + "SIPP_VOLUNTARY_FILING_SOURCE_COLUMNS", + "US_VOLUNTARY_FILING_NONCONSTANT_TAX_UNIT_COLUMNS", + "US_VOLUNTARY_FILING_OUTPUT_COLUMNS", + "US_VOLUNTARY_FILING_STAGE_NAME", + "VOLUNTARY_FILING_ARCHIVED_DERIVATION_URL", + "VOLUNTARY_FILING_ARCHIVED_PARAMETERS_URL", + "VOLUNTARY_FILING_SIPP_DICTIONARY_URL", + "fetch_sipp_2023_voluntary_filing_donor", + "impute_us_voluntary_filing", + "load_sipp_2023_voluntary_filing_donor", + "us_voluntary_filing_signal_gate", + "us_voluntary_filing_stage_spec", + "us_voluntary_filing_summary", + "with_us_voluntary_filing_input", + ), + "microcosm.build.us_runtime.weeks_unemployed": ( + "ASEC_2023_WEEKS_UNEMPLOYED_MEMBER", + "ASEC_2023_WEEKS_UNEMPLOYED_MEMBER_CRC32", + "ASEC_2023_WEEKS_UNEMPLOYED_MEMBER_SHA256", + "ASEC_2023_WEEKS_UNEMPLOYED_MEMBER_SIZE_BYTES", + "ASEC_2023_WEEKS_UNEMPLOYED_RAW_ROWS", + "ASEC_2023_WEEKS_UNEMPLOYED_SOURCE_COLUMNS", + "ASEC_2023_WEEKS_UNEMPLOYED_SOURCE_SHA256", + "ASEC_2023_WEEKS_UNEMPLOYED_SOURCE_YEAR", + "ASEC_2023_WEEKS_UNEMPLOYED_UNIQUE_KEYS", + "ASEC_2023_WEEKS_UNEMPLOYED_WEIGHTED_SOURCE_SHARE", + "ASEC_2023_WEEKS_UNEMPLOYED_WEIGHTED_WEEKS", + "ASEC_2023_WEEKS_UNEMPLOYED_ZIP_SHA256", + "ASEC_2023_WEEKS_UNEMPLOYED_ZIP_SIZE_BYTES", + "ASEC_2023_WEEKS_UNEMPLOYED_ZIP_URL", + "US_WEEKS_UNEMPLOYED_NONCONSTANT_PERSON_COLUMNS", + "US_WEEKS_UNEMPLOYED_OUTPUT_COLUMNS", + "US_WEEKS_UNEMPLOYED_REQUIRED_SOURCE_COLUMNS", + "US_WEEKS_UNEMPLOYED_STAGE_NAME", + "WEEKS_UNEMPLOYED_ARCHIVED_DERIVATION_URL", + "WEEKS_UNEMPLOYED_ARCHIVED_PUF_IMPUTATION_URL", + "WEEKS_UNEMPLOYED_ARCHIVED_SOURCE_URL", + "WEEKS_UNEMPLOYED_DERIVE_PARAMETERS", + "WEEKS_UNEMPLOYED_PUF_IMPUTATION_PARAMETERS", + "WEEKS_UNEMPLOYED_PUF_PREDICTORS", + "WEEKS_UNEMPLOYED_READ_PARAMETERS", + "derive_us_weeks_unemployed_from_manifest", + "fetch_asec_2023_weeks_unemployed_source", + "fill_asec_2022_weeks_unemployed_source", + "impute_us_weeks_unemployed_to_puf_support_from_manifest", + "load_asec_2023_weeks_unemployed_source", + "us_weeks_unemployed_signal_gate", + "us_weeks_unemployed_stage_spec", + "us_weeks_unemployed_summary", + "with_us_weeks_unemployed", + ), + "microcosm.build.us_runtime.wic_claim": ( + "US_WIC_CLAIM_NONCONSTANT_PERSON_COLUMNS", + "US_WIC_CLAIM_OUTPUT_COLUMNS", + "US_WIC_CLAIM_REQUIRED_SOURCE_COLUMNS", + "US_WIC_CLAIM_STAGE_NAME", + "WIC_CLAIM_ARCHIVED_DERIVATION_URL", + "WIC_CLAIM_ARCHIVED_PARAMETERS_URL", + "WIC_CLAIM_ARCHIVED_RANDOMNESS_URL", + "WIC_CLAIM_FNS_SOURCE_URL", + "derive_us_wic_claim_from_manifest", + "us_wic_claim_signal_gate", + "us_wic_claim_stage_spec", + "us_wic_claim_summary", + "with_us_wic_claim_input", + ), + "microcosm.build.us_runtime.workers_compensation": ( + "US_WORKERS_COMPENSATION_NONCONSTANT_PERSON_COLUMNS", + "US_WORKERS_COMPENSATION_OUTPUT_COLUMNS", + "US_WORKERS_COMPENSATION_REQUIRED_SOURCE_COLUMNS", + "US_WORKERS_COMPENSATION_STAGE_NAME", + "WORKERS_COMPENSATION_ARCHIVED_DERIVATION_URL", + "WORKERS_COMPENSATION_ARCHIVED_PUF_IMPUTATION_URL", + "WORKERS_COMPENSATION_ARCHIVED_PUF_OUTPUTS_URL", + "WORKERS_COMPENSATION_ARCHIVED_SOURCE_COLUMNS_URL", + "derive_us_workers_compensation_from_asec", + "derive_us_workers_compensation_from_manifest", + "impute_us_workers_compensation_to_puf_support_from_manifest", + "us_workers_compensation_signal_gate", + "us_workers_compensation_stage_spec", + "us_workers_compensation_summary", + "with_us_workers_compensation", + ), +} +_LAZY_EXPORTS = { + name: (module, name) for module, names in _EXPORT_MODULES.items() for name in names +} + @dataclass(frozen=True) class BuildConfig: @@ -2045,428 +2057,560 @@ def to_manifest(self) -> dict[str, object]: } -#: The US donor graph: every imputation stage's primary survey, with -#: citations. This is the single place the build's sources are declared — -#: the observatory's sources diagram and the dataset card derive from it. -US_DONORS: Mapping[str, DonorSpec] = { - "scf_wealth": DonorSpec( - survey="Fed SCF 2022 + Census SIPP 2023", - source="https://www.federalreserve.gov/econres/scfindex.htm", - notes=( - "SCF anchors signed net worth and one half of household liquid-asset " - "vectors; the immutable SIPP 2023 public-use donor supplies the " - "other half and restores low liquid-asset mass. Auto loans use the " - "full SCF separately." - ), - ), - US_SSI_DISABILITY_CRITERIA_STAGE_NAME: DonorSpec( - survey="Census SIPP", - source="https://www.census.gov/programs-surveys/sipp.html", - notes=( - "Latent under-65 SSI disability/blindness criterion from the " - "pinned full 2023 public-use donor. ASEC and PUF-support people " - "are predicted separately, and only direct under-65 ASEC SSI " - "reporters receive the observed-reporter anchor." - ), - ), - US_SIPP_HEAD_START_STAGE_NAME: DonorSpec( - survey="Census SIPP", - source="https://www.census.gov/programs-surveys/sipp.html", - notes=( - "Direct December nursery/preschool federally sponsored-program " - "responses train a weighted Head Start take-up proxy for ages " - "3--5; strict reported structural negatives exclude hot-decked " - "answers and the prediction is shared by support clones." - ), - ), - US_SSI_TAKE_UP_STAGE_NAME: DonorSpec( - survey="CPS ASEC reported SSI + SSA SSI Monthly Statistics December 2024", - source=SSI_TAKE_UP_SSA_SOURCE_URL, - notes=( - "Direct ASEC SSI_VAL reporters anchor person-level take-up; " - "eligible source-person identities are count-calibrated by age to " - "SSA December 2024 Federal-payment recipient counts and fanned to " - "both support clones." - ), - ), - "sipp_tips": DonorSpec( - survey="Census SIPP", - source="https://www.census.gov/programs-surveys/sipp.html", - notes="Tip income for tipped occupations.", - ), - "org_wages": DonorSpec( - survey="CPS ORG", - source=("https://www2.census.gov/programs-surveys/cps/datasets/2024/basic/"), - notes=( - "Hourly-wage labor-market inputs. Donor load failures abort the " - "build — the silent zero-fallback this stage once had is " - "structurally impossible under StagePlan." - ), - ), - "meps_esi_premiums": DonorSpec( - survey="MEPS-IC", - source="https://meps.ahrq.gov/mepsweb/survey_comp/Insurance.jsp", - notes="Employer-sponsored insurance premium parameters.", - ), - "aca_marketplace_inputs": DonorSpec( - survey="CPS ASEC + CMS Marketplace Open Enrollment PUFs", - source="https://www.cms.gov/marketplace/resources/data/public-use-files", - notes=( - "Marketplace take-up and selected-plan inputs: CPS reported " - "Marketplace coverage and premium reports anchor the records; " - "CMS OEP enrollment, APTC, and metal-level tables provide the " - "calibration targets." - ), - ), - "medicaid_take_up": DonorSpec( - survey="CPS ASEC reported coverage + CMS Medicaid monthly enrollment snapshot", - source="https://data.medicaid.gov/dataset/6165f45b-ca93-5bb5-9d06-db29c692a360", - notes=( - "Medicaid take-up by anchored count-calibration (contract " - "treatment count_calibrated, microcosm #331): CPS-reported " - "Medicaid coverage at interview anchors the flag; the fill is " - "calibrated to CMS December 2024 state enrollment snapshots. " - "Point-in-time semantics per #332; heals the #170 " - "enrollment==eligibility degeneracy." - ), - ), - US_SNAP_STATE_TAKE_UP_STAGE: DonorSpec( - survey=( - "Census CPS ASEC reported receipt + USDA FNS state " - "average-monthly household caseloads" - ), - source="https://www.fns.usda.gov/pd/supplemental-nutrition-assistance-program-snap", - notes=( - "SNAP take-up by anchored count-calibration (contract treatment " - "count_calibrated, microcosm #372): reported ASEC receipt anchors " - "the flag; the fill is calibrated per state to FNS FY2024 " - "average-monthly household counts among eligible non-anchored " - "units, replacing the national snap_take_up fill that bakes in " - "state-dependent CPS underreporting." - ), - ), - US_OTHER_HEALTH_INSURANCE_STAGE_NAME: DonorSpec( - survey="Census CPS ASEC", - source="https://www.census.gov/programs-surveys/cps.html", - notes=( - "Measured PHIP_VAL is reduced by PolicyEngine-calculated CHIP, " - "Marketplace, and Medicaid premiums after take-up stages; an " - "ASEC-trained weighted QRF replaces both premium leaves on the " - "PUF support half." - ), - ), - US_PRIOR_YEAR_INCOME_STAGE_NAME: DonorSpec( - survey="CPS ASEC (prior year)", - source="https://www.census.gov/programs-surveys/cps.html", - notes=( - "Adjacent-year PERIDNUM join for measured prior-year earnings, " - "with Census allocation flags and sentinels enforced. A joint " - "eight-predictor weighted QRF replaces both earnings leaves on " - "the PUF support half; signed self-employment losses survive." - ), - ), - US_IMMIGRATION_STAGE_NAME: DonorSpec( - survey="CPS ASEC + published unauthorized-population estimates", - source=( - "https://www.pewresearch.org/short-reads/2024/07/22/" - "what-we-know-about-unauthorized-immigrants-living-in-the-us/" - ), - notes=( - "SSN card type and immigration status from ASEC citizenship, " - "entry-year, nativity, and program-participation fields via the " - "ASEC-UA residual method (SSRN 4662801), targeted to published " - "undocumented population/worker/student control totals." - ), - ), - US_HOURS_WORKED_STAGE_NAME: DonorSpec( - survey="Census CPS ASEC", - source="https://www.census.gov/programs-surveys/cps.html", - notes=( - "Hours-worked inputs mapped directly from measured ASEC person " - "variables (HRSWK, A_HRS1, WKSWORK) — nothing imputed. Without " - "them the engine defaults every person to 40 weekly hours and " - "hours-conditioned rules (SNAP work requirements) become no-ops." - ), - ), - US_SNAP_TAKE_UP_STAGE_NAME: DonorSpec( - survey="Census CPS ASEC + USDA FNS participation rate estimates", - source="https://www.fns.usda.gov/snap/participation-rates", - notes=( - "SNAP take-up: reported recipients (SPM_SNAPSUB) always take " - "up; non-reporting units drawn to the cited FNS participation " - "rate. Without it the engine defaults every eligible unit to " - "100% take-up." - ), - ), - US_ELIGIBILITY_INPUTS_STAGE_NAME: DonorSpec( - survey="Census CPS ASEC", - source="https://www.census.gov/programs-surveys/cps.html", - notes=( - "SNAP eligibility/exemption inputs mapped directly from " - "measured ASEC person variables (PEDIS*, A_HSCOL/A_FTPT, " - "PEPAR1/PEPAR2, VET_VAL, SSI_VAL) — nothing imputed. Without " - "them disability, student, parent/child, and veteran " - "exemption channels default to False/0." - ), - ), - US_RELATIONSHIP_INPUTS_STAGE_NAME: DonorSpec( - survey="Census CPS ASEC", - source="https://www.census.gov/programs-surveys/cps.html", - notes=( - "Measured household-head and marital-status flags mapped exactly " - "from P_SEQ and A_MARITL; nothing is imputed." - ), - ), - US_MEDICARE_TAKE_UP_STAGE_NAME: DonorSpec( - survey="Census CPS ASEC", - source="https://www.census.gov/programs-surveys/cps.html", - notes=( - "Measured Medicare enrollment mapped exactly from MCARE == 1 and " - "copied onto the PUF support clone; no take-up rate is applied." - ), - ), - US_RETIREMENT_DISTRIBUTION_STAGE_NAME: DonorSpec( - survey="Census CPS ASEC", - source="https://www.census.gov/programs-surveys/cps.html", - notes=( - "ASEC rows map exactly from all four DST_SC*/DST_VAL* pairs; the " - "archived CPS-only QRF replaces the four populated non-IRA leaves " - "on PUF support while preserving IRA channel ownership. Nothing " - "is allocated across accounts." - ), - ), - US_PREGNANCY_STAGE_NAME: DonorSpec( - survey="Census CPS ASEC + CDC natality-derived national pregnancy rate", - source="https://www.cdc.gov/nchs/nvss/births.htm", - notes=( - "Pregnancy seeded among women 15-44 at the national " - "point-in-time rate (births x 39/52 over female 15-44 " - "population), matching the retired pipeline's national " - "fallback; state-level rates are follow-up work (#351). " - "The ASEC does not measure pregnancy." - ), - ), - US_WIC_CLAIM_STAGE_NAME: DonorSpec( - survey="USDA FNS WIC Eligibility and Enrollment Estimates + Census CPS ASEC", - source="https://www.fns.usda.gov/research/wic/eligibility-and-coverage-rates-2022", - notes=( - "Stable person-level claim draws use official CY2022 FNS category " - "coverage rates after the pregnancy and parent-input stages. The " - "all-postpartum rate is used because no hermetic source identifies " - "breastfeeding; nutritional risk remains separately excluded." - ), - ), - US_SNAP_DISCRETIONARY_EXEMPTION_STAGE_NAME: DonorSpec( - survey="Census CPS ASEC + statutory exemption cap (7 U.S.C. 2015(o)(6))", - source="https://www.law.cornell.edu/uscode/text/7/2015#o_6", - notes=( - "ABAWD discretionary exemptions seeded at the statutory cap " - "(8% from FY2024) across potentially covered adults 18-64; " - "the engine intersects with modeled coverage. Assumes full " - "state usage of the cap (#323)." - ), - ), - US_CHILDCARE_STAGE_NAME: DonorSpec( - survey="Census CPS ASEC", - source="https://www.census.gov/programs-surveys/cps.html", - notes=( - "Measured replicated SPM_CHILDCAREXPNS is validated and carried " - "to the SPM-unit childcare leaf. After support expansion, an " - "ASEC-trained weighted QRF replaces only the PUF half and the " - "archived first-person reduction places predictions on SPM units." - ), - ), - US_ADULT_CARE_STAGE_NAME: DonorSpec( - survey="Census CPS ASEC", - source="https://www.census.gov/programs-surveys/cps.html", - notes=( - "Section 21 CDCC adult-care inputs (microcosm#451 item 1): the " - "qualifying flag is the measured PEDISDRS self-care difficulty " - "item; the expense leg is a seeded, weight-targeted draw from " - "the measured ASEC childcare-expense distribution (the same " - "section 21 expense class), restricted to tax units where the " - "statute can bind, with the 21(d)(2) spouse deeming honored. " - "Neither ASEC nor the SIPP 2023 PUF releases an in-household " - "adult-care expenditure amount, so the level proxy is declared " - "in the stage manifest." - ), - ), - US_ENERGY_SUBSIDY_STAGE_NAME: DonorSpec( - survey="Census CPS ASEC", - source="https://www.census.gov/programs-surveys/cps.html", - notes=( - "Measured replicated SPM_ENGVAL is validated and carried to the " - "SPM-unit energy-subsidy leaf. After support expansion, an " - "ASEC-trained weighted QRF replaces only the PUF half and the " - "archived first-person reduction places predictions on SPM units." - ), - ), - US_CHILD_SUPPORT_STAGE_NAME: DonorSpec( - survey="Census CPS ASEC", - source="https://www.census.gov/programs-surveys/cps.html", - notes=( - "Measured annual CSP_VAL receipts and positive CHSP_VAL expenses " - "are carried directly. After PUF tax-detail imputation, one " - "ASEC-trained weighted QRF jointly replaces both person leaves " - "on the PUF support half using the archived predictor subset." - ), - ), - US_DISABILITY_BENEFITS_STAGE_NAME: DonorSpec( - survey="Census CPS ASEC", - source="https://www.census.gov/programs-surveys/cps.html", - notes=( - "Measured annual DIS_VAL1/DIS_VAL2 benefits are retained only " - "when their source code is not workers' compensation. After PUF " - "tax-detail imputation, an ASEC-trained weighted QRF replaces the " - "CPS-only person leaf on the PUF support half." - ), - ), - US_WORKERS_COMPENSATION_STAGE_NAME: DonorSpec( - survey="Census CPS ASEC", - source="https://www.census.gov/programs-surveys/cps.html", - notes=( - "Measured annual WC_VAL is carried directly. After PUF tax-detail " - "imputation, an ASEC-trained weighted QRF replaces the CPS-only " - "person leaf on the PUF support half." - ), - ), - US_WEEKS_UNEMPLOYED_STAGE_NAME: DonorSpec( - survey="Census CPS ASEC", - source="https://www.census.gov/programs-surveys/cps.html", - notes=( - "Measured LKWEEKS is carried directly, including an exact " - "identity-keyed repair of the omitted 2022-income-year column " - "from the pinned official 2023 ASEC archive. An ASEC-trained " - "QRF then replaces only the PUF support half, with the archived " - "unemployment-compensation zero rule." - ), - ), - "puf_tax_detail": DonorSpec( - survey="IRS PUF 2015 (uprated)", - source="https://www.irs.gov/statistics/soi-tax-stats-individual-public-use-microdata-files", - notes=( - "Itemized-deduction detail, versioned processed-PUF Section 199A " - "simulation leaves (carried without redrawing), partnership SE, " - "source-year-AGI E19200 mortgage/non-mortgage split, direct " - "E00800/E03500 alimony, direct E20500 casualty loss, and the E20400 " - "miscellaneous-expense proxy. The pinned processed PUF uprates its " - "raw TY2015 rows before seeded disclosure-record replacement and " - "includes a Forbes-backed 3,900-record open tail; this runtime " - "reconstructs the bounded-record AGI lineage and anchors every " - "Forbes-tail record in the final published AGI band without rerunning " - "Forbes synthesis. Support is clipped to the PUF's realized ranges." - ), - ), - US_EDUCATION_INPUTS_STAGE_NAME: DonorSpec( - survey="IRS PUF 2015 (uprated) + Census CPS ASEC", - source="https://www.irs.gov/statistics/soi-tax-stats-individual-public-use-microdata-files", - notes=( - "Qualified tuition comes from the PUF E03230/E87530 maximum; " - "the published retired path drops the reported AOTC output and its " - "five affirmative factual inputs therefore follow positive tuition; " - "educational assistance carries directly from ASEC ED_VAL." - ), - ), - US_RETIREMENT_CONTRIBUTION_STAGE_NAME: DonorSpec( - survey="Census CPS ASEC + published retirement-contribution shares", - source="https://www.census.gov/programs-surveys/cps.html", - notes=( - "Measured ASEC RETCB_VAL is allocated across five desired " - "retirement-contribution leaves using archived IRS/BEA/Vanguard/" - "PSCA shares, then CPS-trained QRF predictions replace the PUF " - "support half. PolicyEngine-US owns all statutory caps." - ), - ), - "capital_gain_distributions": DonorSpec( - survey="IRS SOI Sales of Capital Assets (TY2015) + Pub 1304 Table 1.4", - source=( - "https://www.irs.gov/statistics/" - "soi-tax-stats-sales-of-capital-assets-reported-on-individual-tax-returns" - ), - notes=( - "Schedule D line 13 capital gain distributions split out of " - "long-term gains as a memo component at the national SOCA-derived " - "share; the direct-1040 route is already a PUF-stage output and " - "the two routes are mutually exclusive on a real return." - ), - ), - US_HOUSING_INPUTS_STAGE_NAME: DonorSpec( - survey="Census ACS 2022", - source="https://www.census.gov/programs-surveys/acs", - notes=( - "Exact ASEC H_TENURE and SPM housing carries plus annual ACS PUMS " - "rent imputation on household heads; reported housing assistance " - "alone receives the archived second-stage PUF-support QRF." - ), - ), - "vehicle_assets": DonorSpec( - survey="Census SIPP", - source="https://www.census.gov/programs-surveys/sipp.html", - notes=( - "Household vehicle count and value from the pinned full 2023 " - "public-use donor. They remain independent policy inputs until " - "the full mixed-source net-worth reconciliation is restored." - ), - ), - US_VOLUNTARY_FILING_STAGE_NAME: DonorSpec( - survey="Census SIPP", - source="https://www.census.gov/programs-surveys/sipp.html", - notes=( - "Measured 2023 SIPP filing and expected-filing responses replace " - "the retired uncited demographic probability table. Reciprocal " - "spouses form one source unit, reported dependents are excluded, " - "and one weighted-QRF prediction is shared across support clones." - ), - ), -} +# Annotations preserve the public type surface without initializing lazy values. +US_DONORS: Mapping[str, DonorSpec] +US_STAGE_NAMES: tuple[str, ...] +US_SOURCE_MANIFEST: SourceManifest +US_SOURCE_STAGE_SPECS: tuple[SourceStageSpec, ...] +US_NONNEGATIVE_SOURCE_OUTPUTS: frozenset[str] +US_SUPPORT_SPINE_MANIFEST: SupportSpineManifest +US_SUPPORT_SPINE_SPEC: SupportSpineSpec + +_EXPORT_LOCK = RLock() + + +def _resolve_export(name: str): + if name in globals(): + return globals()[name] + module, attribute = _LAZY_EXPORTS[name] + # Never hold our lock across Python's module import locks. + value = getattr(import_module(module), attribute) + with _EXPORT_LOCK: + return globals().setdefault(name, value) + + +def _initialize_source_manifest() -> None: + with _EXPORT_LOCK: + if all(name in globals() for name in _SOURCE_EXPORTS): + return + manifest = globals().get("US_SOURCE_MANIFEST") + if manifest is None: + manifest = _load_us_source_manifest() + values = { + "US_SOURCE_MANIFEST": manifest, + "US_SOURCE_STAGE_SPECS": manifest.stages, + "US_NONNEGATIVE_SOURCE_OUTPUTS": frozenset( + output + for stage in manifest.stages + for output in stage.nonnegative_outputs + ), + } + for name, value in values.items(): + globals().setdefault(name, value) + + +def _initialize_support_manifest() -> None: + with _EXPORT_LOCK: + if all(name in globals() for name in _SUPPORT_EXPORTS): + return + manifest = globals().get("US_SUPPORT_SPINE_MANIFEST") + if manifest is None: + manifest = _load_us_support_spine_manifest() + values = { + "US_SUPPORT_SPINE_MANIFEST": manifest, + "US_SUPPORT_SPINE_SPEC": manifest.support_spine, + } + for name, value in values.items(): + globals().setdefault(name, value) -#: Stage order of the US build. Derivation stages (no donor) interleave with -#: the donor imputations; the export/calibration stages close the plan. -US_STAGE_NAMES: tuple[str, ...] = ( - "asec_load", - "unit_assignment", - "derive_cps_carried", - US_PRIOR_YEAR_INCOME_STAGE_NAME, - US_IMMIGRATION_STAGE_NAME, - US_HOURS_WORKED_STAGE_NAME, - US_SNAP_TAKE_UP_STAGE_NAME, - US_RELATIONSHIP_INPUTS_STAGE_NAME, - US_MEDICARE_TAKE_UP_STAGE_NAME, - US_HOUSING_INPUTS_STAGE_NAME, - US_RETIREMENT_DISTRIBUTION_STAGE_NAME, - US_ELIGIBILITY_INPUTS_STAGE_NAME, - US_PREGNANCY_STAGE_NAME, - US_WIC_CLAIM_STAGE_NAME, - US_SNAP_DISCRETIONARY_EXEMPTION_STAGE_NAME, - US_RETIREMENT_CONTRIBUTION_STAGE_NAME, - US_CHILDCARE_STAGE_NAME, - US_ADULT_CARE_STAGE_NAME, - US_ENERGY_SUBSIDY_STAGE_NAME, - US_PUF_SUPPORT_STAGE_NAME, - "puf_tax_detail", - US_CHILD_SUPPORT_STAGE_NAME, - US_DISABILITY_BENEFITS_STAGE_NAME, - US_WORKERS_COMPENSATION_STAGE_NAME, - US_WEEKS_UNEMPLOYED_STAGE_NAME, - US_EDUCATION_INPUTS_STAGE_NAME, - "capital_gain_distributions", - "scf_wealth", - US_SSI_DISABILITY_CRITERIA_STAGE_NAME, - US_SIPP_HEAD_START_STAGE_NAME, - US_SSI_TAKE_UP_STAGE_NAME, - "sipp_tips", - "org_wages", - "meps_esi_premiums", - "mortgage_conversion", - "vehicle_assets", - US_VOLUNTARY_FILING_STAGE_NAME, - "entity_placement", - "aca_marketplace_inputs", - "medicaid_take_up", - US_SNAP_STATE_TAKE_UP_STAGE, - US_OTHER_HEALTH_INSURANCE_STAGE_NAME, - "export", + +_SOURCE_EXPORTS = ( + "US_SOURCE_MANIFEST", + "US_SOURCE_STAGE_SPECS", + "US_NONNEGATIVE_SOURCE_OUTPUTS", ) +_SUPPORT_EXPORTS = ("US_SUPPORT_SPINE_MANIFEST", "US_SUPPORT_SPINE_SPEC") + + +def __getattr__(name: str): + if name in _LAZY_EXPORTS: + return _resolve_export(name) + if name in ("US_DONORS", "US_STAGE_NAMES"): + _initialize_plan() + elif name in _SOURCE_EXPORTS: + _initialize_source_manifest() + elif name in _SUPPORT_EXPORTS: + _initialize_support_manifest() + else: + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") + return globals()[name] + + +def __dir__() -> list[str]: + return sorted(set(globals()) | set(__all__)) + + +def _initialize_plan() -> None: + if "US_DONORS" in globals() and "US_STAGE_NAMES" in globals(): + return + ssi_take_up_ssa_source_url = _resolve_export("SSI_TAKE_UP_SSA_SOURCE_URL") + us_adult_care_stage_name = _resolve_export("US_ADULT_CARE_STAGE_NAME") + us_childcare_stage_name = _resolve_export("US_CHILDCARE_STAGE_NAME") + us_child_support_stage_name = _resolve_export("US_CHILD_SUPPORT_STAGE_NAME") + us_disability_benefits_stage_name = _resolve_export( + "US_DISABILITY_BENEFITS_STAGE_NAME" + ) + us_education_inputs_stage_name = _resolve_export("US_EDUCATION_INPUTS_STAGE_NAME") + us_eligibility_inputs_stage_name = _resolve_export( + "US_ELIGIBILITY_INPUTS_STAGE_NAME" + ) + us_energy_subsidy_stage_name = _resolve_export("US_ENERGY_SUBSIDY_STAGE_NAME") + us_hours_worked_stage_name = _resolve_export("US_HOURS_WORKED_STAGE_NAME") + us_housing_inputs_stage_name = _resolve_export("US_HOUSING_INPUTS_STAGE_NAME") + us_immigration_stage_name = _resolve_export("US_IMMIGRATION_STAGE_NAME") + us_medicare_take_up_stage_name = _resolve_export("US_MEDICARE_TAKE_UP_STAGE_NAME") + us_other_health_insurance_stage_name = _resolve_export( + "US_OTHER_HEALTH_INSURANCE_STAGE_NAME" + ) + us_pregnancy_stage_name = _resolve_export("US_PREGNANCY_STAGE_NAME") + us_prior_year_income_stage_name = _resolve_export("US_PRIOR_YEAR_INCOME_STAGE_NAME") + us_puf_support_stage_name = _resolve_export("US_PUF_SUPPORT_STAGE_NAME") + us_relationship_inputs_stage_name = _resolve_export( + "US_RELATIONSHIP_INPUTS_STAGE_NAME" + ) + us_retirement_contribution_stage_name = _resolve_export( + "US_RETIREMENT_CONTRIBUTION_STAGE_NAME" + ) + us_retirement_distribution_stage_name = _resolve_export( + "US_RETIREMENT_DISTRIBUTION_STAGE_NAME" + ) + us_sipp_head_start_stage_name = _resolve_export("US_SIPP_HEAD_START_STAGE_NAME") + us_snap_discretionary_exemption_stage_name = _resolve_export( + "US_SNAP_DISCRETIONARY_EXEMPTION_STAGE_NAME" + ) + us_snap_state_take_up_stage = _resolve_export("US_SNAP_STATE_TAKE_UP_STAGE") + us_snap_take_up_stage_name = _resolve_export("US_SNAP_TAKE_UP_STAGE_NAME") + us_ssi_disability_criteria_stage_name = _resolve_export( + "US_SSI_DISABILITY_CRITERIA_STAGE_NAME" + ) + us_ssi_take_up_stage_name = _resolve_export("US_SSI_TAKE_UP_STAGE_NAME") + us_voluntary_filing_stage_name = _resolve_export("US_VOLUNTARY_FILING_STAGE_NAME") + us_weeks_unemployed_stage_name = _resolve_export("US_WEEKS_UNEMPLOYED_STAGE_NAME") + us_wic_claim_stage_name = _resolve_export("US_WIC_CLAIM_STAGE_NAME") + us_workers_compensation_stage_name = _resolve_export( + "US_WORKERS_COMPENSATION_STAGE_NAME" + ) + with _EXPORT_LOCK: + us_donors: Mapping[str, DonorSpec] = { + "scf_wealth": DonorSpec( + survey="Fed SCF 2022 + Census SIPP 2023", + source="https://www.federalreserve.gov/econres/scfindex.htm", + notes=( + "SCF anchors signed net worth and one half of household liquid-asset " + "vectors; the immutable SIPP 2023 public-use donor supplies the " + "other half and restores low liquid-asset mass. Auto loans use the " + "full SCF separately." + ), + ), + us_ssi_disability_criteria_stage_name: DonorSpec( + survey="Census SIPP", + source="https://www.census.gov/programs-surveys/sipp.html", + notes=( + "Latent under-65 SSI disability/blindness criterion from the " + "pinned full 2023 public-use donor. ASEC and PUF-support people " + "are predicted separately, and only direct under-65 ASEC SSI " + "reporters receive the observed-reporter anchor." + ), + ), + us_sipp_head_start_stage_name: DonorSpec( + survey="Census SIPP", + source="https://www.census.gov/programs-surveys/sipp.html", + notes=( + "Direct December nursery/preschool federally sponsored-program " + "responses train a weighted Head Start take-up proxy for ages " + "3--5; strict reported structural negatives exclude hot-decked " + "answers and the prediction is shared by support clones." + ), + ), + us_ssi_take_up_stage_name: DonorSpec( + survey="CPS ASEC reported SSI + SSA SSI Monthly Statistics December 2024", + source=ssi_take_up_ssa_source_url, + notes=( + "Direct ASEC SSI_VAL reporters anchor person-level take-up; " + "eligible source-person identities are count-calibrated by age to " + "SSA December 2024 Federal-payment recipient counts and fanned to " + "both support clones." + ), + ), + "sipp_tips": DonorSpec( + survey="Census SIPP", + source="https://www.census.gov/programs-surveys/sipp.html", + notes="Tip income for tipped occupations.", + ), + "org_wages": DonorSpec( + survey="CPS ORG", + source=( + "https://www2.census.gov/programs-surveys/cps/datasets/2024/basic/" + ), + notes=( + "Hourly-wage labor-market inputs. Donor load failures abort the " + "build — the silent zero-fallback this stage once had is " + "structurally impossible under StagePlan." + ), + ), + "meps_esi_premiums": DonorSpec( + survey="MEPS-IC", + source="https://meps.ahrq.gov/mepsweb/survey_comp/Insurance.jsp", + notes="Employer-sponsored insurance premium parameters.", + ), + "aca_marketplace_inputs": DonorSpec( + survey="CPS ASEC + CMS Marketplace Open Enrollment PUFs", + source="https://www.cms.gov/marketplace/resources/data/public-use-files", + notes=( + "Marketplace take-up and selected-plan inputs: CPS reported " + "Marketplace coverage and premium reports anchor the records; " + "CMS OEP enrollment, APTC, and metal-level tables provide the " + "calibration targets." + ), + ), + "medicaid_take_up": DonorSpec( + survey="CPS ASEC reported coverage + CMS Medicaid monthly enrollment snapshot", + source="https://data.medicaid.gov/dataset/6165f45b-ca93-5bb5-9d06-db29c692a360", + notes=( + "Medicaid take-up by anchored count-calibration (contract " + "treatment count_calibrated, microcosm #331): CPS-reported " + "Medicaid coverage at interview anchors the flag; the fill is " + "calibrated to CMS December 2024 state enrollment snapshots. " + "Point-in-time semantics per #332; heals the #170 " + "enrollment==eligibility degeneracy." + ), + ), + us_snap_state_take_up_stage: DonorSpec( + survey=( + "Census CPS ASEC reported receipt + USDA FNS state " + "average-monthly household caseloads" + ), + source="https://www.fns.usda.gov/pd/supplemental-nutrition-assistance-program-snap", + notes=( + "SNAP take-up by anchored count-calibration (contract treatment " + "count_calibrated, microcosm #372): reported ASEC receipt anchors " + "the flag; the fill is calibrated per state to FNS FY2024 " + "average-monthly household counts among eligible non-anchored " + "units, replacing the national snap_take_up fill that bakes in " + "state-dependent CPS underreporting." + ), + ), + us_other_health_insurance_stage_name: DonorSpec( + survey="Census CPS ASEC", + source="https://www.census.gov/programs-surveys/cps.html", + notes=( + "Measured PHIP_VAL is reduced by PolicyEngine-calculated CHIP, " + "Marketplace, and Medicaid premiums after take-up stages; an " + "ASEC-trained weighted QRF replaces both premium leaves on the " + "PUF support half." + ), + ), + us_prior_year_income_stage_name: DonorSpec( + survey="CPS ASEC (prior year)", + source="https://www.census.gov/programs-surveys/cps.html", + notes=( + "Adjacent-year PERIDNUM join for measured prior-year earnings, " + "with Census allocation flags and sentinels enforced. A joint " + "eight-predictor weighted QRF replaces both earnings leaves on " + "the PUF support half; signed self-employment losses survive." + ), + ), + us_immigration_stage_name: DonorSpec( + survey="CPS ASEC + published unauthorized-population estimates", + source=( + "https://www.pewresearch.org/short-reads/2024/07/22/" + "what-we-know-about-unauthorized-immigrants-living-in-the-us/" + ), + notes=( + "SSN card type and immigration status from ASEC citizenship, " + "entry-year, nativity, and program-participation fields via the " + "ASEC-UA residual method (SSRN 4662801), targeted to published " + "undocumented population/worker/student control totals." + ), + ), + us_hours_worked_stage_name: DonorSpec( + survey="Census CPS ASEC", + source="https://www.census.gov/programs-surveys/cps.html", + notes=( + "Hours-worked inputs mapped directly from measured ASEC person " + "variables (HRSWK, A_HRS1, WKSWORK) — nothing imputed. Without " + "them the engine defaults every person to 40 weekly hours and " + "hours-conditioned rules (SNAP work requirements) become no-ops." + ), + ), + us_snap_take_up_stage_name: DonorSpec( + survey="Census CPS ASEC + USDA FNS participation rate estimates", + source="https://www.fns.usda.gov/snap/participation-rates", + notes=( + "SNAP take-up: reported recipients (SPM_SNAPSUB) always take " + "up; non-reporting units drawn to the cited FNS participation " + "rate. Without it the engine defaults every eligible unit to " + "100% take-up." + ), + ), + us_eligibility_inputs_stage_name: DonorSpec( + survey="Census CPS ASEC", + source="https://www.census.gov/programs-surveys/cps.html", + notes=( + "SNAP eligibility/exemption inputs mapped directly from " + "measured ASEC person variables (PEDIS*, A_HSCOL/A_FTPT, " + "PEPAR1/PEPAR2, VET_VAL, SSI_VAL) — nothing imputed. Without " + "them disability, student, parent/child, and veteran " + "exemption channels default to False/0." + ), + ), + us_relationship_inputs_stage_name: DonorSpec( + survey="Census CPS ASEC", + source="https://www.census.gov/programs-surveys/cps.html", + notes=( + "Measured household-head and marital-status flags mapped exactly " + "from P_SEQ and A_MARITL; nothing is imputed." + ), + ), + us_medicare_take_up_stage_name: DonorSpec( + survey="Census CPS ASEC", + source="https://www.census.gov/programs-surveys/cps.html", + notes=( + "Measured Medicare enrollment mapped exactly from MCARE == 1 and " + "copied onto the PUF support clone; no take-up rate is applied." + ), + ), + us_retirement_distribution_stage_name: DonorSpec( + survey="Census CPS ASEC", + source="https://www.census.gov/programs-surveys/cps.html", + notes=( + "ASEC rows map exactly from all four DST_SC*/DST_VAL* pairs; the " + "archived CPS-only QRF replaces the four populated non-IRA leaves " + "on PUF support while preserving IRA channel ownership. Nothing " + "is allocated across accounts." + ), + ), + us_pregnancy_stage_name: DonorSpec( + survey="Census CPS ASEC + CDC natality-derived national pregnancy rate", + source="https://www.cdc.gov/nchs/nvss/births.htm", + notes=( + "Pregnancy seeded among women 15-44 at the national " + "point-in-time rate (births x 39/52 over female 15-44 " + "population), matching the retired pipeline's national " + "fallback; state-level rates are follow-up work (#351). " + "The ASEC does not measure pregnancy." + ), + ), + us_wic_claim_stage_name: DonorSpec( + survey="USDA FNS WIC Eligibility and Enrollment Estimates + Census CPS ASEC", + source="https://www.fns.usda.gov/research/wic/eligibility-and-coverage-rates-2022", + notes=( + "Stable person-level claim draws use official CY2022 FNS category " + "coverage rates after the pregnancy and parent-input stages. The " + "all-postpartum rate is used because no hermetic source identifies " + "breastfeeding; nutritional risk remains separately excluded." + ), + ), + us_snap_discretionary_exemption_stage_name: DonorSpec( + survey="Census CPS ASEC + statutory exemption cap (7 U.S.C. 2015(o)(6))", + source="https://www.law.cornell.edu/uscode/text/7/2015#o_6", + notes=( + "ABAWD discretionary exemptions seeded at the statutory cap " + "(8% from FY2024) across potentially covered adults 18-64; " + "the engine intersects with modeled coverage. Assumes full " + "state usage of the cap (#323)." + ), + ), + us_childcare_stage_name: DonorSpec( + survey="Census CPS ASEC", + source="https://www.census.gov/programs-surveys/cps.html", + notes=( + "Measured replicated SPM_CHILDCAREXPNS is validated and carried " + "to the SPM-unit childcare leaf. After support expansion, an " + "ASEC-trained weighted QRF replaces only the PUF half and the " + "archived first-person reduction places predictions on SPM units." + ), + ), + us_adult_care_stage_name: DonorSpec( + survey="Census CPS ASEC", + source="https://www.census.gov/programs-surveys/cps.html", + notes=( + "Section 21 CDCC adult-care inputs (microcosm#451 item 1): the " + "qualifying flag is the measured PEDISDRS self-care difficulty " + "item; the expense leg is a seeded, weight-targeted draw from " + "the measured ASEC childcare-expense distribution (the same " + "section 21 expense class), restricted to tax units where the " + "statute can bind, with the 21(d)(2) spouse deeming honored. " + "Neither ASEC nor the SIPP 2023 PUF releases an in-household " + "adult-care expenditure amount, so the level proxy is declared " + "in the stage manifest." + ), + ), + us_energy_subsidy_stage_name: DonorSpec( + survey="Census CPS ASEC", + source="https://www.census.gov/programs-surveys/cps.html", + notes=( + "Measured replicated SPM_ENGVAL is validated and carried to the " + "SPM-unit energy-subsidy leaf. After support expansion, an " + "ASEC-trained weighted QRF replaces only the PUF half and the " + "archived first-person reduction places predictions on SPM units." + ), + ), + us_child_support_stage_name: DonorSpec( + survey="Census CPS ASEC", + source="https://www.census.gov/programs-surveys/cps.html", + notes=( + "Measured annual CSP_VAL receipts and positive CHSP_VAL expenses " + "are carried directly. After PUF tax-detail imputation, one " + "ASEC-trained weighted QRF jointly replaces both person leaves " + "on the PUF support half using the archived predictor subset." + ), + ), + us_disability_benefits_stage_name: DonorSpec( + survey="Census CPS ASEC", + source="https://www.census.gov/programs-surveys/cps.html", + notes=( + "Measured annual DIS_VAL1/DIS_VAL2 benefits are retained only " + "when their source code is not workers' compensation. After PUF " + "tax-detail imputation, an ASEC-trained weighted QRF replaces the " + "CPS-only person leaf on the PUF support half." + ), + ), + us_workers_compensation_stage_name: DonorSpec( + survey="Census CPS ASEC", + source="https://www.census.gov/programs-surveys/cps.html", + notes=( + "Measured annual WC_VAL is carried directly. After PUF tax-detail " + "imputation, an ASEC-trained weighted QRF replaces the CPS-only " + "person leaf on the PUF support half." + ), + ), + us_weeks_unemployed_stage_name: DonorSpec( + survey="Census CPS ASEC", + source="https://www.census.gov/programs-surveys/cps.html", + notes=( + "Measured LKWEEKS is carried directly, including an exact " + "identity-keyed repair of the omitted 2022-income-year column " + "from the pinned official 2023 ASEC archive. An ASEC-trained " + "QRF then replaces only the PUF support half, with the archived " + "unemployment-compensation zero rule." + ), + ), + "puf_tax_detail": DonorSpec( + survey="IRS PUF 2015 (uprated)", + source="https://www.irs.gov/statistics/soi-tax-stats-individual-public-use-microdata-files", + notes=( + "Itemized-deduction detail, versioned processed-PUF Section 199A " + "simulation leaves (carried without redrawing), partnership SE, " + "source-year-AGI E19200 mortgage/non-mortgage split, direct " + "E00800/E03500 alimony, direct E20500 casualty loss, and the E20400 " + "miscellaneous-expense proxy. The pinned processed PUF uprates its " + "raw TY2015 rows before seeded disclosure-record replacement and " + "includes a Forbes-backed 3,900-record open tail; this runtime " + "reconstructs the bounded-record AGI lineage and anchors every " + "Forbes-tail record in the final published AGI band without rerunning " + "Forbes synthesis. Support is clipped to the PUF's realized ranges." + ), + ), + us_education_inputs_stage_name: DonorSpec( + survey="IRS PUF 2015 (uprated) + Census CPS ASEC", + source="https://www.irs.gov/statistics/soi-tax-stats-individual-public-use-microdata-files", + notes=( + "Qualified tuition comes from the PUF E03230/E87530 maximum; " + "the published retired path drops the reported AOTC output and its " + "five affirmative factual inputs therefore follow positive tuition; " + "educational assistance carries directly from ASEC ED_VAL." + ), + ), + us_retirement_contribution_stage_name: DonorSpec( + survey="Census CPS ASEC + published retirement-contribution shares", + source="https://www.census.gov/programs-surveys/cps.html", + notes=( + "Measured ASEC RETCB_VAL is allocated across five desired " + "retirement-contribution leaves using archived IRS/BEA/Vanguard/" + "PSCA shares, then CPS-trained QRF predictions replace the PUF " + "support half. PolicyEngine-US owns all statutory caps." + ), + ), + "capital_gain_distributions": DonorSpec( + survey="IRS SOI Sales of Capital Assets (TY2015) + Pub 1304 Table 1.4", + source=( + "https://www.irs.gov/statistics/" + "soi-tax-stats-sales-of-capital-assets-reported-on-individual-tax-returns" + ), + notes=( + "Schedule D line 13 capital gain distributions split out of " + "long-term gains as a memo component at the national SOCA-derived " + "share; the direct-1040 route is already a PUF-stage output and " + "the two routes are mutually exclusive on a real return." + ), + ), + us_housing_inputs_stage_name: DonorSpec( + survey="Census ACS 2022", + source="https://www.census.gov/programs-surveys/acs", + notes=( + "Exact ASEC H_TENURE and SPM housing carries plus annual ACS PUMS " + "rent imputation on household heads; reported housing assistance " + "alone receives the archived second-stage PUF-support QRF." + ), + ), + "vehicle_assets": DonorSpec( + survey="Census SIPP", + source="https://www.census.gov/programs-surveys/sipp.html", + notes=( + "Household vehicle count and value from the pinned full 2023 " + "public-use donor. They remain independent policy inputs until " + "the full mixed-source net-worth reconciliation is restored." + ), + ), + us_voluntary_filing_stage_name: DonorSpec( + survey="Census SIPP", + source="https://www.census.gov/programs-surveys/sipp.html", + notes=( + "Measured 2023 SIPP filing and expected-filing responses replace " + "the retired uncited demographic probability table. Reciprocal " + "spouses form one source unit, reported dependents are excluded, " + "and one weighted-QRF prediction is shared across support clones." + ), + ), + } + us_stage_names: tuple[str, ...] = ( + "asec_load", + "unit_assignment", + "derive_cps_carried", + us_prior_year_income_stage_name, + us_immigration_stage_name, + us_hours_worked_stage_name, + us_snap_take_up_stage_name, + us_relationship_inputs_stage_name, + us_medicare_take_up_stage_name, + us_housing_inputs_stage_name, + us_retirement_distribution_stage_name, + us_eligibility_inputs_stage_name, + us_pregnancy_stage_name, + us_wic_claim_stage_name, + us_snap_discretionary_exemption_stage_name, + us_retirement_contribution_stage_name, + us_childcare_stage_name, + us_adult_care_stage_name, + us_energy_subsidy_stage_name, + us_puf_support_stage_name, + "puf_tax_detail", + us_child_support_stage_name, + us_disability_benefits_stage_name, + us_workers_compensation_stage_name, + us_weeks_unemployed_stage_name, + us_education_inputs_stage_name, + "capital_gain_distributions", + "scf_wealth", + us_ssi_disability_criteria_stage_name, + us_sipp_head_start_stage_name, + us_ssi_take_up_stage_name, + "sipp_tips", + "org_wages", + "meps_esi_premiums", + "mortgage_conversion", + "vehicle_assets", + us_voluntary_filing_stage_name, + "entity_placement", + "aca_marketplace_inputs", + "medicaid_take_up", + us_snap_state_take_up_stage, + us_other_health_insurance_stage_name, + "export", + ) + globals().setdefault("US_DONORS", us_donors) + globals().setdefault("US_STAGE_NAMES", us_stage_names) def _load_us_source_manifest() -> SourceManifest: @@ -2481,15 +2625,6 @@ def _load_us_support_spine_manifest() -> SupportSpineManifest: ) -US_SOURCE_MANIFEST = _load_us_source_manifest() -US_SUPPORT_SPINE_MANIFEST = _load_us_support_spine_manifest() -US_SUPPORT_SPINE_SPEC: SupportSpineSpec = US_SUPPORT_SPINE_MANIFEST.support_spine -US_SOURCE_STAGE_SPECS: tuple[SourceStageSpec, ...] = US_SOURCE_MANIFEST.stages -US_NONNEGATIVE_SOURCE_OUTPUTS: frozenset[str] = frozenset( - output for stage in US_SOURCE_STAGE_SPECS for output in stage.nonnegative_outputs -) - - def us_plan( implementations: Mapping[str, Callable[[Frame], Frame]], ) -> StagePlan: @@ -2510,23 +2645,26 @@ def us_plan( ValueError: If any declared stage lacks an implementation, or an implementation is supplied for an undeclared stage. """ - missing = [name for name in US_STAGE_NAMES if name not in implementations] + _initialize_plan() + stage_names = globals()["US_STAGE_NAMES"] + donors = globals()["US_DONORS"] + missing = [name for name in stage_names if name not in implementations] if missing: raise ValueError( f"us_plan needs an implementation for every declared stage; " f"missing {missing}. There are no stubs or fallbacks by design." ) - unknown = sorted(set(implementations) - set(US_STAGE_NAMES)) + unknown = sorted(set(implementations) - set(stage_names)) if unknown: raise ValueError( f"Unknown stage implementation(s) {unknown}; declared stages " - f"are {list(US_STAGE_NAMES)}." + f"are {list(stage_names)}." ) return StagePlan( Stage( name=name, transform=implementations[name], - donor=US_DONORS.get(name), + donor=donors.get(name), ) - for name in US_STAGE_NAMES + for name in stage_names ) diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/_asec_current_money_codec.py b/packages/microcosm-build/src/microcosm/build/us_runtime/_asec_current_money_codec.py new file mode 100644 index 000000000..dca569472 --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/_asec_current_money_codec.py @@ -0,0 +1,143 @@ +"""Bounded, checksummed codec preserving target-only money source authority.""" + +import struct +from hashlib import sha256 + +from .asec_current_money import ( + _STATE_TOKEN, + HEADER_MAX_BYTES, + MAX_HOUSEHOLDS, + MAX_PERSONS, + AuthenticatedAsecSource, + MoneyBindings, + MoneyField, + ReadyCurrentMoney, + RestatedAsecMoney, + SyntheticReadyCurrentMoney, + _parse, + _require, + _validate_money, + require_complete_current_money, +) + +MAGIC = b"MCAM2024\x01" +_CONTENT_DOMAIN = b"microcosm/asec-current-money-content/1\0" +# 32 person fields, one household observation, 8+1+1+1 bytes per field/row. +PAYLOAD_MAX_BYTES = ( + len(MAGIC) + 4 + HEADER_MAX_BYTES + 11 * (32 * MAX_PERSONS + MAX_HOUSEHOLDS) + 32 +) + + +def _ready_parts( + ready: SyntheticReadyCurrentMoney | ReadyCurrentMoney, +) -> tuple[bytes, ...]: + """One canonical serialization shared by transport and content identity.""" + _require( + type(ready) in (SyntheticReadyCurrentMoney, ReadyCurrentMoney), + "SYNTHETIC_READY_REQUIRED", + ) + _validate_money(ready, ready.bindings.spec, nominal=False) + # Recheck completeness even for a directly constructed or replaced value. + require_complete_current_money( + RestatedAsecMoney(ready.bindings, ready.fields, _token=_STATE_TOKEN), + ready.bindings.spec, + production=type(ready) is ReadyCurrentMoney, + ) + parts = [MAGIC, struct.pack(" str: + """Bind the full canonical header and field body, apart from transport checksum.""" + digest = sha256(_CONTENT_DOMAIN) + for part in _ready_parts(ready): + digest.update(part) + return digest.hexdigest() + + +def encode_current_money( + ready: SyntheticReadyCurrentMoney | ReadyCurrentMoney, +) -> bytes: + """Encode immutable target amounts and evidence, including field-scoped origins. + + Schema 1 retains legacy zero provenance. Schema 2 binds the restored-source + PTOTVAL origin policy through the exact header/spec; the byte widths and + checksum framing are unchanged. Per-field validation rejects swapped codes. + """ + parts = _ready_parts(ready) + checksum = sha256() + for part in parts: + checksum.update(part) + return b"".join((*parts, checksum.digest())) + + +def decode_current_money( + payload: bytes, + expected_bindings: MoneyBindings, + *, + expected: ReadyCurrentMoney | None = None, +) -> SyntheticReadyCurrentMoney | ReadyCurrentMoney: + """Authenticate replay content against issued readiness before state creation. + + Verified source bindings alone cannot authorize a restated amount body. + Synthetic replay retains its binding-only API and refuses an expected object. + """ + _require( + type(payload) is bytes + and len(MAGIC) + 4 + 32 < len(payload) <= PAYLOAD_MAX_BYTES, + "PAYLOAD_SIZE", + ) + _require(type(expected_bindings) is MoneyBindings, "EXPECTED_BINDINGS_REQUIRED") + expected_bindings.__post_init__() + authenticated = type(expected_bindings.spec.source) is AuthenticatedAsecSource + if authenticated: + _require(type(expected) is ReadyCurrentMoney, "EXPECTED_CONTENT_REQUIRED") + _require(expected.bindings == expected_bindings, "EXPECTED_CONTENT_BINDING") + _validate_money(expected, expected_bindings.spec, nominal=False) + else: + _require(expected is None, "EXPECTED_CONTENT_SYNTHETIC") + view = memoryview(payload) + _require(payload.startswith(MAGIC), "PAYLOAD_VERSION") + size = struct.unpack_from(" None: + """Require genuine-summary fields, numeric domains and registered bands. + + This validates evidence shape and the fixed decision policy, not the source + identity or truth of the supplied measurements. Genuine summary functions + emit Python integers/floats and list-valued bands; immutable tuple bands are + also accepted for callers that freeze the same data in memory. + """ + prefix = f"Malformed {family}-input summary" + fields = { + "unique_counts", + *share_bands, + *(name for name, _ in share_bands.values()), + *invariants, + } + if not isinstance(summary, Mapping) or set(summary) != fields: + raise ValueError(f"{prefix}: expected the exact registered field inventory.") + counts = summary["unique_counts"] + if not isinstance(counts, Mapping) or set(counts) != set(outputs): + raise ValueError(f"{prefix}: expected a count for every registered output.") + for name, value in (*counts.items(), *((key, summary[key]) for key in invariants)): + if type(value) is not int or value < 0: + raise ValueError(f"{prefix}: {name!r} must be a nonnegative integer.") + for share_name, (band_name, registered) in share_bands.items(): + share = summary[share_name] + # The closed interval also refuses both infinities and NaN without + # coercing strings, bools, or oversized integer evidence to float64. + if type(share) not in (int, float) or not 0.0 <= share <= 1.0: + raise ValueError(f"{prefix}: {share_name!r} must be finite in [0, 1].") + band = summary[band_name] + if ( + not isinstance(band, (list, tuple)) + or len(band) != 2 + or any(type(value) not in (int, float) for value in band) + or tuple(band) != registered + ): + raise ValueError( + f"{prefix}: {band_name!r} differs from its registered band." + ) diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/acs_2024_housing_universe.json b/packages/microcosm-build/src/microcosm/build/us_runtime/acs_2024_housing_universe.json new file mode 100644 index 000000000..272b560dd --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/acs_2024_housing_universe.json @@ -0,0 +1,23 @@ +{ + "schema_version": 1, + "source_kind": "microcosm.acs_housing_universe_source.v1", + "vintage": 2024, + "dictionary": { + "url": "https://www2.census.gov/programs-surveys/acs/tech_docs/pums/data_dict/PUMS_Data_Dictionary_2024.pdf", + "sha256": "929c2752995b0af1c16d5c64de8cdc43b4aa7d388ee2d45b4b4df90fecce1dff", + "pages": [1, 2, 3, 8, 32, 33] + }, + "vacancy_definition": "acs_2024_source_vacancy_v1", + "vacancy": "TYPEHUGQ=1, NP=0, no linked persons, WGTP=1..9999, blank TEN; separate source vacancy evidence, not a change to the pure classifier", + "gq_definition": "acs_2024_gq_person_placeholder_design_v1", + "gq": "TYPEHUGQ=2/3, NP=1, WGTP=0, blank TEN, one linked person with PWGTP=1..9999; one source person placeholder at DESIGN weight, not a physical facility or calibrated descendant", + "occupied_unknown_tenure": "Retained in occupied HU total with explicitly unclassified tenure; never infer from model tenure_type", + "origin": "CD source household origin: ACS arm, original household member name and SHA256, vintage integer2024, raw SERIALNO string; person additionally retains original person member and normalized SPORDER", + "parser": "UTF8/noBOM, strict comma QUOTE_MINIMAL, lexical projected fields, no trimming/NA inference; same ordered full role headers; every physical record full width", + "encoding": "acs-housing-lexical-projection/1", + "classifier_columns": ["interview_scope", "physical_unit", "household_kind", "tenure_subtype", "occupied_hu", "hu_tenure_class", "unresolved_reasons", "TEN_valid"], + "us_archive_scope_assumption": "50 US states and DC; observed Puerto Rico refuses rather than silently changing scope", + "release_eligible": false, + "calibrated_descendants_approved": false, + "B19001_B25003_activation": false +} diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/acs_housing_universe.py b/packages/microcosm-build/src/microcosm/build/us_runtime/acs_housing_universe.py new file mode 100644 index 000000000..fc4928883 --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/acs_housing_universe.py @@ -0,0 +1,203 @@ +"""Pure, unbound ACS housing-unit semantics; no source authentication. + +ACS TYPEHUGQ/NP observations do not supply the CPS interview-scope or +household-kind observations. Those output axes therefore remain source-specific +not-observed codes, even for a known occupied ACS housing unit. Invalid TEN +payloads are uninterpreted placeholders, never observed zeros. +""" + +from __future__ import annotations + +import numpy as np +import pandas as pd + +__all__ = ["ACSHousingUniverseRefusalError", "classify_acs_housing_universe"] + + +class ACSHousingUniverseRefusalError(ValueError): + """Refuse an invalid input contract without reporting source values or keys.""" + + +def _owned_index(index: pd.Index) -> pd.Index: + """Copy axis storage, reconstructing indexes with shared deep-copy caches.""" + if isinstance(index, pd.RangeIndex): + return pd.RangeIndex(index.start, index.stop, index.step, name=index.name) + if isinstance(index, pd.MultiIndex): + return pd.MultiIndex( + levels=[_owned_index(level) for level in index.levels], + codes=[code.copy() for code in index.codes], + names=list(index.names), + sortorder=index.sortorder, + verify_integrity=False, + ) + if isinstance(index, pd.CategoricalIndex): + return pd.CategoricalIndex( + pd.Categorical.from_codes( + index.codes.copy(), + categories=_owned_index(index.categories), + ordered=index.ordered, + ), + name=index.name, + ) + return index.copy(deep=True) + + +def _header() -> dict: + """Fresh semantic metadata only: no source rows, keys, counts, or pins.""" + return { + "schema_version": 1, + "artifact_kind": "microcosm.acs_housing_universe_unbound.v1", + "source_authentication": "unbound_semantic_classifier", + "release_eligible": False, + "source_mapping": { + "TYPEHUGQ": { + "1": "housing_unit", + "2": "institutional_group_quarters", + "3": "noninstitutional_group_quarters", + }, + "NP": "nonnegative int64; no additional upper bound", + "TEN": "native ACS codes; see tenure_subtype codebook", + "occupied_hu": ( + "TYPEHUGQ=1 and NP>0 establishes occupancy; TYPEHUGQ=2/3 " + "is outside the HU universe; TYPEHUGQ=1 and NP=0 leaves " + "occupancy unresolved and does not assert vacancy" + ), + "hu_tenure_class": ( + "only known occupied HUs: valid TEN=1/2 owner, valid TEN=3/4 " + "renter, invalid TEN unclassified but retained in the HU total" + ), + "interview_scope": "CPS interview observation not supplied by ACS inputs", + "household_kind": "CPS household-kind observation not supplied by ACS inputs", + "ignored_columns": "all other columns, including model tenure_type", + "row_policy": "retain every row in original order, including all GQ rows", + }, + "codebooks": { + "interview_scope": {"0": "not_observed_in_these_ACS_inputs"}, + "physical_unit": {"1": "housing_unit", "2": "group_quarters"}, + "household_kind": {"0": "not_observed_in_these_ACS_inputs"}, + "tenure_subtype": { + "0": "unavailable", + "1": "owned_with_mortgage_or_loan", + "2": "owned_outright", + "3": "paying_rent", + "4": "no_rent_paid", + }, + "occupied_hu": { + "0": "occupancy_not_established", + "1": "occupied_housing_unit", + "2": "outside_housing_unit_universe", + }, + "hu_tenure_class": { + "0": "outside_or_unresolved_occupied_HU_universe", + "1": "owner", + "2": "renter_including_no_rent", + "3": "occupied_HU_tenure_unclassified", + }, + "unresolved_reasons": { + "0": "no_unresolved_reason", + "1": "housing_unit_occupancy_not_established", + "2": "occupied_housing_unit_tenure_unavailable", + }, + "TEN_valid": {"0": "unavailable", "1": "observed_native_TEN"}, + }, + "validity_placeholder_semantics": { + "input": "exact numpy int64 TYPEHUGQ/NP/TEN and positional bool ndarray mask", + "invalid_TEN": ( + "any int64 payload is an uninterpreted placeholder; never an " + "observed zero and never checked against the observed TEN domain" + ), + "output": "uint8 codes; tenure_subtype=0 iff TEN_valid=0 on every row", + "GQ": "unavailable TEN, including a blank, adds no conflict or reason", + "unresolved_reasons": "bit mask: 1 occupancy not established; 2 occupied HU TEN unavailable", + }, + "limits": [ + "no_authenticated_source_or_source_pins", + "no_CPS_interview_or_household_kind_observations", + "no_B19001_income_concept_bridge", + "no_scientific_certification", + "no_stacked_population_or_denominator_certification", + ], + } + + +def classify_acs_housing_universe( + households: pd.DataFrame, *, tenure_valid: np.ndarray +) -> tuple[pd.DataFrame, dict]: + """Classify supplied ACS observations without authenticating their source. + + Require an exact DataFrame with unique columns and native numpy int64 + TYPEHUGQ, NP, and TEN. TYPEHUGQ must be 1/2/3 and NP nonnegative. The + positional validity argument must be an exact one-dimensional bool ndarray + of matching length. Only valid TEN cells are observed and require codes + 1/2/3/4; invalid payloads can be any int64. No coercion or nullable path is + provided. Other columns, including tenure_type, are ignored. + + Return an owned uint8 DataFrame preserving every row and its index, plus a + fresh, data-independent semantic header. Both CPS-specific axes stay zero + (not observed). NP=0 does not establish vacancy. Unknown occupied-HU tenure + stays explicitly unclassified within the total; GQ tenure never changes + the HU universe. The classifier performs no I/O or source authentication. + """ + if type(households) is not pd.DataFrame: + raise ACSHousingUniverseRefusalError("households_must_be_exact_dataframe") + if not households.columns.is_unique: + raise ACSHousingUniverseRefusalError("household_columns_must_be_unique") + required = ("TYPEHUGQ", "NP", "TEN") + if not set(required).issubset(set(households.columns)): + raise ACSHousingUniverseRefusalError("required_ACS_columns_missing") + for column in required: + dtype = households[column].dtype + if not isinstance(dtype, np.dtype) or dtype != np.dtype(np.int64): + raise ACSHousingUniverseRefusalError("ACS_columns_must_be_numpy_int64") + if ( + type(tenure_valid) is not np.ndarray + or tenure_valid.dtype != np.dtype(np.bool_) + or tenure_valid.ndim != 1 + or len(tenure_valid) != len(households) + ): + raise ACSHousingUniverseRefusalError( + "TEN_validity_must_be_matching_bool_vector" + ) + + unit_type = households["TYPEHUGQ"].to_numpy(copy=False) + persons = households["NP"].to_numpy(copy=False) + tenure = households["TEN"].to_numpy(copy=False) + if np.any((unit_type < 1) | (unit_type > 3)): + raise ACSHousingUniverseRefusalError("TYPEHUGQ_outside_domain") + if np.any(persons < 0): + raise ACSHousingUniverseRefusalError("NP_must_be_nonnegative") + observed_tenure = tenure[tenure_valid] + if np.any((observed_tenure < 1) | (observed_tenure > 4)): + raise ACSHousingUniverseRefusalError("valid_TEN_outside_domain") + + housing_unit = unit_type == 1 + occupied = housing_unit & (persons > 0) + occupancy_unknown = housing_unit & (persons == 0) + tenure_subtype = np.zeros(len(households), dtype=np.uint8) + tenure_subtype[tenure_valid] = observed_tenure + occupied_hu = np.zeros(len(households), dtype=np.uint8) + occupied_hu[occupied] = 1 + occupied_hu[~housing_unit] = 2 + hu_tenure_class = np.zeros(len(households), dtype=np.uint8) + hu_tenure_class[occupied & tenure_valid & (tenure_subtype <= 2)] = 1 + hu_tenure_class[occupied & tenure_valid & (tenure_subtype >= 3)] = 2 + hu_tenure_class[occupied & ~tenure_valid] = 3 + reasons = np.zeros(len(households), dtype=np.uint8) + reasons[occupancy_unknown] |= 1 + reasons[occupied & ~tenure_valid] |= 2 + + output = pd.DataFrame( + { + "interview_scope": np.zeros(len(households), dtype=np.uint8), + "physical_unit": np.where(housing_unit, 1, 2).astype(np.uint8), + "household_kind": np.zeros(len(households), dtype=np.uint8), + "tenure_subtype": tenure_subtype, + "occupied_hu": occupied_hu, + "hu_tenure_class": hu_tenure_class, + "unresolved_reasons": reasons, + "TEN_valid": tenure_valid.astype(np.uint8), + }, + index=_owned_index(households.index), + copy=True, + ) + return output, _header() diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/acs_housing_universe_source.py b/packages/microcosm-build/src/microcosm/build/us_runtime/acs_housing_universe_source.py new file mode 100644 index 000000000..104c15936 --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/acs_housing_universe_source.py @@ -0,0 +1,933 @@ +"""Closed ACS source observations, with a same-snapshot native Frame bridge. + +Source projections retain vacancies, both GQ classes and unknown occupied tenure. +Their graph transport is separate; source authentication requires reconstruction +from the two fixed Census archives. No source or model download occurs here. +""" + +from __future__ import annotations + +import csv +import hashlib +import json +import os +import re +import shutil +import stat +import tempfile +import zipfile +from contextlib import contextmanager, suppress +from dataclasses import InitVar, dataclass +from pathlib import Path + +import numpy as np +import pandas as pd + +from microcosm.build.serialization_dtypes import canonicalize_frame_string_dtypes +from microcosm.frame import US_SCHEMA, Frame +from microcosm.graph.canonical import canonical_json +from microcosm.graph.population import dtype_for_token +from microcosm.graph.store import ContentStore + +from .acs_housing_universe import classify_acs_housing_universe +from .acs_inputs import map_acs_native_inputs +from .acs_pums import AcsPumsSource, _validate_person_counts, build_acs_pums_unit_frame +from .acs_sources import load_acs_source_manifest +from .graph_implementation import implementation_hash +from .operator_boundary import assert_operator_free_source_frame +from .source_csv_builtin import capture_csv_reader + +ACS_HU_STAGE = "acs_housing_universe_2024" +ACS_HU_CODEC = "us-acs-housing-universe-2024-v1" +ACS_HU_SOURCE_MAX_BYTES = 8 * 1024**3 +ACS_HU_RECEIPT_MAX_BYTES = 1024**2 +_CHUNK = 1024**2 +_MEMBER_MAX = 8 * 1024**3 +_EXPANDED_MAX = 16 * 1024**3 +_MEMBER_COUNT_MAX = 64 +_RECORD_MAX = 1024**2 +_FIELD_MAX = 64 * 1024 +_STATES = frozenset( + "01 02 04 05 06 08 09 10 11 12 13 15 16 17 18 19 20 21 22 23 24 25 " + "26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 44 45 46 47 48 " + "49 50 51 53 54 55 56".split() +) +_H_FIELDS = ("SERIALNO", "TYPEHUGQ", "NP", "TEN", "WGTP", "PUMA") +_P_FIELDS = ("SERIALNO", "SPORDER", "PWGTP") +_LINEAGE = ("source_member", "source_row_ordinal") +# No source/resource I/O merely to register the callable beside legacy codecs. +# Invented tests replace this private slot; public callers have no pin argument. +_ARCHIVE_PINS = None +_TOKEN = object() + + +class ACSHousingSourceError(ValueError): + """Sanitized source refusal; never print source keys or cells.""" + + +def _require(condition, code): + if not condition: + raise ACSHousingSourceError(code) + + +def _sha(value: bytes) -> str: + return hashlib.sha256(value).hexdigest() + + +def _json(value) -> bytes: + return canonical_json(value) + + +def _parse_json(payload: bytes, cap: int) -> dict: + _require(type(payload) is bytes and len(payload) <= cap, "JSON_SIZE") + try: + value = json.loads(payload) + _require(type(value) is dict and _json(value) == payload, "CANONICAL_JSON") + except RecursionError: + raise ACSHousingSourceError("JSON_DEPTH") from None + return value + + +def _definition() -> tuple[dict, str]: + raw = Path(__file__).with_name("acs_2024_housing_universe.json").read_bytes() + return json.loads(raw), _sha(raw) + + +def _implementation() -> str: + _require(capture_csv_reader(csv) is not None, "SOURCE_CSV_READER_CHANGED") + return implementation_hash(ACS_HU_STAGE) + + +def _pins(): + if _ARCHIVE_PINS is not None: + return _ARCHIVE_PINS + return tuple( + (a.role, a.filename, a.sha256, a.size_bytes) + for a in load_acs_source_manifest().artifacts + ) + + +def _identity(value: os.stat_result): + return tuple( + getattr(value, key) + for key in ("st_dev", "st_ino", "st_size", "st_mtime_ns", "st_ctime_ns") + ) + + +def _path_components(path: Path): + path = path.absolute() + for parent in reversed((path, *path.parents)): + value = parent.lstat() + _require(not stat.S_ISLNK(value.st_mode), "SYMLINK_PATH") + return Path(os.path.abspath(path)) + + +def _directory(path: Path): + path = _path_components(path) + _require(stat.S_ISDIR(path.lstat().st_mode), "DIRECTORY_REQUIRED") + return path + + +def _copy(source: Path, destination: Path, cap: int, *, exact_size=None): + source = _path_components(source) + fd = os.open(source, os.O_RDONLY | os.O_NONBLOCK | os.O_NOFOLLOW) + with os.fdopen(fd, "rb") as stream: + before = os.fstat(stream.fileno()) + _require(stat.S_ISREG(before.st_mode), "REGULAR_FILE_REQUIRED") + _require(before.st_size <= cap, "FILE_TOO_LARGE") + if exact_size is not None: + _require(before.st_size == exact_size, "SOURCE_SIZE") + digest, count = hashlib.sha256(), 0 + with destination.open("xb") as output: + while chunk := stream.read(min(_CHUNK, cap - count + 1)): + count += len(chunk) + _require(count <= cap, "FILE_GREW") + digest.update(chunk) + output.write(chunk) + output.flush() + os.fsync(output.fileno()) + _require( + count == before.st_size + and _identity(before) == _identity(os.fstat(stream.fileno())) + and _identity(before) == _identity(source.lstat()), + "FILE_CHANGED", + ) + _path_components(source) + destination.chmod(0o400) + return digest.hexdigest() + + +def _persisted_sha(path, size): + digest, count = hashlib.sha256(), 0 + fd = os.open(path, os.O_RDONLY | os.O_NONBLOCK | os.O_NOFOLLOW) + with os.fdopen(fd, "rb") as stream: + before = os.fstat(stream.fileno()) + _require( + stat.S_ISREG(before.st_mode) and before.st_size == size, "SNAPSHOT_SIZE" + ) + while chunk := stream.read(min(_CHUNK, size - count + 1)): + count += len(chunk) + _require(count <= size, "SNAPSHOT_GREW") + digest.update(chunk) + _require( + count == size and _identity(before) == _identity(os.fstat(stream.fileno())), + "SNAPSHOT_CHANGED", + ) + return digest.hexdigest() + + +def _write(path: Path, data: bytes, cap: int): + _require(len(data) <= cap, "OUTPUT_SIZE") + with path.open("xb") as stream: + stream.write(data) + stream.flush() + os.fsync(stream.fileno()) + path.chmod(0o400) + _require(_persisted_sha(path, len(data)) == _sha(data), "PERSISTED_OUTPUT") + + +@contextmanager +def _capture(source_dir, snapshot_root): + root = _directory(Path(snapshot_root)) + source = _directory(Path(source_dir)) + _require(not root.is_relative_to(source), "SNAPSHOT_SOURCE_OVERLAP") + pins = _pins() + expected = {p[1] for p in pins} + _require(expected == {"csv_hus.zip", "csv_pus.zip"}, "SOURCE_ROSTER") + _require({p.name for p in source.iterdir()} == expected, "SOURCE_ROSTER") + # Full private projection and selected publication/candidate capture can + # coexist. Existing captures already reduce free space and are not deducted + # a second time. This bound is operational, not a source-size prediction. + required = ( + sum(p[3] for p in pins) + + 2 * ACS_HU_SOURCE_MAX_BYTES + + 2 * ACS_HU_RECEIPT_MAX_BYTES + + 1024**3 + ) + _require(shutil.disk_usage(root).free >= required, "INSUFFICIENT_DISK") + private = Path(tempfile.mkdtemp(prefix="acs-hu-capture-", dir=root)) + paths = {} + try: + for role, name, digest, size in pins: + target = private / name + actual = _copy(source / name, target, size, exact_size=size) + _require(actual == digest, "SOURCE_SHA256") + _require(_persisted_sha(target, size) == digest, "PERSISTED_SOURCE_SHA256") + paths[role] = target + yield private, paths, pins + _require(pins == _pins(), "SOURCE_AUTHORITY_CHANGED") + except Exception as error: + # Captures belong exclusively to this invocation. Keep failed evidence; + # no rename can replace an earlier artifact or caller input. + code = ( + str(error) + if isinstance(error, ACSHousingSourceError) + else "SOURCE_PREPARATION_REFUSED" + ) + # Recording is best effort and strictly subordinate: if the record + # cannot be written, the refusal it was recording still propagates. + with suppress(Exception): + failure = _json({"status": "failed", "reason": code}) + if not (private / "failure.json").exists(): + _write(private / "failure.json", failure, ACS_HU_RECEIPT_MAX_BYTES) + raise + + +def _csv_record(raw: bytes) -> list[str]: + csv_reader = capture_csv_reader(csv) + _require(csv_reader is not None, "SOURCE_CSV_READER_CHANGED") + _require(0 < len(raw) <= _RECORD_MAX, "CSV_RECORD_SIZE") + _require(not raw.startswith(b"\xef\xbb\xbf"), "CSV_BOM") + if raw.endswith(b"\n"): + raw = raw[:-1] + if raw.endswith(b"\r"): + raw = raw[:-1] + _require( + bool(raw) and b"\x00" not in raw and b"\r" not in raw and b"\n" not in raw, + "CSV_PHYSICAL_RECORD", + ) + values = next( + csv_reader( + [raw.decode("utf-8", errors="strict")], + strict=True, + quoting=csv.QUOTE_MINIMAL, + ) + ) + _require( + all(len(v.encode("utf-8")) <= _FIELD_MAX for v in values), "CSV_FIELD_SIZE" + ) + return values + + +def _archive(path: Path, role: str): + """Validate the complete central directory before resolving any member name.""" + prefix = "psam_hus" if role == "household" else "psam_pus" + required = _H_FIELDS if role == "household" else _P_FIELDS + inventory, rows, role_header, fields, total_bytes = [], [], None, None, 0 + with zipfile.ZipFile(path) as archive: + members = archive.infolist() + _require(0 < len(members) <= _MEMBER_COUNT_MAX, "ZIP_MEMBER_COUNT") + names = [m.filename for m in members] + _require( + len({n.casefold() for n in names}) == len(names), "ZIP_DUPLICATE_MEMBER" + ) + for member in members: + name = member.filename + mode = member.external_attr >> 16 + _require( + name not in ("", ".", "..") + and "/" not in name + and "\\" not in name + and "\x00" not in name, + "ZIP_MEMBER_PATH", + ) + _require( + not member.is_dir() and (stat.S_IFMT(mode) in (0, stat.S_IFREG)), + "ZIP_MEMBER_TYPE", + ) + _require( + not member.flag_bits & 1 + and member.compress_type in (zipfile.ZIP_STORED, zipfile.ZIP_DEFLATED), + "ZIP_MEMBER_ENCODING", + ) + _require(0 <= member.file_size <= _MEMBER_MAX, "ZIP_MEMBER_SIZE") + total_bytes += member.file_size + _require(total_bytes <= _EXPANDED_MAX, "ZIP_EXPANDED_SIZE") + _require( + not name.casefold().startswith(prefix) + or name.casefold().endswith(".csv"), + "ZIP_PREFIX_NONCSV", + ) + selected_bytes = 0 + for member in sorted(members, key=lambda m: m.filename): + applicable = member.filename.casefold().startswith(prefix) + digest, count, header_raw, header, record_count = ( + hashlib.sha256(), + 0, + b"", + None, + 0, + ) + with archive.open(member) as stream: + while True: + raw = ( + stream.readline( + min(_RECORD_MAX + 1, member.file_size - count + 1) + ) + if applicable + else stream.read(min(_CHUNK, member.file_size - count + 1)) + ) + if not raw: + break + count += len(raw) + _require(count <= member.file_size, "ZIP_MEMBER_GREW") + digest.update(raw) + if not applicable: + continue + tokens = _csv_record(raw) + if header is None: + header, header_raw = tokens, raw + _require( + all(header) and len(set(header)) == len(header), + "CSV_HEADER", + ) + _require(set(required) <= set(header), "CSV_REQUIRED_FIELDS") + if role == "household": + _require( + bool({"ST", "STATE"} & set(header)), + "CSV_STATE_REQUIRED", + ) + if role_header is None: + role_header = header + fields = ( + ( + *required, + *(n for n in ("ST", "STATE") if n in header), + ) + if role == "household" + else required + ) + _require(header == role_header, "CSV_ROLE_HEADER_MISMATCH") + positions = [header.index(f) for f in fields] + continue + _require(len(tokens) == len(header), "CSV_ROW_WIDTH") + record_count += 1 + row = [ + *(tokens[p] for p in positions), + member.filename, + record_count, + ] + selected_bytes += len(_json(row)) + _require( + selected_bytes <= ACS_HU_SOURCE_MAX_BYTES, + "PROJECTION_TOO_LARGE", + ) + rows.append(row) + _require(count == member.file_size, "ZIP_MEMBER_SIZE_MISMATCH") + _require(not applicable or header is not None, "CSV_HEADER_MISSING") + inventory.append( + { + "name": member.filename, + "compressed_bytes": member.compress_size, + "bytes": count, + "crc32": member.CRC, + "sha256": digest.hexdigest(), + "is_data": applicable, + "header_hex": header_raw.hex(), + "header_sha256": _sha(header_raw), + "header": header, + "rows": record_count, + } + ) + _require(role_header is not None, "ZIP_ROLE_MISSING") + return list((*fields, *_LINEAGE)), rows, inventory + + +def _integer(token, low, high, label): + _require( + type(token) is str + and re.fullmatch(r"[0-9]+", token, flags=re.ASCII) is not None + and len(token) <= 5, + label, + ) + value = int(token) + _require(low <= value <= high, label) + return value + + +def _tables(projection): + _require( + set(projection) + == {"format", "household_columns", "person_columns", "households", "persons"} + and projection["format"] == "acs-housing-lexical-projection/1", + "PROJECTION_SCHEMA", + ) + hcols, pcols = projection["household_columns"], projection["person_columns"] + _require( + hcols + in [ + list((*_H_FIELDS, *state, *_LINEAGE)) + for state in (("ST",), ("STATE",), ("ST", "STATE")) + ] + and pcols == list((*_P_FIELDS, *_LINEAGE)), + "PROJECTION_COLUMNS", + ) + for cols, name in ((hcols, "households"), (pcols, "persons")): + _require(type(projection[name]) is list, "PROJECTION_ROWS") + for row in projection[name]: + _require( + type(row) is list + and len(row) == len(cols) + and all(type(v) is str for v in row[:-1]) + and type(row[-1]) is int + and row[-1] > 0, + "PROJECTION_ROW", + ) + household = pd.DataFrame(projection["households"], columns=hcols) + person = pd.DataFrame(projection["persons"], columns=pcols) + _require(not household.empty, "EMPTY_HOUSEHOLD_SOURCE") + for table in (household, person): + _require( + all( + re.fullmatch(r"2024(?:HU|GQ)[0-9]{7}", value, flags=re.ASCII) + and int(value[-7:]) > 0 + for value in table.SERIALNO + ), + "SERIALNO", + ) + _require(not household.SERIALNO.duplicated().any(), "DUPLICATE_HOUSEHOLD") + _require(set(person.SERIALNO) <= set(household.SERIALNO), "ORPHAN_PERSON") + typed = pd.DataFrame(index=household.index) + for name, low, high in (("TYPEHUGQ", 1, 3), ("NP", 0, 20), ("WGTP", 0, 9999)): + typed[name] = np.asarray( + [_integer(v, low, high, name) for v in household[name]], dtype="int64" + ) + valid = household.TEN.to_numpy() != "" + typed["TEN"] = np.asarray( + [ + _integer(v, 1, 4, "TEN") if present else 0 + for v, present in zip(household.TEN, valid, strict=True) + ], + dtype="int64", + ) + lines = np.asarray( + [ + _integer(v, 1, 20, "SPORDER") + if len(v) <= 2 + else _integer("", 1, 20, "SPORDER") + for v in person.SPORDER + ], + dtype="int64", + ) + pw = np.asarray( + [_integer(v, 1, 9999, "PWGTP") for v in person.PWGTP], dtype="int64" + ) + _require( + not pd.MultiIndex.from_arrays([person.SERIALNO, lines]).duplicated().any(), + "DUPLICATE_PERSON", + ) + state = "ST" if "ST" in household else "STATE" + _require(set(household[state]) <= _STATES, "STATE_SCOPE") + if "ST" in household and "STATE" in household: + _require(household.ST.equals(household.STATE), "STATE_CONFLICT") + for v in household.PUMA: + _require(len(v) == 5, "PUMA") + _integer(v, 100, 81003, "PUMA") + gq = typed.TYPEHUGQ.to_numpy() != 1 + vacant = ~gq & (typed.NP.to_numpy() == 0) + _require( + np.array_equal(household.SERIALNO.str.startswith("2024GQ").to_numpy(), gq), + "SERIALNO_TYPE", + ) + _require(bool((typed.NP.to_numpy()[gq] == 1).all()), "GQ_NP") + _require( + bool((typed.WGTP.to_numpy()[gq] == 0).all()) + and bool((typed.WGTP.to_numpy()[~gq] > 0).all()), + "WGTP_SCOPE", + ) + _require(not bool(valid[gq | vacant].any()), "TEN_NIU_SCOPE") + full_h = typed.assign(SERIALNO=household.SERIALNO) + _validate_person_counts(full_h, person) + codes, _header = classify_acs_housing_universe( + typed.loc[:, ["TYPEHUGQ", "NP", "TEN"]], tenure_valid=valid.astype(bool) + ) + return household, person, typed, lines, pw, codes + + +def _select(projection, serialnos): + household, person, typed, lines, _pw, _codes = _tables(projection) + if serialnos is None: + chosen = tuple(sorted(household.SERIALNO)) + else: + _require( + type(serialnos) is tuple + and bool(serialnos) + and all(type(v) is str for v in serialnos) + and len(set(serialnos)) == len(serialnos), + "SELECTION_KEYS", + ) + _require(set(serialnos) <= set(household.SERIALNO), "SELECTION_UNKNOWN") + chosen = tuple(sorted(serialnos)) + horder = household.sort_values("SERIALNO", kind="stable").index + porder = ( + person.assign(_line=lines) + .sort_values(["SERIALNO", "_line"], kind="stable") + .index + ) + chosen_set = set(chosen) + selected = { + **projection, + "households": [ + projection["households"][i] + for i in horder + if household.SERIALNO[i] in chosen_set + ], + "persons": [ + projection["persons"][i] for i in porder if person.SERIALNO[i] in chosen_set + ], + } + full_counts = { + "households": len(household), + "persons": len(person), + "vacant": int(((typed.TYPEHUGQ == 1) & (typed.NP == 0)).sum()), + "institutional_gq": int((typed.TYPEHUGQ == 2).sum()), + "noninstitutional_gq": int((typed.TYPEHUGQ == 3).sum()), + } + return selected, chosen, full_counts + + +@dataclass(frozen=True) +class AuthenticatedACSHousingSource: + """Owned immutable source bytes; DataFrame access returns independent copies.""" + + projection_json: bytes + receipt_json: bytes + _token: InitVar[object] = None + + def __post_init__(self, _token): + _require(_token is _TOKEN, "AUTHENTICATED_SOURCE_CONSTRUCTOR") + + @property + def households(self): + return _tables(json.loads(self.projection_json))[0] + + @property + def persons(self): + return _tables(json.loads(self.projection_json))[1] + + @property + def codes(self): + return _tables(json.loads(self.projection_json))[5] + + +def _reconstruct(private, paths, pins, serialnos, implementation): + hc, h, hi = _archive(paths["household"], "household") + pc, p, pi = _archive(paths["person"], "person") + full = { + "format": "acs-housing-lexical-projection/1", + "household_columns": hc, + "person_columns": pc, + "households": h, + "persons": p, + } + full_bytes = _json(full) + _write(private / "full-projection.json", full_bytes, ACS_HU_SOURCE_MAX_BYTES) + selected, chosen, counts = _select(full, serialnos) + projection = _json(selected) + _require(len(projection) <= ACS_HU_SOURCE_MAX_BYTES, "PROJECTION_TOO_LARGE") + definition, definition_sha = _definition() + receipt = { + "format": "microcosm.acs_housing_universe_source.v1", + "release_eligible": False, + "vintage": 2024, + "definition_sha256": definition_sha, + "source_definitions": [ + definition["vacancy_definition"], + definition["gq_definition"], + ], + "implementation_sha256": implementation, + "parser_profile": { + "engine": "stdlib_csv_reader", + "encoding": "utf-8", + "quoting": "QUOTE_MINIMAL", + "strict": True, + "csv.field_size_limit": csv.field_size_limit(), + "field_max_bytes": _FIELD_MAX, + "record_max_bytes": _RECORD_MAX, + }, + "archives": [ + {"role": r, "filename": n, "sha256": d, "bytes": s} for r, n, d, s in pins + ], + "members": {"household": hi, "person": pi}, + "full_projection_sha256": _sha(full_bytes), + "full_counts": counts, + "selection": "all" if serialnos is None else "exact_serialnos", + "selected_serialnos_sha256": _sha(_json(chosen)), + "selected_counts": { + "households": len(selected["households"]), + "persons": len(selected["persons"]), + }, + "projection_sha256": _sha(projection), + "projection_bytes": len(projection), + "source_weight_kind": "design", + "calibrated_descendants_approved": False, + } + receipt_bytes = _json(receipt) + _require(len(receipt_bytes) <= ACS_HU_RECEIPT_MAX_BYTES, "RECEIPT_TOO_LARGE") + return AuthenticatedACSHousingSource(projection, receipt_bytes, _token=_TOKEN) + + +def _run_source(source_dir, snapshot_root, serialnos, output_dir, readback): + try: + original = _directory(Path(source_dir)) + destination = Path(os.path.abspath(output_dir)) + _require(not destination.is_relative_to(original), "OUTPUT_SOURCE_OVERLAP") + if readback: + _directory(destination) + else: + _directory(destination.parent) + _require(not destination.exists(), "OUTPUT_EXISTS") + implementation = _implementation() + with _capture(source_dir, snapshot_root) as (private, paths, pins): + result = _reconstruct(private, paths, pins, serialnos, implementation) + _require(_implementation() == implementation, "IMPLEMENTATION_CHANGED") + if readback: + _directory(destination) + _require( + {p.name for p in destination.iterdir()} + == {"projection.json", "receipt.json"}, + "OUTPUT_ROSTER", + ) + for name, expected, cap in ( + ( + "projection.json", + result.projection_json, + ACS_HU_SOURCE_MAX_BYTES, + ), + ("receipt.json", result.receipt_json, ACS_HU_RECEIPT_MAX_BYTES), + ): + target = private / ("candidate-" + name) + # SOURCE_SIZE and FILE_TOO_LARGE are the pinned-archive + # codes. A candidate of the wrong length is a different + # fact, and an operator has to be able to tell them apart, + # so only this call's size refusals are renamed. + try: + digest = _copy( + destination / name, target, cap, exact_size=len(expected) + ) + except ACSHousingSourceError as error: + if str(error) not in ("SOURCE_SIZE", "FILE_TOO_LARGE"): + raise + raise ACSHousingSourceError("RECONSTRUCTION_SIZE") from None + _require( + digest == _sha(expected) + and _persisted_sha(target, len(expected)) == digest, + "RECONSTRUCTION_MISMATCH", + ) + else: + _directory(destination.parent) + destination.mkdir(mode=0o700, exist_ok=False) + _write( + destination / "projection.json", + result.projection_json, + ACS_HU_SOURCE_MAX_BYTES, + ) + _write( + destination / "receipt.json", + result.receipt_json, + ACS_HU_RECEIPT_MAX_BYTES, + ) + return result + except ACSHousingSourceError: + raise + except ( + OSError, + ValueError, + TypeError, + KeyError, + zipfile.BadZipFile, + UnicodeError, + csv.Error, + ): + raise ACSHousingSourceError("SOURCE_PREPARATION_REFUSED") from None + + +def produce_acs_housing_source( + source_dir, *, snapshot_root, output_dir, serialnos=None +): + """Authenticate full parents, retain whole selected keys, publish exclusively.""" + return _run_source(source_dir, snapshot_root, serialnos, output_dir, False) + + +def load_acs_housing_source(source_dir, output_dir, *, snapshot_root, serialnos=None): + """Fresh full-parent reconstruction; candidate files never supply authority.""" + return _run_source(source_dir, snapshot_root, serialnos, output_dir, True) + + +def verify_acs_frame_projection(frame: Frame, projection: dict): + """Check a Frame against source observations, without granting source authority.""" + household, person, typed, lines, pw, _codes = _tables(projection) + h = frame.table("household") + p = frame.person + expected_h = household.loc[typed.NP > 0].reset_index(drop=True) + _require( + list(h.SERIALNO.astype(str)) == list(expected_h.SERIALNO), + "FRAME_HOUSEHOLD_ORIGIN", + ) + indexed = household.set_index("SERIALNO") + observed = indexed.loc[h.SERIALNO.astype(str)] + # The unchanged Frame builder retains TYPEHUGQ/NP/TEN but not raw WGTP. + # WGTP stays lexical in the source artifact and binds the typed design + # weights below, including the distinct GQ PWGTP substitution. + for field in ("TYPEHUGQ", "NP"): + _require( + np.array_equal(h[field].to_numpy(), observed[field].map(int).to_numpy()), + "FRAME_RAW_CONTROLS", + ) + valid = observed.TEN.to_numpy() != "" + _require(np.array_equal(h.TEN.notna().to_numpy(), valid), "FRAME_TEN_VALIDITY") + _require( + np.array_equal( + h.TEN.to_numpy()[valid], observed.TEN[valid].map(int).to_numpy() + ), + "FRAME_TEN_VALUES", + ) + state = "ST" if "ST" in observed else "STATE" + _require( + list(h.ST.astype(str)) == list(observed[state]) + and list(h.PUMA.astype(str)) == list(observed.PUMA), + "FRAME_GEOGRAPHY", + ) + ids = dict(zip(h.SERIALNO.astype(str), h.household_id, strict=True)) + _require(len(p) == len(person), "FRAME_PERSON_COUNT") + expected_ids = np.asarray([ids[s] for s in person.SERIALNO], dtype="int64") + _require( + np.array_equal(p.person_household_id.to_numpy(), expected_ids) + and np.array_equal(p.source_household_id.to_numpy(), expected_ids) + and np.array_equal(p.SPORDER.to_numpy(), lines) + and list(p.source_person_id.astype(str)) == [str(v) for v in lines], + "FRAME_PERSON_ORIGIN", + ) + _require( + np.array_equal(p.PWGTP.to_numpy(), pw) + and bool((p.source_year.to_numpy() == 2024).all()), + "FRAME_PERSON_SOURCE", + ) + _require( + frame.weighted_entities == ("household",) + and frame.weights_for("household").kind.value == "design", + "FRAME_WEIGHT_KIND", + ) + by_serial = dict(zip(person.SERIALNO, pw, strict=True)) + expected_weights = np.asarray( + [ + by_serial[serial] if int(weight) == 0 else int(weight) + for serial, weight in zip(observed.index, observed.WGTP, strict=True) + ], + dtype="float64", + ) + _require( + frame.weights_for("household").values.tobytes() == expected_weights.tobytes(), + "FRAME_DESIGN_WEIGHT", + ) + + +def frame_content_sha256(frame: Frame): + """Typed storage identity of every declared cell, axis, stratum and weight.""" + digest = hashlib.sha256(b"microcosm.acs-frame-storage/1\0") + + def series(values): + digest.update(_json({"dtype": str(values.dtype), "name": values.name})) + mask = values.isna().to_numpy(dtype=bool) + digest.update(mask.tobytes()) + array = values.to_numpy(copy=False) + if isinstance(values.dtype, np.dtype) and not values.dtype.hasobject: + digest.update(np.ascontiguousarray(array).tobytes()) + else: + for value, missing in zip(array, mask, strict=True): + item = ( + None + if missing + else value.item() + if isinstance(value, np.generic) + else value + ) + raw = _json(item) + digest.update(len(raw).to_bytes(8, "little")) + digest.update(raw) + + _require(frame.schema == US_SCHEMA, "FRAME_SCHEMA") + digest.update(_json(dict(frame.metadata))) + _require(not frame.mass_log, "FRAME_MASS_LOG") + for entity in frame.entities: + table = frame.table(entity) + digest.update(_json([entity, list(table.columns)])) + series(pd.Series(table.index.to_numpy(), name=table.index.name)) + for column in table: + series(table[column]) + series(frame.strata) + series(pd.Series(frame.strata.index.to_numpy(), name=frame.strata.index.name)) + for entity in frame.weighted_entities: + digest.update(_json([entity, frame.weights_for(entity).kind.value])) + digest.update(frame.weights_for(entity).values.tobytes()) + return digest.hexdigest() + + +@dataclass(frozen=True) +class PreparedACSHousingPopulation: + frame: Frame + source: AuthenticatedACSHousingSource + receipt_json: bytes + + @property + def receipt(self): + return json.loads(self.receipt_json) + + +def prepare_acs_housing_population(source_dir, *, snapshot_root, serialnos=None): + """Validate the full lexical source; construct only exact selected native rows.""" + try: + serialnos = AcsPumsSource.snapshot_serialnos(serialnos) + implementation = _implementation() + with _capture(source_dir, snapshot_root) as (private, paths, pins): + source = _reconstruct(private, paths, pins, serialnos, implementation) + projection = json.loads(source.projection_json) + _require(bool(projection["persons"]), "EMPTY_GRAPH_POPULATION") + raw, _builder_receipt = build_acs_pums_unit_frame( + AcsPumsSource( + paths["household"], + paths["person"], + vintage=2024, + max_households=None, + ), + serialnos=serialnos, + ) + mapped = map_acs_native_inputs(raw) + frame = mapped.frame + assert_operator_free_source_frame( + frame, + label="ACS HU authenticated source", + native_inputs=mapped.native_inputs, + ) + verify_acs_frame_projection(frame, projection) + before = frame_content_sha256(frame) + transitions = [] + original_dtypes = { + (e, c): repr(frame.table(e)[c].dtype) + for e in frame.entities + for c in frame.table(e) + } + canonicalize_frame_string_dtypes( + frame, boundary="ACS HU graph storage", in_place=True + ) + for entity in frame.entities: + table = frame.table(entity) + for column in table: + original = table[column] + if isinstance( + original.dtype, pd.StringDtype + ) and original.dtype != dtype_for_token("string"): + promoted = original.astype(dtype_for_token("string")) + _require( + original.isna().equals(promoted.isna()) + and np.array_equal( + original[original.notna()].to_numpy(), + promoted[promoted.notna()].to_numpy(), + ), + "GRAPH_STRING_PROMOTION", + ) + table[column] = promoted + if original_dtypes[(entity, column)] != repr(table[column].dtype): + transitions.append( + { + "entity": entity, + "column": column, + "from": original_dtypes[(entity, column)], + "to": repr(table[column].dtype), + } + ) + verify_acs_frame_projection(frame, projection) + _require(_implementation() == implementation, "IMPLEMENTATION_CHANGED") + receipt = { + "format": "microcosm.acs_housing_preparation.v2", + "release_eligible": False, + "source_receipt_sha256": _sha(source.receipt_json), + "projection_sha256": _sha(source.projection_json), + "implementation_sha256": implementation, + "same_snapshot_frame_and_observations": True, + "full_source_frame_before_selection": serialnos is None, + "full_source_lexical_projection": True, + "native_selection_before_person_accumulation": serialnos is not None, + "selection_kind": "all" + if serialnos is None + else "engineering_exact_keys", + "requested_serialnos": serialnos, + "full_source_inclusion_probability": None, + "pre_promotion_frame_sha256": before, + "frame_sha256": frame_content_sha256(frame), + "dtype_transitions": transitions, + "entity_rows": {e: frame.n(e) for e in frame.entities}, + "weight_kind": "design", + "HU_columns": "artifact_only", + } + return PreparedACSHousingPopulation(frame, source, _json(receipt)) + except ACSHousingSourceError: + raise + except ( + OSError, + ValueError, + TypeError, + KeyError, + zipfile.BadZipFile, + UnicodeError, + csv.Error, + ): + raise ACSHousingSourceError("FRAME_PREPARATION_REFUSED") from None + + +def load_graph_acs_housing_universe(path, *, store=None): + """Frame codec with an explicit store and a dedicated private capture child.""" + _require(type(store) is ContentStore, "EXPLICIT_CONTENT_STORE_REQUIRED") + root = _directory(store.root) / "acs-hu-source-captures-v1" + root.mkdir(mode=0o700, exist_ok=True) + return prepare_acs_housing_population(path, snapshot_root=root).frame diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/acs_inputs.py b/packages/microcosm-build/src/microcosm/build/us_runtime/acs_inputs.py index 88b29412f..2d8e8e99e 100644 --- a/packages/microcosm-build/src/microcosm/build/us_runtime/acs_inputs.py +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/acs_inputs.py @@ -247,6 +247,14 @@ def _map_tenure( transformation="ACS TEN enum recode through SPM membership", register=register, ) + # These enum outputs are strings even when the selected source households + # all have unreported tenure. Declare that type without inventing a value; + # generic serialization cannot infer a type from an all-missing object axis. + string_dtype = pd.StringDtype(storage="python", na_value=np.nan) + household["tenure_type"] = household["tenure_type"].astype(string_dtype) + spm_unit["spm_unit_tenure_type"] = spm_unit["spm_unit_tenure_type"].astype( + string_dtype + ) def _map_housing_amounts( diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/acs_native_coverage_binding.py b/packages/microcosm-build/src/microcosm/build/us_runtime/acs_native_coverage_binding.py new file mode 100644 index 000000000..770fdc9bb --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/acs_native_coverage_binding.py @@ -0,0 +1,711 @@ +"""Closed real ACS preparation plus literal coverage issuance, without admission. + +Private process ownership is required in addition to immutable evidence bytes. +This is a native population successor, not a decoder or a graph attachment. +""" + +from __future__ import annotations + +import ast +import builtins +import importlib.util +import json +import sys +import tempfile +from _thread import RLock +from dataclasses import InitVar, dataclass +from pathlib import Path +from types import BuiltinFunctionType, CodeType, FunctionType +from weakref import WeakKeyDictionary + +from microcosm.frame import Frame + +from . import acs_housing_universe_source as housing +from . import acs_person_coverage_authentication as coverage +from . import graph_implementation as implementation + +PROTOCOL = "microcosm.acs-native-coverage-binding.v2" +MAX_ARCHIVE_BYTES = 8 * 1024**3 # combined compressed bytes, before capture +MAX_EXPANDED_BYTES = 16 * 1024**3 # combined, before opening any member +MAX_SOURCE_ROWS = 6_000_000 # per role, before full-source construction +MAX_EVIDENCE_BYTES = 2 * 1024**2 +# Exact accepted direct AGEP -> A_AGE and AGEP -> age implementation, and owners. +# A new transform/owner version requires explicit review of this successor. +_ACCEPTED = { + "acs_pums.py": "6ecf79f0dfb0c0bc0ad0af6be5fa65bd8c2d1e1968009402e4c8dee347de70dc", + "acs_inputs.py": "aa4a8aeaba63dfef2f3e04fb89de59766deb088ed7f4d290aeba0425739916da", + "acs_housing_universe_source.py": "beb46a4a05a13580a868be423809a77946441dcafc3a0f57160561157e93e9a3", + "acs_person_coverage_authentication.py": "475aa795c8a5b49a0dd3405a0877866dddd03012a1f2b9743447fab1fe85bcff", +} +_TOKEN = object() +_ISSUED = WeakKeyDictionary() + +# Only bytecode compilation is reused across producers. Fresh source reads, +# AST checks, local code indexes and loaded-function checks remain mandatory. +_COMPILE_CACHE_MAX_ENTRIES = 128 +_COMPILE_CACHE_MAX_SOURCE_BYTES = 16 * 1024**2 +_COMPILE_CACHE_MAX_ENTRY_BYTES = 1024**2 +_COMPILE_CACHE_COMPILER = compile +_COMPILE_CACHE_DEFAULT_OPTIMIZE = sys.flags.optimize +_COMPILE_CACHE = {} +_COMPILE_CACHE_LOCK = RLock() + + +class ACSNativeCoverageBindingError(ValueError): + """Static refusal without source values, paths or exception chains.""" + + +def _require(condition, code): + if not condition: + raise ACSNativeCoverageBindingError(code) + + +def _clear_compile_cache(): + """Clear only compiled-source outputs, for process-local test isolation.""" + with _COMPILE_CACHE_LOCK: + _COMPILE_CACHE.clear() + + +def _compile_source( + source, filename, mode="exec", *, flags=0, dont_inherit=True, optimize=-1 +): + """Reuse bounded immutable bytecode; never reuse loaded-function validity. + + Non-original compilers and context-dependent or non-exact inputs bypass the + cache. Compiler warning/audit events occur on misses; AST parsing still runs + on every _live_code call. This shares the trusted-process scope of that check. + """ + compiler = compile + eligible = ( + compiler is _COMPILE_CACHE_COMPILER + and type(compiler) is BuiltinFunctionType + and compiler.__module__ == "builtins" + and compiler.__name__ == "compile" + and compiler.__self__ is builtins + and type(source) is bytes + and type(filename) is str + and type(mode) is str + and mode in ("exec", "eval", "single") + and type(flags) is int + and flags >= 0 + and dont_inherit is True + and type(optimize) is int + and optimize in (-1, 0, 1, 2) + and len(source) <= _COMPILE_CACHE_MAX_ENTRY_BYTES + and len(source) <= _COMPILE_CACHE_MAX_SOURCE_BYTES + and _COMPILE_CACHE_MAX_ENTRIES > 0 + ) + key = None + if eligible: + effective_optimize = ( + _COMPILE_CACHE_DEFAULT_OPTIMIZE if optimize == -1 else optimize + ) + key = ( + source, + filename, + mode, + flags, + dont_inherit, + effective_optimize, + id(compiler), + ) + with _COMPILE_CACHE_LOCK: + entry = _COMPILE_CACHE.get(key) + if ( + type(entry) is tuple + and len(entry) == 3 + and type(entry[0]) is tuple + and entry[0] == key + and entry[1] is compiler + and type(entry[2]) is CodeType + ): + return entry[2] + _COMPILE_CACHE.pop(key, None) + + # Compile outside the lock: instrumentation/audit hooks may reenter, and a + # concurrent duplicate miss is harmless. Never retain a failed compilation. + code = compiler( + source, + filename, + mode, + flags=flags, + dont_inherit=dont_inherit, + optimize=optimize, + ) + if key is not None and type(code) is CodeType: + with _COMPILE_CACHE_LOCK: + # A nested/concurrent call may already have filled this key. FIFO + # eviction bounds retained source keys; it is not a hard RSS cap. + if key not in _COMPILE_CACHE: + while _COMPILE_CACHE and ( + len(_COMPILE_CACHE) >= _COMPILE_CACHE_MAX_ENTRIES + or sum(len(item[0]) for item in _COMPILE_CACHE) + len(source) + > _COMPILE_CACHE_MAX_SOURCE_BYTES + ): + del _COMPILE_CACHE[next(iter(_COMPILE_CACHE))] + _COMPILE_CACHE[key] = (key, compiler, code) + return code + + +def _live_code(module, compiled): + """Check loaded Python implementations against their current source bytes. + + Covers module functions, imported function aliases and source class methods; + generated dataclass methods have no source code object. This is a drift + check in a trusted process, not a sandbox against arbitrary Python execution. + """ + path = getattr(module, "__file__", None) + if path is None: + return + tree = ast.parse(Path(path).read_bytes()) + for node in tree.body: + if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef)): + value = getattr(module, node.name, None) + _require( + value is not None and value.__module__ == module.__name__, + "LOADED_PRODUCER", + ) + elif isinstance(node, ast.ImportFrom) and node.module != "__future__": + origin_name = ( + importlib.util.resolve_name( + "." * node.level + (node.module or ""), module.__package__ + ) + if node.level + else node.module + ) + origin = sys.modules.get(origin_name) + for alias in node.names: + if alias.name != "*" and origin is not None: + _require( + getattr(module, alias.asname or alias.name, None) + is getattr(origin, alias.name, None), + "LOADED_PRODUCER", + ) + + def check(function): + if not isinstance(function, FunctionType): + return + origin = sys.modules.get(function.__globals__.get("__name__")) + path = getattr(origin, "__file__", None) + _require(path is not None, "LOADED_PRODUCER") + if function.__code__.co_filename == "": + return # dataclass-generated methods + if path not in compiled: + codes = {} + + def visit(code): + codes[code.co_qualname] = code + for value in code.co_consts: + if isinstance(value, CodeType): + visit(value) + + visit( + _compile_source( + Path(path).read_bytes(), path, "exec", dont_inherit=True + ) + ) + compiled[path] = codes + _require( + function.__globals__ is vars(origin) + and function.__code__ == compiled[path].get(function.__code__.co_qualname), + "LOADED_PRODUCER", + ) + + for cell in function.__closure__ or (): + if isinstance(cell.cell_contents, FunctionType): + check(cell.cell_contents) + + for value in vars(module).values(): + if isinstance(value, FunctionType): + check(value) + elif isinstance(value, type) and value.__module__ == module.__name__: + for method in vars(value).values(): + if isinstance(method, (classmethod, staticmethod)): + method = method.__func__ + if isinstance(method, property): + check(method.fget) + check(method.fset) + else: + check(method) + + +def _producer(): + _require(PROTOCOL == "microcosm.acs-native-coverage-binding.v2", "NATIVE_PROTOCOL") + for name, expected in _ACCEPTED.items(): + _require( + coverage._sha(Path(__file__).with_name(name).read_bytes()) == expected, + "UNREVIEWED_PREPARATION", + ) + manifest = implementation.implementation_manifest(housing.ACS_HU_STAGE) + # Reuse the reviewed complete preparation closure, without changing its + # graph inventory or historical hashes. Include source-only coverage's own + # closure, C _csv provider (libpython where builtin), and this successor. + names = { + "microcosm.build.us_runtime.acs_person_coverage_authentication", + "microcosm.build.us_runtime.acs_person_coverage_columns", + __name__, + } + for item in manifest["modules"]: + package, relative = item.split("/", 1) + name = package + "." + relative.removesuffix(".py").replace("/", ".") + names.add(name.removesuffix(".__init__")) + compiled = {} + for name in sorted(names): + module = sys.modules.get(name) + if module is not None: + _live_code(module, compiled) + return { + "preparation": manifest, + "coverage": coverage._producer(), + "issuer_sha256": coverage._sha(Path(__file__).read_bytes()), + "limits": [ + MAX_ARCHIVE_BYTES, + MAX_EXPANDED_BYTES, + MAX_SOURCE_ROWS, + MAX_EVIDENCE_BYTES, + sys.modules[housing.AcsPumsSource.__module__].MAX_EXACT_HOUSEHOLDS, + sys.modules[housing.AcsPumsSource.__module__].MAX_EXACT_PERSON_ROWS, + ], + "accepted_age_transform": "literal_numeric_identity_AGEP_to_AGEP_A_AGE_age", + } + + +def _archives(pins): + _require( + len(pins) == 2 and {p[0] for p in pins} == {"household", "person"}, + "SOURCE_ROLES", + ) + return [ + {"role": role, "filename": name, "sha256": digest, "bytes": size} + for role, name, digest, size in pins + ] + + +def _members_equal(left, right): + # Owners intentionally have distinct parser profiles and inventory shapes. + # Compare common captured byte/header/row identities, never parser claims. + fields = ("name", "bytes", "sha256", "compressed_bytes", "crc32", "rows") + for role in ("household", "person"): + a, b = left[role], right[role] + _require(len(a) == len(b), "SOURCE_MEMBER_IDENTITY") + for x, y in zip(a, b, strict=True): + _require(all(x[k] == y[k] for k in fields), "SOURCE_MEMBER_IDENTITY") + if y["applicable"]: + _require( + x["header_sha256"] == y["header_sha256"], "SOURCE_MEMBER_IDENTITY" + ) + + +def _preflight(paths, serialnos=None): + # Directory/expanded-byte gates precede member opening. Streaming lexical + # gates then precede preparation's full projection and pandas construction. + expanded = 0 + for role, path in paths.items(): + with coverage.zipfile.ZipFile(path) as archive: + members, _prefix = coverage._members(archive, role) + expanded += sum(member.file_size for member in members) + _require(expanded <= MAX_EXPANDED_BYTES, "EXPANDED_BUDGET") + inventories = {} + if serialnos is None: + # Preserve the whole-source route and its pre-construction row ceiling. + for role, path in paths.items(): + inventories[role], _empty = coverage._inventory(path, role, frozenset()) + _require( + sum(m["rows"] for m in inventories[role]) <= MAX_SOURCE_ROWS, + "SOURCE_ROW_BUDGET", + ) + if role == "person": + _require( + sum(m["rows"] for m in inventories[role]) + <= coverage.literal.MAX_SELECTED_ROWS, + "NATIVE_ROW_BUDGET", + ) + return inventories, None + selected = frozenset(serialnos) + inventories["household"], households = coverage._inventory( + paths["household"], "household", selected + ) + _require( + sum(m["rows"] for m in inventories["household"]) <= MAX_SOURCE_ROWS, + "SOURCE_ROW_BUDGET", + ) + _require(set(households) == selected, "SELECTION_UNKNOWN") + _require(all(0 <= n <= 20 for n in households.values()), "SELECTED_NP") + expected_rows = sum(households.values()) + _require(expected_rows > 0, "EMPTY_SELECTED_POPULATION") + _require(expected_rows <= coverage.literal.MAX_SELECTED_ROWS, "NATIVE_ROW_BUDGET") + # _inventory bounds the actual selected row/body accumulation, independently + # of reported NP. Every source member is still streamed and hashed. + inventories["person"], roster = coverage._inventory( + paths["person"], "person", selected + ) + _require( + sum(m["rows"] for m in inventories["person"]) <= MAX_SOURCE_ROWS, + "SOURCE_ROW_BUDGET", + ) + counts = dict.fromkeys(selected, 0) + for serial, _line in roster: + counts[serial] += 1 + _require(counts == households, "SELECTED_ROSTER") + _require(len(roster) == expected_rows, "SELECTED_ROSTER") + return inventories, (households, roster) + + +@dataclass(frozen=True, slots=True) +class _Owned: + frame: Frame + payload: bytes + prepared: housing.PreparedACSHousingPopulation + literal: coverage.AuthenticatedACSPersonCoverage + snapshots: tuple + pins: tuple + + +@dataclass(frozen=True, slots=True, weakref_slot=True, eq=False) +class AuthenticatedACSNativeCoverage: + """Process-owned immutable evidence; the live Frame is checked on every borrow.""" + + payload: bytes + _token: InitVar[object] = None + + def __post_init__(self, _token): + _require(_token is _TOKEN, "ISSUANCE_CONSTRUCTOR") + + @property + def receipt(self): + return json.loads(_owned(self).payload) + + @property + def frame(self): + verify_acs_native_coverage(self) + return _owned(self).frame + + @property + def coverage_table(self): + return _owned(self).literal.table + + @property + def source_coverage(self): + """The unchanged predecessor still has an ungranted native_binding.""" + return _owned(self).literal + + +def _owned(issuance): + _require(type(issuance) is AuthenticatedACSNativeCoverage, "ISSUANCE_TYPE") + owned = _ISSUED.get(issuance) + _require( + owned is not None + and type(issuance.payload) is bytes + and issuance.payload == owned.payload, + "ISSUANCE_NOT_OWNED", + ) + return owned + + +def _verify_sources(snapshots, pins): + for paths in snapshots: + for role, _name, digest, size in pins: + path = paths[role] + housing._path_components(path) + _require(housing._persisted_sha(path, size) == digest, "SOURCE_CHANGED") + _require(housing._pins() == pins, "SOURCE_AUTHORITY_CHANGED") + + +def _frame_sha256(frame): + # Supplement the unchanged owner's typed-cell identity with pandas axis + # descriptors/attrs and every entity's resolved original DESIGN weights. + details = {"storage": housing.frame_content_sha256(frame), "entities": {}} + for entity in frame.entities: + table = frame.table(entity) + weights = frame.resolve_weights(entity) + details["entities"][entity] = { + "index_type": type(table.index).__name__, + "index_dtype": repr(table.index.dtype), + "column_axis_name": table.columns.name, + "column_axis_dtype": repr(table.columns.dtype), + "dtypes": [repr(dtype) for dtype in table.dtypes], + "attrs": table.attrs, + "allows_duplicate_labels": table.flags.allows_duplicate_labels, + "weight_kind": weights.kind.value, + "weight_dtype": str(weights.values.dtype), + "weight_shape": list(weights.values.shape), + "weight_sha256": coverage._sha(weights.values.tobytes()), + } + details["strata_attrs"] = frame.strata.attrs + details["strata_index_type"] = type(frame.strata.index).__name__ + details["strata_allows_duplicate_labels"] = ( + frame.strata.flags.allows_duplicate_labels + ) + _require(not frame._link_tables, "UNEXPECTED_LINK_TABLES") + return coverage._sha(coverage._json(details, MAX_EVIDENCE_BYTES)) + + +def _verify_prepared_frame(prepared): + recorded = json.loads(prepared.receipt_json)["frame_sha256"] + _require( + housing.frame_content_sha256(prepared.frame) == recorded, + "PREPARATION_FRAME_CHANGED", + ) + return recorded + + +def _verify_frame(owned): + receipt = json.loads(owned.payload) + _require( + housing.frame_content_sha256(owned.frame) == receipt["frame_sha256"] + and _frame_sha256(owned.frame) == receipt["exact_frame_sha256"], + "FRAME_CHANGED", + ) + _require( + _verify_prepared_frame(owned.prepared) == receipt["frame_sha256"], + "PREPARATION_FRAME_CHANGED", + ) + + +def _verify(owned, producer): + receipt = json.loads(owned.payload) + frame, prepared, literal = owned.frame, owned.prepared, owned.literal + _require(receipt["protocol"] == PROTOCOL, "NATIVE_PROTOCOL") + _require(receipt["producer"] == producer, "PRODUCER_CHANGED") + _require( + prepared.frame is frame + and coverage._sha(prepared.receipt_json) + == receipt["preparation_receipt_sha256"] + and coverage._sha(prepared.source.receipt_json) + == receipt["housing_receipt_sha256"] + and coverage._sha(prepared.source.projection_json) + == receipt["projection_sha256"] + and coverage._sha(literal.payload) == receipt["coverage_payload_sha256"], + "EVIDENCE_CHANGED", + ) + _verify_frame(owned) + housing.verify_acs_frame_projection( + frame, json.loads(prepared.source.projection_json) + ) + consistency = coverage.verify_acs_coverage_native_consistency( + literal, frame + ).receipt + _require( + consistency["original_age"]["relation"] == "numeric_identity" + and consistency["original_age"]["matching_rows"] == frame.n("person"), + "ORIGINAL_AGE_IDENTITY", + ) + _verify_sources(owned.snapshots, owned.pins) + _verify_frame(owned) + + +def verify_acs_native_coverage(issuance, frame=None): + """Authenticate this issuance and exact live Frame, sources and producer. + + A separately built equal Frame is not this issuance. Any borrowed mutable + Frame must be verified again at consumption; retaining a borrow bypasses no + checks here but Python cannot intercept arbitrary downstream table reads. + Callers must hold exclusive access to mutable tables during verification + and consumption. Final checks detect changes during archive/producer work; + they do not make pandas tables an atomic concurrent snapshot. + """ + try: + owned = _owned(issuance) + _require(frame is None or frame is owned.frame, "FRAME_NOT_ISSUED") + producer = _producer() + _verify(owned, producer) + _require(_producer() == producer, "PRODUCER_CHANGED") + _verify_frame(owned) + return issuance + except ACSNativeCoverageBindingError: + raise + except Exception: + raise ACSNativeCoverageBindingError("NATIVE_VERIFICATION_REFUSED") from None + + +def issue_acs_native_coverage( + source_dir, *, snapshot_root, serialnos=None, candidate_path=None +): + """Run real closed owners with optional exact engineering household selection. + + The complete selected roster is bounded before native construction. Full + source validation and lexical projection remain. Candidate bytes, if given, + are compared only after independent real reconstruction and never decoded. + """ + try: + serialnos = housing.AcsPumsSource.snapshot_serialnos(serialnos) + # Snapshot path-like inputs once too, before producer/capture work. + source_dir = Path(source_dir).absolute() + snapshot_root = Path(snapshot_root).absolute() + candidate_path = ( + None if candidate_path is None else Path(candidate_path).absolute() + ) + coverage._json( + serialnos, min(MAX_EVIDENCE_BYTES, housing.ACS_HU_RECEIPT_MAX_BYTES) + ) + pins = housing._pins() + archives = _archives(pins) + _require(sum(p[3] for p in pins) <= MAX_ARCHIVE_BYTES, "ARCHIVE_BUDGET") + producer = _producer() + with housing._capture(source_dir, snapshot_root) as (private, paths, captured): + _require(captured == pins, "SOURCE_AUTHORITY_CHANGED") + inventory, selected_roster = _preflight(paths, serialnos) + # These roots are outside the archive-only captured source directory. + roots = [ + Path(tempfile.mkdtemp(prefix=prefix, dir=snapshot_root)) + for prefix in ("acs-native-preparation-", "acs-native-coverage-") + ] + prepared = housing.prepare_acs_housing_population( + private, snapshot_root=roots[0], serialnos=serialnos + ) + _verify_prepared_frame(prepared) + _require( + prepared.receipt["format"] == "microcosm.acs_housing_preparation.v2", + "PREPARATION_PROTOCOL", + ) + _require( + prepared.receipt["requested_serialnos"] + == (None if serialnos is None else list(serialnos)), + "PREPARATION_SELECTION", + ) + keys, _roster_sha = coverage._native_roster(prepared.frame) + if selected_roster is not None: + households, roster = selected_roster + _require( + {s: n for s, n in households.items() if n > 0} + == keys.groupby("SERIALNO").size().to_dict() + and set(roster) + == set(zip(keys.SERIALNO, keys.SPORDER, strict=True)), + "SELECTED_NATIVE_ROSTER", + ) + literal = coverage.load_authenticated_acs_person_coverage( + private, snapshot_root=roots[1], frame=prepared.frame + ) + hreceipt, creceipt = ( + json.loads(prepared.source.receipt_json), + literal.receipt, + ) + _require( + hreceipt["archives"] == creceipt["archives"] == archives, + "SOURCE_ARCHIVE_IDENTITY", + ) + _members_equal(hreceipt["members"], creceipt["members"]) + _require(creceipt["members"] == inventory, "SOURCE_MEMBER_IDENTITY") + relation = literal.native_binding.receipt["original_age"] + _require( + relation["relation"] == "numeric_identity", "ORIGINAL_AGE_IDENTITY" + ) + snapshots = [ + { + role: Path(source_dir).absolute() / name + for role, name, _d, _s in pins + }, + paths, + ] + for root in roots: + children = tuple(root.iterdir()) + _require(len(children) == 1 and children[0].is_dir(), "CAPTURE_ROSTER") + snapshots.append( + {role: children[0] / name for role, name, _d, _s in pins} + ) + receipt = { + "protocol": PROTOCOL, + "producer": producer, + "archives": archives, + "vintage": 2024, + "selection": { + "kind": "all" if serialnos is None else "engineering_exact_keys", + "requested_serialnos": serialnos, + "complete_selected_roster": True, + "person_rows": prepared.frame.n("person"), + "raw_person_keys_sha256": coverage._sha( + coverage._json( + sorted( + zip(keys.SERIALNO, map(int, keys.SPORDER), strict=True) + ), + coverage.MAX_BODY_BYTES, + ) + ), + "vacant_serialnos": None + if selected_roster is None + else sorted(s for s, n in selected_roster[0].items() if n == 0), + "vacancy_status": "source_record_without_population_rows", + "before_person_accumulation_and_unit_assignment": serialnos + is not None, + "full_source_inclusion_probability": None, + "representative_sample": False, + }, + "original_literal_authentication": { + "source_authenticated": True, + "fields": ["AGEP", "MIL", "ESR"], + "raw_literals_preserved": True, + "predecessor_native_binding_authenticated": False, + }, + "native_preparation_issuance": { + "population_binding_authenticated": True, + "producer_executed_by_issuer": True, + "complete_original_nonvacant_roster": serialnos is None, + "complete_selected_original_roster": True, + "source_aliases_and_vintage_checked": True, + "age_relationship": "literal_AGEP_to_native_AGEP_A_AGE_to_model_age_identity", + "matching_age_rows": relation["matching_rows"], + "native_roster_sha256": creceipt["native_consistency"][ + "native_roster_sha256" + ], + }, + "design_anchors": { + "authenticated": True, + "kind": "design", + "source": "WGTP; single-person PWGTP for WGTP=0 GQ placeholders", + "renormalized": False, + "original_weight_literals_projection_sha256": coverage._sha( + prepared.source.projection_json + ), + "weight_sha256": coverage._sha( + prepared.frame.weights_for("household").values.tobytes() + ), + }, + "assumptions": { + "coverage_status": "literal_source_fields_only", + "cross_survey_coverage_equivalence_established": False, + "domain_assignment_authenticated": False, + "period_harmonized": False, + "release_eligible": False, + "graph_attachment": "requires_reviewed_typed_successor", + }, + "capture": { + "owners_share_physical_snapshot": False, + "owners_separately_capture_same_pins": True, + "issuer_preflight_capture_count": 1, + "owner_capture_count": 2, + "full_source_construction_peak_remains": True, + "selected_only_memory": False, + }, + "frame_sha256": housing.frame_content_sha256(prepared.frame), + "exact_frame_sha256": _frame_sha256(prepared.frame), + "preparation_receipt_sha256": coverage._sha(prepared.receipt_json), + "housing_receipt_sha256": coverage._sha(prepared.source.receipt_json), + "projection_sha256": coverage._sha(prepared.source.projection_json), + "coverage_payload_sha256": coverage._sha(literal.payload), + } + payload = coverage._json(receipt, MAX_EVIDENCE_BYTES) + owned = _Owned( + prepared.frame, payload, prepared, literal, tuple(snapshots), pins + ) + if candidate_path is not None: + target = private / "candidate.native-coverage" + digest = housing._copy( + Path(candidate_path), target, len(payload), exact_size=len(payload) + ) + _require( + digest == coverage._sha(payload) + and housing._persisted_sha(target, len(payload)) == digest, + "CANDIDATE_MISMATCH", + ) + _verify(owned, producer) + _require(_producer() == producer, "PRODUCER_CHANGED") + _verify_frame(owned) + # Capture's exit authority check must complete before minting the token. + _verify_frame(owned) + result = AuthenticatedACSNativeCoverage(payload, _token=_TOKEN) + _ISSUED[result] = owned + return result + except ACSNativeCoverageBindingError: + raise + except Exception: + raise ACSNativeCoverageBindingError("NATIVE_ISSUANCE_REFUSED") from None diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/acs_person_coverage_authentication.py b/packages/microcosm-build/src/microcosm/build/us_runtime/acs_person_coverage_authentication.py new file mode 100644 index 000000000..fe0346500 --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/acs_person_coverage_authentication.py @@ -0,0 +1,667 @@ +"""Closed ACS coverage source issuance; native consistency grants no identity. + +The existing prepared ACS container is publicly constructible. Until a closed +prepared/native authority exists, even a consistent Frame receives only an +UngrantedACSNativeBinding. Nothing here attaches columns or prepares a population. +""" + +from __future__ import annotations + +import _csv +import csv +import hashlib +import io +import json +import platform +import re +import stat +import sys +import sysconfig +import zipfile +from dataclasses import InitVar, dataclass +from pathlib import Path +from types import BuiltinFunctionType + +import numpy as np +import pandas as pd + +from microcosm.frame import Frame + +from . import acs_housing_universe_source as custody +from . import acs_person_coverage_columns as literal +from .acs_pums import AcsPumsSource +from .source_csv_builtin import csv_reader_bound + +PROTOCOL = "microcosm.acs-person-coverage-authentication.v1" +MAX_HEADER_BYTES = 1024**2 +MAX_BODY_BYTES = 64 * 1024**2 +MAX_RECORD_BYTES = 400_000 +MAX_CSV_HEADER_BYTES = 64 * 1024 +MAX_TOKEN_BYTES = 64 * 1024 +_MAGIC = b"ACS-COVERAGE/1\n" +_TOKEN = object() +_CSV_READER = _csv.reader +_COLUMNS = (*literal.READ_COLUMNS, "MIL_state", "ESR_state") +_IMPLEMENTATION_FILES = ( + "acs_person_coverage_authentication.py", + "acs_person_coverage_columns.py", + "acs_housing_universe_source.py", + "acs_sources.py", + "acs_pums.py", + "source_csv_builtin.py", + "acs_2024_1yr_sources.json", +) + + +class ACSCoverageAuthenticationError(ValueError): + """Static refusal code, without source paths, keys, tokens or cause chains.""" + + +def _require(condition, code): + if not condition: + raise ACSCoverageAuthenticationError(code) + + +def _sha(raw): + return hashlib.sha256(raw).hexdigest() + + +def _json(value, cap=MAX_HEADER_BYTES): + # ASCII JSON escapes preserve CR/LF/HT and distinguish null from "". Values + # originate in byte-bounded records or the fixed, bounded header inventory. + # Count exact escaped bytes before an encoder can allocate a whole token. + count = 0 + + def charge(size): + nonlocal count + count += size + _require(count <= cap, "CANONICAL_SIZE") + + def visit(item): + if isinstance(item, str): + charge(2) + for char in item: + code = ord(char) + charge( + 2 + if char in '\\"\b\f\n\r\t' + else 1 + if 32 <= code <= 126 + else 6 + if code <= 0xFFFF + else 12 + ) + elif item is None: + charge(4) + elif type(item) is bool: + charge(4 if item else 5) + elif type(item) is int: + charge(len(str(item))) + elif isinstance(item, (list, tuple)): + charge(2 + max(0, len(item) - 1)) + for part in item: + visit(part) + elif type(item) is dict: + charge(2 + max(0, len(item) - 1) + len(item)) + for key, part in item.items(): + _require(type(key) is str, "CANONICAL_KEY") + visit(key) + visit(part) + else: + _require(False, "CANONICAL_TYPE") + + visit(value) + parts, count = [], 0 + for part in json.JSONEncoder( + sort_keys=True, separators=(",", ":"), ensure_ascii=True, allow_nan=False + ).iterencode(value): + count += len(part) + _require(count <= cap, "CANONICAL_SIZE") + parts.append(part.encode("ascii")) + return b"".join(parts) + + +def _producer(): + # Provider bytes alone cannot attest the callable used by either parsing + # pass. Preserve the real C binding across issuance/verification, including + # refusal when both public aliases were replaced before this owner imported. + _require( + type(_CSV_READER) is BuiltinFunctionType + and _CSV_READER.__module__ == "_csv" + and _CSV_READER.__name__ == "reader" + and _CSV_READER.__self__ is _csv + and csv.reader is _csv.reader is _CSV_READER + and literal.csv is custody.csv is csv + and csv_reader_bound(csv), + "CSV_READER_CHANGED", + ) + parser_binary = getattr(_csv, "__file__", None) + if parser_binary is None: + # This runtime builds _csv into libpython. Other builds ship an + # extension or link the parser into the executable itself. + parser_binary = ( + Path(sysconfig.get_config_var("LIBDIR")) + / sysconfig.get_config_var("LDLIBRARY") + if sysconfig.get_config_var("Py_ENABLE_SHARED") + else Path(sys.executable) + ) + return { + "protocol": PROTOCOL, + "files": { + name: _sha(Path(__file__).with_name(name).read_bytes()) + for name in _IMPLEMENTATION_FILES + }, + "python": platform.python_version(), + "pandas": pd.__version__, + "numpy": np.__version__, + "csv_binary_sha256": _sha(Path(parser_binary).read_bytes()), + "stdlib": { + name: _sha(Path(module.__file__).read_bytes()) + for name, module in ( + ("csv", csv), + ("zipfile", zipfile), + ("json", json), + ) + }, + "parser": { + "reader": "_csv.reader:verified_builtin_identity", + "encoding": "utf-8-sig", + "newline": "", + "csv_field_size_limit": csv.field_size_limit(), + "literal_record_chars": literal.MAX_CSV_RECORD_CHARS, + "source_rows": literal.MAX_ROWS, + "selected_rows": literal.MAX_SELECTED_ROWS, + }, + "ceilings": { + "header": MAX_HEADER_BYTES, + "body": MAX_BODY_BYTES, + "record": MAX_RECORD_BYTES, + "csv_header": MAX_CSV_HEADER_BYTES, + "token": MAX_TOKEN_BYTES, + "member": custody._MEMBER_MAX, + "expanded_archive": custody._EXPANDED_MAX, + "member_count": custody._MEMBER_COUNT_MAX, + }, + } + + +def _records(stream): + """Fence raw logical records/tokens before UTF-8 decoding or CSV allocation. + + Quote parity only locates boundaries; the unchanged strict literal parser + subsequently owns CSV validity. CR, LF and CRLF are preserved, including + inside quotes. The conservative token ceiling includes raw CSV quoting. + """ + record = bytearray() + quoted, pending_cr, first, token_bytes = False, False, True, 0 + cap = MAX_CSV_HEADER_BYTES + while block := stream.read(4096): + for byte in block: + if pending_cr: + if byte == 10: + _require(len(record) < cap, "CSV_RECORD_BYTES") + record.append(byte) + yield bytes(record) + record.clear() + first, pending_cr, token_bytes = False, False, 0 + if byte == 10: + continue + cap = MAX_CSV_HEADER_BYTES if first else MAX_RECORD_BYTES + _require(len(record) < cap, "CSV_RECORD_BYTES") + if byte == 44 and not quoted: + token_bytes = 0 + else: + token_bytes += 1 + _require(token_bytes <= MAX_TOKEN_BYTES, "CSV_TOKEN_BYTES") + record.append(byte) + if byte == 34: + quoted = not quoted + if not quoted and byte in (10, 13): + if byte == 13: + pending_cr = True + else: + yield bytes(record) + record.clear() + first, token_bytes = False, 0 + if record: + yield bytes(record) + + +def _decode_record(raw, *, first): + # BOM is only consumed at member start, exactly as in the literal reader. + with io.TextIOWrapper( + io.BytesIO(raw), encoding="utf-8-sig" if first else "utf-8", newline="" + ) as text: + reader = literal._literal_csv_records(text) + result = next(reader, None) + _require(result is not None and next(reader, None) is None, "CSV_RECORD") + return result + + +def _members(archive, role): + # These are the closed housing owner's complete central-directory checks, + # kept local because _archive also invokes its physical-record parser. + # Editing/extracting that owner would change historical custody producers. + prefix = "psam_hus" if role == "household" else "psam_pus" + members = archive.infolist() + _require(0 < len(members) <= custody._MEMBER_COUNT_MAX, "ZIP_MEMBER_COUNT") + names = [m.filename for m in members] + _require(len({n.casefold() for n in names}) == len(names), "ZIP_DUPLICATE_MEMBER") + expanded = 0 + for member in members: + name, mode = member.filename, member.external_attr >> 16 + _require( + name not in ("", ".", "..") + and not any(c in name for c in ("/", "\\", "\x00")), + "ZIP_MEMBER_PATH", + ) + _require( + not member.is_dir() and stat.S_IFMT(mode) in (0, stat.S_IFREG), + "ZIP_MEMBER_TYPE", + ) + _require( + not member.flag_bits & 1 + and member.compress_type in (zipfile.ZIP_STORED, zipfile.ZIP_DEFLATED), + "ZIP_MEMBER_ENCODING", + ) + _require(0 <= member.file_size <= custody._MEMBER_MAX, "ZIP_MEMBER_SIZE") + expanded += member.file_size + _require(expanded <= custody._EXPANDED_MAX, "ZIP_EXPANDED_SIZE") + _require( + not name.casefold().startswith(prefix) or name.casefold().endswith(".csv"), + "ZIP_PREFIX_NONCSV", + ) + _require(any(n.casefold().startswith(prefix) for n in names), "ZIP_ROLE_MISSING") + return sorted(members, key=lambda m: m.filename), prefix + + +def _inventory(path, role, serialnos): + inventory, selected, rows, selected_budget = [], {}, 0, 0 + required = ("SERIALNO", "NP") if role == "household" else literal.READ_COLUMNS + with zipfile.ZipFile(path) as archive: + members, prefix = _members(archive, role) + for member in members: + applicable = member.filename.casefold().startswith(prefix) + digest, count, row_count, header, header_sha = ( + hashlib.sha256(), + 0, + 0, + None, + None, + ) + with archive.open(member) as stream: + if not applicable: + while block := stream.read(min(4096, member.file_size - count + 1)): + count += len(block) + _require(count <= member.file_size, "ZIP_MEMBER_GREW") + digest.update(block) + else: + for raw in _records(stream): + count += len(raw) + _require(count <= member.file_size, "ZIP_MEMBER_GREW") + digest.update(raw) + values = _decode_record(raw, first=header is None) + if header is None: + header, header_sha = values, _sha(raw) + _require( + all(header) + and len(set(header)) == len(header) + and set(required) <= set(header), + "CSV_HEADER", + ) + positions = [header.index(c) for c in required] + continue + _require(len(values) == len(header), "CSV_ROW_WIDTH") + rows += 1 + row_count += 1 + _require(rows <= literal.MAX_ROWS, "SOURCE_ROWS") + cells = [values[p] for p in positions] + if cells[0] not in serialnos: + continue + # Upper bound on escaped row + statuses + lineage before + # the low-level reader allocates its selected DataFrame. + selected_budget += 6 * len(raw) + 1024 + _require( + selected_budget <= MAX_BODY_BYTES, "SELECTED_BODY_BUDGET" + ) + if role == "household": + key = cells[0] + _require(re.fullmatch(r"[0-9]{1,2}", cells[1]), "SOURCE_NP") + value = int(cells[1]) + else: + _require( + re.fullmatch(r"[0-9]{1,2}", cells[1]), "SOURCE_SPORDER" + ) + key = (cells[0], int(cells[1])) + _require(1 <= key[1] <= 20, "SOURCE_SPORDER") + value = [member.filename, row_count] + _require(key not in selected, "SOURCE_DUPLICATE_KEY") + _require( + len(selected) < literal.MAX_SELECTED_ROWS, "SELECTED_ROWS" + ) + selected[key] = value + _require(count == member.file_size, "ZIP_MEMBER_SIZE_MISMATCH") + _require(not applicable or header is not None, "CSV_HEADER_MISSING") + inventory.append( + { + "name": member.filename, + "bytes": count, + "sha256": digest.hexdigest(), + "compressed_bytes": member.compress_size, + "crc32": member.CRC, + "applicable": applicable, + "header_sha256": header_sha, + "rows": row_count, + } + ) + return inventory, selected + + +def _native_roster(frame): + _require(type(frame) is Frame, "NATIVE_FRAME_REQUIRED") + p, h = frame.person, frame.table("household") + _require(0 < len(h) <= len(p) <= literal.MAX_SELECTED_ROWS, "NATIVE_ROWS") + _require(p.columns.is_unique and h.columns.is_unique, "NATIVE_COLUMNS") + _require( + { + "person_id", + "person_household_id", + "SPORDER", + "A_LINENO", + "source_year", + "source_household_id", + "source_person_id", + "source_row_id", + "AGEP", + "A_AGE", + "age", + } + <= set(p) + and {"household_id", "SERIALNO", "NP"} <= set(h), + "NATIVE_COLUMNS", + ) + for table, names in ( + ( + p, + ( + "person_id", + "person_household_id", + "SPORDER", + "A_LINENO", + "source_year", + "source_household_id", + "source_row_id", + ), + ), + (h, ("household_id", "NP")), + ): + for name in names: + _require( + pd.api.types.is_integer_dtype(table[name].dtype) + and not table[name].isna().any(), + "NATIVE_INTEGER", + ) + _require( + not p.person_id.duplicated().any() + and not h.household_id.duplicated().any() + and not p.source_row_id.duplicated().any() + and bool((p.source_row_id >= 0).all()), + "NATIVE_IDS", + ) + _require( + not h.SERIALNO.isna().any() and not h.SERIALNO.duplicated().any(), + "NATIVE_SERIALNO", + ) + _require(set(p.person_household_id) == set(h.household_id), "NATIVE_MEMBERSHIP") + household = h.set_index("household_id") + serials = p.person_household_id.map(household.SERIALNO) + _require(bool((p.source_year == 2024).all()), "NATIVE_VINTAGE") + _require( + np.array_equal(p.SPORDER, p.A_LINENO) + and np.array_equal(p.person_household_id, p.source_household_id) + and list(p.source_person_id) == [str(v) for v in p.SPORDER], + "NATIVE_ALIASES", + ) + keys = literal._keys( + pd.DataFrame({"SERIALNO": serials, "SPORDER": p.SPORDER}) + ).reset_index(drop=True) + counts = p.groupby("person_household_id", sort=False).size() + _require(np.array_equal(counts.loc[h.household_id], h.NP), "NATIVE_HOUSEHOLD_COUNT") + roster = ( + [int(pid), int(hid), serial, int(order), int(row)] + for pid, hid, serial, order, row in zip( + p.person_id, + p.person_household_id, + keys.SERIALNO, + keys.SPORDER, + p.source_row_id, + strict=True, + ) + ) + digest = hashlib.sha256() + for row in roster: + digest.update(_json(row, MAX_RECORD_BYTES - 1) + b"\n") + return keys, digest.hexdigest() + + +def _age_relation(frame, table): + matched, unresolved, mismatch = 0, 0, 0 + for raw, ag, aa, age in zip( + table.AGEP, frame.person.AGEP, frame.person.A_AGE, frame.person.age, strict=True + ): + if re.fullmatch(r"[0-9]{1,2}", raw) is None: + unresolved += 1 + elif all( + isinstance(v, (int, float, np.integer, np.floating)) + and not isinstance(v, (bool, np.bool_)) + and np.isfinite(v) + and v == int(raw) + for v in (ag, aa, age) + ): + matched += 1 + else: + mismatch += 1 + return { + "matching_rows": matched, + "unresolved_literal_rows": unresolved, + "mismatching_rows": mismatch, + "relation": "numeric_identity" + if not unresolved and not mismatch + else "unproven", + "preparation_provenance_authenticated": False, + } + + +@dataclass(frozen=True, slots=True) +class UngrantedACSNativeBinding: + """Owned consistency evidence, explicitly lacking prepared population proof.""" + + receipt_json: bytes + _token: InitVar[object] = None + + def __post_init__(self, _token): + _require(_token is _TOKEN, "BINDING_CONSTRUCTOR") + + @property + def receipt(self): + return json.loads(self.receipt_json) + + +@dataclass(frozen=True, slots=True) +class AuthenticatedACSPersonCoverage: + """Immutable source envelope; candidate bytes cannot construct this type.""" + + payload: bytes + _token: InitVar[object] = None + + def __post_init__(self, _token): + _require(_token is _TOKEN, "SOURCE_CONSTRUCTOR") + + def _parts(self): + start = len(_MAGIC) + 4 + size = int.from_bytes(self.payload[len(_MAGIC) : start], "big") + return self.payload[start : start + size], self.payload[start + size :] + + @property + def receipt(self): + return json.loads(self._parts()[0]) + + @property + def table(self): + rows = [json.loads(row) for row in self._parts()[1].splitlines()] + result = pd.DataFrame([r[: len(_COLUMNS)] for r in rows], columns=_COLUMNS) + for column in _COLUMNS: + result[column] = result[column].astype( + "int64" if column == "SPORDER" else "string" + ) + return result + + @property + def native_binding(self): + return UngrantedACSNativeBinding( + _json( + { + **self.receipt["native_consistency"], + "source_payload_sha256": _sha(self.payload), + "population_binding_authenticated": False, + "missing_authority": "closed_prepared_acs_native_population", + }, + MAX_HEADER_BYTES, + ), + _token=_TOKEN, + ) + + +def verify_acs_coverage_native_consistency(coverage, frame): + """Check the exact live parent; never upgrade consistency to population proof.""" + try: + _require(type(coverage) is AuthenticatedACSPersonCoverage, "SOURCE_TYPE") + _require(coverage.receipt["producer"] == _producer(), "PRODUCER_CHANGED") + _keys, roster = _native_roster(frame) + evidence = coverage.native_binding.receipt + _require( + roster == evidence["native_roster_sha256"] + and custody.frame_content_sha256(frame) == evidence["frame_sha256"], + "NATIVE_FRAME_CHANGED", + ) + _require( + _age_relation(frame, coverage.table) == evidence["original_age"], + "NATIVE_AGE_CHANGED", + ) + return coverage.native_binding + except ACSCoverageAuthenticationError: + raise + except Exception: + raise ACSCoverageAuthenticationError("NATIVE_CONSISTENCY_REFUSED") from None + + +def load_authenticated_acs_person_coverage( + source_dir, *, snapshot_root, frame, candidate_path=None +): + """Reconstruct from closed default archive pins, then compare optional bytes. + + ``frame`` supplies a checked selection, never source or population authority. + No manifest/pin/preparation-receipt argument and no publishing API exists. + Successful and failed private captures retain the housing owner's policy. + """ + try: + _require(type(frame) is Frame, "NATIVE_FRAME_REQUIRED") + before = custody.frame_content_sha256(frame) + keys, roster_sha = _native_roster(frame) + producer = _producer() + with custody._capture(source_dir, snapshot_root) as (private, paths, pins): + _require( + {r for r, _n, _d, _s in pins} == {"household", "person"} + and len(pins) == 2, + "SOURCE_ROLES", + ) + members = {} + members["household"], households = _inventory( + paths["household"], "household", set(keys.SERIALNO) + ) + members["person"], lineage = _inventory( + paths["person"], "person", set(keys.SERIALNO) + ) + _require( + households == keys.groupby("SERIALNO").size().to_dict(), + "SOURCE_HOUSEHOLD_ROSTER", + ) + _require( + set(lineage) == set(zip(keys.SERIALNO, keys.SPORDER, strict=True)), + "SOURCE_PERSON_ROSTER", + ) + table, original_receipt = literal.read_acs_person_coverage_columns( + AcsPumsSource(paths["household"], paths["person"], vintage=2024), + person_keys=keys, + chunksize=min(1000, len(keys)), + ) + body, size = [], 0 + for row in table.itertuples(index=False, name=None): + raw = ( + _json([*row, *lineage[(row[0], row[1])]], MAX_RECORD_BYTES - 1) + + b"\n" + ) + size += len(raw) + _require(size <= MAX_BODY_BYTES, "BODY_SIZE") + body.append(raw) + body = b"".join(body) + header = { + "protocol": PROTOCOL, + "encoding": "ascii-escaped-json-header-and-ndjson-v1", + "producer": producer, + "source_authenticated": True, + "vintage": 2024, + "population_binding_authenticated": False, + "coverage_status": "literal_source_fields_only", + "cross_survey_coverage_equivalence_established": False, + "domain_assignment_authenticated": False, + "period_harmonized": False, + "release_eligible": False, + "archives": [ + {"role": r, "filename": n, "sha256": d, "bytes": s} + for r, n, d, s in pins + ], + "members": members, + "field_contract": literal.coverage_field_contract(), + "original_literal_receipt": original_receipt, + "columns": [*_COLUMNS, "source_member", "source_row_ordinal"], + "body_bytes": len(body), + "body_sha256": _sha(body), + "rows": len(table), + "native_consistency": { + "frame_sha256": before, + "native_roster_sha256": roster_sha, + "complete_households_checked": True, + "source_aliases_checked": True, + "original_age": _age_relation(frame, table), + }, + } + raw_header = _json(header, MAX_HEADER_BYTES) + payload = _MAGIC + len(raw_header).to_bytes(4, "big") + raw_header + body + # No candidate header, digest, DataFrame or claimed producer is parsed. + if candidate_path is not None: + target = private / "candidate.coverage" + actual = custody._copy( + Path(candidate_path), target, len(payload), exact_size=len(payload) + ) + _require( + actual == _sha(payload) + and custody._persisted_sha(target, len(payload)) == actual, + "CANDIDATE_MISMATCH", + ) + for role, _name, digest, size in pins: + _require( + custody._persisted_sha(paths[role], size) == digest, + "SNAPSHOT_CHANGED", + ) + _require(_producer() == producer, "PRODUCER_CHANGED") + _require(_native_roster(frame)[1] == roster_sha, "NATIVE_FRAME_CHANGED") + _require( + custody.frame_content_sha256(frame) == before, "NATIVE_FRAME_CHANGED" + ) + # _capture's authority-stability check must complete before issuance. + return AuthenticatedACSPersonCoverage(payload, _token=_TOKEN) + except ACSCoverageAuthenticationError: + raise + except Exception: + raise ACSCoverageAuthenticationError("SOURCE_RECONSTRUCTION_REFUSED") from None diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/acs_person_coverage_columns.py b/packages/microcosm-build/src/microcosm/build/us_runtime/acs_person_coverage_columns.py new file mode 100644 index 000000000..53b692dad --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/acs_person_coverage_columns.py @@ -0,0 +1,295 @@ +"""Additive, literal ACS person coverage fields for an exact native roster. + +This reader preserves MIL and ESR separately, including their different age +universes. It performs no population-domain allocation and authenticates no +source file. A source owner must bind the archive and requested native keys +before these observations may support a genuine graph. The existing ACS +loader, its columns and its implementation identity are unchanged. +""" + +from __future__ import annotations + +import csv +import hashlib +import io +import json +import re +from pathlib import Path +from zipfile import ZipFile + +import pandas as pd + +from .acs_pums import AcsPumsSource +from .source_csv_builtin import capture_csv_reader + +PROTOCOL = "microcosm.acs-person-coverage-columns.v1" +DICTIONARY_URL = ( + "https://www2.census.gov/programs-surveys/acs/tech_docs/pums/data_dict/" + "PUMS_Data_Dictionary_2024.pdf" +) +DICTIONARY_SHA256 = "929c2752995b0af1c16d5c64de8cdc43b4aa7d388ee2d45b4b4df90fecce1dff" +KEYS = ("SERIALNO", "SPORDER") +READ_COLUMNS = (*KEYS, "AGEP", "MIL", "ESR") +MAX_ROWS = 6_000_000 +MAX_SELECTED_ROWS = 1_000_000 +MAX_CSV_RECORD_CHARS = 100_000 +_NON_CSV_CONTROLS = re.compile(r"[\x00-\x08\x0b\x0c\x0e-\x1f\x7f-\x9f]") + + +def coverage_field_contract() -> dict: + """A fresh source-definition document, suitable for provenance consumers.""" + return { + "protocol": PROTOCOL, + "survey": "ACS", + "survey_year": 2024, + "record_grain": "person", + "read_columns": list(READ_COLUMNS), + "dictionary": {"url": DICTIONARY_URL, "sha256": DICTIONARY_SHA256}, + "fields": { + "MIL": { + "pdf_page": 41, + "minimum_age": 17, + "blank_meaning": "outside age universe", + "codes": { + "1": "active_duty", + "2": "past_active_duty", + "3": "training_only", + "4": "never_served", + }, + }, + "ESR": { + "pdf_page": 56, + "minimum_age": 16, + "blank_meaning": "outside age universe", + "codes": { + "1": "civilian_working", + "2": "civilian_job_absent", + "3": "unemployed", + "4": "armed_forces_working", + "5": "armed_forces_job_absent", + "6": "outside_labor_force", + }, + }, + }, + "cross_survey_residence_equivalence_established": False, + } + + +def _json(value) -> bytes: + return json.dumps( + value, sort_keys=True, separators=(",", ":"), allow_nan=False + ).encode() + + +def _keys(table: pd.DataFrame) -> pd.DataFrame: + """Canonical native identities; no row-position join or key imputation.""" + if ( + not isinstance(table, pd.DataFrame) + or not table.columns.is_unique + or not set(KEYS).issubset(table) + ): + raise ValueError("ACS coverage requires native person keys") + result = table.loc[:, list(KEYS)].copy() + if result.isna().any().any(): + raise ValueError("ACS coverage native keys are missing") + serial = result.SERIALNO + if not serial.map( + lambda x: ( + isinstance(x, str) and re.fullmatch(r"2024(?:HU|GQ)[0-9]{7}", x) is not None + ) + ).all(): + raise ValueError("ACS coverage native serial is not a 2024 HU/GQ identity") + order = result.SPORDER.astype("string") + if not order.str.fullmatch(r"[0-9]{1,2}").all(): + raise ValueError("ACS coverage native person order is not an integer") + result["SPORDER"] = order.astype("int64") + if not result.SPORDER.between(1, 20).all(): + raise ValueError("ACS coverage native person order is outside 1..20") + if result.duplicated(list(KEYS)).any(): + raise ValueError("ACS coverage native person keys repeat") + return result + + +def _field_state(age: str, raw: str, *, minimum_age: int, codes: dict) -> str: + if not re.fullmatch(r"[0-9]{1,2}", age): + return "age_unresolved" + if int(age) < minimum_age: + return "outside_age_universe" if raw == "" else "value_below_age_universe" + if raw == "": + return "missing_in_universe" + return "observed_code" if raw in codes else "unlabelled_code" + + +def _literal_csv_records(stream): + """Check controls before CSV parsing, bounding each complete logical record.""" + csv_reader = capture_csv_reader(csv) + if csv_reader is None: + raise ValueError("SOURCE_CSV_READER_CHANGED") + record_chars = 0 + + def lines(): + nonlocal record_chars + while line := stream.readline(MAX_CSV_RECORD_CHARS - record_chars + 1): + record_chars += len(line) + if record_chars > MAX_CSV_RECORD_CHARS: + raise ValueError("ACS coverage CSV record character bound exceeded") + # HT is literal whitespace; CR/LF delimit records or remain literal + # inside quotes. newline="" prevents universal-newline rewriting. + if _NON_CSV_CONTROLS.search(line): + raise ValueError("ACS coverage CSV contains a forbidden control") + yield line + + reader = csv_reader(lines(), strict=True) + try: + while True: + record_chars = 0 + record = next(reader, None) + if record is None: + return + yield record + except (csv.Error, UnicodeError) as exc: + raise ValueError("ACS coverage invalid literal CSV") from exc + + +def _scan_acs_person_coverage(source, consume_row): + """Exhaust the literal source once before returning its member/row counts. + + The private consumer receives only exact READ_COLUMNS strings. It cannot + signal successful early completion: all members, records and widths are + checked unless an exception refuses the scan. Source custody and requested + roster checks remain the caller's responsibility. + """ + if type(source) is not AcsPumsSource or source.vintage != 2024: + raise ValueError("ACS coverage projection requires the 2024 source type") + rows = 0 + with ZipFile(source.person_zip) as archive: + members = sorted( + name + for name in archive.namelist() + if Path(name).name.lower().startswith("psam_pus") + and name.lower().endswith(".csv") + ) + if not members or len(members) != len(set(members)): + raise ValueError( + "ACS coverage person member roster is missing or duplicated" + ) + for name in members: + with ( + archive.open(name) as member, + io.TextIOWrapper(member, encoding="utf-8-sig", newline="") as stream, + ): + reader = _literal_csv_records(stream) + header = next(reader, []) + if len(header) != len(set(header)) or not set(READ_COLUMNS).issubset( + header + ): + raise ValueError( + "ACS coverage source columns are missing or duplicated" + ) + positions = [header.index(column) for column in READ_COLUMNS] + for record in reader: + rows += 1 + if rows > MAX_ROWS: + raise ValueError("ACS coverage source row bound exceeded") + if len(record) != len(header): + raise ValueError( + "ACS coverage CSV record width differs from header" + ) + consume_row(tuple(record[position] for position in positions)) + return members, rows + + +def read_acs_person_coverage_columns( + source: AcsPumsSource, + *, + person_keys: pd.DataFrame, + chunksize: int = 100_000, +) -> tuple[pd.DataFrame, dict]: + """Stream an additive person projection and align exact selected households. + + ``person_keys`` must include every person in each requested source + household. The read refuses missing, duplicate or extra native person keys + in those households, including differences discovered in later members. + Other households are streamed past. The separate max_households option on + the legacy source is irrelevant here: the exact native roster is authority. + + AGEP, MIL and ESR remain literal strings, including empty Census cells. + Each member is parsed once, with full record widths checked before column + or household selection. Non-CSV C0/C1 controls (including NUL) refuse; + tabs and quoted CR/LF remain literal. Records, including their physical + line endings, are bounded to MAX_CSV_RECORD_CHARS decoded characters. + State columns distinguish printed codes, out-of-universe blanks, missing + in-universe values, unexpected codes and age conflicts. They are evidence + states, not a combined military predicate or survey membership decision. + + Only invented archives have been executed in development. Genuine use + requires source authentication and a separately bounded source-read plan. + """ + if type(source) is not AcsPumsSource or source.vintage != 2024: + raise ValueError("ACS coverage projection requires the 2024 source type") + if type(chunksize) is not int or not 0 < chunksize <= 100_000: + raise ValueError("ACS coverage chunksize must be in 1..100000") + expected = _keys(person_keys).reset_index(drop=True) + if not 0 < len(expected) <= MAX_SELECTED_ROWS: + raise ValueError("ACS coverage selected person count is outside the bound") + retained = frozenset(expected.SERIALNO) + pieces, batch, selected_rows = [], [], 0 + + def consume_row(cells): + nonlocal batch, selected_rows + if cells[0] not in retained: + return + selected_rows += 1 + if selected_rows > len(expected): + raise ValueError("ACS coverage source has extra selected-household people") + batch.append(cells) + if len(batch) == chunksize: + pieces.append(pd.DataFrame(batch, columns=READ_COLUMNS, dtype="string")) + batch = [] + + members, rows = _scan_acs_person_coverage(source, consume_row) + if batch: + pieces.append(pd.DataFrame(batch, columns=READ_COLUMNS, dtype="string")) + if not pieces: + raise ValueError("ACS coverage requested persons are missing") + observed = pd.concat(pieces, ignore_index=True) + observed_keys = _keys(observed) + observed["SPORDER"] = observed_keys.SPORDER + wanted = pd.MultiIndex.from_frame(expected) + actual = pd.MultiIndex.from_frame(observed_keys) + if set(wanted) != set(actual): + raise ValueError("ACS coverage source and requested native roster differ") + result = observed.set_index(list(KEYS)).loc[wanted].reset_index() + contract = coverage_field_contract() + for field, definition in contract["fields"].items(): + result[field + "_state"] = pd.Series( + [ + _field_state( + age, + raw, + minimum_age=definition["minimum_age"], + codes=definition["codes"], + ) + for age, raw in zip(result.AGEP, result[field], strict=True) + ], + dtype="string", + ) + receipt = { + "protocol": PROTOCOL, + "contract_sha256": hashlib.sha256(_json(contract)).hexdigest(), + "source_authenticated": False, + "coverage_status": "literal_source_fields_only", + "release_eligible": False, + "member_names": members, + "source_rows_streamed": rows, + "selected_person_rows": len(result), + "native_roster_sha256": hashlib.sha256( + _json(expected.to_dict("records")) + ).hexdigest(), + "projection_sha256": hashlib.sha256( + _json(result.to_dict("records")) + ).hexdigest(), + "legacy_max_households_applied": False, + "read_columns": list(READ_COLUMNS), + } + return result, receipt diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/acs_population_catalogue.py b/packages/microcosm-build/src/microcosm/build/us_runtime/acs_population_catalogue.py new file mode 100644 index 000000000..44d97c96f --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/acs_population_catalogue.py @@ -0,0 +1,579 @@ +"""Closed raw ACS catalogue before native population or unit construction. + +The full housing lexical projection and raw catalogue remain in memory. Literal +persons are scanned once and records are assembled in whole-household batches; +this does not make national preparation memory-bounded. No domain is classified +here. +""" + +from __future__ import annotations + +import hashlib +import json +import sys +import weakref +from dataclasses import InitVar, dataclass +from pathlib import Path +from types import FunctionType + +from . import acs_housing_universe_source as housing +from . import acs_native_coverage_binding as native +from . import acs_person_coverage_columns as literal +from . import survey_population_domains as domains + +PROTOCOL = "microcosm.acs-source-catalogue.v1" +_ARCHIVE_BYTES = 8 * 1024**3 +_EXPANDED_BYTES = 16 * 1024**3 +_MEMBERS = 64 +_SOURCE_ROWS = 6_000_000 +_BATCH_PEOPLE = 100_000 +_CATALOGUE_BYTES = 2 * 1024**3 +_RECEIPT_BYTES = 1024**2 +_TOKEN = object() +_ISSUED = {} + + +class ACSSourceCatalogueError(ValueError): + """Static refusal, without source identities, paths or cause chains.""" + + +def _require(condition, code): + if not condition: + raise ACSSourceCatalogueError(code) + + +def _local_bytes(): + return { + module.__name__: housing._sha(Path(module.__file__).read_bytes()) + for module in (sys.modules[__name__], domains) + } + + +def _local_runtime(): + """Capture exact loaded code, including generated raw dataclass methods.""" + result = {} + for module in (sys.modules[__name__], domains): + for name, value in vars(module).items(): + if isinstance(value, FunctionType): + result[(module.__name__, name)] = value.__code__ + elif isinstance(value, type) and value.__module__ == module.__name__: + result[(module.__name__, name)] = value + for method, function in vars(value).items(): + if isinstance(function, (staticmethod, classmethod)): + function = function.__func__ + if isinstance(function, property): + function = function.fget + if isinstance(function, FunctionType): + result[(module.__name__, name, method)] = function.__code__ + return result + + +def _immutable_pins(pins): + _require( + type(pins) is tuple + and len(pins) == 2 + and all( + type(pin) is tuple + and len(pin) == 4 + and all(type(value) is str for value in pin[:3]) + and type(pin[3]) is int + for pin in pins + ), + "SOURCE_AUTHORITY_TYPE", + ) + return pins + + +def _final_seals(pins, pin_authority): + # Pure comparisons after the last producer/source I/O, including capture + # exit. Another read would reopen the same interval this seal closes. + # _pins() loads the code manifest when the private override is None. Bind + # that exact authority mode and any override here without reopening I/O. + current = housing._ARCHIVE_PINS + if current is not None: + _immutable_pins(current) + _require( + current == pin_authority and (current is None or current == pins), + "SOURCE_AUTHORITY_CHANGED", + ) + _require(_local_runtime() == _RUNTIME_AUTHORITY, "PRODUCER_CODE_CHANGED") + + +def _producer(): + _require(PROTOCOL == "microcosm.acs-source-catalogue.v1", "PROTOCOL_CHANGED") + _require( + literal.KEYS == ("SERIALNO", "SPORDER") + and literal.READ_COLUMNS == ("SERIALNO", "SPORDER", "AGEP", "MIL", "ESR"), + "LITERAL_CONTRACT_CHANGED", + ) + limits = ( + _ARCHIVE_BYTES, + _EXPANDED_BYTES, + _MEMBERS, + _SOURCE_ROWS, + _BATCH_PEOPLE, + _CATALOGUE_BYTES, + _RECEIPT_BYTES, + ) + ceilings = (8 * 1024**3, 16 * 1024**3, 64, 6_000_000, 100_000, 2 * 1024**3, 1024**2) + _require( + all( + type(v) is int and 0 < v <= cap + for v, cap in zip(limits, ceilings, strict=True) + ), + "LIMITS", + ) + checked = native._producer() + compiled = {} + for module in (domains, sys.modules[__name__]): + native._live_code(module, compiled) + code = _local_bytes() + _require(code == _BYTE_AUTHORITY, "PRODUCER_CODE_CHANGED") + _require(_local_runtime() == _RUNTIME_AUTHORITY, "PRODUCER_CODE_CHANGED") + return { + "native_source_closure": checked, + "catalogue_sha256": code[__name__], + "raw_domain_types_sha256": code[domains.__name__], + "limits": list(limits), + } + + +def _preflight(paths): + """Keep full-source gates separate from native's 1M selected-row gate.""" + expanded = 0 + for role, path in paths.items(): + with native.coverage.zipfile.ZipFile(path) as archive: + members, _prefix = native.coverage._members(archive, role) + _require(len(members) <= _MEMBERS, "MEMBER_LIMIT") + expanded += sum(member.file_size for member in members) + _require(expanded <= _EXPANDED_BYTES, "EXPANDED_LIMIT") + inventories = {} + for role, path in paths.items(): + inventory, empty = native.coverage._inventory(path, role, frozenset()) + _require(not empty, "UNEXPECTED_SELECTED_ROWS") + _require( + sum(member["rows"] for member in inventory) <= _SOURCE_ROWS, + "SOURCE_ROW_LIMIT", + ) + inventories[role] = inventory + return inventories + + +def _source_checks(source_dir, paths, pins): + _require(housing._pins() == pins, "SOURCE_AUTHORITY_CHANGED") + for role, name, digest, size in pins: + for path in (source_dir / name, paths[role]): + housing._path_components(path) + _require(housing._persisted_sha(path, size) == digest, "SOURCE_CHANGED") + + +def _charge(record, digest, size): + raw = native.coverage._json(record, max(0, _CATALOGUE_BYTES - size - 1)) + b"\n" + size += len(raw) + _require(size <= _CATALOGUE_BYTES, "CATALOGUE_BYTE_LIMIT") + digest.update(raw) + return size + + +def _collect(projection, paths): + # Validate the actual persisted full projection, independently of the + # publicly constructible returned housing capsule's mutable attributes. + housing._tables(projection) + hcols = {name: i for i, name in enumerate(projection["household_columns"])} + pcols = {name: i for i, name in enumerate(projection["person_columns"])} + by_household = {} + positions = {} + people = [None] * len(projection["persons"]) + for index, row in enumerate(projection["persons"]): + serial, raw_order = row[pcols["SERIALNO"]], row[pcols["SPORDER"]] + key = (serial, int(raw_order)) + _require(key not in positions, "DUPLICATE_PERSON") + positions[key] = index + by_household.setdefault(serial, []).append(index) + household_keys, started_households = set(), set() + prospective_size = 0 + + def reserve(value, punctuation): + # An empty household's [] gains exactly the encoded person lengths and + # one comma between people. Charge before retaining literal fields. + nonlocal prospective_size + raw = native.coverage._json( + value, max(0, _CATALOGUE_BYTES - prospective_size - punctuation) + ) + prospective_size += len(raw) + punctuation + _require(prospective_size <= _CATALOGUE_BYTES, "CATALOGUE_BYTE_LIMIT") + + def household_record(row, members): + return ( + row[hcols["SERIALNO"]], + row[hcols["TYPEHUGQ"]], + row[hcols["NP"]], + row[hcols["WGTP"]], + row[hcols["source_member"]], + row[hcols["source_row_ordinal"]], + members, + ) + + for row in projection["households"]: + serial, count = row[hcols["SERIALNO"]], int(row[hcols["NP"]]) + _require(serial not in household_keys, "DUPLICATE_HOUSEHOLD") + household_keys.add(serial) + _require(count <= _BATCH_PEOPLE, "HOUSEHOLD_EXCEEDS_BATCH") + reserve(household_record(row, ()), 1) + _require(set(by_household) <= household_keys, "GLOBAL_PERSON_KEYS") + contract = literal.coverage_field_contract() + + def consume_row(cells): + serial, order, age, mil, esr = cells + _require( + all(type(v) is str for v in cells) + and literal.re.fullmatch(r"2024(?:HU|GQ)[0-9]{7}", serial) is not None + and literal.re.fullmatch(r"[0-9]{1,2}", order) is not None + and 1 <= int(order) <= 20, + "LITERAL_PERSON_KEY", + ) + key = (serial, int(order)) + _require(key in positions, "UNEXPECTED_LITERAL_PERSON") + index = positions[key] + _require(people[index] is None, "DUPLICATE_LITERAL_PERSON") + states = { + field: literal._field_state( + age, + raw, + minimum_age=contract["fields"][field]["minimum_age"], + codes=contract["fields"][field]["codes"], + ) + for field, raw in (("MIL", mil), ("ESR", esr)) + } + _require(all(type(v) is str for v in states.values()), "LITERAL_TYPES") + original = projection["persons"][index] + person = ( + original[pcols["SPORDER"]], + age, + esr, + states["ESR"], + mil, + states["MIL"], + original[pcols["PWGTP"]], + original[pcols["source_member"]], + original[pcols["source_row_ordinal"]], + ) + reserve(person, int(serial in started_households)) + people[index] = person + started_households.add(serial) + + if positions: + _members, rows = literal._scan_acs_person_coverage( + housing.AcsPumsSource(paths["household"], paths["person"], vintage=2024), + consume_row, + ) + _require(rows == len(positions), "GLOBAL_PERSON_KEYS") + _require(all(person is not None for person in people), "GLOBAL_PERSON_KEYS") + records, vacancies, batch = [], [], [] + digest, size, batches, batch_people = hashlib.sha256(), 0, 0, 0 + + def consume(): + nonlocal size, batches + count = sum(len(by_household.get(row[hcols["SERIALNO"]], ())) for row in batch) + if count: + # Preserve the prior selected-reader ceiling on each assembly + # batch, even though no selected DataFrame is allocated here. + _require( + count <= min(_BATCH_PEOPLE, literal.MAX_SELECTED_ROWS), + "BATCH_PERSON_LIMIT", + ) + batches += 1 + for row in batch: + serial = row[hcols["SERIALNO"]] + members = tuple(people[index] for index in by_household.get(serial, ())) + record = household_record(row, members) + _require(len(members) == int(record[2]), "HOUSEHOLD_COMPLETENESS") + size = _charge(record, digest, size) + ( + vacancies if int(record[1]) == 1 and int(record[2]) == 0 else records + ).append(record) + + for row in projection["households"]: + count = int(row[hcols["NP"]]) + if batch and batch_people + count > _BATCH_PEOPLE: + consume() + batch, batch_people = [], 0 + batch.append(row) + batch_people += count + if batch: + consume() + _require(size == prospective_size, "CATALOGUE_BYTE_ACCOUNTING") + return ( + tuple(records), + tuple(vacancies), + { + "households": len(household_keys), + "people": len(positions), + "occupied_hu": sum(int(r[1]) == 1 for r in records), + "institutional_gq": sum(int(r[1]) == 2 for r in records), + "noninstitutional_gq": sum(int(r[1]) == 3 for r in records), + "vacancies": len(vacancies), + "literal_batches": batches, + "canonical_record_bytes": size, + "canonical_record_sha256": digest.hexdigest(), + }, + ) + + +def _household(record): + serial, kind, count, weight, _member, _ordinal, people = record + key = domains.HouseholdKey(domains.Source.ACS, 2024, 2024, serial) + return domains.AcsHousehold( + key, + kind, + count, + weight, + tuple( + domains.AcsPerson(order, age, esr, esr_state, mil, mil_state, pwgtp, key) + for order, age, esr, esr_state, mil, mil_state, pwgtp, _pmember, _pordinal in people + ), + ) + + +@dataclass(frozen=True, slots=True) +class _Owned: + receipt: bytes + records: tuple + vacancies: tuple + source_dir: Path + paths: tuple + pins: tuple + pin_authority: tuple | None + producer: bytes + projection_path: Path + projection_bytes: int + projection_sha256: str + + +@dataclass(frozen=True, slots=True, weakref_slot=True, eq=False) +class AuthenticatedACSSourceCatalogue: + """Process-issued compact receipt; raw views are independent frozen values.""" + + payload: bytes + _token: InitVar[object] = None + + def __post_init__(self, _token): + _require(_token is _TOKEN, "ISSUANCE_CONSTRUCTOR") + + def validate(self): + return verify_acs_source_catalogue(self) + + def to_bytes(self): + return _checked(self).receipt + + @property + def receipt(self): + return json.loads(self.to_bytes()) + + @property + def households(self): + return tuple(_household(record) for record in _checked(self).records) + + @property + def exclusion_ledger(self): + return tuple(_household(record) for record in _checked(self).vacancies) + + @property + def lineage(self): + owned = _checked(self) + return { + r[0]: { + "household": (r[4], r[5]), + "persons": tuple((p[0], p[7], p[8]) for p in r[6]), + } + for group in (owned.records, owned.vacancies) + for r in group + } + + +def _lookup(value): + _require(type(value) is AuthenticatedACSSourceCatalogue, "ISSUANCE_TYPE") + entry = _ISSUED.get(id(value)) + _require(entry is not None and entry[0]() is value, "ISSUANCE_NOT_OWNED") + owned = entry[1] + _require( + type(value.payload) is bytes and value.payload == owned.receipt, + "ISSUANCE_CHANGED", + ) + return owned + + +def _checked(value): + try: + owned = _lookup(value) + before = native.coverage._json(_producer(), _RECEIPT_BYTES) + _require(before == owned.producer, "PRODUCER_CHANGED") + _source_checks(owned.source_dir, dict(owned.paths), owned.pins) + _require( + housing._persisted_sha(owned.projection_path, owned.projection_bytes) + == owned.projection_sha256, + "PROJECTION_CHANGED", + ) + _require( + native.coverage._json(_producer(), _RECEIPT_BYTES) == before, + "PRODUCER_CHANGED", + ) + _final_seals(owned.pins, owned.pin_authority) + _require(_lookup(value) is owned, "ISSUANCE_CHANGED") + return owned + except ACSSourceCatalogueError: + raise + except Exception: + raise ACSSourceCatalogueError("CATALOGUE_VERIFICATION_REFUSED") from None + + +def verify_acs_source_catalogue(value): + _checked(value) + return value + + +def issue_acs_source_catalogue(source_dir, *, snapshot_root, candidate=None): + """Execute closed real owners; candidate bytes never supply source authority.""" + try: + source_dir = Path(source_dir).absolute() + snapshot_root = Path(snapshot_root).absolute() + _require( + candidate is None + or (type(candidate) is bytes and len(candidate) <= _RECEIPT_BYTES), + "CANDIDATE_TYPE_SIZE", + ) + pin_authority = housing._ARCHIVE_PINS + if pin_authority is not None: + _immutable_pins(pin_authority) + pins = _immutable_pins(housing._pins()) + archives = native._archives(pins) + _require(sum(p[3] for p in pins) <= _ARCHIVE_BYTES, "ARCHIVE_LIMIT") + producer = _producer() + producer_bytes = native.coverage._json(producer, _RECEIPT_BYTES) + with housing._capture(source_dir, snapshot_root) as (private, paths, captured): + _require(captured == pins, "SOURCE_AUTHORITY_CHANGED") + inventories = _preflight(paths) + source = housing._reconstruct( + private, paths, pins, None, housing._implementation() + ) + source_receipt = json.loads(source.receipt_json) + native._members_equal(source_receipt["members"], inventories) + projection_path = private / "full-projection.json" + with projection_path.open("rb") as stream: + projection_raw = stream.read(housing.ACS_HU_SOURCE_MAX_BYTES + 1) + _require( + len(projection_raw) <= housing.ACS_HU_SOURCE_MAX_BYTES, + "PROJECTION_SIZE", + ) + projection_sha256 = housing._sha(projection_raw) + _require( + projection_sha256 == source_receipt["full_projection_sha256"], + "PROJECTION_CHANGED", + ) + projection_bytes = len(projection_raw) + projection = json.loads(projection_raw) + del projection_raw + records, vacancies, counts = _collect(projection, paths) + _require( + counts["households"] == sum(m["rows"] for m in inventories["household"]) + and counts["people"] == sum(m["rows"] for m in inventories["person"]), + "GLOBAL_SOURCE_COMPLETENESS", + ) + receipt = native.coverage._json( + { + "protocol": PROTOCOL, + "producer": producer, + "archives": archives, + "members": inventories, + "source_authenticated": True, + "population_binding_authenticated": False, + "release_eligible": False, + "domain_assignment_authenticated": False, + "selection_performed": False, + "source_year": 2024, + "survey_year": 2024, + "scope": "all_occupied_hu_and_both_gq_with_separate_vacancies", + "field_contract": literal.coverage_field_contract(), + "complete_original_membership": True, + "original_anchors_preserved": True, + "housing_projection_sha256": projection_sha256, + "counts": counts, + "canonical_record_schema": [ + "SERIALNO", + "TYPEHUGQ", + "NP", + "WGTP", + "household_member", + "household_row", + [ + "SPORDER", + "AGEP", + "ESR", + "ESR_state", + "MIL", + "MIL_state", + "PWGTP", + "person_member", + "person_row", + ], + ], + "canonical_record_order": "housing_source_member_then_row_ordinal", + "execution": { + "native_frame_constructed": False, + "unit_assignment_executed": False, + "full_housing_projection_retained": True, + "full_person_scan_per_literal_batch": False, + "literal_full_scans": int(bool(counts["people"])), + "national_efficiency_claimed": False, + }, + }, + _RECEIPT_BYTES, + ) + if candidate is not None: + _require(candidate == receipt, "CANDIDATE_MISMATCH") + _source_checks(source_dir, paths, pins) + _require( + housing._persisted_sha(projection_path, projection_bytes) + == projection_sha256, + "PROJECTION_CHANGED", + ) + owned = _Owned( + receipt, + records, + vacancies, + source_dir, + tuple(paths.items()), + pins, + pin_authority, + producer_bytes, + projection_path, + projection_bytes, + projection_sha256, + ) + # Capture exit can read the default source manifest. Recheck producer + # bytes/live code after that I/O, then finish with pure final seals. + _require( + native.coverage._json(_producer(), _RECEIPT_BYTES) == producer_bytes, + "PRODUCER_CHANGED", + ) + _final_seals(pins, pin_authority) + result = AuthenticatedACSSourceCatalogue(receipt, _token=_TOKEN) + identity = id(result) + + def cleanup(reference): + entry = _ISSUED.get(identity) + if entry is not None and entry[0] is reference: + del _ISSUED[identity] + + _ISSUED[identity] = (weakref.ref(result, cleanup), owned) + return result + except ACSSourceCatalogueError: + raise + except Exception: + raise ACSSourceCatalogueError("CATALOGUE_ISSUANCE_REFUSED") from None + + +# Authority begins at this import, rather than accepting replacement source and +# matching live code as a newly approved implementation on first issuance. +_BYTE_AUTHORITY = _local_bytes() +_RUNTIME_AUTHORITY = _local_runtime() diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/acs_pums.py b/packages/microcosm-build/src/microcosm/build/us_runtime/acs_pums.py index dd0456fca..f4431a860 100644 --- a/packages/microcosm-build/src/microcosm/build/us_runtime/acs_pums.py +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/acs_pums.py @@ -46,6 +46,8 @@ ACS_2024_1YR_SPINE = "acs_2024_1yr" ACS_2024_1YR_VINTAGE = 2024 DEFAULT_CHUNKSIZE = 100_000 +MAX_EXACT_HOUSEHOLDS = 1_000_000 +MAX_EXACT_PERSON_ROWS = 1_000_000 _HOUSEHOLD_REQUIRED = ( "SERIALNO", @@ -69,6 +71,14 @@ "TAXAMT", "TYPEHUGQ", ) +# 2024 ACS PUMS dictionary p.3: TYPEHUGQ 1 is a housing-unit record, 2 an +# institutional and 3 a noninstitutional group-quarters person record. Only +# occupied code-1 rows survive the vacancy drop in _occupied_households. +_HOUSEHOLD_AXIS_KINDS = { + 1: "occupied_housing_unit", + 2: "institutional_gq_person", + 3: "noninstitutional_gq_person", +} _PERSON_REQUIRED = ( "SERIALNO", "SPORDER", @@ -168,11 +178,36 @@ def __post_init__(self) -> None: if self.max_households is not None and self.max_households <= 0: raise ValueError("max_households must be positive when provided.") + @staticmethod + def snapshot_serialnos(serialnos): + """Freeze exact raw native keys; no sampling probability is implied.""" + if serialnos is None: + return None + if ( + type(serialnos) is not tuple + or not 0 < len(serialnos) <= MAX_EXACT_HOUSEHOLDS + or any( + type(key) is not str + or len(key) != 13 + or key[:6] not in {"2024HU", "2024GQ"} + or not key[6:].isascii() + or not key[6:].isdigit() + or int(key[6:]) == 0 + for key in serialnos + ) + or len(set(serialnos)) != len(serialnos) + ): + raise ValueError( + "ACS exact selection requires bounded unique raw native keys." + ) + return serialnos + def load_acs_pums_tables( source: AcsPumsSource, *, chunksize: int = DEFAULT_CHUNKSIZE, + serialnos: tuple[str, ...] | None = None, ) -> tuple[dict[str, pd.DataFrame], dict[str, Any]]: """Read and key-align the household and person PUMS tables. @@ -180,6 +215,11 @@ def load_acs_pums_tables( missing; this stage never converts an out-of-universe blank to zero. """ + serialnos = AcsPumsSource.snapshot_serialnos(serialnos) + if serialnos is not None and source.max_households is not None: + raise ValueError( + "ACS exact serialnos and max_households are ambiguous together." + ) if chunksize <= 0: raise ValueError("chunksize must be positive.") household, household_members = _read_archive( @@ -207,7 +247,24 @@ def load_acs_pums_tables( raise ValueError(f"ACS duplicate household SERIALNO value(s): {examples}.") household = household.sort_values("SERIALNO", kind="stable").reset_index(drop=True) all_household_serials = frozenset(household["SERIALNO"].tolist()) + full_household = household household, vacant_count = _occupied_households(household) + if serialnos is not None: + if not set(serialnos) <= all_household_serials: + raise ValueError("ACS exact selection contains absent household keys.") + weights = pd.to_numeric(household.WGTP, errors="coerce").to_numpy(dtype=float) + if not np.isfinite(weights).all() or (weights < 0).any(): + raise ValueError("ACS WGTP must be finite and nonnegative.") + _validate_amount_columns( + (household.RNTP, household.GRNTP, household.TAXAMT), household.ADJHSG + ) + household = household.loc[household.SERIALNO.isin(serialnos)].reset_index( + drop=True + ) + if household.NP.sum() > MAX_EXACT_PERSON_ROWS: + raise ValueError( + "ACS selected complete roster exceeds native person budget." + ) if source.max_households is not None and len(household) > source.max_households: household = _smoke_household_selection(household, source.max_households) selected_serials = frozenset(household["SERIALNO"].tolist()) @@ -220,6 +277,7 @@ def load_acs_pums_tables( chunksize=chunksize, valid_serials=all_household_serials, retained_serials=selected_serials, + validate_households=full_household if serialnos is not None else None, ) duplicate_people = person.duplicated(["SERIALNO", "SPORDER"], keep=False) if duplicate_people.any(): @@ -247,6 +305,14 @@ def load_acs_pums_tables( "vacant_household_rows_dropped": vacant_count, "max_households": source.max_households, } + if serialnos is not None: + metadata["exact_selection"] = { + "requested_serialnos": serialnos, + "populated_serialnos": tuple(household.SERIALNO), + "vacant_serialnos": tuple(sorted(set(serialnos) - selected_serials)), + "selection_kind": "engineering_exact_keys", + "full_source_inclusion_probability": None, + } return {"household": household, "person": person}, metadata @@ -254,10 +320,13 @@ def build_acs_pums_unit_frame( source: AcsPumsSource, *, chunksize: int = DEFAULT_CHUNKSIZE, + serialnos: tuple[str, ...] | None = None, ) -> tuple[Frame, dict[str, Any]]: """Construct the ACS 2024 1-year US entity frame.""" - tables, metadata = load_acs_pums_tables(source, chunksize=chunksize) + tables, metadata = load_acs_pums_tables( + source, chunksize=chunksize, serialnos=serialnos + ) household = tables["household"].copy() person = tables["person"].copy() @@ -299,6 +368,11 @@ def build_acs_pums_unit_frame( metadata.update( { "weighted_household_population": frame.weights_for("household").total, + # That total mixes housing-unit and GQ-person design mass. + "household_axis_composition": _household_axis_composition( + household, + household_weights, + ), "relationship_pointer_policy": ( "RELSHIPP reference/spouse pairing; own/adopted/stepchildren " "point to reference person and present spouse; all other " @@ -330,10 +404,12 @@ def _read_archive( chunksize: int, valid_serials: frozenset[str] | None = None, retained_serials: frozenset[str] | None = None, + validate_households: pd.DataFrame | None = None, ) -> tuple[pd.DataFrame, list[str]]: if not path.is_file(): raise FileNotFoundError(f"ACS PUMS archive not found: {path}") pieces: list[pd.DataFrame] = [] + source_roster = {} with ZipFile(path) as archive: members = sorted( name @@ -357,7 +433,7 @@ def _read_archive( usecols = [column for column in (*required, *optional) if column in columns] string_columns = { column: "string" - for column in ("SERIALNO", "ST", "STATE", "PUMA") + for column in ("SERIALNO", "ST", "STATE", "PUMA", "AGEP") if column in usecols } with archive.open(member_name) as member: @@ -369,6 +445,11 @@ def _read_archive( low_memory=False, ) for chunk in reader: + if "AGEP" in chunk: + # Validate original literals before numeric inference or + # selection can hide an unresolved source age. The native + # coverage contract accepts only one/two ASCII digits. + chunk["AGEP"] = _original_source_ages(chunk["AGEP"]) if valid_serials is not None: orphan = ~chunk["SERIALNO"].isin(valid_serials) if orphan.any(): @@ -382,15 +463,88 @@ def _read_archive( "ACS person SERIALNO value(s) missing from the " f"household archive: {examples}." ) + if validate_households is not None: + _validate_source_chunk(chunk, source_roster) if retained_serials is not None: chunk = chunk.loc[chunk["SERIALNO"].isin(retained_serials)] if not chunk.empty: pieces.append(chunk) + if validate_households is not None: + _validate_source_roster(validate_households, source_roster) if not pieces: return pd.DataFrame(columns=[*required, *optional]), members return pd.concat(pieces, ignore_index=True), members +def _original_source_ages(ages: pd.Series) -> np.ndarray: + """Preserve the native coverage owner's literal 0–99 age identity contract.""" + if not ages.str.fullmatch(r"[0-9]{1,2}", na=False).all(): + raise ValueError("ACS original AGEP must contain one or two ASCII digits.") + return ages.to_numpy(dtype=np.int64) + + +def _validate_amount_columns(amounts, factor): + """Keep the native mapper's finite observed-amount/factor gates before filtering.""" + adjustment = pd.to_numeric(factor, errors="coerce").to_numpy(dtype=float) + for values in amounts: + amount = pd.to_numeric(values, errors="coerce").to_numpy(dtype=float) + observed = ~np.isnan(amount) + if (observed & ~np.isfinite(amount)).any(): + raise ValueError("ACS source amount must be finite.") + if (observed & (~np.isfinite(adjustment) | (adjustment <= 0))).any(): + raise ValueError("ACS source adjustment must be finite and positive.") + + +def _validate_source_chunk(person, roster): + # Only global key sets and small household structural summaries survive a + # chunk. Unselected observation rows never enter the accumulated person table. + lines = _required_integer(person, "SPORDER") + marital = _required_integer(person, "MAR") + relations = _required_integer(person, "RELSHIPP") + sexes = _required_integer(person, "SEX") + if not set(marital) <= set(_ACS_TO_CPS_MARITAL_STATUS): + raise ValueError("ACS source MAR contains unsupported codes.") + if not set(relations) <= {20, *_ACS_SPOUSE_CODES, *_ACS_TO_CPS_RELATIONSHIP}: + raise ValueError("ACS source RELSHIPP contains unsupported codes.") + if not set(sexes) <= {1, 2}: + raise ValueError("ACS source SEX contains unsupported codes.") + _validate_amount_columns( + (person.WAGP, person.SEMP, person.INTP, person.RETP, person.SSP, person.SSIP), + person.ADJINC, + ) + weights = pd.to_numeric(person.PWGTP, errors="coerce").to_numpy(dtype=float) + for serial, line, relation, mar, weight in zip( + person.SERIALNO, lines, relations, marital, weights, strict=True + ): + state = roster.setdefault(serial, [set(), 0, 0, True, True, True]) + if line in state[0]: + raise ValueError("ACS duplicate person key in complete source.") + state[0].add(int(line)) + state[1] += int(relation == 20) + state[2] += int(relation in _ACS_SPOUSE_CODES) + state[3] &= relation not in {20, *_ACS_SPOUSE_CODES} or mar == 1 + state[4] &= relation in {37, 38} + state[5] &= bool(np.isfinite(weight) and weight > 0) + + +def _validate_source_roster(household, roster): + for serial, count, kind in household[["SERIALNO", "NP", "TYPEHUGQ"]].itertuples( + index=False, name=None + ): + state = dict.get(roster, serial) + if (len(state[0]) if state else 0) != int(count): + raise ValueError("ACS NP/person row-count mismatch in complete source.") + if not state: + continue + if int(kind) != 1 and not state[5]: + raise ValueError("ACS GQ PWGTP must be finite and positive.") + if not state[4]: + if state[1] != 1 or state[2] > 1: + raise ValueError("ACS source reference/spouse roster is invalid.") + if state[2] and not state[3]: + raise ValueError("ACS source spouse pair must have MAR=1.") + + def _occupied_households(household: pd.DataFrame) -> tuple[pd.DataFrame, int]: people = pd.to_numeric(household["NP"], errors="coerce") if people.isna().any() or (people < 0).any() or (people % 1 != 0).any(): @@ -616,6 +770,52 @@ def _household_weights(household: pd.DataFrame, person: pd.DataFrame) -> Weights return Weights(raw, WeightKind.DESIGN) +def _household_axis_composition( + household: pd.DataFrame, + household_weights: Weights, +) -> dict[str, int | float]: + """Split the loaded household-axis rows and DESIGN mass by TYPEHUGQ. + + The axis mixes occupied physical housing units with one-person group- + quarters person placeholders. Each part retains its own statistical unit; + their mixed sum is not a housing-unit count or a person count. These are + source design weights, with no common-survey or calibrated-population + claim. The rows described are the ones this build + loaded, after the vacancy drop and any ``max_households`` selection. + + Admissibility of the codes and their weights is decided upstream by + ``_occupied_households``; this only partitions what that check accepted, + and refuses if the accepted rows do not partition. + """ + + if "TYPEHUGQ" not in household.columns: + raise ValueError("ACS household-axis composition requires the TYPEHUGQ column.") + design = np.asarray(household_weights.values, dtype=np.float64) + if design.shape[0] != len(household): + raise ValueError( + "ACS household-axis composition needs one DESIGN weight per loaded " + f"household row; got {design.shape[0]} for {len(household)} row(s)." + ) + kind = pd.to_numeric(household["TYPEHUGQ"], errors="coerce").to_numpy( + dtype=np.float64, + na_value=np.nan, + ) + composition: dict[str, int | float] = {} + partitioned = 0 + for code, label in _HOUSEHOLD_AXIS_KINDS.items(): + selected = kind == code + rows = int(selected.sum()) + composition[f"{label}_rows"] = rows + composition[f"{label}_design_weight_total"] = float(design[selected].sum()) + partitioned += rows + if partitioned != len(household): + raise ValueError( + "ACS household-axis composition does not partition the loaded rows: " + f"{partitioned} of {len(household)} carry a described TYPEHUGQ code." + ) + return composition + + def _attach_household_source_columns( frame: Frame, source_household: pd.DataFrame, diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/acs_transfer.py b/packages/microcosm-build/src/microcosm/build/us_runtime/acs_transfer.py index a3fb00a6b..e6cf40c8c 100644 --- a/packages/microcosm-build/src/microcosm/build/us_runtime/acs_transfer.py +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/acs_transfer.py @@ -48,6 +48,10 @@ from microcosm.fit.qrf import detect_regime from microcosm.frame import EntitySchema, Frame, Weights +from .operator_column_contracts import ( + ACS_DERIVED_TRANSFER_INPUTS as ACS_DERIVED_TRANSFER_INPUTS, +) + QRF: Any | None = None __all__ = [ @@ -182,6 +186,7 @@ def _pregnancy_structural_policy_identity(*, enabled: bool) -> dict[str, object] ).hexdigest() return payload + _IMMIGRATION_STATUS_TARGETS = ( "ssn_card_type", "immigration_status_str", @@ -635,14 +640,6 @@ def required_acs_transfer_inputs() -> frozenset[str]: ) -#: Person columns the default transfer DERIVES deterministically after the -#: QRF fits (never fitted themselves). Coverage checks require them on the -#: recipient exactly like declared plan targets. -ACS_DERIVED_TRANSFER_INPUTS: tuple[str, ...] = ( - "schedule_d_capital_gain_distributions", -) - - def acs_derived_transfer_expectations( target_families: TargetFamilies, ) -> dict[str, str]: @@ -1138,8 +1135,8 @@ def _prepare_pregnancy_structural_plan( "through 44." ) - source_codes, group_count, key_column, representatives = ( - _pregnancy_source_groups(table) + source_codes, group_count, key_column, representatives = _pregnancy_source_groups( + table ) eligible_min = np.ones(group_count, dtype=np.int8) eligible_max = np.zeros(group_count, dtype=np.int8) @@ -1449,9 +1446,7 @@ def transfer_acs_inputs( person, pregnancy_plan, ) - pregnancy_entity, pregnancy_family, pregnancy_targets = ( - pregnancy_request - ) + pregnancy_entity, pregnancy_family, pregnancy_targets = pregnancy_request if pregnancy_targets != (_PREGNANCY_TARGET,): # pragma: no cover raise AssertionError( "Pregnancy structural target was not isolated before receipt." diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/asec_2024_native_population.py b/packages/microcosm-build/src/microcosm/build/us_runtime/asec_2024_native_population.py new file mode 100644 index 000000000..7cad3394d --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/asec_2024_native_population.py @@ -0,0 +1,652 @@ +"""Issue selected 2024 ASEC observations with original household DESIGN anchors. + +The closed parent and person-coverage owners still authenticate all three source +cohorts. Only the selected 2024 observations become this new population. Exact +key selection is an engineering operation with no inclusion-probability claim. +No pooling coefficient, age adjustment, coverage classification or allocation is +performed here. Source evidence stays outside the selected Frame's metadata. +""" + +from __future__ import annotations + +import hashlib +import json +import os +import stat +import sys +import weakref +from collections.abc import Mapping +from dataclasses import dataclass +from pathlib import Path +from typing import NamedTuple + +import numpy as np +import pandas as pd + +from microcosm.build.cd_benchmark import origin +from microcosm.frame import US_SCHEMA, Frame, WeightKind, Weights +from microcosm.frame import bundle as frame_owner +from microcosm.frame import weights as weight_owner + +from . import asec_coverage_authentication as coverage_owner +from . import asec_current_money_source as parent_owner +from . import asec_household_coverage_fields as field_owner +from . import asec_original_household_weights as anchor_owner +from . import asec_person_income_source as restoration +from . import graph_context + +ARTIFACT_KIND = "microcosm.us.asec-2024-native-population.v1" +_MAX_HOUSEHOLDS = 100_000 +_MAX_PERSONS = 600_000 +_MAX_EVIDENCE_BYTES = 128 * 1024**2 +_MAX_CHECKPOINT_BYTES = 8 * 1024**3 +_ISSUED: dict[int, tuple[weakref.ReferenceType, bytes, object]] = {} + + +class AsecNativePopulationError(ValueError): + """Static refusal codes; no source paths, native keys or source values.""" + + +def _require(condition, code): + if not condition: + raise AsecNativePopulationError(code) + + +def _sha(value): + return hashlib.sha256(value).hexdigest() + + +def _encode(value): + result = bytearray() + for part in json.JSONEncoder( + ensure_ascii=True, allow_nan=False, sort_keys=True, separators=(",", ":") + ).iterencode(value): + part = part.encode("ascii") + _require(len(result) + len(part) <= _MAX_EVIDENCE_BYTES, "EVIDENCE_SIZE") + result.extend(part) + return bytes(result) + + +def _modules(): + return ( + sys.modules[__name__], + parent_owner, + restoration, + coverage_owner, + anchor_owner, + field_owner, + graph_context, + origin, + frame_owner, + weight_owner, + ) + + +def _implementation(): + runtime = {m.__name__: coverage_owner._runtime_code(m) for m in _modules()} + _require(runtime == _RUNTIME_AUTHORITY, "PRODUCER_CODE_CHANGED") + return { + "kind": ARTIFACT_KIND, + "runtime": runtime, + "code": {m.__name__: _sha(Path(m.__file__).read_bytes()) for m in _modules()}, + "parent": parent_owner._verification_identity(), + "restoration": restoration._implementation(), + "coverage": coverage_owner._implementation(), + "anchors": anchor_owner._implementation(), + "household_fields": field_owner._producer(), + "limits": [ + _MAX_HOUSEHOLDS, + _MAX_PERSONS, + _MAX_EVIDENCE_BYTES, + _MAX_CHECKPOINT_BYTES, + ], + "periods": [2024, 2025, 2024], + } + + +def _path(value): + _require(type(value) is str or isinstance(value, Path), "SOURCE_PATH") + return Path(value).absolute() + + +def _inputs(parent, household, restored, persons, member, selected, candidate): + # Snapshot every caller-owned lookup before fingerprinting or source I/O. + _require(isinstance(persons, Mapping) and len(persons) == 3, "PERSON_PATHS") + paths = {} + for year in persons: + _require( + type(year) is int + and year in (2022, 2023, 2024) + and year not in paths + and len(paths) < 3, + "PERSON_PATHS", + ) + paths[year] = _path(persons[year]) + _require(set(paths) == {2022, 2023, 2024} and len(persons) == 3, "PERSON_PATHS") + if selected is not None: + _require( + type(selected) is tuple and 0 < len(selected) <= _MAX_HOUSEHOLDS, + "SELECTION", + ) + for key in selected: + _require( + type(key) is tuple + and len(key) == 2 + and all(type(v) is int for v in key) + and key[0] == 2024 + and 1 <= key[1] <= 99999, + "SELECTION", + ) + _require(len(set(selected)) == len(selected), "SELECTION") + _require( + candidate is None + or (type(candidate) is bytes and len(candidate) <= _MAX_EVIDENCE_BYTES), + "CANDIDATE", + ) + return ( + _path(parent), + _path(household), + _path(restored), + paths, + _path(member), + selected, + candidate, + ) + + +def _file_identity(path, expected, maximum, size=None): + """Check original input bytes on every borrow, with bounded regular-file I/O.""" + descriptor = os.open(path, os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK) + try: + before = os.fstat(descriptor) + _require( + stat.S_ISREG(before.st_mode) and 0 < before.st_size <= maximum, + "SOURCE_FILE_BOUNDS", + ) + _require(size is None or before.st_size == size, "SOURCE_FILE_SIZE") + digest, count = hashlib.sha256(), 0 + while block := os.read(descriptor, min(1024**2, before.st_size - count + 1)): + count += len(block) + _require(count <= before.st_size, "SOURCE_FILE_CHANGED") + digest.update(block) + after = os.fstat(descriptor) + keys = ("st_dev", "st_ino", "st_size", "st_mtime_ns", "st_ctime_ns") + _require( + count == before.st_size + and digest.hexdigest() == expected + and all(getattr(before, k) == getattr(after, k) for k in keys), + "SOURCE_FILE_CHANGED", + ) + finally: + os.close(descriptor) + + +def _frame_identity(frame): + _require( + type(frame) is Frame and frame.schema == US_SCHEMA and not frame.links, + "FRAME_SHAPE", + ) + _require( + set(frame._tables) == set(US_SCHEMA.entities) + and not frame._link_tables + and set(frame._weights) == set(frame.weighted_entities), + "FRAME_STORAGE", + ) + # The parent signature binds typed cells, ordered axes, metadata, mass log, + # schema and stored weights. Bind additional mutable pandas/ndarray state. + decorations = [] + for name in frame.entities: + table = frame.table(name) + _require(type(table) is pd.DataFrame, "FRAME_TABLE_TYPE") + decorations.append([name, table.attrs, table.flags.allows_duplicate_labels]) + decorations.append( + ["strata", frame.strata.attrs, frame.strata.flags.allows_duplicate_labels] + ) + weights = [] + for entity in frame.weighted_entities: + vector = frame.weights_for(entity) + _require( + type(vector) is Weights and type(vector.values) is np.ndarray, + "FRAME_WEIGHT_TYPE", + ) + weights.append( + [ + entity, + vector.kind.value, + str(vector.values.dtype), + list(vector.values.shape), + vector.values.flags.writeable, + _sha(vector.values.tobytes()), + ] + ) + return _sha( + _encode( + { + "base": parent_owner._frame_signature(frame), + "decorations": decorations, + "weights": weights, + } + ) + ) + + +def _roster(parent, coverage, anchors, fields, selected): + projection = coverage.table() + people = parent.frame.person + _require(len(projection) == len(people), "PARENT_ROSTER") + for column in coverage_owner._PARENT_COLUMNS: + _require( + projection[column].tolist() == people[column].tolist(), "PARENT_ROSTER" + ) + anchor_doc, field_doc = anchors.document, fields.document + _require( + anchor_doc["members"] == field_doc["members"] + and len(anchor_doc["members"]) == 1, + "HOUSEHOLD_MEMBER_BINDING", + ) + member = anchor_doc["members"][0] + source = next(s for s in coverage.receipt["sources"] if s["source_year"] == 2024) + _require( + member["income_year"] == 2024 + and member["survey_year"] == 2025 + and source["survey_year"] == 2025 + and member["archive_sha256"] == source["archive_sha256"], + "COHORT_ARCHIVE_BINDING", + ) + anchor_rows = {r["native_household_id"]: r for r in anchor_doc["records"]} + field_rows = {r["native_household_id"]: r for r in field_doc["records"]} + _require( + len(anchor_rows) == len(anchor_doc["records"]) + and len(field_rows) == len(field_doc["records"]) + and anchor_rows.keys() == field_rows.keys(), + "HOUSEHOLD_ROSTER", + ) + native_to_receiving, receiving_to_native, person_keys = {}, {}, set() + counts = {} + for row in projection.itertuples(index=False): + key = (int(row.source_year), int(row.source_household_id)) + receiving = int(row.person_household_id) + _require( + native_to_receiving.setdefault(key, receiving) == receiving + and receiving_to_native.setdefault(receiving, key) == key, + "HOUSEHOLD_PARTITION", + ) + person_key = (key[0], row.PERIDNUM, int(row.A_LINENO)) + _require(person_key not in person_keys, "PERSON_ROSTER") + person_keys.add(person_key) + counts[key] = counts.get(key, 0) + 1 + all_keys = {key for key in native_to_receiving if key[0] == 2024} + _require(bool(all_keys), "EMPTY_COHORT") + for native, row in field_rows.items(): + anchor = anchor_rows[native] + for column in ( + "income_year", + "survey_year", + "member_id", + "member_sha256", + "member_row_1based", + "origin_key", + "H_SEQ", + "H_HHTYPE", + ): + _require(row[column] == anchor[column], "HOUSEHOLD_PROJECTION_BINDING") + _require( + row["income_year"] == 2024 and row["survey_year"] == 2025, + "HOUSEHOLD_COHORT", + ) + key = (2024, native) + if row["H_HHTYPE"] == "1": + _require( + key in all_keys and row["reported_person_count"] == counts[key], + "HOUSEHOLD_MEMBER_COUNT", + ) + else: + _require(key not in all_keys, "HOUSEHOLD_INTERVIEW_STATUS") + _require(all(k[1] in field_rows for k in all_keys), "MISSING_HOUSEHOLD") + chosen = all_keys if selected is None else set(selected) + _require(chosen <= all_keys and len(chosen) <= _MAX_HOUSEHOLDS, "SELECTION_UNKNOWN") + mask = np.array( + [receiving_to_native[int(h)] in chosen for h in people.person_household_id], + dtype=bool, + ) + _require(0 < int(mask.sum()) <= _MAX_PERSONS, "SELECTED_PERSON_BOUNDS") + selected_ids = set(people.loc[mask, "person_household_id"].tolist()) + ordered_ids = [ + int(h) + for h in parent.frame.table("household").household_id + if h in selected_ids + ] + ordered_keys = tuple(receiving_to_native[h] for h in ordered_ids) + _require(set(ordered_keys) == chosen, "RECEIVING_HOUSEHOLDS") + exact = anchors.original_weight_fractions(ordered_keys) + rows = [] + for receiving, key, fraction in zip(ordered_ids, ordered_keys, exact, strict=True): + row = anchor_rows[key[1]] + expected_origin = origin.origin_key( + origin.AsecHouseholdOrigin( + origin.SourceMember( + member["canonical_member_id"], member["member_sha256"] + ), + 2024, + key[1], + ) + ) + _require(row["origin_key"] == expected_origin, "ORIGIN_BINDING") + rows.append( + { + "household_id": receiving, + "native_key": list(key), + "origin_key": expected_origin, + "HSUP_WGT": row["HSUP_WGT"], + "numerator": row["weight_integer_units"], + "denominator": 100, + "fraction": [fraction.numerator, fraction.denominator], + "source": row, + "household_fields": field_rows[key[1]], + } + ) + roster = projection.loc[mask].to_dict(orient="records") + return mask, np.array([float(f) for f in exact], dtype=np.float64), rows, roster + + +def _descendant(parent, mask, anchors): + """Select observations without using legacy weight slices as source anchors.""" + frame = parent.frame + _require(frame.schema == US_SCHEMA and not frame.links, "PARENT_SCHEMA") + person = frame.person.loc[mask].copy(deep=True) + tables = {"person": person} + for entity in US_SCHEMA.group_entities: + column = US_SCHEMA.membership_column(entity) + ids = set(person[column].tolist()) + # Referenced groups cannot silently retain only part of their members. + _require(not frame.person.loc[~mask, column].isin(ids).any(), "SPLIT_GROUP") + table = frame.table(entity) + tables[entity] = table.loc[ + table[US_SCHEMA.id_column(entity)].isin(ids) + ].reset_index(drop=True) + descendant = Frame( + tables, + US_SCHEMA, + {"household": Weights(anchors, WeightKind.DESIGN)}, + frame.strata.loc[mask].copy(deep=True), + metadata={}, + mass_log=(), + ) + parent_owner._detach_frame_axes(descendant) + return descendant + + +class _State(NamedTuple): + """Tuple storage keeps the external seal immutable even to object.__setattr__.""" + + frame: Frame + parent: object + coverage: object + anchors: object + fields: object + source_files: tuple + producer: bytes + parent_identity: str + frame_identity: str + context: bytes + attached_evidence: tuple + + +def _attached_evidence(parent, coverage, anchors, fields): + """Capture immutable issued bytes; the parent authority also has identity.""" + values = ( + parent.source._validate(), + coverage._header, + coverage._body, + anchor_owner._checked_capsule_payload(anchors), + anchor_owner._checked_capsule_payload(fields), + ) + _require(all(type(value) is bytes for value in values), "ATTACHED_EVIDENCE_CHANGED") + return (parent.source, *values) + + +def _validate_state(state): + _require(_encode(_implementation()) == state.producer, "PRODUCER_CHANGED") + _require(_frame_identity(state.frame) == state.frame_identity, "FRAME_CHANGED") + _require( + _frame_identity(state.parent.frame) == state.parent_identity, "PARENT_CHANGED" + ) + state.parent.validate() + coverage_owner.verify_asec_coverage_parent(state.coverage, state.parent) + anchor_owner.verify_asec_household_weights_source(state.anchors) + field_owner.verify_asec_household_coverage_fields(state.fields) + for path, expected, maximum, size in state.source_files: + _file_identity(path, expected, maximum, size) + # File checks/foreign owner calls may yield. Recheck the actual receiving + # frame and producer after them, not only before final source verification. + _require(_encode(_implementation()) == state.producer, "PRODUCER_CHANGED") + _require( + _frame_identity(state.parent.frame) == state.parent_identity, "PARENT_CHANGED" + ) + _require( + _frame_identity(state.frame) == state.frame_identity + and graph_context.encode_us_frame_context(state.frame) == state.context, + "FRAME_CHANGED", + ) + # These final seal checks perform no source/resource I/O. The immutable + # tuple belongs to the external population seal, not mutable capsule state. + current = _attached_evidence( + state.parent, state.coverage, state.anchors, state.fields + ) + _require( + current[0] is state.attached_evidence[0] + and current[1:] == state.attached_evidence[1:], + "ATTACHED_EVIDENCE_CHANGED", + ) + + +@dataclass(frozen=True, slots=True, weakref_slot=True) +class AuthenticatedAsec2024NativePopulation: + """Process-issued population; every accessor verifies its full live binding.""" + + payload: bytes + + def _checked(self): + try: + entry = _ISSUED.get(id(self)) + _require( + type(self) is AuthenticatedAsec2024NativePopulation + and entry is not None + and entry[0]() is self + and type(self.payload) is bytes + and self.payload == entry[1], + "UNISSUED_OR_CHANGED", + ) + _validate_state(entry[2]) + _require( + _ISSUED.get(id(self)) is entry + and type(self.payload) is bytes + and self.payload == entry[1], + "UNISSUED_OR_CHANGED", + ) + return entry + except AsecNativePopulationError: + raise + except ( + OSError, + ValueError, + TypeError, + KeyError, + AttributeError, + OverflowError, + ): + raise AsecNativePopulationError("NATIVE_BINDING_REFUSAL") from None + + def _state(self): + return self._checked()[2] + + def validate(self): + self._state() + + @property + def frame(self): + return self._state().frame + + @property + def context(self): + return self._state().context + + @property + def receipt(self): + return json.loads(self._checked()[1]) + + def to_bytes(self): + return self._checked()[1] + + +def load_authenticated_asec_2024_native_population( + parent_path, + household_attachment_path, + person_income_attachment_path, + *, + person_member_paths, + household_member_path, + selected_households=None, + candidate=None, +): + """Reconstruct from closed source paths; candidate bytes never grant authority. + + ``selected_households`` is a nonempty unique tuple of ``(2024, H_SEQ)`` + integer keys, or None for the complete 2024 source cohort. This is not a + probability sample. The authenticated parent/coverage still read all cohorts. + """ + try: + ( + parent_path, + household_path, + restored_path, + persons, + member, + selection, + candidate, + ) = _inputs( + parent_path, + household_attachment_path, + person_income_attachment_path, + person_member_paths, + household_member_path, + selected_households, + candidate, + ) + producer = _encode(_implementation()) + parent = restoration.load_authenticated_restored_current_money_source( + parent_path, household_path, restored_path, member_paths=persons + ) + parent_identity = _frame_identity(parent.frame) + coverage = coverage_owner.authenticate_asec_coverage( + parent, member_paths=persons + ) + anchors = anchor_owner.load_authenticated_asec_household_weights({2024: member}) + fields = field_owner.load_authenticated_asec_household_coverage_fields( + {2024: member} + ) + mask, weights, rows, roster = _roster( + parent, coverage, anchors, fields, selection + ) + frame = _descendant(parent, mask, weights) + context = graph_context.encode_us_frame_context(frame) + identity = json.loads(parent.source.identity) + source_files = [ + (parent_path, identity["parent_sha256"], _MAX_CHECKPOINT_BYTES, None), + ( + household_path, + identity["attachment_sha256"], + _MAX_CHECKPOINT_BYTES, + None, + ), + ( + restored_path, + identity["person_income_attachment_sha256"], + _MAX_CHECKPOINT_BYTES, + None, + ), + ] + for source in coverage.receipt["sources"]: + source_files.append( + ( + persons[source["source_year"]], + source["member_sha256"], + source["member_bytes"], + source["member_bytes"], + ) + ) + pin = anchors.document["members"][0] + source_files.append( + (member, pin["member_sha256"], pin["size_bytes"], pin["size_bytes"]) + ) + state = _State( + frame, + parent, + coverage, + anchors, + fields, + tuple(source_files), + producer, + parent_identity, + _frame_identity(frame), + context, + _attached_evidence(parent, coverage, anchors, fields), + ) + payload = _encode( + { + "kind": ARTIFACT_KIND, + "source_year": 2024, + "income_year": 2024, + "survey_year": 2025, + "analysis_year": 2024, + "selection": { + "kind": "complete_2024_cohort" + if selection is None + else "exact_engineering_keys", + "keys": [row["native_key"] for row in rows], + "inclusion_probabilities_known": False, + }, + "households": rows, + "persons": roster, + "raw_age_relation": "original_A_AGE_identity_no_adjustment", + "original_weight_conversion": "float64(Fraction(HSUP_WGT_integer,100))", + "weight_kind": "design", + "frame_sha256": state.frame_identity, + "context_sha256": _sha(context), + "producer_sha256": _sha(producer), + "parent_custody": { + "cohorts": [2022, 2023, 2024], + "population_cohorts": [2024], + "parent_identity": identity, + "frame_sha256": parent_identity, + }, + "person_coverage": coverage.receipt, + "household_weights_sha256": _sha(anchors.payload), + "household_fields_sha256": _sha(fields.payload), + "coverage_classification_performed": False, + "source_share_allocation_performed": False, + "release_eligible": False, + } + ) + _require(candidate is None or candidate == payload, "CANDIDATE_MISMATCH") + _validate_state(state) + result = AuthenticatedAsec2024NativePopulation(payload) + key = id(result) + + def discard(reference): + entry = _ISSUED.get(key) + if entry is not None and entry[0] is reference: + del _ISSUED[key] + + _ISSUED[key] = (weakref.ref(result, discard), payload, state) + result.validate() + return result + except AsecNativePopulationError: + raise + except (OSError, ValueError, TypeError, KeyError, AttributeError, OverflowError): + raise AsecNativePopulationError("NATIVE_SOURCE_REFUSAL") from None + + +# Pins can be substituted explicitly by invented tests. Callable implementation +# changes cannot silently establish a new producer merely by rebuilding receipts. +_RUNTIME_AUTHORITY = {m.__name__: coverage_owner._runtime_code(m) for m in _modules()} diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/asec_checkpoint.py b/packages/microcosm-build/src/microcosm/build/us_runtime/asec_checkpoint.py index 0c6ebe0fa..6299a4762 100644 --- a/packages/microcosm-build/src/microcosm/build/us_runtime/asec_checkpoint.py +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/asec_checkpoint.py @@ -12,6 +12,7 @@ import re from collections.abc import Mapping +from dataclasses import dataclass from pathlib import Path import numpy as np @@ -24,9 +25,15 @@ frame_identity, ) from microcosm.build.serialization_dtypes import canonicalize_frame_string_dtypes +from microcosm.build.us_runtime.education_assistance_source import ( + ASEC_EDUCATION_ASSISTANCE_ARCHIVES, +) from microcosm.build.us_runtime.operator_boundary import ( assert_operator_free_source_frame, ) +from microcosm.build.us_runtime.reported_coverage_source import ( + ASEC_REPORTED_COVERAGE_RAW_COLUMNS, +) from microcosm.frame import US_SCHEMA, Frame, WeightKind, Weights __all__ = [ @@ -34,9 +41,11 @@ "ASEC_RAW_STAGE_CHECKPOINT_FILENAME", "ASEC_RAW_STAGE_OPERATOR_STATUS", "ASEC_RAW_STAGE_SCHEMA_VERSION", + "ASEC_RAW_STAGE_COVERAGE_SCHEMA_VERSION", "ASEC_RAW_STAGE_STAGE", "load_asec_pre_clone_checkpoint", "load_asec_raw_stage_checkpoint", + "load_asec_raw_stage_checkpoint_v4", ] _OUTER_STAGE_ARTIFACT_KIND = "populace_outer_stage_frame" @@ -61,6 +70,9 @@ # artifacts lack the gate column and must fail loudly rather than let # PAW_VAL-only conflation back in (microcosm#591). ASEC_RAW_STAGE_SCHEMA_VERSION = 3 +# V4 is a distinct measured-coverage contract. Never reinterpret a v3 file +# under this schema merely because its columns happen to look compatible. +ASEC_RAW_STAGE_COVERAGE_SCHEMA_VERSION = 4 ASEC_RAW_STAGE_STAGE = "raw_source_mapping" _RAW_STAGE_BINDING_KEYS = frozenset( { @@ -100,6 +112,23 @@ ) +@dataclass(frozen=True) +class _RawStagePolicy: + version: int + coverage_columns: frozenset[str] = frozenset() + + @property + def mapping_columns(self) -> frozenset[str]: + return _RAW_SOURCE_MAPPING_COLUMNS | self.coverage_columns + + +_RAW_STAGE_V3 = _RawStagePolicy(ASEC_RAW_STAGE_SCHEMA_VERSION) +_RAW_STAGE_V4 = _RawStagePolicy( + ASEC_RAW_STAGE_COVERAGE_SCHEMA_VERSION, + frozenset(ASEC_REPORTED_COVERAGE_RAW_COLUMNS), +) + + def load_asec_pre_clone_checkpoint( path: str | Path, ) -> tuple[Frame, dict[str, object]]: @@ -152,11 +181,33 @@ def load_asec_raw_stage_checkpoint( contract even if it happens to carry a structurally valid US ``Frame``. """ + return _load_asec_raw_stage_checkpoint(path, policy=_RAW_STAGE_V3) + + +def load_asec_raw_stage_checkpoint_v4( + path: str | Path, +) -> tuple[Frame, dict[str, object]]: + """Load only v4 measured-coverage sources with registered per-year pins. + + This validates a declared source artifact, not model accuracy or release + eligibility. The caller must independently authenticate the whole file's + digest; shape and registered source pins alone do not authenticate values. + """ + return _load_asec_raw_stage_checkpoint(path, policy=_RAW_STAGE_V4) + + +def _load_asec_raw_stage_checkpoint( + path: str | Path, + *, + policy: _RawStagePolicy, +) -> tuple[Frame, dict[str, object]]: + checkpoint_path = Path(path) loaded = load_frame_checkpoint(checkpoint_path) metadata = _validate_raw_stage_binding( loaded.metadata, path=checkpoint_path, + policy=policy, ) stored_identity = FrameIdentity.from_payload( metadata["identity"], @@ -176,7 +227,11 @@ def load_asec_raw_stage_checkpoint( loaded.frame, label=f"ASEC raw-stage checkpoint {checkpoint_path}", ) - _validate_raw_stage_source_columns(loaded.frame, path=checkpoint_path) + _validate_raw_stage_source_columns( + loaded.frame, path=checkpoint_path, policy=policy + ) + if policy.coverage_columns: + _validate_coverage_v4_binding(loaded.frame, metadata, path=checkpoint_path) source_construction_identity = FrameIdentity.from_payload( metadata["source_construction_identity"], label="ASEC raw-stage source-construction identity", @@ -250,6 +305,7 @@ def _validate_raw_stage_binding( metadata: dict[str, object], *, path: Path, + policy: _RawStagePolicy = _RAW_STAGE_V3, ) -> dict[str, object]: actual_keys = frozenset(metadata) if actual_keys != _RAW_STAGE_BINDING_KEYS: @@ -265,8 +321,10 @@ def _validate_raw_stage_binding( "ASEC artifact." ) schema_version = metadata["schema_version"] - if schema_version != ASEC_RAW_STAGE_SCHEMA_VERSION or isinstance( - schema_version, bool + if ( + schema_version != policy.version + or isinstance(schema_version, bool) + or (policy.coverage_columns and type(schema_version) is not int) ): raise ValueError( f"ASEC raw-stage checkpoint {path} has an unsupported raw-stage " @@ -292,7 +350,9 @@ def _validate_raw_stage_binding( "lowercase SHA-256 digest." ) _validate_source_receipt(metadata["source_receipt"], path=path) - _validate_raw_source_mappings(metadata["raw_source_mappings"], path=path) + _validate_raw_source_mappings( + metadata["raw_source_mappings"], path=path, policy=policy + ) return dict(metadata) @@ -338,17 +398,19 @@ def _validate_source_receipt(receipt: object, *, path: Path) -> None: years.add(year) -def _validate_raw_source_mappings(mappings: object, *, path: Path) -> None: +def _validate_raw_source_mappings( + mappings: object, *, path: Path, policy: _RawStagePolicy = _RAW_STAGE_V3 +) -> None: if not isinstance(mappings, Mapping): raise ValueError( f"ASEC raw-stage checkpoint {path} raw_source_mappings must be an object." ) - if frozenset(mappings) != _RAW_SOURCE_MAPPING_COLUMNS: + if frozenset(mappings) != policy.mapping_columns: raise ValueError( f"ASEC raw-stage checkpoint {path} raw_source_mappings must bind " - f"exactly {sorted(_RAW_SOURCE_MAPPING_COLUMNS)}." + f"exactly {sorted(policy.mapping_columns)}." ) - for column in sorted(_RAW_SOURCE_MAPPING_COLUMNS): + for column in sorted(policy.mapping_columns): mapping = mappings[column] if not isinstance(mapping, Mapping) or frozenset(mapping) != ( _RAW_SOURCE_MAPPING_KEYS @@ -400,9 +462,13 @@ def _validate_raw_source_mappings(mappings: object, *, path: Path) -> None: ) -def _validate_raw_stage_source_columns(frame: Frame, *, path: Path) -> None: +def _validate_raw_stage_source_columns( + frame: Frame, *, path: Path, policy: _RawStagePolicy = _RAW_STAGE_V3 +) -> None: person = frame.table("person") - missing = sorted(_RAW_STAGE_REQUIRED_PERSON_COLUMNS - set(person)) + missing = sorted( + (_RAW_STAGE_REQUIRED_PERSON_COLUMNS | policy.coverage_columns) - set(person) + ) if missing: raise ValueError( f"ASEC raw-stage checkpoint {path} is not input-complete; missing " @@ -458,6 +524,99 @@ def _validate_raw_stage_source_columns(frame: Frame, *, path: Path) -> None: "in {0, 1, 2, 3}." ) + for column in sorted(policy.coverage_columns): + values = pd.to_numeric(person[column], errors="coerce").to_numpy( + dtype=np.float64 + ) + boolean = ( + person[column] + .map(lambda value: isinstance(value, (bool, np.bool_))) + .to_numpy() + ) + if not (np.isfinite(values) & np.isin(values, (1.0, 2.0)) & ~boolean).all(): + raise ValueError( + f"ASEC v4 raw-stage checkpoint {path} {column} must be complete " + "integer recodes in {1, 2}, not booleans." + ) + + +def _validate_coverage_v4_binding( + frame: Frame, metadata: Mapping, *, path: Path +) -> None: + """Cross-check source vintages and actual registered coverage source pins.""" + person = frame.table("person") + normalized_years = pd.to_numeric(person["source_year"]).astype(np.int64) + frame_years = set(normalized_years) + source_years = {source["year"] for source in metadata["source_receipt"]["sources"]} + if frame_years != source_years: + raise ValueError( + f"ASEC v4 checkpoint {path} frame/source receipt year coverage differs." + ) + if frame.weights_for("household").kind is not WeightKind.DESIGN: + raise ValueError( + f"ASEC v4 checkpoint {path} requires household design weights." + ) + peridnum = person["PERIDNUM"].map( + lambda value: value.decode() if isinstance(value, bytes) else value + ) + if not peridnum.map( + lambda value: ( + isinstance(value, str) and re.fullmatch(r"[0-9]{22}", value) is not None + ) + ).all(): + raise ValueError( + f"ASEC v4 checkpoint {path} PERIDNUM must be exact 22-digit strings." + ) + keys = pd.DataFrame({"source_year": normalized_years, "PERIDNUM": peridnum}) + if keys.duplicated().any(): + raise ValueError( + f"ASEC v4 checkpoint {path} repeats a source-year/PERIDNUM identity." + ) + for column in ASEC_REPORTED_COVERAGE_RAW_COLUMNS: + mapping = metadata["raw_source_mappings"][column] + pins = mapping["source_pins"] + years = [pin["income_year"] for pin in pins] + if len(years) != len(set(years)) or set(years) != source_years: + raise ValueError( + f"ASEC v4 {column} source pin year coverage differs or repeats." + ) + for pin in pins: + registered = ASEC_EDUCATION_ASSISTANCE_ARCHIVES.get(pin["income_year"]) + if registered is None or pin != { + "income_year": registered.income_year, + "locator": registered.zip_url, + "member": registered.member, + "member_sha256": registered.member_sha256, + "sha256": registered.zip_sha256, + }: + raise ValueError( + f"ASEC v4 {column} source pin differs from the registered archive." + ) + audit = mapping["audit"] + if set(audit) != {str(year) for year in source_years}: + raise ValueError(f"ASEC v4 {column} audit year coverage differs.") + for year in source_years: + row = audit[str(year)] + if not isinstance(row, Mapping) or set(row) != { + "rows", + "yes_rows", + "no_rows", + "weighted_yes_share", + }: + raise ValueError(f"ASEC v4 {column} audit is malformed.") + counts = [row[key] for key in ("rows", "yes_rows", "no_rows")] + share = row["weighted_yes_share"] + if ( + any(type(count) is not int or count < 0 for count in counts) + or counts[0] != ASEC_EDUCATION_ASSISTANCE_ARCHIVES[year].rows + or counts[1] + counts[2] != counts[0] + or isinstance(share, bool) + or not isinstance(share, (int, float)) + or not np.isfinite(share) + or not 0 <= share <= 1 + ): + raise ValueError(f"ASEC v4 {column} audit counts/share are invalid.") + def _validate_asec_frame( frame: Frame, diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/asec_coverage_authentication.py b/packages/microcosm-build/src/microcosm/build/us_runtime/asec_coverage_authentication.py new file mode 100644 index 000000000..5641051e3 --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/asec_coverage_authentication.py @@ -0,0 +1,754 @@ +"""Closed, source-only PRPERTYP evidence for an exact current-money parent. + +No attachment, public pin override, population preparation or publication API. +The low-level reader remains non-authoritative inside this owned envelope. +""" + +from __future__ import annotations + +import csv +import hashlib +import io +import json +import os +import stat +import struct +import sys +import tempfile +import weakref +from collections.abc import Mapping +from dataclasses import InitVar, dataclass +from importlib import metadata, resources +from pathlib import Path +from types import CodeType, FunctionType + +import pandas as pd + +from . import asec_current_money_source as money +from . import asec_person_coverage_source as literal +from . import source_csv_builtin as _csv_builtin +from .education_assistance_source import ASEC_EDUCATION_ASSISTANCE_ARCHIVES + +ARTIFACT_KIND = "microcosm.asec_coverage_authentication.v1" +MAGIC = b"MCASCOV\x01" +_MEMBER_PINS = tuple( + (year, p.member, p.zip_sha256, p.member_sha256, p.rows, p.member_size_bytes) + for year, p in sorted(ASEC_EDUCATION_ASSISTANCE_ARCHIVES.items()) +) +_HEADER_MAX = 262_144 +_BODY_MAX = 268_435_456 +_ROW_MAX = 1_048_576 +_TOKEN_MAX = 65_536 +_CSV_HEADER_MAX = 65_536 +_MAX_PERSONS = 600_000 +_CHUNK = 65_536 +_INT_COLUMNS = ( + "person_id", + "person_household_id", + "source_year", + "source_household_id", + "A_LINENO", + "A_AGE", +) +_STRING_COLUMNS = ("PERIDNUM", "PRPERTYP", "PRPERTYP_state") +COLUMNS = ( + "person_id", + "person_household_id", + "source_year", + "source_household_id", + "A_LINENO", + "A_AGE", + "PERIDNUM", + "PRPERTYP", + "PRPERTYP_state", +) +_PARENT_COLUMNS = ( + "person_id", + "person_household_id", + "source_year", + "source_household_id", + "A_LINENO", + "A_AGE", + "PERIDNUM", +) +_NUMBERS = struct.Struct("<6q") +_LENGTH = struct.Struct("= 0, "COVERAGE_BODY_BYTES") + if self.quoted: + if byte == 34: + self.quoted, self.after_quote = False, True + elif self.after_quote and byte == 34: + self.quoted, self.after_quote = True, False + elif byte in (10, 13): + if self.header: + self._end_header() + self.row = self.field = 0 + self.column = 0 + self.header = self.after_quote = False + self.start = True + self.after_cr = byte == 13 + elif byte == 44: + self.field = 0 + self.column += 1 + self.start, self.after_quote = True, False + else: + self.quoted = self.start and byte == 34 + self.start = self.after_quote = False + + +def _capture(path, destination, *, size, digest, budget): + """The student-controls exact-size, nonblocking private-capture pattern. + + Add raw record/token limits during that same copy, plus a private inode + identity retained until parsing finishes. No second coordinate parse. + """ + _require( + type(size) is int and 0 < size <= literal.MAX_MEMBER_BYTES, + "COVERAGE_MEMBER_BYTES", + ) + descriptor = os.open(path, os.O_RDONLY | os.O_NONBLOCK | os.O_NOFOLLOW) + try: + before = os.fstat(descriptor) + _require(stat.S_ISREG(before.st_mode), "COVERAGE_SOURCE_FILE_KIND") + _require(before.st_size == size, "COVERAGE_SOURCE_SIZE") + count, hashed, guard = 0, hashlib.sha256(), _CsvBounds(budget) + with ( + os.fdopen(descriptor, "rb", buffering=0, closefd=False) as src, + destination.open("xb") as dst, + ): + while chunk := src.read(min(_CHUNK, size - count + 1)): + count += len(chunk) + _require(count <= size, "COVERAGE_SOURCE_SIZE") + guard.feed(chunk) + hashed.update(chunk) + dst.write(chunk) + dst.flush() + captured = _identity(os.fstat(dst.fileno())) + _require( + count == size and _identity(before) == _identity(os.fstat(descriptor)), + "COVERAGE_SOURCE_CHANGED", + ) + _require(hashed.hexdigest() == digest, "COVERAGE_SOURCE_DIGEST") + return captured + finally: + os.close(descriptor) + + +def _parent_binding(source): + _require( + type(source) is money.AuthenticatedCurrentMoneySource, + "COVERAGE_AUTHENTICATED_PARENT", + ) + source.validate() + return { + "authority_type": "AuthenticatedCurrentMoneySource", + "source_identity_sha256": _sha(source.source.identity), + "frame_sha256": money._frame_signature(source.frame), + "scope_sha256": _sha(source.scope.identity), + } + + +def _partitions(person): + published_to_parent, parent_to_published = {}, {} + for year, native, parent in person[ + ["source_year", "source_household_id", "person_household_id"] + ].itertuples(index=False, name=None): + published = (year, native) + _require(parent > 0, "COVERAGE_HOUSEHOLD_COORDINATE") + _require( + published_to_parent.setdefault(published, parent) == parent, + "COVERAGE_HOUSEHOLD_SPLIT", + ) + _require( + parent_to_published.setdefault(parent, published) == published, + "COVERAGE_HOUSEHOLD_MERGE", + ) + return { + "relation": "bidirectional_partition_bijection", + "published_key": ["source_year", "source_household_id"], + "parent_key": "person_household_id", + "households": len(parent_to_published), + "rows": len(person), + } + + +def _utf8_size(token): + _require(type(token) is str and len(token) <= _TOKEN_MAX, "COVERAGE_TOKEN_BYTES") + size = sum( + 1 if ord(c) < 128 else 2 if ord(c) < 2048 else 3 if ord(c) < 65536 else 4 + for c in token + ) + _require(size <= _TOKEN_MAX, "COVERAGE_TOKEN_BYTES") + return size + + +def _body(table): + _require(0 < len(table) <= _MAX_PERSONS, "COVERAGE_ROWS") + result = bytearray() + for row in table[ + [ + "person_id", + "person_household_id", + "source_year", + "source_household_id", + "A_LINENO", + "A_AGE", + "PERIDNUM", + "PRPERTYP", + "PRPERTYP_state", + ] + ].itertuples(index=False, name=None): + sizes = [_utf8_size(token) for token in row[6:]] + size = _NUMBERS.size + 3 * _LENGTH.size + sum(sizes) + _require(size <= _ROW_MAX, "COVERAGE_ROW_BYTES") + _require(len(result) + _LENGTH.size + size <= _BODY_MAX, "COVERAGE_BODY_BYTES") + result.extend(_LENGTH.pack(size)) + result.extend(_NUMBERS.pack(*row[:6])) + for token, length in zip(row[6:], sizes, strict=True): + result.extend(_LENGTH.pack(length)) + result.extend(token.encode("utf-8")) + return bytes(result) + + +def _decode_rows(body, rows): + _require(type(body) is bytes and len(body) <= _BODY_MAX, "COVERAGE_BODY_BYTES") + _require(type(rows) is int and 0 < rows <= _MAX_PERSONS, "COVERAGE_ROWS") + view, offset = memoryview(body), 0 + for _ in range(rows): + _require(offset + 4 <= len(view), "COVERAGE_ROW_BYTES") + size = _LENGTH.unpack_from(view, offset)[0] + offset += 4 + end = offset + size + _require( + _NUMBERS.size + 12 <= size <= _ROW_MAX and end <= len(view), + "COVERAGE_ROW_BYTES", + ) + values = list(_NUMBERS.unpack_from(view, offset)) + offset += _NUMBERS.size + for _ in _STRING_COLUMNS: + _require(offset + 4 <= end, "COVERAGE_TOKEN_BYTES") + length = _LENGTH.unpack_from(view, offset)[0] + offset += 4 + _require( + length <= _TOKEN_MAX and offset + length <= end, "COVERAGE_TOKEN_BYTES" + ) + values.append(bytes(view[offset : offset + length]).decode("utf-8")) + offset += length + _require(offset == end, "COVERAGE_ROW_BYTES") + yield values + _require(offset == len(view), "COVERAGE_BODY_BYTES") + + +def _bounded_header(header): + # Count ASCII JSON bytes without first allocating escaped strings or a + # serialized header. This vocabulary is owned metadata, never row tokens. + remaining = _HEADER_MAX + + def charge(size): + nonlocal remaining + remaining -= size + _require(remaining >= 0, "COVERAGE_HEADER_BYTES") + + def visit(value, depth=0): + _require(depth <= 16, "COVERAGE_HEADER_DEPTH") + if isinstance(value, str): + charge(2) + for c in value: + charge( + 2 + if c in '\\"\b\f\n\r\t' + else 6 + if ord(c) < 32 or 127 <= ord(c) <= 65535 + else 12 + if ord(c) > 65535 + else 1 + ) + elif isinstance(value, (list, tuple, dict)): + charge(2 + max(0, len(value) - 1)) + if isinstance(value, dict): + for key, item in value.items(): + _require(type(key) is str, "COVERAGE_HEADER_TYPE") + visit(key, depth + 1) + charge(1) + visit(item, depth + 1) + else: + for item in value: + visit(item, depth + 1) + else: + _require( + value is None + or type(value) is bool + or (type(value) is int and -(2**63) <= value < 2**63), + "COVERAGE_HEADER_TYPE", + ) + charge(len(_json(value))) + + visit(header) + return _json(header) + + +@dataclass(frozen=True, eq=False, slots=True, weakref_slot=True) +class AuthenticatedAsecCoverage: + """Issued immutable bytes, with fresh receipt and table views on demand.""" + + _header: bytes + _body: bytes + _token: InitVar[object] = None + + def __post_init__(self, _token): + _require(_token is _TOKEN, "COVERAGE_CONSTRUCTOR_UNAVAILABLE") + _ISSUED[self] = (_sha(self._header), _sha(self._body)) + + def validate(self): + try: + _require( + type(self._header) is bytes and 0 < len(self._header) <= _HEADER_MAX, + "COVERAGE_HEADER_BYTES", + ) + _require( + type(self._body) is bytes and len(self._body) <= _BODY_MAX, + "COVERAGE_BODY_BYTES", + ) + _require( + _ISSUED.get(self) == (_sha(self._header), _sha(self._body)), + "COVERAGE_ISSUED_CONTENT_CHANGED", + ) + header = json.loads(self._header) + _require( + _json(header["implementation"]) == _json(_implementation()), + "COVERAGE_IMPLEMENTATION_CHANGED", + ) + _require(0 < header["rows"] <= _MAX_PERSONS, "COVERAGE_ROWS") + except AsecCoverageAuthenticationError: + raise + except (OSError, ValueError, TypeError, KeyError, AttributeError): + raise AsecCoverageAuthenticationError("COVERAGE_CONTENT_REFUSAL") from None + + @property + def receipt(self): + self.validate() + return json.loads(self._header) + + @property + def content_sha256(self): + self.validate() + return _sha(self._header + self._body) + + def to_bytes(self): + self.validate() + payload = MAGIC + _LENGTH.pack(len(self._header)) + self._header + self._body + return payload + hashlib.sha256(payload).digest() + + def table(self): + self.validate() + table = pd.DataFrame( + _decode_rows(self._body, self.receipt["rows"]), columns=COLUMNS + ) + for name in _STRING_COLUMNS: + table[name] = pd.array(table[name], dtype="string") + return table + + +def verify_asec_coverage_parent(coverage, source): + """Verify the closed artifact against the exact named live receiving parent.""" + try: + _require( + type(coverage) is AuthenticatedAsecCoverage, + "COVERAGE_AUTHENTICATED_ARTIFACT", + ) + coverage.validate() + _require( + coverage.receipt["parent"] == _parent_binding(source), + "COVERAGE_PARENT_CHANGED", + ) + except AsecCoverageAuthenticationError: + raise + except (OSError, ValueError, TypeError, KeyError, AttributeError): + raise AsecCoverageAuthenticationError("COVERAGE_PARENT_REFUSAL") from None + + +def _reconstruct(source, member_paths: Mapping[int, str | Path]): + before = _implementation() + parent = _parent_binding(source) + _require( + isinstance(member_paths, Mapping) + and set(member_paths) == {2022, 2023, 2024} + and all(type(year) is int for year in member_paths) + and all(isinstance(p, (str, Path)) for p in member_paths.values()), + "COVERAGE_MEMBER_PATHS", + ) + _require( + tuple(p[0] for p in _MEMBER_PINS) == (2022, 2023, 2024) + and len({p[1] for p in _MEMBER_PINS}) == 3, + "COVERAGE_PIN_COHORTS", + ) + person = source.frame.person + _require(0 < len(person) <= _MAX_PERSONS, "COVERAGE_ROWS") + _require( + set(_INT_COLUMNS + ("PERIDNUM",)) <= set(person), "COVERAGE_PARENT_COLUMNS" + ) + _require( + all(str(person[name].dtype) == "int64" for name in _INT_COLUMNS), + "COVERAGE_PARENT_DTYPE", + ) + # Build only from the validated parent's owned scope/native columns; never + # from model age or a caller's DataFrame/hash/receipt. + person = person[ + [ + "person_id", + "person_household_id", + "source_year", + "source_household_id", + "A_LINENO", + "A_AGE", + "PERIDNUM", + ] + ].copy(deep=True) + _require( + tuple(person.person_id) == source.scope.person_ids + and tuple(person.person_household_id) == source.scope.person_household_ids + and tuple(person.source_year) == source.scope.person_years, + "COVERAGE_PARENT_COORDINATES", + ) + _require(_parent_binding(source) == parent, "COVERAGE_PARENT_CHANGED") + roster = person[list(literal.ROSTER_COLUMNS)].copy().reset_index(drop=True) + _require( + tuple(roster.PERIDNUM) == source.scope.person_native_keys, + "COVERAGE_PARENT_NATIVE_KEYS", + ) + membership = _partitions(person) + with tempfile.TemporaryDirectory(prefix="microcosm-asec-coverage-") as directory: + paths, identities, sources = {}, {}, [] + budget = [_BODY_MAX] + pins = {pin[0]: pin for pin in _MEMBER_PINS} + for year in (2022, 2023, 2024): + _, member, archive, digest, rows, size = pins[year] + _require( + type(rows) is int + and 0 < rows <= _MAX_PERSONS + and int((roster.source_year == year).sum()) == rows, + "COVERAGE_COHORT_ROWS", + ) + path = Path(directory) / member + identities[year] = _capture( + member_paths[year], path, size=size, digest=digest, budget=budget + ) + paths[year] = path + sources.append( + { + "source_year": year, + "survey_year": year + 1, + "member": member, + "archive_sha256": archive, + "member_sha256": digest, + "member_bytes": size, + "rows": rows, + } + ) + projection, receipt = literal.read_asec_person_coverage_source( + paths, person_roster=roster + ) + for year, path in paths.items(): + _require( + _identity(path.stat(follow_symlinks=False)) == identities[year], + "COVERAGE_PRIVATE_SOURCE_CHANGED", + ) + for name in _INT_COLUMNS[:2]: + projection[name] = person[name].to_numpy(copy=True) + body = _body(projection) + _require(_parent_binding(source) == parent, "COVERAGE_PARENT_CHANGED") + _require( + _json(before) == _json(_implementation()), "COVERAGE_IMPLEMENTATION_CHANGED" + ) + header = { + "artifact_kind": ARTIFACT_KIND, + "encoding": "length_prefixed_utf8_fixed_int64_v1", + "columns": COLUMNS, + "rows": len(projection), + "body_sha256": _sha(body), + "parent": parent, + "sources": sources, + "household_partitions": membership, + "source_authenticated": True, + "named_parent_binding_authenticated": True, + "archive_bytes_read": False, + "source_scope": "all_three_complete_original_person_cohorts", + "age_relation": { + "relation": "original_csv_A_AGE_equals_parent_A_AGE", + "comparison": "strict_unsigned_decimal_int64_identity", + "evidence": "authenticated_private_csv_read_in_this_reconstruction", + "compared_rows": len(projection), + "conflicts": 0, + "model_age_used": False, + }, + "coverage_status": "literal_source_fields_only", + "period_harmonized": False, + "cross_survey_coverage_equivalence_established": False, + "domain_authority": False, + "original_design_weight_semantics_established": False, + "release_eligible": False, + "literal_reader_receipt": receipt, + "implementation": before, + } + return AuthenticatedAsecCoverage(_bounded_header(header), body, _token=_TOKEN) + + +def _compare_candidate(path, payload): + # No candidate header, token, checksum or self-declared source is decoded. + descriptor = os.open(path, os.O_RDONLY | os.O_NONBLOCK | os.O_NOFOLLOW) + try: + before = os.fstat(descriptor) + _require(stat.S_ISREG(before.st_mode), "COVERAGE_CANDIDATE_FILE_KIND") + _require(before.st_size == len(payload), "COVERAGE_CANDIDATE_BYTES") + offset = 0 + with os.fdopen(descriptor, "rb", buffering=0, closefd=False) as handle: + while chunk := handle.read(min(_CHUNK, len(payload) - offset + 1)): + _require( + chunk == memoryview(payload)[offset : offset + len(chunk)], + "COVERAGE_CANONICAL_BYTES", + ) + offset += len(chunk) + _require( + offset == len(payload) + and _identity(before) == _identity(os.fstat(descriptor)), + "COVERAGE_CANDIDATE_CHANGED", + ) + finally: + os.close(descriptor) + + +def authenticate_asec_coverage(source, *, member_paths, candidate_path=None): + """Independently reconstruct before comparing optional candidate bytes.""" + try: + expected = _reconstruct(source, member_paths) + if candidate_path is not None: + _compare_candidate(candidate_path, expected.to_bytes()) + verify_asec_coverage_parent(expected, source) + return expected + except AsecCoverageAuthenticationError: + raise + except ( + OSError, + ValueError, + TypeError, + KeyError, + AttributeError, + OverflowError, + csv.Error, + ): + raise AsecCoverageAuthenticationError("COVERAGE_SOURCE_REFUSAL") from None + + +# The imported literal parser is code authority, not a caller-selectable +# decoder. Refuse monkeypatched parsing/status/field code even before issuance. +_LITERAL_AUTHORITY = _runtime_code(literal) diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/asec_current_money.py b/packages/microcosm-build/src/microcosm/build/us_runtime/asec_current_money.py new file mode 100644 index 000000000..e1b4c99bc --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/asec_current_money.py @@ -0,0 +1,1099 @@ +"""Pure ASEC current-money decoding and price restatement. + +No resource discovery or source authentication happens here. Only the separate +resource loader reads installed code. Authenticated authority can only come from +the separate verified checkpoint loader; synthetic identities remain distinct. +""" + +import json +import struct +import weakref +from dataclasses import InitVar, dataclass +from dataclasses import fields as dataclass_fields +from enum import IntEnum +from hashlib import sha256 +from types import MappingProxyType + +import numpy as np +import pandas as pd + +RECIPE = "us_asec_current_money_ccpiu_2024_v1" +ZERO_POLICY = "frozen_fillna_zero_origin_unresolved" +RESTORED_SOURCE_KIND = "asec_v4_with_household_and_person_income_observations_v1" +ENCODED_ZERO_POLICY = "authenticated_census_csv_encoded_zero_v1" +RESTORED_FIELD_COLUMNS = MappingProxyType({"PTOTVAL": "asec_PTOTVAL"}) +RESTORED_FIELD_ZERO_POLICY = MappingProxyType({"PTOTVAL": ENCODED_ZERO_POLICY}) + + +class ZeroOrigin(IntEnum): + """Source-encoding provenance; no code establishes a respondent answer.""" + + NOT_ZERO = 0 + FROZEN_FILLNA_UNRESOLVED = 1 + AUTHENTICATED_CENSUS_ENCODED = 2 + + +FIELDS = tuple( + "ANN_VAL CAP_VAL CHSP_VAL CSP_VAL DIS_VAL1 DIS_VAL2 DIV_VAL DST_VAL1 DST_VAL1_YNG DST_VAL2 DST_VAL2_YNG ED_VAL FRSE_VAL HTOTVAL INT_VAL OI_VAL PHIP_VAL PMED_VAL PNSN_VAL POTC_VAL PTOTVAL RETCB_VAL RNT_VAL SEMP_VAL SPM_CAPHOUSESUB SPM_CHILDCAREXPNS SPM_ENGVAL SSI_VAL SS_VAL UC_VAL VET_VAL WC_VAL WSAL_VAL".split() +) +RESOURCE_PINS = ( + "09cefd4d08968cf1dc4dcdc787deef88c77b84549dfb0baa6bf9cbd9e3ce7352", + "88cec207a3c73e5de93d677acf16e8db9ddae9761c644c06da92cacf0feb3f63", + "c56058eaf08b68b2add4a5d2b366559ea6cc5238b039031be7094ef1f6457ff7", +) +MICROUNIT_VERSION = "0.1.0" +MICROUNIT_SHA256 = "79007065113f9600601433dc2633f4392c675e28b87a7383ab8a1f7c6279d47e" +RESOURCE_MAX_BYTES = 512 * 1024 +HEADER_MAX_BYTES = 64 * 1024 +MAX_PERSONS = 1_000_000 +MAX_HOUSEHOLDS = 400_000 +_STATE_TOKEN = object() +_SOURCE_TOKEN = object() +# Keep issuance outside publicly writable evidence objects. Keys use identity, +# not dataclass value equality: independently reconstructed evidence stays equal +# without lending its authority to an unissued copy. Weak references retire +# entries with their objects, and retained bytes are bounded by HEADER_MAX_BYTES. +_SOURCE_ISSUANCE: dict[int, tuple[weakref.ReferenceType, bytes]] = {} + + +class MoneyRefusalError(ValueError): + """Sanitized field/reason refusal; never includes row identifiers or values.""" + + def __init__(self, reason: str, field: str = "contract"): + self.reason = reason + self.field = field if field in FIELDS else "contract" + super().__init__(f"{self.field}: {reason}") + + +def _require(condition: bool, reason: str, field: str = "contract") -> None: + if not condition: + raise MoneyRefusalError(reason, field) + + +def _sha(value: bytes) -> str: + return sha256(value).hexdigest() + + +def _json(value: object) -> bytes: + return json.dumps( + value, sort_keys=True, separators=(",", ":"), allow_nan=False, ensure_ascii=True + ).encode("ascii") + + +def _parse(value: bytes, limit: int = RESOURCE_MAX_BYTES) -> dict: + _require(type(value) is bytes and 0 < len(value) <= limit, "JSON_SIZE_OR_TYPE") + + def pairs(items): + result = {} + for key, item in items: + _require(key not in result, "DUPLICATE_JSON_KEY") + result[key] = item + return result + + def constant(_): + raise MoneyRefusalError("NONFINITE_JSON") + + try: + result = json.loads(value, object_pairs_hook=pairs, parse_constant=constant) + except MoneyRefusalError: + raise + except (ValueError, RecursionError) as exc: + raise MoneyRefusalError("MALFORMED_JSON") from exc + _require(type(result) is dict, "JSON_OBJECT_REQUIRED") + return result + + +def _digest(value: str) -> bool: + return ( + type(value) is str + and len(value) == 64 + and all(c in "0123456789abcdef" for c in value) + ) + + +@dataclass(frozen=True) +class AuthenticatedAsecSource: + """Immutable evidence issued only by the actual checkpoint verification loader.""" + + evidence: bytes = b"" + _token: InitVar[object] = None + + def __post_init__(self, _token): + _require(_token is _SOURCE_TOKEN, "AUTHENTICATED_SOURCE_UNAVAILABLE") + self._validate_content() + key = id(self) + + def discard(reference): + issued = _SOURCE_ISSUANCE.get(key) + if issued is not None and issued[0] is reference: + del _SOURCE_ISSUANCE[key] + + _SOURCE_ISSUANCE[key] = (weakref.ref(self, discard), self.evidence) + + def _validate(self): + issued = _SOURCE_ISSUANCE.get(id(self)) + evidence = self.evidence + _require( + type(self) is AuthenticatedAsecSource + and issued is not None + and issued[0]() is self + and type(evidence) is bytes + and evidence == issued[1], + "AUTHENTICATED_SOURCE_ISSUANCE", + ) + return evidence + + def _validate_content(self): + data = _parse(self.evidence, HEADER_MAX_BYTES) + restored = data.get("source_kind") == RESTORED_SOURCE_KIND + extra = ( + { + "person_income_attachment_sha256", + "field_source_columns", + "field_zero_origin_policy", + } + if restored + else set() + ) + _require( + set(data) + == extra + | { + "schema_version", + "source_kind", + "source_authentication", + "parent_sha256", + "attachment_sha256", + "cohorts", + "sidecars", + "field_roster", + "zero_origin_policy", + "scope_sha256", + "input_sha256", + "frame_sha256", + "verification_sha256", + }, + "AUTHENTICATED_SOURCE_SCHEMA", + ) + _require( + data["schema_version"] == (2 if restored else 1) + and data["source_kind"] + == ( + RESTORED_SOURCE_KIND + if restored + else "asec_v4_with_household_observations_v1" + ) + and data["source_authentication"] == "checkpoint_bytes_verified" + and data["field_roster"] == list(FIELDS) + and data["zero_origin_policy"] == ZERO_POLICY, + "AUTHENTICATED_SOURCE_SCHEMA", + ) + _require( + all( + _digest(data[name]) + for name in ( + "parent_sha256", + "attachment_sha256", + "scope_sha256", + "input_sha256", + "frame_sha256", + "verification_sha256", + ) + ), + "SOURCE_PIN", + ) + if restored: + _require( + _digest(data["person_income_attachment_sha256"]) + and data["field_source_columns"] == RESTORED_FIELD_COLUMNS + and data["field_zero_origin_policy"] == RESTORED_FIELD_ZERO_POLICY, + "RESTORED_SOURCE_SCHEMA", + ) + _require(self.evidence == _json(data), "AUTHENTICATED_SOURCE_SCHEMA") + + @property + def identity(self) -> bytes: + # Return the same immutable bytes checked, not a second mutable lookup. + return self._validate() + + +def _restored_source(source) -> bool: + return ( + type(source) is AuthenticatedAsecSource + and _parse(source.identity)["source_kind"] == RESTORED_SOURCE_KIND + ) + + +def _origin_code(spec, field: str) -> ZeroOrigin: + return ( + ZeroOrigin.AUTHENTICATED_CENSUS_ENCODED + if field == "PTOTVAL" and _restored_source(spec.source) + else ZeroOrigin.FROZEN_FILLNA_UNRESOLVED + ) + + +def _header_provenance(spec) -> dict: + if _restored_source(spec.source): + return { + "schema_version": 2, + "field_zero_origin_policy": dict(RESTORED_FIELD_ZERO_POLICY), + } + return {"schema_version": 1} + + +def _authentication(source) -> str: + return ( + "checkpoint_bytes_verified" + if type(source) is AuthenticatedAsecSource + else "synthetic_unverified" + ) + + +@dataclass(frozen=True) +class SyntheticAsecSource: + """Explicit invented pins, never promoted to authenticated source authority.""" + + parent_sha256: str + attachment_sha256: str + cohort_sha256: tuple[tuple[int, str], ...] + sidecar_sha256: tuple[tuple[str, str], ...] + selection_sha256: str + source_kind: str = "asec_v4_with_household_observations_v1" + field_roster: tuple[str, ...] = FIELDS + zero_origin_policy: str = ZERO_POLICY + + def __post_init__(self): + _require( + all( + _digest(v) + for v in ( + self.parent_sha256, + self.attachment_sha256, + self.selection_sha256, + ) + ), + "SOURCE_PIN", + ) + _require( + type(self.cohort_sha256) is tuple and len(self.cohort_sha256) == 3, + "COHORT_PINS", + ) + _require( + all( + type(p) is tuple and len(p) == 2 and type(p[0]) is int and _digest(p[1]) + for p in self.cohort_sha256 + ), + "COHORT_PINS", + ) + _require( + tuple(p[0] for p in self.cohort_sha256) == (2022, 2023, 2024), + "SOURCE_YEAR_MAPPING", + ) + _require( + type(self.sidecar_sha256) is tuple and len(self.sidecar_sha256) == 2, + "SIDECAR_PINS", + ) + _require( + all( + type(p) is tuple and len(p) == 2 and _digest(p[1]) + for p in self.sidecar_sha256 + ), + "SIDECAR_PINS", + ) + _require( + tuple(p[0] for p in self.sidecar_sha256) + == ("ED_VAL.archive", "ED_VAL.member"), + "SIDECAR_PINS", + ) + _require( + self.source_kind == "asec_v4_with_household_observations_v1", + "UNSUPPORTED_PARENT_KIND", + ) + _require( + type(self.field_roster) is tuple and self.field_roster == FIELDS, + "FIELD_ROSTER", + ) + _require(self.zero_origin_policy == ZERO_POLICY, "ZERO_ORIGIN_POLICY") + + @property + def identity(self) -> bytes: + return _json( + { + **{f.name: getattr(self, f.name) for f in dataclass_fields(self)}, + "source_authentication": "synthetic_unverified", + } + ) + + +@dataclass(frozen=True) +class CurrentMoneyResources: + """Explicit immutable resource bytes and inspected-code/runtime claims. + + Pure compilation checks these claims, not installed files. Use the separate + loader for installed verification. Source authentication belongs to the + separate closed checkpoint loader. + """ + + domains: bytes + price: bytes + consumers: bytes + execution_identity: bytes + inspected_version: str + inspected_module_sha256: str + + def __post_init__(self): + _require( + all( + type(x) is bytes + for x in ( + self.domains, + self.price, + self.consumers, + self.execution_identity, + ) + ), + "RESOURCE_BYTES_REQUIRED", + ) + + +@dataclass(frozen=True) +class MoneyDomain: + name: str + entity: str + column: str + grain: str + minimum: int + maximum: int + zero_semantics: str + + +@dataclass(frozen=True) +class CurrentMoneySpec: + """Closed v1 resources and issued source evidence; no mutable user mappings.""" + + resources: CurrentMoneyResources + source: SyntheticAsecSource | AuthenticatedAsecSource + fields: tuple[MoneyDomain, ...] + factors: tuple[tuple[int, str, str], ...] + identity: bytes + + @property + def sha256(self) -> str: + return _sha(self.identity) + + +def compile_asec_current_money_spec( + resources: CurrentMoneyResources, + source_descriptor: SyntheticAsecSource | AuthenticatedAsecSource, +) -> CurrentMoneySpec: + """Compile pinned declarations; this does not change live nominal consumers.""" + _require( + type(resources) is CurrentMoneyResources + and type(source_descriptor) in (SyntheticAsecSource, AuthenticatedAsecSource), + "SYNTHETIC_SOURCE_REQUIRED", + ) + if type(source_descriptor) is AuthenticatedAsecSource: + source_descriptor._validate() + else: + source_descriptor.__post_init__() + resources.__post_init__() + documents = tuple( + _parse(v) for v in (resources.domains, resources.price, resources.consumers) + ) + # Immutable v1 pins close every nested key, domain, writer, year and address. + # A changed convention requires a reviewed resource/recipe revision. + _require( + tuple( + _sha(v) for v in (resources.domains, resources.price, resources.consumers) + ) + == RESOURCE_PINS, + "RESOURCE_FINGERPRINT", + ) + _require( + resources.inspected_version == MICROUNIT_VERSION + and resources.inspected_module_sha256 == MICROUNIT_SHA256, + "MICROUNIT_CONTRACT_MISMATCH", + ) + execution = _parse(resources.execution_identity, HEADER_MAX_BYTES) + _require( + set(execution) == {"modules", "dependencies", "platform"}, "EXECUTION_IDENTITY" + ) + _require( + type(execution["modules"]) is dict + and set(execution["modules"]) + == { + "asec_current_money.py", + "_asec_current_money_codec.py", + "asec_current_money_resources.py", + }, + "EXECUTION_MODULES", + ) + _require( + all(_digest(v) for v in execution["modules"].values()), "EXECUTION_MODULES" + ) + _require( + type(execution["dependencies"]) is dict + and set(execution["dependencies"]) == {"python", "numpy", "pandas"}, + "EXECUTION_DEPENDENCIES", + ) + _require( + type(execution["platform"]) is dict + and set(execution["platform"]) + == {"system", "machine", "byteorder", "python_implementation"}, + "EXECUTION_PLATFORM", + ) + _require( + all( + type(v) is str and 0 < len(v) <= 128 + for group in (execution["dependencies"], execution["platform"]) + for v in group.values() + ), + "EXECUTION_IDENTITY", + ) + domains, price, _ = documents + fields = tuple( + MoneyDomain( + **{ + k: f[k] + for k in ( + "name", + "entity", + "column", + "grain", + "minimum", + "maximum", + "zero_semantics", + ) + } + ) + for f in domains["fields"] + ) + factors = tuple( + ( + y, + f"174.4/{price['cells'][str(y)]}", + struct.pack(" None: + _require(type(spec) is CurrentMoneySpec, "SPEC_TYPE") + _require( + spec == compile_asec_current_money_spec(spec.resources, spec.source), + "SPEC_BINDING", + ) + + +@dataclass(frozen=True) +class AsecMoneyScope: + """Ordered source coordinates supplied explicitly with projected tables. + + Pandas labels are representation only. IDs/native keys here belong to each + positional row, and must be projected in the same permutation as the table. + No original source authentication is inferred from this synthetic contract. + """ + + person_ids: tuple[int, ...] + household_ids: tuple[int, ...] + person_household_ids: tuple[int, ...] + person_spm_ids: tuple[int, ...] + person_years: tuple[int, ...] + household_years: tuple[int, ...] + person_native_keys: tuple[str, ...] + household_native_keys: tuple[str, ...] + + def __post_init__(self): + p, h = len(self.person_ids), len(self.household_ids) + _require(0 < p <= MAX_PERSONS and 0 < h <= MAX_HOUSEHOLDS, "SOURCE_SIZE") + for name in ( + "person_ids", + "household_ids", + "person_household_ids", + "person_spm_ids", + "person_years", + "household_years", + ): + vector = getattr(self, name) + _require( + type(vector) is tuple + and all(type(x) is int and -(2**63) <= x < 2**63 for x in vector), + "SCOPE_INTEGER_VECTOR", + ) + _require( + len(vector) == (h if name.startswith("household") else p), + "SCOPE_LENGTH", + ) + _require(len(set(self.person_ids)) == p, "DUPLICATE_PERSON_ID") + _require(len(set(self.household_ids)) == h, "DUPLICATE_HOUSEHOLD_ID") + _require( + set(self.person_household_ids) == set(self.household_ids), + "HOUSEHOLD_MEMBERSHIP", + ) + _require( + set(self.person_years) <= {2022, 2023, 2024} + and set(self.household_years) <= {2022, 2023, 2024}, + "SOURCE_YEAR_MAPPING", + ) + expected = dict(zip(self.household_ids, self.household_years, strict=True)) + _require( + all( + expected[g] == y + for g, y in zip( + self.person_household_ids, self.person_years, strict=True + ) + ), + "MIXED_HOUSEHOLD_YEAR", + ) + spm_years = {} + for spm, year in zip(self.person_spm_ids, self.person_years, strict=True): + _require(spm not in spm_years or spm_years[spm] == year, "MIXED_SPM_YEAR") + spm_years[spm] = year + for name, years, count in ( + ("person_native_keys", self.person_years, p), + ("household_native_keys", self.household_years, h), + ): + keys = getattr(self, name) + _require( + type(keys) is tuple + and len(keys) == count + and all(type(k) is str and 0 < len(k) <= 256 for k in keys), + "NATIVE_KEYS", + ) + _require( + len(set(zip(years, keys, strict=True))) == count, "DUPLICATE_NATIVE_KEY" + ) + + @property + def identity(self) -> bytes: + # Digests bound the header, while the active scope retains real coordinates. + return _json( + {f.name: _sha(_json(getattr(self, f.name))) for f in dataclass_fields(self)} + ) + + +@dataclass(frozen=True) +class AsecMoneyViews: + """Explicit raw projections plus their ordered coordinate association.""" + + person: pd.DataFrame + household: pd.DataFrame + scope: AsecMoneyScope + + +class CodebookStatus(IntEnum): + MISSING_NULL = 0 + AMOUNT_NONZERO = 1 + ZERO_NONE_OR_NIU = 2 + ZERO_DOLLARS_AS_CODED = 3 + DECLARED_NIU = 4 + + +_ZERO_STATUS = { + "none_or_niu_not_distinguishable_from_amount_alone": CodebookStatus.ZERO_NONE_OR_NIU, + "valid_zero_dollars": CodebookStatus.ZERO_DOLLARS_AS_CODED, + "valid_zero_dollars_or_none_as_described": CodebookStatus.ZERO_DOLLARS_AS_CODED, + "niu": CodebookStatus.DECLARED_NIU, +} +_DTYPES = frozenset( + ( + "int8", + "uint8", + "int16", + "uint16", + "int32", + "uint32", + "int64", + "uint64", + "float64", + "Int8", + "UInt8", + "Int16", + "UInt16", + "Int32", + "UInt32", + "Int64", + "UInt64", + "Float64", + ) +) + + +@dataclass(frozen=True) +class MoneyField: + """Immutable canonical little-endian amounts and three uint8 evidence axes.""" + + name: str + amount_bytes: bytes + status_bytes: bytes + validity_bytes: bytes + zero_origin_bytes: bytes + + @property + def amounts(self) -> np.ndarray: + return np.frombuffer(self.amount_bytes, dtype=" np.ndarray: + return np.frombuffer(self.status_bytes, dtype="u1") + + @property + def validity(self) -> np.ndarray: + return np.frombuffer(self.validity_bytes, dtype="u1") + + @property + def zero_origin(self) -> np.ndarray: + return np.frombuffer(self.zero_origin_bytes, dtype="u1") + + +def _validate_field( + field: MoneyField, + domain: MoneyDomain, + count: int, + *, + nominal: bool, + zero_origin_code: ZeroOrigin, +) -> None: + _require(type(field) is MoneyField and field.name == domain.name, "FIELD_IDENTITY") + buffers = ( + field.amount_bytes, + field.status_bytes, + field.validity_bytes, + field.zero_origin_bytes, + ) + _require( + all(type(v) is bytes for v in buffers) + and tuple(map(len, buffers)) == (8 * count, count, count, count), + "FIELD_BUFFER_SHAPE", + domain.name, + ) + a, s, v, z = field.amounts, field.statuses, field.validity, field.zero_origin + _require( + np.isfinite(a).all() + and np.isin(s, [0, 1, 2, 3, 4]).all() + and np.isin(v, [0, 1]).all() + and np.isin(z, [0, 1, 2]).all(), + "FIELD_ENCODING", + domain.name, + ) + _require( + np.array_equal(v == 0, s == CodebookStatus.MISSING_NULL), + "INVALID_SLOT", + domain.name, + ) + _require( + np.array_equal(a != 0, s == CodebookStatus.AMOUNT_NONZERO) + and not np.signbit(a[a == 0]).any(), + "AMOUNT_STATUS_ENCODING", + domain.name, + ) + expected_zero = _ZERO_STATUS[domain.zero_semantics] + allowed = { + CodebookStatus.MISSING_NULL, + CodebookStatus.AMOUNT_NONZERO, + expected_zero, + } + if domain.name == "ANN_VAL": + allowed.add(CodebookStatus.DECLARED_NIU) + _require(np.isin(s, list(allowed)).all(), "FIELD_STATUS", domain.name) + _require( + type(zero_origin_code) is ZeroOrigin + and zero_origin_code != ZeroOrigin.NOT_ZERO, + "ZERO_ORIGIN_POLICY", + domain.name, + ) + expected_origin = (s == expected_zero).astype("u1") * int(zero_origin_code) + _require(np.array_equal(z, expected_origin), "ZERO_ORIGIN_ENCODING", domain.name) + factor_max = 1.0 if nominal else float("174.4") / float("163.6") + lo = min(domain.minimum, domain.minimum * factor_max) + hi = max(domain.maximum, domain.maximum * factor_max) + _require(((a >= lo) & (a <= hi)).all(), "AMOUNT_DOMAIN", domain.name) + if domain.name == "ANN_VAL": + _require((a >= 0).all(), "AMOUNT_DOMAIN", domain.name) + if nominal: + _require((a == np.floor(a)).all(), "FRACTIONAL_SOURCE", domain.name) + + +@dataclass(frozen=True) +class MoneyBindings: + """Expected replay identity, including the actual raw input fingerprint.""" + + spec: CurrentMoneySpec + header: bytes + + def __post_init__(self): + _spec(self.spec) + data = _parse(self.header, HEADER_MAX_BYTES) + keys = { + "schema_version", + "recipe", + "country", + "semantic", + "target_year", + "source_authentication", + "state", + "spec_sha256", + "scope_sha256", + "input_sha256", + "person_rows", + "household_rows", + "fields", + "dtype", + "evidence_dtype", + "person_year_sha256", + "household_year_sha256", + } + provenance = _header_provenance(self.spec) + _require( + set(data) == keys | set(provenance) and self.header == _json(data), + "HEADER_SCHEMA", + ) + fixed = { + **provenance, + "recipe": RECIPE, + "country": "us", + "semantic": "annual_current_money", + "target_year": 2024, + "source_authentication": _authentication(self.spec.source), + "state": "target_current", + "spec_sha256": self.spec.sha256, + "fields": list(FIELDS), + "dtype": " MoneyField: + return _field(self.fields, name) + + +@dataclass(frozen=True) +class RestatedAsecMoney: + bindings: MoneyBindings + fields: tuple[MoneyField, ...] + _token: InitVar[object] = None + + def __post_init__(self, _token): + _require(_token is _STATE_TOKEN, "STATE_CONSTRUCTOR_UNAVAILABLE") + + @property + def header(self) -> bytes: + return self.bindings.header + + def field(self, name: str) -> MoneyField: + return _field(self.fields, name) + + +@dataclass(frozen=True) +class SyntheticReadyCurrentMoney(RestatedAsecMoney): + """Complete invented fixture amounts. This is NOT production authority.""" + + +class ReadyCurrentMoney(RestatedAsecMoney): + """Complete amounts bound to verified source bytes; not release certification.""" + + def __new__(cls, *args, **kwargs): + _require( + kwargs.get("_token") is _STATE_TOKEN, "PRODUCTION_READINESS_UNAVAILABLE" + ) + return super().__new__(cls) + + +def _field(fields: tuple[MoneyField, ...], name: str) -> MoneyField: + _require(name in FIELDS, "UNKNOWN_FIELD") + return fields[FIELDS.index(name)] + + +def _validate_money(value, spec: CurrentMoneySpec, *, nominal: bool) -> None: + _spec(spec) + _require( + type(value.bindings) is MoneyBindings and value.bindings.spec == spec, + "SPEC_MISMATCH", + ) + value.bindings.__post_init__() + header = _parse(value.bindings.header, HEADER_MAX_BYTES) + _require( + type(value.fields) is tuple and len(value.fields) == len(FIELDS), "FIELD_ROSTER" + ) + for domain, field in zip(spec.fields, value.fields, strict=True): + _validate_field( + field, + domain, + header[domain.entity + "_rows"], + nominal=nominal, + zero_origin_code=_origin_code(spec, domain.name), + ) + + +def _index_identity(index: pd.Index) -> bytes: + # An intentionally narrow, honest representation contract; duplicate/nondefault + # signed-int64 labels are supported positionally. No label-based assignment. + _require( + type(index) in (pd.Index, pd.RangeIndex) and str(index.dtype) == "int64", + "UNSUPPORTED_INDEX", + ) + _require(index.name is None or type(index.name) is str, "UNSUPPORTED_INDEX_NAME") + descriptor = { + "type": type(index).__name__, + "name": index.name, + "values_sha256": _sha(index.to_numpy(dtype=" DecodedAsecMoney: + """Decode explicit raw columns positionally, preserving codebook evidence.""" + _require(type(raw_views) is AsecMoneyViews, "TYPED_STATE") + _spec(spec) + _require( + type(source_scope) is AsecMoneyScope + and type(raw_views.scope) is AsecMoneyScope, + "SCOPE_TYPE", + ) + source_scope.__post_init__() + _require(raw_views.scope == source_scope, "SCOPE_MISMATCH") + p, h = len(source_scope.person_ids), len(source_scope.household_ids) + for table, count in ((raw_views.person, p), (raw_views.household, h)): + _require( + type(table) is pd.DataFrame + and len(table) == count + and table.columns.is_unique, + "VIEW_SHAPE", + ) + scope_identity = _json( + { + "coordinates": _parse(source_scope.identity), + "person_index": _parse(_index_identity(raw_views.person.index)), + "household_index": _parse(_index_identity(raw_views.household.index)), + } + ) + result = [] + inputs = [] + # Sorting native group IDs is only a temporary positional validation index. + spm = np.asarray(source_scope.person_spm_ids, dtype="= 10001) & (values <= 99999) & valid).any(), + "DISPUTED_CODEBOOK_RANGE", + domain.name, + ) + _require( + not (((values < domain.minimum) | (values > domain.maximum)) & valid).any(), + "OUTSIDE_CODEBOOK_RANGE", + domain.name, + ) + # Bind received numeric bytes (including -0) and validity before normalization. + received = values.copy() + received[~valid] = 0.0 + inputs.append( + { + "field": domain.name, + "entity": domain.entity, + "column": domain.column, + "dtype": str(series.dtype), + "values_sha256": _sha(received.tobytes()), + "validity_sha256": _sha(valid.astype("u1").tobytes()), + } + ) + status = np.full(len(values), CodebookStatus.AMOUNT_NONZERO, dtype="u1") + zeros = valid & (values == 0) + status[zeros] = _ZERO_STATUS[domain.zero_semantics] + if domain.name == "ANN_VAL": + niu = valid & (values == -1) + status[niu] = CodebookStatus.DECLARED_NIU + values[niu] = 0.0 + status[~valid] = CodebookStatus.MISSING_NULL + values[~valid | (values == 0)] = 0.0 + field = MoneyField( + domain.name, + values.tobytes(), + status.tobytes(), + valid.astype("u1").tobytes(), + (zeros.astype("u1") * int(_origin_code(spec, domain.name))).tobytes(), + ) + _validate_field( + field, + domain, + len(table), + nominal=True, + zero_origin_code=_origin_code(spec, domain.name), + ) + if domain.grain == "spm_unit_repeated_on_person": + for vector in (values, status, valid, zeros): + sorted_values = vector[order] + _require( + not ((sorted_values[1:] != sorted_values[:-1]) & same).any(), + "INCONSISTENT_SPM_AMOUNT", + domain.name, + ) + result.append(field) + header = _json( + { + **_header_provenance(spec), + "recipe": RECIPE, + "country": "us", + "semantic": "annual_current_money", + "target_year": 2024, + "source_authentication": _authentication(spec.source), + "state": "target_current", + "spec_sha256": spec.sha256, + "scope_sha256": _sha(scope_identity), + "input_sha256": _sha(_json(inputs)), + "person_year_sha256": _sha( + np.asarray(source_scope.person_years, dtype=" RestatedAsecMoney: + """Restate once; target-year bytes and normalized zero bypass arithmetic.""" + _require(type(decoded) is DecodedAsecMoney, "TYPED_STATE") + _validate_money(decoded, spec, nominal=True) + header = _parse(decoded.bindings.header, HEADER_MAX_BYTES) + years = {} + for entity in ("person", "household"): + data = getattr(decoded, entity + "_year_bytes") + _require( + type(data) is bytes and len(data) == header[entity + "_rows"] * 8, + "YEAR_BUFFER", + ) + _require(_sha(data) == header[entity + "_year_sha256"], "YEAR_BINDING") + vector = np.frombuffer(data, dtype=" SyntheticReadyCurrentMoney | ReadyCurrentMoney: + """Require every amount; synthetic evidence cannot request verified readiness.""" + _require(type(production) is bool, "PRODUCTION_FLAG") + verified = type(spec.source) is AuthenticatedAsecSource + _require(not production or verified, "SYNTHETIC_NOT_PRODUCTION") + _require(not verified or production, "AUTHENTICATED_READINESS_REQUIRED") + _require(type(restated) is RestatedAsecMoney, "TYPED_STATE") + _validate_money(restated, spec, nominal=False) + for field in restated.fields: + _require(field.validity.all(), "MISSING_REQUIRED_AMOUNT", field.name) + ready_type = ReadyCurrentMoney if verified else SyntheticReadyCurrentMoney + return ready_type(restated.bindings, restated.fields, _token=_STATE_TOKEN) diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/asec_current_money_consumers_v1.json b/packages/microcosm-build/src/microcosm/build/us_runtime/asec_current_money_consumers_v1.json new file mode 100644 index 000000000..5ac33188e --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/asec_current_money_consumers_v1.json @@ -0,0 +1,628 @@ +{ + "addresses": [ + { + "column": "ANN_VAL", + "consumers": [ + "cps_carried", + "microunit" + ], + "cpi_owner": "us_asec_current_money_ccpiu_2024_v1", + "entity": "person", + "field": "ANN_VAL", + "grain": "person", + "input_basis": "target_current_2024", + "phase": "source_decode_before_current_leaves_and_units", + "raw_alternates": [], + "resulting_surfaces": [ + "taxable_pension_income", + "tax_exempt_pension_income" + ], + "role": "cps_source" + }, + { + "column": "CAP_VAL", + "consumers": [ + "cps_carried", + "microunit" + ], + "cpi_owner": "us_asec_current_money_ccpiu_2024_v1", + "entity": "person", + "field": "CAP_VAL", + "grain": "person", + "input_basis": "target_current_2024", + "phase": "source_decode_before_current_leaves_and_units", + "raw_alternates": [], + "resulting_surfaces": [ + "long_term_capital_gains", + "short_term_capital_gains" + ], + "role": "cps_source" + }, + { + "column": "CHSP_VAL", + "consumers": [ + "child_support" + ], + "cpi_owner": "us_asec_current_money_ccpiu_2024_v1", + "entity": "person", + "field": "CHSP_VAL", + "grain": "person", + "input_basis": "target_current_2024", + "phase": "source_decode_before_current_leaves_and_units", + "raw_alternates": [], + "resulting_surfaces": [ + "child_support_paid" + ], + "role": "cps_source" + }, + { + "column": "CSP_VAL", + "consumers": [ + "child_support" + ], + "cpi_owner": "us_asec_current_money_ccpiu_2024_v1", + "entity": "person", + "field": "CSP_VAL", + "grain": "person", + "input_basis": "target_current_2024", + "phase": "source_decode_before_current_leaves_and_units", + "raw_alternates": [], + "resulting_surfaces": [ + "child_support_received" + ], + "role": "cps_source" + }, + { + "column": "DIS_VAL1", + "consumers": [ + "disability_benefits" + ], + "cpi_owner": "us_asec_current_money_ccpiu_2024_v1", + "entity": "person", + "field": "DIS_VAL1", + "grain": "person", + "input_basis": "target_current_2024", + "phase": "source_decode_before_current_leaves_and_units", + "raw_alternates": [], + "resulting_surfaces": [ + "disability_benefits" + ], + "role": "cps_source" + }, + { + "column": "DIS_VAL2", + "consumers": [ + "disability_benefits" + ], + "cpi_owner": "us_asec_current_money_ccpiu_2024_v1", + "entity": "person", + "field": "DIS_VAL2", + "grain": "person", + "input_basis": "target_current_2024", + "phase": "source_decode_before_current_leaves_and_units", + "raw_alternates": [], + "resulting_surfaces": [ + "disability_benefits" + ], + "role": "cps_source" + }, + { + "column": "DIV_VAL", + "consumers": [ + "cps_carried", + "microunit" + ], + "cpi_owner": "us_asec_current_money_ccpiu_2024_v1", + "entity": "person", + "field": "DIV_VAL", + "grain": "person", + "input_basis": "target_current_2024", + "phase": "source_decode_before_current_leaves_and_units", + "raw_alternates": [], + "resulting_surfaces": [ + "qualified_dividend_income", + "non_qualified_dividend_income" + ], + "role": "cps_source" + }, + { + "column": "DST_VAL1", + "consumers": [ + "cps_carried", + "retirement_distributions" + ], + "cpi_owner": "us_asec_current_money_ccpiu_2024_v1", + "entity": "person", + "field": "DST_VAL1", + "grain": "person", + "input_basis": "target_current_2024", + "phase": "source_decode_before_current_leaves_and_units", + "raw_alternates": [], + "resulting_surfaces": [ + "taxable_ira_distributions", + "retirement_distribution_family" + ], + "role": "cps_source" + }, + { + "column": "DST_VAL1_YNG", + "consumers": [ + "cps_carried", + "retirement_distributions" + ], + "cpi_owner": "us_asec_current_money_ccpiu_2024_v1", + "entity": "person", + "field": "DST_VAL1_YNG", + "grain": "person", + "input_basis": "target_current_2024", + "phase": "source_decode_before_current_leaves_and_units", + "raw_alternates": [], + "resulting_surfaces": [ + "taxable_ira_distributions", + "retirement_distribution_family" + ], + "role": "cps_source" + }, + { + "column": "DST_VAL2", + "consumers": [ + "cps_carried", + "retirement_distributions" + ], + "cpi_owner": "us_asec_current_money_ccpiu_2024_v1", + "entity": "person", + "field": "DST_VAL2", + "grain": "person", + "input_basis": "target_current_2024", + "phase": "source_decode_before_current_leaves_and_units", + "raw_alternates": [], + "resulting_surfaces": [ + "taxable_ira_distributions", + "retirement_distribution_family" + ], + "role": "cps_source" + }, + { + "column": "DST_VAL2_YNG", + "consumers": [ + "cps_carried", + "retirement_distributions" + ], + "cpi_owner": "us_asec_current_money_ccpiu_2024_v1", + "entity": "person", + "field": "DST_VAL2_YNG", + "grain": "person", + "input_basis": "target_current_2024", + "phase": "source_decode_before_current_leaves_and_units", + "raw_alternates": [], + "resulting_surfaces": [ + "taxable_ira_distributions", + "retirement_distribution_family" + ], + "role": "cps_source" + }, + { + "column": "ED_VAL", + "consumers": [ + "education_inputs" + ], + "cpi_owner": "us_asec_current_money_ccpiu_2024_v1", + "entity": "person", + "field": "ED_VAL", + "grain": "person", + "input_basis": "target_current_2024", + "phase": "source_decode_before_current_leaves_and_units", + "raw_alternates": [], + "resulting_surfaces": [ + "educational_assistance" + ], + "role": "cps_source" + }, + { + "column": "FRSE_VAL", + "consumers": [ + "cps_carried", + "microunit" + ], + "cpi_owner": "us_asec_current_money_ccpiu_2024_v1", + "entity": "person", + "field": "FRSE_VAL", + "grain": "person", + "input_basis": "target_current_2024", + "phase": "source_decode_before_current_leaves_and_units", + "raw_alternates": [], + "resulting_surfaces": [ + "farm_operations_income" + ], + "role": "cps_source" + }, + { + "column": "asec_HTOTVAL", + "consumers": [ + "asec_household_observations" + ], + "cpi_owner": "us_asec_current_money_ccpiu_2024_v1", + "entity": "household", + "field": "HTOTVAL", + "grain": "household", + "input_basis": "target_current_2024", + "phase": "source_decode_before_current_leaves_and_units", + "raw_alternates": [], + "resulting_surfaces": [ + "asec_current_household_total_income" + ], + "role": "cps_source" + }, + { + "column": "INT_VAL", + "consumers": [ + "cps_carried", + "microunit" + ], + "cpi_owner": "us_asec_current_money_ccpiu_2024_v1", + "entity": "person", + "field": "INT_VAL", + "grain": "person", + "input_basis": "target_current_2024", + "phase": "source_decode_before_current_leaves_and_units", + "raw_alternates": [], + "resulting_surfaces": [ + "taxable_interest_income" + ], + "role": "cps_source" + }, + { + "column": "OI_VAL", + "consumers": [ + "asec_other_income_split", + "microunit" + ], + "cpi_owner": "us_asec_current_money_ccpiu_2024_v1", + "entity": "person", + "field": "OI_VAL", + "grain": "person", + "input_basis": "target_current_2024", + "phase": "source_decode_before_current_leaves_and_units", + "raw_alternates": [], + "resulting_surfaces": [ + "alimony_income", + "strike_benefits", + "miscellaneous_income" + ], + "role": "cps_source" + }, + { + "column": "PHIP_VAL", + "consumers": [ + "cps_carried" + ], + "cpi_owner": "us_asec_current_money_ccpiu_2024_v1", + "entity": "person", + "field": "PHIP_VAL", + "grain": "person", + "input_basis": "target_current_2024", + "phase": "source_decode_before_current_leaves_and_units", + "raw_alternates": [], + "resulting_surfaces": [ + "health_insurance_premiums_without_medicare_part_b" + ], + "role": "cps_source" + }, + { + "column": "PMED_VAL", + "consumers": [ + "cps_carried" + ], + "cpi_owner": "us_asec_current_money_ccpiu_2024_v1", + "entity": "person", + "field": "PMED_VAL", + "grain": "person", + "input_basis": "target_current_2024", + "phase": "source_decode_before_current_leaves_and_units", + "raw_alternates": [], + "resulting_surfaces": [ + "other_medical_expenses" + ], + "role": "cps_source" + }, + { + "column": "PNSN_VAL", + "consumers": [ + "cps_carried", + "microunit" + ], + "cpi_owner": "us_asec_current_money_ccpiu_2024_v1", + "entity": "person", + "field": "PNSN_VAL", + "grain": "person", + "input_basis": "target_current_2024", + "phase": "source_decode_before_current_leaves_and_units", + "raw_alternates": [], + "resulting_surfaces": [ + "taxable_pension_income", + "tax_exempt_pension_income" + ], + "role": "cps_source" + }, + { + "column": "POTC_VAL", + "consumers": [ + "cps_carried" + ], + "cpi_owner": "us_asec_current_money_ccpiu_2024_v1", + "entity": "person", + "field": "POTC_VAL", + "grain": "person", + "input_basis": "target_current_2024", + "phase": "source_decode_before_current_leaves_and_units", + "raw_alternates": [], + "resulting_surfaces": [ + "over_the_counter_health_expenses" + ], + "role": "cps_source" + }, + { + "column": "PTOTVAL", + "consumers": [ + "microunit" + ], + "cpi_owner": "us_asec_current_money_ccpiu_2024_v1", + "entity": "person", + "field": "PTOTVAL", + "grain": "person", + "input_basis": "target_current_2024", + "phase": "source_decode_before_current_leaves_and_units", + "raw_alternates": [], + "resulting_surfaces": [ + "microunit_current_total_income" + ], + "role": "cps_source" + }, + { + "column": "RETCB_VAL", + "consumers": [ + "retirement_contributions" + ], + "cpi_owner": "us_asec_current_money_ccpiu_2024_v1", + "entity": "person", + "field": "RETCB_VAL", + "grain": "person", + "input_basis": "target_current_2024", + "phase": "source_decode_before_current_leaves_and_units", + "raw_alternates": [], + "resulting_surfaces": [ + "retirement_contributions" + ], + "role": "cps_source" + }, + { + "column": "RNT_VAL", + "consumers": [ + "cps_carried", + "microunit" + ], + "cpi_owner": "us_asec_current_money_ccpiu_2024_v1", + "entity": "person", + "field": "RNT_VAL", + "grain": "person", + "input_basis": "target_current_2024", + "phase": "source_decode_before_current_leaves_and_units", + "raw_alternates": [], + "resulting_surfaces": [ + "rental_income" + ], + "role": "cps_source" + }, + { + "column": "SEMP_VAL", + "consumers": [ + "cps_carried", + "immigration", + "microunit" + ], + "cpi_owner": "us_asec_current_money_ccpiu_2024_v1", + "entity": "person", + "field": "SEMP_VAL", + "grain": "person", + "input_basis": "target_current_2024", + "phase": "source_decode_before_current_leaves_and_units", + "raw_alternates": [], + "resulting_surfaces": [ + "self_employment_income_before_lsr" + ], + "role": "cps_source" + }, + { + "column": "SPM_CAPHOUSESUB", + "consumers": [ + "immigration" + ], + "cpi_owner": "us_asec_current_money_ccpiu_2024_v1", + "entity": "person", + "field": "SPM_CAPHOUSESUB", + "grain": "spm_unit_repeated_on_person", + "input_basis": "target_current_2024", + "phase": "source_decode_before_current_leaves_and_units", + "raw_alternates": [], + "resulting_surfaces": [ + "immigration_amount_predictor" + ], + "role": "cps_source" + }, + { + "column": "SPM_CHILDCAREXPNS", + "consumers": [ + "childcare", + "cps_carried" + ], + "cpi_owner": "us_asec_current_money_ccpiu_2024_v1", + "entity": "person", + "field": "SPM_CHILDCAREXPNS", + "grain": "spm_unit_repeated_on_person", + "input_basis": "target_current_2024", + "phase": "source_decode_before_current_leaves_and_units", + "raw_alternates": [], + "resulting_surfaces": [ + "spm_unit_pre_subsidy_childcare_expenses" + ], + "role": "cps_source" + }, + { + "column": "SPM_ENGVAL", + "consumers": [ + "energy_subsidy" + ], + "cpi_owner": "us_asec_current_money_ccpiu_2024_v1", + "entity": "person", + "field": "SPM_ENGVAL", + "grain": "spm_unit_repeated_on_person", + "input_basis": "target_current_2024", + "phase": "source_decode_before_current_leaves_and_units", + "raw_alternates": [], + "resulting_surfaces": [ + "spm_unit_energy_subsidy" + ], + "role": "cps_source" + }, + { + "column": "SSI_VAL", + "consumers": [ + "acs_release_predictors_reported_ssi_anchor" + ], + "cpi_owner": "us_asec_current_money_ccpiu_2024_v1", + "entity": "person", + "field": "SSI_VAL", + "grain": "person", + "input_basis": "target_current_2024", + "phase": "source_decode_before_current_leaves_and_units", + "raw_alternates": [], + "resulting_surfaces": [ + "asec_current_reported_ssi_predictor" + ], + "role": "cps_source" + }, + { + "column": "SS_VAL", + "consumers": [ + "cps_carried", + "microunit" + ], + "cpi_owner": "us_asec_current_money_ccpiu_2024_v1", + "entity": "person", + "field": "SS_VAL", + "grain": "person", + "input_basis": "target_current_2024", + "phase": "source_decode_before_current_leaves_and_units", + "raw_alternates": [], + "resulting_surfaces": [ + "social_security_retirement", + "social_security_disability", + "social_security_survivors", + "social_security_dependents" + ], + "role": "cps_source" + }, + { + "column": "UC_VAL", + "consumers": [ + "cps_carried", + "microunit" + ], + "cpi_owner": "us_asec_current_money_ccpiu_2024_v1", + "entity": "person", + "field": "UC_VAL", + "grain": "person", + "input_basis": "target_current_2024", + "phase": "source_decode_before_current_leaves_and_units", + "raw_alternates": [], + "resulting_surfaces": [ + "unemployment_compensation" + ], + "role": "cps_source" + }, + { + "column": "VET_VAL", + "consumers": [ + "eligibility_inputs" + ], + "cpi_owner": "us_asec_current_money_ccpiu_2024_v1", + "entity": "person", + "field": "VET_VAL", + "grain": "person", + "input_basis": "target_current_2024", + "phase": "source_decode_before_current_leaves_and_units", + "raw_alternates": [], + "resulting_surfaces": [ + "veterans_benefits" + ], + "role": "cps_source" + }, + { + "column": "WC_VAL", + "consumers": [ + "workers_compensation" + ], + "cpi_owner": "us_asec_current_money_ccpiu_2024_v1", + "entity": "person", + "field": "WC_VAL", + "grain": "person", + "input_basis": "target_current_2024", + "phase": "source_decode_before_current_leaves_and_units", + "raw_alternates": [], + "resulting_surfaces": [ + "workers_compensation" + ], + "role": "cps_source" + }, + { + "column": "WSAL_VAL", + "consumers": [ + "cps_carried", + "immigration", + "microunit", + "voluntary_filing_current_leaf" + ], + "cpi_owner": "us_asec_current_money_ccpiu_2024_v1", + "entity": "person", + "field": "WSAL_VAL", + "grain": "person", + "input_basis": "target_current_2024", + "phase": "source_decode_before_current_leaves_and_units", + "raw_alternates": [], + "resulting_surfaces": [ + "employment_income_before_lsr" + ], + "role": "cps_source" + } + ], + "implementation_status": "declarations_only_live_nominal_consumers_unchanged", + "microunit": { + "amount_fields": [ + "ANN_VAL", + "CAP_VAL", + "DIV_VAL", + "FRSE_VAL", + "INT_VAL", + "OI_VAL", + "PNSN_VAL", + "PTOTVAL", + "RNT_VAL", + "SEMP_VAL", + "SS_VAL", + "UC_VAL", + "WSAL_VAL" + ], + "distribution": "microunit", + "module": "microunit/tax_unit_construction.py", + "module_sha256": "79007065113f9600601433dc2633f4392c675e28b87a7383ab8a1f7c6279d47e", + "version": "0.1.0" + }, + "ownership_policy": "restater_owns_cpi_only_derived_outputs_keep_registered_phase_role_owners", + "routing_policy": "unchanged_source_codes_no_new_DST_universe_exclusion", + "schema_version": 1, + "scope": "current_money_only_no_prior_wages", + "unscaled_fields": [ + "PAW_VAL", + "SPM_SNAPSUB" + ] +} diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/asec_current_money_domains_v1.json b/packages/microcosm-build/src/microcosm/build/us_runtime/asec_current_money_domains_v1.json new file mode 100644 index 000000000..7f09e933e --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/asec_current_money_domains_v1.json @@ -0,0 +1,3418 @@ +{ + "fields": [ + { + "column": "ANN_VAL", + "declared_niu_codes": [ + -1 + ], + "domain": { + "declared_negative_nonmoney_codes": [ + -1 + ], + "declared_niu_codes": [ + -1 + ], + "declared_other_missing_codes": [], + "encoded_range_inclusive": { + "maximum": 999999, + "minimum": -1 + }, + "missing_code_statement": "No additional missing codes are declared in this field entry; this does not certify upstream storage null handling.", + "negative_dollars_permitted": false, + "numeric_type": "integer_US_dollars_in_public_use_dictionary", + "valid_dollar_range_excludes": [], + "valid_dollar_range_inclusive": { + "maximum": 999999, + "minimum": 0 + }, + "zero_semantics": "valid_zero_dollars" + }, + "domain_status": "resolved_numeric_domain_all_three_vintages", + "entity": "person", + "grain": "person", + "maximum": 999999, + "minimum": -1, + "name": "ANN_VAL", + "period": { + "currency": "USD", + "income_year": "source_year", + "observation_period": "annual", + "price_basis": "nominal_income_year", + "survey_year": "source_year + 1" + }, + "temporal_authority": { + "pdf_pages_1based": [ + 2, + 365 + ], + "pdf_sha256": "7f0cb9f791f2737ad12fd552b8cb201f4d1de266eac160322597344e73499fb4", + "scope": "2025 reference document confirms preceding-calendar-year convention; exact cohort-to-survey bindings additionally authenticated by frozen receipt.", + "source_url": "https://www2.census.gov/programs-surveys/cps/techdocs/cpsmar25.pdf" + }, + "universe_validation": "Literal dictionary universe is preserved as evidence, not automatically compiled into a row-exclusion rule.", + "vintages": [ + { + "ascii_length_as_printed": 6, + "ascii_position_as_printed": 430, + "dictionary_spelling": "ANN_VAL", + "income_year": 2022, + "pdf_page_1based": 41, + "pdf_sha256": "66bd6e3fe516233ab63b75c60573451222b3b3d3235d61cfed96f64608de2117", + "printed_page": "6C-22", + "range_header": { + "maximum": 999999, + "minimum": -1 + }, + "source_url": "https://www2.census.gov/programs-surveys/cps/datasets/2023/march/asec2023_ddl_pub_full.pdf", + "survey_year": 2023, + "universe_as_printed": "ANN_YN = 1", + "values_description": "-1 = niu\n0-999999 = dollar amount" + }, + { + "ascii_length_as_printed": 6, + "ascii_position_as_printed": 430, + "dictionary_spelling": "ANN_VAL", + "income_year": 2023, + "pdf_page_1based": 42, + "pdf_sha256": "761c67ea53f5c3264329b3e9ddbdd802826ba4a859122b8a8a2f29ab85b3a840", + "printed_page": "6C-22", + "range_header": { + "maximum": 999999, + "minimum": -1 + }, + "source_url": "https://www2.census.gov/programs-surveys/cps/datasets/2024/march/asec2024_ddl_pub_full.pdf", + "survey_year": 2024, + "universe_as_printed": "ANN_YN = 1", + "values_description": "-1 = niu\n0-999999 = dollar amount" + }, + { + "ascii_length_as_printed": 6, + "ascii_position_as_printed": 438, + "dictionary_spelling": "ANN_VAL", + "income_year": 2024, + "pdf_page_1based": 44, + "pdf_sha256": "5cb80973326ef8b625fbaae70d80b0c641ce5d2b3911abd2fb4427abd5908a6f", + "printed_page": "6C-23", + "range_header": { + "maximum": 999999, + "minimum": -1 + }, + "source_url": "https://www2.census.gov/programs-surveys/cps/datasets/2025/march/asec2025_ddl_pub_full.pdf", + "survey_year": 2025, + "universe_as_printed": "ANN_YN = 1", + "values_description": "-1 = niu\n0-999999 = dollar amount" + } + ], + "zero_semantics": "valid_zero_dollars" + }, + { + "column": "CAP_VAL", + "declared_niu_codes": [ + 0 + ], + "domain": { + "declared_negative_nonmoney_codes": [], + "declared_niu_codes": [ + 0 + ], + "declared_other_missing_codes": [], + "encoded_range_inclusive": { + "maximum": 999999, + "minimum": 0 + }, + "missing_code_statement": "No additional missing codes are declared in this field entry; this does not certify upstream storage null handling.", + "negative_dollars_permitted": false, + "numeric_type": "integer_US_dollars_in_public_use_dictionary", + "valid_dollar_range_excludes": [], + "valid_dollar_range_inclusive": { + "maximum": 999999, + "minimum": 0 + }, + "zero_semantics": "none_or_niu_not_distinguishable_from_amount_alone" + }, + "domain_status": "resolved_numeric_domain_all_three_vintages", + "entity": "person", + "grain": "person", + "maximum": 999999, + "minimum": 0, + "name": "CAP_VAL", + "period": { + "currency": "USD", + "income_year": "source_year", + "observation_period": "annual", + "price_basis": "nominal_income_year", + "survey_year": "source_year + 1" + }, + "temporal_authority": { + "pdf_pages_1based": [ + 2, + 365 + ], + "pdf_sha256": "7f0cb9f791f2737ad12fd552b8cb201f4d1de266eac160322597344e73499fb4", + "scope": "2025 reference document confirms preceding-calendar-year convention; exact cohort-to-survey bindings additionally authenticated by frozen receipt.", + "source_url": "https://www2.census.gov/programs-surveys/cps/techdocs/cpsmar25.pdf" + }, + "universe_validation": "Literal dictionary universe is preserved as evidence, not automatically compiled into a row-exclusion rule.", + "vintages": [ + { + "ascii_length_as_printed": 6, + "ascii_position_as_printed": 437, + "dictionary_spelling": "CAP_VAL", + "income_year": 2022, + "pdf_page_1based": 41, + "pdf_sha256": "66bd6e3fe516233ab63b75c60573451222b3b3d3235d61cfed96f64608de2117", + "printed_page": "6C-22", + "range_header": { + "maximum": 999999, + "minimum": 0 + }, + "source_url": "https://www2.census.gov/programs-surveys/cps/datasets/2023/march/asec2023_ddl_pub_full.pdf", + "survey_year": 2023, + "universe_as_printed": "CAP_YN = 1", + "values_description": "0 = none or niu\n1-999999 = captial gains amount" + }, + { + "ascii_length_as_printed": 6, + "ascii_position_as_printed": 437, + "dictionary_spelling": "CAP_VAL", + "income_year": 2023, + "pdf_page_1based": 42, + "pdf_sha256": "761c67ea53f5c3264329b3e9ddbdd802826ba4a859122b8a8a2f29ab85b3a840", + "printed_page": "6C-22", + "range_header": { + "maximum": 999999, + "minimum": 0 + }, + "source_url": "https://www2.census.gov/programs-surveys/cps/datasets/2024/march/asec2024_ddl_pub_full.pdf", + "survey_year": 2024, + "universe_as_printed": "CAP_YN = 1", + "values_description": "0 = none or niu\n1-999999 = captial gains amount" + }, + { + "ascii_length_as_printed": 6, + "ascii_position_as_printed": 445, + "dictionary_spelling": "CAP_VAL", + "income_year": 2024, + "pdf_page_1based": 44, + "pdf_sha256": "5cb80973326ef8b625fbaae70d80b0c641ce5d2b3911abd2fb4427abd5908a6f", + "printed_page": "6C-23", + "range_header": { + "maximum": 999999, + "minimum": 0 + }, + "source_url": "https://www2.census.gov/programs-surveys/cps/datasets/2025/march/asec2025_ddl_pub_full.pdf", + "survey_year": 2025, + "universe_as_printed": "CAP_YN = 1", + "values_description": "0 = none or niu\n1-999999 = captial gains amount" + } + ], + "zero_semantics": "none_or_niu_not_distinguishable_from_amount_alone" + }, + { + "column": "CHSP_VAL", + "declared_niu_codes": [ + 0 + ], + "domain": { + "declared_negative_nonmoney_codes": [], + "declared_niu_codes": [ + 0 + ], + "declared_other_missing_codes": [], + "encoded_range_inclusive": { + "maximum": 99999, + "minimum": 0 + }, + "missing_code_statement": "No additional missing codes are declared in this field entry; this does not certify upstream storage null handling.", + "negative_dollars_permitted": false, + "numeric_type": "integer_US_dollars_in_public_use_dictionary", + "valid_dollar_range_excludes": [], + "valid_dollar_range_inclusive": { + "maximum": 99999, + "minimum": 1 + }, + "zero_semantics": "niu" + }, + "domain_status": "resolved_numeric_domain_all_three_vintages", + "entity": "person", + "grain": "person", + "maximum": 99999, + "minimum": 0, + "name": "CHSP_VAL", + "period": { + "currency": "USD", + "income_year": "source_year", + "observation_period": "annual", + "price_basis": "nominal_income_year", + "survey_year": "source_year + 1" + }, + "temporal_authority": { + "pdf_pages_1based": [ + 2, + 365 + ], + "pdf_sha256": "7f0cb9f791f2737ad12fd552b8cb201f4d1de266eac160322597344e73499fb4", + "scope": "2025 reference document confirms preceding-calendar-year convention; exact cohort-to-survey bindings additionally authenticated by frozen receipt.", + "source_url": "https://www2.census.gov/programs-surveys/cps/techdocs/cpsmar25.pdf" + }, + "universe_validation": "Literal dictionary universe is preserved as evidence, not automatically compiled into a row-exclusion rule.", + "vintages": [ + { + "ascii_length_as_printed": 5, + "ascii_position_as_printed": 704, + "dictionary_spelling": "CHSP_VAL", + "income_year": 2022, + "pdf_page_1based": 49, + "pdf_sha256": "66bd6e3fe516233ab63b75c60573451222b3b3d3235d61cfed96f64608de2117", + "printed_page": "6C-30", + "range_header": { + "maximum": 99999, + "minimum": 0 + }, + "source_url": "https://www2.census.gov/programs-surveys/cps/datasets/2023/march/asec2023_ddl_pub_full.pdf", + "survey_year": 2023, + "universe_as_printed": "CHSP_YN = 1", + "values_description": "0 = NIU\n1:99999 = amount paid in child support" + }, + { + "ascii_length_as_printed": 5, + "ascii_position_as_printed": 704, + "dictionary_spelling": "CHSP_VAL", + "income_year": 2023, + "pdf_page_1based": 50, + "pdf_sha256": "761c67ea53f5c3264329b3e9ddbdd802826ba4a859122b8a8a2f29ab85b3a840", + "printed_page": "6C-30", + "range_header": { + "maximum": 99999, + "minimum": 0 + }, + "source_url": "https://www2.census.gov/programs-surveys/cps/datasets/2024/march/asec2024_ddl_pub_full.pdf", + "survey_year": 2024, + "universe_as_printed": "CHSP_YN = 1", + "values_description": "0 = NIU\n1:99999 = amount paid in child support" + }, + { + "ascii_length_as_printed": 5, + "ascii_position_as_printed": 712, + "dictionary_spelling": "CHSP_VAL", + "income_year": 2024, + "pdf_page_1based": 52, + "pdf_sha256": "5cb80973326ef8b625fbaae70d80b0c641ce5d2b3911abd2fb4427abd5908a6f", + "printed_page": "6C-31", + "range_header": { + "maximum": 99999, + "minimum": 0 + }, + "source_url": "https://www2.census.gov/programs-surveys/cps/datasets/2025/march/asec2025_ddl_pub_full.pdf", + "survey_year": 2025, + "universe_as_printed": "CHSP_YN = 1", + "values_description": "0 = NIU\n1:99999 = amount paid in child support" + } + ], + "zero_semantics": "niu" + }, + { + "column": "CSP_VAL", + "declared_niu_codes": [ + 0 + ], + "domain": { + "declared_negative_nonmoney_codes": [], + "declared_niu_codes": [ + 0 + ], + "declared_other_missing_codes": [], + "encoded_range_inclusive": { + "maximum": 99999, + "minimum": 0 + }, + "missing_code_statement": "No additional missing codes are declared in this field entry; this does not certify upstream storage null handling.", + "negative_dollars_permitted": false, + "numeric_type": "integer_US_dollars_in_public_use_dictionary", + "valid_dollar_range_excludes": [], + "valid_dollar_range_inclusive": { + "maximum": 99999, + "minimum": 0 + }, + "zero_semantics": "none_or_niu_not_distinguishable_from_amount_alone" + }, + "domain_status": "resolved_numeric_domain_all_three_vintages", + "entity": "person", + "grain": "person", + "maximum": 99999, + "minimum": 0, + "name": "CSP_VAL", + "period": { + "currency": "USD", + "income_year": "source_year", + "observation_period": "annual", + "price_basis": "nominal_income_year", + "survey_year": "source_year + 1" + }, + "temporal_authority": { + "pdf_pages_1based": [ + 2, + 365 + ], + "pdf_sha256": "7f0cb9f791f2737ad12fd552b8cb201f4d1de266eac160322597344e73499fb4", + "scope": "2025 reference document confirms preceding-calendar-year convention; exact cohort-to-survey bindings additionally authenticated by frozen receipt.", + "source_url": "https://www2.census.gov/programs-surveys/cps/techdocs/cpsmar25.pdf" + }, + "universe_validation": "Literal dictionary universe is preserved as evidence, not automatically compiled into a row-exclusion rule.", + "vintages": [ + { + "ascii_length_as_printed": 5, + "ascii_position_as_printed": 710, + "dictionary_spelling": "CSP_VAL", + "income_year": 2022, + "pdf_page_1based": 50, + "pdf_sha256": "66bd6e3fe516233ab63b75c60573451222b3b3d3235d61cfed96f64608de2117", + "printed_page": "6C-31", + "range_header": { + "maximum": 99999, + "minimum": 0 + }, + "source_url": "https://www2.census.gov/programs-surveys/cps/datasets/2023/march/asec2023_ddl_pub_full.pdf", + "survey_year": 2023, + "universe_as_printed": "CSP_YN = 1", + "values_description": "0 = none or niu\n1-99999 = child support" + }, + { + "ascii_length_as_printed": 5, + "ascii_position_as_printed": 710, + "dictionary_spelling": "CSP_VAL", + "income_year": 2023, + "pdf_page_1based": 51, + "pdf_sha256": "761c67ea53f5c3264329b3e9ddbdd802826ba4a859122b8a8a2f29ab85b3a840", + "printed_page": "6C-31", + "range_header": { + "maximum": 99999, + "minimum": 0 + }, + "source_url": "https://www2.census.gov/programs-surveys/cps/datasets/2024/march/asec2024_ddl_pub_full.pdf", + "survey_year": 2024, + "universe_as_printed": "CSP_YN = 1", + "values_description": "0 = none or niu\n1-99999 = child support" + }, + { + "ascii_length_as_printed": 5, + "ascii_position_as_printed": 718, + "dictionary_spelling": "CSP_VAL", + "income_year": 2024, + "pdf_page_1based": 52, + "pdf_sha256": "5cb80973326ef8b625fbaae70d80b0c641ce5d2b3911abd2fb4427abd5908a6f", + "printed_page": "6C-31", + "range_header": { + "maximum": 99999, + "minimum": 0 + }, + "source_url": "https://www2.census.gov/programs-surveys/cps/datasets/2025/march/asec2025_ddl_pub_full.pdf", + "survey_year": 2025, + "universe_as_printed": "CSP_YN = 1", + "values_description": "0 = none or niu\n1-99999 = child support" + } + ], + "zero_semantics": "none_or_niu_not_distinguishable_from_amount_alone" + }, + { + "column": "DIS_VAL1", + "declared_niu_codes": [ + 0 + ], + "domain": { + "declared_negative_nonmoney_codes": [], + "declared_niu_codes": [ + 0 + ], + "declared_other_missing_codes": [], + "encoded_range_inclusive": { + "maximum": 999999, + "minimum": 0 + }, + "missing_code_statement": "No additional missing codes are declared in this field entry; this does not certify upstream storage null handling.", + "negative_dollars_permitted": false, + "numeric_type": "integer_US_dollars_in_public_use_dictionary", + "valid_dollar_range_excludes": [], + "valid_dollar_range_inclusive": { + "maximum": 999999, + "minimum": 0 + }, + "zero_semantics": "none_or_niu_not_distinguishable_from_amount_alone" + }, + "domain_status": "resolved_numeric_domain_all_three_vintages", + "entity": "person", + "grain": "person", + "maximum": 999999, + "minimum": 0, + "name": "DIS_VAL1", + "period": { + "currency": "USD", + "income_year": "source_year", + "observation_period": "annual", + "price_basis": "nominal_income_year", + "survey_year": "source_year + 1" + }, + "temporal_authority": { + "pdf_pages_1based": [ + 2, + 365 + ], + "pdf_sha256": "7f0cb9f791f2737ad12fd552b8cb201f4d1de266eac160322597344e73499fb4", + "scope": "2025 reference document confirms preceding-calendar-year convention; exact cohort-to-survey bindings additionally authenticated by frozen receipt.", + "source_url": "https://www2.census.gov/programs-surveys/cps/techdocs/cpsmar25.pdf" + }, + "universe_validation": "Literal dictionary universe is preserved as evidence, not automatically compiled into a row-exclusion rule.", + "vintages": [ + { + "ascii_length_as_printed": 6, + "ascii_position_as_printed": 457, + "dictionary_spelling": "DIS_VAL1", + "income_year": 2022, + "pdf_page_1based": 42, + "pdf_sha256": "66bd6e3fe516233ab63b75c60573451222b3b3d3235d61cfed96f64608de2117", + "printed_page": "6C-23", + "range_header": { + "maximum": 999999, + "minimum": 0 + }, + "source_url": "https://www2.census.gov/programs-surveys/cps/datasets/2023/march/asec2023_ddl_pub_full.pdf", + "survey_year": 2023, + "universe_as_printed": "DIS_SC1>0", + "values_description": "0 = none or niu\n1-999999 = disability income" + }, + { + "ascii_length_as_printed": 6, + "ascii_position_as_printed": 457, + "dictionary_spelling": "DIS_VAL1", + "income_year": 2023, + "pdf_page_1based": 43, + "pdf_sha256": "761c67ea53f5c3264329b3e9ddbdd802826ba4a859122b8a8a2f29ab85b3a840", + "printed_page": "6C-23", + "range_header": { + "maximum": 999999, + "minimum": 0 + }, + "source_url": "https://www2.census.gov/programs-surveys/cps/datasets/2024/march/asec2024_ddl_pub_full.pdf", + "survey_year": 2024, + "universe_as_printed": "DIS_SC1>0", + "values_description": "0 = none or niu\n1-999999 = disability income" + }, + { + "ascii_length_as_printed": 6, + "ascii_position_as_printed": 465, + "dictionary_spelling": "DIS_VAL1", + "income_year": 2024, + "pdf_page_1based": 44, + "pdf_sha256": "5cb80973326ef8b625fbaae70d80b0c641ce5d2b3911abd2fb4427abd5908a6f", + "printed_page": "6C-23", + "range_header": { + "maximum": 999999, + "minimum": 0 + }, + "source_url": "https://www2.census.gov/programs-surveys/cps/datasets/2025/march/asec2025_ddl_pub_full.pdf", + "survey_year": 2025, + "universe_as_printed": "DIS_SC1>0", + "values_description": "0 = none or niu\n1-999999 = disability income" + } + ], + "zero_semantics": "none_or_niu_not_distinguishable_from_amount_alone" + }, + { + "column": "DIS_VAL2", + "declared_niu_codes": [ + 0 + ], + "domain": { + "declared_negative_nonmoney_codes": [], + "declared_niu_codes": [ + 0 + ], + "declared_other_missing_codes": [], + "encoded_range_inclusive": { + "maximum": 999999, + "minimum": 0 + }, + "missing_code_statement": "No additional missing codes are declared in this field entry; this does not certify upstream storage null handling.", + "negative_dollars_permitted": false, + "numeric_type": "integer_US_dollars_in_public_use_dictionary", + "valid_dollar_range_excludes": [], + "valid_dollar_range_inclusive": { + "maximum": 999999, + "minimum": 0 + }, + "zero_semantics": "none_or_niu_not_distinguishable_from_amount_alone" + }, + "domain_status": "resolved_numeric_domain_all_three_vintages", + "entity": "person", + "grain": "person", + "maximum": 999999, + "minimum": 0, + "name": "DIS_VAL2", + "period": { + "currency": "USD", + "income_year": "source_year", + "observation_period": "annual", + "price_basis": "nominal_income_year", + "survey_year": "source_year + 1" + }, + "temporal_authority": { + "pdf_pages_1based": [ + 2, + 365 + ], + "pdf_sha256": "7f0cb9f791f2737ad12fd552b8cb201f4d1de266eac160322597344e73499fb4", + "scope": "2025 reference document confirms preceding-calendar-year convention; exact cohort-to-survey bindings additionally authenticated by frozen receipt.", + "source_url": "https://www2.census.gov/programs-surveys/cps/techdocs/cpsmar25.pdf" + }, + "universe_validation": "Literal dictionary universe is preserved as evidence, not automatically compiled into a row-exclusion rule.", + "vintages": [ + { + "ascii_length_as_printed": 6, + "ascii_position_as_printed": 463, + "dictionary_spelling": "DIS_VAL2", + "income_year": 2022, + "pdf_page_1based": 42, + "pdf_sha256": "66bd6e3fe516233ab63b75c60573451222b3b3d3235d61cfed96f64608de2117", + "printed_page": "6C-23", + "range_header": { + "maximum": 999999, + "minimum": 0 + }, + "source_url": "https://www2.census.gov/programs-surveys/cps/datasets/2023/march/asec2023_ddl_pub_full.pdf", + "survey_year": 2023, + "universe_as_printed": "DIS_SC2>0", + "values_description": "0 = none or niu\n1-999999 = disability income" + }, + { + "ascii_length_as_printed": 6, + "ascii_position_as_printed": 463, + "dictionary_spelling": "DIS_VAL2", + "income_year": 2023, + "pdf_page_1based": 43, + "pdf_sha256": "761c67ea53f5c3264329b3e9ddbdd802826ba4a859122b8a8a2f29ab85b3a840", + "printed_page": "6C-23", + "range_header": { + "maximum": 999999, + "minimum": 0 + }, + "source_url": "https://www2.census.gov/programs-surveys/cps/datasets/2024/march/asec2024_ddl_pub_full.pdf", + "survey_year": 2024, + "universe_as_printed": "DIS_SC2>0", + "values_description": "0 = none or niu\n1-999999 = disability income" + }, + { + "ascii_length_as_printed": 6, + "ascii_position_as_printed": 471, + "dictionary_spelling": "DIS_VAL2", + "income_year": 2024, + "pdf_page_1based": 44, + "pdf_sha256": "5cb80973326ef8b625fbaae70d80b0c641ce5d2b3911abd2fb4427abd5908a6f", + "printed_page": "6C-23", + "range_header": { + "maximum": 999999, + "minimum": 0 + }, + "source_url": "https://www2.census.gov/programs-surveys/cps/datasets/2025/march/asec2025_ddl_pub_full.pdf", + "survey_year": 2025, + "universe_as_printed": "DIS_SC2>0", + "values_description": "0 = none or niu\n1-999999 = disability income" + } + ], + "zero_semantics": "none_or_niu_not_distinguishable_from_amount_alone" + }, + { + "column": "DIV_VAL", + "declared_niu_codes": [ + 0 + ], + "domain": { + "declared_negative_nonmoney_codes": [], + "declared_niu_codes": [ + 0 + ], + "declared_other_missing_codes": [], + "encoded_range_inclusive": { + "maximum": 999999, + "minimum": 0 + }, + "missing_code_statement": "No additional missing codes are declared in this field entry; this does not certify upstream storage null handling.", + "negative_dollars_permitted": false, + "numeric_type": "integer_US_dollars_in_public_use_dictionary", + "valid_dollar_range_excludes": [], + "valid_dollar_range_inclusive": { + "maximum": 999999, + "minimum": 0 + }, + "zero_semantics": "none_or_niu_not_distinguishable_from_amount_alone" + }, + "domain_status": "resolved_numeric_domain_all_three_vintages", + "entity": "person", + "grain": "person", + "maximum": 999999, + "minimum": 0, + "name": "DIV_VAL", + "period": { + "currency": "USD", + "income_year": "source_year", + "observation_period": "annual", + "price_basis": "nominal_income_year", + "survey_year": "source_year + 1" + }, + "temporal_authority": { + "pdf_pages_1based": [ + 2, + 365 + ], + "pdf_sha256": "7f0cb9f791f2737ad12fd552b8cb201f4d1de266eac160322597344e73499fb4", + "scope": "2025 reference document confirms preceding-calendar-year convention; exact cohort-to-survey bindings additionally authenticated by frozen receipt.", + "source_url": "https://www2.census.gov/programs-surveys/cps/techdocs/cpsmar25.pdf" + }, + "universe_validation": "Literal dictionary universe is preserved as evidence, not automatically compiled into a row-exclusion rule.", + "vintages": [ + { + "ascii_length_as_printed": 6, + "ascii_position_as_printed": 470, + "dictionary_spelling": "DIV_VAL", + "income_year": 2022, + "pdf_page_1based": 42, + "pdf_sha256": "66bd6e3fe516233ab63b75c60573451222b3b3d3235d61cfed96f64608de2117", + "printed_page": "6C-23", + "range_header": { + "maximum": 999999, + "minimum": 0 + }, + "source_url": "https://www2.census.gov/programs-surveys/cps/datasets/2023/march/asec2023_ddl_pub_full.pdf", + "survey_year": 2023, + "universe_as_printed": "DIV_YN = 1", + "values_description": "0 = none or niu\n1-999999 = dividends" + }, + { + "ascii_length_as_printed": 6, + "ascii_position_as_printed": 470, + "dictionary_spelling": "DIV_VAL", + "income_year": 2023, + "pdf_page_1based": 43, + "pdf_sha256": "761c67ea53f5c3264329b3e9ddbdd802826ba4a859122b8a8a2f29ab85b3a840", + "printed_page": "6C-23", + "range_header": { + "maximum": 999999, + "minimum": 0 + }, + "source_url": "https://www2.census.gov/programs-surveys/cps/datasets/2024/march/asec2024_ddl_pub_full.pdf", + "survey_year": 2024, + "universe_as_printed": "DIV_YN = 1", + "values_description": "0 = none or niu\n1-999999 = dividends" + }, + { + "ascii_length_as_printed": 6, + "ascii_position_as_printed": 478, + "dictionary_spelling": "DIV_VAL", + "income_year": 2024, + "pdf_page_1based": 45, + "pdf_sha256": "5cb80973326ef8b625fbaae70d80b0c641ce5d2b3911abd2fb4427abd5908a6f", + "printed_page": "6C-24", + "range_header": { + "maximum": 999999, + "minimum": 0 + }, + "source_url": "https://www2.census.gov/programs-surveys/cps/datasets/2025/march/asec2025_ddl_pub_full.pdf", + "survey_year": 2025, + "universe_as_printed": "DIV_YN = 1", + "values_description": "0 = none or niu\n1-999999 = dividends" + } + ], + "zero_semantics": "none_or_niu_not_distinguishable_from_amount_alone" + }, + { + "column": "DST_VAL1", + "declared_niu_codes": [ + 0 + ], + "domain": { + "declared_negative_nonmoney_codes": [], + "declared_niu_codes": [ + 0 + ], + "declared_other_missing_codes": [], + "encoded_range_inclusive": { + "maximum": 999999, + "minimum": 0 + }, + "missing_code_statement": "No additional missing codes are declared in this field entry; this does not certify upstream storage null handling.", + "negative_dollars_permitted": false, + "numeric_type": "integer_US_dollars_in_public_use_dictionary", + "valid_dollar_range_excludes": [], + "valid_dollar_range_inclusive": { + "maximum": 999999, + "minimum": 0 + }, + "zero_semantics": "none_or_niu_not_distinguishable_from_amount_alone" + }, + "domain_status": "resolved_numeric_domain_all_three_vintages", + "entity": "person", + "grain": "person", + "maximum": 999999, + "minimum": 0, + "name": "DST_VAL1", + "period": { + "currency": "USD", + "income_year": "source_year", + "observation_period": "annual", + "price_basis": "nominal_income_year", + "survey_year": "source_year + 1" + }, + "temporal_authority": { + "pdf_pages_1based": [ + 2, + 365 + ], + "pdf_sha256": "7f0cb9f791f2737ad12fd552b8cb201f4d1de266eac160322597344e73499fb4", + "scope": "2025 reference document confirms preceding-calendar-year convention; exact cohort-to-survey bindings additionally authenticated by frozen receipt.", + "source_url": "https://www2.census.gov/programs-surveys/cps/techdocs/cpsmar25.pdf" + }, + "universe_validation": "Literal dictionary universe is preserved as evidence, not automatically compiled into a row-exclusion rule.", + "vintages": [ + { + "ascii_length_as_printed": 6, + "ascii_position_as_printed": 487, + "dictionary_spelling": "DST_VAL1", + "income_year": 2022, + "pdf_page_1based": 43, + "pdf_sha256": "66bd6e3fe516233ab63b75c60573451222b3b3d3235d61cfed96f64608de2117", + "printed_page": "6C-24", + "range_header": { + "maximum": 999999, + "minimum": 0 + }, + "source_url": "https://www2.census.gov/programs-surveys/cps/datasets/2023/march/asec2023_ddl_pub_full.pdf", + "survey_year": 2023, + "universe_as_printed": "DST_SC1 = 1", + "values_description": "0 = none or niu\n1-999,999 = amount withdrawn or distributed" + }, + { + "ascii_length_as_printed": 6, + "ascii_position_as_printed": 487, + "dictionary_spelling": "DST_VAL1", + "income_year": 2023, + "pdf_page_1based": 44, + "pdf_sha256": "761c67ea53f5c3264329b3e9ddbdd802826ba4a859122b8a8a2f29ab85b3a840", + "printed_page": "6C-24", + "range_header": { + "maximum": 999999, + "minimum": 0 + }, + "source_url": "https://www2.census.gov/programs-surveys/cps/datasets/2024/march/asec2024_ddl_pub_full.pdf", + "survey_year": 2024, + "universe_as_printed": "DST_SC1 = 1", + "values_description": "0 = none or niu\n1-999,999 = amount withdrawn or distributed" + }, + { + "ascii_length_as_printed": 6, + "ascii_position_as_printed": 495, + "dictionary_spelling": "DST_VAL1", + "income_year": 2024, + "pdf_page_1based": 45, + "pdf_sha256": "5cb80973326ef8b625fbaae70d80b0c641ce5d2b3911abd2fb4427abd5908a6f", + "printed_page": "6C-24", + "range_header": { + "maximum": 999999, + "minimum": 0 + }, + "source_url": "https://www2.census.gov/programs-surveys/cps/datasets/2025/march/asec2025_ddl_pub_full.pdf", + "survey_year": 2025, + "universe_as_printed": "DST_SC1 = 1", + "values_description": "0 = none or niu\n1-999,999 = amount withdrawn or distributed" + } + ], + "zero_semantics": "none_or_niu_not_distinguishable_from_amount_alone" + }, + { + "column": "DST_VAL1_YNG", + "declared_niu_codes": [ + 0 + ], + "domain": { + "declared_negative_nonmoney_codes": [], + "declared_niu_codes": [ + 0 + ], + "declared_other_missing_codes": [], + "encoded_range_inclusive": { + "maximum": 999999, + "minimum": 0 + }, + "missing_code_statement": "No additional missing codes are declared in this field entry; this does not certify upstream storage null handling.", + "negative_dollars_permitted": false, + "numeric_type": "integer_US_dollars_in_public_use_dictionary", + "valid_dollar_range_excludes": [], + "valid_dollar_range_inclusive": { + "maximum": 999999, + "minimum": 0 + }, + "zero_semantics": "none_or_niu_not_distinguishable_from_amount_alone" + }, + "domain_status": "resolved_numeric_domain_all_three_vintages", + "entity": "person", + "grain": "person", + "maximum": 999999, + "minimum": 0, + "name": "DST_VAL1_YNG", + "period": { + "currency": "USD", + "income_year": "source_year", + "observation_period": "annual", + "price_basis": "nominal_income_year", + "survey_year": "source_year + 1" + }, + "temporal_authority": { + "pdf_pages_1based": [ + 2, + 365 + ], + "pdf_sha256": "7f0cb9f791f2737ad12fd552b8cb201f4d1de266eac160322597344e73499fb4", + "scope": "2025 reference document confirms preceding-calendar-year convention; exact cohort-to-survey bindings additionally authenticated by frozen receipt.", + "source_url": "https://www2.census.gov/programs-surveys/cps/techdocs/cpsmar25.pdf" + }, + "universe_validation": "Literal dictionary universe is preserved as evidence, not automatically compiled into a row-exclusion rule.", + "vintages": [ + { + "ascii_length_as_printed": 6, + "ascii_position_as_printed": 493, + "dictionary_spelling": "DST_VAL1_YNG", + "income_year": 2022, + "pdf_page_1based": 43, + "pdf_sha256": "66bd6e3fe516233ab63b75c60573451222b3b3d3235d61cfed96f64608de2117", + "printed_page": "6C-24", + "range_header": { + "maximum": 999999, + "minimum": 0 + }, + "source_url": "https://www2.census.gov/programs-surveys/cps/datasets/2023/march/asec2023_ddl_pub_full.pdf", + "survey_year": 2023, + "universe_as_printed": "DST_SC1_YNG = 1", + "values_description": "0 = none or niu\n1- 999,999 = amount withdrawn or distributed" + }, + { + "ascii_length_as_printed": 6, + "ascii_position_as_printed": 493, + "dictionary_spelling": "DST_VAL1_YNG", + "income_year": 2023, + "pdf_page_1based": 44, + "pdf_sha256": "761c67ea53f5c3264329b3e9ddbdd802826ba4a859122b8a8a2f29ab85b3a840", + "printed_page": "6C-24", + "range_header": { + "maximum": 999999, + "minimum": 0 + }, + "source_url": "https://www2.census.gov/programs-surveys/cps/datasets/2024/march/asec2024_ddl_pub_full.pdf", + "survey_year": 2024, + "universe_as_printed": "DST_SC1_YNG = 1", + "values_description": "0 = none or niu\n1- 999,999 = amount withdrawn or distributed" + }, + { + "ascii_length_as_printed": 6, + "ascii_position_as_printed": 501, + "dictionary_spelling": "DST_VAL1_YNG", + "income_year": 2024, + "pdf_page_1based": 45, + "pdf_sha256": "5cb80973326ef8b625fbaae70d80b0c641ce5d2b3911abd2fb4427abd5908a6f", + "printed_page": "6C-24", + "range_header": { + "maximum": 999999, + "minimum": 0 + }, + "source_url": "https://www2.census.gov/programs-surveys/cps/datasets/2025/march/asec2025_ddl_pub_full.pdf", + "survey_year": 2025, + "universe_as_printed": "DST_SC1_YNG = 1", + "values_description": "0 = none or niu\n1- 999,999 = amount withdrawn or distributed" + } + ], + "zero_semantics": "none_or_niu_not_distinguishable_from_amount_alone" + }, + { + "column": "DST_VAL2", + "declared_niu_codes": [ + 0 + ], + "domain": { + "declared_negative_nonmoney_codes": [], + "declared_niu_codes": [ + 0 + ], + "declared_other_missing_codes": [], + "encoded_range_inclusive": { + "maximum": 999999, + "minimum": 0 + }, + "missing_code_statement": "No additional missing codes are declared in this field entry; this does not certify upstream storage null handling.", + "negative_dollars_permitted": false, + "numeric_type": "integer_US_dollars_in_public_use_dictionary", + "valid_dollar_range_excludes": [], + "valid_dollar_range_inclusive": { + "maximum": 999999, + "minimum": 0 + }, + "zero_semantics": "none_or_niu_not_distinguishable_from_amount_alone" + }, + "domain_status": "resolved_numeric_domain_all_three_vintages", + "entity": "person", + "grain": "person", + "maximum": 999999, + "minimum": 0, + "name": "DST_VAL2", + "period": { + "currency": "USD", + "income_year": "source_year", + "observation_period": "annual", + "price_basis": "nominal_income_year", + "survey_year": "source_year + 1" + }, + "temporal_authority": { + "pdf_pages_1based": [ + 2, + 365 + ], + "pdf_sha256": "7f0cb9f791f2737ad12fd552b8cb201f4d1de266eac160322597344e73499fb4", + "scope": "2025 reference document confirms preceding-calendar-year convention; exact cohort-to-survey bindings additionally authenticated by frozen receipt.", + "source_url": "https://www2.census.gov/programs-surveys/cps/techdocs/cpsmar25.pdf" + }, + "universe_validation": "Literal dictionary universe is preserved as evidence, not automatically compiled into a row-exclusion rule.", + "vintages": [ + { + "ascii_length_as_printed": 6, + "ascii_position_as_printed": 499, + "dictionary_spelling": "DST_VAL2", + "income_year": 2022, + "pdf_page_1based": 43, + "pdf_sha256": "66bd6e3fe516233ab63b75c60573451222b3b3d3235d61cfed96f64608de2117", + "printed_page": "6C-24", + "range_header": { + "maximum": 999999, + "minimum": 0 + }, + "source_url": "https://www2.census.gov/programs-surveys/cps/datasets/2023/march/asec2023_ddl_pub_full.pdf", + "survey_year": 2023, + "universe_as_printed": "DST_SC2 = 1", + "values_description": "0 = none or niu\n1- 999,999 = amount withdrawn or distributed" + }, + { + "ascii_length_as_printed": 6, + "ascii_position_as_printed": 499, + "dictionary_spelling": "DST_VAL2", + "income_year": 2023, + "pdf_page_1based": 44, + "pdf_sha256": "761c67ea53f5c3264329b3e9ddbdd802826ba4a859122b8a8a2f29ab85b3a840", + "printed_page": "6C-24", + "range_header": { + "maximum": 999999, + "minimum": 0 + }, + "source_url": "https://www2.census.gov/programs-surveys/cps/datasets/2024/march/asec2024_ddl_pub_full.pdf", + "survey_year": 2024, + "universe_as_printed": "DST_SC2 = 1", + "values_description": "0 = none or niu\n1- 999,999 = amount withdrawn or distributed" + }, + { + "ascii_length_as_printed": 6, + "ascii_position_as_printed": 507, + "dictionary_spelling": "DST_VAL2", + "income_year": 2024, + "pdf_page_1based": 45, + "pdf_sha256": "5cb80973326ef8b625fbaae70d80b0c641ce5d2b3911abd2fb4427abd5908a6f", + "printed_page": "6C-24", + "range_header": { + "maximum": 999999, + "minimum": 0 + }, + "source_url": "https://www2.census.gov/programs-surveys/cps/datasets/2025/march/asec2025_ddl_pub_full.pdf", + "survey_year": 2025, + "universe_as_printed": "DST_SC2 = 1", + "values_description": "0 = none or niu\n1- 999,999 = amount withdrawn or distributed" + } + ], + "zero_semantics": "none_or_niu_not_distinguishable_from_amount_alone" + }, + { + "column": "DST_VAL2_YNG", + "declared_niu_codes": [ + 0 + ], + "domain": { + "declared_negative_nonmoney_codes": [], + "declared_niu_codes": [ + 0 + ], + "declared_other_missing_codes": [], + "encoded_range_inclusive": { + "maximum": 999999, + "minimum": 0 + }, + "missing_code_statement": "No additional missing codes are declared in this field entry; this does not certify upstream storage null handling.", + "negative_dollars_permitted": false, + "numeric_type": "integer_US_dollars_in_public_use_dictionary", + "valid_dollar_range_excludes": [], + "valid_dollar_range_inclusive": { + "maximum": 999999, + "minimum": 0 + }, + "zero_semantics": "none_or_niu_not_distinguishable_from_amount_alone" + }, + "domain_status": "resolved_numeric_domain_all_three_vintages", + "entity": "person", + "grain": "person", + "maximum": 999999, + "minimum": 0, + "name": "DST_VAL2_YNG", + "period": { + "currency": "USD", + "income_year": "source_year", + "observation_period": "annual", + "price_basis": "nominal_income_year", + "survey_year": "source_year + 1" + }, + "temporal_authority": { + "pdf_pages_1based": [ + 2, + 365 + ], + "pdf_sha256": "7f0cb9f791f2737ad12fd552b8cb201f4d1de266eac160322597344e73499fb4", + "scope": "2025 reference document confirms preceding-calendar-year convention; exact cohort-to-survey bindings additionally authenticated by frozen receipt.", + "source_url": "https://www2.census.gov/programs-surveys/cps/techdocs/cpsmar25.pdf" + }, + "universe_validation": "Literal dictionary universe is preserved as evidence, not automatically compiled into a row-exclusion rule.", + "vintages": [ + { + "ascii_length_as_printed": 6, + "ascii_position_as_printed": 505, + "dictionary_spelling": "DST_VAL2_YNG", + "income_year": 2022, + "pdf_page_1based": 43, + "pdf_sha256": "66bd6e3fe516233ab63b75c60573451222b3b3d3235d61cfed96f64608de2117", + "printed_page": "6C-24", + "range_header": { + "maximum": 999999, + "minimum": 0 + }, + "source_url": "https://www2.census.gov/programs-surveys/cps/datasets/2023/march/asec2023_ddl_pub_full.pdf", + "survey_year": 2023, + "universe_as_printed": "DST_SC2_YNG = 1", + "values_description": "0 = none or niu\n1-999,999 = amount withdrawn or distributed" + }, + { + "ascii_length_as_printed": 6, + "ascii_position_as_printed": 505, + "dictionary_spelling": "DST_VAL2_YNG", + "income_year": 2023, + "pdf_page_1based": 44, + "pdf_sha256": "761c67ea53f5c3264329b3e9ddbdd802826ba4a859122b8a8a2f29ab85b3a840", + "printed_page": "6C-24", + "range_header": { + "maximum": 999999, + "minimum": 0 + }, + "source_url": "https://www2.census.gov/programs-surveys/cps/datasets/2024/march/asec2024_ddl_pub_full.pdf", + "survey_year": 2024, + "universe_as_printed": "DST_SC2_YNG = 1", + "values_description": "0 = none or niu\n1-999,999 = amount withdrawn or distributed" + }, + { + "ascii_length_as_printed": 6, + "ascii_position_as_printed": 513, + "dictionary_spelling": "DST_VAL2_YNG", + "income_year": 2024, + "pdf_page_1based": 45, + "pdf_sha256": "5cb80973326ef8b625fbaae70d80b0c641ce5d2b3911abd2fb4427abd5908a6f", + "printed_page": "6C-24", + "range_header": { + "maximum": 999999, + "minimum": 0 + }, + "source_url": "https://www2.census.gov/programs-surveys/cps/datasets/2025/march/asec2025_ddl_pub_full.pdf", + "survey_year": 2025, + "universe_as_printed": "DST_SC2_YNG = 1", + "values_description": "0 = none or niu\n1-999,999 = amount withdrawn or distributed" + } + ], + "zero_semantics": "none_or_niu_not_distinguishable_from_amount_alone" + }, + { + "column": "ED_VAL", + "declared_niu_codes": [ + 0 + ], + "domain": { + "declared_negative_nonmoney_codes": [], + "declared_niu_codes": [ + 0 + ], + "declared_other_missing_codes": [], + "encoded_range_inclusive": { + "maximum": 999999, + "minimum": 0 + }, + "missing_code_statement": "No additional missing codes are declared in this field entry; this does not certify upstream storage null handling.", + "negative_dollars_permitted": false, + "numeric_type": "integer_US_dollars_in_public_use_dictionary", + "valid_dollar_range_excludes": [], + "valid_dollar_range_inclusive": { + "maximum": 999999, + "minimum": 0 + }, + "zero_semantics": "none_or_niu_not_distinguishable_from_amount_alone" + }, + "domain_status": "resolved_numeric_domain_all_three_vintages", + "entity": "person", + "grain": "person", + "maximum": 999999, + "minimum": 0, + "name": "ED_VAL", + "period": { + "currency": "USD", + "income_year": "source_year", + "observation_period": "annual", + "price_basis": "nominal_income_year", + "survey_year": "source_year + 1" + }, + "temporal_authority": { + "pdf_pages_1based": [ + 2, + 365 + ], + "pdf_sha256": "7f0cb9f791f2737ad12fd552b8cb201f4d1de266eac160322597344e73499fb4", + "scope": "2025 reference document confirms preceding-calendar-year convention; exact cohort-to-survey bindings additionally authenticated by frozen receipt.", + "source_url": "https://www2.census.gov/programs-surveys/cps/techdocs/cpsmar25.pdf" + }, + "universe_validation": "Literal dictionary universe is preserved as evidence, not automatically compiled into a row-exclusion rule.", + "vintages": [ + { + "ascii_length_as_printed": 6, + "ascii_position_as_printed": 513, + "dictionary_spelling": "ED_VAL", + "income_year": 2022, + "pdf_page_1based": 43, + "pdf_sha256": "66bd6e3fe516233ab63b75c60573451222b3b3d3235d61cfed96f64608de2117", + "printed_page": "6C-24", + "range_header": { + "maximum": 999999, + "minimum": 0 + }, + "source_url": "https://www2.census.gov/programs-surveys/cps/datasets/2023/march/asec2023_ddl_pub_full.pdf", + "survey_year": 2023, + "universe_as_printed": "ED_YN = 1", + "values_description": "0 = none or niu;\n1- 999,999 = dollar amount" + }, + { + "ascii_length_as_printed": 6, + "ascii_position_as_printed": 513, + "dictionary_spelling": "ED_VAL", + "income_year": 2023, + "pdf_page_1based": 44, + "pdf_sha256": "761c67ea53f5c3264329b3e9ddbdd802826ba4a859122b8a8a2f29ab85b3a840", + "printed_page": "6C-24", + "range_header": { + "maximum": 999999, + "minimum": 0 + }, + "source_url": "https://www2.census.gov/programs-surveys/cps/datasets/2024/march/asec2024_ddl_pub_full.pdf", + "survey_year": 2024, + "universe_as_printed": "ED_YN = 1", + "values_description": "0 = none or niu;\n1- 999,999 = dollar amount" + }, + { + "ascii_length_as_printed": 6, + "ascii_position_as_printed": 521, + "dictionary_spelling": "ED_VAL", + "income_year": 2024, + "pdf_page_1based": 46, + "pdf_sha256": "5cb80973326ef8b625fbaae70d80b0c641ce5d2b3911abd2fb4427abd5908a6f", + "printed_page": "6C-25", + "range_header": { + "maximum": 999999, + "minimum": 0 + }, + "source_url": "https://www2.census.gov/programs-surveys/cps/datasets/2025/march/asec2025_ddl_pub_full.pdf", + "survey_year": 2025, + "universe_as_printed": "ED_YN = 1", + "values_description": "0 = none or niu;\n1- 999,999 = dollar amount" + } + ], + "zero_semantics": "none_or_niu_not_distinguishable_from_amount_alone" + }, + { + "column": "FRSE_VAL", + "declared_niu_codes": [ + 0 + ], + "domain": { + "declared_negative_nonmoney_codes": [], + "declared_niu_codes": [ + 0 + ], + "declared_other_missing_codes": [], + "encoded_range_inclusive": { + "maximum": 9999999, + "minimum": -9999999 + }, + "missing_code_statement": "No additional missing codes are declared in this field entry; this does not certify upstream storage null handling.", + "negative_dollars_permitted": true, + "numeric_type": "integer_US_dollars_in_public_use_dictionary", + "valid_dollar_range_excludes": [], + "valid_dollar_range_inclusive": { + "maximum": 9999999, + "minimum": -9999999 + }, + "zero_semantics": "none_or_niu_not_distinguishable_from_amount_alone" + }, + "domain_status": "resolved_numeric_domain_all_three_vintages", + "entity": "person", + "grain": "person", + "maximum": 9999999, + "minimum": -9999999, + "name": "FRSE_VAL", + "period": { + "currency": "USD", + "income_year": "source_year", + "observation_period": "annual", + "price_basis": "nominal_income_year", + "survey_year": "source_year + 1" + }, + "temporal_authority": { + "pdf_pages_1based": [ + 2, + 365 + ], + "pdf_sha256": "7f0cb9f791f2737ad12fd552b8cb201f4d1de266eac160322597344e73499fb4", + "scope": "2025 reference document confirms preceding-calendar-year convention; exact cohort-to-survey bindings additionally authenticated by frozen receipt.", + "source_url": "https://www2.census.gov/programs-surveys/cps/techdocs/cpsmar25.pdf" + }, + "universe_validation": "Literal dictionary universe is preserved as evidence, not automatically compiled into a row-exclusion rule.", + "vintages": [ + { + "ascii_length_as_printed": 7, + "ascii_position_as_printed": 382, + "dictionary_spelling": "FRSE_VAL", + "income_year": 2022, + "pdf_page_1based": 40, + "pdf_sha256": "66bd6e3fe516233ab63b75c60573451222b3b3d3235d61cfed96f64608de2117", + "printed_page": "6C-21", + "range_header": { + "maximum": 9999999, + "minimum": -9999999 + }, + "source_url": "https://www2.census.gov/programs-surveys/cps/datasets/2023/march/asec2023_ddl_pub_full.pdf", + "survey_year": 2023, + "universe_as_printed": "ERN_YN=1 or FRMOTR=1", + "values_description": "0 = none or niu;\n-9999999-9999999 = farm self employment" + }, + { + "ascii_length_as_printed": 7, + "ascii_position_as_printed": 382, + "dictionary_spelling": "FRSE_VAL", + "income_year": 2023, + "pdf_page_1based": 41, + "pdf_sha256": "761c67ea53f5c3264329b3e9ddbdd802826ba4a859122b8a8a2f29ab85b3a840", + "printed_page": "6C-21", + "range_header": { + "maximum": 9999999, + "minimum": -9999999 + }, + "source_url": "https://www2.census.gov/programs-surveys/cps/datasets/2024/march/asec2024_ddl_pub_full.pdf", + "survey_year": 2024, + "universe_as_printed": "ERN_YN=1 or FRMOTR=1", + "values_description": "0 = none or niu;\n-9999999-9999999 = farm self employment" + }, + { + "ascii_length_as_printed": 7, + "ascii_position_as_printed": 390, + "dictionary_spelling": "FRSE_VAL", + "income_year": 2024, + "pdf_page_1based": 43, + "pdf_sha256": "5cb80973326ef8b625fbaae70d80b0c641ce5d2b3911abd2fb4427abd5908a6f", + "printed_page": "6C-22", + "range_header": { + "maximum": 9999999, + "minimum": -9999999 + }, + "source_url": "https://www2.census.gov/programs-surveys/cps/datasets/2025/march/asec2025_ddl_pub_full.pdf", + "survey_year": 2025, + "universe_as_printed": "ERN_YN=1 or FRMOTR=1", + "values_description": "0 = none or niu;\n-9999999-9999999 = farm self employment" + } + ], + "zero_semantics": "none_or_niu_not_distinguishable_from_amount_alone" + }, + { + "column": "asec_HTOTVAL", + "declared_niu_codes": [], + "domain": { + "declared_negative_nonmoney_codes": [], + "declared_niu_codes": [], + "declared_other_missing_codes": [], + "encoded_range_inclusive": { + "maximum": 99999999, + "minimum": -999999 + }, + "missing_code_statement": "No additional missing codes are declared in this field entry; this does not certify upstream storage null handling.", + "negative_dollars_permitted": true, + "numeric_type": "integer_US_dollars_in_public_use_dictionary", + "valid_dollar_range_excludes": [], + "valid_dollar_range_inclusive": { + "maximum": 99999999, + "minimum": -999999 + }, + "zero_semantics": "valid_zero_dollars_or_none_as_described" + }, + "domain_status": "resolved_numeric_domain_all_three_vintages", + "entity": "household", + "grain": "household", + "maximum": 99999999, + "minimum": -999999, + "name": "HTOTVAL", + "period": { + "currency": "USD", + "income_year": "source_year", + "observation_period": "annual", + "price_basis": "nominal_income_year", + "survey_year": "source_year + 1" + }, + "temporal_authority": { + "pdf_pages_1based": [ + 2, + 365 + ], + "pdf_sha256": "7f0cb9f791f2737ad12fd552b8cb201f4d1de266eac160322597344e73499fb4", + "scope": "2025 reference document confirms preceding-calendar-year convention; exact cohort-to-survey bindings additionally authenticated by frozen receipt.", + "source_url": "https://www2.census.gov/programs-surveys/cps/techdocs/cpsmar25.pdf" + }, + "universe_validation": "Literal dictionary universe is preserved as evidence, not automatically compiled into a row-exclusion rule.", + "vintages": [ + { + "ascii_length_as_printed": 8, + "ascii_position_as_printed": 106, + "dictionary_spelling": "HTOTVAL", + "income_year": 2022, + "pdf_page_1based": 5, + "pdf_sha256": "66bd6e3fe516233ab63b75c60573451222b3b3d3235d61cfed96f64608de2117", + "printed_page": "6A-5", + "range_header": { + "maximum": 99999999, + "minimum": -999999 + }, + "source_url": "https://www2.census.gov/programs-surveys/cps/datasets/2023/march/asec2023_ddl_pub_full.pdf", + "survey_year": 2023, + "universe_as_printed": "All Households", + "values_description": "0 = none\nnegative dollar amount\npositive dollar amount" + }, + { + "ascii_length_as_printed": 8, + "ascii_position_as_printed": 106, + "dictionary_spelling": "HTOTVAL", + "income_year": 2023, + "pdf_page_1based": 5, + "pdf_sha256": "761c67ea53f5c3264329b3e9ddbdd802826ba4a859122b8a8a2f29ab85b3a840", + "printed_page": "6A-5", + "range_header": { + "maximum": 99999999, + "minimum": -999999 + }, + "source_url": "https://www2.census.gov/programs-surveys/cps/datasets/2024/march/asec2024_ddl_pub_full.pdf", + "survey_year": 2024, + "universe_as_printed": "All Households", + "values_description": "0 = none\nnegative dollar amount\npositive dollar amount" + }, + { + "ascii_length_as_printed": 8, + "ascii_position_as_printed": 108, + "dictionary_spelling": "HTOTVAL", + "income_year": 2024, + "pdf_page_1based": 12, + "pdf_sha256": "5cb80973326ef8b625fbaae70d80b0c641ce5d2b3911abd2fb4427abd5908a6f", + "printed_page": "6A-5", + "range_header": { + "maximum": 99999999, + "minimum": -999999 + }, + "source_url": "https://www2.census.gov/programs-surveys/cps/datasets/2025/march/asec2025_ddl_pub_full.pdf", + "survey_year": 2025, + "universe_as_printed": "All Households", + "values_description": "0 = none\nnegative dollar amount\npositive dollar amount" + } + ], + "zero_semantics": "valid_zero_dollars_or_none_as_described" + }, + { + "column": "INT_VAL", + "declared_niu_codes": [ + 0 + ], + "domain": { + "declared_negative_nonmoney_codes": [], + "declared_niu_codes": [ + 0 + ], + "declared_other_missing_codes": [], + "encoded_range_inclusive": { + "maximum": 999999, + "minimum": 0 + }, + "missing_code_statement": "No additional missing codes are declared in this field entry; this does not certify upstream storage null handling.", + "negative_dollars_permitted": false, + "numeric_type": "integer_US_dollars_in_public_use_dictionary", + "valid_dollar_range_excludes": [], + "valid_dollar_range_inclusive": { + "maximum": 999999, + "minimum": 0 + }, + "zero_semantics": "none_or_niu_not_distinguishable_from_amount_alone" + }, + "domain_status": "resolved_numeric_domain_all_three_vintages", + "entity": "person", + "grain": "person", + "maximum": 999999, + "minimum": 0, + "name": "INT_VAL", + "period": { + "currency": "USD", + "income_year": "source_year", + "observation_period": "annual", + "price_basis": "nominal_income_year", + "survey_year": "source_year + 1" + }, + "temporal_authority": { + "pdf_pages_1based": [ + 2, + 365 + ], + "pdf_sha256": "7f0cb9f791f2737ad12fd552b8cb201f4d1de266eac160322597344e73499fb4", + "scope": "2025 reference document confirms preceding-calendar-year convention; exact cohort-to-survey bindings additionally authenticated by frozen receipt.", + "source_url": "https://www2.census.gov/programs-surveys/cps/techdocs/cpsmar25.pdf" + }, + "universe_validation": "Literal dictionary universe is preserved as evidence, not automatically compiled into a row-exclusion rule.", + "vintages": [ + { + "ascii_length_as_printed": 6, + "ascii_position_as_printed": 529, + "dictionary_spelling": "INT_VAL", + "income_year": 2022, + "pdf_page_1based": 44, + "pdf_sha256": "66bd6e3fe516233ab63b75c60573451222b3b3d3235d61cfed96f64608de2117", + "printed_page": "6C-25", + "range_header": { + "maximum": 999999, + "minimum": 0 + }, + "source_url": "https://www2.census.gov/programs-surveys/cps/datasets/2023/march/asec2023_ddl_pub_full.pdf", + "survey_year": 2023, + "universe_as_printed": "INT_YN = 1", + "values_description": "0 = none or niu;\n1- 999,999 = dollar amount" + }, + { + "ascii_length_as_printed": 6, + "ascii_position_as_printed": 529, + "dictionary_spelling": "INT_VAL", + "income_year": 2023, + "pdf_page_1based": 45, + "pdf_sha256": "761c67ea53f5c3264329b3e9ddbdd802826ba4a859122b8a8a2f29ab85b3a840", + "printed_page": "6C-25", + "range_header": { + "maximum": 999999, + "minimum": 0 + }, + "source_url": "https://www2.census.gov/programs-surveys/cps/datasets/2024/march/asec2024_ddl_pub_full.pdf", + "survey_year": 2024, + "universe_as_printed": "INT_YN = 1", + "values_description": "0 = none or niu;\n1- 999,999 = dollar amount" + }, + { + "ascii_length_as_printed": 6, + "ascii_position_as_printed": 537, + "dictionary_spelling": "INT_VAL", + "income_year": 2024, + "pdf_page_1based": 46, + "pdf_sha256": "5cb80973326ef8b625fbaae70d80b0c641ce5d2b3911abd2fb4427abd5908a6f", + "printed_page": "6C-25", + "range_header": { + "maximum": 999999, + "minimum": 0 + }, + "source_url": "https://www2.census.gov/programs-surveys/cps/datasets/2025/march/asec2025_ddl_pub_full.pdf", + "survey_year": 2025, + "universe_as_printed": "INT_YN = 1", + "values_description": "0 = none or niu;\n1- 999,999 = dollar amount" + } + ], + "zero_semantics": "none_or_niu_not_distinguishable_from_amount_alone" + }, + { + "column": "OI_VAL", + "declared_niu_codes": [ + 0 + ], + "domain": { + "declared_negative_nonmoney_codes": [], + "declared_niu_codes": [ + 0 + ], + "declared_other_missing_codes": [], + "encoded_range_inclusive": { + "maximum": 999999, + "minimum": 0 + }, + "missing_code_statement": "No additional missing codes are declared in this field entry; this does not certify upstream storage null handling.", + "negative_dollars_permitted": false, + "numeric_type": "integer_US_dollars_in_public_use_dictionary", + "valid_dollar_range_excludes": [], + "valid_dollar_range_inclusive": { + "maximum": 999999, + "minimum": 0 + }, + "zero_semantics": "none_or_niu_not_distinguishable_from_amount_alone" + }, + "domain_status": "resolved_numeric_domain_all_three_vintages", + "entity": "person", + "grain": "person", + "maximum": 999999, + "minimum": 0, + "name": "OI_VAL", + "period": { + "currency": "USD", + "income_year": "source_year", + "observation_period": "annual", + "price_basis": "nominal_income_year", + "survey_year": "source_year + 1" + }, + "temporal_authority": { + "pdf_pages_1based": [ + 2, + 365 + ], + "pdf_sha256": "7f0cb9f791f2737ad12fd552b8cb201f4d1de266eac160322597344e73499fb4", + "scope": "2025 reference document confirms preceding-calendar-year convention; exact cohort-to-survey bindings additionally authenticated by frozen receipt.", + "source_url": "https://www2.census.gov/programs-surveys/cps/techdocs/cpsmar25.pdf" + }, + "universe_validation": "Literal dictionary universe is preserved as evidence, not automatically compiled into a row-exclusion rule.", + "vintages": [ + { + "ascii_length_as_printed": 6, + "ascii_position_as_printed": 541, + "dictionary_spelling": "OI_VAL", + "income_year": 2022, + "pdf_page_1based": 44, + "pdf_sha256": "66bd6e3fe516233ab63b75c60573451222b3b3d3235d61cfed96f64608de2117", + "printed_page": "6C-25", + "range_header": { + "maximum": 999999, + "minimum": 0 + }, + "source_url": "https://www2.census.gov/programs-surveys/cps/datasets/2023/march/asec2023_ddl_pub_full.pdf", + "survey_year": 2023, + "universe_as_printed": "OI_YN = 1", + "values_description": "0 = none or niu\n1-999999 = other income" + }, + { + "ascii_length_as_printed": 6, + "ascii_position_as_printed": 541, + "dictionary_spelling": "OI_VAL", + "income_year": 2023, + "pdf_page_1based": 45, + "pdf_sha256": "761c67ea53f5c3264329b3e9ddbdd802826ba4a859122b8a8a2f29ab85b3a840", + "printed_page": "6C-25", + "range_header": { + "maximum": 999999, + "minimum": 0 + }, + "source_url": "https://www2.census.gov/programs-surveys/cps/datasets/2024/march/asec2024_ddl_pub_full.pdf", + "survey_year": 2024, + "universe_as_printed": "OI_YN = 1", + "values_description": "0 = none or niu\n1-999999 = other income" + }, + { + "ascii_length_as_printed": 6, + "ascii_position_as_printed": 549, + "dictionary_spelling": "OI_VAL", + "income_year": 2024, + "pdf_page_1based": 47, + "pdf_sha256": "5cb80973326ef8b625fbaae70d80b0c641ce5d2b3911abd2fb4427abd5908a6f", + "printed_page": "6C-26", + "range_header": { + "maximum": 999999, + "minimum": 0 + }, + "source_url": "https://www2.census.gov/programs-surveys/cps/datasets/2025/march/asec2025_ddl_pub_full.pdf", + "survey_year": 2025, + "universe_as_printed": "OI_YN = 1", + "values_description": "0 = none or niu\n1-999999 = other income" + } + ], + "zero_semantics": "none_or_niu_not_distinguishable_from_amount_alone" + }, + { + "column": "PHIP_VAL", + "declared_niu_codes": [], + "domain": { + "declared_negative_nonmoney_codes": [], + "declared_niu_codes": [], + "declared_other_missing_codes": [], + "encoded_range_inclusive": { + "maximum": 999999, + "minimum": 0 + }, + "missing_code_statement": "No additional missing codes are declared in this field entry; this does not certify upstream storage null handling.", + "negative_dollars_permitted": false, + "numeric_type": "integer_US_dollars_in_public_use_dictionary", + "valid_dollar_range_excludes": [], + "valid_dollar_range_inclusive": { + "maximum": 999999, + "minimum": 0 + }, + "zero_semantics": "valid_zero_dollars_or_none_as_described" + }, + "domain_status": "resolved_numeric_domain_all_three_vintages", + "entity": "person", + "grain": "person", + "maximum": 999999, + "minimum": 0, + "name": "PHIP_VAL", + "period": { + "currency": "USD", + "income_year": "source_year", + "observation_period": "annual", + "price_basis": "nominal_income_year", + "survey_year": "source_year + 1" + }, + "temporal_authority": { + "pdf_pages_1based": [ + 2, + 365, + 289 + ], + "pdf_sha256": "7f0cb9f791f2737ad12fd552b8cb201f4d1de266eac160322597344e73499fb4", + "scope": "2025 reference document confirms preceding-calendar-year convention; exact cohort-to-survey bindings additionally authenticated by frozen receipt.", + "source_url": "https://www2.census.gov/programs-surveys/cps/techdocs/cpsmar25.pdf" + }, + "universe_validation": "Literal dictionary universe is preserved as evidence, not automatically compiled into a row-exclusion rule.", + "vintages": [ + { + "ascii_length_as_printed": 6, + "ascii_position_as_printed": 1292, + "dictionary_spelling": "PHIP_VAL", + "income_year": 2022, + "pdf_page_1based": 78, + "pdf_sha256": "66bd6e3fe516233ab63b75c60573451222b3b3d3235d61cfed96f64608de2117", + "printed_page": "6C-59", + "range_header": { + "maximum": 999999, + "minimum": 0 + }, + "source_url": "https://www2.census.gov/programs-surveys/cps/datasets/2023/march/asec2023_ddl_pub_full.pdf", + "survey_year": 2023, + "universe_as_printed": "All Persons", + "values_description": "0 - 999999" + }, + { + "ascii_length_as_printed": 6, + "ascii_position_as_printed": 1288, + "dictionary_spelling": "PHIP_VAL", + "income_year": 2023, + "pdf_page_1based": 79, + "pdf_sha256": "761c67ea53f5c3264329b3e9ddbdd802826ba4a859122b8a8a2f29ab85b3a840", + "printed_page": "6C-59", + "range_header": { + "maximum": 999999, + "minimum": 0 + }, + "source_url": "https://www2.census.gov/programs-surveys/cps/datasets/2024/march/asec2024_ddl_pub_full.pdf", + "survey_year": 2024, + "universe_as_printed": "All Persons", + "values_description": "0 - 999999" + }, + { + "ascii_length_as_printed": 6, + "ascii_position_as_printed": 1455, + "dictionary_spelling": "PHIP_VAL", + "income_year": 2024, + "pdf_page_1based": 83, + "pdf_sha256": "5cb80973326ef8b625fbaae70d80b0c641ce5d2b3911abd2fb4427abd5908a6f", + "printed_page": "6C-62", + "range_header": { + "maximum": 999999, + "minimum": 0 + }, + "source_url": "https://www2.census.gov/programs-surveys/cps/datasets/2025/march/asec2025_ddl_pub_full.pdf", + "survey_year": 2025, + "universe_as_printed": "All Persons", + "values_description": "0 - 999999" + } + ], + "zero_semantics": "valid_zero_dollars_or_none_as_described" + }, + { + "column": "PMED_VAL", + "declared_niu_codes": [], + "domain": { + "declared_negative_nonmoney_codes": [], + "declared_niu_codes": [], + "declared_other_missing_codes": [], + "encoded_range_inclusive": { + "maximum": 999999, + "minimum": 0 + }, + "missing_code_statement": "No additional missing codes are declared in this field entry; this does not certify upstream storage null handling.", + "negative_dollars_permitted": false, + "numeric_type": "integer_US_dollars_in_public_use_dictionary", + "valid_dollar_range_excludes": [], + "valid_dollar_range_inclusive": { + "maximum": 999999, + "minimum": 0 + }, + "zero_semantics": "valid_zero_dollars_or_none_as_described" + }, + "domain_status": "resolved_numeric_domain_all_three_vintages", + "entity": "person", + "grain": "person", + "maximum": 999999, + "minimum": 0, + "name": "PMED_VAL", + "period": { + "currency": "USD", + "income_year": "source_year", + "observation_period": "annual", + "price_basis": "nominal_income_year", + "survey_year": "source_year + 1" + }, + "temporal_authority": { + "pdf_pages_1based": [ + 2, + 365, + 289 + ], + "pdf_sha256": "7f0cb9f791f2737ad12fd552b8cb201f4d1de266eac160322597344e73499fb4", + "scope": "2025 reference document confirms preceding-calendar-year convention; exact cohort-to-survey bindings additionally authenticated by frozen receipt.", + "source_url": "https://www2.census.gov/programs-surveys/cps/techdocs/cpsmar25.pdf" + }, + "universe_validation": "Literal dictionary universe is preserved as evidence, not automatically compiled into a row-exclusion rule.", + "vintages": [ + { + "ascii_length_as_printed": 6, + "ascii_position_as_printed": 1304, + "dictionary_spelling": "PMED_VAL", + "income_year": 2022, + "pdf_page_1based": 79, + "pdf_sha256": "66bd6e3fe516233ab63b75c60573451222b3b3d3235d61cfed96f64608de2117", + "printed_page": "6C-60", + "range_header": { + "maximum": 999999, + "minimum": 0 + }, + "source_url": "https://www2.census.gov/programs-surveys/cps/datasets/2023/march/asec2023_ddl_pub_full.pdf", + "survey_year": 2023, + "universe_as_printed": "All Persons", + "values_description": "0 - 999999" + }, + { + "ascii_length_as_printed": 6, + "ascii_position_as_printed": 1300, + "dictionary_spelling": "PMED_VAL", + "income_year": 2023, + "pdf_page_1based": 79, + "pdf_sha256": "761c67ea53f5c3264329b3e9ddbdd802826ba4a859122b8a8a2f29ab85b3a840", + "printed_page": "6C-59", + "range_header": { + "maximum": 999999, + "minimum": 0 + }, + "source_url": "https://www2.census.gov/programs-surveys/cps/datasets/2024/march/asec2024_ddl_pub_full.pdf", + "survey_year": 2024, + "universe_as_printed": "All Persons", + "values_description": "0 - 999999" + }, + { + "ascii_length_as_printed": 6, + "ascii_position_as_printed": 1467, + "dictionary_spelling": "PMED_VAL", + "income_year": 2024, + "pdf_page_1based": 83, + "pdf_sha256": "5cb80973326ef8b625fbaae70d80b0c641ce5d2b3911abd2fb4427abd5908a6f", + "printed_page": "6C-62", + "range_header": { + "maximum": 999999, + "minimum": 0 + }, + "source_url": "https://www2.census.gov/programs-surveys/cps/datasets/2025/march/asec2025_ddl_pub_full.pdf", + "survey_year": 2025, + "universe_as_printed": "All Persons", + "values_description": "0 - 999999" + } + ], + "zero_semantics": "valid_zero_dollars_or_none_as_described" + }, + { + "column": "PNSN_VAL", + "declared_niu_codes": [ + 0 + ], + "domain": { + "declared_negative_nonmoney_codes": [], + "declared_niu_codes": [ + 0 + ], + "declared_other_missing_codes": [], + "encoded_range_inclusive": { + "maximum": 9999999, + "minimum": 0 + }, + "missing_code_statement": "No additional missing codes are declared in this field entry; this does not certify upstream storage null handling.", + "negative_dollars_permitted": false, + "numeric_type": "integer_US_dollars_in_public_use_dictionary", + "valid_dollar_range_excludes": [], + "valid_dollar_range_inclusive": { + "maximum": 9999999, + "minimum": 0 + }, + "zero_semantics": "none_or_niu_not_distinguishable_from_amount_alone" + }, + "domain_status": "resolved_numeric_domain_all_three_vintages", + "entity": "person", + "grain": "person", + "maximum": 9999999, + "minimum": 0, + "name": "PNSN_VAL", + "period": { + "currency": "USD", + "income_year": "source_year", + "observation_period": "annual", + "price_basis": "nominal_income_year", + "survey_year": "source_year + 1" + }, + "temporal_authority": { + "pdf_pages_1based": [ + 2, + 365 + ], + "pdf_sha256": "7f0cb9f791f2737ad12fd552b8cb201f4d1de266eac160322597344e73499fb4", + "scope": "2025 reference document confirms preceding-calendar-year convention; exact cohort-to-survey bindings additionally authenticated by frozen receipt.", + "source_url": "https://www2.census.gov/programs-surveys/cps/techdocs/cpsmar25.pdf" + }, + "universe_validation": "Literal dictionary universe is preserved as evidence, not automatically compiled into a row-exclusion rule.", + "vintages": [ + { + "ascii_length_as_printed": 7, + "ascii_position_as_printed": 563, + "dictionary_spelling": "PNSN_VAL", + "income_year": 2022, + "pdf_page_1based": 45, + "pdf_sha256": "66bd6e3fe516233ab63b75c60573451222b3b3d3235d61cfed96f64608de2117", + "printed_page": "6C-26", + "range_header": { + "maximum": 9999999, + "minimum": 0 + }, + "source_url": "https://www2.census.gov/programs-surveys/cps/datasets/2023/march/asec2023_ddl_pub_full.pdf", + "survey_year": 2023, + "universe_as_printed": "PEN_YN = 1", + "values_description": "0 = none or niu\n1- 9,999,999 = retirement income" + }, + { + "ascii_length_as_printed": 7, + "ascii_position_as_printed": 563, + "dictionary_spelling": "PNSN_VAL", + "income_year": 2023, + "pdf_page_1based": 46, + "pdf_sha256": "761c67ea53f5c3264329b3e9ddbdd802826ba4a859122b8a8a2f29ab85b3a840", + "printed_page": "6C-26", + "range_header": { + "maximum": 9999999, + "minimum": 0 + }, + "source_url": "https://www2.census.gov/programs-surveys/cps/datasets/2024/march/asec2024_ddl_pub_full.pdf", + "survey_year": 2024, + "universe_as_printed": "PEN_YN = 1", + "values_description": "0 = none or niu\n1- 9,999,999 = retirement income" + }, + { + "ascii_length_as_printed": 7, + "ascii_position_as_printed": 571, + "dictionary_spelling": "PNSN_VAL", + "income_year": 2024, + "pdf_page_1based": 47, + "pdf_sha256": "5cb80973326ef8b625fbaae70d80b0c641ce5d2b3911abd2fb4427abd5908a6f", + "printed_page": "6C-26", + "range_header": { + "maximum": 9999999, + "minimum": 0 + }, + "source_url": "https://www2.census.gov/programs-surveys/cps/datasets/2025/march/asec2025_ddl_pub_full.pdf", + "survey_year": 2025, + "universe_as_printed": "PEN_YN = 1", + "values_description": "0 = none or niu\n1- 9,999,999 = retirement income" + } + ], + "zero_semantics": "none_or_niu_not_distinguishable_from_amount_alone" + }, + { + "column": "POTC_VAL", + "declared_niu_codes": [], + "domain": { + "declared_negative_nonmoney_codes": [], + "declared_niu_codes": [], + "declared_other_missing_codes": [], + "encoded_range_inclusive": { + "maximum": 99999, + "minimum": 0 + }, + "missing_code_statement": "No additional missing codes are declared in this field entry; this does not certify upstream storage null handling.", + "negative_dollars_permitted": false, + "numeric_type": "integer_US_dollars_in_public_use_dictionary", + "valid_dollar_range_excludes": [], + "valid_dollar_range_inclusive": { + "maximum": 99999, + "minimum": 0 + }, + "zero_semantics": "valid_zero_dollars_or_none_as_described" + }, + "domain_status": "resolved_numeric_domain_all_three_vintages", + "entity": "person", + "grain": "person", + "maximum": 99999, + "minimum": 0, + "name": "POTC_VAL", + "period": { + "currency": "USD", + "income_year": "source_year", + "observation_period": "annual", + "price_basis": "nominal_income_year", + "survey_year": "source_year + 1" + }, + "temporal_authority": { + "pdf_pages_1based": [ + 2, + 365, + 289, + 290 + ], + "pdf_sha256": "7f0cb9f791f2737ad12fd552b8cb201f4d1de266eac160322597344e73499fb4", + "scope": "2025 reference document confirms preceding-calendar-year convention; exact cohort-to-survey bindings additionally authenticated by frozen receipt.", + "source_url": "https://www2.census.gov/programs-surveys/cps/techdocs/cpsmar25.pdf" + }, + "universe_validation": "Literal dictionary universe is preserved as evidence, not automatically compiled into a row-exclusion rule.", + "vintages": [ + { + "ascii_length_as_printed": 5, + "ascii_position_as_printed": 1310, + "dictionary_spelling": "POTC_VAL", + "income_year": 2022, + "pdf_page_1based": 79, + "pdf_sha256": "66bd6e3fe516233ab63b75c60573451222b3b3d3235d61cfed96f64608de2117", + "printed_page": "6C-60", + "range_header": { + "maximum": 99999, + "minimum": 0 + }, + "source_url": "https://www2.census.gov/programs-surveys/cps/datasets/2023/march/asec2023_ddl_pub_full.pdf", + "survey_year": 2023, + "universe_as_printed": "All Persons", + "values_description": "0 - 99999" + }, + { + "ascii_length_as_printed": 5, + "ascii_position_as_printed": 1306, + "dictionary_spelling": "POTC_VAL", + "income_year": 2023, + "pdf_page_1based": 79, + "pdf_sha256": "761c67ea53f5c3264329b3e9ddbdd802826ba4a859122b8a8a2f29ab85b3a840", + "printed_page": "6C-59", + "range_header": { + "maximum": 99999, + "minimum": 0 + }, + "source_url": "https://www2.census.gov/programs-surveys/cps/datasets/2024/march/asec2024_ddl_pub_full.pdf", + "survey_year": 2024, + "universe_as_printed": "All Persons", + "values_description": "0 - 99999" + }, + { + "ascii_length_as_printed": 5, + "ascii_position_as_printed": 1473, + "dictionary_spelling": "POTC_VAL", + "income_year": 2024, + "pdf_page_1based": 83, + "pdf_sha256": "5cb80973326ef8b625fbaae70d80b0c641ce5d2b3911abd2fb4427abd5908a6f", + "printed_page": "6C-62", + "range_header": { + "maximum": 99999, + "minimum": 0 + }, + "source_url": "https://www2.census.gov/programs-surveys/cps/datasets/2025/march/asec2025_ddl_pub_full.pdf", + "survey_year": 2025, + "universe_as_printed": "All Persons", + "values_description": "0 - 99999" + } + ], + "zero_semantics": "valid_zero_dollars_or_none_as_described" + }, + { + "column": "PTOTVAL", + "declared_niu_codes": [], + "domain": { + "declared_negative_nonmoney_codes": [], + "declared_niu_codes": [], + "declared_other_missing_codes": [], + "encoded_range_inclusive": { + "maximum": 99999999, + "minimum": -99999 + }, + "missing_code_statement": "No additional missing codes are declared in this field entry; this does not certify upstream storage null handling.", + "negative_dollars_permitted": true, + "numeric_type": "integer_US_dollars_in_public_use_dictionary", + "valid_dollar_range_excludes": [], + "valid_dollar_range_inclusive": { + "maximum": 99999999, + "minimum": -99999 + }, + "zero_semantics": "valid_zero_dollars_or_none_as_described" + }, + "domain_status": "resolved_numeric_domain_all_three_vintages", + "entity": "person", + "grain": "person", + "maximum": 99999999, + "minimum": -99999, + "name": "PTOTVAL", + "period": { + "currency": "USD", + "income_year": "source_year", + "observation_period": "annual", + "price_basis": "nominal_income_year", + "survey_year": "source_year + 1" + }, + "temporal_authority": { + "pdf_pages_1based": [ + 2, + 365 + ], + "pdf_sha256": "7f0cb9f791f2737ad12fd552b8cb201f4d1de266eac160322597344e73499fb4", + "scope": "2025 reference document confirms preceding-calendar-year convention; exact cohort-to-survey bindings additionally authenticated by frozen receipt.", + "source_url": "https://www2.census.gov/programs-surveys/cps/techdocs/cpsmar25.pdf" + }, + "universe_validation": "Literal dictionary universe is preserved as evidence, not automatically compiled into a row-exclusion rule.", + "vintages": [ + { + "ascii_length_as_printed": 8, + "ascii_position_as_printed": 580, + "dictionary_spelling": "PTOTVAL", + "income_year": 2022, + "pdf_page_1based": 45, + "pdf_sha256": "66bd6e3fe516233ab63b75c60573451222b3b3d3235d61cfed96f64608de2117", + "printed_page": "6C-26", + "range_header": { + "maximum": 99999999, + "minimum": -99999 + }, + "source_url": "https://www2.census.gov/programs-surveys/cps/datasets/2023/march/asec2023_ddl_pub_full.pdf", + "survey_year": 2023, + "universe_as_printed": "All Persons aged 15+", + "values_description": "0 = none\nnegative amt = income (loss)\npositive amt = income" + }, + { + "ascii_length_as_printed": 8, + "ascii_position_as_printed": 580, + "dictionary_spelling": "PTOTVAL", + "income_year": 2023, + "pdf_page_1based": 46, + "pdf_sha256": "761c67ea53f5c3264329b3e9ddbdd802826ba4a859122b8a8a2f29ab85b3a840", + "printed_page": "6C-26", + "range_header": { + "maximum": 99999999, + "minimum": -99999 + }, + "source_url": "https://www2.census.gov/programs-surveys/cps/datasets/2024/march/asec2024_ddl_pub_full.pdf", + "survey_year": 2024, + "universe_as_printed": "All Persons aged 15+", + "values_description": "0 = none\nnegative amt = income (loss)\npositive amt = income" + }, + { + "ascii_length_as_printed": 8, + "ascii_position_as_printed": 588, + "dictionary_spelling": "PTOTVAL", + "income_year": 2024, + "pdf_page_1based": 48, + "pdf_sha256": "5cb80973326ef8b625fbaae70d80b0c641ce5d2b3911abd2fb4427abd5908a6f", + "printed_page": "6C-27", + "range_header": { + "maximum": 99999999, + "minimum": -99999 + }, + "source_url": "https://www2.census.gov/programs-surveys/cps/datasets/2025/march/asec2025_ddl_pub_full.pdf", + "survey_year": 2025, + "universe_as_printed": "All Persons aged 15+", + "values_description": "0 = none\nnegative amt = income (loss)\npositive amt = income" + } + ], + "zero_semantics": "valid_zero_dollars_or_none_as_described" + }, + { + "column": "RETCB_VAL", + "declared_niu_codes": [ + 0 + ], + "domain": { + "declared_negative_nonmoney_codes": [], + "declared_niu_codes": [ + 0 + ], + "declared_other_missing_codes": [], + "encoded_range_inclusive": { + "maximum": 99999, + "minimum": 0 + }, + "missing_code_statement": "No additional missing codes are declared in this field entry; this does not certify upstream storage null handling.", + "negative_dollars_permitted": false, + "numeric_type": "integer_US_dollars_in_public_use_dictionary", + "valid_dollar_range_excludes": [], + "valid_dollar_range_inclusive": { + "maximum": 99999, + "minimum": 0 + }, + "zero_semantics": "none_or_niu_not_distinguishable_from_amount_alone" + }, + "domain_status": "resolved_numeric_domain_all_three_vintages", + "entity": "person", + "grain": "person", + "maximum": 99999, + "minimum": 0, + "name": "RETCB_VAL", + "period": { + "currency": "USD", + "income_year": "source_year", + "observation_period": "annual", + "price_basis": "nominal_income_year", + "survey_year": "source_year + 1" + }, + "temporal_authority": { + "pdf_pages_1based": [ + 2, + 365, + 274 + ], + "pdf_sha256": "7f0cb9f791f2737ad12fd552b8cb201f4d1de266eac160322597344e73499fb4", + "scope": "2025 reference document confirms preceding-calendar-year convention; exact cohort-to-survey bindings additionally authenticated by frozen receipt.", + "source_url": "https://www2.census.gov/programs-surveys/cps/techdocs/cpsmar25.pdf" + }, + "universe_validation": "Literal dictionary universe is preserved as evidence, not automatically compiled into a row-exclusion rule.", + "vintages": [ + { + "ascii_length_as_printed": 5, + "ascii_position_as_printed": 592, + "dictionary_spelling": "RETCB_VAL", + "income_year": 2022, + "pdf_page_1based": 46, + "pdf_sha256": "66bd6e3fe516233ab63b75c60573451222b3b3d3235d61cfed96f64608de2117", + "printed_page": "6C-27", + "range_header": { + "maximum": 99999, + "minimum": 0 + }, + "source_url": "https://www2.census.gov/programs-surveys/cps/datasets/2023/march/asec2023_ddl_pub_full.pdf", + "survey_year": 2023, + "universe_as_printed": "RETCB_YN = 1", + "values_description": "0 = none or niu;\n1-99999 = amount contributed" + }, + { + "ascii_length_as_printed": 5, + "ascii_position_as_printed": 592, + "dictionary_spelling": "RETCB_VAL", + "income_year": 2023, + "pdf_page_1based": 47, + "pdf_sha256": "761c67ea53f5c3264329b3e9ddbdd802826ba4a859122b8a8a2f29ab85b3a840", + "printed_page": "6C-27", + "range_header": { + "maximum": 99999, + "minimum": 0 + }, + "source_url": "https://www2.census.gov/programs-surveys/cps/datasets/2024/march/asec2024_ddl_pub_full.pdf", + "survey_year": 2024, + "universe_as_printed": "RETCB_YN = 1", + "values_description": "0 = none or niu;\n1-99999 = amount contributed" + }, + { + "ascii_length_as_printed": 5, + "ascii_position_as_printed": 600, + "dictionary_spelling": "RETCB_VAL", + "income_year": 2024, + "pdf_page_1based": 48, + "pdf_sha256": "5cb80973326ef8b625fbaae70d80b0c641ce5d2b3911abd2fb4427abd5908a6f", + "printed_page": "6C-27", + "range_header": { + "maximum": 99999, + "minimum": 0 + }, + "source_url": "https://www2.census.gov/programs-surveys/cps/datasets/2025/march/asec2025_ddl_pub_full.pdf", + "survey_year": 2025, + "universe_as_printed": "RETCB_YN = 1", + "values_description": "0 = none or niu;\n1-99999 = amount contributed" + } + ], + "zero_semantics": "none_or_niu_not_distinguishable_from_amount_alone" + }, + { + "column": "RNT_VAL", + "declared_niu_codes": [ + 0 + ], + "domain": { + "declared_negative_nonmoney_codes": [], + "declared_niu_codes": [ + 0 + ], + "declared_other_missing_codes": [], + "encoded_range_inclusive": { + "maximum": 999999, + "minimum": -9999 + }, + "missing_code_statement": "No additional missing codes are declared in this field entry; this does not certify upstream storage null handling.", + "negative_dollars_permitted": true, + "numeric_type": "integer_US_dollars_in_public_use_dictionary", + "valid_dollar_range_excludes": [], + "valid_dollar_range_inclusive": { + "maximum": 999999, + "minimum": -9999 + }, + "zero_semantics": "none_or_niu_not_distinguishable_from_amount_alone" + }, + "domain_status": "resolved_numeric_domain_all_three_vintages", + "entity": "person", + "grain": "person", + "maximum": 999999, + "minimum": -9999, + "name": "RNT_VAL", + "period": { + "currency": "USD", + "income_year": "source_year", + "observation_period": "annual", + "price_basis": "nominal_income_year", + "survey_year": "source_year + 1" + }, + "temporal_authority": { + "pdf_pages_1based": [ + 2, + 365 + ], + "pdf_sha256": "7f0cb9f791f2737ad12fd552b8cb201f4d1de266eac160322597344e73499fb4", + "scope": "2025 reference document confirms preceding-calendar-year convention; exact cohort-to-survey bindings additionally authenticated by frozen receipt.", + "source_url": "https://www2.census.gov/programs-surveys/cps/techdocs/cpsmar25.pdf" + }, + "universe_validation": "Literal dictionary universe is preserved as evidence, not automatically compiled into a row-exclusion rule.", + "vintages": [ + { + "ascii_length_as_printed": 6, + "ascii_position_as_printed": 613, + "dictionary_spelling": "RNT_VAL", + "income_year": 2022, + "pdf_page_1based": 47, + "pdf_sha256": "66bd6e3fe516233ab63b75c60573451222b3b3d3235d61cfed96f64608de2117", + "printed_page": "6C-28", + "range_header": { + "maximum": 999999, + "minimum": -9999 + }, + "source_url": "https://www2.census.gov/programs-surveys/cps/datasets/2023/march/asec2023_ddl_pub_full.pdf", + "survey_year": 2023, + "universe_as_printed": "RNT_YN = 1", + "values_description": "0 = none or niu;\n-9999-999999 = rental income" + }, + { + "ascii_length_as_printed": 6, + "ascii_position_as_printed": 613, + "dictionary_spelling": "RNT_VAL", + "income_year": 2023, + "pdf_page_1based": 48, + "pdf_sha256": "761c67ea53f5c3264329b3e9ddbdd802826ba4a859122b8a8a2f29ab85b3a840", + "printed_page": "6C-28", + "range_header": { + "maximum": 999999, + "minimum": -9999 + }, + "source_url": "https://www2.census.gov/programs-surveys/cps/datasets/2024/march/asec2024_ddl_pub_full.pdf", + "survey_year": 2024, + "universe_as_printed": "RNT_YN = 1", + "values_description": "0 = none or niu;\n-9999-999999 = rental income" + }, + { + "ascii_length_as_printed": 6, + "ascii_position_as_printed": 621, + "dictionary_spelling": "RNT_VAL", + "income_year": 2024, + "pdf_page_1based": 49, + "pdf_sha256": "5cb80973326ef8b625fbaae70d80b0c641ce5d2b3911abd2fb4427abd5908a6f", + "printed_page": "6C-28", + "range_header": { + "maximum": 999999, + "minimum": -9999 + }, + "source_url": "https://www2.census.gov/programs-surveys/cps/datasets/2025/march/asec2025_ddl_pub_full.pdf", + "survey_year": 2025, + "universe_as_printed": "RNT_YN = 1", + "values_description": "0 = none or niu;\n-9999-999999 = rental income" + } + ], + "zero_semantics": "none_or_niu_not_distinguishable_from_amount_alone" + }, + { + "column": "SEMP_VAL", + "declared_niu_codes": [ + 0 + ], + "domain": { + "declared_negative_nonmoney_codes": [], + "declared_niu_codes": [ + 0 + ], + "declared_other_missing_codes": [], + "encoded_range_inclusive": { + "maximum": 9999999, + "minimum": -999999 + }, + "missing_code_statement": "No additional missing codes are declared in this field entry; this does not certify upstream storage null handling.", + "negative_dollars_permitted": true, + "numeric_type": "integer_US_dollars_in_public_use_dictionary", + "valid_dollar_range_excludes": [], + "valid_dollar_range_inclusive": { + "maximum": 9999999, + "minimum": -999999 + }, + "zero_semantics": "none_or_niu_not_distinguishable_from_amount_alone" + }, + "domain_status": "resolved_numeric_domain_all_three_vintages", + "entity": "person", + "grain": "person", + "maximum": 9999999, + "minimum": -999999, + "name": "SEMP_VAL", + "period": { + "currency": "USD", + "income_year": "source_year", + "observation_period": "annual", + "price_basis": "nominal_income_year", + "survey_year": "source_year + 1" + }, + "temporal_authority": { + "pdf_pages_1based": [ + 2, + 365 + ], + "pdf_sha256": "7f0cb9f791f2737ad12fd552b8cb201f4d1de266eac160322597344e73499fb4", + "scope": "2025 reference document confirms preceding-calendar-year convention; exact cohort-to-survey bindings additionally authenticated by frozen receipt.", + "source_url": "https://www2.census.gov/programs-surveys/cps/techdocs/cpsmar25.pdf" + }, + "universe_validation": "Literal dictionary universe is preserved as evidence, not automatically compiled into a row-exclusion rule.", + "vintages": [ + { + "ascii_length_as_printed": 7, + "ascii_position_as_printed": 405, + "dictionary_spelling": "SEMP_VAL", + "income_year": 2022, + "pdf_page_1based": 41, + "pdf_sha256": "66bd6e3fe516233ab63b75c60573451222b3b3d3235d61cfed96f64608de2117", + "printed_page": "6C-22", + "range_header": { + "maximum": 9999999, + "minimum": -999999 + }, + "source_url": "https://www2.census.gov/programs-surveys/cps/datasets/2023/march/asec2023_ddl_pub_full.pdf", + "survey_year": 2023, + "universe_as_printed": "ERN_YN=1 or SEOTR=1", + "values_description": "0 = none or niu;\n-999999-9999999 = own business self employment" + }, + { + "ascii_length_as_printed": 7, + "ascii_position_as_printed": 405, + "dictionary_spelling": "SEMP_VAL", + "income_year": 2023, + "pdf_page_1based": 42, + "pdf_sha256": "761c67ea53f5c3264329b3e9ddbdd802826ba4a859122b8a8a2f29ab85b3a840", + "printed_page": "6C-22", + "range_header": { + "maximum": 9999999, + "minimum": -999999 + }, + "source_url": "https://www2.census.gov/programs-surveys/cps/datasets/2024/march/asec2024_ddl_pub_full.pdf", + "survey_year": 2024, + "universe_as_printed": "ERN_YN=1 or SEOTR=1", + "values_description": "0 = none or niu;\n-999999-9999999 = own business self employment" + }, + { + "ascii_length_as_printed": 7, + "ascii_position_as_printed": 413, + "dictionary_spelling": "SEMP_VAL", + "income_year": 2024, + "pdf_page_1based": 43, + "pdf_sha256": "5cb80973326ef8b625fbaae70d80b0c641ce5d2b3911abd2fb4427abd5908a6f", + "printed_page": "6C-22", + "range_header": { + "maximum": 9999999, + "minimum": -999999 + }, + "source_url": "https://www2.census.gov/programs-surveys/cps/datasets/2025/march/asec2025_ddl_pub_full.pdf", + "survey_year": 2025, + "universe_as_printed": "ERN_YN=1 or SEOTR=1", + "values_description": "0 = none or niu;\n-999999-9999999 = own business self employment" + } + ], + "zero_semantics": "none_or_niu_not_distinguishable_from_amount_alone" + }, + { + "column": "SPM_CAPHOUSESUB", + "declared_niu_codes": [], + "domain": { + "declared_negative_nonmoney_codes": [], + "declared_niu_codes": [], + "declared_other_missing_codes": [], + "encoded_range_inclusive": { + "maximum": 99999, + "minimum": 0 + }, + "missing_code_statement": "No additional missing codes are declared in this field entry; this does not certify upstream storage null handling.", + "negative_dollars_permitted": false, + "numeric_type": "integer_US_dollars_in_public_use_dictionary", + "valid_dollar_range_excludes": [], + "valid_dollar_range_inclusive": { + "maximum": 99999, + "minimum": 0 + }, + "zero_semantics": "valid_zero_dollars_or_none_as_described" + }, + "domain_status": "resolved_numeric_domain_all_three_vintages", + "entity": "person", + "grain": "spm_unit_repeated_on_person", + "maximum": 99999, + "minimum": 0, + "name": "SPM_CAPHOUSESUB", + "period": { + "currency": "USD", + "income_year": "source_year", + "observation_period": "annual", + "price_basis": "nominal_income_year", + "survey_year": "source_year + 1" + }, + "temporal_authority": { + "pdf_pages_1based": [ + 2, + 365 + ], + "pdf_sha256": "7f0cb9f791f2737ad12fd552b8cb201f4d1de266eac160322597344e73499fb4", + "scope": "2025 reference document confirms preceding-calendar-year convention; exact cohort-to-survey bindings additionally authenticated by frozen receipt.", + "source_url": "https://www2.census.gov/programs-surveys/cps/techdocs/cpsmar25.pdf" + }, + "universe_validation": "Literal dictionary universe is preserved as evidence, not automatically compiled into a row-exclusion rule.", + "vintages": [ + { + "ascii_length_as_printed": 5, + "ascii_position_as_printed": 1436, + "dictionary_spelling": "SPM_CapHouseSub", + "income_year": 2022, + "pdf_page_1based": 85, + "pdf_sha256": "66bd6e3fe516233ab63b75c60573451222b3b3d3235d61cfed96f64608de2117", + "printed_page": "6C-66", + "range_header": { + "maximum": 99999, + "minimum": 0 + }, + "source_url": "https://www2.census.gov/programs-surveys/cps/datasets/2023/march/asec2023_ddl_pub_full.pdf", + "survey_year": 2023, + "universe_as_printed": "All Persons", + "values_description": "$0 to $99,999" + }, + { + "ascii_length_as_printed": 5, + "ascii_position_as_printed": 1432, + "dictionary_spelling": "SPM_CapHouseSub", + "income_year": 2023, + "pdf_page_1based": 85, + "pdf_sha256": "761c67ea53f5c3264329b3e9ddbdd802826ba4a859122b8a8a2f29ab85b3a840", + "printed_page": "6C-65", + "range_header": { + "maximum": 99999, + "minimum": 0 + }, + "source_url": "https://www2.census.gov/programs-surveys/cps/datasets/2024/march/asec2024_ddl_pub_full.pdf", + "survey_year": 2024, + "universe_as_printed": "All Persons", + "values_description": "$0 to $99,999" + }, + { + "ascii_length_as_printed": 5, + "ascii_position_as_printed": 936, + "dictionary_spelling": "SPM_CapHouseSub", + "income_year": 2024, + "pdf_page_1based": 61, + "pdf_sha256": "5cb80973326ef8b625fbaae70d80b0c641ce5d2b3911abd2fb4427abd5908a6f", + "printed_page": "6C-40", + "range_header": { + "maximum": 99999, + "minimum": 0 + }, + "source_url": "https://www2.census.gov/programs-surveys/cps/datasets/2025/march/asec2025_ddl_pub_full.pdf", + "survey_year": 2025, + "universe_as_printed": "All Persons", + "values_description": "$0 to $99,999" + } + ], + "zero_semantics": "valid_zero_dollars_or_none_as_described" + }, + { + "column": "SPM_CHILDCAREXPNS", + "declared_niu_codes": [], + "domain": { + "declared_negative_nonmoney_codes": [], + "declared_niu_codes": [], + "declared_other_missing_codes": [], + "encoded_range_inclusive": { + "maximum": 999999, + "minimum": 0 + }, + "missing_code_statement": "No additional missing codes are declared in this field entry; this does not certify upstream storage null handling.", + "negative_dollars_permitted": false, + "numeric_type": "integer_US_dollars_in_public_use_dictionary", + "valid_dollar_range_excludes": [], + "valid_dollar_range_inclusive": { + "maximum": 999999, + "minimum": 0 + }, + "zero_semantics": "valid_zero_dollars_or_none_as_described" + }, + "domain_status": "resolved_numeric_domain_all_three_vintages", + "entity": "person", + "grain": "spm_unit_repeated_on_person", + "maximum": 999999, + "minimum": 0, + "name": "SPM_CHILDCAREXPNS", + "period": { + "currency": "USD", + "income_year": "source_year", + "observation_period": "annual", + "price_basis": "nominal_income_year", + "survey_year": "source_year + 1" + }, + "temporal_authority": { + "pdf_pages_1based": [ + 2, + 365, + 304 + ], + "pdf_sha256": "7f0cb9f791f2737ad12fd552b8cb201f4d1de266eac160322597344e73499fb4", + "scope": "2025 reference document confirms preceding-calendar-year convention; exact cohort-to-survey bindings additionally authenticated by frozen receipt.", + "source_url": "https://www2.census.gov/programs-surveys/cps/techdocs/cpsmar25.pdf" + }, + "universe_validation": "Literal dictionary universe is preserved as evidence, not automatically compiled into a row-exclusion rule.", + "vintages": [ + { + "ascii_length_as_printed": 6, + "ascii_position_as_printed": 1447, + "dictionary_spelling": "SPM_ChildcareXpns", + "income_year": 2022, + "pdf_page_1based": 85, + "pdf_sha256": "66bd6e3fe516233ab63b75c60573451222b3b3d3235d61cfed96f64608de2117", + "printed_page": "6C-66", + "range_header": { + "maximum": 999999, + "minimum": 0 + }, + "source_url": "https://www2.census.gov/programs-surveys/cps/datasets/2023/march/asec2023_ddl_pub_full.pdf", + "survey_year": 2023, + "universe_as_printed": "All Persons", + "values_description": "$0 to $999,999" + }, + { + "ascii_length_as_printed": 6, + "ascii_position_as_printed": 1443, + "dictionary_spelling": "SPM_ChildcareXpns", + "income_year": 2023, + "pdf_page_1based": 85, + "pdf_sha256": "761c67ea53f5c3264329b3e9ddbdd802826ba4a859122b8a8a2f29ab85b3a840", + "printed_page": "6C-65", + "range_header": { + "maximum": 999999, + "minimum": 0 + }, + "source_url": "https://www2.census.gov/programs-surveys/cps/datasets/2024/march/asec2024_ddl_pub_full.pdf", + "survey_year": 2024, + "universe_as_printed": "All Persons", + "values_description": "$0 to $999,999" + }, + { + "ascii_length_as_printed": 6, + "ascii_position_as_printed": 947, + "dictionary_spelling": "SPM_ChildcareXpns", + "income_year": 2024, + "pdf_page_1based": 61, + "pdf_sha256": "5cb80973326ef8b625fbaae70d80b0c641ce5d2b3911abd2fb4427abd5908a6f", + "printed_page": "6C-40", + "range_header": { + "maximum": 999999, + "minimum": 0 + }, + "source_url": "https://www2.census.gov/programs-surveys/cps/datasets/2025/march/asec2025_ddl_pub_full.pdf", + "survey_year": 2025, + "universe_as_printed": "All Persons", + "values_description": "$0 to $999,999" + } + ], + "zero_semantics": "valid_zero_dollars_or_none_as_described" + }, + { + "column": "SPM_ENGVAL", + "declared_niu_codes": [], + "domain": { + "authority_range_alternatives": [ + { + "location": "Range header", + "maximum": 10000, + "minimum": 0 + }, + { + "location": "Values description", + "maximum": 99999, + "minimum": 0 + } + ], + "declared_negative_nonmoney_codes": [], + "declared_niu_codes": [], + "declared_other_missing_codes": [], + "definitely_outside_both_declared_ranges": { + "above": 99999, + "below": 0 + }, + "encoded_range_inclusive": null, + "missing_code_statement": "No additional missing codes are declared in this field entry; this does not certify upstream storage null handling.", + "negative_dollars_permitted": false, + "numeric_type": "integer_US_dollars_in_public_use_dictionary", + "uncontested_valid_dollar_range_inclusive": { + "maximum": 10000, + "minimum": 0 + }, + "unresolved_dollar_range_inclusive": { + "maximum": 99999, + "minimum": 10001 + }, + "valid_dollar_range_excludes": [], + "valid_dollar_range_inclusive": null, + "zero_semantics": "valid_zero_dollars_or_none_as_described" + }, + "domain_status": "partial_conflicting_upper_bound_all_three_vintages", + "entity": "person", + "grain": "spm_unit_repeated_on_person", + "maximum": 10000, + "minimum": 0, + "name": "SPM_ENGVAL", + "period": { + "currency": "USD", + "income_year": "source_year", + "observation_period": "annual", + "price_basis": "nominal_income_year", + "survey_year": "source_year + 1" + }, + "temporal_authority": { + "pdf_pages_1based": [ + 2, + 365, + 293 + ], + "pdf_sha256": "7f0cb9f791f2737ad12fd552b8cb201f4d1de266eac160322597344e73499fb4", + "scope": "2025 reference document confirms preceding-calendar-year convention; exact cohort-to-survey bindings additionally authenticated by frozen receipt.", + "source_url": "https://www2.census.gov/programs-surveys/cps/techdocs/cpsmar25.pdf" + }, + "universe_validation": "Literal dictionary universe is preserved as evidence, not automatically compiled into a row-exclusion rule.", + "vintages": [ + { + "ascii_length_as_printed": 5, + "ascii_position_as_printed": 1463, + "dictionary_spelling": "SPM_EngVal", + "income_year": 2022, + "pdf_page_1based": 85, + "pdf_sha256": "66bd6e3fe516233ab63b75c60573451222b3b3d3235d61cfed96f64608de2117", + "printed_page": "6C-66", + "range_header": { + "maximum": 10000, + "minimum": 0 + }, + "source_url": "https://www2.census.gov/programs-surveys/cps/datasets/2023/march/asec2023_ddl_pub_full.pdf", + "survey_year": 2023, + "universe_as_printed": "All Persons", + "values_description": "$0 to $99,999" + }, + { + "ascii_length_as_printed": 5, + "ascii_position_as_printed": 1459, + "dictionary_spelling": "SPM_EngVal", + "income_year": 2023, + "pdf_page_1based": 86, + "pdf_sha256": "761c67ea53f5c3264329b3e9ddbdd802826ba4a859122b8a8a2f29ab85b3a840", + "printed_page": "6C-66", + "range_header": { + "maximum": 10000, + "minimum": 0 + }, + "source_url": "https://www2.census.gov/programs-surveys/cps/datasets/2024/march/asec2024_ddl_pub_full.pdf", + "survey_year": 2024, + "universe_as_printed": "All Persons", + "values_description": "$0 to $99,999" + }, + { + "ascii_length_as_printed": 5, + "ascii_position_as_printed": 963, + "dictionary_spelling": "SPM_EngVal", + "income_year": 2024, + "pdf_page_1based": 61, + "pdf_sha256": "5cb80973326ef8b625fbaae70d80b0c641ce5d2b3911abd2fb4427abd5908a6f", + "printed_page": "6C-40", + "range_header": { + "maximum": 10000, + "minimum": 0 + }, + "source_url": "https://www2.census.gov/programs-surveys/cps/datasets/2025/march/asec2025_ddl_pub_full.pdf", + "survey_year": 2025, + "universe_as_printed": "All Persons", + "values_description": "$0 to $99,999" + } + ], + "zero_semantics": "valid_zero_dollars_or_none_as_described" + }, + { + "column": "SSI_VAL", + "declared_niu_codes": [ + 0 + ], + "domain": { + "declared_negative_nonmoney_codes": [], + "declared_niu_codes": [ + 0 + ], + "declared_other_missing_codes": [], + "encoded_range_inclusive": { + "maximum": 99999, + "minimum": 0 + }, + "missing_code_statement": "No additional missing codes are declared in this field entry; this does not certify upstream storage null handling.", + "negative_dollars_permitted": false, + "numeric_type": "integer_US_dollars_in_public_use_dictionary", + "valid_dollar_range_excludes": [], + "valid_dollar_range_inclusive": { + "maximum": 99999, + "minimum": 0 + }, + "zero_semantics": "none_or_niu_not_distinguishable_from_amount_alone" + }, + "domain_status": "resolved_numeric_domain_all_three_vintages", + "entity": "person", + "grain": "person", + "maximum": 99999, + "minimum": 0, + "name": "SSI_VAL", + "period": { + "currency": "USD", + "income_year": "source_year", + "observation_period": "annual", + "price_basis": "nominal_income_year", + "survey_year": "source_year + 1" + }, + "temporal_authority": { + "pdf_pages_1based": [ + 2, + 365 + ], + "pdf_sha256": "7f0cb9f791f2737ad12fd552b8cb201f4d1de266eac160322597344e73499fb4", + "scope": "2025 reference document confirms preceding-calendar-year convention; exact cohort-to-survey bindings additionally authenticated by frozen receipt.", + "source_url": "https://www2.census.gov/programs-surveys/cps/techdocs/cpsmar25.pdf" + }, + "universe_validation": "Literal dictionary universe is preserved as evidence, not automatically compiled into a row-exclusion rule.", + "vintages": [ + { + "ascii_length_as_printed": 5, + "ascii_position_as_printed": 632, + "dictionary_spelling": "SSI_VAL", + "income_year": 2022, + "pdf_page_1based": 47, + "pdf_sha256": "66bd6e3fe516233ab63b75c60573451222b3b3d3235d61cfed96f64608de2117", + "printed_page": "6C-28", + "range_header": { + "maximum": 99999, + "minimum": 0 + }, + "source_url": "https://www2.census.gov/programs-surveys/cps/datasets/2023/march/asec2023_ddl_pub_full.pdf", + "survey_year": 2023, + "universe_as_printed": "SSI_YN = 1", + "values_description": "0 = none or niu\n1-99999 = supplemental security income" + }, + { + "ascii_length_as_printed": 5, + "ascii_position_as_printed": 632, + "dictionary_spelling": "SSI_VAL", + "income_year": 2023, + "pdf_page_1based": 48, + "pdf_sha256": "761c67ea53f5c3264329b3e9ddbdd802826ba4a859122b8a8a2f29ab85b3a840", + "printed_page": "6C-28", + "range_header": { + "maximum": 99999, + "minimum": 0 + }, + "source_url": "https://www2.census.gov/programs-surveys/cps/datasets/2024/march/asec2024_ddl_pub_full.pdf", + "survey_year": 2024, + "universe_as_printed": "SSI_YN = 1", + "values_description": "0 = none or niu\n1-99999 = supplemental security income" + }, + { + "ascii_length_as_printed": 5, + "ascii_position_as_printed": 640, + "dictionary_spelling": "SSI_VAL", + "income_year": 2024, + "pdf_page_1based": 49, + "pdf_sha256": "5cb80973326ef8b625fbaae70d80b0c641ce5d2b3911abd2fb4427abd5908a6f", + "printed_page": "6C-28", + "range_header": { + "maximum": 99999, + "minimum": 0 + }, + "source_url": "https://www2.census.gov/programs-surveys/cps/datasets/2025/march/asec2025_ddl_pub_full.pdf", + "survey_year": 2025, + "universe_as_printed": "SSI_YN = 1", + "values_description": "0 = none or niu\n1-99999 = supplemental security income" + } + ], + "zero_semantics": "none_or_niu_not_distinguishable_from_amount_alone" + }, + { + "column": "SS_VAL", + "declared_niu_codes": [ + 0 + ], + "domain": { + "declared_negative_nonmoney_codes": [], + "declared_niu_codes": [ + 0 + ], + "declared_other_missing_codes": [], + "encoded_range_inclusive": { + "maximum": 99999, + "minimum": 0 + }, + "missing_code_statement": "No additional missing codes are declared in this field entry; this does not certify upstream storage null handling.", + "negative_dollars_permitted": false, + "numeric_type": "integer_US_dollars_in_public_use_dictionary", + "valid_dollar_range_excludes": [], + "valid_dollar_range_inclusive": { + "maximum": 99999, + "minimum": 0 + }, + "zero_semantics": "none_or_niu_not_distinguishable_from_amount_alone" + }, + "domain_status": "resolved_numeric_domain_all_three_vintages", + "entity": "person", + "grain": "person", + "maximum": 99999, + "minimum": 0, + "name": "SS_VAL", + "period": { + "currency": "USD", + "income_year": "source_year", + "observation_period": "annual", + "price_basis": "nominal_income_year", + "survey_year": "source_year + 1" + }, + "temporal_authority": { + "pdf_pages_1based": [ + 2, + 365 + ], + "pdf_sha256": "7f0cb9f791f2737ad12fd552b8cb201f4d1de266eac160322597344e73499fb4", + "scope": "2025 reference document confirms preceding-calendar-year convention; exact cohort-to-survey bindings additionally authenticated by frozen receipt.", + "source_url": "https://www2.census.gov/programs-surveys/cps/techdocs/cpsmar25.pdf" + }, + "universe_validation": "Literal dictionary universe is preserved as evidence, not automatically compiled into a row-exclusion rule.", + "vintages": [ + { + "ascii_length_as_printed": 5, + "ascii_position_as_printed": 626, + "dictionary_spelling": "SS_VAL", + "income_year": 2022, + "pdf_page_1based": 47, + "pdf_sha256": "66bd6e3fe516233ab63b75c60573451222b3b3d3235d61cfed96f64608de2117", + "printed_page": "6C-28", + "range_header": { + "maximum": 99999, + "minimum": 0 + }, + "source_url": "https://www2.census.gov/programs-surveys/cps/datasets/2023/march/asec2023_ddl_pub_full.pdf", + "survey_year": 2023, + "universe_as_printed": "SS_YN = 1", + "values_description": "0 = none or niu;\n1-99999 = social security" + }, + { + "ascii_length_as_printed": 5, + "ascii_position_as_printed": 626, + "dictionary_spelling": "SS_VAL", + "income_year": 2023, + "pdf_page_1based": 48, + "pdf_sha256": "761c67ea53f5c3264329b3e9ddbdd802826ba4a859122b8a8a2f29ab85b3a840", + "printed_page": "6C-28", + "range_header": { + "maximum": 99999, + "minimum": 0 + }, + "source_url": "https://www2.census.gov/programs-surveys/cps/datasets/2024/march/asec2024_ddl_pub_full.pdf", + "survey_year": 2024, + "universe_as_printed": "SS_YN = 1", + "values_description": "0 = none or niu;\n1-99999 = social security" + }, + { + "ascii_length_as_printed": 5, + "ascii_position_as_printed": 634, + "dictionary_spelling": "SS_VAL", + "income_year": 2024, + "pdf_page_1based": 49, + "pdf_sha256": "5cb80973326ef8b625fbaae70d80b0c641ce5d2b3911abd2fb4427abd5908a6f", + "printed_page": "6C-28", + "range_header": { + "maximum": 99999, + "minimum": 0 + }, + "source_url": "https://www2.census.gov/programs-surveys/cps/datasets/2025/march/asec2025_ddl_pub_full.pdf", + "survey_year": 2025, + "universe_as_printed": "SS_YN = 1", + "values_description": "0 = none or niu;\n1-99999 = social security" + } + ], + "zero_semantics": "none_or_niu_not_distinguishable_from_amount_alone" + }, + { + "column": "UC_VAL", + "declared_niu_codes": [ + 0 + ], + "domain": { + "declared_negative_nonmoney_codes": [], + "declared_niu_codes": [ + 0 + ], + "declared_other_missing_codes": [], + "encoded_range_inclusive": { + "maximum": 99999, + "minimum": 0 + }, + "missing_code_statement": "No additional missing codes are declared in this field entry; this does not certify upstream storage null handling.", + "negative_dollars_permitted": false, + "numeric_type": "integer_US_dollars_in_public_use_dictionary", + "valid_dollar_range_excludes": [], + "valid_dollar_range_inclusive": { + "maximum": 99999, + "minimum": 0 + }, + "zero_semantics": "none_or_niu_not_distinguishable_from_amount_alone" + }, + "domain_status": "resolved_numeric_domain_all_three_vintages", + "entity": "person", + "grain": "person", + "maximum": 99999, + "minimum": 0, + "name": "UC_VAL", + "period": { + "currency": "USD", + "income_year": "source_year", + "observation_period": "annual", + "price_basis": "nominal_income_year", + "survey_year": "source_year + 1" + }, + "temporal_authority": { + "pdf_pages_1based": [ + 2, + 365 + ], + "pdf_sha256": "7f0cb9f791f2737ad12fd552b8cb201f4d1de266eac160322597344e73499fb4", + "scope": "2025 reference document confirms preceding-calendar-year convention; exact cohort-to-survey bindings additionally authenticated by frozen receipt.", + "source_url": "https://www2.census.gov/programs-surveys/cps/techdocs/cpsmar25.pdf" + }, + "universe_validation": "Literal dictionary universe is preserved as evidence, not automatically compiled into a row-exclusion rule.", + "vintages": [ + { + "ascii_length_as_printed": 5, + "ascii_position_as_printed": 664, + "dictionary_spelling": "UC_VAL", + "income_year": 2022, + "pdf_page_1based": 48, + "pdf_sha256": "66bd6e3fe516233ab63b75c60573451222b3b3d3235d61cfed96f64608de2117", + "printed_page": "6C-29", + "range_header": { + "maximum": 99999, + "minimum": 0 + }, + "source_url": "https://www2.census.gov/programs-surveys/cps/datasets/2023/march/asec2023_ddl_pub_full.pdf", + "survey_year": 2023, + "universe_as_printed": "UC_YN = 1", + "values_description": "0 = none or niu\n1-99999 = unemployment compensation" + }, + { + "ascii_length_as_printed": 5, + "ascii_position_as_printed": 664, + "dictionary_spelling": "UC_VAL", + "income_year": 2023, + "pdf_page_1based": 49, + "pdf_sha256": "761c67ea53f5c3264329b3e9ddbdd802826ba4a859122b8a8a2f29ab85b3a840", + "printed_page": "6C-29", + "range_header": { + "maximum": 99999, + "minimum": 0 + }, + "source_url": "https://www2.census.gov/programs-surveys/cps/datasets/2024/march/asec2024_ddl_pub_full.pdf", + "survey_year": 2024, + "universe_as_printed": "UC_YN = 1", + "values_description": "0 = none or niu\n1-99999 = unemployment compensation" + }, + { + "ascii_length_as_printed": 5, + "ascii_position_as_printed": 672, + "dictionary_spelling": "UC_VAL", + "income_year": 2024, + "pdf_page_1based": 50, + "pdf_sha256": "5cb80973326ef8b625fbaae70d80b0c641ce5d2b3911abd2fb4427abd5908a6f", + "printed_page": "6C-29", + "range_header": { + "maximum": 99999, + "minimum": 0 + }, + "source_url": "https://www2.census.gov/programs-surveys/cps/datasets/2025/march/asec2025_ddl_pub_full.pdf", + "survey_year": 2025, + "universe_as_printed": "UC_YN = 1", + "values_description": "0 = none or niu\n1-99999 = unemployment compensation" + } + ], + "zero_semantics": "none_or_niu_not_distinguishable_from_amount_alone" + }, + { + "column": "VET_VAL", + "declared_niu_codes": [ + 0 + ], + "domain": { + "declared_negative_nonmoney_codes": [], + "declared_niu_codes": [ + 0 + ], + "declared_other_missing_codes": [], + "encoded_range_inclusive": { + "maximum": 999999, + "minimum": 0 + }, + "missing_code_statement": "No additional missing codes are declared in this field entry; this does not certify upstream storage null handling.", + "negative_dollars_permitted": false, + "numeric_type": "integer_US_dollars_in_public_use_dictionary", + "valid_dollar_range_excludes": [], + "valid_dollar_range_inclusive": { + "maximum": 999999, + "minimum": 0 + }, + "zero_semantics": "none_or_niu_not_distinguishable_from_amount_alone" + }, + "domain_status": "resolved_numeric_domain_all_three_vintages", + "entity": "person", + "grain": "person", + "maximum": 999999, + "minimum": 0, + "name": "VET_VAL", + "period": { + "currency": "USD", + "income_year": "source_year", + "observation_period": "annual", + "price_basis": "nominal_income_year", + "survey_year": "source_year + 1" + }, + "temporal_authority": { + "pdf_pages_1based": [ + 2, + 365 + ], + "pdf_sha256": "7f0cb9f791f2737ad12fd552b8cb201f4d1de266eac160322597344e73499fb4", + "scope": "2025 reference document confirms preceding-calendar-year convention; exact cohort-to-survey bindings additionally authenticated by frozen receipt.", + "source_url": "https://www2.census.gov/programs-surveys/cps/techdocs/cpsmar25.pdf" + }, + "universe_validation": "Literal dictionary universe is preserved as evidence, not automatically compiled into a row-exclusion rule.", + "vintages": [ + { + "ascii_length_as_printed": 6, + "ascii_position_as_printed": 676, + "dictionary_spelling": "VET_VAL", + "income_year": 2022, + "pdf_page_1based": 48, + "pdf_sha256": "66bd6e3fe516233ab63b75c60573451222b3b3d3235d61cfed96f64608de2117", + "printed_page": "6C-29", + "range_header": { + "maximum": 999999, + "minimum": 0 + }, + "source_url": "https://www2.census.gov/programs-surveys/cps/datasets/2023/march/asec2023_ddl_pub_full.pdf", + "survey_year": 2023, + "universe_as_printed": "VET_YN = 1", + "values_description": "0 = none or niu\n1-999999 = veterans' payments" + }, + { + "ascii_length_as_printed": 6, + "ascii_position_as_printed": 676, + "dictionary_spelling": "VET_VAL", + "income_year": 2023, + "pdf_page_1based": 49, + "pdf_sha256": "761c67ea53f5c3264329b3e9ddbdd802826ba4a859122b8a8a2f29ab85b3a840", + "printed_page": "6C-29", + "range_header": { + "maximum": 999999, + "minimum": 0 + }, + "source_url": "https://www2.census.gov/programs-surveys/cps/datasets/2024/march/asec2024_ddl_pub_full.pdf", + "survey_year": 2024, + "universe_as_printed": "VET_YN = 1", + "values_description": "0 = none or niu\n1-999999 = veterans' payments" + }, + { + "ascii_length_as_printed": 6, + "ascii_position_as_printed": 684, + "dictionary_spelling": "VET_VAL", + "income_year": 2024, + "pdf_page_1based": 51, + "pdf_sha256": "5cb80973326ef8b625fbaae70d80b0c641ce5d2b3911abd2fb4427abd5908a6f", + "printed_page": "6C-30", + "range_header": { + "maximum": 999999, + "minimum": 0 + }, + "source_url": "https://www2.census.gov/programs-surveys/cps/datasets/2025/march/asec2025_ddl_pub_full.pdf", + "survey_year": 2025, + "universe_as_printed": "VET_YN = 1", + "values_description": "0 = none or niu\n1-999999 = veterans' payments" + } + ], + "zero_semantics": "none_or_niu_not_distinguishable_from_amount_alone" + }, + { + "column": "WC_VAL", + "declared_niu_codes": [ + 0 + ], + "domain": { + "declared_negative_nonmoney_codes": [], + "declared_niu_codes": [ + 0 + ], + "declared_other_missing_codes": [], + "encoded_range_inclusive": { + "maximum": 99999, + "minimum": 0 + }, + "missing_code_statement": "No additional missing codes are declared in this field entry; this does not certify upstream storage null handling.", + "negative_dollars_permitted": false, + "numeric_type": "integer_US_dollars_in_public_use_dictionary", + "valid_dollar_range_excludes": [], + "valid_dollar_range_inclusive": { + "maximum": 99999, + "minimum": 0 + }, + "zero_semantics": "none_or_niu_not_distinguishable_from_amount_alone" + }, + "domain_status": "resolved_numeric_domain_all_three_vintages", + "entity": "person", + "grain": "person", + "maximum": 99999, + "minimum": 0, + "name": "WC_VAL", + "period": { + "currency": "USD", + "income_year": "source_year", + "observation_period": "annual", + "price_basis": "nominal_income_year", + "survey_year": "source_year + 1" + }, + "temporal_authority": { + "pdf_pages_1based": [ + 2, + 365 + ], + "pdf_sha256": "7f0cb9f791f2737ad12fd552b8cb201f4d1de266eac160322597344e73499fb4", + "scope": "2025 reference document confirms preceding-calendar-year convention; exact cohort-to-survey bindings additionally authenticated by frozen receipt.", + "source_url": "https://www2.census.gov/programs-surveys/cps/techdocs/cpsmar25.pdf" + }, + "universe_validation": "Literal dictionary universe is preserved as evidence, not automatically compiled into a row-exclusion rule.", + "vintages": [ + { + "ascii_length_as_printed": 5, + "ascii_position_as_printed": 684, + "dictionary_spelling": "WC_VAL", + "income_year": 2022, + "pdf_page_1based": 49, + "pdf_sha256": "66bd6e3fe516233ab63b75c60573451222b3b3d3235d61cfed96f64608de2117", + "printed_page": "6C-30", + "range_header": { + "maximum": 99999, + "minimum": 0 + }, + "source_url": "https://www2.census.gov/programs-surveys/cps/datasets/2023/march/asec2023_ddl_pub_full.pdf", + "survey_year": 2023, + "universe_as_printed": "WC_YN = 1", + "values_description": "0 = none or niu\n1-99999 = worker's compensation" + }, + { + "ascii_length_as_printed": 5, + "ascii_position_as_printed": 684, + "dictionary_spelling": "WC_VAL", + "income_year": 2023, + "pdf_page_1based": 50, + "pdf_sha256": "761c67ea53f5c3264329b3e9ddbdd802826ba4a859122b8a8a2f29ab85b3a840", + "printed_page": "6C-30", + "range_header": { + "maximum": 99999, + "minimum": 0 + }, + "source_url": "https://www2.census.gov/programs-surveys/cps/datasets/2024/march/asec2024_ddl_pub_full.pdf", + "survey_year": 2024, + "universe_as_printed": "WC_YN = 1", + "values_description": "0 = none or niu\n1-99999 = worker's compensation" + }, + { + "ascii_length_as_printed": 5, + "ascii_position_as_printed": 692, + "dictionary_spelling": "WC_VAL", + "income_year": 2024, + "pdf_page_1based": 51, + "pdf_sha256": "5cb80973326ef8b625fbaae70d80b0c641ce5d2b3911abd2fb4427abd5908a6f", + "printed_page": "6C-30", + "range_header": { + "maximum": 99999, + "minimum": 0 + }, + "source_url": "https://www2.census.gov/programs-surveys/cps/datasets/2025/march/asec2025_ddl_pub_full.pdf", + "survey_year": 2025, + "universe_as_printed": "WC_YN = 1", + "values_description": "0 = none or niu\n1-99999 = worker's compensation" + } + ], + "zero_semantics": "none_or_niu_not_distinguishable_from_amount_alone" + }, + { + "column": "WSAL_VAL", + "declared_niu_codes": [ + 0 + ], + "domain": { + "declared_negative_nonmoney_codes": [], + "declared_niu_codes": [ + 0 + ], + "declared_other_missing_codes": [], + "encoded_range_inclusive": { + "maximum": 9999999, + "minimum": 0 + }, + "missing_code_statement": "No additional missing codes are declared in this field entry; this does not certify upstream storage null handling.", + "negative_dollars_permitted": false, + "numeric_type": "integer_US_dollars_in_public_use_dictionary", + "valid_dollar_range_excludes": [], + "valid_dollar_range_inclusive": { + "maximum": 9999999, + "minimum": 0 + }, + "zero_semantics": "none_or_niu_not_distinguishable_from_amount_alone" + }, + "domain_status": "resolved_numeric_domain_all_three_vintages", + "entity": "person", + "grain": "person", + "maximum": 9999999, + "minimum": 0, + "name": "WSAL_VAL", + "period": { + "currency": "USD", + "income_year": "source_year", + "observation_period": "annual", + "price_basis": "nominal_income_year", + "survey_year": "source_year + 1" + }, + "temporal_authority": { + "pdf_pages_1based": [ + 2, + 365 + ], + "pdf_sha256": "7f0cb9f791f2737ad12fd552b8cb201f4d1de266eac160322597344e73499fb4", + "scope": "2025 reference document confirms preceding-calendar-year convention; exact cohort-to-survey bindings additionally authenticated by frozen receipt.", + "source_url": "https://www2.census.gov/programs-surveys/cps/techdocs/cpsmar25.pdf" + }, + "universe_validation": "Literal dictionary universe is preserved as evidence, not automatically compiled into a row-exclusion rule.", + "vintages": [ + { + "ascii_length_as_printed": 7, + "ascii_position_as_printed": 422, + "dictionary_spelling": "WSAL_VAL", + "income_year": 2022, + "pdf_page_1based": 41, + "pdf_sha256": "66bd6e3fe516233ab63b75c60573451222b3b3d3235d61cfed96f64608de2117", + "printed_page": "6C-22", + "range_header": { + "maximum": 9999999, + "minimum": 0 + }, + "source_url": "https://www2.census.gov/programs-surveys/cps/datasets/2023/march/asec2023_ddl_pub_full.pdf", + "survey_year": 2023, + "universe_as_printed": "ERN_YN=1 or WAGEOTR=1", + "values_description": "0 = none or niu;\n1-9999999 = wage and salary" + }, + { + "ascii_length_as_printed": 7, + "ascii_position_as_printed": 422, + "dictionary_spelling": "WSAL_VAL", + "income_year": 2023, + "pdf_page_1based": 42, + "pdf_sha256": "761c67ea53f5c3264329b3e9ddbdd802826ba4a859122b8a8a2f29ab85b3a840", + "printed_page": "6C-22", + "range_header": { + "maximum": 9999999, + "minimum": 0 + }, + "source_url": "https://www2.census.gov/programs-surveys/cps/datasets/2024/march/asec2024_ddl_pub_full.pdf", + "survey_year": 2024, + "universe_as_printed": "ERN_YN=1 or WAGEOTR=1", + "values_description": "0 = none or niu;\n1-9999999 = wage and salary" + }, + { + "ascii_length_as_printed": 7, + "ascii_position_as_printed": 430, + "dictionary_spelling": "WSAL_VAL", + "income_year": 2024, + "pdf_page_1based": 43, + "pdf_sha256": "5cb80973326ef8b625fbaae70d80b0c641ce5d2b3911abd2fb4427abd5908a6f", + "printed_page": "6C-22", + "range_header": { + "maximum": 9999999, + "minimum": 0 + }, + "source_url": "https://www2.census.gov/programs-surveys/cps/datasets/2025/march/asec2025_ddl_pub_full.pdf", + "survey_year": 2025, + "universe_as_printed": "ERN_YN=1 or WAGEOTR=1", + "values_description": "0 = none or niu;\n1-9999999 = wage and salary" + } + ], + "zero_semantics": "none_or_niu_not_distinguishable_from_amount_alone" + } + ], + "max_households": 400000, + "max_persons": 1000000, + "recipe": "us_asec_current_money_ccpiu_2024_v1", + "registry_sha256": "bdeb4d2fa1c37c6064530d560a62b12c515118608845b30d01b06c2eeeda8c6e", + "schema_version": 1, + "source_years": [ + [ + 2022, + 2023 + ], + [ + 2023, + 2024 + ], + [ + 2024, + 2025 + ] + ], + "zero_origin_policy": "frozen_fillna_zero_origin_unresolved" +} diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/asec_current_money_engine_defaults_v1.json b/packages/microcosm-build/src/microcosm/build/us_runtime/asec_current_money_engine_defaults_v1.json new file mode 100644 index 000000000..583c3684d --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/asec_current_money_engine_defaults_v1.json @@ -0,0 +1,34 @@ +{ + "schema_version": 1, + "artifact_kind": "microcosm.us.asec_current_money_engine_defaults.v1", + "engine_package": "policyengine-us", + "engine_version": "1.819.0", + "allowlist": [], + "allowlist_policy": "no engine default is an approved input to an admitted output; a closure leaf this graph does not produce blocks its root", + "admitted_roots": [ + "capital_gains", + "dividend_income", + "employment_income", + "social_security" + ], + "blocked_roots": { + "interest_income": "needs tax_exempt_interest_income, which this slice does not map", + "pension_income": "needs the public pension leaves, which this slice does not map", + "self_employment_income": "needs sstb_self_employment_income_before_lsr, which this slice does not map" + }, + "release_eligible": false, + "scoring_claim": "none; these are engine outputs on an engineering sample, not a tax or benefit score", + "baseline_runtime": { + "policyengine-us": { + "version": "1.819.0", + "files": 12313, + "source_and_parameters_sha256": "12c474205741021bb45ae618f85ddd6d15e8cbb51dece8334f2d52c04500c674" + }, + "policyengine-core": { + "version": "3.31.0", + "files": 175, + "source_and_parameters_sha256": "fbcc6f3d8e86c023c29e0782d95103155828cfa4ef6a80ed6f785f619be6439e" + } + }, + "baseline_runtime_file_policy": "all package Python source and parameters YAML/JSON; sorted relative paths and SHA256; no population data" +} diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/asec_current_money_graph_consumers_v1.json b/packages/microcosm-build/src/microcosm/build/us_runtime/asec_current_money_graph_consumers_v1.json new file mode 100644 index 000000000..cd0270dee --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/asec_current_money_graph_consumers_v1.json @@ -0,0 +1,127 @@ +{ + "all_current_money_consumers_wired": false, + "artifact_kind": "microcosm.us.asec_current_money_graph_consumers.v1", + "blocked_engine_outputs": { + "interest_income": [ + "tax_exempt_interest_income" + ], + "pension_income": [ + "tax_exempt_public_pension_income", + "taxable_public_pension_income" + ], + "self_employment_income": [ + "sstb_self_employment_income_before_lsr" + ] + }, + "engine_defaults_allowlist": [], + "implementation_status": "prepared_current_money_graph_and_microunit_wired_other_nominal_routes_pending", + "pending_nominal_consumer_routes": [ + "acs_release_predictors_reported_ssi_anchor", + "asec_household_observations", + "asec_other_income_split", + "child_support", + "childcare", + "cps_carried", + "disability_benefits", + "education_inputs", + "eligibility_inputs", + "energy_subsidy", + "immigration", + "retirement_contributions", + "retirement_distributions", + "voluntary_filing_current_leaf", + "workers_compensation" + ], + "release_eligible": false, + "schema_version": 1, + "scope": "current_money_only_no_prior_wages", + "source_declarations": { + "binding": "unchanged historical source recipe; this additive record supersedes implementation status only for the explicitly wired routes below; source address roster and T1/money identities are unchanged", + "resource": "asec_current_money_consumers_v1.json", + "sha256": "c56058eaf08b68b2add4a5d2b366559ea6cc5238b039031be7094ef1f6457ff7" + }, + "wired_routes": { + "asec_prepared_current_money_graph": { + "engine_output_storage": "evaluation_artifact_only", + "engine_outputs": [ + "capital_gains", + "dividend_income", + "employment_income", + "social_security" + ], + "module": "microcosm.build.us_runtime.graph_asec_prepared", + "money_fields": [ + "ANN_VAL", + "CAP_VAL", + "DIV_VAL", + "DST_VAL1", + "DST_VAL1_YNG", + "DST_VAL2", + "DST_VAL2_YNG", + "FRSE_VAL", + "INT_VAL", + "OI_VAL", + "PHIP_VAL", + "PMED_VAL", + "PNSN_VAL", + "POTC_VAL", + "RNT_VAL", + "SEMP_VAL", + "SPM_CHILDCAREXPNS", + "SS_VAL", + "UC_VAL", + "WSAL_VAL" + ], + "money_input": "typed selected current-money body; never standalone ReadyCurrentMoney", + "person_leaves": [ + "age", + "alimony_income", + "employment_income_before_lsr", + "farm_operations_income", + "health_insurance_premiums_without_medicare_part_b", + "long_term_capital_gains_before_response", + "miscellaneous_income", + "non_qualified_dividend_income", + "other_medical_expenses", + "over_the_counter_health_expenses", + "qualified_dividend_income", + "rental_income", + "self_employment_income_before_lsr", + "short_term_capital_gains", + "social_security_dependents", + "social_security_disability", + "social_security_retirement", + "social_security_survivors", + "strike_benefits", + "tax_exempt_private_pension_income", + "taxable_interest_income", + "taxable_ira_distributions", + "taxable_private_pension_income", + "unemployment_compensation" + ], + "spm_unit_leaves": [ + "spm_unit_pre_subsidy_childcare_expenses" + ] + }, + "microunit": { + "module": "microcosm.build.us_runtime.asec_current_money_units", + "money_fields": [ + "ANN_VAL", + "CAP_VAL", + "DIV_VAL", + "FRSE_VAL", + "INT_VAL", + "OI_VAL", + "PNSN_VAL", + "PTOTVAL", + "RNT_VAL", + "SEMP_VAL", + "SS_VAL", + "UC_VAL", + "WSAL_VAL" + ], + "money_input": "authenticated full-source ReadyCurrentMoney", + "student_controls": "separately authenticated S attached before tax construction; interview-week proxy only" + } + } +} diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/asec_current_money_graph_resources.py b/packages/microcosm-build/src/microcosm/build/us_runtime/asec_current_money_graph_resources.py new file mode 100644 index 000000000..bab0c7b02 --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/asec_current_money_graph_resources.py @@ -0,0 +1,34 @@ +"""Pinned additive consumer status, separate from the immutable money recipe. + +The source recipe's declarations and execution identity stay byte-for-byte +unchanged. This graph record reports the routes actually wired by this slice; +it grants no source authority and changes no source address or monetary rule. +""" + +from hashlib import sha256 +from importlib import resources + +from .asec_current_money import RESOURCE_PINS, _parse + +GRAPH_CONSUMERS_RESOURCE = "asec_current_money_graph_consumers_v1.json" +GRAPH_CONSUMERS_SHA256 = ( + "67432a10d05e11dd4b97a80cd4f04ed0eb6e56c4c8e21d5dffe87c5f67e27c66" +) + + +def load_graph_current_money_consumers() -> dict: + """Verify graph status and its unchanged source-declaration parent.""" + package = resources.files(__package__) + payload = package.joinpath(GRAPH_CONSUMERS_RESOURCE).read_bytes() + if sha256(payload).hexdigest() != GRAPH_CONSUMERS_SHA256: + raise ValueError("GRAPH_CONSUMERS_FINGERPRINT") + document = _parse(payload) + parent = document["source_declarations"] + if ( + parent["resource"] != "asec_current_money_consumers_v1.json" + or parent["sha256"] != RESOURCE_PINS[2] + or sha256(package.joinpath(parent["resource"]).read_bytes()).hexdigest() + != parent["sha256"] + ): + raise ValueError("GRAPH_CONSUMERS_PARENT_FINGERPRINT") + return document diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/asec_current_money_price_basis_v1.json b/packages/microcosm-build/src/microcosm/build/us_runtime/asec_current_money_price_basis_v1.json new file mode 100644 index 000000000..445650d70 --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/asec_current_money_price_basis_v1.json @@ -0,0 +1,21 @@ +{ + "algorithm": "parse_float64_target_divide_source_then_multiply_nonzero_v1", + "cells": { + "2022": "163.6", + "2023": "170.0", + "2024": "174.4" + }, + "currency": "USD", + "identity_year": "copy_normalized_bytes", + "numeric_scope": "platform_bitwise", + "price_basis": "census_ccpiu_annual_2024", + "schema_version": 1, + "source_finality": "not_stated_in_source", + "target_year": 2024, + "workbook": { + "sha256": "8a5ef078b02c48c9e6414f3ca0cbdfe4a013ce6acf1a3fda42e6bd9c7da49e48", + "size_bytes": 41472, + "url": "https://www2.census.gov/programs-surveys/demo/tables/p60/286/annual-index-value_annual-percent-change.xls" + }, + "zero": "positive_zero_no_arithmetic" +} diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/asec_current_money_resources.py b/packages/microcosm-build/src/microcosm/build/us_runtime/asec_current_money_resources.py new file mode 100644 index 000000000..66d8e6d10 --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/asec_current_money_resources.py @@ -0,0 +1,114 @@ +"""Explicit packaged-resource and installed microunit byte verification. + +This module never imports microunit or runs units. The lower-level packaged +function permits explicit inspected-contract injection for pure synthetic tests; +it does not certify an installed distribution or authenticate source data. +""" + +import platform +import sys +from importlib import metadata, resources + +import numpy as np +import pandas as pd + +from .asec_current_money import ( + MICROUNIT_SHA256, + MICROUNIT_VERSION, + CurrentMoneyResources, + MoneyRefusalError, + _json, + _require, + _sha, +) + +RESOURCE_NAMES = ( + "asec_current_money_domains_v1.json", + "asec_current_money_price_basis_v1.json", + "asec_current_money_consumers_v1.json", +) +MODULE_NAMES = ( + "asec_current_money.py", + "_asec_current_money_codec.py", + "asec_current_money_resources.py", +) + + +def packaged_current_money_resources( + *, inspected_version: str, inspected_module_sha256: str +) -> CurrentMoneyResources: + """Read package resources with explicit claims, for synthetic fixture injection. + + Use load_current_money_resources to verify the actual installed reader. + Neither resource-reading function creates authenticated source authority. + """ + package = resources.files(__package__) + identity = _json( + { + "modules": { + name: _sha(package.joinpath(name).read_bytes()) for name in MODULE_NAMES + }, + "dependencies": { + "python": platform.python_version(), + "numpy": np.__version__, + "pandas": pd.__version__, + }, + "platform": { + "system": platform.system(), + "machine": platform.machine(), + "byteorder": sys.byteorder, + "python_implementation": platform.python_implementation(), + }, + } + ) + return CurrentMoneyResources( + *(package.joinpath(name).read_bytes() for name in RESOURCE_NAMES), + identity, + inspected_version, + inspected_module_sha256, + ) + + +def load_current_money_resources( + *, microunit_search_path: tuple[str, ...] | None = None +) -> CurrentMoneyResources: + """Verify pinned installed version/module bytes without importing microunit. + + An explicit site-packages search path supports a separately pinned local + environment. Paths are operational and never enter the normative identity. + """ + try: + if microunit_search_path is None: + distribution = metadata.distribution("microunit") + else: + _require( + type(microunit_search_path) is tuple + and all(type(p) is str for p in microunit_search_path), + "MICROUNIT_SEARCH_PATH", + ) + matches = [ + d + for d in metadata.distributions(path=microunit_search_path) + if d.metadata.get("Name", "").lower() == "microunit" + ] + _require(len(matches) == 1, "MICROUNIT_DISTRIBUTION_COUNT") + distribution = matches[0] + _require(distribution.version == MICROUNIT_VERSION, "MICROUNIT_VERSION") + relative = "microunit/tax_unit_construction.py" + _require( + distribution.files is not None + and any(str(f) == relative for f in distribution.files), + "MICROUNIT_MODULE_INVENTORY", + ) + path = distribution.locate_file(relative) + _require(path.stat().st_size <= 1024 * 1024, "MICROUNIT_MODULE_SIZE") + module = path.read_bytes() + _require( + len(module) <= 1024 * 1024 and _sha(module) == MICROUNIT_SHA256, + "MICROUNIT_MODULE_SHA256", + ) + except (metadata.PackageNotFoundError, OSError) as exc: + raise MoneyRefusalError("MICROUNIT_UNAVAILABLE") from exc + return packaged_current_money_resources( + inspected_version=distribution.version, inspected_module_sha256=_sha(module) + ) diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/asec_current_money_selection.py b/packages/microcosm-build/src/microcosm/build/us_runtime/asec_current_money_selection.py new file mode 100644 index 000000000..71e4010d7 --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/asec_current_money_selection.py @@ -0,0 +1,549 @@ +"""Typed subset of an authenticated ASEC current-money body. + +The authenticated ``ReadyCurrentMoney`` object never leaves the preparation +process. Its canonical encoded bytes travel as a typed graph artifact, and this +module parses those bytes into a *body* that carries amounts and evidence +without carrying source readiness. Selecting whole households from a body +produces a second typed artifact whose header names its parent and the exact +selected coordinates. + +Neither type subclasses ``RestatedAsecMoney``, so every existing +``type(x) is ReadyCurrentMoney`` check refuses them: a subset can never be +minted as full-source readiness, and full readiness can only originate in the +reviewed source loaders. +""" + +from __future__ import annotations + +import struct +from dataclasses import InitVar, dataclass +from hashlib import sha256 + +import numpy as np + +from microcosm.graph import ArtifactType + +from ._asec_current_money_codec import _CONTENT_DOMAIN +from ._asec_current_money_codec import MAGIC as BODY_MAGIC +from ._asec_current_money_codec import PAYLOAD_MAX_BYTES as BODY_PAYLOAD_MAX_BYTES +from .asec_current_money import ( + FIELDS, + HEADER_MAX_BYTES, + MAX_HOUSEHOLDS, + MAX_PERSONS, + RECIPE, + MoneyField, + MoneyRefusalError, + _digest, + _json, + _parse, + _require, + _sha, +) + +US_ASEC_CURRENT_MONEY_BODY_TYPE = ArtifactType( + "microcosm.us.asec_current_money_body", 1 +) +US_ASEC_PREPARED_RECEIPT_TYPE = ArtifactType("microcosm.us.asec_prepared_receipt", 3) +US_ASEC_SELECTION_TYPE = ArtifactType("microcosm.us.asec_household_selection", 2) +US_ASEC_SELECTED_MONEY_TYPE = ArtifactType( + "microcosm.us.asec_selected_current_money", 1 +) + +SELECTED_MAGIC = b"MCASELM1\x01" +SELECTED_ARTIFACT_KIND = "microcosm.us.asec_selected_current_money.v1" +_ENTITIES = ("person", "household") +_BODY_TOKEN = object() +_SELECTED_TOKEN = object() +_SELECTED_HEADER_KEYS = frozenset( + { + "schema_version", + "artifact_kind", + "release_eligible", + "recipe", + "target_year", + "state", + "semantic", + "source_authentication", + "parent_header_sha256", + "parent_content_sha256", + "prepared_receipt_sha256", + "selection_sha256", + "person_rows", + "household_rows", + "person_identity_sha256", + "household_identity_sha256", + "fields", + "field_entities", + "dtype", + "evidence_dtype", + } +) +# 32 person fields plus one household observation, four evidence axes each. +SELECTED_PAYLOAD_MAX_BYTES = ( + len(SELECTED_MAGIC) + + 4 + + HEADER_MAX_BYTES + + 11 * (32 * MAX_PERSONS + MAX_HOUSEHOLDS) + + 32 +) + + +def _entities(field_entities) -> tuple[str, ...]: + """Validate a producer-declared field/entity roster against the closed fields.""" + _require( + type(field_entities) is tuple and len(field_entities) == len(FIELDS), + "FIELD_ENTITY_ROSTER", + ) + names = [] + for item in field_entities: + _require( + type(item) is tuple + and len(item) == 2 + and type(item[0]) is str + and item[1] in _ENTITIES, + "FIELD_ENTITY_ROSTER", + ) + names.append(item[0]) + _require(tuple(names) == FIELDS, "FIELD_ENTITY_ROSTER") + return tuple(item[1] for item in field_entities) + + +def _decoded_entities(data: dict) -> tuple[str, ...]: + roster = data.get("field_entities") + _require( + type(roster) is list + and all(type(item) is list and len(item) == 2 for item in roster), + "FIELD_ENTITY_ROSTER", + ) + return _entities(tuple(tuple(item) for item in roster)) + + +def _positions(values: np.ndarray, *, rows: int, reason: str) -> np.ndarray: + _require( + type(values) is np.ndarray + and values.dtype == np.dtype("int64") + and values.ndim == 1, + reason, + ) + _require(0 < len(values) <= rows, reason) + _require((values[:-1] < values[1:]).all() if len(values) > 1 else True, reason) + _require(int(values[0]) >= 0 and int(values[-1]) < rows, reason) + return values + + +def _split( + payload: bytes, cursor: int, entities, counts: dict[str, int] +) -> tuple[MoneyField, ...]: + fields = [] + for name, entity in zip(FIELDS, entities, strict=True): + count = counts[entity] + buffers = [] + for width in (8, 1, 1, 1): + buffers.append(payload[cursor : cursor + width * count]) + cursor += width * count + fields.append(MoneyField(name, *buffers)) + return tuple(fields) + + +def _field_shapes(fields, entities, counts: dict[str, int]) -> None: + for field, entity in zip(fields, entities, strict=True): + count = counts[entity] + widths = ( + len(field.amount_bytes), + len(field.status_bytes), + len(field.validity_bytes), + len(field.zero_origin_bytes), + ) + _require(widths == (8 * count, count, count, count), "FIELD_BUFFER_SHAPE") + amounts = field.amounts + _require(np.isfinite(amounts).all(), "FIELD_ENCODING", field.name) + _require( + np.isin(field.statuses, [0, 1, 2, 3, 4]).all() + and np.isin(field.validity, [0, 1]).all() + and np.isin(field.zero_origin, [0, 1, 2]).all(), + "FIELD_ENCODING", + field.name, + ) + _require(field.validity.all(), "MISSING_REQUIRED_AMOUNT", field.name) + + +@dataclass(frozen=True) +class BoundCurrentMoneyBody: + """Amount/evidence buffers bound to a producer-declared body identity. + + This is emphatically not readiness: it carries no spec, no source evidence + and no authenticated authority, only bytes that a named producer already + authenticated inside its own process. + """ + + header: bytes + content_sha256: str + field_entities: tuple[tuple[str, str], ...] + fields: tuple[MoneyField, ...] + person_rows: int + household_rows: int + _token: InitVar[object] = None + + def __post_init__(self, _token): + _require(_token is _BODY_TOKEN, "BODY_CONSTRUCTOR_UNAVAILABLE") + + @property + def header_data(self) -> dict: + return _parse(self.header, HEADER_MAX_BYTES) + + @property + def header_sha256(self) -> str: + return _sha(self.header) + + def field(self, name: str) -> MoneyField: + _require(name in FIELDS, "UNKNOWN_FIELD") + return self.fields[FIELDS.index(name)] + + def entity_of(self, name: str) -> str: + _require(name in FIELDS, "UNKNOWN_FIELD") + return self.field_entities[FIELDS.index(name)][1] + + +@dataclass(frozen=True) +class SelectedCurrentMoney: + """Whole-household subset of a body, bound to its exact selected coordinates.""" + + header: bytes + fields: tuple[MoneyField, ...] + _token: InitVar[object] = None + + def __post_init__(self, _token): + _require(_token is _SELECTED_TOKEN, "SELECTED_CONSTRUCTOR_UNAVAILABLE") + + @property + def header_data(self) -> dict: + return _parse(self.header, HEADER_MAX_BYTES) + + @property + def header_sha256(self) -> str: + return _sha(self.header) + + @property + def person_rows(self) -> int: + return self.header_data["person_rows"] + + @property + def household_rows(self) -> int: + return self.header_data["household_rows"] + + def field(self, name: str) -> MoneyField: + _require(name in FIELDS, "UNKNOWN_FIELD") + return self.fields[FIELDS.index(name)] + + def entity_of(self, name: str) -> str: + _require(name in FIELDS, "UNKNOWN_FIELD") + return self.header_data["field_entities"][FIELDS.index(name)][1] + + def amounts(self, name: str) -> np.ndarray: + """Return a writable copy of one field's selected target-year amounts.""" + return self.field(name).amounts.copy() + + +def parse_current_money_body( + payload: bytes, + *, + expected_header_sha256: str, + expected_content_sha256: str, + field_entities: tuple[tuple[str, str], ...], +) -> BoundCurrentMoneyBody: + """Parse canonical money bytes without reconstructing replayable readiness. + + The caller supplies the producer's header and content digests; both must + match exactly. Digests, not header bytes, are what a producer receipt + carries across a graph edge, so binding on them keeps the consumer honest + without re-transporting the header. This deliberately cannot mint a + ``ReadyCurrentMoney``: the authenticated replay contract stays with + ``decode_current_money``. + """ + _require( + type(payload) is bytes + and len(BODY_MAGIC) + 4 + 32 < len(payload) <= BODY_PAYLOAD_MAX_BYTES, + "PAYLOAD_SIZE", + ) + _require(_digest(expected_header_sha256), "EXPECTED_HEADER_DIGEST") + _require(_digest(expected_content_sha256), "EXPECTED_CONTENT_DIGEST") + entities = _entities(field_entities) + _require(payload.startswith(BODY_MAGIC), "PAYLOAD_VERSION") + size = struct.unpack_from(" SelectedCurrentMoney: + """Slice a body to selected positions and bind the selected coordinates.""" + _require(type(body) is BoundCurrentMoneyBody, "TYPED_BODY_REQUIRED") + _require( + all( + _digest(value) + for value in ( + prepared_receipt_sha256, + selection_sha256, + person_identity_sha256, + household_identity_sha256, + ) + ), + "SELECTION_BINDING_DIGEST", + ) + entities = _entities(body.field_entities) + positions = { + "person": _positions( + person_positions, rows=body.person_rows, reason="PERSON_POSITIONS" + ), + "household": _positions( + household_positions, + rows=body.household_rows, + reason="HOUSEHOLD_POSITIONS", + ), + } + selected = [] + for field, entity in zip(body.fields, entities, strict=True): + index = positions[entity] + selected.append( + MoneyField( + field.name, + field.amounts[index].tobytes(), + field.statuses[index].tobytes(), + field.validity[index].tobytes(), + field.zero_origin[index].tobytes(), + ) + ) + parent = body.header_data + header = _json( + { + "schema_version": 1, + "artifact_kind": SELECTED_ARTIFACT_KIND, + "release_eligible": False, + "recipe": RECIPE, + "target_year": 2024, + "state": "target_current", + "semantic": "annual_current_money", + "source_authentication": parent["source_authentication"], + "parent_header_sha256": body.header_sha256, + "parent_content_sha256": body.content_sha256, + "prepared_receipt_sha256": prepared_receipt_sha256, + "selection_sha256": selection_sha256, + "person_rows": len(positions["person"]), + "household_rows": len(positions["household"]), + "person_identity_sha256": person_identity_sha256, + "household_identity_sha256": household_identity_sha256, + "fields": list(FIELDS), + "field_entities": [list(item) for item in body.field_entities], + "dtype": " None: + data = _parse(selected.header, HEADER_MAX_BYTES) + _require(set(data) == _SELECTED_HEADER_KEYS, "SELECTED_HEADER_SCHEMA") + _require(selected.header == _json(data), "SELECTED_HEADER_SCHEMA") + _require( + data["schema_version"] == 1 + and data["artifact_kind"] == SELECTED_ARTIFACT_KIND + and data["release_eligible"] is False + and data["recipe"] == RECIPE + and data["target_year"] == 2024 + and data["state"] == "target_current" + and data["semantic"] == "annual_current_money" + and data["dtype"] == " tuple[bytes, ...]: + _require(type(selected) is SelectedCurrentMoney, "TYPED_SELECTED_REQUIRED") + _validate_selected(selected) + parts = [ + SELECTED_MAGIC, + struct.pack(" bytes: + """Encode the selected subset with a transport checksum over its bytes.""" + parts = _selected_parts(selected) + checksum = sha256() + for part in parts: + checksum.update(part) + return b"".join((*parts, checksum.digest())) + + +def decode_selected_current_money( + payload: bytes, + *, + expected_parent_header_sha256: str, + expected_parent_content_sha256: str, + expected_prepared_receipt_sha256: str, + expected_selection_sha256: str, +) -> SelectedCurrentMoney: + """Decode a selected subset only against its declared parent and selection.""" + _require( + type(payload) is bytes + and len(SELECTED_MAGIC) + 4 + 32 < len(payload) <= SELECTED_PAYLOAD_MAX_BYTES, + "PAYLOAD_SIZE", + ) + _require( + all( + _digest(value) + for value in ( + expected_parent_header_sha256, + expected_parent_content_sha256, + expected_prepared_receipt_sha256, + expected_selection_sha256, + ) + ), + "SELECTION_BINDING_DIGEST", + ) + _require(payload.startswith(SELECTED_MAGIC), "PAYLOAD_VERSION") + size = struct.unpack_from(" 0 for value in counts.values()), + "SELECTED_ROW_BOUND", + ) + expected_size = start + size + 11 * sum(counts[entity] for entity in entities) + 32 + _require(len(payload) == expected_size, "PAYLOAD_LENGTH") + _require( + sha256(memoryview(payload)[:-32]).digest() == payload[-32:], "PAYLOAD_CHECKSUM" + ) + fields = _split(payload, start + size, entities, counts) + value = SelectedCurrentMoney(header, fields, _token=_SELECTED_TOKEN) + _validate_selected(value) + return value + + +__all__ = [ + "BoundCurrentMoneyBody", + "MoneyRefusalError", + "SELECTED_ARTIFACT_KIND", + "SELECTED_MAGIC", + "SELECTED_PAYLOAD_MAX_BYTES", + "SelectedCurrentMoney", + "US_ASEC_CURRENT_MONEY_BODY_TYPE", + "US_ASEC_PREPARED_RECEIPT_TYPE", + "US_ASEC_SELECTED_MONEY_TYPE", + "US_ASEC_SELECTION_TYPE", + "decode_selected_current_money", + "encode_selected_current_money", + "parse_current_money_body", + "select_current_money", +] diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/asec_current_money_source.py b/packages/microcosm-build/src/microcosm/build/us_runtime/asec_current_money_source.py new file mode 100644 index 000000000..93e846b82 --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/asec_current_money_source.py @@ -0,0 +1,607 @@ +"""Authenticate the reviewed full ASEC v4 plus household-observation attachment. + +No public pin override, file discovery, download, source selection, or unit fit. +Verification is source authority, not evidence that all downstream consumers are +corrected or that a population is eligible for release. +""" + +import hashlib +import os +import tempfile +from collections.abc import Mapping +from dataclasses import InitVar, dataclass +from importlib import metadata, resources +from pathlib import Path + +import numpy as np +import pandas as pd + +from microcosm.build import frame_checkpoint as checkpoint +from microcosm.build.outer_stage_runtime import frame_identity +from microcosm.build.serialization_dtypes import canonicalize_frame_string_dtypes +from microcosm.frame import Frame + +from . import asec_checkpoint, asec_household_observations, operator_boundary +from .asec_current_money import ( + _DTYPES, + _SOURCE_TOKEN, + FIELDS, + ZERO_POLICY, + AsecMoneyScope, + AsecMoneyViews, + AuthenticatedAsecSource, + CurrentMoneySpec, + MoneyRefusalError, + _index_identity, + _json, + _parse, + _require, + _sha, + classify_asec_money, + compile_asec_current_money_spec, + require_complete_current_money, + restate_asec_current_money, +) +from .asec_current_money_resources import load_current_money_resources +from .education_assistance_source import ASEC_EDUCATION_ASSISTANCE_ARCHIVES + +# Exact reviewed bytes from the accepted current-money source contract and the +# completed household-observation receipt. Tests monkeypatch this PRIVATE tuple +# only for invented files; production has no override/constructor for other pins. +_SOURCE_PINS = ( + "e2f2b7495bfcf1448dfb0acb0a17e93f86a6ab8bef8a70ec0a2981983028cbb5", + "f7f086e262d9a1d9ec6fc1a0b7c32e577128578a4fc5b3f14e13fb8711f7ec2d", + ( + (2022, "7ccca976284bb47815d84460cc4f75a0a65d26d7754ab0a0f417de351b3d474e"), + (2023, "cb57817327799f42b741caed5f9be94d04021c2e6809c1ad7bd0686da5428d88"), + (2024, "ec36604cb735a660b51b0b2f90be27d803b5878f3464fb30d0eacead59c1260d"), + ), +) +_LOAD_TOKEN = object() +_ATTACHMENT_COLUMNS = tuple( + "asec_" + c for c in asec_household_observations.ASEC_HOUSEHOLD_OBSERVATION_COLUMNS +) + + +def _stage_verified(path, expected, staging): + """Load only a verified private copy, closing same-path replacement windows.""" + digest = hashlib.sha256() + try: + with Path(path).open("rb") as src, staging.open("xb") as dst: + before = os.fstat(src.fileno()) + while chunk := src.read(1024 * 1024): + digest.update(chunk) + dst.write(chunk) + after = os.fstat(src.fileno()) + _require( + ( + before.st_dev, + before.st_ino, + before.st_size, + before.st_mtime_ns, + before.st_ctime_ns, + ) + == ( + after.st_dev, + after.st_ino, + after.st_size, + after.st_mtime_ns, + after.st_ctime_ns, + ), + "SOURCE_BYTES_CHANGED", + ) + _require(digest.hexdigest() == expected, "SOURCE_BYTES_MISMATCH") + except OSError: + raise MoneyRefusalError("SOURCE_BYTES_UNAVAILABLE") from None + + +def _series_digest(digest, series): + """Use the existing checkpoint's closed scalar vocabulary, one column at a time.""" + spec = checkpoint._series_spec(series, label="authenticated source") + digest.update(_json(spec)) + extension = series.array + data, mask = getattr(extension, "_data", None), getattr(extension, "_mask", None) + if isinstance(data, np.ndarray) and isinstance(mask, np.ndarray): + for array in (data, mask): + value = np.ascontiguousarray(array).tobytes() + digest.update(len(value).to_bytes(8, "little")) + digest.update(value) + elif isinstance(series.dtype, np.dtype) and not series.dtype.hasobject: + value = np.ascontiguousarray(series.to_numpy(copy=False)).tobytes() + digest.update(len(value).to_bytes(8, "little")) + digest.update(value) + else: + # Repeated literal strings are common in full survey source frames. + # Cache their exact length-prefixed bytes within this column only; every + # value is still visited and every later seal rereads the current data. + # Both entry and payload bounds keep distinct or large strings bounded. + string_tokens = {} + token_bytes = 0 + for value in series.to_numpy(dtype=object, copy=False): + if type(value) is str: + token = string_tokens.get(value) + if token is None: + encoded = checkpoint._encode_object_scalar(value) + token = len(encoded).to_bytes(8, "little") + encoded + if len(string_tokens) < 1024 and token_bytes + len(token) <= 262144: + string_tokens[value] = token + token_bytes += len(token) + digest.update(token) + continue + encoded = checkpoint._encode_object_scalar(value) + digest.update(len(encoded).to_bytes(8, "little")) + digest.update(encoded) + + +def _frame_signature(frame): + digest = hashlib.sha256(b"microcosm/asec-current-money-source-frame/1\0") + digest.update(_json(checkpoint._checkpoint_metadata(frame, {}))) + digest.update(_json(_plain_metadata(frame.metadata))) + for _, table in checkpoint._frame_tables(frame): + digest.update( + _json( + { + "columns_dtype": repr(table.columns.dtype), + "columns_type": type(table.columns).__name__, + } + ) + ) + _series_digest( + digest, pd.Series(table.index.to_numpy(copy=False), dtype=table.index.dtype) + ) + for column in table: + _series_digest(digest, table[column]) + _series_digest(digest, frame.strata) + digest.update(_json(frame.strata.name)) + digest.update(_json(checkpoint._index_spec(frame.strata.index, label="strata"))) + _series_digest( + digest, + pd.Series( + frame.strata.index.to_numpy(copy=False), dtype=frame.strata.index.dtype + ), + ) + for entity in frame.weighted_entities: + digest.update(frame.weights_for(entity).values.tobytes()) + return digest.hexdigest() + + +def _owned_index(index): + """Detach axes, including pandas' materialized RangeIndex cache.""" + if isinstance(index, pd.RangeIndex): + # RangeIndex.copy(deep=True) shares its cached ndarray. Reconstruct the + # same logical range so a child cannot write through to a sealed parent. + return pd.RangeIndex(index.start, index.stop, index.step, name=index.name) + return index.copy(deep=True) + + +def _detach_frame_axes(frame): + """Detach axes of an already copied Frame; preserve its data and axis types.""" + for _, table in checkpoint._frame_tables(frame): + table.index = _owned_index(table.index) + table.columns = _owned_index(table.columns) + frame.strata.index = _owned_index(frame.strata.index) + + +def _plain_metadata(value): + if isinstance(value, Mapping): + return [ + "mapping", + [[key, _plain_metadata(item)] for key, item in sorted(value.items())], + ] + if isinstance(value, tuple): + return ["tuple", [_plain_metadata(item) for item in value]] + if isinstance(value, frozenset): + return [ + "frozenset", + sorted((_plain_metadata(item) for item in value), key=_json), + ] + return ["scalar", value] + + +def _verification_identity(): + package = resources.files(__package__) + names = ( + "asec_current_money.py", + "asec_current_money_source.py", + "asec_checkpoint.py", + "asec_household_observations.py", + "education_assistance_source.py", + "reported_coverage_source.py", + "operator_boundary.py", + ) + shared = resources.files("microcosm.build") + frame_package = resources.files("microcosm.frame") + files = {name: _sha(package.joinpath(name).read_bytes()) for name in names} + files.update( + { + "build/" + name: _sha(shared.joinpath(name).read_bytes()) + for name in ( + "frame_checkpoint.py", + "outer_stage_runtime.py", + "serialization_dtypes.py", + ) + } + ) + files.update( + { + "frame/" + path.name: _sha(path.read_bytes()) + for path in frame_package.iterdir() + if path.name.endswith(".py") + } + ) + # This exact live projection is consumed by the existing source-only guard. + projection = { + family: {entity: sorted(columns) for entity, columns in by_entity.items()} + for family, by_entity in operator_boundary.PRE_ASSEMBLY_OPERATOR_OUTPUT_FAMILIES.items() + } + return _sha( + _json( + { + "modules": files, + "operator_owned": projection, + "pins": _SOURCE_PINS, + "dependencies": { + n: metadata.version(n) for n in ("numpy", "pandas", "h5py") + }, + } + ) + ) + + +def _scope(frame): + person = frame.person + keys = asec_household_observations._household_keys(frame) + + def ints(column): + values = person[column] + _require( + values.dtype == np.dtype("int64") and not values.isna().any(), + "SOURCE_COORDINATE_DTYPE", + ) + return tuple(int(v) for v in values.to_numpy()) + + years = ints("source_year") + _require(set(years) == {2022, 2023, 2024}, "SOURCE_YEAR_MAPPING") + return AsecMoneyScope( + ints("person_id"), + tuple(int(v) for v in keys.household_id), + ints("person_household_id"), + ints("person_spm_unit_id"), + years, + tuple(int(v) for v in keys.source_year), + tuple( + value.decode("utf-8") if isinstance(value, bytes) else value + for value in person.PERIDNUM.tolist() + ), + tuple(str(int(v)) for v in keys.H_SEQ), + ) + + +def _views(frame, scope, *, restored=False): + person = frame.person.loc[:, [f for f in FIELDS if f != "HTOTVAL"]].copy() + if restored: + person["PTOTVAL"] = frame.person["asec_PTOTVAL"].to_numpy(copy=True) + return AsecMoneyViews( + person, + frame.table("household").loc[:, ["asec_HTOTVAL"]].copy(), + scope, + ) + + +def _input_binding(views): + scope_sha = _sha( + _json( + { + "coordinates": _parse(views.scope.identity), + "person_index": _parse(_index_identity(views.person.index)), + "household_index": _parse(_index_identity(views.household.index)), + } + ) + ) + inputs = [] + for entity, table in (("person", views.person), ("household", views.household)): + for column, series in table.items(): + field = "HTOTVAL" if entity == "household" else column + _require(str(series.dtype) in _DTYPES, "UNSUPPORTED_DTYPE", field) + values = series.to_numpy(dtype="= 0 + and row["joined_rows"] == int((keys.source_year == year).sum()) + and row["unreferenced_source_rows"] + == row["source_rows"] - row["joined_rows"] + >= 0, + "ATTACHMENT_SOURCE_COVERAGE", + ) + return cohorts + + +def _source_verification_identity(evidence): + from .asec_current_money import RESTORED_SOURCE_KIND + + if evidence["source_kind"] == RESTORED_SOURCE_KIND: + from .asec_person_income_source import _implementation + + return _implementation() + return _verification_identity() + + +@dataclass(frozen=True) +class AuthenticatedCurrentMoneySource: + frame: Frame + scope: AsecMoneyScope + source: AuthenticatedAsecSource + spec: CurrentMoneySpec + _token: InitVar[object] = None + + def __post_init__(self, _token): + _require(_token is _LOAD_TOKEN, "SOURCE_CONSTRUCTOR_UNAVAILABLE") + + def validate(self): + _require( + type(self.source) is AuthenticatedAsecSource, + "AUTHENTICATED_SOURCE_ISSUANCE", + ) + self.source._validate() + evidence = _parse(self.source.identity) + _require( + _source_verification_identity(evidence) == evidence["verification_sha256"], + "SOURCE_IMPLEMENTATION_CHANGED", + ) + try: + _require( + _frame_signature(self.frame) == evidence["frame_sha256"] + and _scope(self.frame) == self.scope, + "SOURCE_CHANGED", + ) + except MoneyRefusalError: + raise + except (ValueError, TypeError, KeyError): + raise MoneyRefusalError("SOURCE_CHANGED") from None + + def views(self): + self.validate() + from .asec_current_money import _restored_source + + return _views(self.frame, self.scope, restored=_restored_source(self.source)) + + def ready(self): + views = self.views() + decoded = classify_asec_money(views, self.spec, self.scope) + return require_complete_current_money( + restate_asec_current_money(decoded, self.spec), self.spec, production=True + ) + + +def load_authenticated_current_money_source(parent_path, attachment_path): + """Issue authority only for the two reviewed full-source files and exact scope.""" + try: + return _load(parent_path, attachment_path) + except MoneyRefusalError: + raise + except (ValueError, TypeError, KeyError, OSError): + raise MoneyRefusalError("SOURCE_CONTRACT_REFUSAL") from None + + +def _load(parent_path, attachment_path): + before = _verification_identity() + with tempfile.TemporaryDirectory(prefix="microcosm-money-source-") as directory: + directory = Path(directory) + parent_copy = directory / "parent.h5" + _stage_verified(parent_path, _SOURCE_PINS[0], parent_copy) + parent, metadata = asec_checkpoint.load_asec_raw_stage_checkpoint_v4( + parent_copy + ) + parent_copy.unlink() + attachment_copy = directory / "attachment.h5" + _stage_verified(attachment_path, _SOURCE_PINS[1], attachment_copy) + loaded = checkpoint.load_frame_checkpoint(attachment_copy) + canonicalize_frame_string_dtypes( + loaded.frame, boundary="ASEC money attachment", in_place=True + ) + cohorts = _validate_attachment(parent, metadata, loaded, _SOURCE_PINS[0]) + sidecars = [] + actual_pins = metadata["raw_source_mappings"]["ED_VAL"]["source_pins"] + _require(len(actual_pins) == 3, "ED_VAL_SOURCE_COVERAGE") + for year in (2022, 2023, 2024): + registered = ASEC_EDUCATION_ASSISTANCE_ARCHIVES[year] + expected = { + "income_year": year, + "locator": registered.zip_url, + "member": registered.member, + "member_sha256": registered.member_sha256, + "sha256": registered.zip_sha256, + } + _require(actual_pins.count(expected) == 1, "ED_VAL_REGISTERED_PIN") + sidecars.append( + { + "income_year": year, + "survey_year": registered.survey_year, + "archive_sha256": registered.zip_sha256, + "member": registered.member, + "member_sha256": registered.member_sha256, + } + ) + frame = loaded.frame + scope = _scope(frame) + views = _views(frame, scope) + scope_sha, input_sha = _input_binding(views) + evidence = { + "schema_version": 1, + "source_kind": "asec_v4_with_household_observations_v1", + "source_authentication": "checkpoint_bytes_verified", + "parent_sha256": _SOURCE_PINS[0], + "attachment_sha256": _SOURCE_PINS[1], + "cohorts": cohorts, + "sidecars": sidecars, + "field_roster": FIELDS, + "zero_origin_policy": ZERO_POLICY, + "scope_sha256": scope_sha, + "input_sha256": input_sha, + "frame_sha256": _frame_signature(frame), + "verification_sha256": before, + } + _require(_verification_identity() == before, "SOURCE_IMPLEMENTATION_CHANGED") + authority = AuthenticatedAsecSource(_json(evidence), _token=_SOURCE_TOKEN) + spec = compile_asec_current_money_spec(load_current_money_resources(), authority) + return AuthenticatedCurrentMoneySource( + frame, scope, authority, spec, _token=_LOAD_TOKEN + ) diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/asec_current_money_units.py b/packages/microcosm-build/src/microcosm/build/us_runtime/asec_current_money_units.py new file mode 100644 index 000000000..3dd275752 --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/asec_current_money_units.py @@ -0,0 +1,411 @@ +"""Actual tax-only reconstruction from verified target-current ASEC money. + +The transient view has a closed read set. All other observed source columns and +US entities remain untouched. This is not a policy impact or release evaluator. +""" + +import inspect +import sys +from dataclasses import InitVar, dataclass +from importlib import metadata, resources +from pathlib import Path + +import numpy as np +import pandas as pd + +from microcosm.build.serialization_dtypes import canonicalize_table_string_dtypes +from microcosm.frame import Frame, Weights +from microcosm.frame import units as frame_units + +from ._asec_current_money_codec import current_money_content_sha256 +from .asec_current_money import ( + MICROUNIT_VERSION, + MoneyRefusalError, + ReadyCurrentMoney, + _json, + _parse, + _require, + _sha, + _validate_money, +) +from .asec_current_money_source import ( + AuthenticatedCurrentMoneySource, + _detach_frame_axes, + _frame_signature, + _same_table, + _source_verification_identity, +) +from .asec_student_controls import ( + ALIASES as STUDENT_ALIASES, +) +from .asec_student_controls import ( + CONTROLS as STUDENT_CONTROLS, +) +from .asec_student_controls import ( + StudentControlsAttachedAsec, +) + +MICROUNIT_CURRENT_MONEY_COLUMNS = tuple( + "WSAL_VAL SEMP_VAL FRSE_VAL INT_VAL DIV_VAL RNT_VAL CAP_VAL UC_VAL OI_VAL ANN_VAL PNSN_VAL SS_VAL PTOTVAL".split() +) +_STRUCTURAL_COLUMNS = ( + "PH_SEQ", + "A_LINENO", + "A_AGE", + "A_MARITL", + "A_SPOUSE", + "PEPAR1", + "PEPAR2", + "A_EXPRRP", +) +_CONTROL_COLUMNS = ( + "A_ENRLW", + "A_FTPT", + "A_HSCOL", + "PEDISDRS", + "PEDISEAR", + "PEDISEYE", + "PEDISOUT", + "PEDISPHY", + "PEDISREM", +) +MICROUNIT_CURRENT_INPUT_COLUMNS = ( + _STRUCTURAL_COLUMNS + _CONTROL_COLUMNS + MICROUNIT_CURRENT_MONEY_COLUMNS +) +TAX_PERSON_OUTPUTS = ( + "person_tax_unit_id", + "tax_unit_role_input", + "is_related_to_head_or_spouse", +) +_RESULT_TOKEN = object() + + +class MicrounitControlRefusalError(MoneyRefusalError): + """Closed control-field diagnostics without changing the money-source schema.""" + + def __init__(self, reason, column): + super().__init__(reason) + self.field = ( + column if column in _STRUCTURAL_COLUMNS + _CONTROL_COLUMNS else "contract" + ) + self.args = (f"{self.field}: {reason}",) + + +def _require_control(condition, reason, column): + if not condition: + raise MicrounitControlRefusalError(reason, column) + + +# Exact 0.1.0 import/runtime closure: __init__ imports core/diagnostics/registry; +# tax construction calls rule_helpers, whose cached threshold reads this YAML. +# No PolicyEngine package is needed for those rules. +_MICROUNIT_EXECUTION_PINS = { + "__init__.py": "6164ebb289659a027d2bdd419781bd1025483bb2baf5576487f8e3f4ac594e01", + "core.py": "95ec837e23e9a61a20bb48bcf20f87ffbfa536aaf0536706d020c6f79dc47701", + "diagnostics.py": "c1a683b98ceac7a44438653eb992240e5d9ed130119f75f69f6bed3a02e6c7c6", + "registry.py": "3ae5442a7bc637f8f786913abae97f63997baba5cedac948a80ce3b00f8cf673", + "rule_helpers.py": "7d32f1b3ab855a21ec0ca2e6d30431c4dcb030ffb250a28ef1978c50ed334243", + "tax_unit_construction.py": "79007065113f9600601433dc2633f4392c675e28b87a7383ab8a1f7c6279d47e", + "data/dependent_gross_income_limit.yaml": "3e20dd8f44d5ff919497fe38c2f68aa97a9f333539fbea7d0ca4f2c29e588da0", +} + + +def _runtime_file_bytes(path): + _require(path.stat().st_size <= 1024 * 1024, "MICROUNIT_EXECUTION_SIZE") + value = path.read_bytes() + _require(len(value) <= 1024 * 1024, "MICROUNIT_EXECUTION_SIZE") + return value + + +def _execution(): + try: + distribution = metadata.distribution("microunit") + _require(distribution.version == MICROUNIT_VERSION, "MICROUNIT_VERSION") + root = Path(distribution.locate_file("microunit")).resolve() + inventory = {str(p) for p in distribution.files or ()} + for name, expected in _MICROUNIT_EXECUTION_PINS.items(): + _require( + "microunit/" + name in inventory + and _sha(_runtime_file_bytes(root / name)) == expected, + "MICROUNIT_EXECUTION_PIN", + ) + import microunit + + _require( + Path(microunit.__file__).resolve() == root / "__init__.py", + "MICROUNIT_IMPORT_ORIGIN", + ) + _require( + inspect.getsourcefile(microunit.construct_tax_units) + == str(root / "tax_unit_construction.py"), + "MICROUNIT_IMPORT_ORIGIN", + ) + for name in _MICROUNIT_EXECUTION_PINS: + if name.endswith(".py"): + module_name = ( + "microunit" if name == "__init__.py" else "microunit." + name[:-3] + ) + module = sys.modules.get(module_name) + _require( + module is not None + and Path(module.__file__).resolve() == root / name, + "MICROUNIT_IMPORT_ORIGIN", + ) + package = resources.files(__package__) + files = { + name: _sha(package.joinpath(name).read_bytes()) + for name in ( + "asec_current_money_units.py", + "asec_current_money_source.py", + "asec_current_money.py", + "_asec_current_money_codec.py", + "asec_current_money_resources.py", + "asec_student_controls.py", + ) + } + files["frame.units"] = _sha(Path(frame_units.__file__).read_bytes()) + identity = { + "version": distribution.version, + "microunit": _MICROUNIT_EXECUTION_PINS, + "adapter_modules": files, + "dependencies": { + n: metadata.version(n) for n in ("numpy", "pandas", "pyyaml") + }, + "year": 2024, + "mode": "policyengine", + } + return microunit.construct_tax_units, _json(identity) + except (metadata.PackageNotFoundError, OSError, ImportError): + raise MoneyRefusalError("MICROUNIT_EXECUTION_UNAVAILABLE") from None + + +def current_money_microunit_view(source, ready): + """Project only reviewed controls and corrected amounts, in actual person order.""" + _require( + type(source) in (AuthenticatedCurrentMoneySource, StudentControlsAttachedAsec) + and type(ready) is ReadyCurrentMoney, + "AUTHENTICATED_MONEY_REQUIRED", + ) + expected = source.ready() + _require( + ready.bindings == expected.bindings and ready.fields == expected.fields, + "MONEY_CONTENT_MISMATCH", + ) + _require(ready.bindings.spec == source.spec, "SPEC_MISMATCH") + _validate_money(ready, source.spec, nominal=False) + _require( + all(field.validity.all() for field in ready.fields), "MISSING_REQUIRED_AMOUNT" + ) + person = source.frame.person + _require( + set(_STRUCTURAL_COLUMNS + _CONTROL_COLUMNS) <= set(person), + "MICROUNIT_INPUT_ROSTER", + ) + view = ( + person.loc[:, _STRUCTURAL_COLUMNS + _CONTROL_COLUMNS] + .copy() + .reset_index(drop=True) + ) + if type(source) is StudentControlsAttachedAsec: + # Only authenticated, code-owned aliases replace the transient logical + # controls. Original source observations, including NaNs, stay untouched. + source.validate() + for name, alias in zip(STUDENT_CONTROLS, STUDENT_ALIASES, strict=True): + view[name] = person[alias].to_numpy(copy=True) + for column in view: + values = view[column] + _require_control( + pd.api.types.is_numeric_dtype(values.dtype) + and not pd.api.types.is_bool_dtype(values.dtype), + "MICROUNIT_CONTROL_DTYPE", + column, + ) + _require_control(not values.isna().any(), "MICROUNIT_CONTROL_MISSING", column) + array = values.to_numpy(dtype=np.float64) + _require_control( + np.isfinite(array).all() and (array == np.floor(array)).all(), + "MICROUNIT_CONTROL_VALUES", + column, + ) + _require( + np.array_equal(view.PH_SEQ.to_numpy(), person.person_household_id.to_numpy()), + "MICROUNIT_HOUSEHOLD_BINDING", + ) + _require( + not view[["PH_SEQ", "A_LINENO"]].duplicated().any() + and (view.A_LINENO > 0).all(), + "MICROUNIT_PERSON_LINES", + ) + for field in MICROUNIT_CURRENT_MONEY_COLUMNS: + view[field] = ready.field(field).amounts.copy() + _require( + tuple(view.columns) == MICROUNIT_CURRENT_INPUT_COLUMNS, "MICROUNIT_INPUT_ROSTER" + ) + return view + + +def _partitions(frame): + grouped = {} + for person, group in zip( + frame.person.person_id, frame.person.person_tax_unit_id, strict=True + ): + grouped.setdefault(int(group), []).append(int(person)) + return tuple(sorted(tuple(sorted(members)) for members in grouped.values())) + + +@dataclass(frozen=True) +class CurrentMoneyTaxUnitResult: + frame: Frame + money: ReadyCurrentMoney + _receipt: bytes + _token: InitVar[object] = None + _student_source: StudentControlsAttachedAsec | None = None + + def __post_init__(self, _token): + _require(_token is _RESULT_TOKEN, "TAX_RESULT_CONSTRUCTOR_UNAVAILABLE") + + @property + def receipt(self): + return _parse(self._receipt) + + def validate(self): + """Check the permitted tax-only reattachment before consuming a live result.""" + receipt = self.receipt + if self._student_source is not None: + _require( + type(self._student_source) is StudentControlsAttachedAsec, + "STUDENT_TAX_PARENT", + ) + self._student_source.validate() + _require( + self.money.bindings == self._student_source.money.bindings + and self.money.fields == self._student_source.money.fields + and receipt["student_controls"] == self._student_source.receipt, + "STUDENT_TAX_PARENT", + ) + else: + _require(receipt["student_controls"] is None, "STUDENT_TAX_PARENT") + _validate_money(self.money, self.money.bindings.spec, nominal=False) + source_evidence = _parse(self.money.bindings.spec.source.identity) + _require( + _source_verification_identity(source_evidence) + == source_evidence["verification_sha256"], + "SOURCE_IMPLEMENTATION_CHANGED", + ) + try: + frame_signature = _frame_signature(self.frame) + except (ValueError, TypeError, KeyError, AssertionError): + raise MoneyRefusalError("TAX_RESULT_CHANGED") from None + _require( + _sha(self.money.header) == receipt["money_header_sha256"] + and current_money_content_sha256(self.money) + == receipt["money_content_sha256"] + and frame_signature == receipt["output_frame_sha256"], + "TAX_RESULT_CHANGED", + ) + _require( + _execution()[1] == _json(receipt["implementation"]), + "MICROUNIT_EXECUTION_CHANGED", + ) + + +def reconstruct_current_money_tax_units(source, ready): + """Run the pinned real engine, graft tax outputs, and verify preservation.""" + try: + return _reconstruct(source, ready) + except MoneyRefusalError: + raise + except (ValueError, TypeError, KeyError, AssertionError): + raise MoneyRefusalError("TAX_RECONSTRUCTION_REFUSAL") from None + + +def _reconstruct(source, ready): + view = current_money_microunit_view(source, ready) + construct, implementation = _execution() + original = source.frame + assignments, tax_unit = frame_units._construct_tax_units( + construct, view, year=2024, mode="policyengine" + ) + tax_unit = canonicalize_table_string_dtypes( + tax_unit, + boundary="ASEC current-money tax reconstruction", + table_name="tax_unit", + ) + _require( + assignments.index.equals(view.index) and len(assignments) == len(view), + "MICROUNIT_OUTPUT_ROWS", + ) + person = original.person.copy(deep=True) + for target, output in zip( + TAX_PERSON_OUTPUTS, + ("TAX_ID", "tax_unit_role_input", "is_related_to_head_or_spouse"), + strict=True, + ): + dtype = person[target].dtype if target in person else assignments[output].dtype + person[target] = pd.array(assignments[output].to_numpy(copy=True), dtype=dtype) + tables = {entity: original.table(entity) for entity in original.entities} + tables["person"] = person + tables["tax_unit"] = tax_unit + result = Frame( + tables, + original.schema, + { + entity: Weights( + original.weights_for(entity).values, original.weights_for(entity).kind + ) + for entity in original.weighted_entities + }, + original.strata, + mass_log=original.mass_log, + metadata=original.metadata, + ) + _detach_frame_axes(result) + source.validate() + _require(_execution()[1] == implementation, "MICROUNIT_EXECUTION_CHANGED") + for entity in original.entities: + if entity not in ("person", "tax_unit"): + _same_table(result.table(entity), original.table(entity)) + untouched = [name for name in original.person if name not in TAX_PERSON_OUTPUTS] + _same_table(result.person[untouched], original.person[untouched]) + pd.testing.assert_series_equal(result.strata, original.strata, check_exact=True) + _require( + result.metadata == original.metadata and result.mass_log == original.mass_log, + "TAX_CONTEXT_PRESERVATION", + ) + for entity in original.weighted_entities: + _require( + result.weights_for(entity).kind == original.weights_for(entity).kind + and result.weights_for(entity).values.tobytes() + == original.weights_for(entity).values.tobytes(), + "TAX_WEIGHT_PRESERVATION", + ) + receipt = { + "schema_version": 1, + "artifact_kind": "microcosm.asec_current_money_tax_units", + "release_eligible": False, + "all_current_money_consumers_wired": False, + "year": 2024, + "mode": "policyengine", + "money_header_sha256": _sha(ready.header), + "money_content_sha256": current_money_content_sha256(ready), + "source_frame_sha256": _parse(source.source.identity)["frame_sha256"], + "student_controls": source.receipt + if type(source) is StudentControlsAttachedAsec + else None, + "output_frame_sha256": _frame_signature(result), + "implementation": _parse(implementation), + "person_outputs": TAX_PERSON_OUTPUTS, + "tax_table_outputs": tuple(tax_unit.columns), + "old_tax_units": original.n("tax_unit"), + "new_tax_units": result.n("tax_unit"), + "old_partition_sha256": _sha(_json(_partitions(original))), + "new_partition_sha256": _sha(_json(_partitions(result))), + "scope_sha256": _parse(ready.header)["scope_sha256"], + } + return CurrentMoneyTaxUnitResult( + result, + ready, + _json(receipt), + _token=_RESULT_TOKEN, + _student_source=source if type(source) is StudentControlsAttachedAsec else None, + ) diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/asec_demographic_source.py b/packages/microcosm-build/src/microcosm/build/us_runtime/asec_demographic_source.py new file mode 100644 index 000000000..0a227ebb3 --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/asec_demographic_source.py @@ -0,0 +1,1519 @@ +"""Explicit ASEC demographic source contract: sex, its allocation flag, and the +household reference-person code, restored from the pinned original CSV cohorts. + +Scope fence. This module declares and implements the contract and its reader. +It is deliberately **not** wired into +:mod:`.asec_prepared_source`: the root adjudication of 2026-09-06 authorised +preparing this source code and tiny fixtures independently and authorised no +additional full-source read and no new binding. Nothing here is executed +against a genuine Census member by this lane. + +What the contract refuses to assume, stated once so a later reader cannot +mistake an omission for an oversight: + +* A declared value range is not measured proof that every delivered token is + inside it. ``A_SEX`` is validated against its own printed codes, and a token + that is not a printed code leaves that person's sex **unbound**. Nothing here + maps "not 2" onto male. +* ``AXSEX`` prints two codes inside a wider declared range. A token in the + range but outside the printed codes is an unresolved allocation provenance, + so it leaves that person's sex unbound rather than being smoothed into + "reported". +* ``A_EXPRRP`` codes 1 and 2 are the printed, self-labelled household + reference person. They are the only admitted head evidence here. +* ``P_SEQ`` labels no code in its own dictionary entry. It is carried and + diagnosed against the reference code, and it is never a fallback: a + household whose reference code is absent, duplicated or indeterminate stays + unbound even when exactly one of its members has ``P_SEQ == 1``. +* ``A_FAMREL`` is a **family** relationship and is not read here. ``A_LINENO`` + is a Basic-CPS roster line number read only as a join crosscheck. Neither + substitutes for the household reference code. + +Every printed fact below was read this session from the three pinned public +dictionaries named in :data:`ASEC_DICTIONARY_AUTHORITY`, at the PDF pages +recorded there, and the four entries are textually identical across the three +years. +""" + +from __future__ import annotations + +import csv +import hashlib +import json +import os +import re +import struct +import tempfile +from collections.abc import Mapping +from dataclasses import InitVar, dataclass +from importlib import metadata, resources +from pathlib import Path +from types import MappingProxyType + +import numpy as np +import pandas as pd + +from microcosm.frame import Frame + +from . import asec_current_money_source as legacy +from . import asec_person_income_source as restoration +from .asec_student_controls import _snapshot +from .education_assistance_source import ASEC_EDUCATION_ASSISTANCE_ARCHIVES + +MAGIC = b"MCASDEMO\x01" +FILENAME = "asec_demographic_source.bin" +ARTIFACT_KIND = "microcosm.asec_demographic_source.v1" +_HEADER_MAX = 262_144 +_MAX_PERSONS = 600_000 +_TOKEN = object() +_COMPOSE_TOKEN = object() + + +class DemographicSourceRefusalError(ValueError): + """Value-free refusal; never carries a row, an identifier or an amount.""" + + def __init__(self, reason: str, field: str = "contract"): + self.reason = reason + self.field = field + super().__init__(f"{field}: {reason}") + + +def _require(condition: bool, reason: str, field: str = "contract") -> None: + if not condition: + raise DemographicSourceRefusalError(reason, field) + + +def _sha(value: bytes) -> str: + return hashlib.sha256(value).hexdigest() + + +def _json(value: object) -> bytes: + return json.dumps( + value, sort_keys=True, separators=(",", ":"), allow_nan=False, ensure_ascii=True + ).encode("ascii") + + +_ASEC_DICTIONARY_URL_TEMPLATE = ( + "https://www2.census.gov/programs-surveys/cps/datasets/" + "{survey_year}/march/asec{survey_year}_ddl_pub_full.pdf" +) +#: Survey year -> the pinned public-use dictionary that prints these entries. +#: The three digests are the same documents the reviewed household +#: reported-income contract already pins; no new pin is introduced here. +ASEC_DICTIONARY_AUTHORITY: Mapping[int, Mapping[str, object]] = MappingProxyType( + { + 2023: MappingProxyType( + { + "url": _ASEC_DICTIONARY_URL_TEMPLATE.format(survey_year=2023), + "sha256": ( + "66bd6e3fe516233ab63b75c60573451222b3b3d3235d61cfed96f64608de2117" + ), + } + ), + 2024: MappingProxyType( + { + "url": _ASEC_DICTIONARY_URL_TEMPLATE.format(survey_year=2024), + "sha256": ( + "761c67ea53f5c3264329b3e9ddbdd802826ba4a859122b8a8a2f29ab85b3a840" + ), + } + ), + 2025: MappingProxyType( + { + "url": _ASEC_DICTIONARY_URL_TEMPLATE.format(survey_year=2025), + "sha256": ( + "5cb80973326ef8b625fbaae70d80b0c641ce5d2b3911abd2fb4427abd5908a6f" + ), + } + ), + } +) + + +@dataclass(frozen=True, eq=False) +class AsecDemographicField: + """One printed dictionary entry, restated exactly, with its own role. + + ``named_codes`` holds only codes the dictionary actually prints. Codes + inside ``printed_range`` that it does not print are exposed as + :attr:`unnamed_in_range_codes` and are never treated as a named value. + """ + + name: str + concept: str + printed_length: int + printed_range: tuple[int, int] + printed_position: int + named_codes: Mapping[int, str] + universe_as_printed: str + role: str + quoted_locator: str + pdf_page_1based_by_survey_year: Mapping[int, int] + + @property + def declared_domain(self) -> tuple[int, ...]: + low, high = self.printed_range + return tuple(range(low, high + 1)) + + @property + def unnamed_in_range_codes(self) -> tuple[int, ...]: + return tuple( + code for code in self.declared_domain if code not in self.named_codes + ) + + @property + def token_pattern(self) -> str: + return rf"[0-9]{{1,{self.printed_length}}}" + + def document(self) -> dict: + return { + "name": self.name, + "concept": self.concept, + "printed_length": self.printed_length, + "printed_range": list(self.printed_range), + "printed_position": self.printed_position, + "named_codes": { + str(code): self.named_codes[code] for code in sorted(self.named_codes) + }, + "unnamed_in_range_codes": list(self.unnamed_in_range_codes), + "universe_as_printed": self.universe_as_printed, + "role": self.role, + "quoted_locator": self.quoted_locator, + "token_pattern": self.token_pattern, + "authority": [ + { + "survey_year": year, + "url": ASEC_DICTIONARY_AUTHORITY[year]["url"], + "sha256": ASEC_DICTIONARY_AUTHORITY[year]["sha256"], + "pdf_page_1based": page, + } + for year, page in sorted(self.pdf_page_1based_by_survey_year.items()) + ], + } + + +A_SEX = AsecDemographicField( + name="A_SEX", + concept="Sex", + printed_length=1, + printed_range=(1, 2), + printed_position=92, + named_codes=MappingProxyType({1: "Male", 2: "Female"}), + universe_as_printed="All Persons", + role="restored_observation", + quoted_locator="A_SEX / Sex / 1 (1:2) 92 / Values: 1 = Male 2 = Female / " + "Universe: All Persons", + pdf_page_1based_by_survey_year=MappingProxyType({2023: 22, 2024: 23, 2025: 24}), +) +AXSEX = AsecDemographicField( + name="AXSEX", + concept="Allocation flag for A_SEX", + printed_length=1, + printed_range=(0, 4), + printed_position=161, + named_codes=MappingProxyType({0: "No change", 4: "Allocated"}), + universe_as_printed="All Persons", + role="restored_observation", + quoted_locator="AXSEX / Allocation flag for A_SEX / 1 (0:4) 161 / " + "Values: 0 = No change 4 = Allocated / Universe: All Persons", + pdf_page_1based_by_survey_year=MappingProxyType({2023: 27, 2024: 28, 2025: 29}), +) +A_EXPRRP = AsecDemographicField( + name="A_EXPRRP", + concept="Expanded relationship code", + printed_length=2, + printed_range=(1, 14), + printed_position=82, + named_codes=MappingProxyType( + { + 1: "Reference person with relatives", + 2: "Reference person without relatives", + 3: "Husband", + 4: "Wife", + 5: "Own child", + 7: "Grandchild", + 8: "Parent", + 9: "Brother/sister", + 10: "Other relative", + 11: "Foster child", + 12: "Nonrelative with relatives", + 13: "Partner/roommate", + 14: "Nonrelative without relatives", + } + ), + universe_as_printed="All Persons", + role="restored_observation", + quoted_locator="A_EXPRRP / Expanded relationship code / 2 (1:14) 82 / " + "Values: 1 = Reference person with relatives 2 = Reference person " + "without relatives 3 = Husband 4 = Wife 5 = Own child 7 = Grandchild " + "8 = Parent 9 = Brother/sister 10 = Other relative 11 = Foster child " + "12 = Nonrelative with relatives 13 = Partner/roommate 14 = Nonrelative " + "without relatives / Universe: All Persons", + pdf_page_1based_by_survey_year=MappingProxyType({2023: 21, 2024: 22, 2025: 23}), +) +P_SEQ = AsecDemographicField( + name="P_SEQ", + concept="Sequence number of person in hhld", + printed_length=2, + printed_range=(0, 16), + printed_position=10, + named_codes=MappingProxyType({}), + universe_as_printed="All Persons", + role="diagnostic_crosscheck", + quoted_locator="P_SEQ / Sequence number of person in hhld / 2 (00:16) 10 / " + "Values: 0-16 / Universe: All Persons", + pdf_page_1based_by_survey_year=MappingProxyType({2023: 20, 2024: 21, 2025: 22}), +) + +#: The four printed entries this contract covers, in output order. +ASEC_DEMOGRAPHIC_FIELDS: tuple[AsecDemographicField, ...] = ( + A_SEX, + AXSEX, + A_EXPRRP, + P_SEQ, +) +OBSERVATIONS: tuple[str, ...] = tuple(field.name for field in ASEC_DEMOGRAPHIC_FIELDS) +ALIASES: tuple[str, ...] = tuple(f"asec_{name}" for name in OBSERVATIONS) +#: The only codes admitted as household reference-person evidence. Both are +#: printed, self-labelled reference-person codes; the split is with/without +#: relatives and is irrelevant to headship itself. +HOUSEHOLD_REFERENCE_CODES: tuple[int, ...] = (1, 2) + +#: Signals deliberately not adopted as semantic proof of headship, each with +#: the reason it is refused. Declared so the refusal survives a later reader. +NON_ADOPTED_HEAD_SIGNALS: tuple[Mapping[str, str], ...] = ( + MappingProxyType( + { + "signal": "P_SEQ == 1", + "kind": "positional_sequence_within_household", + "refusal": ( + "P_SEQ's own dictionary entry labels no code, including 1, as " + "reference person or household head; the ordering statement " + "that would support it is expressly limited to the ASCII file " + "while this restoration reads the CSV member" + ), + "carried_as": "diagnostic_crosscheck", + } + ), + MappingProxyType( + { + "signal": "A_FAMREL == 1", + "kind": "family_relationship", + "refusal": ( + "A_FAMREL is a family relationship scoped to primary-family " + "membership; a subfamily reference person is not its code 1, " + "so it is not a household role and is not read here" + ), + "carried_as": "not_read", + } + ), + MappingProxyType( + { + "signal": "A_LINENO == 1", + "kind": "basic_cps_roster_line_number", + "refusal": ( + "A_LINENO is a separately assigned Basic-CPS roster line " + "number with no source guarantee of row-for-row agreement " + "with any ASEC sequence; it is read only as a join crosscheck" + ), + "carried_as": "join_crosscheck", + } + ), +) + +#: Person-level derived production fields, with their declared code meanings. +SEX_BINDING_STATES: Mapping[int, str] = MappingProxyType( + {0: "unbound", 1: "bound_male", 2: "bound_female"} +) +SEX_UNBOUND_REASONS: Mapping[int, str] = MappingProxyType( + { + 0: "bound", + 1: "sex_code_outside_printed_codes", + 2: "allocation_flag_outside_printed_codes", + } +) +SEX_ALLOCATION_STATES: Mapping[int, str] = MappingProxyType( + {0: "undetermined", 1: "no_change", 2: "census_allocated"} +) +HOUSEHOLD_REFERENCE_STATES: Mapping[int, str] = MappingProxyType( + { + 0: "unbound", + 1: "bound_household_reference_person", + 2: "bound_not_household_reference_person", + } +) +HOUSEHOLD_REFERENCE_UNBOUND_REASONS: Mapping[int, str] = MappingProxyType( + { + 0: "bound", + 1: "relationship_code_outside_printed_codes", + 2: "household_reference_absent", + 3: "household_reference_duplicated", + 4: "household_relationship_code_indeterminate", + } +) +DERIVED: tuple[str, ...] = ( + "asec_sex_binding_state", + "asec_sex_unbound_reason", + "asec_sex_allocation_state", + "asec_household_reference_state", + "asec_household_reference_unbound_reason", +) + +COORDINATES: tuple[str, ...] = ( + "person_id", + "income_year", + "source_household_id", + "A_LINENO", + "A_AGE", +) +#: Every column the artifact carries, in body order. +COLUMNS: tuple[str, ...] = COORDINATES + ALIASES + DERIVED +#: The exact readset: the only columns read from any Census member here. +ASEC_DEMOGRAPHIC_SOURCE_COLUMNS: tuple[str, ...] = ( + "PERIDNUM", + "PH_SEQ", + "A_LINENO", + "A_AGE", + *OBSERVATIONS, +) +_COORDINATE_COLUMNS = ("PH_SEQ", "A_LINENO", "A_AGE") +_MEMBER_PINS = tuple( + (year, p.member, p.zip_sha256, p.member_sha256, p.rows, p.member_size_bytes) + for year, p in sorted(ASEC_EDUCATION_ASSISTANCE_ARCHIVES.items()) +) +#: Income year -> survey year. The observation is the interview household one +#: year after the income year; this module harmonizes no period. +COHORTS: Mapping[int, int] = MappingProxyType( + {year: year + 1 for year, *_rest in _MEMBER_PINS} +) +OBSERVED_PERIOD_KIND = "interview_household_one_year_after_income_year" + + +def _code_document(codes: Mapping[int, str]) -> dict: + """Codes are documented with string keys so a JSON replay compares equal.""" + return {str(code): codes[code] for code in sorted(codes)} + + +def contract_document() -> dict: + """The full declared contract, as it is recorded in every receipt.""" + return { + "schema_version": 1, + "artifact_kind": ARTIFACT_KIND, + "fields": [field.document() for field in ASEC_DEMOGRAPHIC_FIELDS], + "readset": list(ASEC_DEMOGRAPHIC_SOURCE_COLUMNS), + "missing_token_policy": "refuse_source_member_without_filling_or_coercion", + "incumbent_relationship_policy": ( + "A_EXPRRP presence does not attest source observation; compare and " + "diagnose discrepancies without replacing or trusting the incumbent recode" + ), + "production_fields": { + "restored_observations": [ + { + "column": alias, + "source_column": name, + "role": field.role, + "dtype": "int64", + } + for alias, name, field in zip( + ALIASES, OBSERVATIONS, ASEC_DEMOGRAPHIC_FIELDS, strict=True + ) + ], + "derived": [ + { + "column": "asec_sex_binding_state", + "dtype": "int64", + "codes": _code_document(SEX_BINDING_STATES), + }, + { + "column": "asec_sex_unbound_reason", + "dtype": "int64", + "codes": _code_document(SEX_UNBOUND_REASONS), + }, + { + "column": "asec_sex_allocation_state", + "dtype": "int64", + "codes": _code_document(SEX_ALLOCATION_STATES), + }, + { + "column": "asec_household_reference_state", + "dtype": "int64", + "codes": _code_document(HOUSEHOLD_REFERENCE_STATES), + }, + { + "column": "asec_household_reference_unbound_reason", + "dtype": "int64", + "codes": _code_document(HOUSEHOLD_REFERENCE_UNBOUND_REASONS), + }, + ], + }, + "household_reference_codes": list(HOUSEHOLD_REFERENCE_CODES), + "non_adopted_head_signals": [dict(item) for item in NON_ADOPTED_HEAD_SIGNALS], + "household_and_family_roles_separate": True, + "acs_diagnostic_reference_contract": { + "source_column": "RELSHIPP", + "declared_range": [20, 38], + "housing_unit_reference_code": 20, + "group_quarters_codes": { + "37": "institutionalized", + "38": "noninstitutionalized", + }, + "authority": { + "url": "https://www2.census.gov/programs-surveys/acs/tech_docs/pums/data_dict/PUMS_Data_Dictionary_2024.pdf", + "sha256": "929c2752995b0af1c16d5c64de8cdc43b4aa7d388ee2d45b4b4df90fecce1dff", + "pdf_page_1based": 43, + }, + "production_acs_values_changed": False, + }, + "family_relationship_field_read": False, + "line_number_used_for_headship": False, + "sex_binding_requires": [ + "A_SEX in printed codes", + "AXSEX in printed codes", + ], + "household_reference_binding_requires": [ + "every A_EXPRRP in the household inside printed codes", + "exactly one household member carrying code 1 or 2", + ], + "household_membership_verification": { + "grouping_column": HOUSEHOLD_GROUPING_COLUMN, + "published_household_key": list(HOUSEHOLD_MEMBERSHIP_KEY), + "equivalence": "bidirectional_partition_bijection", + "verified_before_classification": True, + "merged_published_households": "refused", + "split_published_households": "refused", + "native_household_id_reuse_across_cohorts": "admitted_and_counted", + }, + "other_than_two_is_male": False, + "cohorts": { + str(year): { + "income_year": year, + "survey_year": survey, + "observed_period_kind": OBSERVED_PERIOD_KIND, + "aged_from_a_date": False, + } + for year, survey in sorted(COHORTS.items()) + }, + "release_eligible": False, + "wired_into_prepared_source": False, + } + + +def _implementation() -> dict: + package = resources.files(__package__) + return { + "schema": 1, + "modules": { + name: _sha(package.joinpath(name).read_bytes()) + for name in ( + "asec_demographic_source.py", + "asec_person_income_source.py", + "asec_student_controls.py", + ) + }, + "legacy_verification": legacy._verification_identity(), + "dependencies": {n: metadata.version(n) for n in ("numpy", "pandas")}, + "member_pins": _MEMBER_PINS, + "contract": contract_document(), + } + + +def read_demographic_member(path, *, rows: int, size: int) -> pd.DataFrame: + """Parse one member's readset as exact tokens; never fill, never coerce. + + Lexical admission comes from the dictionary's own printed field length: + a token must be digits only and no longer than the printed width, so a + two-character ``A_SEX`` or a signed token is refused here rather than + reaching the domain classification as a plausible code. + """ + path = Path(path) + _require(path.stat().st_size == size, "MEMBER_SIZE") + with path.open("r", encoding="utf-8", newline="") as handle: + header = next(csv.reader(handle), []) + _require( + len(header) == len(set(header)) + and set(ASEC_DEMOGRAPHIC_SOURCE_COLUMNS) <= set(header), + "MEMBER_HEADER", + ) + table = pd.read_csv( + path, + usecols=list(ASEC_DEMOGRAPHIC_SOURCE_COLUMNS), + dtype="string", + na_filter=False, + ) + _require(len(table) == rows, "MEMBER_ROWS") + _require( + bool(table.PERIDNUM.str.fullmatch(r"[0-9]{22}").all()), "MEMBER_PERSON_KEY" + ) + for name in _COORDINATE_COLUMNS: + _require( + bool(table[name].str.fullmatch(r"[0-9]+").all()), + "MEMBER_INTEGER", + name, + ) + table[name] = table[name].astype("int64") + for field in ASEC_DEMOGRAPHIC_FIELDS: + _require( + bool(table[field.name].str.fullmatch(field.token_pattern).all()), + "MEMBER_TOKEN_WIDTH", + field.name, + ) + table[field.name] = table[field.name].astype("int64") + _require( + not table.PERIDNUM.duplicated().any() + and not table.duplicated(["PH_SEQ", "A_LINENO"]).any() + and bool((table.PH_SEQ > 0).all()) + and bool((table.A_LINENO > 0).all()), + "MEMBER_COORDINATES", + ) + return table + + +def _counts(codes: np.ndarray, meanings: Mapping[int, str]) -> dict: + return {meanings[code]: int((codes == code).sum()) for code in sorted(meanings)} + + +#: The published household key. ``A_EXPRRP`` names a person's role inside the +#: household the Census published, and that household is identified by its +#: native ``PH_SEQ`` **within one income year**: an unrelated household in +#: another year legitimately carries the same native id. +HOUSEHOLD_MEMBERSHIP_KEY: tuple[str, ...] = ("income_year", "source_household_id") +#: The parent roster column headship is grouped by. It is a remapped id, not +#: the published one, so its partition is proven equal to the key above rather +#: than assumed to be. +HOUSEHOLD_GROUPING_COLUMN = "person_household_id" +#: Counters below that any nonzero value would have refused before an artifact +#: exists. Named so a later reader cannot read their zero as a measurement. +MEMBERSHIP_REFUSAL_COUNTERS: tuple[str, ...] = ( + "grouping_households_spanning_multiple_cohorts", + "grouping_households_spanning_multiple_published_households", + "published_households_spanning_multiple_grouping_households", +) +ZERO_COUNTER_SEMANTICS = ( + "derived from the delivered values, but any nonzero value refuses the run " + "before an artifact exists, so a zero in a successful receipt attests that " + "refusal and is not an independent measurement of the source" +) + + +def _dense_codes(*keys: np.ndarray) -> tuple[np.ndarray, int]: + """Number the distinct key tuples 0..n-1 without combining them numerically. + + Key columns are compared component-wise after a lexicographic sort, so no + two coordinates are ever multiplied or concatenated into one integer and + no width assumption can silently collide two distinct households. + """ + order = np.lexsort(tuple(reversed(keys))) + changed = np.zeros(len(order), dtype=bool) + changed[0] = True + for values in keys: + sorted_values = values[order] + changed[1:] |= sorted_values[1:] != sorted_values[:-1] + codes = np.empty(len(order), dtype=np.int64) + codes[order] = np.cumsum(changed) - 1 + return codes, int(changed.sum()) + + +def _representatives(codes: np.ndarray, count: int) -> np.ndarray: + """One arbitrary but deterministic row position per distinct code.""" + positions = np.zeros(count, dtype=np.int64) + positions[codes] = np.arange(len(codes), dtype=np.int64) + return positions + + +def verify_household_membership( + *, + income_year: np.ndarray, + source_household_id: np.ndarray, + household_id: np.ndarray, +) -> dict: + """Prove the grouping column partitions persons exactly as the publisher did. + + Household reference status is a statement about the household the Census + published: the cardinality of ``A_EXPRRP`` 1/2 is evidence only inside + ``(income_year, source_household_id)``. Classification groups instead by + the parent roster's remapped :data:`HOUSEHOLD_GROUPING_COLUMN`, and a + per-row equality between that roster's ``source_household_id`` and the + member's ``PH_SEQ`` says nothing about how either column *groups*. So the + two partitions are proven identical in both directions here, before any + household verdict is formed. + + Refused in both directions: a grouping household carrying persons from + more than one published household (a merge, whether inside one cohort or + across cohorts), and a published household split across more than one + grouping household. Admitted and counted: the same native household id + reused by unrelated households in different income years, which is + ordinary and is what makes the income year part of the key. + + Returns the aggregate verification recorded in the artifact receipt. It + carries counts, column names and cohort years only; no person, household + or native identifier appears in it or in any refusal raised here. + """ + rows = len(income_year) + _require(0 < rows <= _MAX_PERSONS, "MEMBERSHIP_ROWS") + arrays = { + "income_year": income_year, + "source_household_id": source_household_id, + HOUSEHOLD_GROUPING_COLUMN: household_id, + } + for name, values in arrays.items(): + _require( + isinstance(values, np.ndarray) + and values.dtype == np.dtype("int64") + and values.shape == (rows,), + "MEMBERSHIP_INPUT", + name, + ) + _require( + bool(np.isin(income_year, list(COHORTS)).all()), + "MEMBERSHIP_COHORT", + "income_year", + ) + for name in ("source_household_id", HOUSEHOLD_GROUPING_COLUMN): + _require(bool((arrays[name] > 0).all()), "MEMBERSHIP_COORDINATE", name) + + published, published_count = _dense_codes(income_year, source_household_id) + grouping, grouping_count = _dense_codes(household_id) + edges, edge_count = _dense_codes(grouping, published) + # The relation between the two partitions is exactly its distinct + # (grouping household, published household) pairs; take one row per pair. + edge_rows = _representatives(edges, edge_count) + edge_grouping = grouping[edge_rows] + edge_published = published[edge_rows] + edge_year = income_year[edge_rows] + + _, grouping_degree = np.unique(edge_grouping, return_counts=True) + _, published_degree = np.unique(edge_published, return_counts=True) + cohort_edges, cohort_edge_count = _dense_codes(edge_grouping, edge_year) + _, cohort_degree = np.unique( + edge_grouping[_representatives(cohort_edges, cohort_edge_count)], + return_counts=True, + ) + native_rows = _representatives(published, published_count) + _, native_degree = np.unique(source_household_id[native_rows], return_counts=True) + + collisions = int((cohort_degree > 1).sum()) + merged = int((grouping_degree > 1).sum()) + split = int((published_degree > 1).sum()) + _require(collisions == 0, "MEMBERSHIP_COHORT_COLLISION", HOUSEHOLD_GROUPING_COLUMN) + _require( + merged == 0, "MEMBERSHIP_MERGED_NATIVE_HOUSEHOLDS", HOUSEHOLD_GROUPING_COLUMN + ) + _require( + split == 0, "MEMBERSHIP_SPLIT_NATIVE_HOUSEHOLDS", HOUSEHOLD_GROUPING_COLUMN + ) + # A relation whose edge count equals both vertex counts, with every vertex + # incident to at least one edge by construction, is a bijection. Asserted + # independently of the three counters above so a gap in any one of them + # still cannot let an unequal partition through. + _require( + edge_count == grouping_count == published_count, + "MEMBERSHIP_NOT_BIJECTIVE", + HOUSEHOLD_GROUPING_COLUMN, + ) + + return { + "schema_version": 1, + "grouping_column": HOUSEHOLD_GROUPING_COLUMN, + "published_household_key": list(HOUSEHOLD_MEMBERSHIP_KEY), + "equivalence": "bidirectional_partition_bijection", + "verified_before_classification": True, + "rows": rows, + "grouping_households": grouping_count, + "published_households": published_count, + "membership_edges": edge_count, + # Recomputable from the three counts recorded immediately above it. + "bijective": edge_count == grouping_count == published_count, + "grouping_households_spanning_multiple_cohorts": collisions, + "grouping_households_spanning_multiple_published_households": merged, + "published_households_spanning_multiple_grouping_households": split, + "native_household_ids_reused_across_cohorts": int((native_degree > 1).sum()), + "native_household_id_reuse_across_cohorts_is_admitted": True, + "by_cohort": { + str(year): { + "income_year": int(year), + "survey_year": int(COHORTS[int(year)]), + "rows": int((income_year == year).sum()), + "grouping_households": int( + np.unique(household_id[income_year == year]).size + ), + "published_households": int( + np.unique(published[income_year == year]).size + ), + "native_household_ids": int( + np.unique(source_household_id[income_year == year]).size + ), + } + for year in sorted({int(value) for value in np.unique(income_year)}) + }, + "counters_zero_in_every_successful_receipt": list(MEMBERSHIP_REFUSAL_COUNTERS), + "zero_counter_semantics": ZERO_COUNTER_SEMANTICS, + } + + +@dataclass(frozen=True) +class DemographicClassification: + """Per-person states plus value-free counts. Holds no source identifier.""" + + states: Mapping[str, np.ndarray] + diagnostics: dict + + +def classify_asec_demographic_observations( + *, + income_year: np.ndarray, + household_id: np.ndarray, + a_sex: np.ndarray, + axsex: np.ndarray, + a_exprrp: np.ndarray, + p_seq: np.ndarray, +) -> DemographicClassification: + """Bind only what the printed codes and the household cardinality support. + + Sex binds when the sex code is printed and its allocation flag is printed; + the allocation state is carried alongside, never folded into the value. + Household headship binds when every relationship code in the household is + printed and exactly one member carries a reference-person code. Every other + person is reported unbound with the reason, and ``P_SEQ`` is diagnosed + against the result without ever supplying it. + + ``household_id`` is the grouping column, and the reference-person + cardinality this reads is only evidence about the household the Census + published. The caller must therefore have proven that grouping equal to + ``(income_year, source_household_id)`` with + :func:`verify_household_membership` first; the cohort-uniformity check + below is a weaker self-check and is not that proof. + """ + arrays = { + "income_year": income_year, + "household_id": household_id, + "A_SEX": a_sex, + "AXSEX": axsex, + "A_EXPRRP": a_exprrp, + "P_SEQ": p_seq, + } + rows = len(income_year) + _require(0 < rows <= _MAX_PERSONS, "CLASSIFY_ROWS") + for name, values in arrays.items(): + _require( + isinstance(values, np.ndarray) + and values.dtype == np.dtype("int64") + and values.shape == (rows,), + "CLASSIFY_INPUT", + name, + ) + _require( + bool(np.isin(income_year, list(COHORTS)).all()) + and bool((household_id > 0).all()), + "CLASSIFY_COORDINATES", + ) + + sex_state = np.zeros(rows, dtype=np.int64) + sex_reason = np.zeros(rows, dtype=np.int64) + allocation = np.zeros(rows, dtype=np.int64) + named_sex = np.isin(a_sex, list(A_SEX.named_codes)) + named_flag = np.isin(axsex, list(AXSEX.named_codes)) + allocation[named_flag & (axsex == 0)] = 1 + allocation[named_flag & (axsex == 4)] = 2 + admissible = named_sex & named_flag + sex_state[admissible & (a_sex == 1)] = 1 + sex_state[admissible & (a_sex == 2)] = 2 + sex_reason[~named_sex] = 1 + sex_reason[named_sex & ~named_flag] = 2 + + named_relationship = np.isin(a_exprrp, list(A_EXPRRP.named_codes)) + reference = np.isin(a_exprrp, list(HOUSEHOLD_REFERENCE_CODES)) + head_state = np.zeros(rows, dtype=np.int64) + head_reason = np.zeros(rows, dtype=np.int64) + order = np.argsort(household_id, kind="stable") + grouped = household_id[order] + starts = np.flatnonzero(np.concatenate(([True], grouped[1:] != grouped[:-1]))) + bounds = np.append(starts, len(grouped)) + households = len(starts) + household_verdicts = np.zeros(households, dtype=np.int64) + for index in range(households): + positions = order[bounds[index] : bounds[index + 1]] + _require( + len(np.unique(income_year[positions])) == 1, "CLASSIFY_HOUSEHOLD_COHORT" + ) + unnamed = ~named_relationship[positions] + if unnamed.any(): + head_reason[positions] = 4 + head_reason[positions[unnamed]] = 1 + household_verdicts[index] = 4 + continue + references = positions[reference[positions]] + if len(references) == 0: + head_reason[positions] = 2 + household_verdicts[index] = 2 + continue + if len(references) > 1: + head_reason[positions] = 3 + household_verdicts[index] = 3 + continue + head_state[positions] = 2 + head_state[references[0]] = 1 + household_verdicts[index] = 1 + + # P_SEQ is diagnosed, never consulted. Nothing above reads it. + p_seq_one = p_seq == 1 + in_declared_range = np.isin(p_seq, list(P_SEQ.declared_domain)) + household_years = income_year[order[bounds[:-1]]] + household_p_seq_ones = np.add.reduceat(p_seq_one[order].astype(np.int64), starts) + + diagnostics: dict = { + "rows": rows, + "households": households, + "by_cohort": {}, + "totals": {}, + "p_seq_crosscheck": {}, + } + cohorts = sorted({int(year) for year in np.unique(income_year)}) + for scope, mask, household_mask in [ + ("totals", np.ones(rows, dtype=bool), np.ones(households, dtype=bool)), + *[ + ( + str(year), + income_year == year, + household_years == year, + ) + for year in cohorts + ], + ]: + block = { + "income_year": None if scope == "totals" else int(scope), + "survey_year": None + if scope == "totals" + else int(COHORTS.get(int(scope), int(scope) + 1)), + "observed_period_kind": OBSERVED_PERIOD_KIND, + "rows": int(mask.sum()), + "households": int(household_mask.sum()), + "field_domains": { + field.name: { + "rows_in_declared_range": int( + np.isin(arrays[field.name][mask], field.declared_domain).sum() + ), + "rows_outside_declared_range": int( + ( + ~np.isin(arrays[field.name][mask], field.declared_domain) + ).sum() + ), + "rows_in_named_codes": int( + np.isin(arrays[field.name][mask], list(field.named_codes)).sum() + ), + "rows_in_unnamed_range_codes": int( + np.isin( + arrays[field.name][mask], field.unnamed_in_range_codes + ).sum() + ), + "role": field.role, + } + for field in ASEC_DEMOGRAPHIC_FIELDS + }, + "sex_binding_state": _counts(sex_state[mask], SEX_BINDING_STATES), + "sex_unbound_reason": _counts(sex_reason[mask], SEX_UNBOUND_REASONS), + "sex_allocation_state": _counts(allocation[mask], SEX_ALLOCATION_STATES), + "household_reference_state": _counts( + head_state[mask], HOUSEHOLD_REFERENCE_STATES + ), + "household_reference_unbound_reason": _counts( + head_reason[mask], HOUSEHOLD_REFERENCE_UNBOUND_REASONS + ), + "household_reference_verdict": _counts( + household_verdicts[household_mask], + MappingProxyType( + { + 1: "exactly_one_reference", + 2: "no_reference", + 3: "multiple_references", + 4: "indeterminate_relationship_code", + } + ), + ), + "p_seq_crosscheck": { + "declared_role": P_SEQ.concept, + "named_codes": {}, + "is_semantic_proof_of_headship": False, + "used_as_fallback": False, + "rows_with_p_seq_one": int(p_seq_one[mask].sum()), + "rows_outside_declared_range": int((~in_declared_range[mask]).sum()), + "households_with_one_p_seq_one": int( + (household_p_seq_ones[household_mask] == 1).sum() + ), + "households_with_no_p_seq_one": int( + (household_p_seq_ones[household_mask] == 0).sum() + ), + "households_with_multiple_p_seq_one": int( + (household_p_seq_ones[household_mask] > 1).sum() + ), + "agreement_rows": int((p_seq_one & reference)[mask].sum()), + "p_seq_one_without_reference_code": int( + (p_seq_one & ~reference)[mask].sum() + ), + "reference_code_without_p_seq_one": int( + (reference & ~p_seq_one)[mask].sum() + ), + "unbound_households_with_one_p_seq_one": int( + ((household_verdicts != 1) & (household_p_seq_ones == 1))[ + household_mask + ].sum() + ), + }, + } + if scope == "totals": + diagnostics["totals"] = block + diagnostics["p_seq_crosscheck"] = block["p_seq_crosscheck"] + else: + diagnostics["by_cohort"][scope] = block + diagnostics["family_relationship_field_read"] = False + diagnostics["line_number_used_for_headship"] = False + diagnostics["household_and_family_roles_separate"] = True + return DemographicClassification( + MappingProxyType( + { + "asec_sex_binding_state": sex_state, + "asec_sex_unbound_reason": sex_reason, + "asec_sex_allocation_state": allocation, + "asec_household_reference_state": head_state, + "asec_household_reference_unbound_reason": head_reason, + } + ), + diagnostics, + ) + + +#: The ACS side is *not* rewritten by this contract. These are the pinned 2024 +#: 1-year PUMS codes the existing reader already branches on, restated so the +#: shared classifier below can read the ACS arm without inventing a head. +ACS_OBSERVED_REFERENCE_CODE = 20 +ACS_GROUP_QUARTERS_CODES: tuple[int, int] = (37, 38) +#: What ``acs_pums`` already writes for a group-quarters-only household. It is +#: a printed A_EXPRRP nonrelative code, deliberately not a reference code, so a +#: synthetic group-quarters representative can never be read as an observed +#: housing-unit reference person. +ACS_SYNTHETIC_GROUP_QUARTERS_RELATIONSHIP_CODE = 14 +ACS_REFERENCE_STATES: Mapping[int, str] = MappingProxyType( + { + 0: "unbound", + 1: "observed_housing_unit_reference_person", + 2: "observed_not_reference_person", + 3: "group_quarters_no_observed_reference_person", + } +) + + +def acs_household_reference_states( + *, household_id: np.ndarray, relshipp: np.ndarray +) -> tuple[np.ndarray, dict]: + """Read the ACS arm's own observed reference person; never manufacture one. + + ``RELSHIPP == 20`` is the self-labelled reference person of a housing unit. + Codes 37 and 38 are the institutional and noninstitutional group-quarters + populations; they are neither a reference person nor a relationship to one, + so their households report *no* observed reference person and stay + distinguishable from a housing unit throughout. + """ + rows = len(household_id) + _require(0 < rows <= _MAX_PERSONS, "ACS_ROWS") + for name, values in (("household_id", household_id), ("RELSHIPP", relshipp)): + _require( + isinstance(values, np.ndarray) + and values.dtype == np.dtype("int64") + and values.shape == (rows,), + "ACS_INPUT", + name, + ) + group_quarters = np.isin(relshipp, list(ACS_GROUP_QUARTERS_CODES)) + valid_relationship = np.isin(relshipp, np.arange(20, 39)) + reference = relshipp == ACS_OBSERVED_REFERENCE_CODE + states = np.where(reference, 1, 2).astype(np.int64) + order = np.argsort(household_id, kind="stable") + grouped = household_id[order] + starts = np.flatnonzero(np.concatenate(([True], grouped[1:] != grouped[:-1]))) + bounds = np.append(starts, len(grouped)) + housing_units = 0 + gq_households = 0 + unbound_housing_units = 0 + bad_reference_cardinality = 0 + mixed_households = 0 + invalid_households = 0 + for index in range(len(starts)): + positions = order[bounds[index] : bounds[index + 1]] + if group_quarters[positions].all(): + states[positions] = 3 + gq_households += 1 + continue + housing_units += 1 + bad_reference_cardinality += int(int(reference[positions].sum()) != 1) + mixed_households += int(group_quarters[positions].any()) + invalid_households += int(not valid_relationship[positions].all()) + if ( + group_quarters[positions].any() + or not valid_relationship[positions].all() + or int(reference[positions].sum()) != 1 + ): + states[positions] = 0 + unbound_housing_units += 1 + diagnostics = { + "rows": rows, + "households": len(starts), + "housing_unit_households": housing_units, + "group_quarters_only_households": gq_households, + "unbound_housing_unit_households": unbound_housing_units, + "housing_unit_households_without_exactly_one_reference": bad_reference_cardinality, + "mixed_housing_unit_group_quarters_households": mixed_households, + "households_with_invalid_relationship_codes": invalid_households, + "state_counts": _counts(states, ACS_REFERENCE_STATES), + "group_quarters_rows": int(group_quarters.sum()), + "institutional_group_quarters_rows": int((relshipp == 37).sum()), + "noninstitutional_group_quarters_rows": int((relshipp == 38).sum()), + "manufactured_group_quarters_reference_person": False, + "synthetic_group_quarters_relationship_code": ( + ACS_SYNTHETIC_GROUP_QUARTERS_RELATIONSHIP_CODE + ), + } + return states, diagnostics + + +@dataclass(frozen=True) +class AuthenticatedAsecDemographicSource: + """Immutable owned numeric buffers issued only by closed reconstruction.""" + + _header: bytes + _body: bytes + _token: InitVar[object] = None + + def __post_init__(self, _token): + _require(_token is _TOKEN, "DEMOGRAPHIC_CONSTRUCTOR_UNAVAILABLE") + self.validate() + + @property + def receipt(self) -> dict: + return json.loads(self._header.decode("ascii")) + + @property + def content_sha256(self) -> str: + return _sha(self._header + self._body) + + def validate(self) -> None: + _require( + type(self._header) is bytes and 0 < len(self._header) <= _HEADER_MAX, + "DEMOGRAPHIC_HEADER_SIZE", + ) + header = self.receipt + rows = header["rows"] + _require(type(rows) is int and 0 < rows <= _MAX_PERSONS, "DEMOGRAPHIC_ROWS") + _require( + type(self._body) is bytes + and len(self._body) == rows * len(COLUMNS) * 8 + and _sha(self._body) == header["body_sha256"] + and header["columns"] == list(COLUMNS) + and _json(header["contract"]) == _json(contract_document()) + and _json(header["implementation"]) == _json(_implementation()), + "DEMOGRAPHIC_CONTENT_CHANGED", + ) + + def array(self, name: str) -> np.ndarray: + self.validate() + _require(name in COLUMNS, "DEMOGRAPHIC_COLUMN") + rows = self.receipt["rows"] + offset = COLUMNS.index(name) * rows * 8 + return np.frombuffer(self._body, dtype=" bytes: + value.validate() + payload = ( + MAGIC + struct.pack(" None: + _require( + type(source) is legacy.AuthenticatedCurrentMoneySource, + "DEMOGRAPHIC_AUTHENTICATED_PARENT", + ) + source.validate() + + +def _reconstruct(source, member_paths: Mapping[int, str | Path]): + restoration._paths(member_paths) + _parent(source) + before = _implementation() + person = source.frame.person + rows = len(person) + _require(0 < rows <= _MAX_PERSONS, "DEMOGRAPHIC_ROWS") + _require(not set(ALIASES) & set(person), "DEMOGRAPHIC_ALREADY_ATTACHED") + _require( + set(COORDINATES[2:]) <= set(person) and "person_household_id" in person, + "DEMOGRAPHIC_PARENT_ROSTER", + ) + years = np.asarray(source.scope.person_years, dtype=np.int64) + keys = np.asarray(source.scope.person_native_keys) + output: dict[str, np.ndarray] = { + "person_id": np.asarray(source.scope.person_ids, dtype=np.int64), + "income_year": years, + } + for name in COORDINATES[2:]: + _require( + person[name].dtype == np.dtype("int64"), + "DEMOGRAPHIC_COORDINATE_DTYPE", + name, + ) + output[name] = person[name].to_numpy(copy=True) + # The grouping column is a coordinate of the headship verdict, so it is + # typed here rather than coerced by the conversion that consumes it. + _require( + person[HOUSEHOLD_GROUPING_COLUMN].dtype == np.dtype("int64"), + "DEMOGRAPHIC_COORDINATE_DTYPE", + HOUSEHOLD_GROUPING_COLUMN, + ) + household_id = person[HOUSEHOLD_GROUPING_COLUMN].to_numpy(copy=True) + for alias in ALIASES: + output[alias] = np.empty(rows, dtype=np.int64) + joins = [] + with tempfile.TemporaryDirectory( + prefix="microcosm-demographic-members-" + ) as directory: + for year, member, archive_pin, pin, member_rows, size in _MEMBER_PINS: + staged = Path(directory) / member + _require( + _snapshot(member_paths[year], staged, size=size) == pin, + "DEMOGRAPHIC_SOURCE_BYTES", + ) + raw = read_demographic_member(staged, rows=member_rows, size=size) + staged.unlink() + positions = np.flatnonzero(years == year) + _require(len(positions) == member_rows, "DEMOGRAPHIC_COHORT_ROWS") + recipient = pd.Index(keys[positions]) + _require(not recipient.has_duplicates, "DEMOGRAPHIC_PERSON_KEY") + indices = pd.Index(raw.PERIDNUM).get_indexer(recipient) + _require( + bool((indices >= 0).all()) and len(np.unique(indices)) == member_rows, + "DEMOGRAPHIC_KEY_COVERAGE", + ) + joined = raw.iloc[indices] + coordinate_conflicts = 0 + for name, raw_name in ( + ("source_household_id", "PH_SEQ"), + ("A_LINENO", "A_LINENO"), + ("A_AGE", "A_AGE"), + ): + conflicts = int( + (output[name][positions] != joined[raw_name].to_numpy()).sum() + ) + coordinate_conflicts += conflicts + _require(conflicts == 0, "DEMOGRAPHIC_NATIVE_KEY_OR_AGE", name) + compared = {} + incumbent_conflicts = 0 + relationship_crosscheck = { + "incumbent_present": "A_EXPRRP" in person, + "compared_rows": 0, + "missing_rows": 0, + "mismatch_rows": 0, + "presence_certifies_source_observation": False, + "used_as_semantic_proof": False, + } + for alias, name in zip(ALIASES, OBSERVATIONS, strict=True): + observed = joined[name].to_numpy(dtype=np.int64) + output[alias][positions] = observed + if name not in person: + compared[name] = None + continue + incumbent = person[name].iloc[positions] + known = ~incumbent.isna().to_numpy() + equal = ( + incumbent.to_numpy(dtype=np.float64, na_value=np.nan)[known] + == observed[known] + ) + if name == "A_EXPRRP": + # Older prepared cohorts carry an A_LINENO-derived recode. + # The restored alias owns the CSV observation; an unverified + # incumbent may diagnose a discrepancy but cannot veto it. + relationship_crosscheck.update( + compared_rows=int(known.sum()), + missing_rows=int((~known).sum()), + mismatch_rows=int((~equal).sum()), + ) + compared[name] = int(known.sum()) + continue + conflicts = int((~equal).sum()) + incumbent_conflicts += conflicts + _require(conflicts == 0, "DEMOGRAPHIC_INCUMBENT_CONFLICT", name) + compared[name] = int(known.sum()) + joins.append( + { + "income_year": year, + "survey_year": COHORTS[year], + "member": member, + "archive_sha256": archive_pin, + "member_sha256": pin, + "source_rows": member_rows, + "joined_rows": len(positions), + "unreferenced_source_rows": int( + member_rows - len(np.unique(indices)) + ), + "incumbent_compared_rows": compared, + "incumbent_relationship_crosscheck": relationship_crosscheck, + "accepted_source_missing_tokens": dict.fromkeys(OBSERVATIONS, 0), + "incumbent_absent_columns": sorted( + name for name, value in compared.items() if value is None + ), + "incumbent_conflicts": incumbent_conflicts, + "native_key_or_age_conflicts": coordinate_conflicts, + "counters_zero_in_every_successful_receipt": [ + "accepted_source_missing_tokens", + "incumbent_conflicts", + "native_key_or_age_conflicts", + "unreferenced_source_rows", + ], + "zero_counter_semantics": ZERO_COUNTER_SEMANTICS, + } + ) + membership = verify_household_membership( + income_year=years, + source_household_id=output["source_household_id"], + household_id=household_id, + ) + classification = classify_asec_demographic_observations( + income_year=years, + household_id=household_id, + a_sex=output["asec_A_SEX"], + axsex=output["asec_AXSEX"], + a_exprrp=output["asec_A_EXPRRP"], + p_seq=output["asec_P_SEQ"], + ) + output.update({name: values for name, values in classification.states.items()}) + _parent(source) + _require( + _json(_implementation()) == _json(before), + "DEMOGRAPHIC_IMPLEMENTATION_CHANGED", + ) + buffers = [output[name].astype(" AuthenticatedAsecDemographicSource: + """Reconstruct from pinned members; a candidate supplies bytes, not content.""" + try: + expected = _reconstruct(source, member_paths) + if candidate_path is not None: + payload = _encode(expected) + with tempfile.TemporaryDirectory( + prefix="microcosm-demographic-candidate-" + ) as directory: + digest = _snapshot( + candidate_path, Path(directory) / FILENAME, size=len(payload) + ) + _require(digest == _sha(payload), "DEMOGRAPHIC_CANONICAL_BYTES") + _parent(source) + expected.validate() + return expected + except DemographicSourceRefusalError: + raise + except ( + OSError, + ValueError, + TypeError, + KeyError, + OverflowError, + UnicodeError, + csv.Error, + re.error, + ): + raise DemographicSourceRefusalError("DEMOGRAPHIC_SOURCE_CONTRACT") from None + + +def _attachment_receipt(frame, source, observations) -> dict: + return { + "schema_version": 1, + "artifact_kind": "microcosm.asec_demographic_source_attachment.v1", + "source_identity_sha256": _sha(source.source.identity), + "observations_content_sha256": observations.content_sha256, + "columns": list(ALIASES + DERIVED), + "output_frame_sha256": legacy._frame_signature(frame), + "release_eligible": False, + } + + +def _check_attachment_parents(source, observations) -> None: + _parent(source) + _require( + type(observations) is AuthenticatedAsecDemographicSource, + "DEMOGRAPHIC_AUTHENTICATED_OBSERVATIONS", + ) + observations.validate() + _require( + observations.receipt["source_identity"].encode() == source.source.identity, + "DEMOGRAPHIC_ATTACHMENT_PARENT", + ) + + +@dataclass(frozen=True) +class DemographicSourceAttachedAsec: + """Owned additive Frame beside an unchanged authenticated money parent.""" + + frame: Frame + parent: legacy.AuthenticatedCurrentMoneySource + observations: AuthenticatedAsecDemographicSource + _receipt: bytes + _token: InitVar[object] = None + + def __post_init__(self, _token): + _require( + _token is _COMPOSE_TOKEN, + "DEMOGRAPHIC_ATTACHMENT_CONSTRUCTOR_UNAVAILABLE", + ) + + @property + def receipt(self) -> dict: + return json.loads(self._receipt.decode("ascii")) + + def validate(self) -> None: + try: + _check_attachment_parents(self.parent, self.observations) + _require(type(self.frame) is Frame, "DEMOGRAPHIC_ATTACHMENT_FRAME") + for name in ALIASES + DERIVED: + values = self.frame.person[name] + _require( + values.dtype == np.dtype("int64") + and np.array_equal( + values.to_numpy(), self.observations.array(name) + ), + "DEMOGRAPHIC_ATTACHMENT_OUTPUT", + name, + ) + recovered = restoration._owned_frame(self.frame) + recovered.person.drop(columns=list(ALIASES + DERIVED), inplace=True) + _require( + legacy._frame_signature(recovered) + == self.observations.receipt["source_frame_sha256"] + and self.receipt + == _attachment_receipt(self.frame, self.parent, self.observations), + "DEMOGRAPHIC_ATTACHMENT_CHANGED", + ) + except DemographicSourceRefusalError: + raise + except (ValueError, TypeError, KeyError, AttributeError) as error: + raise DemographicSourceRefusalError( + "DEMOGRAPHIC_ATTACHMENT_CHANGED" + ) from error + + +def attach_asec_demographic_source( + source, observations +) -> DemographicSourceAttachedAsec: + """Append the restored observations and their states; change nothing else.""" + _check_attachment_parents(source, observations) + _require( + not set(ALIASES + DERIVED) & set(source.frame.person), + "DEMOGRAPHIC_ALREADY_ATTACHED", + ) + frame = restoration._owned_frame(source.frame) + for name in ALIASES + DERIVED: + frame.person[name] = observations.array(name).copy() + result = DemographicSourceAttachedAsec( + frame, + source, + observations, + _json(_attachment_receipt(frame, source, observations)), + _token=_COMPOSE_TOKEN, + ) + result.validate() + return result + + +def write_asec_demographic_source(source, *, member_paths, output_dir) -> dict: + """Produce a new immutable local bundle; never replace an existing parent.""" + destination = Path(output_dir) + if destination.exists(): + raise FileExistsError(destination) + value = load_authenticated_asec_demographic_source( + source, member_paths=member_paths + ) + payload = _encode(value) + destination.parent.mkdir(parents=True, exist_ok=True) + with tempfile.TemporaryDirectory( + prefix=".asec-demographic-", dir=destination.parent + ) as directory: + staging = Path(directory) + (staging / FILENAME).write_bytes(payload) + receipt = { + **value.receipt, + "content_sha256": value.content_sha256, + "output_file": FILENAME, + "output_sha256": _sha(payload), + } + (staging / "receipt.json").write_bytes(_json(receipt) + b"\n") + destination.mkdir() + for path in staging.iterdir(): + os.link(path, destination / path.name) + return receipt + + +__all__ = [ + "ACS_GROUP_QUARTERS_CODES", + "ACS_OBSERVED_REFERENCE_CODE", + "ACS_REFERENCE_STATES", + "ACS_SYNTHETIC_GROUP_QUARTERS_RELATIONSHIP_CODE", + "ALIASES", + "ARTIFACT_KIND", + "ASEC_DEMOGRAPHIC_FIELDS", + "ASEC_DEMOGRAPHIC_SOURCE_COLUMNS", + "ASEC_DICTIONARY_AUTHORITY", + "COHORTS", + "COLUMNS", + "DERIVED", + "HOUSEHOLD_GROUPING_COLUMN", + "HOUSEHOLD_MEMBERSHIP_KEY", + "HOUSEHOLD_REFERENCE_CODES", + "HOUSEHOLD_REFERENCE_STATES", + "HOUSEHOLD_REFERENCE_UNBOUND_REASONS", + "MEMBERSHIP_REFUSAL_COUNTERS", + "NON_ADOPTED_HEAD_SIGNALS", + "OBSERVATIONS", + "OBSERVED_PERIOD_KIND", + "SEX_ALLOCATION_STATES", + "SEX_BINDING_STATES", + "SEX_UNBOUND_REASONS", + "ZERO_COUNTER_SEMANTICS", + "AsecDemographicField", + "AuthenticatedAsecDemographicSource", + "DemographicClassification", + "DemographicSourceAttachedAsec", + "DemographicSourceRefusalError", + "acs_household_reference_states", + "attach_asec_demographic_source", + "classify_asec_demographic_observations", + "contract_document", + "load_authenticated_asec_demographic_source", + "read_demographic_member", + "verify_household_membership", + "write_asec_demographic_source", +] diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/asec_engine_evaluation.py b/packages/microcosm-build/src/microcosm/build/us_runtime/asec_engine_evaluation.py new file mode 100644 index 000000000..5648d95a4 --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/asec_engine_evaluation.py @@ -0,0 +1,473 @@ +"""Closure-admitted PolicyEngine-US evaluation of the corrected money leaves. + +An output is admitted only when its complete static input closure is produced by +this graph. The defaults allowlist is empty by decision, so a closure leaf this +slice does not map blocks its root outright rather than being zero-filled or +split into an unreviewed surface. Each admitted root carries a pinned closure +identity; engine or closure drift refuses before any simulation runs. + +Outputs are formula-owned. They are retained in an evaluation artifact and are +never written back as population cells. +""" + +from __future__ import annotations + +import struct +import sys +from dataclasses import dataclass +from hashlib import sha256 +from importlib import resources +from importlib.metadata import distribution +from importlib.util import find_spec +from pathlib import Path + +import numpy as np + +from microcosm.frame import US_SCHEMA, Frame +from microcosm.graph import ArtifactType +from microcosm.graph.canonical import canonical_json + +from .operator_boundary import PRE_ASSEMBLY_OPERATOR_OUTPUT_FAMILIES + +US_ASEC_ENGINE_EVALUATION_TYPE = ArtifactType("microcosm.us.asec_engine_evaluation", 1) +ENGINE_PACKAGE = "policyengine-us" +ENGINE_DEFAULTS_RESOURCE = "asec_current_money_engine_defaults_v1.json" +ENGINE_DEFAULTS_SHA256 = ( + "090dd200ecdb837127dba1d1f22903d152cef8a434085e4ead4d379350ee5b82" +) +ENGINE_DEFAULTS_ARTIFACT_KIND = "microcosm.us.asec_current_money_engine_defaults.v1" +EVALUATION_MAGIC = b"MCASECEV\x01" +EVALUATION_ARTIFACT_KIND = "microcosm.us.asec_engine_evaluation.v1" +EVALUATION_PERIOD = 2024 +EVALUATION_HEADER_MAX_BYTES = 64 * 1024 +EVALUATION_MAX_ROWS = 1_000_000 +_PRIOR_YEAR_MARKERS = ("last_year", "previous_year", "prior_year") + + +class EngineAdmissionError(ValueError): + """A candidate output is not admitted, or a pinned identity drifted.""" + + +def _require(condition: bool, reason: str) -> None: + if not condition: + raise EngineAdmissionError(reason) + + +@dataclass(frozen=True) +class EngineOutputContract: + """Pinned identity of one admitted root's static PolicyEngine-US closure.""" + + root: str + entity: str + engine_version: str + input_leaves: tuple[str, ...] + formula_node_count: int + edge_count: int + sha256: str + + +ADMITTED_ENGINE_OUTPUT_CONTRACTS: tuple[EngineOutputContract, ...] = ( + EngineOutputContract( + root="capital_gains", + entity="person", + engine_version="1.819.0", + input_leaves=( + "long_term_capital_gains_before_response", + "short_term_capital_gains", + ), + formula_node_count=4, + edge_count=6, + sha256="1768cb833b183b3898567910b463e645a2744c2138688c50b5fc1cc17c3319df", + ), + EngineOutputContract( + root="dividend_income", + entity="person", + engine_version="1.819.0", + input_leaves=("non_qualified_dividend_income", "qualified_dividend_income"), + formula_node_count=2, + edge_count=3, + sha256="a27f07536f4909361de7444831f7852a3f976e12d40d6afe4059c6a5fbdfbbe3", + ), + EngineOutputContract( + root="employment_income", + entity="person", + engine_version="1.819.0", + input_leaves=("employment_income_before_lsr",), + formula_node_count=3, + edge_count=4, + sha256="bdcd8da23a79ff228a610a4d0b483f5fd2068d968f31e96c07cea8a86cc3366b", + ), + EngineOutputContract( + root="social_security", + entity="person", + engine_version="1.819.0", + input_leaves=( + "social_security_dependents", + "social_security_disability", + "social_security_retirement", + "social_security_survivors", + ), + formula_node_count=1, + edge_count=4, + sha256="3f5a5304ac1e8c8d785614069867590176325b212a0a080457f80594a6fd53c0", + ), +) +BLOCKED_ENGINE_OUTPUTS: tuple[str, ...] = ( + "interest_income", + "pension_income", + "self_employment_income", +) +ADMITTED_ROOTS: tuple[str, ...] = tuple( + contract.root for contract in ADMITTED_ENGINE_OUTPUT_CONTRACTS +) + + +def load_engine_defaults() -> dict: + """Read and pin-verify the defaults resource; the allowlist must stay empty.""" + payload = ( + resources.files(__package__).joinpath(ENGINE_DEFAULTS_RESOURCE).read_bytes() + ) + _require( + sha256(payload).hexdigest() == ENGINE_DEFAULTS_SHA256, + "ENGINE_DEFAULTS_FINGERPRINT", + ) + import json + + document = json.loads(payload) + _require( + document["schema_version"] == 1 + and document["artifact_kind"] == ENGINE_DEFAULTS_ARTIFACT_KIND, + "ENGINE_DEFAULTS_SCHEMA", + ) + _require(document["allowlist"] == [], "ENGINE_DEFAULTS_ALLOWLIST") + _require( + tuple(sorted(document["admitted_roots"])) == ADMITTED_ROOTS, + "ENGINE_DEFAULTS_ROOTS", + ) + _require( + tuple(sorted(document["blocked_roots"])) == BLOCKED_ENGINE_OUTPUTS, + "ENGINE_DEFAULTS_BLOCKED", + ) + _require(document["engine_package"] == ENGINE_PACKAGE, "ENGINE_DEFAULTS_PACKAGE") + _require(document["release_eligible"] is False, "ENGINE_DEFAULTS_RELEASE_CLAIM") + return document + + +def _runtime_package_identity(package: str, module: str) -> dict: + """Hash installed model code and parameter files, never population data. + + Include relative names so additions, removals and relocation have explicit + semantics. Recompute on every invocation: cache admission must see an edit + made after a previous evaluation in the same process. + """ + installed = distribution(package) + root = Path(installed.locate_file(module)) + files = sorted( + path + for path in root.rglob("*") + if path.is_file() + and ( + path.suffix == ".py" + or ( + "parameters" in path.relative_to(root).parts + and path.suffix in (".yaml", ".yml", ".json") + ) + ) + ) + records = [ + [path.relative_to(root).as_posix(), sha256(path.read_bytes()).hexdigest()] + for path in files + ] + return { + "version": installed.version, + "files": len(records), + "source_and_parameters_sha256": sha256(canonical_json(records)).hexdigest(), + } + + +def _verify_runtime_origin(package: str, module: str) -> None: + """The adapter must import the same distribution whose bytes are pinned.""" + root = Path(distribution(package).locate_file(module)).resolve() + spec = find_spec(module) + _require( + spec is not None + and spec.origin is not None + and Path(spec.origin).resolve() == root / "__init__.py" + and spec.submodule_search_locations is not None + and [Path(path).resolve() for path in spec.submodule_search_locations] + == [root], + "ENGINE_RUNTIME_ORIGIN", + ) + for name, loaded in tuple(sys.modules.items()): + if name != module and not name.startswith(module + "."): + continue + filename = getattr(loaded, "__file__", None) + if filename is not None: + _require( + Path(filename).resolve().is_relative_to(root), "ENGINE_RUNTIME_ORIGIN" + ) + for path in getattr(loaded, "__path__", ()): + _require(Path(path).resolve().is_relative_to(root), "ENGINE_RUNTIME_ORIGIN") + + +def engine_runtime_identity() -> dict: + """Verify the pinned baseline runtime before execution or a cache lookup.""" + document = load_engine_defaults() + for package, module in ( + ("policyengine-us", "policyengine_us"), + ("policyengine-core", "policyengine_core"), + ): + _verify_runtime_origin(package, module) + observed = { + package: _runtime_package_identity(package, module) + for package, module in ( + ("policyengine-us", "policyengine_us"), + ("policyengine-core", "policyengine_core"), + ) + } + _require(observed == document["baseline_runtime"], "ENGINE_RUNTIME_DRIFT") + return {"policy": "installed_baseline_no_reforms", "packages": observed} + + +def _prior_year_leaves() -> frozenset[str]: + family = PRE_ASSEMBLY_OPERATOR_OUTPUT_FAMILIES["prior_year_income"] + return frozenset(name for columns in family.values() for name in columns) + + +def _closure(index, root: str): + try: + return index.variable_dependency_closure(root) + except ValueError as error: + raise EngineAdmissionError(f"UNKNOWN_ENGINE_ROOT:{root}") from error + + +def _check_contract(closure, contract: EngineOutputContract) -> None: + observed = { + "engine_version": closure.engine_version, + "root": closure.root, + "input_leaves": tuple(closure.input_leaves), + "formula_node_count": len(closure.formula_nodes), + "edge_count": len(closure.edges), + "sha256": closure.sha256, + } + expected = { + "engine_version": contract.engine_version, + "root": contract.root, + "input_leaves": contract.input_leaves, + "formula_node_count": contract.formula_node_count, + "edge_count": contract.edge_count, + "sha256": contract.sha256, + } + if observed != expected: + raise EngineAdmissionError( + "PolicyEngine-US closure drifted for " + f"{contract.root!r}; expected={expected}, observed={observed}." + ) + + +def admit_engine_outputs( + index, + *, + produced_leaves, + defaults: dict | None = None, +) -> tuple[EngineOutputContract, ...]: + """Admit only roots whose complete closure this graph produces.""" + document = load_engine_defaults() if defaults is None else defaults + _require(document["allowlist"] == [], "ENGINE_DEFAULTS_ALLOWLIST") + produced = frozenset(produced_leaves) + prior_year = _prior_year_leaves() + for contract in ADMITTED_ENGINE_OUTPUT_CONTRACTS: + closure = _closure(index, contract.root) + _check_contract(closure, contract) + metadata = index.variable_metadata(contract.root) + _require(metadata.entity == contract.entity, "ADMITTED_ROOT_ENTITY") + missing = sorted(set(closure.input_leaves) - produced) + if missing: + raise EngineAdmissionError( + f"Root {contract.root!r} has unproduced closure leaves {missing}; " + "the engine-default allowlist is empty by decision." + ) + reaching = sorted(set(closure.input_leaves) & prior_year) + _require(not reaching, f"PRIOR_YEAR_LEAF_IN_CLOSURE:{contract.root}") + _require( + not any( + marker in leaf + for leaf in closure.input_leaves + for marker in _PRIOR_YEAR_MARKERS + ), + f"PRIOR_YEAR_LEAF_IN_CLOSURE:{contract.root}", + ) + for root in BLOCKED_ENGINE_OUTPUTS: + closure = _closure(index, root) + blocked = sorted(set(closure.input_leaves) - produced) + _require(bool(blocked), f"BLOCKED_ROOT_NOW_COMPLETE:{root}") + return ADMITTED_ENGINE_OUTPUT_CONTRACTS + + +@dataclass(frozen=True) +class EngineEvaluation: + """Engine outputs retained as evidence; never population cells.""" + + header: dict + values: dict[str, np.ndarray] + + @property + def roots(self) -> tuple[str, ...]: + return tuple(sorted(self.values)) + + +def _aggregates(values: np.ndarray) -> dict[str, object]: + return { + "rows": int(values.size), + "nonzero_rows": int(np.count_nonzero(values)), + "unweighted_total": float(values.sum()), + "minimum": float(values.min()), + "maximum": float(values.max()), + } + + +def materialize_engine_outputs( + frame: Frame, + engine, + contracts: tuple[EngineOutputContract, ...], + *, + period: int = EVALUATION_PERIOD, + context_bindings: dict | None = None, +) -> EngineEvaluation: + """Run the real adapter for the admitted roots and retain the arrays only.""" + _require(type(frame) is Frame and frame.schema == US_SCHEMA, "ENGINE_FRAME_SCHEMA") + _require(getattr(engine, "country", None) == "us", "ENGINE_COUNTRY") + _require(period == EVALUATION_PERIOD, "ENGINE_PERIOD") + _require(bool(contracts), "ENGINE_NO_ADMITTED_OUTPUTS") + _require(contracts == ADMITTED_ENGINE_OUTPUT_CONTRACTS, "ENGINE_CONTRACT_ROSTER") + runtime = engine_runtime_identity() + rows = frame.n("person") + _require(0 < rows <= EVALUATION_MAX_ROWS, "ENGINE_ROW_BOUND") + roots = tuple(sorted(contract.root for contract in contracts)) + _require(len(set(roots)) == len(roots), "ENGINE_DUPLICATE_ROOT") + results = engine.materialize(frame, list(roots), period) + _require(set(results) == set(roots), "ENGINE_RESULT_ROSTER") + values: dict[str, np.ndarray] = {} + for root in roots: + array = np.asarray(results[root], dtype="float64") + _require(array.shape == (rows,), "ENGINE_RESULT_SHAPE") + _require(bool(np.isfinite(array).all()), "ENGINE_RESULT_NONFINITE") + values[root] = np.ascontiguousarray(array, dtype=" tuple[bytes, ...]: + _require(type(evaluation) is EngineEvaluation, "TYPED_EVALUATION_REQUIRED") + header = canonical_json(evaluation.header) + _require(0 < len(header) <= EVALUATION_HEADER_MAX_BYTES, "HEADER_SIZE") + roots = tuple(evaluation.header["roots"]) + _require(roots == tuple(sorted(evaluation.values)), "ENGINE_RESULT_ROSTER") + rows = evaluation.header["person_rows"] + parts = [EVALUATION_MAGIC, struct.pack(" bytes: + """Encode the evaluation header and arrays with a transport checksum.""" + parts = _evaluation_parts(evaluation) + checksum = sha256() + for part in parts: + checksum.update(part) + return b"".join((*parts, checksum.digest())) + + +def decode_engine_evaluation(payload: bytes) -> EngineEvaluation: + """Decode a stored evaluation, verifying framing, checksum and shapes.""" + _require( + type(payload) is bytes and len(payload) > len(EVALUATION_MAGIC) + 4 + 32, + "PAYLOAD_SIZE", + ) + _require(payload.startswith(EVALUATION_MAGIC), "PAYLOAD_VERSION") + size = struct.unpack_from(" None: + if not condition: + raise HouseholdCoverageSourceError(reason) + + +def _producer() -> dict: + _require(_csv_builtin.csv_reader_bound(csv), "SOURCE_CSV_READER_CHANGED") + return { + "protocol": _SCHEMA, + "owner_sha256": shared._sha( + resources.files(__package__) + .joinpath("asec_household_coverage_fields.py") + .read_bytes() + ), + "bounded_reader_dependency": shared._implementation(), + "csv_builtin_sha256": shared._sha( + resources.files(__package__).joinpath("source_csv_builtin.py").read_bytes() + ), + "columns": COLUMNS, + "fields": _FIELDS, + "interpretation_notes": _INTERPRETATION_NOTES, + } + + +def _code_state(name: str, token: str, interview_code: int | None) -> dict: + contract = _FIELDS[name] + code, label = None, None + if token == "": + status = "missing" + elif re.fullmatch(rf"[0-9]{{1,{contract['printed_length']}}}", token) is None: + status = "malformed" + else: + code = int(token) + label = contract["labels"].get(str(code)) + status = "valid" if label is not None else "unlabelled" + universe = ( + "in" + if contract["universe_as_printed"] == "All Households" or interview_code == 1 + else "outside" + if interview_code in (2, 3) + else "unresolved" + ) + return { + "code": code, + "label": label, + "code_status": status, + "universe_status": universe, + } + + +def _project(row: Sequence[str]) -> dict: + _require( + all(len(token) <= shared._MAX_FIELD_CHARS for token in row), "MEMBER_FIELD_SIZE" + ) + native = row[0] + _require( + re.fullmatch(r"[0-9]+", native) is not None and 1 <= int(native) <= 99999, + "MEMBER_NATIVE_KEY", + ) + literals = dict(zip(COLUMNS, row, strict=True)) + interview = _code_state("H_HHTYPE", literals["H_HHTYPE"], None) + interview_code = interview["code"] if interview["code_status"] == "valid" else None + states = { + name: _code_state(name, literals[name], interview_code) for name in COLUMNS[1:] + } + hr = ( + states["HRHTYPE"]["code"] + if states["HRHTYPE"]["code_status"] == "valid" + else None + ) + count = ( + states["H_NUMPER"]["code"] + if states["H_NUMPER"]["code_status"] == "valid" + else None + ) + diagnostics = [] + if interview_code == 1: + if hr == 0: + diagnostics.append("interview_with_noninterview_household_type") + if count == 0: + diagnostics.append("interview_with_noninterview_person_count") + elif interview_code in (2, 3): + if hr is not None and hr > 0: + diagnostics.append("noninterview_with_interview_household_type") + if count is not None and count > 0: + diagnostics.append("noninterview_with_positive_person_count") + if states["H_LIVQRT"]["code"] == 11: + diagnostics.append("student_quarters_dictionary_methodology_unresolved") + return { + **literals, + "native_household_id": int(native), + "field_states": states, + "reported_person_count": count + if interview_code == 1 and count is not None and count > 0 + else None, + "diagnostics": diagnostics, + "coverage_interpretation": "unresolved", + } + + +def _read_capture(capture: Path, pin) -> list[dict]: + """Compose existing bounded byte/record primitives with this field projection.""" + csv_reader = _csv_builtin.capture_csv_reader(csv) + _require(csv_reader is not None, "SOURCE_CSV_READER_CHANGED") + result, seen = [], set() + member = SourceMember(pin.canonical_member_id, pin.member_sha256) + with open(capture, "rb", buffering=0, opener=shared._regular_opener) as raw: + before = os.fstat(raw.fileno()) + _require(before.st_size == pin.size_bytes, "CAPTURE_SIZE") + digest = shared._DigestReader(raw, pin.size_bytes) + with io.TextIOWrapper( + io.BufferedReader(digest), encoding="utf-8-sig", newline="" + ) as text: + lines = shared._RecordLines(text) + reader = csv_reader(lines, strict=True) + header = next(reader, []) + _require( + bool(header) + and all(header) + and len(set(header)) == len(header) + and set(COLUMNS) <= set(header), + "MEMBER_HEADER", + ) + positions = [header.index(column) for column in COLUMNS] + while True: + lines.characters = 0 + try: + row = next(reader) + except StopIteration: + break + _require(len(row) == len(header), "MEMBER_ROW_WIDTH") + _require(len(result) < pin.rows, "MEMBER_ROWS") + record = _project([row[i] for i in positions]) + native = record["native_household_id"] + _require(native not in seen, "MEMBER_DUPLICATE_KEY") + seen.add(native) + record.update( + income_year=pin.income_year, + survey_year=pin.survey_year, + member_id=pin.canonical_member_id, + member_sha256=pin.member_sha256, + member_row_1based=len(result) + 1, + origin_key=origin_key( + AsecHouseholdOrigin(member, pin.income_year, native) + ), + ) + result.append(record) + _require(len(result) == pin.rows, "MEMBER_ROWS") + _require( + digest.count == pin.size_bytes + and digest.digest.hexdigest() == pin.member_sha256, + "CAPTURE_CHANGED", + ) + after = os.fstat(raw.fileno()) + _require( + all( + getattr(before, key) == getattr(after, key) + for key in ( + "st_dev", + "st_ino", + "st_size", + "st_mtime_ns", + "st_ctime_ns", + ) + ), + "CAPTURE_CHANGED", + ) + return result + + +@dataclass(frozen=True, slots=True, weakref_slot=True) +class AuthenticatedAsecHouseholdCoverageFields: + """Reader-issued immutable original observations, without Frame authority.""" + + payload: bytes + _token: InitVar[object] = None + _issued_sha256: str = field(init=False, repr=False) + + def __post_init__(self, _token: object) -> None: + _require(_token is _TOKEN, "SOURCE_CONSTRUCTOR") + try: + digest = shared._issue_capsule(self, self.payload) + except shared.HouseholdWeightSourceError as error: + raise HouseholdCoverageSourceError(str(error)) from None + object.__setattr__(self, "_issued_sha256", digest) + + @property + def document(self) -> dict: + """Return a defensive view after checking issuance and current producer.""" + return verify_asec_household_coverage_fields(self) + + def households_for( + self, native_roster: Sequence[tuple[int, int]] + ) -> tuple[dict, ...]: + """Look up exact cohort/native keys; preserve unresolved observations.""" + document = self.document + _require( + isinstance(native_roster, (list, tuple)) + and 0 < len(native_roster) <= shared._MAX_ROWS, + "LOOKUP_ROSTER", + ) + keys = [] + for key in native_roster: + _require( + isinstance(key, (list, tuple)) + and len(key) == 2 + and all(type(value) is int for value in key), + "LOOKUP_ROSTER", + ) + keys.append(tuple(key)) + _require(len(set(keys)) == len(keys), "LOOKUP_ROSTER") + records = { + (row["income_year"], row["native_household_id"]): row + for row in document["records"] + } + _require(all(key in records for key in keys), "LOOKUP_UNKNOWN") + return tuple(records[key] for key in keys) + + +def verify_asec_household_coverage_fields( + source: AuthenticatedAsecHouseholdCoverageFields, +) -> dict: + """Validate a reader-issued capsule; JSON is not an admission credential.""" + _require(type(source) is AuthenticatedAsecHouseholdCoverageFields, "SOURCE_TYPE") + try: + payload = shared._checked_capsule_payload(source) + document = json.loads(payload) + _require( + shared._encode(document["producer"]) == shared._encode(_producer()), + "PRODUCER_CHANGED", + ) + except shared.HouseholdWeightSourceError as error: + raise HouseholdCoverageSourceError(str(error)) from None + return document + + +def load_authenticated_asec_household_coverage_fields( + member_paths: Mapping[int, str | Path], *, candidate: bytes | None = None +) -> AuthenticatedAsecHouseholdCoverageFields: + """Reconstruct whole requested original cohorts, never caller-provided rows.""" + try: + pins = shared._registry() + paths = shared._member_path_snapshot(member_paths, pins) + _require( + candidate is None + or ( + type(candidate) is bytes and len(candidate) <= shared._MAX_PAYLOAD_BYTES + ), + "CANDIDATE_SIZE", + ) + producer_document = _producer() + _require( + shared._encode(producer_document["bounded_reader_dependency"]["registry"]) + == shared._encode([asdict(pin) for pin in pins]), + "REGISTRY_TRANSITION", + ) + producer = shared._encode(producer_document) + selected = [pin for pin in pins if pin.income_year in paths] + _require(bool(selected), "MEMBER_PATHS") + _require(sum(pin.rows for pin in selected) <= shared._MAX_ROWS, "REQUEST_ROWS") + records = [] + with tempfile.TemporaryDirectory( + prefix="asec-household-coverage-fields-" + ) as tmp: + for pin in selected: + capture = Path(tmp) / f"{pin.income_year}.csv" + try: + digest = shared._capture_owner._snapshot( + paths[pin.income_year], capture, size=pin.size_bytes + ) + except (OSError, ValueError, TypeError): + raise HouseholdCoverageSourceError("MEMBER_CAPTURE") from None + _require(digest == pin.member_sha256, "MEMBER_SHA256") + try: + records.extend(_read_capture(capture, pin)) + except ( + HouseholdCoverageSourceError, + shared.HouseholdWeightSourceError, + ): + raise + except (OSError, UnicodeError, csv.Error, ValueError, OverflowError): + raise HouseholdCoverageSourceError("MEMBER_ENCODING") from None + _require(shared._encode(_producer()) == producer, "PRODUCER_CHANGED") + payload = shared._encode( + { + "schema": _SCHEMA, + "producer": json.loads(producer), + "members": [asdict(pin) for pin in selected], + "fields": _FIELDS, + "dictionary_authorities": [ + { + "survey_year": year, + "sha256": digest, + "url": "https://www2.census.gov/programs-surveys/cps/datasets/" + f"{year}/march/asec{year}_ddl_pub_full.pdf", + } + for year, _page, digest in shared._DICTIONARIES + ], + "interpretation_notes": _INTERPRETATION_NOTES, + "native_roster_sha256": shared._sha( + shared._encode( + [ + [row["income_year"], row["native_household_id"]] + for row in records + ] + ) + ), + "projection_sha256": shared._sha(shared._encode(records)), + "records": records, + "source_authenticated": True, + "population_binding_authenticated": False, + "release_eligible": False, + "coverage_classification_performed": False, + } + ) + _require(candidate is None or candidate == payload, "CANDIDATE_MISMATCH") + return AuthenticatedAsecHouseholdCoverageFields(payload, _token=_TOKEN) + except shared.HouseholdWeightSourceError as error: + raise HouseholdCoverageSourceError(str(error)) from None diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/asec_household_observations.py b/packages/microcosm-build/src/microcosm/build/us_runtime/asec_household_observations.py new file mode 100644 index 000000000..c0ba8eb56 --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/asec_household_observations.py @@ -0,0 +1,392 @@ +"""Attach authenticated raw ASEC household observations by exact original keys. + +The caller authenticates the v4 checkpoint bytes and loads its actual Frame and +metadata with ``load_asec_raw_stage_checkpoint_v4`` before calling this helper. +Structural checks here do not authenticate arbitrary caller-supplied Frame +values. The augmented source has a new attachment receipt; it is not the input +checkpoint's v4 content identity. No graph registration or price conversion runs. +""" + +from __future__ import annotations + +import hashlib +import json +import os +import re +from collections.abc import Mapping, Sequence +from dataclasses import dataclass +from pathlib import Path +from typing import BinaryIO + +import h5py +import numpy as np +import pandas as pd + +from microcosm.build.outer_stage_runtime import FrameIdentity, frame_identity +from microcosm.build.us_runtime import asec_checkpoint +from microcosm.build.us_runtime.operator_boundary import ( + assert_operator_free_source_frame, +) +from microcosm.frame import Frame + +ASEC_HOUSEHOLD_OBSERVATION_COLUMNS = ( + "HTOTVAL", + "H_LIVQRT", + "HRHTYPE", + "H_HHTYPE", + "H_SEQ", + "H_TENURE", + "GESTFIPS", +) +_OUTPUT_COLUMNS = tuple( + f"asec_{column}" for column in ASEC_HOUSEHOLD_OBSERVATION_COLUMNS +) +_SHA256 = re.compile(r"[0-9a-f]{64}") +_CONTRACT_LABEL = Path("authenticated-v4-frame") + + +def _require_sha(value: object, label: str) -> str: + if not isinstance(value, str) or _SHA256.fullmatch(value) is None: + raise ValueError(f"{label} must be a lowercase SHA-256 digest.") + return value + + +@dataclass(frozen=True) +class AsecHouseholdObservationSource: + """Explicit raw household file binding to the checkpoint's income year.""" + + year: int + path: Path + sha256: str + + def __post_init__(self) -> None: + if type(self.year) is not int: + raise ValueError("ASEC household source income year must be an integer.") + _require_sha(self.sha256, "ASEC household source pin") + object.__setattr__(self, "path", Path(self.path)) + + +@dataclass(frozen=True) +class AsecHouseholdObservationResult: + """The augmented full source Frame and a reproducible, value-free receipt.""" + + frame: Frame + _receipt_json: str + + @property + def receipt(self) -> dict[str, object]: + """Return an independent JSON-safe copy of the immutable receipt.""" + return json.loads(self._receipt_json) + + +def _stat_identity(stat: os.stat_result) -> tuple[int, ...]: + return (stat.st_dev, stat.st_ino, stat.st_size, stat.st_mtime_ns, stat.st_ctime_ns) + + +def _hash_file(handle: BinaryIO) -> str: + handle.seek(0) + return hashlib.file_digest(handle, "sha256").hexdigest() + + +def _hard_child(group: h5py.Group, name: str) -> h5py.Group | h5py.Dataset: + if not isinstance(group.get(name, getlink=True), h5py.HardLink): + raise ValueError(f"ASEC household HDF requires local hard-linked {name!r}.") + child = group[name] + if isinstance(child, h5py.Dataset) and (child.is_virtual or child.external): + raise ValueError("ASEC household HDF cannot read virtual or external storage.") + return child + + +def _dataset(group: h5py.Group, name: str) -> h5py.Dataset: + child = _hard_child(group, name) + if not isinstance(child, h5py.Dataset): + raise ValueError(f"ASEC household HDF {name!r} must be a dataset.") + return child + + +def _column_names(group: h5py.Group, name: str) -> tuple[str, ...]: + dataset = _dataset(group, name) + if dataset.ndim != 1 or dataset.dtype.kind != "S": + raise ValueError("ASEC household column metadata must be fixed-width strings.") + try: + values = tuple(value.decode("utf-8") for value in dataset[()]) + except UnicodeDecodeError as error: + raise ValueError("ASEC household column metadata must be UTF-8.") from error + if len(values) != len(set(values)) or any(not name for name in values): + raise ValueError("ASEC household column metadata is empty or duplicated.") + return values + + +def _read_numeric_household( + handle: BinaryIO, + *, + columns: Sequence[str] = ASEC_HOUSEHOLD_OBSERVATION_COLUMNS, + require_same_block: bool = False, +) -> pd.DataFrame: + """Read selected primitive observations without decoding any object block.""" + if ( + isinstance(columns, str) + or not columns + or any(not isinstance(name, str) or not name for name in columns) + or len(set(columns)) != len(columns) + ): + raise ValueError("ASEC household numeric projection requires unique names.") + requested = tuple(columns) + with h5py.File(handle, "r") as h5: + group = _hard_child(h5, "household") + if not isinstance(group, h5py.Group): + raise ValueError("ASEC household HDF lacks its household group.") + if group.attrs.get("pandas_type") != b"frame" or group.attrs.get("ndim") != 2: + raise ValueError( + "ASEC household HDF requires the pandas fixed frame format." + ) + nblocks = group.attrs.get("nblocks") + if ( + isinstance(nblocks, (bool, np.bool_)) + or not isinstance(nblocks, (int, np.integer)) + or nblocks < 1 + ): + raise ValueError("ASEC household HDF has an invalid block count.") + columns = _column_names(group, "axis0") + if set(requested) - set(columns): + raise ValueError("ASEC household HDF lacks required observed columns.") + axis1 = _dataset(group, "axis1") + if axis1.ndim != 1: + raise ValueError("ASEC household HDF row axis must be one-dimensional.") + rows = axis1.shape[0] + if not rows: + raise ValueError("ASEC household HDF has no household rows.") + all_items = [] + output = {} + for index in range(int(nblocks)): + names = _column_names(group, f"block{index}_items") + all_items.extend(names) + wanted = [i for i, name in enumerate(names) if name in requested] + if not wanted: + # Do not even open an unrelated object/VLARRAY value dataset. + continue + if require_same_block and output: + raise ValueError( + "ASEC household selected observations require one shared block." + ) + values = _dataset(group, f"block{index}_values") + if values.dtype != np.dtype(" pd.DataFrame: + with source.path.open("rb", buffering=0) as handle: + before = _stat_identity(os.fstat(handle.fileno())) + if ( + _stat_identity(source.path.stat()) != before + or _hash_file(handle) != source.sha256 + ): + raise ValueError(f"ASEC household source {source.year} SHA-256 mismatch.") + handle.seek(0) + table = _read_numeric_household(handle) + if ( + _hash_file(handle) != source.sha256 + or _stat_identity(os.fstat(handle.fileno())) != before + or _stat_identity(source.path.stat()) != before + ): + raise ValueError( + f"ASEC household source {source.year} changed during reading." + ) + if table.H_SEQ.duplicated().any(): + raise ValueError(f"ASEC household source {source.year} repeats H_SEQ.") + return table + + +def _int64_column(table: pd.DataFrame, name: str) -> np.ndarray: + if name not in table or table[name].dtype != np.dtype("int64"): + raise ValueError( + f"ASEC household join requires complete ordinary int64 {name}." + ) + return table[name].to_numpy(copy=False) + + +def _validate_input(frame: Frame, metadata: Mapping[str, object]) -> str: + if not isinstance(frame, Frame) or not isinstance(metadata, Mapping): + raise TypeError( + "ASEC household attachment requires an actual v4 Frame and metadata." + ) + asec_checkpoint._validate_raw_stage_binding( + metadata, + path=_CONTRACT_LABEL, + policy=asec_checkpoint._RAW_STAGE_V4, + ) + asec_checkpoint._validate_asec_frame( + frame, path=_CONTRACT_LABEL, artifact_label="ASEC v4 source" + ) + assert_operator_free_source_frame(frame, label="ASEC household observation input") + asec_checkpoint._validate_raw_stage_source_columns( + frame, + path=_CONTRACT_LABEL, + policy=asec_checkpoint._RAW_STAGE_V4, + ) + asec_checkpoint._validate_coverage_v4_binding(frame, metadata, path=_CONTRACT_LABEL) + actual = frame_identity(frame) + for key in ("identity", "source_construction_identity"): + if FrameIdentity.from_payload(metadata[key], label=key) != actual: + raise ValueError( + "ASEC household input structural identity differs from v4 metadata." + ) + if set(_OUTPUT_COLUMNS) & set(frame.table("household").columns): + raise ValueError( + "ASEC household observation attachment refuses existing output columns." + ) + return actual.sha256 + + +def _household_keys(frame: Frame) -> pd.DataFrame: + person, household = frame.person, frame.table("household") + person_keys = pd.DataFrame( + { + "household_id": _int64_column(person, "person_household_id"), + "source_year": _int64_column(person, "source_year"), + "H_SEQ": _int64_column(person, "source_household_id"), + } + ) + keys = person_keys.drop_duplicates() + if keys.household_id.duplicated().any(): + raise ValueError("ASEC persons disagree on the source key within a household.") + if keys.duplicated(["source_year", "H_SEQ"]).any(): + raise ValueError( + "ASEC raw source key is claimed by multiple normalized households." + ) + ids = _int64_column(household, "household_id") + if set(keys.household_id) != set(ids): + raise ValueError( + "ASEC household source keys do not cover actual household IDs." + ) + return keys.set_index("household_id").loc[ids].reset_index() + + +def with_asec_household_observations( + frame: Frame, + *, + checkpoint_metadata: Mapping[str, object], + checkpoint_sha256: str, + sources: Sequence[AsecHouseholdObservationSource], +) -> AsecHouseholdObservationResult: + """Augment the actual authenticated v4 source without changing its input. + + ``checkpoint_sha256`` records the caller's authenticated input-byte pin; this + in-memory helper cannot verify that file. Explicit household sources must + match all per-year pins in the loaded checkpoint receipt. Every raw source + is authenticated locally before and after numeric-only reading. Source codes + remain unclassified; nominal signed amounts remain exact int64 observations. + """ + checkpoint_sha256 = _require_sha(checkpoint_sha256, "Input checkpoint pin") + input_identity = _validate_input(frame, checkpoint_metadata) + declared = { + item["year"]: item["sha256"] + for item in checkpoint_metadata["source_receipt"]["sources"] + } + if any( + not isinstance(source, AsecHouseholdObservationSource) for source in sources + ): + raise TypeError("ASEC household sources require explicit typed file bindings.") + actual = {source.year: source.sha256 for source in sources} + if len(actual) != len(sources) or actual != declared: + raise ValueError("ASEC household source years/pins differ from the v4 receipt.") + keys = _household_keys(frame) + household = frame.table("household").copy() + _int64_column(household, "state_fips") + _int64_column(household, "H_TENURE") + outputs = { + name: np.empty(len(household), dtype=np.int64) for name in _OUTPUT_COLUMNS + } + source_receipts = [] + for source in sorted(sources, key=lambda value: value.year): + raw = _load_source(source) + positions = np.flatnonzero(keys.source_year.to_numpy() == source.year) + source_positions = pd.Index(raw.H_SEQ).get_indexer(keys.H_SEQ.iloc[positions]) + if (source_positions < 0).any(): + raise ValueError( + f"ASEC household source {source.year} lacks a referenced H_SEQ." + ) + joined = raw.iloc[source_positions] + for raw_name, existing_name in ( + ("H_TENURE", "H_TENURE"), + ("GESTFIPS", "state_fips"), + ): + if not np.array_equal( + joined[raw_name].to_numpy(), + household[existing_name].iloc[positions].to_numpy(), + ): + raise ValueError( + f"ASEC household joined {raw_name} disagrees with carried {existing_name}." + ) + for column in ASEC_HOUSEHOLD_OBSERVATION_COLUMNS: + outputs[f"asec_{column}"][positions] = joined[column].to_numpy(copy=False) + source_receipts.append( + { + "income_year": source.year, + "sha256": source.sha256, + "source_rows": len(raw), + "joined_rows": len(positions), + "unreferenced_source_rows": len(raw) - len(positions), + } + ) + for name, values in outputs.items(): + household[name] = values + tables = {entity: frame.table(entity) for entity in frame.entities} + tables["household"] = household + augmented = Frame( + tables, + frame.schema, + {entity: frame.weights_for(entity) for entity in frame.weighted_entities}, + frame.strata, + mass_log=frame.mass_log, + metadata=frame.metadata, + ) + assert_operator_free_source_frame( + augmented, label="ASEC household observation output" + ) + if frame_identity(augmented).sha256 != input_identity: + raise AssertionError("ASEC household attachment changed structural identity.") + receipt = { + "schema_version": 1, + "artifact_kind": "microcosm.asec_household_observation_attachment", + "input_checkpoint_sha256": checkpoint_sha256, + "input_structural_identity_sha256": input_identity, + "reader": "pandas_fixed_hdf_numeric_int64_v1", + "join_keys": ["income_year", "H_SEQ"], + "sources": source_receipts, + "household_identity": { + "rows": len(household), + "id_dtype": "int64", + "ordered_ids_sha256": hashlib.sha256( + household.household_id.to_numpy(dtype=" None: + if not condition: + raise HousingStatusRefusalError(reason) + + +def _json(value: object) -> bytes: + return json.dumps( + value, sort_keys=True, separators=(",", ":"), allow_nan=False + ).encode() + + +def _parse(value: bytes) -> dict: + _require(type(value) is bytes and 0 < len(value) <= HEADER_MAX_BYTES, "HEADER_SIZE") + try: + parsed = json.loads(value) + _require(type(parsed) is dict and _json(parsed) == value, "HEADER_CANONICAL") + return parsed + except (ValueError, TypeError, UnicodeError, RecursionError): + raise HousingStatusRefusalError("HEADER_FORMAT") from None + + +def _sha(value: bytes) -> str: + return hashlib.sha256(value).hexdigest() + + +def _codebook() -> dict: + return { + "status": { + "unknown": 0, + "receipt": 1, + "nonreceipt": 2, + "not_in_universe": 3, + "conflict": 4, + }, + "route": { + "unresolved": 0, + "public_housing": 1, + "lower_rent": 2, + "both_negative": 3, + "not_in_universe": 4, + }, + "group_quarters": {"not_gq": 0, "gq": 1, "unavailable": 2}, + "conflict_bits": { + "excluded_scope": 1, + "parent_route": 2, + "public_allocation": 4, + "lower_allocation": 8, + "context": 16, + }, + "unknown_reason_bits": {"parent_answer": 1, "lower_answer": 2, "scope": 4}, + "quality": { + "not_applicable": 0, + "no_allocation_code_origin_unresolved": 1, + "allocated": 2, + "conflict": 3, + }, + "validity": "1 iff substantive response; receipt_valid additionally requires consistent known receipt/nonreceipt", + "zero_origin": {"not_zero": 0, "frozen_zero_origin_unresolved": 1}, + } + + +def _definition_identity() -> str: + return _sha( + _json( + { + "module": _sha( + resources.files(__package__) + .joinpath("asec_housing_status.py") + .read_bytes() + ), + "codebook": _codebook(), + "dictionaries": DICTIONARIES, + "zero_origin_evidence": ZERO_ORIGIN_EVIDENCE, + "dependencies": { + name: metadata.version(name) for name in ("numpy", "pandas") + }, + } + ) + ) + + +def _derive(raw: dict[str, np.ndarray]) -> dict[str, np.ndarray]: + """Vectorized dictionary-domain checks and explicit interpretation axes.""" + n = len(raw["household_id"]) + _require(0 < n <= MAX_HOUSEHOLDS, "ROW_COUNT") + _require( + all(v.shape == (n,) and v.dtype == np.dtype("int64") for v in raw.values()), + "RAW_INT64", + ) + for name, maximum in ( + ("HPUBLIC", 2), + ("HLORENT", 2), + ("I_HPUBLI", 1), + ("I_HLOREN", 1), + ): + _require(bool(((raw[name] >= 0) & (raw[name] <= maximum)).all()), "CODE_DOMAIN") + for name, maximum in (("H_TENURE", 3), ("HRHTYPE", 10), ("H_HHTYPE", 3)): + # Zero context remains unresolved under the frozen fill provenance. + _require( + bool(((raw[name] >= 0) & (raw[name] <= maximum)).all()), "CONTEXT_DOMAIN" + ) + _require(bool(np.isin(raw["income_year"], [2022, 2023, 2024]).all()), "INCOME_YEAR") + _require(bool((raw["survey_year"] == raw["income_year"] + 1).all()), "SURVEY_YEAR") + _require(bool((raw["H_SEQ"] > 0).all()), "NATIVE_KEY") + _require(len(np.unique(raw["household_id"])) == n, "DUPLICATE_HOUSEHOLD_ID") + keys = pd.MultiIndex.from_arrays([raw["income_year"], raw["H_SEQ"]]) + _require(keys.is_unique, "DUPLICATE_NATIVE_KEY") + public, lower = raw["HPUBLIC"], raw["HLORENT"] + interview = raw["H_HHTYPE"] == 1 + noninterview = raw["H_HHTYPE"] >= 2 + known = interview & np.isin(raw["H_TENURE"], [2, 3]) + excluded = noninterview | (interview & (raw["H_TENURE"] == 1)) + scope_unknown = ~(known | excluded) + context_conflict = (interview & (raw["HRHTYPE"] == 0)) | ( + noninterview & ((raw["HRHTYPE"] > 0) | (raw["H_TENURE"] > 0)) + ) + parent_conflict = (lower > 0) & (public != 2) + public_alloc_conflict = (raw["I_HPUBLI"] == 1) & ((public == 0) | excluded) + lower_alloc_conflict = (raw["I_HLOREN"] == 1) & ( + (lower == 0) | (public != 2) | excluded + ) + conflicts = ( + ((excluded & ((public > 0) | (lower > 0))).astype("uint8")) + | (parent_conflict.astype("uint8") * 2) + | (public_alloc_conflict.astype("uint8") * 4) + | (lower_alloc_conflict.astype("uint8") * 8) + | (context_conflict.astype("uint8") * 16) + ) + status = np.zeros(n, dtype="uint8") + route = np.zeros(n, dtype="uint8") + for mask, state, path in ( + (excluded, 3, 4), + (known & (public == 1) & (lower == 0), 1, 1), + (known & (public == 2) & (lower == 1), 1, 2), + (known & (public == 2) & (lower == 2), 2, 3), + ): + status[mask], route[mask] = state, path + status[conflicts > 0] = 4 + route[conflicts > 0] = 0 + unknown = (known & (public == 0)).astype("uint8") + unknown |= (known & (public == 2) & (lower == 0)).astype("uint8") * 2 + unknown |= scope_unknown.astype("uint8") * 4 + quality = [] + for answer, flag, invalid in ( + (public, raw["I_HPUBLI"], public_alloc_conflict), + (lower, raw["I_HLOREN"], lower_alloc_conflict), + ): + q = np.where(answer > 0, np.where(flag == 1, 2, 1), 0).astype("uint8") + q[invalid] = 3 + quality.append(q) + return { + "status": status, + "route": route, + "receipt_valid": np.isin(status, [1, 2]).astype("uint8"), + "group_quarters": np.where( + raw["HRHTYPE"] == 0, 2, np.isin(raw["HRHTYPE"], [9, 10]) + ).astype("uint8"), + "conflicts": conflicts, + "unknown_reasons": unknown, + "public_quality": quality[0], + "lower_quality": quality[1], + "HPUBLIC_valid": (public > 0).astype("uint8"), + "HLORENT_valid": (lower > 0).astype("uint8"), + **{ + name + "_zero_origin": (raw[name] == 0).astype("uint8") + for name in OBSERVED_COLUMNS + }, + } + + +@dataclass(frozen=True) +class _HousingStatus: + header: bytes + buffers: tuple[bytes, ...] + _token: InitVar[object] = None + + def __post_init__(self, _token): + _require(_token is _STATUS_TOKEN, "STATUS_CONSTRUCTOR_UNAVAILABLE") + + @property + def header_data(self) -> dict: + """Independent provenance copy; modifying it cannot change the artifact.""" + return _parse(self.header) + + def array(self, name: str) -> np.ndarray: + """Return an immutable byte-backed view in declared household order.""" + if name not in COLUMNS: + raise KeyError(name) + return np.frombuffer( + self.buffers[COLUMNS.index(name)], + dtype=" None: + """Recheck shape, actual classifier closure, and all derived evidence.""" + _require( + type(self) in (SyntheticHousingStatus, AuthenticatedHousingStatus), + "STATUS_TYPE", + ) + data = self.header_data + _require( + data["definition_sha256"] == _definition_identity(), "DEFINITION_CHANGED" + ) + n = data["household_rows"] + _require(type(n) is int and 0 < n <= MAX_HOUSEHOLDS, "ROW_COUNT") + _require( + type(self.buffers) is tuple and len(self.buffers) == len(COLUMNS), + "BUFFER_ROSTER", + ) + for name, buf in zip(COLUMNS, self.buffers, strict=True): + _require( + type(buf) is bytes + and len(buf) == n * (8 if name in RAW_COLUMNS else 1), + "BUFFER_SIZE", + ) + _require( + data["source_authentication"] + == ( + "checkpoint_bytes_verified" + if type(self) is AuthenticatedHousingStatus + else "synthetic_unverified" + ), + "STATUS_AUTHENTICATION", + ) + raw = {name: self.array(name) for name in RAW_COLUMNS} + derived = _derive(raw) + _require( + all( + derived[name].tobytes() == self.buffers[COLUMNS.index(name)] + for name in DERIVED_COLUMNS + ), + "DERIVED_EVIDENCE", + ) + if type(self) is AuthenticatedHousingStatus: + from .asec_housing_status_source import _implementation + + _require( + data["implementation"] == _implementation(), "IMPLEMENTATION_CHANGED" + ) + + @property + def content_sha256(self) -> str: + """Complete header and raw/derived buffer identity, without a large join.""" + self.validate() + digest = hashlib.sha256(b"microcosm/asec-housing-status-content/1\0") + for part in _parts(self): + digest.update(part) + return digest.hexdigest() + + +class SyntheticHousingStatus(_HousingStatus): + """Invented observations; never accepted by the authenticated attachment.""" + + +class AuthenticatedHousingStatus(_HousingStatus): + """Only the fixed-cohort source loader may issue this status evidence.""" + + +def _issue( + raw: pd.DataFrame, *, source: dict | None = None, implementation: dict | None = None +) -> _HousingStatus: + _require( + type(raw) is pd.DataFrame + and raw.columns.is_unique + and set(raw.columns) == set(RAW_COLUMNS), + "RAW_COLUMNS", + ) + _require( + all(raw[name].dtype == np.dtype("int64") for name in RAW_COLUMNS), "RAW_INT64" + ) + # Copy into immutable bytes before either validating values or deriving output. + buffers = tuple( + raw[name].to_numpy(dtype=" SyntheticHousingStatus: + """Classify explicitly invented rows; this API never mints source authority.""" + return _issue(observations) + + +def _parts(status): + yield MAGIC + yield struct.pack(" bytes: + """Serialize primitive buffers with a bounded header and transport checksum.""" + _require( + type(status) in (SyntheticHousingStatus, AuthenticatedHousingStatus), + "STATUS_TYPE", + ) + status.validate() + parts = tuple(_parts(status)) + digest = hashlib.sha256() + for part in parts: + digest.update(part) + return b"".join((*parts, digest.digest())) + + +def decode_housing_status( + payload: bytes, *, expected: _HousingStatus +) -> _HousingStatus: + """Require independently issued COMPLETE expected content before replay authority.""" + _require( + type(expected) in (SyntheticHousingStatus, AuthenticatedHousingStatus), + "EXPECTED_STATUS", + ) + expected.validate() + _require( + type(payload) is bytes + and len(MAGIC) + 4 + 32 < len(payload) <= PAYLOAD_MAX_BYTES, + "PAYLOAD_SIZE", + ) + view = memoryview(payload) + _require(payload.startswith(MAGIC), "PAYLOAD_VERSION") + size = struct.unpack_from(" str: + """Write an artifact and return its complete-content identity.""" + Path(path).write_bytes(encode_housing_status(status)) + return status.content_sha256 + + +def read_housing_status( + path: str | Path, *, expected: _HousingStatus +) -> _HousingStatus: + """Bound disk reads and require independently source-issued expected content.""" + with Path(path).open("rb") as handle: + payload = handle.read(PAYLOAD_MAX_BYTES + 1) + return decode_housing_status(payload, expected=expected) diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/asec_housing_status_source.py b/packages/microcosm-build/src/microcosm/build/us_runtime/asec_housing_status_source.py new file mode 100644 index 000000000..eb0152e31 --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/asec_housing_status_source.py @@ -0,0 +1,338 @@ +"""Fixed-cohort numeric source authentication and tax-result composition. + +This module does not calculate housing benefits. Public source authority requires +an existing verified ASEC source and privately staged exact cohort bytes. +""" + +from __future__ import annotations + +import tempfile +from collections.abc import Mapping +from dataclasses import InitVar, dataclass +from importlib import resources +from pathlib import Path + +import numpy as np +import pandas as pd + +from microcosm.frame import Frame, Weights + +from . import asec_current_money_source as money_source +from . import asec_household_observations as household_reader +from . import asec_housing_status as status_module +from .asec_current_money import MoneyRefusalError +from .asec_current_money_units import CurrentMoneyTaxUnitResult +from .asec_housing_status import ( + CONTEXT_COLUMNS, + DERIVED_COLUMNS, + OBSERVED_COLUMNS, + RAW_COLUMNS, + AuthenticatedHousingStatus, + HousingStatusRefusalError, + _json, + _parse, + _require, + _sha, +) +from .operator_boundary import assert_operator_free_source_frame + +_READ_COLUMNS = ("H_SEQ", "H_YEAR") + CONTEXT_COLUMNS + OBSERVED_COLUMNS +ATTACHED_COLUMNS = tuple("asec_housing_observed_" + name for name in DERIVED_COLUMNS) +_COMPOSE_TOKEN = object() + + +def _implementation() -> dict: + """Actual classifier, reader, source, Frame, codec and composition closure.""" + package = resources.files(__package__) + return { + "source_verification_sha256": money_source._verification_identity(), + "definition_sha256": status_module._definition_identity(), + "modules": { + name: _sha(package.joinpath(name).read_bytes()) + for name in ( + "asec_housing_status_source.py", + "asec_housing_status.py", + "asec_current_money.py", + "_asec_current_money_codec.py", + "asec_current_money_units.py", + ) + }, + } + + +def _read_cohort(path: Path) -> pd.DataFrame: + with path.open("rb") as handle: + return household_reader._read_numeric_household( + handle, columns=_READ_COLUMNS, require_same_block=True + ) + + +def load_authenticated_housing_status( + source: money_source.AuthenticatedCurrentMoneySource, + *, + cohort_paths: Mapping[int, str | Path], +) -> AuthenticatedHousingStatus: + """Join declared full cohort snapshots to the verified source's native keys. + + No caller pin override. This does not certify release or annual participation. + The source's sealed scope supplies ordered IDs/year/native keys; the freshly + authenticated cohort context must also equal the carried attachment context. + """ + try: + return _load(source, cohort_paths) + except HousingStatusRefusalError: + raise + except MoneyRefusalError as error: + raise HousingStatusRefusalError(error.reason) from None + except (OSError, ValueError, KeyError, TypeError, OverflowError): + raise HousingStatusRefusalError("SOURCE_CONTRACT_REFUSAL") from None + + +def _load(source, cohort_paths: Mapping[int, str | Path]): + _require( + type(source) is money_source.AuthenticatedCurrentMoneySource, + "AUTHENTICATED_SOURCE", + ) + source.validate() + before = _implementation() + evidence = _parse(source.source.identity) + pins = tuple((year, pin) for year, pin in evidence["cohorts"]) + _require( + isinstance(cohort_paths, Mapping) + and set(cohort_paths) == {year for year, _ in pins} + and all(type(year) is int for year in cohort_paths), + "COHORT_PATHS", + ) + # Scope tuples were sealed from the actual Frame by the source loader. + # They cannot borrow mutable pandas index/column backing arrays. + scope = source.scope + output = { + "household_id": np.asarray(scope.household_ids, dtype=np.int64), + "income_year": np.asarray(scope.household_years, dtype=np.int64), + "survey_year": np.empty(len(scope.household_ids), dtype=np.int64), + "H_SEQ": np.asarray( + [int(key) for key in scope.household_native_keys], dtype=np.int64 + ), + **{ + name: np.empty(len(scope.household_ids), dtype=np.int64) + for name in CONTEXT_COLUMNS + OBSERVED_COLUMNS + }, + } + carried = ( + source.frame.table("household") + .loc[:, ["household_id"] + ["asec_" + n for n in CONTEXT_COLUMNS]] + .copy(deep=True) + ) + carried.index = carried.index.copy(deep=True) + carried.columns = carried.columns.copy(deep=True) + _require( + np.array_equal(carried.household_id.to_numpy(), output["household_id"]), + "HOUSEHOLD_ORDER", + ) + _require( + all(carried[name].dtype == np.dtype("int64") for name in carried), + "CARRIED_INT64", + ) + joins = [] + with tempfile.TemporaryDirectory(prefix="microcosm-housing-status-") as directory: + for year, pin in pins: + staged = Path(directory) / f"cohort-{year}.h5" + money_source._stage_verified(cohort_paths[year], pin, staged) + raw = _read_cohort(staged) + staged.unlink() + _require( + not raw.H_SEQ.duplicated().any() and bool((raw.H_SEQ > 0).all()), + "NATIVE_KEY", + ) + _require(bool((raw.H_YEAR == year + 1).all()), "SURVEY_YEAR") + positions = np.flatnonzero(output["income_year"] == year) + source_positions = pd.Index(raw.H_SEQ).get_indexer( + output["H_SEQ"][positions] + ) + _require(bool((source_positions >= 0).all()), "SOURCE_KEY_COVERAGE") + joined = raw.iloc[source_positions] + for name in CONTEXT_COLUMNS: + _require( + np.array_equal( + joined[name].to_numpy(), + carried["asec_" + name].iloc[positions].to_numpy(), + ), + "CARRIED_CONTEXT", + ) + output["survey_year"][positions] = joined.H_YEAR.to_numpy() + for name in CONTEXT_COLUMNS + OBSERVED_COLUMNS: + output[name][positions] = joined[name].to_numpy() + joins.append( + { + "income_year": year, + "sha256": pin, + "source_rows": len(raw), + "joined_rows": len(positions), + "unreferenced_source_rows": len(raw) - len(positions), + } + ) + source.validate() + _require(_implementation() == before, "IMPLEMENTATION_CHANGED") + return status_module._issue( + pd.DataFrame(output, columns=RAW_COLUMNS), + source={"identity": source.source.identity.decode(), "joins": joins}, + implementation=before, + ) + + +def _owned_frame(frame: Frame) -> Frame: + # Frame copies native buffers; explicitly detach axes and weight vectors too. + # This is one complete result copy, not a partial/fabricated US population. + strata = frame.strata.copy(deep=True) + strata.index = money_source._owned_index(strata.index) + result = Frame( + { + **{entity: frame.table(entity) for entity in frame.entities}, + **{name: frame.link(name) for name in frame.links}, + }, + frame.schema, + { + entity: Weights( + frame.weights_for(entity).values, frame.weights_for(entity).kind + ) + for entity in frame.weighted_entities + }, + strata, + mass_log=frame.mass_log, + metadata=frame.metadata, + ) + money_source._detach_frame_axes(result) + return result + + +def _check_parent_status(parent, status): + _require(type(parent) is CurrentMoneyTaxUnitResult, "TAX_RESULT_REQUIRED") + _require(type(status) is AuthenticatedHousingStatus, "AUTHENTICATED_STATUS") + parent.validate() + status.validate() + _require( + status.header_data["source"]["identity"].encode() + == parent.money.bindings.spec.source.identity, + "STATUS_SOURCE_BINDING", + ) + household = parent.frame.table("household") + _require( + np.array_equal(household.household_id.to_numpy(), status.array("household_id")), + "STATUS_HOUSEHOLD_BINDING", + ) + for name in CONTEXT_COLUMNS: + _require( + np.array_equal(household["asec_" + name].to_numpy(), status.array(name)), + "STATUS_CONTEXT_BINDING", + ) + + +def validate_attached_frame( + frame: Frame, + *, + parent: CurrentMoneyTaxUnitResult, + status: AuthenticatedHousingStatus, +) -> None: + """Verify a live or checkpoint-reloaded composition against both trusted parents.""" + try: + _check_parent_status(parent, status) + _require(type(frame) is Frame, "COMPOSED_FRAME") + frame.revalidate() + assert_operator_free_source_frame(frame, label="ASEC housing status attachment") + household = frame.table("household") + _require(set(ATTACHED_COLUMNS) <= set(household), "ATTACHMENT_COLUMNS") + for name, attached in zip(DERIVED_COLUMNS, ATTACHED_COLUMNS, strict=True): + _require( + household[attached].dtype == np.dtype("uint8") + and household[attached].to_numpy().tobytes() + == status.array(name).tobytes(), + "ATTACHMENT_OUTPUT", + ) + # Drop ONLY our columns, then compare the existing full Frame signature. + recovered = _owned_frame(frame) + recovered.table("household").drop(columns=list(ATTACHED_COLUMNS), inplace=True) + _require( + money_source._frame_signature(recovered) + == parent.receipt["output_frame_sha256"], + "PARENT_FRAME_RECOVERY", + ) + parent.validate() + except HousingStatusRefusalError: + raise + except MoneyRefusalError as error: + raise HousingStatusRefusalError(error.reason) from None + except (ValueError, TypeError, KeyError, AssertionError): + raise HousingStatusRefusalError("ATTACHMENT_CONTRACT_REFUSAL") from None + + +@dataclass(frozen=True) +class HousingStatusAttachedAsec: + """Separate composed result retaining the identical sealed tax/money parents.""" + + frame: Frame + tax_result: CurrentMoneyTaxUnitResult + status: AuthenticatedHousingStatus + _receipt: bytes + _token: InitVar[object] = None + + def __post_init__(self, _token): + _require(_token is _COMPOSE_TOKEN, "ATTACHMENT_CONSTRUCTOR_UNAVAILABLE") + + @property + def money(self): + return self.tax_result.money + + @property + def receipt(self): + return _parse(self._receipt) + + def validate(self) -> None: + validate_attached_frame(self.frame, parent=self.tax_result, status=self.status) + _require( + self.receipt["housing_content_sha256"] == self.status.content_sha256 + and self.receipt["tax_receipt_sha256"] + == _sha(_json(self.tax_result.receipt)) + and self.receipt["output_frame_sha256"] + == money_source._frame_signature(self.frame), + "COMPOSED_RESULT_CHANGED", + ) + + +def attach_housing_status( + tax_result: CurrentMoneyTaxUnitResult, status: AuthenticatedHousingStatus +) -> HousingStatusAttachedAsec: + """Compose source-status evidence after actual tax reconstruction, without mutation.""" + try: + _check_parent_status(tax_result, status) + _require( + not set(ATTACHED_COLUMNS) & set(tax_result.frame.table("household")), + "EXISTING_ATTACHMENT_COLUMNS", + ) + result = _owned_frame(tax_result.frame) + _require( + money_source._frame_signature(result) + == tax_result.receipt["output_frame_sha256"], + "PARENT_CAPTURE_CHANGED", + ) + for name, output in zip(DERIVED_COLUMNS, ATTACHED_COLUMNS, strict=True): + result.table("household")[output] = status.array(name).copy() + validate_attached_frame(result, parent=tax_result, status=status) + receipt = { + "schema_version": 1, + "artifact_kind": "microcosm.asec_housing_status_attachment.v1", + "release_eligible": False, + "tax_receipt_sha256": _sha(_json(tax_result.receipt)), + "housing_content_sha256": status.content_sha256, + "parent_frame_sha256": tax_result.receipt["output_frame_sha256"], + "output_frame_sha256": money_source._frame_signature(result), + "outputs": ATTACHED_COLUMNS, + } + return HousingStatusAttachedAsec( + result, tax_result, status, _json(receipt), _token=_COMPOSE_TOKEN + ) + except HousingStatusRefusalError: + raise + except MoneyRefusalError as error: + raise HousingStatusRefusalError(error.reason) from None + except (ValueError, TypeError, KeyError, AssertionError): + raise HousingStatusRefusalError("ATTACHMENT_CONTRACT_REFUSAL") from None diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/asec_housing_universe.py b/packages/microcosm-build/src/microcosm/build/us_runtime/asec_housing_universe.py new file mode 100644 index 000000000..e67327f8d --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/asec_housing_universe.py @@ -0,0 +1,423 @@ +"""Source-specific occupied-HU evidence; never a population or release verdict.""" + +from __future__ import annotations + +import hashlib +import json +import os +import stat +import struct +from dataclasses import InitVar, dataclass +from importlib import metadata, resources +from pathlib import Path + +import numpy as np +import pandas as pd + +from .asec_housing_status import DICTIONARIES, ZERO_ORIGIN_EVIDENCE, ZERO_POLICY + +RAW_COLUMNS = ( + "household_id", + "income_year", + "H_SEQ", + "H_HHTYPE", + "H_LIVQRT", + "HRHTYPE", + "H_TENURE", +) +CONTEXT_COLUMNS = ("H_HHTYPE", "H_LIVQRT", "HRHTYPE", "H_TENURE") +DERIVED_COLUMNS = ( + "interview_scope", + "physical_unit", + "household_kind", + "tenure_subtype", + "occupied_hu", + "hu_tenure_class", + "unresolved_reasons", +) +COLUMNS = RAW_COLUMNS + DERIVED_COLUMNS +MAGIC = b"MCAHUNIV\x01" +HEADER_MAX_BYTES = 65536 +MAX_HOUSEHOLDS = 400_000 +PAYLOAD_MAX_BYTES = ( + len(MAGIC) + + 4 + + HEADER_MAX_BYTES + + MAX_HOUSEHOLDS * (8 * len(RAW_COLUMNS) + len(DERIVED_COLUMNS)) + + 32 +) +_UNIVERSE_TOKEN = object() + + +class HousingUniverseRefusalError(ValueError): + """Sanitized reason for unavailable or inconsistent source evidence.""" + + +def _require(condition: bool, reason: str) -> None: + if not condition: + raise HousingUniverseRefusalError(reason) + + +def _json(value: object) -> bytes: + return json.dumps( + value, sort_keys=True, separators=(",", ":"), allow_nan=False + ).encode() + + +def _sha(value: bytes) -> str: + return hashlib.sha256(value).hexdigest() + + +def _parse(value: bytes) -> dict: + _require(type(value) is bytes and 0 < len(value) <= HEADER_MAX_BYTES, "HEADER_SIZE") + try: + result = json.loads(value) + _require(type(result) is dict and _json(result) == value, "HEADER_CANONICAL") + return result + except (ValueError, TypeError, UnicodeError, RecursionError): + raise HousingUniverseRefusalError("HEADER_FORMAT") from None + + +def _codebook() -> dict: + return { + "interview_scope": { + "unavailable": 0, + "interview": 1, + "type_a_noninterview": 2, + "type_bc_noninterview": 3, + }, + "physical_unit": {"unavailable": 0, "housing_unit": 1, "other_unit": 2}, + "household_kind": { + "source_zero_or_not_established": 0, + "primary_household": 1, + "group_quarters": 2, + }, + "tenure_subtype": { + "niu_or_zero_unresolved": 0, + "owned_or_being_bought": 1, + "cash_renter": 2, + "no_cash_rent": 3, + }, + "occupied_hu": { + "not_established": 0, + "occupied_housing_unit": 1, + "outside_housing_unit": 2, + }, + "hu_tenure_class": { + "outside_or_unestablished_hu": 0, + "owner": 1, + "renter": 2, + "occupied_hu_unresolved_tenure": 3, + }, + "unresolved_reason_bits": { + "outside_interview_scope": 1, + "interview_zero": 2, + "quarters_zero": 4, + "household_kind_zero": 8, + "hu_quarters_gq_kind": 16, + "context_conflict": 32, + "hu_tenure_zero": 64, + "other_quarters_primary_kind": 128, + }, + } + + +def _definition_identity() -> str: + return _sha( + _json( + { + "module": _sha( + resources.files(__package__) + .joinpath("asec_housing_universe.py") + .read_bytes() + ), + "codebook": _codebook(), + "dictionaries": DICTIONARIES, + "zero_origin_evidence": ZERO_ORIGIN_EVIDENCE, + "dependencies": { + name: metadata.version(name) for name in ("numpy", "pandas") + }, + } + ) + ) + + +def _derive(raw: dict[str, np.ndarray]) -> dict[str, np.ndarray]: + n = len(raw["household_id"]) + _require(0 < n <= MAX_HOUSEHOLDS, "ROW_COUNT") + _require( + set(raw) == set(RAW_COLUMNS) + and all(v.shape == (n,) and v.dtype == np.dtype("int64") for v in raw.values()), + "RAW_INT64", + ) + for name, maximum in ( + ("H_HHTYPE", 3), + ("H_LIVQRT", 12), + ("HRHTYPE", 10), + ("H_TENURE", 3), + ): + _require( + bool(((raw[name] >= 0) & (raw[name] <= maximum)).all()), "CONTEXT_DOMAIN" + ) + _require(bool(np.isin(raw["income_year"], (2022, 2023, 2024)).all()), "INCOME_YEAR") + _require(bool((raw["H_SEQ"] > 0).all()), "NATIVE_KEY") + _require(len(np.unique(raw["household_id"])) == n, "DUPLICATE_HOUSEHOLD_ID") + _require( + pd.MultiIndex.from_arrays([raw["income_year"], raw["H_SEQ"]]).is_unique, + "DUPLICATE_NATIVE_KEY", + ) + interview = raw["H_HHTYPE"] == 1 + noninterview = raw["H_HHTYPE"] >= 2 + quarters, kind, tenure = raw["H_LIVQRT"], raw["HRHTYPE"], raw["H_TENURE"] + hu_quarters = (quarters >= 1) & (quarters <= 7) + other_quarters = quarters >= 8 + primary = (kind >= 1) & (kind <= 8) + gq = kind >= 9 + context_conflict = (interview & (kind == 0)) | ( + noninterview & ((kind > 0) | (tenure > 0)) + ) + hu = interview & hu_quarters & primary & ~context_conflict + outside = interview & other_quarters & gq & ~context_conflict + reasons = noninterview.astype("uint8") + for mask, bit in ( + (raw["H_HHTYPE"] == 0, 2), + (interview & (quarters == 0), 4), + (interview & (kind == 0), 8), + (interview & hu_quarters & gq, 16), + (context_conflict, 32), + (hu & (tenure == 0), 64), + (interview & other_quarters & primary, 128), + ): + reasons |= mask.astype("uint8") * bit + return { + "interview_scope": raw["H_HHTYPE"].astype("uint8"), + "physical_unit": np.where( + hu_quarters, 1, np.where(other_quarters, 2, 0) + ).astype("uint8"), + "household_kind": np.where(primary, 1, np.where(gq, 2, 0)).astype("uint8"), + "tenure_subtype": tenure.astype("uint8"), + "occupied_hu": np.where(hu, 1, np.where(outside, 2, 0)).astype("uint8"), + "hu_tenure_class": np.where( + hu, np.where(tenure == 1, 1, np.where(tenure >= 2, 2, 3)), 0 + ).astype("uint8"), + "unresolved_reasons": reasons, + } + + +@dataclass(frozen=True) +class _HousingUniverse: + header: bytes + buffers: tuple[bytes, ...] + _token: InitVar[object] = None + + def __post_init__(self, _token): + _require(_token is _UNIVERSE_TOKEN, "UNIVERSE_CONSTRUCTOR_UNAVAILABLE") + + @property + def header_data(self) -> dict: + return _parse(self.header) + + def array(self, name: str) -> np.ndarray: + if name not in COLUMNS: + raise KeyError(name) + return np.frombuffer( + self.buffers[COLUMNS.index(name)], + dtype=" None: + """Recompute every derived code from immutable raw evidence.""" + _require( + type(self) in (SyntheticHousingUniverse, AuthenticatedHousingUniverse), + "UNIVERSE_TYPE", + ) + data = self.header_data + try: + _require( + data["definition_sha256"] == _definition_identity(), + "DEFINITION_CHANGED", + ) + n = data["household_rows"] + _require(type(n) is int and 0 < n <= MAX_HOUSEHOLDS, "ROW_COUNT") + _require( + type(self.buffers) is tuple and len(self.buffers) == len(COLUMNS), + "BUFFER_ROSTER", + ) + for name, buf in zip(COLUMNS, self.buffers, strict=True): + _require( + type(buf) is bytes + and len(buf) == n * (8 if name in RAW_COLUMNS else 1), + "BUFFER_SIZE", + ) + authenticated = type(self) is AuthenticatedHousingUniverse + _require( + data["source_authentication"] + == ( + "checkpoint_bytes_verified" + if authenticated + else "synthetic_unverified" + ), + "UNIVERSE_AUTHENTICATION", + ) + derived = _derive({name: self.array(name) for name in RAW_COLUMNS}) + _require( + all( + derived[name].tobytes() == self.buffers[COLUMNS.index(name)] + for name in DERIVED_COLUMNS + ), + "DERIVED_EVIDENCE", + ) + if authenticated: + from .asec_housing_universe_source import _implementation + + _require( + data["implementation"] == _implementation(), + "IMPLEMENTATION_CHANGED", + ) + except (KeyError, TypeError, OverflowError): + raise HousingUniverseRefusalError("UNIVERSE_CONTRACT") from None + + @property + def content_sha256(self) -> str: + self.validate() + digest = hashlib.sha256(b"microcosm/asec-housing-universe-content/1\0") + for part in _parts(self): + digest.update(part) + return digest.hexdigest() + + +class SyntheticHousingUniverse(_HousingUniverse): + """Invented observations, with no source authority.""" + + +class AuthenticatedHousingUniverse(_HousingUniverse): + """Issued only from an authenticated current-money source.""" + + +def _issue(raw: pd.DataFrame, *, source=None, implementation=None) -> _HousingUniverse: + _require( + type(raw) is pd.DataFrame + and raw.columns.is_unique + and set(raw.columns) == set(RAW_COLUMNS), + "RAW_COLUMNS", + ) + _require( + all(raw[name].dtype == np.dtype("int64") for name in RAW_COLUMNS), "RAW_INT64" + ) + _require(0 < len(raw) <= MAX_HOUSEHOLDS, "ROW_COUNT") + buffers = tuple( + raw[name].to_numpy(dtype=" SyntheticHousingUniverse: + """Classify invented rows without granting source authority.""" + return _issue(observations) + + +def _parts(value): + yield MAGIC + yield struct.pack(" bytes: + """Canonical primitive bytes with a bounded header and transport checksum.""" + _require( + type(value) in (SyntheticHousingUniverse, AuthenticatedHousingUniverse), + "UNIVERSE_TYPE", + ) + value.validate() + payload = b"".join(_parts(value)) + return payload + hashlib.sha256(payload).digest() + + +def decode_housing_universe( + payload: bytes, *, expected: _HousingUniverse +) -> _HousingUniverse: + """Only independently issued complete expected evidence confers replay authority.""" + _require( + type(expected) in (SyntheticHousingUniverse, AuthenticatedHousingUniverse), + "EXPECTED_UNIVERSE", + ) + expected.validate() + _require( + type(payload) is bytes + and len(MAGIC) + 4 + 32 < len(payload) <= PAYLOAD_MAX_BYTES, + "PAYLOAD_SIZE", + ) + # Compare before parsing any candidate header or buffer; no alternate encoding. + _require(payload == encode_housing_universe(expected), "EXPECTED_CONTENT") + return type(expected)(expected.header, expected.buffers, _token=_UNIVERSE_TOKEN) + + +def write_housing_universe(value: _HousingUniverse, path: str | Path) -> str: + """Create an immutable artifact; never overwrite an existing path.""" + payload = encode_housing_universe(value) + try: + with Path(path).open("xb") as handle: + handle.write(payload) + except OSError: + raise HousingUniverseRefusalError("ARTIFACT_WRITE") from None + return value.content_sha256 + + +def read_housing_universe( + path: str | Path, *, expected: _HousingUniverse +) -> _HousingUniverse: + """Bound regular-file reads; no FIFO, device or candidate header parsing.""" + _require( + type(expected) in (SyntheticHousingUniverse, AuthenticatedHousingUniverse), + "EXPECTED_UNIVERSE", + ) + size = len(encode_housing_universe(expected)) + descriptor = None + try: + descriptor = os.open(path, os.O_RDONLY | os.O_NONBLOCK) + info = os.fstat(descriptor) + _require(stat.S_ISREG(info.st_mode) and info.st_size == size, "ARTIFACT_FILE") + with os.fdopen(descriptor, "rb") as handle: + descriptor = None + payload = handle.read(size + 1) + return decode_housing_universe(payload, expected=expected) + except (OSError, TypeError): + raise HousingUniverseRefusalError("ARTIFACT_READ") from None + finally: + if descriptor is not None: + os.close(descriptor) diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/asec_housing_universe_source.py b/packages/microcosm-build/src/microcosm/build/us_runtime/asec_housing_universe_source.py new file mode 100644 index 000000000..23deb4385 --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/asec_housing_universe_source.py @@ -0,0 +1,336 @@ +"""Authenticate carried ASEC HU evidence and preserve the chosen parent chain.""" + +from __future__ import annotations + +from dataclasses import InitVar, dataclass +from importlib import resources + +import numpy as np +import pandas as pd + +from microcosm.frame import Frame + +from . import asec_current_money_source as money_source +from . import asec_housing_status_source as housing_source +from . import asec_housing_universe as universe_module +from .asec_current_money import MoneyRefusalError +from .asec_current_money_units import CurrentMoneyTaxUnitResult +from .asec_housing_status import HousingStatusRefusalError +from .asec_housing_status_source import HousingStatusAttachedAsec, _owned_frame +from .asec_housing_universe import ( + CONTEXT_COLUMNS, + DERIVED_COLUMNS, + RAW_COLUMNS, + AuthenticatedHousingUniverse, + HousingUniverseRefusalError, + _json, + _parse, + _require, + _sha, +) +from .operator_boundary import assert_operator_free_source_frame + +ATTACHED_COLUMNS = tuple("asec_housing_universe_" + name for name in DERIVED_COLUMNS) +_COMPOSE_TOKEN = object() + + +def _implementation() -> dict: + """Bind new interpretation/composition without editing any old verifier roster.""" + package = resources.files(__package__) + return { + "source_verification_sha256": money_source._verification_identity(), + "definition_sha256": universe_module._definition_identity(), + "housing_implementation": housing_source._implementation(), + "modules": { + name: _sha(package.joinpath(name).read_bytes()) + for name in ("asec_housing_universe_source.py", "asec_housing_universe.py") + }, + } + + +def load_authenticated_housing_universe( + source: money_source.AuthenticatedCurrentMoneySource, +) -> AuthenticatedHousingUniverse: + """Read only the already-authenticated household attachment, with no new joins.""" + try: + _require( + type(source) is money_source.AuthenticatedCurrentMoneySource, + "AUTHENTICATED_SOURCE", + ) + source.validate() + implementation = _implementation() + scope = source.scope + table = source.frame.table("household") + columns = ("household_id",) + tuple( + "asec_" + name for name in ("H_SEQ",) + CONTEXT_COLUMNS + ) + _require( + table.columns.is_unique and set(columns) <= set(table), "CARRIED_COLUMNS" + ) + _require( + all(table[name].dtype == np.dtype("int64") for name in columns), + "CARRIED_INT64", + ) + raw = pd.DataFrame( + { + "household_id": np.asarray(scope.household_ids, dtype="int64"), + "income_year": np.asarray(scope.household_years, dtype="int64"), + "H_SEQ": np.asarray( + [int(key) for key in scope.household_native_keys], dtype="int64" + ), + **{ + name: table["asec_" + name].to_numpy(copy=True) + for name in CONTEXT_COLUMNS + }, + }, + columns=RAW_COLUMNS, + ) + _require( + np.array_equal(table.household_id.to_numpy(), raw.household_id.to_numpy()), + "HOUSEHOLD_ORDER", + ) + _require( + np.array_equal(table.asec_H_SEQ.to_numpy(), raw.H_SEQ.to_numpy()), + "NATIVE_KEY_BINDING", + ) + result = universe_module._issue( + raw, + source={ + "identity": source.source.identity.decode(), + "carried_columns": columns, + "income_year_origin": "authenticated_money_scope", + }, + implementation=implementation, + ) + source.validate() + _require(_implementation() == implementation, "IMPLEMENTATION_CHANGED") + return result + except HousingUniverseRefusalError: + raise + except MoneyRefusalError as error: + raise HousingUniverseRefusalError(error.reason) from None + except (ValueError, TypeError, KeyError, OverflowError, AssertionError, OSError): + raise HousingUniverseRefusalError("SOURCE_CONTRACT_REFUSAL") from None + + +def _tax_parent(parent) -> CurrentMoneyTaxUnitResult: + _require( + type(parent) in (CurrentMoneyTaxUnitResult, HousingStatusAttachedAsec), + "TAX_OR_HOUSING_PARENT_REQUIRED", + ) + return parent if type(parent) is CurrentMoneyTaxUnitResult else parent.tax_result + + +def _check_parent(parent, universe): + tax = _tax_parent(parent) + _require(type(universe) is AuthenticatedHousingUniverse, "AUTHENTICATED_UNIVERSE") + parent.validate() + universe.validate() + _require( + universe.header_data["source"]["identity"].encode() + == tax.money.bindings.spec.source.identity, + "UNIVERSE_SOURCE_BINDING", + ) + household = parent.frame.table("household") + _require( + household.household_id.dtype == np.dtype("int64") + and np.array_equal( + household.household_id.to_numpy(), universe.array("household_id") + ), + "UNIVERSE_HOUSEHOLD_BINDING", + ) + _require( + _parse(tax.money.header)["household_year_sha256"] + == _sha(universe.array("income_year").tobytes()), + "UNIVERSE_YEAR_BINDING", + ) + for name in ("H_SEQ",) + CONTEXT_COLUMNS: + column = household["asec_" + name] + _require( + column.dtype == np.dtype("int64") + and column.to_numpy().tobytes() == universe.array(name).tobytes(), + "UNIVERSE_CONTEXT_BINDING", + ) + + +def validate_attached_frame( + frame: Frame, + *, + parent: CurrentMoneyTaxUnitResult | HousingStatusAttachedAsec, + universe: AuthenticatedHousingUniverse, +) -> None: + """Check a whole owned parent extension; a selected child cannot inherit this authority.""" + try: + _check_parent(parent, universe) + _require(type(frame) is Frame, "COMPOSED_FRAME") + frame.revalidate() + assert_operator_free_source_frame( + frame, label="ASEC housing universe attachment" + ) + household = frame.table("household") + _require(set(ATTACHED_COLUMNS) <= set(household), "ATTACHMENT_COLUMNS") + for name, attached in zip(DERIVED_COLUMNS, ATTACHED_COLUMNS, strict=True): + column = household[attached] + _require( + column.dtype == np.dtype("uint8") + and column.to_numpy().tobytes() == universe.array(name).tobytes(), + "ATTACHMENT_OUTPUT", + ) + recovered = _owned_frame(frame) + recovered.table("household").drop(columns=list(ATTACHED_COLUMNS), inplace=True) + _require( + money_source._frame_signature(recovered) + == parent.receipt["output_frame_sha256"], + "PARENT_FRAME_RECOVERY", + ) + parent.validate() + universe.validate() + except HousingUniverseRefusalError: + raise + except MoneyRefusalError as error: + raise HousingUniverseRefusalError(error.reason) from None + except HousingStatusRefusalError: + raise HousingUniverseRefusalError("HOUSING_PARENT_REFUSAL") from None + except (ValueError, TypeError, KeyError, AssertionError): + raise HousingUniverseRefusalError("ATTACHMENT_CONTRACT_REFUSAL") from None + + +@dataclass(frozen=True) +class HousingUniverseAttachedAsec: + """Owned HU extension retaining its exact tax or housing-status parent.""" + + frame: Frame + parent: CurrentMoneyTaxUnitResult | HousingStatusAttachedAsec + universe: AuthenticatedHousingUniverse + _receipt: bytes + _token: InitVar[object] = None + + def __post_init__(self, _token): + _require(_token is _COMPOSE_TOKEN, "ATTACHMENT_CONSTRUCTOR_UNAVAILABLE") + + @property + def tax_result(self): + return _tax_parent(self.parent) + + @property + def money(self): + return self.tax_result.money + + @property + def receipt(self): + return _parse(self._receipt) + + def validate(self) -> None: + validate_attached_frame(self.frame, parent=self.parent, universe=self.universe) + _require( + self.receipt == _receipt(self.parent, self.universe, self.frame), + "COMPOSED_RESULT_CHANGED", + ) + + +def _receipt(parent, universe, frame) -> dict: + return { + "schema_version": 1, + "artifact_kind": "microcosm.asec_housing_universe_attachment.v1", + "release_eligible": False, + "parent_kind": type(parent).__name__, + "parent_receipt_sha256": _sha(_json(parent.receipt)), + "tax_receipt_sha256": _sha(_json(_tax_parent(parent).receipt)), + "money_header_sha256": _sha(parent.money.header), + "universe_content_sha256": universe.content_sha256, + "parent_frame_sha256": parent.receipt["output_frame_sha256"], + "output_frame_sha256": money_source._frame_signature(frame), + "outputs": list(ATTACHED_COLUMNS), + } + + +def attach_housing_universe( + parent: CurrentMoneyTaxUnitResult | HousingStatusAttachedAsec, + universe: AuthenticatedHousingUniverse, +) -> HousingUniverseAttachedAsec: + """Preserve all chosen parent columns, including student and subsidy evidence.""" + try: + _check_parent(parent, universe) + _require( + not set(ATTACHED_COLUMNS) & set(parent.frame.table("household")), + "EXISTING_ATTACHMENT_COLUMNS", + ) + frame = _owned_frame(parent.frame) + _require( + money_source._frame_signature(frame) + == parent.receipt["output_frame_sha256"], + "PARENT_CAPTURE_CHANGED", + ) + for name, alias in zip(DERIVED_COLUMNS, ATTACHED_COLUMNS, strict=True): + frame.table("household")[alias] = universe.array(name).copy() + validate_attached_frame(frame, parent=parent, universe=universe) + receipt = _receipt(parent, universe, frame) + result = HousingUniverseAttachedAsec( + frame, parent, universe, _json(receipt), _token=_COMPOSE_TOKEN + ) + result.validate() + return result + except HousingUniverseRefusalError: + raise + except MoneyRefusalError as error: + raise HousingUniverseRefusalError(error.reason) from None + except HousingStatusRefusalError: + raise HousingUniverseRefusalError("HOUSING_PARENT_REFUSAL") from None + except (ValueError, TypeError, KeyError, AssertionError): + raise HousingUniverseRefusalError("ATTACHMENT_CONTRACT_REFUSAL") from None + + +def verify_housing_universe_rows( + household_table: pd.DataFrame, + universe: AuthenticatedHousingUniverse, + *, + lineage_ids: np.ndarray, +) -> None: + """Verify source-derived cells of subsets/clones, without issuing Frame authority. + + ``lineage_ids`` are the caller's original source household IDs, repeated for + clones. Their custody/assembly provenance is a separate obligation. This + checks carried H_SEQ/context and derived values, not a selected Frame receipt. + """ + try: + _require( + type(universe) is AuthenticatedHousingUniverse, "AUTHENTICATED_UNIVERSE" + ) + universe.validate() + _require( + type(household_table) is pd.DataFrame and household_table.columns.is_unique, + "HOUSEHOLD_TABLE", + ) + _require( + type(lineage_ids) is np.ndarray + and lineage_ids.dtype == np.dtype("int64") + and lineage_ids.shape == (len(household_table),), + "LINEAGE_INT64", + ) + positions = pd.Index(universe.array("household_id")).get_indexer(lineage_ids) + _require(bool((positions >= 0).all()), "LINEAGE_COVERAGE") + if "household_source_id" in household_table: + carried_ids = household_table.household_source_id + _require( + carried_ids.dtype == np.dtype("int64") + and np.array_equal(carried_ids.to_numpy(), lineage_ids), + "LINEAGE_BINDING", + ) + for name, column, dtype in ( + *((n, "asec_" + n, "int64") for n in ("H_SEQ",) + CONTEXT_COLUMNS), + *( + (n, c, "uint8") + for n, c in zip(DERIVED_COLUMNS, ATTACHED_COLUMNS, strict=True) + ), + ): + values = household_table[column] + _require( + values.dtype == np.dtype(dtype) + and np.array_equal(values.to_numpy(), universe.array(name)[positions]), + "ROW_EVIDENCE", + ) + universe.validate() + except HousingUniverseRefusalError: + raise + except (ValueError, TypeError, KeyError, AssertionError): + raise HousingUniverseRefusalError("ROW_CONTRACT_REFUSAL") from None diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/asec_income_observations.py b/packages/microcosm-build/src/microcosm/build/us_runtime/asec_income_observations.py new file mode 100644 index 000000000..0c0283d17 --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/asec_income_observations.py @@ -0,0 +1,574 @@ +"""Closed PAW source observations, independent of the sealed 33-field money body. + +Only exact source reconstruction issues this authority. Candidate replay compares +canonical expected bytes before parsing any candidate. No Frame columns are added. +""" + +from __future__ import annotations + +import csv +import hashlib +import os +import stat +import struct +import tempfile +from collections.abc import Mapping +from dataclasses import InitVar, dataclass +from importlib import metadata, resources +from pathlib import Path + +import numpy as np +import pandas as pd + +from . import asec_current_money_source as legacy +from . import asec_person_income_source as restoration +from ._asec_current_money_codec import current_money_content_sha256 +from .asec_current_money import ( + RESOURCE_PINS, + RESTORED_SOURCE_KIND, + MoneyRefusalError, + ReadyCurrentMoney, + _json, + _parse, + _require, + _sha, + _validate_money, +) +from .asec_student_controls import _snapshot +from .education_assistance_source import ASEC_EDUCATION_ASSISTANCE_ARCHIVES + +MAGIC = b"MCAINCOB\x01" +FILENAME = "asec_income_observations.bin" +ARTIFACT_KIND = "microcosm.us.asec_income_observations.v1" +_HEADER_MAX = 65536 +_MAX_PERSONS = 600_000 +INCOME_PAYLOAD_MAX_BYTES = len(MAGIC) + 4 + _HEADER_MAX + 8 * 8 * _MAX_PERSONS + 32 +COORDINATES = ("person_id", "income_year", "source_household_id", "A_LINENO", "A_AGE") +COLUMNS = COORDINATES + ("PAW_YN", "PAW_VAL", "PAW_VAL_2024_price") +COLUMN_DTYPES = { + name: "little_endian_float64" if name == COLUMNS[-1] else "little_endian_int64" + for name in COLUMNS +} +_READ_COLUMNS = ("PERIDNUM", "PH_SEQ", "A_LINENO", "A_AGE", "PAW_VAL", "PAW_YN") +_MEMBER_PINS = tuple( + (year, p.member, p.zip_sha256, p.member_sha256, p.rows, p.member_size_bytes) + for year, p in sorted(ASEC_EDUCATION_ASSISTANCE_ARCHIVES.items()) +) +RESOURCE = "asec_income_observations_v1.json" +RESOURCE_SHA256 = "6e37855ddd6642d02231efc664644029ed6131885eb743a00197261bc5a6a51f" +ZERO_CLASSES = ( + "observed_amount", + "niu_below_receipt_age", + "reported_receipt_zero_amount", + "reported_no_receipt", + "niu_in_age_universe", +) +_TOKEN = object() +_CSV_DIALECT = "excel" +_CSV_ENCODING = "utf-8" +_PRICE_TARGET = "174.4" +_PRICE_YEARS = (2022, 2023, 2024) + + +def parser_profile() -> dict: + """Read the effective CSV parser limits and dialect, without changing them. + + ``csv.field_size_limit`` is process state that silently changes which member + files parse. Binding it into the source identity makes any change to it a + different artifact identity instead of an invisible reparse. + """ + # _read_member uses csv.reader's built-in default, which is independent of + # the mutable named-dialect registry. Inspect that same reader's dialect. + dialect = csv.reader(()).dialect + return { + "header_engine": "stdlib_csv_reader", + "table_engine": "pandas_read_csv", + "dialect": _CSV_DIALECT, + "encoding": _CSV_ENCODING, + "newline": "", + "delimiter": dialect.delimiter, + "quotechar": dialect.quotechar, + "doublequote": bool(dialect.doublequote), + "skipinitialspace": bool(dialect.skipinitialspace), + "strict": bool(dialect.strict), + "quoting": int(dialect.quoting), + "csv.field_size_limit": csv.field_size_limit(), + } + + +def price_factors(contract) -> tuple[tuple[int, str, str], ...]: + """The single restatement-factor derivation, shared with graph transport.""" + return tuple( + ( + year, + f"{_PRICE_TARGET}/{contract['price']['cells'][str(year)]}", + struct.pack( + " 0).all()) + and bool((table.A_LINENO > 0).all()), + "INCOME_MEMBER_COORDINATES", + ) + _validate_observations( + table.A_AGE.to_numpy(), table.PAW_YN.to_numpy(), table.PAW_VAL.to_numpy() + ) + return table + + +def _validate_observations( + age: np.ndarray, receipt_code: np.ndarray, nominal: np.ndarray +): + _require( + all( + type(a) is np.ndarray + and a.dtype == np.dtype("int64") + and a.ndim == 1 + and a.shape == nominal.shape + for a in (age, receipt_code, nominal) + ), + "INCOME_OBSERVATION_ARRAYS", + ) + _require( + bool(((nominal >= 0) & (nominal <= 99999)).all()), + "INCOME_CODE_DOMAIN", + "PAW_VAL", + ) + _require( + bool(np.isin(receipt_code, [0, 1, 2]).all()), "INCOME_CODE_DOMAIN", "PAW_YN" + ) + _require( + bool((receipt_code[nominal != 0] == 1).all()) + and bool((nominal[receipt_code != 1] == 0).all()) + and bool((receipt_code[age < 15] == 0).all()) + and bool((nominal[age < 15] == 0).all()), + "INCOME_SOURCE_UNIVERSE", + ) + + +def _restate_amounts(nominal, years, factors): + """The single PAW restatement path, also used to verify graph transport.""" + amounts = nominal.astype("= 15) & (receipt_code == 1)] = 2 + result[zero & (age >= 15) & (receipt_code == 2)] = 3 + result[zero & (age >= 15) & (receipt_code == 0)] = 4 + return result + + +@dataclass(frozen=True) +class AuthenticatedIncomeObservations: + """Immutable source authority; only the complete reconstructed join issues it.""" + + _header: bytes + _body: bytes + _token: InitVar[object] = None + + def __post_init__(self, _token): + _require(_token is _TOKEN, "INCOME_CONSTRUCTOR_UNAVAILABLE") + self.validate() + + @property + def receipt(self): + return _parse(self._header) + + @property + def content_sha256(self): + return _sha(self._header + self._body) + + def validate(self): + _require( + type(self._header) is bytes and 0 < len(self._header) <= _HEADER_MAX, + "INCOME_HEADER_SIZE", + ) + header = self.receipt + _require( + set(header) == _HEADER_KEYS and _json(header) == self._header, + "INCOME_HEADER_SCHEMA", + ) + rows = header["rows"] + _require(type(rows) is int and 0 < rows <= _MAX_PERSONS, "INCOME_ROWS") + _require( + header["schema_version"] == 1 + and header["artifact_kind"] == ARTIFACT_KIND + and type(self._body) is bytes + and len(self._body) == rows * len(COLUMNS) * 8 + and _sha(self._body) == header["body_sha256"] + and header["columns"] == list(COLUMNS) + and header["column_dtypes"] == COLUMN_DTYPES + and _json(header["implementation"]) == _json(_implementation()) + and header["column_sha256"] + == { + name: _sha(self._body[i * rows * 8 : (i + 1) * rows * 8]) + for i, name in enumerate(COLUMNS) + }, + "INCOME_CONTENT_CHANGED", + ) + + def array(self, name): + self.validate() + _require(name in COLUMNS, "INCOME_COLUMN") + n = self.receipt["rows"] + return np.frombuffer( + self._body, + dtype="= 0).all()) and len(np.unique(indices)) == rows, + "INCOME_KEY_COVERAGE", + ) + joined = raw.iloc[indices] + for name, raw_name in ( + ("source_household_id", "PH_SEQ"), + ("A_LINENO", "A_LINENO"), + ("A_AGE", "A_AGE"), + ): + _require( + np.array_equal( + output[name][positions], joined[raw_name].to_numpy() + ), + "INCOME_NATIVE_KEY_OR_AGE", + ) + incumbent = person.PAW_VAL.iloc[positions] + known = ~incumbent.isna().to_numpy() + observed = joined.PAW_VAL.to_numpy(dtype=np.int64) + _require( + np.array_equal( + incumbent.to_numpy(dtype=np.float64, na_value=np.nan)[known], + observed[known], + ), + "INCOME_INCUMBENT_CONFLICT", + "PAW_VAL", + ) + for name in ("PAW_YN", "PAW_VAL"): + output[name][positions] = joined[name].to_numpy(dtype=np.int64) + classes = zero_origin_classes( + output["A_AGE"][positions], output["PAW_YN"][positions], observed + ) + counts = { + name: int((classes == i).sum()) for i, name in enumerate(ZERO_CLASSES) + } + joins.append( + { + "income_year": year, + "survey_year": year + 1, + "member": member, + "archive_sha256": archive_pin, + "member_sha256": pin, + "source_rows": rows, + "joined_rows": len(positions), + "unreferenced_source_rows": 0, + "incumbent_compared_rows": {"PAW_VAL": int(known.sum())}, + "incumbent_conflicts": 0, + "native_key_or_age_conflicts": 0, + "zero_origin_counts": counts, + "paw_yn_zero_at_age_15_plus": counts["niu_in_age_universe"], + } + ) + output["PAW_VAL_2024_price"] = _restate_amounts( + output["PAW_VAL"], years, ready.bindings.spec.factors + ) + _parent(source, ready) + _require(_json(_implementation()) == _json(before), "INCOME_IMPLEMENTATION_CHANGED") + buffers = [ + output[name] + .astype(" None: + if not condition: + raise HouseholdWeightSourceError(reason) + + +def _sha(payload: bytes) -> str: + return hashlib.sha256(payload).hexdigest() + + +def _issue_capsule(source: object, payload: bytes) -> str: + """Seal this identity outside writable capsule state without retaining it.""" + _require( + type(payload) is bytes and len(payload) <= _MAX_PAYLOAD_BYTES, "SOURCE_BYTES" + ) + digest = _sha(payload) + identity = id(source) + + def discard(reference): + entry = _CAPSULE_ISSUANCE.get(identity) + if entry is not None and entry[0] is reference: + del _CAPSULE_ISSUANCE[identity] + + # WeakKeyDictionary would merge distinct, value-equal frozen dataclasses. + _CAPSULE_ISSUANCE[identity] = (weakref.ref(source, discard), digest) + return digest + + +def _checked_capsule_payload(source: object) -> bytes: + """Return exactly the checked bytes; copies and deserialization are unissued.""" + entry = _CAPSULE_ISSUANCE.get(id(source)) + _require(entry is not None and entry[0]() is source, "SOURCE_MUTATION") + payload = getattr(source, "payload", None) + _require( + type(payload) is bytes + and len(payload) <= _MAX_PAYLOAD_BYTES + and _sha(payload) == entry[1], + "SOURCE_MUTATION", + ) + return payload + + +def _member_path_snapshot( + member_paths: Mapping[int, str | Path], + pins: tuple[_origin.AsecNativeMemberPin, ...], +) -> dict[int, Path]: + """Bound the closed cohort snapshot before allocation and each value lookup.""" + _require(isinstance(member_paths, Mapping), "MEMBER_PATHS") + try: + count = len(member_paths) + _require(0 < count <= len(pins), "MEMBER_PATHS") + allowed = {pin.income_year for pin in pins} + paths = {} + for year in member_paths: + _require( + len(paths) < count + and type(year) is int + and year in allowed + and year not in paths, + "MEMBER_PATHS", + ) + path = member_paths[year] + _require(isinstance(path, (str, Path)), "MEMBER_PATHS") + paths[year] = Path(path) + _require(len(paths) == count == len(member_paths), "MEMBER_PATHS") + except (TypeError, ValueError, RuntimeError, KeyError, OverflowError): + raise HouseholdWeightSourceError("MEMBER_PATHS") from None + return paths + + +def _encode(value: object) -> bytes: + """Canonical JSON with a bound during encoding, not after a huge allocation.""" + encoder = json.JSONEncoder( + ensure_ascii=True, allow_nan=False, sort_keys=True, separators=(",", ":") + ) + result = bytearray() + for fragment in encoder.iterencode(value): + encoded = fragment.encode("ascii") + _require(len(result) + len(encoded) <= _MAX_PAYLOAD_BYTES, "ENCODING_SIZE") + result.extend(encoded) + return bytes(result) + + +def _registry() -> tuple[_origin.AsecNativeMemberPin, ...]: + pins = tuple(_origin._ASEC_MEMBER_PINS) + _require(0 < len(pins) <= 3, "MEMBER_REGISTRY") + try: + for pin in pins: + _origin.AsecNativeMemberPin(**asdict(pin)) + _require( + pin.income_year in (2022, 2023, 2024) + and pin.size_bytes <= _MAX_MEMBER_BYTES + and pin.rows <= _MAX_ROWS, + "MEMBER_REGISTRY", + ) + _require( + len({p.income_year for p in pins}) == len(pins) + and len({p.canonical_member_id for p in pins}) == len(pins), + "MEMBER_REGISTRY", + ) + except (TypeError, ValueError, AttributeError): + raise HouseholdWeightSourceError("MEMBER_REGISTRY") from None + return tuple(sorted(pins, key=lambda pin: pin.income_year)) + + +def _implementation() -> dict: + """Bind the live owner, registry, private capture, codec and field policy.""" + _require(_csv_builtin.csv_reader_bound(csv), "SOURCE_CSV_READER_CHANGED") + parser_binary = getattr(_csv, "__file__", None) + if parser_binary is None: + parser_binary = ( + Path(sysconfig.get_config_var("LIBDIR")) + / sysconfig.get_config_var("LDLIBRARY") + if sysconfig.get_config_var("Py_ENABLE_SHARED") + else Path(sys.executable) + ) + return { + "schema": _SCHEMA, + "modules": { + name: _sha(resources.files(__package__).joinpath(name).read_bytes()) + for name in ( + "asec_original_household_weights.py", + "source_csv_builtin.py", + "native_household_origin.py", + "asec_student_controls.py", + "education_assistance_source.py", + ) + }, + "origin_codec_sha256": _sha( + resources.files("microcosm.build.cd_benchmark") + .joinpath("origin.py") + .read_bytes() + ), + "python": sys.version, + "csv_python_sha256": _sha(Path(csv.__file__).read_bytes()), + "csv_native_origin": _csv.__spec__.origin, + "csv_native_provider": Path(parser_binary).name, + "csv_native_sha256": _sha(Path(parser_binary).read_bytes()), + "registry": [asdict(pin) for pin in _registry()], + "fields": _FIELD_CONTRACT, + "dictionaries": _DICTIONARIES, + "limits": [ + _MAX_MEMBER_BYTES, + _MAX_ROWS, + _MAX_RECORD_CHARS, + _MAX_FIELD_CHARS, + _MAX_PAYLOAD_BYTES, + ], + } + + +class _DigestReader(io.RawIOBase): + """Bound and hash the actual captured bytes consumed by UTF-8 decoding.""" + + def __init__(self, raw: io.FileIO, size: int): + self.raw = raw + self.size = size + self.count = 0 + self.digest = hashlib.sha256() + + def readable(self) -> bool: + return True + + def readinto(self, buffer) -> int: + limit = min(len(buffer), self.size - self.count + 1) + count = self.raw.readinto(memoryview(buffer)[:limit]) + self.count += count + _require(self.count <= self.size, "CAPTURE_SIZE") + self.digest.update(memoryview(buffer)[:count]) + return count + + +class _RecordLines: + """Limit the complete logical CSV record, including embedded newlines.""" + + def __init__(self, text: io.TextIOWrapper): + self.text = text + self.characters = 0 + + def __iter__(self): + return self + + def __next__(self) -> str: + line = self.text.readline(_MAX_RECORD_CHARS - self.characters + 1) + if not line: + raise StopIteration + self.characters += len(line) + _require(self.characters <= _MAX_RECORD_CHARS, "MEMBER_RECORD_SIZE") + return line + + +def _record_fields(tokens: Sequence[str]) -> dict: + native_token, household_type, weight_token = tokens + _require(all(len(t) <= _MAX_FIELD_CHARS for t in tokens), "MEMBER_FIELD_SIZE") + _require( + re.fullmatch(r"[0-9]+", native_token) is not None + and 1 <= int(native_token) <= 99999, + "MEMBER_NATIVE_KEY", + ) + if household_type == "1": + universe = "in" + elif re.fullmatch(r"(?:0|[2-9])", household_type): + # Establish only that the printed weight predicate is false, not the + # meaning/validity of any other household classification. + universe = "outside" + else: + universe = "unresolved" + units = None + if weight_token == "": + state = "missing" + elif re.fullmatch(r"[0-9]+", weight_token) is None: + state = "malformed" + else: + units = int(weight_token) + state = "integer" if units <= 999999999 else "out_of_range" + return { + "H_SEQ": native_token, + "native_household_id": int(native_token), + "H_HHTYPE": household_type, + "HSUP_WGT": weight_token, + "universe_status": universe, + "weight_token_status": state, + "weight_integer_units": units, + "weight_denominator": 100 if units is not None else None, + "weight_status": ( + "valid_in_universe" + if universe == "in" and state == "integer" + else "unresolved_weight" + if universe == "in" + else "outside_universe" + if universe == "outside" + else "unresolved_universe" + ), + } + + +def _regular_opener(path, flags): + fd = os.open(path, flags | os.O_NONBLOCK) + try: + _require(stat.S_ISREG(os.fstat(fd).st_mode), "CAPTURE_KIND") + except BaseException: + os.close(fd) + raise + return fd + + +def _read_capture(capture: Path, pin: _origin.AsecNativeMemberPin) -> list[dict]: + csv_reader = _csv_builtin.capture_csv_reader(csv) + _require(csv_reader is not None, "SOURCE_CSV_READER_CHANGED") + member = SourceMember(pin.canonical_member_id, pin.member_sha256) + result, seen = [], set() + with open(capture, "rb", buffering=0, opener=_regular_opener) as raw: + before = os.fstat(raw.fileno()) + _require(before.st_size == pin.size_bytes, "CAPTURE_SIZE") + digest_reader = _DigestReader(raw, pin.size_bytes) + with io.TextIOWrapper( + io.BufferedReader(digest_reader), encoding="utf-8-sig", newline="" + ) as text: + lines = _RecordLines(text) + reader = csv_reader(lines, strict=True) + header = next(reader, []) + _require( + bool(header) + and all(header) + and len(set(header)) == len(header) + and set(_COLUMNS) <= set(header), + "MEMBER_HEADER", + ) + positions = [header.index(column) for column in _COLUMNS] + while True: + lines.characters = 0 + try: + row = next(reader) + except StopIteration: + break + _require(len(row) == len(header), "MEMBER_ROW_WIDTH") + _require(len(result) < pin.rows, "MEMBER_ROWS") + record = _record_fields([row[i] for i in positions]) + native = record["native_household_id"] + _require(native not in seen, "MEMBER_DUPLICATE_KEY") + seen.add(native) + record.update( + income_year=pin.income_year, + survey_year=pin.survey_year, + member_id=pin.canonical_member_id, + member_sha256=pin.member_sha256, + member_row_1based=len(result) + 1, + origin_key=origin_key( + AsecHouseholdOrigin(member, pin.income_year, native) + ), + ) + result.append(record) + _require(len(result) == pin.rows, "MEMBER_ROWS") + _require( + digest_reader.count == pin.size_bytes + and digest_reader.digest.hexdigest() == pin.member_sha256, + "CAPTURE_CHANGED", + ) + after = os.fstat(raw.fileno()) + _require( + all( + getattr(before, attribute) == getattr(after, attribute) + for attribute in ( + "st_dev", + "st_ino", + "st_size", + "st_mtime_ns", + "st_ctime_ns", + ) + ), + "CAPTURE_CHANGED", + ) + return result + + +@dataclass(frozen=True, slots=True, weakref_slot=True) +class AuthenticatedAsecHouseholdWeights: + """Reader-issued immutable source evidence, not a population-weight vector.""" + + payload: bytes + _token: InitVar[object] = None + _issued_sha256: str = field(init=False, repr=False) + + def __post_init__(self, _token: object) -> None: + _require(_token is _TOKEN, "SOURCE_CONSTRUCTOR") + object.__setattr__(self, "_issued_sha256", _issue_capsule(self, self.payload)) + + @property + def document(self) -> dict: + """Fresh defensive decoding, checked against the live source producer.""" + return verify_asec_household_weights_source(self) + + def original_weight_fractions( + self, native_roster: Sequence[tuple[int, int]] + ) -> tuple[Fraction, ...]: + """Exact weights for known in-universe keys; no Frame binding or stamping.""" + doc = self.document + _require( + isinstance(native_roster, (list, tuple)) + and 0 < len(native_roster) <= _MAX_ROWS, + "ANCHOR_ROSTER", + ) + keys = [] + for key in native_roster: + _require( + isinstance(key, (list, tuple)) + and len(key) == 2 + and all(type(value) is int for value in key), + "ANCHOR_ROSTER", + ) + keys.append(tuple(key)) + _require(len(set(keys)) == len(keys), "ANCHOR_ROSTER") + records = { + (row["income_year"], row["native_household_id"]): row + for row in doc["records"] + } + values = [] + for key in keys: + _require(key in records, "ANCHOR_UNKNOWN") + row = records[key] + _require(row["weight_status"] == "valid_in_universe", "ANCHOR_UNRESOLVED") + values.append(Fraction(row["weight_integer_units"], 100)) + return tuple(values) + + +def verify_asec_household_weights_source( + source: AuthenticatedAsecHouseholdWeights, +) -> dict: + """Validate reader-issued evidence; arbitrary JSON/digests cannot issue it.""" + _require(type(source) is AuthenticatedAsecHouseholdWeights, "SOURCE_TYPE") + payload = _checked_capsule_payload(source) + doc = json.loads(payload) + _require(_encode(doc["producer"]) == _encode(_implementation()), "PRODUCER_CHANGED") + return doc + + +def load_authenticated_asec_household_weights( + member_paths: Mapping[int, str | Path], *, candidate: bytes | None = None +) -> AuthenticatedAsecHouseholdWeights: + """Reconstruct complete requested cohorts from closed original member pins. + + Mapping keys are income/source years, not survey years. Any nonempty subset + of the closed cohorts is explicit; every requested original member is read + completely. Optional candidate bytes must equal the independent canonical + reconstruction. There is no public pin override or candidate-only admission. + """ + pins = _registry() + paths = _member_path_snapshot(member_paths, pins) + _require( + candidate is None + or (type(candidate) is bytes and len(candidate) <= _MAX_PAYLOAD_BYTES), + "CANDIDATE_SIZE", + ) + producer = _implementation() + _require(producer["registry"] == [asdict(pin) for pin in pins], "PRODUCER_CHANGED") + implementation = _encode(producer) + selected = [pin for pin in pins if pin.income_year in paths] + records = [] + with tempfile.TemporaryDirectory(prefix="asec-original-household-weights-") as tmp: + for pin in selected: + capture = Path(tmp) / f"{pin.income_year}.csv" + try: + digest = _capture_owner._snapshot( + paths[pin.income_year], capture, size=pin.size_bytes + ) + except (OSError, ValueError, TypeError): + raise HouseholdWeightSourceError("MEMBER_CAPTURE") from None + _require(digest == pin.member_sha256, "MEMBER_SHA256") + try: + records.extend(_read_capture(capture, pin)) + except HouseholdWeightSourceError: + raise + except (OSError, UnicodeError, csv.Error, ValueError, OverflowError): + raise HouseholdWeightSourceError("MEMBER_ENCODING") from None + _require(_encode(_implementation()) == implementation, "PRODUCER_CHANGED") + payload = _encode( + { + "schema": _SCHEMA, + "producer": json.loads(implementation), + "members": [asdict(pin) for pin in selected], + "fields": _FIELD_CONTRACT, + "dictionary_authorities": [ + { + "survey_year": year, + "pdf_page_1based": page, + "printed_page": "6A-1", + "sha256": digest, + "url": "https://www2.census.gov/programs-surveys/cps/datasets/" + f"{year}/march/asec{year}_ddl_pub_full.pdf", + } + for year, page, digest in _DICTIONARIES + ], + "native_roster_sha256": _sha( + _encode([[r["income_year"], r["native_household_id"]] for r in records]) + ), + "projection_sha256": _sha(_encode(records)), + "records": records, + "source_authenticated": True, + "population_binding_authenticated": False, + "release_eligible": False, + "period_and_coverage_policy": "not_selected_by_this_source_owner", + } + ) + _require(candidate is None or candidate == payload, "CANDIDATE_MISMATCH") + return AuthenticatedAsecHouseholdWeights(payload, _token=_TOKEN) diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/asec_person_coverage_source.py b/packages/microcosm-build/src/microcosm/build/us_runtime/asec_person_coverage_source.py new file mode 100644 index 000000000..eef9b20ae --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/asec_person_coverage_source.py @@ -0,0 +1,299 @@ +"""Bounded, non-authoritative original-CSV ASEC PRPERTYP projection. + +Only invented cohorts have been exercised. This reader neither authenticates +source bytes nor attaches observations to a Frame. Genuine use requires a +separately reviewed authentication/binding wrapper and source-read approval. +""" + +from __future__ import annotations + +import csv +import hashlib +import io +import json +import os +import re +import stat +from collections.abc import Mapping +from pathlib import Path + +import pandas as pd + +from . import source_csv_builtin as _csv_builtin + +PROTOCOL = "microcosm.asec_person_coverage_source.v1" +# Existing ASEC adapters use income/source year as the cohort key. This is an +# identity convention only, not a decision about the observation's period. +SOURCE_YEARS = (2022, 2023, 2024) +KEYS = ("source_year", "PERIDNUM") +CROSSCHECKS = ("source_household_id", "A_LINENO", "A_AGE") +ROSTER_COLUMNS = KEYS + CROSSCHECKS +READ_COLUMNS = ("PERIDNUM", "PH_SEQ", "A_LINENO", "A_AGE", "PRPERTYP") +MAX_PERSONS = 600_000 +MAX_MEMBER_BYTES = 1_000_000_000 +_INT64_MAX = 2**63 - 1 + + +class CoverageSourceRefusalError(ValueError): + """Static refusal codes only; no path, key, row or source token.""" + + +def _require(condition: bool, reason: str) -> None: + if not condition: + raise CoverageSourceRefusalError(reason) + + +def coverage_field_contract() -> dict: + """Fresh metadata from the public dictionaries reviewed on 2026-09-07. + + The review's receipt.json and field-pages.json establish these definitions; + their document hashes are metadata, not original CSV authentication pins. + """ + documents = ( + (2023, 26, "66bd6e3fe516233ab63b75c60573451222b3b3d3235d61cfed96f64608de2117"), + (2024, 27, "761c67ea53f5c3264329b3e9ddbdd802826ba4a859122b8a8a2f29ab85b3a840"), + (2025, 28, "5cb80973326ef8b625fbaae70d80b0c641ce5d2b3911abd2fb4427abd5908a6f"), + ) + return { + "protocol": PROTOCOL, + "survey": "CPS ASEC", + "grain": "person", + "field": "PRPERTYP", + "concept_as_printed": "Type of person record recode", + "universe_as_printed": "All Persons", + "printed_length": 1, + "printed_position": 155, + "printed_range": [-4, 3], + "codes": { + "1": "Child household member", + "2": "Adult civilian household member", + "3": "Adult Armed Forces household member", + }, + "unlabelled_in_range_codes": [-4, -3, -2, -1, 0], + "blank_meaning": "unresolved", + "authority": [ + { + "survey_year": year, + "source_year": year - 1, + "url": "https://www2.census.gov/programs-surveys/cps/datasets/" + f"{year}/march/asec{year}_ddl_pub_full.pdf", + "sha256": digest, + "pdf_page_1based": page, + "printed_page": "6C-7", + } + for year, page, digest in documents + ], + "join_keys": list(KEYS), + "crosschecks": dict(zip(CROSSCHECKS, READ_COLUMNS[1:4], strict=True)), + "roster_scope": "all_person_rows_in_each_of_the_three_source_cohorts", + "period_harmonized": False, + "cross_survey_coverage_equivalence_established": False, + } + + +def _state(token: str) -> str: + if token in ("1", "2", "3"): + return "observed_code" + if token == "": + return "blank_unresolved" + if token in ("-4", "-3", "-2", "-1", "0"): + return "unlabelled_in_range" + if re.fullmatch(r"-?(?:0|[1-9][0-9]*)", token) and token != "-0": + return "out_of_range" + return "malformed_token" + + +def _roster(person_roster: pd.DataFrame) -> pd.DataFrame: + _require( + isinstance(person_roster, pd.DataFrame) + and person_roster.columns.is_unique + and set(ROSTER_COLUMNS) <= set(person_roster.columns), + "ROSTER_COLUMNS", + ) + _require(0 < len(person_roster) <= MAX_PERSONS, "ROSTER_ROWS") + result = person_roster.loc[:, list(ROSTER_COLUMNS)].copy().reset_index(drop=True) + _require(not result.isna().any().any(), "ROSTER_MISSING") + _require( + all(str(result[name].dtype) == "int64" for name in (KEYS[0], *CROSSCHECKS)), + "ROSTER_INTEGER_DTYPE", + ) + _require(set(result.source_year) == set(SOURCE_YEARS), "ROSTER_COHORTS") + _require( + result.PERIDNUM.map( + lambda value: ( + isinstance(value, str) and re.fullmatch(r"[0-9]{22}", value) is not None + ) + ).all(), + "ROSTER_PERSON_KEY", + ) + _require( + (result.source_household_id > 0).all() + and (result.A_LINENO > 0).all() + and (result.A_AGE >= 0).all(), + "ROSTER_COORDINATES", + ) + _require(not result.duplicated(list(KEYS)).any(), "ROSTER_DUPLICATE_KEY") + _require( + not result.duplicated([KEYS[0], *CROSSCHECKS[:2]]).any(), + "ROSTER_DUPLICATE_COORDINATES", + ) + return result + + +def _coordinate(token: str) -> int: + _require(re.fullmatch(r"[0-9]{1,19}", token) is not None, "MEMBER_INTEGER") + number = int(token) + _require(number <= _INT64_MAX, "MEMBER_INTEGER") + return number + + +def _digest(table: pd.DataFrame) -> str: + # Stream canonical records into the hash; never log native values or retain + # a second full JSON copy. Row order is part of the projection identity. + digest = hashlib.sha256() + digest.update(json.dumps(list(table.columns), separators=(",", ":")).encode()) + for row in table.itertuples(index=False, name=None): + digest.update(b"\n") + digest.update(json.dumps(row, separators=(",", ":"), allow_nan=False).encode()) + return digest.hexdigest() + + +def _regular_member_opener(path, flags): + # Check the opened descriptor, not a race-prone path snapshot. Nonblocking + # open lets us refuse a FIFO even when no writer has opened its other end. + descriptor = os.open(path, flags | os.O_NONBLOCK) + try: + snapshot = os.fstat(descriptor) + _require(stat.S_ISREG(snapshot.st_mode), "MEMBER_FILE_TYPE") + _require(snapshot.st_size <= MAX_MEMBER_BYTES, "MEMBER_BYTES") + except BaseException: + os.close(descriptor) + raise + return descriptor + + +class _BoundedMember(io.RawIOBase): + """Bound raw reads beneath buffering/decoding; the caller owns the raw file.""" + + def __init__(self, raw): + super().__init__() + self._raw = raw + self._remaining = MAX_MEMBER_BYTES + + def readable(self): + return True + + def readinto(self, buffer): + # One excess byte distinguishes exact-limit EOF from a growing member. + # Never pass that probe byte to the decoder or issue an unbounded read. + count = self._raw.readinto(memoryview(buffer)[: self._remaining + 1]) + self._remaining -= count + _require(self._remaining >= 0, "MEMBER_BYTES") + return count + + +def read_asec_person_coverage_source( + member_paths: Mapping[int, str | Path], *, person_roster: pd.DataFrame +) -> tuple[pd.DataFrame, dict]: + """Read exact whole cohorts, returning literals in requested native order. + + Paths are keyed by source/income years 2022, 2023, 2024 (survey years + 2023, 2024, 2025), following the existing ASEC source adapters. The caller + supplies PERIDNUM and int64 source_year/source_household_id/A_LINENO/A_AGE. + Every CSV row must match exactly one requested person in its own cohort; + PH_SEQ, A_LINENO and A_AGE crosscheck the requested native coordinates. + Age is only a crosscheck: it never supplies or overrides PRPERTYP meaning. + + All PRPERTYP cells remain strings, including blanks and malformed tokens. + The state column describes lexical/printed-code evidence, not coverage or + eligibility. Receipt digests bind this local projection, not source custody. + """ + csv_reader = _csv_builtin.capture_csv_reader(csv) + _require(csv_reader is not None, "SOURCE_CSV_READER_CHANGED") + _require( + isinstance(member_paths, Mapping) + and set(member_paths) == set(SOURCE_YEARS) + and all(type(year) is int for year in member_paths) + and all(isinstance(path, (str, Path)) for path in member_paths.values()), + "MEMBER_PATHS", + ) + expected = _roster(person_roster) + lookup = { + row[:2]: (position, row[2:]) + for position, row in enumerate(expected.itertuples(index=False, name=None)) + } + tokens = [""] * len(expected) + seen = set() + cohort_counts = [] + for year in SOURCE_YEARS: + coordinates = set() + rows = 0 + try: + with ( + open( + member_paths[year], + "rb", + buffering=0, + opener=_regular_member_opener, + ) as raw, + io.BufferedReader(_BoundedMember(raw)) as bounded, + io.TextIOWrapper(bounded, encoding="utf-8", newline="") as handle, + ): + reader = csv_reader(handle, strict=True) + header = next(reader, []) + _require( + len(header) == len(set(header)) + and set(READ_COLUMNS) <= set(header), + "MEMBER_HEADER", + ) + indices = [header.index(name) for name in READ_COLUMNS] + for cells in reader: + rows += 1 + _require(len(seen) < MAX_PERSONS, "MEMBER_ROWS") + _require(len(cells) == len(header), "MEMBER_ROW_WIDTH") + key, household, line, age, token = (cells[i] for i in indices) + _require( + re.fullmatch(r"[0-9]{22}", key) is not None, "MEMBER_PERSON_KEY" + ) + native_key = (year, key) + _require(native_key not in seen, "MEMBER_DUPLICATE_KEY") + _require(native_key in lookup, "MEMBER_EXTRA_KEY") + actual = tuple( + _coordinate(value) for value in (household, line, age) + ) + _require(actual[0] > 0 and actual[1] > 0, "MEMBER_COORDINATES") + _require( + actual[:2] not in coordinates, "MEMBER_DUPLICATE_COORDINATES" + ) + position, wanted = lookup[native_key] + _require(actual == wanted, "MEMBER_CROSSCHECK") + coordinates.add(actual[:2]) + seen.add(native_key) + tokens[position] = token + except CoverageSourceRefusalError: + raise + except (OSError, UnicodeError, csv.Error, ValueError, OverflowError): + raise CoverageSourceRefusalError("MEMBER_READ") from None + cohort_counts.append( + {"source_year": year, "survey_year": year + 1, "rows": rows} + ) + _require(len(seen) == len(expected), "MEMBER_MISSING_KEY") + result = expected.copy() + result["PRPERTYP"] = pd.array(tokens, dtype="string") + result["PRPERTYP_state"] = pd.array( + [_state(token) for token in tokens], dtype="string" + ) + receipt = { + "protocol": PROTOCOL, + "source_authenticated": False, + "population_binding_authenticated": False, + "coverage_status": "literal_source_fields_only", + "release_eligible": False, + "contract": coverage_field_contract(), + "read_columns": list(READ_COLUMNS), + "cohorts": cohort_counts, + "person_rows": len(result), + "native_roster_sha256": _digest(expected), + "projection_sha256": _digest(result), + } + return result, receipt diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/asec_person_income_source.py b/packages/microcosm-build/src/microcosm/build/us_runtime/asec_person_income_source.py new file mode 100644 index 000000000..245a8caef --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/asec_person_income_source.py @@ -0,0 +1,488 @@ +"""Restore an observed person total from closed, authenticated Census members. + +P (v4) and H (household attachment) remain immutable. T appends one nominal +observation. A candidate T digest is a locator identity, never source authority: +verification reconstructs all of T from authenticated parents and CSV members. +""" + +from __future__ import annotations + +import csv +import hashlib +import os +import tempfile +from collections.abc import Mapping +from dataclasses import dataclass +from importlib import resources +from pathlib import Path + +import numpy as np +import pandas as pd + +from microcosm.build import frame_checkpoint as checkpoint +from microcosm.frame import Frame, Weights + +from . import asec_current_money as money_schema +from . import asec_current_money_source as legacy +from .asec_current_money import MoneyRefusalError, _json, _require, _sha +from .education_assistance_source import ASEC_EDUCATION_ASSISTANCE_ARCHIVES + +CHECKPOINT_FILENAME = "asec_person_income_source.checkpoint.h5" +ARTIFACT_KIND = "microcosm.asec_person_income_observation_attachment" +OBSERVED_COLUMN = "asec_PTOTVAL" +_READ_COLUMNS = ("PERIDNUM", "PH_SEQ", "A_LINENO", "A_AGE", "PTOTVAL") +# Closed code-owned source pins. No public caller pin authority exists. +_MEMBER_PINS = tuple( + (year, p.member, p.zip_sha256, p.member_sha256, p.rows, p.member_size_bytes) + for year, p in sorted(ASEC_EDUCATION_ASSISTANCE_ARCHIVES.items()) +) +_ATTACHMENT_MAX_BYTES = 2_000_000_000 +_ENCODING_CONTRACT = "independently_reconstructed_canonical_checkpoint_bytes_v1" + + +def _implementation() -> str: + return _sha( + _json( + { + "schema": 1, + "legacy_verification": legacy._verification_identity(), + "schema_module": _sha( + resources.files(__package__) + .joinpath("asec_current_money.py") + .read_bytes() + ), + "source_kind": money_schema.RESTORED_SOURCE_KIND, + "field_columns": dict(money_schema.RESTORED_FIELD_COLUMNS), + "field_zero_policy": dict(money_schema.RESTORED_FIELD_ZERO_POLICY), + "reader_producer": _sha( + resources.files(__package__) + .joinpath("asec_person_income_source.py") + .read_bytes() + ), + "member_pins": _MEMBER_PINS, + "columns": _READ_COLUMNS, + "output": OBSERVED_COLUMN, + "origin": "authenticated_census_csv_encoded_zero_v1", + "encoding_contract": _ENCODING_CONTRACT, + } + ) + ) + + +def _paths(member_paths): + _require( + isinstance(member_paths, Mapping) + and set(member_paths) == {2022, 2023, 2024} + and all(type(y) is int for y in member_paths) + and all(isinstance(p, (str, Path)) for p in member_paths.values()), + "MEMBER_PATHS", + ) + + +def _read_member(path: Path, *, rows: int, size: int) -> pd.DataFrame: + _require(path.stat().st_size == size, "MEMBER_SIZE") + with path.open("r", encoding="utf-8", newline="") as handle: + header = next(csv.reader(handle), []) + _require( + len(header) == len(set(header)) and set(_READ_COLUMNS) <= set(header), + "MEMBER_HEADER", + ) + # Preserve source tokens until validation. In particular, never fillna and + # never let permissive numeric coercion turn absent observations into zeros. + table = pd.read_csv( + path, usecols=list(_READ_COLUMNS), dtype="string", na_filter=False + ) + _require(len(table) == rows, "MEMBER_ROWS") + _require( + bool(table.PERIDNUM.str.fullmatch(r"[0-9]{22}").all()), "MEMBER_PERSON_KEY" + ) + for column in _READ_COLUMNS[1:]: + _require( + bool(table[column].str.fullmatch(r"-?[0-9]+").all()), + "MEMBER_INTEGER" if column == "PTOTVAL" else f"MEMBER_INTEGER_{column}", + column, + ) + table[column] = table[column].astype("int64") + _require( + bool(table.PTOTVAL.between(-99999, 99999999).all()), + "MEMBER_AMOUNT_DOMAIN", + "PTOTVAL", + ) + _require( + not table.PERIDNUM.duplicated().any() + and not table.duplicated(["PH_SEQ", "A_LINENO"]).any(), + "MEMBER_DUPLICATE_KEY", + ) + return table + + +def _owned_frame(parent: Frame) -> Frame: + strata = parent.strata.copy(deep=True) + strata.index = legacy._owned_index(strata.index) + result = Frame( + { + **{e: parent.table(e) for e in parent.entities}, + **{n: parent.link(n) for n in parent.links}, + }, + parent.schema, + { + e: Weights(parent.weights_for(e).values, parent.weights_for(e).kind) + for e in parent.weighted_entities + }, + strata, + mass_log=parent.mass_log, + metadata=parent.metadata, + ) + legacy._detach_frame_axes(result) + return result + + +def _reconstruct(source, member_paths: Mapping[int, str | Path]): + _paths(member_paths) + source.validate() + before = _implementation() + parent = source.frame + person = parent.person + _require(OBSERVED_COLUMN not in person, "RESTORATION_ALREADY_ATTACHED") + _require( + all( + c in person for c in ("source_household_id", "A_LINENO", "A_AGE", "PTOTVAL") + ), + "RESTORATION_SOURCE_COLUMNS", + ) + for c in ("source_household_id", "A_LINENO", "A_AGE"): + _require( + person[c].dtype == np.dtype("int64") and not person[c].isna().any(), + "RESTORATION_COORDINATE_DTYPE", + ) + years = np.asarray(source.scope.person_years, dtype=np.int64) + keys = np.asarray(source.scope.person_native_keys) + output = np.empty(len(person), dtype=np.int64) + joins = [] + with tempfile.TemporaryDirectory( + prefix="microcosm-person-income-members-" + ) as directory: + for year, member, archive_pin, pin, rows, size in _MEMBER_PINS: + staged = Path(directory) / member + legacy._stage_verified(member_paths[year], pin, staged) + raw = _read_member(staged, rows=rows, size=size) + staged.unlink() + positions = np.flatnonzero(years == year) + _require(len(positions) == rows, "RESTORATION_COHORT_ROWS") + recipient_keys = pd.Index(keys[positions]) + _require(not recipient_keys.has_duplicates, "RESTORATION_PERSON_KEY") + indices = pd.Index(raw.PERIDNUM).get_indexer(recipient_keys) + _require( + bool((indices >= 0).all()) and len(np.unique(indices)) == rows, + "RESTORATION_KEY_COVERAGE", + ) + joined = raw.iloc[indices] + for source_col, raw_col in ( + ("source_household_id", "PH_SEQ"), + ("A_LINENO", "A_LINENO"), + ("A_AGE", "A_AGE"), + ): + _require( + np.array_equal( + person[source_col].iloc[positions].to_numpy(), + joined[raw_col].to_numpy(), + ), + "RESTORATION_NATIVE_KEY_OR_AGE", + ) + incumbent = person.PTOTVAL.iloc[positions] + observed = joined.PTOTVAL.to_numpy(dtype=np.int64) + known = ~incumbent.isna().to_numpy() + _require( + np.array_equal( + incumbent.to_numpy(dtype=np.float64, na_value=np.nan)[known], + observed[known], + ), + "RESTORATION_INCUMBENT_CONFLICT", + "PTOTVAL", + ) + # The full current cohort has an incumbent and must be checked, not + # silently treated as another missing-column restoration. + if year == 2024: + _require( + bool(known.all()), "RESTORATION_INCUMBENT_INCOMPLETE", "PTOTVAL" + ) + output[positions] = observed + joins.append( + { + "income_year": year, + "survey_year": year + 1, + "member": member, + "archive_sha256": archive_pin, + "member_sha256": pin, + "source_rows": rows, + "joined_rows": len(positions), + "unreferenced_source_rows": 0, + "incumbent_compared_rows": int(known.sum()), + "incumbent_conflicts": 0, + "native_key_or_age_conflicts": 0, + } + ) + result = _owned_frame(parent) + result.person[OBSERVED_COLUMN] = output + source.validate() + _require(_implementation() == before, "RESTORATION_IMPLEMENTATION_CHANGED") + evidence = source.source.evidence + from .asec_current_money import _parse + + parent_evidence = _parse(evidence) + receipt = { + "schema_version": 1, + "artifact_kind": ARTIFACT_KIND, + "original_v4_sha256": parent_evidence["parent_sha256"], + "household_attachment_sha256": parent_evidence["attachment_sha256"], + "parent_frame_sha256": legacy._frame_signature(parent), + "output_frame_sha256": legacy._frame_signature(result), + "reader": "authenticated_census_csv_exact_integer_no_fill_v1", + "join_keys": ["source_year", "PERIDNUM"], + "crosscheck_columns": ["source_household_id", "A_LINENO", "A_AGE"], + "sources": joins, + "output_column": OBSERVED_COLUMN, + "logical_field": "PTOTVAL", + "nominal_basis": "income_year_us_dollars", + "dtype": "int64", + "observation_sha256": _sha(output.astype(" _VerifiedRestoration: + """Authenticate and reconstruct T fully; this function does not issue money.""" + try: + return _verify( + parent_path, + household_attachment_path, + person_income_attachment_path, + member_paths, + ) + except MoneyRefusalError: + raise + except ( + OSError, + ValueError, + TypeError, + KeyError, + OverflowError, + UnicodeError, + csv.Error, + StopIteration, + ): + raise MoneyRefusalError("RESTORATION_SOURCE_CONTRACT") from None + + +def restore_asec_person_income_source( + parent_path: str | Path, + household_attachment_path: str | Path, + *, + member_paths: Mapping[int, str | Path], + output_dir: str | Path, +) -> dict[str, object]: + """Write a new local T and receipt from closed sources; never replace parents.""" + destination = Path(output_dir) + if destination.exists(): + raise FileExistsError(destination) + try: + _paths(member_paths) + source = legacy.load_authenticated_current_money_source( + parent_path, household_attachment_path + ) + frame, receipt = _reconstruct(source, member_paths) + destination.parent.mkdir(parents=True, exist_ok=True) + with tempfile.TemporaryDirectory( + prefix=".asec-person-income-", dir=destination.parent + ) as directory: + staging = Path(directory) + output = staging / CHECKPOINT_FILENAME + checkpoint.write_frame_checkpoint( + output, frame, metadata=_document(receipt) + ) + verified = _verify( + parent_path, household_attachment_path, output, member_paths + ) + report = { + **receipt, + "output_sha256": verified.attachment_sha256, + "output_file": CHECKPOINT_FILENAME, + } + (staging / "restoration.receipt.json").write_bytes(_json(report) + b"\n") + # Exclusive final directory creation avoids replacing even a raced + # empty directory. No historical artifact is opened for writing. + destination.mkdir() + for path in staging.iterdir(): + os.link(path, destination / path.name) + return report + except FileExistsError: + raise + except MoneyRefusalError: + raise + except ( + OSError, + ValueError, + TypeError, + KeyError, + OverflowError, + UnicodeError, + csv.Error, + StopIteration, + ): + raise MoneyRefusalError("RESTORATION_SOURCE_CONTRACT") from None + + +def load_authenticated_restored_current_money_source( + parent_path: str | Path, + household_attachment_path: str | Path, + person_income_attachment_path: str | Path, + *, + member_paths: Mapping[int, str | Path], +) -> legacy.AuthenticatedCurrentMoneySource: + """Issue restored authority only after closed-source complete reconstruction.""" + from .asec_current_money import ( + _SOURCE_TOKEN, + RESTORED_FIELD_COLUMNS, + RESTORED_FIELD_ZERO_POLICY, + RESTORED_SOURCE_KIND, + AuthenticatedAsecSource, + _parse, + compile_asec_current_money_spec, + ) + from .asec_current_money_resources import load_current_money_resources + + checked = verify_asec_person_income_source( + parent_path, + household_attachment_path, + person_income_attachment_path, + member_paths=member_paths, + ) + frame = checked.frame + scope = legacy._scope(frame) + views = legacy._views(frame, scope, restored=True) + scope_sha, input_sha = legacy._input_binding(views) + evidence = _parse(checked.parent_source.source.identity) + evidence.update( + schema_version=2, + source_kind=RESTORED_SOURCE_KIND, + person_income_attachment_sha256=checked.attachment_sha256, + field_source_columns=dict(RESTORED_FIELD_COLUMNS), + field_zero_origin_policy=dict(RESTORED_FIELD_ZERO_POLICY), + scope_sha256=scope_sha, + input_sha256=input_sha, + frame_sha256=legacy._frame_signature(frame), + verification_sha256=checked.receipt["implementation_sha256"], + ) + checked.parent_source.validate() + _require( + _implementation() == evidence["verification_sha256"], + "RESTORATION_IMPLEMENTATION_CHANGED", + ) + authority = AuthenticatedAsecSource(_json(evidence), _token=_SOURCE_TOKEN) + spec = compile_asec_current_money_spec(load_current_money_resources(), authority) + result = legacy.AuthenticatedCurrentMoneySource( + frame, scope, authority, spec, _token=legacy._LOAD_TOKEN + ) + result.validate() + return result diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/asec_population_catalogue.py b/packages/microcosm-build/src/microcosm/build/us_runtime/asec_population_catalogue.py new file mode 100644 index 000000000..03c83d4d7 --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/asec_population_catalogue.py @@ -0,0 +1,684 @@ +"""Authenticate the complete 2024 ASEC catalogue before population construction. + +The closed parent still authenticates three cohorts. Only original 2024 records +are exposed here. No descendant Frame, unit assignment, domain classification, +selection, or weight allocation is performed. Household literals are retained; +A_AGE and A_LINENO are canonical strings of authenticated numeric originals. +""" + +from __future__ import annotations + +import hashlib +import json +import operator +import sys +import weakref +from collections.abc import Mapping +from dataclasses import asdict, dataclass +from itertools import chain +from pathlib import Path +from typing import NamedTuple + +from . import asec_2024_native_population as native +from . import survey_population_domains as domains + +ARTIFACT_KIND = "microcosm.us.asec-source-catalogue.v1" +_MAX_ENVELOPE_BYTES = 1024**2 +_MAX_RECORD_BYTES = 128 * 1024**2 +_ISSUED: dict[int, tuple[weakref.ReferenceType, bytes, object]] = {} + + +class AsecSourceCatalogueError(ValueError): + """Static refusal codes, without source paths or source values.""" + + +def _require(condition, code): + if not condition: + raise AsecSourceCatalogueError(code) + + +def _sha(value): + return hashlib.sha256(value).hexdigest() + + +def _encode(value): + # Only compact identities and individual records use this encoder. The + # complete catalogue is hashed incrementally, never encoded as another body. + result = bytearray() + encoder = json.JSONEncoder( + ensure_ascii=True, allow_nan=False, sort_keys=True, separators=(",", ":") + ) + for part in encoder.iterencode(value): + part = part.encode("ascii") + _require(len(result) + len(part) <= _MAX_ENVELOPE_BYTES, "ENVELOPE_SIZE") + result.extend(part) + return bytes(result) + + +def _implementation(): + modules = (sys.modules[__name__], domains) + runtime = {m.__name__: native.coverage_owner._runtime_code(m) for m in modules} + code = {m.__name__: _sha(Path(m.__file__).read_bytes()) for m in modules} + _require( + runtime == _RUNTIME_AUTHORITY and code == _BYTE_AUTHORITY, + "PRODUCER_CODE_CHANGED", + ) + return { + "kind": ARTIFACT_KIND, + "runtime": runtime, + "code": code, + "source_owner": native._implementation(), + "limits": [_MAX_ENVELOPE_BYTES, _MAX_RECORD_BYTES], + "periods": [2024, 2025], + } + + +def _source_authority(): + """Snapshot exact code-owned source pin values without file/resource I/O. + + Canonical bytes detach the seal from mutable registry objects. The original + household registry helper validates its records but reads no source files. + """ + return _encode( + { + "parent": native.parent_owner._SOURCE_PINS, + "restoration": native.restoration._MEMBER_PINS, + "coverage": native.coverage_owner._MEMBER_PINS, + "household": [asdict(pin) for pin in native.anchor_owner._registry()], + } + ) + + +@dataclass(frozen=True, slots=True) +class UnrepresentedAsecHousehold: + """Original noninterview household, absent from the authenticated roster. + + This is a custody ledger, not a scientific coverage classification. No empty + person list is supplied as a claim about the household's actual membership. + """ + + key: domains.HouseholdKey + h_hhtype: str + hrhtype: str + h_livqrt: str + h_numper: str + hsup_wgt: str + reason: str = "original_noninterview_not_in_authenticated_person_roster" + + +def _key_record(key): + _require(type(key) is domains.HouseholdKey, "RECORD_TYPE") + _require(key.source is domains.Source.ASEC, "RECORD_SOURCE") + return [key.source.value, key.source_year, key.survey_year, key.native_id] + + +def _record(row): + _require( + type(row) in (domains.AsecHousehold, UnrepresentedAsecHousehold), + "RECORD_TYPE", + ) + result = { + "key": _key_record(row.key), + "H_HHTYPE": row.h_hhtype, + "HRHTYPE": row.hrhtype, + "H_LIVQRT": row.h_livqrt, + "H_NUMPER": row.h_numper, + "HSUP_WGT": row.hsup_wgt, + } + if type(row) is UnrepresentedAsecHousehold: + result["unrepresented_reason"] = row.reason + else: + _require(type(row.persons) is tuple, "RECORD_TYPE") + persons = [] + for person in row.persons: + _require(type(person) is domains.AsecPerson, "RECORD_TYPE") + persons.append( + [ + person.peridnum, + person.a_lineno, + person.age, + person.prpertyp, + person.prpertyp_state, + _key_record(person.household_key), + ] + ) + result["persons"] = persons + return result + + +class _RecordsMemo(NamedTuple): + """The issued record roots, their projected leaves, and the identity streamed.""" + + households: tuple + ledger: tuple + household_leaves: tuple + ledger_leaves: tuple + identity: tuple + + +def _records_identity(households, ledger, *, memo=False): + """Stream every canonical record; optionally bind a leaf snapshot memo. + + With ``memo=True`` the result is ``(identity, memo)``. The memo's leaves are + projected from the exact rows encoded in this same pass, so it can only + stand for this identity. It is ``None`` when any record carries a leaf + that is not an exact ``str``/``int`` or the ASEC source member. + """ + _require(type(households) is tuple and type(ledger) is tuple, "RECORD_TYPE") + digest, count, size, persons = hashlib.sha256(), 0, 0, 0 + household_leaves, ledger_leaves, eligible = [], [], memo + for kind, rows in (("household", households), ("unrepresented", ledger)): + for row in rows: + data = _encode([kind, _record(row)]) + b"\n" + size += len(data) + _require(size <= _MAX_RECORD_BYTES, "RECORD_SIZE") + digest.update(data) + count += 1 + if kind == "household": + persons += len(row.persons) + if eligible: + projected = _eligible_leaves(row, kind) + if projected is None: + eligible = False + elif kind == "household": + household_leaves.append(projected) + else: + ledger_leaves.append(projected) + identity = (digest.hexdigest(), size, count, persons) + if not memo: + return identity + if not eligible: + return identity, None + return identity, _RecordsMemo( + households, ledger, tuple(household_leaves), tuple(ledger_leaves), identity + ) + + +_KEY_LEAVES = operator.attrgetter("source", "source_year", "survey_year", "native_id") +_HOUSEHOLD_LITERALS = operator.attrgetter( + "h_hhtype", "hrhtype", "h_livqrt", "h_numper", "hsup_wgt" +) +_HOUSEHOLD_LEAVES = operator.attrgetter( + "key", "h_hhtype", "hrhtype", "h_livqrt", "h_numper", "hsup_wgt", "persons" +) +_PERSON_LITERALS = operator.attrgetter( + "peridnum", "a_lineno", "age", "prpertyp", "prpertyp_state" +) +_PERSON_LEAVES = operator.attrgetter( + "peridnum", "a_lineno", "age", "prpertyp", "prpertyp_state", "household_key" +) +_LEDGER_LITERALS = operator.attrgetter( + "h_hhtype", "hrhtype", "h_livqrt", "h_numper", "hsup_wgt", "reason" +) +_LEDGER_LEAVES = operator.attrgetter( + "key", "h_hhtype", "hrhtype", "h_livqrt", "h_numper", "hsup_wgt", "reason" +) + + +def _key_leaves(key): + # The encoder writes ``source.value``; an enum member is a mutable object, + # so its value object is a leaf in its own right. Classes are leaves too: + # ``__class__`` can be reassigned in place, and the encoder refuses + # anything but the exact record types on every full pass. + source, *rest = _KEY_LEAVES(key) + return (type(key), source, source.value, *rest) + + +def _household_leaves(row): + key, *literals, persons = _HOUSEHOLD_LEAVES(row) + return ( + type(row), + key, + *_key_leaves(key), + *literals, + type(persons), + persons, + *chain.from_iterable( + (type(person), *_PERSON_LEAVES(person), *_key_leaves(person.household_key)) + for person in persons + ), + ) + + +def _ledger_leaves(row): + key, *literals = _LEDGER_LEAVES(row) + return (type(row), key, *_key_leaves(key), *literals) + + +def _eligible_key(key): + return ( + type(key) is domains.HouseholdKey + and key.source is domains.Source.ASEC + and type(key.source.value) is str + and type(key.source_year) is int + and type(key.survey_year) is int + and type(key.native_id) is str + ) + + +def _eligible_leaves(row, kind): + """Project one record's leaves when every one is an exact immutable value.""" + if not _eligible_key(row.key): + return None + if kind == "household": + if type(row) is not domains.AsecHousehold or type(row.persons) is not tuple: + return None + literals = list(_HOUSEHOLD_LITERALS(row)) + for person in row.persons: + if type(person) is not domains.AsecPerson or not _eligible_key( + person.household_key + ): + return None + literals.extend(_PERSON_LITERALS(person)) + leaves = _household_leaves(row) + else: + if type(row) is not UnrepresentedAsecHousehold: + return None + literals = _LEDGER_LITERALS(row) + leaves = _ledger_leaves(row) + if any(type(item) is not str for item in literals): + return None + return leaves + + +def _leaves_unchanged(rows, snapshot, project): + if len(rows) != len(snapshot): + return False + for row, expected in zip(rows, snapshot, strict=True): + try: + current = project(row) + except (AttributeError, TypeError): + return False + if len(current) != len(expected) or not all( + map(operator.is_, current, expected) + ): + return False + return True + + +def _memoized_records_identity( + households, ledger, memo=None, *, expected_identity=None +): + """Reuse the issued records identity only over the same unchanged leaves. + + A hit needs the exact issued household and ledger tuples, a memo identity + equal to the owner's retained one, and every projected leaf (literals, + keys, person rosters and their members' fields) to be the very object seen + while the identity streamed. Frozen dataclasses stay writable through + ``object.__setattr__``, so the leaves carry the proof, not the records. + Anything else falls back to the full canonical encoding and its refusals. + """ + if ( + type(households) is tuple + and type(ledger) is tuple + and type(memo) is _RecordsMemo + and households is memo.households + and ledger is memo.ledger + and type(memo.household_leaves) is tuple + and type(memo.ledger_leaves) is tuple + and type(memo.identity) is tuple + and memo.identity == expected_identity + and _leaves_unchanged(households, memo.household_leaves, _household_leaves) + and _leaves_unchanged(ledger, memo.ledger_leaves, _ledger_leaves) + ): + return memo.identity + return _records_identity(households, ledger) + + +def _bound_records_identity(rows, roster): + """Stream complete receiving/native keys, source positions and exact anchors.""" + digest, count, size = hashlib.sha256(), 0, 0 + for kind, values in (("household", rows), ("person", roster)): + for row in values: + data = _encode([kind, row]) + b"\n" + size += len(data) + _require(size <= _MAX_RECORD_BYTES, "RECORD_SIZE") + digest.update(data) + count += 1 + return {"sha256": digest.hexdigest(), "canonical_bytes": size, "records": count} + + +def _rows(rows, roster, anchors, fields): + people = {} + for person in roster: + people.setdefault(int(person["source_household_id"]), []).append(person) + households = [] + for row in sorted(rows, key=lambda r: r["source"]["member_row_1based"]): + original = row["household_fields"] + key = domains.HouseholdKey(domains.Source.ASEC, 2024, 2025, original["H_SEQ"]) + members = tuple( + domains.AsecPerson( + p["PERIDNUM"], + str(int(p["A_LINENO"])), + str(int(p["A_AGE"])), + p["PRPERTYP"], + p["PRPERTYP_state"], + key, + ) + for p in people[row["native_key"][1]] + ) + households.append( + domains.AsecHousehold( + key, + original["H_HHTYPE"], + original["HRHTYPE"], + original["H_LIVQRT"], + original["H_NUMPER"], + row["HSUP_WGT"], + members, + ) + ) + # _roster(None) already authenticates every original member row and refuses + # interview rows absent from the represented roster or noninterviews in it. + weights = {r["H_SEQ"]: r for r in anchors.document["records"]} + ledger = tuple( + UnrepresentedAsecHousehold( + domains.HouseholdKey(domains.Source.ASEC, 2024, 2025, r["H_SEQ"]), + r["H_HHTYPE"], + r["HRHTYPE"], + r["H_LIVQRT"], + r["H_NUMPER"], + weights[r["H_SEQ"]]["HSUP_WGT"], + ) + for r in fields.document["records"] + if r["H_HHTYPE"] != "1" + ) + return tuple(households), ledger + + +class _State(NamedTuple): + parent: object + parent_frame: object + coverage: object + anchors: object + fields: object + source_files: tuple + producer: bytes + source_authority: bytes + parent_identity: str + attached_evidence: tuple + households: tuple + ledger: tuple + records_identity: tuple + records_memo: _RecordsMemo | None = None + + +def _current_records_identity(state): + return _memoized_records_identity( + state.households, + state.ledger, + state.records_memo, + expected_identity=state.records_identity, + ) + + +def _validate_state(state): + _require(_source_authority() == state.source_authority, "SOURCE_AUTHORITY_CHANGED") + _require(_encode(_implementation()) == state.producer, "PRODUCER_CHANGED") + _require( + state.parent.frame is state.parent_frame + and native._frame_identity(state.parent.frame) == state.parent_identity, + "PARENT_CHANGED", + ) + _require( + _current_records_identity(state) == state.records_identity, "RECORDS_CHANGED" + ) + state.parent.validate() + native.coverage_owner.verify_asec_coverage_parent(state.coverage, state.parent) + native.anchor_owner.verify_asec_household_weights_source(state.anchors) + native.field_owner.verify_asec_household_coverage_fields(state.fields) + for path, expected, maximum, size in state.source_files: + native._file_identity(path, expected, maximum, size) + _require(_encode(_implementation()) == state.producer, "PRODUCER_CHANGED") + _require( + state.parent.frame is state.parent_frame + and native._frame_identity(state.parent.frame) == state.parent_identity, + "PARENT_CHANGED", + ) + _require( + _current_records_identity(state) == state.records_identity, "RECORDS_CHANGED" + ) + # Pure final seals after source/resource I/O; no recursively repeated reads. + current = native._attached_evidence( + state.parent, state.coverage, state.anchors, state.fields + ) + _require( + current[0] is state.attached_evidence[0] + and current[1:] == state.attached_evidence[1:], + "ATTACHED_EVIDENCE_CHANGED", + ) + # Producer helpers read registry values before later helper source-code I/O. + # A current borrow must refuse a late registry change, without waiting for + # the next producer pass. This final comparison is pure and bounded. + _require(_source_authority() == state.source_authority, "SOURCE_AUTHORITY_CHANGED") + + +@dataclass(frozen=True, slots=True, weakref_slot=True) +class AuthenticatedAsecSourceCatalogue: + """Process-issued immutable records with checked borrows and compact evidence.""" + + payload: bytes + + def _checked(self): + try: + entry = _ISSUED.get(id(self)) + _require( + type(self) is AuthenticatedAsecSourceCatalogue + and entry is not None + and entry[0]() is self + and type(self.payload) is bytes + and self.payload == entry[1], + "UNISSUED_OR_CHANGED", + ) + _validate_state(entry[2]) + _require( + _ISSUED.get(id(self)) is entry + and type(self.payload) is bytes + and self.payload == entry[1], + "UNISSUED_OR_CHANGED", + ) + return entry + except AsecSourceCatalogueError: + raise + except ( + OSError, + ValueError, + TypeError, + KeyError, + AttributeError, + OverflowError, + ): + raise AsecSourceCatalogueError("CATALOGUE_BINDING_REFUSAL") from None + + @property + def households(self) -> tuple[domains.AsecHousehold, ...]: + return self._checked()[2].households + + @property + def exclusion_ledger(self) -> tuple[UnrepresentedAsecHousehold, ...]: + return self._checked()[2].ledger + + @property + def receipt(self) -> dict: + return json.loads(self._checked()[1]) + + def to_bytes(self) -> bytes: + return self._checked()[1] + + def validate(self) -> None: + self._checked() + + +def verify_asec_source_catalogue(value: object) -> AuthenticatedAsecSourceCatalogue: + _require(type(value) is AuthenticatedAsecSourceCatalogue, "UNISSUED_OR_CHANGED") + value.validate() + return value + + +def issue_asec_source_catalogue( + parent_path: str | Path, + household_attachment_path: str | Path, + person_income_attachment_path: str | Path, + *, + person_member_paths: Mapping[int, str | Path], + household_member_path: str | Path, + candidate: bytes | None = None, +) -> AuthenticatedAsecSourceCatalogue: + """Reconstruct the complete 2024 source catalogue from authenticated paths. + + Candidate bytes are compared only after actual reconstruction. No caller + Frame, record list, weights or receipt can grant source authority. + """ + try: + _require( + candidate is None + or (type(candidate) is bytes and len(candidate) <= _MAX_ENVELOPE_BYTES), + "CANDIDATE_SIZE_OR_TYPE", + ) + parent_path, household_path, restored_path, persons, member, _, candidate = ( + native._inputs( + parent_path, + household_attachment_path, + person_income_attachment_path, + person_member_paths, + household_member_path, + None, + candidate, + ) + ) + source_authority = _source_authority() + producer = _encode(_implementation()) + parent = native.restoration.load_authenticated_restored_current_money_source( + parent_path, household_path, restored_path, member_paths=persons + ) + parent_identity = native._frame_identity(parent.frame) + coverage = native.coverage_owner.authenticate_asec_coverage( + parent, member_paths=persons + ) + anchors = native.anchor_owner.load_authenticated_asec_household_weights( + {2024: member} + ) + fields = native.field_owner.load_authenticated_asec_household_coverage_fields( + {2024: member} + ) + _, _, rows, roster = native._roster(parent, coverage, anchors, fields, None) + households, ledger = _rows(rows, roster, anchors, fields) + records_identity, records_memo = _records_identity( + households, ledger, memo=True + ) + identity = json.loads(parent.source.identity) + files = [ + ( + parent_path, + identity["parent_sha256"], + native._MAX_CHECKPOINT_BYTES, + None, + ), + ( + household_path, + identity["attachment_sha256"], + native._MAX_CHECKPOINT_BYTES, + None, + ), + ( + restored_path, + identity["person_income_attachment_sha256"], + native._MAX_CHECKPOINT_BYTES, + None, + ), + ] + sources = coverage.receipt["sources"] + for source in sources: + files.append( + ( + persons[source["source_year"]], + source["member_sha256"], + source["member_bytes"], + source["member_bytes"], + ) + ) + pin = anchors.document["members"][0] + files.append( + (member, pin["member_sha256"], pin["size_bytes"], pin["size_bytes"]) + ) + attached = native._attached_evidence(parent, coverage, anchors, fields) + state = _State( + parent, + parent.frame, + coverage, + anchors, + fields, + tuple(files), + producer, + source_authority, + parent_identity, + attached, + households, + ledger, + records_identity, + records_memo, + ) + payload = _encode( + { + "kind": ARTIFACT_KIND, + "source_year": 2024, + "survey_year": 2025, + "counts": { + "households": len(households), + "persons": records_identity[3], + "unrepresented_households": len(ledger), + }, + "records": { + "sha256": records_identity[0], + "canonical_bytes": records_identity[1], + "count": records_identity[2], + }, + "native_binding": _bound_records_identity(rows, roster), + "producer_sha256": _sha(producer), + "source_authority_sha256": _sha(source_authority), + "parent_custody": { + "cohorts": [2022, 2023, 2024], + "catalogue_cohorts": [2024], + "identity": identity, + "frame_sha256": parent_identity, + }, + "source_members": sources, + "household_member": pin, + "attached_capsules_sha256": [_sha(value) for value in attached[1:]], + "age_representation": "canonical_integer_string_of_authenticated_original_A_AGE_not_retained_CSV_lexeme", + "line_representation": "canonical_integer_string_of_authenticated_original_A_LINENO_not_retained_CSV_lexeme", + "household_key_representation": "original_H_SEQ_CSV_lexeme", + "classification_performed": False, + "weights_applied": False, + "selection_performed": False, + "release_eligible": False, + } + ) + _require(candidate is None or candidate == payload, "CANDIDATE_MISMATCH") + _validate_state(state) + result = AuthenticatedAsecSourceCatalogue(payload) + key = id(result) + + def discard(reference): + entry = _ISSUED.get(key) + if entry is not None and entry[0] is reference: + del _ISSUED[key] + + _ISSUED[key] = (weakref.ref(result, discard), payload, state) + result.validate() + return result + except AsecSourceCatalogueError: + raise + except (OSError, ValueError, TypeError, KeyError, AttributeError, OverflowError): + raise AsecSourceCatalogueError("CATALOGUE_SOURCE_REFUSAL") from None + + +_RUNTIME_AUTHORITY = { + m.__name__: native.coverage_owner._runtime_code(m) + for m in (sys.modules[__name__], domains) +} + +_BYTE_AUTHORITY = { + m.__name__: _sha(Path(m.__file__).read_bytes()) + for m in (sys.modules[__name__], domains) +} diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/asec_prepared_source.py b/packages/microcosm-build/src/microcosm/build/us_runtime/asec_prepared_source.py new file mode 100644 index 000000000..0aa7ae532 --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/asec_prepared_source.py @@ -0,0 +1,567 @@ +"""Full-source ASEC preparation for the current-money graph slice. + +The graph source is one directory holding exactly the reviewed restoration +inputs: the raw-stage parent P, the household-observation attachment H, the +restored person-income attachment T with its receipt, the three housing cohort +HDFs, and the three official Census PERSON CSV members. No ZIP archive is +accepted and no file is discovered by pattern: the roster below is the contract. + +The executor keys a directory source by relative name plus file bytes, which is +a cache identity, not an authentication. Authentication stays where it already +is — the pinned digests inside the reviewed loaders and the canonical byte +replay of T — and this module simply refuses to run unless those loaders issue +authority. + +The authenticated ``ReadyCurrentMoney`` exists only inside this call. What +leaves is the canonical encoded body, a receipt of identities, and an +operator-free frame whose dtypes have been promoted to graph tokens. +""" + +from __future__ import annotations + +import os +import stat +from dataclasses import dataclass +from importlib import resources +from pathlib import Path + +import numpy as np +import pandas as pd + +from microcosm.build.serialization_dtypes import canonicalize_frame_string_dtypes +from microcosm.frame import US_SCHEMA, Frame +from microcosm.graph.population import dtype_for_token, token_for_dtype + +from . import asec_current_money_source as money_source +from . import asec_housing_status_source as housing_source +from . import asec_housing_universe as housing_universe +from . import asec_housing_universe_source as housing_universe_source +from . import asec_income_observations as income_source +from . import asec_person_income_source as person_income +from . import asec_student_controls as student_source +from ._asec_current_money_codec import ( + current_money_content_sha256, + encode_current_money, +) +from .asec_checkpoint import ASEC_RAW_STAGE_CHECKPOINT_FILENAME +from .asec_current_money import ( + ENCODED_ZERO_POLICY, + FIELDS, + RESTORED_SOURCE_KIND, + ZERO_POLICY, + MoneyRefusalError, + ReadyCurrentMoney, + _json, + _parse, + _sha, +) +from .asec_current_money import ( + _digest as _digest_shape, +) +from .asec_current_money_units import reconstruct_current_money_tax_units +from .asec_housing_status_source import ( + attach_housing_status, + load_authenticated_housing_status, +) +from .education_assistance_source import ASEC_EDUCATION_ASSISTANCE_ARCHIVES +from .operator_boundary import assert_operator_free_source_frame + +PREPARED_SOURCE_KIND = "us_asec_prepared_current_money_v3" +PREPARED_RECEIPT_SCHEMA = "microcosm.us.asec_prepared_receipt.v3" +PREPARATION_BOUNDARY = "US graph ASEC prepared source" +SOURCE_YEARS: tuple[int, ...] = (2022, 2023, 2024) +PARENT_FILENAME = ASEC_RAW_STAGE_CHECKPOINT_FILENAME +HOUSEHOLD_ATTACHMENT_FILENAME = "asec_household_observations.checkpoint.h5" +PERSON_INCOME_FILENAME = person_income.CHECKPOINT_FILENAME +RESTORATION_RECEIPT_FILENAME = "restoration.receipt.json" +COHORT_FILENAMES: tuple[tuple[int, str], ...] = tuple( + (year, f"asec_household_cohort_{year}.h5") for year in SOURCE_YEARS +) +MEMBER_FILENAMES: tuple[tuple[int, str], ...] = tuple( + (year, ASEC_EDUCATION_ASSISTANCE_ARCHIVES[year].member) for year in SOURCE_YEARS +) +PREPARED_SOURCE_FILES: tuple[str, ...] = ( + PARENT_FILENAME, + HOUSEHOLD_ATTACHMENT_FILENAME, + PERSON_INCOME_FILENAME, + RESTORATION_RECEIPT_FILENAME, + *(name for _year, name in COHORT_FILENAMES), + *(name for _year, name in MEMBER_FILENAMES), +) +RESTORATION_RECEIPT_MAX_BYTES = 256 * 1024 +_INTEGER_WIDTH_TARGETS = frozenset({"int8", "int16", "uint8", "uint16", "uint32"}) +_NULLABLE_INTEGER_TARGETS = frozenset( + {"Int8", "Int16", "Int32", "UInt8", "UInt16", "UInt32", "UInt64"} +) + + +class PreparedSourceRefusalError(ValueError): + """Sanitized preparation refusal; never carries rows, values or paths.""" + + +def _require(condition: bool, reason: str) -> None: + if not condition: + raise PreparedSourceRefusalError(reason) + + +def _roster(source_dir) -> dict[str, Path]: + directory = Path(source_dir) + _require(directory.is_dir(), "SOURCE_DIRECTORY") + present = sorted(entry.name for entry in directory.iterdir()) + _require(present == sorted(PREPARED_SOURCE_FILES), "SOURCE_FILE_ROSTER") + paths = {} + for name in PREPARED_SOURCE_FILES: + path = directory / name + _require(path.is_file() and not path.is_symlink(), "SOURCE_REGULAR_FILE") + paths[name] = path + return paths + + +RESTORATION_READER = "authenticated_census_csv_exact_integer_no_fill_v1" +RESTORATION_JOIN_KEYS = ["source_year", "PERIDNUM"] +RESTORATION_CROSSCHECK_COLUMNS = ["source_household_id", "A_LINENO", "A_AGE"] +RESTORATION_LOGICAL_FIELD = "PTOTVAL" +RESTORATION_NOMINAL_BASIS = "income_year_us_dollars" +RESTORATION_ZERO_CLAIM = "encoded_source_value_not_respondent_answer" +_RESTORATION_RECEIPT_KEYS = frozenset( + { + "artifact_kind", + "crosscheck_columns", + "dtype", + "encoding_contract", + "household_attachment_sha256", + "implementation_sha256", + "join_keys", + "logical_field", + "nominal_basis", + "observation_sha256", + "original_v4_sha256", + "output_column", + "output_file", + "output_frame_sha256", + "output_sha256", + "parent_frame_sha256", + "reader", + "schema_version", + "sources", + "zero_claim", + "zero_origin_policy", + } +) +_RESTORATION_SOURCE_KEYS = frozenset( + { + "archive_sha256", + "income_year", + "incumbent_compared_rows", + "incumbent_conflicts", + "joined_rows", + "member", + "member_sha256", + "native_key_or_age_conflicts", + "source_rows", + "survey_year", + "unreferenced_source_rows", + } +) + + +def _restoration_sources(rows, source) -> None: + _require( + type(rows) is list and len(rows) == len(SOURCE_YEARS), "RESTORATION_SOURCES" + ) + for row, pins in zip(rows, person_income._MEMBER_PINS, strict=True): + year, member, archive_pin, member_pin, source_rows, _size = pins + positions = np.flatnonzero(np.asarray(source.scope.person_years) == year) + known_incumbents = int( + source.frame.person.PTOTVAL.iloc[positions].notna().sum() + ) + _require( + type(row) is dict and set(row) == _RESTORATION_SOURCE_KEYS, + "RESTORATION_SOURCES", + ) + _require( + row["income_year"] == year + and row["survey_year"] == year + 1 + and row["member"] == member + and row["archive_sha256"] == archive_pin + and row["member_sha256"] == member_pin + and row["source_rows"] == source_rows, + "RESTORATION_SOURCE_PINS", + ) + _require( + all( + type(row[name]) is int and row[name] >= 0 + for name in ( + "joined_rows", + "unreferenced_source_rows", + "incumbent_compared_rows", + ) + ) + and row["joined_rows"] == len(positions) == source_rows + and row["unreferenced_source_rows"] == 0 + and row["incumbent_compared_rows"] == known_incumbents + and row["incumbent_conflicts"] == 0 + and row["native_key_or_age_conflicts"] == 0, + "RESTORATION_SOURCE_COVERAGE", + ) + + +def _restoration_receipt(path: Path, evidence: dict, source) -> dict: + """Bind the shipped restoration receipt to the authority the loader issued.""" + # Refuse special files even if the roster check raced with a replacement, + # and bound allocation before parsing an untrusted sidecar. + descriptor = os.open(path, os.O_RDONLY | os.O_NONBLOCK) + try: + info = os.fstat(descriptor) + _require(stat.S_ISREG(info.st_mode), "RESTORATION_RECEIPT") + _require( + 0 < info.st_size <= RESTORATION_RECEIPT_MAX_BYTES, "RESTORATION_RECEIPT" + ) + with os.fdopen(descriptor, "rb", closefd=False) as stream: + payload = stream.read(RESTORATION_RECEIPT_MAX_BYTES + 1) + finally: + os.close(descriptor) + _require(0 < len(payload) <= RESTORATION_RECEIPT_MAX_BYTES, "RESTORATION_RECEIPT") + _require(payload.endswith(b"\n"), "RESTORATION_RECEIPT") + try: + receipt = _parse(payload[:-1], RESTORATION_RECEIPT_MAX_BYTES) + except MoneyRefusalError as error: + raise PreparedSourceRefusalError("RESTORATION_RECEIPT") from error + _require(_json(receipt) == payload[:-1], "RESTORATION_RECEIPT_CANONICAL") + _require(set(receipt) == _RESTORATION_RECEIPT_KEYS, "RESTORATION_RECEIPT_SCHEMA") + _require( + receipt["schema_version"] == 1 + and receipt["artifact_kind"] == person_income.ARTIFACT_KIND + and receipt["output_file"] == PERSON_INCOME_FILENAME + and receipt["output_column"] == person_income.OBSERVED_COLUMN + and receipt["encoding_contract"] == person_income._ENCODING_CONTRACT + and receipt["reader"] == RESTORATION_READER + and receipt["join_keys"] == RESTORATION_JOIN_KEYS + and receipt["crosscheck_columns"] == RESTORATION_CROSSCHECK_COLUMNS + and receipt["logical_field"] == RESTORATION_LOGICAL_FIELD + and receipt["nominal_basis"] == RESTORATION_NOMINAL_BASIS + and receipt["dtype"] == "int64" + and receipt["zero_origin_policy"] == ENCODED_ZERO_POLICY + and receipt["zero_claim"] == RESTORATION_ZERO_CLAIM, + "RESTORATION_RECEIPT_SCHEMA", + ) + _require( + all( + _digest_shape(receipt[name]) + for name in ( + "observation_sha256", + "output_frame_sha256", + "parent_frame_sha256", + ) + ), + "RESTORATION_RECEIPT_DIGEST", + ) + _restoration_sources(receipt["sources"], source) + # The reconstruction the loader just performed is the authority; the shipped + # receipt must agree with the evidence it issued, digest for digest. + _require( + receipt["output_sha256"] == evidence["person_income_attachment_sha256"] + and receipt["original_v4_sha256"] == evidence["parent_sha256"] + and receipt["household_attachment_sha256"] == evidence["attachment_sha256"] + and receipt["implementation_sha256"] == evidence["verification_sha256"], + "RESTORATION_RECEIPT_BINDING", + ) + # T1's pinned producer only appends the observed person-income column to an + # owned copy of the authenticated parent. Recover that exact parent view + # without reading another source or changing the original T1 authority. + # The sidecar must describe those reconstructed facts, not merely contain + # syntactically valid digest strings beside an authentic checkpoint. + source.validate() + observed = source.frame.person[person_income.OBSERVED_COLUMN] + _require(observed.dtype == np.dtype("int64"), "RESTORATION_OBSERVATION_DTYPE") + _require( + receipt["observation_sha256"] + == _sha(observed.to_numpy(dtype=" str: + try: + return token_for_dtype(dtype) + except Exception as error: # PopulationError is a ValueError subclass + del error + name = getattr(dtype, "name", None) + if isinstance(dtype, pd.api.extensions.ExtensionDtype): + _require(name in _NULLABLE_INTEGER_TARGETS, "UNPROMOTABLE_DTYPE") + return "Int64" + normalized = np.dtype(dtype) + _require( + normalized.name in _INTEGER_WIDTH_TARGETS or normalized.name == "float16", + "UNPROMOTABLE_DTYPE", + ) + return "float64" if normalized.name == "float16" else "int64" + + +def _promote(frame: Frame) -> tuple[dict[str, str], ...]: + """Promote every column to its graph dtype token, value- and null-preserving.""" + canonicalize_frame_string_dtypes( + frame, boundary=PREPARATION_BOUNDARY, in_place=True + ) + transitions: list[dict[str, str]] = [] + for entity in US_SCHEMA.entities: + table = frame.table(entity) + for column in list(table.columns): + _require(isinstance(column, str), "COLUMN_NAME") + original = table[column] + token = _target_token(original.dtype) + target = dtype_for_token(token) + if original.dtype == target: + continue + promoted = original.astype(target) + _require(original.isna().equals(promoted.isna()), "DTYPE_PROMOTION_NULLS") + observed = original.notna() + _require( + np.array_equal( + original[observed].to_numpy(dtype=object), + promoted[observed].to_numpy(dtype=object), + ), + "DTYPE_PROMOTION_VALUES", + ) + table[column] = promoted + transitions.append( + { + "entity": entity, + "column": column, + "from": str(original.dtype), + "to": token, + } + ) + for entity in US_SCHEMA.entities: + table = frame.table(entity) + for column in table.columns: + token_for_dtype(table[column].dtype) + return tuple(transitions) + + +def _entity_rows(frame: Frame) -> dict[str, int]: + return {entity: frame.n(entity) for entity in US_SCHEMA.entities} + + +def prepared_source_implementation_identity() -> str: + """Bind this preparation's own code plus the reviewed loaders it calls.""" + package = resources.files(__package__) + modules = { + name: _sha(package.joinpath(name).read_bytes()) + for name in ( + "asec_prepared_source.py", + "asec_income_observations.py", + "asec_current_money_selection.py", + "cps_carried_current.py", + "graph_housing_universe.py", + ) + } + return _sha( + _json( + { + "schema": 3, + "source_kind": PREPARED_SOURCE_KIND, + "modules": modules, + "money_source_verification": money_source._verification_identity(), + "restoration_verification": person_income._implementation(), + "student_controls_verification": student_source._implementation(), + "income_observations_verification": income_source._implementation(), + "housing_verification": housing_source._implementation(), + "housing_universe_verification": housing_universe_source._implementation(), + "file_roster": list(PREPARED_SOURCE_FILES), + "boundary": PREPARATION_BOUNDARY, + } + ) + ) + + +@dataclass(frozen=True) +class PreparedAsecPopulation: + """An operator-free prepared frame with its money body and identity receipt.""" + + frame: Frame + money_payload: bytes + receipt: dict + field_entities: tuple[tuple[str, str], ...] + housing_universe_payload: bytes + income_observations_payload: bytes + + @property + def receipt_payload(self) -> bytes: + return _json(self.receipt) + + @property + def receipt_sha256(self) -> str: + return _sha(self.receipt_payload) + + +def prepare_asec_current_money_population(source_dir) -> PreparedAsecPopulation: + """Load, reconstruct, attach and promote the full authenticated population.""" + paths = _roster(source_dir) + identity = prepared_source_implementation_identity() + source = person_income.load_authenticated_restored_current_money_source( + paths[PARENT_FILENAME], + paths[HOUSEHOLD_ATTACHMENT_FILENAME], + paths[PERSON_INCOME_FILENAME], + member_paths={year: paths[name] for year, name in MEMBER_FILENAMES}, + ) + evidence = _parse(source.source.identity) + _require(evidence["source_kind"] == RESTORED_SOURCE_KIND, "RESTORED_SOURCE_KIND") + _require(evidence["zero_origin_policy"] == ZERO_POLICY, "ZERO_ORIGIN_POLICY") + _require(evidence["field_roster"] == list(FIELDS), "FIELD_ROSTER") + restoration = _restoration_receipt( + paths[RESTORATION_RECEIPT_FILENAME], evidence, source + ) + ready = source.ready() + _require(type(ready) is ReadyCurrentMoney, "AUTHENTICATED_READINESS_REQUIRED") + income = income_source.load_authenticated_income_observations( + source, + ready, + member_paths={year: paths[name] for year, name in MEMBER_FILENAMES}, + ) + # S is separate source authority. Its public loader authenticates the same + # official members against the unchanged T1 and money parents; attachment + # adds two aliases while retaining original missing controls untouched. + controls = student_source.load_authenticated_student_controls( + source, + ready, + member_paths={year: paths[name] for year, name in MEMBER_FILENAMES}, + ) + with_students = student_source.attach_student_controls(source, ready, controls) + tax = reconstruct_current_money_tax_units(with_students, ready) + # Housing still authenticates original T1, never the transient tax/S view. + status = load_authenticated_housing_status( + source, cohort_paths={year: paths[name] for year, name in COHORT_FILENAMES} + ) + attached = attach_housing_status(tax, status) + attached.validate() + # HU authenticates original T1 and retains the complete S/tax/housing parent. + # Its strict uint8 attachment validation runs before graph width promotion. + universe = housing_universe_source.load_authenticated_housing_universe(source) + with_universe = housing_universe_source.attach_housing_universe(attached, universe) + with_universe.validate() + frame = housing_source._owned_frame(with_universe.frame) + pre_promotion = money_source._frame_signature(frame) + _require( + pre_promotion == with_universe.receipt["output_frame_sha256"], "PARENT_CAPTURE" + ) + transitions = _promote(frame) + post_promotion = money_source._frame_signature(frame) + # The graph carries metadata as a typed context artifact, and the store's + # frame codec does not persist Frame.metadata at all. A non-empty prepared + # metadata mapping would be silently dropped, so refuse instead. + _require(dict(frame.metadata) == {}, "PREPARED_FRAME_METADATA") + _require(frame.mass_log == (), "PREPARED_FRAME_MASS_LOG") + _require(frame.weighted_entities == ("household",), "PREPARED_FRAME_WEIGHTS") + frame.revalidate() + assert_operator_free_source_frame(frame, label=PREPARATION_BOUNDARY) + _require( + prepared_source_implementation_identity() == identity, + "PREPARATION_IMPLEMENTATION_CHANGED", + ) + money_payload = encode_current_money(ready) + universe_payload = housing_universe.encode_housing_universe(universe) + income_payload = income_source.encode_income_observations(income) + field_entities = tuple( + (domain.name, domain.entity) for domain in ready.bindings.spec.fields + ) + receipt = { + "schema": PREPARED_RECEIPT_SCHEMA, + "source_kind": PREPARED_SOURCE_KIND, + "restored_source_kind": evidence["source_kind"], + "release_eligible": False, + "all_current_money_consumers_wired": False, + "source_evidence_sha256": _sha(source.source.identity), + "t1_sha256": evidence["person_income_attachment_sha256"], + "income_observations": { + "parser_profile": income_source.parser_profile(), + "header_sha256": _sha(income._header), + "content_sha256": income.content_sha256, + "payload_sha256": _sha(income_payload), + "rows": income.receipt["rows"], + "columns": list(income_source.COLUMNS), + "kind": income_source.ARTIFACT_KIND, + "scope_sha256": income.receipt["scope_sha256"], + }, + "student_controls_receipt_sha256": _sha(_json(controls.receipt)), + "student_controls_content_sha256": controls.content_sha256, + "student_controls_attachment_receipt_sha256": _sha( + _json(with_students.receipt) + ), + "student_controls_reference_period": controls.receipt["reference_period"], + "annual_five_month_student_status_validated": False, + "money_header_sha256": _sha(ready.header), + "money_content_sha256": current_money_content_sha256(ready), + "money_spec_sha256": ready.bindings.spec.sha256, + "field_entities": [list(item) for item in field_entities], + "scope_sha256": _parse(ready.header)["scope_sha256"], + "tax_receipt_sha256": _sha(_json(tax.receipt)), + "tax_old_partition_sha256": tax.receipt["old_partition_sha256"], + "tax_new_partition_sha256": tax.receipt["new_partition_sha256"], + "housing_content_sha256": status.content_sha256, + "housing_attachment_receipt_sha256": _sha(_json(attached.receipt)), + "pre_housing_universe_frame_sha256": attached.receipt["output_frame_sha256"], + "housing_universe": { + "header_sha256": _sha(universe.header), + "content_sha256": universe.content_sha256, + "payload_sha256": _sha(universe_payload), + "definition_sha256": universe.header_data["definition_sha256"], + "attachment_receipt_sha256": _sha(_json(with_universe.receipt)), + "parent_kind": "HousingStatusAttachedAsec", + "source_period_kind": "interview_household_universe", + "native_dtype": "uint8", + "graph_dtype": "int64", + "aliases": list(housing_universe_source.ATTACHED_COLUMNS), + }, + "restoration_receipt_sha256": _sha(_json(restoration)), + "pre_promotion_frame_sha256": pre_promotion, + "post_promotion_frame_sha256": post_promotion, + "dtype_transitions": [dict(item) for item in transitions], + "entity_rows": _entity_rows(frame), + "implementation_sha256": identity, + "file_roster": list(PREPARED_SOURCE_FILES), + } + return PreparedAsecPopulation( + frame, money_payload, receipt, field_entities, universe_payload, income_payload + ) + + +def load_graph_asec_prepared(path, *, store=None) -> Frame: + """Registered source codec: the prepared operator-free population only. + + The executor verifies that this loader is installed and content-keys the + directory; it never calls it. The declared computation that invokes the + preparation is the CREATE kernel, which needs the money body and receipt + this signature cannot return, so the two paths share one implementation + rather than preparing the population twice. + """ + del store + return prepare_asec_current_money_population(path).frame + + +__all__ = [ + "COHORT_FILENAMES", + "HOUSEHOLD_ATTACHMENT_FILENAME", + "MEMBER_FILENAMES", + "PARENT_FILENAME", + "PERSON_INCOME_FILENAME", + "PREPARATION_BOUNDARY", + "PREPARED_RECEIPT_SCHEMA", + "PREPARED_SOURCE_FILES", + "PREPARED_SOURCE_KIND", + "PreparedAsecPopulation", + "PreparedSourceRefusalError", + "RESTORATION_RECEIPT_FILENAME", + "load_graph_asec_prepared", + "prepare_asec_current_money_population", + "prepared_source_implementation_identity", +] diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/asec_raw_stage_v4.py b/packages/microcosm-build/src/microcosm/build/us_runtime/asec_raw_stage_v4.py new file mode 100644 index 000000000..5e60f0a0e --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/asec_raw_stage_v4.py @@ -0,0 +1,194 @@ +"""Explicit local v3-to-v4 exact-source restoration, never schema relabeling.""" + +from __future__ import annotations + +import argparse +import copy +import hashlib +import json +import os +import platform +import re +import tempfile +from collections.abc import Mapping +from importlib.metadata import version +from pathlib import Path + +from microcosm.build.frame_checkpoint import write_frame_checkpoint +from microcosm.build.us_runtime import asec_checkpoint, reported_coverage_source +from microcosm.frame import Frame + + +def _file_sha256(path: Path) -> str: + with path.open("rb") as stream: + return hashlib.file_digest(stream, "sha256").hexdigest() + + +def restore_asec_raw_stage_v4( + checkpoint_path: str | Path, + *, + expected_sha256: str, + coverage_paths: Mapping[int, str | Path], + output_dir: str | Path, +) -> dict[str, object]: + """Authenticate and restore all seven recodes into a new local bundle. + + The input is strictly v3 and operator-untouched. Every pooled income year + must have an explicit local official member or ZIP; this API never fetches. + Existing nonmissing recodes must agree with that source. The output directory + must not exist, so neither historical artifacts nor receipts are overwritten. + + Whole-file hashes authenticate immutable local inputs relative to caller pins; + they do not establish trust in an unknown producer. Structural Frame identity + alone does not bind person values. Source code and serialization versions are + recorded, but no country model runs and no model accuracy is certified here. + """ + source_path = Path(checkpoint_path).expanduser().resolve() + destination = Path(output_dir).expanduser().resolve() + if destination.exists(): + raise FileExistsError(destination) + if not isinstance(expected_sha256, str) or not re.fullmatch( + r"[0-9a-f]{64}", expected_sha256 + ): + raise ValueError("expected_sha256 must be a lowercase SHA-256 digest") + if _file_sha256(source_path) != expected_sha256: + raise ValueError("ASEC v3 input SHA-256 mismatch") + frame, metadata = asec_checkpoint.load_asec_raw_stage_checkpoint(source_path) + if _file_sha256(source_path) != expected_sha256: + raise ValueError("ASEC v3 input SHA-256 changed during loading") + years = tuple(sorted({int(year) for year in frame.table("person")["source_year"]})) + receipt_years = {source["year"] for source in metadata["source_receipt"]["sources"]} + if set(years) != receipt_years: + raise ValueError("ASEC v3 frame/source receipt year coverage differs") + if set(coverage_paths) != set(years) or any( + type(year) is not int or not isinstance(path, (str, Path)) + for year, path in coverage_paths.items() + ): + raise ValueError( + "explicit local coverage paths must cover exactly the pooled years" + ) + sidecar = reported_coverage_source.load_asec_reported_coverage_sources( + coverage_paths, income_years=years + ) + person = reported_coverage_source.fill_asec_reported_coverage_source( + frame.table("person"), sidecar + ) + restored = Frame( + { + entity: person if entity == "person" else frame.table(entity) + for entity in frame.entities + }, + frame.schema, + {entity: frame.weights_for(entity) for entity in frame.weighted_entities}, + frame.strata, + mass_log=frame.mass_log, + metadata=frame.metadata, + ) + binding = copy.deepcopy(metadata) + binding["schema_version"] = asec_checkpoint.ASEC_RAW_STAGE_COVERAGE_SCHEMA_VERSION + pins = reported_coverage_source.ASEC_EDUCATION_ASSISTANCE_ARCHIVES + source_pins = [ + { + "income_year": year, + "locator": pins[year].zip_url, + "member": pins[year].member, + "sha256": pins[year].zip_sha256, + "member_sha256": pins[year].member_sha256, + } + for year in years + ] + audit = sidecar.attrs["source_audit"] + for column in reported_coverage_source.ASEC_REPORTED_COVERAGE_RAW_COLUMNS: + binding["raw_source_mappings"][column] = { + "column": column, + "entity": "person", + "operation": "exact_source_join", + "join_keys": ["source_year", "PERIDNUM"], + "source_pins": source_pins, + "audit": { + str(year): { + "rows": audit[year]["rows"], + **audit[year]["columns"][column], + } + for year in years + }, + } + destination.parent.mkdir(parents=True, exist_ok=True) + with tempfile.TemporaryDirectory( + prefix=".asec-v4-", dir=destination.parent + ) as temp: + temporary = Path(temp) + output_path = temporary / asec_checkpoint.ASEC_RAW_STAGE_CHECKPOINT_FILENAME + write_frame_checkpoint(output_path, restored, metadata=binding) + # Validate the actual serialization, not only the in-memory construction. + checked, checked_binding = asec_checkpoint.load_asec_raw_stage_checkpoint_v4( + output_path + ) + del checked, checked_binding + implementations = { + Path(module.__file__).name: _file_sha256(Path(module.__file__)) + for module in (asec_checkpoint, reported_coverage_source) + } + implementations[Path(__file__).name] = _file_sha256(Path(__file__)) + receipt = { + "schema": "microcosm.asec_raw_stage_restoration.v1", + "input_schema_version": 3, + "output_schema_version": 4, + "input_sha256": expected_sha256, + "output_sha256": _file_sha256(output_path), + "output_file": output_path.name, + "operation": "authenticated_exact_source_join", + "source_pins": source_pins, + "source_years": list(years), + "person_rows": len(person), + "implementation_sha256": implementations, + "runtime_versions": { + "python": platform.python_version(), + **{ + name: version(name) + for name in ("microcosm-build", "pandas", "numpy", "h5py") + }, + }, + "model_execution": False, + "release_certification": False, + "pipeline_lineage_sha256": metadata["pipeline_sha256"], + } + (temporary / "restoration.receipt.json").write_text( + json.dumps(receipt, sort_keys=True, indent=2, allow_nan=False) + "\n" + ) + if destination.exists(): + raise FileExistsError(destination) + os.rename(temporary, destination) + return receipt + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--input", type=Path, required=True) + parser.add_argument("--expected-sha256", required=True) + parser.add_argument( + "--coverage", action="append", required=True, metavar="YEAR=LOCAL_PATH" + ) + parser.add_argument("--output-dir", type=Path, required=True) + args = parser.parse_args(argv) + paths = {} + for item in args.coverage: + year_text, separator, path = item.partition("=") + if not separator or not year_text.isdecimal() or not path: + parser.error("--coverage must be YEAR=LOCAL_PATH") + year = int(year_text) + if year in paths: + parser.error("--coverage repeats a year") + paths[year] = Path(path) + receipt = restore_asec_raw_stage_v4( + args.input, + expected_sha256=args.expected_sha256, + coverage_paths=paths, + output_dir=args.output_dir, + ) + print(json.dumps(receipt, sort_keys=True, indent=2)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/asec_student_controls.py b/packages/microcosm-build/src/microcosm/build/us_runtime/asec_student_controls.py new file mode 100644 index 000000000..ddf127896 --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/asec_student_controls.py @@ -0,0 +1,511 @@ +"""Closed Census interview-week controls, separate from the T1 money source. + +These observations do not establish annual or five-month tax-student status. +Candidate bytes never supply parsed content or authority: replay reconstructs S +from the authenticated T1 parent and the code-owned official CSV members. +""" + +from __future__ import annotations + +import csv +import hashlib +import os +import stat +import struct +import tempfile +from collections.abc import Mapping +from dataclasses import InitVar, dataclass +from importlib import metadata, resources +from pathlib import Path + +import numpy as np +import pandas as pd + +from microcosm.frame import Frame + +from . import asec_current_money_source as legacy +from . import asec_person_income_source as restoration +from ._asec_current_money_codec import current_money_content_sha256 +from .asec_current_money import ( + RESTORED_SOURCE_KIND, + MoneyRefusalError, + ReadyCurrentMoney, + _json, + _parse, + _require, + _sha, + _validate_money, +) +from .education_assistance_source import ASEC_EDUCATION_ASSISTANCE_ARCHIVES + +CONTROLS = ("A_ENRLW", "A_FTPT") +ALIASES = ("asec_A_ENRLW", "asec_A_FTPT") +COORDINATES = ("person_id", "income_year", "source_household_id", "A_LINENO", "A_AGE") +COLUMNS = COORDINATES + CONTROLS +_READ_COLUMNS = ("PERIDNUM", "PH_SEQ", "A_LINENO", "A_AGE") + CONTROLS +_MEMBER_PINS = tuple( + (year, p.member, p.zip_sha256, p.member_sha256, p.rows, p.member_size_bytes) + for year, p in sorted(ASEC_EDUCATION_ASSISTANCE_ARCHIVES.items()) +) +MAGIC = b"MCASTUD\x01" +FILENAME = "asec_student_controls.bin" +_HEADER_MAX = 65536 +_MAX_PERSONS = 600_000 +_TOKEN = object() +_COMPOSE_TOKEN = object() + + +def _implementation(): + package = resources.files(__package__) + return { + "schema": 1, + "t1_verification_sha256": restoration._implementation(), + "modules": { + name: _sha(package.joinpath(name).read_bytes()) + for name in ("asec_student_controls.py", "_asec_current_money_codec.py") + }, + "dependencies": {n: metadata.version(n) for n in ("numpy", "pandas")}, + "member_pins": _MEMBER_PINS, + "aliases": dict(zip(CONTROLS, ALIASES, strict=True)), + } + + +def _parent(source, ready, *, reconstruct_money=False): + _require( + type(source) is legacy.AuthenticatedCurrentMoneySource + and type(ready) is ReadyCurrentMoney, + "STUDENT_AUTHENTICATED_PARENT", + ) + source.validate() + _require( + _parse(source.source.identity)["source_kind"] == RESTORED_SOURCE_KIND, + "STUDENT_RESTORED_PARENT_REQUIRED", + ) + _require(ready.bindings.spec == source.spec, "STUDENT_MONEY_PARENT") + _validate_money(ready, source.spec, nominal=False) + if reconstruct_money: + expected = source.ready() + _require( + ready.bindings == expected.bindings and ready.fields == expected.fields, + "STUDENT_MONEY_PARENT", + ) + + +def _snapshot(path, destination, *, size): + """Exact-size private capture; no candidate parser and no unbounded copy.""" + digest = hashlib.sha256() + # Opening a FIFO in ordinary blocking mode would wait before fstat could + # reject it. Use one nonblocking descriptor, then admit only regular files. + descriptor = os.open(Path(path), os.O_RDONLY | os.O_NONBLOCK) + try: + before = os.fstat(descriptor) + _require(stat.S_ISREG(before.st_mode), "STUDENT_SOURCE_FILE_KIND") + _require(before.st_size == size, "STUDENT_SOURCE_SIZE") + count = 0 + with ( + os.fdopen(descriptor, "rb", closefd=False) as src, + destination.open("xb") as dst, + ): + while chunk := src.read(min(1024 * 1024, size - count + 1)): + count += len(chunk) + _require(count <= size, "STUDENT_SOURCE_SIZE") + digest.update(chunk) + dst.write(chunk) + after = os.fstat(descriptor) + attrs = ("st_dev", "st_ino", "st_size", "st_mtime_ns", "st_ctime_ns") + _require( + count == size + and all(getattr(before, a) == getattr(after, a) for a in attrs), + "STUDENT_SOURCE_CHANGED", + ) + finally: + os.close(descriptor) + return digest.hexdigest() + + +def _read_member(path, *, rows): + with path.open("r", encoding="utf-8", newline="") as handle: + header = next(csv.reader(handle), []) + _require( + len(header) == len(set(header)) and set(_READ_COLUMNS) <= set(header), + "STUDENT_MEMBER_HEADER", + ) + table = pd.read_csv( + path, usecols=list(_READ_COLUMNS), dtype="string", na_filter=False + ) + _require(len(table) == rows, "STUDENT_MEMBER_ROWS") + _require( + bool(table.PERIDNUM.str.fullmatch(r"[0-9]{22}").all()), + "STUDENT_MEMBER_PERSON_KEY", + ) + for name in _READ_COLUMNS[1:]: + _require( + bool(table[name].str.fullmatch(r"[0-9]+").all()), + "STUDENT_MEMBER_INTEGER", + name, + ) + table[name] = table[name].astype("int64") + _require( + not table.PERIDNUM.duplicated().any() + and not table.duplicated(["PH_SEQ", "A_LINENO"]).any() + and bool((table.PH_SEQ > 0).all()) + and bool((table.A_LINENO > 0).all()), + "STUDENT_MEMBER_COORDINATES", + ) + for name in CONTROLS: + _require(bool(table[name].isin([0, 1, 2]).all()), "STUDENT_CODE_DOMAIN", name) + _require( + bool((table.A_ENRLW[~table.A_AGE.between(16, 54)] == 0).all()) + and bool((table.A_FTPT[table.A_ENRLW == 1] != 0).all()) + and bool((table.A_FTPT[table.A_ENRLW != 1] == 0).all()), + "STUDENT_SOURCE_UNIVERSE", + ) + return table + + +@dataclass(frozen=True) +class AuthenticatedStudentControls: + """Immutable owned numeric buffers; only closed source reconstruction issues S.""" + + _header: bytes + _body: bytes + _token: InitVar[object] = None + + def __post_init__(self, _token): + _require(_token is _TOKEN, "STUDENT_CONSTRUCTOR_UNAVAILABLE") + self.validate() + + @property + def receipt(self): + return _parse(self._header) + + @property + def content_sha256(self): + return _sha(self._header + self._body) + + def validate(self): + _require( + type(self._header) is bytes and 0 < len(self._header) <= _HEADER_MAX, + "STUDENT_HEADER_SIZE", + ) + header = self.receipt + rows = header["rows"] + _require(type(rows) is int and 0 < rows <= _MAX_PERSONS, "STUDENT_ROWS") + _require( + type(self._body) is bytes + and len(self._body) == rows * len(COLUMNS) * 8 + and _sha(self._body) == header["body_sha256"] + and header["columns"] == list(COLUMNS) + and header["aliases"] == dict(zip(CONTROLS, ALIASES, strict=True)) + and _json(header["implementation"]) == _json(_implementation()), + "STUDENT_CONTENT_CHANGED", + ) + + def array(self, name): + self.validate() + _require(name in COLUMNS, "STUDENT_COLUMN") + n = self.receipt["rows"] + offset = COLUMNS.index(name) * n * 8 + # A bytes-backed buffer cannot be made writeable by the caller. + return np.frombuffer(self._body, dtype="= 0).all()) and len(np.unique(indices)) == rows, + "STUDENT_KEY_COVERAGE", + ) + joined = raw.iloc[indices] + for name, raw_name in ( + ("source_household_id", "PH_SEQ"), + ("A_LINENO", "A_LINENO"), + ("A_AGE", "A_AGE"), + ): + _require( + np.array_equal( + output[name][positions], joined[raw_name].to_numpy() + ), + "STUDENT_NATIVE_KEY_OR_AGE", + ) + compared = {} + for name in CONTROLS: + incumbent = person[name].iloc[positions] + known = ~incumbent.isna().to_numpy() + observed = joined[name].to_numpy(dtype=np.int64) + _require( + np.array_equal( + incumbent.to_numpy(dtype=np.float64, na_value=np.nan)[known], + observed[known], + ), + "STUDENT_INCUMBENT_CONFLICT", + name, + ) + if year == 2024: + _require(bool(known.all()), "STUDENT_INCUMBENT_INCOMPLETE", name) + output[name][positions] = observed + compared[name] = int(known.sum()) + joins.append( + { + "income_year": year, + "survey_year": year + 1, + "member": member, + "archive_sha256": archive_pin, + "member_sha256": pin, + "source_rows": rows, + "joined_rows": len(positions), + "unreferenced_source_rows": 0, + "incumbent_compared_rows": compared, + "incumbent_conflicts": 0, + "native_key_or_age_conflicts": 0, + } + ) + _parent(source, ready) + _require( + _json(_implementation()) == _json(before), "STUDENT_IMPLEMENTATION_CHANGED" + ) + buffers = [output[name].astype(" tuple[bytes, bytes]: + """Normalize exact original API responses and mapping bytes, with a receipt. + + Each ordered item is ``(state, block_response, single_state_total_response)``. + There is exactly one item per selected state, drawn from the 50 states plus + DC. Expected API requests appear in the descriptive receipt with repeated + ``in`` parameters preserved and no credential parameter. The caller owns + acquisition and publisher qualification; the response cannot prove its URL. + + Response byte limits precede JSON parsing; the row limit is enforced after + JSON allocation. CD ZIP metadata and inflation follow the existing bounded + source reader. All rows, including zero-population blocks, are counted; + every positive block is retained exactly once and reconciled to its separate + state response. No county/tract aggregation or synthetic PL file is used. + """ + _require( + type(state_fips) is tuple + and bool(state_fips) + and all( + type(state) is str and state in sources._GEO_MEMBERS for state in state_fips + ) + and state_fips == tuple(sorted(set(state_fips))), + "STATE_ROSTER", + ) + identities = normalized._sources(source_ids) + bounds = { + "max_source_bytes": max_source_bytes, + "max_member_bytes": max_member_bytes, + "max_total_geography_bytes": max_total_geography_bytes, + "max_zip_members": max_zip_members, + "max_line_bytes": max_line_bytes, + "max_response_rows": max_response_rows, + } + _require( + all(type(value) is int and 0 < value <= 2**63 - 1 for value in bounds.values()), + "BOUNDS", + ) + expanded = [0] + puma_payload, puma_provenance = _raw_source( + tract_to_puma, bounds=bounds, expanded=expanded + ) + cd_payload, cd_provenance = sources._selected_zip_member( + cd_archive, sources.CD_MEMBER, bounds=bounds, expanded=expanded + ) + cd_statistics, unassigned = {"source_records": 0, "delegate_records": 0}, set() + districts = parse_national_cd_bef( + sources._cd_lines( + cd_payload, + max_line_bytes=max_line_bytes, + statistics=cd_statistics, + unassigned=unassigned, + ) + ) + _require(not unassigned.intersection(districts), "CD_CONFLICTING_UNASSIGNED") + _require( + cd_statistics["source_records"] == len(districts) + len(unassigned), + "CD_RECONCILIATION", + ) + cd_provenance.update( + source_id=identities["district"], + **cd_statistics, + assigned_blocks=len(districts), + unassigned_blocks=len(unassigned), + delegate_normalization="98_to_00", + district_relation="official_tabulation", + ) + del cd_payload, unassigned + pumas = parse_tract_to_puma_relationship( + sources._lines(puma_payload, encoding="utf-8", max_line_bytes=max_line_bytes), + allowed_state_fips=frozenset(state_fips), + ) + puma_provenance.update( + source_id=identities["puma"], selected_state_tract_mappings=len(pumas) + ) + del puma_payload + + iterator, exhausted = iter(population_responses), object() + population, population_provenance, state_totals = {}, [], {} + for state in state_fips: + item = next(iterator, exhausted) + _require( + type(item) is tuple and len(item) == 3 and item[0] == state, + "POPULATION_SOURCE_ROSTER", + ) + block_payload, block_provenance = _raw_source( + item[1], bounds=bounds, expanded=expanded + ) + total_payload, total_provenance = _raw_source( + item[2], bounds=bounds, expanded=expanded + ) + blocks, statistics = _state_population( + block_payload, total_payload, state=state, max_rows=max_response_rows + ) + _require(not population.keys() & blocks.keys(), "OVERLAPPING_STATES") + population.update(blocks) + state_totals[state] = statistics["state_population"] + population_provenance.append( + { + "source_id": identities["population"], + "state_fips": state, + "blocks": {**block_provenance, "request": _request(state, blocks=True)}, + "state_total": { + **total_provenance, + "request": _request(state, blocks=False), + }, + **statistics, + } + ) + del item, block_payload, total_payload, blocks + _require(next(iterator, exhausted) is exhausted, "POPULATION_SOURCE_ROSTER") + payload = normalized.assemble_atomic_block_support( + block_population=population, + cd_by_block=districts, + puma_by_tract=pumas, + source_ids=identities, + ) + _require(len(payload) <= RAW_BYTES_MAX_BYTES, "SUPPORT_BYTES") + support = decode_atomic_support(payload) + output_totals = dict.fromkeys(state_fips, 0) + for state, value in zip( + support.arrays["state"], support.arrays["population"], strict=True + ): + output_totals[str(state)] += int(value) + ordered_blocks = sorted(population) + _require( + support.arrays["area"].tolist() == [f"{block:015d}" for block in ordered_blocks] + and support.arrays["population"].tolist() + == [population[block] for block in ordered_blocks] + and support.arrays["puma"].tolist() + == [f"{pumas[block // 10000]:07d}" for block in ordered_blocks] + and support.arrays["district"].tolist() + == [f"{districts[block]:04d}" for block in ordered_blocks] + and output_totals == state_totals + and len(support.arrays["area"]) == len(population), + "OUTPUT_RECONCILIATION", + ) + receipt = canonical_json( + { + "protocol": PROTOCOL, + "state_fips": list(state_fips), + "source_ids": identities, + "limits": bounds, + "sources": { + "population": population_provenance, + "district": cd_provenance, + "puma": puma_provenance, + }, + "reconciliation": { + "state_population": state_totals, + "output_state_population": output_totals, + "population_total": sum(state_totals.values()), + "populated_blocks": len(population), + "selected_geography_bytes": expanded[0], + }, + "support_sha256": sources._sha(payload), + "exact_input_hashes_verified": True, + "publisher_provenance_established": False, + "request_origin_verified": False, + "source_admission_issued": False, + "release_eligible": False, + } + ) + return payload, receipt diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/atomic_block_sources.py b/packages/microcosm-build/src/microcosm/build/us_runtime/atomic_block_sources.py new file mode 100644 index 000000000..aea5b8366 --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/atomic_block_sources.py @@ -0,0 +1,411 @@ +"""Pinned Census source bytes to normalized atomic-block support, without I/O. + +The caller establishes publisher identity and obtains the bytes. This adapter +checks those bytes against explicit pins, checks full ZIP member rosters, and +decompresses only the selected geography members. Unselected PL segment contents +are neither opened nor CRC-checked. No source or release admission is issued. +""" + +from __future__ import annotations + +import hashlib +import re +from collections.abc import Iterable, Mapping +from dataclasses import dataclass +from io import BytesIO +from pathlib import PurePosixPath +from zipfile import ZIP_DEFLATED, ZIP_STORED, BadZipFile, ZipFile +from zlib import error as zlib_error + +from microcosm.build.atomic_geography import decode_atomic_support +from microcosm.graph.canonical import canonical_json +from microcosm.graph.codecs import RAW_BYTES_MAX_BYTES + +from . import atomic_block_support as normalized +from .block_ladder_sources import ( + US_STATES, + parse_national_cd_bef, + parse_pl_geo_blocks, +) +from .puma_ladder_sources import parse_tract_to_puma_relationship + +PROTOCOL = "microcosm.us.atomic-block-sources.v1" +CD_MEMBER = "NationalCD119.txt" +_GEO_MEMBERS = {state: f"{usps.lower()}geo2020.pl" for state, usps, _ in US_STATES} +_CHUNK_BYTES = 64 * 1024 + + +@dataclass(frozen=True, slots=True) +class AtomicBlockSourceBytes: + """Literal bytes and expected digest/roster; construction conveys no authority.""" + + payload: bytes + sha256: str + zip_members: tuple[str, ...] = () + + +def _require(condition, reason): + if not condition: + raise ValueError("ATOMIC_BLOCK_SOURCES_" + reason) + + +def _sha(payload): + return hashlib.sha256(payload).hexdigest() + + +def _checked_source(record, *, archive, max_source_bytes): + _require(type(record) is AtomicBlockSourceBytes, "SOURCE_TYPE") + payload, digest, members = record.payload, record.sha256, record.zip_members + _require( + type(payload) is bytes and 0 < len(payload) <= max_source_bytes, + "SOURCE_BYTES", + ) + _require( + type(digest) is str + and re.fullmatch(r"[0-9a-f]{64}", digest) is not None + and _sha(payload) == digest, + "SOURCE_DIGEST", + ) + _require(type(members) is tuple, "MEMBER_ROSTER") + if archive: + _require( + bool(members) + and all( + type(name) is str + and 0 < len(name) <= 255 + and "\0" not in name + and "\\" not in name + and not name.endswith("/") + and not PurePosixPath(name).is_absolute() + and all(part not in {"", ".", ".."} for part in name.split("/")) + for name in members + ) + and len(set(members)) == len(members), + "MEMBER_ROSTER", + ) + else: + _require(members == (), "RAW_MEMBER_ROSTER") + return payload, digest, members + + +def _selected_zip_member(record, member, *, bounds, expanded): + source_payload, source_digest, expected_members = _checked_source( + record, archive=True, max_source_bytes=bounds["max_source_bytes"] + ) + provenance = {"sha256": source_digest, "size_bytes": len(source_payload)} + _require( + member in expected_members + and len(expected_members) <= bounds["max_zip_members"], + "SELECTED_MEMBER", + ) + try: + with ZipFile(BytesIO(source_payload)) as archive: + members = archive.infolist() + names = tuple(info.filename for info in members) + _require( + len(names) == len(expected_members) + and len(set(names)) == len(names) + and all(info.orig_filename == info.filename for info in members) + and set(names) == set(expected_members), + "ZIP_MEMBER_ROSTER", + ) + info = archive.getinfo(member) + _require( + not info.is_dir() + and not info.flag_bits & 1 + and info.compress_type in {ZIP_STORED, ZIP_DEFLATED}, + "ZIP_MEMBER_FORMAT", + ) + _require( + 0 < info.file_size <= bounds["max_member_bytes"], "ZIP_MEMBER_SIZE" + ) + _require( + expanded[0] + info.file_size <= bounds["max_total_geography_bytes"], + "TOTAL_GEOGRAPHY_BYTES", + ) + # ZipExtFile validates the selected member's CRC when exhausted. + # No other member is opened, even to check its CRC. + output = BytesIO() + with archive.open(info) as stream: + while chunk := stream.read(_CHUNK_BYTES): + _require( + output.tell() + len(chunk) <= info.file_size, + "ZIP_MEMBER_SIZE", + ) + output.write(chunk) + payload = output.getvalue() + _require(len(payload) == info.file_size, "ZIP_MEMBER_SIZE") + expanded[0] += len(payload) + provenance.update( + { + "zip_members": sorted(names), + "selected_member": member, + "selected_member_sha256": _sha(payload), + "selected_member_size_bytes": len(payload), + "crc_checked_members": [member], + "unselected_member_contents_read": False, + } + ) + except (BadZipFile, EOFError, RuntimeError, NotImplementedError, zlib_error): + raise ValueError("ATOMIC_BLOCK_SOURCES_ZIP_INVALID") from None + return payload, provenance + + +def _lines(payload, *, encoding, max_line_bytes): + stream = BytesIO(payload) + while line := stream.readline(max_line_bytes + 1): + _require(len(line) <= max_line_bytes and b"\0" not in line, "TEXT_LINE") + yield line.decode(encoding) + + +def _pl_lines(payload, *, state, max_line_bytes, statistics): + """Guard parser skips: count every block, including zero-population rows.""" + seen = set() + for line in _lines(payload, encoding="latin-1", max_line_bytes=max_line_bytes): + if not line.strip(): + continue + fields = line.rstrip("\r\n").split("|") + _require(len(fields) == 97, "PL_ROW_WIDTH") + statistics["geography_rows"] += 1 + level = fields[2] + if level in {"040", "750"}: + raw = fields[90].strip() + _require(re.fullmatch(r"[0-9]+", raw) is not None, "PL_POPULATION") + population = int(raw) + if level == "040": + statistics["state_rows"] += 1 + _require(statistics["state_rows"] == 1, "PL_STATE_ROWS") + statistics["state_population"] = population + else: + geoid = fields[9].strip() + _require( + re.fullmatch(r"[0-9]{15}", geoid) is not None + and geoid.startswith(state), + "PL_BLOCK_STATE", + ) + _require(geoid not in seen, "PL_DUPLICATE_BLOCK") + seen.add(geoid) + statistics["block_rows"] += 1 + statistics["zero_population_blocks"] += int(population == 0) + yield line + + +def _cd_lines(payload, *, max_line_bytes, statistics, unassigned): + first = True + for line in _lines(payload, encoding="latin-1", max_line_bytes=max_line_bytes): + if not line.strip(): + continue + if first: + first = False + else: + parts = line.strip().split(",") + _require(len(parts) == 2, "CD_ROW_WIDTH") + statistics["source_records"] += 1 + geoid, district = (part.strip() for part in parts) + if district == "ZZ": + _require(re.fullmatch(r"[0-9]{15}", geoid) is not None, "CD_BLOCK") + block = int(geoid) + _require(block not in unassigned, "CD_DUPLICATE_BLOCK") + unassigned.add(block) + statistics["delegate_records"] += int(district == "98") + yield line + + +def assemble_atomic_block_sources( + *, + pl_archives: Iterable[tuple[str, AtomicBlockSourceBytes]], + cd_archive: AtomicBlockSourceBytes, + tract_to_puma: AtomicBlockSourceBytes, + state_fips: tuple[str, ...], + source_ids: Mapping[str, str], + max_source_bytes: int = 512 * 1024**2, + max_member_bytes: int = 2 * 1024**3, + max_total_geography_bytes: int = 32 * 1024**3, + max_zip_members: int = 16, + max_line_bytes: int = 64 * 1024, +) -> tuple[bytes, bytes]: + """Return deterministic normalized support and a canonical source receipt. + + ``state_fips`` is a nonempty sorted tuple drawn from the 50 states plus DC. + ``pl_archives`` yields exactly that ordered roster, one archive at a time; + only the current PL archive/member is retained while accumulating block maps. + Required members are ``{usps_lower}geo2020.pl`` and ``NationalCD119.txt``. + The tract relationship is raw UTF-8 text. ZIP target text uses the maintained + source readers' Latin-1 convention. All positive-POP100 blocks are retained; + zero-population exclusions are counted in each state's receipt. + + The source byte cap and hash check precede ZIP metadata parsing. Member + roster/count and expanded-byte limits precede selected-member inflation and + text parsing, with a cumulative bound on selected geography plus the raw + relationship. The member-count limit does not bound ZipFile's initial + metadata allocation. No unselected segment is decompressed or parsed. + Normalized output must fit the receiving raw-bytes-v1 codec's 64 MiB cap. + + Matching caller-supplied hashes proves byte integrity, not that the caller's + pins identify an official publisher, a current source, or an admitted dataset. + """ + _require( + type(state_fips) is tuple + and bool(state_fips) + and all(type(state) is str and state in _GEO_MEMBERS for state in state_fips) + and state_fips == tuple(sorted(set(state_fips))), + "STATE_ROSTER", + ) + identities = normalized._sources(source_ids) + bounds = { + "max_source_bytes": max_source_bytes, + "max_member_bytes": max_member_bytes, + "max_total_geography_bytes": max_total_geography_bytes, + "max_zip_members": max_zip_members, + "max_line_bytes": max_line_bytes, + } + _require( + all(type(value) is int and 0 < value <= 2**63 - 1 for value in bounds.values()), + "BOUNDS", + ) + puma_payload, puma_digest, _members = _checked_source( + tract_to_puma, archive=False, max_source_bytes=max_source_bytes + ) + puma_provenance = {"sha256": puma_digest, "size_bytes": len(puma_payload)} + _require( + len(puma_payload) <= min(max_member_bytes, max_total_geography_bytes), + "TOTAL_GEOGRAPHY_BYTES", + ) + expanded = [len(puma_payload)] + cd_payload, cd_provenance = _selected_zip_member( + cd_archive, CD_MEMBER, bounds=bounds, expanded=expanded + ) + cd_statistics = {"source_records": 0, "delegate_records": 0} + unassigned = set() + districts = parse_national_cd_bef( + _cd_lines( + cd_payload, + max_line_bytes=max_line_bytes, + statistics=cd_statistics, + unassigned=unassigned, + ) + ) + _require(not unassigned.intersection(districts), "CD_CONFLICTING_UNASSIGNED") + _require( + cd_statistics["source_records"] == len(districts) + len(unassigned), + "CD_RECONCILIATION", + ) + cd_provenance.update( + source_id=identities["district"], + **cd_statistics, + assigned_blocks=len(districts), + unassigned_blocks=len(unassigned), + delegate_normalization="98_to_00", + district_relation="official_tabulation", + ) + del cd_payload, unassigned + pumas = parse_tract_to_puma_relationship( + _lines(puma_payload, encoding="utf-8", max_line_bytes=max_line_bytes), + allowed_state_fips=frozenset(state_fips), + ) + puma_provenance.update( + source_id=identities["puma"], selected_state_tract_mappings=len(pumas) + ) + + iterator, exhausted = iter(pl_archives), object() + population, population_provenance, state_totals = {}, [], {} + for state in state_fips: + item = next(iterator, exhausted) + _require( + type(item) is tuple and len(item) == 2 and item[0] == state, + "PL_SOURCE_ROSTER", + ) + record = item[1] + payload, provenance = _selected_zip_member( + record, _GEO_MEMBERS[state], bounds=bounds, expanded=expanded + ) + statistics = { + "geography_rows": 0, + "state_rows": 0, + "state_population": 0, + "block_rows": 0, + "zero_population_blocks": 0, + } + state_population = parse_pl_geo_blocks( + _pl_lines( + payload, + state=state, + max_line_bytes=max_line_bytes, + statistics=statistics, + ), + state_fips=state, + ) + _require( + statistics["state_rows"] == 1 + and statistics["state_population"] == sum(state_population.values()) + and statistics["block_rows"] + == len(state_population) + statistics["zero_population_blocks"], + "PL_RECONCILIATION", + ) + _require( + not population.keys() & state_population.keys(), "PL_OVERLAPPING_STATES" + ) + population.update(state_population) + state_totals[state] = statistics["state_population"] + provenance.update( + source_id=identities["population"], + state_fips=state, + populated_blocks=len(state_population), + **statistics, + ) + population_provenance.append(provenance) + del item, record, payload, state_population + _require(next(iterator, exhausted) is exhausted, "PL_SOURCE_ROSTER") + payload = normalized.assemble_atomic_block_support( + block_population=population, + cd_by_block=districts, + puma_by_tract=pumas, + source_ids=identities, + ) + _require(len(payload) <= RAW_BYTES_MAX_BYTES, "SUPPORT_BYTES") + support = decode_atomic_support(payload) + output_totals = dict.fromkeys(state_fips, 0) + for state, value in zip( + support.arrays["state"], support.arrays["population"], strict=True + ): + output_totals[str(state)] += int(value) + ordered_blocks = sorted(population) + _require( + support.arrays["area"].tolist() == [f"{block:015d}" for block in ordered_blocks] + and support.arrays["population"].tolist() + == [population[block] for block in ordered_blocks] + and support.arrays["puma"].tolist() + == [f"{pumas[block // 10000]:07d}" for block in ordered_blocks] + and support.arrays["district"].tolist() + == [f"{districts[block]:04d}" for block in ordered_blocks] + and output_totals == state_totals + and len(support.arrays["area"]) == len(population), + "OUTPUT_RECONCILIATION", + ) + receipt = canonical_json( + { + "protocol": PROTOCOL, + "state_fips": list(state_fips), + "source_ids": identities, + "limits": bounds, + "sources": { + "population": population_provenance, + "district": cd_provenance, + "puma": puma_provenance, + }, + "reconciliation": { + "state_population": state_totals, + "output_state_population": output_totals, + "population_total": sum(state_totals.values()), + "populated_blocks": len(population), + "selected_geography_bytes": expanded[0], + }, + "support_sha256": _sha(payload), + "exact_input_hashes_verified": True, + "unselected_zip_member_contents_read": False, + "publisher_provenance_established": False, + "source_admission_issued": False, + "release_eligible": False, + } + ) + return payload, receipt diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/atomic_block_support.py b/packages/microcosm-build/src/microcosm/build/us_runtime/atomic_block_support.py new file mode 100644 index 000000000..20fa3686c --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/atomic_block_support.py @@ -0,0 +1,276 @@ +"""Normalize US block mappings for shared geography operators; no new kernel. + +Publisher parsers supply block POP100, the CD block-equivalency mapping and the +2020 tract-to-PUMA mapping. Source authentication remains upstream. This module +retains block rows and supplies a country declaration; assignment and geographic +derivation are executed by the shared country-neutral operators. +""" + +from __future__ import annotations + +from collections.abc import Mapping, Sequence +from numbers import Integral + +import numpy as np + +from microcosm.build.atomic_geography import ( + encode_atomic_support, + validate_assignment_spec, +) + +SYSTEM = "us_census_block_2020" +SOURCE = "us_atomic_block_support" +_STATES = frozenset( + { + 1, + 2, + 4, + 5, + 6, + 8, + 9, + 10, + 11, + 12, + 13, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31, + 32, + 33, + 34, + 35, + 36, + 37, + 38, + 39, + 40, + 41, + 42, + 44, + 45, + 46, + 47, + 48, + 49, + 50, + 51, + 53, + 54, + 55, + 56, + } +) + + +def _require(condition, message): + if not condition: + raise ValueError("US atomic support: " + message) + + +def _integer(value, *, minimum=0, maximum=2**53): + return ( + isinstance(value, Integral) + and not isinstance(value, (bool, np.bool_)) + and minimum <= value <= maximum + ) + + +def _sources(source_ids): + _require( + isinstance(source_ids, Mapping) + and set(source_ids) == {"population", "district", "puma"} + and all( + isinstance(v, str) and v and v.strip() == v for v in source_ids.values() + ), + "three explicit nonempty source identities required", + ) + return dict(source_ids) + + +def assemble_atomic_block_support( + *, + block_population: Mapping, + cd_by_block: Mapping, + puma_by_tract: Mapping, + source_ids: Mapping, +) -> bytes: + """Keep each supplied block, with string codes and source-labeled mappings. + + Population is the declared 2020-person sampling proxy. This does not imply + household counts or support for subsequent construction in unpopulated + 2020 blocks. Existing PL parsers supply populated blocks only. Extra mapping + entries may cover unselected areas; every supplied support row must map. + """ + sources = _sources(source_ids) + _require( + all( + isinstance(m, Mapping) + for m in (block_population, cd_by_block, puma_by_tract) + ) + and bool(block_population), + "nonempty block support and mappings required", + ) + for block, population in block_population.items(): + _require( + _integer(block, minimum=10**13, maximum=10**15 - 1) + and int(block) // 10**13 in _STATES, + "invalid block GEOID", + ) + _require(_integer(population), "invalid population weight") + blocks = sorted(int(block) for block in block_population) + pumas, districts = [], [] + for block in blocks: + state, tract = block // 10**13, block // 10000 + puma, district = puma_by_tract.get(tract), cd_by_block.get(block) + _require( + _integer(puma, minimum=1, maximum=9999999) + and int(puma) // 100000 == state + and int(puma) % 100000 > 0, + "missing or inconsistent tract-to-PUMA mapping", + ) + _require( + _integer(district, minimum=100, maximum=5699) + and int(district) // 100 == state + and int(district) % 100 <= 53, + "missing or inconsistent block-to-district mapping", + ) + pumas.append(f"{int(puma):07d}") + districts.append(f"{int(district):04d}") + arrays = { + "area": np.asarray([f"{v:015d}" for v in blocks]), + "state": np.asarray([f"{v // 10**13:02d}" for v in blocks]), + "county": np.asarray([f"{v // 10**10:05d}" for v in blocks]), + "tract": np.asarray([f"{v // 10000:011d}" for v in blocks]), + "puma": np.asarray(pumas), + "district": np.asarray(districts), + "population": np.asarray( + [int(block_population[v]) for v in blocks], dtype=np.int64 + ), + } + columns = { + name: { + "kind": "code", + "source": sources["population"], + "vintage": "2020", + "relation": "exact", + } + for name in ("area", "state", "county", "tract") + } + columns.update( + { + "puma": { + "kind": "code", + "source": sources["puma"], + "vintage": "2020", + "relation": "exact", + }, + "district": { + "kind": "code", + "source": sources["district"], + "vintage": "119th_congress", + "relation": "official_tabulation", + }, + "population": { + "kind": "weight", + "source": sources["population"], + "basis": "2020_census_persons", + }, + } + ) + return encode_atomic_support( + { + "version": 1, + "system": SYSTEM, + "level": "block", + "code_system": "census_geoid", + "vintage": "2020", + "columns": columns, + }, + arrays, + ) + + +def assignment_definition( + *, + identity: Sequence[str], + state_column: str, + puma_column: str | None, + source_ids: Mapping, + seed: int, +) -> dict: + """Declare one block draw constrained by normalized observed state/PUMA. + + Input codes must already be strings from the survey's source projection. + Callers declare stable source plus completed initial-clone identity columns. + Assignment follows those clones; subsequent views retain the block and all + functionally derived geographies. + """ + sources = _sources(source_ids) + _require(type(seed) is int and 0 <= seed < 2**63, "invalid seed") + _require(not isinstance(identity, (str, bytes)), "identity must be columns") + constraints = [{"input": state_column, "support": "state", "required": True}] + if puma_column is not None: + constraints.append({"input": puma_column, "support": "puma", "required": False}) + layers = [ + { + "input": name, + "output": output, + "vintage": vintage, + "relation": relation, + "source": sources[source], + } + for name, output, vintage, relation, source in ( + ("state", "assigned_state_fips", "2020", "exact", "population"), + ("county", "county_fips", "2020", "exact", "population"), + ("tract", "census_tract_geoid", "2020", "exact", "population"), + ("puma", "assigned_puma_geoid", "2020", "exact", "puma"), + ( + "district", + "congressional_district_geoid", + "119th_congress", + "official_tabulation", + "district", + ), + ) + ] + return validate_assignment_spec( + { + "version": 1, + "identity": list(identity), + "stream": ["sha256-u53-v1", "us-atomic-geography", 0, seed], + "outputs": { + "area": "census_block_geoid", + "system": "geography_system", + "basis": "geography_assignment_basis", + }, + "systems": [ + { + "id": SYSTEM, + "level": "block", + "code_system": "census_geoid", + "vintage": "2020", + "source": SOURCE, + "selector": {}, + "constraints": constraints, + "observed_area": None, + "stages": [{"level": "area", "weight": "population"}], + "layers": layers, + } + ], + } + ) diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/cd_reference.py b/packages/microcosm-build/src/microcosm/build/us_runtime/cd_reference.py new file mode 100644 index 000000000..41a120d73 --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/cd_reference.py @@ -0,0 +1,652 @@ +"""Public CD references and uncertainty, without target or candidate activation. + +This inventory retains published observations. Crosswalk shares describe alignment +exposure only; this module neither allocates fiscal cells nor rescales state totals. +""" + +from __future__ import annotations + +import argparse +import csv +import hashlib +import io +import math +import re +import sys +import zipfile +from collections import Counter, defaultdict +from pathlib import Path +from xml.etree import ElementTree as ET + +from .cd_reference_sources import ( + TABLES, + capture_request, + census_requests, + read_census_key, + strict_json, + write_json, +) + +US_STATE_FIPS = frozenset( + "01 02 04 05 06 08 09 10 11 12 13 15 16 17 18 19 20 21 22 23 24 25 26 " + "27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 44 45 46 47 48 49 50 51 53 54 55 56".split() +) +AT_LARGE_117 = frozenset("02 10 11 30 38 46 50 56".split()) +SENTINELS = { + "-666666666": "insufficient_sample", + "-999999999": "insufficient_geographic_sample", + "-888888888": "not_applicable_or_available", + "-222222222": "moe_insufficient_sample", + "-333333333": "moe_open_interval", + "-555555555": "controlled_estimate", +} +ANNOTATIONS = { + "-": "insufficient_sample", + "N": "insufficient_geographic_sample", + "(X)": "not_applicable_or_available", + "**": "moe_insufficient_sample", + "***": "moe_open_interval", + "*****": "controlled_estimate", +} +AUTHORITIES = { + "acs_annotations": "https://www.census.gov/data/developers/data-sets/acs-1year/notes-on-acs-estimate-and-annotation-values.html", + "acs_annotation_precedence": "https://www.census.gov/data/developers/data-sets/acs-1year/notes-on-acs-api-variable-types.html", + "acs_geography": "https://www.census.gov/programs-surveys/acs/geography-acs/geography-boundaries-by-year/2024.html", + "acs_confidence": "https://www.census.gov/programs-surveys/acs/methodology/sample-size-and-data-quality/sample-size-definitions.html", + "irs_guide": "https://www.irs.gov/pub/irs-soi/22incddocguide.docx", +} + + +def classify_acs_value(raw, annotation) -> dict: + """Annotation fields take precedence; retain sentinels and conflicts verbatim.""" + sentinel = SENTINELS.get(str(raw)) + annotation_class = None + if annotation not in (None, ""): + if not isinstance(annotation, str): + raise ValueError("ACS annotation must be text or null") + annotation_class = ANNOTATIONS.get(annotation) + if annotation_class is None: + annotation_class = ( + "open_interval" + if re.fullmatch(r"[\d,.]+[+-]", annotation) + else "unknown_annotation" + ) + classification = annotation_class or sentinel + numeric = None + if classification is None: + if raw in (None, ""): + classification = "missing" + else: + try: + numeric = float(raw) + if not math.isfinite(numeric): + raise ValueError + except (ValueError, TypeError): + raise ValueError("unrecognized ACS numeric value") from None + classification = "numeric" + return { + "raw": raw, + "annotation": annotation, + "class": classification, + "numeric_value": numeric, + "annotation_conflict": bool( + sentinel and annotation_class and sentinel != annotation_class + ), + } + + +def _district_id(state: str, district: str, congress: int) -> str: + if not re.fullmatch(r"\d{2}", state) or not re.fullmatch(r"\d{2}", district): + raise ValueError("invalid district identity") + return f"500{congress - 100:02d}00US{state}{'00' if district == '98' else district}" + + +def acs_table_inventory(table: str, metadata: dict, data_rows: list) -> dict: + if table not in TABLES: + raise ValueError("undeclared ACS table") + variables = metadata["variables"] + stems = sorted( + key[:-1] + for key in variables + if key.startswith(table + "_") and key.endswith("E") + ) + if not stems or not data_rows: + raise ValueError("ACS table has no estimate variables or data") + header = data_rows[0] + if len(header) != len(set(header)): + raise ValueError("duplicate ACS header") + required = {"NAME", "state", "congressional district"} + required.update( + stem + suffix for stem in stems for suffix in ("E", "M", "EA", "MA") + ) + missing = sorted(required - set(header)) + if missing: + raise ValueError("missing ACS columns: " + ", ".join(missing)) + for stem in stems: + if any(stem + suffix not in variables for suffix in ("E", "M", "EA", "MA")): + raise ValueError("missing ACS variable metadata") + cells, geographies, excluded = [], {}, [] + seen = set() + counts = {"estimate": Counter(), "moe": Counter()} + for values in data_rows[1:]: + if len(values) != len(header): + raise ValueError("ACS row width mismatch") + row = dict(zip(header, values, strict=True)) + state, district = row["state"], row["congressional district"] + identity = _district_id(state, district, 119) + if identity in seen: + raise ValueError("duplicate geography in ACS table") + seen.add(identity) + if state not in US_STATE_FIPS: + excluded.append( + { + "state_fips": state, + "district": district, + "name": row["NAME"], + "reason": "outside_us_50_states_dc", + } + ) + continue + geographies[identity] = { + "geography_id": identity, + "state_fips": state, + "published_district": district, + "name": row["NAME"], + "published_geo_id": row.get("GEO_ID"), + } + for stem in stems: + estimate = classify_acs_value(row[stem + "E"], row[stem + "EA"]) + moe = classify_acs_value(row[stem + "M"], row[stem + "MA"]) + if moe["numeric_value"] is not None and moe["numeric_value"] < 0: + raise ValueError("negative numeric ACS MOE") + counts["estimate"][estimate["class"]] += 1 + counts["moe"][moe["class"]] += 1 + cells.append( + { + "geography_id": identity, + "variable": stem, + "estimate": estimate, + "moe": moe, + } + ) + return { + "table": table, + "year": 2024, + "congress": 119, + "confidence_level": 0.9, + "uncertainty": { + "kind": "published_margin_of_error", + "variance_conversion": "not_performed", + "controlled_estimates": "raw sentinel retained; no invented variance", + }, + "grouping": "subject_age_and_sex_published_groups" + if table == "S0101" + else "detailed_sex_by_age_published_groups" + if table == "B01001" + else "published_detailed_table_cells", + "reservation": { + "status": "reserved_same_source_holdout" + if table in ("B19001", "B25003") + else "inventory_only", + "covers_derived_versions": True, + "targets_allowed": False, + "tuning_allowed": False, + "source_independent": False, + "reason": "ACS tabulations and ACS PUMS share a source; no candidate scoring or tuning here", + }, + "lineage": "published", + "geography_ids": sorted(geographies), + "geographies": [geographies[key] for key in sorted(geographies)], + "excluded_geographies": excluded, + "variables": { + stem: { + suffix: variables[stem + suffix] for suffix in ("E", "M", "EA", "MA") + } + for stem in stems + }, + "classification_counts": {key: dict(value) for key, value in counts.items()}, + "cells": sorted( + cells, key=lambda cell: (cell["geography_id"], cell["variable"]) + ), + } + + +def irs_csv_inventory(payload: bytes) -> dict: + reader = csv.DictReader(io.StringIO(payload.decode("utf-8-sig"))) + header = reader.fieldnames or [] + identity_fields = {"STATEFIPS", "STATE", "CONG_DISTRICT", "agi_stub"} + if len(header) != len(set(header)) or not identity_fields.issubset(header): + raise ValueError("IRS CSV identity columns missing or duplicated") + metrics = [field for field in header if field not in identity_fields] + counts = {field: Counter() for field in metrics} + rows, seen = [], set() + stubs = Counter() + for row in reader: + if None in row or None in row.values(): + raise ValueError("IRS row width mismatch") + state, district, stub = row["STATEFIPS"], row["CONG_DISTRICT"], row["agi_stub"] + identity = state, district, stub + if identity in seen: + raise ValueError("duplicate IRS source row") + seen.add(identity) + _district_id(state, district, 117) + if state not in US_STATE_FIPS | {"00"} or not re.fullmatch(r"\d+", stub): + raise ValueError("IRS source row outside declared scope") + if state == "00" and district != "00": + raise ValueError("national IRS row has district code") + level = ( + "national" + if state == "00" + else "state" + if district == "00" + else "congressional_district" + ) + rows.append( + { + "state_fips": state, + "state": row["STATE"], + "district": district, + "agi_stub": stub, + "published_geography_level": level, + "published_geography_id": _district_id(state, district, 117) + if level == "congressional_district" + else None, + "at_large_proxy_geography_id": _district_id(state, "00", 117) + if state in AT_LARGE_117 and district == "00" + else None, + } + ) + stubs[stub] += 1 + for field in metrics: + raw = row[field] + if raw in ("", "**"): + classification = ( + "missing" if raw == "" else "suppression_or_combination_marker" + ) + else: + try: + value = float(raw) + if not math.isfinite(value): + raise ValueError + except ValueError: + raise ValueError("unrecognized IRS metric cell") from None + classification = ( + "numeric_zero_disclosure_unknown" + if value == 0 + else "numeric_disclosure_unknown" + ) + counts[field][classification] += 1 + return { + "tax_year": 2022, + "processing_year": 2023, + "congress": 117, + "lineage": "published", + "rows": rows, + "row_count": len(rows), + "columns": header, + "metric_classification_counts": { + field: dict(count) for field, count in counts.items() + }, + "agi_stub_inventory": { + "counts": dict(sorted(stubs.items())), + "guide_documented_codes": list(range(11)), + "labels_status": "needs_method_resolution", + "note": "Retain published codes. Guide section G describes 0–10; fetched CSV contains 0–9. Do not infer income-bin labels.", + }, + "uncertainty": { + "variance_status": "not_published_needs_method_resolution", + "confidence_level": None, + "note": "Guide section C describes a population; footnote 1 identifies an SOI sample input. No variance is inferred.", + }, + "disclosure": { + "numeric_zero_proves_unsuppressed": False, + "csv_cell_suppression_status": "not_identifiable_from_numeric_cells", + "published_rules": [ + "fewer than 20 returns: combine adjacent AGI cells", + "district item totals under 20 excluded", + "dominant single-return items suppressed using unpublished threshold", + ], + "xlsx_markers": "kept separately by sheet and cell; not joined to CSV AGI slices", + }, + "units": { + "monetary_amounts": "thousands_of_dollars", + "rounding_increment": None, + "rounding_status": "not_established_by_unit_label", + }, + "coverage_caveats": [ + "tax filers only", + "tax address may differ from residence", + "ZIP-based district assignment", + "US national total is not a congressional district", + "at-large state totals are explicit proxies, not directly published district observations", + ], + } + + +def crosswalk_alignment(rows: list[dict]) -> dict: + """Summarize incoming target composition, not fiscal allocation error.""" + incoming, outgoing = defaultdict(list), defaultdict(list) + seen = set() + for row in rows: + source, target = row["source_geography_id"], row["target_geography_id"] + if not re.fullmatch(r"5001700US\d{4}", source) or not re.fullmatch( + r"5001900US\d{4}", target + ): + raise ValueError("crosswalk geography vintage mismatch") + if source[9:11] not in US_STATE_FIPS or source[9:11] != target[9:11]: + raise ValueError("crosswalk outside US50+DC or crosses state") + if (source, target) in seen: + raise ValueError("duplicate crosswalk pair") + seen.add((source, target)) + population, weight = float(row["pair_population"]), float(row["weight"]) + if ( + not math.isfinite(population) + or population <= 0 + or not math.isfinite(weight) + or weight <= 0 + ): + raise ValueError( + "crosswalk population and weight must be positive and finite" + ) + incoming[target].append((source, population)) + outgoing[source].append((population, weight)) + if not seen: + raise ValueError("empty crosswalk") + for pairs in outgoing.values(): + total = math.fsum(population for population, _ in pairs) + if not math.isfinite(total) or any( + not math.isclose(weight, population / total, rel_tol=1e-9, abs_tol=1e-12) + for population, weight in pairs + ): + raise ValueError("crosswalk outgoing weights disagree with pair population") + targets = [] + for target, pairs in sorted(incoming.items()): + total = math.fsum(population for _, population in pairs) + contributions = [ + { + "source_geography_id": source, + "pair_population": population, + "incoming_share": population / total, + } + for source, population in sorted(pairs) + ] + maximum = max(item["incoming_share"] for item in contributions) + targets.append( + { + "target_geography_id": target, + "pair_population": total, + "contributions": contributions, + "max_incoming_share": maximum, + "non_dominant_incoming_share": 1 - maximum, + "known_allocation_error": None, + } + ) + return { + "source_congress": 117, + "target_congress": 119, + "population_vintage": "2020_tabulation_blocks", + "basis": "pair_population_normalized_per_target", + "interpretation": "alignment exposure, not known allocation error", + "targets": targets, + } + + +def xlsx_disclosure_inventory(payload: bytes) -> dict: + ns = {"x": "http://schemas.openxmlformats.org/spreadsheetml/2006/main"} + with zipfile.ZipFile(io.BytesIO(payload)) as archive: + shared = ET.fromstring(archive.read("xl/sharedStrings.xml")) + strings = ["".join(item.itertext()) for item in shared.findall("x:si", ns)] + cells = [] + for name in sorted(archive.namelist()): + if not re.fullmatch(r"xl/worksheets/sheet\d+\.xml", name): + continue + root = ET.fromstring(archive.read(name)) + for cell in root.findall(".//x:c", ns): + if cell.attrib.get("t") == "s": + value = cell.findtext("x:v", namespaces=ns) + text = strings[int(value)] if value is not None else "" + else: + text = ( + "".join(cell.find("x:is", ns).itertext()) + if cell.find("x:is", ns) is not None + else cell.findtext("x:v", default="", namespaces=ns) + ) + if "**" in text: + cells.append( + { + "sheet_part": name, + "cell": cell.attrib["r"], + "text": text, + "class": "suppression_or_combination_marker" + if text.strip() == "**" + else "disclosure_note", + } + ) + return { + "markers": cells, + "csv_cell_mapping": "not_performed", + "note": "Published workbook cells and disclosure notes remain separate from CSV AGI rows.", + } + + +def _pin_local(output: Path, payload: bytes, suffix: str, **metadata) -> dict: + digest = hashlib.sha256(payload).hexdigest() + relative = f"raw/{digest}{suffix}" + path = output / relative + path.parent.mkdir(parents=True, exist_ok=True) + if path.exists() and path.read_bytes() != payload: + raise ValueError("local source digest collision") + if not path.exists(): + path.write_bytes(payload) + return {**metadata, "sha256": digest, "size_bytes": len(payload), "path": relative} + + +def build_bundle( + output: Path, + irs_authority: Path, + crosswalk: Path, + crosswalk_provenance: Path, + *, + offline: bool = False, + api_key: str | None = None, + fetch=None, +) -> dict: + """Create or replay an immutable inventory. No candidate values are accepted.""" + output = Path(output) + captures, tables = [], {} + for request in census_requests(): + descriptor = capture_request( + output, + request, + offline=offline, + api_key=api_key, + **({"fetch": fetch} if fetch else {}), + ) + captures.append(descriptor) + tables.setdefault(request["table"], {})[request["kind"]] = strict_json( + (output / descriptor["path"]).read_bytes() + ) + acs = { + table: acs_table_inventory( + table, tables[table]["metadata"], tables[table]["data"] + ) + for table in TABLES + } + expected = acs["S0101"]["geography_ids"] + if any(item["geography_ids"] != expected for item in acs.values()): + raise ValueError("ACS table geography universes differ") + index_payload = (irs_authority / "source-index.json").read_bytes() + index = strict_json(index_payload) + by_name = {item["filename"]: item for item in index} + if len(by_name) != len(index): + raise ValueError("duplicate IRS authority source identity") + local, payloads = [], {} + for name in ("22incd.csv", "22incdall.xlsx"): + descriptor = by_name[name] + if descriptor["url"] != "https://www.irs.gov/pub/irs-soi/" + name: + raise ValueError("IRS authority URL mismatch") + payload = (irs_authority / name).read_bytes() + if ( + hashlib.sha256(payload).hexdigest() != descriptor["sha256"] + or len(payload) != descriptor["size_bytes"] + ): + raise ValueError("IRS authority source digest mismatch") + payloads[name] = payload + local.append( + _pin_local( + output, payload, Path(name).suffix, url=descriptor["url"], filename=name + ) + ) + guide = (irs_authority / "22incddocguide.docx").read_bytes() + if ( + hashlib.sha256(guide).hexdigest() + != (irs_authority / "22incddocguide.sha256").read_text().split()[0] + ): + raise ValueError("IRS guide digest mismatch") + local.append( + _pin_local( + output, + guide, + ".docx", + url=AUTHORITIES["irs_guide"], + filename="22incddocguide.docx", + ) + ) + local.append(_pin_local(output, index_payload, ".json", role="irs_source_index")) + with zipfile.ZipFile(io.BytesIO(guide)) as archive: + guide_xml = archive.read("word/document.xml") + guide_text = "\n".join(ET.fromstring(guide_xml).itertext()).encode() + local.append( + _pin_local( + output, + guide_text, + ".txt", + role="irs_guide_extracted_text", + derived_from_sha256=hashlib.sha256(guide).hexdigest(), + derivation="word/document.xml text nodes in document order", + ) + ) + crosswalk_bytes, provenance_bytes = ( + crosswalk.read_bytes(), + crosswalk_provenance.read_bytes(), + ) + provenance = strict_json(provenance_bytes) + if any( + provenance.get(key) != value + for key, value in { + "source_geography_vintage": "117th_congress", + "target_geography_vintage": "119th_congress", + "block_vintage": "2020_tabulation_blocks", + }.items() + ): + raise ValueError("crosswalk provenance vintage mismatch") + if hashlib.sha256(crosswalk_bytes).hexdigest() != provenance["crosswalk_sha256"]: + raise ValueError("crosswalk provenance digest mismatch") + local.append(_pin_local(output, crosswalk_bytes, ".csv", role="crosswalk")) + local.append( + _pin_local(output, provenance_bytes, ".json", role="crosswalk_provenance") + ) + alignment = crosswalk_alignment( + list(csv.DictReader(io.StringIO(crosswalk_bytes.decode()))) + ) + irs = irs_csv_inventory(payloads["22incd.csv"]) + irs["workbook_disclosure"] = xlsx_disclosure_inventory(payloads["22incdall.xlsx"]) + published = sorted( + { + row["published_geography_id"] + for row in irs["rows"] + if row["published_geography_id"] + } + ) + proxies = sorted( + { + row["at_large_proxy_geography_id"] + for row in irs["rows"] + if row["at_large_proxy_geography_id"] + } + ) + crosswalk_sources = { + item["source_geography_id"] + for target in alignment["targets"] + for item in target["contributions"] + } + targets = {item["target_geography_id"] for item in alignment["targets"]} + bundle = { + "schema_version": 1, + "kind": "us_cd_reference_inventory", + "purpose": "reference_inventory_only", + "candidate_evaluation": "not_performed", + "target_activation": False, + "authorities": AUTHORITIES, + "scope": { + "country": "us", + "states": "50_states_and_dc", + "puerto_rico": "excluded", + "acs_year": 2024, + "target_congress": 119, + "irs_tax_year": 2022, + "irs_congress": 117, + }, + "sources": {"census": captures, "local": local}, + "acs": acs, + "irs": irs, + "alignment": alignment, + "reconciliation": { + "irs_published_district_ids": published, + "irs_at_large_state_proxy_ids": proxies, + "irs_missing_crosswalk_source_ids": sorted( + crosswalk_sources - set(published) - set(proxies) + ), + "irs_unmapped_source_ids": sorted( + (set(published) | set(proxies)) - crosswalk_sources + ), + "acs_missing_crosswalk_target_ids": sorted(targets - set(expected)), + "acs_unmapped_target_ids": sorted(set(expected) - targets), + }, + "lineage_products": { + "published": "included", + "crosswalked_values": "not_generated", + "state_rescaled_values": "not_generated", + }, + } + write_json(output / "reference-inventory.json", bundle) + return bundle + + +def main(argv=None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--output-dir", type=Path, required=True) + parser.add_argument("--irs-authority-dir", type=Path, required=True) + parser.add_argument("--crosswalk", type=Path, required=True) + parser.add_argument("--crosswalk-provenance", type=Path, required=True) + parser.add_argument("--offline", action="store_true") + parser.add_argument("--census-key-service") + parser.add_argument("--census-key-account") + args = parser.parse_args(argv) + if bool(args.census_key_service) != bool(args.census_key_account): + parser.error("credential service and account must be supplied together") + try: + key = ( + read_census_key(args.census_key_service, args.census_key_account) + if args.census_key_service and not args.offline + else None + ) + bundle = build_bundle( + args.output_dir, + args.irs_authority_dir, + args.crosswalk, + args.crosswalk_provenance, + offline=args.offline, + api_key=key, + ) + except (ValueError, OSError, KeyError, zipfile.BadZipFile) as error: + print(f"Reference bundle failed: {error}", file=sys.stderr) + return 1 + print( + f"Reference inventory saved: {args.output_dir / 'reference-inventory.json'}; {len(bundle['acs']['S0101']['geography_ids'])} ACS districts; no candidate evaluation" + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/cd_reference_sources.py b/packages/microcosm-build/src/microcosm/build/us_runtime/cd_reference_sources.py new file mode 100644 index 000000000..9a1e9b137 --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/cd_reference_sources.py @@ -0,0 +1,188 @@ +"""Pinned public ACS acquisition; credentials never enter stored provenance.""" + +from __future__ import annotations + +import hashlib +import json +import subprocess +from datetime import UTC, datetime +from pathlib import Path +from urllib.error import HTTPError, URLError +from urllib.parse import urlencode +from urllib.request import HTTPRedirectHandler, Request, build_opener + +TABLES = ("S0101", "B01001", "B19001", "B25003") +MAX_RESPONSE_BYTES = 32 * 1024 * 1024 + + +def census_requests() -> list[dict]: + """The complete, fixed public reference request surface.""" + requests = [] + for table in TABLES: + dataset = "acs/acs1/subject" if table == "S0101" else "acs/acs1" + base = f"https://api.census.gov/data/2024/{dataset}" + for kind, url in ( + ("metadata", f"{base}/groups/{table}.json"), + ( + "data", + base + + "?" + + urlencode( + { + "get": f"group({table})", + "for": "congressional district:*", + "in": "state:*", + } + ), + ), + ): + requests.append( + { + "table": table, + "kind": kind, + "year": 2024, + "dataset": dataset, + "url": url, + } + ) + return requests + + +def strict_json(payload: bytes): + def pairs(items): + result = {} + for key, value in items: + if key in result: + raise ValueError("duplicate JSON key") + result[key] = value + return result + + def nonfinite(_): + raise ValueError("nonfinite JSON number") + + try: + return json.loads(payload, object_pairs_hook=pairs, parse_constant=nonfinite) + except (UnicodeError, json.JSONDecodeError): + raise ValueError("source is not valid UTF-8 JSON") from None + + +class _NoRedirect(HTTPRedirectHandler): + def redirect_request(self, *_args, **_kwargs): + return None + + +def fetch_census(url: str, api_key: str | None) -> bytes: + """Fetch a declared request, suppressing URL-bearing transport errors.""" + if url not in {item["url"] for item in census_requests()}: + raise ValueError("request is not declared") + request_url = url + if api_key: + request_url += ("&" if "?" in url else "?") + urlencode({"key": api_key}) + try: + with build_opener(_NoRedirect()).open( + Request(request_url, headers={"User-Agent": "Microcosm-CD-reference/1"}), + timeout=90, + ) as response: + payload = response.read(MAX_RESPONSE_BYTES + 1) + except HTTPError as error: + raise ValueError(f"Census request failed (HTTP {error.code})") from None + except (URLError, OSError, ValueError): + raise ValueError("Census request failed (transport error)") from None + if len(payload) > MAX_RESPONSE_BYTES: + raise ValueError("Census response exceeds size limit") + return payload + + +def read_census_key(service: str, account: str) -> str: + """Read only the explicitly requested agent-secret into process memory.""" + try: + result = subprocess.run( + ["agent-secret", "get", service, account], + capture_output=True, + check=False, + timeout=30, + ) + except (OSError, subprocess.SubprocessError): + raise ValueError("Census credential helper failed") from None + if result.returncode or not result.stdout.strip(): + raise ValueError("Census credential unavailable") + try: + return result.stdout.decode("utf-8").strip() + except UnicodeError: + raise ValueError("Census credential is not text") from None + + +def write_json(path: Path, value) -> None: + payload = ( + json.dumps( + value, sort_keys=True, separators=(",", ":"), allow_nan=False + ).encode() + + b"\n" + ) + if path.exists(): + if path.read_bytes() != payload: + raise ValueError(f"refusing to replace immutable artifact: {path.name}") + return + path.parent.mkdir(parents=True, exist_ok=True) + with path.open("xb") as stream: + stream.write(payload) + + +def capture_request( + root: Path, + request: dict, + *, + api_key: str | None = None, + fetch=fetch_census, + offline: bool = False, +) -> dict: + """Fetch once, pin exact response bytes, then verify on every replay.""" + if request not in census_requests(): + raise ValueError("request metadata is not declared") + root = Path(root) + request_key = hashlib.sha256(request["url"].encode()).hexdigest() + descriptor_path = root / "requests" / f"{request_key}.json" + if descriptor_path.exists(): + descriptor = strict_json(descriptor_path.read_bytes()) + digest = descriptor.get("sha256", "") + expected_path = f"raw/{digest}.json" + if ( + len(digest) != 64 + or any(char not in "0123456789abcdef" for char in digest) + or descriptor.get("path") != expected_path + or any(descriptor.get(key) != value for key, value in request.items()) + ): + raise ValueError("cached request descriptor mismatch") + payload = (root / expected_path).read_bytes() + if ( + hashlib.sha256(payload).hexdigest() != digest + or len(payload) != descriptor["size_bytes"] + ): + raise ValueError("cached source digest mismatch") + strict_json(payload) + return descriptor + if offline: + raise ValueError("offline source is missing") + payload = fetch(request["url"], api_key) + if api_key and api_key.encode() in payload: + raise ValueError("response contains credential; refusing to persist") + strict_json(payload) + digest = hashlib.sha256(payload).hexdigest() + relative_path = f"raw/{digest}.json" + raw_path = root / relative_path + raw_path.parent.mkdir(parents=True, exist_ok=True) + if raw_path.exists(): + if raw_path.read_bytes() != payload: + raise ValueError("raw source digest collision") + else: + with raw_path.open("xb") as stream: + stream.write(payload) + descriptor = { + **request, + "sha256": digest, + "path": relative_path, + "size_bytes": len(payload), + "retrieved_at": datetime.now(UTC).isoformat(), + } + write_json(descriptor_path, descriptor) + return descriptor diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/common_frame_export_contract.py b/packages/microcosm-build/src/microcosm/build/us_runtime/common_frame_export_contract.py new file mode 100644 index 000000000..f7671534f --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/common_frame_export_contract.py @@ -0,0 +1,234 @@ +"""Pure export comparison against a supplied common parent; no source issuer. + +The host must authenticate the actual enriched Population and calibration +ancestry, including targets, ordered weights and geographic scope. A supplied +reference, specification or returned digest cannot establish that ancestry. +The host must also seal those inputs across writer/readback I/O and finish its +owner checks before accepting an export. This module performs no I/O or solve. +""" + +from __future__ import annotations + +import hashlib + +import numpy as np +import pandas as pd + +from microcosm.frame import Frame, MassChange, WeightKind, Weights +from microcosm.graph.canonical import canonical_json +from microcosm.graph.population import storage_equal + +PROTOCOL = "microcosm.us.supplied-parent-export-comparison.v1" +MAX_SPECIFICATION_BYTES = 1024 * 1024 + + +class RetainedFrameExportError(ValueError): + """An export differs from its supplied parent or declared weight/scope inputs.""" + + +def _require(condition, code): + if not condition: + raise RetainedFrameExportError(code) + + +def _vector(value, dtype, code): + _require( + type(value) is np.ndarray and value.ndim == 1 and value.dtype == dtype, + code, + ) + return value + + +def _sha(payload): + return hashlib.sha256(payload).hexdigest() + + +def _array_binding(value): + _require(value.dtype.kind in ("i", "u", "f"), "NUMERICAL_BINDING_DTYPE") + return { + "dtype": value.dtype.str, + "rows": len(value), + "sha256": _sha(np.ascontiguousarray(value).tobytes()), + } + + +def _same_series(expected, actual, *, readback): + """Allow only masked-cell codec canonicalization on logical readback.""" + if expected.dtype != actual.dtype or len(expected) != len(actual): + return False + expected_null = expected.isna().to_numpy(dtype=np.bool_) + actual_null = actual.isna().to_numpy(dtype=np.bool_) + if not np.array_equal(expected_null, actual_null): + return False + if readback and expected.dtype == object: + # Checkpoint supports object-string strata. Raw object pointers cannot + # identify values across a serialization boundary. Arbitrary object + # equality is outside this small numerical/string contract. + left = expected.to_numpy()[~expected_null].tolist() + right = actual.to_numpy()[~actual_null].tolist() + return all(type(v) is str for v in (*left, *right)) and left == right + return storage_equal(expected, actual, ~expected_null if readback else None) + + +def verify_retained_frame_export( + parent: Frame, + candidate: Frame, + *, + parent_reference: str, + ordered_household_ids: np.ndarray, + calibrated_weights: np.ndarray, + calibration_specification: bytes, + scope_household_ids: np.ndarray | None = None, + prune_zero_weight: bool = True, + comparison: str = "storage", + expected_binding: bytes | None = None, +) -> bytes: + """Compare complete retained inputs and bind supplied numerical export inputs. + + IDs retain the parent's exact signed/unsigned integer dtype; weights are + exact float64 arrays in the *whole parent* + household order. Scope IDs must be a unique subset in that same order; + None explicitly denotes all households. Pruning removes exactly zero + weights, with no tolerance or default origin-quality requirement. Household + selection carries all its persons and precisely their referenced groups. + + Call before writing, then on the serializer's logical readback with + comparison="frame-checkpoint-readback" and the prior expected_binding. + That closed mode allows only existing + codec normalization under null masks; known values, dtype and masks remain + exact. Extra or missing columns and record reordering refuse. Pandas row + indices may be reset: entity IDs, not incidental indices, define alignment. + + No Population ownership, design anchor, mass ledger, target interpretation, + geographic predicate meaning or source authenticity is established here. + The host must bind parent_reference to actual graph ancestry and verify the + specification/scope against that ancestry; opaque bytes are not authority. + Empty support, scopes with no positive weight, and experimental separate + link tables refuse explicitly, matching the Frame weight contract. + """ + _require(isinstance(parent, Frame) and isinstance(candidate, Frame), "FRAME_TYPE") + _require( + type(parent_reference) is str and bool(parent_reference), "PARENT_REFERENCE" + ) + _require( + type(calibration_specification) is bytes + and 0 < len(calibration_specification) <= MAX_SPECIFICATION_BYTES, + "CALIBRATION_SPECIFICATION", + ) + _require(type(prune_zero_weight) is bool, "OPTIONS") + _require( + type(comparison) is str + and comparison in ("storage", "frame-checkpoint-readback"), + "COMPARISON_MODE", + ) + readback = comparison == "frame-checkpoint-readback" + _require( + expected_binding is None or type(expected_binding) is bytes, "BINDING_TYPE" + ) + _require(parent.schema == candidate.schema, "SCHEMA") + _require( + not parent.schema.links and parent.links == candidate.links == (), + "SEPARATE_LINK_TABLES_UNSUPPORTED", + ) + _require("household" in parent.schema.group_entities, "HOUSEHOLD_SCHEMA") + _require( + "household" in parent.weighted_entities + and parent.weighted_entities == candidate.weighted_entities, + "WEIGHT_TOPOLOGY", + ) + _require(parent.metadata == candidate.metadata, "FRAME_METADATA") + household_id = parent.schema.entity_id_column("household") + parent_ids = parent.table("household")[household_id].to_numpy() + _require(parent_ids.dtype.kind in ("i", "u"), "INTEGER_HOUSEHOLD_IDS") + ids = _vector(ordered_household_ids, parent_ids.dtype, "HOUSEHOLD_ID_VECTOR") + weights = _vector(calibrated_weights, np.dtype("float64"), "WEIGHT_VECTOR") + _require( + parent_ids.dtype == ids.dtype + and np.array_equal(parent_ids, ids) + and len(np.unique(ids)) == len(ids), + "COMPLETE_ORDERED_HOUSEHOLD_IDS", + ) + _require( + weights.shape == ids.shape + and np.isfinite(weights).all() + and (weights >= 0).all(), + "FINITE_ALIGNED_WEIGHTS", + ) + if scope_household_ids is None: + scope = ids + scope_mask = np.ones(len(ids), dtype=np.bool_) + else: + scope = _vector(scope_household_ids, ids.dtype, "SCOPE_ID_VECTOR") + scope_mask = np.isin(ids, scope) + _require(np.array_equal(ids[scope_mask], scope), "ORDERED_SCOPE_SUBSET") + keep = scope_mask & (weights > 0) if prune_zero_weight else scope_mask + _require(keep.any() and (weights[keep] > 0).any(), "EMPTY_EXPORT_SUPPORT") + membership = parent.schema.membership_column("household") + comparison_parent = parent.with_weights( + "household", + Weights(weights, WeightKind.CALIBRATED), + mass=MassChange( + factor=None, + reason="Temporary supplied-weight view for retained export comparison", + ), + ) + selected = comparison_parent.select( + np.isin(parent.person[membership].to_numpy(), ids[keep]) + ) + _require(selected.entities == candidate.entities, "ENTITY_ROSTER") + entity_ids = {} + for entity in selected.entities: + expected, actual = selected.table(entity), candidate.table(entity) + _require( + not actual.columns.has_duplicates + and tuple(expected.columns) == tuple(actual.columns), + "COLUMN_ROSTER:" + entity, + ) + id_column = parent.schema.entity_id_column(entity) + _require( + _same_series(expected[id_column], actual[id_column], readback=False), + "RETAINED_ENTITY_IDS:" + entity, + ) + for name in expected.columns: + _require( + _same_series(expected[name], actual[name], readback=readback), + "RETAINED_INPUT:" + entity + "." + str(name), + ) + entity_ids[entity] = _array_binding(expected[id_column].to_numpy()) + _require( + selected.strata.name == candidate.strata.name + and _same_series(selected.strata, candidate.strata, readback=readback), + "RETAINED_STRATA", + ) + for entity in selected.weighted_entities: + actual = candidate.weights_for(entity) + if entity == "household": + expected_values, expected_kind = weights[keep], WeightKind.CALIBRATED + else: + expected_weight = selected.weights_for(entity) + expected_values, expected_kind = ( + expected_weight.values, + expected_weight.kind, + ) + _require( + actual.kind is expected_kind + and storage_equal(pd.Series(expected_values), pd.Series(actual.values)), + "RETAINED_WEIGHTS:" + entity, + ) + binding = canonical_json( + { + "protocol": PROTOCOL, + "scope": "comparison_against_supplied_parent_only", + "parent_reference": parent_reference, + "actual_graph_calibration_ancestry_verified": False, + "release_eligible": False, + "calibration_specification_sha256": _sha(calibration_specification), + "complete_ordered_household_ids": _array_binding(ids), + "complete_calibrated_weights": _array_binding(weights), + "scope_household_ids": _array_binding(scope), + "prune_zero_weight": prune_zero_weight, + "retained_entity_ids": entity_ids, + } + ) + _require(expected_binding is None or binding == expected_binding, "EXPORT_BINDING") + return binding diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/congressional_district_vintage.py b/packages/microcosm-build/src/microcosm/build/us_runtime/congressional_district_vintage.py index 3182ef056..e1b5f7210 100644 --- a/packages/microcosm-build/src/microcosm/build/us_runtime/congressional_district_vintage.py +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/congressional_district_vintage.py @@ -7,6 +7,7 @@ import json from collections.abc import Iterable, Mapping from importlib.resources import files +from io import BytesIO from pathlib import Path from typing import Any @@ -96,6 +97,21 @@ def default_congressional_district_vintage_crosswalk_path() -> Path: ) +def decode_congressional_district_vintage_crosswalk(payload: bytes) -> pd.DataFrame: + """Decode immutable CSV artifact bytes under the existing crosswalk contract. + + The caller separately declares whether national roster validation is + required. Generic crosswalks retain the same behavior as the file loader. + """ + if not isinstance(payload, bytes): + raise TypeError("CD vintage crosswalk payload must be immutable bytes.") + return _prepare_crosswalk( + pd.read_csv(BytesIO(payload)), + source_prefix=SOURCE_CONGRESSIONAL_DISTRICT_PREFIX, + target_prefix=CURRENT_CONGRESSIONAL_DISTRICT_PREFIX, + ) + + def load_default_congressional_district_vintage_crosswalk() -> pd.DataFrame: """Load the packaged Census-built default 117th->119th CD crosswalk. diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/cps_carried.py b/packages/microcosm-build/src/microcosm/build/us_runtime/cps_carried.py index 82c172e56..12ff2315d 100644 --- a/packages/microcosm-build/src/microcosm/build/us_runtime/cps_carried.py +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/cps_carried.py @@ -9,6 +9,7 @@ from __future__ import annotations from collections.abc import Mapping +from dataclasses import dataclass import numpy as np import pandas as pd @@ -21,6 +22,9 @@ PAW_TYPE_TANF_CODES, fill_asec_public_assistance_type_source, ) +from microcosm.build.us_runtime.reported_coverage_source import ( + fill_asec_reported_coverage_source, +) from microcosm.frame import US_SCHEMA, Frame __all__ = [ @@ -35,6 +39,8 @@ "US_REPORTED_COVERAGE_VINTAGE_GATE_MIN_ROWS", "WIC_CARRIER_ADJUDICATION_URL", "derive_us_cps_carried_inputs", + "derive_us_cps_carried_tables", + "CpsCarriedTables", "reported_snap_receipt_by_spm_unit", "reported_tanf_enrollment_by_spm_unit", "reported_wic_receipt_carrier", @@ -132,17 +138,75 @@ } ) +# Actual person reads for the no-sidecar measured mapping. Optional restoration +# remains a separate explicit argument on the legacy/table entrypoints. +CPS_CARRIED_RAW_PERSON_COLUMNS = frozenset( + { + "A_AGE", + "A_SEX", + "WICYN", + "WSAL_VAL", + "SEMP_VAL", + "INT_VAL", + "DIV_VAL", + "CAP_VAL", + "SS_VAL", + "RESNSS1", + "RESNSS2", + "PNSN_VAL", + "ANN_VAL", + "OI_VAL", + "OI_OFF", + "RNT_VAL", + "FRSE_VAL", + "UC_VAL", + "PHIP_VAL", + "PMED_VAL", + "POTC_VAL", + "NOW_MRK", + "NOW_NONM", + "NOW_MCAID", + "NOW_GRP", + "NOW_CHAMPVA", + "NOW_MIL", + "NOW_VACARE", + "NOW_OTHMT", + "NOW_IHSFLG", + "PAW_VAL", + "PAW_TYP", + "SPM_SNAPSUB", + "SPM_CHILDCAREXPNS", + *(f"DST_SC{suffix}" for suffix in ("1", "2", "1_YNG", "2_YNG")), + *(f"DST_VAL{suffix}" for suffix in ("1", "2", "1_YNG", "2_YNG")), + } +) +CPS_CARRIED_BOOLEAN_INPUTS = frozenset( + { + "is_female", + "receives_wic", + "receives_tanf", + "receives_snap", + *US_REPORTED_COVERAGE_PERSON_INPUTS, + } +) + def derive_us_cps_carried_inputs( frame: Frame, *, public_assistance_type_source: pd.DataFrame | None = None, + reported_coverage_source: pd.DataFrame | None = None, ) -> Frame: """Carry raw CPS ASEC values onto PE input leaves. - Existing leaf input columns are preserved, making the transform idempotent. - The transform refuses to run on a non-US frame and never creates - formula-owned aggregate variables. + Existing leaves keep the original presence rules; the raw other-income + pair intentionally recomputes its three split leaves. The transform + refuses a non-US frame and never creates formula-owned aggregates. + + Optional ``reported_coverage_source`` restores seven measured at-interview + recodes before deriving coverage leaves. It requires an exact source join; + conflicting observations fail. Native ``NOW_GRP`` and ``NOW_MRK`` remain + unchanged. These observed inputs do not certify a fitted model or release. ``PAW_VAL``, ``SPM_SNAPSUB``, and ``WICYN`` are annual reported facts, while the engine's ``receives_tanf``, ``receives_snap``, and @@ -164,7 +228,49 @@ def derive_us_cps_carried_inputs( if frame.schema != US_SCHEMA: raise ValueError("CPS-carried derivations require the US schema.") tables = {entity: frame.table(entity).copy() for entity in frame.entities} - person = tables["person"] + produced = derive_us_cps_carried_tables( + tables["person"], + tables["spm_unit"], + public_assistance_type_source=public_assistance_type_source, + reported_coverage_source=reported_coverage_source, + ) + tables["person"] = produced.person + tables["spm_unit"] = produced.spm_unit + + return Frame( + tables, + frame.schema, + {entity: frame.weights_for(entity) for entity in frame.weighted_entities}, + frame.strata, + mass_log=frame.mass_log, + metadata=frame.metadata, + ) + + +@dataclass(frozen=True) +class CpsCarriedTables: + """The two actual entity tables touched by the CPS-carried operator.""" + + person: pd.DataFrame + spm_unit: pd.DataFrame + + +def derive_us_cps_carried_tables( + person: pd.DataFrame, + spm_unit: pd.DataFrame, + *, + public_assistance_type_source: pd.DataFrame | None = None, + reported_coverage_source: pd.DataFrame | None = None, +) -> CpsCarriedTables: + """Run the original measured mapping on isolated person and SPM views. + + Group outputs align by actual SPM IDs. Monetary amounts retain their + existing nominal source-year basis; this operation does no price alignment. + """ + person = person.copy(deep=True) + spm_unit = spm_unit.copy(deep=True) + if reported_coverage_source is not None: + person = fill_asec_reported_coverage_source(person, reported_coverage_source) _fill_missing(person, "age", _source(person, "A_AGE")) _fill_bool_missing(person, "is_female", _integer_source(person, "A_SEX") == 2) @@ -222,7 +328,6 @@ def derive_us_cps_carried_inputs( _fill_missing(person, "taxable_ira_distributions", _ira_distributions(person)) person = derive_us_alimony_from_asec(person) - tables["person"] = person direct_sources: Mapping[str, str] = { "rental_income": "RNT_VAL", @@ -241,10 +346,10 @@ def derive_us_cps_carried_inputs( _fill_health_coverage_inputs(person) _fill_spm_unit_reported_enrollment_inputs( person, - tables["spm_unit"], + spm_unit, public_assistance_type_source=public_assistance_type_source, ) - _fill_spm_unit_childcare_inputs(person, tables["spm_unit"]) + _fill_spm_unit_childcare_inputs(person, spm_unit) formula_owned = sorted(CPS_CARRIED_FORMULA_OWNED_COLUMNS.intersection(person)) if formula_owned: @@ -253,14 +358,7 @@ def derive_us_cps_carried_inputs( f"columns: {formula_owned}." ) - return Frame( - tables, - frame.schema, - {entity: frame.weights_for(entity) for entity in frame.weighted_entities}, - frame.strata, - mass_log=frame.mass_log, - metadata=frame.metadata, - ) + return CpsCarriedTables(person, spm_unit) def reported_tanf_enrollment_by_spm_unit( diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/cps_carried_current.py b/packages/microcosm-build/src/microcosm/build/us_runtime/cps_carried_current.py new file mode 100644 index 000000000..7270593ea --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/cps_carried_current.py @@ -0,0 +1,377 @@ +"""Corrected CPS monetary leaves derived from a typed selected-money subset. + +Amounts come only from the selected current-money artifact; routing codes come +only from declared raw code columns. A raw dollar column is never an alternate +input here: the derivation refuses one outright, so a corrected leaf cannot +silently fall back to the nominal source value. + +The split fractions, the RESNSS reason routing with its age-62 fallback, the +OI_OFF 20/12 partition, the DST code-4 IRA rule and the SPM childcare grain are +the legacy modelling choices, reused unchanged and imported from +``cps_carried``/``alimony`` rather than redeclared. Only the amounts they +consume change: nominal source dollars become the 2024-basis restated amounts +the money recipe already produced. +""" + +from __future__ import annotations + +from collections.abc import Mapping +from dataclasses import dataclass + +import numpy as np +import pandas as pd + +from .alimony import US_ASEC_OTHER_INCOME_OUTPUT_COLUMNS, derive_us_alimony_from_asec +from .asec_current_money import FIELDS +from .asec_current_money_selection import SelectedCurrentMoney +from .cps_carried import ( + LONG_TERM_CAPITAL_GAIN_FRACTION, + QUALIFIED_DIVIDEND_FRACTION, + TAXABLE_INTEREST_FRACTION, + TAXABLE_PENSION_FRACTION, +) + +CPS_CARRIED_CURRENT_ROUTING_COLUMNS: tuple[str, ...] = ( + "A_AGE", + "DST_SC1", + "DST_SC1_YNG", + "DST_SC2", + "DST_SC2_YNG", + "OI_OFF", + "RESNSS1", + "RESNSS2", +) +CPS_CARRIED_CURRENT_MONEY_FIELDS: tuple[str, ...] = ( + "ANN_VAL", + "CAP_VAL", + "DIV_VAL", + "DST_VAL1", + "DST_VAL1_YNG", + "DST_VAL2", + "DST_VAL2_YNG", + "FRSE_VAL", + "INT_VAL", + "OI_VAL", + "PHIP_VAL", + "PMED_VAL", + "PNSN_VAL", + "POTC_VAL", + "RNT_VAL", + "SEMP_VAL", + "SPM_CHILDCAREXPNS", + "SS_VAL", + "UC_VAL", + "WSAL_VAL", +) +CPS_CARRIED_CURRENT_PERSON_LEAVES: tuple[str, ...] = ( + "age", + "alimony_income", + "employment_income_before_lsr", + "farm_operations_income", + "health_insurance_premiums_without_medicare_part_b", + "long_term_capital_gains_before_response", + "miscellaneous_income", + "non_qualified_dividend_income", + "other_medical_expenses", + "over_the_counter_health_expenses", + "qualified_dividend_income", + "rental_income", + "self_employment_income_before_lsr", + "short_term_capital_gains", + "social_security_dependents", + "social_security_disability", + "social_security_retirement", + "social_security_survivors", + "strike_benefits", + "tax_exempt_private_pension_income", + "taxable_interest_income", + "taxable_ira_distributions", + "taxable_private_pension_income", + "unemployment_compensation", +) +CPS_CARRIED_CURRENT_SPM_UNIT_LEAVES: tuple[str, ...] = ( + "spm_unit_pre_subsidy_childcare_expenses", +) +CPS_CARRIED_CURRENT_CONTRACT_SCHEMA = "microcosm.us.cps-carried-current-leaves.v1" +_IRA_DISTRIBUTION_CODE = 4 +_SOCIAL_SECURITY_RETIREMENT_AGE_FALLBACK = 62 + + +class CpsCarriedCurrentRefusalError(ValueError): + """Sanitized refusal; never carries row identifiers or amounts.""" + + +def _require(condition: bool, reason: str) -> None: + if not condition: + raise CpsCarriedCurrentRefusalError(reason) + + +def _routing(routing, rows: int) -> dict[str, np.ndarray]: + _require(type(routing) is pd.DataFrame, "ROUTING_TABLE") + _require(len(routing) == rows, "ROUTING_ROW_ALIGNMENT") + columns = tuple(routing.columns) + _require(len(set(columns)) == len(columns), "ROUTING_ROSTER") + # A source dollar column can never reach this derivation, even by accident. + _require( + not any(name in FIELDS or name.endswith("_VAL") for name in columns), + "RAW_MONEY_INPUT_REFUSED", + ) + _require( + tuple(sorted(columns)) == CPS_CARRIED_CURRENT_ROUTING_COLUMNS, "ROUTING_ROSTER" + ) + values = {} + for name in CPS_CARRIED_CURRENT_ROUTING_COLUMNS: + series = routing[name] + _require(pd.api.types.is_integer_dtype(series.dtype), "ROUTING_DTYPE") + _require(not bool(series.isna().any()), "ROUTING_MISSING") + values[name] = series.to_numpy(dtype="int64", copy=True) + return values + + +def _amounts(selected: SelectedCurrentMoney, rows: int) -> dict[str, np.ndarray]: + values = {} + for name in CPS_CARRIED_CURRENT_MONEY_FIELDS: + _require(selected.entity_of(name) == "person", "MONEY_FIELD_ENTITY") + amounts = selected.amounts(name) + _require(len(amounts) == rows, "MONEY_ROW_ALIGNMENT") + _require(bool(np.isfinite(amounts).all()), "MONEY_NONFINITE") + values[name] = amounts + return values + + +def _social_security(amount: np.ndarray, routing) -> dict[str, np.ndarray]: + """Reuse the legacy RESNSS reason routing and its age-62 fallback exactly.""" + reason_1 = routing["RESNSS1"] + reason_2 = routing["RESNSS2"] + age = routing["A_AGE"] + is_retirement = (reason_1 == 1) | (reason_2 == 1) + is_disability = (reason_1 == 2) | (reason_2 == 2) + is_survivor = np.isin(reason_1, [3, 5]) | np.isin(reason_2, [3, 5]) + is_dependent = np.isin(reason_1, [4, 6, 7]) | np.isin(reason_2, [4, 6, 7]) + unclassified = ( + (amount > 0) & ~is_retirement & ~is_disability & ~is_survivor & ~is_dependent + ) + threshold = _SOCIAL_SECURITY_RETIREMENT_AGE_FALLBACK + return { + "social_security_retirement": np.where( + is_retirement | (unclassified & (age >= threshold)), amount, 0.0 + ), + "social_security_disability": np.where( + (is_disability & ~is_retirement) | (unclassified & (age < threshold)), + amount, + 0.0, + ), + "social_security_survivors": np.where( + is_survivor & ~is_retirement & ~is_disability, amount, 0.0 + ), + "social_security_dependents": np.where( + is_dependent & ~is_retirement & ~is_disability & ~is_survivor, amount, 0.0 + ), + } + + +def _ira_distributions(amounts, routing, rows: int) -> np.ndarray: + values = np.zeros(rows, dtype=np.float64) + for suffix in ("1", "2", "1_YNG", "2_YNG"): + code = routing[f"DST_SC{suffix}"] + values += np.where( + code == _IRA_DISTRIBUTION_CODE, amounts[f"DST_VAL{suffix}"], 0.0 + ) + return values + + +def _other_income(amount: np.ndarray, routing) -> dict[str, np.ndarray]: + """Delegate the OI_OFF 20/12 partition to the registered legacy helper.""" + table = pd.DataFrame({"OI_VAL": amount, "OI_OFF": routing["OI_OFF"]}) + split = derive_us_alimony_from_asec(table) + return { + name: split[name].to_numpy(dtype="float64", copy=True) + for name in US_ASEC_OTHER_INCOME_OUTPUT_COLUMNS + } + + +def _childcare(amount: np.ndarray, membership: np.ndarray, ids: np.ndarray): + """Carry the SPM-unit childcare value; members must already agree exactly.""" + _require( + type(membership) is np.ndarray + and membership.dtype == np.dtype("int64") + and membership.shape == amount.shape, + "SPM_MEMBERSHIP", + ) + _require( + type(ids) is np.ndarray and ids.dtype == np.dtype("int64") and ids.ndim == 1, + "SPM_IDS", + ) + _require(len(ids) == len(set(ids.tolist())), "SPM_IDS") + _require(np.array_equal(np.sort(ids), ids), "SPM_IDS") + _require(np.array_equal(np.unique(membership), ids), "SPM_MEMBERSHIP_COVERAGE") + positions = np.searchsorted(ids, membership) + values = np.zeros(len(ids), dtype=np.float64) + seen = np.zeros(len(ids), dtype=bool) + for position, value in zip(positions.tolist(), amount.tolist(), strict=True): + if seen[position]: + # The source repeats one SPM value on every member; a disagreement + # is a source contract failure, never something to reduce away. + _require(values[position] == value, "INCONSISTENT_SPM_AMOUNT") + continue + values[position] = value + seen[position] = True + return values + + +@dataclass(frozen=True) +class CpsCarriedCurrentLeaves: + """Corrected person and SPM-unit leaves, aligned to the selected rows.""" + + person: Mapping[str, np.ndarray] + spm_unit: Mapping[str, np.ndarray] + + +CPS_CURRENT_PREDICTOR_MONEY_FIELDS = ( + "WSAL_VAL", + "SEMP_VAL", + "INT_VAL", + "DIV_VAL", + "CAP_VAL", +) +CPS_CURRENT_PREDICTOR_PERSON_LEAVES = ( + "employment_income_before_lsr", + "self_employment_income_before_lsr", + "taxable_interest_income", + "qualified_dividend_income", + "non_qualified_dividend_income", + "short_term_capital_gains", + "long_term_capital_gains_before_response", +) + + +def derive_cps_current_predictor_leaves(amounts): + """Pure five-field split, sharing the maintained current-money judgments. + + Callers qualify the actual money owner or modeled target artifacts. This + numerical function grants no source authority and never fills an unknown. + Inputs are aligned physical float64 arrays, already in the declared dollar + basis. INT/DIV/CAP fitted for ACS remain modeled, including their splits. + """ + _require(isinstance(amounts, Mapping), "PREDICTOR_MONEY_MAPPING") + _require( + set(amounts) == set(CPS_CURRENT_PREDICTOR_MONEY_FIELDS), + "PREDICTOR_MONEY_ROSTER", + ) + lengths = set() + for value in amounts.values(): + _require( + type(value) is np.ndarray + and value.ndim == 1 + and value.dtype == np.dtype("float64"), + "PREDICTOR_MONEY_TYPE", + ) + _require(bool(np.isfinite(value).all()), "PREDICTOR_MONEY_UNKNOWN") + lengths.add(len(value)) + _require(len(lengths) == 1 and next(iter(lengths)) > 0, "PREDICTOR_MONEY_AXIS") + dividends, gains = amounts["DIV_VAL"], amounts["CAP_VAL"] + return { + "employment_income_before_lsr": amounts["WSAL_VAL"].copy(), + "self_employment_income_before_lsr": amounts["SEMP_VAL"].copy(), + "taxable_interest_income": amounts["INT_VAL"] * TAXABLE_INTEREST_FRACTION, + "qualified_dividend_income": dividends * QUALIFIED_DIVIDEND_FRACTION, + "non_qualified_dividend_income": dividends * (1 - QUALIFIED_DIVIDEND_FRACTION), + "long_term_capital_gains_before_response": gains + * LONG_TERM_CAPITAL_GAIN_FRACTION, + "short_term_capital_gains": gains * (1 - LONG_TERM_CAPITAL_GAIN_FRACTION), + } + + +def derive_cps_carried_current_leaves( + selected: SelectedCurrentMoney, + *, + routing: pd.DataFrame, + spm_membership: np.ndarray, + spm_ids: np.ndarray, +) -> CpsCarriedCurrentLeaves: + """Derive the corrected monetary CPS leaves plus ``age`` from selected money.""" + _require(type(selected) is SelectedCurrentMoney, "TYPED_SELECTED_REQUIRED") + rows = selected.person_rows + codes = _routing(routing, rows) + amounts = _amounts(selected, rows) + # The money recipe already contributes zero for a declared-NIU annuity, so + # the pension base is the plain sum of the two restated amounts. + pensions = amounts["PNSN_VAL"] + amounts["ANN_VAL"] + person: dict[str, np.ndarray] = { + "age": codes["A_AGE"].astype("float64"), + **derive_cps_current_predictor_leaves( + {name: amounts[name] for name in CPS_CURRENT_PREDICTOR_MONEY_FIELDS} + ), + "taxable_private_pension_income": pensions * TAXABLE_PENSION_FRACTION, + "tax_exempt_private_pension_income": pensions * (1 - TAXABLE_PENSION_FRACTION), + "taxable_ira_distributions": _ira_distributions(amounts, codes, rows), + "rental_income": amounts["RNT_VAL"], + "farm_operations_income": amounts["FRSE_VAL"], + "unemployment_compensation": amounts["UC_VAL"], + "health_insurance_premiums_without_medicare_part_b": amounts["PHIP_VAL"], + "other_medical_expenses": amounts["PMED_VAL"], + "over_the_counter_health_expenses": amounts["POTC_VAL"], + } + person.update(_social_security(amounts["SS_VAL"], codes)) + person.update(_other_income(amounts["OI_VAL"], codes)) + spm_unit = { + "spm_unit_pre_subsidy_childcare_expenses": _childcare( + amounts["SPM_CHILDCAREXPNS"], spm_membership, spm_ids + ) + } + _require( + tuple(sorted(person)) == CPS_CARRIED_CURRENT_PERSON_LEAVES, "PERSON_LEAF_ROSTER" + ) + _require( + tuple(sorted(spm_unit)) == CPS_CARRIED_CURRENT_SPM_UNIT_LEAVES, + "SPM_UNIT_LEAF_ROSTER", + ) + for values in (*person.values(), *spm_unit.values()): + _require(values.dtype == np.dtype("float64"), "LEAF_DTYPE") + _require(bool(np.isfinite(values).all()), "LEAF_NONFINITE") + for name, values in person.items(): + _require(len(values) == rows, "LEAF_ROW_ALIGNMENT") + del name + return CpsCarriedCurrentLeaves(person, spm_unit) + + +def cps_carried_current_leaf_contract() -> dict: + """Return the recorded derivation contract for these corrected leaves.""" + return { + "schema": CPS_CARRIED_CURRENT_CONTRACT_SCHEMA, + "amount_source": "us_asec_selected_current_money", + "amount_basis": "target_current_2024", + "raw_dollar_alternates": [], + "money_fields": list(CPS_CARRIED_CURRENT_MONEY_FIELDS), + "routing_columns": list(CPS_CARRIED_CURRENT_ROUTING_COLUMNS), + "person_leaves": list(CPS_CARRIED_CURRENT_PERSON_LEAVES), + "spm_unit_leaves": list(CPS_CARRIED_CURRENT_SPM_UNIT_LEAVES), + "dtype": "float64", + "model_assumptions": { + "taxable_interest_fraction": TAXABLE_INTEREST_FRACTION, + "qualified_dividend_fraction": QUALIFIED_DIVIDEND_FRACTION, + "taxable_pension_fraction": TAXABLE_PENSION_FRACTION, + "long_term_capital_gain_fraction": LONG_TERM_CAPITAL_GAIN_FRACTION, + "annuity_niu_contributes_zero": True, + "signed_losses_retained": ["SEMP_VAL", "RNT_VAL", "FRSE_VAL"], + "social_security_reason_routing": "RESNSS1/RESNSS2 with age-62 fallback", + "social_security_age_fallback": _SOCIAL_SECURITY_RETIREMENT_AGE_FALLBACK, + "other_income_split": "derive_us_alimony_from_asec on OI_VAL/OI_OFF", + "ira_distribution_code": _IRA_DISTRIBUTION_CODE, + "spm_childcare_grain": "spm_unit_repeated_on_person, member-consistent", + }, + "release_eligible": False, + } + + +__all__ = [ + "CPS_CARRIED_CURRENT_CONTRACT_SCHEMA", + "CPS_CARRIED_CURRENT_MONEY_FIELDS", + "CPS_CARRIED_CURRENT_PERSON_LEAVES", + "CPS_CARRIED_CURRENT_ROUTING_COLUMNS", + "CPS_CARRIED_CURRENT_SPM_UNIT_LEAVES", + "CpsCarriedCurrentLeaves", + "CpsCarriedCurrentRefusalError", + "cps_carried_current_leaf_contract", + "derive_cps_carried_current_leaves", +] diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/current_acs_income_anchor_source.py b/packages/microcosm-build/src/microcosm/build/us_runtime/current_acs_income_anchor_source.py new file mode 100644 index 000000000..b206f709c --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/current_acs_income_anchor_source.py @@ -0,0 +1,441 @@ +"""Observed ACS income anchors from the retained original source archive. + +Values and projections are descriptive, not source issuers. Consumers retain the +original preparation and requalify after their last relevant I/O. This module +does not decompose income, create tax inputs, or fill any analytical zeros. +""" + +from __future__ import annotations + +import csv +import hashlib +import json +import re +import tempfile +import zipfile +from dataclasses import dataclass +from pathlib import Path + +import numpy as np +import pandas as pd + +from . import acs_housing_universe_source as housing +from . import acs_person_coverage_authentication as records +from . import graph_full_puf_enrichment as physical +from . import source_csv_builtin +from . import support_provenance as provenance +from . import survey_population_preparation as preparation + +PROTOCOL = "microcosm.us.current-acs-income-anchors.v1" +COLUMNS = ("SERIALNO", "SPORDER", "INTP", "RETP", "ADJINC", "AGEP", "FINTP", "FRETP") +ANCHORS = ( + ("INTP", "FINTP", "property_income", "acs_interest_dividend_rental_income"), + ("RETP", "FRETP", "retirement_income", "acs_retirement_income"), +) +STRING_DTYPE = pd.StringDtype(storage="python", na_value=pd.NA) + + +def require(condition, reason): + if not condition: + raise ValueError("ACS_INCOME_ANCHOR_" + reason) + + +def _literal(value): + require(type(value) is str and len(value) <= 64, "LITERAL_TYPE_OR_BOUND") + return value + + +def parse_anchor( + token: str, *, age: str, adjustment: str, allocation: str, field: str +) -> dict: + """Classify source literals separately from native numeric storage. + + Allocation is provenance, not amount validity. Published domains describe + released records; their disclosure bounds are not latent income bounds. + """ + require(field in ("INTP", "RETP"), "FIELD") + token, age, adjustment, allocation = map( + _literal, (token, age, adjustment, allocation) + ) + require(re.fullmatch(r"[0-9]{1,2}", age, re.ASCII) is not None, "AGE_LITERAL") + parsed_age = int(age) + factor_valid = ( + re.fullmatch(r"[0-9]{1,7}", adjustment, re.ASCII) is not None + and int(adjustment) > 0 + ) + status, amount = "observed", None + if token == "": + status = ( + "outside_universe_blank" if parsed_age < 15 else "missing_source_amount" + ) + elif re.fullmatch(r"-?[0-9]+", token, re.ASCII) is None: + status = "malformed_source_amount" + else: + raw = int(token) + in_domain = ( + raw == 0 or 4 <= raw <= 999999 or (field == "INTP" and -10000 <= raw <= -4) + ) + if not in_domain: + status = "outside_published_domain" + elif parsed_age < 15: + status = "outside_universe_observation" + elif not factor_valid: + status = "invalid_adjustment" + else: + amount = float( + np.float64(raw) * (np.float64(int(adjustment)) / 1_000_000.0) + ) + return { + "amount": amount, + "known": status == "observed", + "status": status, + "allocation_status": { + "0": "not_allocated", + "1": "allocated", + "": "allocation_missing", + }.get(allocation, "allocation_unrecognized"), + "adjustment_known": factor_valid, + "in_income_universe": parsed_age >= 15, + } + + +def _key(row): + require( + re.fullmatch(r"2024(?:HU|GQ)[0-9]{7}", row["SERIALNO"], re.ASCII) is not None, + "SOURCE_HOUSEHOLD_KEY", + ) + require( + re.fullmatch(r"[0-9]{1,2}", row["SPORDER"], re.ASCII) is not None + and 1 <= int(row["SPORDER"]) <= 20, + "SOURCE_PERSON_KEY", + ) + return row["SERIALNO"], int(row["SPORDER"]) + + +def _scan(stream, wanted, selected, *, maximum): + require(source_csv_builtin.capture_csv_reader(csv) is not None, "CSV_BINDING") + header, count = None, 0 + for raw in records._records(stream): + values = records._decode_record(raw, first=header is None) + if header is None: + header = values + require( + bool(header) + and all(header) + and len(set(header)) == len(header) + and set(COLUMNS) <= set(header), + "SOURCE_HEADER", + ) + positions = [header.index(c) for c in COLUMNS] + continue + count += 1 + require(count <= maximum and len(values) == len(header), "SOURCE_ROW_SHAPE") + row = dict(zip(COLUMNS, (values[p] for p in positions), strict=True)) + require(all(len(v) <= 64 for v in row.values()), "SOURCE_TOKEN_BOUND") + key = _key(row) + if key in wanted: + require(key not in selected, "DUPLICATE_SELECTED_SOURCE_KEY") + selected[key] = row + require(header is not None, "SOURCE_HEADER") + return count + + +def _origins(state, document): + payload = document["origins"]["persons"] + full = pd.DataFrame(payload["rows"], columns=payload["columns"]) + people = state.frame.person + require( + full.person_id.dtype == np.dtype("int64") + and full.person_id.is_unique + and np.array_equal(full.person_id.to_numpy(), people.person_id.to_numpy()), + "ORIGIN_AXIS", + ) + channel = provenance.support_channel_column("person") + native_id = provenance.spine_source_id_column("person") + require( + np.array_equal(full.source.to_numpy(), people[channel].to_numpy()) + and np.array_equal( + full.selected_receiving_person_id.to_numpy(), people[native_id].to_numpy() + ), + "ORIGIN_FRAME_IDENTITY", + ) + origins = full.loc[full.source.eq("acs")].copy().set_index("person_id", drop=True) + require( + len(origins) > 0 + and origins.source_year.eq(2024).all() + and origins.survey_year.eq(2024).all(), + "ORIGIN_PERIOD", + ) + native = state.source_frames[0].person + require( + np.array_equal( + origins.selected_receiving_person_id.to_numpy(), native.person_id.to_numpy() + ), + "ORIGIN_NATIVE_AXIS", + ) + keys = [] + for _, row in origins.iterrows(): + require( + all( + type(v) is str + for v in ( + row.raw_native_household_id, + row.raw_native_person_id, + row.native_line_numeric_original, + ) + ), + "ORIGIN_LITERAL_TYPE", + ) + key = _key( + { + "SERIALNO": row.raw_native_household_id, + "SPORDER": row.native_line_numeric_original, + } + ) + require(row.raw_native_person_id == str(key[1]), "ORIGIN_NATIVE_PERSON_KEY") + keys.append(key) + require(len(set(keys)) == len(keys), "ORIGIN_DUPLICATE") + origins["anchor_source_key"] = keys + return origins + + +def _raw_table(origins, selected): + require(set(selected) == set(origins.anchor_source_key), "SELECTED_SOURCE_ROSTER") + raw = pd.DataFrame( + [selected[k] for k in origins.anchor_source_key], + index=origins.index.copy(), + columns=COLUMNS, + dtype=object, + ) + require( + all( + _key(row) == key + for row, key in zip( + raw.to_dict("records"), origins.anchor_source_key, strict=True + ) + ), + "SOURCE_COORDINATE_CHANGED", + ) + return raw + + +def _native_numbers(tokens): + # Match the existing mapper's numeric-coercion semantics for identity only; + # this operation does not determine source amount knownness. + return pd.to_numeric(pd.Series(tokens, dtype=object), errors="coerce").to_numpy( + dtype=np.float64, na_value=np.nan + ) + + +def _same_number(left, right): + if pd.isna(left): + return bool(pd.isna(right)) + if isinstance(left, (bool, np.bool_)) or not isinstance( + left, (int, float, np.integer, np.floating) + ): + return False + return np.float64(left).view("uint64") == np.float64(right).view("uint64") + + +def _compare_retained(raw, origins, native): + people = native.person + require( + raw.index.equals(origins.index) + and np.array_equal( + origins.selected_receiving_person_id.to_numpy(), people.person_id.to_numpy() + ), + "RETAINED_AXIS", + ) + households = native.table("household").set_index("household_id", drop=False) + require( + households.index.is_unique + and np.array_equal( + origins.selected_receiving_household_id.to_numpy(), + people.person_household_id.to_numpy(), + ), + "RETAINED_HOUSEHOLD_ID", + ) + serials = households.SERIALNO.reindex(people.person_household_id.to_numpy()) + require( + np.array_equal(raw.SERIALNO.to_numpy(), serials.to_numpy()) + and np.array_equal(raw.SPORDER.map(int).to_numpy(), people.SPORDER.to_numpy()), + "RETAINED_SOURCE_ID", + ) + numbers = {c: _native_numbers(raw[c]) for c in ("INTP", "RETP", "ADJINC", "AGEP")} + for column, expected_numbers in numbers.items(): + for literal, stored, expected in zip( + raw[column], people[column], expected_numbers, strict=True + ): + same = ( + stored == literal + if type(stored) is str + else _same_number(stored, expected) + ) + require(bool(same), "RETAINED_RAW_" + column) + adjustment = numbers["ADJINC"] + for column, _flag, _prefix, output in ANCHORS: + require(people[output].dtype == np.dtype("float64"), "RETAINED_ADJUSTED_DTYPE") + expected = numbers[column] * (adjustment / 1_000_000.0) + actual = people[output].to_numpy(copy=False) + require( + np.array_equal(np.isnan(expected), np.isnan(actual)) + and np.array_equal( + expected[~np.isnan(expected)].view("uint64"), + actual[~np.isnan(actual)].view("uint64"), + ), + "RETAINED_ADJUSTED_BITS", + ) + + +def _parsed_table(raw, origins): + result = raw.copy(deep=True) + result["native_person_id"] = origins.selected_receiving_person_id.to_numpy( + copy=True + ) + result["source_year"] = 2024 + result["dollar_year"] = 2024 + for column, flag, prefix, _output in ANCHORS: + values = [ + parse_anchor( + r[column], + age=r["AGEP"], + adjustment=r["ADJINC"], + allocation=r[flag], + field=column, + ) + for r in raw.to_dict("records") + ] + parsed = pd.DataFrame(values, index=raw.index) + for name in parsed: + dtype = ( + "Float64" + if name == "amount" + else ( + bool + if name in ("known", "adjustment_known", "in_income_universe") + else STRING_DTYPE + ) + ) + result[prefix + "_" + name] = pd.array(parsed[name], dtype=dtype) + return result + + +@dataclass(frozen=True) +class QualifiedAcsIncomeAnchors: + """Detached source observations, not a substitute for the preparation.""" + + anchors: pd.DataFrame + projection: bytes + evidence: dict + + +def income_anchor_seal(qualified: QualifiedAcsIncomeAnchors) -> tuple: + require(type(qualified) is QualifiedAcsIncomeAnchors, "QUALIFIED_TYPE") + require( + qualified.projection + == qualified.anchors.reset_index().to_json(orient="table", index=False).encode() + and qualified.evidence["projection_sha256"] + == hashlib.sha256(qualified.projection).hexdigest(), + "PROJECTION_BINDING", + ) + return ( + physical._table_stamp(qualified.anchors), + qualified.projection, + json.dumps( + qualified.evidence, sort_keys=True, separators=(",", ":"), allow_nan=False + ), + ) + + +def _capture_person(root, pin, wanted, total_rows): + """Return literal values from an owned capture, after cleanup I/O completes. + + Caller-supplied pins grant no authority here; the qualifier binds them to + the retained catalogue and rechecks that original owner after this returns. + """ + _role, name, digest, size = pin + selected, count = {}, 0 + with tempfile.TemporaryDirectory( + prefix="microcosm-acs-income-anchor-" + ) as directory: + captured = Path(directory) / name + require( + housing._copy(root / "acs" / name, captured, size, exact_size=size) + == digest, + "ACS_CAPTURE_DIGEST", + ) + with zipfile.ZipFile(captured) as archive: + members, prefix = records._members(archive, "person") + for item in members: + with archive.open(item) as stream: + if item.filename.casefold().startswith(prefix): + count += _scan( + stream, + wanted, + selected, + maximum=total_rows - count, + ) + else: + while stream.read(65536): + pass + require( + count == total_rows and housing._persisted_sha(captured, size) == digest, + "ACS_CAPTURE_CHANGED", + ) + return selected + + +def qualify_current_acs_income_anchors(source_preparation) -> QualifiedAcsIncomeAnchors: + """Read exact original ACS members and recheck the real owner before return.""" + require( + type(source_preparation) + is preparation.AuthenticatedSurveyPopulationPreparation, + "PREPARATION_TYPE", + ) + entry = source_preparation._checked() + state, document = entry[2], json.loads(entry[1]) + origins = _origins(state, document) + owned = preparation.acs_catalogue._lookup(state.catalogues[0]) + catalogue = json.loads(owned.receipt) + pins = tuple(p for p in owned.pins if p[0] == "person") + require( + len(pins) == 1 and catalogue["source_year"] == catalogue["survey_year"] == 2024, + "ACS_PIN_OR_PERIOD", + ) + selected = _capture_person( + state.root, + pins[0], + set(origins.anchor_source_key), + catalogue["counts"]["people"], + ) + raw = _raw_table(origins, selected) + _compare_retained(raw, origins, state.source_frames[0]) + anchors = _parsed_table(raw, origins) + projection = anchors.reset_index().to_json(orient="table", index=False).encode() + evidence = { + "protocol": PROTOCOL, + "preparation_sha256": hashlib.sha256(entry[1]).hexdigest(), + "acs_catalogue_sha256": hashlib.sha256(owned.receipt).hexdigest(), + "acs_person_archive_sha256": pins[0][2], + "selected_persons": len(anchors), + "source_year": 2024, + "dollar_year": 2024, + "income_period": "rolling_12_months", + "projection_sha256": hashlib.sha256(projection).hexdigest(), + "source_admission_issued": False, + "unallocated_observation_claim": False, + "analytical_universe_zeros_supplied": False, + "decomposition_performed": False, + "release_eligible": False, + } + result = QualifiedAcsIncomeAnchors(anchors, projection, evidence) + seal = income_anchor_seal(result) + require(source_preparation._checked() is entry, "PREPARATION_CHANGED") + preparation._pure_final(state) + require( + preparation._ISSUED.get(id(source_preparation)) is entry + and preparation.acs_catalogue._lookup(state.catalogues[0]) is owned, + "FINAL_OWNER", + ) + require(income_anchor_seal(result) == seal, "FINAL_VALUES_CHANGED") + return result diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/current_asec_demographics.py b/packages/microcosm-build/src/microcosm/build/us_runtime/current_asec_demographics.py new file mode 100644 index 000000000..12b88afe4 --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/current_asec_demographics.py @@ -0,0 +1,379 @@ +"""Current survey source projection for ASEC sex and household state. + +Reuse the maintained demographic source owner for A_SEX/AXSEX. Add only the +missing GESTFIPS household projection through the original weight owner's +bounded capture/CSV primitives and closed member registry. Returned numerical +values are not source authority; a graph host must re-run this live qualifier +and compare materialized descendants, including after replay. +""" + +from __future__ import annotations + +import csv +import io +import json +import os +import re +import tempfile +from dataclasses import dataclass +from pathlib import Path + +import numpy as np +import pandas as pd + +from . import asec_demographic_source as demographic +from . import asec_original_household_weights as household +from . import source_csv_builtin +from . import survey_population_preparation as preparation_owner +from .support_provenance import spine_source_id_column, support_channel_column + +PROTOCOL = "microcosm.us.current-asec-demographic-projection.v1" +STATE_COLUMNS = ("H_SEQ", "GESTFIPS") +STATE_CONTRACT = { + "field": "GESTFIPS", + "concept": "State FIPS code", + "entity": "household", + "survey_year": 2025, + "income_year": 2024, + "dictionary_url": "https://www2.census.gov/programs-surveys/cps/datasets/2025/march/asec2025_ddl_pub_full.pdf", + "dictionary_sha256": "5cb80973326ef8b625fbaae70d80b0c641ce5d2b3911abd2fb4427abd5908a6f", + "pdf_page_1based": 8, + "printed_page": "6A-1", + "printed_length": 2, + "printed_position": 44, + "printed_range": [1, 56], + "universe": "All Households", + "canonical_mapping": "numeric_identity", + "unassigned_jurisdiction_validation": "separate_geography_gate", + "unknown_policy": "preserve_token_and_unbound_state; never_use_carried_state_as_fallback", +} + + +def _require(condition, reason): + if not condition: + raise ValueError("CURRENT_ASEC_DEMOGRAPHICS_" + reason) + + +def _state_token(token): + """Preserve an unknown literal rather than interpreting it as a state.""" + _require( + type(token) is str and len(token) <= household._MAX_FIELD_CHARS, + "STATE_TOKEN_BOUND", + ) + if token == "": + return None, "missing" + if re.fullmatch(r"[0-9]{1,2}", token, re.ASCII) is None: + return None, "malformed" + code = int(token) + return ( + (code, "in_printed_range") + if 1 <= code <= 56 + else (None, "outside_printed_range") + ) + + +def _read_state_capture(path, pin): + """Low-level literal reader; a caller-supplied pin grants no authority.""" + reader = source_csv_builtin.capture_csv_reader(csv) + _require(reader is not None, "CSV_READER_CHANGED") + result = {} + with open(path, "rb", buffering=0, opener=household._regular_opener) as raw: + before = os.fstat(raw.fileno()) + _require(before.st_size == pin.size_bytes, "CAPTURE_SIZE") + digest = household._DigestReader(raw, pin.size_bytes) + with io.TextIOWrapper( + io.BufferedReader(digest), encoding="utf-8-sig", newline="" + ) as text: + lines = household._RecordLines(text) + records = reader(lines, strict=True) + header = next(records, []) + _require( + bool(header) + and all(header) + and len(header) == len(set(header)) + and set(STATE_COLUMNS) <= set(header), + "STATE_HEADER", + ) + positions = [header.index(c) for c in STATE_COLUMNS] + while True: + lines.characters = 0 + try: + row = next(records) + except StopIteration: + break + _require( + len(row) == len(header) and len(result) < pin.rows, + "STATE_ROW_SHAPE", + ) + key, token = (row[i] for i in positions) + _require( + re.fullmatch(r"[0-9]{1,5}", key, re.ASCII) is not None + and 1 <= int(key) <= 99999, + "HOUSEHOLD_KEY", + ) + native = int(key) + _require(native not in result, "HOUSEHOLD_DUPLICATE_KEY") + code, status = _state_token(token) + result[native] = { + "H_SEQ": key, + "GESTFIPS": token, + "state_code": code, + "status": status, + "member_row_1based": len(result) + 1, + } + _require(len(result) == pin.rows, "STATE_ROW_COUNT") + _require( + digest.count == pin.size_bytes + and digest.digest.hexdigest() == pin.member_sha256, + "STATE_CAPTURE_CHANGED", + ) + after = os.fstat(raw.fileno()) + _require( + all( + getattr(before, k) == getattr(after, k) + for k in ( + "st_dev", + "st_ino", + "st_size", + "st_mtime_ns", + "st_ctime_ns", + ) + ), + "STATE_CAPTURE_CHANGED", + ) + return result + + +def _load_current_state(member_path, expected_member): + """Closed registry lookup; expected member is retained native-owner evidence.""" + pins = household._registry() + selected = [pin for pin in pins if pin.income_year == 2024] + _require(len(selected) == 1, "CURRENT_STATE_REGISTRY") + pin = selected[0] + _require( + pin.survey_year == 2025 + and pin.canonical_member_id == expected_member["canonical_member_id"] + and pin.member_sha256 == expected_member["member_sha256"] + and pin.archive_sha256 == expected_member["archive_sha256"], + "STATE_NATIVE_MEMBER_BINDING", + ) + registry = household._encode(household._implementation()) + paths = household._member_path_snapshot({2024: member_path}, pins) + with tempfile.TemporaryDirectory(prefix="asec-current-state-") as directory: + capture = Path(directory) / "hhpub25.csv" + _require( + household._capture_owner._snapshot( + paths[2024], capture, size=pin.size_bytes + ) + == pin.member_sha256, + "STATE_SOURCE_SHA256", + ) + rows = _read_state_capture(capture, pin) + _require( + household._encode(household._implementation()) == registry, + "STATE_REGISTRY_CHANGED", + ) + return rows, { + "canonical_member_id": pin.canonical_member_id, + "member_sha256": pin.member_sha256, + "archive_sha256": pin.archive_sha256, + "bytes": pin.size_bytes, + "rows": pin.rows, + } + + +@dataclass(frozen=True) +class CurrentAsecDemographicValues: + """Source-derived selected arrays; no independent authority or launch verdict.""" + + person: pd.DataFrame + household: pd.DataFrame + receipt: bytes + + +def qualify_current_asec_demographics(preparation): + """Reconstruct the real parent sex owner and current household state source. + + The selected current survey is joined after full-source reconstruction. + Missing/unlabelled sex allocation or state codes remain explicitly unknown; + no non-2→male or carried-state fallback is permitted. + """ + _require( + type(preparation) is preparation_owner.AuthenticatedSurveyPopulationPreparation, + "PREPARATION_TYPE", + ) + entry = preparation._checked() + state = entry[2] + native = state.native[1] + native_entry = preparation_owner.asec_native._ISSUED.get(id(native)) + _require( + native_entry is not None + and native_entry[0]() is native + and native.payload == native_entry[1], + "NATIVE_ISSUANCE", + ) + parent = native_entry[2].parent + native_receipt = json.loads(native_entry[1]) + _require( + native_receipt["source_year"] == native_receipt["income_year"] == 2024 + and native_receipt["survey_year"] == 2025, + "NATIVE_PERIOD", + ) + observed = demographic.load_authenticated_asec_demographic_source( + parent, + member_paths={ + year: state.root / "asec" / f"pppub{year - 1999}.csv" + for year in (2022, 2023, 2024) + }, + ) + sex_receipt = observed.receipt + _require( + sex_receipt["source_identity"] == parent.source.identity.decode(), + "DEMOGRAPHIC_PARENT_IDENTITY", + ) + fields = native_entry[2].fields.document + _require(len(fields["members"]) == 1, "STATE_MEMBER_ROSTER") + states, state_pin = _load_current_state( + state.root / "asec" / "hhpub25.csv", fields["members"][0] + ) + source_households = { + row["household_id"]: row for row in native_receipt["households"] + } + households = state.frame.table("household") + selected_hh = households.loc[ + households[support_channel_column("household")].eq("asec") + ] + hrows = [] + for stacked, source_hh in selected_hh[ + ["household_id", spine_source_id_column("household")] + ].itertuples(index=False, name=None): + _require(source_hh in source_households, "HOUSEHOLD_NATIVE_JOIN") + current = source_households[source_hh] + year, key = current["native_key"] + _require(year == 2024 and key in states, "HOUSEHOLD_SOURCE_JOIN") + row = states[key] + _require( + row["H_SEQ"] == current["household_fields"]["H_SEQ"], + "HOUSEHOLD_LITERAL_KEY", + ) + hrows.append( + ( + int(stacked), + int(source_hh), + key, + row["GESTFIPS"], + row["state_code"], + row["status"], + ) + ) + htable = pd.DataFrame( + hrows, + columns=( + "household_id", + "native_household_id", + "H_SEQ_integer", + "GESTFIPS", + "state_fips", + "state_status", + ), + ).set_index("household_id", drop=False) + htable["state_fips"] = pd.array(htable.state_fips, dtype="Int64") + htable["state_known"] = htable.state_fips.notna() + full_ids, years = observed.array("person_id"), observed.array("income_year") + _require( + len(set(zip(years, full_ids, strict=True))) == len(full_ids), "SEX_SOURCE_AXIS" + ) + lookup = pd.MultiIndex.from_arrays((years, full_ids)) + person = state.frame.person + selected = person.loc[person[support_channel_column("person")].eq("asec")] + original_ids = selected[spine_source_id_column("person")].to_numpy(dtype=np.int64) + positions = lookup.get_indexer( + pd.MultiIndex.from_arrays( + (np.full(len(selected), 2024, dtype=np.int64), original_ids) + ) + ) + _require( + (positions >= 0).all() and len(set(positions)) == len(positions), + "SEX_NATIVE_JOIN", + ) + ptable = pd.DataFrame( + { + "person_id": selected.person_id.to_numpy(dtype=np.int64), + "native_person_id": original_ids, + }, + index=pd.Index(selected.person_id.to_numpy(), name="person_id"), + ) + for name in ( + "asec_A_SEX", + "asec_AXSEX", + "asec_sex_binding_state", + "asec_sex_allocation_state", + ): + ptable[name] = observed.array(name)[positions] + binding = ptable.asec_sex_binding_state.to_numpy() + _require(np.isin(binding, (0, 1, 2)).all(), "SEX_BINDING_DOMAIN") + ptable["is_female"] = pd.array( + [False if code == 1 else True if code == 2 else None for code in binding], + dtype="boolean", + ) + ptable["sex_known"] = binding != 0 + ptable["sex_universe"] = "All Persons" + ptable["sex_origin"] = np.where( + ptable.asec_sex_allocation_state.eq(2), + "census_allocated", + np.where( + ptable.asec_sex_allocation_state.eq(1), + "source_no_change", + "unresolved_allocation", + ), + ) + receipt = preparation_owner._encode( + { + "protocol": PROTOCOL, + "preparation_sha256": preparation_owner._sha(entry[1]), + "native_population_sha256": preparation_owner._sha(native_entry[1]), + "demographic_source_content_sha256": observed.content_sha256, + "demographic_source_receipt": sex_receipt, + "state_source": state_pin, + "state_contract": STATE_CONTRACT, + "selected_person_ids": ptable.person_id.tolist(), + "selected_household_ids": htable.household_id.tolist(), + "sex_known_persons": int(ptable.sex_known.sum()), + "sex_unknown_persons": int((~ptable.sex_known).sum()), + "state_known_households": int(htable.state_known.sum()), + "state_unknown_households": int((~htable.state_known).sum()), + "sex_observation_year": 2025, + "state_observation_year": 2025, + "income_year": 2024, + "state_is_income_year_residence_claim": False, + "selected_household_state_tokens": htable.loc[ + :, + [ + "household_id", + "native_household_id", + "H_SEQ_integer", + "GESTFIPS", + "state_status", + ], + ].values.tolist(), + "person_projection_sha256": preparation_owner._sha( + ptable.reset_index(drop=True).to_json(orient="table").encode() + ), + "household_projection_sha256": preparation_owner._sha( + htable.reset_index(drop=True).to_json(orient="table").encode() + ), + "source_admission_issued": False, + "release_eligible": False, + } + ) + observed.validate() + preparation_owner._pure_final(state) + _require( + preparation_owner._ISSUED.get(id(preparation)) is entry + and preparation.payload == entry[1] + and preparation_owner.asec_native._ISSUED.get(id(native)) is native_entry + and native.payload == native_entry[1] + and native_entry[2].parent is parent, + "FINAL_ISSUANCE", + ) + return CurrentAsecDemographicValues(ptable, htable, receipt) diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/current_asec_unemployment_source.py b/packages/microcosm-build/src/microcosm/build/us_runtime/current_asec_unemployment_source.py new file mode 100644 index 000000000..7eb0d827c --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/current_asec_unemployment_source.py @@ -0,0 +1,347 @@ +"""Receipt-universe projection from the retained current ASEC original. + +UC_VAL's numeric zero is not itself evidence of nonreceipt. This module keeps +the source amount separate from a canonical amount whose reporting basis is +known. Detached projection values grant no source authority: consuming hosts +requalify the retained preparation and compare these values before/after I/O. +""" + +from __future__ import annotations + +import csv +import hashlib +import json +import re +import tempfile +from dataclasses import dataclass +from pathlib import Path + +import numpy as np +import pandas as pd + +from . import asec_coverage_authentication as coverage +from . import asec_current_money as money +from . import source_csv_builtin +from . import survey_population_preparation as source +from .support_provenance import spine_source_id_column, support_channel_column + +PROTOCOL = "microcosm.us.current-asec-unemployment-source.v1" +READ_COLUMNS = ("PERIDNUM", "PH_SEQ", "A_LINENO", "A_AGE", "UC_VAL", "UC_YN") +DICTIONARY = { + "url": "https://www2.census.gov/programs-surveys/cps/datasets/2025/march/asec2025_ddl_pub_full.pdf", + "sha256": "5cb80973326ef8b625fbaae70d80b0c641ce5d2b3911abd2fb4427abd5908a6f", + "pdf_page_1based": 50, + "printed_page": "6C-29", + "receipt_field": "UC_YN", + "receipt_codes": {"0": "niu", "1": "yes", "2": "no"}, + "receipt_universe": "All Persons aged 15+", + "amount_field": "UC_VAL", + "amount_universe": "UC_YN = 1", + "zero_code": "none or niu", +} + + +def require(condition, reason): + if not condition: + raise ValueError("CURRENT_ASEC_UNEMPLOYMENT_" + reason) + + +def reporting_basis(amount, age, receipt_tokens): + """Classify invented or observed arrays without issuing source authority. + + Invalid/missing receipt literals are unresolved, rather than recoded to no. + Contradictions remain inspectable source observations, excluded from the + modeled donor and canonical amount. Age only gates the printed universe; + it never creates an income zero. + """ + amounts = np.asarray(amount) + ages = np.asarray(age) + tokens = tuple(receipt_tokens) + require( + amounts.dtype == np.dtype("float64") + and ages.dtype == np.dtype("float64") + and amounts.ndim == ages.ndim == 1 + and len(amounts) == len(ages) == len(tokens), + "ARRAY_CONTRACT", + ) + require( + np.isfinite(ages).all() + and ((ages >= 0) & (ages <= 99) & (ages == np.floor(ages))).all(), + "AGE_DOMAIN", + ) + require( + all(type(t) is str and len(t) <= 64 for t in tokens) + and not np.isinf(amounts).any() + and ( + (amounts[np.isfinite(amounts)] >= 0) + & (amounts[np.isfinite(amounts)] <= 99999) + ).all(), + "AMOUNT_OR_TOKEN_DOMAIN", + ) + result = pd.DataFrame({"source_amount": amounts.copy(), "receipt_literal": tokens}) + result["source_reporting_universe"] = ages >= 15 + result["receipt_code_known"] = [t in ("0", "1", "2") for t in tokens] + result["receipt_code"] = pd.array( + [int(t) if t in ("0", "1", "2") else pd.NA for t in tokens], dtype="Int8" + ) + canonical = np.full(len(amounts), np.nan, dtype=np.float64) + labels = [] + for i, (value, years, token) in enumerate(zip(amounts, ages, tokens, strict=True)): + if years < 15: + label = ( + "outside_reporting_universe" + if token == "0" and (np.isnan(value) or value == 0) + else "contradictory_outside_reporting_universe" + ) + elif token == "": + label = "missing_receipt_literal" + elif token not in ("0", "1", "2"): + label = "unrecognized_receipt_literal" + elif np.isnan(value): + label = "missing_amount" + elif token == "0": + label = "niu" if value == 0 else "contradictory_niu_positive" + elif token == "1": + label = "known_receipt" if value > 0 else "ambiguous_recipient_zero" + if value > 0: + canonical[i] = value + else: + label = "known_nonreceipt" if value == 0 else "contradictory_no_positive" + if value == 0: + canonical[i] = value + labels.append(label) + result["reporting_status"] = labels + result["canonical_amount"] = canonical + result["canonical_amount_known"] = np.isfinite(canonical) + return result + + +def _read_capture(path, *, rows): + """Bounded literal reader; its path and row count do not establish authority.""" + reader = source_csv_builtin.capture_csv_reader(csv) + require(reader is not None, "CSV_READER_CHANGED") + records, keys, coordinates = [], set(), set() + with Path(path).open("r", encoding="utf-8-sig", newline="") as handle: + stream = reader(handle, strict=True) + header = next(stream, []) + require( + bool(header) + and all(header) + and len(header) == len(set(header)) + and set(READ_COLUMNS) <= set(header), + "HEADER", + ) + positions = [header.index(c) for c in READ_COLUMNS] + for row in stream: + require(len(row) == len(header) and len(records) < rows, "ROW_SHAPE") + record = dict(zip(READ_COLUMNS, (row[i] for i in positions), strict=True)) + key = record["PERIDNUM"] + require(re.fullmatch(r"[0-9]{22}", key, re.ASCII) is not None, "PERSON_KEY") + for name, width in (("PH_SEQ", 5), ("A_LINENO", 2), ("A_AGE", 2)): + require( + re.fullmatch(r"[0-9]{1," + str(width) + "}", record[name], re.ASCII) + is not None, + "COORDINATE:" + name, + ) + pair = (int(record["PH_SEQ"]), int(record["A_LINENO"])) + require( + min(pair) > 0 and pair not in coordinates and key not in keys, + "DUPLICATE_OR_INVALID_COORDINATE", + ) + require( + record["UC_VAL"] == "" + or re.fullmatch(r"[0-9]{1,5}", record["UC_VAL"], re.ASCII) is not None, + "AMOUNT_TOKEN", + ) + require(len(record["UC_YN"]) <= 64, "RECEIPT_TOKEN_BOUND") + records.append(record) + keys.add(key) + coordinates.add(pair) + require(len(records) == rows, "ROW_COUNT") + return pd.DataFrame(records).set_index("PERIDNUM", drop=False) + + +def _file_sha(path): + digest = hashlib.sha256() + with Path(path).open("rb") as handle: + while chunk := handle.read(1_048_576): + digest.update(chunk) + return digest.hexdigest() + + +def _amount_observations(field, positions, literals): + """Join literal validity; missing backing storage never becomes a survey zero. + + Today's retained parent requires complete ready money before issuing a + preparation. This helper keeps that prerequisite separate from the literal + join, so a missing field is represented correctly if that scope later grows. + """ + require(type(field) is money.MoneyField, "MONEY_FIELD_TYPE") + positions = np.asarray(positions) + require( + positions.dtype == np.dtype("int64") and positions.ndim == 1, "MONEY_POSITIONS" + ) + tokens = tuple(literals) + require( + len(tokens) == len(positions) + and all( + type(t) is str and (t == "" or re.fullmatch(r"[0-9]{1,5}", t, re.ASCII)) + for t in tokens + ), + "AMOUNT_TOKEN", + ) + valid = field.validity[positions] == 1 + require( + np.array_equal(valid, np.array([t != "" for t in tokens])), + "CURRENT_AMOUNT_SOURCE_VALIDITY", + ) + amounts = field.amounts[positions].copy() + literal_values = np.asarray([float(t) if t else np.nan for t in tokens]) + require( + np.array_equal(amounts[valid], literal_values[valid]), + "CURRENT_AMOUNT_SOURCE_IDENTITY", + ) + amounts[~valid] = np.nan + return amounts + + +@dataclass(frozen=True) +class CurrentAsecUnemploymentValues: + person: pd.DataFrame + evidence: dict + + +def qualify_current_asec_unemployment(preparation): + """Capture the exact current source member retained by the original owner.""" + require( + type(preparation) is source.AuthenticatedSurveyPopulationPreparation, + "PREPARATION_TYPE", + ) + entry = preparation._checked() + state = entry[2] + native = state.native[1] + issued = source.asec_native._ISSUED.get(id(native)) + require( + issued is not None and issued[0]() is native and issued[1] == native.payload, + "NATIVE_ISSUANCE", + ) + parent = issued[2].parent + ready = parent.ready() + header = json.loads(ready.header) + require( + header["target_year"] == 2024 and header["semantic"] == "annual_current_money", + "MONEY_PERIOD", + ) + native_document = json.loads(issued[1]) + require( + native_document["source_year"] == native_document["income_year"] == 2024 + and native_document["survey_year"] == 2025, + "NATIVE_PERIOD", + ) + pins = [p for p in coverage._MEMBER_PINS if p[0] == 2024] + require(len(pins) == 1, "SOURCE_REGISTRY") + year, member, archive, digest, rows, size = pins[0] + retained = [ + s for s in issued[2].coverage.receipt["sources"] if s["source_year"] == year + ] + require( + len(retained) == 1 + and retained[0]["member"] == member + and retained[0]["archive_sha256"] == archive + and retained[0]["member_sha256"] == digest + and retained[0]["rows"] == rows + and retained[0]["member_bytes"] == size, + "NATIVE_MEMBER_BINDING", + ) + with tempfile.TemporaryDirectory(prefix="microcosm-current-uc-") as tmp: + captured = Path(tmp) / member + identity = coverage._capture( + state.root / "asec" / member, + captured, + size=size, + digest=digest, + budget=[coverage._BODY_MAX], + ) + raw = _read_capture(captured, rows=rows) + require( + coverage._identity(captured.stat(follow_symlinks=False)) == identity + and _file_sha(captured) == digest, + "CAPTURE_CHANGED", + ) + positions = np.flatnonzero(np.asarray(parent.scope.person_years) == 2024) + keys = np.asarray(parent.scope.person_native_keys)[positions] + require( + len(keys) == rows and len(set(keys)) == rows and set(keys) == set(raw.index), + "COMPLETE_CURRENT_SOURCE_JOIN", + ) + ordered = raw.loc[keys] + parent_people = parent.frame.person.iloc[positions] + for raw_name, parent_name in ( + ("PH_SEQ", "source_household_id"), + ("A_LINENO", "A_LINENO"), + ("A_AGE", "A_AGE"), + ): + require( + np.array_equal( + ordered[raw_name].astype("int64").to_numpy(), + parent_people[parent_name].to_numpy(), + ), + "PARENT_COORDINATE_IDENTITY", + ) + field = ready.field("UC_VAL") + amounts = _amount_observations(field, positions, ordered.UC_VAL) + basis = reporting_basis( + amounts, ordered.A_AGE.to_numpy(dtype="float64"), ordered.UC_YN + ) + basis.index = pd.Index( + np.asarray(parent.scope.person_ids)[positions], name="native_person_id" + ) + basis["amount_status"] = field.statuses[positions] + basis["amount_validity"] = field.validity[positions] + basis["zero_origin"] = field.zero_origin[positions] + people = state.frame.person + selected = people.loc[people[support_channel_column("person")].eq("asec")] + native_ids = selected[spine_source_id_column("person")].to_numpy() + require( + len(set(native_ids)) == len(native_ids) and set(native_ids) <= set(basis.index), + "SELECTED_NATIVE_JOIN", + ) + out = basis.loc[native_ids].copy() + out["native_person_id"] = native_ids + out.index = pd.Index(selected.person_id.to_numpy(), name="person_id") + evidence = { + "protocol": PROTOCOL, + "dictionary": json.loads(json.dumps(DICTIONARY)), + "preparation_sha256": hashlib.sha256(entry[1]).hexdigest(), + "asec_native_sha256": hashlib.sha256(issued[1]).hexdigest(), + "money_header_sha256": hashlib.sha256(ready.header).hexdigest(), + "source_member_sha256": digest, + "read_columns": list(READ_COLUMNS), + "complete_current_source_rows": rows, + "selected_rows": len(out), + "projection_sha256": hashlib.sha256( + out.to_json(orient="table").encode() + ).hexdigest(), + "policy": "known_yes_positive_or_no_zero_only; ambiguous_NIU_missing_contradictions_retained_unknown", + "unallocated_observation_claim": False, + "source_admission_issued": False, + "release_eligible": False, + } + require( + preparation._checked()[1] == entry[1] and parent.ready().header == ready.header, + "SOURCE_CHANGED", + ) + source._pure_final(state) + require( + source._ISSUED.get(id(preparation)) is entry + and source.asec_native._ISSUED.get(id(native)) is issued + and issued[2].parent is parent + and native.payload == issued[1], + "FINAL_OWNER", + ) + require( + hashlib.sha256(out.to_json(orient="table").encode()).hexdigest() + == evidence["projection_sha256"], + "FINAL_PROJECTION_CHANGED", + ) + return CurrentAsecUnemploymentValues(out, evidence) diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/current_social_security_source.py b/packages/microcosm-build/src/microcosm/build/us_runtime/current_social_security_source.py new file mode 100644 index 000000000..8445b6304 --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/current_social_security_source.py @@ -0,0 +1,420 @@ +"""Current survey Social Security totals, reason literals and unknown shares. + +An additive projection from a live authenticated survey preparation. The +current ASEC original member is captured against the maintained closed member +registry; carried RESNSS columns are never source evidence. ACS SSP remains a +combined source amount. This module fits no model and issues no admission. +""" + +from __future__ import annotations + +import csv +import hashlib +import json +import re +import tempfile +from dataclasses import dataclass +from pathlib import Path + +import numpy as np +import pandas as pd + +from . import asec_coverage_authentication as coverage +from . import source_csv_builtin +from . import survey_population_preparation as source +from . import survey_social_security as basis_owner +from .support_provenance import spine_source_id_column, support_channel_column + +PROTOCOL = "microcosm.us.current-social-security-source.v1" +READ_COLUMNS = ( + "PERIDNUM", + "PH_SEQ", + "A_LINENO", + "A_AGE", + "SS_VAL", + "SS_YN", + "RESNSS1", + "RESNSS2", + "RESNSSA", + "I_SSVAL", + "I_SSYN", +) +WIDTHS = { + "PH_SEQ": 5, + "A_LINENO": 2, + "A_AGE": 2, + "SS_VAL": 5, + "SS_YN": 1, + "RESNSS1": 1, + "RESNSS2": 1, + "RESNSSA": 1, + "I_SSVAL": 2, + "I_SSYN": 2, +} +DICTIONARY = { + "url": "https://www2.census.gov/programs-surveys/cps/datasets/2025/march/asec2025_ddl_pub_full.pdf", + "sha256": "5cb80973326ef8b625fbaae70d80b0c641ce5d2b3911abd2fb4427abd5908a6f", + "pdf_pages_1based": { + "reasons": 48, + "total_and_recipiency": 49, + "allocation_values": [53, 56], + "allocation_fields": [57, 59], + }, +} + + +def require(condition, reason): + if not condition: + raise ValueError("CURRENT_SOCIAL_SECURITY_" + reason) + + +def _sha(value): + return hashlib.sha256(value).hexdigest() + + +def _file_sha(path): + digest = hashlib.sha256() + with Path(path).open("rb") as handle: + while chunk := handle.read(1_048_576): + digest.update(chunk) + return digest.hexdigest() + + +def _numeric(series): + require( + pd.api.types.is_numeric_dtype(series.dtype) + and not pd.api.types.is_bool_dtype(series.dtype) + and not pd.api.types.is_complex_dtype(series.dtype), + "PHYSICAL_NUMERIC_TYPE", + ) + return series.to_numpy(dtype=np.float64, na_value=np.nan, copy=True) + + +def _read_capture(path, *, rows): + """Bounded literal parser; a path or row count confers no source authority.""" + reader = source_csv_builtin.capture_csv_reader(csv) + require(reader is not None, "CSV_READER_CHANGED") + records, seen, coordinates = [], set(), set() + with Path(path).open("r", encoding="utf-8-sig", newline="") as handle: + stream = reader(handle, strict=True) + header = next(stream, []) + require( + bool(header) + and all(header) + and len(header) == len(set(header)) + and set(READ_COLUMNS) <= set(header), + "HEADER", + ) + indices = [header.index(name) for name in READ_COLUMNS] + for row in stream: + require(len(row) == len(header) and len(records) < rows, "ROW_SHAPE") + selected = [row[i] for i in indices] + require( + re.fullmatch(r"[0-9]{22}", selected[0], re.ASCII) is not None, + "PERSON_KEY", + ) + record = dict(zip(READ_COLUMNS, selected, strict=True)) + for name, width in WIDTHS.items(): + token = record[name] + # Missing measurements remain literal unknowns. Coordinates + # cannot be missing because they establish the source join. + require( + token == "" + or re.fullmatch(r"[0-9]{1," + str(width) + "}", token, re.ASCII) + is not None, + "TOKEN_WIDTH:" + name, + ) + require( + all(record[n] != "" for n in ("PH_SEQ", "A_LINENO", "A_AGE")), + "COORDINATE_MISSING", + ) + pair = (int(record["PH_SEQ"]), int(record["A_LINENO"])) + require( + min(pair) > 0 and pair not in coordinates and selected[0] not in seen, + "COORDINATE_DUPLICATE", + ) + coordinates.add(pair) + seen.add(selected[0]) + records.append(record) + require(len(records) == rows, "ROW_COUNT") + return pd.DataFrame(records, columns=READ_COLUMNS).set_index("PERIDNUM", drop=False) + + +def _codes(tokens, allowed): + values = np.full(len(tokens), np.nan, dtype=np.float64) + for i, token in enumerate(tokens): + require(type(token) is str, "CODE_TOKEN_TYPE") + if token != "": + require(re.fullmatch(r"[0-9]+", token, re.ASCII) is not None, "CODE_TOKEN") + require(int(token) in allowed, "CODE_DOMAIN") + values[i] = int(token) + return values + + +def _allocation_labels(raw): + total = _codes(raw.I_SSVAL, {0, 11, 12, 13, 14, 15}) + receipt = _codes(raw.I_SSYN, {0, 10, 11}) + reason = _codes(raw.RESNSSA, set(range(10))) + return pd.DataFrame( + { + "amount_allocation_code": total, + "recipiency_allocation_code": receipt, + "reason_allocation_code": reason, + "allocation_origin": [ + "unresolved_allocation_provenance" + if not np.isfinite([a, b, c]).all() + else "publisher_allocated" + if any(v != 0 for v in (a, b, c)) + else "publisher_no_allocation" + for a, b, c in zip(total, receipt, reason, strict=True) + ], + }, + index=raw.index, + ) + + +@dataclass(frozen=True) +class CurrentSocialSecurityProjection: + """Computed data, requalified by the host before execution and replay.""" + + person: pd.DataFrame + asec_literals: pd.DataFrame + evidence: dict + + +def qualify_current_social_security(preparation): + require( + type(preparation) is source.AuthenticatedSurveyPopulationPreparation, + "PREPARATION_TYPE", + ) + entry = preparation._checked() + state = entry[2] + acs_owned = source.acs_native._owned(state.native[0]) + acs_receipt = json.loads(acs_owned.payload) + require(acs_receipt["vintage"] == 2024, "ACS_NATIVE_PERIOD") + native = state.native[1] + issued = source.asec_native._ISSUED.get(id(native)) + require( + issued is not None and issued[0]() is native and issued[1] == native.payload, + "NATIVE_ISSUANCE", + ) + parent = issued[2].parent + ready = parent.ready() + header = json.loads(ready.header) + require( + header["target_year"] == 2024 and header["semantic"] == "annual_current_money", + "MONEY_PERIOD", + ) + native_document = json.loads(issued[1]) + require( + native_document["source_year"] == native_document["income_year"] == 2024 + and native_document["survey_year"] == 2025, + "NATIVE_PERIOD", + ) + selected_pins = [pin for pin in coverage._MEMBER_PINS if pin[0] == 2024] + require(len(selected_pins) == 1, "SOURCE_REGISTRY") + year, member, archive, digest, rows, size = selected_pins[0] + native_sources = issued[2].coverage.receipt["sources"] + retained = [s for s in native_sources if s["source_year"] == year] + require( + len(retained) == 1 + and retained[0]["member"] == member + and retained[0]["archive_sha256"] == archive + and retained[0]["member_sha256"] == digest + and retained[0]["rows"] == rows + and retained[0]["member_bytes"] == size, + "NATIVE_MEMBER_BINDING", + ) + with tempfile.TemporaryDirectory(prefix="microcosm-current-ss-") as tmp: + captured = Path(tmp) / member + identity = coverage._capture( + state.root / "asec" / member, + captured, + size=size, + digest=digest, + # _capture's budget bounds its encoded coverage projection, not + # CSV bytes. Short valid source rows can expand in that format. + # Use the source owner's existing closed projection bound; the + # exact captured CSV size and digest remain separate checks. + budget=[coverage._BODY_MAX], + ) + raw = _read_capture(captured, rows=rows) + require( + coverage._identity(captured.stat(follow_symlinks=False)) == identity, + "CAPTURE_CHANGED", + ) + require(_file_sha(captured) == digest, "CAPTURE_DIGEST") + + scope = parent.scope + current_positions = np.flatnonzero(np.asarray(scope.person_years) == 2024) + keys = np.asarray(scope.person_native_keys)[current_positions] + require( + len(keys) == rows and len(set(keys)) == rows and set(keys) == set(raw.index), + "COMPLETE_CURRENT_SOURCE_JOIN", + ) + ordered = raw.loc[keys].copy() + pp = parent.frame.person.iloc[current_positions] + for native_name, parent_name in ( + ("PH_SEQ", "source_household_id"), + ("A_LINENO", "A_LINENO"), + ("A_AGE", "A_AGE"), + ): + require( + np.array_equal( + ordered[native_name].astype("int64").to_numpy(), + pp[parent_name].to_numpy(), + ), + "PARENT_COORDINATE_IDENTITY", + ) + field = ready.field("SS_VAL") + require((field.validity[current_positions] == 1).all(), "CURRENT_AMOUNT_UNKNOWN") + amount = field.amounts[current_positions].copy() + require( + amount.dtype == np.dtype("float64") + and np.isfinite(amount).all() + and (amount >= 0).all(), + "CURRENT_AMOUNT_DOMAIN", + ) + literal_amount = _codes(ordered.SS_VAL, range(100000)) + require(np.array_equal(literal_amount, amount), "CURRENT_AMOUNT_SOURCE_IDENTITY") + reason_1 = _codes(ordered.RESNSS1, range(9)) + reason_2 = _codes(ordered.RESNSS2, range(9)) + recipiency = _codes(ordered.SS_YN, {0, 1, 2}) + report_amount, components, allowed, labels = basis_owner.asec_reporting_basis( + amount, + ordered.A_AGE.to_numpy(dtype=np.float64), + recipiency, + reason_1, + reason_2, + ) + allocation = _allocation_labels(ordered) + + people = state.frame.person + require( + people.person_id.dtype == np.dtype("int64") + and people.person_id.is_unique + and (people.person_id >= 0).all(), + "PERSON_AXIS", + ) + channel = people[support_channel_column("person")] + require(set(channel) == {"acs", "asec"}, "CHANNEL_ROSTER") + out = pd.DataFrame(index=pd.Index(people.person_id, name="person_id")) + out["native_person_id"] = people[spine_source_id_column("person")].to_numpy() + out["source"] = channel.to_numpy() + out["social_security_source_total"] = np.nan + out["source_reporting_universe"] = False + out["source_reporting_unit"] = "person_report_record" + for c in basis_owner.COMPONENTS: + out[c] = np.nan + out["allowed_" + c] = True + out["basis_origin"] = "unresolved" + out["allocation_origin"] = "unresolved_allocation_provenance" + original_ids = np.asarray(scope.person_ids)[current_positions] + lookup = {int(pid): i for i, pid in enumerate(original_ids)} + asec_ids = out.index[out.source.eq("asec")] + require( + all(int(v) in lookup for v in out.loc[asec_ids, "native_person_id"]), + "SELECTED_NATIVE_JOIN", + ) + take = np.asarray([lookup[int(v)] for v in out.loc[asec_ids, "native_person_id"]]) + require(len(set(take)) == len(take), "SELECTED_NATIVE_DUPLICATE") + out.loc[asec_ids, "social_security_source_total"] = report_amount[take] + out.loc[asec_ids, "source_reporting_universe"] = ( + ordered.A_AGE.to_numpy(dtype=np.float64)[take] >= 15 + ) + out.loc[asec_ids, "source_reporting_unit"] = ( + "person_report_may_combine_family_payments" + ) + for j, c in enumerate(basis_owner.COMPONENTS): + out.loc[asec_ids, c] = components[take, j] + out.loc[asec_ids, "allowed_" + c] = allowed[take, j] + out.loc[asec_ids, "basis_origin"] = np.asarray(labels)[take] + out.loc[asec_ids, "allocation_origin"] = allocation.allocation_origin.to_numpy()[ + take + ] + + acs = people.loc[channel.eq("acs")] + require( + {"SSP", "AGEP", "ADJINC", "acs_social_security_income"} <= set(acs), + "ACS_SOURCE_COLUMNS", + ) + raw_ss = _numeric(acs.SSP) + adjusted = _numeric(acs.acs_social_security_income) + age = _numeric(acs.AGEP) + factor = _numeric(acs.ADJINC) / 1_000_000 + require( + np.isfinite(age).all() + and (age == np.floor(age)).all() + and ((age >= 0) & (age <= 99)).all(), + "ACS_AGE", + ) + require(np.isfinite(factor).all() and (factor > 0).all(), "ACS_PRICE_FACTOR") + eligible = age >= 15 + out.loc[acs.person_id, "source_reporting_universe"] = eligible + require( + np.isnan(raw_ss[~eligible]).all() and np.isnan(adjusted[~eligible]).all(), + "ACS_OUTSIDE_REPORTING_UNIVERSE_OBSERVATION", + ) + require( + np.isfinite(raw_ss[eligible]).all() + and ( + (raw_ss[eligible] == 0) + | ((raw_ss[eligible] >= 4) & (raw_ss[eligible] <= 55000)) + ).all() + and (raw_ss[eligible] == np.floor(raw_ss[eligible])).all(), + "ACS_SSP_DOMAIN", + ) + require( + np.array_equal(adjusted[eligible], raw_ss[eligible] * factor[eligible]), + "ACS_SSP_ADJUSTMENT_IDENTITY", + ) + # Below-age-15 SSP is explicitly out of the question universe. Keep it + # unknown here; a separate reviewed reporting-unit convention is needed. + out.loc[acs.person_id, "social_security_source_total"] = adjusted + zero_ids = acs.person_id.to_numpy()[eligible & (raw_ss == 0)] + out.loc[zero_ids, list(basis_owner.COMPONENTS)] = 0.0 + out.loc[zero_ids, ["allowed_" + c for c in basis_owner.COMPONENTS]] = False + out.loc[zero_ids, "basis_origin"] = "known_total_zero" + out.loc[acs.person_id.to_numpy()[eligible & (raw_ss > 0)], "basis_origin"] = ( + "acs_combined_positive_requires_model" + ) + out.loc[acs.person_id.to_numpy()[~eligible], "basis_origin"] = ( + "acs_below15_outside_reporting_universe" + ) + evidence = { + "protocol": PROTOCOL, + "dictionary": json.loads(json.dumps(DICTIONARY)), + "preparation_sha256": _sha(entry[1]), + "asec_native_sha256": _sha(issued[1]), + "acs_native_sha256": _sha(acs_owned.payload), + "acs_survey_year": 2024, + "money_header_sha256": _sha(ready.header), + "source_member_sha256": digest, + "read_columns": list(READ_COLUMNS), + "complete_current_source_rows": rows, + "asec_income_year": 2024, + "asec_interview_year": 2025, + "price_basis_year": 2024, + "acs_income_window": "rolling_prior_12_months_at_2024_interview", + "acs_price_adjustment": "SSP*(ADJINC/1000000)", + "acs_dictionary": { + "url": "https://www2.census.gov/programs-surveys/acs/tech_docs/pums/data_dict/PUMS_Data_Dictionary_2024.pdf", + "sha256": "929c2752995b0af1c16d5c64de8cdc43b4aa7d388ee2d45b4b4df90fecce1dff", + "ssp_pdf_page_1based": 45, + "fssp_pdf_page_1based": 130, + }, + "acs_allocation_flag": "FSSP_not_in_current_native_projection; provenance_unresolved", + "asec_component_mapping": "unique_reported_component_only; no priority or age fallback", + "knownness": "unknown components and outside-universe SSP retained as unknown", + "reporting_grain": "source person report; ASEC may combine family payments", + "individual_beneficiary_assignment_claim": False, + "below15_convention": "both surveys retain unknown beneficiary totals; ASEC zero literal is NIU", + "observed_component_amounts_claim": False, + "source_admission_issued": False, + "release_eligible": False, + } + require( + preparation._checked()[1] == entry[1] and parent.ready().header == ready.header, + "SOURCE_CHANGED", + ) + return CurrentSocialSecurityProjection(out, ordered, evidence) diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/current_survey_amounts.py b/packages/microcosm-build/src/microcosm/build/us_runtime/current_survey_amounts.py new file mode 100644 index 000000000..78dd8e4fa --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/current_survey_amounts.py @@ -0,0 +1,459 @@ +"""Qualified current-money targets and source-keyed clone attachment values. + +The closed family has one UC route and one conditional health-cost route. +Values are derived from live retained owners; this module issues no authority. +""" + +from __future__ import annotations + +from dataclasses import dataclass, fields + +import numpy as np +import pandas as pd + +from microcosm.fit import _graph_legacy_qrf as codec +from microcosm.fit import model_input +from microcosm.frame import WeightKind + +from . import current_asec_unemployment_source as unemployment +from . import current_survey_predictors as predictors +from . import graph_full_puf_enrichment as physical +from . import graph_survey_puf55 as parent_host +from . import support_provenance as provenance + +PROTOCOL = "microcosm.us.current-survey-amount-family.v1" +SEED = 579 + + +@dataclass(frozen=True) +class AmountGroup: + key: str + fields: tuple[tuple[str, str], ...] + + @property + def targets(self): + return tuple("survey_amount_target_" + raw for raw, _ in self.fields) + + +GROUPS = ( + AmountGroup("unemployment", (("UC_VAL", "unemployment_compensation"),)), + AmountGroup( + "health_costs", + ( + ("PHIP_VAL", "health_insurance_premiums_without_medicare_part_b"), + ("PMED_VAL", "other_medical_expenses"), + ("POTC_VAL", "over_the_counter_health_expenses"), + ), + ), +) +UC_REPORT_COLUMNS = ( + "source_amount", + "receipt_literal", + "source_reporting_universe", + "receipt_code_known", + "receipt_code", + "reporting_status", + "canonical_amount_known", + "amount_status", + "amount_validity", + "zero_origin", +) + + +def require(condition, reason): + if not condition: + raise ValueError("CURRENT_SURVEY_AMOUNTS_" + reason) + + +def selected_groups(names): + require( + type(names) is tuple + and bool(names) + and len(set(names)) == len(names) + and all(type(n) is str for n in names), + "GROUP_NAMES", + ) + result = tuple(g for g in GROUPS if g.key in names) + require(tuple(g.key for g in result) == names, "GROUP_ROSTER_OR_ORDER") + return result + + +@dataclass(frozen=True) +class GroupValues: + spec: AmountGroup + donor_frame: object + donor_columns: pd.DataFrame + matrix: bytes + keep: np.ndarray + + +@dataclass(frozen=True) +class QualifiedSurveyAmounts: + projection: bytes + evidence: dict + source_frame: object + origins: pd.DataFrame + native: pd.DataFrame + reports: pd.DataFrame + features: tuple[str, ...] + groups: tuple[GroupValues, ...] + + +def seal(value): + require(type(value) is QualifiedSurveyAmounts, "QUALIFIED_TYPE") + return ( + value.projection, + codec.encode_json(value.evidence), + value.features, + physical._population_stamp( + parent_host.population_ops.Population.from_frame( + value.source_frame, "survey_amounts.source" + ) + ), + *( + physical._table_stamp(t) + for t in (value.origins, value.native, value.reports) + ), + tuple( + ( + g.spec, + g.matrix, + g.keep.dtype.str, + g.keep.tobytes(), + physical._table_stamp(g.donor_columns), + physical._population_stamp( + parent_host.population_ops.Population.from_frame( + g.donor_frame, "survey_amounts.donor" + ) + ), + ) + for g in value.groups + ), + ) + + +def origin_digest(origins): + """Bind the original source axis and columns even when both name person_id.""" + axis = pd.Index(predictors._ids(origins.index.to_series(index=origins.index))) + columns = ( + origins.reset_index(drop=True).to_json(orient="table", index=False).encode() + ) + return codec.sha( + codec.encode_json( + { + "index_name": origins.index.name, + "index_sha256": codec.sha(axis.to_numpy(dtype="= 15 + require(keep.any(), "NO_QUALIFIED_DONORS:" + spec.key) + require(recipient.any(), "NO_QUALIFIED_RECIPIENTS:" + spec.key) + donor_frame = base.source_frame.select(keep) + require( + donor_frame.resolve_weights("person").kind is WeightKind.DESIGN, + "DONOR_DESIGN_WEIGHTS", + ) + donor_columns = features.loc[ids[keep]].copy() + for target, output in zip(spec.targets, outputs, strict=True): + donor_columns[target] = native.loc[ids[keep], output].to_numpy() + require( + np.array_equal( + donor_frame.person.person_id.to_numpy(), ids[keep].to_numpy() + ), + "DONOR_AXIS", + ) + recipient_matrix = model_input.encode_recipient_matrix( + features.loc[ids[recipient]], + entity="person", + entity_ids=ids[recipient].to_numpy(dtype="= 0).all(), + "DRAW_AXIS_OR_VALUES", + ) + require( + qualified.origins.loc[draw.index, "source"].eq("acs").all(), "DRAW_SOURCE" + ) + for target, (_, output) in zip( + group.spec.targets, group.spec.fields, strict=True + ): + completed.loc[draw.index, output] = draw[target].to_numpy() + all_columns = pd.concat([completed, qualified.reports], axis=1) + require(not set(all_columns) & set(people), "ATTACH_OWNERSHIP_COLLISION") + index = pd.Index(ids, name="person_id") + return { + ("person", name): pd.Series( + all_columns[name].reindex(original).array.copy(), index=index, name=name + ) + for name in all_columns + } diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/current_survey_geography.py b/packages/microcosm-build/src/microcosm/build/us_runtime/current_survey_geography.py new file mode 100644 index 000000000..f463b6412 --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/current_survey_geography.py @@ -0,0 +1,313 @@ +"""Qualified observed geography and stable household draw keys for both surveys. + +This additive, values-only projection borrows current authenticated sources. +It does not assign a location, mutate a Frame, or issue source or Population +authority. Hosts must requalify the projection before execution and replay. +""" + +from __future__ import annotations + +import hashlib +import json +import re +from dataclasses import dataclass + +import pandas as pd + +from microcosm.graph.population import dtype_for_token + +from . import current_asec_demographics as demographics +from . import survey_population_preparation as source +from .support_provenance import spine_source_id_column, support_channel_column + +PROTOCOL = "microcosm.us.current-survey-geography.v1" +COLUMNS = ( + "survey_geography_origin_key", + "survey_observed_state", + "survey_observed_puma", +) +MAX_HOUSEHOLDS = 64 * 1024**2 // 128 +MAX_RECEIPT_BYTES = 64 * 1024 + + +def _require(condition, reason): + if not condition: + raise ValueError("CURRENT_SURVEY_GEOGRAPHY_" + reason) + + +def _sha(value): + return hashlib.sha256(value).hexdigest() + + +def _origin_key(row): + channel = row["source"] + _require(type(channel) is str and channel in {"acs", "asec"}, "SOURCE") + _require( + type(row["source_year"]) is int + and row["source_year"] == 2024 + and type(row["survey_year"]) is int + and row["survey_year"] == (2024 if channel == "acs" else 2025), + "SOURCE_PERIOD", + ) + raw = row["raw_native_id"] + _require(type(raw) is str and 0 < len(raw) <= 128, "NATIVE_KEY") + # Preserve the source literal, including ASEC's leading zeroes. Receiving + # integer ids are coordinates only and never enter the keyed draw stream. + return json.dumps( + [channel, row["source_year"], row["survey_year"], raw], + ensure_ascii=True, + separators=(",", ":"), + allow_nan=False, + ) + + +def _project(origins, households, acs, asec): + """Pure alignment from just-qualified values, with no authority of its own.""" + _require( + type(origins) is list + and 0 < len(origins) <= MAX_HOUSEHOLDS + and len(origins) == len(households), + "HOUSEHOLD_COUNT", + ) + _require( + str(households.household_id.dtype) == "int64" + and households.household_id.is_unique + and households.household_id.ge(0).all(), + "HOUSEHOLD_AXIS", + ) + channel_column = support_channel_column("household") + native_column = spine_source_id_column("household") + _require(set(households[channel_column]) == {"acs", "asec"}, "CHANNEL_ROSTER") + _require( + {"household_id", "SERIALNO", "ST", "PUMA", "puma_geoid", "puma"} <= set(acs) + and acs.household_id.is_unique, + "ACS_SOURCE_COLUMNS", + ) + _require( + { + "household_id", + "native_household_id", + "H_SEQ_integer", + "GESTFIPS", + "state_fips", + "state_status", + "state_known", + } + <= set(asec) + and asec.household_id.is_unique, + "ASEC_SOURCE_COLUMNS", + ) + acs_by_id = acs.set_index("household_id", drop=False) + asec_by_id = asec.set_index("household_id", drop=False) + origin_by_id, keys = {}, set() + for row in origins: + _require(type(row) is dict, "ORIGIN_RECORD") + household_id = row["household_id"] + _require( + type(household_id) is int + and household_id >= 0 + and household_id not in origin_by_id + and type(row["selected_receiving_household_id"]) is int + and row["selected_receiving_household_id"] >= 0, + "ORIGIN_COORDINATE", + ) + key = _origin_key(row) + _require(key not in keys, "ORIGIN_DUPLICATE") + keys.add(key) + origin_by_id[household_id] = (row, key) + _require(set(origin_by_id) == set(households.household_id), "ORIGIN_ROSTER") + rows, used_acs, used_asec = [], set(), set() + for household_id, channel, native_id in households[ + ["household_id", channel_column, native_column] + ].itertuples(index=False, name=None): + row, key = origin_by_id[household_id] + _require( + channel == row["source"] + and native_id == row["selected_receiving_household_id"], + "RECEIVING_ORIGIN_JOIN", + ) + puma = None + if channel == "acs": + _require(native_id in acs_by_id.index, "ACS_NATIVE_JOIN") + original = acs_by_id.loc[native_id] + _require(original.SERIALNO == row["raw_native_id"], "ACS_LITERAL_KEY") + state, local_puma = original.ST, original.PUMA + _require( + type(state) is str + and re.fullmatch(r"[0-9]{2}", state, re.ASCII) is not None + and int(state) > 0 + and type(local_puma) is str + and re.fullmatch(r"[0-9]{5}", local_puma, re.ASCII) is not None + and int(local_puma) > 0, + "ACS_LITERAL_GEOGRAPHY", + ) + puma = state + local_puma + _require( + original.puma_geoid == puma and original.puma == puma, + "ACS_PUMA_IDENTITY", + ) + used_acs.add(native_id) + else: + _require(household_id in asec_by_id.index, "ASEC_RECEIVING_JOIN") + original = asec_by_id.loc[household_id] + raw = row["raw_native_id"] + _require( + re.fullmatch(r"[0-9]+", raw, re.ASCII) is not None + and int(raw) == original.H_SEQ_integer + and native_id == original.native_household_id, + "ASEC_LITERAL_KEY", + ) + code, status = demographics._state_token(original.GESTFIPS) + _require( + original.state_status == status + and bool(original.state_known) == (code is not None) + and ( + pd.isna(original.state_fips) + if code is None + else original.state_fips == code + ), + "ASEC_STATE_IDENTITY", + ) + state = None if code is None else str(code).zfill(2) + used_asec.add(household_id) + rows.append((int(household_id), key, state, puma)) + _require( + used_acs == set(acs.household_id) and used_asec == set(asec.household_id), + "COMPLETE_NATIVE_ROSTER", + ) + return tuple(rows) + + +def _projection_digest(household): + """Stream bounded row encodings; the receipt never contains source keys.""" + _require( + tuple(household.columns) == COLUMNS + and household.index.name == "household_id" + and str(household.index.dtype) == "int64" + and household.index.is_unique + and 0 < len(household) <= MAX_HOUSEHOLDS + and all(household[c].dtype == dtype_for_token("string") for c in COLUMNS), + "PROJECTION_STORAGE", + ) + digest = hashlib.sha256() + for row in household.itertuples(index=True, name=None): + values = [int(row[0]), *(None if pd.isna(v) else v for v in row[1:])] + encoded = source._encode(values, maximum=4096) + digest.update(len(encoded).to_bytes(4, "big")) + digest.update(encoded) + return digest.hexdigest() + + +def _check_asec_projection(values, receipt): + """Bind the detached household values to their producer's exact receipt.""" + _require(type(receipt) is bytes and values.receipt == receipt, "ASEC_RECEIPT") + document = json.loads(receipt) + _require( + _sha(values.household.reset_index(drop=True).to_json(orient="table").encode()) + == document.get("household_projection_sha256"), + "ASEC_PROJECTION_DIGEST", + ) + + +@dataclass(frozen=True) +class CurrentSurveyGeographyValues: + """Detached values; retained output bytes grant no downstream admission.""" + + household: pd.DataFrame + receipt: bytes + + +def qualify_current_survey_geography(preparation): + """Read current source geography, aligned to selected receiving households.""" + _require( + type(preparation) is source.AuthenticatedSurveyPopulationPreparation, + "PREPARATION_TYPE", + ) + entry = preparation._checked() + state = entry[2] + origins = json.loads(entry[1])["origins"]["households"] + acs_native, acs_frame = state.native[0], state.source_frames[0] + source.acs_native.verify_acs_native_coverage(acs_native, acs_frame) + acs_owned = source.acs_native._owned(acs_native) + acs_payload = acs_owned.payload + _require(json.loads(acs_payload)["vintage"] == 2024, "ACS_NATIVE_PERIOD") + asec = demographics.qualify_current_asec_demographics(preparation) + asec_receipt = asec.receipt + _check_asec_projection(asec, asec_receipt) + projected = _project( + origins, + state.frame.table("household"), + acs_frame.table("household"), + asec.household, + ) + table = pd.DataFrame( + { + column: pd.array( + [row[i + 1] for row in projected], dtype=dtype_for_token("string") + ) + for i, column in enumerate(COLUMNS) + }, + index=pd.Index( + [row[0] for row in projected], name="household_id", dtype="int64" + ), + ) + projection_sha256 = _projection_digest(table) + receipt = source._encode( + { + "protocol": PROTOCOL, + "preparation_sha256": _sha(entry[1]), + "acs_native_sha256": _sha(acs_payload), + "asec_demographic_sha256": _sha(asec_receipt), + "projection_sha256": projection_sha256, + "columns": list(COLUMNS), + "households": len(table), + "acs_households": len(acs_frame.table("household")), + "asec_households": len(asec.household), + "state_unknown_households": int(table[COLUMNS[1]].isna().sum()), + "puma_observed_households": int(table[COLUMNS[2]].notna().sum()), + "draw_identity": ["source", "source_year", "survey_year", "raw_native_id"], + "draw_key_encoding": "compact ASCII JSON array; source literal unchanged", + "acs_survey_year": 2024, + "asec_survey_year": 2025, + "asec_income_year": 2024, + "state_is_income_year_residence_claim": False, + "observed_state": "ACS ST; current ASEC GESTFIPS; two-digit strings", + "observed_puma": "ACS ST+PUMA; ASEC remains unknown", + "unknown_state": "preserved; no carried-state fallback", + "unassigned_jurisdiction_validation": "separate geography gate", + "source_admission_issued": False, + "population_admission_issued": False, + "release_eligible": False, + }, + maximum=MAX_RECEIPT_BYTES, + ) + result = CurrentSurveyGeographyValues(table, receipt) + # Complete owner I/O before pure comparisons. A callback during the last + # source check must not mutate a retained input or detached output unnoticed. + source.acs_native.verify_acs_native_coverage(acs_native, acs_frame) + final_entry = preparation._checked() + _require( + final_entry is entry + and source._ISSUED.get(id(preparation)) is entry + and preparation.payload == entry[1] + and source.acs_native._ISSUED.get(acs_native) is acs_owned + and acs_native.payload == acs_payload + and acs_owned.frame is acs_frame + and asec.receipt == asec_receipt, + "FINAL_ISSUANCE", + ) + source._pure_final(state) + _check_asec_projection(asec, asec_receipt) + _require( + _project( + origins, + state.frame.table("household"), + acs_frame.table("household"), + asec.household, + ) + == projected + and result.receipt == receipt + and _projection_digest(result.household) == projection_sha256, + "FINAL_PROJECTION", + ) + return result diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/current_survey_health_coverage.py b/packages/microcosm-build/src/microcosm/build/us_runtime/current_survey_health_coverage.py new file mode 100644 index 000000000..066a13553 --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/current_survey_health_coverage.py @@ -0,0 +1,245 @@ +"""Interview-date coverage recodes; detached values grant no source authority. + +Only literal source yes/no codes determine coverage. Allocation is separate +provenance, and a broader ACS item never becomes a narrower canonical flag. +""" + +from __future__ import annotations + +from dataclasses import dataclass + +import numpy as np +import pandas as pd + +from . import support_provenance as provenance +from .cps_carried import US_REPORTED_COVERAGE_PERSON_INPUTS + +PROTOCOL = "microcosm.us.current-survey-health-coverage.v1" +STRING_DTYPE = pd.StringDtype(storage="python", na_value=pd.NA) + + +@dataclass(frozen=True) +class CoverageField: + output: str + asec: str + acs: str | None + acs_gap: str | None = None + + @property + def columns(self): + return (self.output, self.output + "__known", self.output + "__source_status") + + +# Separate NOW_CAID from the broader NOW_MCAID legacy mapping. These are +# source recodes, not benefit eligibility or prior-calendar-year coverage. +FIELDS = ( + CoverageField("has_esi", "NOW_GRP", "HINS1"), + CoverageField( + "has_marketplace_health_coverage_at_interview", + "NOW_MRK", + None, + "direct_purchase_marketplace_split_unobserved", + ), + CoverageField( + "has_non_marketplace_direct_purchase_health_coverage_at_interview", + "NOW_NONM", + None, + "direct_purchase_marketplace_split_unobserved", + ), + CoverageField( + "has_medicaid_health_coverage_at_interview", + "NOW_CAID", + None, + "means_tested_program_split_unobserved", + ), + CoverageField( + "has_other_means_tested_health_coverage_at_interview", + "NOW_OTHMT", + None, + "means_tested_program_split_unobserved", + ), + CoverageField( + "has_tricare_health_coverage_at_interview", + "NOW_MIL", + None, + "military_coverage_type_unresolved", + ), + CoverageField( + "has_champva_health_coverage_at_interview", + "NOW_CHAMPVA", + None, + "champva_not_separately_observed", + ), + CoverageField( + "has_va_health_coverage_at_interview", + "NOW_VACARE", + None, + "va_champva_recode_scope_unresolved", + ), + CoverageField( + "has_indian_health_service_coverage_at_interview", "NOW_IHSFLG", "HINS7" + ), +) +ASEC_VALUE_COLUMNS = tuple(f.asec for f in FIELDS) + ("NOW_MCAID",) +ASEC_ALLOCATION_COLUMNS = tuple("I_" + c for c in ASEC_VALUE_COLUMNS) +ACS_VALUE_COLUMNS = tuple("HINS" + str(i) for i in range(1, 8)) +ACS_ALLOCATION_COLUMNS = tuple("FHINS" + str(i) + "P" for i in range(1, 8)) +ACS_EDIT_COLUMNS = ("FHINS3C", "FHINS4C", "FHINS5C") +RAW_COLUMNS = ( + *ASEC_VALUE_COLUMNS, + *ASEC_ALLOCATION_COLUMNS, + *ACS_VALUE_COLUMNS, + *ACS_ALLOCATION_COLUMNS, + *ACS_EDIT_COLUMNS, +) +SOURCE_PREFIX = "health_source_" + + +def require(condition, reason): + if not condition: + raise ValueError("CURRENT_SURVEY_HEALTH_" + reason) + + +def _field(name): + require( + set(f.output for f in FIELDS) == set(US_REPORTED_COVERAGE_PERSON_INPUTS), + "PROFILE_ROSTER", + ) + matches = [f for f in FIELDS if f.output == name] + require(len(matches) == 1, "FIELD") + return matches[0] + + +def recode(tokens, allocations, *, survey): + """Keep missing, unrecognized, allocated and reported source states apart. + + This classifier accepts literal strings only. A valid source yes/no value + survives unknown allocation provenance, without claiming a raw response. + Neither blank nor an out-of-dictionary code means no coverage. + """ + require(survey in ("asec", "acs"), "SURVEY") + values, flags = tuple(tokens), tuple(allocations) + require( + len(values) == len(flags) + and all(type(x) is str and len(x) <= 64 for x in (*values, *flags)), + "LITERAL_CONTRACT", + ) + allocation_labels = ( + {"0": "reported", "1": "hotdeck", "2": "logical", "3": "whole_unit"} + if survey == "asec" + else {"0": "not_allocated", "1": "allocated"} + ) + canonical, known, status = [], [], [] + for token, allocation in zip(values, flags, strict=True): + valid = token in ("1", "2") + canonical.append(token == "1" if valid else pd.NA) + known.append(valid) + if not valid: + status.append( + "missing_source_value" if token == "" else "unrecognized_source_value" + ) + else: + method = allocation_labels.get(allocation, "allocation_unknown") + status.append(("source_yes_" if token == "1" else "source_no_") + method) + return pd.DataFrame( + { + "value": pd.array(canonical, dtype="boolean"), + "known": np.asarray(known, dtype=bool), + "status": pd.array(status, dtype=STRING_DTYPE), + } + ) + + +def recode_field(raw, name): + """Transform one field on the full original-person axis, retaining gaps.""" + spec = _field(name) + require( + type(raw) is pd.DataFrame + and raw.index.is_unique + and raw.index.name == "person_id" + and {"source", *RAW_COLUMNS} <= set(raw) + and raw.source.isin(("acs", "asec")).all(), + "RAW_AXIS", + ) + result = pd.DataFrame(index=raw.index.copy()) + result[spec.output] = pd.array([pd.NA] * len(raw), dtype="boolean") + result[spec.output + "__known"] = False + result[spec.output + "__source_status"] = pd.array( + [pd.NA] * len(raw), dtype=STRING_DTYPE + ) + for survey, raw_name in (("asec", spec.asec), ("acs", spec.acs)): + mask = raw.source.eq(survey) + if raw_name is None: + result.loc[mask, spec.output + "__source_status"] = ( + "semantic_gap:" + spec.acs_gap + ) + continue + flag_name = "I_" + raw_name if survey == "asec" else "F" + raw_name + "P" + part = recode(raw.loc[mask, raw_name], raw.loc[mask, flag_name], survey=survey) + for output, value in zip(spec.columns, part, strict=True): + result.loc[mask, output] = part[value].array + return result + + +def source_columns(raw): + """Namespace literal observations so they cannot impersonate output leaves.""" + require(raw.index.name == "person_id" and raw.index.is_unique, "RAW_AXIS") + result = pd.DataFrame(index=raw.index.copy()) + for name in RAW_COLUMNS: + require(all(x is None or type(x) is str for x in raw[name]), "RAW_LITERAL_TYPE") + result[SOURCE_PREFIX + name] = pd.array(raw[name], dtype=STRING_DTYPE) + for name in ("source", "source_year", "survey_year"): + result[SOURCE_PREFIX + name] = ( + pd.array(raw[name], dtype=STRING_DTYPE) + if name == "source" + else raw[name].array.copy() + ) + return result + + +def attach_columns(origins, receiving, columns): + """Fan original observations out to exactly their unchanged two clones. + + The owning host authenticates origins and source columns before calling. + Existing fields are never overwritten, including another family's leaves. + """ + require( + type(origins) is pd.DataFrame + and type(columns) is pd.DataFrame + and origins.index.is_unique + and origins.index.name == "person_id" + and columns.index.equals(origins.index) + and columns.columns.is_unique, + "ATTACH_SOURCE_AXIS", + ) + people = receiving.person + ids = people.person_id + original = people[provenance.support_source_id_column("person")].to_numpy() + clone = people[provenance.support_clone_index_column("person")].to_numpy() + native = people[provenance.spine_source_id_column("person")].to_numpy() + channel = people[provenance.support_channel_column("person")].to_numpy() + require( + ids.dtype == np.dtype("int64") + and ids.is_unique + and original.dtype == clone.dtype == native.dtype == np.dtype("int64") + and len(original) == 2 * len(origins) + and set(original) == set(origins.index), + "CLONE_SOURCE_ROSTER", + ) + expected = origins.reindex(original) + require( + np.array_equal(native, expected.native_person_id.to_numpy()) + and np.array_equal(channel, expected.source.to_numpy()), + "CLONE_SOURCE_IDENTITY", + ) + pairs = pd.MultiIndex.from_arrays((original, clone)) + require(pairs.is_unique and np.isin(clone, (0, 1)).all(), "CLONE_PAIR") + require(not set(columns) & set(people), "ATTACH_OWNERSHIP_COLLISION") + return { + ("person", name): pd.Series( + columns[name].reindex(original).array.copy(), + index=pd.Index(ids.to_numpy(), name="person_id"), + name=name, + ) + for name in columns + } diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/current_survey_health_source.py b/packages/microcosm-build/src/microcosm/build/us_runtime/current_survey_health_source.py new file mode 100644 index 000000000..f9b31c825 --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/current_survey_health_source.py @@ -0,0 +1,343 @@ +"""Health-code projection from the retained original ACS and ASEC sources. + +The preparation issues source identity. This module only borrows it, captures +its pinned original members and returns descriptive values. A host must retain +that owner and requalify after its last relevant I/O before returning output. +""" + +from __future__ import annotations + +import csv +import hashlib +import json +import re +import tempfile +import zipfile +from dataclasses import dataclass +from pathlib import Path + +import numpy as np +import pandas as pd + +from . import acs_housing_universe_source as housing +from . import acs_person_coverage_authentication as records +from . import asec_coverage_authentication as asec +from . import current_survey_health_coverage as health +from . import source_csv_builtin +from . import survey_population_preparation as source + +require = health.require +ASEC_KEYS = ("PERIDNUM", "PH_SEQ", "A_LINENO") +ACS_KEYS = ("SERIALNO", "SPORDER") +ASEC_COLUMNS = (*ASEC_KEYS, *health.ASEC_VALUE_COLUMNS, *health.ASEC_ALLOCATION_COLUMNS) +ACS_COLUMNS = ( + *ACS_KEYS, + *health.ACS_VALUE_COLUMNS, + *health.ACS_ALLOCATION_COLUMNS, + *health.ACS_EDIT_COLUMNS, +) + + +def _key(row, survey): + if survey == "asec": + require( + re.fullmatch(r"[0-9]{22}", row["PERIDNUM"], re.ASCII) is not None, + "ASEC_PERSON_KEY", + ) + require( + re.fullmatch(r"[0-9]{1,5}", row["PH_SEQ"], re.ASCII) is not None, + "ASEC_HOUSEHOLD_KEY", + ) + require( + re.fullmatch(r"[0-9]{1,2}", row["A_LINENO"], re.ASCII) is not None + and 1 <= int(row["A_LINENO"]) <= 20 + and int(row["PH_SEQ"]) > 0, + "ASEC_LINE_KEY", + ) + return (int(row["PH_SEQ"]), row["PERIDNUM"], int(row["A_LINENO"])) + require( + re.fullmatch(r"2024(?:HU|GQ)[0-9]{7}", row["SERIALNO"], re.ASCII) is not None, + "ACS_HOUSEHOLD_KEY", + ) + require( + re.fullmatch(r"[0-9]{1,2}", row["SPORDER"], re.ASCII) is not None + and 1 <= int(row["SPORDER"]) <= 20, + "ACS_LINE_KEY", + ) + return (row["SERIALNO"], str(int(row["SPORDER"])), int(row["SPORDER"])) + + +def _scan(stream, *, survey, wanted, selected, maximum): + """Exhaust a byte-bounded literal member; selected records remain strings. + + This helper grants no authority to a file, row count or caller key list. + A complete authenticated source/roster is bound by the enclosing qualifier. + """ + require( + survey in ("asec", "acs") + and source_csv_builtin.capture_csv_reader(csv) is not None, + "SCANNER_OR_CSV_BINDING", + ) + columns = ASEC_COLUMNS if survey == "asec" else ACS_COLUMNS + header, count = None, 0 + for raw in records._records(stream): + values = records._decode_record(raw, first=header is None) + if header is None: + header = values + require( + bool(header) + and all(header) + and len(set(header)) == len(header) + and set(columns) <= set(header), + "SOURCE_HEADER", + ) + positions = [header.index(c) for c in columns] + continue + count += 1 + require(count <= maximum and len(values) == len(header), "SOURCE_ROW_SHAPE") + row = dict(zip(columns, (values[p] for p in positions), strict=True)) + key = _key(row, survey) + require(all(len(row[c]) <= 64 for c in columns), "SOURCE_TOKEN_BOUND") + if key in wanted: + require(key not in selected, "DUPLICATE_SELECTED_SOURCE_KEY") + selected[key] = row + require(header is not None, "SOURCE_HEADER") + return count + + +def _origins(preparation_frame, document): + payload = document["origins"]["persons"] + original = pd.DataFrame(payload["rows"], columns=payload["columns"]) + required = { + "person_id", + "source", + "source_year", + "survey_year", + "raw_native_household_id", + "raw_native_person_id", + "native_line_numeric_original", + "selected_receiving_person_id", + } + require( + required <= set(original) + and original.person_id.dtype == np.dtype("int64") + and original.person_id.is_unique, + "ORIGIN_ROSTER", + ) + original = original.set_index("person_id", drop=True) + people = preparation_frame.person + require( + np.array_equal(original.index.to_numpy(), people.person_id.to_numpy()) + and np.array_equal( + original.source.to_numpy(), + people[health.provenance.support_channel_column("person")].to_numpy(), + ) + and np.array_equal( + original.selected_receiving_person_id.to_numpy(), + people[health.provenance.spine_source_id_column("person")].to_numpy(), + ), + "ORIGIN_FRAME_IDENTITY", + ) + require( + original.source.isin(("acs", "asec")).all() + and original.source_year.eq(2024).all() + and original.survey_year.eq( + original.source.map({"acs": 2024, "asec": 2025}) + ).all(), + "ORIGIN_PERIOD", + ) + original["native_person_id"] = original.selected_receiving_person_id + keys = {} + for pid, row in original.iterrows(): + household = row.raw_native_household_id + require( + type(household) is str + and type(row.raw_native_person_id) is str + and type(row.native_line_numeric_original) is str, + "ORIGIN_COORDINATE_TYPE", + ) + line = row.native_line_numeric_original + if row.source == "asec": + key = _key( + { + "PH_SEQ": household, + "PERIDNUM": row.raw_native_person_id, + "A_LINENO": line, + }, + "asec", + ) + else: + key = _key({"SERIALNO": household, "SPORDER": line}, "acs") + require(row.raw_native_person_id == key[1], "ORIGIN_PERSON_KEY") + require((row.source, key) not in keys, "DUPLICATE_ORIGIN") + keys[row.source, key] = pid + return original, keys + + +def _combine(origins, keys, selected): + require(set(selected) == {"acs", "asec"}, "SOURCE_ARMS") + require( + all(set(selected[s]) == {k for arm, k in keys if arm == s} for s in selected), + "SELECTED_SOURCE_ROSTER", + ) + raw = origins.loc[:, ["source", "source_year", "survey_year"]].copy() + for column in health.RAW_COLUMNS: + raw[column] = pd.Series([None] * len(raw), index=raw.index, dtype=object) + for (survey, key), pid in keys.items(): + row = selected[survey][key] + require(_key(row, survey) == key, "SOURCE_COORDINATE_CHANGED") + for column in health.RAW_COLUMNS: + if column in row: + raw.at[pid, column] = row[column] + return raw + + +@dataclass(frozen=True) +class QualifiedSurveyHealthCoverage: + """A detached projection, not a substitute for the retained source owner.""" + + source_frame: object + origins: pd.DataFrame + raw: pd.DataFrame + projection: bytes + evidence: dict + + +def qualify_current_survey_health(preparation): + """Re-read pinned originals without native Frame or engine reconstruction. + + This entry point does read original microdata when called by an authorized + build. Tests can exercise the same path with privately pinned invented bytes. + """ + require( + type(preparation) is source.AuthenticatedSurveyPopulationPreparation, + "PREPARATION_TYPE", + ) + entry = preparation._checked() + state = entry[2] + document = json.loads(entry[1]) + origins, keys = _origins(state.frame, document) + native = state.native[1] + issued = source.asec_native._ISSUED.get(id(native)) + require( + issued is not None and issued[0]() is native and issued[1] == native.payload, + "ASEC_NATIVE_ISSUANCE", + ) + native_document = json.loads(issued[1]) + require( + native_document["source_year"] == native_document["income_year"] == 2024 + and native_document["survey_year"] == 2025, + "ASEC_NATIVE_PERIOD", + ) + pin = tuple(p for p in asec._MEMBER_PINS if p[0] == 2024) + require(len(pin) == 1, "ASEC_PIN_ROSTER") + year, member, archive_digest, digest, rows, size = pin[0] + retained = tuple( + s for s in issued[2].coverage.receipt["sources"] if s["source_year"] == year + ) + require( + len(retained) == 1 + and all( + retained[0][k] == v + for k, v in ( + ("member", member), + ("archive_sha256", archive_digest), + ("member_sha256", digest), + ("rows", rows), + ("member_bytes", size), + ) + ), + "ASEC_MEMBER_BINDING", + ) + acs_owned = source.acs_catalogue._lookup(state.catalogues[0]) + acs_doc = json.loads(acs_owned.receipt) + person_pins = tuple(p for p in acs_owned.pins if p[0] == "person") + require( + len(person_pins) == 1 + and acs_doc["source_year"] == acs_doc["survey_year"] == 2024, + "ACS_PIN_OR_PERIOD", + ) + _role, acs_name, acs_digest, acs_size = person_pins[0] + selected = {"asec": {}, "acs": {}} + with tempfile.TemporaryDirectory(prefix="microcosm-health-source-") as temporary: + directory = Path(temporary) + captured = directory / member + identity = asec._capture( + state.root / "asec" / member, + captured, + size=size, + digest=digest, + budget=[asec._BODY_MAX], + ) + with captured.open("rb") as stream: + count = _scan( + stream, + survey="asec", + wanted={k for s, k in keys if s == "asec"}, + selected=selected["asec"], + maximum=rows, + ) + require( + count == rows + and asec._identity(captured.stat(follow_symlinks=False)) == identity + and housing._persisted_sha(captured, size) == digest, + "ASEC_CAPTURE_CHANGED", + ) + captured = directory / acs_name + require( + housing._copy( + state.root / "acs" / acs_name, captured, acs_size, exact_size=acs_size + ) + == acs_digest, + "ACS_CAPTURE_DIGEST", + ) + count = 0 + with zipfile.ZipFile(captured) as archive: + members, prefix = records._members(archive, "person") + for item in members: + with archive.open(item) as stream: + if item.filename.casefold().startswith(prefix): + count += _scan( + stream, + survey="acs", + wanted={k for s, k in keys if s == "acs"}, + selected=selected["acs"], + maximum=acs_doc["counts"]["people"] - count, + ) + else: + while stream.read(65536): + pass # Exhaust auxiliary members so their CRC is checked. + require( + count == acs_doc["counts"]["people"] + and housing._persisted_sha(captured, acs_size) == acs_digest, + "ACS_CAPTURE_CHANGED", + ) + raw = _combine(origins, keys, selected) + projection = raw.reset_index().to_json(orient="table", index=False).encode() + evidence = { + "protocol": health.PROTOCOL, + "preparation_sha256": hashlib.sha256(entry[1]).hexdigest(), + "asec_native_sha256": hashlib.sha256(issued[1]).hexdigest(), + "acs_catalogue_sha256": hashlib.sha256(acs_owned.receipt).hexdigest(), + "asec_person_member_sha256": digest, + "acs_person_archive_sha256": acs_digest, + "asec_coverage_observation_year": 2025, + "acs_coverage_observation_year": 2024, + "projection_sha256": hashlib.sha256(projection).hexdigest(), + "selected_rows": len(raw), + "source_admission_issued": False, + "unallocated_observation_claim": False, + "release_eligible": False, + } + # The original issuer checks source bytes after the last capture/cleanup I/O. + require(preparation._checked() is entry, "PREPARATION_CHANGED") + source._pure_final(state) + require( + source._ISSUED.get(id(preparation)) is entry + and source.asec_native._ISSUED.get(id(native)) is issued + and source.acs_catalogue._lookup(state.catalogues[0]) is acs_owned, + "FINAL_OWNER", + ) + return QualifiedSurveyHealthCoverage( + state.frame, origins, raw, projection, evidence + ) diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/current_survey_predictors.py b/packages/microcosm-build/src/microcosm/build/us_runtime/current_survey_predictors.py new file mode 100644 index 000000000..9d4768a0b --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/current_survey_predictors.py @@ -0,0 +1,638 @@ +"""Source-qualified current ASEC to ACS financial predictor preparation. + +This is a survey-multispine operation before PUF enrichment. The live retained +preparation authenticates source observations; serialized evidence never issues +source authority. Financial ACS leaves are modeled from three current ASEC +money totals, with the maintained split assumptions applied after the draws. +""" + +from __future__ import annotations + +import json +from dataclasses import dataclass, fields + +import numpy as np +import pandas as pd + +from microcosm.fit import _graph_legacy_qrf as codec +from microcosm.fit import model_input +from microcosm.frame import WeightKind + +from . import acs_income_universe as universe +from . import cps_carried_current as leaves +from . import current_survey_geography as observed_geography +from . import graph_puf_diagnostic_consumer as host +from . import support_provenance as provenance + +source = host.survey_source +FEATURES = ( + "survey_predictor_age", + "survey_predictor_employment_income", + "survey_predictor_self_employment_income", +) +DEMOGRAPHIC_FEATURES = ( + *FEATURES, + "survey_predictor_is_female", + "survey_predictor_state_fips", +) +TARGETS = ("survey_current_INT_VAL", "survey_current_DIV_VAL", "survey_current_CAP_VAL") +MONEY_FIELDS = leaves.CPS_CURRENT_PREDICTOR_MONEY_FIELDS +OUTPUTS = ( + *leaves.CPS_CURRENT_PREDICTOR_PERSON_LEAVES, + "tax_exempt_interest_income", +) +PROTOCOL = "microcosm.us.current-survey-predictor-completion.v1" +PHASE = "survey_multispine_current_financial_completion" +SEED = 578 + + +def require(condition, reason): + if not condition: + raise ValueError("CURRENT_SURVEY_PREDICTOR_" + reason) + + +def feature_columns(demographic_conditioning=False): + require(type(demographic_conditioning) is bool, "DEMOGRAPHIC_OPTION") + return DEMOGRAPHIC_FEATURES if demographic_conditioning else FEATURES + + +def _check_demographic_values(values, receipt): + """Detached projections must still match their source producer's receipt.""" + require( + type(values) is observed_geography.demographics.CurrentAsecDemographicValues + and type(receipt) is bytes + and values.receipt == receipt, + "DEMOGRAPHIC_RECEIPT", + ) + document = json.loads(receipt) + for entity in ("person", "household"): + table = getattr(values, entity) + require( + codec.sha(table.reset_index(drop=True).to_json(orient="table").encode()) + == document[f"{entity}_projection_sha256"], + "DEMOGRAPHIC_PROJECTION_CHANGED", + ) + + +def _demographic_features(preparation, entry): + """Bind selected source sex and observation-time state, without fallback. + + The maintained ASEC owner retains unknown literals and allocation status. + This opt-in model refuses those unknowns rather than completing them. State + uses the shared literal geography join, not an assigned block's state. + """ + state = entry[2] + values = observed_geography.demographics.qualify_current_asec_demographics( + preparation + ) + receipt = values.receipt + _check_demographic_values(values, receipt) + document = json.loads(receipt) + require( + document["preparation_sha256"] == codec.sha(entry[1]) + and document["sex_observation_year"] == 2025 + and document["state_observation_year"] == 2025 + and document["income_year"] == 2024, + "DEMOGRAPHIC_PERIOD", + ) + acs_frame = state.source_frames[0] + source.acs_native.verify_acs_native_coverage(state.native[0], acs_frame) + acs_payload = source.acs_native._owned(state.native[0]).payload + require(json.loads(acs_payload)["vintage"] == 2024, "ACS_DEMOGRAPHIC_PERIOD") + # This pure operation verifies literal household keys and the complete + # selected source roster using the just-qualified ASEC household projection. + households = observed_geography._project( + json.loads(entry[1])["origins"]["households"], + state.frame.table("household"), + acs_frame.table("household"), + values.household, + ) + people = state.frame.person + index = pd.Index(_ids(people.person_id), name="person_id") + asec = people[provenance.support_channel_column("person")].eq("asec") + native_ids = people[provenance.spine_source_id_column("person")] + require( + values.person.index.is_unique + and set(values.person.index) == set(index[asec]) + and np.array_equal( + values.person.reindex(index[asec]).native_person_id.to_numpy(), + native_ids.loc[asec].to_numpy(), + ), + "DEMOGRAPHIC_ASEC_PERSON_JOIN", + ) + asec_person = values.person.reindex(index[asec]) + require( + asec_person.sex_known.all() and asec_person.is_female.notna().all(), + "DEMOGRAPHIC_UNKNOWN", + ) + original = acs_frame.person.set_index("person_id", drop=False) + require( + original.index.is_unique + and set(original.index) == set(native_ids.loc[~asec]) + and {"SEX", "is_female"} <= set(original), + "DEMOGRAPHIC_ACS_PERSON_JOIN", + ) + acs_person = original.reindex(native_ids.loc[~asec].to_numpy()) + sex = _numeric(acs_person.SEX) + require( + np.isin(sex, (1, 2)).all() + and pd.api.types.is_bool_dtype(acs_person.is_female.dtype) + and acs_person.is_female.notna().all() + and np.array_equal(acs_person.is_female.to_numpy(), sex == 2), + "ACS_SEX_IDENTITY", + ) + female = np.empty(len(index), dtype=np.float64) + female[asec] = asec_person.is_female.to_numpy(dtype=np.float64) + female[~asec] = (sex == 2).astype(np.float64) + states = {row[0]: row[2] for row in households} + selected_states = people.person_household_id.map(states) + require(selected_states.notna().all(), "DEMOGRAPHIC_UNKNOWN") + features = pd.DataFrame( + { + DEMOGRAPHIC_FEATURES[-2]: female, + DEMOGRAPHIC_FEATURES[-1]: selected_states.to_numpy(dtype=np.float64), + }, + index=index, + ) + evidence = { + "asec_projection_sha256": codec.sha(receipt), + "acs_native_sha256": codec.sha(acs_payload), + "asec_sex_and_state_observation_year": 2025, + "acs_sex_and_state_observation_year": 2024, + "state_is_income_year_residence_claim": False, + "asec_allocated_sex": "retained_when_source_owner_binding_is_known", + "acs_sex_allocation_provenance": "unresolved; no_unallocated_value_claim", + "unknown_policy": "refuse_opted_in_fit; no_fill_or_carried_fallback", + "state_encoding": "numeric_source_FIPS_split_predictor; not_geographic_distance", + "state_domain": "source_dictionary_codes; not_atomic_geography_admission", + "features_sha256": codec.sha(features.to_numpy(dtype="= 0).all()) and len(set(result)) == len(result), "ID_AXIS") + return result + + +def _acs_earnings(frame): + """Verify both native adjustment identities before the named NIU operator.""" + person = frame.person + mask = person[provenance.support_channel_column("person")].eq("acs") + selected = person.loc[mask] + require(len(selected) > 0, "ACS_EMPTY") + require(selected.source_year.astype(str).eq("2024").all(), "ACS_PERIOD") + age = _numeric(selected.age) + raw_age = _numeric(selected.AGEP) + require( + np.array_equal(age, raw_age) + and ((age >= 0) & (age <= 99) & (age == np.floor(age))).all(), + "ACS_AGE_IDENTITY", + ) + adjustment = _numeric(selected.ADJINC, nullable=True) + require(np.isfinite(adjustment).all() and (adjustment > 0).all(), "ACS_ADJINC") + for output, raw in universe.ACS_PUMS_EARNINGS_SOURCE_COLUMNS.items(): + require(raw in selected and output in selected, "ACS_EARNINGS_COLUMNS") + original = _numeric(selected[raw], nullable=True) + carried = _numeric(selected[output], nullable=True) + observed = ~np.isnan(original) + require(not (observed & (age < 15)).any(), "ACS_OUTSIDE_UNIVERSE_OBSERVED") + require( + np.array_equal(observed, ~np.isnan(carried)), "ACS_EARNINGS_MISSINGNESS" + ) + if raw == "WAGP": + require((original[observed] >= 0).all(), "ACS_WAGE_DOMAIN") + expected = original * (adjustment / 1_000_000.0) + require( + np.array_equal( + expected[observed].view("uint64"), carried[observed].view("uint64") + ), + "ACS_EARNINGS_ADJUSTMENT", + ) + require(not ((age >= 15) & ~observed).any(), "ACS_ELIGIBLE_EARNINGS_UNKNOWN") + applied = universe.apply_acs_pums_earnings_universe_zeros( + frame, + person_scope=mask, + boundary="current survey financial predictor preparation", + ) + return applied.frame, applied.receipt + + +@dataclass(frozen=True) +class QualifiedSurveyPredictors: + """Computed values; callers requalify live owners at materialization/replay.""" + + projection: bytes + matrix: bytes + source_frame: object + donor_frame: object + donor_columns: pd.DataFrame + native_money: pd.DataFrame + origins: pd.DataFrame + evidence: dict + demographic_conditioning: bool = False + geography_config_payload: bytes | None = None + geography_validation: bytes | None = None + + +def _qualified_seal(value): + """Seal the actual returned values across the last source/support borrow.""" + require(type(value) is QualifiedSurveyPredictors, "QUALIFIED_VALUES_TYPE") + geography = host.survey_budget.geography + tables = [] + for table in (value.donor_columns, value.native_money, value.origins): + require(type(table) is pd.DataFrame, "QUALIFIED_TABLE_TYPE") + metadata = codec.encode_json( + { + "columns": list(table.columns), + "column_axis": [ + type(table.columns).__name__, + str(table.columns.dtype), + list(table.columns.names), + ], + "index_axis": [ + type(table.index).__name__, + str(table.index.dtype), + list(table.index.names), + ], + "shape": list(table.shape), + } + ) + selected = np.ones(len(table), dtype=np.bool_) + parts = tuple( + (str(series.dtype), geography._storage_parts(series, selected)) + for series in ( + pd.Series(table.index.to_numpy(copy=False)), + *(table[c] for c in table), + ) + ) + tables.append((metadata, parts)) + return ( + value.projection, + value.matrix, + value.demographic_conditioning, + value.geography_config_payload, + value.geography_validation, + codec.encode_json(value.evidence), + tuple( + geography._population_stamp( + host.survey_budget.Population.from_frame( + frame, "survey_predictors.detached_values" + ) + ) + for frame in (value.source_frame, value.donor_frame) + ), + tuple(tables), + ) + + +def qualify_current_survey_predictors( + preparation, + allocated_population, + clone_population, + *, + demographic_conditioning=False, + geography_config=None, +): + """Read the actual full-parent money owner once, then seal all retained state. + + ASEC donors retain original household design weights, before allocation or + cloning. No legacy raw nominal dollar column can stand in for current money. + The source periods are distinct: ASEC 2025 interview/2024 annual income, + restated to 2024 price basis; ACS 2024 rolling prior-12-month amounts adjusted + by ADJINC. This model uses that explicitly declared temporal harmonization; + it does not claim the observation windows are equivalent. + """ + predictors = feature_columns(demographic_conditioning) + config_payload = host.survey_budget._config_payload(geography_config) + require( + type(preparation) is source.AuthenticatedSurveyPopulationPreparation, + "PREPARATION_TYPE", + ) + entry = preparation._checked() + state = entry[2] + view = source.CheckedSurveyPopulationView( + entry[1], state.context, state.frame, state.plan, json.loads(entry[1]) + ) + _, allocation, geography_binding = host.survey_budget._initial( + view, + allocated_population, + clone_population, + preparation=preparation, + geography_config=geography_config, + _with_geography_binding=True, + ) + seals = tuple( + host.survey_budget._population_identity(p) + for p in (allocated_population, clone_population) + ) + native_entry = source.asec_native._ISSUED.get(id(state.native[1])) + require( + native_entry is not None + and native_entry[0]() is state.native[1] + and state.native[1].payload == native_entry[1], + "NATIVE_ISSUANCE", + ) + parent = native_entry[2].parent + demographics = ( + _demographic_features(preparation, entry) if demographic_conditioning else None + ) + ready = parent.ready() + header = json.loads(ready.header) + require( + header["target_year"] == 2024 and header["semantic"] == "annual_current_money", + "CURRENT_MONEY_PERIOD", + ) + corrected, universe_receipt = _acs_earnings(state.frame) + people = corrected.person + pids = _ids(people.person_id) + channels = people[provenance.support_channel_column("person")].astype(str) + require(set(channels) == {"asec", "acs"}, "SOURCE_CHANNELS") + asec = channels.eq("asec").to_numpy() + acs = ~asec + native_ids = people[provenance.spine_source_id_column("person")].to_numpy( + dtype=np.int64 + ) + require( + set(native_ids[asec]) == set(state.source_frames[1].person.person_id), + "SELECTED_ASEC_ROSTER", + ) + scope = parent.scope + positions = {pid: i for i, pid in enumerate(scope.person_ids)} + require(len(positions) == len(scope.person_ids), "PARENT_ROSTER") + require(all(int(pid) in positions for pid in native_ids[asec]), "PARENT_MEMBER") + take = np.array([positions[int(pid)] for pid in native_ids[asec]], dtype=np.int64) + require(all(scope.person_years[int(i)] == 2024 for i in take), "CURRENT_COHORT") + index = pd.Index(pids, name="person_id") + money = pd.DataFrame(np.nan, index=index, columns=MONEY_FIELDS, dtype=np.float64) + field_evidence = {} + for name in MONEY_FIELDS: + field = ready.field(name) + domain = next(d for d in parent.spec.fields if d.name == name) + require(domain.entity == "person", "MONEY_ENTITY") + amounts = field.amounts[take] + valid = field.validity[take] + require( + (valid == 1).all() and np.isfinite(amounts).all(), "CURRENT_MONEY_UNKNOWN" + ) + money.loc[pids[asec], name] = amounts + field_evidence[name] = { + "domain": {f.name: getattr(domain, f.name) for f in fields(domain)}, + "amount_sha256": codec.sha(amounts.astype(" bytes: + return json.dumps( + value, sort_keys=True, separators=(",", ":"), allow_nan=False + ).encode() + + +def _registry_json(registry: TargetRegistry) -> str: + return _json( + {"country": registry.country, "specs": [asdict(s) for s in registry]} + ).decode() + + +def _validate_registry(registry: TargetRegistry) -> None: + if registry.country != "us" or not len(registry): + raise ValueError("A nonempty US demographic registry is required.") + scopes = set() + for spec in registry: + metadata = spec.metadata + table = metadata.get("table") + if table in RESERVED_FAMILIES or spec.family in RESERVED_FAMILIES: + raise ValueError("Income and tenure tables are reserved holdouts.") + if table not in {"S0101", "B01001"} or spec.family != f"acs.{table}": + raise ValueError( + "Only explicit ACS S0101/B01001 demographic families are supported." + ) + cell = re.fullmatch( + r"S0101_C01_(\d{3})" if table == "S0101" else r"B01001_(\d{3})", spec.name + ) + if cell is None or not 1 <= int(cell[1]) <= (19 if table == "S0101" else 49): + raise ValueError( + "A supported count cell must identify each demographic target." + ) + if ( + set(metadata) != _METADATA + or metadata["role"] != "calibration" + or metadata["geography"] != "0100000US" + or metadata["universe"] != "population" + or metadata["evidence_scope"] not in {"invented", "source_documented"} + or re.fullmatch(r"[a-f0-9]{64}", metadata["reference_sha256"]) is None + or spec.entity != "household" + or spec.period != "2024" + or spec.signed + or spec.value < 0 + or spec.filter is not None + or spec.tolerance is not None + or (spec.se is not None and not np.isfinite(spec.se)) + ): + raise ValueError( + "Demographic target scope must be explicit national population counts for 2024." + ) + if spec.hierarchy is None or spec.hierarchy != ( + age_module.expected_demographic_hierarchy( + table, + variable=spec.name, + label=spec.hierarchy.target.label, + geography=metadata["geography"], + ) + ): + raise ValueError( + "Each demographic target must carry exactly its own national " + "Census ACS calibration hierarchy." + ) + scopes.add(metadata["evidence_scope"]) + if len(scopes) != 1: + raise ValueError( + "Invented and source-documented evidence scopes cannot be mixed." + ) + + +def _registry_from_json(text: str) -> TargetRegistry: + if not isinstance(text, str) or len(text.encode()) > 1_048_576: + raise ValueError("A bounded canonical registry declaration is required.") + document = json.loads(text) + if not isinstance(document, dict) or set(document) != {"country", "specs"}: + raise ValueError("Unsupported registry declaration shape.") + specs = [] + for raw in document["specs"]: + if not isinstance(raw, dict): + raise ValueError("Unsupported registry declaration shape.") + hierarchy = raw.get("hierarchy") + if hierarchy is not None: + raw = {**raw, "hierarchy": CalibrationHierarchy.from_dict(hierarchy)} + specs.append(TargetSpec(**raw)) + registry = TargetRegistry(specs, country=document["country"]) + if text != _registry_json(registry): + raise ValueError( + "Registry declaration must have its canonical typed representation." + ) + _validate_registry(registry) + return registry + + +def _solver_options(params) -> dict: + if ( + set(params) != _PARAMS + or params["mass"] != "free" + or params["weight_anchor"] != "design" + ): + raise ValueError( + "The development calibration requires free mass and an original design cap." + ) + if type(params["epochs"]) is not int or params["epochs"] < 1: + raise ValueError("epochs must be a positive integer.") + for key, minimum in ( + ("learning_rate", 0), + ("max_weight_ratio", 1), + ("max_initial_weight_ratio", 1), + ): + value = params[key] + if ( + type(value) not in {int, float} + or not np.isfinite(value) + or value < minimum + or (key == "learning_rate" and value == 0) + ): + raise ValueError(f"Unsupported {key}.") + return { + **{key: params[key] for key in ("epochs", "learning_rate", "mass")}, + "max_weight_ratio": params["max_initial_weight_ratio"], + } + + +def demographic_calibration_node( + registry: TargetRegistry, + *, + base: str, + node_id: str = "national.demographic_calibration", + epochs: int, + learning_rate: float, + max_weight_ratio: float, + max_initial_weight_ratio: float, +) -> Node: + """Freeze a caller's explicit registry and declare its actual count inputs. + + Source citations/digests are declarations here, not independently verified + acquisitions or permission to repurpose held-out evidence. Activating a + genuine registry remains a separate, predeclared build decision. + + max_initial_weight_ratio bounds the solver relative to incoming importance + weights. max_weight_ratio separately guards the result relative to original + design weights in the executor. These anchors need not be equal. The kernel + cannot inspect undeclared design weights; a violated design cap refuses the + graph result rather than silently changing the solver's answer. + """ + frozen = _registry_from_json(_registry_json(registry)) + params = dict( + registry=_registry_json(frozen), + epochs=epochs, + learning_rate=learning_rate, + max_weight_ratio=max_weight_ratio, + max_initial_weight_ratio=max_initial_weight_ratio, + mass="free", + weight_anchor="design", + ) + _solver_options(params) + return Node( + id=node_id, + kernel=DemographicCalibrationKernel.ref, + inputs=(Slice("household", tuple(dict.fromkeys(s.measure for s in frozen))),), + params=params, + base=base, + structural=StructuralDelta.REWEIGHT, + weights=WeightTransition("household", "calibrated", mass="free"), + mass="free", + artifact_outputs=_OUTPUTS, + ) + + +class DemographicCalibrationKernel(KernelBase): + """Call the real solver and persist its complete v6 diagnostic artifact.""" + + ref = "us.demographic_calibration@1" + capabilities = Capabilities( + determinism=Determinism.DETERMINISTIC, + numeric=Numeric.PLATFORM_BITWISE, + seed_source=SeedSource.NONE, + structural=StructuralDelta.REWEIGHT, + consumes_se=False, + dependencies=("numpy", "pandas", "scipy", "torch"), + ) + + def implementation_hash(self) -> str: + return hashlib.sha256( + _json( + { + "adapter": source_hash( + sys.modules[__name__], + age_module, + registry_module, + diagnostics_module, + attribution_module, + solve_module, + kernels_module, + monetary_binding_module, + protocol_module, + dependencies=self.capabilities.dependencies, + ), + "solver": kernels_module.CALIBRATE_ADAM.implementation_hash(), + } + ) + ).hexdigest() + + def run(self, context: KernelContext) -> KernelResult: + registry = _registry_from_json(context.params["registry"]) + options = _solver_options(context.params) + expected = demographic_calibration_node( + registry, + base=context.node.base, + node_id=context.node.id, + **{ + k: context.params[k] + for k in ( + "epochs", + "learning_rate", + "max_weight_ratio", + "max_initial_weight_ratio", + ) + }, + ) + if context.node.normative() != expected.normative(): + raise ValueError("Calibration node differs from its complete declaration.") + if context.weights["household"].kind is not WeightKind.IMPORTANCE: + raise ValueError("Calibration requires the declared importance weights.") + if not np.all(context.weights["household"].values > 0): + raise ValueError("Calibration requires strictly positive incoming weights.") + table = context.tables["household"] + for spec in registry: + dtype = table[spec.measure].dtype + if not is_numeric_dtype(dtype) or is_complex_dtype(dtype): + raise ValueError("Demographic measures require numeric count columns.") + values = table[spec.measure].to_numpy(dtype=float, na_value=np.nan) + if not np.all( + np.isfinite(values) & (values >= 0) & (values == np.floor(values)) + ): + raise ValueError( + "Demographic measures must be nonnegative finite integer counts." + ) + frame = kernels_module._frame_from_context(context, "household") + result = calibrate( + frame, + registry.to_target_set(), + weight_entity="household", + method="adam", + seed=0, + **options, + ) + if result.skipped: + raise ValueError("Demographic calibration cannot skip declared targets.") + anchors = { + "solver_weight_anchor": "incoming_importance", + "solver_max_weight_ratio": context.params["max_initial_weight_ratio"], + "executor_weight_anchor": "original_design", + "executor_max_weight_ratio": context.params["max_weight_ratio"], + } + payload = _json( + diagnostics_payload(result, target_registry=registry, build=anchors) + ) + return KernelResult( + weights=result.frame.weights_for("household"), + artifacts={"diagnostics": payload}, + receipt={ + "scope": "national_demographic_calibration_development", + "evidence_scope": registry.specs[0].metadata["evidence_scope"], + "registry_sha256": hashlib.sha256( + context.params["registry"].encode() + ).hexdigest(), + "registry_version": registry.version, + "diagnostics_sha256": hashlib.sha256(payload).hexdigest(), + "reserved_families": sorted(RESERVED_FAMILIES), + "consumes_standard_errors": False, + **anchors, + "release_eligible": False, + "national_validation": "not_evaluated", + "congressional_district_validation": "not_evaluated", + }, + ) diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/eligibility_inputs.py b/packages/microcosm-build/src/microcosm/build/us_runtime/eligibility_inputs.py index dc7e69826..a1e157806 100644 --- a/packages/microcosm-build/src/microcosm/build/us_runtime/eligibility_inputs.py +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/eligibility_inputs.py @@ -52,6 +52,7 @@ from __future__ import annotations +from collections.abc import Mapping from importlib.resources import files import numpy as np @@ -69,6 +70,9 @@ SourceRuntimeError, run_source_stage, ) +from microcosm.build.us_runtime._person_signal_summary import ( + validate_person_signal_summary, +) from microcosm.frame import Frame from microcosm.frame.units import US_SCHEMA @@ -78,6 +82,11 @@ "US_ELIGIBILITY_INPUTS_REQUIRED_SOURCE_COLUMNS", "US_ELIGIBILITY_INPUTS_STAGE_NAME", "derive_us_eligibility_inputs_from_manifest", + "prepare_us_eligibility_person", + "us_eligibility_inputs_person_carries_signal", + "us_eligibility_inputs_gate_from_summary", + "us_eligibility_inputs_person_gate", + "us_eligibility_inputs_person_summary", "us_eligibility_inputs_signal_gate", "us_eligibility_inputs_stage_spec", "us_eligibility_inputs_summary", @@ -273,6 +282,108 @@ def _disabled_carries_signal(person: pd.DataFrame) -> bool: return values.nunique() > 1 +def us_eligibility_inputs_person_carries_signal(person: pd.DataFrame) -> bool: + """Return the complete legacy pass-through decision on the real person table. + + All five outputs must exist, and observed disability must vary. Preserve + the current partial-null rule; other output variation, raw source columns + and weight resolution belong to separate operations and are not inferred. + """ + have_all = all( + column in person.columns for column in US_ELIGIBILITY_INPUTS_OUTPUT_COLUMNS + ) + return have_all and _disabled_carries_signal(person) + + +def _validated_eligibility_weights( + person: pd.DataFrame, weights: np.ndarray +) -> np.ndarray: + """Return ``weights`` as a float64 array aligned 1:1 with ``person``.""" + + array = np.asarray(weights, dtype=np.float64) + if array.ndim != 1 or len(array) != len(person): + raise ValueError( + "US eligibility-inputs weights must be a 1-D array aligned 1:1 " + f"with the person table ({len(person)} row(s)); got shape " + f"{array.shape}." + ) + if not np.isfinite(array).all(): + raise ValueError("US eligibility-inputs weights must be finite.") + if (array < 0.0).any(): + raise ValueError("US eligibility-inputs weights must be nonnegative.") + return array + + +def prepare_us_eligibility_person( + person: pd.DataFrame, + weights: np.ndarray, + *, + seed: int, + time_period: int, +) -> pd.DataFrame: + """Derive measured ASEC eligibility inputs onto a copy of ``person``. + + The deterministic table-helper behind :func:`with_us_eligibility_inputs`: + it always runs the ``eligibility_inputs`` source-manifest stage over + ``person`` and ``weights`` and returns the aligned result. It does not + decide whether the surface already carries signal — that + idempotence/pass-through decision belongs to the Frame wrapper. + + Args: + person: The actual person table, carrying the raw ASEC source + columns this stage reads (see + ``US_ELIGIBILITY_INPUTS_REQUIRED_SOURCE_COLUMNS``), in its own + index and row order. + weights: Person weights aligned 1:1 with ``person``'s rows (same + length and row order; need not be reindexed by ``person_id``). + seed: Build-wide imputation seed threaded to the source-stage + runtime (the derivation itself is deterministic). + time_period: The dataset's time period. + + Returns: + A copy of ``person`` with all five eligibility columns attached + (``is_disabled``, ``is_blind``, and + ``is_full_time_college_student`` as ``bool``; + ``own_children_in_household`` and ``veterans_benefits`` as + ``float64``), in ``person``'s original index and row order. + + Raises: + ValueError: If ``weights`` does not align 1:1 with ``person``, is + not finite/nonnegative, or the stage output does not cover + every person. + SourceRuntimeError: If required raw ASEC column(s) are missing or + malformed. + """ + + weight_values = _validated_eligibility_weights(person, weights) + stage_person = person.copy(deep=True) + stage_person[_PERSON_WEIGHT_COLUMN] = weight_values + output = run_source_stage( + us_eligibility_inputs_stage_spec(), + tables={"person": stage_person}, + operation_handlers={ + "derive_eligibility_inputs": derive_us_eligibility_inputs_from_manifest, + }, + config=SourceRuntimeConfig(seed=int(seed), target_year=int(time_period)), + ) + aligned = output.set_index("person_id").reindex(person["person_id"]) + for column in US_ELIGIBILITY_INPUTS_OUTPUT_COLUMNS: + if aligned[column].isna().any(): + raise ValueError( + f"US eligibility-inputs stage output does not cover every " + f"person for {column!r}." + ) + + result = person.copy(deep=True) + bool_columns = ("is_disabled", "is_blind", "is_full_time_college_student") + for column in US_ELIGIBILITY_INPUTS_OUTPUT_COLUMNS: + if column in bool_columns: + result[column] = aligned[column].to_numpy(dtype=bool) + else: + result[column] = aligned[column].to_numpy(dtype=np.float64) + return result + + def with_us_eligibility_inputs(frame: Frame, *, seed: int, time_period: int) -> Frame: """Run the ``eligibility_inputs`` manifest stage over a US frame. @@ -302,37 +413,17 @@ def with_us_eligibility_inputs(frame: Frame, *, seed: int, time_period: int) -> if frame.schema != US_SCHEMA: raise ValueError("US eligibility inputs require the US schema.") person = frame.table("person") - have_all = all( - column in person.columns for column in US_ELIGIBILITY_INPUTS_OUTPUT_COLUMNS - ) - if have_all and _disabled_carries_signal(person): + if us_eligibility_inputs_person_carries_signal(person): return frame - stage_person = person.copy(deep=True) - stage_person[_PERSON_WEIGHT_COLUMN] = frame.resolve_weights("person").values - output = run_source_stage( - us_eligibility_inputs_stage_spec(), - tables={"person": stage_person}, - operation_handlers={ - "derive_eligibility_inputs": derive_us_eligibility_inputs_from_manifest, - }, - config=SourceRuntimeConfig(seed=int(seed), target_year=int(time_period)), + new_person = prepare_us_eligibility_person( + person, + frame.resolve_weights("person").values, + seed=seed, + time_period=time_period, ) - aligned = output.set_index("person_id").reindex(person["person_id"]) - for column in US_ELIGIBILITY_INPUTS_OUTPUT_COLUMNS: - if aligned[column].isna().any(): - raise ValueError( - f"US eligibility-inputs stage output does not cover every " - f"person for {column!r}." - ) - tables = {entity: frame.table(entity).copy() for entity in frame.entities} - bool_columns = ("is_disabled", "is_blind", "is_full_time_college_student") - for column in US_ELIGIBILITY_INPUTS_OUTPUT_COLUMNS: - if column in bool_columns: - tables["person"][column] = aligned[column].to_numpy(dtype=bool) - else: - tables["person"][column] = aligned[column].to_numpy(dtype=np.float64) + tables["person"] = new_person return Frame( tables, frame.schema, @@ -343,15 +434,31 @@ def with_us_eligibility_inputs(frame: Frame, *, seed: int, time_period: int) -> ) -def us_eligibility_inputs_summary(frame: Frame) -> dict[str, object]: - """Weighted eligibility-share summary for gates and release manifests.""" +def us_eligibility_inputs_person_summary( + person: pd.DataFrame, weights: np.ndarray +) -> dict[str, object]: + """Weighted eligibility-share summary for gates and release manifests. - person = frame.table("person") - weights = np.asarray(frame.resolve_weights("person").values, dtype=np.float64) - total_weight = float(weights.sum()) + The real-person-table counterpart of :func:`us_eligibility_inputs_summary`, + for callers (e.g. graph adapters) that hold a person table and an + explicit weight vector without a :class:`~microcosm.frame.Frame`. + + Args: + person: The actual person table, already carrying the five + eligibility columns. + weights: Person weights aligned 1:1 with ``person``'s rows. + + Returns: + The same summary payload as :func:`us_eligibility_inputs_summary`. + """ + + weight_values = _validated_eligibility_weights(person, weights) + total_weight = float(weight_values.sum()) def _share(mask: np.ndarray) -> float: - return float(weights[mask].sum()) / total_weight if total_weight > 0 else 0.0 + return ( + float(weight_values[mask].sum()) / total_weight if total_weight > 0 else 0.0 + ) disabled = person["is_disabled"].astype(bool).to_numpy() student = person["is_full_time_college_student"].astype(bool).to_numpy() @@ -377,31 +484,51 @@ def _share(mask: np.ndarray) -> float: } -def us_eligibility_inputs_signal_gate(frame: Frame) -> GateResult: - """Require the eligibility surface to carry plausible distributions. +def us_eligibility_inputs_summary(frame: Frame) -> dict[str, object]: + """Weighted eligibility-share summary for gates and release manifests.""" - Fails when a column is missing or constant, or when the weighted - disabled, full-time-student, parent, or veterans'-payment shares leave - their plausibility bands — each of which reproduces (or inverts) the - everyone-defaults failure of microcosm #244. + return us_eligibility_inputs_person_summary( + frame.table("person"), frame.resolve_weights("person").values + ) + + +def us_eligibility_inputs_gate_from_summary( + summary: Mapping[str, object], +) -> GateResult: + """Check eligibility-input plausibility bands and signal from a summary. + + The pure decision core of :func:`us_eligibility_inputs_signal_gate`, + factored out so graph adapters can reuse the incumbent checks — same + bands, order, and meaning — against a summary computed off the real + person table (see :func:`us_eligibility_inputs_person_summary`) + without a :class:`~microcosm.frame.Frame`. Assumes the caller has + already confirmed the five output columns are present; missing columns + are a separate failure mode (see + :func:`us_eligibility_inputs_person_gate`). + + Raises: + ValueError: If required fields/counts are missing, measurements are + malformed, or supplied bands differ from the registered policy. """ - person = frame.table("person") + validate_person_signal_summary( + summary, + family="eligibility", + outputs=US_ELIGIBILITY_INPUTS_OUTPUT_COLUMNS, + share_bands={ + "disabled_share": ("disabled_share_band", _DISABLED_SHARE_BAND), + "full_time_college_student_share": ( + "full_time_student_share_band", + _FULL_TIME_STUDENT_SHARE_BAND, + ), + "parent_share": ("parent_share_band", _PARENT_SHARE_BAND), + "veterans_benefits_share": ( + "veterans_benefits_share_band", + _VETERANS_BENEFITS_SHARE_BAND, + ), + }, + ) failures: list[str] = [] - missing = [ - column - for column in US_ELIGIBILITY_INPUTS_OUTPUT_COLUMNS - if column not in person.columns - ] - if missing: - return GateResult( - name="eligibility_inputs_signal", - passed=False, - failures=(f"person columns missing: {missing}.",), - details={"missing": missing}, - ) - - summary = us_eligibility_inputs_summary(frame) for column, count in summary["unique_counts"].items(): if count < 2: failures.append( @@ -434,3 +561,58 @@ def us_eligibility_inputs_signal_gate(frame: Frame) -> GateResult: failures=tuple(failures), details=summary, ) + + +def us_eligibility_inputs_person_gate( + person: pd.DataFrame, weights: np.ndarray +) -> GateResult: + """Require the eligibility surface to carry plausible distributions. + + The real-person-table counterpart of + :func:`us_eligibility_inputs_signal_gate`, composing + :func:`us_eligibility_inputs_person_summary` and + :func:`us_eligibility_inputs_gate_from_summary` exactly as the Frame + wrapper does. + """ + + missing = [ + column + for column in US_ELIGIBILITY_INPUTS_OUTPUT_COLUMNS + if column not in person.columns + ] + if missing: + return GateResult( + name="eligibility_inputs_signal", + passed=False, + failures=(f"person columns missing: {missing}.",), + details={"missing": missing}, + ) + summary = us_eligibility_inputs_person_summary(person, weights) + return us_eligibility_inputs_gate_from_summary(summary) + + +def us_eligibility_inputs_signal_gate(frame: Frame) -> GateResult: + """Require the eligibility surface to carry plausible distributions. + + Fails when a column is missing or constant, or when the weighted + disabled, full-time-student, parent, or veterans'-payment shares leave + their plausibility bands — each of which reproduces (or inverts) the + everyone-defaults failure of microcosm #244. + """ + + person = frame.table("person") + missing = [ + column + for column in US_ELIGIBILITY_INPUTS_OUTPUT_COLUMNS + if column not in person.columns + ] + if missing: + return GateResult( + name="eligibility_inputs_signal", + passed=False, + failures=(f"person columns missing: {missing}.",), + details={"missing": missing}, + ) + + summary = us_eligibility_inputs_summary(frame) + return us_eligibility_inputs_gate_from_summary(summary) diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/full_puf_enrichment.py b/packages/microcosm-build/src/microcosm/build/us_runtime/full_puf_enrichment.py new file mode 100644 index 000000000..4efdda196 --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/full_puf_enrichment.py @@ -0,0 +1,800 @@ +"""Complete canonical PUF enrichment boundaries; no source admission is issued. + +An upstream source owner supplies interpreted, period-aligned canonical donor +columns and a complete survey recipient. This module does not read raw PUF, +reinterpret E19200, choose missing-value aliases, or manufacture source evidence. +It reuses the maintained ordered QRF protocol and person/tax-unit finalizer. +Graph hosts must authenticate actual typed edges and retain their independent +full-population replay checks. These value checks do not replace either duty. +""" + +from __future__ import annotations + +from collections.abc import Mapping +from dataclasses import dataclass +from enum import Enum + +import numpy as np +import pandas as pd + +from microcosm.fit import _graph_legacy_qrf as codec +from microcosm.fit import model_input, qrf, qrf_target +from microcosm.fit.graph_legacy_apply_matrix import decode_matrix_apply_state +from microcosm.fit.graph_legacy_qrf import ( + legacy_qrf_apply_matrix_nodes, + legacy_qrf_train_nodes, +) +from microcosm.frame import Frame + +from . import puf_support as support + +PERSON_OUTPUTS = tuple(support.PUF_TAX_DETAIL_DEFAULT_PERSON_OUTPUTS) +TAX_UNIT_OUTPUTS = tuple(support.PUF_TAX_DETAIL_DEFAULT_TAX_UNIT_OUTPUTS) +TARGETS = (*PERSON_OUTPUTS, *TAX_UNIT_OUTPUTS) +PREDICTORS = tuple(support.PUF_TAX_DETAIL_DEFAULT_PREDICTORS) +PHASE = "us_full_canonical_puf_enrichment" +PUF59_PREDICTORS = ( + "puf_2015_filing_status_code", + "puf_2015_capped_return_size", + *PREDICTORS[2:], +) +SURVEY_SS_TOTAL_PREDICTOR = "puf_conditioning_social_security_total" +SURVEY_SS_COMPONENTS = tuple(support.PUF_TAX_DETAIL_SOCIAL_SECURITY_COMPONENT_OUTPUTS) +PUF55_SURVEY_SS_PREDICTORS = (*PUF59_PREDICTORS, SURVEY_SS_TOTAL_PREDICTOR) +PUF55_SURVEY_SS_PERSON_OUTPUTS = tuple( + target for target in PERSON_OUTPUTS if target not in SURVEY_SS_COMPONENTS +) + + +# Compatibility constants above continue to describe FULL65. PUF59 keeps the +# observed person home_mortgage_interest; only these detailed tax-unit fields +# are reserved for the independent downstream SCF producer in that profile. +SCF_MORTGAGE_OUTPUTS = ( + "first_home_mortgage_balance", + "second_home_mortgage_balance", + "first_home_mortgage_interest", + "second_home_mortgage_interest", + "first_home_mortgage_origination_year", + "second_home_mortgage_origination_year", +) +PUF59_TAX_UNIT_OUTPUTS = tuple( + target for target in TAX_UNIT_OUTPUTS if target not in SCF_MORTGAGE_OUTPUTS +) + + +class PufOutputProfile(Enum): + """Closed output contracts; arbitrary target subsets are not profiles.""" + + FULL65 = "full65" + PUF59 = "puf59" + PUF55_SURVEY_SS = "puf55_survey_ss" + PUF55_SURVEY_SS_NO_TOTAL = "puf55_survey_ss_no_total" + + @property + def predictors(self): + if self is PUF55_SURVEY_SS: + return PUF55_SURVEY_SS_PREDICTORS + return PREDICTORS if self is FULL65 else PUF59_PREDICTORS + + @property + def source_predictors(self): + # Independently measured source leaves, never derived here from generic + # filing status or actual recipient membership. The six monetary + # predictors retain their original ordered canonical arithmetic aliases. + if self is PUF55_SURVEY_SS: + return (*self.predictors[:2], SURVEY_SS_TOTAL_PREDICTOR) + return () if self is FULL65 else self.predictors[:2] + + @property + def donor_auxiliary_columns(self): + # Explicit capacity for donor Boolean incidence validation. Return-only + # source owners declare the bound (1 for 0/1 return incidence); person + # tables supply their membership capacity. This is not a physical person + # count observed in a return, nor a predictor or QRF conditioning input. + return () if self is FULL65 else ("puf_person_incidence_capacity",) + + @property + def person_outputs(self): + if self in (PUF55_SURVEY_SS, PUF55_SURVEY_SS_NO_TOTAL): + return PUF55_SURVEY_SS_PERSON_OUTPUTS + return PERSON_OUTPUTS + + @property + def tax_unit_outputs(self): + return TAX_UNIT_OUTPUTS if self is FULL65 else PUF59_TAX_UNIT_OUTPUTS + + @property + def targets(self): + return (*self.person_outputs, *self.tax_unit_outputs) + + @property + def phase(self): + # Legacy kernel parameter schemas are exact: phase carries the profile + # identity into every train/apply key and receipt without widening them. + # FULL65 retains its existing declarations and cache identities. + return PHASE if self is FULL65 else PHASE + "." + self.value + + +FULL65 = PufOutputProfile.FULL65 +PUF59 = PufOutputProfile.PUF59 +PUF55_SURVEY_SS = PufOutputProfile.PUF55_SURVEY_SS +PUF55_SURVEY_SS_NO_TOTAL = PufOutputProfile.PUF55_SURVEY_SS_NO_TOTAL + + +def require_puf_output_profile(profile): + """Require an explicit enum member; never infer a profile from missing data.""" + _require(type(profile) is PufOutputProfile, "PUF_OUTPUT_PROFILE") + return profile + + +def _require(condition, code): + if not condition: + raise ValueError(code) + + +def _numeric(values, *, label, boolean=False, nullable=False): + """Validate physical values before conversion; never parse strings as money.""" + series = pd.Series(values, copy=False) + if not nullable: + _require(not series.isna().any(), f"PUF_UNKNOWN:{label}") + observed = series.dropna() + if boolean: + valid = observed.map(lambda v: isinstance(v, (bool, np.bool_))) + else: + valid = observed.map( + lambda v: ( + not isinstance(v, (bool, np.bool_)) + and isinstance(v, (int, float, np.integer, np.floating)) + ) + ) + _require(bool(valid.all()), f"PUF_PHYSICAL_TYPE:{label}") + if not boolean: + # Do not let integer source identities/amounts silently lose bits at + # the explicit float64 model boundary. + _require( + all( + not isinstance(v, (int, np.integer)) or abs(int(v)) <= 2**53 + for v in observed + ), + f"PUF_FLOAT64_INTEGER_RANGE:{label}", + ) + numeric = series.to_numpy(dtype=np.float64, na_value=np.nan) + _require( + bool(np.isfinite(numeric[~series.isna().to_numpy()]).all()), + f"PUF_NONFINITE:{label}", + ) + return numeric + + +def _profile_source_values(table, *, profile): + """Validate the explicit measured leaves without implementing their producer. + + Physical/knownness checks and strict source qualification remain separate. + These are representation domains only; filing-class disclosure caps and + widow(er) coarsening belong to the independent source measurement receipt. + """ + if not profile.source_predictors: + return () + measured = tuple( + _numeric(table[column], label=column) for column in profile.source_predictors + ) + status, size = measured[:2] + _require( + bool(np.isin(status, [1, 2, 3, 4]).all()), "PUF_PROFILE_FILING_STATUS_DOMAIN" + ) + _require( + bool(((size >= 1) & (size == np.floor(size))).all()), + "PUF_PROFILE_RETURN_SIZE_DOMAIN", + ) + if profile is PUF55_SURVEY_SS: + _require(bool((measured[2] >= 0).all()), "PUF_PROFILE_SOCIAL_SECURITY_DOMAIN") + return measured + + +def _ids(values, label): + series = pd.Series(values, copy=False) + _require(series.dtype == np.dtype("int64"), f"PUF_ID_DTYPE:{label}") + result = series.to_numpy(copy=True) + _require(bool((result > 0).all()), f"PUF_ID_DOMAIN:{label}") + return result + + +def _known(values, known, columns, label): + _require( + isinstance(known, pd.DataFrame) + and known.index.equals(values.index) + and tuple(known.columns) == tuple(columns), + f"PUF_KNOWNNESS_AXIS:{label}", + ) + for column in columns: + mask = known[column] + _numeric(mask, label=f"{label}.{column}.known", boolean=True) + _require(bool(mask.all()), f"PUF_UNKNOWN:{label}.{column}") + + +def canonical_full_puf_donor( + person, + tax_unit, + *, + person_known, + tax_unit_known, + person_targets_at_tax_unit=(), + profile=FULL65, +): + """Reduce canonical columns after upstream interpretation, with no imputation. + + Some person destinations are observed only as return totals, for example + self-employed pension contributions. Such targets must be explicitly listed + in ``person_targets_at_tax_unit`` and supplied on the tax-unit table. A + destination's eventual grain never changes the declared donor grain. + Knownness covers every consumed nonstructural field, including true zeros. + Raw-to-canonical judgments belong upstream. PUF59 does not consume or + derive the six detailed mortgage fields reserved for the SCF producer. + When all person destinations are return totals, ``person=None`` avoids + inventing donor persons. FULL65 retains its canonical return-table + ``tax_unit_person_count`` input. PUF59 instead requires the source-declared + ``puf_person_incidence_capacity`` and its knownness (1 for return-level + 0/1 QBI incidence). Person-table PUF59 donors use membership as capacity; + neither case labels the bound as an observed physical return person count. + """ + profile = require_puf_output_profile(profile) + _require( + type(person_targets_at_tax_unit) is tuple + and len(set(person_targets_at_tax_unit)) == len(person_targets_at_tax_unit) + and set(person_targets_at_tax_unit) <= set(profile.person_outputs), + "PUF_DONOR_GRAIN_DECLARATION", + ) + pcols = tuple( + c for c in profile.person_outputs if c not in person_targets_at_tax_unit + ) + return_only = person is None + _require( + not return_only + or ( + person_targets_at_tax_unit == profile.person_outputs + and person_known is None + ), + "PUF_RETURN_ONLY_GRAIN", + ) + return_capacity_column = ( + profile.donor_auxiliary_columns[0] + if profile.donor_auxiliary_columns + else "tax_unit_person_count" + ) + tcols = ( + "weight", + "filing_status_code", + *((return_capacity_column,) if return_only else ()), + *person_targets_at_tax_unit, + *profile.tax_unit_outputs, + *profile.source_predictors, + ) + tables = [(tax_unit, ("tax_unit_id", *tcols), "tax_unit")] + if not return_only: + tables.append((person, ("person_id", "person_tax_unit_id", *pcols), "person")) + for table, required, label in tables: + _require( + isinstance(table, pd.DataFrame) + and table.index.is_unique + and table.columns.is_unique + and set(required) <= set(table.columns), + f"PUF_DONOR_COLUMNS:{label}", + ) + _require( + (return_only or not set(person_targets_at_tax_unit) & set(person.columns)) + and not set(pcols) & set(tax_unit.columns), + "PUF_DONOR_GRAIN_COLLISION", + ) + tids = _ids(tax_unit.tax_unit_id, "tax_unit") + _require( + len(tids) > 0 and len(set(tids)) == len(tids), + "PUF_DONOR_MEMBERSHIP", + ) + if not return_only: + pids = _ids(person.person_id, "person") + links = _ids(person.person_tax_unit_id, "person_tax_unit_id") + _require( + len(set(pids)) == len(pids) and set(links) == set(tids), + "PUF_DONOR_MEMBERSHIP", + ) + _known(person, person_known, pcols, "person") + _known(tax_unit, tax_unit_known, tcols, "tax_unit") + # Validate EVERY source cell before groupby can skip a missing member. + if not return_only: + normalized = pd.DataFrame(index=person.index) + for column in pcols: + normalized[column] = _numeric( + person[column], + label=f"person.{column}", + boolean=column in support._PUF_TAX_DETAIL_BOOLEAN_PERSON_OUTPUTS, + ) + grouped = normalized.groupby(links, sort=False).sum(min_count=1).reindex(tids) + count = pd.Series(links).value_counts(sort=False).reindex(tids).to_numpy() + else: + count = _numeric(tax_unit[return_capacity_column], label=return_capacity_column) + _require( + bool(((count >= 1) & (count == np.floor(count))).all()), + "PUF_PERSON_INCIDENCE_CAPACITY_DOMAIN" + if profile.donor_auxiliary_columns + else "PUF_PERSON_COUNT_DOMAIN", + ) + donor = pd.DataFrame(index=tax_unit.index) + for column in (*person_targets_at_tax_unit, *profile.tax_unit_outputs): + donor[column] = _numeric(tax_unit[column], label=f"tax_unit.{column}") + for column in pcols: + donor[column] = _numeric(grouped[column], label=f"reduced.{column}") + status = _numeric(tax_unit.filing_status_code, label="filing_status_code") + _require(bool(np.isin(status, [1, 2, 3, 4, 5]).all()), "PUF_FILING_STATUS_DOMAIN") + for column in ( + c + for c in profile.person_outputs + if c in support._PUF_TAX_DETAIL_BOOLEAN_PERSON_OUTPUTS + ): + values = donor[column].to_numpy() + _require( + bool( + ((values >= 0) & (values <= count) & (values == np.floor(values))).all() + ), + f"PUF_BOOLEAN_COUNT_DOMAIN:{column}", + ) + for column in ( + c + for c in profile.tax_unit_outputs + if c in support._PUF_TAX_DETAIL_DISCRETE_TAX_UNIT_OUTPUTS + ): + values = donor[column].to_numpy() + _require( + bool( + ( + (values == 0) + | ( + (values >= 1000) + & (values <= 9999) + & (values == np.floor(values)) + ) + ).all() + ), + f"PUF_YEAR_DOMAIN:{column}", + ) + # PUF59 matches the survey's total Schedule C measurement. The QBI owner + # partitions that total between ordinary and SSTB destinations, so neither + # component alone represents the predictor. FULL65 keeps its historical + # ordinary-component alias for compatibility with accepted prior evidence. + schedule_c = donor.self_employment_income_before_lsr.to_numpy() + if profile is not FULL65: + schedule_c = ( + schedule_c + donor.sstb_self_employment_income_before_lsr.to_numpy() + ) + # These are canonical arithmetic aliases, after complete cell validation. + feature_values = ( + status, + count.astype(np.float64), + donor.employment_income_before_lsr.to_numpy(), + schedule_c, + donor.taxable_interest_income.to_numpy(), + donor.qualified_dividend_income.to_numpy() + + donor.non_qualified_dividend_income.to_numpy(), + donor.short_term_capital_gains.to_numpy(), + donor.long_term_capital_gains_before_response.to_numpy(), + ) + # PUF59 first two features are already source-measured. Never label generic + # status/membership arithmetic as the PUF2015 disclosure measurement. + if profile.source_predictors: + measured = _profile_source_values(tax_unit, profile=profile) + feature_values = ( + *measured[:2], + *feature_values[2:], + *measured[2:], + ) + for name, values in zip(profile.predictors, feature_values, strict=True): + donor[name] = _numeric(values, label=name) + donor["weight"] = _numeric(tax_unit.weight, label="weight") + weights = donor.weight.to_numpy() + _require( + bool((weights >= 0).all()) and np.isfinite(weights.sum()) and weights.sum() > 0, + "PUF_DONOR_WEIGHT_DOMAIN", + ) + for column in profile.donor_auxiliary_columns: + donor[column] = count.astype(np.float64) + return donor.loc[ + :, + [ + *profile.predictors, + *profile.targets, + "weight", + *profile.donor_auxiliary_columns, + ], + ].copy() + + +def _recipient_matrix(frame, *, predictor_known, profile=FULL65): + profile = require_puf_output_profile(profile) + tax_unit, person = frame.table("tax_unit"), frame.table("person") + mask = support.puf_tax_detail_clone_mask(tax_unit, entity="tax_unit") + ids = _ids(tax_unit.loc[mask, "tax_unit_id"], "recipient") + selected_index = pd.Index(ids, name="tax_unit_id") + _require(len(ids) > 0, "PUF_NO_RECIPIENTS") + _require( + isinstance(predictor_known, pd.DataFrame) + and predictor_known.index.equals(selected_index), + "PUF_RECIPIENT_KNOWNNESS_AXIS", + ) + _known(predictor_known, predictor_known, profile.predictors, "recipient_predictor") + person_mask = person.person_tax_unit_id.isin(ids) + # The maintained resolver below owns ACS source applicability and its exact + # age-15 universe zeros. Screen physical numeric types first, since pandas + # numeric conversion otherwise accepts strings, booleans or datetime values. + for name in profile.predictors: + plan = support._strict_predictor_source_plan( + name, tax_unit=tax_unit, person=person + ) + if name in profile.source_predictors: + _require( + plan.entity == "tax_unit" and plan.columns == (name,), + "PUF_PROFILE_PREDICTOR_SOURCE:" + name, + ) + if plan.source_column == "filing_status_code" or plan.entity == "derived": + continue + table = person if plan.entity == "person" else tax_unit + rows = person_mask if plan.entity == "person" else mask + for column in plan.columns: + if column in table: + _numeric( + table.loc[rows, column], + label=f"recipient.{plan.entity}.{column}", + nullable=True, + ) + features, universe = support._strict_recipient_predictor_surface( + frame, mask, profile.predictors, person_outputs=profile.person_outputs + ) + selected = features.loc[mask, list(profile.predictors)].copy() + selected.index = selected_index + selected = selected.astype("float64") + _profile_source_values(selected, profile=profile) + return ( + model_input.encode_recipient_matrix( + selected, entity="tax_unit", entity_ids=ids + ), + universe, + ) + + +@dataclass(frozen=True) +class FullPufInputs: + """Selected model surfaces, excluding nonmodel donor validation columns.""" + + donor: pd.DataFrame + donor_frame: Frame + matrix: bytes + recipient_universe: Mapping[str, object] + profile: PufOutputProfile = FULL65 + + +def _validated_model_donor(donor, *, profile): + """Select the exact validated donor surface; issue no source/model authority. + + This is the existing values check, including auxiliary incidence capacity. + The receiving host must still authenticate source/period construction and + check the maintained donor Frame conversion without permitting coercion. + """ + _require( + isinstance(donor, pd.DataFrame) + and tuple(donor.columns) + == ( + *profile.predictors, + *profile.targets, + "weight", + *profile.donor_auxiliary_columns, + ), + "PUF_FULL_DONOR_ROSTER", + ) + for column in donor: + _numeric(donor[column], label=f"donor.{column}") + _profile_source_values(donor, profile=profile) + model_counts = donor[profile.predictors[1]].to_numpy() + counts = ( + donor[profile.donor_auxiliary_columns[0]].to_numpy() + if profile.donor_auxiliary_columns + else model_counts + ) + weights = donor["weight"].to_numpy() + _require( + len(donor) > 0 + and donor.index.is_unique + and bool(((counts >= 1) & (counts == np.floor(counts))).all()) + and bool(np.isin(donor[profile.predictors[0]], [1, 2, 3, 4, 5]).all()) + and bool((weights >= 0).all()) + and np.isfinite(weights.sum()) + and weights.sum() > 0, + "PUF_DONOR_MODEL_DOMAIN", + ) + for column in ( + c + for c in profile.person_outputs + if c in support._PUF_TAX_DETAIL_BOOLEAN_PERSON_OUTPUTS + ): + values = donor[column].to_numpy() + _require( + bool( + ( + (values >= 0) & (values <= counts) & (values == np.floor(values)) + ).all() + ), + f"PUF_BOOLEAN_COUNT_DOMAIN:{column}", + ) + for column in ( + c + for c in profile.tax_unit_outputs + if c in support._PUF_TAX_DETAIL_DISCRETE_TAX_UNIT_OUTPUTS + ): + values = donor[column].to_numpy() + _require( + bool( + ( + (values == 0) + | ( + (values >= 1000) + & (values <= 9999) + & (values == np.floor(values)) + ) + ).all() + ), + f"PUF_YEAR_DOMAIN:{column}", + ) + model_donor = donor.loc[:, [*profile.predictors, *profile.targets, "weight"]].copy() + return model_donor + + +def prepare_full_puf_inputs(frame, donor, *, predictor_known, profile=FULL65): + """Prepare selected surfaces without imputing absent source values. + + PUF59's canonical donor carries ``puf_person_incidence_capacity`` solely + for Boolean incidence validation. Return-only capacity is source declared; + person-table capacity comes from membership. The disclosure-capped return- + size predictor cannot serve as that bound. Only model predictors, selected + targets and weight reach the maintained donor Frame machinery. + """ + profile = require_puf_output_profile(profile) + model_donor = _validated_model_donor(donor, profile=profile) + matrix, universe = _recipient_matrix( + frame, predictor_known=predictor_known, profile=profile + ) + # Every input is now physically numeric, finite, and known. The maintained + # helper constructs the actual weighted donor Frame and repeats its strict + # source checks; its historical fillna operation has no missing values to fill. + inputs = support.prepare_us_puf_tax_detail_chain_inputs( + frame, + model_donor, + predictors=profile.predictors, + person_outputs=profile.person_outputs, + tax_unit_outputs=profile.tax_unit_outputs, + require_complete_recipient_predictors=True, + ) + _require( + inputs.donor.equals(model_donor) + and inputs.recipient_predictor_universe == universe, + "PUF_PREPARATION_CHANGED_CANONICAL_INPUT", + ) + return FullPufInputs(inputs.donor, inputs.donor_frame, matrix, universe, profile) + + +def full_puf_train_apply_nodes( + *, + donor_population, + recipient_population, + matrix_producer, + seed, + n_estimators, + zero_atol, + prefix="puf_full", + profile=FULL65, +): + """Declare every selected real fit/draw with ordered raw predecessors.""" + profile = require_puf_output_profile(profile) + fits = legacy_qrf_train_nodes( + prefix + ".fit", + population=donor_population, + entity="tax_unit", + predictors=profile.predictors, + targets=profile.targets, + seed=seed, + n_estimators=n_estimators, + zero_atol=zero_atol, + phase=profile.phase, + ) + applies = legacy_qrf_apply_matrix_nodes( + prefix + ".apply", + population=recipient_population, + fit_nodes=fits, + matrix_producer=matrix_producer, + seed=seed, + phase=profile.phase, + ) + return fits, applies + + +def decode_full_puf_draws( + *, + matrix, + matrix_producer_key, + raw_draws, + apply_state, + training_state, + seed, + profile=FULL65, +): + """Check typed payloads against the selected complete model/raw history. + + The exact ordered target roster is intrinsic to fitted model and application + states. Even the common first target has a different model identity across + profiles; truncated FULL65 artifacts cannot stand in for PUF59 artifacts. + """ + profile = require_puf_output_profile(profile) + _require(codec._hash(matrix_producer_key), "PUF_MATRIX_PRODUCER_KEY") + prepared = model_input.decode_recipient_matrix(matrix) + packet = decode_matrix_apply_state(apply_state) + application, chain = codec.read_application( + codec.encode_json(packet["application"]) + ) + training, fitted = codec.read_training(training_state) + fitted_values = fitted.to_dict() + applied_values = chain.to_dict() + _require( + prepared.entity == chain.entity == "tax_unit" + and tuple(prepared.features.columns) + == tuple(chain.predictors) + == profile.predictors + and tuple(chain.targets) == tuple(chain.completed_targets) == profile.targets + and tuple(fitted_values["completed_targets"]) == profile.targets + and all( + fitted_values[key] == applied_values[key] + for key in ( + "predictors", + "targets", + "entity", + "weight_kind", + "weight_sha256", + "model_config", + "donor_index", + ) + ) + and application["models"] == training["models"] + and application["seed"] == seed + and chain.recipient_index == qrf._index_identity(prepared.features.index), + "PUF_FULL_CHAIN_IDENTITY", + ) + _require( + packet["matrix_sha256"] == codec.sha(matrix) + and packet["matrix_producer_key"] == matrix_producer_key, + "PUF_FULL_MATRIX_BINDING", + ) + _require( + isinstance(raw_draws, Mapping) and tuple(raw_draws) == profile.targets, + "PUF_RAW_ROSTER", + ) + _require( + application["raw_targets"] + == [ + {"target": target, "sha256": codec.sha(raw_draws[target])} + for target in profile.targets + ], + "PUF_RAW_HISTORY", + ) + return pd.DataFrame( + { + target: codec.read_raw_target( + raw_draws[target], target=target, index=prepared.features.index + ) + for target in profile.targets + }, + index=prepared.features.index, + ) + + +def _check_fitted_model_donor(donor_frame, *, training_state, last_model, profile): + """Bind complete consumed donor values to an upstream-trusted final model. + + The caller must establish actual producer authority before this trusted + decoder is reached. A caller-selected hash or public receipt is insufficient. + This reuses the existing whole-chain value check and grants no admission. + """ + # The final fitted target consumed every preceding donor outcome. Binding + # its actual trusted producer bytes therefore checks the entire donor value + # surface, including fields used only by sparsity/tail finalization. A donor + # with the same index/weights but altered values cannot adjust these draws. + training, fitted_state = codec.read_training(training_state) + last = qrf_target.LegacyQRFTargetArtifact.from_trusted_bytes( + last_model, expected_sha256=training["models"][-1]["sha256"] + ) + _require( + last.target == profile.targets[-1] + and last.next_training_state == fitted_state + and last.training_id == training["models"][-1]["training_id"], + "PUF_FINAL_MODEL_BINDING", + ) + resolved = qrf._resolve_qrf_fit_input( + donor_frame, list(profile.predictors), list(profile.targets), "design" + ) + qrf.RegimeGatedQRF._validate_chain_donor(last.training_state._chain(), resolved) + _require( + last.donor_sha256 + == qrf_target._consumed_values_sha256( + resolved.table, (*profile.predictors, *profile.targets) + ), + "PUF_DONOR_CONSUMED_BYTES", + ) + + +def finalize_full_puf( + frame, + donor, + *, + predictor_known, + matrix, + matrix_producer_key, + raw_draws, + apply_state, + training_state, + last_model, + seed, + profile=FULL65, +): + """Use actual finalizer judgments only after the complete raw chain validates. + + Returns a candidate Frame plus tail-cap/universe evidence. A graph placement + owner must expose every changed person/tax-unit column, attach only on the + PUF masks, and run the existing full-population replay verifier afterwards. + This function itself never creates a live Population/source qualification. + """ + profile = require_puf_output_profile(profile) + inputs = prepare_full_puf_inputs( + frame, donor, predictor_known=predictor_known, profile=profile + ) + _require(matrix == inputs.matrix, "PUF_RECIPIENT_MATRIX_CHANGED") + raw = decode_full_puf_draws( + matrix=matrix, + matrix_producer_key=matrix_producer_key, + raw_draws=raw_draws, + apply_state=apply_state, + training_state=training_state, + seed=seed, + profile=profile, + ) + _check_fitted_model_donor( + inputs.donor_frame, + training_state=training_state, + last_model=last_model, + profile=profile, + ) + mask = support.puf_tax_detail_clone_mask(frame.table("tax_unit"), entity="tax_unit") + # Explicit adapter between separate graph entity-ID and pandas row indexes; + # matrix recomputation above proves the ordered IDs before this relabeling. + raw.index = frame.table("tax_unit").index[mask] + caps = [] + result = support.finalize_us_puf_tax_detail_predictions( + frame, + inputs.donor, + raw.copy(deep=True), + person_outputs=profile.person_outputs, + tax_unit_outputs=profile.tax_unit_outputs, + tail_bound_diagnostics=caps, + absent_cells=support.PUF_ABSENT_CELLS_PRESERVE_NULLS, + ) + return result, { + "scope": profile.phase, + "output_profile": profile.value, + "predictor_order": list(profile.predictors), + "donor_auxiliary_columns": list(profile.donor_auxiliary_columns), + "person_target_count": len(profile.person_outputs), + "tax_unit_target_count": len(profile.tax_unit_outputs), + "target_count": len(profile.targets), + "target_order": list(profile.targets), + "recipient_universe": dict(inputs.recipient_universe), + "tail_bounds": caps, + "matrix_sha256": codec.sha(matrix), + "raw_target_sha256": { + target: codec.sha(raw_draws[target]) for target in profile.targets + }, + "source_admission_issued": False, + "release_eligible": False, + } diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/graph_acs_housing_universe.py b/packages/microcosm-build/src/microcosm/build/us_runtime/graph_acs_housing_universe.py new file mode 100644 index 000000000..2929f225e --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/graph_acs_housing_universe.py @@ -0,0 +1,698 @@ +"""Typed ACS source evidence and exact whole-household graph selection. + +Only CREATE authenticates source bytes. Selection keeps the original evidence as +an ancestor and emits selected identities/positions; it never mints source +authority or asserts calibrated representation. +""" + +from __future__ import annotations + +import json +import struct +from dataclasses import InitVar, dataclass, replace +from pathlib import Path + +import numpy as np +import pandas as pd + +from microcosm.frame import US_SCHEMA +from microcosm.graph import ( + ArtifactInput, + ArtifactOutput, + ArtifactType, + Capabilities, + Determinism, + Graph, + KernelBase, + KernelContext, + KernelRegistry, + KernelResult, + Node, + Numeric, + Slice, + SourceRef, + StructuralDelta, +) +from microcosm.graph.canonical import canonical_json +from microcosm.graph.keys import opaque_artifact_key + +from . import acs_housing_universe_source as source +from .graph_context import ( + US_FRAME_CONTEXT_TYPE, + encode_us_frame_context, + us_frame_from_context, +) +from .graph_implementation import ( + STAGE_DEPENDENCIES, + implementation_hash, + implementation_manifest, +) +from .graph_sources import frame_column_declarations + +ACS_HU_PAYLOAD_MAX_BYTES = 64 * 1024**2 +ACS_HU_HEADER_MAX_BYTES = 65536 +MAGIC = bytes.fromhex("4d43414353485502") +US_ACS_HOUSING_UNIVERSE_TYPE = ArtifactType("microcosm.us.acs_housing_universe", 2) +US_ACS_HOUSING_PREPARATION_TYPE = ArtifactType( + "microcosm.us.acs_housing_preparation", 2 +) +US_ACS_HOUSING_SELECTION_TYPE = ArtifactType("microcosm.us.acs_housing_selection", 2) +CREATE_NODE = "acs_housing_create" +SELECT_NODE = "acs_housing_select" +SOURCE_NAME = "acs_housing_source" +PHASE = "acs_housing_universe_graph_v2" +_TOKEN = object() +_CODES = ( + "interview_scope", + "physical_unit", + "household_kind", + "tenure_subtype", + "occupied_hu", + "hu_tenure_class", + "unresolved_reasons", + "TEN_valid", +) +_HEADER_FIELDS = frozenset( + { + "format", + "release_eligible", + "source_receipt_bytes", + "projection_bytes", + "household_rows", + "person_rows", + "code_columns", + "body_bytes", + "source_receipt_sha256", + "projection_sha256", + "prepared_receipt_sha256", + "frame_context_sha256", + "frame_sha256", + "definition_sha256", + "implementation_sha256", + } +) +_ARTIFACT_OUTPUTS = ( + ArtifactOutput("frame_context", US_FRAME_CONTEXT_TYPE), + ArtifactOutput("prepared_receipt", US_ACS_HOUSING_PREPARATION_TYPE), + ArtifactOutput("housing_universe", US_ACS_HOUSING_UNIVERSE_TYPE), +) +_PREPARED_FIELDS = frozenset( + { + "format", + "release_eligible", + "source_receipt_sha256", + "projection_sha256", + "implementation_sha256", + "same_snapshot_frame_and_observations", + "full_source_frame_before_selection", + "full_source_lexical_projection", + "native_selection_before_person_accumulation", + "selection_kind", + "requested_serialnos", + "full_source_inclusion_probability", + "pre_promotion_frame_sha256", + "frame_sha256", + "dtype_transitions", + "entity_rows", + "weight_kind", + "HU_columns", + } +) + + +def _require(value, code): + source._require(value, "GRAPH_ACS_" + code) + + +@dataclass(frozen=True) +class BoundACSHousingEvidence: + """Immutable graph transport, explicitly not AuthenticatedACSHousingSource.""" + + header_json: bytes + source_receipt_json: bytes + projection_json: bytes + codes: bytes + _token: InitVar[object] = None + + def __post_init__(self, _token): + _require(_token is _TOKEN, "BOUND_CONSTRUCTOR") + + +def _verify_preparation(prepared, receipt, household, frame_context): + """Check the v2 construction claim without minting source authority.""" + _require( + set(prepared) == _PREPARED_FIELDS + and prepared["format"] == "microcosm.acs_housing_preparation.v2" + and prepared["release_eligible"] is False + and prepared["HU_columns"] == "artifact_only" + and prepared["weight_kind"] == "design" + and prepared["same_snapshot_frame_and_observations"] is True + and prepared["full_source_lexical_projection"] is True + and prepared["full_source_inclusion_probability"] is None, + "PREPARED_SCHEMA", + ) + requested = prepared["requested_serialnos"] + subset = requested is not None + _require( + not subset + or ( + type(requested) is list + and bool(requested) + and all(type(key) is str for key in requested) + and len(set(requested)) == len(requested) + ), + "PREPARED_SELECTION", + ) + chosen = tuple(sorted(household.SERIALNO)) + _require( + prepared["full_source_frame_before_selection"] is (not subset) + and prepared["native_selection_before_person_accumulation"] is subset + and prepared["selection_kind"] + == ("engineering_exact_keys" if subset else "all") + and receipt["selection"] == ("exact_serialnos" if subset else "all") + and (not subset or tuple(sorted(requested)) == chosen) + and receipt["selected_serialnos_sha256"] == source._sha(canonical_json(chosen)), + "PREPARED_SELECTION", + ) + context = source._parse_json(frame_context, ACS_HU_PAYLOAD_MAX_BYTES) + rows = prepared["entity_rows"] + entities = context.get("entities") + _require( + type(rows) is dict + and type(entities) is dict + and set(rows) == set(entities) == set(US_SCHEMA.entities) + and all( + type(rows[entity]) is int + and rows[entity] >= 0 + and type(entities[entity]) is dict + and type(entities[entity].get("rows")) is int + and rows[entity] == entities[entity]["rows"] + for entity in US_SCHEMA.entities + ), + "PREPARED_CONTEXT", + ) + + +def encode_acs_housing_evidence(prepared): + _require( + type(prepared) is source.PreparedACSHousingPopulation + and type(prepared.source) is source.AuthenticatedACSHousingSource, + "PREPARED_TYPE", + ) + receipt = source._parse_json(prepared.receipt_json, source.ACS_HU_RECEIPT_MAX_BYTES) + observation = prepared.source + projection = source._parse_json( + observation.projection_json, ACS_HU_PAYLOAD_MAX_BYTES + ) + _h, _p, _t, _lines, _pw, codes = source._tables(projection) + source.verify_acs_frame_projection(prepared.frame, projection) + _require( + receipt["source_receipt_sha256"] == source._sha(observation.receipt_json) + and receipt["projection_sha256"] == source._sha(observation.projection_json) + and receipt["frame_sha256"] == source.frame_content_sha256(prepared.frame), + "PREPARED_BINDING", + ) + context = encode_us_frame_context(prepared.frame) + _verify_preparation( + receipt, + source._parse_json(observation.receipt_json, source.ACS_HU_RECEIPT_MAX_BYTES), + _h, + context, + ) + native = b"".join(codes[n].to_numpy(dtype="uint8").tobytes() for n in _CODES) + body = observation.receipt_json + observation.projection_json + native + header = { + "format": "microcosm.acs_housing_graph_evidence.v2", + "release_eligible": False, + "source_receipt_bytes": len(observation.receipt_json), + "projection_bytes": len(observation.projection_json), + "household_rows": len(_h), + "person_rows": len(_p), + "code_columns": list(_CODES), + "body_bytes": len(body), + "source_receipt_sha256": source._sha(observation.receipt_json), + "projection_sha256": source._sha(observation.projection_json), + "prepared_receipt_sha256": source._sha(prepared.receipt_json), + "frame_context_sha256": source._sha(context), + "frame_sha256": receipt["frame_sha256"], + "definition_sha256": source._definition()[1], + "implementation_sha256": source._implementation(), + } + encoded = canonical_json(header) + _require(len(encoded) <= ACS_HU_HEADER_MAX_BYTES, "HEADER_SIZE") + payload = MAGIC + struct.pack(" 0 + and nhouse > 0, + "SECTION_SIZE", + ) + _require( + header["body_bytes"] == nreceipt + nprojection + nhouse * len(_CODES) + and len(payload) == start + size + header["body_bytes"] + 32, + "BODY_SIZE", + ) + cursor = start + size + receipt_bytes = payload[cursor : cursor + nreceipt] + projection_bytes = payload[cursor + nreceipt : cursor + nreceipt + nprojection] + codes = payload[cursor + nreceipt + nprojection : -32] + receipt = source._parse_json(receipt_bytes, source.ACS_HU_RECEIPT_MAX_BYTES) + projection = source._parse_json(projection_bytes, ACS_HU_PAYLOAD_MAX_BYTES) + prepared = source._parse_json(prepared_receipt, source.ACS_HU_RECEIPT_MAX_BYTES) + household, person, _typed, _lines, _pw, derived = source._tables(projection) + _require( + source._sha(prepared_receipt) == header["prepared_receipt_sha256"] + and source._sha(frame_context) == header["frame_context_sha256"], + "CONTEXT_BINDING", + ) + _verify_preparation(prepared, receipt, household, frame_context) + _require( + source._sha(receipt_bytes) + == header["source_receipt_sha256"] + == prepared["source_receipt_sha256"] + and source._sha(projection_bytes) + == header["projection_sha256"] + == prepared["projection_sha256"] + == receipt["projection_sha256"], + "SOURCE_BINDING", + ) + _require( + header["definition_sha256"] + == receipt["definition_sha256"] + == source._definition()[1] + and header["implementation_sha256"] + == prepared["implementation_sha256"] + == receipt["implementation_sha256"] + == source._implementation(), + "IMPLEMENTATION_BINDING", + ) + _require(header["frame_sha256"] == prepared["frame_sha256"], "FRAME_BINDING") + # Coverage of the pins is otherwise only transitive, through the + # implementation hash over the archive manifest. Comparing them here + # makes the evidence say what it rests on. + _require( + receipt["format"] == "microcosm.acs_housing_universe_source.v1" + and receipt["release_eligible"] is False + and receipt["vintage"] == 2024 + and receipt["archives"] + == [ + {"role": r, "filename": n, "sha256": d, "bytes": s} + for r, n, d, s in source._pins() + ], + "SOURCE_PINS", + ) + _require( + len(household) == nhouse == receipt["selected_counts"]["households"] + and len(person) + == header["person_rows"] + == receipt["selected_counts"]["persons"], + "ROW_COUNTS", + ) + _require( + codes + == b"".join( + derived[name].to_numpy(dtype="uint8").tobytes() for name in _CODES + ), + "NATIVE_CODES", + ) + return BoundACSHousingEvidence( + raw_header, receipt_bytes, projection_bytes, codes, _token=_TOKEN + ) + except source.ACSHousingSourceError: + raise + except ( + ValueError, + TypeError, + KeyError, + OverflowError, + struct.error, + AttributeError, + ): + raise source.ACSHousingSourceError("GRAPH_ACS_TRANSPORT_CONTRACT") from None + + +def _artifact(context, name, expected_type): + value = context.artifacts.get(name) + _require(value is not None and value.type == expected_type, "ARTIFACT_TYPE") + declarations = [item for item in context.node.artifact_inputs if item.name == name] + _require( + len(declarations) == 1 + and value.key + == opaque_artifact_key(value.producer_key, declarations[0].artifact), + "ARTIFACT_KEY", + ) + return value + + +def _keys(value): + if value is None: + return None + _require( + isinstance(value, (tuple, list)) + and bool(value) + and all(type(v) is str for v in value) + and len(set(value)) == len(value), + "SELECTION_KEYS", + ) + return tuple(value) + + +def _slices(columns): + # The compiler does not treat implicit entity IDs as owned data columns. + # Structural-only groups are reconstructed from declared person membership + # and checked against the parent's typed ID digest, never read as fake cells. + return tuple( + Slice(entity, tuple(c.column for c in columns if c.entity == entity)) + for entity in US_SCHEMA.entities + if any(c.entity == entity for c in columns) + ) + + +def _complete_structural_tables(context, frame_context): + document = json.loads(frame_context) + tables = dict(context.tables) + _require("person" in tables, "PERSON_VIEW") + for group in US_SCHEMA.group_entities: + if group in tables: + continue + id_column = US_SCHEMA.entity_id_column(group) + _require( + document["entities"][group]["columns"] == [id_column], + "UNDECLARED_GROUP_CELLS", + ) + membership = tables["person"][US_SCHEMA.membership_column(group)] + _require(membership.dtype == np.dtype("int64"), "MEMBERSHIP_DTYPE") + tables[group] = pd.DataFrame({id_column: np.unique(membership.to_numpy())}) + # us_frame_from_context checks every reconstructed group's actual IDs + # against the original producer's ordered-ID digest before Frame creation. + return replace(context, tables=tables) + + +class _Kernel(KernelBase): + def implementation_hash(self): + return implementation_hash(source.ACS_HU_STAGE) + + +class ACSHousingCreateKernel(_Kernel): + ref = "us.acs_housing.create@2" + capabilities = Capabilities( + determinism=Determinism.DETERMINISTIC, + numeric=Numeric.PLATFORM_BITWISE, + structural=StructuralDelta.CREATE, + dependencies=STAGE_DEPENDENCIES[source.ACS_HU_STAGE], + ) + + def __init__(self, snapshot_root): + self.snapshot_root = Path(snapshot_root) + + def run(self, context: KernelContext): + node = context.node + _require( + node.kernel == self.ref + and node.structural is StructuralDelta.CREATE + and tuple(node.sources) == (SOURCE_NAME,) + and not node.inputs + and not node.artifact_inputs + and node.artifact_outputs == _ARTIFACT_OUTPUTS, + "CREATE_DECLARATION", + ) + _require( + set(context.params) == {"phase", "serialnos"} + and context.params["phase"] == PHASE, + "PARAMS", + ) + prepared = source.prepare_acs_housing_population( + context.sources[SOURCE_NAME], + snapshot_root=self.snapshot_root, + serialnos=_keys(context.params["serialnos"]), + ) + _require( + frame_column_declarations(prepared.frame) == node.outputs, + "COLUMN_INVENTORY", + ) + payload = encode_acs_housing_evidence(prepared) + return KernelResult( + frame=prepared.frame, + artifacts={ + "frame_context": encode_us_frame_context(prepared.frame), + "prepared_receipt": prepared.receipt_json, + "housing_universe": payload, + }, + receipt={ + "phase": PHASE, + "prepared": prepared.receipt, + "housing_universe_sha256": source._sha(payload), + "implementation": implementation_manifest(source.ACS_HU_STAGE), + "release_eligible": False, + }, + ) + + +class ACSHousingSelectKernel(_Kernel): + ref = "us.acs_housing.select@2" + capabilities = Capabilities( + determinism=Determinism.DETERMINISTIC, + numeric=Numeric.PLATFORM_BITWISE, + structural=StructuralDelta.FILTER, + dependencies=STAGE_DEPENDENCIES[source.ACS_HU_STAGE], + ) + + def run(self, context: KernelContext): + node = context.node + _require( + node.kernel == self.ref + and node.structural is StructuralDelta.FILTER + and node.mass == "declared" + and not node.outputs + and not node.sources, + "SELECT_DECLARATION", + ) + _require( + set(context.params) == {"phase", "serialnos"} + and context.params["phase"] == PHASE, + "PARAMS", + ) + _require( + set(context.artifacts) + == {"frame_context", "prepared_receipt", "housing_universe"} + and node.artifact_outputs + == ( + ArtifactOutput("frame_context", US_FRAME_CONTEXT_TYPE), + ArtifactOutput("selection", US_ACS_HOUSING_SELECTION_TYPE), + ), + "SELECT_ARTIFACTS", + ) + values = [ + _artifact(context, n, t) + for n, t in ( + ("frame_context", US_FRAME_CONTEXT_TYPE), + ("prepared_receipt", US_ACS_HOUSING_PREPARATION_TYPE), + ("housing_universe", US_ACS_HOUSING_UNIVERSE_TYPE), + ) + ] + _require( + len({v.producer_key for v in values}) == 1 + and len({a.producer for a in node.artifact_inputs}) == 1 + and next(iter(node.artifact_inputs)).producer == node.base, + "SAME_PRODUCER", + ) + bound = bind_acs_housing_evidence( + values[2].payload, + prepared_receipt=values[1].payload, + frame_context=values[0].payload, + ) + frame = us_frame_from_context( + _complete_structural_tables(context, values[0].payload) + ) + _require( + tuple(node.inputs) == _slices(frame_column_declarations(frame)), + "COMPLETE_FRAME_DECLARATION", + ) + header = json.loads(bound.header_json) + _require( + source.frame_content_sha256(frame) == header["frame_sha256"], "FRAME_CELLS" + ) + projection = json.loads(bound.projection_json) + source.verify_acs_frame_projection(frame, projection) + selected_projection, chosen, _counts = source._select( + projection, _keys(context.params["serialnos"]) + ) + household = frame.table("household") + household_keep = household.SERIALNO.isin(chosen).to_numpy() + ids = household.household_id.to_numpy()[household_keep] + keep = frame.person.person_household_id.isin(ids) + _require(bool(keep.any()), "EMPTY_SELECTION") + selected = frame.select(keep) + source.verify_acs_frame_projection(selected, selected_projection) + positions = { + "person": np.flatnonzero(keep).astype("int64"), + "household": np.flatnonzero(household_keep).astype("int64"), + } + _require( + selected.weights_for("household").values.tobytes() + == frame.weights_for("household").values[positions["household"]].tobytes(), + "SELECTED_WEIGHT_BYTES", + ) + before_mass, after_mass = frame.stratum_mass(), selected.stratum_mass() + native_households = dict( + zip( + selected.table("household").household_id, + selected.table("household").SERIALNO, + strict=True, + ) + ) + selection = { + "format": "microcosm.acs_housing_selection.v2", + "release_eligible": False, + "representative": False, + "source_authority": "original_CREATE_only", + "producer_key": values[0].producer_key, + "source_payload_sha256": source._sha(values[2].payload), + "source_prepared_receipt_sha256": source._sha(values[1].payload), + "source_frame_context_sha256": source._sha(values[0].payload), + "selected_frame_sha256": source.frame_content_sha256(selected), + "selected_frame_context_sha256": source._sha( + encode_us_frame_context(selected) + ), + "selected_projection_sha256": source._sha( + canonical_json(selected_projection) + ), + "selected_serialnos_sha256": source._sha(canonical_json(chosen)), + "positions": {e: a.tolist() for e, a in positions.items()}, + "frame_identity_scope": "actual_CREATE_frame", + "native_keys": { + "household": selected.table("household").SERIALNO.tolist(), + "person": [ + [native_households[household_id], int(line)] + for household_id, line in selected.person[ + ["person_household_id", "SPORDER"] + ].itertuples(index=False, name=None) + ], + }, + "ordered_ids": { + e: selected.table(e)[US_SCHEMA.entity_id_column(e)].tolist() + for e in US_SCHEMA.entities + }, + "weight_kind": "design", + "mass_policy": "declared_no_normalization", + } + return KernelResult( + keep=pd.Series( + keep.to_numpy(dtype=bool), + index=pd.Index(frame.person.person_id.to_numpy(), name="person_id"), + dtype=bool, + ), + artifacts={ + "frame_context": encode_us_frame_context(selected), + "selection": canonical_json(selection), + }, + receipt={ + "phase": PHASE, + "implementation": implementation_manifest(source.ACS_HU_STAGE), + "selection_sha256": source._sha(canonical_json(selection)), + "release_eligible": False, + "mass": { + "policy": "declared", + "before": float(before_mass.sum()), + "after": float(after_mass.sum()), + "stratum_before": { + str(k): float(v) for k, v in before_mass.items() + }, + "stratum_after": {str(k): float(v) for k, v in after_mass.items()}, + }, + }, + ) + + +def acs_housing_graph(columns, *, serialnos=None, selected_serialnos=None): + """Explicit seed source selection followed by exact further household pruning.""" + columns = tuple(columns) + _keys(serialnos) + _keys(selected_serialnos) + create = Node( + id=CREATE_NODE, + kernel=ACSHousingCreateKernel.ref, + structural=StructuralDelta.CREATE, + sources=(SOURCE_NAME,), + outputs=columns, + params={"phase": PHASE, "serialnos": serialnos}, + artifact_outputs=_ARTIFACT_OUTPUTS, + ) + select = Node( + id=SELECT_NODE, + kernel=ACSHousingSelectKernel.ref, + structural=StructuralDelta.FILTER, + mass="declared", + base=CREATE_NODE, + inputs=_slices(columns), + params={"phase": PHASE, "serialnos": selected_serialnos}, + artifact_inputs=tuple( + ArtifactInput(a.name, CREATE_NODE, a.name, a.type) + for a in _ARTIFACT_OUTPUTS + ), + artifact_outputs=( + ArtifactOutput("frame_context", US_FRAME_CONTEXT_TYPE), + ArtifactOutput("selection", US_ACS_HOUSING_SELECTION_TYPE), + ), + ) + return Graph( + country="us", + sources=(SourceRef(SOURCE_NAME, codec=source.ACS_HU_CODEC),), + nodes=(create, select), + ) + + +def acs_housing_registry(*, snapshot_root): + registry = KernelRegistry() + registry.register(ACSHousingCreateKernel(snapshot_root)) + registry.register(ACSHousingSelectKernel()) + return registry diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/graph_asec_income.py b/packages/microcosm-build/src/microcosm/build/us_runtime/graph_asec_income.py new file mode 100644 index 000000000..addadd581 --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/graph_asec_income.py @@ -0,0 +1,665 @@ +"""Typed PAW transport and reported-income accounting, never source authority. + +The six results are source observations and an accounting residual. They are +not simulated SSI/TANF, ACS other income, or a completed eight-category bridge. +""" + +from __future__ import annotations + +import struct +from dataclasses import InitVar, dataclass + +import numpy as np +import pandas as pd + +from microcosm.graph import ( + ArtifactOutput, + ArtifactType, + Capabilities, + Determinism, + KernelBase, + KernelContext, + KernelResult, + Numeric, + Owned, + Slice, + StructuralDelta, +) +from microcosm.graph.canonical import canonical_json + +from . import asec_income_observations as source +from . import graph_context +from .asec_current_money import MoneyRefusalError, _json, _parse, _require, _sha +from .asec_current_money_selection import ( + US_ASEC_PREPARED_RECEIPT_TYPE, + US_ASEC_SELECTED_MONEY_TYPE, + US_ASEC_SELECTION_TYPE, + SelectedCurrentMoney, + decode_selected_current_money, + encode_selected_current_money, +) +from .graph_context import US_FRAME_CONTEXT_TYPE +from .graph_implementation import ( + STAGE_DEPENDENCIES, + implementation_hash, + implementation_manifest, +) + +US_ASEC_INCOME_OBSERVATIONS_TYPE = ArtifactType( + "microcosm.us.asec_income_observations", 1 +) +US_ASEC_REPORTED_INCOME_TYPE = ArtifactType("microcosm.us.asec_reported_income", 3) +STAGE = "asec_prepared_v3" +REPORTED_INCOME_NODE = "asec_prepared.reported_income" +RESULT_COLUMNS = tuple( + "asec_reported_" + name + "_2024_price" + for name in ( + "wage_income", + "net_self_employment_income", + "ssi", + "cash_public_assistance", + "total_income", + "income_unallocated", + ) +) +CONTRIBUTING_FIELDS = ("WSAL_VAL", "SEMP_VAL", "FRSE_VAL", "SSI_VAL", "PTOTVAL") +COORDINATES = ("person_id", "income_year", "A_AGE") +EVIDENCE_COLUMNS = tuple( + f"{name}.{axis}" + for name in CONTRIBUTING_FIELDS + for axis in ("status", "validity", "zero_origin") +) + ( + "PAW_VAL.status", + "PAW_VAL.validity", + "PAW_VAL.zero_origin", + "PAW_VAL.source_encoding_class", +) +ACCOUNTING_COLUMNS = COORDINATES + RESULT_COLUMNS + EVIDENCE_COLUMNS +ACCOUNTING_DTYPES = tuple([" BoundIncomeObservations: + """Admit graph transport only against the independently issued CREATE receipt.""" + try: + _require( + type(payload) is bytes + and len(source.MAGIC) + 4 + 32 + < len(payload) + <= source.INCOME_PAYLOAD_MAX_BYTES, + "BOUND_INCOME_SIZE", + ) + binding = prepared_receipt["income_observations"] + _require( + set(binding) == _BINDING_KEYS + and prepared_receipt["schema"] == "microcosm.us.asec_prepared_receipt.v3" + and prepared_receipt["source_kind"] == "us_asec_prepared_current_money_v3" + and prepared_receipt["release_eligible"] is False + and binding["kind"] == source.ARTIFACT_KIND + and binding["columns"] == list(source.COLUMNS), + "BOUND_INCOME_PREPARATION", + ) + _require(_sha(payload) == binding["payload_sha256"], "BOUND_INCOME_PAYLOAD") + _require( + payload.startswith(source.MAGIC) + and _sha(payload[:-32]) == payload[-32:].hex(), + "BOUND_INCOME_FRAMING", + ) + length = struct.unpack_from("= 0).all()) + and len(np.unique(positions)) == len(positions) + and (len(positions) < 2 or bool((np.diff(positions) > 0).all())) + and _sha(positions.tobytes()) == person_positions_sha256, + "ACCOUNTING_SOURCE_POSITIONS", + ) + _require( + np.array_equal(income_years, income.array("income_year")[positions]), + "ACCOUNTING_SOURCE_COHORTS", + ) + _require( + all(selected.entity_of(name) == "person" for name in CONTRIBUTING_FIELDS), + "ACCOUNTING_FIELD_ENTITY", + ) + w = selected.field("WSAL_VAL").amounts + e = selected.field("SEMP_VAL").amounts + selected.field("FRSE_VAL").amounts + s = selected.field("SSI_VAL").amounts + p = income.array("PAW_VAL_2024_price")[positions] + t = selected.field("PTOTVAL").amounts + u = t - (((w + e) + s) + p) + results = (w, e, s, p, t, u) + _require(all(np.isfinite(a).all() for a in results), "ACCOUNTING_NONFINITE") + age = income.array("A_AGE")[positions] + nominal, yn = income.array("PAW_VAL")[positions], income.array("PAW_YN")[positions] + buffers = [ + a.astype(" KernelResult: + from . import graph_asec_prepared as bridge + + bridge._phase(context) + node = context.node + _require( + node.kernel == self.ref + and node.structural is StructuralDelta.NONE + and not node.sources, + "ACCOUNTING_NODE", + ) + _require( + node.inputs == (Slice("person", ("source_year",)),) + and node.outputs == reported_income_declarations(), + "ACCOUNTING_DECLARATIONS", + ) + _require( + set(context.artifacts) + == { + "frame_context", + "selection", + "selected_current_money", + "prepared_receipt", + "income_observations", + }, + "ACCOUNTING_ARTIFACT_INPUTS", + ) + _require( + node.artifact_outputs + == ( + ArtifactOutput("reported_income", US_ASEC_REPORTED_INCOME_TYPE), + ArtifactOutput("frame_context", US_FRAME_CONTEXT_TYPE), + ), + "ACCOUNTING_ARTIFACT_OUTPUTS", + ) + values = { + name: bridge._artifact(context, name, kind) + for name, kind in ( + ("frame_context", US_FRAME_CONTEXT_TYPE), + ("selection", US_ASEC_SELECTION_TYPE), + ("selected_current_money", US_ASEC_SELECTED_MONEY_TYPE), + ("prepared_receipt", US_ASEC_PREPARED_RECEIPT_TYPE), + ("income_observations", US_ASEC_INCOME_OBSERVATIONS_TYPE), + ) + } + selection_producer = bridge._same_producer( + context, ("frame_context", "selection", "selected_current_money") + ) + source_producer = bridge._same_producer( + context, ("prepared_receipt", "income_observations") + ) + document = bridge._document(values["frame_context"]) + person = context.tables["person"] + bridge._check_identity(document, "person", person) + _require( + not set(RESULT_COLUMNS) & set(document["entities"]["person"]["columns"]), + "OWNED_LEAF_INCUMBENT", + ) + prepared = _parse(values["prepared_receipt"].payload) + selection = _parse(values["selection"].payload) + _require( + canonical_json(prepared) == values["prepared_receipt"].payload + and canonical_json(selection) == values["selection"].payload + and selection["schema"] == bridge.SELECTION_SCHEMA, + "ACCOUNTING_RECEIPTS", + ) + _require( + selection["parent"]["producer_key"] == source_producer + and selection["parent"]["prepared_receipt_sha256"] + == _sha(values["prepared_receipt"].payload), + "ACCOUNTING_SELECTION_PARENT", + ) + for entity, declared in document["entities"].items(): + _require( + all( + selection["entities"][entity][key] == declared[key] + for key in ("rows", "ordered_ids_sha256") + ), + "ACCOUNTING_SELECTION_IDENTITY", + ) + selected = decode_selected_current_money( + values["selected_current_money"].payload, + expected_parent_header_sha256=prepared["money_header_sha256"], + expected_parent_content_sha256=prepared["money_content_sha256"], + expected_prepared_receipt_sha256=_sha(values["prepared_receipt"].payload), + expected_selection_sha256=_sha(values["selection"].payload), + ) + _require( + selected.header_data["person_identity_sha256"] + == document["entities"]["person"]["ordered_ids_sha256"] + and selected.header_data["household_identity_sha256"] + == document["entities"]["household"]["ordered_ids_sha256"], + "ACCOUNTING_MONEY_COORDINATES", + ) + income = bind_income_observations( + values["income_observations"].payload, prepared_receipt=prepared + ) + _require( + person.source_year.dtype == np.dtype("int64"), + "ACCOUNTING_SOURCE_YEAR_DTYPE", + ) + result = derive_reported_income( + selected, + income, + person_ids=person.person_id.to_numpy(), + income_years=person.source_year.to_numpy(), + person_positions_sha256=selection["person_positions_sha256"], + prepared_receipt_sha256=_sha(values["prepared_receipt"].payload), + selection_sha256=_sha(values["selection"].payload), + selected_money_sha256=_sha(values["selected_current_money"].payload), + income_payload_sha256=_sha(values["income_observations"].payload), + source_producer_key=source_producer, + selection_producer_key=selection_producer, + frame_context_sha256=_sha(values["frame_context"].payload), + ) + index = pd.Index(person.person_id.to_numpy(), name="person_id") + columns = { + ("person", name): pd.Series( + result.array(name), index=index, dtype="float64" + ) + for name in RESULT_COLUMNS + } + document["entities"]["person"]["columns"].extend(RESULT_COLUMNS) + payload = encode_reported_income(result) + return KernelResult( + columns=columns, + artifacts={ + "reported_income": payload, + "frame_context": canonical_json(document), + }, + receipt={ + "phase": bridge.ASEC_PREPARED_PHASE, + "implementation": implementation_manifest(STAGE), + "accounting_sha256": _sha(payload), + "arithmetic_order": ARITHMETIC_ORDER, + "person_rows": len(person), + "release_eligible": False, + }, + ) diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/graph_asec_prepared.py b/packages/microcosm-build/src/microcosm/build/us_runtime/graph_asec_prepared.py new file mode 100644 index 000000000..db6a84363 --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/graph_asec_prepared.py @@ -0,0 +1,1136 @@ +"""The five-node ASEC slice: source, selection, leaves, engine and reported income. + +One directory source holds the reviewed restoration inputs. The CREATE kernel +prepares the whole authenticated population from it; a FILTER kernel draws a +seeded whole-household engineering sample and slices the money body to exactly +the selected coordinates; a pure kernel derives the corrected monetary CPS +leaves from that slice alone; and a final kernel runs the real PolicyEngine-US +adapter for the four outputs whose complete input closure this slice produces. + +Two boundaries are load-bearing. The authenticated ``ReadyCurrentMoney`` never +leaves the CREATE process: what travels is its canonical encoded body, typed as +a subset artifact that no ``type(x) is ReadyCurrentMoney`` check accepts. And +engine outputs are formula-owned, so they are retained in an evaluation +artifact and never written back as population cells. + +Nothing here is a release, a calibration, a representative sample, or a tax or +benefit score. Every receipt says so. +""" + +from __future__ import annotations + +import json +from collections.abc import Sequence + +import numpy as np +import pandas as pd + +from microcosm.build.frame_sampling import EXACT_COUNT_RULE, sample_frame_households +from microcosm.frame import US_SCHEMA, EntitySchema, Frame, Weights +from microcosm.frame.adapters.policyengine_us import ( + PolicyEngineUSEngine, + PolicyEngineUSVariableMetadataIndex, +) +from microcosm.graph import ( + ArtifactInput, + ArtifactOutput, + Capabilities, + Determinism, + Graph, + KernelBase, + KernelContext, + KernelRegistry, + KernelResult, + Node, + Numeric, + Owned, + SeedSource, + Slice, + SourceRef, + StructuralDelta, +) +from microcosm.graph.canonical import canonical_json + +from . import asec_housing_status_source as housing_source +from . import graph_context +from .asec_current_money import _sha +from .asec_current_money_graph_resources import load_graph_current_money_consumers +from .asec_current_money_selection import ( + US_ASEC_CURRENT_MONEY_BODY_TYPE, + US_ASEC_PREPARED_RECEIPT_TYPE, + US_ASEC_SELECTED_MONEY_TYPE, + US_ASEC_SELECTION_TYPE, + decode_selected_current_money, + encode_selected_current_money, + parse_current_money_body, + select_current_money, +) +from .asec_engine_evaluation import ( + ADMITTED_ROOTS, + BLOCKED_ENGINE_OUTPUTS, + US_ASEC_ENGINE_EVALUATION_TYPE, + admit_engine_outputs, + encode_engine_evaluation, + engine_runtime_identity, + materialize_engine_outputs, +) +from .asec_prepared_source import ( + PREPARED_SOURCE_FILES, + PREPARED_SOURCE_KIND, + prepare_asec_current_money_population, +) +from .cps_carried_current import ( + CPS_CARRIED_CURRENT_PERSON_LEAVES, + CPS_CARRIED_CURRENT_ROUTING_COLUMNS, + CPS_CARRIED_CURRENT_SPM_UNIT_LEAVES, + cps_carried_current_leaf_contract, + derive_cps_carried_current_leaves, +) +from .graph_asec_income import ( + REPORTED_INCOME_NODE, + RESULT_COLUMNS, + US_ASEC_INCOME_OBSERVATIONS_TYPE, + US_ASEC_REPORTED_INCOME_TYPE, + USAsecReportedIncomeKernel, + reported_income_declarations, +) +from .graph_context import US_FRAME_CONTEXT_TYPE +from .graph_housing_universe import ( + HOUSEHOLD_EVIDENCE_COLUMNS, + US_ASEC_HOUSING_UNIVERSE_TYPE, + bind_housing_universe, + verify_graph_housing_rows, +) +from .graph_implementation import ( + STAGE_DEPENDENCIES, + implementation_hash, + implementation_manifest, +) +from .graph_sources import frame_column_declarations + +ASEC_PREPARED_STAGE = "asec_prepared_v3" +ASEC_PREPARED_PHASE = "prepare_asec_current_money_slice" +ASEC_PREPARED_CODEC = "us-asec-prepared-current-money-v3" +ASEC_PREPARED_SOURCE_NAME = "asec_prepared" +ASEC_PREPARED_SOURCE = SourceRef( + ASEC_PREPARED_SOURCE_NAME, + ASEC_PREPARED_CODEC, + "Reviewed ASEC restoration inputs: P, H, T with its receipt, three housing " + "cohort HDFs and three official Census PERSON CSV members.", +) +ASEC_PREPARED_DEPENDENCIES = STAGE_DEPENDENCIES[ASEC_PREPARED_STAGE] + +PREFIX = "asec_prepared" +CREATE_NODE = f"{PREFIX}.create" +SELECT_NODE = f"{PREFIX}.select_households" +LEAVES_NODE = f"{PREFIX}.cps_carried_current" +ENGINE_NODE = f"{PREFIX}.engine_current_money" + +#: The one household column the selection and engine nodes declare. They need +#: the household table and its design weights; the executor exposes an entity's +#: weights only to a node that declares that entity. This is the first column +#: the reviewed housing attachment guarantees, so a changed attachment fails +#: closed here instead of silently reading something else. It is a carrier: it +#: never reaches the corrected leaves and never reaches the engine frame. +HOUSEHOLD_WEIGHT_CARRIER = housing_source.ATTACHED_COLUMNS[0] + +SELECTION_SCHEMA = "microcosm.us.asec_household_selection.v2" +SELECTION_SOURCE_LABEL = "US ASEC prepared current money" +SELECTION_STRATUM_COLUMN = "source_year" +LEAF_DTYPE = "float64" +_ENTITIES = US_SCHEMA.entities +_PERSON_ID = US_SCHEMA.entity_id_column("person") + + +class PreparedGraphError(ValueError): + """A declaration, binding or identity this slice refuses to execute on.""" + + +def _require(condition: bool, reason: str) -> None: + if not condition: + raise PreparedGraphError(reason) + + +def _artifact(context: KernelContext, name: str, expected): + """Return one declared typed artifact, refusing an aliased or retyped edge.""" + value = context.artifacts.get(name) + edges = [edge for edge in context.node.artifact_inputs if edge.name == name] + _require( + value is not None + and len(edges) == 1 + and edges[0].type == expected + and value.type == expected, + f"ARTIFACT_EDGE:{name}", + ) + _require( + isinstance(value.producer_key, str) and len(value.producer_key) == 64, + f"ARTIFACT_PRODUCER:{name}", + ) + return value + + +def _same_producer(context: KernelContext, names: Sequence[str]) -> str: + """Require a set of edges to come from one actual producing node execution.""" + producers = {context.artifacts[name].producer_key for name in names} + edges = {edge.name: edge.producer for edge in context.node.artifact_inputs} + _require(len(producers) == 1, "ARTIFACT_PRODUCER_SPLIT") + _require(len({edges[name] for name in names}) == 1, "ARTIFACT_PRODUCER_SPLIT") + return next(iter(producers)) + + +def _phase(context: KernelContext, extra: tuple[str, ...] = ()) -> None: + _require(set(context.params) == {"phase", *extra}, "NODE_PARAMS") + _require(context.params["phase"] == ASEC_PREPARED_PHASE, "NODE_PHASE") + + +def _document(value) -> dict: + document = graph_context._decode(value.payload) + graph_context._mass_records(document["mass_log"]) + _require(canonical_json(document) == value.payload, "CONTEXT_CANONICAL") + _require(document["metadata"] == {}, "PREPARED_CONTEXT_METADATA") + _require(document["mass_log"] == [], "PREPARED_CONTEXT_MASS_LOG") + _require(document["weight_sources"] == {"household": "design"}, "WEIGHT_AUTHORITY") + return document + + +def _identity(ids: np.ndarray, entity: str) -> dict[str, object]: + """Row identity of an id vector, using the graph context's own digest.""" + column = US_SCHEMA.entity_id_column(entity) + return graph_context._row_identity( + pd.DataFrame({column: np.asarray(ids, dtype="int64")}), entity + ) + + +def _check_identity(document: dict, entity: str, table: pd.DataFrame) -> None: + declared = document["entities"][entity] + actual = graph_context._row_identity(table, entity) + _require( + all(declared[key] == value for key, value in actual.items()), + f"CONTEXT_IDENTITY:{entity}", + ) + _require( + not set(table.columns) - set(declared["columns"]), f"CONTEXT_COLUMNS:{entity}" + ) + + +def _group_ids(person: pd.DataFrame, group: str) -> np.ndarray: + """The exact id inventory a group table holds for these persons. + + ``Frame`` validates that a group table's ids are the sorted distinct values + of the person membership column, and ``Frame.select`` prunes to exactly + that set. Reconstructing the inventory from membership is therefore the + same operation, not an approximation; every reconstruction below is checked + against the producer's typed context before it is used. + """ + membership = person[US_SCHEMA.membership_column(group)].to_numpy(dtype="int64") + return np.unique(membership) + + +def _minimal_frame(person: pd.DataFrame, household: pd.DataFrame, weights, strata): + """A person/household view for the sampler; no group beyond household.""" + columns = [_PERSON_ID, US_SCHEMA.membership_column("household")] + extra = [name for name in (SELECTION_STRATUM_COLUMN,) if name in person] + return Frame( + { + "person": person.loc[:, columns + extra].copy(deep=True), + "household": household.loc[ + :, [US_SCHEMA.entity_id_column("household")] + ].copy(deep=True), + }, + EntitySchema(group_entities=("household",)), + { + "household": Weights( + np.asarray(weights.values, dtype="float64"), weights.kind + ) + }, + strata.copy(deep=True), + ) + + +def _household_strata(person: pd.DataFrame, household_ids: np.ndarray) -> np.ndarray: + """One declared source-year stratum per household row; mixed years refuse.""" + years = person[SELECTION_STRATUM_COLUMN] + _require(pd.api.types.is_integer_dtype(years.dtype), "STRATUM_DTYPE") + _require(not bool(years.isna().any()), "STRATUM_MISSING") + frame = pd.DataFrame( + { + "household": person[US_SCHEMA.membership_column("household")].to_numpy( + dtype="int64" + ), + "year": years.to_numpy(dtype="int64"), + } + ) + distinct = frame.drop_duplicates() + _require(not bool(distinct["household"].duplicated().any()), "MIXED_YEAR_HOUSEHOLD") + lookup = dict( + zip(distinct["household"].tolist(), distinct["year"].tolist(), strict=True) + ) + _require(set(lookup) == set(household_ids.tolist()), "STRATUM_COVERAGE") + return np.asarray( + [str(lookup[int(value)]) for value in household_ids], dtype=object + ) + + +class _PreparedKernel(KernelBase): + """Every kernel of this slice shares one reviewed implementation scope.""" + + def implementation_hash(self) -> str: + return implementation_hash(ASEC_PREPARED_STAGE) + + +class USAsecPreparedCreateKernel(_PreparedKernel): + """Prepare the whole authenticated ASEC population from one source directory.""" + + ref = "us.asec_prepared.create@3" + capabilities = Capabilities( + determinism=Determinism.DETERMINISTIC, + numeric=Numeric.PLATFORM_BITWISE, + structural=StructuralDelta.CREATE, + dependencies=ASEC_PREPARED_DEPENDENCIES, + ) + + def run(self, context: KernelContext) -> KernelResult: + _phase(context) + node = context.node + _require(node.kernel == self.ref, "NODE_KERNEL") + _require(tuple(node.sources) == (ASEC_PREPARED_SOURCE_NAME,), "NODE_SOURCES") + _require(not node.inputs and not node.artifact_inputs, "NODE_INPUTS") + _require( + node.artifact_outputs + == ( + ArtifactOutput("frame_context", US_FRAME_CONTEXT_TYPE), + ArtifactOutput("current_money", US_ASEC_CURRENT_MONEY_BODY_TYPE), + ArtifactOutput("prepared_receipt", US_ASEC_PREPARED_RECEIPT_TYPE), + ArtifactOutput("housing_universe", US_ASEC_HOUSING_UNIVERSE_TYPE), + ArtifactOutput("income_observations", US_ASEC_INCOME_OBSERVATIONS_TYPE), + ), + "NODE_ARTIFACT_OUTPUTS", + ) + prepared = prepare_asec_current_money_population( + context.sources[ASEC_PREPARED_SOURCE_NAME] + ) + _require( + frame_column_declarations(prepared.frame) == node.outputs, + "PREPARED_COLUMN_INVENTORY", + ) + receipt = prepared.receipt + consumers = load_graph_current_money_consumers() + _require(receipt["source_kind"] == PREPARED_SOURCE_KIND, "PREPARED_SOURCE_KIND") + _require(receipt["file_roster"] == list(PREPARED_SOURCE_FILES), "FILE_ROSTER") + return KernelResult( + frame=prepared.frame, + artifacts={ + "frame_context": graph_context.encode_us_frame_context(prepared.frame), + "current_money": prepared.money_payload, + "prepared_receipt": prepared.receipt_payload, + "housing_universe": prepared.housing_universe_payload, + "income_observations": prepared.income_observations_payload, + }, + receipt={ + "phase": ASEC_PREPARED_PHASE, + "implementation": implementation_manifest(ASEC_PREPARED_STAGE), + "prepared": receipt, + "current_money_consumers": consumers, + "release_eligible": False, + }, + ) + + +class USAsecPreparedSelectionKernel(_PreparedKernel): + """Draw the seeded whole-household sample and slice the money body to it.""" + + ref = "us.asec_prepared.select_households@3" + capabilities = Capabilities( + determinism=Determinism.SEEDED, + numeric=Numeric.PLATFORM_BITWISE, + seed_source=SeedSource.PARAM, + structural=StructuralDelta.FILTER, + dependencies=ASEC_PREPARED_DEPENDENCIES, + ) + + def run(self, context: KernelContext) -> KernelResult: + _phase(context, ("fraction", "seed")) + node = context.node + _require(node.kernel == self.ref, "NODE_KERNEL") + _require(not node.outputs and not node.sources, "NODE_OUTPUTS") + _require(node.structural is StructuralDelta.FILTER, "NODE_STRUCTURAL") + _require(node.mass == "declared", "NODE_MASS") + _require( + set(context.tables) == {"person", "household"} + and set(context.artifacts) + == { + "frame_context", + "current_money", + "prepared_receipt", + "housing_universe", + }, + "NODE_INPUTS", + ) + _require( + node.inputs + == ( + Slice("person", (SELECTION_STRATUM_COLUMN,)), + Slice( + "household", (HOUSEHOLD_WEIGHT_CARRIER, *HOUSEHOLD_EVIDENCE_COLUMNS) + ), + ), + "NODE_SLICES", + ) + _require( + node.artifact_outputs + == ( + ArtifactOutput("frame_context", US_FRAME_CONTEXT_TYPE), + ArtifactOutput("selection", US_ASEC_SELECTION_TYPE), + ArtifactOutput("selected_current_money", US_ASEC_SELECTED_MONEY_TYPE), + ), + "NODE_ARTIFACT_OUTPUTS", + ) + bound = _artifact(context, "frame_context", US_FRAME_CONTEXT_TYPE) + money = _artifact(context, "current_money", US_ASEC_CURRENT_MONEY_BODY_TYPE) + receipt_value = _artifact( + context, "prepared_receipt", US_ASEC_PREPARED_RECEIPT_TYPE + ) + producer_key = _same_producer( + context, + ("frame_context", "current_money", "prepared_receipt", "housing_universe"), + ) + document = _document(bound) + person, household = context.tables["person"], context.tables["household"] + for entity, table in (("person", person), ("household", household)): + _check_identity(document, entity, table) + _require( + document["weight_sources"] == {"household": "design"}, "WEIGHT_AUTHORITY" + ) + _require( + context.weights["household"].kind.value == "design", "WEIGHT_AUTHORITY" + ) + prepared = json.loads(receipt_value.payload) + _require( + canonical_json(prepared) == receipt_value.payload, "PREPARED_RECEIPT_JSON" + ) + _require( + prepared["source_kind"] == PREPARED_SOURCE_KIND + and prepared["release_eligible"] is False, + "PREPARED_RECEIPT_SCHEMA", + ) + _require( + prepared["entity_rows"]["person"] == document["entities"]["person"]["rows"] + and prepared["entity_rows"]["household"] + == document["entities"]["household"]["rows"], + "PREPARED_RECEIPT_ROWS", + ) + body = parse_current_money_body( + money.payload, + expected_header_sha256=prepared["money_header_sha256"], + expected_content_sha256=prepared["money_content_sha256"], + field_entities=tuple(tuple(item) for item in prepared["field_entities"]), + ) + _require( + body.person_rows == document["entities"]["person"]["rows"] + and body.household_rows == document["entities"]["household"]["rows"], + "BODY_ROW_ALIGNMENT", + ) + household_ids = household[US_SCHEMA.entity_id_column("household")].to_numpy( + dtype="int64" + ) + strata = _household_strata(person, household_ids) + universe_value = _artifact( + context, "housing_universe", US_ASEC_HOUSING_UNIVERSE_TYPE + ) + universe = bind_housing_universe( + universe_value.payload, prepared_receipt=prepared + ) + verify_graph_housing_rows( + household, + universe, + positions=np.arange(len(household), dtype="int64"), + income_years=strata.astype("int64"), + ) + view = _minimal_frame( + person, household, context.weights["household"], context.strata + ) + sampled, selection = sample_frame_households( + view, + fraction=context.params["fraction"], + seed=context.params["seed"], + source_name=SELECTION_SOURCE_LABEL, + unit_strata=strata, + unit_noun="household", + floor_context="the ASEC prepared current-money engineering sample", + ) + selected_household_ids = sampled.table("household")[ + US_SCHEMA.entity_id_column("household") + ].to_numpy(dtype="int64") + membership = person[US_SCHEMA.membership_column("household")].to_numpy( + dtype="int64" + ) + keep = np.isin(membership, selected_household_ids) + person_positions = np.flatnonzero(keep).astype("int64") + household_positions = np.flatnonzero( + np.isin(household_ids, selected_household_ids) + ).astype("int64") + _require(len(person_positions) > 0, "EMPTY_SELECTION") + _require( + np.array_equal(household_ids[household_positions], selected_household_ids), + "HOUSEHOLD_POSITIONS", + ) + # Whole households only: no selected household may leave a member behind. + _require( + np.array_equal( + person[_PERSON_ID].to_numpy(dtype="int64")[keep], + sampled.person[_PERSON_ID].to_numpy(dtype="int64"), + ), + "WHOLE_HOUSEHOLD", + ) + parent_weights = np.asarray( + context.weights["household"].values, dtype="float64" + ) + selected_weights = np.asarray( + sampled.weights_for("household").values, dtype="float64" + ) + _require( + selected_weights.tobytes() == parent_weights[household_positions].tobytes(), + "SELECTED_WEIGHT_BYTES", + ) + entities = {"person": person[_PERSON_ID].to_numpy(dtype="int64")[keep]} + for group in US_SCHEMA.group_entities: + entities[group] = _group_ids(person.loc[keep], group) + _require( + np.array_equal(entities["household"], selected_household_ids), + "SELECTED_HOUSEHOLD_INVENTORY", + ) + identities = { + entity: _identity(ids, entity) for entity, ids in entities.items() + } + before_mass = view.stratum_mass() + after_mass = sampled.stratum_mass() + payload = canonical_json( + { + "schema": SELECTION_SCHEMA, + "phase": ASEC_PREPARED_PHASE, + "release_eligible": False, + "representative": False, + "claim": "engineering_sample_only_no_district_or_calibration_claim", + "stratum_column": SELECTION_STRATUM_COLUMN, + "stratum_grain": "household_source_year", + "exact_count_rule": EXACT_COUNT_RULE, + "mass_normalization": "none", + "selection": _json(selection), + "entities": { + entity: { + "rows": identities[entity]["rows"], + "ordered_ids_sha256": identities[entity]["ordered_ids_sha256"], + } + for entity in _ENTITIES + }, + "person_positions_sha256": _sha(person_positions.tobytes()), + "household_positions_sha256": _sha(household_positions.tobytes()), + "parent": { + "producer_key": producer_key, + "frame_context_sha256": _sha(bound.payload), + "prepared_receipt_sha256": _sha(receipt_value.payload), + "money_header_sha256": prepared["money_header_sha256"], + "money_content_sha256": prepared["money_content_sha256"], + "housing_universe_payload_sha256": _sha(universe_value.payload), + }, + "mass": { + "policy": "declared", + "before": float(before_mass.sum()), + "after": float(after_mass.sum()), + }, + } + ) + selected = select_current_money( + body, + person_positions=person_positions, + household_positions=household_positions, + prepared_receipt_sha256=_sha(receipt_value.payload), + selection_sha256=_sha(payload), + person_identity_sha256=identities["person"]["ordered_ids_sha256"], + household_identity_sha256=identities["household"]["ordered_ids_sha256"], + ) + for entity in _ENTITIES: + document["entities"][entity].update(identities[entity]) + return KernelResult( + keep=pd.Series( + keep, + index=pd.Index(person[_PERSON_ID].to_numpy(), name=_PERSON_ID), + dtype=bool, + ), + artifacts={ + "frame_context": canonical_json(document), + "selection": payload, + "selected_current_money": encode_selected_current_money(selected), + }, + receipt={ + "phase": ASEC_PREPARED_PHASE, + "implementation": implementation_manifest(ASEC_PREPARED_STAGE), + "selection_sha256": _sha(payload), + "selected_money_sha256": _sha(encode_selected_current_money(selected)), + "release_eligible": False, + "mass": { + "policy": "declared", + "before": float(before_mass.sum()), + "after": float(after_mass.sum()), + "stratum_before": { + str(key): float(value) for key, value in before_mass.items() + }, + "stratum_after": { + str(key): float(value) for key, value in after_mass.items() + }, + }, + }, + ) + + +def _json(value: object) -> object: + """Normalize a receipt for canonical JSON without inventing string casts.""" + return graph_context._json_data(value) + + +class USAsecPreparedLeavesKernel(_PreparedKernel): + """Derive the corrected monetary CPS leaves from the selected money alone.""" + + ref = "us.asec_prepared.cps_carried_current@3" + capabilities = Capabilities( + determinism=Determinism.DETERMINISTIC, + numeric=Numeric.PLATFORM_BITWISE, + dependencies=ASEC_PREPARED_DEPENDENCIES, + ) + + def run(self, context: KernelContext) -> KernelResult: + _phase(context) + node = context.node + _require(node.kernel == self.ref, "NODE_KERNEL") + _require(not node.sources, "NODE_SOURCES") + _require(node.structural is StructuralDelta.NONE, "NODE_STRUCTURAL") + _require( + node.inputs + == ( + Slice("person", CPS_CARRIED_CURRENT_ROUTING_COLUMNS), + Slice("household", HOUSEHOLD_EVIDENCE_COLUMNS), + ), + "NODE_SLICES", + ) + _require(node.outputs == owned_leaf_declarations(), "NODE_OUTPUTS") + _require( + set(context.artifacts) + == { + "frame_context", + "selection", + "selected_current_money", + "prepared_receipt", + "housing_universe", + }, + "NODE_ARTIFACT_INPUTS", + ) + _require( + node.artifact_outputs + == (ArtifactOutput("frame_context", US_FRAME_CONTEXT_TYPE),), + "NODE_ARTIFACT_OUTPUTS", + ) + bound = _artifact(context, "frame_context", US_FRAME_CONTEXT_TYPE) + selection_value = _artifact(context, "selection", US_ASEC_SELECTION_TYPE) + money = _artifact( + context, "selected_current_money", US_ASEC_SELECTED_MONEY_TYPE + ) + receipt_value = _artifact( + context, "prepared_receipt", US_ASEC_PREPARED_RECEIPT_TYPE + ) + _same_producer( + context, ("frame_context", "selection", "selected_current_money") + ) + universe_value = _artifact( + context, "housing_universe", US_ASEC_HOUSING_UNIVERSE_TYPE + ) + _same_producer(context, ("prepared_receipt", "housing_universe")) + document = _document(bound) + person, spm_unit = context.tables["person"], context.tables["spm_unit"] + for entity, table in (("person", person), ("spm_unit", spm_unit)): + _check_identity(document, entity, table) + selection = json.loads(selection_value.payload) + _require(canonical_json(selection) == selection_value.payload, "SELECTION_JSON") + _require(selection["schema"] == SELECTION_SCHEMA, "SELECTION_SCHEMA") + prepared = json.loads(receipt_value.payload) + _require( + selection["parent"]["prepared_receipt_sha256"] + == _sha(receipt_value.payload) + and selection["parent"]["producer_key"] == receipt_value.producer_key, + "SELECTION_PARENT_BINDING", + ) + _require( + selection["parent"]["housing_universe_payload_sha256"] + == _sha(universe_value.payload), + "SELECTION_HOUSING_UNIVERSE_BINDING", + ) + universe = bind_housing_universe( + universe_value.payload, prepared_receipt=prepared + ) + household = context.tables["household"] + _check_identity(document, "household", household) + positions = ( + pd.Index(universe.array("household_id")) + .get_indexer(household.household_id.to_numpy()) + .astype("int64") + ) + _require( + _sha(positions.tobytes()) == selection["household_positions_sha256"], + "SELECTED_HOUSING_UNIVERSE_POSITIONS", + ) + verify_graph_housing_rows(household, universe, positions=positions) + # The complete selected-coordinate check: the selection artifact, the + # producer's typed context and the actual rows must name one row set. + for entity in _ENTITIES: + declared = document["entities"][entity] + _require( + selection["entities"][entity]["rows"] == declared["rows"] + and selection["entities"][entity]["ordered_ids_sha256"] + == declared["ordered_ids_sha256"], + f"SELECTION_IDENTITY:{entity}", + ) + selected = decode_selected_current_money( + money.payload, + expected_parent_header_sha256=prepared["money_header_sha256"], + expected_parent_content_sha256=prepared["money_content_sha256"], + expected_prepared_receipt_sha256=_sha(receipt_value.payload), + expected_selection_sha256=_sha(selection_value.payload), + ) + header = selected.header_data + _require( + header["person_identity_sha256"] + == document["entities"]["person"]["ordered_ids_sha256"] + and header["household_identity_sha256"] + == document["entities"]["household"]["ordered_ids_sha256"], + "SELECTED_COORDINATES", + ) + _require( + selected.person_rows == len(person) + and selected.household_rows == document["entities"]["household"]["rows"], + "SELECTED_ROW_ALIGNMENT", + ) + incumbents = sorted( + f"{entity}.{column}" + for entity, column in _owned_coordinates() + if column in document["entities"][entity]["columns"] + ) + _require(not incumbents, "OWNED_LEAF_INCUMBENT") + routing = person.loc[:, list(CPS_CARRIED_CURRENT_ROUTING_COLUMNS)] + spm_ids = spm_unit[US_SCHEMA.entity_id_column("spm_unit")].to_numpy( + dtype="int64" + ) + leaves = derive_cps_carried_current_leaves( + selected, + routing=routing, + spm_membership=person[US_SCHEMA.membership_column("spm_unit")].to_numpy( + dtype="int64" + ), + spm_ids=spm_ids, + ) + person_index = pd.Index(person[_PERSON_ID].to_numpy(), name=_PERSON_ID) + spm_index = pd.Index(spm_ids, name=US_SCHEMA.entity_id_column("spm_unit")) + columns = { + ("person", name): pd.Series(values, index=person_index, dtype=LEAF_DTYPE) + for name, values in leaves.person.items() + } + columns.update( + { + ("spm_unit", name): pd.Series(values, index=spm_index, dtype=LEAF_DTYPE) + for name, values in leaves.spm_unit.items() + } + ) + for entity, column in _owned_coordinates(): + document["entities"][entity]["columns"].append(column) + return KernelResult( + columns=columns, + artifacts={"frame_context": canonical_json(document)}, + receipt={ + "phase": ASEC_PREPARED_PHASE, + "implementation": implementation_manifest(ASEC_PREPARED_STAGE), + "contract": cps_carried_current_leaf_contract(), + "selected_money_sha256": _sha(money.payload), + "selection_sha256": _sha(selection_value.payload), + "person_rows": len(person), + "spm_unit_rows": len(spm_ids), + "release_eligible": False, + }, + ) + + +class USAsecPreparedEngineKernel(_PreparedKernel): + """Run the real PolicyEngine-US adapter for the admitted formula-owned roots.""" + + ref = "us.asec_prepared.engine_current_money@3" + capabilities = Capabilities( + determinism=Determinism.DETERMINISTIC, + numeric=Numeric.PLATFORM_BITWISE, + dependencies=( + *ASEC_PREPARED_DEPENDENCIES, + "policyengine-us", + "policyengine-core", + ), + ) + + def implementation_hash(self) -> str: + # The executor checks this identity before cache lookup. Runtime or + # baseline parameter drift therefore refuses even when run() is unused. + return _sha( + canonical_json( + { + "implementation": super().implementation_hash(), + "runtime": engine_runtime_identity(), + } + ) + ) + + def __init__(self, engine=None) -> None: + # Only a real adapter instance is accepted; a substitute would make the + # evaluation artifact a claim about something other than the engine. + if engine is not None and type(engine) is not PolicyEngineUSEngine: + raise PreparedGraphError("ENGINE_ADAPTER_TYPE") + self._engine = engine + + def run(self, context: KernelContext) -> KernelResult: + _phase(context, ("period", "engine_version")) + node = context.node + _require(node.kernel == self.ref, "NODE_KERNEL") + _require(not node.sources, "NODE_SOURCES") + _require(node.structural is StructuralDelta.NONE, "NODE_STRUCTURAL") + # Engine outputs are formula-owned; this node owns no population cell. + _require(not node.outputs, "ENGINE_OWNS_NO_CELLS") + _require( + node.inputs + == ( + Slice("person", CPS_CARRIED_CURRENT_PERSON_LEAVES), + Slice("spm_unit", CPS_CARRIED_CURRENT_SPM_UNIT_LEAVES), + Slice("household", (HOUSEHOLD_WEIGHT_CARRIER,)), + ), + "NODE_SLICES", + ) + _require(set(context.artifacts) == {"frame_context"}, "NODE_ARTIFACT_INPUTS") + _require( + node.artifact_outputs + == (ArtifactOutput("engine_evaluation", US_ASEC_ENGINE_EVALUATION_TYPE),), + "NODE_ARTIFACT_OUTPUTS", + ) + bound = _artifact(context, "frame_context", US_FRAME_CONTEXT_TYPE) + document = _document(bound) + for entity in ("person", "spm_unit", "household"): + _check_identity(document, entity, context.tables[entity]) + for entity, column in _owned_coordinates(): + _require( + column in document["entities"][entity]["columns"], + f"LEAF_NOT_IN_CONTEXT:{entity}.{column}", + ) + frame = self._engine_frame(context, document) + index = PolicyEngineUSVariableMetadataIndex() + contracts = admit_engine_outputs( + index, produced_leaves=CPS_CARRIED_CURRENT_PERSON_LEAVES + ) + _require( + all( + contract.engine_version == context.params["engine_version"] + for contract in contracts + ), + "DECLARED_ENGINE_VERSION", + ) + evaluation = materialize_engine_outputs( + frame, + PolicyEngineUSEngine() if self._engine is None else self._engine, + contracts, + period=context.params["period"], + context_bindings={ + "frame_context_sha256": _sha(bound.payload), + "frame_context_producer_key": bound.producer_key, + "produced_leaves": list(CPS_CARRIED_CURRENT_PERSON_LEAVES), + "spm_unit_leaves": list(CPS_CARRIED_CURRENT_SPM_UNIT_LEAVES), + }, + ) + payload = encode_engine_evaluation(evaluation) + return KernelResult( + artifacts={"engine_evaluation": payload}, + receipt={ + "phase": ASEC_PREPARED_PHASE, + "implementation": implementation_manifest(ASEC_PREPARED_STAGE), + "engine": { + "package": evaluation.header["engine_package"], + "version": evaluation.header["engine_version"], + "period": evaluation.header["period"], + "roots": list(ADMITTED_ROOTS), + "blocked_roots": list(BLOCKED_ENGINE_OUTPUTS), + "closures": evaluation.header["closures"], + "recorded_defaults": [], + "aggregates": evaluation.header["aggregates"], + }, + "evaluation_sha256": _sha(payload), + "cells_written": 0, + "release_eligible": False, + "claim": "engineering_evidence_only_no_tax_or_benefit_score", + }, + ) + + def _engine_frame(self, context: KernelContext, document: dict) -> Frame: + """Rebuild the selected frame carrying only leaves and id carriers. + + Every raw ASEC column, every routing code and the household carrier are + left behind: what the engine receives is exactly the corrected leaves + plus the structure they are indexed by. + """ + person = context.tables["person"] + keep = [ + _PERSON_ID, + *(US_SCHEMA.membership_column(group) for group in US_SCHEMA.group_entities), + *CPS_CARRIED_CURRENT_PERSON_LEAVES, + ] + _require(set(person.columns) == set(keep), "ENGINE_PERSON_COLUMNS") + tables = {"person": person.loc[:, keep].copy(deep=True)} + for group in US_SCHEMA.group_entities: + ids = _group_ids(person, group) + declared = document["entities"][group] + actual = _identity(ids, group) + _require( + all(declared[key] == value for key, value in actual.items()), + f"ENGINE_GROUP_IDENTITY:{group}", + ) + tables[group] = pd.DataFrame({US_SCHEMA.entity_id_column(group): ids}) + for group in ("household", "spm_unit"): + view = context.tables[group] + _require( + np.array_equal( + view[US_SCHEMA.entity_id_column(group)].to_numpy(dtype="int64"), + tables[group][US_SCHEMA.entity_id_column(group)].to_numpy(), + ), + f"ENGINE_GROUP_VIEW:{group}", + ) + spm = context.tables["spm_unit"] + for name in CPS_CARRIED_CURRENT_SPM_UNIT_LEAVES: + tables["spm_unit"][name] = spm[name].to_numpy(dtype="float64", copy=True) + weights = context.weights["household"] + _require(weights.kind.value == "design", "ENGINE_WEIGHT_AUTHORITY") + return Frame( + tables, + US_SCHEMA, + { + "household": Weights( + np.asarray(weights.values, dtype="float64"), weights.kind + ) + }, + ) + + +def _owned_coordinates() -> tuple[tuple[str, str], ...]: + return ( + *(("person", name) for name in CPS_CARRIED_CURRENT_PERSON_LEAVES), + *(("spm_unit", name) for name in CPS_CARRIED_CURRENT_SPM_UNIT_LEAVES), + ) + + +def owned_leaf_declarations() -> tuple[Owned, ...]: + """The cells the corrected-leaves node owns, in one canonical order.""" + return tuple( + Owned(entity, column, LEAF_DTYPE) for entity, column in _owned_coordinates() + ) + + +def us_asec_prepared_graph( + columns: Sequence[Owned], + *, + fraction: float, + seed: int, + period: int = 2024, + engine_version: str = "1.819.0", +) -> Graph: + """Declare the five-node slice over the caller's prepared column inventory.""" + inventory = {(owned.entity, owned.column) for owned in columns} + _require(len(inventory) == len(tuple(columns)), "COLUMN_INVENTORY_REPEATS") + _require( + ("household", HOUSEHOLD_WEIGHT_CARRIER) in inventory, "MISSING_WEIGHT_CARRIER" + ) + missing = sorted( + name + for name in (SELECTION_STRATUM_COLUMN, *CPS_CARRIED_CURRENT_ROUTING_COLUMNS) + if ("person", name) not in inventory + ) + _require(not missing, f"MISSING_DECLARED_INPUTS:{missing}") + _require( + all(("household", name) in inventory for name in HOUSEHOLD_EVIDENCE_COLUMNS), + "MISSING_HOUSING_UNIVERSE_INPUTS", + ) + # A nominal incumbent can be neither preserved nor silently rewritten. + incumbent = sorted( + f"{entity}.{column}" + for entity, column in _owned_coordinates() + if (entity, column) in inventory + ) + incumbent.extend( + f"person.{name}" for name in RESULT_COLUMNS if ("person", name) in inventory + ) + _require(not incumbent, f"OWNED_LEAF_INCUMBENT:{incumbent}") + parent_artifacts = ( + ArtifactInput( + "frame_context", CREATE_NODE, "frame_context", US_FRAME_CONTEXT_TYPE + ), + ArtifactInput( + "current_money", + CREATE_NODE, + "current_money", + US_ASEC_CURRENT_MONEY_BODY_TYPE, + ), + ArtifactInput( + "prepared_receipt", + CREATE_NODE, + "prepared_receipt", + US_ASEC_PREPARED_RECEIPT_TYPE, + ), + ArtifactInput( + "housing_universe", + CREATE_NODE, + "housing_universe", + US_ASEC_HOUSING_UNIVERSE_TYPE, + ), + ) + create = Node( + id=CREATE_NODE, + kernel=USAsecPreparedCreateKernel.ref, + structural=StructuralDelta.CREATE, + sources=(ASEC_PREPARED_SOURCE_NAME,), + outputs=tuple(columns), + params={"phase": ASEC_PREPARED_PHASE}, + artifact_outputs=( + ArtifactOutput("frame_context", US_FRAME_CONTEXT_TYPE), + ArtifactOutput("current_money", US_ASEC_CURRENT_MONEY_BODY_TYPE), + ArtifactOutput("prepared_receipt", US_ASEC_PREPARED_RECEIPT_TYPE), + ArtifactOutput("housing_universe", US_ASEC_HOUSING_UNIVERSE_TYPE), + ArtifactOutput("income_observations", US_ASEC_INCOME_OBSERVATIONS_TYPE), + ), + ) + select = Node( + id=SELECT_NODE, + kernel=USAsecPreparedSelectionKernel.ref, + base=CREATE_NODE, + structural=StructuralDelta.FILTER, + mass="declared", + inputs=( + Slice("person", (SELECTION_STRATUM_COLUMN,)), + Slice("household", (HOUSEHOLD_WEIGHT_CARRIER, *HOUSEHOLD_EVIDENCE_COLUMNS)), + ), + params={"phase": ASEC_PREPARED_PHASE, "fraction": fraction, "seed": seed}, + artifact_inputs=parent_artifacts, + artifact_outputs=( + ArtifactOutput("frame_context", US_FRAME_CONTEXT_TYPE), + ArtifactOutput("selection", US_ASEC_SELECTION_TYPE), + ArtifactOutput("selected_current_money", US_ASEC_SELECTED_MONEY_TYPE), + ), + ) + leaves = Node( + id=LEAVES_NODE, + kernel=USAsecPreparedLeavesKernel.ref, + population=SELECT_NODE, + inputs=( + Slice("person", CPS_CARRIED_CURRENT_ROUTING_COLUMNS), + Slice("household", HOUSEHOLD_EVIDENCE_COLUMNS), + ), + outputs=owned_leaf_declarations(), + params={"phase": ASEC_PREPARED_PHASE}, + artifact_inputs=( + ArtifactInput( + "frame_context", SELECT_NODE, "frame_context", US_FRAME_CONTEXT_TYPE + ), + ArtifactInput( + "selection", SELECT_NODE, "selection", US_ASEC_SELECTION_TYPE + ), + ArtifactInput( + "selected_current_money", + SELECT_NODE, + "selected_current_money", + US_ASEC_SELECTED_MONEY_TYPE, + ), + parent_artifacts[2], + parent_artifacts[3], + ), + artifact_outputs=(ArtifactOutput("frame_context", US_FRAME_CONTEXT_TYPE),), + ) + engine = Node( + id=ENGINE_NODE, + kernel=USAsecPreparedEngineKernel.ref, + population=SELECT_NODE, + inputs=( + Slice("person", CPS_CARRIED_CURRENT_PERSON_LEAVES), + Slice("spm_unit", CPS_CARRIED_CURRENT_SPM_UNIT_LEAVES), + Slice("household", (HOUSEHOLD_WEIGHT_CARRIER,)), + ), + params={ + "phase": ASEC_PREPARED_PHASE, + "period": period, + "engine_version": engine_version, + }, + artifact_inputs=( + ArtifactInput( + "frame_context", LEAVES_NODE, "frame_context", US_FRAME_CONTEXT_TYPE + ), + ), + artifact_outputs=( + ArtifactOutput("engine_evaluation", US_ASEC_ENGINE_EVALUATION_TYPE), + ), + ) + reported = Node( + id=REPORTED_INCOME_NODE, + kernel=USAsecReportedIncomeKernel.ref, + population=SELECT_NODE, + inputs=(Slice("person", ("source_year",)),), + outputs=reported_income_declarations(), + params={"phase": ASEC_PREPARED_PHASE}, + artifact_inputs=( + *leaves.artifact_inputs[:3], + parent_artifacts[2], + ArtifactInput( + "income_observations", + CREATE_NODE, + "income_observations", + US_ASEC_INCOME_OBSERVATIONS_TYPE, + ), + ), + artifact_outputs=( + ArtifactOutput("reported_income", US_ASEC_REPORTED_INCOME_TYPE), + ArtifactOutput("frame_context", US_FRAME_CONTEXT_TYPE), + ), + ) + return Graph( + country="us", + sources=(ASEC_PREPARED_SOURCE,), + nodes=(create, select, leaves, engine, reported), + ) + + +def us_asec_prepared_registry(engine=None) -> KernelRegistry: + """Register exactly this slice's five kernels.""" + registry = KernelRegistry() + registry.register(USAsecPreparedCreateKernel()) + registry.register(USAsecPreparedSelectionKernel()) + registry.register(USAsecPreparedLeavesKernel()) + registry.register(USAsecPreparedEngineKernel(engine)) + registry.register(USAsecReportedIncomeKernel()) + return registry + + +__all__ = [ + "ASEC_PREPARED_CODEC", + "ASEC_PREPARED_DEPENDENCIES", + "ASEC_PREPARED_PHASE", + "ASEC_PREPARED_SOURCE", + "ASEC_PREPARED_SOURCE_NAME", + "ASEC_PREPARED_STAGE", + "CREATE_NODE", + "ENGINE_NODE", + "HOUSEHOLD_WEIGHT_CARRIER", + "LEAVES_NODE", + "PreparedGraphError", + "SELECTION_SCHEMA", + "SELECT_NODE", + "USAsecPreparedCreateKernel", + "USAsecPreparedEngineKernel", + "USAsecPreparedLeavesKernel", + "USAsecPreparedSelectionKernel", + "owned_leaf_declarations", + "us_asec_prepared_graph", + "us_asec_prepared_registry", +] diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/graph_atomic_survey_clone.py b/packages/microcosm-build/src/microcosm/build/us_runtime/graph_atomic_survey_clone.py new file mode 100644 index 000000000..f65c71236 --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/graph_atomic_survey_clone.py @@ -0,0 +1,56 @@ +"""Assign shared geography after the complete combined-survey support clone. + +This declaration has no country-specific geography kernel or source admission. +Callers provide normalized, qualified observed columns and declared support. +""" + +from microcosm.build.atomic_geography import validate_assignment_spec +from microcosm.build.graph_atomic_geography import atomic_geography_nodes + +from .graph_combined_clone import us_combined_survey_clone_nodes + +ASSIGNMENT_IDENTITY = ( + "survey_geography_origin_key", + "household_support_clone_index", +) + + +def atomic_survey_clone_nodes( + definition, + columns, + *, + base, + geography_prefix="geography", + clone_prefix="combined_survey_puf_support_clone", +): + """Complete initial clones, then draw on each stable source/role identity. + + The base carries qualified observed constraints, not assigned geography. + The clone-index Slice depends on its ownership claim; the population edge + depends on EXPAND. Numeric remapped household IDs are only row coordinates. + """ + columns = tuple(columns) + definition = validate_assignment_spec(definition) + if tuple(definition.get("identity", ())) != ASSIGNMENT_IDENTITY: + raise ValueError( + "US postclone geography requires its stable source/clone identity." + ) + clones = us_combined_survey_clone_nodes( + columns, + base=base, + source_channels=("acs", "asec"), + prefix=clone_prefix, + ) + inventory = {(o.entity, o.column): o for o in columns} + inventory.update({(o.entity, o.column): o for o in clones[1].outputs}) + origin = inventory.get(("household", ASSIGNMENT_IDENTITY[0])) + if origin is None or origin.dtype != "string": + raise ValueError("US postclone geography requires a string source identity.") + geography = atomic_geography_nodes( + definition, + tuple(inventory.values()), + base=clones[0].id, + prefix=geography_prefix, + emit_validation_artifact=True, + ) + return (*clones, *geography) diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/graph_atomic_survey_financial.py b/packages/microcosm-build/src/microcosm/build/us_runtime/graph_atomic_survey_financial.py new file mode 100644 index 000000000..d89986093 --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/graph_atomic_survey_financial.py @@ -0,0 +1,893 @@ +"""Checked atomic survey geography plus current financial development output. + +This nineteen-node composition retains the raw allocation and the pre-financial +clone separately. It grants neither a budget successor nor PUF recipient or +release authority. Support bytes establish integrity, not publisher provenance. +""" + +from __future__ import annotations + +import json +import sys +import weakref +from dataclasses import dataclass, replace +from types import FunctionType, SimpleNamespace + +from microcosm.fit import qrf_target +from microcosm.fit.graph_legacy_apply_matrix import LegacyQRFApplyMatrixKernel +from microcosm.fit.graph_legacy_train import LegacyQRFTrainKernel +from microcosm.graph import KernelResult, compile_graph, run_graph +from microcosm.graph import population as population_ops +from microcosm.graph.artifact_edges import typed_contracts +from microcosm.graph.executor import _all_node_keys, _source_paths_and_keys +from microcosm.graph.keys import _capabilities_projection +from microcosm.graph.population import Population +from microcosm.graph.serialize import graph_to_json + +from . import graph_atomic_survey_population as atomic +from . import graph_current_survey_predictors as financial + +values = financial.values +codec = financial.codec +survey = atomic.survey +reconstruction = atomic.reconstruction +require = values.require +RUN_PROTOCOL = "microcosm.us.atomic-survey-financial-run.v1" +_ISSUED_RUNS = {} + + +@dataclass(frozen=True) +class AtomicSurveyFinancialRunValues: + """Development output, retaining the independently admitted prefix values.""" + + prefix: atomic.AtomicSurveyPopulationRunValues + financial_population: Population + manifest: object + compiled: object + store: object + kernels: object + sources: dict + projection: bytes + matrix: bytes + + def checked_view(self): + """Recheck a completed actual run; public dataclass copies are unissued.""" + return check_atomic_survey_financial_run(self) + + +@dataclass(frozen=True) +class CheckedAtomicSurveyFinancialRun: + """Descriptive values; authority remains in the actual issued run handle.""" + + payload: bytes + digest: str + population: Population + + +@dataclass(frozen=True) +class _FinancialRunState: + prefix: object + prefix_objects: tuple + preparation_entry: tuple + financial_population: Population + populations: tuple + manifest: object + manifest_bytes: bytes + prefix_manifest_bytes: bytes + compiled: object + declaration: str + prefix_declaration: str + store: object + kernels: object + source_items: tuple + config_bytes: bytes + projection: bytes + matrix: bytes + pins: bytes + n_estimators: int + demographic_conditioning: bool + source_keys: tuple + keys: tuple + implementations: tuple + artifact_hashes: tuple + manifest_populations: tuple + prefix_manifest_populations: tuple + live: dict + + +def _manifest_population_seals(manifest, compiled): + """Include transient attached Frames, which portable JSON deliberately omits.""" + return tuple( + ( + version, + reconstruction._population_stamp( + Population.from_frame( + manifest.population(version), + version, + mass_ledger=manifest.mass_ledger(version), + ) + ), + ) + for version in sorted(set(compiled.versions.values())) + ) + + +def _run_entry(run): + entry = _ISSUED_RUNS.get(id(run)) + require( + type(run) is AtomicSurveyFinancialRunValues + and entry is not None + and entry[0]() is run, + "UNISSUED_FINANCIAL_RUN", + ) + return entry + + +def _run_document(run, state): + """Portable ancestry omits timing/cache hits and private physical seals.""" + return codec.encode_json( + { + "protocol": RUN_PROTOCOL, + "graph_sha256": codec.sha(state.declaration.encode()), + "manifest_key": state.manifest.key, + "preparation_sha256": codec.sha(state.preparation_entry[1]), + "geography_config_sha256": codec.sha(state.config_bytes), + "projection_sha256": codec.sha(state.projection), + "matrix_sha256": codec.sha(state.matrix), + "host_edges": codec.decode_json(state.pins), + "node_keys": dict(state.keys), + "artifact_payload_sha256": [list(row) for row in state.artifact_hashes], + "financial_frame_sha256": values.source._frame_identity( + run.financial_population.frame + ), + "financial_version": run.financial_population.version, + "financial_owners": sorted( + (e, c, writer) + for (e, c), writer in run.financial_population.owners.items() + ), + "owned_columns": list(values.OUTPUTS), + "demographic_conditioning": state.demographic_conditioning, + "n_estimators": state.n_estimators, + "release_eligible": False, + } + ) + + +def _pure_run(run, entry): + """Check retained graph/source/Population state after all external I/O.""" + require(_run_entry(run) is entry, "FINAL_FINANCIAL_RUN_ISSUANCE") + state = entry[2] + prefix = state.prefix + require( + run.prefix is prefix + and run.financial_population is state.financial_population + and run.manifest is state.manifest + and run.compiled is state.compiled + and run.store is state.store + and run.kernels is state.kernels + and run.projection == state.projection + and run.matrix == state.matrix + and tuple(sorted(run.sources.items())) == state.source_items + and tuple(sorted(prefix.sources.items())) == state.source_items + and all( + a is b + for a, b in zip( + ( + prefix.preparation, + prefix.manifest, + prefix.compiled, + prefix.store, + prefix.kernels, + prefix.sources, + prefix.geography_config, + ), + state.prefix_objects, + strict=True, + ) + ) + and values.host.survey_budget._config_payload(prefix.geography_config) + == state.config_bytes + and run.manifest.to_json_bytes() == state.manifest_bytes + and prefix.manifest.to_json_bytes() == state.prefix_manifest_bytes + and graph_to_json(run.compiled.graph) == state.declaration + and graph_to_json(prefix.compiled.graph) == state.prefix_declaration + and run.compiled == compile_graph(run.compiled.graph) + and prefix.compiled == compile_graph(prefix.compiled.graph), + "FINANCIAL_RUN_BINDINGS_CHANGED", + ) + require( + _manifest_population_seals(run.manifest, run.compiled) + == state.manifest_populations + and _manifest_population_seals(prefix.manifest, prefix.compiled) + == state.prefix_manifest_populations, + "FINANCIAL_RUN_ATTACHED_POPULATION_CHANGED", + ) + require( + values.source._ISSUED.get(id(prefix.preparation)) is state.preparation_entry + and prefix.preparation.payload == state.preparation_entry[1], + "FINANCIAL_RUN_SOURCE_CHANGED", + ) + values.source._pure_final(state.preparation_entry[2]) + for name, population, stamp in state.populations: + actual = ( + run.financial_population if name == "financial" else getattr(prefix, name) + ) + require( + actual is population and reconstruction._population_stamp(actual) == stamp, + "FINANCIAL_RUN_POPULATION_CHANGED", + ) + require( + _run_document(run, state) == entry[1] and _live() == state.live, + "FINAL_FINANCIAL_RUN_SEAL", + ) + require(_run_entry(run) is entry, "FINAL_FINANCIAL_RUN_ISSUANCE") + + +def check_atomic_survey_financial_run(run): + """Requalify existing artifacts and live owners without fitting or execution.""" + entry = _run_entry(run) + _pure_run(run, entry) + state, prefix = entry[2], entry[2].prefix + _, source_keys = _source_paths_and_keys(run.compiled, run.sources, run.store) + keys, implementations = _all_node_keys(run.compiled, run.kernels, source_keys) + require( + tuple(sorted(source_keys.items())) == state.source_keys + and tuple(sorted(keys.items())) == state.keys + and tuple(sorted(implementations.items())) == state.implementations, + "FINANCIAL_RUN_IMPLEMENTATIONS_CHANGED", + ) + loaded = _artifacts( + run.manifest, run.compiled, run.store, run.kernels, keys, implementations + ) + require( + tuple( + sorted( + (node, name, codec.sha(payload)) + for (node, name), payload in loaded.items() + ) + ) + == state.artifact_hashes, + "FINANCIAL_RUN_ARTIFACT_CHANGED", + ) + # These exact fitted artifacts were checked at actual execution/required + # replay before issuance. Rechecking their identities needs no new pickle + # decode or fit; the materialized verifier freshly derives current values. + financial.verify_materialized_current_survey_predictors( + prefix.preparation, + prefix.allocated_population, + prefix.clone_population, + population=run.financial_population, + projection=state.projection, + matrix=state.matrix, + matrix_producer_key=keys[financial.PROJECTION_NODE], + raw_draws=tuple( + loaded[f"{financial.APPLY_PREFIX}.{i:03d}", "raw_draw"] for i in range(3) + ), + apply_states=tuple( + loaded[f"{financial.APPLY_PREFIX}.{i:03d}", "apply_state"] for i in range(3) + ), + host_pins=codec.decode_json(state.pins), + n_estimators=state.n_estimators, + demographic_conditioning=state.demographic_conditioning, + geography_config=prefix.geography_config, + ) + result = CheckedAtomicSurveyFinancialRun( + entry[1], codec.sha(entry[1]), run.financial_population + ) + _pure_run(run, entry) + return result + + +def _issue_run( + result, + *, + preparation_entry, + pins, + n_estimators, + demographic_conditioning, + source_keys, + keys, + implementations, + loaded, + live, +): + """Called only after this runner's complete materialization/replay checks.""" + prefix = result.prefix + populations = tuple( + (name, population, reconstruction._population_stamp(population)) + for name, population in ( + *( + (name, getattr(prefix, name)) + for name in ( + "allocated_population", + "observed_population", + "expanded_population", + "geography_population", + "clone_population", + ) + ), + ("financial", result.financial_population), + ) + ) + state = _FinancialRunState( + prefix, + ( + prefix.preparation, + prefix.manifest, + prefix.compiled, + prefix.store, + prefix.kernels, + prefix.sources, + prefix.geography_config, + ), + preparation_entry, + result.financial_population, + populations, + result.manifest, + result.manifest.to_json_bytes(), + prefix.manifest.to_json_bytes(), + result.compiled, + graph_to_json(result.compiled.graph), + graph_to_json(prefix.compiled.graph), + result.store, + result.kernels, + tuple(sorted(result.sources.items())), + prefix.geography_config.to_bytes(), + result.projection, + result.matrix, + codec.encode_json(pins), + n_estimators, + demographic_conditioning, + tuple(sorted(source_keys.items())), + tuple(sorted(keys.items())), + tuple(sorted(implementations.items())), + tuple( + sorted( + (node, name, codec.sha(payload)) + for (node, name), payload in loaded.items() + ) + ), + _manifest_population_seals(result.manifest, result.compiled), + _manifest_population_seals(prefix.manifest, prefix.compiled), + live, + ) + identifier = id(result) + + def forget(reference): + entry = _ISSUED_RUNS.get(identifier) + if entry is not None and entry[0] is reference: + _ISSUED_RUNS.pop(identifier, None) + + reference = weakref.ref(result, forget) + _ISSUED_RUNS[identifier] = (reference, _run_document(result, state), state) + _pure_run(result, _run_entry(result)) + + +def _live(): + """Pure final fence over this composition and its existing owner closure.""" + result = dict(values.host.survey_budget._live()) + modules = ( + sys.modules[__name__], + atomic, + financial, + values, + codec, + qrf_target, + financial.qrf, + financial.model_input, + sys.modules[LegacyQRFTrainKernel.__module__], + sys.modules[LegacyQRFApplyMatrixKernel.__module__], + ) + for module in modules: + for name, value in vars(module).items(): + if isinstance(value, FunctionType): + result[module.__name__, name] = values.source._function_seal(value) + elif isinstance(value, type) and value.__module__ == module.__name__: + result[module.__name__, name] = value + for method, function in vars(value).items(): + if isinstance(function, (staticmethod, classmethod)): + function = function.__func__ + if isinstance(function, property): + function = function.fget + if isinstance(function, FunctionType): + result[module.__name__, name, method] = ( + values.source._function_seal(function) + ) + result["financial_contract"] = values.source._runtime_marker( + ( + RUN_PROTOCOL, + values.FEATURES, + values.DEMOGRAPHIC_FEATURES, + values.TARGETS, + values.OUTPUTS, + values.PROTOCOL, + values.PHASE, + values.SEED, + ) + ) + return result + + +def _artifacts(manifest, compiled, store, kernels, keys, implementations): + """Read only the declared outputs after binding their actual producer keys.""" + require(set(manifest.nodes) == set(compiled.order), "ATOMIC_NODE_ROSTER") + loaded = {} + for node_id in compiled.order: + node, record = compiled.graph.node(node_id), manifest.node(node_id) + kernel = kernels.get(node.kernel) + require( + record.key == keys[node_id] + and record.kernel_ref == node.kernel + and record.kernel_impl_hash == implementations[node_id] + and _capabilities_projection(record.capabilities) + == _capabilities_projection(kernel.capabilities) + and record.typed_artifacts + == typed_contracts(compiled, node, keys, kernels), + "ATOMIC_ARTIFACT_PRODUCER", + ) + require( + set(record.opaque_artifacts) == {o.name for o in node.artifact_outputs}, + "ATOMIC_ARTIFACT_ROSTER", + ) + for output in node.artifact_outputs: + payload = store.load_bytes(record.opaque_artifacts[output.name]) + survey._final_artifact( + manifest, + store, + node_id=node_id, + name=output.name, + type_=output.type, + payload=payload, + capabilities=kernel.capabilities, + ) + loaded[node_id, output.name] = payload + return loaded + + +def _model_receipts(nodes, donor, qualified, loaded, matrix_key): + """Bind fitted checkpoints to the exact source-derived design-weight donor. + + The start-chain operation only resolves inputs and initializes RNG state; + this verifier does not fit a second model. Pickles are decoded only after + the caller has checked the actual store and typed graph producer closure. + """ + fits = tuple(n for n in nodes if n.kernel == LegacyQRFTrainKernel.ref) + first = fits[0] + model_frame = codec.model_frame( + SimpleNamespace(weights={"person": donor.frame.resolve_weights("person")}), + first.inputs[0], + donor.frame.person, + ) + predictors = values.feature_columns(qualified.demographic_conditioning) + model = financial.qrf.RegimeGatedQRF( + seed=values.SEED, + n_estimators=first.params["n_estimators"], + zero_atol=0, + max_samples_leaf=None, + ) + before = qrf_target.LegacyQRFTrainingState.from_chain( + model.start_chain( + model_frame, list(predictors), list(values.TARGETS), weights="design" + ) + ) + receipts, history = {}, [] + for i, fit in enumerate(fits): + target = values.TARGETS[i] + payload = loaded[fit.id, "model"] + packet, after = codec.read_training(loaded[fit.id, "training_state"]) + require( + len(packet["models"]) == i + 1 + and packet["models"][:i] == history + and packet["models"][-1]["sha256"] == codec.sha(payload), + "ATOMIC_TRAINING_HISTORY", + ) + fitted = qrf_target.LegacyQRFTargetArtifact.from_trusted_bytes( + payload, expected_sha256=packet["models"][-1]["sha256"] + ) + require( + fitted.training_state == before + and fitted.next_training_state == after + and fitted.donor_sha256 + == qrf_target._consumed_values_sha256( + model_frame.person, (*predictors, *values.TARGETS[:i], target) + ), + "ATOMIC_TRAINING_DONOR", + ) + history = [ + *history, + { + "target": target, + "sha256": codec.sha(payload), + "training_id": fitted.training_id, + }, + ] + require(packet["models"] == history, "ATOMIC_TRAINING_HISTORY") + before = after + receipts[fit.id] = { + "phase": values.PHASE, + "target": target, + "training_id": fitted.training_id, + "model_sha256": codec.sha(payload), + "donor_rows": len(model_frame.person), + "entity": "person", + "weight_kind": "design", + "regime": fitted.regime, + } + apply_id = f"{financial.APPLY_PREFIX}.{i:03d}" + application = financial.decode_matrix_apply_state( + loaded[apply_id, "apply_state"] + ) + require( + application["application"]["models"] == history, + "ATOMIC_APPLY_MODEL_HISTORY", + ) + receipts[apply_id] = { + "phase": values.PHASE, + "target": target, + "recipient_rows": len( + financial.model_input.decode_recipient_matrix(qualified.matrix).features + ), + "entity": "person", + "model_sha256": codec.sha(payload), + "raw_sha256": codec.sha(loaded[apply_id, "raw_draw"]), + "regime": fitted.regime, + "matrix_sha256": codec.sha(qualified.matrix), + "matrix_producer_key": matrix_key, + } + return receipts + + +def run_atomic_survey_financial( + source_dir, + *, + snapshot_root, + store_root, + fraction, + seed, + geography_config, + demographic_conditioning=False, + n_estimators=100, + resume="auto", + return_values=False, +): + """Execute and verify all nineteen nodes, including required cache replay.""" + require(type(return_values) is bool, "RETURN_VALUES_FLAG") + values.feature_columns(demographic_conditioning) + live = _live() + config_bytes = values.host.survey_budget._config_payload(geography_config) + require(config_bytes is not None, "ATOMIC_GEOGRAPHY_REQUIRED") + prefix = atomic.run_atomic_survey_population( + source_dir, + snapshot_root=snapshot_root, + store_root=store_root, + fraction=fraction, + seed=seed, + geography_config=geography_config, + resume=resume, + return_values=True, + ) + entry = prefix.preparation._checked() + prefix_objects = ( + prefix.preparation, + prefix.manifest, + prefix.compiled, + prefix.store, + prefix.kernels, + prefix.sources, + ) + prefix_manifest_bytes = prefix.manifest.to_json_bytes() + prefix_declaration = graph_to_json(prefix.compiled.graph) + retained = { + name: ( + getattr(prefix, name), + reconstruction._population_stamp(getattr(prefix, name)), + ) + for name in ( + "allocated_population", + "observed_population", + "expanded_population", + "geography_population", + "clone_population", + ) + } + qualified = values.qualify_current_survey_predictors( + prefix.preparation, + prefix.allocated_population, + prefix.clone_population, + demographic_conditioning=demographic_conditioning, + geography_config=geography_config, + ) + projection_bytes, matrix_bytes = qualified.projection, qualified.matrix + geography = reconstruction.reconstruct_atomic_survey_geography( + prefix.preparation, prefix.allocated_population, geography_config + ) + pins, prefix_artifacts = {}, {} + for node_id, record in prefix.manifest.nodes.items(): + for name, key in record.opaque_artifacts.items(): + prefix_artifacts[node_id, name] = prefix.store.load_bytes(key) + for edge in ( + *financial.host.current_survey_host_edges(), + financial._geography_edge(), + ): + record = prefix.manifest.node(edge.producer) + pins[edge.name] = { + "producer_key": record.key, + "artifact_key": record.opaque_artifacts[edge.artifact], + "payload_sha256": codec.sha(prefix_artifacts[edge.producer, edge.artifact]), + } + nodes = financial.current_survey_predictor_nodes( + qualified, + prefix.clone_population.frame, + host_pins=pins, + n_estimators=n_estimators, + ) + compiled = compile_graph( + replace(prefix.compiled.graph, nodes=(*prefix.compiled.graph.nodes, *nodes)) + ) + require( + len(prefix.compiled.order) == 9 and len(compiled.order) == 19, + "ATOMIC_COMPILER_ROSTER", + ) + gate_edge = financial._geography_edge() + require( + gate_edge.producer in compiled.predecessors[financial.DONOR_NODE] + and prefix_artifacts[gate_edge.producer, gate_edge.artifact] + == qualified.geography_validation, + "ATOMIC_GEOGRAPHY_GATE_EDGE", + ) + declaration = graph_to_json(compiled.graph) + kernels, store, sources = prefix.kernels, prefix.store, dict(prefix.sources) + source_items = tuple(sorted(sources.items())) + for cls in ( + financial.CurrentSurveyPredictorProjectionKernel, + financial.CurrentSurveyPredictorDonorFilterKernel, + financial.CurrentSurveyPredictorDonorColumnsKernel, + financial.CurrentSurveyPredictorAttachKernel, + ): + kernels.register( + cls( + prefix.preparation, + prefix.allocated_population, + prefix.clone_population, + host_pins=pins, + n_estimators=n_estimators, + demographic_conditioning=demographic_conditioning, + geography_config=geography_config, + ) + ) + kernels.register(LegacyQRFTrainKernel()) + kernels.register(LegacyQRFApplyMatrixKernel()) + _, source_keys = _source_paths_and_keys(compiled, sources, store) + keys, implementations = _all_node_keys(compiled, kernels, source_keys) + base_expected = { + survey.CREATE_NODE: Population.from_frame(entry[2].frame, survey.CREATE_NODE), + survey.ALLOCATION_NODE: prefix.allocated_population, + **{s.node.id: s.population for s in geography.stages}, + } + receipts = { + **{n: r.receipt for n, r in prefix.manifest.nodes.items()}, + **{s.node.id: json.loads(s.receipt) for s in geography.stages}, + } + donor_node = compiled.graph.node(financial.DONOR_NODE) + donor = population_ops.patch( + base_expected[survey.CREATE_NODE], + donor_node, + KernelResult(frame=qualified.donor_frame), + ) + receipts[donor_node.id] = { + "selection": "ASEC_native_whole_households", + "fit_weight_kind": "design", + "release_eligible": False, + } + columns_node = compiled.graph.node(financial.DONOR_COLUMNS_NODE) + donor_columns = population_ops.patch( + donor, + columns_node, + KernelResult( + columns={ + ("person", c): qualified.donor_columns[c] + for c in ( + *values.feature_columns(demographic_conditioning), + *values.TARGETS, + ) + } + ), + ) + receipts[financial.PROJECTION_NODE] = qualified.evidence + receipts[columns_node.id] = qualified.evidence + observed, observed_stamps = {}, {} + + def observe(node_id, population): + require(node_id not in observed, "ATOMIC_OBSERVER_DUPLICATE") + observed[node_id] = population + observed_stamps[node_id] = reconstruction._population_stamp(population) + + manifest = run_graph( + compiled, + sources=sources, + store=store, + kernels=kernels, + resume=resume, + _population_observer=observe, + ) + require(tuple(observed) == compiled.order, "ATOMIC_OBSERVER_ROSTER") + loaded = _artifacts(manifest, compiled, store, kernels, keys, implementations) + require( + all(loaded[k] == v for k, v in prefix_artifacts.items()), + "ATOMIC_PREFIX_ARTIFACTS", + ) + require( + loaded[financial.PROJECTION_NODE, "projection"] == projection_bytes + and loaded[financial.PROJECTION_NODE, "matrix"] == matrix_bytes, + "ATOMIC_SOURCE_ARTIFACTS", + ) + raw = tuple( + loaded[f"{financial.APPLY_PREFIX}.{i:03d}", "raw_draw"] for i in range(3) + ) + applications = tuple( + loaded[f"{financial.APPLY_PREFIX}.{i:03d}", "apply_state"] for i in range(3) + ) + matrix_key = keys[financial.PROJECTION_NODE] + drawn = financial.read_current_survey_draws( + matrix_bytes, + matrix_key, + raw, + applications, + demographic_conditioning=demographic_conditioning, + ) + receipts.update( + _model_receipts(nodes, donor_columns, qualified, loaded, matrix_key) + ) + receipts[financial.ATTACH_NODE] = { + **qualified.evidence, + "raw_sha256": [codec.sha(r) for r in raw], + "all_output_cells_available": True, + "ACS_financial_origin": "modeled", + "paired_draws": "source_origin_join", + "host_weights_changed": False, + } + expected, current = {}, {} + for node_id in compiled.order: + node, version = compiled.graph.node(node_id), compiled.versions[node_id] + if node_id == donor_node.id: + population = donor + elif node_id == columns_node.id: + population = donor_columns + elif node_id == financial.ATTACH_NODE: + population = population_ops.patch( + current[version], + node, + KernelResult( + columns=values.complete_predictor_columns( + qualified, prefix.clone_population.frame, drawn + ) + ), + ) + elif node_id in base_expected: + population = base_expected[node_id] + else: + population = current[version] + expected[node_id] = current[version] = population + atomic.same_replayed_population(population, observed[node_id]) + expected_stamps = { + n: reconstruction._population_stamp(p) for n, p in expected.items() + } + states = atomic._states(compiled, kernels, source_keys, expected, receipts) + survey._check_node_states(manifest, states) + result = AtomicSurveyFinancialRunValues( + prefix, + observed[financial.ATTACH_NODE], + manifest, + compiled, + store, + kernels, + sources, + projection_bytes, + matrix_bytes, + ) + # Implementation/source/store I/O precedes the last owner/support borrow. + current_keys, current_implementations = _all_node_keys( + compiled, kernels, source_keys + ) + require( + current_keys == keys and current_implementations == implementations, + "ATOMIC_FINAL_IMPLEMENTATIONS", + ) + financial.verify_materialized_current_survey_predictors( + prefix.preparation, + prefix.allocated_population, + prefix.clone_population, + population=result.financial_population, + projection=projection_bytes, + matrix=matrix_bytes, + matrix_producer_key=matrix_key, + raw_draws=raw, + apply_states=applications, + host_pins=pins, + n_estimators=n_estimators, + demographic_conditioning=demographic_conditioning, + geography_config=geography_config, + ) + values.source._pure_final(entry[2]) + require( + values.source._ISSUED.get(id(prefix.preparation)) is entry + and prefix.preparation.payload == entry[1] + and prefix.geography_config is geography_config + and values.host.survey_budget._config_payload(geography_config) == config_bytes + and result.prefix is prefix + and result.manifest is manifest + and result.compiled is compiled + and result.store is store + and result.kernels is kernels + and result.financial_population is observed[financial.ATTACH_NODE] + and result.projection == projection_bytes + and result.matrix == matrix_bytes + and all( + a is b + for a, b in zip( + ( + prefix.preparation, + prefix.manifest, + prefix.compiled, + prefix.store, + prefix.kernels, + prefix.sources, + ), + prefix_objects, + strict=True, + ) + ) + and prefix.manifest.to_json_bytes() == prefix_manifest_bytes + and graph_to_json(prefix.compiled.graph) == prefix_declaration + and prefix.compiled == compile_graph(prefix.compiled.graph) + and tuple(sorted(result.sources.items())) == source_items + and tuple(sorted(prefix.sources.items())) == source_items + and graph_to_json(compiled.graph) == declaration + and compiled == compile_graph(compiled.graph) + and _live() == live, + "ATOMIC_FINAL_BINDINGS", + ) + for name, (population, stamp) in retained.items(): + require( + getattr(prefix, name) is population + and reconstruction._population_stamp(population) == stamp, + "ATOMIC_FINAL_PREFIX_MUTATION", + ) + for version, population in ( + (survey.CREATE_NODE, base_expected[survey.CREATE_NODE]), + (survey.ALLOCATION_NODE, prefix.observed_population), + (atomic.clone.COMBINED_CLONE_NODE, prefix.clone_population), + ): + survey._same_frame(population.frame, prefix.manifest.population(version)) + require( + population.mass_ledger == prefix.manifest.mass_ledger(version), + "ATOMIC_FINAL_PREFIX_LEDGER", + ) + for node_id, population in expected.items(): + require( + reconstruction._population_stamp(population) == expected_stamps[node_id] + and reconstruction._population_stamp(observed[node_id]) + == observed_stamps[node_id], + "ATOMIC_FINAL_POPULATION_MUTATION", + ) + atomic.same_replayed_population(population, observed[node_id]) + for version, population in current.items(): + survey._same_frame(population.frame, manifest.population(version)) + require( + population.mass_ledger == manifest.mass_ledger(version), + "ATOMIC_FINAL_LEDGER", + ) + survey._check_node_states(manifest, states) + _issue_run( + result, + preparation_entry=entry, + pins=pins, + n_estimators=n_estimators, + demographic_conditioning=demographic_conditioning, + source_keys=source_keys, + keys=keys, + implementations=implementations, + loaded=loaded, + live=live, + ) + return result if return_values else manifest diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/graph_atomic_survey_population.py b/packages/microcosm-build/src/microcosm/build/us_runtime/graph_atomic_survey_population.py new file mode 100644 index 000000000..5bd732768 --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/graph_atomic_survey_population.py @@ -0,0 +1,334 @@ +"""Complete initial support clones before assigning qualified survey geography. + +The existing runner independently admits the raw allocation. This extension +reconstructs every added column from the live preparation and pinned normalized +support, then checks the complete graph populations on cold and warm execution. +Normalized support integrity does not establish its publisher provenance. +""" + +from __future__ import annotations + +import json +from dataclasses import dataclass +from types import SimpleNamespace + +from microcosm.build.graph_atomic_geography import ( + ATOMIC_SUPPORT_TYPE as ATOMIC_SUPPORT_TYPE, +) +from microcosm.build.graph_atomic_geography import ( + AtomicSupportImportKernel as AtomicSupportImportKernel, +) +from microcosm.build.graph_atomic_geography import ( + register_atomic_geography_kernels, +) +from microcosm.graph import ( + Graph, + SourceRef, + StructuralDelta, + compile_graph, + run_graph, +) +from microcosm.graph.artifact_edges import typed_contracts +from microcosm.graph.codecs import load_raw_bytes +from microcosm.graph.executor import ( + _all_node_keys, + _input_writers, + _source_paths_and_keys, + _tolerance_writer_payload, +) +from microcosm.graph.keys import ( + _capabilities_projection, + artifact_key, + frame_key, + opaque_artifact_key, + weights_key, +) +from microcosm.graph.keys import seed as node_seed +from microcosm.graph.manifest import _freeze_json +from microcosm.graph.population import ( + Population, + mass_record_receipt, + weight_cap_receipt, +) + +from . import graph_combined_clone as clone +from . import graph_current_survey_geography as projection +from . import graph_survey_population as survey +from . import survey_atomic_geography as reconstruction +from .survey_population_replay import same_replayed_population + + +@dataclass(frozen=True) +class AtomicSurveyPopulationRunValues: + """Actual run values, without a new source or release certificate.""" + + manifest: object + preparation: object + allocated_population: Population + observed_population: Population + expanded_population: Population + geography_population: Population + clone_population: Population + geography_config: reconstruction.AtomicSurveyReconstruction + compiled: object + store: object + kernels: object + sources: dict + + +def _states(compiled, kernels, source_keys, expected_populations, raw_receipts): + """Bind independent domain receipts to current implementation and graph keys.""" + keys, implementations = _all_node_keys(compiled, kernels, source_keys) + states, writer_receipts = {}, {} + for node_id in compiled.order: + node = compiled.graph.node(node_id) + population = expected_populations[node_id] + key = keys[node_id] + structural = node.structural is not StructuralDelta.NONE + cells = ( + tuple( + (e, str(c)) + for e in population.frame.entities + for c in population.frame.table(e) + ) + if structural + else tuple((o.entity, o.column) for o in node.outputs) + ) + weight_entity = ( + node.weights.entity + if node.weights is not None + else node.params.get("expand_weight_entity") + if node.structural is StructuralDelta.EXPAND + else None + ) + capabilities = _capabilities_projection(kernels.get(node.kernel).capabilities) + receipt = dict(raw_receipts[node_id]) + receipt["capabilities"] = dict(capabilities) + writers = _tolerance_writer_payload( + _input_writers(compiled, node_id, receipts=writer_receipts) + ) + if writers: + receipt["capabilities"]["tolerance_writers"] = writers + if structural and node.structural is not StructuralDelta.CREATE: + receipt["mass"] = { + **receipt.get("mass", {}), + **mass_record_receipt(population.mass_ledger[-1]), + } + receipt.update(weight_cap_receipt(population, node)) + states[node_id] = { + "key": key, + "ref": node.kernel, + "capabilities": capabilities, + "implementation": implementations[node_id], + "typed_artifacts": typed_contracts(compiled, node, keys, kernels), + "seed": node_seed(key), + "frame_key": frame_key(key) if structural else None, + "weight_key": weights_key(key, weight_entity) if weight_entity else None, + "artifacts": {(e, c): artifact_key(key, e, c) for e, c in cells}, + "opaque_artifacts": { + o.name: opaque_artifact_key(key, o.name) for o in node.artifact_outputs + }, + "receipt": _freeze_json(receipt), + } + writer_receipts[node_id] = SimpleNamespace(receipt=states[node_id]["receipt"]) + return states + + +def run_atomic_survey_population( + source_dir, + *, + snapshot_root, + store_root, + fraction, + seed, + geography_config, + resume="auto", + return_values=False, +): + """Assign each completed source/clone household and independently verify replay.""" + survey._require(type(return_values) is bool, "RETURN_VALUES_FLAG") + prefix = survey.run_authenticated_survey_population( + source_dir, + snapshot_root=snapshot_root, + store_root=store_root, + fraction=fraction, + seed=seed, + resume=resume, + clones=False, + return_values=True, + ) + source_owner = survey._source_owner() + preparation_entry = prefix.preparation._checked() + geography = reconstruction.reconstruct_atomic_survey_geography( + prefix.preparation, prefix.allocated_population, geography_config + ) + _, view = survey._checked_preparation(prefix.preparation) + instructions = survey.allocation_instructions( + view.selection_plan, view.receipt["origins"]["households"] + ) + _, allocated_context, allocation_payload, _, _ = survey._allocation_output( + view.frame, view.context, instructions, survey._sha(view.payload) + ) + prefix_artifacts = { + (survey.CREATE_NODE, "preparation"): view.payload, + (survey.CREATE_NODE, "frame_context"): view.context, + (survey.ALLOCATION_NODE, "allocation"): allocation_payload, + (survey.ALLOCATION_NODE, "frame_context"): allocated_context, + } + additions = geography.nodes + compiled = compile_graph( + Graph( + "us", + ( + *prefix.compiled.graph.sources, + *(SourceRef(name, "raw-bytes-v1") for name, _ in geography.sources), + ), + (*prefix.compiled.graph.nodes, *additions), + ) + ) + survey._require(len(compiled.order) == 9, "ATOMIC_COMPILER_ROSTER") + sources = {**prefix.sources, **dict(geography.sources)} + kernels, store = prefix.kernels, prefix.store + store.codecs.register_bytes("raw-bytes-v1", load_raw_bytes) + kernels.register(projection.CurrentSurveyGeographyKernel(prefix.preparation)) + register_atomic_geography_kernels(kernels) + clone.register_us_combined_survey_clone_kernels(kernels) + expected = { + survey.CREATE_NODE: Population.from_frame( + prefix.manifest.population(survey.CREATE_NODE), survey.CREATE_NODE + ), + survey.ALLOCATION_NODE: prefix.allocated_population, + **{stage.node.id: stage.population for stage in geography.stages}, + } + receipts = { + **{name: record.receipt for name, record in prefix.manifest.nodes.items()}, + **{stage.node.id: json.loads(stage.receipt) for stage in geography.stages}, + } + expected_stamps = { + name: reconstruction._population_stamp(value) + for name, value in expected.items() + } + raw_stamp = reconstruction._population_stamp(prefix.allocated_population) + source_items = tuple(sorted(sources.items())) + config_bytes = geography_config.to_bytes() + _, source_keys = _source_paths_and_keys(compiled, sources, store) + states = _states(compiled, kernels, source_keys, expected, receipts) + observed, observed_stamps = {}, {} + + def observe(node_id, population): + survey._require(node_id not in observed, "DUPLICATE_POPULATION_OBSERVATION") + same_replayed_population(expected[node_id], population) + observed[node_id] = population + observed_stamps[node_id] = reconstruction._population_stamp(population) + + manifest = run_graph( + compiled, + sources=sources, + store=store, + kernels=kernels, + resume=resume, + _population_observer=observe, + ) + survey._require(tuple(observed) == compiled.order, "POPULATION_OBSERVER_COVERAGE") + survey._check_node_states(manifest, states) + for stage in geography.stages: + for name, payload in stage.artifacts: + output = next(o for o in stage.node.artifact_outputs if o.name == name) + survey._final_artifact( + manifest, + store, + node_id=stage.node.id, + name=name, + type_=output.type, + payload=payload, + capabilities=kernels.get(stage.node.kernel).capabilities, + ) + # Validate prefix artifacts separately from column materialization. + for (node_id, name), payload in prefix_artifacts.items(): + key = prefix.manifest.node(node_id).opaque_artifacts[name] + survey._require( + manifest.node(node_id).opaque_artifacts.get(name) == key, + "ATOMIC_PREFIX_ARTIFACT_KEY", + ) + survey._require( + store.load_bytes(key) == payload, "ATOMIC_PREFIX_ARTIFACT_BYTES" + ) + geography_terminal = geography.nodes[-1].id + result = AtomicSurveyPopulationRunValues( + manifest=manifest, + preparation=prefix.preparation, + allocated_population=prefix.allocated_population, + observed_population=observed[projection.NODE], + expanded_population=observed[clone.COMBINED_CLONE_CLAIM_NODE], + geography_population=observed[geography_terminal], + # This historical field is the terminal receiving population. The + # explicit expanded_population is the preassignment clone snapshot. + clone_population=observed[geography_terminal], + geography_config=geography_config, + compiled=compiled, + store=store, + kernels=kernels, + sources=sources, + ) + current_keys, current_implementations = _all_node_keys( + compiled, kernels, source_keys + ) + survey._require( + current_keys == {name: value["key"] for name, value in states.items()} + and current_implementations + == {name: value["implementation"] for name, value in states.items()}, + "ATOMIC_FINAL_IMPLEMENTATIONS", + ) + # Finish source/support I/O before final comparisons of returned objects. + fresh = reconstruction.reconstruct_atomic_survey_geography( + prefix.preparation, prefix.allocated_population, geography_config + ) + survey._require( + source_owner._ISSUED.get(id(prefix.preparation)) is preparation_entry + and prefix.preparation.payload == preparation_entry[1], + "ATOMIC_FINAL_PREPARATION", + ) + source_owner._pure_final(preparation_entry[2]) + survey._require( + fresh.config_sha256 == geography.config_sha256 + and fresh.projection_receipt == geography.projection_receipt + and fresh.support_payload == geography.support_payload + and fresh.definition == geography.definition + and geography_config.to_bytes() == config_bytes + and result.geography_config is geography_config + and result.preparation is prefix.preparation + and result.allocated_population is prefix.allocated_population + and result.observed_population is observed[projection.NODE] + and result.expanded_population is observed[clone.COMBINED_CLONE_CLAIM_NODE] + and result.geography_population is observed[geography_terminal] + and result.clone_population is result.geography_population + and result.manifest is manifest + and result.compiled is compiled + and result.store is store + and result.kernels is kernels + and tuple(sorted(result.sources.items())) == source_items + and reconstruction._population_stamp(prefix.allocated_population) == raw_stamp, + "ATOMIC_FINAL_RECONSTRUCTION", + ) + same_replayed_population(fresh.observed_population, result.observed_population) + same_replayed_population(fresh.expanded_population, result.expanded_population) + same_replayed_population(fresh.population, result.geography_population) + for node_id, population in expected.items(): + survey._require( + reconstruction._population_stamp(population) == expected_stamps[node_id] + and reconstruction._population_stamp(observed[node_id]) + == observed_stamps[node_id], + "ATOMIC_FINAL_POPULATION_MUTATION", + ) + same_replayed_population(population, observed[node_id]) + latest = {} + for node_id in compiled.order: + latest[compiled.versions[node_id]] = node_id + for version, node_id in latest.items(): + survey._same_frame(expected[node_id].frame, manifest.population(version)) + survey._require( + manifest.mass_ledger(version) == expected[node_id].mass_ledger, + "ATOMIC_FINAL_MASS_LEDGER", + ) + survey._check_node_states(manifest, states) + return result if return_values else manifest diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/graph_combined_clone.py b/packages/microcosm-build/src/microcosm/build/us_runtime/graph_combined_clone.py new file mode 100644 index 000000000..c9b1276fe --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/graph_combined_clone.py @@ -0,0 +1,830 @@ +"""The combined-survey PUF-support clone, as a real graph ``EXPAND`` stage. + +What this stage is +------------------ + +It takes an already combined ASEC + ACS population — every entity native, every +``*_support_clone_index`` zero, household weights already IMPORTANCE — and +returns the two-role host the PUF detail pass needs: the untouched native role +plus exactly one full-attachment detail clone of every household, with each +incoming household weight split evenly across the pair. + +It is only that. No PUF donor is read, no quantile forest is fitted, no draw is +taken, no amount is placed, no tax or benefit value is computed, and no prior +wage stage runs. The donor sampling weight ``S006`` never touches a host weight: +this stage never sees a donor at all. The profile is development; nothing here +is release eligible. + +Why it is an ``EXPAND`` and not a Frame handed to the executor +------------------------------------------------------------- + +:func:`~.puf_support.clone_us_frame_for_puf_support` already performs this +operation directly, and it stays the behavioral authority: the kernel calls it +and validates its output with +:func:`~.puf_support.validate_puf_clone_attachment`, exactly as the direct path +does. What the kernel returns to the executor, though, is not that Frame. It +returns what :class:`~microcosm.graph.KernelResult` declares for a structural +node — per-entity clone lineage, the declared cell overlays, and explicit +weights — and ``microcosm.graph.population._patch_expand`` does the structural +work: it carries every column from each copied row, remaps every copied +person's five memberships onto the copied groups, appends the strata, installs +the weights and records the mass ledger. A Frame smuggled through as an opaque +artifact would make the executor's lineage, membership, storage, weight and +mass checks vacuous, so this stage does not do that. + +The ID relationship between the two paths +----------------------------------------- + +The executor's contract is that every base row survives with its own id and new +rows get new ids. The direct operator honours the same rule on this input: for a +preassembled frame it copies the native block unchanged and shifts only the +detail block by a decimal multiplier. The two paths therefore agree on raw ids +here — but the stage does not depend on that. Lineage is derived from the +operator's own output through the ``(entity_source_id, clone_index)`` bijection, +with the positional pairing checked independently, and the receipt publishes a +digest of that bijection. A comparison against the direct operator is a +comparison by declared source and role, not an assumption about integers. + +What the node key binds +----------------------- + +The kernel's implementation hash covers this module, the clone operator's module +and the provenance owner's module, plus the pinned ``numpy``/``pandas`` +versions, so an edit to the operator moves every cached identity. The node's own +key additionally binds its declared parameters, the artifact keys of the +provenance columns it reads and — through ``population_input['base']`` — the +whole base population version: its amounts, its ids, its row order, its +provenance and its incoming weights. Changing any of those changes this node's +key, so warm reuse can never serve a clone of a different population. +""" + +from __future__ import annotations + +import hashlib +from collections.abc import Mapping, Sequence + +import numpy as np +import pandas as pd + +from microcosm.frame import US_SCHEMA, Frame, WeightKind, Weights +from microcosm.graph import ( + Capabilities, + Determinism, + KernelBase, + KernelContext, + KernelRegistry, + KernelResult, + Node, + Numeric, + Owned, + SeedSource, + Slice, + StructuralDelta, + source_hash, +) +from microcosm.graph.canonical import canonical_json +from microcosm.graph.population import token_for_dtype + +from . import puf_support as _puf_support_module +from . import support_provenance as _support_provenance_module +from .puf_support import ( + clone_us_frame_for_puf_support, + validate_puf_clone_attachment, +) +from .support_provenance import ( + BASE_ASEC_SUPPORT_CHANNEL, + PUF_TAX_DETAIL_CLONE_INDEX, + PUF_TAX_DETAIL_SUPPORT_CHANNEL, + spine_assembly_manifest, + spine_provenance_counts, + spine_source_id_column, + support_channel_column, + support_clone_index_column, + support_source_id_column, +) + +__all__ = [ + "COMBINED_CLONE_ATTACHMENT_FRACTION", + "COMBINED_CLONE_ATTACHMENT_SEED", + "COMBINED_CLONE_CELL_DTYPE", + "COMBINED_CLONE_CLAIM_NODE", + "COMBINED_CLONE_NODE", + "COMBINED_CLONE_PHASE", + "COMBINED_CLONE_PREFIX", + "COMBINED_CLONE_PROFILE", + "COMBINED_CLONE_ROLES", + "COMBINED_CLONE_WEIGHT_ENTITY", + "COMBINED_CLONE_WEIGHT_KIND", + "COMBINED_CLONE_WEIGHT_SPLIT_DENOMINATOR", + "CombinedCloneError", + "USCombinedSurveyCloneClaimKernel", + "USCombinedSurveyCloneExpandKernel", + "combined_clone_provenance_columns", + "register_us_combined_survey_clone_kernels", + "us_combined_survey_clone_nodes", +] + + +class CombinedCloneError(ValueError): + """A combined-survey clone declaration or input violates the contract.""" + + +#: Descriptive phase label carried in the node params and every receipt. +COMBINED_CLONE_PHASE = "us_combined_survey_puf_support_clone" +COMBINED_CLONE_PREFIX = "combined_survey_puf_support_clone" +COMBINED_CLONE_NODE = COMBINED_CLONE_PREFIX +COMBINED_CLONE_CLAIM_NODE = f"{COMBINED_CLONE_PREFIX}.owned" + +#: This stage is an engineering intermediate, not a candidate for release. +COMBINED_CLONE_PROFILE = "development" + +#: The two operator roles the clone writes: the incoming native role keeps +#: clone index 0, the single detail copy takes ``PUF_TAX_DETAIL_CLONE_INDEX``. +#: These are operator roles, never source channels: the ASEC and ACS source +#: channels are carried unchanged into *both* roles. +COMBINED_CLONE_ROLES = (BASE_ASEC_SUPPORT_CHANNEL, PUF_TAX_DETAIL_SUPPORT_CHANNEL) + +#: Full one-copy attachment only. A seeded partial arm is a separate, explicitly +#: declared control and is deliberately not reachable from this stage. +COMBINED_CLONE_ATTACHMENT_FRACTION = 1.0 +COMBINED_CLONE_ATTACHMENT_SEED = 0 + +#: Each incoming household weight is split across exactly two roles. +COMBINED_CLONE_WEIGHT_SPLIT_DENOMINATOR = 2 +COMBINED_CLONE_WEIGHT_ENTITY = "household" +COMBINED_CLONE_WEIGHT_KIND = WeightKind.IMPORTANCE + +#: The clone-index columns are the only cells this stage writes. +COMBINED_CLONE_CELL_DTYPE = "int64" + +_PARAM_KEYS = ( + "clone_attachment_fraction", + "clone_attachment_seed", + "clone_roles", + "expand_cells", + "expand_weight_entity", + "expand_weight_kind", + "phase", + "profile", + "release_eligible", + "source_channels", + "weight_split_denominator", +) + +_LINEAGE_DIGEST_DOMAIN = b"microcosm.us.combined-survey-clone-lineage.v1\0" + + +def combined_clone_provenance_columns(entity: str) -> tuple[str, ...]: + """The four provenance columns this stage reads on ``entity``. + + Order is fixed so a node declaration is a stable, reviewable literal. + """ + + return ( + support_source_id_column(entity), + spine_source_id_column(entity), + support_channel_column(entity), + support_clone_index_column(entity), + ) + + +def _require(condition: bool, reason: str) -> None: + if not condition: + raise CombinedCloneError(f"{COMBINED_CLONE_PHASE}: {reason}") + + +def _expected_expand_cells() -> tuple[tuple[str, str, str], ...]: + return tuple( + (entity, support_clone_index_column(entity), COMBINED_CLONE_CELL_DTYPE) + for entity in US_SCHEMA.entities + ) + + +def _validated_source_channels(value: object) -> tuple[str, ...]: + _require(isinstance(value, tuple), "SOURCE_CHANNELS_TYPE") + channels = tuple(value) # type: ignore[arg-type] + _require( + len(channels) >= 2 + and len(set(channels)) == len(channels) + and all(isinstance(name, str) and bool(name) for name in channels), + "SOURCE_CHANNELS_VALUE", + ) + _require(tuple(sorted(channels)) == channels, "SOURCE_CHANNELS_ORDER") + return channels + + +def _digest(payload: object) -> str: + return hashlib.sha256(_LINEAGE_DIGEST_DOMAIN + canonical_json(payload)).hexdigest() + + +def _int64_ids(table: pd.DataFrame, column: str, *, reason: str) -> np.ndarray: + values = table[column] + _require(str(values.dtype) == "int64", reason) + return values.to_numpy(dtype=np.int64, copy=True) + + +def _entity_lineage( + before: Frame, + after: Frame, + entity: str, +) -> tuple[pd.Series, dict[str, object]]: + """Derive one entity's new-id -> copied-id lineage and its receipt facts. + + The pairing is by ``(entity_source_id, clone_index)``: the detail row whose + assembly-unique source id is *s* copies the native row whose source id is + *s*. Nothing here reads or reconstructs an ID-offset rule. The positional + pairing the operator happens to produce is checked against that bijection + rather than trusted in its place. + """ + + id_column = US_SCHEMA.entity_id_column(entity) + source_column = support_source_id_column(entity) + clone_column = support_clone_index_column(entity) + + before_table = before.table(entity) + after_table = after.table(entity) + before_ids = _int64_ids(before_table, id_column, reason=f"BASE_ID_DTYPE:{entity}") + after_ids = _int64_ids(after_table, id_column, reason=f"CLONE_ID_DTYPE:{entity}") + clone_index = after_table[clone_column].to_numpy(dtype=np.int64, copy=True) + source_ids = _int64_ids( + after_table, source_column, reason=f"CLONE_SOURCE_ID_DTYPE:{entity}" + ) + + unexpected = sorted( + set(int(value) for value in np.unique(clone_index)) + - {0, PUF_TAX_DETAIL_CLONE_INDEX} + ) + _require(not unexpected, f"CLONE_ROLE_UNEXPECTED:{entity}:{unexpected}") + + native = clone_index == 0 + detail = clone_index == PUF_TAX_DETAIL_CLONE_INDEX + _require(int(native.sum()) == len(before_ids), f"NATIVE_ROW_COUNT:{entity}") + _require(int(detail.sum()) == len(before_ids), f"DETAIL_ROW_COUNT:{entity}") + _require( + np.array_equal(after_ids[native], before_ids), f"NATIVE_IDS_PRESERVED:{entity}" + ) + + native_sources = source_ids[native] + detail_sources = source_ids[detail] + _require( + len(np.unique(native_sources)) == len(native_sources), + f"NATIVE_SOURCE_IDS_UNIQUE:{entity}", + ) + _require( + len(np.unique(detail_sources)) == len(detail_sources), + f"DETAIL_SOURCE_IDS_UNIQUE:{entity}", + ) + native_by_source = pd.Index(native_sources) + positions = native_by_source.get_indexer(detail_sources) + _require(bool((positions >= 0).all()), f"DETAIL_SOURCE_ID_UNMATCHED:{entity}") + + target_ids = after_ids[detail] + copied_ids = before_ids[positions] + _require( + not len(np.intersect1d(target_ids, before_ids)), + f"TARGET_ID_COLLIDES_WITH_BASE:{entity}", + ) + _require( + len(np.unique(target_ids)) == len(target_ids), f"TARGET_IDS_UNIQUE:{entity}" + ) + # Independent cross-check: the operator emits the detail block row-aligned + # to the native block, so position i of the detail block must resolve to + # position i of the native block. A disagreement means the bijection and + # the physical layout describe different pairings; refuse rather than pick. + _require( + np.array_equal(positions, np.arange(len(positions), dtype=np.int64)), + f"LINEAGE_POSITION_DISAGREES:{entity}", + ) + + lineage = pd.Series( + copied_ids, + index=pd.Index(target_ids, name=id_column, dtype=before_table[id_column].dtype), + dtype=before_table[id_column].dtype, + name=id_column, + ) + facts = { + "base_rows": int(len(before_ids)), + "expanded_rows": int(len(after_ids)), + "native_rows": int(native.sum()), + "detail_rows": int(detail.sum()), + "lineage_sha256": _digest( + [ + [int(source), int(PUF_TAX_DETAIL_CLONE_INDEX), int(target)] + for source, target in zip(detail_sources, target_ids, strict=True) + ] + ), + "native_source_ids_sha256": _digest( + [int(value) for value in native_sources.tolist()] + ), + } + return lineage, facts + + +def _channel_mass(frame: Frame, channels: Sequence[str]) -> dict[str, float]: + """Weighted household mass per source channel, for the receipt only.""" + + table = frame.table(COMBINED_CLONE_WEIGHT_ENTITY) + values = np.asarray( + frame.weights_for(COMBINED_CLONE_WEIGHT_ENTITY).values, dtype=np.float64 + ) + labels = table[support_channel_column(COMBINED_CLONE_WEIGHT_ENTITY)].astype(str) + return { + channel: float(values[labels.eq(channel).to_numpy()].sum()) + for channel in channels + } + + +class USCombinedSurveyCloneExpandKernel(KernelBase): + """Run the registered clone operator and hand the executor its lineage.""" + + ref = "us.combined_survey.puf_support_clone.expand@1" + capabilities = Capabilities( + determinism=Determinism.DETERMINISTIC, + numeric=Numeric.BITWISE, + seed_source=SeedSource.NONE, + structural=StructuralDelta.EXPAND, + dependencies=("numpy", "pandas"), + ) + + def implementation_hash(self) -> str: + # The behavior of this node is this module plus the operator it calls + # and the provenance owner whose columns define both roles. Hashing + # only this file would let an operator edit reuse a stale clone. + return source_hash( + type(self), + _puf_support_module, + _support_provenance_module, + dependencies=self.capabilities.dependencies, + ) + + # ------------------------------------------------------------------ + + def _validated_params(self, context: KernelContext) -> tuple[str, ...]: + params = context.params + _require(set(params) == set(_PARAM_KEYS), f"NODE_PARAMS:{sorted(params)}") + _require(params["phase"] == COMBINED_CLONE_PHASE, "NODE_PHASE") + _require(params["profile"] == COMBINED_CLONE_PROFILE, "NODE_PROFILE") + _require(params["release_eligible"] is False, "NODE_RELEASE_ELIGIBLE") + _require( + params["clone_attachment_fraction"] == COMBINED_CLONE_ATTACHMENT_FRACTION, + "NODE_ATTACHMENT_FRACTION", + ) + _require( + params["clone_attachment_seed"] == COMBINED_CLONE_ATTACHMENT_SEED, + "NODE_ATTACHMENT_SEED", + ) + _require(tuple(params["clone_roles"]) == COMBINED_CLONE_ROLES, "NODE_ROLES") + _require( + params["weight_split_denominator"] + == COMBINED_CLONE_WEIGHT_SPLIT_DENOMINATOR, + "NODE_WEIGHT_SPLIT", + ) + _require( + params["expand_weight_entity"] == COMBINED_CLONE_WEIGHT_ENTITY, + "NODE_WEIGHT_ENTITY", + ) + _require( + params["expand_weight_kind"] == COMBINED_CLONE_WEIGHT_KIND.value, + "NODE_WEIGHT_KIND", + ) + _require( + tuple(params["expand_cells"]) == _expected_expand_cells(), + "NODE_EXPAND_CELLS", + ) + return _validated_source_channels(params["source_channels"]) + + def _validated_declaration(self, context: KernelContext) -> None: + node = context.node + _require(node.kernel == self.ref, "NODE_KERNEL") + _require(node.structural is StructuralDelta.EXPAND, "NODE_STRUCTURAL") + _require(node.entrants is False, "NODE_ENTRANTS") + _require(node.mass == "conserve", "NODE_MASS") + _require(not node.sources, "NODE_SOURCES") + _require(not node.artifact_inputs, "NODE_ARTIFACT_INPUTS") + _require(not node.artifact_outputs, "NODE_ARTIFACT_OUTPUTS") + declared = { + (slice_.entity, column) + for slice_ in node.inputs + for column in slice_.columns + } + expected = { + (entity, column) + for entity in US_SCHEMA.entities + for column in combined_clone_provenance_columns(entity) + } + _require(declared == expected, f"NODE_INPUTS:{sorted(declared ^ expected)}") + _require( + all(slice_.rows == "all" for slice_ in node.inputs), "NODE_INPUT_ROW_MASK" + ) + + def _minimal_frame( + self, context: KernelContext, channels: tuple[str, ...] + ) -> Frame: + """Rebuild the combined host from the declared slices only. + + The frame carries the four provenance columns, the structural id and + membership columns the executor always projects, the household weights + and the strata — nothing else. Every amount the population holds is + carried by the executor from the copied rows, so the operator does not + need to see one, and this kernel deliberately cannot read one. + """ + + missing = [ + entity for entity in US_SCHEMA.entities if entity not in context.tables + ] + _require(not missing, f"MISSING_SLICES:{missing}") + tables = { + entity: context.tables[entity].copy(deep=True) + for entity in US_SCHEMA.entities + } + _require( + COMBINED_CLONE_WEIGHT_ENTITY in context.weights, "MISSING_HOUSEHOLD_WEIGHTS" + ) + household_weights = context.weights[COMBINED_CLONE_WEIGHT_ENTITY] + _require( + household_weights.kind is COMBINED_CLONE_WEIGHT_KIND, + f"INCOMING_WEIGHT_KIND:{household_weights.kind.value}", + ) + self._require_inherited_weights(context, tables, household_weights) + metadata = spine_assembly_manifest(tables, channels=channels) + frame = Frame( + tables, + US_SCHEMA, + {COMBINED_CLONE_WEIGHT_ENTITY: household_weights}, + context.strata.copy(deep=True), + metadata=metadata, + ) + observed = { + str(value) + for entity in US_SCHEMA.entities + for value in frame.table(entity)[support_channel_column(entity)].unique() + } + _require(observed == set(channels), f"SOURCE_CHANNELS_LIVE:{sorted(observed)}") + return frame + + @staticmethod + def _require_inherited_weights( + context: KernelContext, + tables: Mapping[str, pd.DataFrame], + household_weights: Weights, + ) -> None: + """Refuse a base that weights anything but the household explicitly. + + ``KernelContext.weights`` resolves inheritance, so it cannot by itself + say which entity stores a vector. What it can say is whether every + other entity's effective weights are exactly the household weights + broadcast through membership; anything else is a base this stage would + silently mis-split, because the executor carries a non-weight-entity's + weights unhalved while the operator halves them. The residual case — a + stored vector numerically equal to the inherited one — cannot pass the + executor's ``conserve`` person-mass ledger either. + """ + + household = tables[COMBINED_CLONE_WEIGHT_ENTITY] + by_id = pd.Series( + np.asarray(household_weights.values, dtype=np.float64), + index=pd.Index(household["household_id"].to_numpy()), + ) + person = tables[US_SCHEMA.person_entity] + person_weights = by_id.reindex( + person[US_SCHEMA.membership_column(COMBINED_CLONE_WEIGHT_ENTITY)].to_numpy() + ).to_numpy(dtype=np.float64) + for entity, weights in context.weights.items(): + if entity == COMBINED_CLONE_WEIGHT_ENTITY: + continue + values = np.asarray(weights.values, dtype=np.float64) + if entity == US_SCHEMA.person_entity: + expected = person_weights + else: + member = pd.DataFrame( + { + "group": person[US_SCHEMA.membership_column(entity)].to_numpy(), + "weight": person_weights, + } + ) + grouped = member.groupby("group")["weight"] + _require( + bool((grouped.nunique() == 1).all()), + f"AMBIGUOUS_INHERITED_WEIGHTS:{entity}", + ) + expected = ( + grouped.first() + .reindex(tables[entity][US_SCHEMA.entity_id_column(entity)]) + .to_numpy(dtype=np.float64) + ) + _require( + values.shape == expected.shape and np.array_equal(values, expected), + f"NON_INHERITED_WEIGHTS:{entity}", + ) + + # ------------------------------------------------------------------ + + def run(self, context: KernelContext) -> KernelResult: + self._validated_declaration(context) + channels = self._validated_params(context) + before = self._minimal_frame(context, channels) + + after = clone_us_frame_for_puf_support( + before, + channels=COMBINED_CLONE_ROLES, + clone_attachment_fraction=COMBINED_CLONE_ATTACHMENT_FRACTION, + clone_attachment_seed=COMBINED_CLONE_ATTACHMENT_SEED, + ) + authority = validate_puf_clone_attachment( + after, + boundary=f"{COMBINED_CLONE_PHASE} clone output", + expected_fraction=COMBINED_CLONE_ATTACHMENT_FRACTION, + expected_seed=COMBINED_CLONE_ATTACHMENT_SEED, + ) + _require( + authority["authority_form"] == "full_clone_identity_no_manifest", + "ATTACHMENT_AUTHORITY_FORM", + ) + + lineage: dict[str, pd.Series] = {} + entity_facts: dict[str, object] = {} + for entity in US_SCHEMA.entities: + lineage[entity], entity_facts[entity] = _entity_lineage( + before, after, entity + ) + + columns: dict[tuple[str, str], pd.Series] = {} + for entity, column, dtype in _expected_expand_cells(): + incumbent = before.table(entity)[column] + _require( + token_for_dtype(incumbent.dtype) == dtype, + f"CELL_DTYPE:{entity}.{column}", + ) + table = after.table(entity) + values = table[column] + _require(str(values.dtype) == dtype, f"CLONE_CELL_DTYPE:{entity}.{column}") + columns[(entity, column)] = pd.Series( + values.to_numpy(dtype=np.int64, copy=True), + index=pd.Index( + table[US_SCHEMA.entity_id_column(entity)].to_numpy(copy=True), + name=US_SCHEMA.entity_id_column(entity), + ), + name=column, + dtype=dtype, + ) + + weights = self._split_weights(before, after, lineage) + receipt = self._receipt(before, after, channels, authority, entity_facts) + return KernelResult( + columns=columns, expand=lineage, weights=weights, receipt=receipt + ) + + @staticmethod + def _split_weights( + before: Frame, after: Frame, lineage: Mapping[str, pd.Series] + ) -> Weights: + """Household weights in the executor's target order, pair-sum checked.""" + + entity = COMBINED_CLONE_WEIGHT_ENTITY + id_column = US_SCHEMA.entity_id_column(entity) + after_weights = after.weights_for(entity) + _require(after_weights.kind is COMBINED_CLONE_WEIGHT_KIND, "CLONE_WEIGHT_KIND") + by_id = pd.Series( + np.asarray(after_weights.values, dtype=np.float64), + index=pd.Index(after.table(entity)[id_column].to_numpy(copy=True)), + ) + base_ids = before.table(entity)[id_column].to_numpy(copy=True) + target_ids = np.concatenate( + [base_ids, lineage[entity].index.to_numpy(copy=True)] + ) + ordered = by_id.reindex(target_ids) + _require(not bool(ordered.isna().any()), "WEIGHT_TARGET_ALIGNMENT") + values = ordered.to_numpy(dtype=np.float64) + + incoming = np.asarray( + before.weights_for(entity).values, dtype=np.float64, copy=True + ) + native = values[: len(base_ids)] + detail = values[len(base_ids) :] + # Require an exact pair sum rather than a tolerance that could hide + # a rescale. Halving can lose the low bit of a subnormal double; + # such a weight refuses here instead of silently losing mass. + _require(np.array_equal(native + detail, incoming), "PAIR_WEIGHT_SUM") + _require(np.array_equal(native, detail), "PAIR_WEIGHT_SYMMETRY") + return Weights(values, after_weights.kind) + + @staticmethod + def _receipt( + before: Frame, + after: Frame, + channels: tuple[str, ...], + authority: Mapping[str, object], + entity_facts: Mapping[str, object], + ) -> dict[str, object]: + """Counts, digests, labels and platform-sensitive mass summaries. + + Floating reductions here and in ``_channel_mass`` are receipt evidence, + outside the BITWISE population-output promise. Their last bits can + differ across platforms, so receipt/manifest hashes are scoped too. + """ + + return { + "phase": COMBINED_CLONE_PHASE, + "profile": COMBINED_CLONE_PROFILE, + "release_eligible": False, + "operator": { + "module": _puf_support_module.__name__, + "clone": clone_us_frame_for_puf_support.__name__, + "validator": validate_puf_clone_attachment.__name__, + }, + "source_channels": list(channels), + "clone_roles": list(COMBINED_CLONE_ROLES), + "clone_attachment": { + "fraction": COMBINED_CLONE_ATTACHMENT_FRACTION, + "seed": COMBINED_CLONE_ATTACHMENT_SEED, + "authority_form": authority["authority_form"], + "eligible_household_count": int(authority["eligible_household_count"]), + "realized_household_count": int(authority["realized_household_count"]), + "exact_count_rule": authority["exact_count_rule"], + "selected_household_source_ids_sha256": authority[ + "selected_household_source_ids_sha256" + ], + }, + "entities": {entity: entity_facts[entity] for entity in US_SCHEMA.entities}, + "provenance_counts": { + "base": spine_provenance_counts( + before, boundary=f"{COMBINED_CLONE_PHASE} base" + ), + "expanded": spine_provenance_counts( + after, boundary=f"{COMBINED_CLONE_PHASE} expanded" + ), + }, + "household_weights": { + "kind": COMBINED_CLONE_WEIGHT_KIND.value, + # The operator divides by its role count. This literal is + # the re-validated declaration of that two-role contract, + # not a separate arithmetic control for the operator. + "split_denominator": COMBINED_CLONE_WEIGHT_SPLIT_DENOMINATOR, + "base_total": float( + before.weights_for(COMBINED_CLONE_WEIGHT_ENTITY).total + ), + "expanded_total": float( + after.weights_for(COMBINED_CLONE_WEIGHT_ENTITY).total + ), + "base_by_source_channel": _channel_mass(before, channels), + "expanded_by_source_channel": _channel_mass(after, channels), + }, + "donor_inputs": { + "puf_donor_read": False, + "donor_sampling_weight_in_host_weights": False, + "draws_or_tax_benefit_values_written": False, + "prior_wage_stage": False, + }, + } + + +class USCombinedSurveyCloneClaimKernel(KernelBase): + """Own the clone-index cells the ``EXPAND`` materialized, unchanged. + + A copied row's clone index differs from its source's, which the executor + only accepts when a same-version claimant declares that coordinate a + full-cell rewrite. This kernel is that claimant: it re-emits exactly the + values the ``EXPAND`` installed, so ownership becomes explicit without any + second opinion about what the clone index is. + """ + + ref = "us.combined_survey.puf_support_clone.claim@1" + capabilities = Capabilities( + determinism=Determinism.DETERMINISTIC, + numeric=Numeric.BITWISE, + seed_source=SeedSource.NONE, + ) + + def implementation_hash(self) -> str: + return source_hash( + type(self), + _support_provenance_module, + dependencies=self.capabilities.dependencies, + ) + + def run(self, context: KernelContext) -> KernelResult: + node = context.node + expected = { + (entity, support_clone_index_column(entity)) + for entity in US_SCHEMA.entities + } + owned = {(output.entity, output.column) for output in node.outputs} + _require(owned == expected, f"CLAIM_OUTPUTS:{sorted(owned ^ expected)}") + _require( + all(output.rewrite and output.rows == "all" for output in node.outputs), + "CLAIM_REWRITE", + ) + columns: dict[tuple[str, str], pd.Series] = {} + for output in node.outputs: + table = context.tables[output.entity] + id_column = US_SCHEMA.entity_id_column(output.entity) + _require( + str(table[output.column].dtype) == output.dtype, + f"CLAIM_DTYPE:{output.entity}.{output.column}", + ) + columns[(output.entity, output.column)] = pd.Series( + table[output.column].to_numpy(dtype=np.int64, copy=True), + index=pd.Index(table[id_column].to_numpy(copy=True), name=id_column), + name=output.column, + dtype=output.dtype, + ) + return KernelResult( + columns=columns, + receipt={ + "phase": COMBINED_CLONE_PHASE, + "claimed_cells": sorted( + f"{entity}.{column}" for entity, column in owned + ), + }, + ) + + +def us_combined_survey_clone_nodes( + columns: Sequence[Owned], + *, + base: str, + source_channels: Sequence[str], + prefix: str = COMBINED_CLONE_PREFIX, +) -> tuple[Node, ...]: + """Declare the clone ``EXPAND`` and the claim node that owns its cells. + + Args: + columns: The base population version's full column inventory. Only the + four provenance columns of each entity are read; the inventory is + required so a missing or wrongly typed provenance column is a + declaration-time refusal rather than a run-time surprise. + base: The population version holding the combined native host. + source_channels: The exact source channels the host carries, sorted. + Both survey arms must be named; this is what the kernel then + requires the live rows to match. + prefix: Node-id prefix, so a caller may declare more than one profile. + + Returns: + ``(expand_node, claim_node)``. The claim node's population is the + expand node, so the claimed cells are owned inside the cloned version. + """ + + _require(bool(base), "BASE_REQUIRED") + channels = _validated_source_channels(tuple(source_channels)) + inventory = {(owned.entity, owned.column): owned for owned in columns} + _require(len(inventory) == len(tuple(columns)), "COLUMN_INVENTORY_REPEATS") + for entity in US_SCHEMA.entities: + for column in combined_clone_provenance_columns(entity): + owned = inventory.get((entity, column)) + _require(owned is not None, f"MISSING_PROVENANCE_COLUMN:{entity}.{column}") + cells = _expected_expand_cells() + for entity, column, dtype in cells: + _require( + inventory[(entity, column)].dtype == dtype, + f"CLONE_INDEX_DTYPE:{entity}.{column}", + ) + + expand_node = f"{prefix}" + claim_node = f"{prefix}.owned" + expand = Node( + id=expand_node, + kernel=USCombinedSurveyCloneExpandKernel.ref, + inputs=tuple( + Slice(entity, combined_clone_provenance_columns(entity)) + for entity in US_SCHEMA.entities + ), + params={ + "clone_attachment_fraction": COMBINED_CLONE_ATTACHMENT_FRACTION, + "clone_attachment_seed": COMBINED_CLONE_ATTACHMENT_SEED, + "clone_roles": COMBINED_CLONE_ROLES, + "expand_cells": cells, + "expand_weight_entity": COMBINED_CLONE_WEIGHT_ENTITY, + "expand_weight_kind": COMBINED_CLONE_WEIGHT_KIND.value, + "phase": COMBINED_CLONE_PHASE, + "profile": COMBINED_CLONE_PROFILE, + "release_eligible": False, + "source_channels": channels, + "weight_split_denominator": COMBINED_CLONE_WEIGHT_SPLIT_DENOMINATOR, + }, + structural=StructuralDelta.EXPAND, + base=base, + mass="conserve", + description=( + "Attach one whole-household PUF-detail clone to the combined " + "ASEC/ACS host and split every household weight across the pair." + ), + ) + claim = Node( + id=claim_node, + kernel=USCombinedSurveyCloneClaimKernel.ref, + outputs=tuple( + Owned(entity, column, dtype, rewrite=True) + for entity, column, dtype in cells + ), + population=expand_node, + params={"phase": COMBINED_CLONE_PHASE}, + description="Own the clone-index cells the clone stage materialized.", + ) + return (expand, claim) + + +def register_us_combined_survey_clone_kernels(registry: KernelRegistry) -> None: + """Register both kernels of the bounded combined-survey clone stage.""" + + registry.register(USCombinedSurveyCloneExpandKernel()) + registry.register(USCombinedSurveyCloneClaimKernel()) diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/graph_composed_asec_binding.py b/packages/microcosm-build/src/microcosm/build/us_runtime/graph_composed_asec_binding.py new file mode 100644 index 000000000..7ce6a9b24 --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/graph_composed_asec_binding.py @@ -0,0 +1,1404 @@ +"""Bind the carried prepared-ASEC evidence to the ASEC rows of a composed population. + +:mod:`.graph_composed_population` reads both authenticated sources whole and +carries the prepared arm's four typed evidence artifacts plus its own +``asec_frame_context`` unchanged. Those artifacts are **positional over the whole +prepared ASEC population**, not over the composed rows, so the accepted +prepared-slice kernels cannot attach: their contracts require an empty context +metadata, an empty mass history, a design household weight and prepared-receipt +rows equal to the population's rows. A harmonized composed population satisfies +none of them. Those refusals are real, are reproduced in the test module, and +nothing here weakens them. + +What this module adds is the missing edge and nothing else: one node that +resolves which composed rows came from the ASEC arm and which source row each +one is, and publishes that resolution as a typed artifact together with a +:class:`~.asec_current_money_selection.SelectedCurrentMoney` sliced to exactly +those source coordinates by the same reviewed selector the prepared slice uses. + +How rows are resolved +--------------------- +By source arm and the arm's own pre-remap identifier, never by composed row +position and never by the post-offset ``*_source_id``. For each entity the node +reads ``*_support_channel`` to find the arm and ``*_spine_source_id`` for the +arm's original ID, then locates each original ID in the arm-ordered identity the +carried evidence itself declares: ``income_observations``' ``person_id`` for +persons and ``housing_universe``' ``household_id`` for households. Both buffers +are bound to the prepared receipt before they are used, so that ordering is +receipt-authenticated rather than assumed. The resolution refuses an empty, +duplicated, unresolved or non-monotone mapping, and refuses a cohort +disagreement: the composed ``source_year`` must equal the source ``income_year`` +at the resolved positions, and the income artifact's own three declared +crosschecks must agree cell by cell. + +Nothing is assumed about where the arm's rows sit. They need not be a prefix, +need not be contiguous, and group tables may be sorted or interleaved with the +other arm's; the mask is read, never constructed from a row range. Every +reconstruction is then checked back against ``source_origin``: the origin +document's per-entity provenance digests are recomputed against the actual +composed tables, so a document that describes a different frame refuses. + +Why this module may read source provenance +------------------------------------------ +Resolving "which arm did this row come from" *is* this node's operation, so it +is a reviewed source-spine provenance owner. It applies no population treatment: +it writes one boolean arm-membership cell per bound entity and no measurement. +The measurements live in :mod:`.graph_composed_asec_measures`, consume this +node's typed artifact and its declared row mask, and stay source-blind. +""" + +from __future__ import annotations + +import json +import struct +from collections.abc import Iterable, Mapping, Sequence +from dataclasses import InitVar, dataclass + +import numpy as np +import pandas as pd + +from microcosm.frame import US_SCHEMA +from microcosm.graph import ( + ArtifactInput, + ArtifactOutput, + Capabilities, + Determinism, + KernelBase, + KernelContext, + KernelResult, + Node, + Numeric, + Owned, + Slice, + StructuralDelta, +) +from microcosm.graph.canonical import canonical_json + +from .asec_current_money import ( + HEADER_MAX_BYTES, + MAX_HOUSEHOLDS, + MAX_PERSONS, + _json, + _parse, + _sha, +) +from .asec_current_money_selection import ( + US_ASEC_CURRENT_MONEY_BODY_TYPE, + US_ASEC_PREPARED_RECEIPT_TYPE, + US_ASEC_SELECTED_MONEY_TYPE, + encode_selected_current_money, + parse_current_money_body, + select_current_money, +) +from .asec_prepared_source import PREPARED_SOURCE_KIND +from .graph_asec_income import ( + US_ASEC_INCOME_OBSERVATIONS_TYPE, + bind_income_observations, +) +from .graph_asec_prepared import PreparedGraphError, _artifact, _same_producer +from .graph_composed_contracts import ( + BIND_NODE as BIND_NODE, +) +from .graph_composed_contracts import ( + US_COMPOSED_ASEC_ARM_ROWS_TYPE as US_COMPOSED_ASEC_ARM_ROWS_TYPE, +) +from .graph_composed_contracts import ( + US_COMPOSED_ASEC_BINDING_TYPE as US_COMPOSED_ASEC_BINDING_TYPE, +) +from .graph_composed_population import ( + CREATE_NODE, + US_COMPOSED_SOURCE_ORIGIN_TYPE, + bind_composed_source_origin, +) +from .graph_context import US_FRAME_CONTEXT_TYPE, _decode, _mass_records, _row_identity +from .graph_housing_universe import ( + HOUSEHOLD_EVIDENCE_COLUMNS, + US_ASEC_HOUSING_UNIVERSE_TYPE, + bind_housing_universe, + verify_graph_housing_rows, +) +from .graph_implementation import ( + STAGE_DEPENDENCIES, + implementation_hash, + implementation_manifest, +) +from .stacked_spine import ACS_STACKED_SUPPORT_CHANNEL +from .support_provenance import ( + BASE_ASEC_SUPPORT_CHANNEL, + spine_source_id_column, + support_channel_column, + support_source_id_column, +) + +COMPOSED_ASEC_STAGE = "composed_asec_binding_v1" +COMPOSED_ASEC_PHASE = "bind_composed_asec_evidence" +COMPOSED_ASEC_DEPENDENCIES = STAGE_DEPENDENCIES[COMPOSED_ASEC_STAGE] + + +#: The arm this binding resolves. The other arm is named only so an unexpected +#: third channel refuses; no ACS row is read, written or measured here. +ARM_CHANNEL = BASE_ASEC_SUPPORT_CHANNEL +_OTHER_CHANNEL = ACS_STACKED_SUPPORT_CHANNEL + +#: The declared boolean arm-membership cell, entity-prefixed because ``Frame`` +#: requires column names to be globally unique across entity tables. Every +#: measurement node writes under this mask, so the executor — not a convention — +#: is what keeps the opposite arm's storage byte-identical: +#: ``population._patch_columns`` refuses any change outside an ``Owned`` mask, +#: missingness included. +ARM_ROW_SUFFIX = "composed_asec_row" +ARM_ROW_DTYPE = "bool" + + +def arm_row_column(entity: str) -> str: + """Return the entity-prefixed arm-membership cell name.""" + _require(entity in US_SCHEMA.entities, f"ARM_ROW_ENTITY:{entity}") + return f"{entity}_{ARM_ROW_SUFFIX}" + + +#: The entities a measurement of this arm addresses: persons, the households +#: whose carried housing evidence is verified against the source, and the SPM +#: units the corrected childcare leaf is grained on. +BOUND_ENTITIES = ("person", "household", "spm_unit") +#: The two entities the carried evidence declares an ordered source identity for. +_ID_ENTITIES = ("person", "household") + +BINDING_SCHEMA = "microcosm.us.composed-asec-binding.v1" + +#: The income artifact's own declared crosschecks, verified row by row between +#: the composed cells and the source buffers at the resolved positions. +PERSON_COORDINATE_CROSSCHECKS = ("source_household_id", "A_LINENO", "A_AGE") +COHORT_COLUMN = "source_year" + +#: Demographic columns a later calibration wants on both arms and which this +#: stage deliberately does not bind: each names the exact source column the +#: reviewed ASEC mapping needs, the mapping itself, and what the native ACS arm +#: uses instead. Nothing is imputed, aged or defaulted. +#: +#: Whether the prepared arm carries the source column is an **observed property +#: of the population in hand**, recorded per run rather than asserted: the +#: invented fixture parent has neither column, and the genuine prepared arm has +#: both. Presence is not a licence — binding either column still needs a +#: reviewed cross-arm mapping decision that this stage may not make — so what +#: fails closed here is an attempted **write** of one of these columns +#: (``ASEC_DEMOGRAPHIC_COLUMN_BOUND``), not the source column's presence. Keying +#: the refusal on presence would have refused every genuine run while proving +#: nothing about what this stage binds. +#: Each entry is (column, required source columns, diagnostic-only source +#: columns, the explicit source contract that would supply it, the mappings +#: deliberately not adopted as proof, the caveats on reading presence as +#: observation, the ACS arm's own observed mapping). +#: The required columns are the ones an *explicit* binding needs, which is not +#: the same as the columns some existing mapping happens to read: the root +#: adjudication of 2026-09-06 refused ``P_SEQ == 1`` as semantic proof of +#: headship and required sex to carry its allocation provenance, so those +#: mappings are recorded here as not adopted rather than as the requirement. +_UNBOUND_DEMOGRAPHICS = ( + ( + "is_female", + ("A_SEX", "AXSEX"), + (), + "asec_demographic_source: A_SEX printed codes 1 = Male and 2 = Female, " + "admitted only with AXSEX printed codes 0 = No change or 4 = Allocated; " + "any other token leaves the person unbound", + ( + ( + "cps_carried.derive_us_cps_carried_inputs (is_female = A_SEX == 2)", + "maps every token other than 2 onto male, so an unprinted code " + "would be admitted as male, and it reads no allocation flag", + ), + ), + ( + "A_SEX reaches the prepared person roster as a carried source " + "column, but its presence attests the column, not that every " + "delivered token is one of the two printed codes", + ), + "SEX == 2 on the native ACS arm, whose pinned dictionary prints the " + "same 1 = Male / 2 = Female convention", + ), + ( + "is_household_head", + ("A_EXPRRP",), + ("P_SEQ",), + "asec_demographic_source: A_EXPRRP printed codes 1 = Reference person " + "with relatives and 2 = Reference person without relatives, with " + "exactly one such person per household and every household relationship " + "token inside the printed named codes", + ( + ( + "relationship_inputs (is_household_head = P_SEQ == 1)", + "P_SEQ's dictionary entry labels no code as reference person, " + "and the ordering statement behind it is expressly limited to " + "the ASCII file while this arm is restored from the CSV member", + ), + ( + "A_FAMREL == 1", + "a family relationship scoped to primary-family membership, " + "which a subfamily reference person does not carry; household " + "and family roles stay separate", + ), + ( + "asec_pool relationship recode (A_LINENO == 1)", + "a separately assigned Basic-CPS roster line number with no " + "source guarantee of agreement with any ASEC sequence", + ), + ), + ( + "asec_pool._with_relationship_recode derives A_EXPRRP from " + "A_LINENO == 1 whenever the locked input omits it, so the column's " + "presence on a prepared arm does not certify an observed source " + "value for that cohort", + "the native ACS arm's A_EXPRRP is likewise derived by acs_pums " + "from RELSHIPP, and its group-quarters households carry the " + "nonrelative code 14 rather than any reference-person code", + ), + "RELSHIPP == 20 observed on the native ACS arm's housing units only; " + "RELSHIPP 37 and 38 are the group-quarters populations and carry no " + "observed reference person", + ), +) +#: The population columns this stage records as unbound, and therefore may not +#: own on any node. Public because the measurement module's own declarations are +#: checked against it. +UNBOUND_DEMOGRAPHIC_COLUMNS = tuple(column for column, *_rest in _UNBOUND_DEMOGRAPHICS) + +_ARTIFACT_TYPES = ( + ("frame_context", US_FRAME_CONTEXT_TYPE), + ("source_origin", US_COMPOSED_SOURCE_ORIGIN_TYPE), + ("asec_frame_context", US_FRAME_CONTEXT_TYPE), + ("current_money", US_ASEC_CURRENT_MONEY_BODY_TYPE), + ("prepared_receipt", US_ASEC_PREPARED_RECEIPT_TYPE), + ("housing_universe", US_ASEC_HOUSING_UNIVERSE_TYPE), + ("income_observations", US_ASEC_INCOME_OBSERVATIONS_TYPE), +) +#: The prepared context and the four evidence artifacts are positional over one +#: prepared population, so they must come from one producing node execution. +_SOURCE_EDGES = ( + "asec_frame_context", + "current_money", + "housing_universe", + "income_observations", + "prepared_receipt", +) +#: Every edge the composed CREATE produces, the origin document included. The +#: origin is positional over that same CREATE execution, so it is held to the +#: same producer identity as the evidence. Its content is independently +#: recomputed against the actual tables (``ORIGIN_DOES_NOT_DESCRIBE_THIS_FRAME``) +#: and that check is unchanged — but recomputation cannot tell two CREATE +#: executions describing identical tables apart, so the producer key the +#: document records would otherwise be carried unverified. +_CREATE_EDGES = (*_SOURCE_EDGES, "source_origin") + +_BINDING_KEYS = frozenset( + { + "arm", + "arm_channel", + "arm_row_columns", + "certified", + "claim", + "cohorts", + "composed", + "composed_is_whole_arm", + "demographics", + "inputs", + "origin", + "phase", + "prepared_arm_rows", + "release_eligible", + "resolution", + "schema", + } +) +_ARM_KEYS = frozenset({"rows", "composed_ordered_ids_sha256", "mask_sha256"}) +_ARM_ID_KEYS = _ARM_KEYS | { + "original_ordered_ids_sha256", + "source_positions_sha256", +} + + +def _require(condition: bool, reason: str) -> None: + if not condition: + raise PreparedGraphError(reason) + + +def refuse_unbound_demographic_writes(columns: Iterable[str]) -> None: + """Refuse if this stage would own a demographic its binding records as open. + + This is the fail-closed half of the recorded gap. The binding document says + ``is_female`` and ``is_household_head`` are not bound on this population; + that statement is only honest while no node of the stage writes them, so + every owned-column roster of the stage passes through here. A reviewed + mapping that started producing one of them would refuse at declaration time + rather than silently turning a recorded gap into an unreviewed binding. + """ + for column in columns: + _require( + column not in UNBOUND_DEMOGRAPHIC_COLUMNS, + f"ASEC_DEMOGRAPHIC_COLUMN_BOUND:{column}", + ) + + +def _is_digest(value: object) -> bool: + return ( + isinstance(value, str) + and len(value) == 64 + and not set(value) - set("0123456789abcdef") + ) + + +@dataclass(frozen=True) +class ComposedAsecBinding: + """One resolved arm binding: the row masks, the source rows and its document.""" + + document: Mapping[str, object] + context_document: Mapping[str, object] + masks: Mapping[str, np.ndarray] + positions: Mapping[str, np.ndarray] + original_ids: Mapping[str, np.ndarray] + income_years: np.ndarray + selected_money: bytes + + @property + def payload(self) -> bytes: + return canonical_json(self.document) + + +def arm_mask(table: pd.DataFrame, entity: str) -> np.ndarray: + """Which rows of a composed entity table came from the ASEC arm.""" + column = support_channel_column(entity) + _require(column in table, f"ARM_CHANNEL_COLUMN:{entity}") + channels = table[column] + _require(not bool(channels.isna().any()), f"ARM_CHANNEL_MISSING:{entity}") + values = channels.astype(str).to_numpy() + _require( + set(values.tolist()) <= {ARM_CHANNEL, _OTHER_CHANNEL}, + f"ARM_CHANNEL_UNKNOWN:{entity}", + ) + mask = values == ARM_CHANNEL + _require(bool(mask.any()), f"ARM_EMPTY:{entity}") + return mask + + +def _original_ids(table: pd.DataFrame, entity: str, mask: np.ndarray) -> np.ndarray: + """The arm's own pre-remap identifiers, never the post-offset source IDs.""" + column = spine_source_id_column(entity) + _require(column in table, f"ARM_SPINE_SOURCE_ID:{entity}") + return _int64_view(table.loc[mask, column], reason=f"ARM_SPINE_SOURCE_ID:{entity}") + + +def _resolved_positions( + original_ids: np.ndarray, arm_ids: np.ndarray, *, entity: str +) -> np.ndarray: + """Locate each original ID in the source's own ordered identity. + + Every failure mode refuses instead of dropping a row: a source identity that + repeats an ID is ambiguous, an original ID the source never declares is + unresolved, and a non-increasing mapping cannot be a whole-household slice of + the source buffers, which the reviewed selector requires to be strictly + ordered. The order is a property of the actual mapping, checked here; it is + never imposed by sorting the rows. + """ + _require(len(original_ids) > 0, f"BINDING_EMPTY:{entity}") + _require( + len(np.unique(arm_ids)) == len(arm_ids), f"BINDING_AMBIGUOUS_SOURCE:{entity}" + ) + _require( + len(np.unique(original_ids)) == len(original_ids), + f"BINDING_AMBIGUOUS_ROWS:{entity}", + ) + positions = pd.Index(arm_ids).get_indexer(original_ids).astype("int64") + _require(bool((positions >= 0).all()), f"BINDING_UNRESOLVED:{entity}") + _require( + len(positions) < 2 or bool((np.diff(positions) > 0).all()), + f"BINDING_NON_MONOTONE:{entity}", + ) + return positions + + +def _int64_view(series: pd.Series, *, reason: str) -> np.ndarray: + """Narrow the stack's nullable storage back to int64 on this arm's own rows. + + Stacking widens an arm-specific integer column to ``Int64`` so the opposite + arm's absence stays a declared null. On the arm's own rows nothing may be + missing and the reviewed verifiers compare native ``int64`` buffers, so the + narrowing is explicit and refuses rather than filling. + """ + _require(pd.api.types.is_integer_dtype(series.dtype), f"{reason}_DTYPE") + _require(not bool(series.isna().any()), f"{reason}_MISSING") + return series.to_numpy(dtype="int64") + + +def _arm_household_evidence( + household: pd.DataFrame, mask: np.ndarray, original_ids: np.ndarray +) -> pd.DataFrame: + """The arm's carried housing evidence restated under its own source identity. + + ``verify_graph_housing_rows`` compares native source buffers against a + household view keyed by ``household_id``. On a composed population that + column is the assembly-remapped identity, so this view restates the arm's own + pre-remap identity instead. The evidence columns themselves are carried + unchanged; only the stack's nullable storage is narrowed. + """ + view = pd.DataFrame(index=pd.RangeIndex(int(mask.sum()))) + view["household_id"] = original_ids + for column in HOUSEHOLD_EVIDENCE_COLUMNS: + _require(column in household, f"ARM_HOUSING_EVIDENCE:{column}") + view[column] = _int64_view( + household.loc[mask, column], reason=f"ARM_HOUSING_EVIDENCE:{column}" + ) + return view + + +def _arm_household_cohorts( + person: pd.DataFrame, + person_mask: np.ndarray, + income_years: np.ndarray, + composed_household_ids: np.ndarray, +) -> np.ndarray: + """One source income year per ASEC household; a mixed household refuses. + + The household evidence is cohort-scoped, so the housing verifier is given the + year the arm's own persons declare rather than the year the source buffer + would have supplied — a household whose members disagree is a real source + contract failure and must not be reduced away. + """ + membership = person.loc[ + person_mask, US_SCHEMA.membership_column("household") + ].to_numpy(dtype="int64") + pairs = pd.DataFrame({"household": membership, "year": income_years}) + distinct = pairs.drop_duplicates() + _require( + not bool(distinct["household"].duplicated().any()), "ARM_MIXED_YEAR_HOUSEHOLD" + ) + lookup = dict( + zip(distinct["household"].tolist(), distinct["year"].tolist(), strict=True) + ) + _require( + set(lookup) == set(composed_household_ids.tolist()), + "ARM_HOUSEHOLD_COHORT_COVERAGE", + ) + return np.asarray( + [lookup[int(value)] for value in composed_household_ids], dtype="int64" + ) + + +def _composed_document(payload: bytes) -> dict: + """Decode the composed population's typed context without relaxing anything. + + The prepared slice's ``_document`` refuses exactly the three things a + composed population always has — assembly metadata, a mass history and an + importance household weight — so it cannot be reused. This decoder keeps the + canonical-bytes, normative-metadata and mass-record validation that + ``graph_context`` owns and asserts the composed shape positively. + """ + document = _decode(payload) + _mass_records(document["mass_log"]) + _require(canonical_json(document) == payload, "COMPOSED_CONTEXT_CANONICAL") + _require(set(document["weight_sources"]) == {"household"}, "COMPOSED_WEIGHT_ENTITY") + return document + + +def _prepared_document(payload: bytes) -> dict: + """The prepared arm's own pre-stack context, which the evidence is ordered in.""" + document = _decode(payload) + _mass_records(document["mass_log"]) + _require(canonical_json(document) == payload, "PREPARED_CONTEXT_CANONICAL") + _require(document["metadata"] == {}, "PREPARED_CONTEXT_METADATA") + _require(document["mass_log"] == [], "PREPARED_CONTEXT_MASS_LOG") + _require( + document["weight_sources"] == {"household": "design"}, + "PREPARED_WEIGHT_AUTHORITY", + ) + return document + + +def _origin_entity_digests(tables: Mapping[str, pd.DataFrame]) -> dict: + """Recompute the origin document's per-entity mapping from the actual rows. + + ``bind_composed_source_origin`` deliberately verifies no digest against any + population, because it receives only bytes. A consumer that relies on the + mapping has to recompute it against the frame in hand; this is that + recomputation, in the producer's own order and encoding. + """ + digests = {} + for entity in US_SCHEMA.entities: + table = tables[entity] + channel_column = support_channel_column(entity) + spine_column = spine_source_id_column(entity) + source_column = support_source_id_column(entity) + for column in (channel_column, spine_column, source_column): + _require(column in table, f"ORIGIN_RECOMPUTE_COLUMN:{entity}.{column}") + channels = table[channel_column].astype(str).tolist() + spine_ids = table[spine_column].to_numpy(dtype="int64").tolist() + ids = table[source_column].to_numpy(dtype="int64").tolist() + outputs = ( + table[US_SCHEMA.entity_id_column(entity)].to_numpy(dtype="int64").tolist() + ) + digests[entity] = { + "rows_by_channel": { + channel: int(channels.count(channel)) + for channel in sorted(set(channels)) + }, + "ordered_channels_sha256": _sha(canonical_json(channels)), + "ordered_spine_source_ids_sha256": _sha(canonical_json(spine_ids)), + "ordered_source_to_output_sha256": _sha( + canonical_json( + [ + [channel, spine_id, source_id, output_id] + for channel, spine_id, source_id, output_id in zip( + channels, spine_ids, ids, outputs, strict=True + ) + ] + ) + ), + } + return digests + + +def _demographics(prepared_columns: Sequence[str], cohorts: Mapping[str, int]) -> dict: + """What a cross-arm demographic can honestly claim on this population today. + + ``age`` is bound through the reviewed corrected-leaf mapping whose only input + is the routing column ``A_AGE``. Its ASEC observation is the interview + household one year after the income year — the reference period the income + artifact declares and this node verifies cell by cell — while the native ACS + arm's ``age`` is observed in its own vintage. The two are **not** harmonized + here and no person is aged from any date; the periods are recorded per cohort + so a calibration decides rather than inherits an assumption. + + ``is_female`` and ``is_household_head`` stay unbound. What their entries say + about the source is **observed on the prepared arm actually in hand**, and + is stated both ways: which required source columns this population carries + and which it does not. That observation is evidence, not authority — + ``bound_by_this_stage`` is ``False`` either way, and + :func:`refuse_unbound_demographic_writes` is what keeps it true. + + Each entry also records the explicit source contract that would supply the + column, the mappings deliberately **not** adopted as proof of it, and the + diagnostic-only columns that are crosschecked against that contract and + never substituted for it. + """ + present = set(prepared_columns) + unbound = [] + for ( + column, + sources, + diagnostics, + contract, + not_adopted, + caveats, + acs_mapping, + ) in _UNBOUND_DEMOGRAPHICS: + found = sorted(name for name in sources if name in present) + absent = sorted(name for name in sources if name not in present) + unbound.append( + { + "column": column, + "required_asec_source_columns": list(sources), + "present_in_prepared_arm": found, + "absent_from_prepared_arm": absent, + "diagnostic_only_source_columns": list(diagnostics), + "diagnostic_only_present_in_prepared_arm": sorted( + name for name in diagnostics if name in present + ), + "diagnostic_only_is_semantic_proof": False, + "bound_by_this_stage": False, + "explicit_asec_source_contract": contract, + "asec_mappings_not_adopted": [ + {"mapping": mapping, "refusal": refusal} + for mapping, refusal in not_adopted + ], + "presence_certifies_observation": False, + "presence_caveats": list(caveats), + "acs_arm_mapping": acs_mapping, + "reason": ( + "required_asec_source_column_absent_from_this_prepared_population" + if absent + else "no_reviewed_cross_arm_mapping_decision_for_this_column" + ), + "resolution": "independent_source_decision_required", + } + ) + bound = [ + { + "column": "age", + "dtype": "float64", + "asec_source_column": "A_AGE", + "mapping": "derive_cps_carried_current_leaves", + "observed_period_kind": "interview_household_one_year_after_income_year", + "observed_period_by_cohort": { + year: {"income_year": int(year), "asec_survey_year": int(year) + 1} + for year in sorted(cohorts) + }, + "aged_from_a_date": False, + "cross_arm_period_harmonized": False, + "acs_arm_note": ( + "the native ACS arm's age is carried unchanged in its own " + "vintage and is never rewritten by this binding" + ), + } + ] + refuse_unbound_demographic_writes(item["column"] for item in bound) + return {"bound": bound, "unbound": unbound} + + +def resolve_composed_asec_binding( + tables: Mapping[str, pd.DataFrame], + *, + context_payload: bytes, + origin_payload: bytes, + prepared_context_payload: bytes, + money_payload: bytes, + receipt_payload: bytes, + housing_payload: bytes, + income_payload: bytes, + producers: Mapping[str, str], +) -> ComposedAsecBinding: + """Resolve the ASEC arm's composed rows against the carried source evidence.""" + context = _composed_document(context_payload) + origin = bind_composed_source_origin(origin_payload) + prepared_context = _prepared_document(prepared_context_payload) + receipt = json.loads(receipt_payload) + _require(canonical_json(receipt) == receipt_payload, "PREPARED_RECEIPT_JSON") + _require( + receipt["source_kind"] == PREPARED_SOURCE_KIND + and receipt["release_eligible"] is False, + "PREPARED_RECEIPT_SCHEMA", + ) + for entity in _ID_ENTITIES: + _require( + int(receipt["entity_rows"][entity]) + == int(prepared_context["entities"][entity]["rows"]), + f"PREPARED_EVIDENCE_ROWS:{entity}", + ) + income = bind_income_observations(income_payload, prepared_receipt=receipt) + universe = bind_housing_universe(housing_payload, prepared_receipt=receipt) + body = parse_current_money_body( + money_payload, + expected_header_sha256=receipt["money_header_sha256"], + expected_content_sha256=receipt["money_content_sha256"], + field_entities=tuple(tuple(item) for item in receipt["field_entities"]), + ) + _require( + body.person_rows == int(receipt["entity_rows"]["person"]) + and body.household_rows == int(receipt["entity_rows"]["household"]), + "BODY_ROW_ALIGNMENT", + ) + + for entity in US_SCHEMA.entities: + _require(entity in tables, f"COMPOSED_TABLE_MISSING:{entity}") + declared = context["entities"][entity] + actual = _row_identity(tables[entity], entity) + _require( + all(declared[key] == value for key, value in actual.items()), + f"COMPOSED_CONTEXT_IDENTITY:{entity}", + ) + _require( + _origin_entity_digests(tables) == origin["assembled"], + "ORIGIN_DOES_NOT_DESCRIBE_THIS_FRAME", + ) + + masks = {entity: arm_mask(tables[entity], entity) for entity in US_SCHEMA.entities} + alignment = origin["asec_evidence_alignment"] + for entity in US_SCHEMA.entities: + _require( + int(alignment["composed_rows"][entity]) == int(masks[entity].sum()), + f"ORIGIN_COMPOSED_ROWS:{entity}", + ) + whole = bool(alignment["composed_is_whole_arm"]) + _require( + whole == (alignment["composed_rows"] == alignment["arm_rows"]), + "ORIGIN_WHOLE_ARM_FLAG", + ) + for entity in _ID_ENTITIES: + _require( + int(alignment["evidence_rows"][entity]) + == int(receipt["entity_rows"][entity]), + f"ORIGIN_EVIDENCE_ROWS:{entity}", + ) + _require( + int(masks[entity].sum()) <= int(receipt["entity_rows"][entity]), + f"ARM_OVERFLOW:{entity}", + ) + + original = { + entity: _original_ids(tables[entity], entity, masks[entity]) + for entity in _ID_ENTITIES + } + arm_identity = { + "person": income.array("person_id").astype("int64"), + "household": universe.array("household_id").astype("int64"), + } + for entity in _ID_ENTITIES: + _require( + len(arm_identity[entity]) == int(receipt["entity_rows"][entity]), + f"ARM_IDENTITY_ROWS:{entity}", + ) + positions = { + entity: _resolved_positions( + original[entity], arm_identity[entity], entity=entity + ) + for entity in _ID_ENTITIES + } + + person = tables["person"] + income_years = _int64_view( + person.loc[masks["person"], COHORT_COLUMN], reason="ARM_COHORT" + ) + _require( + np.array_equal(income_years, income.array("income_year")[positions["person"]]), + "BINDING_COHORT_MISMATCH", + ) + for column in PERSON_COORDINATE_CROSSCHECKS: + _require(column in person, f"ARM_CROSSCHECK_COLUMN:{column}") + _require( + np.array_equal( + _int64_view( + person.loc[masks["person"], column], + reason=f"ARM_CROSSCHECK:{column}", + ), + income.array(column)[positions["person"]], + ), + f"BINDING_CROSSCHECK:{column}", + ) + composed_household_ids = ( + tables["household"] + .loc[masks["household"], US_SCHEMA.entity_id_column("household")] + .to_numpy(dtype="int64") + ) + verify_graph_housing_rows( + _arm_household_evidence( + tables["household"], masks["household"], original["household"] + ), + universe, + positions=positions["household"], + income_years=_arm_household_cohorts( + person, masks["person"], income_years, composed_household_ids + ), + ) + + identities = { + entity: _row_identity( + pd.DataFrame({US_SCHEMA.entity_id_column(entity): original[entity]}), entity + ) + for entity in _ID_ENTITIES + } + cohorts = { + str(int(year)): int((income_years == year).sum()) + for year in np.unique(income_years).tolist() + } + arm = {} + for entity in BOUND_ENTITIES: + table = tables[entity] + id_column = US_SCHEMA.entity_id_column(entity) + composed_ids = table.loc[masks[entity], id_column].to_numpy(dtype="int64") + record = { + "rows": int(masks[entity].sum()), + "composed_ordered_ids_sha256": _row_identity( + pd.DataFrame({id_column: composed_ids}), entity + )["ordered_ids_sha256"], + "mask_sha256": _sha(masks[entity].tobytes()), + } + if entity in _ID_ENTITIES: + record["original_ordered_ids_sha256"] = identities[entity][ + "ordered_ids_sha256" + ] + record["source_positions_sha256"] = _sha(positions[entity].tobytes()) + arm[entity] = record + + document = { + "schema": BINDING_SCHEMA, + "phase": COMPOSED_ASEC_PHASE, + "release_eligible": False, + "certified": False, + "claim": "engineering_binding_only_no_calibration_or_district_claim", + "arm_channel": ARM_CHANNEL, + "arm_row_columns": { + entity: arm_row_column(entity) for entity in BOUND_ENTITIES + }, + "resolution": { + "keys": ["support_channel", "spine_source_id", COHORT_COLUMN], + "source_identity": { + "household": "housing_universe.household_id", + "person": "income_observations.person_id", + }, + "crosschecks": list(PERSON_COORDINATE_CROSSCHECKS), + "uses_composed_row_position": False, + "uses_post_offset_source_id": False, + }, + "composed": { + entity: { + "rows": int(context["entities"][entity]["rows"]), + "ordered_ids_sha256": context["entities"][entity]["ordered_ids_sha256"], + } + for entity in US_SCHEMA.entities + }, + "arm": arm, + "prepared_arm_rows": { + entity: int(receipt["entity_rows"][entity]) for entity in _ID_ENTITIES + }, + "composed_is_whole_arm": whole, + "cohorts": cohorts, + "demographics": _demographics( + prepared_context["entities"]["person"]["columns"], cohorts + ), + "inputs": { + "asec_frame_context_sha256": _sha(prepared_context_payload), + "current_money_sha256": _sha(money_payload), + "frame_context_sha256": _sha(context_payload), + "housing_universe_sha256": _sha(housing_payload), + "income_observations_sha256": _sha(income_payload), + "money_content_sha256": receipt["money_content_sha256"], + "money_header_sha256": receipt["money_header_sha256"], + "prepared_receipt_sha256": _sha(receipt_payload), + "producers": {name: producers[name] for name in sorted(producers)}, + "source_origin_sha256": _sha(origin_payload), + }, + "origin": { + "preparation_sha256": origin["preparation_sha256"], + "sample_fraction": float(origin["sampling"]["sample_fraction"]), + "sample_seed": int(origin["sampling"]["sample_seed"]), + }, + } + payload = canonical_json(document) + selected = select_current_money( + body, + person_positions=positions["person"], + household_positions=positions["household"], + prepared_receipt_sha256=_sha(receipt_payload), + selection_sha256=_sha(payload), + person_identity_sha256=identities["person"]["ordered_ids_sha256"], + household_identity_sha256=identities["household"]["ordered_ids_sha256"], + ) + for entity in BOUND_ENTITIES: + context["entities"][entity]["columns"].append(arm_row_column(entity)) + return ComposedAsecBinding( + document=document, + context_document=context, + masks={entity: masks[entity] for entity in BOUND_ENTITIES}, + positions=positions, + original_ids=original, + income_years=income_years, + selected_money=encode_selected_current_money(selected), + ) + + +def bind_composed_asec_document(payload: bytes) -> dict: + """Decode the binding artifact as a typed shape, refusing a malformed one. + + Like the origin binder this checks canonical bytes and declared shape only. + It verifies no digest against any population; a consumer holding the frame + recomputes what it needs. + """ + document = json.loads(payload) + _require(canonical_json(document) == payload, "BINDING_CANONICAL") + _require(isinstance(document, dict), "BINDING_DOCUMENT") + _require(set(document) == _BINDING_KEYS, "BINDING_KEYS") + _require(document["schema"] == BINDING_SCHEMA, "BINDING_SCHEMA") + _require(document["phase"] == COMPOSED_ASEC_PHASE, "BINDING_PHASE") + _require(document["arm_channel"] == ARM_CHANNEL, "BINDING_ARM_CHANNEL") + _require( + document["arm_row_columns"] + == {entity: arm_row_column(entity) for entity in BOUND_ENTITIES}, + "BINDING_ARM_ROW_COLUMNS", + ) + _require( + document["release_eligible"] is False and document["certified"] is False, + "BINDING_RELEASE_CLAIM", + ) + _require( + set(document["composed"]) == set(US_SCHEMA.entities), + "BINDING_COMPOSED_ENTITIES", + ) + _require(set(document["arm"]) == set(BOUND_ENTITIES), "BINDING_ARM_ENTITIES") + for entity, record in document["arm"].items(): + expected = _ARM_ID_KEYS if entity in _ID_ENTITIES else _ARM_KEYS + _require(set(record) == expected, f"BINDING_ARM_KEYS:{entity}") + _require( + type(record["rows"]) is int and record["rows"] > 0, + f"BINDING_ARM_ROWS:{entity}", + ) + _require( + all( + _is_digest(value) + for key, value in record.items() + if key.endswith("_sha256") + ), + f"BINDING_ARM_DIGEST:{entity}", + ) + resolution = document["resolution"] + _require( + resolution["uses_composed_row_position"] is False + and resolution["uses_post_offset_source_id"] is False + and resolution["keys"] == ["support_channel", "spine_source_id", COHORT_COLUMN] + and resolution["crosschecks"] == list(PERSON_COORDINATE_CROSSCHECKS), + "BINDING_RESOLUTION", + ) + demographics = document["demographics"] + _require( + set(demographics) == {"bound", "unbound"} + and bool(demographics["bound"]) + and all(item["aged_from_a_date"] is False for item in demographics["bound"]), + "BINDING_DEMOGRAPHICS", + ) + # What the unbound entries must satisfy is that they are unbound and that + # they say the whole truth about the source, not that the source lacked the + # column: a genuine prepared arm does carry some of these, and a document + # recording that is correct. Every required column is accounted for exactly + # once, on one side or the other. A diagnostic-only column is held apart + # from the requirement and is never allowed to claim it proves anything. + _require( + [item["column"] for item in demographics["unbound"]] + == list(UNBOUND_DEMOGRAPHIC_COLUMNS) + and all( + item["bound_by_this_stage"] is False + and sorted( + [*item["present_in_prepared_arm"], *item["absent_from_prepared_arm"]] + ) + == sorted(item["required_asec_source_columns"]) + and not set(item["present_in_prepared_arm"]) + & set(item["absent_from_prepared_arm"]) + and item["diagnostic_only_is_semantic_proof"] is False + and not set(item["diagnostic_only_source_columns"]) + & set(item["required_asec_source_columns"]) + and set(item["diagnostic_only_present_in_prepared_arm"]) + <= set(item["diagnostic_only_source_columns"]) + and bool(item["explicit_asec_source_contract"]) + and bool(item["asec_mappings_not_adopted"]) + and item["presence_certifies_observation"] is False + and bool(item["presence_caveats"]) + and all( + bool(entry["mapping"]) and bool(entry["refusal"]) + for entry in item["asec_mappings_not_adopted"] + ) + for item in demographics["unbound"] + ), + "BINDING_UNBOUND_DEMOGRAPHICS", + ) + # Presence varies with the prepared roster. The required columns and their + # semantics do not: a transported document cannot redefine this registry. + declared_presence = [ + name + for item in demographics["unbound"] + for name in ( + *item["present_in_prepared_arm"], + *item["diagnostic_only_present_in_prepared_arm"], + ) + ] + _require( + demographics["unbound"] == _demographics(declared_presence, {})["unbound"], + "BINDING_UNBOUND_DEMOGRAPHICS", + ) + refuse_unbound_demographic_writes(item["column"] for item in demographics["bound"]) + _require( + all( + _is_digest(value) + for key, value in document["inputs"].items() + if key.endswith("_sha256") + ) + and all( + _is_digest(value) for value in document["inputs"]["producers"].values() + ), + "BINDING_INPUT_DIGESTS", + ) + _require( + type(document["composed_is_whole_arm"]) is bool + and bool(document["cohorts"]) + and all( + type(count) is int and count > 0 for count in document["cohorts"].values() + ), + "BINDING_COHORTS", + ) + return document + + +#: The resolved source rows, as a framed binary artifact rather than a JSON list. +#: The measurement nodes need the arm's original IDs and its source positions to +#: reconstruct the reviewed accounting, and they must get them without reading a +#: provenance column. This carries exactly those four int64 buffers, bound to the +#: binding document that already digests every one of them. +ARM_ROWS_MAGIC = b"MCCASECR\x01" +ARM_ROWS_KIND = "microcosm.us.composed_asec_arm_rows.v1" +_ARM_ROWS_BUFFERS = ( + ("person", "source_positions"), + ("person", "original_ids"), + ("household", "source_positions"), + ("household", "original_ids"), +) +_ARM_ROWS_HEADER_KEYS = frozenset( + { + "schema_version", + "artifact_kind", + "release_eligible", + "binding_sha256", + "buffers", + "rows", + "dtype", + } +) +ARM_ROWS_PAYLOAD_MAX_BYTES = ( + len(ARM_ROWS_MAGIC) + + 4 + + HEADER_MAX_BYTES + + 16 * (MAX_PERSONS + MAX_HOUSEHOLDS) + + 32 +) +_ARM_ROWS_TOKEN = object() + + +@dataclass(frozen=True) +class BoundComposedAsecArmRows: + """The arm's resolved source rows, admitted only against its binding document.""" + + header: bytes + buffers: tuple[bytes, ...] + _token: InitVar[object] = None + + def __post_init__(self, _token): + _require(_token is _ARM_ROWS_TOKEN, "ARM_ROWS_CONSTRUCTOR") + + def array(self, entity: str, name: str) -> np.ndarray: + _require((entity, name) in _ARM_ROWS_BUFFERS, "ARM_ROWS_BUFFER") + return np.frombuffer( + self.buffers[_ARM_ROWS_BUFFERS.index((entity, name))], dtype=" tuple[bytes, tuple[bytes, ...]]: + values = {"source_positions": positions, "original_ids": original_ids} + buffers = tuple( + np.ascontiguousarray(values[name][entity], dtype=" bytes: + """Frame the resolved source rows with a transport checksum over their bytes.""" + header, buffers = _arm_rows_parts( + positions, original_ids, binding_sha256=binding_sha256 + ) + payload = ( + ARM_ROWS_MAGIC + struct.pack(" BoundComposedAsecArmRows: + """Admit the resolved rows only against the binding document that digests them. + + Every buffer is re-digested here in the binding document's own encodings: the + original IDs through the graph context's ordered-identity digest and the + source positions through their raw bytes. A payload that does not reproduce + both, for the exact row counts the binding declares, is refused. + """ + _require( + type(payload) is bytes + and len(ARM_ROWS_MAGIC) + 4 + 32 < len(payload) <= ARM_ROWS_PAYLOAD_MAX_BYTES, + "ARM_ROWS_SIZE", + ) + _require(payload.startswith(ARM_ROWS_MAGIC), "ARM_ROWS_MAGIC") + _require(_sha(payload[:-32]) == payload[-32:].hex(), "ARM_ROWS_CHECKSUM") + size = struct.unpack_from("= 0).all()) + and (len(places) < 2 or bool((np.diff(places) > 0).all())), + f"ARM_ROWS_ORDER:{entity}", + ) + declared = binding_document["arm"][entity] + _require( + _row_identity( + pd.DataFrame({US_SCHEMA.entity_id_column(entity): ids}), entity + )["ordered_ids_sha256"] + == declared["original_ordered_ids_sha256"], + f"ARM_ROWS_IDENTITY:{entity}", + ) + _require( + _sha(places.tobytes()) == declared["source_positions_sha256"], + f"ARM_ROWS_POSITIONS:{entity}", + ) + return bound + + +def arm_row_declarations() -> tuple[Owned, ...]: + """The boolean arm-membership cells this node owns, in one canonical order.""" + return tuple( + Owned(entity, arm_row_column(entity), ARM_ROW_DTYPE) + for entity in BOUND_ENTITIES + ) + + +def bind_node_inputs() -> tuple[Slice, ...]: + """The declared views the resolution reads, and nothing else.""" + provenance = tuple( + ( + support_channel_column(entity), + spine_source_id_column(entity), + support_source_id_column(entity), + ) + for entity in US_SCHEMA.entities + ) + extra = { + "person": (COHORT_COLUMN, *PERSON_COORDINATE_CROSSCHECKS), + "household": HOUSEHOLD_EVIDENCE_COLUMNS, + } + return tuple( + Slice(entity, (*columns, *extra.get(entity, ()))) + for entity, columns in zip(US_SCHEMA.entities, provenance, strict=True) + ) + + +def bind_node_artifact_inputs(*, population_context: str) -> tuple[ArtifactInput, ...]: + """The population's own context plus the composed CREATE's carried evidence.""" + return ( + ArtifactInput( + "frame_context", population_context, "frame_context", US_FRAME_CONTEXT_TYPE + ), + *( + ArtifactInput(name, CREATE_NODE, name, kind) + for name, kind in _ARTIFACT_TYPES + if name != "frame_context" + ), + ) + + +_BIND_ARTIFACTS = ( + ArtifactOutput("frame_context", US_FRAME_CONTEXT_TYPE), + ArtifactOutput("asec_binding", US_COMPOSED_ASEC_BINDING_TYPE), + ArtifactOutput("arm_rows", US_COMPOSED_ASEC_ARM_ROWS_TYPE), + ArtifactOutput("selected_current_money", US_ASEC_SELECTED_MONEY_TYPE), +) + + +def composed_asec_bind_node(*, population: str, population_context: str) -> Node: + """Declare the one binding node over an already composed population version.""" + return Node( + id=BIND_NODE, + kernel=USComposedAsecBindKernel.ref, + population=population, + inputs=bind_node_inputs(), + outputs=arm_row_declarations(), + params={"phase": COMPOSED_ASEC_PHASE}, + artifact_inputs=bind_node_artifact_inputs( + population_context=population_context + ), + artifact_outputs=_BIND_ARTIFACTS, + ) + + +class USComposedAsecBindKernel(KernelBase): + """Resolve the ASEC arm's composed rows and slice its evidence to them.""" + + ref = "us.composed_population.asec_bind@1" + capabilities = Capabilities( + determinism=Determinism.DETERMINISTIC, + numeric=Numeric.PLATFORM_BITWISE, + dependencies=COMPOSED_ASEC_DEPENDENCIES, + ) + + def implementation_hash(self) -> str: + return implementation_hash(COMPOSED_ASEC_STAGE) + + def run(self, context: KernelContext) -> KernelResult: + node = context.node + _require(node.kernel == self.ref, "NODE_KERNEL") + _require(set(context.params) == {"phase"}, "NODE_PARAMS") + _require(context.params["phase"] == COMPOSED_ASEC_PHASE, "NODE_PHASE") + _require(node.structural is StructuralDelta.NONE, "NODE_STRUCTURAL") + _require(not node.sources, "NODE_SOURCES") + _require(node.inputs == bind_node_inputs(), "NODE_SLICES") + _require(node.outputs == arm_row_declarations(), "NODE_OUTPUTS") + _require( + set(context.artifacts) == {name for name, _kind in _ARTIFACT_TYPES}, + "NODE_ARTIFACT_INPUTS", + ) + _require(node.artifact_outputs == _BIND_ARTIFACTS, "NODE_ARTIFACT_OUTPUTS") + _require( + { + edge.producer + for edge in node.artifact_inputs + if edge.name in _CREATE_EDGES + } + == {CREATE_NODE}, + "NODE_EVIDENCE_PRODUCER", + ) + values = { + name: _artifact(context, name, kind) for name, kind in _ARTIFACT_TYPES + } + create_producer = _same_producer(context, _CREATE_EDGES) + binding = resolve_composed_asec_binding( + {entity: context.tables[entity] for entity in US_SCHEMA.entities}, + context_payload=values["frame_context"].payload, + origin_payload=values["source_origin"].payload, + prepared_context_payload=values["asec_frame_context"].payload, + money_payload=values["current_money"].payload, + receipt_payload=values["prepared_receipt"].payload, + housing_payload=values["housing_universe"].payload, + income_payload=values["income_observations"].payload, + producers={ + "population": values["frame_context"].producer_key, + "prepared_source": create_producer, + "source_origin": values["source_origin"].producer_key, + }, + ) + columns = {} + for entity in BOUND_ENTITIES: + id_column = US_SCHEMA.entity_id_column(entity) + index = pd.Index( + context.tables[entity][id_column].to_numpy(), name=id_column + ) + columns[(entity, arm_row_column(entity))] = pd.Series( + binding.masks[entity], index=index, dtype=ARM_ROW_DTYPE + ) + payload = binding.payload + return KernelResult( + columns=columns, + artifacts={ + "frame_context": canonical_json(binding.context_document), + "asec_binding": payload, + "arm_rows": encode_composed_asec_arm_rows( + binding.positions, + binding.original_ids, + binding_sha256=_sha(payload), + ), + "selected_current_money": binding.selected_money, + }, + receipt={ + "phase": COMPOSED_ASEC_PHASE, + "implementation": implementation_manifest(COMPOSED_ASEC_STAGE), + "binding_sha256": _sha(payload), + "selected_money_sha256": _sha(binding.selected_money), + "arm_rows": { + entity: int(binding.masks[entity].sum()) + for entity in BOUND_ENTITIES + }, + "cohorts": binding.document["cohorts"], + "demographics": binding.document["demographics"], + "release_eligible": False, + "certified": False, + }, + ) + + +__all__ = [ + "ARM_CHANNEL", + "ARM_ROW_DTYPE", + "ARM_ROW_SUFFIX", + "BINDING_SCHEMA", + "BIND_NODE", + "BOUND_ENTITIES", + "COHORT_COLUMN", + "COMPOSED_ASEC_DEPENDENCIES", + "COMPOSED_ASEC_PHASE", + "COMPOSED_ASEC_STAGE", + "PERSON_COORDINATE_CROSSCHECKS", + "UNBOUND_DEMOGRAPHIC_COLUMNS", + "US_COMPOSED_ASEC_ARM_ROWS_TYPE", + "US_COMPOSED_ASEC_BINDING_TYPE", + "BoundComposedAsecArmRows", + "ComposedAsecBinding", + "USComposedAsecBindKernel", + "arm_mask", + "arm_row_column", + "arm_row_declarations", + "bind_composed_asec_arm_rows", + "bind_composed_asec_document", + "bind_node_artifact_inputs", + "bind_node_inputs", + "composed_asec_bind_node", + "encode_composed_asec_arm_rows", + "refuse_unbound_demographic_writes", + "resolve_composed_asec_binding", +] diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/graph_composed_asec_measures.py b/packages/microcosm-build/src/microcosm/build/us_runtime/graph_composed_asec_measures.py new file mode 100644 index 000000000..a945506b4 --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/graph_composed_asec_measures.py @@ -0,0 +1,652 @@ +"""Corrected ASEC monetary and demographic measurements over composed ASEC rows. + +Two pure nodes hang off :mod:`.graph_composed_asec_binding`. Each writes only +under the binding's declared boolean arm mask, so the executor holds every +native ACS cell — value and missingness alike — byte-identical: a masked +``Owned`` is checked by ``population._patch_columns``, which refuses any change +to non-owned storage. Where a column is new the opposite arm keeps a declared +null; nothing is inferred as zero. + +Neither kernel reads a source-channel or spine-source-ID column. They receive +the arm's rows through the declared mask and the arm's source coordinates +through the typed ``arm_rows`` artifact, both already resolved and digested by +the binding node. The arithmetic is the accepted one, unchanged and imported: +:func:`~.cps_carried_current.derive_cps_carried_current_leaves` for the +corrected leaves (which is also where ``age`` comes from, mapped from ``A_AGE``) +and :func:`~.graph_asec_income.derive_reported_income` for the six reported +observations. Raw source observations and their quality/zero/allocation axes +stay inside those artifacts; only the computed quantities become cells. + +Nothing here is a release, a calibration, a transfer or a score. +""" + +from __future__ import annotations + +import json + +import numpy as np +import pandas as pd + +from microcosm.frame import US_SCHEMA +from microcosm.graph import ( + ArtifactInput, + ArtifactOutput, + Capabilities, + Determinism, + Graph, + KernelBase, + KernelContext, + KernelRegistry, + KernelResult, + Node, + Numeric, + Owned, + Slice, + StructuralDelta, +) +from microcosm.graph.canonical import canonical_json + +from .asec_current_money import _sha +from .asec_current_money_selection import ( + US_ASEC_PREPARED_RECEIPT_TYPE, + US_ASEC_SELECTED_MONEY_TYPE, + decode_selected_current_money, +) +from .cps_carried_current import ( + CPS_CARRIED_CURRENT_PERSON_LEAVES, + CPS_CARRIED_CURRENT_ROUTING_COLUMNS, + CPS_CARRIED_CURRENT_SPM_UNIT_LEAVES, + cps_carried_current_leaf_contract, + derive_cps_carried_current_leaves, +) +from .graph_asec_income import ( + RESULT_COLUMNS, + US_ASEC_INCOME_OBSERVATIONS_TYPE, + US_ASEC_REPORTED_INCOME_TYPE, + bind_income_observations, + derive_reported_income, + encode_reported_income, +) +from .graph_asec_prepared import PreparedGraphError, _artifact, _same_producer +from .graph_composed_asec_binding import ( + BIND_NODE, + COHORT_COLUMN, + COMPOSED_ASEC_DEPENDENCIES, + COMPOSED_ASEC_PHASE, + COMPOSED_ASEC_STAGE, + US_COMPOSED_ASEC_ARM_ROWS_TYPE, + US_COMPOSED_ASEC_BINDING_TYPE, + USComposedAsecBindKernel, + arm_row_column, + bind_composed_asec_arm_rows, + bind_composed_asec_document, + composed_asec_bind_node, + refuse_unbound_demographic_writes, +) +from .graph_composed_contracts import ( + LEAVES_NODE as LEAVES_NODE, +) +from .graph_composed_contracts import ( + REPORTED_INCOME_NODE as REPORTED_INCOME_NODE, +) +from .graph_composed_population import ( + COMPOSED_SOURCES, + CREATE_NODE, + GEOGRAPHY_PHASE, + HARMONIZE_NODE, + composed_population_nodes, + composed_population_registry, +) +from .graph_context import US_FRAME_CONTEXT_TYPE, _decode, _mass_records, _row_identity +from .graph_geography import LOOKUP_SOURCES, us_geography_nodes +from .graph_implementation import implementation_hash, implementation_manifest + +LEAF_DTYPE = "float64" + +#: Corrected leaves the native ACS arm already carries under the same name. +#: On those the masked write fills the ASEC rows and the executor holds every +#: ACS cell byte-identical; on every other leaf the column is new and the ACS +#: rows stay declared-null. Any other incumbent is unreviewed and refuses. +ACS_SHARED_LEAVES = ( + "age", + "employment_income_before_lsr", + "self_employment_income_before_lsr", +) + + +def _require(condition: bool, reason: str) -> None: + if not condition: + raise PreparedGraphError(reason) + + +def _leaf_coordinates() -> tuple[tuple[str, str], ...]: + return ( + *(("person", name) for name in CPS_CARRIED_CURRENT_PERSON_LEAVES), + *(("spm_unit", name) for name in CPS_CARRIED_CURRENT_SPM_UNIT_LEAVES), + ) + + +def composed_leaf_declarations() -> tuple[Owned, ...]: + """The corrected-leaf cells, owned only at the bound arm's rows. + + The binding document records ``is_female`` and ``is_household_head`` as + unbound on this population. That record stays honest only while no node of + the stage writes them, so the declaration itself refuses a leaf roster that + would (``ASEC_DEMOGRAPHIC_COLUMN_BOUND``) rather than letting a reviewed + mapping quietly turn a recorded gap into an unreviewed binding. + """ + coordinates = _leaf_coordinates() + refuse_unbound_demographic_writes(column for _entity, column in coordinates) + return tuple( + Owned(entity, column, LEAF_DTYPE, rows=arm_row_column(entity)) + for entity, column in coordinates + ) + + +def composed_reported_income_declarations() -> tuple[Owned, ...]: + """The six reported-income cells, owned only at the bound arm's rows.""" + refuse_unbound_demographic_writes(RESULT_COLUMNS) + return tuple( + Owned("person", name, LEAF_DTYPE, rows=arm_row_column("person")) + for name in RESULT_COLUMNS + ) + + +def _document(value) -> dict: + """Decode the producing node's context, keeping every graph-context check.""" + document = _decode(value.payload) + _mass_records(document["mass_log"]) + _require(canonical_json(document) == value.payload, "CONTEXT_CANONICAL") + return document + + +def _bound_population(document: dict, binding: dict) -> None: + """The context and the binding must describe one composed population.""" + for entity in US_SCHEMA.entities: + declared = document["entities"][entity] + recorded = binding["composed"][entity] + _require( + declared["rows"] == recorded["rows"] + and declared["ordered_ids_sha256"] == recorded["ordered_ids_sha256"], + f"BINDING_POPULATION_IDENTITY:{entity}", + ) + for entity, column in binding["arm_row_columns"].items(): + _require( + column == arm_row_column(entity) + and column in document["entities"][entity]["columns"], + f"BINDING_ARM_COLUMN:{entity}", + ) + + +def _arm_rows(table: pd.DataFrame, entity: str, binding: dict) -> np.ndarray: + """The masked view's own ids, checked against the binding's recorded arm.""" + id_column = US_SCHEMA.entity_id_column(entity) + ids = table[id_column].to_numpy(dtype="int64") + recorded = binding["arm"][entity] + _require(len(ids) == recorded["rows"], f"ARM_VIEW_ROWS:{entity}") + _require( + _row_identity(pd.DataFrame({id_column: ids}), entity)["ordered_ids_sha256"] + == recorded["composed_ordered_ids_sha256"], + f"ARM_VIEW_IDENTITY:{entity}", + ) + column = arm_row_column(entity) + _require( + column in table and bool(table[column].to_numpy().all()), + f"ARM_VIEW_MASK:{entity}", + ) + return ids + + +def _selected_money(binding: dict, prepared: dict, values): + """Decode the arm's money slice against the binding it was cut for.""" + selected = decode_selected_current_money( + values["selected_current_money"].payload, + expected_parent_header_sha256=prepared["money_header_sha256"], + expected_parent_content_sha256=prepared["money_content_sha256"], + expected_prepared_receipt_sha256=_sha(values["prepared_receipt"].payload), + expected_selection_sha256=_sha(values["asec_binding"].payload), + ) + header = selected.header_data + _require( + header["person_identity_sha256"] + == binding["arm"]["person"]["original_ordered_ids_sha256"] + and header["household_identity_sha256"] + == binding["arm"]["household"]["original_ordered_ids_sha256"], + "SELECTED_ARM_COORDINATES", + ) + _require( + selected.person_rows == binding["arm"]["person"]["rows"] + and selected.household_rows == binding["arm"]["household"]["rows"], + "SELECTED_ARM_ROWS", + ) + return selected + + +def _binding(values) -> dict: + binding = bind_composed_asec_document(values["asec_binding"].payload) + _require( + binding["inputs"]["prepared_receipt_sha256"] + == _sha(values["prepared_receipt"].payload), + "BINDING_PREPARED_RECEIPT", + ) + _require( + binding["inputs"]["producers"]["prepared_source"] + == values["prepared_receipt"].producer_key, + "BINDING_SOURCE_PRODUCER", + ) + return binding + + +def _prepared(value) -> dict: + """The producing CREATE's own receipt, re-read as canonical bytes.""" + document = json.loads(value.payload) + _require(canonical_json(document) == value.payload, "PREPARED_RECEIPT_JSON") + _require(document["release_eligible"] is False, "PREPARED_RECEIPT_SCHEMA") + return document + + +def _new_columns(document: dict, coordinates) -> None: + """Record only the coordinates this stage actually adds to the population.""" + for entity, column in coordinates: + columns = document["entities"][entity]["columns"] + if column not in columns: + columns.append(column) + + +class USComposedAsecLeavesKernel(KernelBase): + """Derive the corrected monetary and age leaves over the bound arm's rows.""" + + ref = "us.composed_population.asec_cps_carried_current@1" + capabilities = Capabilities( + determinism=Determinism.DETERMINISTIC, + numeric=Numeric.PLATFORM_BITWISE, + dependencies=COMPOSED_ASEC_DEPENDENCIES, + ) + + def implementation_hash(self) -> str: + return implementation_hash(COMPOSED_ASEC_STAGE) + + def run(self, context: KernelContext) -> KernelResult: + node = context.node + _require(node.kernel == self.ref, "NODE_KERNEL") + _require(set(context.params) == {"phase"}, "NODE_PARAMS") + _require(context.params["phase"] == COMPOSED_ASEC_PHASE, "NODE_PHASE") + _require(node.structural is StructuralDelta.NONE, "NODE_STRUCTURAL") + _require(not node.sources, "NODE_SOURCES") + _require(node.inputs == _leaves_inputs(), "NODE_SLICES") + _require(node.outputs == composed_leaf_declarations(), "NODE_OUTPUTS") + _require( + set(context.artifacts) + == { + "frame_context", + "asec_binding", + "arm_rows", + "selected_current_money", + "prepared_receipt", + }, + "NODE_ARTIFACT_INPUTS", + ) + _require( + node.artifact_outputs + == (ArtifactOutput("frame_context", US_FRAME_CONTEXT_TYPE),), + "NODE_ARTIFACT_OUTPUTS", + ) + values = { + name: _artifact(context, name, kind) + for name, kind in ( + ("frame_context", US_FRAME_CONTEXT_TYPE), + ("asec_binding", US_COMPOSED_ASEC_BINDING_TYPE), + ("arm_rows", US_COMPOSED_ASEC_ARM_ROWS_TYPE), + ("selected_current_money", US_ASEC_SELECTED_MONEY_TYPE), + ("prepared_receipt", US_ASEC_PREPARED_RECEIPT_TYPE), + ) + } + _same_producer( + context, + ("frame_context", "asec_binding", "arm_rows", "selected_current_money"), + ) + binding = _binding(values) + prepared = _prepared(values["prepared_receipt"]) + document = _document(values["frame_context"]) + _bound_population(document, binding) + incumbent = sorted( + f"{entity}.{column}" + for entity, column in _leaf_coordinates() + if column in document["entities"][entity]["columns"] + ) + _require( + set(incumbent) <= {f"person.{name}" for name in ACS_SHARED_LEAVES}, + f"UNREVIEWED_LEAF_INCUMBENT:{incumbent}", + ) + person, spm_unit = context.tables["person"], context.tables["spm_unit"] + person_ids = _arm_rows(person, "person", binding) + spm_ids = _arm_rows(spm_unit, "spm_unit", binding) + selected = _selected_money(binding, prepared, values) + rows = bind_composed_asec_arm_rows( + values["arm_rows"].payload, binding_document=binding + ) + _require( + len(rows.array("person", "original_ids")) == len(person_ids), + "ARM_ROWS_PERSON_ALIGNMENT", + ) + leaves = derive_cps_carried_current_leaves( + selected, + routing=person.loc[:, list(CPS_CARRIED_CURRENT_ROUTING_COLUMNS)], + spm_membership=person[US_SCHEMA.membership_column("spm_unit")].to_numpy( + dtype="int64" + ), + spm_ids=spm_ids, + ) + person_index = pd.Index(person_ids, name=US_SCHEMA.entity_id_column("person")) + spm_index = pd.Index(spm_ids, name=US_SCHEMA.entity_id_column("spm_unit")) + columns = { + ("person", name): pd.Series(values_, index=person_index, dtype=LEAF_DTYPE) + for name, values_ in leaves.person.items() + } + columns.update( + { + ("spm_unit", name): pd.Series( + values_, index=spm_index, dtype=LEAF_DTYPE + ) + for name, values_ in leaves.spm_unit.items() + } + ) + _new_columns(document, _leaf_coordinates()) + return KernelResult( + columns=columns, + artifacts={"frame_context": canonical_json(document)}, + receipt={ + "phase": COMPOSED_ASEC_PHASE, + "implementation": implementation_manifest(COMPOSED_ASEC_STAGE), + "contract": cps_carried_current_leaf_contract(), + "binding_sha256": _sha(values["asec_binding"].payload), + "selected_money_sha256": _sha(values["selected_current_money"].payload), + "arm_person_rows": int(len(person_ids)), + "arm_spm_unit_rows": int(len(spm_ids)), + "acs_shared_incumbents": incumbent, + "demographics": binding["demographics"], + "release_eligible": False, + "certified": False, + }, + ) + + +class USComposedAsecReportedIncomeKernel(KernelBase): + """Reconstruct the six reported observations over the bound arm's rows.""" + + ref = "us.composed_population.asec_reported_income@1" + capabilities = Capabilities( + determinism=Determinism.DETERMINISTIC, + numeric=Numeric.PLATFORM_BITWISE, + dependencies=COMPOSED_ASEC_DEPENDENCIES, + ) + + def implementation_hash(self) -> str: + return implementation_hash(COMPOSED_ASEC_STAGE) + + def run(self, context: KernelContext) -> KernelResult: + node = context.node + _require(node.kernel == self.ref, "NODE_KERNEL") + _require(set(context.params) == {"phase"}, "NODE_PARAMS") + _require(context.params["phase"] == COMPOSED_ASEC_PHASE, "NODE_PHASE") + _require(node.structural is StructuralDelta.NONE, "NODE_STRUCTURAL") + _require(not node.sources, "NODE_SOURCES") + _require(node.inputs == _reported_income_inputs(), "NODE_SLICES") + _require( + node.outputs == composed_reported_income_declarations(), "NODE_OUTPUTS" + ) + _require( + set(context.artifacts) + == { + "frame_context", + "asec_binding", + "arm_rows", + "selected_current_money", + "prepared_receipt", + "income_observations", + }, + "NODE_ARTIFACT_INPUTS", + ) + _require( + node.artifact_outputs + == ( + ArtifactOutput("reported_income", US_ASEC_REPORTED_INCOME_TYPE), + ArtifactOutput("frame_context", US_FRAME_CONTEXT_TYPE), + ), + "NODE_ARTIFACT_OUTPUTS", + ) + values = { + name: _artifact(context, name, kind) + for name, kind in ( + ("frame_context", US_FRAME_CONTEXT_TYPE), + ("asec_binding", US_COMPOSED_ASEC_BINDING_TYPE), + ("arm_rows", US_COMPOSED_ASEC_ARM_ROWS_TYPE), + ("selected_current_money", US_ASEC_SELECTED_MONEY_TYPE), + ("prepared_receipt", US_ASEC_PREPARED_RECEIPT_TYPE), + ("income_observations", US_ASEC_INCOME_OBSERVATIONS_TYPE), + ) + } + binding_producer = _same_producer( + context, ("asec_binding", "arm_rows", "selected_current_money") + ) + source_producer = _same_producer( + context, ("prepared_receipt", "income_observations") + ) + binding = _binding(values) + prepared = _prepared(values["prepared_receipt"]) + document = _document(values["frame_context"]) + _bound_population(document, binding) + _require( + not set(RESULT_COLUMNS) & set(document["entities"]["person"]["columns"]), + "OWNED_LEAF_INCUMBENT", + ) + person = context.tables["person"] + person_ids = _arm_rows(person, "person", binding) + selected = _selected_money(binding, prepared, values) + rows = bind_composed_asec_arm_rows( + values["arm_rows"].payload, binding_document=binding + ) + income = bind_income_observations( + values["income_observations"].payload, prepared_receipt=prepared + ) + _require( + person[COHORT_COLUMN].dtype == np.dtype("int64"), "ARM_SOURCE_YEAR_DTYPE" + ) + result = derive_reported_income( + selected, + income, + # The arm's own original person identity, resolved and digested by + # the binding node; the composed row ids never enter the accounting. + person_ids=rows.array("person", "original_ids"), + income_years=person[COHORT_COLUMN].to_numpy(dtype="int64"), + person_positions_sha256=binding["arm"]["person"]["source_positions_sha256"], + prepared_receipt_sha256=_sha(values["prepared_receipt"].payload), + selection_sha256=_sha(values["asec_binding"].payload), + selected_money_sha256=_sha(values["selected_current_money"].payload), + income_payload_sha256=_sha(values["income_observations"].payload), + source_producer_key=source_producer, + selection_producer_key=binding_producer, + frame_context_sha256=_sha(values["frame_context"].payload), + ) + index = pd.Index(person_ids, name=US_SCHEMA.entity_id_column("person")) + columns = { + ("person", name): pd.Series( + result.array(name), index=index, dtype=LEAF_DTYPE + ) + for name in RESULT_COLUMNS + } + _new_columns(document, tuple(("person", name) for name in RESULT_COLUMNS)) + payload = encode_reported_income(result) + return KernelResult( + columns=columns, + artifacts={ + "reported_income": payload, + "frame_context": canonical_json(document), + }, + receipt={ + "phase": COMPOSED_ASEC_PHASE, + "implementation": implementation_manifest(COMPOSED_ASEC_STAGE), + "accounting_sha256": _sha(payload), + "binding_sha256": _sha(values["asec_binding"].payload), + "arm_person_rows": int(len(person_ids)), + "release_eligible": False, + "certified": False, + }, + ) + + +def _leaves_inputs() -> tuple[Slice, ...]: + return ( + Slice( + "person", + (*CPS_CARRIED_CURRENT_ROUTING_COLUMNS, arm_row_column("person")), + rows=arm_row_column("person"), + ), + Slice( + "spm_unit", + (arm_row_column("spm_unit"),), + rows=arm_row_column("spm_unit"), + ), + ) + + +def _reported_income_inputs() -> tuple[Slice, ...]: + return ( + Slice( + "person", + (COHORT_COLUMN, arm_row_column("person")), + rows=arm_row_column("person"), + ), + ) + + +def composed_asec_measure_nodes(*, population: str, population_context: str): + """Bind the arm, then the corrected leaves, then the reported observations.""" + bind = composed_asec_bind_node( + population=population, population_context=population_context + ) + binding_artifacts = ( + ArtifactInput( + "asec_binding", BIND_NODE, "asec_binding", US_COMPOSED_ASEC_BINDING_TYPE + ), + ArtifactInput( + "arm_rows", BIND_NODE, "arm_rows", US_COMPOSED_ASEC_ARM_ROWS_TYPE + ), + ArtifactInput( + "selected_current_money", + BIND_NODE, + "selected_current_money", + US_ASEC_SELECTED_MONEY_TYPE, + ), + ) + prepared_receipt = ArtifactInput( + "prepared_receipt", + CREATE_NODE, + "prepared_receipt", + US_ASEC_PREPARED_RECEIPT_TYPE, + ) + leaves = Node( + id=LEAVES_NODE, + kernel=USComposedAsecLeavesKernel.ref, + population=population, + inputs=_leaves_inputs(), + outputs=composed_leaf_declarations(), + params={"phase": COMPOSED_ASEC_PHASE}, + artifact_inputs=( + ArtifactInput( + "frame_context", BIND_NODE, "frame_context", US_FRAME_CONTEXT_TYPE + ), + *binding_artifacts, + prepared_receipt, + ), + artifact_outputs=(ArtifactOutput("frame_context", US_FRAME_CONTEXT_TYPE),), + ) + reported = Node( + id=REPORTED_INCOME_NODE, + kernel=USComposedAsecReportedIncomeKernel.ref, + population=population, + inputs=_reported_income_inputs(), + outputs=composed_reported_income_declarations(), + params={"phase": COMPOSED_ASEC_PHASE}, + artifact_inputs=( + ArtifactInput( + "frame_context", LEAVES_NODE, "frame_context", US_FRAME_CONTEXT_TYPE + ), + *binding_artifacts, + prepared_receipt, + ArtifactInput( + "income_observations", + CREATE_NODE, + "income_observations", + US_ASEC_INCOME_OBSERVATIONS_TYPE, + ), + ), + artifact_outputs=( + ArtifactOutput("reported_income", US_ASEC_REPORTED_INCOME_TYPE), + ArtifactOutput("frame_context", US_FRAME_CONTEXT_TYPE), + ), + ) + return (bind, leaves, reported) + + +def composed_asec_population_graph( + columns, + *, + sample_fraction: float, + sample_seed: int, + geography: bool = True, +) -> Graph: + """The composed development population plus this arm's bound measurements.""" + composed = composed_population_nodes( + columns, sample_fraction=sample_fraction, sample_seed=sample_seed + ) + if not geography: + return Graph( + country="us", + sources=COMPOSED_SOURCES, + nodes=( + *composed, + *composed_asec_measure_nodes( + population=HARMONIZE_NODE, population_context=HARMONIZE_NODE + ), + ), + ) + geography_nodes = us_geography_nodes( + columns, base=HARMONIZE_NODE, context_producer=HARMONIZE_NODE + ) + return Graph( + country="us", + sources=(*COMPOSED_SOURCES, *LOOKUP_SOURCES), + nodes=( + *composed, + *geography_nodes, + *composed_asec_measure_nodes( + population=f"{GEOGRAPHY_PHASE}.boundary", + population_context=GEOGRAPHY_PHASE, + ), + ), + ) + + +def composed_asec_population_registry(*, geography: bool = True) -> KernelRegistry: + """The composed registry plus exactly this stage's three kernels.""" + registry = composed_population_registry(geography=geography) + registry.register(USComposedAsecBindKernel()) + registry.register(USComposedAsecLeavesKernel()) + registry.register(USComposedAsecReportedIncomeKernel()) + return registry + + +__all__ = [ + "ACS_SHARED_LEAVES", + "LEAF_DTYPE", + "LEAVES_NODE", + "REPORTED_INCOME_NODE", + "USComposedAsecLeavesKernel", + "USComposedAsecReportedIncomeKernel", + "composed_asec_measure_nodes", + "composed_asec_population_graph", + "composed_asec_population_registry", + "composed_leaf_declarations", + "composed_reported_income_declarations", +] diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/graph_composed_contracts.py b/packages/microcosm-build/src/microcosm/build/us_runtime/graph_composed_contracts.py new file mode 100644 index 000000000..2f49f6e89 --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/graph_composed_contracts.py @@ -0,0 +1,16 @@ +"""Source-free declarations shared by composed producers and their consumers. + +These are the existing legacy producer IDs and artifact types. Importing their +names must not select legacy source assembly, evaluation, or country resources. +The defining producers re-export the same objects for compatibility; operations +and source authority remain in those producers. +""" + +from microcosm.graph import ArtifactType + +CREATE_NODE = "composed_population.prepare" +BIND_NODE = "composed_population.asec_bind" +LEAVES_NODE = "composed_population.asec_cps_carried_current" +REPORTED_INCOME_NODE = "composed_population.asec_reported_income" +US_COMPOSED_ASEC_BINDING_TYPE = ArtifactType("microcosm.us.composed_asec_binding", 1) +US_COMPOSED_ASEC_ARM_ROWS_TYPE = ArtifactType("microcosm.us.composed_asec_arm_rows", 1) diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/graph_composed_population.py b/packages/microcosm-build/src/microcosm/build/us_runtime/graph_composed_population.py new file mode 100644 index 000000000..b07496f8e --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/graph_composed_population.py @@ -0,0 +1,751 @@ +"""Compose the prepared ASEC population with the native ACS population. + +This is a separately versioned seam. The reviewed ``asec_raw_stage`` assembly +kernel in :mod:`.graph_sources` keeps its kernel ref, contract, params, sources +and node ids unchanged; nothing here rewrites them. What changes is only which +authenticated ASEC source the ASEC arm reads: the reviewed prepared +current-money directory (``us-asec-prepared-current-money-v3``) instead of the +raw-stage checkpoint. + +Node *keys* are a different matter and do move. Every stage manifest binds the +inventory and the bytes of ``graph_implementation.py``, which is in every +stage's module list, and declaring a new stage necessarily edits both. So every +existing US node key changes and every existing US cache is invalidated — +exactly as when any other attested module changes. + +Why this is a two-source ``CREATE`` and not two parent populations +------------------------------------------------------------------ +``Node.base`` is a single population version. A ``CREATE`` node may declare no +base at all, a structural node may not declare a second ``population``, and a +tuple ``base`` is refused as "not a structural node". So the executor cannot +structurally combine two parent populations; an ``EXPAND`` over one arm can +only invent entrant rows, never bind the other arm's rows or lineage. The +composed population is therefore one ``CREATE`` over two declared sources — +exactly the shape :class:`~.graph_sources.USAssemblyCreateKernel` already uses. +The refusals are recorded in the review root's two-parent probe. + +The stack itself is not reimplemented. Sampling, stacking, provenance and +allocation stay in :mod:`.stacked_spine` / :mod:`.spine_assembly`, and the +harmonization node reuses :class:`~.graph_sources.USSpineHarmonizeKernel` +unchanged by emitting the same two typed producer artifacts it declares. + +Nothing here is a release, a calibration, a certified population, a transfer or +a score. It is an engineering intermediate whose receipts say so. +""" + +from __future__ import annotations + +import json +from collections.abc import Mapping, Sequence +from dataclasses import dataclass + +import numpy as np +import pandas as pd + +from microcosm.build.serialization_dtypes import ( + CANONICAL_STRING_DTYPE, + canonicalize_frame_string_dtypes, +) +from microcosm.frame import US_SCHEMA, Frame +from microcosm.graph import ( + ArtifactInput, + ArtifactOutput, + ArtifactType, + Capabilities, + Determinism, + Graph, + KernelBase, + KernelContext, + KernelRegistry, + KernelResult, + Node, + Numeric, + Owned, + SeedSource, + Slice, + SourceRef, + StructuralDelta, + WeightTransition, +) +from microcosm.graph.canonical import canonical_json + +from .asec_current_money import _sha +from .asec_current_money_selection import ( + US_ASEC_CURRENT_MONEY_BODY_TYPE, + US_ASEC_PREPARED_RECEIPT_TYPE, +) +from .asec_prepared_source import ( + PREPARED_SOURCE_FILES, + PREPARED_SOURCE_KIND, + prepare_asec_current_money_population, +) +from .graph_asec_income import US_ASEC_INCOME_OBSERVATIONS_TYPE +from .graph_asec_prepared import ( + ASEC_PREPARED_CODEC, + ASEC_PREPARED_SOURCE_NAME, + PreparedGraphError, +) +from .graph_composed_contracts import CREATE_NODE as CREATE_NODE +from .graph_context import US_FRAME_CONTEXT_TYPE, _json_data, encode_us_frame_context +from .graph_geography import ( + GEOGRAPHY_PHASE, + LOOKUP_SOURCES, + register_us_geography_kernels, + us_geography_nodes, +) +from .graph_housing_universe import US_ASEC_HOUSING_UNIVERSE_TYPE +from .graph_implementation import ( + STAGE_DEPENDENCIES, + implementation_hash, + implementation_manifest, +) +from .graph_sources import ( + ACS_CODEC, + ASSEMBLY_PHASE, + US_STACK_PREPARATION_TYPE, + USSpineHarmonizeKernel, + _canonical_assembly, + frame_column_declarations, + load_graph_acs, + us_source_codecs, +) +from .stacked_spine import ACS_STACKED_SUPPORT_CHANNEL, prepare_stacked_spine +from .support_provenance import ( + BASE_ASEC_SUPPORT_CHANNEL, + spine_source_id_column, + support_channel_column, + support_source_id_column, +) + +COMPOSED_STAGE = "composed_population_v1" +COMPOSED_PHASE = "compose_prepared_asec_native_acs" +ACS_NATIVE_SOURCE_NAME = "acs_native" +COMPOSED_DEPENDENCIES = STAGE_DEPENDENCIES[COMPOSED_STAGE] + +PREFIX = "composed_population" +HARMONIZE_NODE = PREFIX + +US_COMPOSED_SOURCE_ORIGIN_TYPE = ArtifactType( + "microcosm.us.composed_population_source_origin", 1 +) + +COMPOSED_SOURCES = ( + SourceRef( + ASEC_PREPARED_SOURCE_NAME, + ASEC_PREPARED_CODEC, + "Reviewed ASEC restoration inputs: P, H, T with its receipt, three " + "housing cohort HDFs and three official Census PERSON CSV members.", + ), + SourceRef( + ACS_NATIVE_SOURCE_NAME, + ACS_CODEC, + "2024 ACS one-year national PUMS archives.", + ), +) + +#: Native source-record identity carried by each arm, on both entities. These +#: are the columns a later node must be able to bind back to an actual archive +#: member, so the origin artifact digests them in addition to — never instead of +#: — the arm's own pre-remap IDs. Every declared column must be present: a +#: silently skipped one would leave an arm unbindable while still receipting. +NATIVE_IDENTITY_COLUMNS = { + BASE_ASEC_SUPPORT_CHANNEL: ( + ("household", "asec_H_SEQ"), + ("person", "PERIDNUM"), + ("person", "PH_SEQ"), + ), + ACS_STACKED_SUPPORT_CHANNEL: ( + ("household", "SERIALNO"), + ("person", "source_household_id"), + ("person", "source_person_id"), + ), +} +#: The per-arm cohort/vintage axis. ASEC restores three income years into one +#: prepared population, so a single scalar vintage would misdescribe it. +COHORT_COLUMN = "source_year" + +_SOURCE_ORIGIN_SCHEMA = "microcosm.us.composed-source-origin.v1" + + +@dataclass(frozen=True) +class ComposedPopulationResult: + """A direct-call composition and everything its CREATE kernel emits.""" + + frame: Frame + preparation: object + dtype_transitions: tuple[dict[str, str], ...] + storage_bridge: tuple[dict[str, str], ...] + source_origin: dict[str, object] + asec_frame_context: bytes + money_payload: bytes + receipt_payload: bytes + housing_universe_payload: bytes + income_observations_payload: bytes + prepared_receipt: Mapping[str, object] + + +def _require(condition: bool, reason: str) -> None: + if not condition: + raise PreparedGraphError(reason) + + +def bridge_prepared_string_storage(frame: Frame) -> tuple[dict[str, str], ...]: + """Put the prepared arm on the pre-assembly string storage, in place. + + The one explicit schema difference between the two arms. The prepared + source hands back graph column storage (``string[python, pd.NA]``) because + its own CREATE writes graph cells directly; every pre-assembly build + boundary, and therefore the native ACS arm, uses ``CANONICAL_STRING_DTYPE`` + (``string[python, nan]``). ``_stack_spine_tables`` requires shared columns + to have identical dtypes, so one arm has to move. Moving the prepared arm + down leaves the ACS arm byte-identical to today's production assembly, and + :func:`~.graph_sources._canonical_assembly` promotes the assembled frame + back to graph storage afterwards under its own null/value equality checks. + + This is a physical storage bridge only: no value, no missingness and no + column is added, dropped or filled. + """ + transitions = [] + for entity in US_SCHEMA.entities: + table = frame.table(entity) + for column in table.columns: + dtype = table[column].dtype + if isinstance(dtype, pd.StringDtype) and dtype != CANONICAL_STRING_DTYPE: + transitions.append( + { + "entity": entity, + "column": column, + "from": str(dtype), + "to": str(CANONICAL_STRING_DTYPE), + } + ) + before = { + (item["entity"], item["column"]): frame.table(item["entity"])[item["column"]] + .isna() + .to_numpy(copy=True) + for item in transitions + } + canonicalize_frame_string_dtypes( + frame, boundary="US composed population source bridge", in_place=True + ) + for (entity, column), missing in before.items(): + actual = frame.table(entity)[column] + _require( + actual.dtype == CANONICAL_STRING_DTYPE + and np.array_equal(actual.isna().to_numpy(), missing), + f"STRING_BRIDGE_MISSINGNESS:{entity}.{column}", + ) + return tuple(transitions) + + +def _arm_origin(source: Frame, channel: str) -> dict[str, object]: + """Digest one arm's native identity and cohort axis before assembly.""" + identity = {} + for entity, column in NATIVE_IDENTITY_COLUMNS[channel]: + table = source.table(entity) + _require(column in table, f"ARM_NATIVE_IDENTITY:{channel}.{entity}.{column}") + series = table[column] + identity[f"{entity}.{column}"] = { + "dtype": str(series.dtype), + "rows": int(len(series)), + "missing": int(series.isna().sum()), + "ordered_sha256": _sha( + canonical_json( + [ + None if value is None or value is pd.NA else str(value) + for value in series.tolist() + ] + ) + ), + } + person = source.table("person") + _require(COHORT_COLUMN in person, f"ARM_COHORT_COLUMN:{channel}") + years = person[COHORT_COLUMN] + counts = years.value_counts(dropna=False).to_dict() + cohorts = { + ("missing" if pd.isna(year) else str(int(year))): int(count) + for year, count in counts.items() + } + cohorts = {key: cohorts[key] for key in sorted(cohorts)} + return { + "channel": channel, + "rows": {entity: int(source.n(entity)) for entity in source.entities}, + "household_mass": float(source.weights_for("household").total), + "weight_kind": source.weights_for("household").kind.value, + "native_identity": identity, + "person_cohorts": cohorts, + "columns": { + entity: sorted(source.table(entity).columns) for entity in source.entities + }, + } + + +def _source_origin( + frame: Frame, + sources: Mapping[str, Frame], + *, + sampling: Mapping[str, object], + preparation_sha256: str, + prepared_receipt: Mapping[str, object], +) -> dict[str, object]: + """Bind each output row to the arm and the arm's own pre-remap source ID. + + The mapping is read back off the assembled frame's own receipt-validated + support-provenance columns, so it records what assembly actually did rather + than restating what the caller intended. Both provenance IDs are digested: + ``*_spine_source_id`` is the arm's raw pre-remap identifier and is what + binds an output row back to its source row, while ``*_source_id`` is the + assembly-unique pre-clone identifier the clone stage will later offset. At + CREATE the latter equals the entity ID, so digesting it alone would bind + nothing. + """ + mapping: dict[str, object] = {} + for entity in US_SCHEMA.entities: + table = frame.table(entity) + channel_column = support_channel_column(entity) + spine_column = spine_source_id_column(entity) + source_column = support_source_id_column(entity) + _require(channel_column in table, f"ORIGIN_CHANNEL_MISSING:{entity}") + _require(spine_column in table, f"ORIGIN_SPINE_SOURCE_ID_MISSING:{entity}") + _require(source_column in table, f"ORIGIN_SOURCE_ID_MISSING:{entity}") + channels = table[channel_column].astype(str).tolist() + spine_ids = table[spine_column].to_numpy(dtype="int64").tolist() + ids = table[source_column].to_numpy(dtype="int64").tolist() + outputs = ( + table[US_SCHEMA.entity_id_column(entity)].to_numpy(dtype="int64").tolist() + ) + mapping[entity] = { + "rows_by_channel": { + channel: int(channels.count(channel)) + for channel in sorted(set(channels)) + }, + "ordered_channels_sha256": _sha(canonical_json(channels)), + "ordered_spine_source_ids_sha256": _sha(canonical_json(spine_ids)), + "ordered_source_to_output_sha256": _sha( + canonical_json( + [ + [channel, spine_id, source_id, output_id] + for channel, spine_id, source_id, output_id in zip( + channels, spine_ids, ids, outputs, strict=True + ) + ] + ) + ), + } + asec = sources[BASE_ASEC_SUPPORT_CHANNEL] + composed_asec = { + entity: int( + ( + frame.table(entity)[support_channel_column(entity)] + == BASE_ASEC_SUPPORT_CHANNEL + ).sum() + ) + for entity in US_SCHEMA.entities + } + return { + "schema": _SOURCE_ORIGIN_SCHEMA, + "phase": COMPOSED_PHASE, + "preparation_sha256": preparation_sha256, + "sampling": { + "sample_fraction": float(sampling["sample_fraction"]), + "sample_seed": int(sampling["sample_seed"]), + "survey_samples": { + channel: { + key: sample[key] + for key in ( + "realized_household_count", + "selected_household_ids_sha256", + "incoming_household_mass", + "sampled_household_mass", + ) + } + for channel, sample in sampling["survey_samples"].items() + }, + }, + # The four carried prepared artifacts describe the whole prepared ASEC + # population, not the composed rows. They coincide only when the draw + # keeps the arm entire. Recording both counts makes the difference + # readable instead of leaving a later consumer to assume alignment. + "asec_evidence_alignment": { + "basis": "full_prepared_population", + "evidence_rows": { + entity: int(count) + for entity, count in sorted(prepared_receipt["entity_rows"].items()) + }, + "arm_rows": {entity: int(asec.n(entity)) for entity in asec.entities}, + "composed_rows": composed_asec, + "composed_is_whole_arm": all( + composed_asec[entity] == int(asec.n(entity)) + for entity in US_SCHEMA.entities + ), + }, + "arms": { + channel: _arm_origin(source, channel) + for channel, source in sorted(sources.items()) + }, + "assembled": mapping, + "release_eligible": False, + } + + +def compose_from_sources( + asec_prepared_path, + acs_path, + *, + sample_fraction: float, + sample_seed: int, +) -> ComposedPopulationResult: + """The direct-call parity oracle the CREATE kernel shares. + + Source preparation is deliberately separate from sampling: the prepared + ASEC directory and the ACS archives are read whole, and only then does + :func:`prepare_stacked_spine` draw its declared seeded sample. Reading a + fraction of a source is not what this does and would not cost a fraction + of a whole-source read. + """ + prepared = prepare_asec_current_money_population(asec_prepared_path) + asec = prepared.frame + # The carried evidence is positional over the whole prepared population. + # If the receipt and the arm ever disagreed, everything downstream that + # reads the evidence by position would be silently misaligned. + _require( + { + entity: int(count) + for entity, count in prepared.receipt["entity_rows"].items() + } + == {entity: int(asec.n(entity)) for entity in prepared.receipt["entity_rows"]}, + "PREPARED_EVIDENCE_ROWS", + ) + acs = load_graph_acs(acs_path) + storage_bridge = bridge_prepared_string_storage(asec) + # Encoded before stacking, so the identity it declares is the prepared + # arm's own ordered identity, which is what the evidence is positional in. + asec_context = encode_us_frame_context(asec) + arms = { + BASE_ASEC_SUPPORT_CHANNEL: asec, + ACS_STACKED_SUPPORT_CHANNEL: acs, + } + stacked = prepare_stacked_spine( + asec, acs, sample_fraction=sample_fraction, sample_seed=sample_seed + ) + transitions = _canonical_assembly(stacked.frame, (asec, acs)) + preparation_payload = canonical_json(_json_data(stacked.receipt)) + return ComposedPopulationResult( + frame=stacked.frame, + preparation=stacked.receipt, + dtype_transitions=transitions, + storage_bridge=storage_bridge, + source_origin=_source_origin( + stacked.frame, + arms, + sampling=stacked.receipt["sampling"], + preparation_sha256=_sha(preparation_payload), + prepared_receipt=prepared.receipt, + ), + asec_frame_context=asec_context, + money_payload=prepared.money_payload, + receipt_payload=prepared.receipt_payload, + housing_universe_payload=prepared.housing_universe_payload, + income_observations_payload=prepared.income_observations_payload, + prepared_receipt=prepared.receipt, + ) + + +class USComposedPopulationCreateKernel(KernelBase): + """Read both authenticated sources whole and stack them into one spine.""" + + ref = "us.composed_population.prepare@1" + capabilities = Capabilities( + determinism=Determinism.SEEDED, + numeric=Numeric.PLATFORM_BITWISE, + seed_source=SeedSource.PARAM, + structural=StructuralDelta.CREATE, + dependencies=COMPOSED_DEPENDENCIES, + ) + + def implementation_hash(self) -> str: + # Runs before cache lookup, including resume=require, exactly as the + # raw-stage assembly kernel does: a renamed or substituted public + # loader cannot claim the declared codec implementation. + us_source_codecs() + return implementation_hash(COMPOSED_STAGE) + + def run(self, context: KernelContext) -> KernelResult: + node = context.node + _require(node.kernel == self.ref, "NODE_KERNEL") + _require( + set(context.params) == {"phase", "sample_fraction", "sample_seed"}, + "NODE_PARAMS", + ) + _require(context.params["phase"] == COMPOSED_PHASE, "NODE_PHASE") + _require( + tuple(node.sources) == (ASEC_PREPARED_SOURCE_NAME, ACS_NATIVE_SOURCE_NAME), + "NODE_SOURCES", + ) + _require(not node.inputs and not node.artifact_inputs, "NODE_INPUTS") + _require(node.artifact_outputs == _CREATE_ARTIFACTS, "NODE_ARTIFACT_OUTPUTS") + result = compose_from_sources( + context.sources[ASEC_PREPARED_SOURCE_NAME], + context.sources[ACS_NATIVE_SOURCE_NAME], + sample_fraction=context.params["sample_fraction"], + sample_seed=context.params["sample_seed"], + ) + receipt = result.prepared_receipt + _require(receipt["source_kind"] == PREPARED_SOURCE_KIND, "PREPARED_SOURCE_KIND") + _require( + receipt["file_roster"] == list(PREPARED_SOURCE_FILES), + "PREPARED_FILE_ROSTER", + ) + _require( + frame_column_declarations(result.frame) == node.outputs, + "COMPOSED_COLUMN_INVENTORY", + ) + return KernelResult( + frame=result.frame, + artifacts={ + "frame_context": encode_us_frame_context(result.frame), + "preparation": canonical_json(_json_data(result.preparation)), + "source_origin": canonical_json(result.source_origin), + "asec_frame_context": result.asec_frame_context, + "current_money": result.money_payload, + "prepared_receipt": result.receipt_payload, + "housing_universe": result.housing_universe_payload, + "income_observations": result.income_observations_payload, + }, + receipt={ + "phase": COMPOSED_PHASE, + "implementation": implementation_manifest(COMPOSED_STAGE), + "preparation": result.preparation, + "prepared": receipt, + "dtype_transitions": result.dtype_transitions, + "storage_bridge": result.storage_bridge, + "release_eligible": False, + "certified": False, + }, + ) + + +_CREATE_ARTIFACTS = ( + ArtifactOutput("frame_context", US_FRAME_CONTEXT_TYPE), + ArtifactOutput("preparation", US_STACK_PREPARATION_TYPE), + ArtifactOutput("source_origin", US_COMPOSED_SOURCE_ORIGIN_TYPE), + # The prepared arm's own typed context, encoded before stacking. The four + # evidence artifacts below are positional over that population, so a later + # binding node needs its ordered identity to align them to composed rows. + ArtifactOutput("asec_frame_context", US_FRAME_CONTEXT_TYPE), + ArtifactOutput("current_money", US_ASEC_CURRENT_MONEY_BODY_TYPE), + ArtifactOutput("prepared_receipt", US_ASEC_PREPARED_RECEIPT_TYPE), + ArtifactOutput("housing_universe", US_ASEC_HOUSING_UNIVERSE_TYPE), + ArtifactOutput("income_observations", US_ASEC_INCOME_OBSERVATIONS_TYPE), +) + + +#: Top-level keys the origin document must carry, exactly. +_ORIGIN_KEYS = frozenset( + { + "schema", + "phase", + "preparation_sha256", + "sampling", + "asec_evidence_alignment", + "arms", + "assembled", + "release_eligible", + } +) +_ORIGIN_ENTITY_KEYS = frozenset( + { + "rows_by_channel", + "ordered_channels_sha256", + "ordered_spine_source_ids_sha256", + "ordered_source_to_output_sha256", + } +) + + +def bind_composed_source_origin(payload: bytes) -> dict[str, object]: + """Decode the origin artifact as a typed shape, refusing a malformed one. + + This checks canonical bytes and the document's declared shape: its exact + key set, its schema/phase, that both arms and every US entity are present + with well-formed digests and counts, and that it makes no release claim. It + verifies **no** digest against any population — it cannot, since it receives + only bytes. A consumer that needs the mapping to be true of a frame in hand + must recompute the digests against that frame. + """ + document = json.loads(payload) + _require(canonical_json(document) == payload, "ORIGIN_CANONICAL") + _require(isinstance(document, dict), "ORIGIN_DOCUMENT") + _require(set(document) == _ORIGIN_KEYS, "ORIGIN_KEYS") + _require(document["schema"] == _SOURCE_ORIGIN_SCHEMA, "ORIGIN_SCHEMA") + _require(document["phase"] == COMPOSED_PHASE, "ORIGIN_PHASE") + _require(_is_digest(document["preparation_sha256"]), "ORIGIN_PREPARATION_DIGEST") + _require(document["release_eligible"] is False, "ORIGIN_RELEASE_CLAIM") + channels = {BASE_ASEC_SUPPORT_CHANNEL, ACS_STACKED_SUPPORT_CHANNEL} + _require(set(document["arms"]) == channels, "ORIGIN_ARMS") + sampling = document["sampling"] + _require( + set(sampling) == {"sample_fraction", "sample_seed", "survey_samples"} + and set(sampling["survey_samples"]) == channels, + "ORIGIN_SAMPLING", + ) + alignment = document["asec_evidence_alignment"] + _require( + set(alignment) + == { + "basis", + "evidence_rows", + "arm_rows", + "composed_rows", + "composed_is_whole_arm", + } + and alignment["basis"] == "full_prepared_population" + and isinstance(alignment["composed_is_whole_arm"], bool), + "ORIGIN_ALIGNMENT", + ) + _require(set(document["assembled"]) == set(US_SCHEMA.entities), "ORIGIN_ENTITIES") + for entity, record in document["assembled"].items(): + _require(set(record) == _ORIGIN_ENTITY_KEYS, f"ORIGIN_ENTITY_KEYS:{entity}") + _require( + all( + _is_digest(record[key]) + for key in _ORIGIN_ENTITY_KEYS + if key.endswith("_sha256") + ), + f"ORIGIN_ENTITY_DIGEST:{entity}", + ) + _require( + set(record["rows_by_channel"]) <= channels + and all( + isinstance(count, int) and count >= 0 + for count in record["rows_by_channel"].values() + ), + f"ORIGIN_ENTITY_ROWS:{entity}", + ) + return document + + +def _is_digest(value: object) -> bool: + return ( + isinstance(value, str) + and len(value) == 64 + and not set(value) - set("0123456789abcdef") + ) + + +def composed_population_nodes( + columns: Sequence[Owned], *, sample_fraction: float, sample_seed: int +) -> tuple[Node, ...]: + """The two composition nodes: whole-source CREATE, then reused harmonize.""" + inventory = {(owned.entity, owned.column) for owned in columns} + _require(len(inventory) == len(tuple(columns)), "COLUMN_INVENTORY_REPEATS") + for entity in US_SCHEMA.entities: + _require( + (entity, support_channel_column(entity)) in inventory, + f"MISSING_SUPPORT_CHANNEL:{entity}", + ) + _require( + (entity, support_source_id_column(entity)) in inventory, + f"MISSING_SUPPORT_SOURCE_ID:{entity}", + ) + return ( + Node( + id=CREATE_NODE, + kernel=USComposedPopulationCreateKernel.ref, + structural=StructuralDelta.CREATE, + sources=(ASEC_PREPARED_SOURCE_NAME, ACS_NATIVE_SOURCE_NAME), + outputs=tuple(columns), + params={ + "phase": COMPOSED_PHASE, + "sample_fraction": sample_fraction, + "sample_seed": sample_seed, + }, + artifact_outputs=_CREATE_ARTIFACTS, + ), + Node( + id=HARMONIZE_NODE, + kernel=USSpineHarmonizeKernel.ref, + base=CREATE_NODE, + structural=StructuralDelta.REWEIGHT, + inputs=( + Slice("household", (support_channel_column("household"),)), + Slice("person", (support_channel_column("person"),)), + ), + weights=WeightTransition("household", "importance", mass="declared"), + mass="declared", + # The reused production harmonization kernel owns this phase name; + # composing a different ASEC source does not make its operation a + # different one, and its contract is deliberately not re-versioned. + params={"phase": ASSEMBLY_PHASE}, + artifact_inputs=( + ArtifactInput( + "frame_context", CREATE_NODE, "frame_context", US_FRAME_CONTEXT_TYPE + ), + ArtifactInput( + "preparation", + CREATE_NODE, + "preparation", + US_STACK_PREPARATION_TYPE, + ), + ), + artifact_outputs=(ArtifactOutput("frame_context", US_FRAME_CONTEXT_TYPE),), + ), + ) + + +def composed_population_graph( + columns: Sequence[Owned], + *, + sample_fraction: float, + sample_seed: int, + geography: bool = True, +) -> Graph: + """Declare the composed development population, never a release pool.""" + nodes = composed_population_nodes( + columns, sample_fraction=sample_fraction, sample_seed=sample_seed + ) + if not geography: + return Graph(country="us", sources=COMPOSED_SOURCES, nodes=nodes) + return Graph( + country="us", + sources=(*COMPOSED_SOURCES, *LOOKUP_SOURCES), + nodes=( + *nodes, + *us_geography_nodes( + columns, base=HARMONIZE_NODE, context_producer=HARMONIZE_NODE + ), + ), + ) + + +def composed_population_registry(*, geography: bool = True) -> KernelRegistry: + """Register exactly the kernels this graph runs; two of three are reused.""" + registry = KernelRegistry() + registry.register(USComposedPopulationCreateKernel()) + registry.register(USSpineHarmonizeKernel()) + if geography: + register_us_geography_kernels(registry) + return registry + + +__all__ = [ + "ACS_NATIVE_SOURCE_NAME", + "COHORT_COLUMN", + "COMPOSED_DEPENDENCIES", + "COMPOSED_PHASE", + "COMPOSED_SOURCES", + "COMPOSED_STAGE", + "CREATE_NODE", + "GEOGRAPHY_PHASE", + "HARMONIZE_NODE", + "NATIVE_IDENTITY_COLUMNS", + "US_COMPOSED_SOURCE_ORIGIN_TYPE", + "ComposedPopulationResult", + "USComposedPopulationCreateKernel", + "bind_composed_source_origin", + "bridge_prepared_string_storage", + "compose_from_sources", + "composed_population_graph", + "composed_population_nodes", + "composed_population_registry", +] diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/graph_context.py b/packages/microcosm-build/src/microcosm/build/us_runtime/graph_context.py new file mode 100644 index 000000000..c1732455a --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/graph_context.py @@ -0,0 +1,278 @@ +"""Identity-bound metadata for real US operators in the population graph. + +Population cells, weights and strata remain executor-owned. This typed artifact +carries the metadata and prior mass records that existing US operators require, +so they cannot silently reload context from an unrelated mutable checkpoint. +Reconstruction uses only the node's declared table views; it grants no access to +undeclared columns or a live population object. +""" + +from __future__ import annotations + +import hashlib +import json +from collections.abc import Mapping +from dataclasses import asdict + +import numpy as np +import pandas as pd + +from microcosm.frame import US_SCHEMA, Frame, MassChangeRecord +from microcosm.graph import ArtifactType, KernelContext +from microcosm.graph.canonical import canonical_json + +US_FRAME_CONTEXT_TYPE = ArtifactType("microcosm.us.frame_context", 1) + +_FIELDS = {"schema_version", "entities", "metadata", "mass_log", "weight_sources"} +_ENTITY_FIELDS = {"columns", "rows", "id_dtype", "ordered_ids_sha256"} +_MASS_FIELDS = {"entity", "old_total", "new_total", "declared_factor", "reason"} + +# These five receipts are read as authority by the existing US operators. +# Operational observations belong in run receipts, outside this keyed edge. +# An additional normative receipt requires an explicit contract revision here. +US_NORMATIVE_METADATA_KEYS = frozenset( + { + "us_spine_assembly_manifest", + "us_stacked_spine_manifest", + "us_puf_clone_attachment_manifest", + "us_late_producer_transition_authority", + "acs_pums_earnings_universe_application", + } +) +_RUN_FIELDS = frozenset( + { + "created_at", + "updated_at", + "timestamp", + "started_at", + "finished_at", + "elapsed_seconds", + "wall_time_seconds", + "duration_seconds", + "peak_rss", + "hostname", + "host", + "pid", + "run_id", + "path", + "checkpoint_dir", + "output_dir", + "timings", + "cache_hit", + "cache_hits", + } +) + + +def _normative_metadata(metadata: Mapping[str, object]) -> object: + unknown = set(metadata) - US_NORMATIVE_METADATA_KEYS + if unknown: + raise ValueError( + f"US graph context has undeclared metadata: {sorted(unknown)}." + ) + + def validate(value: object) -> None: + if isinstance(value, Mapping): + if set(value) & _RUN_FIELDS: + raise ValueError("US graph context cannot contain run-level metadata.") + for item in value.values(): + validate(item) + elif isinstance(value, list | tuple): + for item in value: + validate(item) + + validate(metadata) + return _json_data(metadata) + + +def _json_data(value: object) -> object: + """Normalize NumPy metadata without coercing unknown objects to strings.""" + if isinstance(value, np.generic): + return _json_data(value.item()) + if isinstance(value, np.ndarray): + return _json_data(value.tolist()) + if isinstance(value, Mapping): + if any(not isinstance(key, str) for key in value): + raise ValueError("US graph context metadata requires string mapping keys.") + return {key: _json_data(item) for key, item in value.items()} + if isinstance(value, list | tuple): + return [_json_data(item) for item in value] + # canonical_json validates scalar types and refuses nonfinite numbers. + return value + + +def _row_identity(table: pd.DataFrame, entity: str) -> dict[str, object]: + column = US_SCHEMA.entity_id_column(entity) + if column not in table: + raise ValueError(f"US graph context {entity} identity column is missing.") + ids = table[column] + if not pd.api.types.is_integer_dtype(ids.dtype) or ids.isna().any(): + raise ValueError(f"US graph context {entity} identity must be integral.") + if len(ids) and (int(ids.min()) < 0 or int(ids.max()) > np.iinfo(np.int64).max): + raise ValueError(f"US graph context {entity} identity exceeds int64 bounds.") + values = ids.to_numpy(dtype=" bytes: + """Encode current US metadata and row identities, without population cells.""" + if frame.schema != US_SCHEMA: + raise ValueError("US graph context requires the US entity schema.") + entities = {} + for entity in US_SCHEMA.entities: + table = frame.table(entity) + if entity in US_SCHEMA.group_entities and ( + not isinstance(table.index, pd.RangeIndex) + or table.index.start != 0 + or table.index.stop != len(table) + or table.index.step != 1 + or table.index.name is not None + ): + # Version 1 carries group IDs, not pandas index descriptors. An + # identity FILTER uses Frame.select, which resets group indices. + # Refuse an unsupported representation before emitting authority + # that downstream person-only nodes cannot independently inspect. + raise ValueError( + f"US graph context {entity} requires a default unnamed RangeIndex." + ) + if any(not isinstance(column, str) for column in table.columns): + raise ValueError("US graph context column names must be strings.") + entities[entity] = { + "columns": list(table.columns), + **_row_identity(table, entity), + } + document = { + "schema_version": US_FRAME_CONTEXT_TYPE.schema_version, + "entities": entities, + "metadata": _normative_metadata(frame.metadata), + "mass_log": [_json_data(asdict(record)) for record in frame.mass_log], + "weight_sources": { + entity: frame.weights_for(entity).kind.value + for entity in frame.weighted_entities + }, + } + return canonical_json(document) + + +def _decode(payload: bytes) -> dict[str, object]: + def unique_pairs(pairs): + result = {} + for key, value in pairs: + if key in result: + raise ValueError(f"Duplicate US graph context field {key!r}.") + result[key] = value + return result + + try: + document = json.loads(payload, object_pairs_hook=unique_pairs) + canonical_json(document) + except (UnicodeDecodeError, json.JSONDecodeError, TypeError, ValueError) as error: + raise ValueError( + "US graph context must be finite, unambiguous JSON." + ) from error + if ( + not isinstance(document, dict) + or set(document) != _FIELDS + or type(document["schema_version"]) is not int + or document["schema_version"] != US_FRAME_CONTEXT_TYPE.schema_version + ): + raise ValueError("US graph context has an unsupported schema.") + entities = document["entities"] + if not isinstance(entities, dict) or set(entities) != set(US_SCHEMA.entities): + raise ValueError("US graph context must describe exactly the US entities.") + for entity, value in entities.items(): + if not isinstance(value, dict) or set(value) != _ENTITY_FIELDS: + raise ValueError(f"US graph context {entity} identity is malformed.") + columns = value["columns"] + if ( + not isinstance(columns, list) + or any(not isinstance(column, str) or not column for column in columns) + or len(set(columns)) != len(columns) + or US_SCHEMA.entity_id_column(entity) not in columns + or type(value["rows"]) is not int + or value["rows"] < 0 + ): + raise ValueError(f"US graph context {entity} schema is malformed.") + if not isinstance(document["metadata"], dict): + raise ValueError("US graph context metadata must be an object.") + _normative_metadata(document["metadata"]) + weight_sources = document["weight_sources"] + if ( + not isinstance(weight_sources, dict) + or not weight_sources + or set(weight_sources) - set(US_SCHEMA.entities) + or any(not isinstance(kind, str) for kind in weight_sources.values()) + ): + raise ValueError("US graph context weight sources are malformed.") + if not isinstance(document["mass_log"], list): + raise ValueError("US graph context mass_log must be an array.") + return document + + +def _mass_records(raw: list[object]) -> tuple[MassChangeRecord, ...]: + records = [] + for value in raw: + if not isinstance(value, dict) or set(value) != _MASS_FIELDS: + raise ValueError("US graph context mass record is malformed.") + if value["entity"] not in US_SCHEMA.entities or not isinstance( + value["reason"], str + ): + raise ValueError("US graph context mass record has invalid authority.") + for field in ("old_total", "new_total", "declared_factor"): + number = value[field] + if field == "declared_factor" and number is None: + continue + if type(number) not in (int, float) or number < 0: + raise ValueError("US graph context mass record is invalid.") + records.append(MassChangeRecord(**value)) + return tuple(records) + + +def us_frame_from_context( + context: KernelContext, *, artifact_alias: str = "frame_context" +) -> Frame: + """Restore an isolated US operator view from declared cells and typed context. + + The complete row identity of each entity must match the artifact. Columns + omitted by the node stay omitted. Callers needing a narrower row population + must declare a structural graph stage and publish its own context first. + """ + artifact = context.artifacts.get(artifact_alias) + if artifact is None or artifact.type != US_FRAME_CONTEXT_TYPE: + raise ValueError("US graph context artifact has a missing or incorrect type.") + document = _decode(artifact.payload) + if set(context.tables) != set(US_SCHEMA.entities): + raise ValueError("US graph context requires declared views for every entity.") + tables = {} + for entity in US_SCHEMA.entities: + table = context.tables[entity] + declared = document["entities"][entity] + actual = _row_identity(table, entity) + if any(declared[key] != value for key, value in actual.items()): + raise ValueError(f"US graph context {entity} identity does not match.") + if set(table.columns) - set(declared["columns"]): + raise ValueError(f"US graph context {entity} includes unknown columns.") + ordered = [column for column in declared["columns"] if column in table] + tables[entity] = table.loc[:, ordered].copy(deep=True) + weights = {} + for entity, kind in document["weight_sources"].items(): + value = context.weights.get(entity) + if value is None or value.kind.value != kind: + raise ValueError( + f"US graph context has no matching {entity} weight source." + ) + weights[entity] = value + return Frame( + tables, + US_SCHEMA, + weights, + context.strata.copy(deep=True), + mass_log=_mass_records(document["mass_log"]), + metadata=document["metadata"], + ) diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/graph_current_survey_geography.py b/packages/microcosm-build/src/microcosm/build/us_runtime/graph_current_survey_geography.py new file mode 100644 index 000000000..416c79650 --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/graph_current_survey_geography.py @@ -0,0 +1,277 @@ +"""Project authenticated survey geography into declared household cells. + +This nonstructural node qualifies observed inputs only. Shared atomic operators +own location assignment; the country runner owns receiving-Population and warm +replay admission. Preparation bytes and descriptive receipts grant no authority. +""" + +from __future__ import annotations + +import json + +import pandas as pd + +from microcosm.frame import WeightKind +from microcosm.graph import ( + ArtifactInput, + Capabilities, + Determinism, + KernelBase, + KernelResult, + Node, + Numeric, + Owned, + Slice, + source_hash, +) +from microcosm.graph.canonical import canonical_json + +from . import graph_survey_population as source_graph + +NODE = "survey_population.observed_geography" +REF = "us.survey_population.observed_geography@1" +PHASE = "us.survey_population.observed_geography.v1" +OUTPUT_COLUMNS = ( + "survey_geography_origin_key", + "survey_observed_state", + "survey_observed_puma", +) + + +def _require(condition, reason): + if not condition: + raise ValueError("SURVEY_GEOGRAPHY_PROJECTION_" + reason) + + +def _qualifier(): + # Declaration-only imports need not load retained source owners. + from . import current_survey_geography + + return current_survey_geography + + +def current_survey_geography_node( + *, + preparation_sha256, + projection_receipt_sha256, + population, + node_id=NODE, +): + """Declare three new observed columns; digest arguments authenticate nothing.""" + _require( + type(population) is str + and 0 < len(population) <= 256 + and type(node_id) is str + and 0 < len(node_id) <= 256 + and population != node_id, + "NODE_NAMES", + ) + return Node( + id=node_id, + kernel=REF, + population=population, + inputs=(Slice("household", source_graph._provenance_columns("household")),), + outputs=tuple(Owned("household", name, "string") for name in OUTPUT_COLUMNS), + params={ + "phase": PHASE, + "preparation_sha256": source_graph._digest(preparation_sha256), + "projection_receipt_sha256": source_graph._digest( + projection_receipt_sha256 + ), + }, + artifact_inputs=( + ArtifactInput( + "preparation", + source_graph.CREATE_NODE, + "preparation", + source_graph.PREPARATION_TYPE, + ), + ), + description="Project qualified observed state, PUMA and stable source draw keys.", + ) + + +def _check_context(context, *, payload, source_frame, receipt_sha256): + expected = current_survey_geography_node( + preparation_sha256=source_graph._sha(payload), + projection_receipt_sha256=receipt_sha256, + population=context.node.population, + node_id=context.node.id, + ) + _require( + context.node == expected and dict(context.params) == dict(expected.params), + "DECLARATION", + ) + _require( + not context.sources + and set(context.artifacts) == {"preparation"} + and set(context.tables) == {"household"}, + "CONTEXT_ROSTER", + ) + source_graph._artifact( + context, "preparation", source_graph.PREPARATION_TYPE, payload + ) + table = context.tables["household"] + columns = ("household_id", *source_graph._provenance_columns("household")) + _require(tuple(table.columns) == columns, "CONTEXT_COLUMNS") + try: + pd.testing.assert_frame_equal( + table, + source_frame.table("household").loc[:, list(columns)], + check_exact=True, + ) + except (AssertionError, ValueError, TypeError): + raise ValueError("SURVEY_GEOGRAPHY_PROJECTION_HOUSEHOLD_ORIGIN") from None + _require( + set(context.weights) == {"household"} + and context.weights["household"].kind is WeightKind.IMPORTANCE + and len(context.weights["household"].values) == len(table), + "ALLOCATED_WEIGHTS_REQUIRED", + ) + + +def _check_projection(qualifier, projection, *, payload, receipt_sha256): + _require( + type(projection) is qualifier.CurrentSurveyGeographyValues + and qualifier.COLUMNS == OUTPUT_COLUMNS + and type(projection.receipt) is bytes + and 0 < len(projection.receipt) <= qualifier.MAX_RECEIPT_BYTES + and source_graph._sha(projection.receipt) == receipt_sha256, + "PROJECTION_RECEIPT", + ) + document = json.loads(projection.receipt) + _require( + document["protocol"] == qualifier.PROTOCOL + and document["preparation_sha256"] == source_graph._sha(payload) + and document["columns"] == list(OUTPUT_COLUMNS) + and document["households"] == len(projection.household) + and document["projection_sha256"] + == qualifier._projection_digest(projection.household) + and document["source_admission_issued"] is False + and document["population_admission_issued"] is False + and document["release_eligible"] is False, + "QUALIFIED_PROJECTION", + ) + return document + + +class CurrentSurveyGeographyKernel(KernelBase): + """Retain the live preparation; independently qualify every executed output.""" + + ref = REF + capabilities = Capabilities( + determinism=Determinism.DETERMINISTIC, + numeric=Numeric.PLATFORM_BITWISE, + dependencies=source_graph.STAGE_DEPENDENCIES, + ) + + def __init__(self, preparation): + self._preparation = preparation + + def implementation_hash(self): + qualifier = _qualifier() + demographics = qualifier.demographics + # Preserve the source stage's maintained dependency fence, and bind the + # additional actual projection functions and their demographic owners. + return source_graph._sha( + canonical_json( + { + "source_stage": source_graph._Kernel.implementation_hash(self), + "projection": source_hash( + type(self), + qualifier, + qualifier.qualify_current_survey_geography, + demographics, + demographics.qualify_current_asec_demographics, + demographics.demographic, + demographics.demographic.load_authenticated_asec_demographic_source, + demographics.demographic._snapshot, + demographics.household, + demographics.source_csv_builtin, + qualifier.dtype_for_token, + dependencies=self.capabilities.dependencies, + ), + } + ) + ) + + def run(self, context): + preparation = self._preparation + owner = source_graph._source_owner() + _require( + type(preparation) is owner.AuthenticatedSurveyPopulationPreparation, + "PREPARATION_TYPE", + ) + entry = preparation._checked() + payload, state = entry[1], entry[2] + receipt_sha256 = source_graph._digest( + context.params.get("projection_receipt_sha256") + ) + _check_context( + context, + payload=payload, + source_frame=state.frame, + receipt_sha256=receipt_sha256, + ) + qualifier = _qualifier() + projection = qualifier.qualify_current_survey_geography(preparation) + document = _check_projection( + qualifier, projection, payload=payload, receipt_sha256=receipt_sha256 + ) + _require( + projection.household.index.tolist() + == context.tables["household"].household_id.tolist(), + "PROJECTION_HOUSEHOLD_ORDER", + ) + receipt = { + "phase": PHASE, + "preparation_sha256": source_graph._sha(payload), + "projection_receipt_sha256": receipt_sha256, + "source_projection": document, + "population_admission_issued": False, + "release_eligible": False, + } + receipt_bytes = canonical_json(receipt) + result = KernelResult( + columns={ + ("household", name): projection.household[name].copy(deep=True) + for name in OUTPUT_COLUMNS + }, + receipt=receipt, + ) + # Finish source-owner I/O before rechecking mutable source views, + # projected context and every detached returned column/receipt. + final_entry = preparation._checked() + _require( + self._preparation is preparation + and final_entry is entry + and owner._ISSUED.get(id(preparation)) is entry + and preparation.payload == payload, + "FINAL_ISSUANCE", + ) + owner._pure_final(state) + _check_context( + context, + payload=payload, + source_frame=state.frame, + receipt_sha256=receipt_sha256, + ) + _check_projection( + qualifier, projection, payload=payload, receipt_sha256=receipt_sha256 + ) + _require( + set(result.columns) == {("household", name) for name in OUTPUT_COLUMNS} + and result.frame is result.keep is result.expand is result.weights is None + and result.strata is None + and not result.artifacts + and canonical_json(result.receipt) == receipt_bytes, + "FINAL_RESULT", + ) + returned = pd.DataFrame( + {name: result.columns[("household", name)] for name in OUTPUT_COLUMNS} + ) + _require( + qualifier._projection_digest(returned) == document["projection_sha256"], + "FINAL_OUTPUT", + ) + return result diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/graph_current_survey_health.py b/packages/microcosm-build/src/microcosm/build/us_runtime/graph_current_survey_health.py new file mode 100644 index 000000000..73f4c0b67 --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/graph_current_survey_health.py @@ -0,0 +1,431 @@ +"""Private health fragment for the one country enrichment host. + +The host retains the actual checked PUF run, preparation, kernel registry and +source files. This fragment never issues a run or creates a second receiving +branch. Its final ordinary attachment preserves the incoming population version. +""" + +from __future__ import annotations + +import hashlib +import json +import sys +from dataclasses import replace +from types import SimpleNamespace + +import numpy as np +import pandas as pd + +from microcosm.frame import Frame +from microcosm.graph import ( + ArtifactInput, + ArtifactOutput, + ArtifactType, + ArtifactValue, + Capabilities, + Determinism, + KernelBase, + KernelResult, + Node, + Numeric, + Owned, + Slice, + StructuralDelta, + source_hash, +) +from microcosm.graph import population as population_ops + +from . import current_survey_health_coverage as health +from . import current_survey_health_source as source +from . import graph_full_puf_enrichment as physical +from .graph_survey_population import SOURCE_NAME + +qualify_health_coverage = source.qualify_current_survey_health +require = health.require +SOURCE_NODE = "survey_health.source" +RAW_NODE = "survey_health.raw_columns" +RECODE_PREFIX = "survey_health.recode." +ATTACH_NODE = "survey_health.attach" +PROJECTION_TYPE = ArtifactType("microcosm.us.current_survey_health_projection", 1) +COLUMNS_TYPE = ArtifactType("microcosm.us.current_survey_health_columns", 1) +ATTACHMENT_TYPE = ArtifactType("microcosm.us.current_survey_health_attachment", 1) + + +def _json(value): + return json.dumps( + value, sort_keys=True, separators=(",", ":"), allow_nan=False + ).encode() + + +def _table_bytes(table): + return table.to_json(orient="table").encode() + + +def health_coverage_seal(qualified): + require(type(qualified) is source.QualifiedSurveyHealthCoverage, "QUALIFIED_TYPE") + require( + qualified.projection + == qualified.raw.reset_index().to_json(orient="table", index=False).encode() + and qualified.evidence["projection_sha256"] + == hashlib.sha256(qualified.projection).hexdigest(), + "PROJECTION_BINDING", + ) + return ( + source.source._frame_identity(qualified.source_frame), + physical._table_stamp(qualified.origins), + physical._table_stamp(qualified.raw), + qualified.projection, + _json(qualified.evidence), + ) + + +def _source_edge(): + return ArtifactInput( + "health_projection", SOURCE_NODE, "projection", PROJECTION_TYPE + ) + + +def _column_artifact(node_id, alias): + return ArtifactInput(alias, node_id, "columns", COLUMNS_TYPE) + + +def _dtype(series): + return population_ops.token_for_dtype(series.dtype) + + +def _outputs(table): + return tuple(Owned("person", name, _dtype(table[name])) for name in table) + + +def _completed(qualified): + return pd.concat( + [ + health.source_columns(qualified.raw), + *(health.recode_field(qualified.raw, f.output) for f in health.FIELDS), + ], + axis=1, + ) + + +def _params(qualified): + return { + "protocol": health.PROTOCOL, + "projection_sha256": hashlib.sha256(qualified.projection).hexdigest(), + "source_evidence": _json(qualified.evidence).decode(), + } + + +def health_coverage_nodes(qualified, *, receiving_version, after): + health_coverage_seal(qualified) + require( + type(receiving_version) is str + and receiving_version + and type(after) is ArtifactInput, + "FRAGMENT_INPUT", + ) + params = _params(qualified) + raw = health.source_columns(qualified.raw) + create = Node( + SOURCE_NODE, + "us.survey_health.source@1", + structural=StructuralDelta.CREATE, + sources=(SOURCE_NAME,), + outputs=(Owned("person", "health_native_person_id", "int64"),), + params=params, + artifact_inputs=(after,), + artifact_outputs=(ArtifactOutput("projection", PROJECTION_TYPE),), + description="Borrow original ACS/ASEC person support and retain authenticated current-coverage source projection.", + ) + literal = Node( + RAW_NODE, + "us.survey_health.raw_columns@1", + population=SOURCE_NODE, + outputs=_outputs(raw), + params=params, + artifact_inputs=(_source_edge(),), + artifact_outputs=(ArtifactOutput("columns", COLUMNS_TYPE),), + description="Expose literal coverage, allocation and edit codes with survey observation years.", + ) + recodes = [] + for field in health.FIELDS: + inputs = [ + health.SOURCE_PREFIX + "source", + health.SOURCE_PREFIX + field.asec, + health.SOURCE_PREFIX + "I_" + field.asec, + ] + if field.acs: + inputs += [ + health.SOURCE_PREFIX + field.acs, + health.SOURCE_PREFIX + "F" + field.acs + "P", + ] + recodes.append( + Node( + RECODE_PREFIX + field.output, + "us.survey_health.recode@1", + population=SOURCE_NODE, + inputs=(Slice("person", tuple(inputs)),), + outputs=_outputs(health.recode_field(qualified.raw, field.output)), + params={**params, "field": field.output}, + artifact_inputs=( + _source_edge(), + _column_artifact(RAW_NODE, "health_raw_columns"), + ), + artifact_outputs=(ArtifactOutput("columns", COLUMNS_TYPE),), + description=f"Current coverage: {field.asec}; ACS {field.acs or field.acs_gap}. Missing codes remain unknown.", + ) + ) + identity = tuple( + f("person") + for f in ( + health.provenance.support_source_id_column, + health.provenance.support_clone_index_column, + health.provenance.spine_source_id_column, + health.provenance.support_channel_column, + ) + ) + attach = Node( + ATTACH_NODE, + "us.survey_health.attach@1", + population=receiving_version, + inputs=(Slice("person", identity),), + outputs=_outputs(_completed(qualified)), + params=params, + artifact_inputs=( + after, + _source_edge(), + _column_artifact(RAW_NODE, "health_raw_columns"), + *( + _column_artifact(n.id, "health_field_" + str(i)) + for i, n in enumerate(recodes) + ), + ), + artifact_outputs=(ArtifactOutput("attachment", ATTACHMENT_TYPE),), + description="Attach only exact source-person observations and explicit unknowns to both PUF clones; preserve all incoming fields.", + ) + return (create, literal, *recodes, attach) + + +def _expected_artifacts(qualified): + return { + (SOURCE_NODE, "projection"): qualified.projection, + (RAW_NODE, "columns"): _table_bytes(health.source_columns(qualified.raw)), + **{ + (RECODE_PREFIX + f.output, "columns"): _table_bytes( + health.recode_field(qualified.raw, f.output) + ) + for f in health.FIELDS + }, + } + + +def _check_artifacts(node, artifacts, qualified): + require(set(artifacts) == {a.name for a in node.artifact_inputs}, "ARTIFACT_ROSTER") + expected = _expected_artifacts(qualified) + for edge in node.artifact_inputs: + value = artifacts[edge.name] + require( + type(value) is ArtifactValue + and value.type == edge.type + and type(value.payload) is bytes, + "ARTIFACT_TYPE", + ) + if (edge.producer, edge.artifact) in expected: + require( + value.payload == expected[edge.producer, edge.artifact], + "ARTIFACT_PAYLOAD", + ) + # The incoming amount edge is authenticated by the owning host, which + # retains its node/store keys. The fragment never treats its JSON as an issuer. + + +def _result(qualified, node, people): + if node.id == SOURCE_NODE: + original = source.source._copy_source(qualified.source_frame) + # A projection branch keeps original support/design and structural IDs; + # its coverage cells are introduced visibly by the following nodes. + tables = {} + for entity in original.entities: + columns = [original.schema.entity_id_column(entity)] + if entity == original.schema.person_entity: + columns += [ + original.schema.membership_column(e) + for e in original.schema.group_entities + ] + tables[entity] = original.table(entity).loc[:, columns].copy() + require( + np.array_equal( + qualified.origins.index.to_numpy(), original.person.person_id.to_numpy() + ), + "CREATE_ORIGIN_AXIS", + ) + tables["person"]["health_native_person_id"] = ( + qualified.origins.native_person_id.to_numpy(copy=True) + ) + projected = Frame( + tables, + original.schema, + dict(original._weights), + original.strata, + metadata=original.metadata, + mass_log=original.mass_log, + ) + return KernelResult( + frame=projected, artifacts={"projection": qualified.projection} + ) + if node.id == RAW_NODE: + table = health.source_columns(qualified.raw) + elif node.id.startswith(RECODE_PREFIX): + table = health.recode_field(qualified.raw, node.params["field"]) + else: + require(node.id == ATTACH_NODE, "NODE_ID") + columns = health.attach_columns( + qualified.origins, SimpleNamespace(person=people), _completed(qualified) + ) + return KernelResult( + columns=columns, + artifacts={ + "attachment": _json( + { + "protocol": health.PROTOCOL, + "projection_sha256": hashlib.sha256( + qualified.projection + ).hexdigest(), + "receiving_version": node.population, + "rows": len(people), + "source_admission_issued": False, + "release_eligible": False, + } + ) + }, + ) + require( + people is not None + and np.array_equal(people.person_id.to_numpy(), table.index.to_numpy()), + "SOURCE_PERSON_AXIS", + ) + return KernelResult( + columns={("person", c): table[c].copy() for c in table}, + artifacts={"columns": _table_bytes(table)}, + ) + + +def expected_health_population(node_id, incoming, *, qualified, node, artifacts): + """Reconstruct the complete output, independently of executor cache results.""" + require(node_id == node.id, "EXPECTED_NODE") + _check_artifacts(node, artifacts, qualified) + result = _result( + qualified, node, None if incoming is None else incoming.frame.person + ) + if node.structural is StructuralDelta.CREATE: + require(incoming is None, "CREATE_INCOMING") + return population_ops.Population.from_frame(result.frame, node.id) + require(incoming is not None, "ORDINARY_INCOMING") + return population_ops.patch(incoming, node, result) + + +class _HealthKernel(KernelBase): + capabilities = Capabilities( + Determinism.DETERMINISTIC, + numeric=Numeric.PLATFORM_BITWISE, + dependencies=("numpy", "pandas"), + ) + + def __init__(self, qualified, nodes, require_current, *, create=False, ref): + self.ref, self.qualified, self.nodes = ref, qualified, nodes + self.require_current, self.seal = ( + require_current, + health_coverage_seal(qualified), + ) + if create: + self.capabilities = replace( + self.capabilities, structural=StructuralDelta.CREATE + ) + + def implementation_hash(self): + return source_hash( + sys.modules[__name__], + health, + source, + physical, + source.source, + source.housing, + source.records, + source.asec, + source.source_csv_builtin, + sys.modules[health.__package__ + ".cps_carried"], + dependencies=self.capabilities.dependencies, + ) + + def run(self, context): + self.require_current() + require(health_coverage_seal(self.qualified) == self.seal, "QUALIFIED_CHANGED") + require( + context.node in self.nodes and context.node.kernel == self.ref, + "KERNEL_NODE", + ) + _check_artifacts(context.node, context.artifacts, self.qualified) + if context.node.id.startswith(RECODE_PREFIX): + expected = health.source_columns(self.qualified.raw) + actual = context.tables["person"].set_index("person_id") + for selection in context.node.inputs: + for column in selection.columns: + require( + population_ops.storage_equal(actual[column], expected[column]), + "RECODE_SOURCE_SLICE", + ) + result = _result(self.qualified, context.node, context.tables.get("person")) + result_seal = ( + physical._table_stamp( + pd.concat( + [v.rename(c) for (_e, c), v in result.columns.items()], axis=1 + ) + ) + if result.columns + else None + ) + frame_seal = ( + source.source._frame_identity(result.frame) + if result.frame is not None + else None + ) + artifacts = tuple(sorted(result.artifacts.items())) + self.require_current() + require(health_coverage_seal(self.qualified) == self.seal, "QUALIFIED_CHANGED") + require( + ( + physical._table_stamp( + pd.concat( + [v.rename(c) for (_e, c), v in result.columns.items()], axis=1 + ) + ) + if result.columns + else None + ) + == result_seal + and ( + source.source._frame_identity(result.frame) + if result.frame is not None + else None + ) + == frame_seal + and tuple(sorted(result.artifacts.items())) == artifacts, + "RESULT_CHANGED", + ) + return result + + +def health_coverage_kernels(qualified, *, receiving_version, after, require_current): + nodes = health_coverage_nodes( + qualified, receiving_version=receiving_version, after=after + ) + require(callable(require_current), "HOST_CALLBACK") + return tuple( + _HealthKernel( + qualified, + nodes, + require_current, + create=ref == "us.survey_health.source@1", + ref=ref, + ) + for ref in dict.fromkeys(n.kernel for n in nodes) + ) diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/graph_current_survey_predictors.py b/packages/microcosm-build/src/microcosm/build/us_runtime/graph_current_survey_predictors.py new file mode 100644 index 000000000..b6d9cfd66 --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/graph_current_survey_predictors.py @@ -0,0 +1,616 @@ +"""Graph training, draw and attachment of current survey financial predictors. + +The ASEC fitting branch leaves the source CREATE before importance allocation; +its household-derived person weights remain DESIGN. No PUF clone is a donor. +All fitted totals, prior-target conditioning and final source-origin joins are +visible typed graph dependencies. Cold and replay results require the same +materialized verifier, independently of cache or node receipt labels. +""" + +from __future__ import annotations + +import sys + +import numpy as np +import pandas as pd + +from microcosm.fit import _graph_legacy_qrf as codec +from microcosm.fit import model_input, qrf +from microcosm.fit.graph_legacy_apply_matrix import ( + MATRIX_APPLY_STATE_TYPE, + decode_matrix_apply_state, +) +from microcosm.fit.graph_legacy_qrf import ( + legacy_qrf_apply_matrix_nodes, + legacy_qrf_train_nodes, +) +from microcosm.frame import US_SCHEMA +from microcosm.graph import ( + ArtifactInput, + ArtifactOutput, + ArtifactType, + Capabilities, + Determinism, + KernelResult, + Node, + Numeric, + Owned, + Slice, + StructuralDelta, + source_hash, +) +from microcosm.graph import population as population_ops + +from . import current_survey_predictors as values +from . import survey_population_replay as replay + +host = values.host +require = values.require +PROJECTION_NODE = "survey_predictors.source_projection" +DONOR_NODE = "survey_predictors.asec_design_donor" +DONOR_COLUMNS_NODE = "survey_predictors.asec_current_columns" +FIT_PREFIX = "survey_predictors.fit" +APPLY_PREFIX = "survey_predictors.apply" +ATTACH_NODE = "survey_predictors.attach" +PROJECTION_TYPE = ArtifactType("microcosm.us.current_survey_predictor_projection", 1) + + +def _inputs(frame): + result = [] + for entity in US_SCHEMA.entities: + structural = {US_SCHEMA.entity_id_column(entity)} + if entity == "person": + structural.update( + US_SCHEMA.membership_column(e) for e in US_SCHEMA.group_entities + ) + result.append( + Slice(entity, tuple(c for c in frame.table(entity) if c not in structural)) + ) + return tuple(result) + + +def _projection_edges(): + return ( + ArtifactInput("projection", PROJECTION_NODE, "projection", PROJECTION_TYPE), + ArtifactInput( + "matrix", PROJECTION_NODE, "matrix", model_input.RECIPIENT_MATRIX_TYPE + ), + ) + + +def _attach_edges(): + result = list(_projection_edges()) + for i in range(len(values.TARGETS)): + result.extend( + ( + ArtifactInput( + f"raw_{i:03d}", + f"{APPLY_PREFIX}.{i:03d}", + "raw_draw", + codec.RAW_TARGET_TYPE, + ), + ArtifactInput( + f"state_{i:03d}", + f"{APPLY_PREFIX}.{i:03d}", + "apply_state", + MATRIX_APPLY_STATE_TYPE, + ), + ) + ) + return tuple(result) + + +def _geography_edge(): + return ArtifactInput( + "geography_validation", + "geography.gate", + "validation", + host.survey_budget.geography.atomic_graph.ATOMIC_GEOGRAPHY_VALIDATION_TYPE, + ) + + +def _params(qualified, host_pins, n_estimators): + require(type(n_estimators) is int and n_estimators > 0, "TREE_COUNT") + values.feature_columns(qualified.demographic_conditioning) + geography_enabled = qualified.geography_config_payload is not None + require( + (type(qualified.geography_validation) is bytes) + if geography_enabled + else qualified.geography_validation is None, + "GEOGRAPHY_VALIDATION_VALUES", + ) + names = {e.name for e in host.current_survey_host_edges()} + if geography_enabled: + names.add(_geography_edge().name) + require( + type(host_pins) is dict and set(host_pins) == names, + "HOST_PINS", + ) + for pin in host_pins.values(): + require( + type(pin) is dict + and set(pin) == {"producer_key", "artifact_key", "payload_sha256"} + and all(codec._hash(v) for v in pin.values()), + "HOST_PIN_DIGESTS", + ) + return { + "protocol": values.PROTOCOL, + "source_projection_sha256": codec.sha(qualified.projection), + "host_edges": codec.encode_json(host_pins).decode(), + "seed": values.SEED, + "n_estimators": n_estimators, + "demographic_conditioning": qualified.demographic_conditioning, + "geography_config_sha256": None + if qualified.geography_config_payload is None + else codec.sha(qualified.geography_config_payload), + } + + +def current_survey_predictor_nodes( + qualified, clone_frame, *, host_pins, n_estimators=100 +): + require( + type(qualified) is values.QualifiedSurveyPredictors, "QUALIFIED_VALUES_TYPE" + ) + params = _params(qualified, host_pins, n_estimators) + predictors = values.feature_columns(qualified.demographic_conditioning) + projection = Node( + PROJECTION_NODE, + CurrentSurveyPredictorProjectionKernel.ref, + # This ordinary node must live outside CREATE: its typed allocation + # evidence is downstream of that version's structural allocation node. + population=DONOR_NODE, + inputs=_inputs(qualified.donor_frame), + params=params, + artifact_inputs=host.current_survey_host_edges(), + artifact_outputs=( + ArtifactOutput("projection", PROJECTION_TYPE), + ArtifactOutput("matrix", model_input.RECIPIENT_MATRIX_TYPE), + ), + ) + donor = Node( + DONOR_NODE, + CurrentSurveyPredictorDonorFilterKernel.ref, + base=host.survey_graph.CREATE_NODE, + structural=StructuralDelta.FILTER, + inputs=_inputs(qualified.source_frame), + mass="free", + params=params, + # Branch from the authenticated CREATE before its weight transition. + # The later projection depends on this FILTER, never conversely. + artifact_inputs=( + host.current_survey_host_edges()[0], + *( + (_geography_edge(),) + if qualified.geography_config_payload is not None + else () + ), + ), + description="Select native ASEC whole households before allocation; retain original design weights for survey financial fitting.", + ) + columns = Node( + DONOR_COLUMNS_NODE, + CurrentSurveyPredictorDonorColumnsKernel.ref, + population=DONOR_NODE, + inputs=_inputs(qualified.donor_frame), + params=params, + outputs=tuple( + Owned("person", c, "float64") for c in (*predictors, *values.TARGETS) + ), + artifact_inputs=_projection_edges(), + ) + fit = legacy_qrf_train_nodes( + FIT_PREFIX, + population=DONOR_NODE, + entity="person", + predictors=predictors, + targets=values.TARGETS, + seed=values.SEED, + phase=values.PHASE, + n_estimators=n_estimators, + zero_atol=0, + ) + apply = legacy_qrf_apply_matrix_nodes( + APPLY_PREFIX, + population=host.survey_clone.COMBINED_CLONE_NODE, + fit_nodes=fit, + matrix_producer=PROJECTION_NODE, + seed=values.SEED, + phase=values.PHASE, + ) + attach = Node( + ATTACH_NODE, + CurrentSurveyPredictorAttachKernel.ref, + population=host.survey_clone.COMBINED_CLONE_NODE, + inputs=_inputs(clone_frame), + params=params, + outputs=tuple( + Owned("person", c, "float64", rewrite=(c in clone_frame.person)) + for c in values.OUTPUTS + ), + artifact_inputs=_attach_edges(), + ) + return (projection, donor, columns, *fit, *apply, attach) + + +class _Kernel(host._CurrentSurveyKernel): + def __init__( + self, + preparation, + allocated_population, + clone_population, + *, + host_pins, + n_estimators=100, + demographic_conditioning=False, + geography_config=None, + ): + self.preparation = preparation + self.allocated_population = allocated_population + self.clone_population = clone_population + self.host_pins = codec.decode_json(codec.encode_json(host_pins)) + self.n_estimators = n_estimators + values.feature_columns(demographic_conditioning) + self.demographic_conditioning = demographic_conditioning + self.geography_config = geography_config + self.geography_config_payload = host.survey_budget._config_payload( + geography_config + ) + + def implementation_hash(self): + demographics = values.observed_geography.demographics + return codec.sha( + codec.encode_json( + { + "host": super().implementation_hash(), + "predictors": source_hash( + sys.modules[__name__], + values, + values.leaves, + values.universe, + values.observed_geography, + demographics, + demographics.qualify_current_asec_demographics, + demographics.demographic, + demographics.demographic.load_authenticated_asec_demographic_source, + demographics.demographic._snapshot, + demographics.household, + demographics.source_csv_builtin, + population_ops, + replay, + qrf, + model_input, + sys.modules[decode_matrix_apply_state.__module__], + sys.modules[legacy_qrf_train_nodes.__module__], + # The optional postclone geography admission invokes + # the budget owner's complete reconstruction closure. + *host.survey_budget._modules(), + dependencies=self.capabilities.dependencies, + ), + } + ) + ) + + def _qualified(self, context): + require(not context.sources, "UNDECLARED_SOURCE") + require( + host.survey_budget._config_payload(self.geography_config) + == self.geography_config_payload, + "GEOGRAPHY_CONFIG_CHANGED", + ) + result = values.qualify_current_survey_predictors( + self.preparation, + self.allocated_population, + self.clone_population, + demographic_conditioning=self.demographic_conditioning, + geography_config=self.geography_config, + ) + require( + result.geography_config_payload == self.geography_config_payload, + "GEOGRAPHY_CONFIG_CHANGED", + ) + nodes = current_survey_predictor_nodes( + result, + self.clone_population.frame, + host_pins=self.host_pins, + n_estimators=self.n_estimators, + ) + expected = {n.id: n for n in nodes} + require(context.node == expected.get(context.node.id), "NODE_DECLARATION") + require( + set(context.artifacts) == {e.name for e in context.node.artifact_inputs}, + "ARTIFACT_ROSTER", + ) + if context.node.id == DONOR_NODE: + edge = host.current_survey_host_edges()[0] + require(edge.name == "preparation", "PREPARATION_EDGE") + value = host.shared.artifact(context, edge.name, edge.type) + require( + self.host_pins[edge.name] + == { + "producer_key": value.producer_key, + "artifact_key": value.key, + "payload_sha256": codec.sha(value.payload), + } + and value.payload == self.preparation.payload, + "DONOR_PREPARATION_BINDING", + ) + if result.geography_config_payload is not None: + edge = _geography_edge() + value = host.shared.artifact(context, edge.name, edge.type) + require( + self.host_pins[edge.name] + == { + "producer_key": value.producer_key, + "artifact_key": value.key, + "payload_sha256": codec.sha(value.payload), + } + and type(value.payload) is bytes + and value.payload == result.geography_validation, + "GEOGRAPHY_VALIDATION_BINDING", + ) + elif context.node.id != PROJECTION_NODE: + projection = host.shared.artifact(context, "projection", PROJECTION_TYPE) + matrix = host.shared.artifact( + context, "matrix", model_input.RECIPIENT_MATRIX_TYPE + ) + host.shared.siblings(context, ("projection", "matrix")) + require( + projection.payload == result.projection + and matrix.payload == result.matrix, + "SOURCE_ARTIFACT_BYTES", + ) + return result + + +class CurrentSurveyPredictorProjectionKernel(_Kernel): + ref = "us.survey_predictors.source_projection@1" + + def run(self, context): + qualified = self._qualified(context) + host._current_context_frame(context, qualified.donor_frame) + loaded = {} + for edge in host.current_survey_host_edges(): + value = host.shared.artifact(context, edge.name, edge.type) + require( + self.host_pins[edge.name] + == { + "producer_key": value.producer_key, + "artifact_key": value.key, + "payload_sha256": codec.sha(value.payload), + }, + "HOST_EDGE_PIN", + ) + loaded[edge.name] = value.payload + host.shared.siblings(context, ("allocation", "frame_context")) + require( + loaded["preparation"] == self.preparation.payload + and codec.sha(loaded["allocation"]) + == qualified.evidence["allocation_sha256"] + and codec.sha(loaded["frame_context"]) + == codec.decode_json(loaded["allocation"])["output_context_sha256"], + "HOST_EDGE_BYTES", + ) + return KernelResult( + artifacts={"projection": qualified.projection, "matrix": qualified.matrix}, + receipt=qualified.evidence, + ) + + +class CurrentSurveyPredictorDonorFilterKernel(_Kernel): + ref = "us.survey_predictors.asec_design_donor@1" + capabilities = Capabilities( + Determinism.DETERMINISTIC, + structural=StructuralDelta.FILTER, + numeric=Numeric.PLATFORM_BITWISE, + dependencies=host._CurrentSurveyKernel.capabilities.dependencies, + ) + + def run(self, context): + self._qualified(context) + # CREATE carries raw ACS NIU blanks. The qualified feature frame has + # separately receipted materialized zeros and is not this context. + original = self.preparation._checked()[2].frame + host._current_context_frame(context, original) + person = original.person + keep = ( + person[values.provenance.support_channel_column("person")] + .eq("asec") + .to_numpy() + ) + return KernelResult( + keep=pd.Series( + keep, + index=pd.Index(person.person_id.to_numpy(), name="person_id"), + dtype=bool, + ), + receipt={ + "selection": "ASEC_native_whole_households", + "fit_weight_kind": "design", + "release_eligible": False, + }, + ) + + +class CurrentSurveyPredictorDonorColumnsKernel(_Kernel): + ref = "us.survey_predictors.asec_current_columns@1" + + def run(self, context): + qualified = self._qualified(context) + host._current_context_frame(context, qualified.donor_frame) + return KernelResult( + columns={ + ("person", c): qualified.donor_columns[c] + for c in ( + *values.feature_columns(qualified.demographic_conditioning), + *values.TARGETS, + ) + }, + receipt=qualified.evidence, + ) + + +def read_current_survey_draws( + matrix, + matrix_producer_key, + raw_draws, + apply_states, + *, + demographic_conditioning=False, +): + predictors = values.feature_columns(demographic_conditioning) + require( + codec._hash(matrix_producer_key) + and type(raw_draws) is tuple + and type(apply_states) is tuple + and len(raw_draws) == len(apply_states) == len(values.TARGETS), + "DRAW_CHAIN_ROSTER", + ) + prepared = model_input.decode_recipient_matrix(matrix) + require( + prepared.entity == "person" and tuple(prepared.features.columns) == predictors, + "MATRIX_FEATURE_ROSTER", + ) + result = pd.DataFrame(index=prepared.features.index) + models = [] + for i, target in enumerate(values.TARGETS): + packet = decode_matrix_apply_state(apply_states[i]) + require( + packet["matrix_sha256"] == codec.sha(matrix) + and packet["matrix_producer_key"] == matrix_producer_key, + "DRAW_MATRIX_BINDING", + ) + application, chain = codec.read_application( + codec.encode_json(packet["application"]) + ) + require( + chain.entity == "person" + and tuple(chain.predictors) == predictors + and tuple(chain.targets) == values.TARGETS + and tuple(chain.completed_targets) == values.TARGETS[: i + 1] + and chain.recipient_index == qrf._index_identity(prepared.features.index) + and application["seed"] == values.SEED + and application["raw_targets"] + == [ + {"target": name, "sha256": codec.sha(raw)} + for name, raw in zip( + values.TARGETS[: i + 1], raw_draws[: i + 1], strict=True + ) + ], + "DRAW_CHAIN_BINDING", + ) + current_models = application["models"] + require( + current_models[:i] == models and len(current_models) == i + 1, + "DRAW_MODEL_HISTORY", + ) + models = current_models + values_array = codec.read_raw_target( + raw_draws[i], target=target, index=prepared.features.index + ) + require(np.isfinite(values_array).all(), "DRAW_UNKNOWN") + result[target] = values_array + return result + + +class CurrentSurveyPredictorAttachKernel(_Kernel): + ref = "us.survey_predictors.attach@1" + + def run(self, context): + qualified = self._qualified(context) + host._current_context_frame(context, self.clone_population.frame) + matrix = host.shared.artifact( + context, "matrix", model_input.RECIPIENT_MATRIX_TYPE + ) + raw, states = [], [] + for i in range(len(values.TARGETS)): + r, s = f"raw_{i:03d}", f"state_{i:03d}" + host.shared.siblings(context, (r, s)) + raw.append(host.shared.artifact(context, r, codec.RAW_TARGET_TYPE).payload) + states.append( + host.shared.artifact(context, s, MATRIX_APPLY_STATE_TYPE).payload + ) + drawn = read_current_survey_draws( + matrix.payload, + matrix.producer_key, + tuple(raw), + tuple(states), + demographic_conditioning=self.demographic_conditioning, + ) + columns = values.complete_predictor_columns( + qualified, self.clone_population.frame, drawn + ) + return KernelResult( + columns=columns, + receipt={ + **qualified.evidence, + "raw_sha256": [codec.sha(r) for r in raw], + "all_output_cells_available": True, + "ACS_financial_origin": "modeled", + "paired_draws": "source_origin_join", + "host_weights_changed": False, + }, + ) + + +def verify_materialized_current_survey_predictors( + preparation, + allocated_population, + clone_population, + *, + population, + projection, + matrix, + matrix_producer_key, + raw_draws, + apply_states, + host_pins, + n_estimators=100, + demographic_conditioning=False, + geography_config=None, +): + """Use actual executor-observed Population and authenticated typed artifacts. + + Call only after graph store/type/producer-key checks, both cold and replay. + Requalifies the current source owners, then compares every column, identity, + owner, metadata value, weight and mass ledger of the expected full result. + """ + qualified = values.qualify_current_survey_predictors( + preparation, + allocated_population, + clone_population, + demographic_conditioning=demographic_conditioning, + geography_config=geography_config, + ) + require( + projection == qualified.projection and matrix == qualified.matrix, + "MATERIALIZED_SOURCE", + ) + drawn = read_current_survey_draws( + matrix, + matrix_producer_key, + raw_draws, + apply_states, + demographic_conditioning=demographic_conditioning, + ) + attach = current_survey_predictor_nodes( + qualified, + clone_population.frame, + host_pins=host_pins, + n_estimators=n_estimators, + )[-1] + expected = population_ops.patch( + clone_population, + attach, + KernelResult( + columns=values.complete_predictor_columns( + qualified, clone_population.frame, drawn + ) + ), + ) + replay.same_replayed_population(expected, population) + return { + **qualified.evidence, + "all_output_cells_available": True, + "paired_draws": "source_origin_join", + } diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/graph_current_survey_puf_transfer.py b/packages/microcosm-build/src/microcosm/build/us_runtime/graph_current_survey_puf_transfer.py new file mode 100644 index 000000000..e1ce9a0f3 --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/graph_current_survey_puf_transfer.py @@ -0,0 +1,407 @@ +"""Current survey-first E00900 diagnostic placement, not donor admission. + +The source-qualified host is the accepted ACS+ASEC allocation and whole-household +clone. Donor qualification belongs to a separately reviewed upstream producer. +This module writes a development tax-unit diagnostic, never an engine input. +The mandatory materialized check also runs after replay, using the retained live +preparation and the real executor-observed Population. +""" + +from __future__ import annotations + +import sys + +import numpy as np +import pandas as pd + +from microcosm.fit import model_input +from microcosm.fit.graph_legacy_apply_matrix import ( + MATRIX_APPLY_STATE_TYPE, + decode_matrix_apply_state, +) +from microcosm.fit.graph_legacy_qrf import ( + legacy_qrf_apply_matrix_nodes, + legacy_qrf_train_nodes, +) +from microcosm.frame import US_SCHEMA +from microcosm.graph import ( + ArtifactInput, + ArtifactOutput, + ArtifactType, + KernelResult, + Node, + Owned, + Slice, + source_hash, +) +from microcosm.graph import population as population_ops + +from . import graph_puf_diagnostic_consumer as host +from . import survey_population_replay as replay + +shared, detail, codec, require = host.shared, host.detail, host.codec, host.require +SCOPE = "current_survey_e00900_development_not_tax_inputs" +PLACEMENT_TYPE = ArtifactType("microcosm.us.current_survey_puf_placement", 1) +PLACEMENT_NODE = "puf_diagnostic.current_survey_placement" +APPLY_PREFIX = "puf_diagnostic.current_survey_apply" +ATTACH_NODE = "puf_diagnostic.current_survey_attach" +SEED = 578 + + +def _inputs(frame): + values = [] + for entity in US_SCHEMA.entities: + structural = {US_SCHEMA.entity_id_column(entity)} + if entity == US_SCHEMA.person_entity: + structural.update( + US_SCHEMA.membership_column(e) for e in US_SCHEMA.group_entities + ) + values.append( + Slice(entity, tuple(c for c in frame.table(entity) if c not in structural)) + ) + return tuple(values) + + +def _edges(): + return ( + ArtifactInput( + "projection", + host.CURRENT_PROJECTION_NODE, + "projection", + host.CURRENT_PROJECTION_TYPE, + ), + ArtifactInput( + "matrix", + host.CURRENT_MATRIX_NODE, + "matrix", + model_input.RECIPIENT_MATRIX_TYPE, + ), + ) + + +def _attach_edges(): + return ( + *_edges(), + ArtifactInput("placement", PLACEMENT_NODE, "placement", PLACEMENT_TYPE), + ArtifactInput( + "raw_draw", APPLY_PREFIX + ".000", "raw_draw", codec.RAW_TARGET_TYPE + ), + ArtifactInput( + "apply_state", APPLY_PREFIX + ".000", "apply_state", MATRIX_APPLY_STATE_TYPE + ), + ) + + +def _placement_node(frame): + require( + detail.MASK not in frame.table("tax_unit") + and detail.OUTPUT not in frame.table("tax_unit"), + "SURVEY_TRANSFER_ALREADY_PRESENT", + ) + return Node( + PLACEMENT_NODE, + CurrentSurveyPlacementKernel.ref, + population=host.survey_clone.COMBINED_CLONE_NODE, + inputs=_inputs(frame), + outputs=(Owned("tax_unit", detail.MASK, "bool"),), + params={"scope": SCOPE}, + artifact_inputs=_edges(), + artifact_outputs=(ArtifactOutput("placement", PLACEMENT_TYPE),), + ) + + +def _attach_node(frame): + inputs = tuple( + Slice( + s.entity, (*s.columns, detail.MASK) if s.entity == "tax_unit" else s.columns + ) + for s in _inputs(frame) + ) + return Node( + ATTACH_NODE, + CurrentSurveyAttachKernel.ref, + population=host.survey_clone.COMBINED_CLONE_NODE, + inputs=inputs, + outputs=(Owned("tax_unit", detail.OUTPUT, "float64", rows=detail.MASK),), + params={"scope": SCOPE}, + artifact_inputs=_attach_edges(), + ) + + +def _binding(projection, matrix, matrix_producer_key, frame, mask): + return codec.encode_json( + { + "scope": SCOPE, + "projection_sha256": codec.sha(projection), + "matrix_sha256": codec.sha(matrix), + "matrix_producer_key": matrix_producer_key, + "features": list(detail.FEATURES), + "targets": [detail.TARGET], + "output": detail.OUTPUT, + "selected_ids": frame.table("tax_unit") + .tax_unit_id.to_numpy()[mask] + .tolist(), + "source_admission": detail.SOURCE_ADMISSION, + "release_eligible": False, + } + ) + + +def _mask_result(frame, mask): + return KernelResult( + columns={ + ("tax_unit", detail.MASK): pd.Series( + mask, index=frame.table("tax_unit").tax_unit_id, dtype=bool + ), + } + ) + + +def _raw_values(matrix, matrix_producer_key, raw_draw, apply_state): + packet = decode_matrix_apply_state(apply_state) + require( + packet["matrix_sha256"] == codec.sha(matrix) + and packet["matrix_producer_key"] == matrix_producer_key, + "SURVEY_TRANSFER_MATRIX_PRODUCER", + ) + application, chain = codec.read_application( + codec.encode_json(packet["application"]) + ) + require( + tuple(chain.completed_targets) == (detail.TARGET,) + and tuple(chain.predictors) == detail.FEATURES + and application["seed"] == SEED + and application["raw_targets"] + == [ + { + "target": detail.TARGET, + "sha256": codec.sha(raw_draw), + } + ], + "SURVEY_TRANSFER_RAW_STATE", + ) + prepared = model_input.decode_recipient_matrix(matrix) + require( + chain.entity == prepared.entity == "tax_unit" + and tuple(chain.targets) == (detail.TARGET,) + and chain.recipient_index + == shared.qrf._index_identity(prepared.features.index), + "SURVEY_TRANSFER_RAW_INDEX", + ) + values = codec.read_raw_target( + raw_draw, target=detail.TARGET, index=prepared.features.index + ) + require(np.isfinite(values).all(), "SURVEY_TRANSFER_NONFINITE_DRAW") + return pd.Series(values, index=prepared.entity_ids, dtype="float64") + + +class _CurrentTransferKernel(host._CurrentSurveyKernel): + def __init__(self, preparation, allocated_population, clone_population): + self.preparation = preparation + self.allocated_population = allocated_population + self.clone_population = clone_population + + def implementation_hash(self): + # Independent extension closure; do not relabel or refresh survey inventory. + return codec.sha( + codec.encode_json( + { + "host": super().implementation_hash(), + "transfer": source_hash( + sys.modules[__name__], + population_ops, + replay, + sys.modules[decode_matrix_apply_state.__module__], + sys.modules[legacy_qrf_apply_matrix_nodes.__module__], + dependencies=self.capabilities.dependencies, + ), + } + ) + ) + + def _qualified(self, context): + expected_projection, expected_matrix, mask, evidence = ( + host.qualify_current_survey_host( + self.preparation, self.allocated_population, self.clone_population + ) + ) + projection = shared.artifact( + context, "projection", host.CURRENT_PROJECTION_TYPE + ) + matrix = shared.artifact(context, "matrix", model_input.RECIPIENT_MATRIX_TYPE) + require( + projection.payload == expected_projection + and matrix.payload == expected_matrix, + "SURVEY_TRANSFER_HOST_ARTIFACTS", + ) + return projection, matrix, mask, evidence + + +class CurrentSurveyPlacementKernel(_CurrentTransferKernel): + ref = "us.puf_diagnostic.current_survey_placement@1" + + def run(self, context): + require( + context.node == _placement_node(self.clone_population.frame) + and not context.sources + and set(context.artifacts) == {"projection", "matrix"}, + "SURVEY_TRANSFER_PLACEMENT_DECLARATION", + ) + projection, matrix, mask, evidence = self._qualified(context) + host._current_context_frame(context, self.clone_population.frame) + result = _mask_result(self.clone_population.frame, mask) + return KernelResult( + columns=result.columns, + artifacts={ + "placement": _binding( + projection.payload, + matrix.payload, + matrix.producer_key, + self.clone_population.frame, + mask, + ) + }, + receipt={ + **evidence, + "scope": SCOPE, + "source_admission": detail.SOURCE_ADMISSION, + }, + ) + + +class CurrentSurveyAttachKernel(_CurrentTransferKernel): + ref = "us.puf_diagnostic.current_survey_attach@1" + + def run(self, context): + require( + context.node == _attach_node(self.clone_population.frame) + and not context.sources + and set(context.artifacts) + == {"projection", "matrix", "placement", "raw_draw", "apply_state"}, + "SURVEY_TRANSFER_ATTACH_DECLARATION", + ) + projection, matrix, mask, evidence = self._qualified(context) + placement = shared.artifact(context, "placement", PLACEMENT_TYPE) + raw = shared.artifact(context, "raw_draw", codec.RAW_TARGET_TYPE) + state = shared.artifact(context, "apply_state", MATRIX_APPLY_STATE_TYPE) + shared.siblings(context, ("raw_draw", "apply_state")) + require( + placement.payload + == _binding( + projection.payload, + matrix.payload, + matrix.producer_key, + self.clone_population.frame, + mask, + ), + "SURVEY_TRANSFER_PLACEMENT_BINDING", + ) + # Comparison-only normal patch; no new source/successor owner is issued. + masked = population_ops.patch( + self.clone_population, + _placement_node(self.clone_population.frame), + _mask_result(self.clone_population.frame, mask), + ) + host._current_context_frame(context, masked.frame) + values = _raw_values( + matrix.payload, matrix.producer_key, raw.payload, state.payload + ) + return KernelResult( + columns={("tax_unit", detail.OUTPUT): values}, + receipt={ + **evidence, + "scope": SCOPE, + "source_admission": detail.SOURCE_ADMISSION, + "raw_sha256": codec.sha(raw.payload), + "native_output_cells_written": 0, + "host_weights_changed": False, + }, + ) + + +def current_survey_transfer_nodes(frame, *, fit_nodes): + """Connect the independently qualified donor fits to the accepted host matrix. + + This E00900 mechanism does not admit a donor source. A future complete PUF + chain uses the same maintained train/apply family with its full ordered + targets and separately reviewed person/tax-unit finalization contract. + """ + require( + type(fit_nodes) is tuple and len(fit_nodes) == 1, "SURVEY_TRANSFER_FIT_CHAIN" + ) + fit = fit_nodes[0] + expected = legacy_qrf_train_nodes( + "__comparison__", + population=fit.population, + entity="tax_unit", + predictors=detail.FEATURES, + targets=(detail.TARGET,), + seed=SEED, + phase=SCOPE, + n_estimators=2, + zero_atol=0, + )[0] + from dataclasses import replace + + require(fit == replace(expected, id=fit.id), "SURVEY_TRANSFER_FIT_DECLARATION") + applies = legacy_qrf_apply_matrix_nodes( + APPLY_PREFIX, + population=host.survey_clone.COMBINED_CLONE_NODE, + fit_nodes=fit_nodes, + matrix_producer=host.CURRENT_MATRIX_NODE, + seed=SEED, + phase=SCOPE, + ) + return (_placement_node(frame), *applies, _attach_node(frame)) + + +def verify_materialized_current_survey_transfer( + preparation, + allocated_population, + clone_population, + *, + population, + projection, + matrix, + matrix_producer_key, + placement, + raw_draw, + apply_state, +): + """Mandatory after actual typed-artifact/key/store checks, cold and replay. + + Inputs are the actual loaded payloads and producer key, not receipt labels. + Last source I/O requalifies the retained host. Pure comparison then verifies + every native/clone cell, axis, metadata value, weight, owner and mass ledger. + Reconstructed expected populations are comparison values, never source owners. + """ + require(codec._hash(matrix_producer_key), "SURVEY_TRANSFER_MATRIX_KEY") + expected_projection, expected_matrix, mask, evidence = ( + host.qualify_current_survey_host( + preparation, allocated_population, clone_population + ) + ) + require( + projection == expected_projection and matrix == expected_matrix, + "SURVEY_TRANSFER_MATERIALIZED_HOST", + ) + require( + placement + == _binding( + projection, matrix, matrix_producer_key, clone_population.frame, mask + ), + "SURVEY_TRANSFER_PLACEMENT_BINDING", + ) + values = _raw_values(matrix, matrix_producer_key, raw_draw, apply_state) + expected = population_ops.patch( + clone_population, + _placement_node(clone_population.frame), + _mask_result(clone_population.frame, mask), + ) + expected = population_ops.patch( + expected, + _attach_node(clone_population.frame), + KernelResult(columns={("tax_unit", detail.OUTPUT): values}), + ) + replay.same_replayed_population(expected, population) + return {**evidence, "scope": SCOPE, "source_admission": detail.SOURCE_ADMISSION} diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/graph_fiscal_dense_calibration.py b/packages/microcosm-build/src/microcosm/build/us_runtime/graph_fiscal_dense_calibration.py new file mode 100644 index 000000000..1c16e0cc8 --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/graph_fiscal_dense_calibration.py @@ -0,0 +1,602 @@ +"""Numerical dense fiscal calibration; complete-parent admission stays external. + +The caller supplies original DESIGN anchors and origin-derived absolute caps. +This module transports and checks those numbers but cannot authenticate their +source or the complete Population. The host must compare original anchors and +admit the complete parent and successor on cold execution, replay and return. + +One existing grouped Adam/free-mass solve preserves every initial zero exactly. +There is no pruning, restart, repeated calibration or row-cap relaxation. CSR +rows enter the existing public Target callable API one at a time: O(N) temporary +row memory, plus sparse storage, rather than a dense target-by-household table. +The existing solver chooses sparse/dense Torch storage for the resulting matrix; +its capped scaled-MAPE objective uses float32 and accepted weights use float64. +""" + +from __future__ import annotations + +import hashlib +import json +import sys +from dataclasses import dataclass, replace + +import numpy as np +import pandas as pd + +from microcosm.calibrate import TargetSet, calibrate, diagnostics_payload, group_bounds +from microcosm.calibrate import kernels as calibration_kernels +from microcosm.calibrate import target as target_module +from microcosm.frame import WeightKind, Weights +from microcosm.graph import ( + ArtifactInput, + ArtifactOutput, + ArtifactType, + Capabilities, + Determinism, + KernelBase, + KernelResult, + Node, + Numeric, + SeedSource, + StructuralDelta, + WeightTransition, + source_hash, +) +from microcosm.graph.canonical import canonical_json, normative +from microcosm.graph.keys import opaque_artifact_key + +from . import graph_fiscal_measurement as measurement +from .demographic_calibration_graph import DIAGNOSTICS_TYPE + +BOUNDS_TYPE = ArtifactType("microcosm.us.fiscal_calibration_numeric_bounds", 1) +BOUNDS_PROTOCOL = "microcosm.us.fiscal-calibration-numeric-bounds.v1" +ORIGIN_TYPE = ArtifactType("microcosm.us.fiscal_calibration_origin_diagnostics", 1) +MAX_BYTES = 64 * 1024**2 +_FIELDS = { + "protocol", + "population", + "budget_sha256", + "household_ids", + "original_design", + "incoming", + "group_indices", + "group_upper", + "row_upper", +} + + +def _require(condition, reason): + if not condition: + raise ValueError("FISCAL_CALIBRATION_" + reason) + + +def _sha(payload): + return hashlib.sha256(payload).hexdigest() + + +def _digest(value): + return ( + type(value) is str + and len(value) == 64 + and all(c in "0123456789abcdef" for c in value) + ) + + +def _same_array(a, b): + return a.dtype == b.dtype and a.shape == b.shape and a.tobytes() == b.tobytes() + + +@dataclass(frozen=True) +class FiscalCalibrationBounds: + """Decoded numerical references only; never an issued sampling budget.""" + + document: dict + household_ids: np.ndarray + original_design: Weights + incoming: Weights + grouped: group_bounds.GroupedUpperBounds + row_upper: np.ndarray + + +def decode_fiscal_calibration_bounds(payload: bytes) -> FiscalCalibrationBounds: + """Read bounded, canonical typed-ID and exact float64 reference arrays.""" + _require(type(payload) is bytes and 0 < len(payload) <= MAX_BYTES, "BOUNDS_PAYLOAD") + try: + raw = json.loads(payload) + except (ValueError, UnicodeError): + raise ValueError("FISCAL_CALIBRATION_BOUNDS_JSON") from None + _require(type(raw) is dict and set(raw) == _FIELDS, "BOUNDS_FIELDS") + _require(canonical_json(raw) == payload, "BOUNDS_CANONICAL") + _require(raw["protocol"] == BOUNDS_PROTOCOL, "BOUNDS_PROTOCOL") + _require(_digest(raw["budget_sha256"]), "BUDGET_DIGEST") + _require( + type(raw["population"]) is str and 0 < len(raw["population"]) <= 256, + "POPULATION", + ) + ids = measurement._read_array(raw["household_ids"], {" 0 and len(set(ids.tolist())) == len(ids), "HOUSEHOLD_IDS") + vectors = { + name: measurement._read_array(raw[name], {"= 0) for v in vectors.values()), + "REFERENCE_VALUES", + ) + _require( + all( + len(vectors[n]) == len(ids) + for n in ("original_design", "incoming", "row_upper") + ), + "REFERENCE_ALIGNMENT", + ) + _require(np.any(vectors["incoming"] > 0), "EMPTY_POSITIVE_SUPPORT") + groups = group_bounds.GroupedUpperBounds( + ids, + measurement._read_array(raw["group_indices"], {" bytes: + """Freeze explicitly supplied original references without issuing authority. + + ``row_upper`` is already the original source-reference cap, not an inferred + multiple of incoming or cloned DESIGN values. Its provenance and arithmetic + belong to the original budget owner. Full uint64 IDs and row order survive. + """ + _require( + type(original_design_weights) is Weights + and original_design_weights.kind is WeightKind.DESIGN, + "DESIGN_KIND", + ) + _require( + type(incoming_weights) is Weights + and incoming_weights.kind is WeightKind.IMPORTANCE, + "IMPORTANCE_KIND", + ) + _require( + type(grouped_upper_bounds) is group_bounds.GroupedUpperBounds, "GROUP_TYPE" + ) + _require( + type(household_ids) is np.ndarray + and household_ids.dtype.str in {" 0), + "FIXED_ZERO_SUPPORT", + ) + + +def verify_fiscal_calibration_bounds( + payload, + *, + household_ids, + incoming_weights, + original_design_weights, +) -> FiscalCalibrationBounds: + """Compare numeric bytes to a host's live references; not source admission. + + The host must obtain original_design_weights from retained Population + anchors and separately authenticate the original group and row caps. + """ + bounds = decode_fiscal_calibration_bounds(payload) + _require(_same_array(bounds.household_ids, household_ids), "HOUSEHOLD_ALIGNMENT") + _require( + type(incoming_weights) is Weights + and incoming_weights.kind is WeightKind.IMPORTANCE, + "IMPORTANCE_KIND", + ) + _require( + _same_array(bounds.incoming.values, incoming_weights.values), "INCOMING_ANCHOR" + ) + _require( + type(original_design_weights) is Weights + and original_design_weights.kind is WeightKind.DESIGN + and _same_array(bounds.original_design.values, original_design_weights.values), + "DESIGN_ANCHOR", + ) + return bounds + + +def _measurement_node(raw): + params = raw["params"] + result = measurement.fiscal_measurement_node( + measurement._registry(json.loads(params["registry"])), + population=raw["population"], + node_id=raw["id"], + input_columns={s["entity"]: tuple(s["columns"]) for s in raw["inputs"]}, + contract_targets=json.loads(params["contract_targets"]), + advertised_scopes=measurement._scopes(json.loads(params["advertised_scopes"])), + geography_vintage=params["geography_vintage"], + period=params["period"], + model_outputs=tuple(params["model_outputs"]), + ) + _require( + canonical_json(normative(result)) == canonical_json(raw), "MEASUREMENT_NODE" + ) + return result + + +def fiscal_dense_calibration_node( + *, + measurement_node: Node, + bounds_node: str, + epochs: int = 256, + learning_rate: float = 0.02, + node_id: str = "us.fiscal_dense_calibration", +) -> Node: + """Declare one weight-only, fixed-zero grouped solve on the measured parent.""" + _require(type(epochs) is int and 1 <= epochs <= 10000, "EPOCHS") + _require( + type(learning_rate) in (int, float) + and np.isfinite(learning_rate) + and 0 < learning_rate <= 1, + "LEARNING_RATE", + ) + raw = canonical_json(normative(measurement_node)) + _require(len(raw) <= 2 * measurement.MAX_DECLARATION_BYTES, "DECLARATION_SIZE") + verified = _measurement_node(json.loads(raw)) + return Node( + node_id, + FiscalDenseCalibrationKernel.ref, + base=verified.population, + inputs=verified.inputs, + structural=StructuralDelta.REWEIGHT, + weights=WeightTransition("household", "calibrated", mass="free"), + mass="free", + params={ + "measurement_node": raw.decode(), + "epochs": epochs, + "learning_rate": learning_rate, + "method": "adam", + "seed": 0, + "grouped_preserve_zeros": True, + "l0_lambda": 0.0, + "l1_lambda": 0.0, + "l2_lambda": 0.0, + "l2_anchor": "initial", + "weight_anchor": "explicit_original_sampling_references", + "cap_enforcement": "numeric_group_projection_and_final_row_refusal", + }, + artifact_inputs=( + ArtifactInput( + "measurement", verified.id, "measurement", measurement.MEASUREMENT_TYPE + ), + ArtifactInput("bounds", bounds_node, "numeric_bounds", BOUNDS_TYPE), + ), + artifact_outputs=( + ArtifactOutput("diagnostics", DIAGNOSTICS_TYPE), + ArtifactOutput("origin_diagnostics", ORIGIN_TYPE), + ), + ) + + +def _artifact(context, name, type_, output): + value = context.artifacts[name] + _require( + value.type == type_ + and value.key == opaque_artifact_key(value.producer_key, output), + "ARTIFACT_EDGE", + ) + return value.payload + + +class FiscalDenseCalibrationKernel(KernelBase): + """Use the existing solver and independently check the compiled measurement.""" + + ref = "us.fiscal_dense_calibration@1" + capabilities = Capabilities( + Determinism.DETERMINISTIC, + numeric=Numeric.PLATFORM_BITWISE, + seed_source=SeedSource.NONE, + structural=StructuralDelta.REWEIGHT, + consumes_se=False, + dependencies=("numpy", "pandas", "scipy", "torch"), + ) + + def implementation_hash(self): + return _sha( + canonical_json( + { + "adapter": source_hash( + sys.modules[__name__], + measurement, + group_bounds, + target_module, + measurement.TargetSpec, + measurement.CalibrationHierarchy, + calibration_kernels.diagnostics_module.assemble_target_loss_attribution, + dependencies=self.capabilities.dependencies, + ), + "solver": calibration_kernels.CALIBRATE_ADAM.implementation_hash(), + } + ) + ) + + def run(self, context): + params = context.params + _require( + tuple(a.name for a in context.node.artifact_inputs) + == ("measurement", "bounds"), + "ALIASES", + ) + measured_node = _measurement_node(json.loads(params["measurement_node"])) + expected = fiscal_dense_calibration_node( + measurement_node=measured_node, + bounds_node=context.node.artifact_inputs[1].producer, + epochs=params["epochs"], + learning_rate=params["learning_rate"], + node_id=context.node.id, + ) + _require( + normative(context.node) == normative(expected) + and dict(params) == dict(expected.params), + "DECLARATION", + ) + _require( + context.weights["household"].kind is WeightKind.IMPORTANCE, + "IMPORTANCE_REQUIRED", + ) + _require(not context.sources, "UNDECLARED_SOURCE") + raw_measurement = _artifact( + context, "measurement", measurement.MEASUREMENT_TYPE, "measurement" + ) + measured_context = replace( + context, node=measured_node, params=measured_node.params, artifacts={} + ) + measured = measurement.verify_fiscal_measurement( + raw_measurement, + context=measured_context, + expected_sha256=_sha(raw_measurement), + ) + raw_bounds = _artifact(context, "bounds", BOUNDS_TYPE, "numeric_bounds") + bounds = decode_fiscal_calibration_bounds(raw_bounds) + _require( + bounds.document["population"] == context.node.base, "BOUNDS_POPULATION" + ) + _require( + _same_array(bounds.household_ids, measured.household_ids), + "HOUSEHOLD_ALIGNMENT", + ) + _require( + _same_array(bounds.incoming.values, context.weights["household"].values), + "INCOMING_ANCHOR", + ) + table = pd.DataFrame({"household_id": measured.household_ids.copy()}) + work = calibration_kernels._frame_from_context( + replace( + context, + tables={"household": table}, + weights={"household": bounds.incoming}, + ), + "household", + ) + targets = [] + for i, spec in enumerate(measured.registry): + # The public callable API materializes only this one sparse row. + def row(_frame, position=i): + return measured.matrix[position : position + 1].toarray().ravel() + + targets.append( + replace(spec.to_target(), entity="household", measure=row, filter=None) + ) + result = calibrate( + work, + TargetSet(targets), + weight_entity="household", + method="adam", + seed=0, + epochs=params["epochs"], + learning_rate=params["learning_rate"], + mass="free", + max_weight_ratio=None, + target_records=None, + l0_lambda=0.0, + l1_lambda=0.0, + l2_lambda=0.0, + l2_anchor="initial", + grouped_upper_bounds=bounds.grouped, + grouped_preserve_zeros=True, + ) + _require( + not result.skipped + and result.problem.names + == tuple(s.to_target().row_name for s in measured.registry), + "TARGET_ALIGNMENT", + ) + _require( + _same_array(result.problem.target_vector, measured.target_values) + and result.problem.matrix.shape == measured.matrix.shape + and all( + _same_array( + getattr(result.problem.matrix, k), getattr(measured.matrix, k) + ) + for k in ("indptr", "indices", "data") + ), + "MATRIX_IDENTITY", + ) + _require( + _same_array(result.initial_weights, bounds.incoming.values), + "SOLVER_INITIAL", + ) + weights = result.frame.weights_for("household") + _require(weights.kind is WeightKind.CALIBRATED, "CALIBRATED_KIND") + check_fiscal_calibration_weights(bounds, weights.values) + accepted = weights.values.tobytes() + _require( + np.isfinite(result.loss_trajectory).all() + and np.isfinite(result.final_loss), + "NONFINITE_LOSS", + ) + initial_estimates = measured.matrix @ bounds.incoming.values + final_estimates = measured.matrix @ weights.values + _require( + np.isfinite(initial_estimates).all() + and np.isfinite(final_estimates).all() + and np.isfinite(final_estimates - measured.target_values).all(), + "NONFINITE_RESIDUAL", + ) + _require( + _same_array( + np.asarray([d.initial_estimate for d in result.diagnostics]), + initial_estimates, + ) + and _same_array( + np.asarray([d.final_estimate for d in result.diagnostics]), + final_estimates, + ), + "DIAGNOSTIC_ESTIMATES", + ) + build = { + "measurement_sha256": _sha(raw_measurement), + "numeric_bounds_sha256": _sha(raw_bounds), + "budget_sha256": bounds.document["budget_sha256"], + "original_design_sha256": _sha(bounds.original_design.values.tobytes()), + "incoming_sha256": _sha(bounds.incoming.values.tobytes()), + "accepted_weight_sha256": _sha(accepted), + "constraint_digest": bounds.grouped.digest, + "matrix_matches_measurement": True, + "numeric_measurement_entity": "household", + "original_measurement_entities": { + s.name: s.entity for s in measured.registry + }, + "weight_anchor": params["weight_anchor"], + "cap_enforcement": params["cap_enforcement"], + } + diagnostics = canonical_json( + diagnostics_payload( + result, + target_registry=measured.registry, + build=build, + ) + ) + origin = canonical_json( + { + "protocol": "microcosm.us.fiscal-calibration-origin-diagnostics.v1", + **build, + "household_ids": measurement._array(bounds.household_ids), + "accepted_weights": measurement._array(weights.values), + "group_totals": measurement._array( + bounds.grouped.totals(weights.values) + ), + "group_upper": measurement._array(bounds.grouped.absolute_bounds), + "group_diagnostics": bounds.grouped.diagnostics( + weights.values, + result.options["grouped_upper_bounds"][ + "last_corrected_group_count" + ], + ), + "fixed_zero_rows": int(np.count_nonzero(bounds.incoming.values == 0)), + "active_rows": int(np.count_nonzero(weights.values > 0)), + "binding_row_caps": int( + np.count_nonzero(weights.values == bounds.row_upper) + ), + "all_row_caps_satisfied": True, + "release_eligible": False, + "source_admission": "required_from_complete_parent_owner", + } + ) + _require( + len(diagnostics) <= MAX_BYTES and len(origin) <= MAX_BYTES, + "DIAGNOSTIC_SIZE", + ) + output = KernelResult( + weights=weights, + artifacts={"diagnostics": diagnostics, "origin_diagnostics": origin}, + receipt={ + **build, + "diagnostics_sha256": _sha(diagnostics), + "origin_diagnostics_sha256": _sha(origin), + "release_eligible": False, + "source_admission": "required_from_complete_parent_owner", + "scope": "one_numeric_dense_fiscal_solve_no_ancestry_admission", + }, + ) + check_fiscal_calibration_weights( + decode_fiscal_calibration_bounds(raw_bounds), weights.values + ) + _require(weights.values.tobytes() == accepted, "FINAL_WEIGHT_BYTES") + _require( + _same_array( + work.table("household").household_id.to_numpy(), bounds.household_ids + ) + and _same_array( + result.frame.table("household").household_id.to_numpy(), + bounds.household_ids, + ), + "FINAL_HOUSEHOLD_ALIGNMENT", + ) + measurement.verify_fiscal_measurement( + raw_measurement, + context=measured_context, + expected_sha256=_sha(raw_measurement), + ) + return output diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/graph_fiscal_measurement.py b/packages/microcosm-build/src/microcosm/build/us_runtime/graph_fiscal_measurement.py new file mode 100644 index 000000000..6b6ce6bde --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/graph_fiscal_measurement.py @@ -0,0 +1,684 @@ +"""Declared fiscal measurements on one supplied US population, without fitting. + +V1 supports shared-interpreter value variables, sums and same-entity predicates. +Every target also has an explicit national/state/CD household scope. Specialized +providers, counterfactuals, band interpretation and monetary-binding receipts +are refused. This is not full fiscal-registry coverage or a population issuer. +The host owns source/target activation and complete-enrichment ancestry. + +Computed roots use the existing real US adapter only after its static closure +is explicitly present, with no missing cells or default allowlist. Measurements +and model outputs live only in private tables and a bounded CSR artifact. +""" + +from __future__ import annotations + +import hashlib +import json +import sys +from dataclasses import asdict, dataclass + +import numpy as np +import pandas as pd +from scipy import sparse + +from microcosm.build import target_materialization +from microcosm.calibrate import TargetRegistry, TargetSpec, matrix +from microcosm.calibrate import target as target_math +from microcosm.calibrate.hierarchy import CalibrationHierarchy, HierarchyGeography +from microcosm.frame import US_SCHEMA, Frame +from microcosm.frame.adapters import policyengine_us +from microcosm.graph import ( + ArtifactOutput, + ArtifactType, + Capabilities, + Determinism, + KernelBase, + KernelContext, + KernelResult, + Node, + Numeric, + SeedSource, + Slice, + source_hash, +) +from microcosm.graph.canonical import canonical_json, normative +from microcosm.graph.executor import _context_digest + +from . import asec_engine_evaluation as runtime + +MEASUREMENT_TYPE = ArtifactType("microcosm.us.declared_fiscal_measurement", 1) +PROTOCOL = "microcosm.us.declared-fiscal-measurement.v1" +MAX_BYTES = 64 * 1024**2 +MAX_DECLARATION_BYTES = 1024**2 +_BINDING_KEYS = {"value_variable", "value_expression", "filters", "from_entity"} +_LEVELS = {"national", "state", "congressional_district"} + + +def _require(condition, reason): + if not condition: + raise ValueError("FISCAL_MEASUREMENT_" + reason) + + +def _sha(payload): + return hashlib.sha256(payload).hexdigest() + + +def _json(value): + return canonical_json(value).decode() + + +def _registry_document(registry): + return {"country": registry.country, "specs": [asdict(s) for s in registry]} + + +def _registry(raw): + _require(set(raw) == {"country", "specs"}, "REGISTRY_FIELDS") + specs = [] + for item in raw["specs"]: + hierarchy = item.get("hierarchy") + specs.append( + TargetSpec( + **{ + **item, + "hierarchy": None + if hierarchy is None + else CalibrationHierarchy.from_dict(hierarchy), + } + ) + ) + result = TargetRegistry(specs, country=raw["country"]) + _require(result.country == "us" and len(result) > 0, "US_REGISTRY_REQUIRED") + _require(_json(_registry_document(result)) == _json(raw), "REGISTRY_CANONICAL") + return result + + +@dataclass(frozen=True) +class FiscalGeographyScope: + """An explicit interpretation of a target hierarchy's geographic scope. + + The national scope uses no column/value; state and CD scopes use the named + integer household geography columns. The host admits the boundary vintage. + """ + + geography: HierarchyGeography + column: str | None = None + value: int | None = None + + def __post_init__(self): + level = self.geography.level + _require(level in _LEVELS, "GEOGRAPHY_LEVEL") + expected = { + "national": None, + "state": "state_fips", + "congressional_district": "congressional_district_geoid", + }[level] + _require(self.column == expected, "GEOGRAPHY_COLUMN") + _require( + self.value is None if expected is None else type(self.value) is int, + "GEOGRAPHY_VALUE", + ) + + +def _scopes(raw): + scopes = tuple( + FiscalGeographyScope( + HierarchyGeography(**r["geography"]), r["column"], r["value"] + ) + for r in raw + ) + _require({s.geography.level for s in scopes} == _LEVELS, "ADVERTISED_LEVELS") + keys = [(s.geography.level, s.geography.id) for s in scopes] + _require(len(set(keys)) == len(keys), "DUPLICATE_SCOPE") + _require( + len({(s.column, s.value) for s in scopes}) == len(scopes), + "DUPLICATE_SCOPE_PREDICATE", + ) + _require( + sum(s.geography.level == "national" for s in scopes) == 1, "NATIONAL_SCOPE" + ) + return scopes + + +def _model_contract(roots): + if not roots: + return {"evaluation": "none", "roots": []} + identity = runtime.engine_runtime_identity() + index = policyengine_us.PolicyEngineUSVariableMetadataIndex() + contracts = [] + for root in roots: + closure = index.variable_dependency_closure(root) + contracts.append( + { + "closure": asdict(closure), + "entity": index.variable_metadata(root).entity, + "leaves": { + leaf: index.variable_metadata(leaf).entity + for leaf in closure.input_leaves + }, + } + ) + return {"evaluation": "real_baseline", "runtime": identity, "roots": contracts} + + +def fiscal_measurement_node( + registry: TargetRegistry, + *, + population: str, + input_columns: dict[str, tuple[str, ...]], + contract_targets: dict, + advertised_scopes: tuple[FiscalGeographyScope, ...], + geography_vintage: str, + period: int, + model_outputs: tuple[str, ...] = (), + node_id: str = "us.fiscal_measurement", +) -> Node: + """Declare a bounded measurement stage; supplied targets are not activated here. + + Use the same model_outputs on FiscalMeasurementKernel. Full US structural + person membership is retained through its required input slice. Unread + group tables are reconstructed as sorted IDs only, using Frame's exact + membership contract; none of their values is inferred or used. + Every required input leaf is checked against this declaration before model + evaluation. Neither this node nor its artifact admits a complete parent. + """ + _require(type(period) is int and period > 0, "PERIOD") + _require(type(geography_vintage) is str and bool(geography_vintage), "VINTAGE") + _require( + type(model_outputs) is tuple + and all(type(r) is str and r for r in model_outputs) + and len(set(model_outputs)) == len(model_outputs), + "MODEL_OUTPUTS", + ) + registry = _registry(_registry_document(registry)) + scopes = _scopes([asdict(s) for s in advertised_scopes]) + entities = (US_SCHEMA.person_entity, *US_SCHEMA.group_entities) + _require(set(input_columns) <= set(entities), "INPUT_ENTITIES") + columns = {entity: tuple(input_columns.get(entity, ())) for entity in entities} + _require(bool(columns["person"]), "DECLARED_PERSON_PROJECTION") + _require( + all(len(set(names)) == len(names) for names in columns.values()), + "DUPLICATE_INPUT", + ) + _require( + {s.column for s in scopes if s.column} <= set(columns["household"]), + "DECLARED_GEOGRAPHY", + ) + keys = {(s.geography.level, s.geography.id) for s in scopes} + target_scopes = set() + used_bindings = set() + measurements = {} + for spec in registry: + _require( + spec.period == period and spec.hierarchy is not None, + "TARGET_PERIOD_HIERARCHY", + ) + key = (spec.hierarchy.geography.level, spec.hierarchy.geography.id) + _require(key in keys, "TARGET_SCOPE") + target_scopes.add(key) + _require( + "monetary_binding" not in spec.metadata, "UNSUPPORTED_MONETARY_BINDING" + ) + name = spec.metadata.get("contract_target_id") + _require(name in contract_targets, "MISSING_BINDING") + used_bindings.add(name) + target = contract_targets[name] + _require( + set(target) == {"bindings"} and set(target["bindings"]) == {"policyengine"}, + "BINDING_SHAPE", + ) + binding = target["bindings"]["policyengine"] + coordinate = (spec.entity, spec.measure) + _require( + coordinate not in measurements + or measurements[coordinate] == _json(binding), + "CONFLICTING_MEASUREMENT_BINDINGS", + ) + measurements[coordinate] = _json(binding) + _require(set(binding) <= _BINDING_KEYS, "UNSUPPORTED_BINDING") + _require( + ("value_variable" in binding) != ("value_expression" in binding), + "VALUE_BINDING", + ) + _require( + binding.get("from_entity", spec.entity) == spec.entity, "BINDING_ENTITY" + ) + _require(spec.entity in entities, "TARGET_ENTITY") + _require( + spec.measure != f"{spec.entity}_id" + and not ( + spec.entity == "person" + and spec.measure in {f"person_{e}_id" for e in US_SCHEMA.group_entities} + ), + "STRUCTURAL_MEASURE_COLLISION", + ) + _require(spec.measure not in columns[spec.entity], "INPUT_MEASURE_COLLISION") + _require(spec.measure not in model_outputs, "MODEL_MEASURE_COLLISION") + for predicate in binding.get("filters", ()): + _require( + set(predicate) + in ({"variable", "operator", "value"}, {"variable", "equals"}), + "PREDICATE_FIELDS", + ) + _require(used_bindings == set(contract_targets), "UNUSED_BINDINGS") + _require(target_scopes == keys, "UNCOVERED_ADVERTISED_SCOPE") + model = _model_contract(model_outputs) + for contract in model["roots"]: + root = contract["closure"]["root"] + _require(root not in columns[contract["entity"]], "FORMULA_INPUT_COLLISION") + _require( + not ( + set(contract["closure"]["formula_nodes"]) + & {name for names in columns.values() for name in names} + ), + "FORMULA_INPUT_COLLISION", + ) + for leaf, entity in contract["leaves"].items(): + _require(leaf in columns[entity], "UNDECLARED_MODEL_LEAF:" + leaf) + params = { + "registry": _json(_registry_document(registry)), + "contract_targets": _json(contract_targets), + "advertised_scopes": _json([asdict(s) for s in scopes]), + "geography_vintage": geography_vintage, + "period": period, + "model_outputs": model_outputs, + "model_contract": _json(model), + } + _require(len(_json(params).encode()) <= MAX_DECLARATION_BYTES, "DECLARATION_BOUND") + return Node( + node_id, + FiscalMeasurementKernel.ref, + population=population, + inputs=tuple(Slice(e, columns[e]) for e in entities if columns[e]), + params=params, + artifact_outputs=(ArtifactOutput("measurement", MEASUREMENT_TYPE),), + ) + + +class _Adapter: + def __init__(self, frame): + self.tables = {e: frame.table(e).copy(deep=True) for e in frame.entities} + + def column(self, entity, variable): + if variable == f"{entity}_count": + return np.ones(len(self.tables[entity]), dtype=np.float64) + return self.tables[entity][variable].to_numpy(copy=True) + + def has_column(self, entity, variable): + return variable in self.tables[entity] + + def require_known_predicate(self, entity, variable): + # A missing comparison input is unknown, even when NumPy would turn + # the comparison into False or treat NaN as a nonzero target mask. + _require( + not pd.isna(self.column(entity, variable)).any(), + "MISSING_PREDICATE_INPUT:" + entity + "." + variable, + ) + + def set_column(self, entity, variable, values): + _require(variable not in self.tables[entity], "MEASUREMENT_COLUMN_COLLISION") + array = np.asarray(values) + _require(array.shape == (len(self.tables[entity]),), "MEASUREMENT_ALIGNMENT") + self.tables[entity][variable] = array + + +def _array(array): + array = np.ascontiguousarray(array) + return {"dtype": array.dtype.str, "hex": array.tobytes().hex()} + + +def _read_array(raw, dtypes): + _require(type(raw) is dict and set(raw) == {"dtype", "hex"}, "ARRAY_FIELDS") + _require(raw["dtype"] in dtypes and type(raw["hex"]) is str, "ARRAY_DTYPE") + try: + values = np.frombuffer(bytes.fromhex(raw["hex"]), dtype=raw["dtype"]).copy() + except (ValueError, TypeError): + raise ValueError("FISCAL_MEASUREMENT_ARRAY_ENCODING") from None + _require(_array(values) == raw, "ARRAY_CANONICAL") + return values + + +@dataclass(frozen=True) +class FiscalMeasurement: + """Decoded numbers and declarations only; no source or release authority.""" + + document: dict + registry: TargetRegistry + household_ids: np.ndarray + matrix: sparse.csr_array + target_values: np.ndarray + + +def decode_fiscal_measurement(payload: bytes) -> FiscalMeasurement: + """Decode bounded, canonical numerical bytes without loading models/files.""" + _require(type(payload) is bytes and 0 < len(payload) <= MAX_BYTES, "PAYLOAD_BOUND") + raw = json.loads(payload) + _require(canonical_json(raw) == payload, "CANONICAL") + _require( + set(raw) + == { + "protocol", + "node", + "declaration", + "projection_sha256", + "household_ids", + "indptr", + "indices", + "data", + "target_values", + "support", + }, + "PAYLOAD_FIELDS", + ) + _require(raw["protocol"] == PROTOCOL, "PROTOCOL") + registry = _registry(json.loads(raw["declaration"]["registry"])) + ids = _read_array(raw["household_ids"], {" 0 and len(set(ids.tolist())) == len(ids), "HOUSEHOLD_IDS") + indptr = _read_array(raw["indptr"], {"= 0) + and indptr[-1] == len(indices) == len(data), + "CSR_SHAPE", + ) + _require( + np.all((indices >= 0) & (indices < len(ids))) and np.isfinite(data).all(), + "CSR_VALUES", + ) + problem = sparse.csr_array((data, indices, indptr), shape=(len(registry), len(ids))) + _require(problem.has_canonical_format and np.all(data != 0), "CSR_CANONICAL") + _require( + type(raw["support"]) is list and len(raw["support"]) == len(registry), + "SUPPORT_SHAPE", + ) + for spec, row in zip(registry, raw["support"], strict=True): + _require( + type(row) is dict + and set(row) + == { + "target", + "positive_weight_positive_rows", + "positive_weight_negative_rows", + }, + "SUPPORT_FIELDS", + ) + _require(row["target"] == spec.to_target().row_name, "SUPPORT_TARGET") + for key in ("positive_weight_positive_rows", "positive_weight_negative_rows"): + _require( + type(row[key]) is int and 0 <= row[key] <= len(ids), "SUPPORT_COUNT" + ) + return FiscalMeasurement(raw, registry, ids, problem, values) + + +def verify_fiscal_measurement(payload, *, context, expected_sha256): + """Bind a graph-selected artifact to its exact declared input projection. + + The host must obtain expected_sha256 from the matching actual graph record; + an arbitrary supplied hash cannot establish producer or parent authority. + """ + _require(_sha(payload) == expected_sha256, "ARTIFACT_DIGEST") + result = decode_fiscal_measurement(payload) + _require( + _json(result.document["node"]) == _json(normative(context.node)), + "NODE_BINDING", + ) + _require( + _json(result.document["declaration"]) == _json(dict(context.params)), + "DECLARATION_BINDING", + ) + _require( + result.document["projection_sha256"] == _context_digest(context).hex(), + "PROJECTION_BINDING", + ) + expected = context.tables["household"]["household_id"].to_numpy() + _require( + result.household_ids.dtype == expected.dtype + and np.array_equal(result.household_ids, expected), + "HOUSEHOLD_ALIGNMENT", + ) + masks = _geography_masks( + context.tables["household"], + _scopes(json.loads(context.params["advertised_scopes"])), + ) + weights = context.weights["household"].values + for i, spec in enumerate(result.registry): + row = result.matrix[i : i + 1] + mask = masks[spec.hierarchy.geography.level, spec.hierarchy.geography.id] + _require(mask[row.indices].all(), "GEOGRAPHIC_ROW_SUPPORT") + _require( + _support(spec, row, weights) == result.document["support"][i], + "SUPPORT_BINDING", + ) + return result + + +def _geography_masks(household, scopes): + masks = {} + for column in ("state_fips", "congressional_district_geoid"): + values = household[column] + _require( + values.dtype.kind in "iu" and not values.isna().any(), "GEOGRAPHY_DTYPE" + ) + _require( + np.array_equal( + household.congressional_district_geoid.to_numpy() // 100, + household.state_fips.to_numpy(), + ), + "STATE_CD_MAPPING", + ) + for scope in scopes: + mask = ( + np.ones(len(household), dtype=bool) + if scope.column is None + else household[scope.column].to_numpy() == scope.value + ) + _require(mask.any(), "EMPTY_ADVERTISED_GEOGRAPHY:" + scope.geography.id) + masks[scope.geography.level, scope.geography.id] = mask + for level in ("state", "congressional_district"): + covered = np.sum( + [mask for (scope_level, _), mask in masks.items() if scope_level == level], + axis=0, + ) + _require(np.all(covered == 1), "UNCOVERED_POPULATION_GEOGRAPHY") + return masks + + +def _support(spec, row, weights): + positive = int(np.count_nonzero((row.data > 0) & (weights[row.indices] > 0))) + negative = int(np.count_nonzero((row.data < 0) & (weights[row.indices] > 0))) + _require( + spec.value == 0 or (positive > 0 if spec.value > 0 else negative > 0), + "UNSUPPORTED_TARGET:" + spec.name, + ) + return { + "target": spec.to_target().row_name, + "positive_weight_positive_rows": positive, + "positive_weight_negative_rows": negative, + } + + +class FiscalMeasurementKernel(KernelBase): + """Materialize declared measures and compile the real household CSR system.""" + + ref = "us.declared_fiscal_measurement@1" + capabilities = Capabilities( + Determinism.DETERMINISTIC, + numeric=Numeric.PLATFORM_BITWISE, + seed_source=SeedSource.NONE, + dependencies=("numpy", "pandas", "scipy"), + ) + + def __init__(self, *, model_outputs=()): + self.model_outputs = model_outputs + + def implementation_hash(self): + return _sha( + canonical_json( + { + "implementation": source_hash( + sys.modules[__name__], + target_materialization, + matrix, + target_math, + TargetSpec, + Frame, + policyengine_us, + # The adapter delegates table/weight materialization + # to this helper's separate defining module. + policyengine_us.engine_tables, + runtime, + _context_digest, + dependencies=self.capabilities.dependencies, + ), + "model": _model_contract(self.model_outputs), + } + ) + ) + + def run(self, context: KernelContext) -> KernelResult: + params = context.params + _require(params["model_outputs"] == self.model_outputs, "KERNEL_MODEL_OUTPUTS") + registry = _registry(json.loads(params["registry"])) + scopes = _scopes(json.loads(params["advertised_scopes"])) + expected = fiscal_measurement_node( + registry, + population=context.node.population, + input_columns={s.entity: s.columns for s in context.node.inputs}, + contract_targets=json.loads(params["contract_targets"]), + advertised_scopes=scopes, + geography_vintage=params["geography_vintage"], + period=params["period"], + model_outputs=self.model_outputs, + node_id=context.node.id, + ) + _require(context.node.normative() == expected.normative(), "NODE_DECLARATION") + projection = _context_digest(context).hex() + tables = {e: t.copy(deep=True) for e, t in context.tables.items()} + for entity in US_SCHEMA.group_entities: + if entity not in tables: + tables[entity] = pd.DataFrame( + { + f"{entity}_id": np.unique( + tables["person"][f"person_{entity}_id"].to_numpy() + ), + } + ) + frame = Frame( + tables, + US_SCHEMA, + {"household": context.weights["household"]}, + strata=context.strata.copy(), + ) + adapter = _Adapter(frame) + model = json.loads(params["model_contract"]) + if self.model_outputs: + for contract in model["roots"]: + for leaf, entity in contract["leaves"].items(): + values = frame.table(entity)[leaf] + _require(not values.isna().any(), "MISSING_MODEL_INPUT:" + leaf) + if pd.api.types.is_numeric_dtype(values.dtype): + _require( + np.isfinite(values.to_numpy()).all(), + "NONFINITE_MODEL_INPUT", + ) + outputs = policyengine_us.PolicyEngineUSEngine().materialize( + frame, self.model_outputs, params["period"] + ) + _require(set(outputs) == set(self.model_outputs), "MODEL_RESULT_ROSTER") + for contract in model["roots"]: + output = np.asarray(outputs[contract["closure"]["root"]]) + _require( + output.dtype.kind in "biuf" and np.isfinite(output).all(), + "MODEL_RESULT_VALUES", + ) + adapter.set_column( + contract["entity"], + contract["closure"]["root"], + outputs[contract["closure"]["root"]], + ) + bindings = json.loads(params["contract_targets"]) + predicates = { + (spec.entity, predicate["variable"]) + for spec in registry + for predicate in bindings[spec.metadata["contract_target_id"]]["bindings"][ + "policyengine" + ].get("filters", ()) + } + for entity, variable in sorted(predicates): + adapter.require_known_predicate(entity, variable) + prepared = target_materialization.materialize_target_bindings( + adapter, + registry, + bindings, + period=params["period"], + ) + _require( + not prepared.skipped, "UNMATERIALIZED_TARGETS:" + str(prepared.skipped) + ) + # TargetSpec.filter is evaluated by the CSR compiler, independently + # of contract binding predicates, and may use a prepared measure. + for entity, variable in sorted( + {(s.entity, s.filter) for s in registry if s.filter is not None} + ): + adapter.require_known_predicate(entity, variable) + work = Frame( + adapter.tables, US_SCHEMA, {"household": context.weights["household"]} + ) + problem = matrix.build_constraint_matrix( + work, registry.to_target_set(), weight_entity="household" + ) + _require( + not problem.skipped + and problem.names == tuple(s.to_target().row_name for s in registry), + "TARGET_ALIGNMENT", + ) + household = frame.table("household") + masks = _geography_masks(household, scopes) + rows, support = [], [] + incoming = problem.initial_weights.values + for i, spec in enumerate(registry): + mask = masks[spec.hierarchy.geography.level, spec.hierarchy.geography.id] + row = problem.matrix[i : i + 1].multiply(mask).tocsr() + row.eliminate_zeros() + rows.append(row) + support.append(_support(spec, row, incoming)) + csr = sparse.vstack(rows, format="csr") + _require(csr.nnz * 48 + frame.n("household") * 16 < MAX_BYTES, "MATRIX_BOUND") + payload = canonical_json( + { + "protocol": PROTOCOL, + "node": normative(context.node), + "declaration": dict(params), + "projection_sha256": projection, + "household_ids": _array(household.household_id.to_numpy()), + "indptr": _array(csr.indptr.astype(" str: + return implementation_hash("geography") + + +class USGeographyLookupKernel(_USGeographyKernel): + ref = "us.production.geography_lookup@2" + capabilities = Capabilities( + determinism=Determinism.DETERMINISTIC, + numeric=Numeric.PLATFORM_BITWISE, + dependencies=STAGE_DEPENDENCIES["geography"], + ) + + def run(self, context: KernelContext) -> KernelResult: + if context.params != {"phase": GEOGRAPHY_PHASE}: + raise ValueError("Unsupported US geography lookup parameters.") + ladder_bytes = load_source_bytes( + "raw-bytes-v1", context.sources["us_puma_ladder_2020"] + ) + crosswalk_bytes = load_source_bytes( + "raw-bytes-v1", context.sources["us_cd_crosswalk_117_119"] + ) + ladder, expected = _lookups(ladder_bytes, crosswalk_bytes) + return KernelResult( + artifacts={"ladder": ladder_bytes, "crosswalk": crosswalk_bytes}, + receipt={ + "phase": GEOGRAPHY_PHASE, + "implementation": implementation_manifest("geography"), + "district_vintage": CURRENT_CONGRESSIONAL_DISTRICT_VINTAGE, + "district_count": len(expected), + "puma_count": len(ladder), + "ladder_sha256": hashlib.sha256(ladder_bytes).hexdigest(), + "crosswalk_sha256": hashlib.sha256(crosswalk_bytes).hexdigest(), + }, + ) + + +class USGeographyBoundaryKernel(_USGeographyKernel): + """Preserve all rows while opening the geography rewrite version.""" + + ref = "us.production.geography_boundary@2" + capabilities = Capabilities( + determinism=Determinism.DETERMINISTIC, + structural=StructuralDelta.FILTER, + dependencies=STAGE_DEPENDENCIES["geography"], + ) + + def run(self, context: KernelContext) -> KernelResult: + if context.params != {"phase": GEOGRAPHY_PHASE}: + raise ValueError("US geography boundary requires its registered phase.") + person = context.tables["person"] + return KernelResult( + receipt={"implementation": implementation_manifest("geography")}, + keep=pd.Series( + True, index=pd.Index(person["person_id"], name="person_id"), dtype=bool + ), + ) + + +class USGeographyAssignKernel(_USGeographyKernel): + ref = "us.production.geography_assign@2" + capabilities = Capabilities( + determinism=Determinism.SEEDED, + seed_source=SeedSource.PARAM, + numeric=Numeric.PLATFORM_BITWISE, + dependencies=STAGE_DEPENDENCIES["geography"], + ) + + def run(self, context: KernelContext) -> KernelResult: + if context.params != { + "phase": GEOGRAPHY_PHASE, + "seed": 0, + "assign_tract": False, + }: + raise ValueError( + "US geography requires the declared seed and joint county/CD contract." + ) + artifact = context.artifacts.get("frame_context") + if artifact is None or artifact.type != US_FRAME_CONTEXT_TYPE: + raise ValueError("US geography requires a typed frame context.") + document = _decode(artifact.payload) + _mass_records(document["mass_log"]) + household = context.tables["household"] + declared = document["entities"]["household"] + if any( + declared[key] != value + for key, value in _row_identity(household, "household").items() + ): + raise ValueError("US geography household identity does not match context.") + if set(household.columns) - set(declared["columns"]): + raise ValueError("US geography has undeclared household input columns.") + if ( + document["weight_sources"].get("household") + != context.weights["household"].kind.value + ): + raise ValueError( + "US geography requires the matching household weight source." + ) + ladder, expected = _lookups( + context.artifacts["ladder"].payload, context.artifacts["crosswalk"].payload + ) + table = assign_us_puma_ladder( + household, + ladder, + seed=0, + assign_tract=False, + expected_congressional_district_vintage=CURRENT_CONGRESSIONAL_DISTRICT_VINTAGE, + ) + columns = {} + for owned in context.node.outputs: + if ( + owned.entity != "household" + or owned.column not in US_PUMA_LADDER_COLUMNS + ): + raise ValueError("US geography has an unsupported owned output.") + original = table[owned.column] + converted = original.astype(dtype_for_token(owned.dtype)) + if not original.isna().equals(converted.isna()) or not np.array_equal( + original.to_numpy(), converted.to_numpy() + ): + raise ValueError( + "US geography storage conversion changed values or missingness." + ) + table[owned.column] = converted + columns[(owned.entity, owned.column)] = pd.Series( + converted.array, + index=pd.Index(table["household_id"], name="household_id"), + ) + if set(column for _, column in columns) != set(US_PUMA_LADDER_COLUMNS): + raise ValueError( + "US geography must own exactly its three household outputs." + ) + # This operator changes only these three cells on each household. The + # graph boundary preserves every row, weight and membership. Carry the + # authenticated descriptors for untouched entities without copying + # their population tables into this household-only operator. + for column in US_PUMA_LADDER_COLUMNS: + if column not in declared["columns"]: + declared["columns"].append(column) + return KernelResult( + columns=columns, + artifacts={"frame_context": canonical_json(document)}, + receipt={ + "phase": GEOGRAPHY_PHASE, + "implementation": implementation_manifest("geography"), + "seed": 0, + "assign_tract": False, + "district_vintage": CURRENT_CONGRESSIONAL_DISTRICT_VINTAGE, + "target_district_count": len(expected), + "summary": us_puma_ladder_assignment_summary( + table, + ladder, + weight_values=context.weights["household"].values, + ), + }, + ) + + +class USGeographyGateKernel(_USGeographyKernel): + ref = "us.production.geography_gate@2" + capabilities = Capabilities( + determinism=Determinism.DETERMINISTIC, + role=KernelRole.GATE, + numeric=Numeric.PLATFORM_BITWISE, + dependencies=STAGE_DEPENDENCIES["geography"], + ) + + def run(self, context: KernelContext) -> KernelResult: + if context.params != {"phase": GEOGRAPHY_PHASE}: + raise ValueError("US geography gate requires its registered phase.") + gate = us_puma_ladder_gate( + context.tables["household"], + context.weights["household"].values, + assign_tract=False, + ) + ladder, _ = _lookups( + context.artifacts["ladder"].payload, context.artifacts["crosswalk"].payload + ) + joint_gate = us_puma_ladder_joint_support_gate( + context.tables["household"], + ladder, + assign_tract=False, + expected_congressional_district_vintage=CURRENT_CONGRESSIONAL_DISTRICT_VINTAGE, + ) + return KernelResult( + receipt={ + "outcome": "pass" if gate.passed and joint_gate.passed else "fail", + "evidence": GateReport((gate, joint_gate)).to_manifest(), + "scope": "geography_assignment_integrity", + } + ) + + +def us_geography_nodes( + columns: Sequence[Owned], *, base: str, context_producer: str +) -> tuple[Node, ...]: + """Append a declared production geography stage after source assembly. + + This checks assignment integrity, not CD calibration or release readiness. + """ + inventory = {(owned.entity, owned.column): owned for owned in columns} + if len(inventory) != len(columns): + raise ValueError("US geography input column declarations repeat coordinates.") + if ("person", "age") not in inventory: + raise ValueError("US geography requires the assembled person age column.") + if ("household", "state_fips") not in inventory: + raise ValueError("US geography requires household state_fips.") + input_columns = ("state_fips",) + ( + ("puma",) if ("household", "puma") in inventory else () + ) + boundary = f"{GEOGRAPHY_PHASE}.boundary" + lookup = f"{GEOGRAPHY_PHASE}.lookup" + output_types = { + "puma": "string", + CONGRESSIONAL_DISTRICT_GEOID_COLUMN: "int64", + "county_fips": "string", + } + outputs = [] + for column, dtype in output_types.items(): + incumbent = inventory.get(("household", column)) + if incumbent: + if incumbent.dtype not in ( + {"int64", "Int64"} if dtype == "int64" else {"string"} + ): + raise ValueError(f"Unsupported incumbent geography dtype: {column}.") + dtype = incumbent.dtype + outputs.append(Owned("household", column, dtype, rewrite=incumbent is not None)) + return ( + Node( + id=lookup, + kernel=USGeographyLookupKernel.ref, + population=base, + sources=tuple(source.name for source in LOOKUP_SOURCES), + params={"phase": GEOGRAPHY_PHASE}, + artifact_outputs=( + ArtifactOutput("ladder", US_PUMA_LOOKUP_TYPE), + ArtifactOutput("crosswalk", US_CD_CROSSWALK_TYPE), + ), + ), + Node( + id=boundary, + kernel=USGeographyBoundaryKernel.ref, + base=base, + structural=StructuralDelta.FILTER, + inputs=(Slice("person", ("age",)),), + params={"phase": GEOGRAPHY_PHASE}, + ), + Node( + id=GEOGRAPHY_PHASE, + kernel=USGeographyAssignKernel.ref, + population=boundary, + inputs=(Slice("household", input_columns),), + outputs=tuple(outputs), + params={"phase": GEOGRAPHY_PHASE, "seed": 0, "assign_tract": False}, + artifact_inputs=( + ArtifactInput( + "frame_context", + context_producer, + "frame_context", + US_FRAME_CONTEXT_TYPE, + ), + ArtifactInput("ladder", lookup, "ladder", US_PUMA_LOOKUP_TYPE), + ArtifactInput("crosswalk", lookup, "crosswalk", US_CD_CROSSWALK_TYPE), + ), + artifact_outputs=(ArtifactOutput("frame_context", US_FRAME_CONTEXT_TYPE),), + ), + Node( + id=f"{GEOGRAPHY_PHASE}.gate", + kernel=USGeographyGateKernel.ref, + population=boundary, + inputs=(Slice("household", ("state_fips", *US_PUMA_LADDER_COLUMNS)),), + params={"phase": GEOGRAPHY_PHASE}, + artifact_inputs=( + ArtifactInput("ladder", lookup, "ladder", US_PUMA_LOOKUP_TYPE), + ArtifactInput("crosswalk", lookup, "crosswalk", US_CD_CROSSWALK_TYPE), + ), + ), + ) + + +def register_us_geography_kernels(registry: KernelRegistry) -> None: + for kernel in ( + USGeographyLookupKernel(), + USGeographyBoundaryKernel(), + USGeographyAssignKernel(), + USGeographyGateKernel(), + ): + registry.register(kernel) diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/graph_housing_universe.py b/packages/microcosm-build/src/microcosm/build/us_runtime/graph_housing_universe.py new file mode 100644 index 000000000..c8bbdf3c0 --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/graph_housing_universe.py @@ -0,0 +1,230 @@ +"""Non-authoritative HU transport bound to CREATE and selected graph ancestry. + +Parsing these bytes never issues AuthenticatedHousingUniverse or Frame authority. +The graph supplies the producer and preparation-receipt custody; this module +checks its native bytes and the exact int64 promotion of selected carried cells. +""" + +from __future__ import annotations + +import struct +from dataclasses import InitVar, dataclass + +import numpy as np +import pandas as pd + +from microcosm.graph import ArtifactType + +from . import asec_housing_universe as hu +from . import asec_housing_universe_source as source + +US_ASEC_HOUSING_UNIVERSE_TYPE = ArtifactType( + "microcosm.us.asec_bound_housing_universe", 1 +) +HOUSEHOLD_EVIDENCE_COLUMNS = ( + *("asec_" + name for name in ("H_SEQ",) + hu.CONTEXT_COLUMNS), + *source.ATTACHED_COLUMNS, +) +_TOKEN = object() +_HEADER_KEYS = frozenset( + { + "schema_version", + "artifact_kind", + "source_authentication", + "release_eligible", + "source_period_kind", + "zero_origin_policy", + "zero_origin_evidence", + "household_rows", + "columns", + "codebook", + "dictionaries", + "definition_sha256", + "source", + "implementation", + "limitations", + } +) + + +ACCEPTED_PREPARED_RECEIPT_SCHEMAS = ("microcosm.us.asec_prepared_receipt.v3",) +ACCEPTED_PREPARED_SOURCE_KINDS = ("us_asec_prepared_current_money_v3",) + + +@dataclass(frozen=True) +class BoundHousingUniverse: + """Immutable native buffers with graph bindings, no full-source authority.""" + + header: bytes + buffers: tuple[bytes, ...] + _token: InitVar[object] = None + + def __post_init__(self, _token): + hu._require(_token is _TOKEN, "BOUND_HU_CONSTRUCTOR") + + def array(self, name: str) -> np.ndarray: + hu._require(name in hu.COLUMNS, "BOUND_HU_COLUMN") + return np.frombuffer( + self.buffers[hu.COLUMNS.index(name)], + dtype=" BoundHousingUniverse: + """Validate transport against a graph-provided receipt, not a source decoder.""" + try: + hu._require( + type(payload) is bytes + and len(hu.MAGIC) + 4 + 32 < len(payload) <= hu.PAYLOAD_MAX_BYTES, + "BOUND_HU_SIZE", + ) + binding = prepared_receipt["housing_universe"] + hu._require( + prepared_receipt["schema"] in ACCEPTED_PREPARED_RECEIPT_SCHEMAS + and prepared_receipt["source_kind"] in ACCEPTED_PREPARED_SOURCE_KINDS + and prepared_receipt["release_eligible"] is False + and binding["parent_kind"] == "HousingStatusAttachedAsec" + and binding["source_period_kind"] == "interview_household_universe" + and binding["native_dtype"] == "uint8" + and binding["graph_dtype"] == "int64" + and binding["aliases"] == list(source.ATTACHED_COLUMNS), + "BOUND_HU_PREPARATION_SCHEMA", + ) + hu._require(hu._sha(payload) == binding["payload_sha256"], "BOUND_HU_PAYLOAD") + hu._require(payload.startswith(hu.MAGIC), "BOUND_HU_MAGIC") + hu._require(hu._sha(payload[:-32]) == payload[-32:].hex(), "BOUND_HU_CHECKSUM") + size = struct.unpack_from(" None: + """Compare graph int64 rows with native evidence; no cast or authority upgrade.""" + hu._require(type(body) is BoundHousingUniverse, "BOUND_HU_TYPE") + hu._require( + type(positions) is np.ndarray + and positions.dtype == np.dtype("int64") + and positions.shape == (len(household),) + and bool( + ((positions >= 0) & (positions < len(body.array("household_id")))).all() + ) + and (len(positions) < 2 or bool((np.diff(positions) > 0).all())), + "BOUND_HU_POSITIONS", + ) + hu._require(household.columns.is_unique, "BOUND_HU_TABLE_COLUMNS") + for name, column in ( + ("household_id", "household_id"), + *((n, "asec_" + n) for n in ("H_SEQ",) + hu.CONTEXT_COLUMNS), + *zip(hu.DERIVED_COLUMNS, source.ATTACHED_COLUMNS, strict=True), + ): + hu._require(column in household, "BOUND_HU_CARRIED_COLUMN") + values = household[column] + hu._require( + values.dtype == np.dtype("int64") + and np.array_equal(values.to_numpy(), body.array(name)[positions]), + "BOUND_HU_CARRIED_VALUES", + ) + if income_years is not None: + hu._require( + type(income_years) is np.ndarray + and income_years.dtype == np.dtype("int64") + and np.array_equal(income_years, body.array("income_year")[positions]), + "BOUND_HU_COHORTS", + ) + + +__all__ = [ + "BoundHousingUniverse", + "HOUSEHOLD_EVIDENCE_COLUMNS", + "US_ASEC_HOUSING_UNIVERSE_TYPE", + "bind_housing_universe", + "verify_graph_housing_rows", +] diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/graph_implementation.py b/packages/microcosm-build/src/microcosm/build/us_runtime/graph_implementation.py new file mode 100644 index 000000000..e715b47de --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/graph_implementation.py @@ -0,0 +1,532 @@ +"""Reviewed, path-independent implementation identities for the US source graph. + +Whole modules remain the unit of code identity. The packaged inventory is a +reviewed dependency fence, not a Python sandbox or inferred call-graph proof. +Its import/use/resource expressions prevent silently extending a reviewed route +to an unbound helper. Updating that inventory requires a scope review. +""" + +from __future__ import annotations + +import ast +import csv +import hashlib +import importlib.metadata as importlib_metadata +import importlib.util +import inspect +import json +import sys +from collections.abc import Mapping +from functools import lru_cache +from pathlib import Path +from types import MappingProxyType + +import pandas as pd + +STAGE_DEPENDENCIES = MappingProxyType( + { + "asec_codec_v4": ("numpy", "pandas", "h5py"), + "acs_codec_2024": ("numpy", "pandas", "microunit", "PyYAML", "pyarrow"), + "acs_housing_universe_2024": ( + "numpy", + "pandas", + "microunit", + "PyYAML", + "pyarrow", + ), + # Full authenticated source catalogues, one household draw, selected + # native construction and one allocation. This deliberately excludes + # legacy prepared-source income observations and engine resources. + "authenticated_survey_population_v1": ( + "numpy", + "pandas", + "microunit", + "tables", + "h5py", + "PyYAML", + "pyarrow", + ), + "assembly_prepare": ( + "numpy", + "pandas", + "microunit", + "tables", + "h5py", + "PyYAML", + "pyarrow", + ), + "assembly_harmonize": ("numpy", "pandas"), + "geography": ("numpy", "pandas"), + # The ASEC prepared current-money slice. PolicyEngine-US is deliberately + # absent: its version is a normative parameter of the evaluation node and + # a pinned field of every admitted closure contract, so engine drift + # refuses there. Putting it here instead would make a secrets-free lane + # without the engine extra unable to hash any stage of this graph. + "asec_prepared_v3": ( + "numpy", + "pandas", + "microunit", + "tables", + "h5py", + "PyYAML", + "pyarrow", + ), + # The composed prepared-ASEC/native-ACS population. It runs both source + # scopes in one CREATE, so its module inventory is their union and its + # dependency set is theirs (identical in both). PolicyEngine-US is + # absent for the same reason it is absent from the prepared scope: no + # engine node lives in this graph. + "composed_population_v1": ( + "numpy", + "pandas", + "microunit", + "tables", + "h5py", + "PyYAML", + "pyarrow", + ), + # Binding the carried prepared-ASEC evidence to the composed ASEC rows + # and deriving the reviewed corrected leaves and reported observations + # over them. It runs no source loader of its own, but its resolution and + # every verifier it reuses live inside the composing scope, so its module + # inventory is that scope's plus the two binding modules and its + # dependency set is identical. + "composed_asec_binding_v1": ( + "numpy", + "pandas", + "microunit", + "tables", + "h5py", + "PyYAML", + "pyarrow", + ), + } +) +_CODECS = MappingProxyType( + { + "us-asec-raw-stage-v4": ( + "asec_codec_v4", + "microcosm.build.us_runtime.graph_sources:load_graph_asec", + ), + "us-acs-native-2024-v1": ( + "acs_codec_2024", + "microcosm.build.us_runtime.graph_sources:load_graph_acs", + ), + "us-asec-prepared-current-money-v3": ( + "asec_prepared_v3", + "microcosm.build.us_runtime.asec_prepared_source:load_graph_asec_prepared", + ), + "raw-bytes-v1": ("geography", "microcosm.graph.codecs:load_raw_bytes"), + "us-acs-housing-universe-2024-v1": ( + "acs_housing_universe_2024", + "microcosm.build.us_runtime.acs_housing_universe_source:load_graph_acs_housing_universe", + ), + } +) +_RESOURCE_CALLS = frozenset( + { + "open", + "read_bytes", + "read_text", + "read_csv", + "read_parquet", + "read_hdf", + "read_excel", + "read_json", + "read_table", + "read_feather", + "File", + "loadtxt", + "genfromtxt", + "load", + "safe_load", + "files", + "joinpath", + "import_module", + "__import__", + "exec", + "eval", + } +) + + +# These local lazy entry points are dependencies even though no ImportFrom +# binds their names. Record references AND complete calls in their caller scope: +# passing the dispatcher around must not hide its use, and changing a selected +# symbol must require review. This adds tripwires; it grants no scope exemption. +_LOCAL_IMPORT_DISPATCHERS = { + "microcosm.build/us_runtime/graph_sources.py": frozenset( + { + "_stacked_alias", + "__getattr__", + "assemble_stacked_spine", + "prepare_stacked_spine", + "harmonize_stacked_spine_weights", + } + ), +} + + +def _canonical(value: object) -> bytes: + return json.dumps( + value, sort_keys=True, separators=(",", ":"), allow_nan=False + ).encode() + + +def _digest(payload: bytes) -> str: + return hashlib.sha256(payload).hexdigest() + + +def _package_roots() -> dict[str, Path]: + # Resolve packages, never import leaf modules (including optional adapters) + # merely to hash their files. Editable and wheel installs use the same names. + roots = {} + for package in ( + "microcosm.build", + "microcosm.frame", + "microcosm.graph", + "microunit", + ): + spec = importlib.util.find_spec(package) + if spec is None and package == "microunit": + continue # Only the ACS/CREATE scopes require this dependency. + locations = () if spec is None else spec.submodule_search_locations or () + if len(locations) != 1: + raise ValueError( + f"US implementation package inventory unavailable: {package}." + ) + roots[package] = Path(next(iter(locations))).resolve() + return roots + + +@lru_cache(maxsize=256) +def _dependency_details(payload: bytes, name: str, covered: tuple[str, ...]) -> dict: + """Static tripwire cached by exact bytes, never by path, mtime or version.""" + tree = ast.parse(payload) + imports, aliases, unbound_imports = set(), {}, set() + package, relative = name.split("/", 1) + module = package + "." + relative.removesuffix(".py").replace("/", ".") + parent = module.rpartition(".")[0] + for node in ast.walk(tree): + if isinstance(node, (ast.Import, ast.ImportFrom)): + for alias in node.names: + if alias.name == "*": + raise ValueError("Unclassified wildcard dependency import.") + target = ( + alias.name + if isinstance(node, ast.Import) + else ( + importlib.util.resolve_name( + "." * node.level + (node.module or ""), parent + ) + if node.level + else node.module + ) + ) + is_stdlib = target.split(".")[0] in sys.stdlib_module_names + if not is_stdlib: + imports.add(target) + is_bound = target in covered or any( + target == item[1:] or target.startswith(item[1:] + ".") + for item in covered + if item.startswith("*") + ) + if not is_stdlib and not is_bound: + aliases[alias.asname or alias.name.split(".")[0]] = target + unbound_imports.add(f"import:{target}:{alias.name}") + resources, uses = set(), set(unbound_imports) + dispatchers = _LOCAL_IMPORT_DISPATCHERS.get(name, ()) + + class Visitor(ast.NodeVisitor): + scope = "" + + def visit_FunctionDef(self, node): + previous = self.scope + self.scope = f"{previous}.{node.name}" + self.generic_visit(node) + self.scope = previous + + visit_AsyncFunctionDef = visit_FunctionDef # noqa: N815 - AST visitor API + visit_ClassDef = visit_FunctionDef # noqa: N815 - AST visitor API + + def visit_Name(self, node): + if isinstance(node.ctx, ast.Load) and node.id in dispatchers: + resources.add( + f"{self.scope}:{ast.dump(node, include_attributes=False)}" + ) + if isinstance(node.ctx, ast.Load) and node.id in aliases: + uses.add(f"{self.scope}:{aliases[node.id]}:{node.id}") + + def visit_Call(self, node): + name = ( + node.func.attr + if isinstance(node.func, ast.Attribute) + else getattr(node.func, "id", None) + ) + # Bind reflective selection too, including the dispatcher's + # getattr(import_module(...), name) target expression. + if ( + name in _RESOURCE_CALLS + or name in dispatchers + or (dispatchers and name == "getattr") + ): + resources.add( + f"{self.scope}:{ast.dump(node, include_attributes=False)}" + ) + self.generic_visit(node) + + Visitor().visit(tree) + return { + "imports": sorted(imports), + "unbound_uses": sorted(uses), + "resource_accesses": sorted(resources), + } + + +def _covered_imports(name: str, inventory: dict) -> tuple[str, ...]: + """Omit a symbol tripwire only if every scope using this file binds it.""" + scopes = [spec for spec in inventory["stages"].values() if name in spec["modules"]] + modules = set.intersection(*(set(spec["modules"]) for spec in scopes)) + dependencies = set.intersection(*(set(spec["dependencies"]) for spec in scopes)) + covered = set() + for module in modules: + package, relative = module.split("/", 1) + resolved = package + "." + relative.removesuffix(".py").replace("/", ".") + covered.add(resolved.removesuffix(".__init__")) + covered.update( + "*" + ("yaml" if dependency == "PyYAML" else dependency) + for dependency in dependencies + ) + return tuple(sorted(covered)) + + +def _dependency_contract(payload: bytes, name: str, covered: tuple[str, ...]) -> dict: + details = _dependency_details(payload, name, covered) + return { + "imports": details["imports"], + "unbound_uses_sha256": _digest(_canonical(details["unbound_uses"])), + "resource_accesses_sha256": _digest(_canonical(details["resource_accesses"])), + } + + +def _inventory(roots: Mapping[str, Path]) -> tuple[dict, bytes]: + payload = ( + roots["microcosm.build"] / "us_runtime/graph_implementation_inventory.json" + ).read_bytes() + value = json.loads(payload) + if value["schema"] != "microcosm.us.implementation-inventory.v1": + raise ValueError("Unsupported US implementation inventory schema.") + if set(value["stages"]) != set(STAGE_DEPENDENCIES): + raise ValueError("US implementation stage dependency inventory differs.") + for name, dependencies in STAGE_DEPENDENCIES.items(): + if value["stages"][name]["dependencies"] != list(dependencies): + raise ValueError(f"US implementation dependency inventory differs: {name}.") + imports = { + name for contract in value["contracts"].values() for name in contract["imports"] + } + if set(value["import_classifications"]) != imports: + raise ValueError("US implementation import classification inventory differs.") + return value, payload + + +def _validate_package_roster(package: str, root: Path, inventory: dict) -> None: + actual = {path.relative_to(root).as_posix() for path in root.rglob("*.py")} + expected = inventory["package_rosters"][package] + if len(expected) != len(set(expected)) or actual != set(expected): + raise ValueError(f"US implementation Python inventory differs: {package}.") + if package == "microunit": + # py.typed is packaging metadata. Everything else must be explicitly + # bound, including an added non-Python rule/data file. + actual_resources = { + path.relative_to(root).as_posix() + for path in root.rglob("*") + if path.is_file() + and path.suffix != ".py" + and "__pycache__" not in path.parts + and path.name != "py.typed" + } + if actual_resources != set(inventory["package_resources"][package]): + raise ValueError("US implementation resource inventory differs: microunit.") + + +def _projections() -> dict: + from . import operator_boundary + + def columns(value): + if isinstance(value, Mapping): + return {key: columns(item) for key, item in sorted(value.items())} + if not isinstance(value, (tuple, list, frozenset, set)) or not all( + isinstance(item, str) for item in value + ): + raise ValueError( + "US source registry projection requires string column sets." + ) + return sorted(value) + + return { + name: columns(getattr(operator_boundary, name)) + for name in ( + "PRE_ASSEMBLY_OPERATOR_OUTPUT_FAMILIES", + "FORMULA_OWNED_SOURCE_COLUMNS", + ) + } + + +def _loader_ref(loader: object, expected: str, roots: Mapping[str, Path]) -> None: + actual = ( + f"{getattr(loader, '__module__', '')}:{getattr(loader, '__qualname__', '')}" + ) + module, _ = expected.split(":") + package = next(name for name in roots if module.startswith(name + ".")) + path = roots[package] / ( + module.removeprefix(package + ".").replace(".", "/") + ".py" + ) + # A correctly named function imported from another checkout is not the + # implementation whose file bytes this manifest promises. + source = inspect.getsourcefile(loader) if inspect.isfunction(loader) else None + if actual != expected or source is None or Path(source).resolve() != path.resolve(): + raise ValueError( + f"US source codec loader differs from declared reference: {expected}." + ) + + +def validate_source_codecs(codecs) -> None: + """Bind US registry names to real loaders; generic graph codec APIs stay unchanged.""" + roots = _package_roots() + # Explicit declared selectors keep this registry inspection source-blind. + try: + loaders = ( + codecs.get("us-asec-raw-stage-v4"), + codecs.get("us-acs-native-2024-v1"), + codecs.get("us-asec-prepared-current-money-v3"), + codecs.get("raw-bytes-v1"), + codecs.get("us-acs-housing-universe-2024-v1"), + ) + except Exception as error: + raise ValueError("US source codec loader missing.") from error + for (_, (_, expected)), loader in zip(_CODECS.items(), loaders, strict=True): + _loader_ref(loader, expected, roots) + + +def implementation_manifest(stage: str) -> dict: + """Return small auditable identities, never source data or absolute paths.""" + if stage not in STAGE_DEPENDENCIES: + raise ValueError(f"Unclassified US implementation stage: {stage}.") + roots = _package_roots() + inventory, inventory_bytes = _inventory(roots) + spec = inventory["stages"][stage] + modules, resources = {}, {} + packages = {name.split("/", 1)[0] for name in spec["modules"]} + if not packages <= roots.keys(): + raise ValueError("US implementation dependency package inventory unavailable.") + for package in packages - {"microcosm.build"}: + _validate_package_roster(package, roots[package], inventory) + if len(spec["modules"]) != len(set(spec["modules"])): + raise ValueError("Duplicate US implementation module inventory.") + for name in spec["modules"]: + package, relative = name.split("/", 1) + path = roots[package] / relative + if not path.is_file(): + raise ValueError(f"US implementation module inventory missing: {name}.") + payload = path.read_bytes() + actual = _dependency_contract(payload, name, _covered_imports(name, inventory)) + if actual != inventory["contracts"][name]: + raise ValueError(f"Unclassified US dependency/resource contract: {name}.") + modules[name] = _digest(payload) + for name in spec["resources"]: + package, relative = name.split("/", 1) + resources[name] = _digest((roots[package] / relative).read_bytes()) + manifest = { + "schema": "microcosm.us.implementation-manifest.v1", + "stage": stage, + "inventory_sha256": _digest(inventory_bytes), + "modules": modules, + "resources": resources, + "dependencies": { + name: importlib_metadata.version(name) for name in STAGE_DEPENDENCIES[stage] + }, + "projections": _projections() if spec["source_boundary_projection"] else {}, + "runtime_options": ( + { + "pandas.mode.string_storage": pd.get_option("mode.string_storage"), + "pandas.future.infer_string": pd.get_option("future.infer_string"), + "acs_reader_resolved_string_storage": pd.StringDtype().storage, + } + if stage + in { + "acs_codec_2024", + "assembly_prepare", + "acs_housing_universe_2024", + "authenticated_survey_population_v1", + "composed_population_v1", + "composed_asec_binding_v1", + } + else {} + ), + "codecs": {}, + } + if stage in { + "asec_prepared_v3", + "acs_housing_universe_2024", + "authenticated_survey_population_v1", + "composed_population_v1", + "composed_asec_binding_v1", + }: + # These stages parse pinned official CSV members with the stdlib reader. + # csv's field size limit is process state, so it belongs in the identity + # of each stage that reads under it. Whole-module and inventory hashes + # separately record code changes, including additions of new stages. + manifest["runtime_options"]["csv.field_size_limit"] = csv.field_size_limit() + if stage in { + "assembly_prepare", + "composed_population_v1", + "composed_asec_binding_v1", + }: + bound = ( + ("asec_codec_v4", "acs_codec_2024") + if stage == "assembly_prepare" + # The composed CREATE reads the prepared directory, not the + # raw-stage checkpoint, so it binds that codec's identity instead. + # The binding stage reads no source, but every artifact it admits + # was produced under those two codecs, so it binds them too rather + # than claiming an identity narrower than its actual inputs. + else ("asec_prepared_v3", "acs_codec_2024") + ) + for name, (codec_stage, loader) in _CODECS.items(): + if codec_stage in bound: + manifest["codecs"][name] = { + "loader": loader, + "implementation_sha256": implementation_hash(codec_stage), + } + elif stage == "authenticated_survey_population_v1": + # The standalone factory registers one real closure over an operational + # capture directory. Its whole module belongs to this stage. The legacy + # five-codec registry and its strict validator retain their contract. + manifest["codecs"] = { + "us-survey-population-source-v1": { + "loader": ( + "microcosm.build.us_runtime.graph_survey_population:" + "survey_population_source_codecs..load" + ) + } + } + elif stage in ( + "asec_codec_v4", + "acs_codec_2024", + "geography", + "asec_prepared_v3", + "acs_housing_universe_2024", + ): + manifest["codecs"] = { + name: {"loader": loader} + for name, (codec_stage, loader) in _CODECS.items() + if codec_stage == stage + } + return manifest + + +def implementation_hash(stage: str) -> str: + return _digest( + b"microcosm.us.scoped-implementation.v1\0" + + _canonical(implementation_manifest(stage)) + ) diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/graph_implementation_inventory.json b/packages/microcosm-build/src/microcosm/build/us_runtime/graph_implementation_inventory.json new file mode 100644 index 000000000..f3832710b --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/graph_implementation_inventory.json @@ -0,0 +1,2466 @@ +{ + "contracts": { + "microcosm.build/cd_benchmark/canonical.py": { + "imports": [], + "resource_accesses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", + "unbound_uses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945" + }, + "microcosm.build/cd_benchmark/origin.py": { + "imports": [ + "microcosm.build.cd_benchmark.canonical" + ], + "resource_accesses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", + "unbound_uses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945" + }, + "microcosm.build/frame_checkpoint.py": { + "imports": [ + "h5py", + "microcosm.frame", + "numpy", + "pandas" + ], + "resource_accesses_sha256": "6e84799b53beb6b39981b770f65412a60e3808f6416bda934c0017ad36196e13", + "unbound_uses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945" + }, + "microcosm.build/frame_sampling.py": { + "imports": [ + "microcosm.frame", + "numpy", + "pandas" + ], + "resource_accesses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", + "unbound_uses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945" + }, + "microcosm.build/gates.py": { + "imports": [ + "microcosm.build.ledger_targets", + "microcosm.build.us_runtime.stacked_spine", + "microcosm.calibrate.registry", + "microcosm.calibrate.solve", + "microcosm.frame", + "numpy", + "pandas" + ], + "resource_accesses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", + "unbound_uses_sha256": "06b1202277975ae894fc8ece00dbac4b51129c78ab41d44d45aa86969f8e940e" + }, + "microcosm.build/outer_stage_runtime.py": { + "imports": [ + "microcosm.build.frame_checkpoint", + "microcosm.frame", + "numpy", + "pandas" + ], + "resource_accesses_sha256": "a0d0bd839844e63cf2581413187e20748db921b657051a72a022ea6185176fd0", + "unbound_uses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945" + }, + "microcosm.build/serialization_dtypes.py": { + "imports": [ + "microcosm.frame", + "numpy", + "pandas" + ], + "resource_accesses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", + "unbound_uses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945" + }, + "microcosm.build/survey_allocation.py": { + "imports": [ + "microcosm.frame", + "numpy" + ], + "resource_accesses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", + "unbound_uses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945" + }, + "microcosm.build/survey_domain_sample.py": { + "imports": [ + "microcosm.build.survey_allocation", + "microcosm.frame", + "numpy" + ], + "resource_accesses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", + "unbound_uses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945" + }, + "microcosm.build/table_identity.py": { + "imports": [ + "microcosm.build.serialization_dtypes", + "numpy", + "pandas" + ], + "resource_accesses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", + "unbound_uses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945" + }, + "microcosm.build/us_runtime/_asec_current_money_codec.py": { + "imports": [ + "microcosm.build.us_runtime.asec_current_money" + ], + "resource_accesses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", + "unbound_uses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945" + }, + "microcosm.build/us_runtime/acs_housing_universe.py": { + "imports": [ + "numpy", + "pandas" + ], + "resource_accesses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", + "unbound_uses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945" + }, + "microcosm.build/us_runtime/acs_housing_universe_source.py": { + "imports": [ + "microcosm.build.serialization_dtypes", + "microcosm.build.us_runtime.acs_housing_universe", + "microcosm.build.us_runtime.acs_inputs", + "microcosm.build.us_runtime.acs_pums", + "microcosm.build.us_runtime.acs_sources", + "microcosm.build.us_runtime.graph_implementation", + "microcosm.build.us_runtime.operator_boundary", + "microcosm.build.us_runtime.source_csv_builtin", + "microcosm.frame", + "microcosm.graph.canonical", + "microcosm.graph.population", + "microcosm.graph.store", + "numpy", + "pandas" + ], + "resource_accesses_sha256": "a8d2536f4bb75f36e40995185901ce8867ce1c01478efba980a75d0bd778b3f7", + "unbound_uses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945" + }, + "microcosm.build/us_runtime/acs_inputs.py": { + "imports": [ + "microcosm.frame", + "numpy", + "pandas" + ], + "resource_accesses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", + "unbound_uses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945" + }, + "microcosm.build/us_runtime/acs_native_coverage_binding.py": { + "imports": [ + "microcosm.build.us_runtime", + "microcosm.frame" + ], + "resource_accesses_sha256": "493c1a38451cc17c161232077fcea53211dc46f91695d6ec2485d755da52b291", + "unbound_uses_sha256": "937ac46755604aef55c18570df2520f92e51dd0486b93e2316a160cdaa67ad6a" + }, + "microcosm.build/us_runtime/acs_person_coverage_authentication.py": { + "imports": [ + "microcosm.build.us_runtime", + "microcosm.build.us_runtime.acs_pums", + "microcosm.build.us_runtime.source_csv_builtin", + "microcosm.frame", + "numpy", + "pandas" + ], + "resource_accesses_sha256": "27da902db7790fcf13f79f383532fb1eaffea49c118f54f76870c36cad264004", + "unbound_uses_sha256": "dc548c6590471125d90a86717034d6ed275b6e98115676512a623e4b9f7dc323" + }, + "microcosm.build/us_runtime/acs_person_coverage_columns.py": { + "imports": [ + "microcosm.build.us_runtime.acs_pums", + "microcosm.build.us_runtime.source_csv_builtin", + "pandas" + ], + "resource_accesses_sha256": "94d488b944fbeaa44d07120dd12f7f1b01e7e20faddf2e3daac77d905c2eca88", + "unbound_uses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945" + }, + "microcosm.build/us_runtime/acs_population_catalogue.py": { + "imports": [ + "microcosm.build.us_runtime" + ], + "resource_accesses_sha256": "8899ffc02d4808966ab99d6897940046a3d422acb99014c0f3cb44591978c775", + "unbound_uses_sha256": "c175b4f5a567b65b62146020a7b0ae39aa9d32d57bdf0839618b13d1003721a1" + }, + "microcosm.build/us_runtime/acs_pums.py": { + "imports": [ + "microcosm.build.serialization_dtypes", + "microcosm.frame", + "microcosm.frame.units", + "numpy", + "pandas" + ], + "resource_accesses_sha256": "3de7aae8590dfd67cfd50c129c5fca3b3ba9eb8b2444ed59f84f92ff94a735ee", + "unbound_uses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945" + }, + "microcosm.build/us_runtime/acs_sources.py": { + "imports": [ + "microcosm.build.us_runtime.acs_pums" + ], + "resource_accesses_sha256": "1f6355e38c0cc1c36391097afe9917115275b0e5585a9429af6007419dbee036", + "unbound_uses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945" + }, + "microcosm.build/us_runtime/alimony.py": { + "imports": [ + "microcosm.build.gates", + "microcosm.build.source_manifest", + "microcosm.build.us_runtime.support_provenance", + "microcosm.frame", + "numpy", + "pandas" + ], + "resource_accesses_sha256": "905273f67cd9edd6939cb6467fd9466c837f90603e4fff9365c09b42326ce7b9", + "unbound_uses_sha256": "c07a32776a6c50eb55aa704fa8671cf2e5a029c88ccd2c86e418ed0310d6fd43" + }, + "microcosm.build/us_runtime/asec_2024_native_population.py": { + "imports": [ + "microcosm.build.cd_benchmark", + "microcosm.build.us_runtime", + "microcosm.frame", + "numpy", + "pandas" + ], + "resource_accesses_sha256": "466e46803aecf67189f31031cdb4eb9287af37669e5fa0b36610788d62564e46", + "unbound_uses_sha256": "71463df4f645cd2a98f537a212187b4d1e52bffcc4d2317defe6968d7f287608" + }, + "microcosm.build/us_runtime/asec_checkpoint.py": { + "imports": [ + "microcosm.build.frame_checkpoint", + "microcosm.build.outer_stage_runtime", + "microcosm.build.serialization_dtypes", + "microcosm.build.us_runtime.education_assistance_source", + "microcosm.build.us_runtime.operator_boundary", + "microcosm.build.us_runtime.reported_coverage_source", + "microcosm.frame", + "numpy", + "pandas" + ], + "resource_accesses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", + "unbound_uses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945" + }, + "microcosm.build/us_runtime/asec_coverage_authentication.py": { + "imports": [ + "microcosm.build.us_runtime", + "microcosm.build.us_runtime.education_assistance_source", + "pandas" + ], + "resource_accesses_sha256": "f1826cc233390a0a2b3e97a9905d95d51df58edb6bcee521e8893be26d37969d", + "unbound_uses_sha256": "f03f18df090746827bf57705d6bcc5bc28fcaaf07c16832c8450ac3f28c1c492" + }, + "microcosm.build/us_runtime/asec_current_money.py": { + "imports": [ + "numpy", + "pandas" + ], + "resource_accesses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", + "unbound_uses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945" + }, + "microcosm.build/us_runtime/asec_current_money_graph_resources.py": { + "imports": [ + "microcosm.build.us_runtime.asec_current_money" + ], + "resource_accesses_sha256": "eaea1354548803c511b49d0f3c3d9d88bb50700553056a34e88a2c314de2282e", + "unbound_uses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945" + }, + "microcosm.build/us_runtime/asec_current_money_resources.py": { + "imports": [ + "microcosm.build.us_runtime.asec_current_money", + "numpy", + "pandas" + ], + "resource_accesses_sha256": "2a2ed530efb66ac76ba8757f6bd5bd7492e586fe2f28c6d2eddada8aed722f78", + "unbound_uses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945" + }, + "microcosm.build/us_runtime/asec_current_money_selection.py": { + "imports": [ + "microcosm.build.us_runtime._asec_current_money_codec", + "microcosm.build.us_runtime.asec_current_money", + "microcosm.graph", + "numpy" + ], + "resource_accesses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", + "unbound_uses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945" + }, + "microcosm.build/us_runtime/asec_current_money_source.py": { + "imports": [ + "microcosm.build", + "microcosm.build.outer_stage_runtime", + "microcosm.build.serialization_dtypes", + "microcosm.build.us_runtime", + "microcosm.build.us_runtime.asec_current_money", + "microcosm.build.us_runtime.asec_current_money_resources", + "microcosm.build.us_runtime.asec_person_income_source", + "microcosm.build.us_runtime.education_assistance_source", + "microcosm.frame", + "numpy", + "pandas" + ], + "resource_accesses_sha256": "10c92df9a22413a1f5bdee648fc1a79edefc13d469c1f915c14b2c3f90eb605d", + "unbound_uses_sha256": "edeb60d6acaab0ae263edf0537a6de3887a07bbd00382f545191e41cb1a1e278" + }, + "microcosm.build/us_runtime/asec_current_money_units.py": { + "imports": [ + "microcosm.build.serialization_dtypes", + "microcosm.build.us_runtime._asec_current_money_codec", + "microcosm.build.us_runtime.asec_current_money", + "microcosm.build.us_runtime.asec_current_money_source", + "microcosm.build.us_runtime.asec_student_controls", + "microcosm.frame", + "microunit", + "numpy", + "pandas" + ], + "resource_accesses_sha256": "5bebc4222c02f5c2f4e165df30b178cea156109dfbff6d6d88e5aed190e34e9c", + "unbound_uses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945" + }, + "microcosm.build/us_runtime/asec_engine_evaluation.py": { + "imports": [ + "microcosm.build.us_runtime.operator_boundary", + "microcosm.frame", + "microcosm.graph", + "microcosm.graph.canonical", + "numpy" + ], + "resource_accesses_sha256": "ef73c9051031f964f8066b33572925566d6e8e7757d633429d48f67951d462ec", + "unbound_uses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945" + }, + "microcosm.build/us_runtime/asec_household_coverage_fields.py": { + "imports": [ + "microcosm.build.cd_benchmark.origin", + "microcosm.build.us_runtime" + ], + "resource_accesses_sha256": "57a3c65a298ad7f131570a5873ebb67b964c566b1f6368636b53e4a4c16c39f3", + "unbound_uses_sha256": "7a523e77a972d9ee60ba7ad78d95c13660d89df0920c6e82ecb98f38292c7cc8" + }, + "microcosm.build/us_runtime/asec_household_observations.py": { + "imports": [ + "h5py", + "microcosm.build.outer_stage_runtime", + "microcosm.build.us_runtime", + "microcosm.build.us_runtime.operator_boundary", + "microcosm.frame", + "numpy", + "pandas" + ], + "resource_accesses_sha256": "c44acab8834834c2e938718ae7f8da07bc2a3ee1e147c4f5d1ca40c7654d1180", + "unbound_uses_sha256": "17ce33d5d0b830d29e75dbee4a5c87d6f5b08415ac1d421f23aa46bb73b1c8f8" + }, + "microcosm.build/us_runtime/asec_housing_status.py": { + "imports": [ + "microcosm.build.us_runtime.asec_housing_status_source", + "numpy", + "pandas" + ], + "resource_accesses_sha256": "11b2b6becb66ad05e2cc9d4728cb56c348701971f0dcf4cd491d56a8c813b23f", + "unbound_uses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945" + }, + "microcosm.build/us_runtime/asec_housing_status_source.py": { + "imports": [ + "microcosm.build.us_runtime", + "microcosm.build.us_runtime.asec_current_money", + "microcosm.build.us_runtime.asec_current_money_units", + "microcosm.build.us_runtime.asec_housing_status", + "microcosm.build.us_runtime.operator_boundary", + "microcosm.frame", + "numpy", + "pandas" + ], + "resource_accesses_sha256": "bebe07e6f115689ca38ae68c2d7f69df4ab09cfaad3326e622f9f7b8a9dc89a0", + "unbound_uses_sha256": "ec593873f78e53325d93fc7f8cd3422f3935c0bec2a60120738bd3e606e065fc" + }, + "microcosm.build/us_runtime/asec_housing_universe.py": { + "imports": [ + "microcosm.build.us_runtime.asec_housing_status", + "microcosm.build.us_runtime.asec_housing_universe_source", + "numpy", + "pandas" + ], + "resource_accesses_sha256": "ac86369b087151b3b538064a5d1dcfaa533bd2adf1948c1132fb6ace4a2365d0", + "unbound_uses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945" + }, + "microcosm.build/us_runtime/asec_housing_universe_source.py": { + "imports": [ + "microcosm.build.us_runtime", + "microcosm.build.us_runtime.asec_current_money", + "microcosm.build.us_runtime.asec_current_money_units", + "microcosm.build.us_runtime.asec_housing_status", + "microcosm.build.us_runtime.asec_housing_status_source", + "microcosm.build.us_runtime.asec_housing_universe", + "microcosm.build.us_runtime.operator_boundary", + "microcosm.frame", + "numpy", + "pandas" + ], + "resource_accesses_sha256": "0f622ceee1c600d15eb314ecf7b88121f6c707b9cf8703c2965af95240ec73ee", + "unbound_uses_sha256": "a8cb1277b205b614033078bca03e9c055d190134adc1172bcae5294c76d2737e" + }, + "microcosm.build/us_runtime/asec_income_observations.py": { + "imports": [ + "microcosm.build.us_runtime", + "microcosm.build.us_runtime._asec_current_money_codec", + "microcosm.build.us_runtime.asec_current_money", + "microcosm.build.us_runtime.asec_student_controls", + "microcosm.build.us_runtime.education_assistance_source", + "numpy", + "pandas" + ], + "resource_accesses_sha256": "92fb0be32eb3167e2c7fdcff8ba929ef0ad3b8a49d50eb5c19a5e7431d84fe23", + "unbound_uses_sha256": "21636adcde1832a321b968f7959d8c75331d08831f9ed093003a26eb450acecd" + }, + "microcosm.build/us_runtime/asec_original_household_weights.py": { + "imports": [ + "microcosm.build.cd_benchmark.origin", + "microcosm.build.us_runtime" + ], + "resource_accesses_sha256": "8c98f1d18560b79086a6ce71868f5800b3d6b8a214bb20446cafe3e3204e6180", + "unbound_uses_sha256": "d01a0598bee95a0bc72aa37ad70ec054cc52e17e07f426f9cb3d25ca464d1c9a" + }, + "microcosm.build/us_runtime/asec_person_coverage_source.py": { + "imports": [ + "microcosm.build.us_runtime", + "pandas" + ], + "resource_accesses_sha256": "d048b0a585f81a8902dc11ef2fa5da96c8874f77802885fdaa08c7e69764994b", + "unbound_uses_sha256": "5a52b6865fa982890fc246f9fcf2568fb77fa725623e9759dd3342e26fae2ae7" + }, + "microcosm.build/us_runtime/asec_person_income_source.py": { + "imports": [ + "microcosm.build", + "microcosm.build.us_runtime", + "microcosm.build.us_runtime.asec_current_money", + "microcosm.build.us_runtime.asec_current_money_resources", + "microcosm.build.us_runtime.education_assistance_source", + "microcosm.frame", + "numpy", + "pandas" + ], + "resource_accesses_sha256": "ef03627f20e0d35ccaacd16397281e6bbe5b1f765a55f58cc4702a3fe2dfe2a3", + "unbound_uses_sha256": "d3ed60989ff82d6e0c96de69dcd77106689c19ecdb7f96c57a4f6d15bd0d415d" + }, + "microcosm.build/us_runtime/asec_population_catalogue.py": { + "imports": [ + "microcosm.build.us_runtime" + ], + "resource_accesses_sha256": "2622beb3c86b2a9e1cea2db1c6b34a1ec2d2cf29824c3d3e276ea41baeb4f93b", + "unbound_uses_sha256": "138ac0967786b481d52704c91d5d5cefc4e39f92d7cab4154215bc77a981ef6e" + }, + "microcosm.build/us_runtime/asec_prepared_source.py": { + "imports": [ + "microcosm.build.serialization_dtypes", + "microcosm.build.us_runtime", + "microcosm.build.us_runtime._asec_current_money_codec", + "microcosm.build.us_runtime.asec_checkpoint", + "microcosm.build.us_runtime.asec_current_money", + "microcosm.build.us_runtime.asec_current_money_units", + "microcosm.build.us_runtime.asec_housing_status_source", + "microcosm.build.us_runtime.education_assistance_source", + "microcosm.build.us_runtime.operator_boundary", + "microcosm.frame", + "microcosm.graph.population", + "numpy", + "pandas" + ], + "resource_accesses_sha256": "d53265b60115118b47ecd636a9fb01a878791d7c9fb1deff86ea21e35947b9c6", + "unbound_uses_sha256": "57f904134e8a69655e1b36357fd288a9ab69966fe10f27dd4f7607c80fba93af" + }, + "microcosm.build/us_runtime/asec_student_controls.py": { + "imports": [ + "microcosm.build.us_runtime", + "microcosm.build.us_runtime._asec_current_money_codec", + "microcosm.build.us_runtime.asec_current_money", + "microcosm.build.us_runtime.education_assistance_source", + "microcosm.frame", + "numpy", + "pandas" + ], + "resource_accesses_sha256": "d20705bfb2110cfc573f493e988e0dbc47b5441e2bda83ef5425110adab2d4e8", + "unbound_uses_sha256": "983062f97faeff20a1761770968d967cc871520de8decbbd98c8018067475796" + }, + "microcosm.build/us_runtime/congressional_district_geography.py": { + "imports": [ + "microcosm.frame", + "numpy", + "pandas" + ], + "resource_accesses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", + "unbound_uses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945" + }, + "microcosm.build/us_runtime/congressional_district_vintage.py": { + "imports": [ + "microcosm.calibrate.geography_constants", + "numpy", + "pandas" + ], + "resource_accesses_sha256": "cba8c434f995a8a867f6b67951909b2038954efc13cf25c04ea958ffac6a8f61", + "unbound_uses_sha256": "d49a89e818222a896afaaed38a0f448e33b38a5656f2eb9591e1bd9195eed687" + }, + "microcosm.build/us_runtime/cps_carried.py": { + "imports": [ + "microcosm.build.us_runtime.alimony", + "microcosm.build.us_runtime.public_assistance_type_source", + "microcosm.build.us_runtime.reported_coverage_source", + "microcosm.frame", + "numpy", + "pandas" + ], + "resource_accesses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", + "unbound_uses_sha256": "d2c7c444dbfd99bf73f141320b4a1d4a6eb409a07dbc84d36984f1bc5c95e288" + }, + "microcosm.build/us_runtime/cps_carried_current.py": { + "imports": [ + "microcosm.build.us_runtime.alimony", + "microcosm.build.us_runtime.asec_current_money", + "microcosm.build.us_runtime.asec_current_money_selection", + "microcosm.build.us_runtime.cps_carried", + "numpy", + "pandas" + ], + "resource_accesses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", + "unbound_uses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945" + }, + "microcosm.build/us_runtime/education_assistance_source.py": { + "imports": [ + "numpy", + "pandas" + ], + "resource_accesses_sha256": "27781733e25a9a4e98fb2ee8fe937391e7a301cf22f3ed1b2faa9bf8524831a0", + "unbound_uses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945" + }, + "microcosm.build/us_runtime/geography_ladder.py": { + "imports": [ + "microcosm.build.gates", + "microcosm.build.us_runtime.congressional_district_geography", + "microcosm.frame", + "numpy", + "pandas" + ], + "resource_accesses_sha256": "539fedb954503afc210fa2e9ad841638d8fc2fc1ab6e5070d21827cb896246fc", + "unbound_uses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945" + }, + "microcosm.build/us_runtime/graph_acs_housing_universe.py": { + "imports": [ + "microcosm.build.us_runtime", + "microcosm.build.us_runtime.graph_context", + "microcosm.build.us_runtime.graph_implementation", + "microcosm.build.us_runtime.graph_sources", + "microcosm.frame", + "microcosm.graph", + "microcosm.graph.canonical", + "microcosm.graph.keys", + "numpy", + "pandas" + ], + "resource_accesses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", + "unbound_uses_sha256": "fb7b9295aae7f0e4db8fc053fa9fdb192fdb5176f4db5d0aa80bb0c2e61d8a86" + }, + "microcosm.build/us_runtime/graph_asec_income.py": { + "imports": [ + "microcosm.build.us_runtime", + "microcosm.build.us_runtime.asec_current_money", + "microcosm.build.us_runtime.asec_current_money_selection", + "microcosm.build.us_runtime.graph_context", + "microcosm.build.us_runtime.graph_implementation", + "microcosm.graph", + "microcosm.graph.canonical", + "numpy", + "pandas" + ], + "resource_accesses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", + "unbound_uses_sha256": "79c5158f94d6ba24554bdfe39ec48a02debebd761b4b67ae40caa22a7c471dd2" + }, + "microcosm.build/us_runtime/graph_asec_prepared.py": { + "imports": [ + "microcosm.build.frame_sampling", + "microcosm.build.us_runtime", + "microcosm.build.us_runtime.asec_current_money", + "microcosm.build.us_runtime.asec_current_money_graph_resources", + "microcosm.build.us_runtime.asec_current_money_selection", + "microcosm.build.us_runtime.asec_engine_evaluation", + "microcosm.build.us_runtime.asec_prepared_source", + "microcosm.build.us_runtime.cps_carried_current", + "microcosm.build.us_runtime.graph_asec_income", + "microcosm.build.us_runtime.graph_context", + "microcosm.build.us_runtime.graph_housing_universe", + "microcosm.build.us_runtime.graph_implementation", + "microcosm.build.us_runtime.graph_sources", + "microcosm.frame", + "microcosm.frame.adapters.policyengine_us", + "microcosm.graph", + "microcosm.graph.canonical", + "numpy", + "pandas" + ], + "resource_accesses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", + "unbound_uses_sha256": "1c0575822ea8e45ece5c94da38b4593df28eb7d96df04cece468174f1a38ff2d" + }, + "microcosm.build/us_runtime/graph_composed_asec_binding.py": { + "imports": [ + "microcosm.build.us_runtime.asec_current_money", + "microcosm.build.us_runtime.asec_current_money_selection", + "microcosm.build.us_runtime.asec_prepared_source", + "microcosm.build.us_runtime.graph_asec_income", + "microcosm.build.us_runtime.graph_asec_prepared", + "microcosm.build.us_runtime.graph_composed_contracts", + "microcosm.build.us_runtime.graph_composed_population", + "microcosm.build.us_runtime.graph_context", + "microcosm.build.us_runtime.graph_housing_universe", + "microcosm.build.us_runtime.graph_implementation", + "microcosm.build.us_runtime.stacked_spine", + "microcosm.build.us_runtime.support_provenance", + "microcosm.frame", + "microcosm.graph", + "microcosm.graph.canonical", + "numpy", + "pandas" + ], + "resource_accesses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", + "unbound_uses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945" + }, + "microcosm.build/us_runtime/graph_composed_asec_measures.py": { + "imports": [ + "microcosm.build.us_runtime.asec_current_money", + "microcosm.build.us_runtime.asec_current_money_selection", + "microcosm.build.us_runtime.cps_carried_current", + "microcosm.build.us_runtime.graph_asec_income", + "microcosm.build.us_runtime.graph_asec_prepared", + "microcosm.build.us_runtime.graph_composed_asec_binding", + "microcosm.build.us_runtime.graph_composed_contracts", + "microcosm.build.us_runtime.graph_composed_population", + "microcosm.build.us_runtime.graph_context", + "microcosm.build.us_runtime.graph_geography", + "microcosm.build.us_runtime.graph_implementation", + "microcosm.frame", + "microcosm.graph", + "microcosm.graph.canonical", + "numpy", + "pandas" + ], + "resource_accesses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", + "unbound_uses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945" + }, + "microcosm.build/us_runtime/graph_composed_contracts.py": { + "imports": [ + "microcosm.graph" + ], + "resource_accesses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", + "unbound_uses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945" + }, + "microcosm.build/us_runtime/graph_composed_population.py": { + "imports": [ + "microcosm.build.serialization_dtypes", + "microcosm.build.us_runtime.asec_current_money", + "microcosm.build.us_runtime.asec_current_money_selection", + "microcosm.build.us_runtime.asec_prepared_source", + "microcosm.build.us_runtime.graph_asec_income", + "microcosm.build.us_runtime.graph_asec_prepared", + "microcosm.build.us_runtime.graph_composed_contracts", + "microcosm.build.us_runtime.graph_context", + "microcosm.build.us_runtime.graph_geography", + "microcosm.build.us_runtime.graph_housing_universe", + "microcosm.build.us_runtime.graph_implementation", + "microcosm.build.us_runtime.graph_sources", + "microcosm.build.us_runtime.stacked_spine", + "microcosm.build.us_runtime.support_provenance", + "microcosm.frame", + "microcosm.graph", + "microcosm.graph.canonical", + "numpy", + "pandas" + ], + "resource_accesses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", + "unbound_uses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945" + }, + "microcosm.build/us_runtime/graph_context.py": { + "imports": [ + "microcosm.frame", + "microcosm.graph", + "microcosm.graph.canonical", + "numpy", + "pandas" + ], + "resource_accesses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", + "unbound_uses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945" + }, + "microcosm.build/us_runtime/graph_geography.py": { + "imports": [ + "microcosm.build.gates", + "microcosm.build.us_runtime.congressional_district_geography", + "microcosm.build.us_runtime.congressional_district_vintage", + "microcosm.build.us_runtime.graph_context", + "microcosm.build.us_runtime.graph_implementation", + "microcosm.build.us_runtime.puma_ladder", + "microcosm.graph", + "microcosm.graph.canonical", + "microcosm.graph.codecs", + "microcosm.graph.kernel", + "microcosm.graph.population", + "numpy", + "pandas" + ], + "resource_accesses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", + "unbound_uses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945" + }, + "microcosm.build/us_runtime/graph_housing_universe.py": { + "imports": [ + "microcosm.build.us_runtime", + "microcosm.graph", + "numpy", + "pandas" + ], + "resource_accesses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", + "unbound_uses_sha256": "457bc0cc87c6883ba9423769b70238b0e4ea060bca16c94d03379de01ac882c6" + }, + "microcosm.build/us_runtime/graph_implementation.py": { + "imports": [ + "microcosm.build.us_runtime", + "pandas" + ], + "resource_accesses_sha256": "454f9e3c1f7f1b840429f7c14c16dac67ab8219df0c9c32cf5807352b5c338ce", + "unbound_uses_sha256": "b45c8251bea93efe1847308338b21c6f8e74852564ad1a76ba5c384bd6de99a1" + }, + "microcosm.build/us_runtime/graph_sources.py": { + "imports": [ + "microcosm.build.serialization_dtypes", + "microcosm.build.us_runtime.acs_housing_universe_source", + "microcosm.build.us_runtime.acs_inputs", + "microcosm.build.us_runtime.acs_pums", + "microcosm.build.us_runtime.asec_checkpoint", + "microcosm.build.us_runtime.asec_prepared_source", + "microcosm.build.us_runtime.graph_context", + "microcosm.build.us_runtime.graph_implementation", + "microcosm.build.us_runtime.operator_boundary", + "microcosm.build.us_runtime.support_provenance", + "microcosm.frame", + "microcosm.graph", + "microcosm.graph.canonical", + "microcosm.graph.codecs", + "microcosm.graph.population", + "numpy", + "pandas" + ], + "resource_accesses_sha256": "627540b59d564b58991a32dc38aed03b9e2c852a3e9326792f21ae5beef19ebb", + "unbound_uses_sha256": "e9036a4821c20237bd98219383026cf440cdb29ac1b78f7b1bafd9be99b1dd30" + }, + "microcosm.build/us_runtime/graph_survey_population.py": { + "imports": [ + "microcosm.build.us_runtime", + "microcosm.build.us_runtime.graph_combined_clone", + "microcosm.build.us_runtime.graph_context", + "microcosm.build.us_runtime.graph_implementation", + "microcosm.build.us_runtime.graph_sources", + "microcosm.build.us_runtime.support_provenance", + "microcosm.build.us_runtime.survey_catalogue_selection", + "microcosm.build.us_runtime.survey_population_domains", + "microcosm.frame", + "microcosm.graph", + "microcosm.graph.artifact_edges", + "microcosm.graph.codecs", + "microcosm.graph.executor", + "microcosm.graph.keys", + "microcosm.graph.manifest", + "microcosm.graph.population", + "numpy", + "pandas" + ], + "resource_accesses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", + "unbound_uses_sha256": "5a6799ab18a561e0d4fdc4e08b11656a6100216df59e268e27e0ad4d38b1c994" + }, + "microcosm.build/us_runtime/native_household_origin.py": { + "imports": [ + "microcosm.build.cd_benchmark.origin", + "microcosm.build.us_runtime", + "microcosm.build.us_runtime.asec_student_controls", + "microcosm.build.us_runtime.education_assistance_source", + "microcosm.build.us_runtime.support_provenance", + "microcosm.frame", + "microcosm.graph.canonical", + "numpy" + ], + "resource_accesses_sha256": "740699f0e8648d061a7057844a4aabf04bb5ce2df7118fa4256dc2d215820c93", + "unbound_uses_sha256": "d4cc434fc5fa5a17c6fa46592dbeb8019d061927ff218c618036288221e76503" + }, + "microcosm.build/us_runtime/operator_boundary.py": { + "imports": [ + "microcosm.build.us_runtime.adult_care", + "microcosm.build.us_runtime.child_support", + "microcosm.build.us_runtime.childcare", + "microcosm.build.us_runtime.congressional_district_geography", + "microcosm.build.us_runtime.cps_carried", + "microcosm.build.us_runtime.disability_benefits", + "microcosm.build.us_runtime.education_inputs", + "microcosm.build.us_runtime.eligibility_inputs", + "microcosm.build.us_runtime.energy_subsidy", + "microcosm.build.us_runtime.geography_ladder", + "microcosm.build.us_runtime.hours_worked", + "microcosm.build.us_runtime.housing_inputs", + "microcosm.build.us_runtime.immigration", + "microcosm.build.us_runtime.medicare_take_up", + "microcosm.build.us_runtime.operator_column_contracts", + "microcosm.build.us_runtime.pregnancy", + "microcosm.build.us_runtime.prior_year_income", + "microcosm.build.us_runtime.qbi_inputs", + "microcosm.build.us_runtime.relationship_inputs", + "microcosm.build.us_runtime.retirement_contributions", + "microcosm.build.us_runtime.retirement_distributions", + "microcosm.build.us_runtime.scf_wealth", + "microcosm.build.us_runtime.weeks_unemployed", + "microcosm.build.us_runtime.wic_claim", + "microcosm.build.us_runtime.workers_compensation", + "microcosm.frame", + "pandas" + ], + "resource_accesses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", + "unbound_uses_sha256": "f3e52e09d1c5b005d16f083e716966bf60087b72eba71a149a8128faa7551617" + }, + "microcosm.build/us_runtime/operator_column_contracts.py": { + "imports": [], + "resource_accesses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", + "unbound_uses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945" + }, + "microcosm.build/us_runtime/puf_support.py": { + "imports": [ + "microcosm.build.gates", + "microcosm.build.us_runtime.acs_income_universe", + "microcosm.build.us_runtime.operator_column_contracts", + "microcosm.build.us_runtime.puf_e01000_reconciliation", + "microcosm.build.us_runtime.puf_interest_components", + "microcosm.build.us_runtime.qbi_inputs", + "microcosm.build.us_runtime.support_provenance", + "microcosm.frame", + "microcosm.frame.adapters.policyengine_us", + "microcosm.frame.schema", + "numpy", + "pandas" + ], + "resource_accesses_sha256": "851b8041c69b1b721dcdc02d1ce312976a87e45b66fc4aca59d0ec538ab62b44", + "unbound_uses_sha256": "3b7e8d1e071256065b1c8a43c7fea2f9f39378b10d1613f76bfa9cbbf0cff6ad" + }, + "microcosm.build/us_runtime/puma_ladder.py": { + "imports": [ + "microcosm.build.gates", + "microcosm.build.us_runtime.congressional_district_geography", + "microcosm.build.us_runtime.geography_ladder", + "microcosm.build.us_runtime.puma_ladder_sources", + "microcosm.frame", + "numpy", + "pandas" + ], + "resource_accesses_sha256": "056fbb3baf3f3008aa654612e700957b3ecb795b8da9460be5e9a884930c36a7", + "unbound_uses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945" + }, + "microcosm.build/us_runtime/puma_ladder_sources.py": { + "imports": [ + "numpy" + ], + "resource_accesses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", + "unbound_uses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945" + }, + "microcosm.build/us_runtime/reported_coverage_source.py": { + "imports": [ + "microcosm.build.us_runtime.education_assistance_source", + "numpy", + "pandas" + ], + "resource_accesses_sha256": "4b87bba3a98f67416ff34353df950e2638c86d56170d03d372f62578149f3e2e", + "unbound_uses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945" + }, + "microcosm.build/us_runtime/source_csv_builtin.py": { + "imports": [], + "resource_accesses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", + "unbound_uses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945" + }, + "microcosm.build/us_runtime/spine_assembly.py": { + "imports": [ + "microcosm.build.us_runtime.operator_column_contracts", + "microcosm.build.us_runtime.support_provenance", + "microcosm.frame", + "numpy", + "pandas" + ], + "resource_accesses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", + "unbound_uses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945" + }, + "microcosm.build/us_runtime/stacked_spine.py": { + "imports": [ + "microcosm.build.frame_sampling", + "microcosm.build.gates", + "microcosm.build.serialization_dtypes", + "microcosm.build.source_manifest", + "microcosm.build.table_identity", + "microcosm.build.us_runtime", + "microcosm.build.us_runtime.acs_income_universe", + "microcosm.build.us_runtime.acs_transfer", + "microcosm.build.us_runtime.late_producer_dag", + "microcosm.build.us_runtime.multispine_pool", + "microcosm.build.us_runtime.operator_boundary", + "microcosm.build.us_runtime.post_transfer_calibration", + "microcosm.build.us_runtime.puf_aggregate_records", + "microcosm.build.us_runtime.puf_capital_gains_tail", + "microcosm.build.us_runtime.puf_interest_components", + "microcosm.build.us_runtime.puf_qrf_chain", + "microcosm.build.us_runtime.puf_support", + "microcosm.build.us_runtime.spine_assembly", + "microcosm.build.us_runtime.support_provenance", + "microcosm.build.us_runtime.us_late_overlap_ownership", + "microcosm.build.us_runtime.us_late_producer_registry", + "microcosm.fit", + "microcosm.frame", + "numpy", + "pandas" + ], + "resource_accesses_sha256": "3839407286d44ddbf3f207c63dcb92e1c0b45e2eec2e88a09c25ebb3a4711430", + "unbound_uses_sha256": "062e621661c109d963f2356bcaa67a3da0906c8e66af1dcd81122d7ce05bd454" + }, + "microcosm.build/us_runtime/support_provenance.py": { + "imports": [ + "microcosm.build.gates", + "microcosm.build.us_runtime.cps_carried", + "numpy", + "pandas" + ], + "resource_accesses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", + "unbound_uses_sha256": "bfcbcc3d41a567ee0d5b015c6715c3fb659937713db37d3b2d74e5fa2ddeead3" + }, + "microcosm.build/us_runtime/survey_catalogue_selection.py": { + "imports": [ + "microcosm.build.survey_domain_sample", + "microcosm.build.us_runtime" + ], + "resource_accesses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", + "unbound_uses_sha256": "00c2ba97f0fb73d788c8f9c6e42ab90a67c5a2f7fe947d2232441aa6b03ae27c" + }, + "microcosm.build/us_runtime/survey_observed_age.py": { + "imports": [ + "numpy", + "pandas" + ], + "resource_accesses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", + "unbound_uses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945" + }, + "microcosm.build/us_runtime/survey_population_domains.py": { + "imports": [], + "resource_accesses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", + "unbound_uses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945" + }, + "microcosm.build/us_runtime/survey_population_preparation.py": { + "imports": [ + "microcosm.build", + "microcosm.build.us_runtime", + "microcosm.build.us_runtime.support_provenance", + "microcosm.frame", + "microcosm.graph", + "microcosm.graph.population", + "numpy", + "pandas" + ], + "resource_accesses_sha256": "ccfed1c1acff5a1c50b538424dc129930f69c374d9841e233b3fb593a440c3aa", + "unbound_uses_sha256": "29c09f6fcb25ce8bc6111dd0dc76f3501d9a6b635929e94b80835889d296ef91" + }, + "microcosm.frame/__init__.py": { + "imports": [ + "microcosm.frame.accounting", + "microcosm.frame.bundle", + "microcosm.frame.materialize", + "microcosm.frame.rules", + "microcosm.frame.schema", + "microcosm.frame.units", + "microcosm.frame.weights" + ], + "resource_accesses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", + "unbound_uses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945" + }, + "microcosm.frame/accounting.py": { + "imports": [ + "microcosm.frame.bundle", + "numpy", + "pandas", + "pandas.api.types" + ], + "resource_accesses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", + "unbound_uses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945" + }, + "microcosm.frame/adapters/__init__.py": { + "imports": [], + "resource_accesses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", + "unbound_uses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945" + }, + "microcosm.frame/adapters/_policyengine_us_source_index.py": { + "imports": [ + "microcosm.frame.schema", + "yaml" + ], + "resource_accesses_sha256": "a64f5650fc19608459f4e1c034e1f7e692de8ca77e4e2e111f8dd40559ce481f", + "unbound_uses_sha256": "6da25f88c6cadb5b1522da2e977a71e749dd54c6cb1d58fb1c6224161e9e42be" + }, + "microcosm.frame/adapters/axiom.py": { + "imports": [ + "axiom_rules_engine", + "microcosm.frame.bundle", + "microcosm.frame.materialize", + "microcosm.frame.rules", + "microcosm.frame.schema", + "numpy", + "pandas" + ], + "resource_accesses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", + "unbound_uses_sha256": "9f0928dfa308893a0724f2e22e2676993aa7084c83115c4fdaecc341e515bd76" + }, + "microcosm.frame/adapters/policyengine_uk.py": { + "imports": [ + "microcosm.frame.bundle", + "microcosm.frame.materialize", + "microcosm.frame.rules", + "microcosm.frame.schema", + "numpy", + "policyengine_uk", + "policyengine_uk.data" + ], + "resource_accesses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", + "unbound_uses_sha256": "366f98c5d3507744c7f9667e7323821c41358a47c56bd83f7ca9c677756c7094" + }, + "microcosm.frame/adapters/policyengine_us.py": { + "imports": [ + "microcosm.frame.adapters._policyengine_us_source_index", + "microcosm.frame.bundle", + "microcosm.frame.materialize", + "microcosm.frame.rules", + "microcosm.frame.schema", + "microcosm.frame.units", + "numpy", + "pandas", + "policyengine_us", + "policyengine_us.data" + ], + "resource_accesses_sha256": "b87a963f56b28cef5b93690cec90f1ff2e935075283e553fdf7af693f8bf3204", + "unbound_uses_sha256": "175c1d0a9621cee21f400b0c3e4618a40eb2fdb795adbf42b16488ea2dcf5b23" + }, + "microcosm.frame/bundle.py": { + "imports": [ + "microcosm.frame.schema", + "microcosm.frame.weights", + "numpy", + "pandas" + ], + "resource_accesses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", + "unbound_uses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945" + }, + "microcosm.frame/kernels.py": { + "imports": [ + "microcosm.frame.bundle", + "microcosm.frame.rules", + "microcosm.frame.schema", + "microcosm.graph", + "numpy", + "pandas" + ], + "resource_accesses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", + "unbound_uses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945" + }, + "microcosm.frame/materialize.py": { + "imports": [ + "microcosm.frame.bundle", + "numpy", + "pandas" + ], + "resource_accesses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", + "unbound_uses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945" + }, + "microcosm.frame/rules.py": { + "imports": [ + "microcosm.frame.bundle", + "microcosm.frame.schema", + "numpy", + "pandas" + ], + "resource_accesses_sha256": "bb3739bfa9c1bdb290ce2e1136dfd73bc4f2c40715a822ea87e3c91ef51a5760", + "unbound_uses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945" + }, + "microcosm.frame/schema.py": { + "imports": [], + "resource_accesses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", + "unbound_uses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945" + }, + "microcosm.frame/units.py": { + "imports": [ + "microcosm.frame.bundle", + "microcosm.frame.schema", + "microcosm.frame.weights", + "microunit", + "microunit.units", + "numpy", + "pandas" + ], + "resource_accesses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", + "unbound_uses_sha256": "e40d25e078da274aaaa43639b66ac70df5a8b87b6b637f46d00069e73bff3e73" + }, + "microcosm.frame/weights.py": { + "imports": [ + "numpy" + ], + "resource_accesses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", + "unbound_uses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945" + }, + "microcosm.graph/__init__.py": { + "imports": [ + "microcosm.graph.codecs", + "microcosm.graph.decl", + "microcosm.graph.errors", + "microcosm.graph.executor", + "microcosm.graph.explain", + "microcosm.graph.kernel", + "microcosm.graph.keys", + "microcosm.graph.manifest", + "microcosm.graph.population", + "microcosm.graph.randomness", + "microcosm.graph.serialize", + "microcosm.graph.store", + "microcosm.graph.view" + ], + "resource_accesses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", + "unbound_uses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945" + }, + "microcosm.graph/artifact_edges.py": { + "imports": [ + "microcosm.graph.decl", + "microcosm.graph.errors", + "microcosm.graph.kernel", + "microcosm.graph.keys" + ], + "resource_accesses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", + "unbound_uses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945" + }, + "microcosm.graph/availability.py": { + "imports": [ + "microcosm.graph.decl", + "microcosm.graph.kernel" + ], + "resource_accesses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", + "unbound_uses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945" + }, + "microcosm.graph/canonical.py": { + "imports": [ + "microcosm.graph.decl" + ], + "resource_accesses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", + "unbound_uses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945" + }, + "microcosm.graph/codecs.py": { + "imports": [ + "microcosm.frame", + "microcosm.graph.store", + "numpy", + "pandas" + ], + "resource_accesses_sha256": "4adeae91a0658854557dc4d2bf01d09ad531726de7cec890f00258a2dbe81da2", + "unbound_uses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945" + }, + "microcosm.graph/decl.py": { + "imports": [], + "resource_accesses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", + "unbound_uses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945" + }, + "microcosm.graph/errors.py": { + "imports": [], + "resource_accesses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", + "unbound_uses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945" + }, + "microcosm.graph/executor.py": { + "imports": [ + "microcosm.frame", + "microcosm.graph", + "microcosm.graph.artifact_edges", + "microcosm.graph.availability", + "microcosm.graph.canonical", + "microcosm.graph.codecs", + "microcosm.graph.decl", + "microcosm.graph.errors", + "microcosm.graph.kernel", + "microcosm.graph.keys", + "microcosm.graph.manifest", + "microcosm.graph.population", + "microcosm.graph.store", + "numpy", + "pandas" + ], + "resource_accesses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", + "unbound_uses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945" + }, + "microcosm.graph/explain.py": { + "imports": [ + "microcosm.graph.availability", + "microcosm.graph.decl", + "microcosm.graph.manifest", + "microcosm.graph.population", + "microcosm.graph.view" + ], + "resource_accesses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", + "unbound_uses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945" + }, + "microcosm.graph/kernel.py": { + "imports": [ + "microcosm.frame", + "microcosm.graph.decl", + "numpy", + "pandas" + ], + "resource_accesses_sha256": "313aceccf09abe24eace2f6d432f41960afb9ed7e80ebe890a10cc41557d87d3", + "unbound_uses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945" + }, + "microcosm.graph/keys.py": { + "imports": [ + "microcosm.graph.canonical", + "microcosm.graph.decl", + "microcosm.graph.kernel" + ], + "resource_accesses_sha256": "5950b5602a721f5334ac91d4027a61707cf58600a827a6f1a4e3b73e690f9748", + "unbound_uses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945" + }, + "microcosm.graph/manifest.py": { + "imports": [ + "microcosm.frame", + "microcosm.graph.artifact_edges", + "microcosm.graph.availability", + "microcosm.graph.canonical", + "microcosm.graph.decl", + "microcosm.graph.errors", + "microcosm.graph.kernel", + "microcosm.graph.population", + "microcosm.graph.store", + "pandas" + ], + "resource_accesses_sha256": "13c805cbaa1c2fd026afc6574c4c08dd71143a68f787d616852d5efcbb151c82", + "unbound_uses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945" + }, + "microcosm.graph/population.py": { + "imports": [ + "microcosm.frame", + "microcosm.graph.canonical", + "microcosm.graph.decl", + "microcosm.graph.kernel", + "microcosm.graph.store", + "numpy", + "pandas" + ], + "resource_accesses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", + "unbound_uses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945" + }, + "microcosm.graph/randomness.py": { + "imports": [ + "microcosm.graph.canonical", + "numpy" + ], + "resource_accesses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", + "unbound_uses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945" + }, + "microcosm.graph/serialize.py": { + "imports": [ + "microcosm.graph.canonical", + "microcosm.graph.decl" + ], + "resource_accesses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", + "unbound_uses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945" + }, + "microcosm.graph/store.py": { + "imports": [ + "microcosm.frame", + "microcosm.frame.bundle", + "microcosm.graph.errors", + "numpy", + "pandas" + ], + "resource_accesses_sha256": "99afee26ca660c678dcbd7117eb2e64d70a0343ead9a602fa5819222aca02bc3", + "unbound_uses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945" + }, + "microcosm.graph/view.py": { + "imports": [ + "microcosm.graph.canonical", + "microcosm.graph.decl", + "microcosm.graph.manifest" + ], + "resource_accesses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", + "unbound_uses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945" + }, + "microunit/__init__.py": { + "imports": [ + "microunit.core", + "microunit.diagnostics", + "microunit.registry", + "microunit.rule_helpers", + "microunit.tax_unit_construction" + ], + "resource_accesses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", + "unbound_uses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945" + }, + "microunit/core.py": { + "imports": [ + "pandas" + ], + "resource_accesses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", + "unbound_uses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945" + }, + "microunit/diagnostics.py": { + "imports": [ + "microunit.core", + "pandas" + ], + "resource_accesses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", + "unbound_uses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945" + }, + "microunit/registry.py": { + "imports": [], + "resource_accesses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", + "unbound_uses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945" + }, + "microunit/rule_helpers.py": { + "imports": [ + "yaml" + ], + "resource_accesses_sha256": "d880e849f439db697d177523c8c7e92775b109fc42d2f71372e2702dd9c19749", + "unbound_uses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945" + }, + "microunit/tax_unit_construction.py": { + "imports": [ + "microunit.rule_helpers", + "numpy", + "pandas" + ], + "resource_accesses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", + "unbound_uses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945" + }, + "microunit/units/__init__.py": { + "imports": [ + "microunit.units.medicaid", + "microunit.units.passthrough", + "microunit.units.programs", + "microunit.units.snap", + "microunit.units.spm", + "microunit.units.tax" + ], + "resource_accesses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", + "unbound_uses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945" + }, + "microunit/units/_helpers.py": { + "imports": [ + "pandas" + ], + "resource_accesses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", + "unbound_uses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945" + }, + "microunit/units/medicaid.py": { + "imports": [ + "microunit.core", + "microunit.units.programs", + "pandas" + ], + "resource_accesses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", + "unbound_uses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945" + }, + "microunit/units/passthrough.py": { + "imports": [ + "microunit.core", + "pandas" + ], + "resource_accesses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", + "unbound_uses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945" + }, + "microunit/units/programs.py": { + "imports": [ + "microunit.core", + "microunit.units.spm", + "pandas" + ], + "resource_accesses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", + "unbound_uses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945" + }, + "microunit/units/snap.py": { + "imports": [ + "microunit.core", + "microunit.units._helpers", + "microunit.units.programs", + "pandas" + ], + "resource_accesses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", + "unbound_uses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945" + }, + "microunit/units/spm.py": { + "imports": [ + "microunit.core", + "microunit.units._helpers", + "pandas" + ], + "resource_accesses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", + "unbound_uses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945" + }, + "microunit/units/tax.py": { + "imports": [ + "microunit.core", + "microunit.tax_unit_construction", + "microunit.units._helpers", + "pandas" + ], + "resource_accesses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", + "unbound_uses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945" + } + }, + "import_classifications": { + "axiom_rules_engine": "inactive shared-module or optional-adapter route; scoped-use tripwire retained", + "h5py": "versioned dependency in explicit stages", + "microcosm.build": "package namespace import of explicitly bound frame_checkpoint; other package exports are inactive in this slice", + "microcosm.build.cd_benchmark": "package-relative original source-member identity alias; origin and its pure canonical helper are explicitly whole-module bound in authenticated_survey_population_v1", + "microcosm.build.cd_benchmark.canonical": "whole-module pure canonical JSON and source digest validation in authenticated_survey_population_v1; no benchmark targets or data", + "microcosm.build.cd_benchmark.origin": "whole-module original source-member identity in authenticated_survey_population_v1; no benchmark targets or data", + "microcosm.build.frame_checkpoint": "whole-module in explicit stages", + "microcosm.build.frame_sampling": "whole-module in explicit stages", + "microcosm.build.gates": "whole-module in explicit stages", + "microcosm.build.ledger_targets": "inactive shared-module or optional-adapter route; scoped-use tripwire retained", + "microcosm.build.outer_stage_runtime": "whole-module in explicit stages", + "microcosm.build.serialization_dtypes": "whole-module in explicit stages", + "microcosm.build.source_manifest": "inactive legacy alimony source-stage declaration; scoped-use and resource-access tripwires retained", + "microcosm.build.survey_allocation": "whole-module pure domain share and original-anchor contracts in authenticated_survey_population_v1", + "microcosm.build.survey_domain_sample": "whole-module deterministic per-domain household selection in authenticated_survey_population_v1", + "microcosm.build.table_identity": "whole-module in explicit stages", + "microcosm.build.us_runtime": "package-relative live aliases for original money/schema/checkpoint/household/person-income/housing verifiers, separate student controls and graph context are explicitly module-bound in asec_prepared_v3; other shared-module or optional routes remain inactive with scoped-use tripwires; v2 adds package-relative HU definition/source aliases and non-authoritative transport, each explicitly whole-module bound. The authenticated survey stage additionally binds its exact catalogue, selected-native, original-source, pure-selection and preparation aliases; optional PUF clone validation remains downstream and bound by the clone operator identity.", + "microcosm.build.us_runtime._asec_current_money_codec": "whole-module in asec_prepared_v3; original canonical full-source replay unchanged", + "microcosm.build.us_runtime.acs_housing_universe": "Whole-module in acs_housing_universe_2024; only source capture/projection/native Frame and typed custody are live. Other stages retain scoped-use tripwires for registry availability.", + "microcosm.build.us_runtime.acs_housing_universe_source": "Whole-module in acs_housing_universe_2024; only source capture/projection/native Frame and typed custody are live. Other stages retain scoped-use tripwires for registry availability.", + "microcosm.build.us_runtime.acs_income_universe": "inactive shared-module or optional-adapter route; scoped-use tripwire retained", + "microcosm.build.us_runtime.acs_inputs": "whole-module in explicit stages", + "microcosm.build.us_runtime.acs_pums": "whole-module in explicit stages", + "microcosm.build.us_runtime.acs_sources": "Whole-module in acs_housing_universe_2024; only source capture/projection/native Frame and typed custody are live. Other stages retain scoped-use tripwires for registry availability.", + "microcosm.build.us_runtime.acs_transfer": "live registry projection; other shared-module routes inactive", + "microcosm.build.us_runtime.adult_care": "live registry projection; other shared-module routes inactive", + "microcosm.build.us_runtime.alimony": "whole-module in asec_prepared_v3 for pure ASEC split; source manifest and other nominal routes inactive", + "microcosm.build.us_runtime.asec_checkpoint": "whole-module in explicit stages", + "microcosm.build.us_runtime.asec_current_money": "whole-module in asec_prepared_v3; immutable recipe source identity", + "microcosm.build.us_runtime.asec_current_money_graph_resources": "whole-module in asec_prepared_v3; additive status loader leaves original T1/money resource identity unchanged", + "microcosm.build.us_runtime.asec_current_money_resources": "whole-module in asec_prepared_v3; original recipe and installed microunit verification unchanged", + "microcosm.build.us_runtime.asec_current_money_selection": "whole-module in asec_prepared_v3; selected body has provenance but no full-source authority", + "microcosm.build.us_runtime.asec_current_money_source": "whole-module in asec_prepared_v3; authenticated original source verification", + "microcosm.build.us_runtime.asec_current_money_units": "whole-module in asec_prepared_v3; real microunit with separate authenticated student controls", + "microcosm.build.us_runtime.asec_engine_evaluation": "whole-module in asec_prepared_v3; pinned complete installed formula closures and empty defaults allowlist", + "microcosm.build.us_runtime.asec_housing_status": "whole-module in asec_prepared_v3; status definition and sealed authenticated artifact", + "microcosm.build.us_runtime.asec_housing_status_source": "whole-module in asec_prepared_v3; housing source remains original T1", + "microcosm.build.us_runtime.asec_housing_universe": "Whole-module in asec_prepared_v3; native HU source definition/ownership and non-authoritative graph transport plus selected-row validation; no ACS or selected-Frame authority.", + "microcosm.build.us_runtime.asec_housing_universe_source": "Whole-module in asec_prepared_v3; native HU source definition/ownership and non-authoritative graph transport plus selected-row validation; no ACS or selected-Frame authority.", + "microcosm.build.us_runtime.asec_person_income_source": "whole-module in asec_prepared_v3; unchanged T1 restoration verifier closure", + "microcosm.build.us_runtime.asec_prepared_source": "whole-module in asec_prepared_v3; registry-only import in other graph_sources scopes retains scoped-use tripwire", + "microcosm.build.us_runtime.asec_student_controls": "whole-module in asec_prepared_v3; separate S authority before tax units, never part of original T1 verifier", + "microcosm.build.us_runtime.child_support": "live registry projection; other shared-module routes inactive", + "microcosm.build.us_runtime.childcare": "live registry projection; other shared-module routes inactive", + "microcosm.build.us_runtime.congressional_district_geography": "whole-module in explicit stages", + "microcosm.build.us_runtime.congressional_district_vintage": "whole-module in explicit stages", + "microcosm.build.us_runtime.cps_carried": "whole-module in asec_prepared_v3 for split constants; nominal consumer route inactive", + "microcosm.build.us_runtime.cps_carried_current": "whole-module in asec_prepared_v3; typed corrected money amounts and declared raw routing only", + "microcosm.build.us_runtime.disability_benefits": "live registry projection; other shared-module routes inactive", + "microcosm.build.us_runtime.education_assistance_source": "whole-module in explicit stages", + "microcosm.build.us_runtime.education_inputs": "live registry projection; other shared-module routes inactive", + "microcosm.build.us_runtime.eligibility_inputs": "live registry projection; other shared-module routes inactive", + "microcosm.build.us_runtime.energy_subsidy": "live registry projection; other shared-module routes inactive", + "microcosm.build.us_runtime.geography_ladder": "whole-module in explicit stages", + "microcosm.build.us_runtime.graph_asec_income": "Whole-module in asec_prepared_v3; PAW source evidence, bound reported-income accounting or shared typed graph declarations. No source authority is minted from graph transport.", + "microcosm.build.us_runtime.graph_asec_prepared": "whole-module in explicit stages", + "microcosm.build.us_runtime.graph_combined_clone": "optional downstream clone declaration/registration; clone kernels retain their own implementation identities; no donor or target resource in authenticated_survey_population_v1", + "microcosm.build.us_runtime.graph_composed_asec_binding": "whole-module in composed_asec_binding_v1; composed-arm resolution, its typed binding/arm-row transport and the declared row mask. Reads source provenance by charter and mints no source authority.", + "microcosm.build.us_runtime.graph_composed_contracts": "Pure composed producer IDs and artifact types; whole module bound by both legacy composed stages and explicit PUF extension hashes. No resources or operation dispatch.", + "microcosm.build.us_runtime.graph_composed_population": "whole-module in composed_population_v1 and composed_asec_binding_v1; the composing CREATE's declarations, node ids and typed origin binder. Adds no source route of its own.", + "microcosm.build.us_runtime.graph_context": "whole-module in explicit stages", + "microcosm.build.us_runtime.graph_geography": "whole-module in explicit stages", + "microcosm.build.us_runtime.graph_housing_universe": "Whole-module in asec_prepared_v3; native HU source definition/ownership and non-authoritative graph transport plus selected-row validation; no ACS or selected-Frame authority.", + "microcosm.build.us_runtime.graph_implementation": "whole-module in explicit stages", + "microcosm.build.us_runtime.graph_sources": "whole-module in explicit graph stages; prepared slice uses frame column declarations and validated codec registration only; authenticated survey preparation uses only frame column declarations and lossless storage canonicalization, not legacy source codecs or assembly treatment", + "microcosm.build.us_runtime.hours_worked": "live registry projection; other shared-module routes inactive", + "microcosm.build.us_runtime.housing_inputs": "live registry projection; other shared-module routes inactive", + "microcosm.build.us_runtime.immigration": "live registry projection; other shared-module routes inactive", + "microcosm.build.us_runtime.late_producer_dag": "inactive shared-module or optional-adapter route; scoped-use tripwire retained", + "microcosm.build.us_runtime.medicare_take_up": "live registry projection; other shared-module routes inactive", + "microcosm.build.us_runtime.multispine_pool": "inactive shared-module or optional-adapter route; scoped-use tripwire retained", + "microcosm.build.us_runtime.operator_boundary": "whole-module in explicit stages", + "microcosm.build.us_runtime.operator_column_contracts": "Pure shared column declarations bound by every source/assembly scope that consumes them; no runtime operator or resource loading.", + "microcosm.build.us_runtime.post_transfer_calibration": "inactive shared-module or optional-adapter route; scoped-use tripwire retained", + "microcosm.build.us_runtime.pregnancy": "live registry projection; other shared-module routes inactive", + "microcosm.build.us_runtime.prior_year_income": "live registry projection; other shared-module routes inactive", + "microcosm.build.us_runtime.public_assistance_type_source": "inactive legacy cps_carried consumer route; split constants used by corrected slice do not call this reader", + "microcosm.build.us_runtime.puf_aggregate_records": "inactive shared-module or optional-adapter route; scoped-use tripwire retained", + "microcosm.build.us_runtime.puf_capital_gains_tail": "live registry projection; other shared-module routes inactive", + "microcosm.build.us_runtime.puf_e01000_reconciliation": "inactive shared-module or optional-adapter route; scoped-use tripwire retained", + "microcosm.build.us_runtime.puf_interest_components": "inactive shared-module or optional-adapter route; scoped-use tripwire retained", + "microcosm.build.us_runtime.puf_qrf_chain": "inactive shared-module or optional-adapter route; scoped-use tripwire retained", + "microcosm.build.us_runtime.puf_support": "whole-module in explicit stages", + "microcosm.build.us_runtime.puma_ladder": "whole-module in explicit stages", + "microcosm.build.us_runtime.puma_ladder_sources": "whole-module in explicit stages", + "microcosm.build.us_runtime.qbi_inputs": "live registry projection; other shared-module routes inactive", + "microcosm.build.us_runtime.relationship_inputs": "live registry projection; other shared-module routes inactive", + "microcosm.build.us_runtime.reported_coverage_source": "whole-module in explicit stages", + "microcosm.build.us_runtime.retirement_contributions": "live registry projection; other shared-module routes inactive", + "microcosm.build.us_runtime.retirement_distributions": "live registry projection; other shared-module routes inactive", + "microcosm.build.us_runtime.scf_wealth": "live registry projection; other shared-module routes inactive", + "microcosm.build.us_runtime.source_csv_builtin": "Whole-module in acs_housing_universe_2024; stdlib-only checked CSV builtin capture, without source or resource access.", + "microcosm.build.us_runtime.spine_assembly": "whole-module in explicit stages", + "microcosm.build.us_runtime.stacked_spine": "whole-module in explicit stages", + "microcosm.build.us_runtime.support_provenance": "whole-module in explicit stages", + "microcosm.build.us_runtime.survey_catalogue_selection": "whole-module pure complete-catalogue selection plan in authenticated_survey_population_v1", + "microcosm.build.us_runtime.survey_population_domains": "whole-module raw source-record classification and prespecified development allocation declaration in authenticated_survey_population_v1", + "microcosm.build.us_runtime.us_late_overlap_ownership": "inactive shared-module or optional-adapter route; scoped-use tripwire retained", + "microcosm.build.us_runtime.us_late_producer_registry": "inactive shared-module or optional-adapter route; scoped-use tripwire retained", + "microcosm.build.us_runtime.weeks_unemployed": "live registry projection; other shared-module routes inactive", + "microcosm.build.us_runtime.wic_claim": "live registry projection; other shared-module routes inactive", + "microcosm.build.us_runtime.workers_compensation": "live registry projection; other shared-module routes inactive", + "microcosm.calibrate.geography_constants": "inactive shared-module or optional-adapter route; scoped-use tripwire retained", + "microcosm.calibrate.registry": "inactive shared-module or optional-adapter route; scoped-use tripwire retained", + "microcosm.calibrate.solve": "inactive shared-module or optional-adapter route; scoped-use tripwire retained", + "microcosm.fit": "inactive shared-module or optional-adapter route; scoped-use tripwire retained", + "microcosm.frame": "whole-module in explicit stages", + "microcosm.frame.accounting": "whole-module in explicit stages", + "microcosm.frame.adapters._policyengine_us_source_index": "whole-module in explicit stages", + "microcosm.frame.adapters.policyengine_us": "whole-module in explicit stages", + "microcosm.frame.bundle": "whole-module in explicit stages", + "microcosm.frame.materialize": "whole-module in explicit stages", + "microcosm.frame.rules": "whole-module in explicit stages", + "microcosm.frame.schema": "whole-module in explicit stages", + "microcosm.frame.units": "whole-module in explicit stages", + "microcosm.frame.weights": "whole-module in explicit stages", + "microcosm.graph": "whole-module in explicit stages", + "microcosm.graph.artifact_edges": "whole-module in explicit stages", + "microcosm.graph.availability": "whole-module in explicit stages", + "microcosm.graph.canonical": "whole-module in explicit stages", + "microcosm.graph.codecs": "whole-module in explicit stages", + "microcosm.graph.decl": "whole-module in explicit stages", + "microcosm.graph.errors": "whole-module in explicit stages", + "microcosm.graph.executor": "whole-module in explicit stages", + "microcosm.graph.explain": "whole-module in explicit stages", + "microcosm.graph.kernel": "whole-module in explicit stages", + "microcosm.graph.keys": "whole-module in explicit stages", + "microcosm.graph.manifest": "whole-module in explicit stages", + "microcosm.graph.population": "whole-module in explicit stages", + "microcosm.graph.randomness": "whole-module in explicit stages", + "microcosm.graph.serialize": "whole-module in explicit stages", + "microcosm.graph.store": "whole-module in explicit stages", + "microcosm.graph.view": "whole-module in explicit stages", + "microunit": "whole-module in explicit stages", + "microunit.core": "whole-module in explicit stages", + "microunit.diagnostics": "whole-module in explicit stages", + "microunit.registry": "whole-module in explicit stages", + "microunit.rule_helpers": "whole-module in explicit stages", + "microunit.tax_unit_construction": "whole-module in explicit stages", + "microunit.units": "whole-module in explicit stages", + "microunit.units._helpers": "whole-module in explicit stages", + "microunit.units.medicaid": "whole-module in explicit stages", + "microunit.units.passthrough": "whole-module in explicit stages", + "microunit.units.programs": "whole-module in explicit stages", + "microunit.units.snap": "whole-module in explicit stages", + "microunit.units.spm": "whole-module in explicit stages", + "microunit.units.tax": "whole-module in explicit stages", + "numpy": "versioned dependency in explicit stages", + "pandas": "versioned dependency in explicit stages", + "pandas.api.types": "versioned dependency in explicit stages", + "policyengine_uk": "inactive shared-module or optional-adapter route; scoped-use tripwire retained", + "policyengine_uk.data": "inactive shared-module or optional-adapter route; scoped-use tripwire retained", + "policyengine_us": "live evaluation only, explicitly pinned installed runtime and parameter identity; inactive in other shared adapter routes", + "policyengine_us.data": "live evaluation dataset adapter only, bound by pinned installed PE-US runtime identity; inactive in other stages", + "yaml": "versioned dependency in explicit stages" + }, + "package_resources": { + "microunit": [ + "data/dependent_gross_income_limit.yaml" + ] + }, + "package_rosters": { + "microcosm.frame": [ + "__init__.py", + "accounting.py", + "adapters/__init__.py", + "adapters/_policyengine_us_source_index.py", + "adapters/axiom.py", + "adapters/policyengine_uk.py", + "adapters/policyengine_us.py", + "bundle.py", + "kernels.py", + "materialize.py", + "rules.py", + "schema.py", + "units.py", + "weights.py" + ], + "microcosm.graph": [ + "__init__.py", + "artifact_edges.py", + "availability.py", + "canonical.py", + "codecs.py", + "decl.py", + "errors.py", + "executor.py", + "explain.py", + "kernel.py", + "keys.py", + "manifest.py", + "population.py", + "randomness.py", + "serialize.py", + "store.py", + "view.py" + ], + "microunit": [ + "__init__.py", + "core.py", + "diagnostics.py", + "registry.py", + "rule_helpers.py", + "tax_unit_construction.py", + "units/__init__.py", + "units/_helpers.py", + "units/medicaid.py", + "units/passthrough.py", + "units/programs.py", + "units/snap.py", + "units/spm.py", + "units/tax.py" + ] + }, + "reviewed_base": "c71e31654faf0e3f874e68e8c100b09c7eb9955f", + "schema": "microcosm.us.implementation-inventory.v1", + "scope_notes": { + "asec_prepared_engine": "PE-US/Core are live only in the evaluation kernel. That kernel adds their dependencies and verifies pinned installed implementation/parameter bytes in its implementation identity before cache lookup. The common prepared scope can be hashed without engine extras; its local adapter/source-index modules and empty-defaults resource are bound.", + "asec_prepared_v3": "Full P/H/T1 restoration, unchanged money recipe, separate S authentication/attachment before tax units, housing authentication against original T1, lossless dtype promotion, typed whole-household selection, corrected CPS leaves and four admitted PE outputs. Graph/Frame/microunit rosters are bound conservatively. Only declared source codec registration and frame column declarations run from graph_sources; ACS and stack assembly routes remain inactive. Alimony's pure ASEC split and cps_carried's split constants are bound, while their legacy consumer/source-manifest routes are inactive. V2 additionally authenticates HU from original T1, attaches it after housing status, preserves the pre-HU parent signature and carries native HU bytes plus selected-row ancestry with exact int64 promotion; pure ACS HU remains inactive. V3 additionally issues separate PAW_VAL/PAW_YN observations from the same official members and a derived six-column reported-income branch. The 33-field body, T1/S definitions, selected-money v1 and selection v2 payload contract remain sealed; historical v2 artifacts are preserved, but v3 runtime keys invalidate old caches.", + "authenticated_survey_population_v1": "Complete ACS and ASEC catalogues authenticate original source records before one domain draw. Selected native issuers produce original DESIGN anchors, which are stacked without share normalization; the graph applies share/inclusion probability once. Explicit narrow source closure includes pure origin canonical encoding and the original household/person custody helpers, not the legacy prepared ASEC resource union. Original source JSONs and microunit YAML are the only packaged data. The isolated real source-codec loader is declared separately from the unchanged global five codecs. The runner reconstructs source authority on cold and warm invocations and checks actual populations, design ancestry, owners, weight kinds, mass ledgers and typed artifact descriptors. Optional PUF support cloning follows over both survey arms under existing clone implementation identities. No donor fit/draw, policy engine, calibration or target measurement is part of this stage. Initial source and graph snapshots remain preliminary pending integrated invented execution and independent review.", + "bound_modules": "Whole bytes of explicitly named modules; helper-only parsing never imports adapters.", + "composed_asec_binding_v1": "Binding the carried prepared-ASEC evidence to the composed ASEC rows and deriving the reviewed corrected leaves and reported observations over them. This stage reads no source; every artifact it admits was produced by composed_population_v1, and every verifier, selector and derivation it reuses already lives in that scope, so its module inventory is exactly that scope's plus the two binding modules and its dependency set is identical. Because the inventory is a superset of composed_population_v1 and therefore of each of its parents, no existing module's covered-import set moves. It binds the same two codecs the composing CREATE binds rather than claiming an identity narrower than its actual inputs.", + "composed_population_v1": "The composed prepared-ASEC/native-ACS population. One CREATE runs both reviewed source scopes, so this stage's module inventory is exactly the union of assembly_prepare, asec_prepared_v3 and geography plus the composing module, and its dependency set is theirs. Because the union is a superset of each parent, no existing module's covered-import set and therefore no existing stage identity moves. It binds the prepared codec and the ACS codec it actually reads; the raw-stage codec is not bound because this stage never reads it. Sampling, stacking, provenance and importance allocation remain in the unchanged stacked_spine/spine_assembly modules, and harmonization reuses the assembly_harmonize scope through its own kernel. No engine, clone, transfer or calibration route is active in this scope.", + "current_money_resources": "Original source recipe and consumers declaration bytes remain unchanged to preserve authenticated T1/money identities. The additive graph consumer resource is separately pinned and supersedes only implementation status for the graph slice and microunit; all old nominal consumer routes remain pending.", + "occupied_housing_review_base": "HU v1 source definition and attachment are exact approved 4b26e59b additions on d4bddcb6ac2aba52903cd8dace90330cee4a21e1; only the v2 prepared graph/transport wiring is new. Original source-verifier modules and resources are unchanged.", + "operator_boundary": "Resolved output/formula registries projected; provider bodies are not called by source construction.", + "other_imports": "Canonical non-stdlib import modules are explicitly classified. Ordinary symbols from a module/dependency bound in every using scope need no inventory edit. Only unbound-helper uses retain a lexical-scope fingerprint; resource-access expressions retain their own fingerprint. Expanded AST details are audit output, not packaged data. Neither fingerprint is a proof of dynamic closure.", + "resources": "Microunit rule YAML is bound. Default packaged geography/source-stage resources belong to inactive legacy entrypoints; graph lookups are explicit SourceRefs." + }, + "stages": { + "acs_codec_2024": { + "dependencies": [ + "numpy", + "pandas", + "microunit", + "PyYAML", + "pyarrow" + ], + "modules": [ + "microcosm.build/serialization_dtypes.py", + "microcosm.build/us_runtime/acs_inputs.py", + "microcosm.build/us_runtime/acs_pums.py", + "microcosm.build/us_runtime/graph_context.py", + "microcosm.build/us_runtime/graph_implementation.py", + "microcosm.build/us_runtime/graph_sources.py", + "microcosm.build/us_runtime/operator_boundary.py", + "microcosm.build/us_runtime/operator_column_contracts.py", + "microcosm.frame/__init__.py", + "microcosm.frame/accounting.py", + "microcosm.frame/adapters/__init__.py", + "microcosm.frame/adapters/_policyengine_us_source_index.py", + "microcosm.frame/adapters/axiom.py", + "microcosm.frame/adapters/policyengine_uk.py", + "microcosm.frame/adapters/policyengine_us.py", + "microcosm.frame/bundle.py", + "microcosm.frame/kernels.py", + "microcosm.frame/materialize.py", + "microcosm.frame/rules.py", + "microcosm.frame/schema.py", + "microcosm.frame/units.py", + "microcosm.frame/weights.py", + "microcosm.graph/__init__.py", + "microcosm.graph/artifact_edges.py", + "microcosm.graph/availability.py", + "microcosm.graph/canonical.py", + "microcosm.graph/codecs.py", + "microcosm.graph/decl.py", + "microcosm.graph/errors.py", + "microcosm.graph/executor.py", + "microcosm.graph/explain.py", + "microcosm.graph/kernel.py", + "microcosm.graph/keys.py", + "microcosm.graph/manifest.py", + "microcosm.graph/population.py", + "microcosm.graph/randomness.py", + "microcosm.graph/serialize.py", + "microcosm.graph/store.py", + "microcosm.graph/view.py", + "microunit/__init__.py", + "microunit/core.py", + "microunit/diagnostics.py", + "microunit/registry.py", + "microunit/rule_helpers.py", + "microunit/tax_unit_construction.py", + "microunit/units/__init__.py", + "microunit/units/_helpers.py", + "microunit/units/medicaid.py", + "microunit/units/passthrough.py", + "microunit/units/programs.py", + "microunit/units/snap.py", + "microunit/units/spm.py", + "microunit/units/tax.py" + ], + "resources": [ + "microunit/data/dependent_gross_income_limit.yaml" + ], + "source_boundary_projection": true + }, + "acs_housing_universe_2024": { + "dependencies": [ + "numpy", + "pandas", + "microunit", + "PyYAML", + "pyarrow" + ], + "modules": [ + "microcosm.build/serialization_dtypes.py", + "microcosm.build/us_runtime/acs_housing_universe.py", + "microcosm.build/us_runtime/acs_housing_universe_source.py", + "microcosm.build/us_runtime/acs_inputs.py", + "microcosm.build/us_runtime/acs_pums.py", + "microcosm.build/us_runtime/acs_sources.py", + "microcosm.build/us_runtime/graph_acs_housing_universe.py", + "microcosm.build/us_runtime/graph_context.py", + "microcosm.build/us_runtime/graph_implementation.py", + "microcosm.build/us_runtime/graph_sources.py", + "microcosm.build/us_runtime/operator_boundary.py", + "microcosm.build/us_runtime/operator_column_contracts.py", + "microcosm.build/us_runtime/source_csv_builtin.py", + "microcosm.frame/__init__.py", + "microcosm.frame/accounting.py", + "microcosm.frame/adapters/__init__.py", + "microcosm.frame/adapters/_policyengine_us_source_index.py", + "microcosm.frame/adapters/axiom.py", + "microcosm.frame/adapters/policyengine_uk.py", + "microcosm.frame/adapters/policyengine_us.py", + "microcosm.frame/bundle.py", + "microcosm.frame/kernels.py", + "microcosm.frame/materialize.py", + "microcosm.frame/rules.py", + "microcosm.frame/schema.py", + "microcosm.frame/units.py", + "microcosm.frame/weights.py", + "microcosm.graph/__init__.py", + "microcosm.graph/artifact_edges.py", + "microcosm.graph/availability.py", + "microcosm.graph/canonical.py", + "microcosm.graph/codecs.py", + "microcosm.graph/decl.py", + "microcosm.graph/errors.py", + "microcosm.graph/executor.py", + "microcosm.graph/explain.py", + "microcosm.graph/kernel.py", + "microcosm.graph/keys.py", + "microcosm.graph/manifest.py", + "microcosm.graph/population.py", + "microcosm.graph/randomness.py", + "microcosm.graph/serialize.py", + "microcosm.graph/store.py", + "microcosm.graph/view.py", + "microunit/__init__.py", + "microunit/core.py", + "microunit/diagnostics.py", + "microunit/registry.py", + "microunit/rule_helpers.py", + "microunit/tax_unit_construction.py", + "microunit/units/__init__.py", + "microunit/units/_helpers.py", + "microunit/units/medicaid.py", + "microunit/units/passthrough.py", + "microunit/units/programs.py", + "microunit/units/snap.py", + "microunit/units/spm.py", + "microunit/units/tax.py" + ], + "resources": [ + "microcosm.build/us_runtime/acs_2024_1yr_sources.json", + "microcosm.build/us_runtime/acs_2024_housing_universe.json", + "microunit/data/dependent_gross_income_limit.yaml" + ], + "source_boundary_projection": true + }, + "asec_codec_v4": { + "dependencies": [ + "numpy", + "pandas", + "h5py" + ], + "modules": [ + "microcosm.build/frame_checkpoint.py", + "microcosm.build/outer_stage_runtime.py", + "microcosm.build/serialization_dtypes.py", + "microcosm.build/us_runtime/asec_checkpoint.py", + "microcosm.build/us_runtime/education_assistance_source.py", + "microcosm.build/us_runtime/graph_context.py", + "microcosm.build/us_runtime/graph_implementation.py", + "microcosm.build/us_runtime/graph_sources.py", + "microcosm.build/us_runtime/operator_boundary.py", + "microcosm.build/us_runtime/operator_column_contracts.py", + "microcosm.build/us_runtime/reported_coverage_source.py", + "microcosm.frame/__init__.py", + "microcosm.frame/accounting.py", + "microcosm.frame/adapters/__init__.py", + "microcosm.frame/adapters/_policyengine_us_source_index.py", + "microcosm.frame/adapters/axiom.py", + "microcosm.frame/adapters/policyengine_uk.py", + "microcosm.frame/adapters/policyengine_us.py", + "microcosm.frame/bundle.py", + "microcosm.frame/kernels.py", + "microcosm.frame/materialize.py", + "microcosm.frame/rules.py", + "microcosm.frame/schema.py", + "microcosm.frame/units.py", + "microcosm.frame/weights.py", + "microcosm.graph/__init__.py", + "microcosm.graph/artifact_edges.py", + "microcosm.graph/availability.py", + "microcosm.graph/canonical.py", + "microcosm.graph/codecs.py", + "microcosm.graph/decl.py", + "microcosm.graph/errors.py", + "microcosm.graph/executor.py", + "microcosm.graph/explain.py", + "microcosm.graph/kernel.py", + "microcosm.graph/keys.py", + "microcosm.graph/manifest.py", + "microcosm.graph/population.py", + "microcosm.graph/randomness.py", + "microcosm.graph/serialize.py", + "microcosm.graph/store.py", + "microcosm.graph/view.py" + ], + "resources": [], + "source_boundary_projection": true + }, + "asec_prepared_v3": { + "dependencies": [ + "numpy", + "pandas", + "microunit", + "tables", + "h5py", + "PyYAML", + "pyarrow" + ], + "modules": [ + "microcosm.build/frame_checkpoint.py", + "microcosm.build/frame_sampling.py", + "microcosm.build/gates.py", + "microcosm.build/outer_stage_runtime.py", + "microcosm.build/serialization_dtypes.py", + "microcosm.build/table_identity.py", + "microcosm.build/us_runtime/_asec_current_money_codec.py", + "microcosm.build/us_runtime/alimony.py", + "microcosm.build/us_runtime/asec_checkpoint.py", + "microcosm.build/us_runtime/asec_current_money.py", + "microcosm.build/us_runtime/asec_current_money_graph_resources.py", + "microcosm.build/us_runtime/asec_current_money_resources.py", + "microcosm.build/us_runtime/asec_current_money_selection.py", + "microcosm.build/us_runtime/asec_current_money_source.py", + "microcosm.build/us_runtime/asec_current_money_units.py", + "microcosm.build/us_runtime/asec_engine_evaluation.py", + "microcosm.build/us_runtime/asec_household_observations.py", + "microcosm.build/us_runtime/asec_housing_status.py", + "microcosm.build/us_runtime/asec_housing_status_source.py", + "microcosm.build/us_runtime/asec_housing_universe.py", + "microcosm.build/us_runtime/asec_housing_universe_source.py", + "microcosm.build/us_runtime/asec_income_observations.py", + "microcosm.build/us_runtime/asec_person_income_source.py", + "microcosm.build/us_runtime/asec_prepared_source.py", + "microcosm.build/us_runtime/asec_student_controls.py", + "microcosm.build/us_runtime/cps_carried.py", + "microcosm.build/us_runtime/cps_carried_current.py", + "microcosm.build/us_runtime/education_assistance_source.py", + "microcosm.build/us_runtime/graph_asec_income.py", + "microcosm.build/us_runtime/graph_asec_prepared.py", + "microcosm.build/us_runtime/graph_context.py", + "microcosm.build/us_runtime/graph_housing_universe.py", + "microcosm.build/us_runtime/graph_implementation.py", + "microcosm.build/us_runtime/graph_sources.py", + "microcosm.build/us_runtime/operator_boundary.py", + "microcosm.build/us_runtime/operator_column_contracts.py", + "microcosm.build/us_runtime/reported_coverage_source.py", + "microcosm.build/us_runtime/support_provenance.py", + "microcosm.frame/__init__.py", + "microcosm.frame/accounting.py", + "microcosm.frame/adapters/__init__.py", + "microcosm.frame/adapters/_policyengine_us_source_index.py", + "microcosm.frame/adapters/axiom.py", + "microcosm.frame/adapters/policyengine_uk.py", + "microcosm.frame/adapters/policyengine_us.py", + "microcosm.frame/bundle.py", + "microcosm.frame/kernels.py", + "microcosm.frame/materialize.py", + "microcosm.frame/rules.py", + "microcosm.frame/schema.py", + "microcosm.frame/units.py", + "microcosm.frame/weights.py", + "microcosm.graph/__init__.py", + "microcosm.graph/artifact_edges.py", + "microcosm.graph/availability.py", + "microcosm.graph/canonical.py", + "microcosm.graph/codecs.py", + "microcosm.graph/decl.py", + "microcosm.graph/errors.py", + "microcosm.graph/executor.py", + "microcosm.graph/explain.py", + "microcosm.graph/kernel.py", + "microcosm.graph/keys.py", + "microcosm.graph/manifest.py", + "microcosm.graph/population.py", + "microcosm.graph/randomness.py", + "microcosm.graph/serialize.py", + "microcosm.graph/store.py", + "microcosm.graph/view.py", + "microunit/__init__.py", + "microunit/core.py", + "microunit/diagnostics.py", + "microunit/registry.py", + "microunit/rule_helpers.py", + "microunit/tax_unit_construction.py", + "microunit/units/__init__.py", + "microunit/units/_helpers.py", + "microunit/units/medicaid.py", + "microunit/units/passthrough.py", + "microunit/units/programs.py", + "microunit/units/snap.py", + "microunit/units/spm.py", + "microunit/units/tax.py" + ], + "resources": [ + "microcosm.build/us_runtime/asec_current_money_consumers_v1.json", + "microcosm.build/us_runtime/asec_current_money_domains_v1.json", + "microcosm.build/us_runtime/asec_current_money_engine_defaults_v1.json", + "microcosm.build/us_runtime/asec_current_money_graph_consumers_v1.json", + "microcosm.build/us_runtime/asec_current_money_price_basis_v1.json", + "microcosm.build/us_runtime/asec_income_observations_v1.json", + "microunit/data/dependent_gross_income_limit.yaml" + ], + "source_boundary_projection": true + }, + "assembly_harmonize": { + "dependencies": [ + "numpy", + "pandas" + ], + "modules": [ + "microcosm.build/frame_sampling.py", + "microcosm.build/gates.py", + "microcosm.build/serialization_dtypes.py", + "microcosm.build/table_identity.py", + "microcosm.build/us_runtime/graph_context.py", + "microcosm.build/us_runtime/graph_implementation.py", + "microcosm.build/us_runtime/graph_sources.py", + "microcosm.build/us_runtime/operator_column_contracts.py", + "microcosm.build/us_runtime/puf_support.py", + "microcosm.build/us_runtime/spine_assembly.py", + "microcosm.build/us_runtime/stacked_spine.py", + "microcosm.build/us_runtime/support_provenance.py", + "microcosm.frame/__init__.py", + "microcosm.frame/accounting.py", + "microcosm.frame/adapters/__init__.py", + "microcosm.frame/adapters/_policyengine_us_source_index.py", + "microcosm.frame/adapters/axiom.py", + "microcosm.frame/adapters/policyengine_uk.py", + "microcosm.frame/adapters/policyengine_us.py", + "microcosm.frame/bundle.py", + "microcosm.frame/kernels.py", + "microcosm.frame/materialize.py", + "microcosm.frame/rules.py", + "microcosm.frame/schema.py", + "microcosm.frame/units.py", + "microcosm.frame/weights.py", + "microcosm.graph/__init__.py", + "microcosm.graph/artifact_edges.py", + "microcosm.graph/availability.py", + "microcosm.graph/canonical.py", + "microcosm.graph/codecs.py", + "microcosm.graph/decl.py", + "microcosm.graph/errors.py", + "microcosm.graph/executor.py", + "microcosm.graph/explain.py", + "microcosm.graph/kernel.py", + "microcosm.graph/keys.py", + "microcosm.graph/manifest.py", + "microcosm.graph/population.py", + "microcosm.graph/randomness.py", + "microcosm.graph/serialize.py", + "microcosm.graph/store.py", + "microcosm.graph/view.py" + ], + "resources": [], + "source_boundary_projection": false + }, + "assembly_prepare": { + "dependencies": [ + "numpy", + "pandas", + "microunit", + "tables", + "h5py", + "PyYAML", + "pyarrow" + ], + "modules": [ + "microcosm.build/frame_checkpoint.py", + "microcosm.build/frame_sampling.py", + "microcosm.build/gates.py", + "microcosm.build/outer_stage_runtime.py", + "microcosm.build/serialization_dtypes.py", + "microcosm.build/table_identity.py", + "microcosm.build/us_runtime/acs_inputs.py", + "microcosm.build/us_runtime/acs_pums.py", + "microcosm.build/us_runtime/asec_checkpoint.py", + "microcosm.build/us_runtime/education_assistance_source.py", + "microcosm.build/us_runtime/graph_context.py", + "microcosm.build/us_runtime/graph_implementation.py", + "microcosm.build/us_runtime/graph_sources.py", + "microcosm.build/us_runtime/operator_boundary.py", + "microcosm.build/us_runtime/operator_column_contracts.py", + "microcosm.build/us_runtime/puf_support.py", + "microcosm.build/us_runtime/reported_coverage_source.py", + "microcosm.build/us_runtime/spine_assembly.py", + "microcosm.build/us_runtime/stacked_spine.py", + "microcosm.build/us_runtime/support_provenance.py", + "microcosm.frame/__init__.py", + "microcosm.frame/accounting.py", + "microcosm.frame/adapters/__init__.py", + "microcosm.frame/adapters/_policyengine_us_source_index.py", + "microcosm.frame/adapters/axiom.py", + "microcosm.frame/adapters/policyengine_uk.py", + "microcosm.frame/adapters/policyengine_us.py", + "microcosm.frame/bundle.py", + "microcosm.frame/kernels.py", + "microcosm.frame/materialize.py", + "microcosm.frame/rules.py", + "microcosm.frame/schema.py", + "microcosm.frame/units.py", + "microcosm.frame/weights.py", + "microcosm.graph/__init__.py", + "microcosm.graph/artifact_edges.py", + "microcosm.graph/availability.py", + "microcosm.graph/canonical.py", + "microcosm.graph/codecs.py", + "microcosm.graph/decl.py", + "microcosm.graph/errors.py", + "microcosm.graph/executor.py", + "microcosm.graph/explain.py", + "microcosm.graph/kernel.py", + "microcosm.graph/keys.py", + "microcosm.graph/manifest.py", + "microcosm.graph/population.py", + "microcosm.graph/randomness.py", + "microcosm.graph/serialize.py", + "microcosm.graph/store.py", + "microcosm.graph/view.py", + "microunit/__init__.py", + "microunit/core.py", + "microunit/diagnostics.py", + "microunit/registry.py", + "microunit/rule_helpers.py", + "microunit/tax_unit_construction.py", + "microunit/units/__init__.py", + "microunit/units/_helpers.py", + "microunit/units/medicaid.py", + "microunit/units/passthrough.py", + "microunit/units/programs.py", + "microunit/units/snap.py", + "microunit/units/spm.py", + "microunit/units/tax.py" + ], + "resources": [ + "microunit/data/dependent_gross_income_limit.yaml" + ], + "source_boundary_projection": true + }, + "authenticated_survey_population_v1": { + "dependencies": [ + "numpy", + "pandas", + "microunit", + "tables", + "h5py", + "PyYAML", + "pyarrow" + ], + "modules": [ + "microcosm.build/cd_benchmark/canonical.py", + "microcosm.build/cd_benchmark/origin.py", + "microcosm.build/frame_checkpoint.py", + "microcosm.build/outer_stage_runtime.py", + "microcosm.build/serialization_dtypes.py", + "microcosm.build/survey_allocation.py", + "microcosm.build/survey_domain_sample.py", + "microcosm.build/us_runtime/_asec_current_money_codec.py", + "microcosm.build/us_runtime/acs_housing_universe.py", + "microcosm.build/us_runtime/acs_housing_universe_source.py", + "microcosm.build/us_runtime/acs_inputs.py", + "microcosm.build/us_runtime/acs_native_coverage_binding.py", + "microcosm.build/us_runtime/acs_person_coverage_authentication.py", + "microcosm.build/us_runtime/acs_person_coverage_columns.py", + "microcosm.build/us_runtime/acs_population_catalogue.py", + "microcosm.build/us_runtime/acs_pums.py", + "microcosm.build/us_runtime/acs_sources.py", + "microcosm.build/us_runtime/asec_2024_native_population.py", + "microcosm.build/us_runtime/asec_checkpoint.py", + "microcosm.build/us_runtime/asec_coverage_authentication.py", + "microcosm.build/us_runtime/asec_current_money.py", + "microcosm.build/us_runtime/asec_current_money_resources.py", + "microcosm.build/us_runtime/asec_current_money_source.py", + "microcosm.build/us_runtime/asec_household_coverage_fields.py", + "microcosm.build/us_runtime/asec_household_observations.py", + "microcosm.build/us_runtime/asec_original_household_weights.py", + "microcosm.build/us_runtime/asec_person_coverage_source.py", + "microcosm.build/us_runtime/asec_person_income_source.py", + "microcosm.build/us_runtime/asec_population_catalogue.py", + "microcosm.build/us_runtime/asec_student_controls.py", + "microcosm.build/us_runtime/education_assistance_source.py", + "microcosm.build/us_runtime/graph_acs_housing_universe.py", + "microcosm.build/us_runtime/graph_context.py", + "microcosm.build/us_runtime/graph_implementation.py", + "microcosm.build/us_runtime/graph_sources.py", + "microcosm.build/us_runtime/graph_survey_population.py", + "microcosm.build/us_runtime/native_household_origin.py", + "microcosm.build/us_runtime/operator_boundary.py", + "microcosm.build/us_runtime/operator_column_contracts.py", + "microcosm.build/us_runtime/reported_coverage_source.py", + "microcosm.build/us_runtime/source_csv_builtin.py", + "microcosm.build/us_runtime/spine_assembly.py", + "microcosm.build/us_runtime/support_provenance.py", + "microcosm.build/us_runtime/survey_catalogue_selection.py", + "microcosm.build/us_runtime/survey_observed_age.py", + "microcosm.build/us_runtime/survey_population_domains.py", + "microcosm.build/us_runtime/survey_population_preparation.py", + "microcosm.frame/__init__.py", + "microcosm.frame/accounting.py", + "microcosm.frame/adapters/__init__.py", + "microcosm.frame/adapters/_policyengine_us_source_index.py", + "microcosm.frame/adapters/axiom.py", + "microcosm.frame/adapters/policyengine_uk.py", + "microcosm.frame/adapters/policyengine_us.py", + "microcosm.frame/bundle.py", + "microcosm.frame/kernels.py", + "microcosm.frame/materialize.py", + "microcosm.frame/rules.py", + "microcosm.frame/schema.py", + "microcosm.frame/units.py", + "microcosm.frame/weights.py", + "microcosm.graph/__init__.py", + "microcosm.graph/artifact_edges.py", + "microcosm.graph/availability.py", + "microcosm.graph/canonical.py", + "microcosm.graph/codecs.py", + "microcosm.graph/decl.py", + "microcosm.graph/errors.py", + "microcosm.graph/executor.py", + "microcosm.graph/explain.py", + "microcosm.graph/kernel.py", + "microcosm.graph/keys.py", + "microcosm.graph/manifest.py", + "microcosm.graph/population.py", + "microcosm.graph/randomness.py", + "microcosm.graph/serialize.py", + "microcosm.graph/store.py", + "microcosm.graph/view.py", + "microunit/__init__.py", + "microunit/core.py", + "microunit/diagnostics.py", + "microunit/registry.py", + "microunit/rule_helpers.py", + "microunit/tax_unit_construction.py", + "microunit/units/__init__.py", + "microunit/units/_helpers.py", + "microunit/units/medicaid.py", + "microunit/units/passthrough.py", + "microunit/units/programs.py", + "microunit/units/snap.py", + "microunit/units/spm.py", + "microunit/units/tax.py" + ], + "resources": [ + "microcosm.build/us_runtime/acs_2024_1yr_sources.json", + "microcosm.build/us_runtime/acs_2024_housing_universe.json", + "microcosm.build/us_runtime/asec_current_money_consumers_v1.json", + "microcosm.build/us_runtime/asec_current_money_domains_v1.json", + "microcosm.build/us_runtime/asec_current_money_price_basis_v1.json", + "microunit/data/dependent_gross_income_limit.yaml" + ], + "source_boundary_projection": true + }, + "composed_asec_binding_v1": { + "dependencies": [ + "numpy", + "pandas", + "microunit", + "tables", + "h5py", + "PyYAML", + "pyarrow" + ], + "modules": [ + "microcosm.build/frame_checkpoint.py", + "microcosm.build/frame_sampling.py", + "microcosm.build/gates.py", + "microcosm.build/outer_stage_runtime.py", + "microcosm.build/serialization_dtypes.py", + "microcosm.build/table_identity.py", + "microcosm.build/us_runtime/_asec_current_money_codec.py", + "microcosm.build/us_runtime/acs_inputs.py", + "microcosm.build/us_runtime/acs_pums.py", + "microcosm.build/us_runtime/alimony.py", + "microcosm.build/us_runtime/asec_checkpoint.py", + "microcosm.build/us_runtime/asec_current_money.py", + "microcosm.build/us_runtime/asec_current_money_graph_resources.py", + "microcosm.build/us_runtime/asec_current_money_resources.py", + "microcosm.build/us_runtime/asec_current_money_selection.py", + "microcosm.build/us_runtime/asec_current_money_source.py", + "microcosm.build/us_runtime/asec_current_money_units.py", + "microcosm.build/us_runtime/asec_engine_evaluation.py", + "microcosm.build/us_runtime/asec_household_observations.py", + "microcosm.build/us_runtime/asec_housing_status.py", + "microcosm.build/us_runtime/asec_housing_status_source.py", + "microcosm.build/us_runtime/asec_housing_universe.py", + "microcosm.build/us_runtime/asec_housing_universe_source.py", + "microcosm.build/us_runtime/asec_income_observations.py", + "microcosm.build/us_runtime/asec_person_income_source.py", + "microcosm.build/us_runtime/asec_prepared_source.py", + "microcosm.build/us_runtime/asec_student_controls.py", + "microcosm.build/us_runtime/congressional_district_geography.py", + "microcosm.build/us_runtime/congressional_district_vintage.py", + "microcosm.build/us_runtime/cps_carried.py", + "microcosm.build/us_runtime/cps_carried_current.py", + "microcosm.build/us_runtime/education_assistance_source.py", + "microcosm.build/us_runtime/geography_ladder.py", + "microcosm.build/us_runtime/graph_asec_income.py", + "microcosm.build/us_runtime/graph_asec_prepared.py", + "microcosm.build/us_runtime/graph_composed_asec_binding.py", + "microcosm.build/us_runtime/graph_composed_asec_measures.py", + "microcosm.build/us_runtime/graph_composed_contracts.py", + "microcosm.build/us_runtime/graph_composed_population.py", + "microcosm.build/us_runtime/graph_context.py", + "microcosm.build/us_runtime/graph_geography.py", + "microcosm.build/us_runtime/graph_housing_universe.py", + "microcosm.build/us_runtime/graph_implementation.py", + "microcosm.build/us_runtime/graph_sources.py", + "microcosm.build/us_runtime/operator_boundary.py", + "microcosm.build/us_runtime/operator_column_contracts.py", + "microcosm.build/us_runtime/puf_support.py", + "microcosm.build/us_runtime/puma_ladder.py", + "microcosm.build/us_runtime/puma_ladder_sources.py", + "microcosm.build/us_runtime/reported_coverage_source.py", + "microcosm.build/us_runtime/spine_assembly.py", + "microcosm.build/us_runtime/stacked_spine.py", + "microcosm.build/us_runtime/support_provenance.py", + "microcosm.frame/__init__.py", + "microcosm.frame/accounting.py", + "microcosm.frame/adapters/__init__.py", + "microcosm.frame/adapters/_policyengine_us_source_index.py", + "microcosm.frame/adapters/axiom.py", + "microcosm.frame/adapters/policyengine_uk.py", + "microcosm.frame/adapters/policyengine_us.py", + "microcosm.frame/bundle.py", + "microcosm.frame/kernels.py", + "microcosm.frame/materialize.py", + "microcosm.frame/rules.py", + "microcosm.frame/schema.py", + "microcosm.frame/units.py", + "microcosm.frame/weights.py", + "microcosm.graph/__init__.py", + "microcosm.graph/artifact_edges.py", + "microcosm.graph/availability.py", + "microcosm.graph/canonical.py", + "microcosm.graph/codecs.py", + "microcosm.graph/decl.py", + "microcosm.graph/errors.py", + "microcosm.graph/executor.py", + "microcosm.graph/explain.py", + "microcosm.graph/kernel.py", + "microcosm.graph/keys.py", + "microcosm.graph/manifest.py", + "microcosm.graph/population.py", + "microcosm.graph/randomness.py", + "microcosm.graph/serialize.py", + "microcosm.graph/store.py", + "microcosm.graph/view.py", + "microunit/__init__.py", + "microunit/core.py", + "microunit/diagnostics.py", + "microunit/registry.py", + "microunit/rule_helpers.py", + "microunit/tax_unit_construction.py", + "microunit/units/__init__.py", + "microunit/units/_helpers.py", + "microunit/units/medicaid.py", + "microunit/units/passthrough.py", + "microunit/units/programs.py", + "microunit/units/snap.py", + "microunit/units/spm.py", + "microunit/units/tax.py" + ], + "resources": [ + "microcosm.build/us_runtime/asec_current_money_consumers_v1.json", + "microcosm.build/us_runtime/asec_current_money_domains_v1.json", + "microcosm.build/us_runtime/asec_current_money_engine_defaults_v1.json", + "microcosm.build/us_runtime/asec_current_money_graph_consumers_v1.json", + "microcosm.build/us_runtime/asec_current_money_price_basis_v1.json", + "microcosm.build/us_runtime/asec_income_observations_v1.json", + "microunit/data/dependent_gross_income_limit.yaml" + ], + "source_boundary_projection": true + }, + "composed_population_v1": { + "dependencies": [ + "numpy", + "pandas", + "microunit", + "tables", + "h5py", + "PyYAML", + "pyarrow" + ], + "modules": [ + "microcosm.build/frame_checkpoint.py", + "microcosm.build/frame_sampling.py", + "microcosm.build/gates.py", + "microcosm.build/outer_stage_runtime.py", + "microcosm.build/serialization_dtypes.py", + "microcosm.build/table_identity.py", + "microcosm.build/us_runtime/_asec_current_money_codec.py", + "microcosm.build/us_runtime/acs_inputs.py", + "microcosm.build/us_runtime/acs_pums.py", + "microcosm.build/us_runtime/alimony.py", + "microcosm.build/us_runtime/asec_checkpoint.py", + "microcosm.build/us_runtime/asec_current_money.py", + "microcosm.build/us_runtime/asec_current_money_graph_resources.py", + "microcosm.build/us_runtime/asec_current_money_resources.py", + "microcosm.build/us_runtime/asec_current_money_selection.py", + "microcosm.build/us_runtime/asec_current_money_source.py", + "microcosm.build/us_runtime/asec_current_money_units.py", + "microcosm.build/us_runtime/asec_engine_evaluation.py", + "microcosm.build/us_runtime/asec_household_observations.py", + "microcosm.build/us_runtime/asec_housing_status.py", + "microcosm.build/us_runtime/asec_housing_status_source.py", + "microcosm.build/us_runtime/asec_housing_universe.py", + "microcosm.build/us_runtime/asec_housing_universe_source.py", + "microcosm.build/us_runtime/asec_income_observations.py", + "microcosm.build/us_runtime/asec_person_income_source.py", + "microcosm.build/us_runtime/asec_prepared_source.py", + "microcosm.build/us_runtime/asec_student_controls.py", + "microcosm.build/us_runtime/congressional_district_geography.py", + "microcosm.build/us_runtime/congressional_district_vintage.py", + "microcosm.build/us_runtime/cps_carried.py", + "microcosm.build/us_runtime/cps_carried_current.py", + "microcosm.build/us_runtime/education_assistance_source.py", + "microcosm.build/us_runtime/geography_ladder.py", + "microcosm.build/us_runtime/graph_asec_income.py", + "microcosm.build/us_runtime/graph_asec_prepared.py", + "microcosm.build/us_runtime/graph_composed_contracts.py", + "microcosm.build/us_runtime/graph_composed_population.py", + "microcosm.build/us_runtime/graph_context.py", + "microcosm.build/us_runtime/graph_geography.py", + "microcosm.build/us_runtime/graph_housing_universe.py", + "microcosm.build/us_runtime/graph_implementation.py", + "microcosm.build/us_runtime/graph_sources.py", + "microcosm.build/us_runtime/operator_boundary.py", + "microcosm.build/us_runtime/operator_column_contracts.py", + "microcosm.build/us_runtime/puf_support.py", + "microcosm.build/us_runtime/puma_ladder.py", + "microcosm.build/us_runtime/puma_ladder_sources.py", + "microcosm.build/us_runtime/reported_coverage_source.py", + "microcosm.build/us_runtime/spine_assembly.py", + "microcosm.build/us_runtime/stacked_spine.py", + "microcosm.build/us_runtime/support_provenance.py", + "microcosm.frame/__init__.py", + "microcosm.frame/accounting.py", + "microcosm.frame/adapters/__init__.py", + "microcosm.frame/adapters/_policyengine_us_source_index.py", + "microcosm.frame/adapters/axiom.py", + "microcosm.frame/adapters/policyengine_uk.py", + "microcosm.frame/adapters/policyengine_us.py", + "microcosm.frame/bundle.py", + "microcosm.frame/kernels.py", + "microcosm.frame/materialize.py", + "microcosm.frame/rules.py", + "microcosm.frame/schema.py", + "microcosm.frame/units.py", + "microcosm.frame/weights.py", + "microcosm.graph/__init__.py", + "microcosm.graph/artifact_edges.py", + "microcosm.graph/availability.py", + "microcosm.graph/canonical.py", + "microcosm.graph/codecs.py", + "microcosm.graph/decl.py", + "microcosm.graph/errors.py", + "microcosm.graph/executor.py", + "microcosm.graph/explain.py", + "microcosm.graph/kernel.py", + "microcosm.graph/keys.py", + "microcosm.graph/manifest.py", + "microcosm.graph/population.py", + "microcosm.graph/randomness.py", + "microcosm.graph/serialize.py", + "microcosm.graph/store.py", + "microcosm.graph/view.py", + "microunit/__init__.py", + "microunit/core.py", + "microunit/diagnostics.py", + "microunit/registry.py", + "microunit/rule_helpers.py", + "microunit/tax_unit_construction.py", + "microunit/units/__init__.py", + "microunit/units/_helpers.py", + "microunit/units/medicaid.py", + "microunit/units/passthrough.py", + "microunit/units/programs.py", + "microunit/units/snap.py", + "microunit/units/spm.py", + "microunit/units/tax.py" + ], + "resources": [ + "microcosm.build/us_runtime/asec_current_money_consumers_v1.json", + "microcosm.build/us_runtime/asec_current_money_domains_v1.json", + "microcosm.build/us_runtime/asec_current_money_engine_defaults_v1.json", + "microcosm.build/us_runtime/asec_current_money_graph_consumers_v1.json", + "microcosm.build/us_runtime/asec_current_money_price_basis_v1.json", + "microcosm.build/us_runtime/asec_income_observations_v1.json", + "microunit/data/dependent_gross_income_limit.yaml" + ], + "source_boundary_projection": true + }, + "geography": { + "dependencies": [ + "numpy", + "pandas" + ], + "modules": [ + "microcosm.build/gates.py", + "microcosm.build/serialization_dtypes.py", + "microcosm.build/us_runtime/congressional_district_geography.py", + "microcosm.build/us_runtime/congressional_district_vintage.py", + "microcosm.build/us_runtime/geography_ladder.py", + "microcosm.build/us_runtime/graph_context.py", + "microcosm.build/us_runtime/graph_geography.py", + "microcosm.build/us_runtime/graph_implementation.py", + "microcosm.build/us_runtime/puma_ladder.py", + "microcosm.build/us_runtime/puma_ladder_sources.py", + "microcosm.frame/__init__.py", + "microcosm.frame/accounting.py", + "microcosm.frame/adapters/__init__.py", + "microcosm.frame/adapters/_policyengine_us_source_index.py", + "microcosm.frame/adapters/axiom.py", + "microcosm.frame/adapters/policyengine_uk.py", + "microcosm.frame/adapters/policyengine_us.py", + "microcosm.frame/bundle.py", + "microcosm.frame/kernels.py", + "microcosm.frame/materialize.py", + "microcosm.frame/rules.py", + "microcosm.frame/schema.py", + "microcosm.frame/units.py", + "microcosm.frame/weights.py", + "microcosm.graph/__init__.py", + "microcosm.graph/artifact_edges.py", + "microcosm.graph/availability.py", + "microcosm.graph/canonical.py", + "microcosm.graph/codecs.py", + "microcosm.graph/decl.py", + "microcosm.graph/errors.py", + "microcosm.graph/executor.py", + "microcosm.graph/explain.py", + "microcosm.graph/kernel.py", + "microcosm.graph/keys.py", + "microcosm.graph/manifest.py", + "microcosm.graph/population.py", + "microcosm.graph/randomness.py", + "microcosm.graph/serialize.py", + "microcosm.graph/store.py", + "microcosm.graph/view.py" + ], + "resources": [], + "source_boundary_projection": false + } + } +} diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/graph_national_age_counts.py b/packages/microcosm-build/src/microcosm/build/us_runtime/graph_national_age_counts.py new file mode 100644 index 000000000..c30472fa5 --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/graph_national_age_counts.py @@ -0,0 +1,451 @@ +"""Disjoint household age counts, and the composition that calibrates them. + +This is an ordinary source-blind population operator. It declares exactly what +it reads — each person's age and household membership, and the household ids — +and owns one int64 count column per declared age band. It opens no file, reads +no source-channel or spine column, imports no rules engine, calculates nothing +from a model, and never touches weights: these are unweighted head counts, and +the graph executor owns every weight in the build. + +The graph contract splits that read surface in two, and the declaration follows +it rather than working around it. ``age`` is an owned cell, so it travels in an +input :class:`~microcosm.graph.Slice`. Entity ids and person-to-group +memberships are structural: the executor puts them in every context table and +no node owns them, so a ``Slice`` cannot name them. They are declared instead +in the node's normative parameters, where they still enter the node key, and +the kernel refuses a context whose household view carries anything beyond the +declared id column. + +Every included person is counted exactly once. Group-quarters people are +included because the activated target universe is the whole resident +population, and their household membership is preserved rather than dropped. +The bands are declared, contiguous from zero, disjoint, and closed by a single +open top band, so an age outside every band is impossible by construction and a +person can never fall into two. + +Refusals are explicit; nothing is silently dropped. An unknown, non-integer, +negative or non-finite age refuses the node, as does a membership that names no +household row, a duplicated household id, or a household row that no person +belongs to. The last one matches the population contract itself, which requires +each group row to be referenced by at least one person — a memberless household +is a broken population, not a zero-count household. + +:func:`national_age_calibration_nodes` composes this operator with the existing +`demographic_calibration_node`, unchanged. It emits no US frame-context +artifact: a context written before reweighting must not be relabelled after it, +so consumers read the executor-owned calibrated population and the diagnostics +edge instead. +""" + +from __future__ import annotations + +import json +import sys + +import numpy as np +import pandas as pd +from pandas.api.types import ( + is_bool_dtype, + is_complex_dtype, + is_integer_dtype, + is_numeric_dtype, +) + +from microcosm.frame import US_SCHEMA +from microcosm.graph import ( + Capabilities, + Determinism, + KernelBase, + KernelContext, + KernelResult, + Node, + Numeric, + Owned, + SeedSource, + Slice, + StructuralDelta, + source_hash, +) + +from .demographic_calibration_graph import demographic_calibration_node +from .national_age_activation import NATIONAL_AGE_ACTIVATION, AgeBand + +__all__ = [ + "AGE_COUNT_SCHEMA_VERSION", + "SUPPORTED_AGE_CONVENTION", + "NationalAgeCountKernel", + "national_age_count_node", + "national_age_calibration_nodes", +] + +#: Revision of the band-schema parameter's canonical form. +AGE_COUNT_SCHEMA_VERSION = 1 + +#: The one convention this operator implements: the person's ``age`` column is +#: read verbatim as completed years observed at that person's own source +#: interview. No birth date is consulted, no one is aged forward, and no claim +#: is made that the person was observed in the target period. +SUPPORTED_AGE_CONVENTION = "observed_interview_age_completed_years" +if SUPPORTED_AGE_CONVENTION != NATIONAL_AGE_ACTIVATION.age_convention_id: + raise ValueError("The age-count operator and target activation must agree on age.") + +# Operational input envelope for this development connection, not an assertion +# that every source age within it is observed or valid. Outliers refuse rather +# than silently entering the open top band. +MAX_SUPPORTED_AGE = 120 + +_AGE_COLUMN = "age" +_PERSON = US_SCHEMA.person_entity +_HOUSEHOLD = "household" +_HOUSEHOLD_ID = US_SCHEMA.entity_id_column(_HOUSEHOLD) +_PERSON_HOUSEHOLD_ID = US_SCHEMA.membership_column(_HOUSEHOLD) + +#: Every person column the kernel reads: the owned age cell plus the structural +#: household membership. Declared normatively so the read surface is in the key. +_PERSON_COLUMNS = (_AGE_COLUMN, _PERSON_HOUSEHOLD_ID) +#: Every household column the kernel reads: the structural id, and nothing else. +_HOUSEHOLD_COLUMNS = (_HOUSEHOLD_ID,) + +_PARAMS = frozenset( + { + "bands", + "person_columns", + "household_columns", + "age_convention", + "schema_version", + "maximum_age", + } +) + + +def _canonical(value: object) -> str: + return json.dumps(value, sort_keys=True, separators=(",", ":"), allow_nan=False) + + +def _bands_param(bands: tuple[AgeBand, ...]) -> str: + return _canonical([[band.column, band.low, band.high] for band in bands]) + + +def _bands_from_param(text: object) -> tuple[tuple[str, int, int | None], ...]: + """Parse and fully re-validate the declared band schema. + + The kernel trusts nothing it did not check here: the schema must be the + canonical form of a contiguous, disjoint, zero-based partition closed by + one open top band, with distinct column names. + """ + + if not isinstance(text, str) or len(text.encode()) > 65_536: + raise ValueError("A bounded canonical age-band declaration is required.") + rows = json.loads(text) + if not isinstance(rows, list) or not rows: + raise ValueError("The age-band declaration must be a nonempty list.") + bands: list[tuple[str, int, int | None]] = [] + expected_low = 0 + for index, row in enumerate(rows): + if ( + not isinstance(row, list) + or len(row) != 3 + or not isinstance(row[0], str) + or not row[0] + or type(row[1]) is not int + ): + raise ValueError("Each age band declares a column, a low and a high.") + column, low, high = row + last = index == len(rows) - 1 + if (high is None) is not last: + raise ValueError("Exactly the final age band is an open interval.") + if high is not None and (type(high) is not int or high < low): + raise ValueError("A closed age band needs an integer high at or above low.") + if low != expected_low: + raise ValueError("Age bands must be contiguous and start at zero.") + expected_low = None if high is None else high + 1 + bands.append((column, low, high)) + if len({column for column, _, _ in bands}) != len(bands): + raise ValueError("Age bands must own distinct count columns.") + if _canonical([[c, lo, hi] for c, lo, hi in bands]) != text: + raise ValueError("The age-band declaration must be in canonical form.") + return tuple(bands) + + +def _node_from_schema(schema: str, *, population: str, node_id: str) -> Node: + """The single declaration site, shared by the factory and the kernel.""" + + bands = _bands_from_param(schema) + return Node( + id=node_id, + kernel=NationalAgeCountKernel.ref, + population=population, + inputs=(Slice(_PERSON, (_AGE_COLUMN,)),), + outputs=tuple(Owned(_HOUSEHOLD, column, "int64") for column, _, _ in bands), + params={ + "bands": schema, + "person_columns": _PERSON_COLUMNS, + "household_columns": _HOUSEHOLD_COLUMNS, + "age_convention": SUPPORTED_AGE_CONVENTION, + "schema_version": AGE_COUNT_SCHEMA_VERSION, + "maximum_age": MAX_SUPPORTED_AGE, + }, + ) + + +def national_age_count_node( + *, + population: str, + bands: tuple[AgeBand, ...] = NATIONAL_AGE_ACTIVATION.bands, + node_id: str = "national.age_counts", +) -> Node: + """Declare the household age-count operator over one population version. + + Args: + population: The population version whose rows the counts live in. + bands: The activated bands, in order. Their canonical form enters the + node key, so an altered schema is a different node. + node_id: The node's id. + + Returns: + The node owning one int64 count column per band. + """ + + return _node_from_schema( + _bands_param(bands), population=population, node_id=node_id + ) + + +class NationalAgeCountKernel(KernelBase): + """Count each person once into their household's declared age band.""" + + ref = "us.national_age_counts@1" + capabilities = Capabilities( + determinism=Determinism.DETERMINISTIC, + numeric=Numeric.BITWISE, + seed_source=SeedSource.NONE, + structural=StructuralDelta.NONE, + consumes_se=False, + dependencies=("numpy", "pandas"), + ) + + def implementation_hash(self) -> str: + return source_hash( + sys.modules[__name__], dependencies=self.capabilities.dependencies + ) + + def run(self, context: KernelContext) -> KernelResult: + params = context.params + if set(params) != _PARAMS: + raise ValueError("The age-count node declares exactly its band schema.") + if params["age_convention"] != SUPPORTED_AGE_CONVENTION: + raise ValueError( + "This operator implements only " + f"{SUPPORTED_AGE_CONVENTION!r}; it cannot honour " + f"{params['age_convention']!r}." + ) + if params["schema_version"] != AGE_COUNT_SCHEMA_VERSION: + raise ValueError("Unsupported age-band schema version.") + if ( + params["person_columns"] != _PERSON_COLUMNS + or params["household_columns"] != _HOUSEHOLD_COLUMNS + ): + raise ValueError( + "This operator reads exactly " + f"{_PERSON_COLUMNS} on {_PERSON!r} and {_HOUSEHOLD_COLUMNS} on " + f"{_HOUSEHOLD!r}." + ) + bands = _bands_from_param(params["bands"]) + expected = _node_from_schema( + params["bands"], + population=context.node.population or "", + node_id=context.node.id, + ) + if context.node.normative() != expected.normative(): + raise ValueError("Age-count node differs from its complete declaration.") + + household = context.tables[_HOUSEHOLD] + person = context.tables[_PERSON] + # The household view is exactly its id column: no weight, no source + # channel, no other attribute is visible to this operator. + if tuple(household.columns) != _HOUSEHOLD_COLUMNS: + raise ValueError( + f"The household view must carry exactly {_HOUSEHOLD_COLUMNS}, " + f"got {tuple(household.columns)}." + ) + missing = [column for column in _PERSON_COLUMNS if column not in person] + if missing: + raise ValueError(f"The person view is missing {missing}.") + household_ids = _ids(household, _HOUSEHOLD_ID, what="household id") + index = pd.Index(household_ids, name=_HOUSEHOLD_ID) + if index.has_duplicates: + raise ValueError("Household ids must be unique.") + # Align by id, never by row order: neither axis is assumed sequential, + # sorted, or shared between the two tables. + positions = index.get_indexer( + _ids(person, _PERSON_HOUSEHOLD_ID, what="household membership") + ) + if np.any(positions < 0): + missing = np.unique(person[_PERSON_HOUSEHOLD_ID].to_numpy()[positions < 0])[ + :5 + ] + raise ValueError( + "Every person's household membership must resolve to a household " + f"row; unresolved ids include {missing.tolist()}." + ) + + age = _ages(person) + band_of_person = np.full(len(person), -1, dtype=np.int64) + for ordinal, (_, low, high) in enumerate(bands): + inside = age >= low if high is None else (age >= low) & (age <= high) + if np.any(band_of_person[inside] >= 0): + raise ValueError("Declared age bands overlap.") + band_of_person[inside] = ordinal + if np.any(band_of_person < 0): + raise ValueError("Declared age bands do not cover every observed age.") + + counts = np.bincount( + band_of_person * len(index) + positions, + minlength=len(bands) * len(index), + ).astype(np.int64, copy=False) + counts = counts.reshape(len(bands), len(index)) + if int(counts.sum()) != len(person): + raise ValueError("Every person must be counted into exactly one band.") + people = counts.sum(axis=0) + if np.any(people == 0): + # The population contract requires every group row to be referenced + # by a person, so this is a broken population, not an empty house. + raise ValueError( + "Every household row must contain at least one person; " + f"{int((people == 0).sum())} household(s) contain none." + ) + return KernelResult( + columns={ + (_HOUSEHOLD, column): pd.Series( + counts[ordinal], index=index, dtype="int64" + ) + for ordinal, (column, _, _) in enumerate(bands) + }, + receipt={ + "scope": "national_age_counts_development", + "age_convention": SUPPORTED_AGE_CONVENTION, + "bands": params["bands"], + "people_counted": int(counts.sum()), + "households": int(len(index)), + "maximum_observed_age": int(age.max()) if len(age) else None, + "minimum_observed_age": int(age.min()) if len(age) else None, + "maximum_supported_age": MAX_SUPPORTED_AGE, + "consumes_weights": False, + "universe": "every person on the population, group quarters included", + "release_eligible": False, + }, + ) + + +def _ids(table: pd.DataFrame, column: str, *, what: str) -> np.ndarray: + series = table[column] + if ( + is_bool_dtype(series.dtype) + or is_complex_dtype(series.dtype) + or not is_numeric_dtype(series.dtype) + ): + raise ValueError(f"The {what} column must be an integer column.") + if series.isna().any(): + raise ValueError(f"The {what} column contains missing values.") + values = series.to_numpy(dtype="float64", na_value=np.nan) + if not np.all(np.isfinite(values) & (values == np.floor(values))): + raise ValueError(f"The {what} column must hold whole numbers.") + # Preserve exact integer ids without a float round trip, and refuse casts + # that would wrap or saturate into another household's identity. The open + # float upper bound matters: float64 cannot represent int64's maximum. + if is_integer_dtype(series.dtype): + limits = np.iinfo(np.int64) + outside = ((series < limits.min) | (series > limits.max)).any() + else: + outside = np.any(values < -(2**63)) or np.any(values >= 2**63) + if outside: + raise ValueError(f"The {what} column must fit signed int64 exactly.") + return series.to_numpy(dtype="int64") + + +def _ages(person: pd.DataFrame) -> np.ndarray: + """Ages as nonnegative whole years; anything else refuses the node.""" + + series = person[_AGE_COLUMN] + if ( + is_bool_dtype(series.dtype) + or is_complex_dtype(series.dtype) + or not is_numeric_dtype(series.dtype) + ): + raise ValueError("Person age must be a real numeric column.") + if series.isna().any(): + raise ValueError( + f"{int(series.isna().sum())} person age(s) are unknown; an age-count " + "node refuses rather than dropping people." + ) + values = series.to_numpy(dtype="float64", na_value=np.nan) + if not np.all(np.isfinite(values)): + raise ValueError("Person age must be finite.") + if np.any(values < 0): + raise ValueError("Person age must not be negative.") + if np.any(values > MAX_SUPPORTED_AGE): + raise ValueError( + f"Person age exceeds the declared maximum of {MAX_SUPPORTED_AGE}." + ) + if not np.all(values == np.floor(values)): + raise ValueError("Person age must be a whole number of completed years.") + return values + + +def national_age_calibration_nodes( + *, + base: str, + registry, + epochs: int, + learning_rate: float, + max_weight_ratio: float, + max_initial_weight_ratio: float, + bands: tuple[AgeBand, ...] = NATIONAL_AGE_ACTIVATION.bands, + count_node_id: str = "national.age_counts", + calibration_node_id: str = "national.demographic_calibration", +) -> tuple[Node, Node]: + """Compose the age-count operator with the existing calibration node. + + The count node lives in ``base``'s population version and owns the columns; + the unchanged `demographic_calibration_node` reweights over ``base`` and + reads exactly those columns, so the count node is its predecessor. + + Args: + base: The population version carrying household importance weights. + registry: The activated :class:`TargetRegistry`. Its measures must be + exactly the declared band columns, in order — a registry and a band + schema that disagree are refused here rather than half-wired. + epochs: Solver epochs. + learning_rate: Solver learning rate. + max_weight_ratio: The executor's cap against original design weights. + max_initial_weight_ratio: The solver's cap against incoming weights. + bands: The activated bands, in order. + count_node_id: Id of the count node. + calibration_node_id: Id of the calibration node. + + Returns: + ``(count_node, calibration_node)`` in dependency order. + + Raises: + ValueError: If the registry's measures are not the band columns. + """ + + measures = tuple(spec.measure for spec in registry) + columns = tuple(band.column for band in bands) + if measures != columns: + raise ValueError( + "The registry's measures must be exactly the declared band columns " + f"in order; got {measures} against {columns}." + ) + return ( + national_age_count_node(population=base, bands=bands, node_id=count_node_id), + demographic_calibration_node( + registry, + base=base, + node_id=calibration_node_id, + epochs=epochs, + learning_rate=learning_rate, + max_weight_ratio=max_weight_ratio, + max_initial_weight_ratio=max_initial_weight_ratio, + ), + ) diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/graph_native_household_origin.py b/packages/microcosm-build/src/microcosm/build/us_runtime/graph_native_household_origin.py new file mode 100644 index 000000000..3d52c78a7 --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/graph_native_household_origin.py @@ -0,0 +1,415 @@ +"""Graph source producers and population-grain native household lineage. + +Only source kernels open original members. Consumers preserve the executor's +typed producer edges when decoding transport; a nominal artifact type by itself +is not source authentication. These nodes issue no benchmark approval or score. +""" + +from __future__ import annotations + +import json +import tempfile +from collections.abc import Sequence +from pathlib import Path + +from microcosm.frame import US_SCHEMA, Frame +from microcosm.graph import ( + ArtifactInput, + ArtifactOutput, + ArtifactType, + Capabilities, + Determinism, + KernelBase, + KernelRegistry, + KernelResult, + Node, + Numeric, + Owned, + Slice, + SourceRef, +) +from microcosm.graph.artifact_edges import value_from_descriptor +from microcosm.graph.canonical import canonical_json +from microcosm.graph.keys import opaque_artifact_key + +from . import native_household_origin as native +from .graph_native_origin_implementation import ( + DEPENDENCIES, + implementation_hash, + implementation_manifest, +) + +SOURCE_TYPE = ArtifactType("microcosm.us.native_household_origins", 1) +BINDING_TYPE = ArtifactType("microcosm.us.population_household_origins", 1) +SOURCE_OUTPUT = (ArtifactOutput("origins", SOURCE_TYPE),) +BINDING_OUTPUT = (ArtifactOutput("binding", BINDING_TYPE),) +PREFIX = "native_household_origins" +ACS_NODE = PREFIX + ".acs" +ASEC_NODE = PREFIX + ".asec" +ACS_SOURCE = "native_origin_acs" +ASEC_SOURCES = tuple(f"native_origin_asec_{year}" for year in (2022, 2023, 2024)) +SOURCES = ( + SourceRef(ACS_SOURCE, native.acs.ACS_HU_CODEC), + *(SourceRef(name, "raw-bytes-v1") for name in ASEC_SOURCES), +) + + +class _Kernel(KernelBase): + capabilities = Capabilities( + determinism=Determinism.DETERMINISTIC, + numeric=Numeric.PLATFORM_BITWISE, + dependencies=DEPENDENCIES, + ) + + def implementation_hash(self): + return implementation_hash() + + +class NativeACSOriginKernel(_Kernel): + """Authenticate original ACS archive members through the accepted codec.""" + + ref = "us.native_household_origin.acs@1" + + def run(self, context): + native._require( + tuple(context.node.sources) == (ACS_SOURCE,) + and not context.node.inputs + and not context.node.artifact_inputs + and context.node.artifact_outputs == SOURCE_OUTPUT + and not context.params, + "ACS_ORIGIN_DECLARATION", + ) + with tempfile.TemporaryDirectory(prefix="graph-native-acs-") as tmp: + root = Path(tmp).resolve() + snapshots = root / "snapshots" + snapshots.mkdir() + source = native.produce_acs_native_origins( + context.sources[ACS_SOURCE], + snapshot_root=snapshots, + output_dir=root / "projection", + ) + return _source_result(source) + + +class NativeASECOriginKernel(_Kernel): + """Authenticate the registered original HOUSEHOLD CSV projections.""" + + ref = "us.native_household_origin.asec@1" + + def run(self, context): + native._require( + tuple(context.node.sources) == ASEC_SOURCES + and not context.node.inputs + and not context.node.artifact_inputs + and context.node.artifact_outputs == SOURCE_OUTPUT + and not context.params, + "ASEC_ORIGIN_DECLARATION", + ) + source = native.produce_asec_native_origins( + { + year: context.sources[name] + for year, name in zip((2022, 2023, 2024), ASEC_SOURCES, strict=True) + } + ) + return _source_result(source) + + +def _source_result(source): + document = source.document + return KernelResult( + artifacts={"origins": source.payload}, + receipt={ + "phase": "native_household_origin_source", + "arm": document["arm"], + "households": len(document["records"]), + "source_projection_sha256": native._sha(source.payload), + "source_receipt_sha256": document["source_receipt_sha256"], + "implementation": implementation_manifest(), + "release_eligible": False, + }, + ) + + +def _artifact(context, name, expected_type, output): + value = context.artifacts[name] + native._require( + value.type == expected_type + and value.key == opaque_artifact_key(value.producer_key, output), + "ORIGIN_ARTIFACT_EDGE", + ) + return value + + +class PopulationOriginBindingKernel(_Kernel): + """Bind carried source evidence to every current row and clone membership.""" + + ref = "us.native_household_origin.bind@1" + + def run(self, context): + node = context.node + native._require( + not node.sources + and not node.outputs + and node.artifact_outputs == BINDING_OUTPUT + and set(context.params) == {"columns", "parent_binding"}, + "ORIGIN_BIND_DECLARATION", + ) + expected = { + entity: [US_SCHEMA.entity_id_column(entity)] + for entity in US_SCHEMA.entities + } + expected[US_SCHEMA.person_entity].extend( + US_SCHEMA.membership_column(group) for group in US_SCHEMA.group_entities + ) + for entity, column, _dtype in context.params["columns"]: + expected[entity].append(column) + native._require( + set(context.tables) == set(expected) + and all( + set(context.tables[e]) == set(columns) + for e, columns in expected.items() + ), + "ORIGIN_BIND_COLUMN_INVENTORY", + ) + # Graph context supplies IDs and memberships implicitly. All remaining + # source cells must be declared by the integrating graph. The returned + # content digest must also be verified against the complete materialized + # population: this projected context cannot discover omitted columns. + tables = { + e: context.tables[e].loc[:, columns] for e, columns in expected.items() + } + frame = Frame(tables, US_SCHEMA, dict(context.weights), context.strata) + aliases = {"acs", "asec"} + if context.params["parent_binding"]: + aliases.add("parent_binding") + native._require( + set(context.artifacts) == aliases, "ORIGIN_BIND_ARTIFACT_ROSTER" + ) + values = [ + _artifact(context, arm, SOURCE_TYPE, "origins") for arm in ("acs", "asec") + ] + # Typed immutable transport was content-checked by the executor. This + # does not reopen sources or mint an independent source-authentication + # claim; the concrete producer keys are carried into the new artifact. + sources = [] + for arm, value in zip(("acs", "asec"), values, strict=True): + native._require( + native._source_document(value.payload)["arm"] == arm, + "ORIGIN_SOURCE_ARM", + ) + sources.append( + native.AuthenticatedNativeOriginSource( + value.payload, _token=native._TOKEN + ) + ) + parent = None + if "parent_binding" in aliases: + value = _artifact(context, "parent_binding", BINDING_TYPE, "binding") + doc = json.loads(value.payload) + native._require( + canonical_json(doc) == value.payload + and doc["schema"] == native.BINDING_SCHEMA, + "ORIGIN_PARENT_PAYLOAD", + ) + parent = native.PopulationOriginBinding( + value.payload, _token=native._BOUND_TOKEN + ) + result = native.bind_population_origins(frame, sources=sources, parent=parent) + document = result.document + document["graph_parent_edges"] = { + alias: { + "producer_key": value.producer_key, + "artifact_key": value.key, + "payload_sha256": native._sha(value.payload), + } + for alias, value in context.artifacts.items() + } + document["population_node"] = node.population + payload = canonical_json(document) + return KernelResult( + artifacts={"binding": payload}, + receipt={ + "phase": "native_household_origin_binding", + "summary": result.summary, + "binding_sha256": native._sha(payload), + "parent_artifact_keys": { + alias: value.key for alias, value in context.artifacts.items() + }, + "implementation": implementation_manifest(), + "release_eligible": False, + }, + ) + + +def native_origin_source_nodes(*, population: str) -> tuple[Node, ...]: + """Declare source-only producers at an explicit graph population version. + + The graph requires the version to disambiguate structural branches. These + kernels declare no population columns and change no rows. + """ + return ( + Node( + ACS_NODE, + NativeACSOriginKernel.ref, + population=population, + sources=(ACS_SOURCE,), + artifact_outputs=SOURCE_OUTPUT, + ), + Node( + ASEC_NODE, + NativeASECOriginKernel.ref, + population=population, + sources=ASEC_SOURCES, + artifact_outputs=SOURCE_OUTPUT, + ), + ) + + +def native_origin_binding_node( + columns: Sequence[Owned], + *, + population: str, + node_id: str, + parent_binding_node: str | None = None, +) -> Node: + """Bind the exact declared full population; optionally extend native ancestry.""" + grouped = {entity: [] for entity in US_SCHEMA.entities} + for cell in columns: + grouped[cell.entity].append(cell.column) + native._require( + all(grouped.values()) and all(len(v) == len(set(v)) for v in grouped.values()), + "ORIGIN_COLUMN_DECLARATION", + ) + artifacts = [ + ArtifactInput("acs", ACS_NODE, "origins", SOURCE_TYPE), + ArtifactInput("asec", ASEC_NODE, "origins", SOURCE_TYPE), + ] + if parent_binding_node is not None: + artifacts.append( + ArtifactInput( + "parent_binding", parent_binding_node, "binding", BINDING_TYPE + ) + ) + return Node( + node_id, + PopulationOriginBindingKernel.ref, + population=population, + inputs=tuple( + Slice(entity, tuple(values)) for entity, values in grouped.items() + ), + artifact_inputs=tuple(artifacts), + artifact_outputs=BINDING_OUTPUT, + params={ + "columns": tuple((c.entity, c.column, c.dtype) for c in columns), + "parent_binding": parent_binding_node is not None, + }, + ) + + +def register_native_origin_kernels(registry: KernelRegistry) -> None: + """Register only the three new kernels, without editing upstream identities.""" + for kernel in ( + NativeACSOriginKernel(), + NativeASECOriginKernel(), + PopulationOriginBindingKernel(), + ): + registry.register(kernel) + + +def verify_materialized_population_origins( + manifest, + store, + *, + population_node: str, + binding_node: str, + parent_binding_node: str | None = None, +) -> native.PopulationOriginBinding: + """Verify graph evidence against its complete materialized population. + + This is a mandatory integration boundary: a kernel cannot know whether its + declared slices omitted another population column. The manifest must come + from the reviewed graph and store; matching local receipts do not constitute + an external producer authorization or approve any benchmark evaluation. + No original source is reopened, and no kernel is executed here. + """ + expected_implementation = implementation_hash() + + def read(node_id, output, expected_type, expected_kernel): + receipt = manifest.nodes[node_id] + descriptor = receipt.typed_artifacts["outputs"][output] + payload = store.load_bytes(receipt.opaque_artifacts[output]) + value = value_from_descriptor(payload, descriptor) + native._require( + receipt.kernel_ref == expected_kernel + and receipt.kernel_impl_hash == expected_implementation + and descriptor["producer"] == node_id + and descriptor["artifact"] == output + and value.producer_key == receipt.key + and value.key == receipt.opaque_artifacts[output] + and value.type == expected_type, + "ORIGIN_MATERIALIZED_PRODUCER", + ) + return value + + try: + values = { + arm: read(node_id, "origins", SOURCE_TYPE, kernel.ref) + for arm, node_id, kernel in ( + ("acs", ACS_NODE, NativeACSOriginKernel), + ("asec", ASEC_NODE, NativeASECOriginKernel), + ) + } + sources = [ + native.AuthenticatedNativeOriginSource(value.payload, _token=native._TOKEN) + for value in values.values() + ] + parent = None + if parent_binding_node is not None: + values["parent_binding"] = read( + parent_binding_node, + "binding", + BINDING_TYPE, + PopulationOriginBindingKernel.ref, + ) + parent = native.PopulationOriginBinding( + values["parent_binding"].payload, _token=native._BOUND_TOKEN + ) + # The native parent's own projection must also describe its actual + # complete population. This boundary supports the first full clone + # (or one native household selection), not an implicit ancestry DAG. + verify_materialized_population_origins( + manifest, + store, + population_node=parent.document["population_node"], + binding_node=parent_binding_node, + ) + artifact = read( + binding_node, "binding", BINDING_TYPE, PopulationOriginBindingKernel.ref + ) + actual = native.PopulationOriginBinding( + artifact.payload, _token=native._BOUND_TOKEN + ) + document = actual.document + frame = manifest.population(population_node) + native.verify_population_origin_binding(frame, actual) + expected = native.bind_population_origins( + frame, sources=sources, parent=parent + ).document + expected["population_node"] = population_node + expected["graph_parent_edges"] = { + alias: { + "producer_key": value.producer_key, + "artifact_key": value.key, + "payload_sha256": native._sha(value.payload), + } + for alias, value in values.items() + } + native._require( + document == expected and canonical_json(document) == artifact.payload, + "ORIGIN_MATERIALIZED_LINEAGE", + ) + return actual + except native.NativeOriginError: + raise + except (ValueError, TypeError, KeyError, IndexError, OverflowError): + raise native.NativeOriginError("ORIGIN_MATERIALIZED_CONTRACT") from None diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/graph_native_origin_implementation.py b/packages/microcosm-build/src/microcosm/build/us_runtime/graph_native_origin_implementation.py new file mode 100644 index 000000000..26977b24c --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/graph_native_origin_implementation.py @@ -0,0 +1,111 @@ +"""Closed additional identity for native-origin nodes; upstream bytes stay fixed. + +The shared v1 service and inventory are unchanged. Their complete composition +and ACS-source scopes are validated before adding this independently classified +inventory. A new origin helper or registry changes only these extension keys. +""" + +from __future__ import annotations + +import csv +import hashlib +import json + +from . import native_household_origin as native +from .graph_implementation import ( + STAGE_DEPENDENCIES, + _canonical, + _covered_imports, + _dependency_contract, + _package_roots, +) +from .graph_implementation import implementation_manifest as upstream_manifest + +STAGE = "native_household_origin_v1" +BASE_STAGES = ("composed_asec_binding_v1", "acs_housing_universe_2024") +DEPENDENCIES = STAGE_DEPENDENCIES[BASE_STAGES[0]] +INVENTORY_FILE = "us_runtime/native_origin_graph_inventory.json" +INVENTORY_SCHEMA = "microcosm.us.native-origin-extension-inventory.v1" +EXTRA_MODULES = ( + "microcosm.build/cd_benchmark/canonical.py", + "microcosm.build/cd_benchmark/origin.py", + "microcosm.build/us_runtime/native_household_origin.py", + "microcosm.build/us_runtime/graph_native_household_origin.py", + "microcosm.build/us_runtime/graph_native_origin_implementation.py", +) + + +def implementation_manifest(): + """Verify all live classified dependencies without opening any survey file.""" + bases = {stage: upstream_manifest(stage) for stage in BASE_STAGES} + roots = _package_roots() + payload = (roots["microcosm.build"] / INVENTORY_FILE).read_bytes() + inventory = json.loads(payload) + native._require( + set(inventory) + == { + "schema", + "stage", + "base_stages", + "dependencies", + "extra_modules", + "contracts", + "import_classifications", + } + and inventory["schema"] == INVENTORY_SCHEMA + and inventory["stage"] == STAGE + and inventory["base_stages"] == list(BASE_STAGES) + and inventory["dependencies"] == list(DEPENDENCIES) + and inventory["extra_modules"] == list(EXTRA_MODULES) + and set(inventory["contracts"]) == set(EXTRA_MODULES), + "ORIGIN_IMPLEMENTATION_INVENTORY", + ) + imports = { + name + for contract in inventory["contracts"].values() + for name in contract["imports"] + } + native._require( + set(inventory["import_classifications"]) == imports + and all( + isinstance(label, str) and bool(label.strip()) + for label in inventory["import_classifications"].values() + ), + "ORIGIN_IMPORT_CLASSIFICATION", + ) + modules = set(EXTRA_MODULES) + for base in bases.values(): + modules.update(base["modules"]) + scope = { + "stages": { + STAGE: {"modules": sorted(modules), "dependencies": list(DEPENDENCIES)} + } + } + hashes = {} + for name in EXTRA_MODULES: + package, relative = name.split("/", 1) + code = (roots[package] / relative).read_bytes() + actual = _dependency_contract(code, name, _covered_imports(name, scope)) + native._require( + actual == inventory["contracts"][name], "ORIGIN_UNCLASSIFIED_DEPENDENCY" + ) + hashes[name] = hashlib.sha256(code).hexdigest() + return { + "schema": "microcosm.us.native-origin-extension-implementation.v1", + "stage": STAGE, + "inventory_sha256": hashlib.sha256(payload).hexdigest(), + "modules": hashes, + "upstream_implementations": bases, + "dependencies": dict(bases[BASE_STAGES[0]]["dependencies"]), + "acs_source_implementation": native.acs._implementation(), + "asec_household_member_registry": native.asec_member_registry(), + "csv_field_size_limit": csv.field_size_limit(), + } + + +def implementation_hash(): + """Identity of the additional implementation, not a replacement upstream key.""" + return hashlib.sha256( + b"microcosm.us.native-origin-extension-implementation.v1\0" + + _canonical(implementation_manifest()) + ).hexdigest() diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/graph_puf55_canonical_donor.py b/packages/microcosm-build/src/microcosm/build/us_runtime/graph_puf55_canonical_donor.py new file mode 100644 index 000000000..086c0032e --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/graph_puf55_canonical_donor.py @@ -0,0 +1,635 @@ +"""One source-backed canonical PUF55 donor CREATE, shared by both routes. + +All ordinary returns reach one nine-predictor model Frame. An eight-predictor +training Slice can omit the transported SS total without constructing another +donor cohort. Technical model persons are not source persons. The emitted +receipts describe construction; the eventual host must verify actual source +keys, producer implementations and typed ancestry before trusting any model. +""" + +from __future__ import annotations + +import hashlib +import json +import sys +from collections.abc import Mapping +from dataclasses import fields, is_dataclass +from functools import partial +from pathlib import Path +from types import FunctionType + +import numpy as np + +from microcosm.graph import ( + ArtifactOutput, + ArtifactType, + Capabilities, + Determinism, + KernelBase, + KernelResult, + Node, + Numeric, + Owned, + SeedSource, + StructuralDelta, + codecs, + source_hash, +) +from microcosm.graph import canonical as graph_canonical +from microcosm.graph.canonical import canonical_json +from microcosm.graph.population import Population + +from . import full_puf_enrichment as full +from . import graph_full_puf_enrichment as physical +from . import puf55_canonical_donor as projection +from . import puf55_route_finalization as numerical +from . import puf59_canonical as canonical +from . import puf59_canonical_artifact as envelope +from . import puf_full_source as source +from . import puf_full_source_graph as source_graph +from . import puf_interest_components as interest +from . import puf_raw_source as raw + +CANONICAL_DONOR_NODE = "survey_puf55.canonical_donor" +CANONICAL_DONOR_TYPE = ArtifactType("microcosm.us.puf59_canonical_return", 2) +DONOR_PROJECTION_TYPE = ArtifactType("microcosm.us.puf55_donor_projection", 1) +PHASE = "us.survey_puf55.canonical_donor.v1" +MAX_PROJECTION_BYTES = 128 * 1024 + + +def _require(condition, reason): + if not condition: + raise ValueError("PUF55_CANONICAL_SOURCE_" + reason) + + +def _sha(payload): + return hashlib.sha256(payload).hexdigest() + + +def _pin(pin): + _require(type(pin) is raw.SourcePin, "PIN_TYPE") + values = tuple(getattr(pin, field.name) for field in fields(raw.SourcePin)) + _require( + all(type(value) is str for value in values[:3]) + and type(pin.bytes) is int + and all(type(value) is str for value in values[4:7]) + and type(pin.delivered_header) is tuple + and all(type(value) is str for value in pin.delivered_header) + and type(pin.data_records) is int, + "PIN_FIELDS", + ) + return values + + +def _definition(definition): + """Pure check against reconstructed literals, never caller-made authority.""" + _require(type(definition) is raw.PufRawSourceDefinition, "DEFINITION_TYPE") + payload, route, digest = definition.canonical, definition.route, definition.sha256 + _require( + type(payload) is bytes + and type(route) is str + and route in ("packaged", "test_fixture") + and type(digest) is str + and _sha(payload) == digest + and canonical_json(raw._thawed(definition.document)) == payload, + "DEFINITION_BYTES", + ) + rebuilt = raw._definition_from_document(json.loads(payload), route=route) + actual = ( + route, + payload, + digest, + _pin(definition.main), + _pin(definition.demographic), + ) + _require( + actual + == ( + rebuilt.route, + rebuilt.canonical, + rebuilt.sha256, + _pin(rebuilt.main), + _pin(rebuilt.demographic), + ), + "DEFINITION_PINS", + ) + return actual + + +def _fresh_definition(fixture_definition): + packaged_bytes = raw._packaged_bytes() + _require(type(packaged_bytes) is bytes, "PACKAGED_BYTES") + packaged = raw._definition_from_document( + json.loads(packaged_bytes), route="packaged" + ) + _definition(packaged) + if fixture_definition is None: + return packaged, _sha(packaged_bytes) + _definition(fixture_definition) + document = json.loads(fixture_definition.canonical) + _require( + fixture_definition.route == "test_fixture" + and document.get("authority") == "invented_fixture_nonauthority" + and not {fixture_definition.main.sha256, fixture_definition.demographic.sha256} + & {packaged.main.sha256, packaged.demographic.sha256}, + "FIXTURE_AUTHORITY", + ) + # Reproduce the maintained fixture contract with the freshly read packaged + # pins, without consulting or accepting its process-global _PACKAGED cache. + return raw._definition_from_document(document, route="test_fixture"), _sha( + packaged_bytes + ) + + +def _recipe(seed, growth_scheme): + _require(type(seed) is int and 0 <= seed < 2**64, "SEED") + _require( + type(growth_scheme) is str and growth_scheme in ("family_observed", "cpi_only"), + "GROWTH_SCHEME", + ) + qbi, growth = canonical.qbi, canonical.growth + parameters = qbi._json(qbi.model_parameters()).encode() + _require(_sha(parameters) == qbi.PARAMETERS_SHA256, "QBI_PARAMETERS") + _require( + _sha(growth._RECIPE_JSON.encode()) == growth.RECIPE_SHA256 + and canonical_json(growth.growth_recipe()) == canonical_json(growth._RECIPE), + "GROWTH_RECIPE", + ) + runtime_bands = interest.puf_e19200_agi_bands_runtime_identity() + return { + "phase": PHASE, + "seed": seed, + "growth_scheme": growth_scheme, + "source_statistical_year": 2015, + "money_year": 2024, + "qbi_parameters_sha256": qbi.PARAMETERS_SHA256, + "growth_recipe_sha256": growth.RECIPE_SHA256, + "interest_asset_sha256": canonical.INTEREST_ASSET_SHA256, + "interest_band_facts_sha256": canonical.INTEREST_BAND_FACTS_SHA256, + "interest_runtime_sha256": runtime_bands["sha256"], + "profile": full.PUF55_SURVEY_SS.value, + "ordinary_cohort": "complete; zero design weights retained", + } + + +def _params(definition, seed, growth_scheme, packaged_sha256): + return { + **_recipe(seed, growth_scheme), + "definition": definition.params_text, + "definition_sha256": definition.sha256, + "packaged_definition_bytes_sha256": packaged_sha256, + "source_pins": canonical_json( + (_pin(definition.main), _pin(definition.demographic)) + ).decode("ascii"), + } + + +def _node(definition, params): + profile = full.require_puf_output_profile(full.PUF55_SURVEY_SS) + _require(len(profile.predictors) == 9 and len(profile.targets) == 55, "PROFILE") + return Node( + CANONICAL_DONOR_NODE, + CanonicalPuf55DonorKernel.ref, + structural=StructuralDelta.CREATE, + sources=(definition.main.source_name, definition.demographic.source_name), + outputs=tuple( + Owned("tax_unit", name, "float64") + for name in (*profile.predictors, *profile.targets) + ), + params=params, + artifact_outputs=( + ArtifactOutput("full_return_source", source_graph.FULL_RETURN_SOURCE_TYPE), + ArtifactOutput("canonical_donor", CANONICAL_DONOR_TYPE), + ArtifactOutput("donor_projection", DONOR_PROJECTION_TYPE), + ), + description="Construct one complete source-pinned modeled PUF55 donor; nine/eight training Slices share its cohort.", + ) + + +def canonical_puf55_donor_node( + *, seed=578, growth_scheme="family_observed", fixture_definition=None +): + definition, packaged_sha = _fresh_definition(fixture_definition) + return _node(definition, _params(definition, seed, growth_scheme, packaged_sha)) + + +def _modules(): + # Include actual consumed codecs, numerical source/recipe owners and Frame + # construction/storage helpers. No survey issuer or country engine is used. + modules = ( + sys.modules[__name__], + raw, + source, + source_graph, + canonical, + canonical.qbi, + canonical.growth, + envelope, + projection, + interest, + full, + full.support, + numerical, + full.codec, + full.qrf_target, + physical, + physical.operator_column_contracts, + physical.population_ops, + physical.store_ops, + codecs, + graph_canonical, + sys.modules[full.Frame.__module__], + sys.modules[full.support.EntitySchema.__module__], + sys.modules[full.support.Weights.__module__], + ) + return tuple(dict.fromkeys(modules)) + + +def _marker(value, depth=0): + """Snapshot borrowed code/configuration as an immutable object graph. + + Functions can close over a namespace that contains the same function (for + example generated class annotation machinery). Record an object's local + reference before descending, so a back edge is explicit rather than an + unbounded expansion. Each new call captures all reachable contents again; + the memo is neither a source cache nor an exemption for mutable objects. + """ + seen = {} + + def visit(item, level): + kind = type(item) + if kind in (type(None), bool, int, str, bytes, float): + _require(level <= 16, "LIVE_DEPTH") + return kind, item.hex() if kind is float else item + previous = seen.get(id(item)) + if previous is not None: + _require(previous[0] is item, "LIVE_REFERENCE") + return "reference", previous[1] + _require(level <= 16, "LIVE_DEPTH") + reference = len(seen) + # Keep the object alive while traversing borrowed mappings/dataclasses, + # preventing an identity from being recycled during this snapshot. + seen[id(item)] = (item, reference) + if kind in (tuple, list): + contents = tuple(visit(child, level + 1) for child in item) + elif kind in (set, frozenset): + contents = frozenset(visit(child, level + 1) for child in item) + elif isinstance(item, Mapping): + contents = tuple( + (visit(k, level + 1), visit(v, level + 1)) for k, v in item.items() + ) + elif kind is np.ndarray and not item.dtype.hasobject: + contents = item.dtype.str, item.shape, item.tobytes() + elif is_dataclass(item) and not isinstance(item, type): + contents = tuple( + (field.name, visit(getattr(item, field.name), level + 1)) + for field in fields(item) + ) + elif kind is FunctionType: + # Keep identity as well as immutable code/default/closure contents; + # comparing retained mutable functions to themselves is not a seal. + # Marshal bytes also encode interpreter reference-sharing state; + # retaining a returned literal can change them without a code edit. + # Code objects and their constant graph are immutable. Keep the + # object alive, retain its identity, and snapshot public fields; + # mutable function defaults/closures still use visit below. + code_object = item.__code__ + code = ( + code_object, + id(code_object), + code_object.co_argcount, + code_object.co_posonlyargcount, + code_object.co_kwonlyargcount, + code_object.co_nlocals, + code_object.co_stacksize, + code_object.co_flags, + code_object.co_code, + code_object.co_consts, + code_object.co_names, + code_object.co_varnames, + code_object.co_freevars, + code_object.co_cellvars, + code_object.co_filename, + code_object.co_name, + code_object.co_qualname, + code_object.co_firstlineno, + code_object.co_linetable, + code_object.co_exceptiontable, + ) + defaults = visit(item.__defaults__, level + 1) + kwdefaults = visit(item.__kwdefaults__, level + 1) + closure = [] + for cell in item.__closure__ or (): + try: + cell_value = cell.cell_contents + except ValueError: + closure.append(("empty_cell",)) + else: + closure.append(("cell", visit(cell_value, level + 1))) + contents = ( + item, + code, + defaults, + kwdefaults, + tuple(closure), + ) + else: + contents = id(item) + return "object", reference, kind, contents + + return visit(value, depth) + + +def _live(): + result = [] + for module in _modules(): + result.append(("module", module, module.__name__, module.__file__)) + for name, value in vars(module).items(): + if name.startswith("__") or (module is raw and name == "_PACKAGED"): + continue + if isinstance(value, type) and value.__module__ == module.__name__: + members = [] + for member, function in vars(value).items(): + if isinstance(function, (staticmethod, classmethod)): + function = function.__func__ + if isinstance(function, property): + function = function.fget + if type(function) is FunctionType: + members.append((member, _marker(function))) + elif not member.startswith("__"): + members.append((member, _marker(function))) + result.append((module.__name__, name, value, tuple(members))) + else: + result.append((module.__name__, name, _marker(value))) + result.append( + ( + "rng", + tuple( + _marker(value) + for value in ( + np.random.default_rng, + np.random.Generator, + np.random.PCG64, + np.random.SeedSequence, + ) + ), + ) + ) + return tuple(result) + + +def _frame_seal(frame): + return physical._population_stamp( + Population.from_frame(frame, CANONICAL_DONOR_NODE) + ) + + +class CanonicalPuf55DonorKernel(KernelBase): + ref = "us.survey_puf55.canonical_donor@1" + capabilities = Capabilities( + Determinism.SEEDED, + numeric=Numeric.PLATFORM_BITWISE, + seed_source=SeedSource.PARAM, + structural=StructuralDelta.CREATE, + dependencies=("numpy", "pandas"), + ) + + def __init__( + self, *, seed=578, growth_scheme="family_observed", fixture_definition=None + ): + definition, packaged_sha = _fresh_definition(fixture_definition) + self._definition = definition + self.source_codecs = raw.puf_raw_source_codecs(definition) + self.source_refs = raw.us_puf_raw_source_refs(definition) + params = canonical_json(_params(definition, seed, growth_scheme, packaged_sha)) + # Retain primitive pins and immutable baselines before any future borrow. + self._state = ( + definition, + _definition(definition), + params, + packaged_sha, + seed, + growth_scheme, + self.implementation_hash(), + _live(), + ) + self._check_state(self._state) + + def implementation_hash(self): + # The executor calls this on required replay too. Resource reads cannot + # live only in run(), which is deliberately skipped on a cache hit. + state = getattr(self, "_state", None) + if state is not None: + self._check_state(state) + packaged = raw._packaged_bytes() + asset = interest.puf_e19200_interest_components_asset_identity() + code = source_hash(*_modules(), dependencies=self.capabilities.dependencies) + if state is not None: + self._check_state(state) + _require( + type(packaged) is bytes + and asset["asset_sha256"] == canonical.INTEREST_ASSET_SHA256 + and asset["agi_bands"] + == interest.puf_e19200_agi_bands_runtime_identity()["agi_bands"], + "IMPLEMENTATION_RESOURCES", + ) + return _sha( + canonical_json( + { + "code": code, + "packaged_definition_bytes_sha256": _sha(packaged), + "interest_asset": asset, + "csv_acceptance": raw.csv_acceptance_profile(), + } + ) + ) + + def _check_state(self, state): + definition, definition_seal, params, packaged_sha, seed, scheme, _, live = state + _require( + self._state is state + and self._definition is definition + and _definition(definition) == definition_seal + and canonical_json(_params(definition, seed, scheme, packaged_sha)) + == params + and self.source_refs == raw.us_puf_raw_source_refs(definition) + and _live() == live, + "LIVE_STATE_CHANGED", + ) + expected = (definition.main, definition.demographic) + _require( + type(self.source_codecs) is codecs.SourceCodecRegistry + and not self.source_codecs.names(), + "CODEC_MODE", + ) + _require( + set(self.source_codecs.bytes_names()) == {p.codec for p in expected}, + "CODEC_ROSTER", + ) + for pin in expected: + loader = self.source_codecs.get(pin.codec) + _require( + type(loader) is partial + and loader.func is raw.load_pinned_puf_bytes + and loader.args == () + and set(loader.keywords) == {"pin"} + and loader.keywords["pin"] is pin + and _pin(loader.keywords["pin"]) == _pin(pin), + "CODEC_BINDING", + ) + + def _context(self, context, state, expected_paths=None): + definition, _, params, *_ = state + _require( + context.node == _node(definition, json.loads(params)) + and canonical_json(dict(context.params)) == params + and not context.tables + and not context.weights + and not context.artifacts + and len(context.strata) == 0 + and set(context.sources) + == {definition.main.source_name, definition.demographic.source_name}, + "DECLARATION_OR_CONTEXT", + ) + paths = tuple( + (p.source_name, context.sources[p.source_name]) + for p in (definition.main, definition.demographic) + ) + _require( + all(type(path) is type(Path()) and path.is_absolute() for _, path in paths), + "SOURCE_PATH", + ) + _require( + expected_paths is None or paths == expected_paths, "SOURCE_PATH_CHANGED" + ) + return paths + + def _read_sources(self, paths, state): + definition = state[0] + self._check_state(state) + return tuple( + codecs.load_source_bytes(pin.codec, path, registry=self.source_codecs) + for pin, (_, path) in zip( + (definition.main, definition.demographic), paths, strict=True + ) + ) + + def _read_resources(self, state): + packaged_bytes = raw._packaged_bytes() + asset = interest.puf_e19200_interest_components_asset_identity() + implementation = self.implementation_hash() + # All file/metadata reads precede these pure comparisons. + self._check_state(state) + _require( + type(packaged_bytes) is bytes + and _sha(packaged_bytes) == state[3] + and implementation == state[6], + "SOURCE_CODE_OR_RESOURCE_CHANGED", + ) + _require( + asset["asset_sha256"] == canonical.INTEREST_ASSET_SHA256 + and asset["agi_bands"] + == interest.puf_e19200_agi_bands_runtime_identity()["agi_bands"], + "INTEREST_RESOURCE", + ) + + def run(self, context): + state = self._state + self._check_state(state) + paths = self._context(context, state) + self._read_resources(state) + buffers = self._read_sources(paths, state) + definition, _, params, _, seed, scheme, *_ = state + decoded = source.decode_full_puf_source(*buffers, definition) + raw_payload = source.encode_full_puf_source(decoded) + constructed = canonical.construct_canonical_puf59( + decoded, + interest_bands=interest.US_PUF_E19200_AGI_BANDS, + interest_asset_sha256=canonical.INTEREST_ASSET_SHA256, + seed=seed, + growth_scheme=scheme, + ) + canonical_payload = envelope.encode_canonical_puf59( + constructed, expected_growth_scheme=scheme + ) + donor, evidence = projection.canonical_puf55_donor_from_artifact( + canonical_payload, + expected_artifact_sha256=_sha(canonical_payload), + expected_growth_scheme=scheme, + profile=full.PUF55_SURVEY_SS, + ) + profile = full.require_puf_output_profile(full.PUF55_SURVEY_SS) + selected = full._validated_model_donor(donor, profile=profile) + frame = numerical._model_donor_frame(selected) + ids = decoded.status["RECID"][decoded.ordinary] + _require( + np.array_equal(donor.index.to_numpy(), ids) + and np.array_equal(frame.table("tax_unit").index.to_numpy(), ids) + and frame.weights_for("tax_unit").values.tobytes() + == ( + decoded.status["S006"][decoded.ordinary].astype(np.float64) / 100 + ).tobytes() + and all(dtype == np.dtype("float64") for dtype in selected.dtypes), + "ORDINARY_COHORT_OR_CONVERSION", + ) + frame_sha = _frame_seal(frame) + receipt = { + **json.loads(params), + "protocol": PHASE, + "source_route": definition.route, + "source_rows": len(decoded.status["RECID"]), + "ordinary_rows": len(ids), + "zero_weight_rows": int((frame.weights_for("tax_unit").values == 0).sum()), + "full_return_source_sha256": _sha(raw_payload), + "canonical_donor_sha256": _sha(canonical_payload), + "projection": evidence, + "model_frame_sha256": frame_sha, + "source_recid_sha256": _sha(ids.astype(" list[str]: + return sorted(set(globals()) | _STACKED_ALIASES) + + +ASEC_CODEC = "us-asec-raw-stage-v4" +ACS_CODEC = "us-acs-native-2024-v1" +ASEC_PREPARED_CODEC = "us-asec-prepared-current-money-v3" +ASSEMBLY_PHASE = "assemble_stacked_spine" +US_STACK_PREPARATION_TYPE = ArtifactType("microcosm.us.stacked_spine_preparation", 1) +US_SOURCE_DEPENDENCIES = STAGE_DEPENDENCIES["assembly_prepare"] + + +@dataclass(frozen=True) +class GraphAssemblyResult: + frame: Frame + receipt: object + dtype_transitions: tuple[dict[str, str], ...] + + +def _canonical_assembly(frame: Frame, sources: tuple[Frame, ...]): + """Represent source-declared integer/bool/string absence with nullable storage. + + Legacy union assembly fills absent non-float columns with object None. + The graph has no arbitrary-object dtype. Promotion is determined by the + actual source dtype, never by guessing from the assembled observations. + Values and the missing mask must be identical after the explicit boundary. + """ + transitions = [] + for entity in US_SCHEMA.entities: + table = frame.table(entity) + for column in table: + original = table[column] + if not pd.api.types.is_object_dtype(original.dtype): + continue + dtypes = [ + source.table(entity)[column].dtype + for source in sources + if column in source.table(entity) + ] + if not dtypes: + continue # graph-added provenance strings are handled below + if all(pd.api.types.is_bool_dtype(dtype) for dtype in dtypes): + target = "boolean" + elif all(pd.api.types.is_integer_dtype(dtype) for dtype in dtypes): + target = "Int64" + elif all(isinstance(dtype, pd.StringDtype) for dtype in dtypes): + target = "string" + else: + continue + promoted = original.astype(dtype_for_token(target)) + if not original.isna().equals(promoted.isna()): + raise ValueError( + f"US graph dtype promotion changed nulls: {entity}.{column}." + ) + observed = original.notna() + if not np.array_equal( + original[observed].to_numpy(), promoted[observed].to_numpy() + ): + raise ValueError( + f"US graph dtype promotion changed values: {entity}.{column}." + ) + table[column] = promoted + transitions.append( + {"entity": entity, "column": column, "from": "object", "to": target} + ) + canonicalize_frame_string_dtypes(frame, boundary="US graph assembly", in_place=True) + # Build checkpoint strings use NumPy NaN; the graph column codec requires + # nullable pandas strings. This is an explicit physical storage boundary. + for entity in US_SCHEMA.entities: + table = frame.table(entity) + for column in table: + original = table[column] + if isinstance( + original.dtype, pd.StringDtype + ) and original.dtype != dtype_for_token("string"): + promoted = original.astype(dtype_for_token("string")) + if not original.isna().equals(promoted.isna()): + raise ValueError( + f"US graph string promotion changed nulls: {entity}.{column}." + ) + observed = original.notna() + if not np.array_equal( + original[observed].to_numpy(), promoted[observed].to_numpy() + ): + raise ValueError( + f"US graph string promotion changed values: {entity}.{column}." + ) + table[column] = promoted + transitions.append( + { + "entity": entity, + "column": column, + "from": "string[python,nan]", + "to": "string[python,pd.NA]", + } + ) + return tuple(transitions) + + +def load_graph_asec(path: Path, *, store=None) -> Frame: + return _decode_graph_asec(path, store=store) + + +def _decode_graph_asec(path: Path, *, store=None) -> Frame: + del store + frame, _binding = load_asec_raw_stage_checkpoint_v4(path) + return frame + + +def load_graph_acs(path: Path, *, store=None) -> Frame: + return _decode_graph_acs(path, store=store) + + +def _decode_graph_acs(path: Path, *, store=None) -> Frame: + del store + expected = {"csv_hus.zip", "csv_pus.zip"} + if not path.is_dir() or {entry.name for entry in path.iterdir()} != expected: + raise ValueError("US ACS graph source must contain exactly the two archives.") + if any(not (path / name).is_file() for name in expected): + raise ValueError("US ACS graph source archives must be regular files.") + source = AcsPumsSource(path / "csv_hus.zip", path / "csv_pus.zip", vintage=2024) + raw, _receipt = build_acs_pums_unit_frame(source) + mapped = map_acs_native_inputs(raw) + assert_operator_free_source_frame( + mapped.frame, label="US graph ACS source", native_inputs=mapped.native_inputs + ) + return mapped.frame + + +def us_source_codecs() -> SourceCodecRegistry: + codecs = SourceCodecRegistry() + codecs.register(ASEC_CODEC, load_graph_asec) + codecs.register(ACS_CODEC, load_graph_acs) + # The prepared current-money directory source. Its loader lives with the + # preparation it names, so registering it here adds no import cycle and no + # second preparation path. + codecs.register(ASEC_PREPARED_CODEC, load_graph_asec_prepared) + codecs.register_bytes("raw-bytes-v1", load_raw_bytes) + codecs.register(ACS_HU_CODEC, load_graph_acs_housing_universe) + validate_source_codecs(codecs) + return codecs + + +def assembly_from_sources( + asec_path: Path, acs_path: Path, *, sample_fraction: float, sample_seed: int +): + """The direct-call parity oracle and the CREATE kernel share this operation.""" + asec = load_graph_asec(asec_path) + acs = load_graph_acs(acs_path) + result = _stacked_alias("assemble_stacked_spine")( + asec, acs, sample_fraction=sample_fraction, sample_seed=sample_seed + ) + transitions = _canonical_assembly(result.frame, (asec, acs)) + return GraphAssemblyResult(result.frame, result.receipt, transitions) + + +def frame_column_declarations(frame: Frame) -> tuple[Owned, ...]: + """Inventory data cells; the graph owns IDs and membership implicitly.""" + if frame.schema != US_SCHEMA: + raise ValueError("US graph columns require the US schema.") + return tuple( + Owned(entity, column, token_for_dtype(table[column].dtype)) + for entity in US_SCHEMA.entities + for table in (frame.table(entity),) + for column in table.columns + if column != US_SCHEMA.entity_id_column(entity) + and not ( + entity == US_SCHEMA.person_entity + and column + in { + US_SCHEMA.membership_column(group) for group in US_SCHEMA.group_entities + } + ) + ) + + +def _source_implementation_hash() -> str: + """Compatibility spelling for the new preparation identity only.""" + # This runs before cache lookup, including resume=require. A renamed or + # substituted public loader cannot claim the declared codec implementation. + us_source_codecs() + return implementation_hash("assembly_prepare") + + +class USAssemblyCreateKernel(KernelBase): + ref = "us.production.spine_prepare@2" + capabilities = Capabilities( + determinism=Determinism.SEEDED, + numeric=Numeric.PLATFORM_BITWISE, + seed_source=SeedSource.PARAM, + structural=StructuralDelta.CREATE, + dependencies=US_SOURCE_DEPENDENCIES, + ) + + def implementation_hash(self) -> str: + return _source_implementation_hash() + + def run(self, context: KernelContext) -> KernelResult: + if set(context.params) != {"phase", "sample_fraction", "sample_seed"}: + raise ValueError("US assembly parameters have an unsupported contract.") + if context.params["phase"] != ASSEMBLY_PHASE: + raise ValueError("US assembly has an incorrect outer phase.") + asec = load_graph_asec(context.sources["asec_raw_stage"]) + acs = load_graph_acs(context.sources["acs_native"]) + result = _stacked_alias("prepare_stacked_spine")( + asec, + acs, + sample_fraction=context.params["sample_fraction"], + sample_seed=context.params["sample_seed"], + ) + transitions = _canonical_assembly(result.frame, (asec, acs)) + if frame_column_declarations(result.frame) != context.node.outputs: + raise ValueError( + "US assembly source columns differ from the declared inventory." + ) + return KernelResult( + frame=result.frame, + receipt={ + "phase": ASSEMBLY_PHASE, + "implementation": implementation_manifest("assembly_prepare"), + "preparation": result.receipt, + "dtype_transitions": transitions, + }, + artifacts={ + "frame_context": encode_us_frame_context(result.frame), + "preparation": canonical_json(_json_data(result.receipt)), + }, + ) + + +class USSpineHarmonizeKernel(KernelBase): + ref = "us.production.spine_harmonize@2" + capabilities = Capabilities( + determinism=Determinism.DETERMINISTIC, + numeric=Numeric.PLATFORM_BITWISE, + structural=StructuralDelta.REWEIGHT, + dependencies=STAGE_DEPENDENCIES["assembly_harmonize"], + ) + + def implementation_hash(self) -> str: + return implementation_hash("assembly_harmonize") + + def run(self, context: KernelContext) -> KernelResult: + if context.params != {"phase": ASSEMBLY_PHASE}: + raise ValueError( + "US spine harmonization has an unsupported phase contract." + ) + bound = context.artifacts["frame_context"] + prepared = context.artifacts["preparation"] + if ( + bound.type != US_FRAME_CONTEXT_TYPE + or prepared.type != US_STACK_PREPARATION_TYPE + or bound.producer_key != prepared.producer_key + ): + raise ValueError( + "US spine preparation/context must share a typed producer." + ) + document = _decode(bound.payload) + if document["weight_sources"] != {"household": "design"}: + raise ValueError( + "US spine harmonization requires its explicit DESIGN anchor." + ) + for entity in ("person", "household"): + if any( + document["entities"][entity][key] != value + for key, value in _row_identity(context.tables[entity], entity).items() + ): + raise ValueError(f"US spine {entity} context identity differs.") + # This is canonical, producer-owned JSON, not a self-authored declaration. + import json + + preparation = json.loads(prepared.payload) + if canonical_json(preparation) != prepared.payload: + raise ValueError("US spine preparation must be canonical JSON.") + if preparation["assembly_metadata"] != document["metadata"]: + raise ValueError("US spine preparation and context metadata differ.") + result = _stacked_alias("harmonize_stacked_spine_weights")( + household=context.tables["household"], + weights=context.weights["household"], + preparation=preparation, + ) + # The graph's declared mass is person mass by source stratum. Compute + # it with the same Frame reducer from minimal declared membership views. + tables = { + "person": context.tables["person"][["person_id", "person_household_id"]], + "household": context.tables["household"][["household_id"]], + } + schema = EntitySchema(group_entities=("household",)) + before = Frame( + tables, schema, {"household": context.weights["household"]}, context.strata + ) + after = Frame(tables, schema, {"household": result.weights}, context.strata) + before_mass, after_mass = before.stratum_mass(), after.stratum_mass() + document["metadata"] = _normative_metadata(result.metadata) + document["mass_log"] = [ + _json_data(asdict(item)) for item in result.legacy_mass_log + ] + document["weight_sources"] = {"household": "importance"} + return KernelResult( + weights=result.weights, + artifacts={"frame_context": canonical_json(document)}, + receipt={ + "phase": ASSEMBLY_PHASE, + "implementation": implementation_manifest("assembly_harmonize"), + "assembly": result.receipt, + "mass": { + "policy": "declared", + "before": float(before_mass.sum()), + "after": float(after_mass.sum()), + "stratum_before": before_mass.to_dict(), + "stratum_after": after_mass.to_dict(), + }, + }, + ) + + +def us_assembly_graph( + columns: Sequence[Owned], *, sample_fraction: float, sample_seed: int +) -> Graph: + """Construct only the assembly development graph, never a release pool.""" + return Graph( + country="us", + sources=( + SourceRef( + "asec_raw_stage", ASEC_CODEC, "Operator-free ASEC raw-stage checkpoint." + ), + SourceRef( + "acs_native", ACS_CODEC, "2024 ACS one-year national PUMS archives." + ), + ), + nodes=( + Node( + id=f"{ASSEMBLY_PHASE}.prepare", + kernel=USAssemblyCreateKernel.ref, + outputs=tuple(columns), + structural=StructuralDelta.CREATE, + sources=("asec_raw_stage", "acs_native"), + params={ + "phase": ASSEMBLY_PHASE, + "sample_fraction": sample_fraction, + "sample_seed": sample_seed, + }, + artifact_outputs=( + ArtifactOutput("frame_context", US_FRAME_CONTEXT_TYPE), + ArtifactOutput("preparation", US_STACK_PREPARATION_TYPE), + ), + ), + Node( + id=ASSEMBLY_PHASE, + kernel=USSpineHarmonizeKernel.ref, + base=f"{ASSEMBLY_PHASE}.prepare", + structural=StructuralDelta.REWEIGHT, + inputs=( + Slice("household", (support_channel_column("household"),)), + Slice("person", (support_channel_column("person"),)), + ), + weights=WeightTransition("household", "importance", mass="declared"), + mass="declared", + params={"phase": ASSEMBLY_PHASE}, + artifact_inputs=( + ArtifactInput( + "frame_context", + f"{ASSEMBLY_PHASE}.prepare", + "frame_context", + US_FRAME_CONTEXT_TYPE, + ), + ArtifactInput( + "preparation", + f"{ASSEMBLY_PHASE}.prepare", + "preparation", + US_STACK_PREPARATION_TYPE, + ), + ), + artifact_outputs=( + ArtifactOutput("frame_context", US_FRAME_CONTEXT_TYPE), + ), + ), + ), + ) + + +def us_assembly_registry() -> KernelRegistry: + kernels = KernelRegistry() + kernels.register(USAssemblyCreateKernel()) + kernels.register(USSpineHarmonizeKernel()) + return kernels diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/graph_survey_age_artifact.py b/packages/microcosm-build/src/microcosm/build/us_runtime/graph_survey_age_artifact.py new file mode 100644 index 000000000..b59f3f4df --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/graph_survey_age_artifact.py @@ -0,0 +1,221 @@ +"""Unweighted age counts as an ordered artifact, without population columns. + +The actual NationalAgeCountKernel performs the measurement. Its household +roster is the sorted unique membership of the declared person projection; +Frame requires every household to be referenced. Downstream admission must +also compare the artifact's ordered IDs with the complete receiving population. +This artifact is a measurement, not source authority or target activation. +""" + +from __future__ import annotations + +import hashlib +import json +import sys +from dataclasses import dataclass, replace + +import numpy as np +import pandas as pd + +from microcosm.graph import ( + ArtifactOutput, + ArtifactType, + KernelBase, + KernelResult, + source_hash, +) + +from . import graph_national_age_counts as ages +from . import national_age_activation as activation + +COUNTS_TYPE = ArtifactType("microcosm.us.survey_household_age_counts", 1) +MAGIC = b"MCUSAGE1\n" +MAX_BYTES = 64 * 1024**2 +MAX_HEADER_BYTES = 65_536 +MAX_PEOPLE = 10_000_000 +_COLUMNS = tuple(b.column for b in activation.NATIONAL_AGE_ACTIVATION.bands) +_WIDTH = 1 + len(_COLUMNS) +_FIELDS = frozenset( + {"protocol", "population", "columns", "rows", "people", "age_convention"} +) + + +def _require(condition, reason): + if not condition: + raise ValueError("SURVEY_AGE_ARTIFACT_" + reason) + + +def _json(value): + return json.dumps( + value, sort_keys=True, separators=(",", ":"), allow_nan=False + ).encode() + + +def _shape(rows, people): + _require(type(rows) is int and type(people) is int, "INTEGER_SHAPE") + _require(0 < rows <= people <= MAX_PEOPLE, "SHAPE") + _require( + len(MAGIC) + 4 + MAX_HEADER_BYTES + rows * _WIDTH * 8 <= MAX_BYTES, "LIMIT" + ) + + +def _validate(header: dict, values: np.ndarray): + _require(type(header) is dict and set(header) == _FIELDS, "HEADER") + _require( + header["protocol"] == "microcosm.us.survey-household-age-counts.v1", "PROTOCOL" + ) + _require( + type(header["population"]) is str and 0 < len(header["population"]) <= 256, + "POPULATION", + ) + _require(header["columns"] == list(_COLUMNS), "COLUMNS") + _require(header["age_convention"] == ages.SUPPORTED_AGE_CONVENTION, "CONVENTION") + _shape(header["rows"], header["people"]) + _require( + type(values) is np.ndarray + and values.dtype == np.dtype(" ids[:-1]), "ORDERED_IDS") + _require(np.all(counts >= 0) and np.all(counts <= header["people"]), "COUNTS") + _require(np.all(np.any(counts > 0, axis=1)), "EMPTY_HOUSEHOLD") + # The shape and per-cell limits above bound this sum well below int64 max. + _require(int(counts.sum(dtype=np.int64)) == header["people"], "PEOPLE_TOTAL") + + +@dataclass(frozen=True) +class SurveyAgeCountValues: + """Decoded values only. Typed ancestry and current-population checks are separate.""" + + population: str + household_ids: np.ndarray + counts: pd.DataFrame + people: int + + +def decode_survey_age_counts(payload: bytes) -> SurveyAgeCountValues: + _require( + type(payload) is bytes and len(MAGIC) + 4 < len(payload) <= MAX_BYTES, "PAYLOAD" + ) + _require(payload.startswith(MAGIC), "MAGIC") + offset = len(MAGIC) + length = int.from_bytes(payload[offset : offset + 4], "big") + _require( + 0 < length <= MAX_HEADER_BYTES and offset + 4 + length <= len(payload), + "HEADER_LIMIT", + ) + offset += 4 + raw = payload[offset : offset + length] + try: + header = json.loads(raw) + except (ValueError, UnicodeError): + raise ValueError("SURVEY_AGE_ARTIFACT_HEADER_JSON") from None + _require(type(header) is dict and set(header) == _FIELDS, "HEADER") + _require(_json(header) == raw, "CANONICAL_HEADER") + _shape(header["rows"], header["people"]) + offset += length + _require(len(payload) - offset == header["rows"] * _WIDTH * 8, "BYTE_COUNT") + values = np.frombuffer(payload, dtype=" bytes: + """Project numbers from bounded bytes; this pure operation authenticates nothing.""" + document = _document(payload) + ids = document.get("household_ids") + groups = document.get("group_indices") + records = document.get("origins") + _require( + type(ids) is list + and 0 < len(ids) <= numeric.MAX_ROWS + and type(groups) is list + and len(groups) == len(ids) + and type(records) is list + and 0 < len(records) <= budgets.MAX_GROUPS + and type(document.get("group_count")) is int + and document["group_count"] == len(records) + and len(ids) == 2 * len(records), + "COMPLETE_CLONE_SHAPE", + ) + _require( + all(type(i) is int and -(2**63) <= i < 2**63 for i in ids) + and len(set(ids)) == len(ids), + "HOUSEHOLD_IDS", + ) + positions = {i: position for position, i in enumerate(ids)} + incoming, row_upper = [None] * len(ids), [None] * len(ids) + upper, seen = [], set() + for group, record in enumerate(records): + _require(type(record) is dict, "ORIGIN_RECORD") + # Validate before duplicating per-origin tokens into per-row output. + numeric._vector([record.get("design_bound_float64_hex")], 1) + numeric._vector([record.get("upper_float64_hex")], 1) + members, weights = ( + record.get("members"), + record.get("incoming_clone_float64_bytes"), + ) + _require( + type(members) is list + and len(members) == 2 + and type(weights) is list + and len(weights) == 2, + "COMPLETE_ROLES", + ) + roles = set() + for member, weight_bytes in zip(members, weights, strict=True): + _require( + type(member) is list + and len(member) == 2 + and type(member[0]) is int + and member[0] in positions + and member[0] not in seen + and type(member[1]) is int + and member[1] in (0, 1), + "MEMBERSHIP", + ) + seen.add(member[0]) + roles.add(member[1]) + position = positions[member[0]] + _require( + type(groups[position]) is int and groups[position] == group, + "GROUP_ORDER", + ) + _require( + type(weight_bytes) is str + and len(weight_bytes) == 16 + and all(c in "0123456789abcdef" for c in weight_bytes), + "WEIGHT_BYTES", + ) + value = np.frombuffer(bytes.fromhex(weight_bytes), dtype=np.float64)[0] + incoming[position] = float(value).hex() + row_upper[position] = record.get("design_bound_float64_hex") + _require(roles == {0, 1}, "COMPLETE_ROLES") + upper.append(record.get("upper_float64_hex")) + _require(seen == set(ids), "COMPLETE_MEMBERSHIP") + output = graph._bounded_json( + { + "protocol": numeric.BOUNDS_PROTOCOL, + "budget_sha256": _sha(payload), + "population": clone.COMBINED_CLONE_NODE, + "household_ids": ids, + "group_indices": groups, + "group_upper_hex": upper, + "row_upper_hex": row_upper, + "incoming_hex": incoming, + }, + numeric.MAX_BYTES, + ) + _require(len(output) <= numeric.MAX_BYTES, "NUMERIC_LIMIT") + numeric.decode_numeric_survey_bounds(output) + return output + + +def survey_sampling_budget_node(*, budget_sha256): + _require(numeric._digest(budget_sha256), "BUDGET_DIGEST") + return Node( + BUDGET_NODE, + SurveySamplingBudgetKernel.ref, + population=clone.COMBINED_CLONE_NODE, + # This provenance input forces the actual ownership claim to precede + # the budget transport. It is not an imputation feature. + inputs=(Slice("household", (support_clone_index_column("household"),)),), + params={"budget_sha256": budget_sha256, "authority": "country_runner_required"}, + artifact_inputs=( + ArtifactInput( + "preparation", graph.CREATE_NODE, "preparation", graph.PREPARATION_TYPE + ), + ArtifactInput( + "allocation", graph.ALLOCATION_NODE, "allocation", graph.ALLOCATION_TYPE + ), + ), + artifact_outputs=( + ArtifactOutput("budget", budgets.BUDGET_TYPE), + ArtifactOutput("numeric_bounds", numeric.BOUNDS_TYPE), + ), + ) + + +class SurveySamplingBudgetKernel(KernelBase): + """Carry immutable caller values with checked source and clone projections.""" + + ref = "us.survey_sampling_budget@1" + capabilities = Capabilities( + determinism=Determinism.DETERMINISTIC, + numeric=Numeric.PLATFORM_BITWISE, + dependencies=("numpy", "pandas"), + ) + + def __init__(self, payload): + _document(payload) + self._payload = payload + + def implementation_hash(self): + return source_hash( + sys.modules[__name__], + budgets, + numeric, + numeric.group_bounds, + support_clone_index_column, + graph, + dependencies=self.capabilities.dependencies, + ) + + def run(self, context): + payload = self._payload + document = _document(payload) + expected = survey_sampling_budget_node(budget_sha256=_sha(payload)) + _require(context.node.normative() == expected.normative(), "DECLARATION") + _require(dict(context.params) == dict(expected.params), "PARAMETERS") + _require( + set(context.tables) == set(context.weights) == {"household"} + and not context.sources + and set(context.artifacts) == {"preparation", "allocation"}, + "CONTEXT", + ) + for edge in expected.artifact_inputs: + value = context.artifacts[edge.name] + _require( + value.type == edge.type + and value.key == opaque_artifact_key(value.producer_key, edge.artifact) + and _sha(value.payload) == document[edge.name + "_sha256"], + "SOURCE_EDGE", + ) + numbers_raw = numeric_survey_budget_payload(payload) + numbers = numeric.decode_numeric_survey_bounds(numbers_raw) + household = context.tables["household"] + role_column = support_clone_index_column("household") + _require(set(household.columns) == {"household_id", role_column}, "PROJECTION") + _require( + tuple(household.household_id) == numbers.grouped.household_ids, + "ORDERED_IDS", + ) + roles = { + member[0]: member[1] + for record in document["origins"] + for member in record["members"] + } + _require( + tuple(household[role_column]) + == tuple(roles[i] for i in household.household_id), + "CLONE_ROLES", + ) + weights = context.weights["household"] + _require( + weights.kind is WeightKind.IMPORTANCE + and weights.values.dtype == numbers.incoming.dtype + and weights.values.tobytes() == numbers.incoming.tobytes(), + "ACTUAL_INCOMING_WEIGHTS", + ) + result = KernelResult( + artifacts={"budget": payload, "numeric_bounds": numbers_raw}, + receipt={ + "budget_sha256": _sha(payload), + "numeric_bounds_sha256": _sha(numbers_raw), + "constraint_digest": numbers.grouped.digest, + "group_count": numbers.grouped.group_count, + "release_eligible": False, + "source_admission": "required_from_country_runner", + }, + ) + _require( + type(self._payload) is bytes and self._payload == payload, "FINAL_PAYLOAD" + ) + return result diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/graph_survey_calibration.py b/packages/microcosm-build/src/microcosm/build/us_runtime/graph_survey_calibration.py new file mode 100644 index 000000000..b3d839f8c --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/graph_survey_calibration.py @@ -0,0 +1,417 @@ +"""Source-blind age calibration over typed numeric budget and count artifacts. + +These kernels do not authenticate a survey. The country runner must reconstruct +the source-qualified budget and independently admit the complete population on +cold execution, cached restoration and final return. Frozen incoming weights +are numeric artifact values, never replacement original DESIGN anchors. +""" + +from __future__ import annotations + +import hashlib +import json +import sys +from dataclasses import dataclass, replace + +import numpy as np + +from microcosm.calibrate import calibrate, diagnostics_payload, group_bounds +from microcosm.frame import WeightKind, Weights +from microcosm.graph import ( + ArtifactInput, + ArtifactOutput, + ArtifactType, + Capabilities, + Determinism, + KernelBase, + KernelResult, + Node, + Numeric, + SeedSource, + StructuralDelta, + WeightTransition, + source_hash, +) +from microcosm.graph.canonical import canonical_json +from microcosm.graph.keys import opaque_artifact_key + +from . import demographic_calibration_graph as demographic +from . import graph_survey_age_artifact as ages +from . import survey_age_activation as age_activation + +BOUNDS_TYPE = ArtifactType("microcosm.us.survey_calibration_numeric_bounds", 1) +BOUNDS_PROTOCOL = "microcosm.us.survey-calibration-numeric-bounds.v1" +MAX_BYTES = 64 * 1024**2 +MAX_ROWS = MAX_BYTES // 128 +CALIBRATION_NODE = "survey.age_calibration" +_FIELDS = frozenset( + { + "protocol", + "budget_sha256", + "population", + "household_ids", + "group_indices", + "group_upper_hex", + "row_upper_hex", + "incoming_hex", + } +) + + +def _require(condition, reason): + if not condition: + raise ValueError("SURVEY_CALIBRATION_" + reason) + + +def _sha(payload): + return hashlib.sha256(payload).hexdigest() + + +def _digest(value): + return ( + type(value) is str + and len(value) == 64 + and all(c in "0123456789abcdef" for c in value) + ) + + +def _vector(raw, length): + _require(type(raw) is list and len(raw) == length, "VECTOR_LENGTH") + _require(all(type(v) is str and 0 < len(v) <= 32 for v in raw), "FLOAT_TOKEN") + try: + values = np.array([float.fromhex(v) for v in raw], dtype=np.float64) + except (ValueError, OverflowError): + raise ValueError("SURVEY_CALIBRATION_FLOAT_TOKEN") from None + _require( + np.all(np.isfinite(values) & (values >= 0)) + and all(float(v).hex() == text for v, text in zip(values, raw, strict=True)), + "FLOAT_VALUES", + ) + return values + + +@dataclass(frozen=True) +class NumericSurveyBounds: + """Decoded numbers only; neither a source certificate nor a live budget.""" + + budget_sha256: str + population: str + grouped: group_bounds.GroupedUpperBounds + incoming: np.ndarray + row_upper: np.ndarray + + +def decode_numeric_survey_bounds(payload: bytes) -> NumericSurveyBounds: + _require(type(payload) is bytes and 0 < len(payload) <= MAX_BYTES, "PAYLOAD") + try: + document = json.loads(payload) + except (ValueError, UnicodeError): + raise ValueError("SURVEY_CALIBRATION_JSON") from None + _require(type(document) is dict and set(document) == _FIELDS, "FIELDS") + _require(canonical_json(document) == payload, "CANONICAL") + _require(document["protocol"] == BOUNDS_PROTOCOL, "PROTOCOL") + _require(_digest(document["budget_sha256"]), "BUDGET_DIGEST") + _require( + type(document["population"]) is str and 0 < len(document["population"]) <= 256, + "POPULATION", + ) + ids, indices = document["household_ids"], document["group_indices"] + _require(type(ids) is list and 0 < len(ids) <= MAX_ROWS, "ROW_COUNT") + _require( + all(type(v) is int and -(2**63) <= v < 2**63 for v in ids) + and all(a < b for a, b in zip(ids, ids[1:], strict=False)), + "ORDERED_IDS", + ) + _require(type(indices) is list and len(indices) == len(ids), "GROUP_INDICES") + raw_upper = document["group_upper_hex"] + _require(type(raw_upper) is list and 0 < len(raw_upper) <= len(ids), "GROUP_COUNT") + _require( + all(type(v) is int and 0 <= v < len(raw_upper) for v in indices), + "GROUP_INDICES", + ) + upper = _vector(raw_upper, len(raw_upper)) + incoming = _vector(document["incoming_hex"], len(ids)) + row_upper = _vector(document["row_upper_hex"], len(ids)) + grouped = group_bounds.GroupedUpperBounds(ids, indices, upper) + grouped.check(incoming, positive=False) + _require(np.all(incoming <= row_upper), "INITIAL_ROW_CAP") + _require(np.any(incoming > 0), "EMPTY_POSITIVE_SUPPORT") + return NumericSurveyBounds( + document["budget_sha256"], document["population"], grouped, incoming, row_upper + ) + + +def check_numeric_survey_weights(bounds, values): + """Independent numerical checks, including strict retained zero support.""" + _require(type(bounds) is NumericSurveyBounds, "BOUNDS_TYPE") + _require( + type(values) is np.ndarray + and values.dtype == np.dtype("float64") + and values.shape == bounds.incoming.shape, + "WEIGHT_ARRAY", + ) + bounds.grouped.check(values, positive=False) + _require(np.all(values <= bounds.row_upper), "ROW_REFERENCE_CAP") + _require(np.array_equal(values == 0, bounds.incoming == 0), "FIXED_ZERO_SUPPORT") + + +def survey_age_calibration_node( + registry, *, base, budget_node, count_node, epochs, learning_rate +): + frozen = demographic._registry_from_json(demographic._registry_json(registry)) + _require( + tuple(s.measure for s in frozen) == ages._COLUMNS + and all(s.metadata["evidence_scope"] == "invented" for s in frozen), + "INVENTED_AGE_REGISTRY_REQUIRED", + ) + return _survey_age_calibration_node( + frozen, + base=base, + budget_node=budget_node, + count_node=count_node, + epochs=epochs, + learning_rate=learning_rate, + ) + + +def survey_age_development_node( + registry, + *, + activation_binding, + base, + budget_node, + count_node, + epochs, + learning_rate, +): + """Declare source-documented numbers; the country runner activates sources.""" + frozen = age_activation.validate_survey_age_registry(registry, activation_binding) + return _survey_age_calibration_node( + frozen, + base=base, + budget_node=budget_node, + count_node=count_node, + epochs=epochs, + learning_rate=learning_rate, + activation_binding=activation_binding, + ) + + +def _survey_age_calibration_node( + frozen, + *, + base, + budget_node, + count_node, + epochs, + learning_rate, + activation_binding=None, +): + _require(type(epochs) is int and 1 <= epochs <= 1000, "EPOCHS") + _require( + type(learning_rate) in (int, float) + and np.isfinite(learning_rate) + and 0 < learning_rate <= 1, + "LEARNING_RATE", + ) + activation_params = {} + if activation_binding is not None: + declaration = age_activation.declaration_from_binding(activation_binding) + activation_params["activation"] = canonical_json( + age_activation.activation_binding(declaration) + ).decode() + return Node( + CALIBRATION_NODE, + SurveyAgeCalibrationKernel.ref, + base=base, + structural=StructuralDelta.REWEIGHT, + weights=WeightTransition("household", "calibrated", mass="free"), + mass="free", + params={ + **activation_params, + "registry": demographic._registry_json(frozen), + "epochs": epochs, + "learning_rate": learning_rate, + "weight_anchor": "source_qualified_sampling_reference", + "cap_enforcement": "mandatory_country_runner_cold_cache_and_final", + "grouped_preserve_zeros": True, + "method": "adam", + "seed": 0, + "l2_lambda": 0.0, + "l2_anchor": "initial", + "initial_anchor_meaning": "frozen_incoming_clone_weights", + }, + artifact_inputs=( + ArtifactInput("bounds", budget_node, "numeric_bounds", BOUNDS_TYPE), + ArtifactInput("counts", count_node, "counts", ages.COUNTS_TYPE), + ), + artifact_outputs=(ArtifactOutput("diagnostics", demographic.DIAGNOSTICS_TYPE),), + ) + + +def calibration_receipt_scope(params): + """Identify numerical evidence scope without claiming source admission.""" + return ( + "source_documented_survey_age_calibration_numbers_only" + if "activation" in params + else "invented_survey_age_calibration_numbers_only" + ) + + +def calibration_activation_binding(params): + """Decode the bounded canonical-text profile without acquiring its sources.""" + if "activation" not in params: + return None + value = params["activation"] + _require(type(value) is str and 0 < len(value.encode()) <= 65536, "ACTIVATION_TEXT") + try: + binding = json.loads(value) + encoded = canonical_json(binding).decode() + except (ValueError, TypeError, RecursionError, OverflowError): + raise ValueError("SURVEY_CALIBRATION_ACTIVATION_JSON") from None + _require(encoded == value, "ACTIVATION_CANONICAL") + age_activation.declaration_from_binding(binding) + return binding + + +def _artifact(context, name, type_, output): + value = context.artifacts[name] + _require( + value.type == type_ + and value.key == opaque_artifact_key(value.producer_key, output) + and type(value.payload) is bytes, + "ARTIFACT_EDGE", + ) + return value.payload + + +class SurveyAgeCalibrationKernel(KernelBase): + """Real fixed-zero grouped Adam on a minimal numeric work Frame.""" + + ref = "us.survey_age_calibration@1" + capabilities = Capabilities( + determinism=Determinism.DETERMINISTIC, + numeric=Numeric.PLATFORM_BITWISE, + seed_source=SeedSource.NONE, + structural=StructuralDelta.REWEIGHT, + consumes_se=False, + dependencies=("numpy", "pandas", "scipy", "torch"), + ) + + def implementation_hash(self): + return _sha( + canonical_json( + { + "adapter": source_hash( + sys.modules[__name__], + group_bounds, + dependencies=self.capabilities.dependencies, + ), + "age_artifact": ages.SurveyAgeCountArtifactKernel().implementation_hash(), + "activation_declaration": source_hash( + age_activation, + age_activation.age, + age_activation.survey_age_requests, + ), + "solver": demographic.DemographicCalibrationKernel().implementation_hash(), + } + ) + ) + + def run(self, context): + aliases = context.node.artifact_inputs + _require(tuple(a.name for a in aliases) == ("bounds", "counts"), "ALIASES") + registry = demographic._registry_from_json(context.params["registry"]) + factory = survey_age_calibration_node + profile_options = {} + if "activation" in context.params: + factory = survey_age_development_node + profile_options["activation_binding"] = calibration_activation_binding( + context.params + ) + expected = factory( + registry, + **profile_options, + base=context.node.base, + budget_node=aliases[0].producer, + count_node=aliases[1].producer, + epochs=context.params["epochs"], + learning_rate=context.params["learning_rate"], + ) + _require(context.node.normative() == expected.normative(), "DECLARATION") + _require(dict(context.params) == dict(expected.params), "PARAMETERS") + _require( + not context.tables and not context.weights and not context.sources, + "CONTEXT", + ) + bounds_raw = _artifact(context, "bounds", BOUNDS_TYPE, "numeric_bounds") + counts_raw = _artifact(context, "counts", ages.COUNTS_TYPE, "counts") + bounds = decode_numeric_survey_bounds(bounds_raw) + counts = ages.decode_survey_age_counts(counts_raw) + _require( + bounds.population == counts.population == context.node.base + and tuple(counts.household_ids) == bounds.grouped.household_ids, + "COMPLETE_ORDERED_POPULATION", + ) + table = counts.counts.copy(deep=True) + table.insert(0, "household_id", counts.household_ids.copy()) + incoming = Weights(bounds.incoming.copy(), WeightKind.IMPORTANCE) + work = demographic.kernels_module._frame_from_context( + replace( + context, tables={"household": table}, weights={"household": incoming} + ), + "household", + ) + result = calibrate( + work, + registry.to_target_set(), + weight_entity="household", + method="adam", + seed=0, + epochs=context.params["epochs"], + learning_rate=context.params["learning_rate"], + mass="free", + max_weight_ratio=None, + grouped_upper_bounds=bounds.grouped, + grouped_preserve_zeros=True, + l2_lambda=context.params["l2_lambda"], + l2_anchor=context.params["l2_anchor"], + ) + _require(not result.skipped, "SKIPPED_TARGETS") + weights = result.frame.weights_for("household") + accepted = weights.values.tobytes() + check_numeric_survey_weights(bounds, weights.values) + anchors = { + "budget_sha256": bounds.budget_sha256, + "numeric_bounds_sha256": _sha(bounds_raw), + "counts_sha256": _sha(counts_raw), + "accepted_weight_sha256": _sha(accepted), + "constraint_digest": bounds.grouped.digest, + "weight_anchor": expected.params["weight_anchor"], + "cap_enforcement": expected.params["cap_enforcement"], + "fixed_zero_rows": int(np.count_nonzero(bounds.incoming == 0)), + } + payload = canonical_json( + diagnostics_payload(result, target_registry=registry, build=anchors) + ) + output = KernelResult( + weights=weights, + artifacts={"diagnostics": payload}, + receipt={ + **anchors, + "diagnostics_sha256": _sha(payload), + "scope": calibration_receipt_scope(expected.params), + "release_eligible": False, + "source_admission": "required_from_country_runner", + }, + ) + # Validation follows diagnostics and return-object materialization. + fresh = decode_numeric_survey_bounds(bounds_raw) + check_numeric_survey_weights(fresh, weights.values) + _require(weights.values.tobytes() == accepted, "FINAL_WEIGHT_BYTES") + _require( + tuple(result.frame.table("household").household_id) + == fresh.grouped.household_ids, + "FINAL_HOUSEHOLD_IDS", + ) + return output diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/graph_survey_population.py b/packages/microcosm-build/src/microcosm/build/us_runtime/graph_survey_population.py new file mode 100644 index 000000000..624a088d8 --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/graph_survey_population.py @@ -0,0 +1,1259 @@ +"""Graph declarations for a source-authenticated, selected survey population. + +The catalogue draw precedes native construction. CREATE therefore materializes +the selected combined population, and a separate REWEIGHT applies source shares +and inverse inclusion. Neither decoded evidence nor these declarations grants +source, population, or release authority. +""" + +from __future__ import annotations + +import hashlib +import json +import math +from collections.abc import Sequence +from dataclasses import asdict, dataclass +from fractions import Fraction +from pathlib import Path +from types import SimpleNamespace + +import numpy as np +import pandas as pd + +from microcosm.frame import US_SCHEMA, MassChange, WeightKind, Weights +from microcosm.graph import ( + ArtifactInput, + ArtifactOutput, + ArtifactType, + Capabilities, + Determinism, + KernelBase, + KernelResult, + Node, + Numeric, + Owned, + Ownership, + Slice, + StructuralDelta, + WeightTransition, +) + +from . import graph_context +from .graph_context import US_FRAME_CONTEXT_TYPE +from .support_provenance import ( + spine_source_id_column, + support_channel_column, + support_clone_index_column, + support_source_id_column, +) +from .survey_catalogue_selection import CatalogueSelectionPlan, SelectedHousehold +from .survey_population_domains import HouseholdKey, Source + +PHASE = "us.authenticated_survey_population.v1" +SOURCE_NAME = "survey_population_source" +SOURCE_CODEC = "us-survey-population-source-v1" +CREATE_NODE = "survey_population.create" +ALLOCATION_NODE = "survey_population.allocate" +CREATE_REF = "us.survey_population.create@1" +ALLOCATION_REF = "us.survey_population.allocate@1" +PREPARATION_TYPE = ArtifactType("microcosm.us.survey_population_preparation", 2) +ALLOCATION_TYPE = ArtifactType("microcosm.us.survey_population_allocation", 1) +PREPARATION_MAX_BYTES = 64 * 1024**2 +ALLOCATION_MAX_BYTES = 64 * 1024**2 +STAGE = "authenticated_survey_population_v1" +STAGE_DEPENDENCIES = ( + "numpy", + "pandas", + "microunit", + "tables", + "h5py", + "PyYAML", + "pyarrow", +) +_HOUSEHOLD_ORIGIN_FIELDS = frozenset( + { + "household_id", + "source", + "source_year", + "survey_year", + "raw_native_id", + "selected_receiving_household_id", + "original_anchor", + } +) + + +@dataclass(frozen=True, slots=True) +class AllocationInstruction: + """Exact arithmetic values, without source or population authority.""" + + household_id: int + key: HouseholdKey + selected_receiving_household_id: int + original_anchor: Fraction + share: Fraction + inclusion_probability: Fraction + + @property + def importance_weight(self): + return self.original_anchor * self.share / self.inclusion_probability + + +class SurveyPopulationGraphError(ValueError): + """Static graph refusal without source values or local paths.""" + + +def _require(condition, code): + if not condition: + raise SurveyPopulationGraphError(code) + + +def _digest(value): + _require( + type(value) is str + and len(value) == 64 + and not set(value) - set("0123456789abcdef"), + "DIGEST", + ) + return value + + +def _draw_parameters(fraction, seed): + _require(type(fraction) is Fraction and 0 < fraction <= 1, "FRACTION") + _require(type(seed) is int and 0 <= seed < 2**64, "SEED") + return (fraction.numerator, fraction.denominator), seed + + +def _fraction_pair(value): + _require( + type(value) in (list, tuple) + and len(value) == 2 + and all(type(v) is int for v in value) + and value[1] > 0, + "FRACTION_PAIR", + ) + number = Fraction(*value) + _require((number.numerator, number.denominator) == tuple(value), "FRACTION_PAIR") + return number + + +def allocation_instructions(plan, household_origins): + """Join checked preparation values; this pure join authenticates nothing. + + The runner obtains both arguments from a freshly issued preparation. Keeping + this operation separate lets the allocation kernel hold immutable values + instead of a source Frame or a mutable decoded preparation document. + """ + _require(type(plan) is CatalogueSelectionPlan, "SELECTION_PLAN") + _require(type(plan.selected) is tuple and plan.selected, "EMPTY_SELECTION") + _require(type(household_origins) in (list, tuple), "HOUSEHOLD_ORIGINS") + _require(len(household_origins) == len(plan.selected), "ORIGIN_COUNT") + # A prospective record bound precedes the two maps and output tuple. The + # streaming transport encoder enforces its exact bound separately. + _require(len(plan.selected) <= ALLOCATION_MAX_BYTES // 128, "ALLOCATION_LIMIT") + selected = {} + for row in plan.selected: + _require( + type(row) is SelectedHousehold and type(row.key) is HouseholdKey, + "SELECTED_TYPE", + ) + _require( + type(row.key.source) is Source + and type(row.key.source_year) is int + and row.key.source_year == 2024 + and type(row.key.survey_year) is int + and row.key.survey_year == (2024 if row.key.source is Source.ACS else 2025) + and type(row.key.native_id) is str + and 0 < len(row.key.native_id) <= 128, + "SELECTED_KEY", + ) + _require(row.key not in selected, "SELECTED_DUPLICATE") + _require( + type(row.original_design_weight) is Fraction + and row.original_design_weight >= 0 + and type(row.share) is Fraction + and 0 < row.share <= 1 + and type(row.inclusion_probability) is Fraction + and 0 < row.inclusion_probability <= 1, + "SELECTED_ARITHMETIC", + ) + selected[row.key] = row + output, seen_ids, seen_native, seen_receiving = [], set(), set(), set() + for origin in household_origins: + _require( + type(origin) is dict and set(origin) == _HOUSEHOLD_ORIGIN_FIELDS, + "ORIGIN_FIELDS", + ) + _require( + type(origin["source"]) is str and origin["source"] in {"acs", "asec"}, + "ORIGIN_SOURCE", + ) + _require( + all( + type(origin[field]) is int + for field in ( + "household_id", + "source_year", + "survey_year", + "selected_receiving_household_id", + ) + ), + "ORIGIN_INTEGER", + ) + _require( + all( + 0 <= origin[field] < 2**63 + for field in ("household_id", "selected_receiving_household_id") + ), + "ORIGIN_INTEGER", + ) + _require(type(origin["raw_native_id"]) is str, "ORIGIN_NATIVE_KEY") + key = HouseholdKey( + Source(origin["source"]), + origin["source_year"], + origin["survey_year"], + origin["raw_native_id"], + ) + row = selected.get(key) + _require(row is not None, "ORIGIN_NATIVE_KEY") + anchor = _fraction_pair(origin["original_anchor"]) + _require(anchor == row.original_design_weight, "ORIGIN_ANCHOR") + hh_id = origin["household_id"] + receiving = (key.source, origin["selected_receiving_household_id"]) + _require( + hh_id not in seen_ids + and key not in seen_native + and receiving not in seen_receiving, + "ORIGIN_DUPLICATE", + ) + seen_ids.add(hh_id) + seen_native.add(key) + seen_receiving.add(receiving) + instruction = AllocationInstruction( + hh_id, key, receiving[1], anchor, row.share, row.inclusion_probability + ) + _finite_weight(anchor) + _finite_weight(instruction.importance_weight) + output.append(instruction) + return tuple(output) + + +def _finite_weight(value): + try: + number = float(value) + except (OverflowError, ValueError): + raise SurveyPopulationGraphError("FLOAT_WEIGHT") from None + _require( + math.isfinite(number) and number >= 0 and (value == 0 or number > 0), + "FLOAT_WEIGHT", + ) + return number + + +def _bounded_json(value, limit): + """Canonical UTF-8 encoding with a bound checked before every append.""" + _require(type(limit) is int and 0 < limit <= 64 * 1024**2, "TRANSPORT_LIMIT") + result = bytearray() + encoder = json.JSONEncoder( + sort_keys=True, separators=(",", ":"), ensure_ascii=False, allow_nan=False + ) + try: + for piece in encoder.iterencode(value): + # All variable source tokens admitted to allocation are <=128 chars; + # this pre-encoding bound also protects against unexpected strings. + _require(len(piece) <= limit - len(result), "TRANSPORT_LIMIT") + encoded = piece.encode("utf-8") + _require(len(encoded) <= limit - len(result), "TRANSPORT_LIMIT") + result.extend(encoded) + except SurveyPopulationGraphError: + raise + except (TypeError, ValueError, UnicodeError): + raise SurveyPopulationGraphError("TRANSPORT_ENCODING") from None + return bytes(result) + + +def _source_owner(): + # The owner remains independent of this graph adapter. Declaration-only + # imports do not load source capture owners or their runtime resources. + from . import survey_population_preparation + + return survey_population_preparation + + +def _sha(payload): + return hashlib.sha256(payload).hexdigest() + + +def _checked_preparation(preparation): + owner = _source_owner() + _require( + type(preparation) is owner.AuthenticatedSurveyPopulationPreparation, + "PREPARATION_TYPE", + ) + view = owner.AuthenticatedSurveyPopulationPreparation.checked_view(preparation) + payload, context = view.payload, view.context + _require( + type(payload) is bytes and 0 < len(payload) <= PREPARATION_MAX_BYTES, + "PREPARATION_BYTES", + ) + _require( + type(context) is bytes and 0 < len(context) <= PREPARATION_MAX_BYTES, + "CONTEXT_BYTES", + ) + return owner, view + + +def _artifact(context, name, expected_type, expected_payload): + from microcosm.graph.keys import opaque_artifact_key + + value = context.artifacts.get(name) + edges = [edge for edge in context.node.artifact_inputs if edge.name == name] + _require( + value is not None + and value.type == expected_type + and len(edges) == 1 + and edges[0].producer == CREATE_NODE + and edges[0].artifact == name + and value.key == opaque_artifact_key(value.producer_key, name) + and type(value.payload) is bytes + and value.payload == expected_payload, + "ARTIFACT_BINDING", + ) + return value + + +def _instruction_document(row): + def pair(value): + return [value.numerator, value.denominator] + + return { + "household_id": row.household_id, + "source": row.key.source.value, + "source_year": row.key.source_year, + "survey_year": row.key.survey_year, + "raw_native_id": row.key.native_id, + "selected_receiving_household_id": row.selected_receiving_household_id, + "original_anchor": pair(row.original_anchor), + "share": pair(row.share), + "inclusion_probability": pair(row.inclusion_probability), + "importance_multiplier": pair(row.share / row.inclusion_probability), + "importance_weight_float64_hex": _finite_weight(row.importance_weight).hex(), + } + + +def _allocation_payload( + instructions, *, preparation_sha256, input_context, output_context +): + """Stream rows individually; never construct a selected-population JSON tree.""" + metadata = { + "protocol": "microcosm.us.survey-population-allocation.v1", + "preparation_sha256": preparation_sha256, + "input_context_sha256": _sha(input_context), + "output_context_sha256": _sha(output_context), + "weight_transition": ["design", "importance"], + "release_eligible": False, + } + tail = b"]," + _bounded_json(metadata, 4096)[1:] + output = bytearray(b'{"households":[') + for index, row in enumerate(instructions): + raw = _bounded_json(_instruction_document(row), 4096) + separator = b"," if index else b"" + _require( + len(output) + len(separator) + len(raw) + len(tail) <= ALLOCATION_MAX_BYTES, + "ALLOCATION_LIMIT", + ) + output.extend(separator) + output.extend(raw) + _require(len(output) + len(tail) <= ALLOCATION_MAX_BYTES, "ALLOCATION_LIMIT") + output.extend(tail) + return bytes(output) + + +def _verify_allocation_view(frame, instructions): + _require(frame.weighted_entities == ("household",), "WEIGHT_SOURCES") + weights = frame.weights_for("household") + _require(weights.kind is WeightKind.DESIGN, "DESIGN_REQUIRED") + household = frame.table("household") + _require(len(household) == len(instructions), "HOUSEHOLD_COUNT") + _require(household.household_id.dtype == np.dtype("int64"), "HOUSEHOLD_ID_DTYPE") + _require( + tuple(household.household_id) == tuple(r.household_id for r in instructions), + "HOUSEHOLD_ORDER", + ) + _require( + tuple(household[support_channel_column("household")]) + == tuple(r.key.source.value for r in instructions), + "HOUSEHOLD_CHANNEL", + ) + _require( + tuple(household[spine_source_id_column("household")]) + == tuple(r.selected_receiving_household_id for r in instructions), + "RECEIVING_ID", + ) + for entity in US_SCHEMA.entities: + table = frame.table(entity) + _require( + table[support_clone_index_column(entity)].dtype == np.dtype("int64") + and bool(table[support_clone_index_column(entity)].eq(0).all()), + "PRECLONE_REQUIRED", + ) + expected = np.asarray( + [_finite_weight(r.original_anchor) for r in instructions], dtype="float64" + ) + _require( + weights.values.dtype == expected.dtype + and weights.values.tobytes() == expected.tobytes(), + "ORIGINAL_ANCHORS", + ) + + +def _allocation_output(frame, context_bytes, instructions, preparation_sha256): + _verify_allocation_view(frame, instructions) + values = np.asarray( + [_finite_weight(r.importance_weight) for r in instructions], dtype="float64" + ) + weights = Weights(values, WeightKind.IMPORTANCE) + allocated = frame.with_weights( + "household", + weights, + mass=MassChange( + factor=None, reason=f"{PHASE}: declared source shares and inverse inclusion" + ), + ) + context = graph_context._decode(context_bytes) + _require( + context["weight_sources"] == {"household": "design"}, "CONTEXT_WEIGHT_KIND" + ) + context["weight_sources"] = {"household": "importance"} + context["mass_log"] = [ + graph_context._json_data(asdict(r)) for r in allocated.mass_log + ] + new_context = _bounded_json(context, PREPARATION_MAX_BYTES) + payload = _allocation_payload( + instructions, + preparation_sha256=preparation_sha256, + input_context=context_bytes, + output_context=new_context, + ) + before, after = frame.stratum_mass(), allocated.stratum_mass() + receipt = { + "phase": PHASE, + "preparation_sha256": preparation_sha256, + "allocation_sha256": _sha(payload), + "release_eligible": False, + "frame_mass_log_append": [ + graph_context._json_data(asdict(r)) + for r in allocated.mass_log[len(frame.mass_log) :] + ], + "mass": { + "policy": "declared", + "before": float(before.sum()), + "after": float(after.sum()), + "stratum_before": before.to_dict(), + "stratum_after": after.to_dict(), + }, + } + return weights, new_context, payload, receipt, allocated + + +class _Kernel(KernelBase): + def implementation_hash(self): + from .graph_implementation import ( + STAGE_DEPENDENCIES as STAGES, + ) + from .graph_implementation import ( + implementation_hash, + ) + + _require(STAGES[STAGE] == STAGE_DEPENDENCIES, "STAGE_DEPENDENCIES") + return implementation_hash(STAGE) + + +class SurveyPopulationCreateKernel(_Kernel): + ref = CREATE_REF + capabilities = Capabilities( + determinism=Determinism.DETERMINISTIC, + numeric=Numeric.PLATFORM_BITWISE, + structural=StructuralDelta.CREATE, + dependencies=STAGE_DEPENDENCIES, + ) + + def __init__(self, preparation, *, source_dir): + self._preparation = preparation + self._source_dir = Path(source_dir).absolute() + + def run(self, context): + from .graph_sources import frame_column_declarations + + preparation = self._preparation + owner, view = _checked_preparation(preparation) + payload, context_bytes, plan = view.payload, view.context, view.selection_plan + expected, _ = survey_population_nodes( + frame_column_declarations(view.frame), + preparation_sha256=_sha(payload), + fraction=plan.fraction, + seed=plan.seed, + ) + _require( + context.node == expected and dict(context.params) == dict(expected.params), + "CREATE_DECLARATION", + ) + _require( + set(context.sources) == {SOURCE_NAME} + and Path(context.sources[SOURCE_NAME]) + == self._source_dir.resolve(strict=True), + "SOURCE_PATH", + ) + owner.verify_survey_population_preparation(preparation) + return KernelResult( + frame=view.frame, + artifacts={"frame_context": context_bytes, "preparation": payload}, + receipt={ + "phase": PHASE, + "preparation_sha256": _sha(payload), + "source_reconstruction": "fresh_before_graph_execution", + "release_eligible": False, + }, + ) + + +class SurveyPopulationAllocationKernel(_Kernel): + ref = ALLOCATION_REF + capabilities = Capabilities( + determinism=Determinism.DETERMINISTIC, + numeric=Numeric.PLATFORM_BITWISE, + structural=StructuralDelta.REWEIGHT, + dependencies=STAGE_DEPENDENCIES, + ) + + def __init__(self, instructions, *, preparation_bytes, context_bytes): + _require( + type(instructions) is tuple + and instructions + and all(type(row) is AllocationInstruction for row in instructions), + "INSTRUCTIONS", + ) + _require( + type(preparation_bytes) is bytes + and 0 < len(preparation_bytes) <= PREPARATION_MAX_BYTES, + "PREPARATION_BYTES", + ) + _require( + type(context_bytes) is bytes + and 0 < len(context_bytes) <= PREPARATION_MAX_BYTES, + "CONTEXT_BYTES", + ) + self._instructions = instructions + self._preparation_bytes = preparation_bytes + self._context_bytes = context_bytes + + def run(self, context): + node = context.node + digest = _sha(self._preparation_bytes) + _require( + node.id == ALLOCATION_NODE + and node.kernel == self.ref + and node.structural is StructuralDelta.REWEIGHT + and node.base == CREATE_NODE + and node.mass == "declared" + and not node.outputs + and not node.sources + and node.weights + == WeightTransition("household", "importance", mass="declared") + and node.inputs + == tuple(Slice(e, _provenance_columns(e)) for e in US_SCHEMA.entities) + and node.artifact_inputs + == ( + ArtifactInput( + "frame_context", CREATE_NODE, "frame_context", US_FRAME_CONTEXT_TYPE + ), + ArtifactInput( + "preparation", CREATE_NODE, "preparation", PREPARATION_TYPE + ), + ) + and node.artifact_outputs + == ( + ArtifactOutput("frame_context", US_FRAME_CONTEXT_TYPE), + ArtifactOutput("allocation", ALLOCATION_TYPE), + ) + and dict(context.params) == {"phase": PHASE, "preparation_sha256": digest}, + "ALLOCATION_DECLARATION", + ) + _require( + set(context.artifacts) == {"frame_context", "preparation"}, "ARTIFACTS" + ) + bound = _artifact( + context, "frame_context", US_FRAME_CONTEXT_TYPE, self._context_bytes + ) + prepared = _artifact( + context, "preparation", PREPARATION_TYPE, self._preparation_bytes + ) + _require(bound.producer_key == prepared.producer_key, "ARTIFACT_PRODUCER") + original = graph_context.us_frame_from_context(context) + weights, new_context, payload, receipt, _allocated = _allocation_output( + original, bound.payload, self._instructions, digest + ) + return KernelResult( + weights=weights, + artifacts={"frame_context": new_context, "allocation": payload}, + receipt=receipt, + ) + + +def survey_population_source_codecs(*, snapshot_root): + """One real source codec, isolated from the legacy five-codec registry.""" + from microcosm.graph.codecs import SourceCodecRegistry + + snapshot_root = Path(snapshot_root) + + def load(path, *, store=None): + del store + owner = _source_owner() + fraction, seed = owner.read_survey_population_request(path) + preparation = owner.prepare_authenticated_survey_population( + path, snapshot_root=snapshot_root, fraction=fraction, seed=seed + ) + owner, view = _checked_preparation(preparation) + owner.verify_survey_population_preparation(preparation) + return view.frame + + codecs = SourceCodecRegistry() + codecs.register(SOURCE_CODEC, load) + return codecs + + +def _same_frame(expected, actual): + """Full semantic identity; mutable-storage source seals stay in their owner.""" + _require( + expected.schema == actual.schema + and expected.entities == actual.entities + and expected.links == actual.links + and expected.metadata == actual.metadata + and expected.mass_log == actual.mass_log + and expected.weighted_entities == actual.weighted_entities, + "MATERIALIZED_FRAME_CONTEXT", + ) + try: + for entity in expected.entities: + pd.testing.assert_frame_equal( + expected.table(entity), + actual.table(entity), + check_exact=True, + check_flags=True, + ) + for link in expected.links: + pd.testing.assert_frame_equal( + expected.link(link), + actual.link(link), + check_exact=True, + check_flags=True, + ) + pd.testing.assert_series_equal(expected.strata, actual.strata, check_exact=True) + except (AssertionError, ValueError, TypeError): + raise SurveyPopulationGraphError("MATERIALIZED_FRAME_VALUES") from None + for entity in expected.weighted_entities: + left, right = expected.weights_for(entity), actual.weights_for(entity) + _require( + left.kind is right.kind + and left.values.dtype == right.values.dtype + and left.values.tobytes() == right.values.tobytes(), + "MATERIALIZED_FRAME_WEIGHTS", + ) + + +def _verify_cloned_frame(before, actual, design_weights): + from . import puf_support + + _require( + before.schema == actual.schema + and before.entities == actual.entities + and before.links == actual.links == () + and before.metadata == actual.metadata + and before.mass_log == actual.mass_log + and actual.weighted_entities == ("household",), + "CLONE_CONTEXT", + ) + multiplier = puf_support._id_multiplier_for_frame(before) + try: + # Execute the real operator's row transform one entity at a time. This + # checks every carried cell, missing value, dtype, native key and remapped + # membership without constructing another full cloned Frame. + for entity in before.entities: + expected = puf_support._clone_preassembled_entity_table( + before.table(entity), + entity=entity, + schema=before.schema, + id_multiplier=multiplier, + ) + pd.testing.assert_frame_equal( + expected, actual.table(entity), check_exact=True, check_flags=True + ) + expected_strata = pd.concat([before.strata, before.strata], ignore_index=True) + pd.testing.assert_series_equal(expected_strata, actual.strata, check_exact=True) + except (AssertionError, ValueError, TypeError): + raise SurveyPopulationGraphError("CLONE_FRAME_VALUES") from None + weights = actual.weights_for("household") + expected_weights = np.tile(before.weights_for("household").values / 2, 2) + _require( + weights.kind is WeightKind.IMPORTANCE + and weights.values.tobytes() == expected_weights.tobytes(), + "CLONE_WEIGHTS", + ) + # The graph duplicates original design anchors along lineage. It halves + # current IMPORTANCE weights, never the original design evidence. + expected_design = np.tile(design_weights, 2) + puf_support.validate_puf_clone_attachment( + actual, boundary=PHASE, expected_fraction=1.0, expected_seed=0 + ) + return expected_design + + +def _check_design_anchors(population, expected): + _require(set(population.design_weights) == {"household"}, "DESIGN_ANCHOR_SOURCES") + actual = population.design_weights["household"] + _require( + type(actual) is np.ndarray + and actual.dtype == expected.dtype + and actual.shape == expected.shape + and actual.tobytes() == expected.tobytes(), + "DESIGN_ANCHOR_LINEAGE", + ) + + +def _check_population_state(population, *, version, owners, kind, ledger): + from microcosm.graph.population import MassRecord + + _require( + type(population.version) is str and population.version == version, + "POPULATION_VERSION", + ) + _require( + dict(population.owners) == owners + and all(type(value) is str for value in population.owners.values()), + "POPULATION_OWNERS", + ) + _require( + dict(population.weight_kind) == {"household": kind} + and type(population.weight_kind["household"]) is WeightKind, + "POPULATION_WEIGHT_KIND", + ) + _require( + type(population.mass_ledger) is tuple + and all(type(record) is MassRecord for record in population.mass_ledger) + and population.mass_ledger == ledger, + "POPULATION_MASS_LEDGER", + ) + + +def _final_artifact(manifest, store, *, node_id, name, type_, payload, capabilities): + from microcosm.graph.artifact_edges import descriptor + from microcosm.graph.keys import opaque_artifact_key + + node = manifest.node(node_id) + expected = descriptor( + producer=node_id, + artifact=name, + type_=type_, + producer_key=node.key, + capabilities=capabilities, + ) + _require( + node.typed_artifacts["outputs"].get(name) == expected, + "FINAL_ARTIFACT_DESCRIPTOR", + ) + key = opaque_artifact_key(node.key, name) + _require(node.opaque_artifacts.get(name) == key, "FINAL_ARTIFACT_KEY") + actual = store.load_bytes(key) + _require(type(actual) is bytes and actual == payload, "FINAL_ARTIFACT_BYTES") + + +def _check_node_states(manifest, expected): + from microcosm.graph.keys import _capabilities_projection + + _require(set(manifest.nodes) == set(expected), "MANIFEST_NODE_SET") + for node_id, state in expected.items(): + node = manifest.node(node_id) + _require( + node.key == state["key"] + and node.kernel_ref == state["ref"] + and type(node.capabilities) is Capabilities + and _capabilities_projection(node.capabilities) == state["capabilities"] + and node.kernel_impl_hash == state["implementation"] + and node.typed_artifacts == state["typed_artifacts"] + and type(node.seed) is int + and node.seed == state["seed"] + and node.frame_key == state["frame_key"] + and node.weight_key == state["weight_key"] + and node.artifacts == state["artifacts"] + and node.opaque_artifacts == state["opaque_artifacts"] + and node.receipt == state["receipt"] + and node.legacy_capabilities is False, + "MANIFEST_NODE_STATE", + ) + + +@dataclass(frozen=True) +class SurveyPopulationRunValues: + """Values retained from one completed run, without additional authority. + + These are the actual objects used by the runner, not decoded substitutes. + Downstream source-qualified boundaries must check the issued preparation + and complete populations themselves. Freezing this container does not seal + its contents, authenticate a copy, or admit a later graph descendant. + """ + + manifest: object + preparation: object + allocated_population: object + clone_population: object | None + compiled: object + store: object + kernels: object + sources: dict + + +def run_authenticated_survey_population( + source_dir, + *, + snapshot_root, + store_root, + fraction, + seed, + resume="auto", + clones=True, + return_values=False, +): + """Reconstruct sources, then execute and verify the real selected graph. + + Warm execution can skip graph kernels; it still reconstructs both complete + catalogues and the selected native populations. Returned manifests are + development build records. Decoding their bytes grants no source authority. + ``return_values=True`` returns the live composition values with the manifest; + it does not bypass any final checks or issue a downstream admission. + """ + from microcosm.graph import ( + ContentStore, + Graph, + KernelRegistry, + SourceRef, + compile_graph, + run_graph, + ) + from microcosm.graph.artifact_edges import typed_contracts + from microcosm.graph.executor import ( + _all_node_keys, + _expand_declared_payload, + _expand_rewrite_coordinates, + _input_writers, + _source_paths_and_keys, + _tolerance_writer_payload, + ) + from microcosm.graph.keys import ( + _capabilities_projection, + artifact_key, + frame_key, + opaque_artifact_key, + weights_key, + ) + from microcosm.graph.keys import seed as node_seed + from microcosm.graph.manifest import _freeze_json + from microcosm.graph.population import ( + _mass_record, + expand_lineage_receipt, + expand_writes_receipt, + mass_record_receipt, + weight_cap_receipt, + ) + + from .graph_sources import frame_column_declarations + + _draw_parameters(fraction, seed) + _require(type(clones) is bool, "CLONES_FLAG") + _require(type(return_values) is bool, "RETURN_VALUES_FLAG") + # Do not resolve source symlinks before their authenticated owner checks them. + source_dir, snapshot_root, store_root = ( + Path(path).absolute() for path in (source_dir, snapshot_root, store_root) + ) + owner = _source_owner() + preparation = owner.prepare_authenticated_survey_population( + source_dir, snapshot_root=snapshot_root, fraction=fraction, seed=seed + ) + owner, view = _checked_preparation(preparation) + # Resolve '..' only after the source owner has checked the raw path and + # refused symlinks. The executor uses this same canonical path identity. + source_dir = source_dir.resolve(strict=True) + payload, context_bytes, prepared_frame = view.payload, view.context, view.frame + instructions = allocation_instructions( + view.selection_plan, view.receipt["origins"]["households"] + ) + columns = frame_column_declarations(prepared_frame) + nodes = survey_population_nodes( + columns, preparation_sha256=_sha(payload), fraction=fraction, seed=seed + ) + _weights, allocated_context, allocation_payload, allocation_receipt, allocated = ( + _allocation_output(prepared_frame, context_bytes, instructions, _sha(payload)) + ) + design = np.array(prepared_frame.weights_for("household").values, copy=True) + kernels = KernelRegistry() + kernels.register(SurveyPopulationCreateKernel(preparation, source_dir=source_dir)) + kernels.register( + SurveyPopulationAllocationKernel( + instructions, preparation_bytes=payload, context_bytes=context_bytes + ) + ) + clone_nodes = () + if clones: + from . import graph_combined_clone as clone + from . import puf_support + from .graph_combined_clone import ( + register_us_combined_survey_clone_kernels, + us_combined_survey_clone_nodes, + ) + + clone_nodes = us_combined_survey_clone_nodes( + columns, base=ALLOCATION_NODE, source_channels=("acs", "asec") + ) + register_us_combined_survey_clone_kernels(kernels) + compiled = compile_graph( + Graph("us", (SourceRef(SOURCE_NAME, SOURCE_CODEC),), (*nodes, *clone_nodes)) + ) + store = ContentStore( + store_root, codecs=survey_population_source_codecs(snapshot_root=snapshot_root) + ) + observed = {} + expected_states = {} + expected_writer_receipts = {} + allocation_ledger = ( + _mass_record( + prepared_frame, + allocated, + nodes[1], + KernelResult(receipt=allocation_receipt), + "declared", + ), + ) + all_cells = tuple( + (entity, str(column)) + for entity in prepared_frame.entities + for column in prepared_frame.table(entity) + ) + + def observe(node_id, population): + owner.verify_survey_population_preparation(preparation) + if node_id == CREATE_NODE: + _same_frame(prepared_frame, population.frame) + expected_design = design + owners = dict.fromkeys(all_cells, CREATE_NODE) + kind, ledger = WeightKind.DESIGN, () + elif node_id == ALLOCATION_NODE: + _same_frame(allocated, population.frame) + expected_design = design + owners = dict.fromkeys(all_cells, ALLOCATION_NODE) + kind, ledger = WeightKind.IMPORTANCE, allocation_ledger + elif node_id in {node.id for node in clone_nodes}: + expected_design = _verify_cloned_frame(allocated, population.frame, design) + owners = dict.fromkeys(all_cells, clone_nodes[0].id) + if node_id == clone_nodes[1].id: + owners.update( + {(o.entity, o.column): node_id for o in clone_nodes[1].outputs} + ) + # Full cloned cells have been checked against the real operator row + # transform. Derive mass from these values, never the cached ledger. + kind = WeightKind.IMPORTANCE + ledger = ( + *allocation_ledger, + _mass_record( + allocated, + population.frame, + clone_nodes[0], + KernelResult(), + "conserve", + ), + ) + else: + raise SurveyPopulationGraphError("UNEXPECTED_POPULATION") + _check_design_anchors(population, expected_design) + state = dict( + version=compiled.versions[node_id], owners=owners, kind=kind, ledger=ledger + ) + _check_population_state(population, **state) + node = compiled.graph.node(node_id) + if node_id == CREATE_NODE: + receipt = { + "phase": PHASE, + "preparation_sha256": _sha(payload), + "source_reconstruction": "fresh_before_graph_execution", + "release_eligible": False, + } + elif node_id == ALLOCATION_NODE: + receipt = dict(allocation_receipt) + elif node_id == clone_nodes[0].id: + lineage, entity_facts = {}, {} + for entity in US_SCHEMA.entities: + lineage[entity], entity_facts[entity] = clone._entity_lineage( + allocated, population.frame, entity + ) + authority = puf_support.validate_puf_clone_attachment( + population.frame, boundary=PHASE, expected_fraction=1.0, expected_seed=0 + ) + receipt = clone.USCombinedSurveyCloneExpandKernel._receipt( + allocated, population.frame, ("acs", "asec"), authority, entity_facts + ) + receipt["expand"] = expand_lineage_receipt(lineage) + receipt["expand_declared"] = _expand_declared_payload(node) + receipt["expand_writes"] = expand_writes_receipt( + allocated, + population.frame, + node, + receipt, + rewrite_coordinates=_expand_rewrite_coordinates(compiled, node), + ) + else: + receipt = { + "phase": clone.COMBINED_CLONE_PHASE, + "claimed_cells": sorted( + f"{output.entity}.{output.column}" for output in node.outputs + ), + } + receipt["capabilities"] = dict(expected_nodes[node_id]["capabilities"]) + writers = _tolerance_writer_payload( + _input_writers(compiled, node_id, receipts=expected_writer_receipts) + ) + if writers: + receipt["capabilities"]["tolerance_writers"] = writers + if node.structural not in {StructuralDelta.NONE, StructuralDelta.CREATE}: + receipt["mass"] = { + **receipt.get("mass", {}), + **mass_record_receipt(ledger[-1]), + } + receipt.update(weight_cap_receipt(population, node)) + expected_nodes[node_id]["receipt"] = _freeze_json(receipt) + # Value-only input for the generic writer analysis. This is neither a + # returned NodeReceipt nor a source/Frame authority object. + expected_writer_receipts[node_id] = SimpleNamespace( + receipt=expected_nodes[node_id]["receipt"] + ) + _require(node_id not in observed, "DUPLICATE_POPULATION_OBSERVATION") + observed[node_id] = population + expected_states[node_id] = state + + # Derive expectations independently of returned/cache receipts. This is one + # additional streaming source-key pass, not another draw or native build. + _paths, source_keys = _source_paths_and_keys( + compiled, {SOURCE_NAME: source_dir}, store + ) + keys, implementations = _all_node_keys(compiled, kernels, source_keys) + expected_nodes = {} + for node in compiled.graph.nodes: + key = keys[node.id] + structural = node.structural is not StructuralDelta.NONE + cells = ( + all_cells + if structural + else tuple((output.entity, output.column) for output in node.outputs) + ) + weight_entity = ( + node.weights.entity + if node.weights is not None + else node.params.get("expand_weight_entity") + if node.structural is StructuralDelta.EXPAND + else None + ) + expected_nodes[node.id] = { + "key": key, + "ref": node.kernel, + "capabilities": _capabilities_projection( + kernels.get(node.kernel).capabilities + ), + "implementation": implementations[node.id], + "typed_artifacts": typed_contracts(compiled, node, keys, kernels), + "seed": node_seed(key), + "frame_key": frame_key(key) if structural else None, + "weight_key": weights_key(key, weight_entity) if weight_entity else None, + "artifacts": {(e, c): artifact_key(key, e, c) for e, c in cells}, + "opaque_artifacts": { + output.name: opaque_artifact_key(key, output.name) + for output in node.artifact_outputs + }, + } + manifest = run_graph( + compiled, + sources={SOURCE_NAME: source_dir}, + store=store, + kernels=kernels, + resume=resume, + _population_observer=observe, + ) + _require(tuple(observed) == compiled.order, "POPULATION_OBSERVER_COVERAGE") + _check_node_states(manifest, expected_nodes) + # Explicit store reads are necessary: the observer sees real populations, + # while typed byte artifacts pass through a different materialization path. + for node_id, name, type_, raw, capabilities in ( + ( + CREATE_NODE, + "preparation", + PREPARATION_TYPE, + payload, + SurveyPopulationCreateKernel.capabilities, + ), + ( + CREATE_NODE, + "frame_context", + US_FRAME_CONTEXT_TYPE, + context_bytes, + SurveyPopulationCreateKernel.capabilities, + ), + ( + ALLOCATION_NODE, + "allocation", + ALLOCATION_TYPE, + allocation_payload, + SurveyPopulationAllocationKernel.capabilities, + ), + ( + ALLOCATION_NODE, + "frame_context", + US_FRAME_CONTEXT_TYPE, + allocated_context, + SurveyPopulationAllocationKernel.capabilities, + ), + ): + _final_artifact( + manifest, + store, + node_id=node_id, + name=name, + type_=type_, + payload=raw, + capabilities=capabilities, + ) + # Construct the optional value container before terminal checks. A caller + # still needs the downstream issuer's independent reconstruction; this is + # deliberately not another source certificate or a manifest decoder. + result = ( + SurveyPopulationRunValues( + manifest=manifest, + preparation=preparation, + allocated_population=observed[ALLOCATION_NODE], + clone_population=observed[clone_nodes[-1].id] if clone_nodes else None, + compiled=compiled, + store=store, + kernels=kernels, + sources={SOURCE_NAME: source_dir}, + ) + if return_values + else manifest + ) + # Finish all potentially expensive owner I/O before the final pure checks + # of graph-owned Frames. A callback during the last producer/source check + # must not mutate an already-observed derivative and escape detection. + _owner, final_view = _checked_preparation(preparation) + _require( + final_view.payload == payload + and final_view.context == context_bytes + and final_view.frame is prepared_frame, + "FINAL_PREPARATION_SEAL", + ) + _same_frame(prepared_frame, manifest.population(CREATE_NODE)) + _same_frame(allocated, manifest.population(ALLOCATION_NODE)) + for node_id in (CREATE_NODE, ALLOCATION_NODE): + _check_design_anchors(observed[node_id], design) + for node in clone_nodes: + cloned_design = _verify_cloned_frame( + allocated, manifest.population(compiled.versions[node.id]), design + ) + _check_design_anchors(observed[node.id], cloned_design) + for node_id, state in expected_states.items(): + _check_population_state(observed[node_id], **state) + _require( + manifest.mass_ledger(compiled.versions[node_id]) == state["ledger"], + "FINAL_MASS_LEDGER", + ) + _check_node_states(manifest, expected_nodes) + return result + + +def _provenance_columns(entity): + return ( + support_channel_column(entity), + support_source_id_column(entity), + spine_source_id_column(entity), + support_clone_index_column(entity), + ) + + +def survey_population_nodes( + columns: Sequence[Owned], *, preparation_sha256: str, fraction: Fraction, seed: int +) -> tuple[Node, Node]: + """Declare selected CREATE then allocation, without an invented FILTER. + + The digest names checked preparation bytes; accepting it here authenticates + nothing. The country runner must reconstruct the preparation and verify the + materialized populations and artifacts on both cold and cached execution. + """ + _require(type(columns) in (tuple, list), "COLUMNS") + columns = tuple(columns) + _require(columns and all(type(column) is Owned for column in columns), "COLUMNS") + _require( + all( + column.entity in US_SCHEMA.entities + and column.rows == "all" + and column.ownership is Ownership.PRODUCED + and column.rewrite is False + for column in columns + ), + "CREATE_COLUMN_CONTRACT", + ) + inventory = {(column.entity, column.column): column for column in columns} + _require(len(inventory) == len(columns), "DUPLICATE_COLUMNS") + for entity in US_SCHEMA.entities: + for name in _provenance_columns(entity): + column = inventory.get((entity, name)) + _require(column is not None, "MISSING_PROVENANCE_COLUMN") + _require( + column.dtype + == ("string" if name == support_channel_column(entity) else "int64"), + "PROVENANCE_DTYPE", + ) + fraction_pair, seed = _draw_parameters(fraction, seed) + digest = _digest(preparation_sha256) + create = Node( + id=CREATE_NODE, + kernel=CREATE_REF, + structural=StructuralDelta.CREATE, + sources=(SOURCE_NAME,), + outputs=columns, + params={ + "phase": PHASE, + "preparation_sha256": digest, + "fraction": fraction_pair, + "sample_seed": seed, + }, + artifact_outputs=( + ArtifactOutput("frame_context", US_FRAME_CONTEXT_TYPE), + ArtifactOutput("preparation", PREPARATION_TYPE), + ), + ) + allocate = Node( + id=ALLOCATION_NODE, + kernel=ALLOCATION_REF, + base=CREATE_NODE, + structural=StructuralDelta.REWEIGHT, + inputs=tuple( + Slice(entity, _provenance_columns(entity)) for entity in US_SCHEMA.entities + ), + weights=WeightTransition("household", "importance", mass="declared"), + mass="declared", + params={"phase": PHASE, "preparation_sha256": digest}, + artifact_inputs=( + ArtifactInput( + "frame_context", CREATE_NODE, "frame_context", US_FRAME_CONTEXT_TYPE + ), + ArtifactInput("preparation", CREATE_NODE, "preparation", PREPARATION_TYPE), + ), + artifact_outputs=( + ArtifactOutput("frame_context", US_FRAME_CONTEXT_TYPE), + ArtifactOutput("allocation", ALLOCATION_TYPE), + ), + ) + return create, allocate diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/graph_survey_puf55.py b/packages/microcosm-build/src/microcosm/build/us_runtime/graph_survey_puf55.py new file mode 100644 index 000000000..4de2e1c52 --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/graph_survey_puf55.py @@ -0,0 +1,645 @@ +"""Extend an issued postclone nineteen-node financial run with two PUF55 routes. + +The existing financial run is an explicit prerequisite. This host compiles and +executes the extension once; it does not claim that the upstream run's entire +lifetime used one execution. No donor or FILTER issuer is run in preparation. +Required replay is a separate explicit call over the same source paths/store. +""" + +from __future__ import annotations + +import weakref +from dataclasses import dataclass, replace +from pathlib import Path + +from microcosm.fit.graph_legacy_apply_matrix import LegacyQRFApplyMatrixKernel +from microcosm.fit.graph_legacy_train import LegacyQRFTrainKernel +from microcosm.frame import US_SCHEMA +from microcosm.graph import ( + ContentStore, + KernelRegistry, + artifact_edges, + codecs, + compile_graph, + run_graph, +) +from microcosm.graph.executor import _all_node_keys, _source_paths_and_keys +from microcosm.graph.serialize import graph_to_json + +from . import graph_puf55_route_attachment as attach + +require = attach.require +financial, recipient, canonical = attach.financial, attach.recipient, attach.canonical +physical, codec, population_ops = attach.physical, attach.codec, attach.population_ops +RUN_PROTOCOL = "microcosm.us.survey-puf55-checked-run.v1" +_ISSUED_RUNS = {} + + +@dataclass(frozen=True) +class SurveyPuf55Run: + """Actual result handle; constructor/copies do not issue run authority.""" + + financial_run: financial.AtomicSurveyFinancialRunValues + population: population_ops.Population + manifest: object + compiled: object + store: ContentStore + kernels: KernelRegistry + sources: tuple[tuple[str, Path], ...] + receipt: bytes + + def checked_view(self): + """Recheck this retained handle without fitting or graph execution.""" + return check_survey_puf55_run(self) + + +@dataclass(frozen=True) +class CheckedSurveyPuf55Run: + """Descriptive values; only the original run retains in-process authority. + + A downstream owner must check that run immediately before consumption and + after its last relevant I/O, before returning or exporting a successor. + Reconstructing this view from JSON or copying it cannot authorize a run. + Neither this view nor the run grants calibration or release admission. + """ + + payload: bytes + digest: str + population: population_ops.Population + + +@dataclass(frozen=True) +class _RunState: + boundary: attach.Boundary + population: population_ops.Population + population_stamp: str + expected: population_ops.Population + expected_stamp: str + manifest: object + manifest_bytes: str + manifest_populations: tuple + receipt: bytes + artifact_hashes: tuple + + +def _run_entry(run): + entry = _ISSUED_RUNS.get(id(run)) + require( + type(run) is SurveyPuf55Run and entry is not None and entry[0]() is run, + "UNISSUED_PUF55_RUN", + ) + return entry + + +def _forget_run(run, entry): + if _ISSUED_RUNS.get(id(run)) is entry: + _ISSUED_RUNS.pop(id(run)) + + +def _run_document(state): + """Describe checked ancestry without granting authority to portable bytes.""" + boundary = state.boundary + return codec.encode_json( + { + "protocol": RUN_PROTOCOL, + "graph_sha256": codec.sha(boundary.declaration.encode()), + "manifest_key": state.manifest.key, + "financial_run_sha256": codec.sha(boundary.entry[1]), + "receipt_sha256": codec.sha(state.receipt), + "node_count": len(boundary.compiled.order), + "node_keys": dict(boundary.keys), + "source_keys": dict(boundary.source_keys), + "artifact_payload_sha256": [list(row) for row in state.artifact_hashes], + "population_version": state.population.version, + "population_physical_sha256": state.population_stamp, + "source_admission_issued": False, + "population_admission_issued": False, + "release_eligible": False, + } + ) + + +def _pure_run(run, entry): + """Check the retained output after all external reads have completed.""" + require(_run_entry(run) is entry, "FINAL_PUF55_RUN_ISSUANCE") + state, boundary = entry[2], entry[2].boundary + require( + run.financial_run is boundary.run + and run.population is state.population + and run.manifest is state.manifest + and run.compiled is boundary.compiled + and run.store is boundary.store + and run.kernels is boundary.kernels + and run.sources is boundary.paths + and run.receipt == state.receipt, + "PUF55_RUN_BINDINGS_CHANGED", + ) + require( + physical._population_stamp(run.population) == state.population_stamp + and physical._population_stamp(state.expected) == state.expected_stamp, + "PUF55_RUN_POPULATION_CHANGED", + ) + require( + set(run.manifest.populations) + == set(run.manifest.mass_ledgers) + == set(run.compiled.versions.values()), + "PUF55_RUN_MANIFEST_ROSTER_CHANGED", + ) + require( + run.manifest.to_json() == state.manifest_bytes + and _heterogeneous_manifest_seals(run.manifest, run.compiled) + == state.manifest_populations, + "PUF55_RUN_MANIFEST_CHANGED", + ) + boundary.pure() + require( + graph_to_json(run.compiled.graph) == boundary.declaration + and run.compiled == compile_graph(run.compiled.graph), + "PUF55_RUN_DECLARATION_CHANGED", + ) + physical.replay.same_replayed_population(state.expected, run.population) + require(_run_document(state) == entry[1], "PUF55_RUN_DOCUMENT_CHANGED") + require(_run_entry(run) is entry, "FINAL_PUF55_RUN_ISSUANCE") + + +def check_survey_puf55_run(run): + """Requalify actual source/store ancestry, then seal the complete output. + + Does not fit, execute a graph, decode model pickles, or reconstruct the PUF + donor. The actual execution already verified those exact artifact bytes. + A failed check revokes this handle; restoring its fields cannot reissue it. + Downstream owners must recheck before consumption and after their last + relevant I/O before returning/exporting a successor. A successful check is + a point-in-time claim, not a lease permitting future unchecked mutation. + """ + entry = _run_entry(run) + try: + _pure_run(run, entry) + state, boundary = entry[2], entry[2].boundary + boundary.borrow() + loaded = financial._artifacts( + run.manifest, + boundary.compiled, + boundary.store, + boundary.kernels, + dict(boundary.keys), + dict(boundary.implementations), + ) + boundary.borrow() + require( + tuple(sorted((n, a, codec.sha(p)) for (n, a), p in loaded.items())) + == state.artifact_hashes, + "PUF55_RUN_ARTIFACT_CHANGED", + ) + result = CheckedSurveyPuf55Run(entry[1], codec.sha(entry[1]), run.population) + _pure_run(run, entry) + return result + except BaseException: + _forget_run(run, entry) + raise + + +def _issue_run(output, state): + """Retain only a fully verified execution, with no new baseline reads.""" + identifier = id(output) + require(identifier not in _ISSUED_RUNS, "PUF55_RUN_REISSUANCE") + + def forget(reference): + entry = _ISSUED_RUNS.get(identifier) + if entry is not None and entry[0] is reference: + _ISSUED_RUNS.pop(identifier) + + entry = (weakref.ref(output, forget), _run_document(state), state) + _ISSUED_RUNS[identifier] = entry + try: + _pure_run(output, entry) + except BaseException: + _forget_run(output, entry) + raise + return entry + + +def _registry(existing, donor): + require(type(existing) is codecs.SourceCodecRegistry, "UPSTREAM_CODEC_REGISTRY") + registry = codecs.SourceCodecRegistry() + for old in (existing, donor): + for name, loader in old.as_mapping().items(): + require( + name not in (*registry.names(), *registry.bytes_names()), + "CODEC_COLLISION", + ) + registry.register(name, loader) + for name, loader in old.as_bytes_mapping().items(): + require( + name not in (*registry.names(), *registry.bytes_names()), + "CODEC_COLLISION", + ) + registry.register_bytes(name, loader) + return registry + + +def _construct( + financial_run, donor_sources, *, fixture_definition, seed, n_estimators, zero_atol +): + """Construct actual declarations and bind actual source keys without running.""" + financial.check_atomic_survey_financial_run(financial_run) + require(len(financial_run.compiled.order) == 19, "UPSTREAM_NINETEEN_NODES") + require(type(seed) is int and 0 <= seed < 2**64, "SEED") + require(type(n_estimators) is int and n_estimators > 0, "TREE_COUNT") + qualified = recipient.values.qualify_puf55_survey_recipients(financial_run) + donor = canonical.CanonicalPuf55DonorKernel( + seed=seed, fixture_definition=fixture_definition + ) + require( + type(donor_sources) is dict + and set(donor_sources) == {s.name for s in donor.source_refs}, + "DONOR_SOURCE_ROSTER", + ) + donor_paths = { + name: Path(path).resolve(strict=True) for name, path in donor_sources.items() + } + require(not set(donor_paths) & set(financial_run.sources), "SOURCE_NAME_COLLISION") + boundary = attach.Boundary( + financial_run, + qualified, + donor, + donor_paths, + seed=seed, + n_estimators=n_estimators, + zero_atol=zero_atol, + ) + recipient_nodes = recipient.puf55_survey_recipient_nodes(qualified) + nodes = ( + *financial_run.compiled.graph.nodes, + *recipient_nodes, + boundary.donor_node, + boundary.nodes[0], + *(node for route in boundary.routes for node in (*route.fits, *route.applies)), + *boundary.nodes[1:], + ) + graph = replace( + financial_run.compiled.graph, + sources=(*financial_run.compiled.graph.sources, *donor.source_refs), + nodes=nodes, + ) + compiled = compile_graph(graph) + require(len(compiled.order) == 25 + 110 * len(boundary.routes), "NODE_COUNT") + order = compiled.order + require( + order.index(financial.financial.ATTACH_NODE) + < order.index(recipient.PROJECTION_NODE) + < order.index(recipient.MATRIX_NODE) + < order.index(attach.FILTER_NODE) + < order.index(attach.MASK_NODE) + < order.index(attach.ATTACH_NODE), + "EXTENSION_ORDER", + ) + # Private registries preserve the original issued run's retained containers. + kernels = KernelRegistry() + for kernel in financial_run.kernels.as_mapping().values(): + kernels.register(kernel) + for kernel in ( + recipient.Puf55SurveyRecipientProjectionKernel(financial_run), + recipient.Puf55SurveyRecipientMatrixKernel(financial_run), + donor, + attach.SurveyPuf55KeepAllKernel(boundary), + attach.SurveyPuf55MaskKernel(boundary), + attach.SurveyPuf55AttachKernel(boundary), + ): + require(kernel.ref not in kernels.refs(), "KERNEL_COLLISION") + kernels.register(kernel) + for kernel in (LegacyQRFTrainKernel(), LegacyQRFApplyMatrixKernel()): + if kernel.ref not in kernels.refs(): + kernels.register(kernel) + else: + require(type(kernels.get(kernel.ref)) is type(kernel), "LEGACY_KERNEL_TYPE") + store = ContentStore( + financial_run.store.root, + codecs=_registry(financial_run.store.codecs, donor.source_codecs), + ) + paths, sources = _source_paths_and_keys( + compiled, {**financial_run.sources, **donor_paths}, store + ) + keys, implementations = _all_node_keys(compiled, kernels, sources) + # The preserved prefix must have the actual issued keys, not merely the + # same user-facing IDs. New donor sources are unused by the original nodes. + state = financial._run_entry(financial_run)[2] + require( + all(keys[n] == k for n, k in state.keys) + and all(implementations[n] == k for n, k in state.implementations), + "UPSTREAM_PRODUCER_KEYS", + ) + boundary.bind(compiled, store, kernels, paths, sources, keys, implementations) + boundary.borrow() + return boundary + + +def _heterogeneous_manifest_seals(manifest, compiled): + """Seal every attached version of this mixed-schema run, donor included. + + The survey helper stamps through ``source._frame_identity``, which requires + the six US_SCHEMA entity groups. This host attaches a tax_unit-only PUF + donor CREATE, so that helper cannot seal this manifest at all. The physical + seal is schema-aware and already seals exactly this donor frame elsewhere + (``canonical._frame_seal``). It records link *names* but never link bodies, + so a frame carrying link tables is refused here rather than sealed blind. + Reconstructed versions carry default owners and anchors derived from their + stored weights. This manifest-content seal does not recover independently + retained original owner/anchor/history evidence. + """ + seals = [] + for version in sorted(set(compiled.versions.values())): + frame = manifest.population(version) + require(not frame.links, "MANIFEST_SEAL_LINK_TABLES") + seals.append( + ( + version, + physical._population_stamp( + population_ops.Population.from_frame( + frame, version, mass_ledger=manifest.mass_ledger(version) + ) + ), + ) + ) + return tuple(seals) + + +def _check_replayed_survey_manifest(expected_manifest, actual_manifest, compiled): + """Compare an exact upstream survey roster across store representations. + + The caller keeps the original issued manifest and its physical lifetime + seals. These Frame/ledger views reconstruct the same default owners and + design anchors as the former manifest stamp; they do not recover true + execution ownership. Actual observed Populations are checked separately. + Manifest access may materialize lazy attachments, so the host brackets this + comparison with its retained-owner checks and keeps its final physical seal. + """ + versions = tuple(sorted(set(compiled.versions.values()))) + wanted = set(versions) + require( + expected_manifest.country + == actual_manifest.country + == compiled.graph.country + == "us", + "UPSTREAM_SURVEY_COUNTRY", + ) + require( + bool(versions) + and set(expected_manifest.nodes) == set(compiled.order) + and set(compiled.order) <= set(actual_manifest.nodes) + and set(expected_manifest.populations) == wanted + and set(expected_manifest.mass_ledgers) == wanted + and wanted <= set(actual_manifest.populations) + and wanted <= set(actual_manifest.mass_ledgers), + "UPSTREAM_SURVEY_ROSTER", + ) + for version in versions: + expected = expected_manifest.population(version) + actual = actual_manifest.population(version) + require(expected.schema == actual.schema == US_SCHEMA, "UPSTREAM_SURVEY_SCHEMA") + physical.replay.same_replayed_population( + population_ops.Population.from_frame( + expected, version, mass_ledger=expected_manifest.mass_ledger(version) + ), + population_ops.Population.from_frame( + actual, version, mass_ledger=actual_manifest.mass_ledger(version) + ), + ) + + +def _loaded_values(boundary, manifest, loaded, node): + """Only construct typed inputs from the already-checked real manifest.""" + record = manifest.node(node.id) + return { + edge.name: artifact_edges.value_from_descriptor( + loaded[edge.producer, edge.artifact], + record.typed_artifacts["inputs"][edge.name], + ) + for edge in node.artifact_inputs + } + + +def _reconstruct(boundary, observed, values, result): + """Use original retained upstream and independently source-derived donor.""" + run = boundary.run + # The original issuer retains final views per structural version. Its + # existing nineteen-node keys/types are checked above; the complete retained + # final views below bind the unchanged prefix, without inventing a second + # upstream observer history or re-executing a preparation graph. + physical.replay.same_replayed_population( + run.financial_population, observed[recipient.PROJECTION_NODE] + ) + physical.replay.same_replayed_population( + run.financial_population, observed[recipient.MATRIX_NODE] + ) + physical.replay.same_replayed_population( + boundary.expected, observed[attach.FILTER_NODE] + ) + donor_frame, donors, donor_seal = boundary.canonical_donor(values) + require( + attach._donor_seal(donor_frame, donors) == donor_seal, "DONOR_RESULT_CHANGED" + ) + donor_node = boundary.donor_node + # CREATE assigns every loaded column, including linkage, to this version. + donor_population = population_ops.Population.from_frame(donor_frame, donor_node.id) + physical.replay.same_replayed_population(donor_population, observed[donor_node.id]) + for route in boundary.routes: + for node in route.fits: + physical.replay.same_replayed_population( + donor_population, observed[node.id] + ) + for node in route.applies: + # Ordinary fit/apply nodes own no Population cells. Compiler order + # can put the independent mask before any apply, so compare with + # the preceding state of this version, not an assumed fixed order. + require(not node.outputs and node.weights is None, "CHAIN_POPULATION_WRITE") + current = {} + for node_id in boundary.compiled.order: + node = boundary.compiled.graph.node(node_id) + version = boundary.compiled.versions[node_id] + if node_id in run.compiled.order: + current[version] = observed[node_id] + continue + if node_id == donor_node.id: + expected = donor_population + elif node_id == attach.FILTER_NODE: + expected = boundary.expected + elif node_id == attach.MASK_NODE: + expected = population_ops.patch( + current[version], node, attach._mask_result(boundary.expected.frame) + ) + elif node_id == attach.ATTACH_NODE: + expected = population_ops.patch(current[version], node, result) + else: + expected = current[version] + physical.replay.same_replayed_population(expected, observed[node_id]) + current[version] = expected + return current[attach.FILTER_NODE] + + +def run_survey_puf55( + financial_run, + *, + donor_sources, + seed=578, + n_estimators=100, + zero_atol=1e-8, + fixture_definition=None, + resume="auto", +): + """Run one extension over an actual issued run; no pre-run or donor injection. + + ``fixture_definition`` is the canonical owner's explicit invented-source + route. It cannot claim packaged-source admission. A separate call with + ``resume='require'`` must reproduce every source/producer and all values. + """ + require(resume in ("auto", "require"), "RESUME") + boundary = _construct( + financial_run, + donor_sources, + fixture_definition=fixture_definition, + seed=seed, + n_estimators=n_estimators, + zero_atol=zero_atol, + ) + observed, stamps = {}, {} + + def observe(node_id, population): + require(node_id not in observed, "OBSERVER_DUPLICATE") + observed[node_id] = population + stamps[node_id] = physical._population_stamp(population) + + manifest = run_graph( + boundary.compiled, + sources=dict(boundary.paths), + store=boundary.store, + kernels=boundary.kernels, + resume=resume, + _population_observer=observe, + ) + require(tuple(observed) == boundary.compiled.order, "OBSERVER_ROSTER") + if resume == "require": + require( + all(manifest.node(n).hit for n in boundary.compiled.order), "REQUIRED_HITS" + ) + # Complete source-key/implementation/type ancestry before loading any + # trusted model or independently invoking the replay numerical verifier. + boundary.borrow() + loaded = financial._artifacts( + manifest, + boundary.compiled, + boundary.store, + boundary.kernels, + dict(boundary.keys), + dict(boundary.implementations), + ) + # The mixed-manifest physical baseline includes donor and survey versions + # and is captured once. Cross-store comparison uses the still-issued + # upstream manifest as expected, retaining its original lifetime seals. + full_seals = _heterogeneous_manifest_seals(manifest, boundary.compiled) + boundary.pure() + _check_replayed_survey_manifest( + financial_run.manifest, manifest, financial_run.compiled + ) + boundary.pure() + values = _loaded_values(boundary, manifest, loaded, boundary.nodes[2]) + boundary.projections(values) + if boundary.computed is None: + result = boundary.finalize(values) + else: + boundary.pure() + result = boundary.computed[0] + require( + loaded[attach.ATTACH_NODE, "finalization"] == result.artifacts["finalization"], + "FINALIZATION_ARTIFACT", + ) + expected = _reconstruct(boundary, observed, values, result) + population = observed[attach.ATTACH_NODE] + physical.replay.same_replayed_population(expected, population) + receipt = codec.encode_json( + { + "protocol": attach.PROTOCOL, + "node_count": len(boundary.compiled.order), + "profiles": [r.profile.value for r in boundary.routes], + "financial_run_sha256": codec.sha(boundary.entry[1]), + "manifest_key": manifest.key, + "one_extension_execution": True, + "upstream_already_issued": True, + "complete_population_compared": True, + "source_admission_issued": False, + "population_admission_issued": False, + "release_eligible": False, + } + ) + output = SurveyPuf55Run( + financial_run, + population, + manifest, + boundary.compiled, + boundary.store, + boundary.kernels, + boundary.paths, + receipt, + ) + manifest_bytes = manifest.to_json() + expected_stamp = physical._population_stamp(expected) + artifact_hashes = tuple( + sorted((n, a, codec.sha(p)) for (n, a), p in loaded.items()) + ) + # Last reads include actual source owners/code, manifest store identities, + # and every returned Population. Nothing below the fence performs I/O. + boundary.borrow() + fresh = financial._artifacts( + manifest, + boundary.compiled, + boundary.store, + boundary.kernels, + dict(boundary.keys), + dict(boundary.implementations), + ) + boundary.borrow() + require( + tuple(sorted((n, a, codec.sha(p)) for (n, a), p in fresh.items())) + == artifact_hashes, + "LATE_ARTIFACT_CHANGED", + ) + boundary.pure() + require( + output.financial_run is financial_run + and output.population is population + and output.manifest is manifest + and output.compiled is boundary.compiled + and output.store is boundary.store + and output.kernels is boundary.kernels + and output.sources == boundary.paths + and output.receipt == receipt + and manifest.to_json() == manifest_bytes + and _heterogeneous_manifest_seals(manifest, boundary.compiled) == full_seals + and physical._population_stamp(expected) == expected_stamp + and all(physical._population_stamp(observed[n]) == stamps[n] for n in observed), + "FINAL_OUTPUT_CHANGED", + ) + physical.replay.same_replayed_population(expected, population) + state = _RunState( + boundary, + population, + stamps[attach.ATTACH_NODE], + expected, + expected_stamp, + manifest, + manifest_bytes, + full_seals, + receipt, + artifact_hashes, + ) + # Baselines were captured before the preceding last-I/O fence. The entry + # retains the final output, not all 245 executor-observed snapshots. + try: + entry = _issue_run(output, state) + _pure_run(output, entry) + except BaseException: + candidate = _ISSUED_RUNS.get(id(output)) + if candidate is not None and candidate[0]() is output: + _forget_run(output, candidate) + raise + return output diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/graph_us_survey_enrichment.py b/packages/microcosm-build/src/microcosm/build/us_runtime/graph_us_survey_enrichment.py new file mode 100644 index 000000000..c9c3ae590 --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/graph_us_survey_enrichment.py @@ -0,0 +1,1043 @@ +"""Fixed US post-PUF host for source-qualified survey enrichment fragments. + +The owning boundary retains a real checked PUF run and authenticated source +projection. Models use separate original-design donor branches; attachment +adds the declared amount and coverage families to the existing receiving frame. +""" + +from __future__ import annotations + +import sys +import weakref +from dataclasses import dataclass, replace +from types import FunctionType, SimpleNamespace + +import pandas as pd + +from microcosm.fit import _graph_legacy_qrf as codec +from microcosm.fit import model_input, qrf, qrf_target +from microcosm.fit.graph_legacy_apply_matrix import ( + MATRIX_APPLY_STATE_TYPE, + decode_matrix_apply_state, +) +from microcosm.fit.graph_legacy_qrf import ( + legacy_qrf_apply_matrix_nodes, + legacy_qrf_train_nodes, +) +from microcosm.graph import ( + ArtifactInput, + ArtifactOutput, + ArtifactType, + Capabilities, + ContentStore, + Determinism, + KernelBase, + KernelRegistry, + KernelResult, + Node, + Numeric, + Owned, + StructuralDelta, + codecs, + compile_graph, + run_graph, + source_hash, +) +from microcosm.graph import population as population_ops +from microcosm.graph.executor import _all_node_keys, _source_paths_and_keys +from microcosm.graph.serialize import graph_to_json + +from . import current_survey_amounts as values +from . import graph_current_survey_health as health_graph +from . import graph_current_survey_predictors as predictor_graph + +parent = values.parent_host +physical = values.physical +require = values.require +PROJECTION_NODE = "survey_amounts.source_projection" +ATTACH_NODE = "survey_amounts.attach" +PROJECTION_TYPE = ArtifactType("microcosm.us.current_survey_amount_projection", 1) +ATTACHMENT_TYPE = ArtifactType("microcosm.us.current_survey_amount_attachment", 1) +_ISSUED = {} + + +def _live(): + result = [] + for module in ( + sys.modules[__name__], + values, + values.unemployment, + health_graph, + health_graph.health, + health_graph.source, + ): + for name, item in vars(module).items(): + if type(item) is FunctionType: + result.append( + ( + module.__name__, + name, + values.predictors.source._function_seal(item), + ) + ) + elif isinstance(item, type) and item.__module__ == module.__name__: + result.append((module.__name__, name, item)) + for member, function in vars(item).items(): + if isinstance(function, (classmethod, staticmethod)): + function = function.__func__ + if isinstance(function, property): + function = function.fget + if type(function) is FunctionType: + result.append( + ( + module.__name__, + name, + member, + values.predictors.source._function_seal(function), + ) + ) + result.append( + ( + "amount_configuration", + values.SEED, + tuple((g.key, g.fields, g.targets) for g in values.GROUPS), + values.UC_REPORT_COLUMNS, + values.unemployment.PROTOCOL, + values.unemployment.READ_COLUMNS, + codec.encode_json(values.unemployment.DICTIONARY), + ) + ) + result.append( + ( + "health_configuration", + health_graph.health.PROTOCOL, + tuple( + (f.output, f.asec, f.acs, f.acs_gap) for f in health_graph.health.FIELDS + ), + health_graph.health.RAW_COLUMNS, + health_graph.health.SOURCE_PREFIX, + health_graph.source.ASEC_COLUMNS, + health_graph.source.ACS_COLUMNS, + ) + ) + return tuple(result) + + +def _amount_edge(): + return ArtifactInput( + "amount_attachment", ATTACH_NODE, "attachment", ATTACHMENT_TYPE + ) + + +def _upstream_edge(): + return ArtifactInput( + "puf_finalization", + parent.attach.ATTACH_NODE, + "finalization", + parent.attach.FINALIZATION_TYPE, + ) + + +def _projection_edge(): + return ArtifactInput("projection", PROJECTION_NODE, "projection", PROJECTION_TYPE) + + +def _ids(group): + base = "survey_amounts." + group.spec.key + return base + ".donor", base + ".columns", base + ".fit", base + ".apply" + + +def amount_nodes(qualified, receiving, *, parent_digest, n_estimators): + require( + type(n_estimators) is int and n_estimators > 0 and codec._hash(parent_digest), + "NODE_PARAMETERS", + ) + params = { + "projection_sha256": codec.sha(qualified.projection), + "parent_sha256": parent_digest, + "n_estimators": n_estimators, + "groups": tuple(g.spec.key for g in qualified.groups), + "protocol": values.PROTOCOL, + } + nodes = [ + Node( + PROJECTION_NODE, + CurrentSurveyAmountProjectionKernel.ref, + population=parent.attach.FILTER_NODE, + inputs=predictor_graph._inputs(receiving), + params=params, + artifact_inputs=(_upstream_edge(),), + artifact_outputs=( + ArtifactOutput("projection", PROJECTION_TYPE), + *( + ArtifactOutput( + g.spec.key + "_matrix", model_input.RECIPIENT_MATRIX_TYPE + ) + for g in qualified.groups + ), + ), + description="Qualify original current survey amount and reporting-universe evidence; preserve source unknowns.", + ) + ] + attach_edges = [_projection_edge(), _upstream_edge()] + for group in qualified.groups: + donor, columns, fit_prefix, apply_prefix = _ids(group) + group_params = {**params, "group": group.spec.key} + matrix_edge = ArtifactInput( + group.spec.key + "_matrix", + PROJECTION_NODE, + group.spec.key + "_matrix", + model_input.RECIPIENT_MATRIX_TYPE, + ) + nodes.extend( + ( + Node( + donor, + CurrentSurveyAmountDonorKernel.ref, + base=values.predictors.host.survey_graph.CREATE_NODE, + structural=StructuralDelta.FILTER, + mass="free", + inputs=predictor_graph._inputs(qualified.source_frame), + params=group_params, + artifact_inputs=(_projection_edge(),), + description="Select source ASEC persons with jointly known targets; retain original design weights before allocation and clones.", + ), + Node( + columns, + CurrentSurveyAmountColumnsKernel.ref, + population=donor, + inputs=predictor_graph._inputs(group.donor_frame), + params=group_params, + outputs=tuple( + Owned("person", c, "float64") + for c in (*qualified.features, *group.spec.targets) + ), + artifact_inputs=(_projection_edge(),), + description="Map qualified current amounts and source predictors without numeric or reporting-unknown fills.", + ), + ) + ) + fits = legacy_qrf_train_nodes( + fit_prefix, + population=donor, + entity="person", + predictors=qualified.features, + targets=group.spec.targets, + seed=values.SEED, + n_estimators=n_estimators, + zero_atol=0, + phase="current_survey_amounts." + group.spec.key, + ) + applies = legacy_qrf_apply_matrix_nodes( + apply_prefix, + population=parent.attach.FILTER_NODE, + fit_nodes=fits, + matrix_producer=PROJECTION_NODE, + matrix_artifact=group.spec.key + "_matrix", + seed=values.SEED, + phase="current_survey_amounts." + group.spec.key, + ) + nodes.extend((*fits, *applies)) + attach_edges.append(matrix_edge) + for i, node in enumerate(applies): + attach_edges.extend( + ( + ArtifactInput( + f"{group.spec.key}_raw_{i}", + node.id, + "raw_draw", + codec.RAW_TARGET_TYPE, + ), + ArtifactInput( + f"{group.spec.key}_state_{i}", + node.id, + "apply_state", + MATRIX_APPLY_STATE_TYPE, + ), + ) + ) + columns = pd.concat([qualified.native, qualified.reports], axis=1) + nodes.append( + Node( + ATTACH_NODE, + CurrentSurveyAmountAttachKernel.ref, + population=parent.attach.FILTER_NODE, + inputs=predictor_graph._inputs(receiving), + params=params, + outputs=tuple(Owned("person", c, str(columns[c].dtype)) for c in columns), + artifact_inputs=tuple(attach_edges), + artifact_outputs=(ArtifactOutput("attachment", ATTACHMENT_TYPE),), + description="Join source observations and ACS conditional draws to both support clones; retain all PUF and source columns, geography, weights and ledger.", + ) + ) + return tuple(nodes) + + +class Boundary: + """Internal retained-value seam; a detached projection cannot construct it.""" + + def __init__(self, run, *, groups, n_estimators): + self.run = run + self.parent_view = parent.check_survey_puf55_run(run) + self.parent_entry = parent._run_entry(run) + self.qualified = values.qualify_current_survey_amounts(run, groups=groups) + self.qualified_stamp = values.seal(self.qualified) + self.parent_stamp = physical._population_stamp(run.population) + self.preparation = run.financial_run.prefix.preparation + self.health = health_graph.qualify_health_coverage(self.preparation) + self.health_stamp = health_graph.health_coverage_seal(self.health) + self.n_estimators = n_estimators + self.amount_nodes = amount_nodes( + self.qualified, + run.population.frame, + parent_digest=self.parent_view.digest, + n_estimators=n_estimators, + ) + self.health_nodes = health_graph.health_coverage_nodes( + self.health, + receiving_version=parent.attach.FILTER_NODE, + after=_amount_edge(), + ) + self.nodes = (*self.amount_nodes, *self.health_nodes) + self.declaration = tuple(self.nodes) + self.live = _live() + self.compiled = self.kernels = self.store = None + self.paths = self.source_keys = self.keys = self.implementations = None + self.parent_objects = ( + run.population, + run.compiled, + run.manifest, + run.store, + run.kernels, + run.sources, + ) + + def pure(self): + parent._pure_run(self.run, self.parent_entry) + require( + parent._run_entry(self.run) is self.parent_entry + and all( + a is b + for a, b in zip( + self.parent_objects, + ( + self.run.population, + self.run.compiled, + self.run.manifest, + self.run.store, + self.run.kernels, + self.run.sources, + ), + strict=True, + ) + ) + and physical._population_stamp(self.run.population) == self.parent_stamp + and values.seal(self.qualified) == self.qualified_stamp + and self.run.financial_run.prefix.preparation is self.preparation + and health_graph.health_coverage_seal(self.health) == self.health_stamp + and self.nodes == self.declaration + and _live() == self.live, + "BOUNDARY_CHANGED", + ) + require( + self.amount_nodes + == amount_nodes( + self.qualified, + self.run.population.frame, + parent_digest=self.parent_view.digest, + n_estimators=self.n_estimators, + ) + and self.health_nodes + == health_graph.health_coverage_nodes( + self.health, + receiving_version=parent.attach.FILTER_NODE, + after=_amount_edge(), + ) + and self.nodes == (*self.amount_nodes, *self.health_nodes), + "BOUNDARY_DECLARATIONS", + ) + if self.compiled is not None: + require( + all( + a is b + for a, b in zip( + self.bound_objects, + (self.compiled, self.store, self.kernels, self.store.codecs), + strict=True, + ) + ) + and tuple(self.kernels.as_mapping().items()) == self.kernel_items + and tuple(self.store.codecs.as_mapping().items()) == self.codec_items + and tuple(self.store.codecs.as_bytes_mapping().items()) + == self.bytes_codec_items, + "BOUND_REGISTRY_CHANGED", + ) + require( + self.compiled == compile_graph(self.compiled.graph) + and graph_to_json(self.compiled.graph) == self.graph_json, + "COMPILED_CHANGED", + ) + + def borrow(self): + require( + parent.check_survey_puf55_run(self.run).payload == self.parent_view.payload, + "PARENT_IDENTITY", + ) + if self.compiled is not None: + paths, source_keys = _source_paths_and_keys( + self.compiled, dict(self.paths), self.store + ) + keys, implementations = _all_node_keys( + self.compiled, self.kernels, source_keys + ) + require( + tuple(sorted(paths.items())) == self.paths + and tuple(sorted(source_keys.items())) == self.source_keys + and tuple(sorted(keys.items())) == self.keys + and tuple(sorted(implementations.items())) == self.implementations, + "SOURCE_OR_IMPLEMENTATION_CHANGED", + ) + self.pure() + + def context(self, context): + # Entry/final host fences check the complete PUF owner. These ordinary + # fragment calls consume retained, source-qualified values and check + # their pure seals; they do not repeatedly reread the full PUF pipeline. + self.pure() + require( + not context.sources + and context.node in self.nodes + and set(context.artifacts) + == {e.name for e in context.node.artifact_inputs}, + "CONTEXT_DECLARATION", + ) + for edge in context.node.artifact_inputs: + value = values.predictors.host.shared.artifact( + context, edge.name, edge.type + ) + require( + value.producer_key == dict(self.keys)[edge.producer], + "ARTIFACT_PRODUCER_KEY", + ) + if edge.name == "projection": + require(value.payload == self.qualified.projection, "PROJECTION_BYTES") + if edge == _upstream_edge(): + record = self.run.manifest.node(edge.producer) + payload = self.run.store.load_bytes( + record.opaque_artifacts[edge.artifact] + ) + require( + value.producer_key == record.key + and value.key == record.opaque_artifacts[edge.artifact] + and value.payload == payload, + "PARENT_ARTIFACT", + ) + self.pure() + return self.qualified + + def requalify(self): + """Reconstruct owned source transformations at the host's final I/O fence.""" + fresh = values.qualify_current_survey_amounts( + self.run, groups=tuple(g.spec.key for g in self.qualified.groups) + ) + require( + values.seal(fresh) == self.qualified_stamp, "SOURCE_REQUALIFICATION_CHANGED" + ) + fresh_health = health_graph.qualify_health_coverage(self.preparation) + require( + health_graph.health_coverage_seal(fresh_health) == self.health_stamp, + "HEALTH_SOURCE_REQUALIFICATION_CHANGED", + ) + self.pure() + + +class _Kernel(KernelBase): + capabilities = Capabilities( + Determinism.DETERMINISTIC, + numeric=Numeric.PLATFORM_BITWISE, + dependencies=predictor_graph._Kernel.capabilities.dependencies, + ) + + def __init__(self, boundary): + self.boundary = boundary + + def implementation_hash(self): + return source_hash( + sys.modules[__name__], + values, + values.unemployment, + health_graph, + parent, + physical, + predictor_graph, + values.predictors, + values.unemployment.coverage, + values.unemployment.source_csv_builtin, + population_ops, + model_input, + qrf, + qrf_target, + dependencies=self.capabilities.dependencies, + ) + + +class CurrentSurveyAmountProjectionKernel(_Kernel): + ref = "us.survey_amounts.source_projection@1" + + def run(self, context): + qualified = self.boundary.context(context) + values.predictors.host._current_context_frame( + context, self.boundary.run.population.frame + ) + return KernelResult( + artifacts={ + "projection": qualified.projection, + **{g.spec.key + "_matrix": g.matrix for g in qualified.groups}, + }, + receipt=qualified.evidence, + ) + + +class CurrentSurveyAmountDonorKernel(_Kernel): + ref = "us.survey_amounts.source_donor@1" + capabilities = replace(_Kernel.capabilities, structural=StructuralDelta.FILTER) + + def run(self, context): + qualified = self.boundary.context(context) + values.predictors.host._current_context_frame(context, qualified.source_frame) + group = next( + g for g in qualified.groups if g.spec.key == context.params["group"] + ) + return KernelResult( + keep=pd.Series( + group.keep.copy(), + index=pd.Index( + qualified.source_frame.person.person_id.to_numpy(), name="person_id" + ), + ), + receipt={ + "group": group.spec.key, + "donor_persons": int(group.keep.sum()), + "selection": "joint_known_current_ASEC_person_targets", + "weight_kind": "design", + }, + ) + + +class CurrentSurveyAmountColumnsKernel(_Kernel): + ref = "us.survey_amounts.source_columns@1" + + def run(self, context): + qualified = self.boundary.context(context) + group = next( + g for g in qualified.groups if g.spec.key == context.params["group"] + ) + values.predictors.host._current_context_frame(context, group.donor_frame) + return KernelResult( + columns={ + ("person", c): group.donor_columns[c] for c in group.donor_columns + }, + receipt={ + "group": group.spec.key, + "projection_sha256": codec.sha(qualified.projection), + }, + ) + + +def read_draws(qualified, artifacts): + """Check target history, exact matrix producer and immutable draw bytes.""" + draws = {} + for group in qualified.groups: + name = group.spec.key + matrix_value = artifacts[name + "_matrix"] + require(matrix_value.payload == group.matrix, "MATRIX_BYTES") + matrix = model_input.decode_recipient_matrix(group.matrix) + require(tuple(matrix.features) == qualified.features, "MATRIX_FEATURES") + result = pd.DataFrame(index=matrix.features.index) + models, raw_history = [], [] + for i, target in enumerate(group.spec.targets): + raw = artifacts[f"{name}_raw_{i}"].payload + state_value = artifacts[f"{name}_state_{i}"] + require( + state_value.producer_key == artifacts[f"{name}_raw_{i}"].producer_key, + "RAW_STATE_SIBLINGS", + ) + packet = decode_matrix_apply_state(state_value.payload) + application, chain = codec.read_application( + codec.encode_json(packet["application"]) + ) + raw_history.append({"target": target, "sha256": codec.sha(raw)}) + require( + packet["matrix_sha256"] == codec.sha(group.matrix) + and packet["matrix_producer_key"] == matrix_value.producer_key + and chain.entity == "person" + and tuple(chain.predictors) == qualified.features + and tuple(chain.targets) == group.spec.targets + and tuple(chain.completed_targets) == group.spec.targets[: i + 1] + and chain.recipient_index == qrf._index_identity(matrix.features.index) + and application["seed"] == values.SEED + and application["raw_targets"] == raw_history + and application["models"][:i] == models + and len(application["models"]) == i + 1, + "DRAW_CHAIN", + ) + models = application["models"] + result[target] = codec.read_raw_target( + raw, target=target, index=matrix.features.index + ) + draws[name] = result + return draws + + +def _attachment_result(boundary, artifacts): + qualified = boundary.qualified + require( + artifacts["projection"].payload == qualified.projection, "ATTACHMENT_PROJECTION" + ) + draws = read_draws(qualified, artifacts) + columns = values.attach_columns(qualified, boundary.run.population.frame, draws) + receipt = { + "protocol": values.PROTOCOL, + "parent_sha256": boundary.parent_view.digest, + "projection_sha256": codec.sha(qualified.projection), + "draw_sha256": { + n: codec.sha(v.payload) for n, v in artifacts.items() if "_raw_" in n + }, + "source_unknowns_preserved": True, + "weights_changed": False, + "prior_wages_consumed": False, + "release_eligible": False, + } + return KernelResult( + columns=columns, + artifacts={"attachment": codec.encode_json(receipt)}, + receipt=receipt, + ) + + +class CurrentSurveyAmountAttachKernel(_Kernel): + ref = "us.survey_amounts.attach@1" + + def run(self, context): + self.boundary.context(context) + values.predictors.host._current_context_frame( + context, self.boundary.run.population.frame + ) + result = _attachment_result(self.boundary, context.artifacts) + self.boundary.pure() + return result + + +def _construct(run, *, groups, n_estimators): + boundary = Boundary(run, groups=groups, n_estimators=n_estimators) + compiled = compile_graph( + replace(run.compiled.graph, nodes=(*run.compiled.graph.nodes, *boundary.nodes)) + ) + kernels = KernelRegistry() + for kernel in run.kernels.as_mapping().values(): + kernels.register(kernel) + for cls in ( + CurrentSurveyAmountProjectionKernel, + CurrentSurveyAmountDonorKernel, + CurrentSurveyAmountColumnsKernel, + CurrentSurveyAmountAttachKernel, + ): + require(cls.ref not in kernels.refs(), "KERNEL_COLLISION") + kernels.register(cls(boundary)) + for kernel in health_graph.health_coverage_kernels( + boundary.health, + receiving_version=parent.attach.FILTER_NODE, + after=_amount_edge(), + require_current=boundary.pure, + ): + require(kernel.ref not in kernels.refs(), "HEALTH_KERNEL_COLLISION") + kernels.register(kernel) + registry = parent._registry(run.store.codecs, codecs.SourceCodecRegistry()) + store = ContentStore(run.store.root, codecs=registry) + paths, source_keys = _source_paths_and_keys(compiled, dict(run.sources), store) + keys, implementations = _all_node_keys(compiled, kernels, source_keys) + require( + all( + keys[n] == run.manifest.node(n).key + and implementations[n] == run.manifest.node(n).kernel_impl_hash + and compiled.graph.node(n) == run.compiled.graph.node(n) + for n in run.compiled.order + ), + "PREFIX_DECLARATION_OR_KEY", + ) + boundary.compiled, boundary.kernels, boundary.store = compiled, kernels, store + boundary.paths = tuple(sorted(paths.items())) + boundary.source_keys = tuple(sorted(source_keys.items())) + boundary.keys = tuple(sorted(keys.items())) + boundary.implementations = tuple(sorted(implementations.items())) + boundary.graph_json = graph_to_json(compiled.graph) + boundary.bound_objects = (compiled, store, kernels, store.codecs) + boundary.kernel_items = tuple(kernels.as_mapping().items()) + boundary.codec_items = tuple(store.codecs.as_mapping().items()) + boundary.bytes_codec_items = tuple(store.codecs.as_bytes_mapping().items()) + boundary.borrow() + return boundary + + +def _verify_models(boundary, loaded, donors): + """Verify persisted models against actual donor values after artifact admission.""" + for group in boundary.qualified.groups: + donor_id, _, fit_prefix, apply_prefix = _ids(group) + donor = donors[donor_id] + first = boundary.compiled.graph.node(fit_prefix + ".000") + model_frame = codec.model_frame( + SimpleNamespace(weights={"person": donor.frame.resolve_weights("person")}), + first.inputs[0], + donor.frame.person, + ) + model = qrf.RegimeGatedQRF( + seed=values.SEED, + n_estimators=boundary.n_estimators, + zero_atol=0, + max_samples_leaf=None, + ) + before = qrf_target.LegacyQRFTrainingState.from_chain( + model.start_chain( + model_frame, + list(boundary.qualified.features), + list(group.spec.targets), + weights="design", + ) + ) + history = [] + for i, target in enumerate(group.spec.targets): + node = f"{fit_prefix}.{i:03d}" + payload = loaded[node, "model"] + packet, after = codec.read_training(loaded[node, "training_state"]) + fitted = qrf_target.LegacyQRFTargetArtifact.from_trusted_bytes( + payload, expected_sha256=codec.sha(payload) + ) + require( + fitted.training_state == before + and fitted.next_training_state == after + and fitted.donor_sha256 + == qrf_target._consumed_values_sha256( + model_frame.person, + (*boundary.qualified.features, *group.spec.targets[:i], target), + ), + "TRAINING_DONOR", + ) + history.append( + { + "target": target, + "sha256": codec.sha(payload), + "training_id": fitted.training_id, + } + ) + require( + packet["models"] == history + and decode_matrix_apply_state( + loaded[f"{apply_prefix}.{i:03d}", "apply_state"] + )["application"]["models"] + == history, + "TRAINING_APPLY_HISTORY", + ) + before = after + + +@dataclass(frozen=True) +class SurveyEnrichmentRun: + parent_run: parent.SurveyPuf55Run + population: population_ops.Population + manifest: object + compiled: object + store: ContentStore + kernels: KernelRegistry + sources: tuple + receipt: bytes + + def checked_view(self): + return check_survey_enrichment_run(self) + + +@dataclass(frozen=True) +class CheckedSurveyEnrichmentRun: + """Descriptive values; authority remains in the original retained host.""" + + payload: bytes + digest: str + population: population_ops.Population + + +def _run_seal(run): + return ( + run.receipt, + run.manifest.to_json(), + graph_to_json(run.compiled.graph), + physical._population_stamp(run.population), + parent._heterogeneous_manifest_seals(run.manifest, run.compiled), + tuple(sorted(run.manifest.populations)), + tuple(sorted(run.manifest.mass_ledgers)), + ) + + +def check_survey_enrichment_run(run): + entry = _ISSUED.get(id(run)) + require( + type(run) is SurveyEnrichmentRun and entry is not None and entry[0]() is run, + "UNISSUED_RUN", + ) + _, boundary, stamp, objects, artifacts = entry + try: + boundary.borrow() + require( + all( + a is b + for a, b in zip( + objects, + ( + run.parent_run, + run.population, + run.manifest, + run.compiled, + run.store, + run.kernels, + run.sources, + ), + strict=True, + ) + ), + "RUN_OBJECTS", + ) + loaded = parent.financial._artifacts( + run.manifest, + run.compiled, + run.store, + run.kernels, + dict(boundary.keys), + dict(boundary.implementations), + ) + require( + tuple(sorted((n, a, codec.sha(p)) for (n, a), p in loaded.items())) + == artifacts, + "RUN_ARTIFACTS", + ) + boundary.borrow() + require(_run_seal(run) == stamp, "RUN_CHANGED") + boundary.pure() + except BaseException: + if _ISSUED.get(id(run)) is entry: + del _ISSUED[id(run)] + raise + return CheckedSurveyEnrichmentRun( + run.receipt, codec.sha(run.receipt), run.population + ) + + +def run_us_survey_enrichment( + run, *, groups=("unemployment", "health_costs"), n_estimators=100, resume="auto" +): + """Execute one extension and verify its complete observed parent and output.""" + require(resume in ("auto", "require"), "RESUME") + boundary = _construct(run, groups=groups, n_estimators=n_estimators) + observed, stamps = {}, {} + + def observe(node_id, population): + require(node_id not in observed, "OBSERVER_DUPLICATE") + observed[node_id] = population + stamps[node_id] = physical._population_stamp(population) + + manifest = run_graph( + boundary.compiled, + sources=dict(boundary.paths), + store=boundary.store, + kernels=boundary.kernels, + resume=resume, + _population_observer=observe, + ) + require(tuple(observed) == boundary.compiled.order, "OBSERVER_ROSTER") + if resume == "require": + require(all(record.hit for record in manifest.nodes.values()), "REQUIRED_HITS") + boundary.borrow() + loaded = parent.financial._artifacts( + manifest, + boundary.compiled, + boundary.store, + boundary.kernels, + dict(boundary.keys), + dict(boundary.implementations), + ) + for node in run.compiled.graph.nodes: + old = run.manifest.node(node.id) + for name, key in old.opaque_artifacts.items(): + require( + loaded[node.id, name] == run.store.load_bytes(key), "PREFIX_ARTIFACT" + ) + physical.replay.same_replayed_population( + run.population, observed[parent.attach.ATTACH_NODE] + ) + require( + loaded[PROJECTION_NODE, "projection"] == boundary.qualified.projection, + "SOURCE_PROJECTION", + ) + attach = boundary.compiled.graph.node(ATTACH_NODE) + artifacts = parent._loaded_values(boundary, manifest, loaded, attach) + result = _attachment_result(boundary, artifacts) + require( + loaded[ATTACH_NODE, "attachment"] == result.artifacts["attachment"], + "ATTACHMENT_ARTIFACT", + ) + # Compare every unchanged prefix terminal Frame and ledger against the + # retained parent manifest. Full execution owners/design anchors on the + # receiving population are checked separately above against the original. + prefix_terminal = {} + for node_id in run.compiled.order: + prefix_terminal[run.compiled.versions[node_id]] = observed[node_id] + for version, population in prefix_terminal.items(): + physical.replay.same_replayed_population( + population_ops.Population.from_frame( + population.frame, version, mass_ledger=population.mass_ledger + ), + population_ops.Population.from_frame( + run.manifest.population(version), + version, + mass_ledger=run.manifest.mass_ledger(version), + ), + ) + current, donors = {}, {} + health_ids = {n.id for n in boundary.health_nodes} + group_nodes = {_ids(g)[0]: g for g in boundary.qualified.groups} + column_nodes = {_ids(g)[1]: g for g in boundary.qualified.groups} + original = population_ops.Population.from_frame( + boundary.qualified.source_frame, values.predictors.host.survey_graph.CREATE_NODE + ) + for node_id in boundary.compiled.order: + node = boundary.compiled.graph.node(node_id) + version = boundary.compiled.versions[node_id] + if node_id in run.compiled.order: + current[version] = observed[node_id] + continue + if node_id in health_ids: + health_artifacts = parent._loaded_values(boundary, manifest, loaded, node) + expected = health_graph.expected_health_population( + node_id, + current.get(version), + qualified=boundary.health, + node=node, + artifacts=health_artifacts, + ) + # Bind every persisted source/recode/attachment artifact to its + # independent domain result, including attachment metadata. + expected_result = health_graph._result( + boundary.health, + node, + None if current.get(version) is None else current[version].frame.person, + ) + require( + all( + loaded[node_id, name] == payload + for name, payload in expected_result.artifacts.items() + ), + "HEALTH_RESULT_ARTIFACT", + ) + elif node_id in group_nodes: + expected = population_ops.patch( + original, node, KernelResult(frame=group_nodes[node_id].donor_frame) + ) + elif node_id in column_nodes: + group = column_nodes[node_id] + expected = population_ops.patch( + current[version], + node, + KernelResult( + columns={ + ("person", c): group.donor_columns[c] + for c in group.donor_columns + } + ), + ) + donors[_ids(group)[0]] = expected + elif node_id == ATTACH_NODE: + expected = population_ops.patch(current[version], node, result) + else: + expected = current[version] + physical.replay.same_replayed_population(expected, observed[node_id]) + current[version] = expected + _verify_models(boundary, loaded, donors) + for version, population in current.items(): + physical.replay.same_replayed_population( + population_ops.Population.from_frame( + population.frame, version, mass_ledger=population.mass_ledger + ), + population_ops.Population.from_frame( + manifest.population(version), + version, + mass_ledger=manifest.mass_ledger(version), + ), + ) + receipt = codec.encode_json( + { + "protocol": values.PROTOCOL, + "parent_sha256": boundary.parent_view.digest, + "manifest_key": manifest.key, + "node_count": len(boundary.compiled.order), + "groups": list(groups), + "complete_population_compared": True, + "projection_sha256": codec.sha(boundary.qualified.projection), + "attachment_sha256": codec.sha(result.artifacts["attachment"]), + "health_projection_sha256": codec.sha(boundary.health.projection), + "health_attachment_sha256": codec.sha( + loaded[health_graph.ATTACH_NODE, "attachment"] + ), + "health_fields": [f.output for f in health_graph.health.FIELDS], + "release_eligible": False, + } + ) + output = SurveyEnrichmentRun( + run, + observed[health_graph.ATTACH_NODE], + manifest, + boundary.compiled, + boundary.store, + boundary.kernels, + boundary.paths, + receipt, + ) + objects = ( + output.parent_run, + output.population, + output.manifest, + output.compiled, + output.store, + output.kernels, + output.sources, + ) + stamp = _run_seal(output) + artifact_hashes = tuple( + sorted((n, a, codec.sha(p)) for (n, a), p in loaded.items()) + ) + boundary.requalify() + boundary.borrow() + fresh = parent.financial._artifacts( + manifest, + boundary.compiled, + boundary.store, + boundary.kernels, + dict(boundary.keys), + dict(boundary.implementations), + ) + boundary.borrow() + require( + tuple(sorted((n, a, codec.sha(p)) for (n, a), p in fresh.items())) + == artifact_hashes + and _run_seal(output) == stamp + and all(physical._population_stamp(observed[n]) == stamps[n] for n in observed), + "FINAL_OUTPUT", + ) + boundary.pure() + require(id(output) not in _ISSUED, "RUN_ALREADY_ISSUED") + ident = id(output) + + def forget(ref): + old = _ISSUED.get(ident) + if old is not None and old[0] is ref: + del _ISSUED[ident] + + entry = (weakref.ref(output, forget), boundary, stamp, objects, artifact_hashes) + _ISSUED[ident] = entry + return output diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/hours_worked.py b/packages/microcosm-build/src/microcosm/build/us_runtime/hours_worked.py index df08aa857..528957104 100644 --- a/packages/microcosm-build/src/microcosm/build/us_runtime/hours_worked.py +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/hours_worked.py @@ -59,7 +59,7 @@ SourceRuntimeError, run_source_stage, ) -from microcosm.frame import Frame +from microcosm.frame import Frame, Weights from microcosm.frame.units import US_SCHEMA __all__ = [ @@ -70,10 +70,13 @@ "US_HOURS_WORKED_REQUIRED_SOURCE_COLUMNS", "US_HOURS_WORKED_STAGE_NAME", "derive_us_hours_worked_from_manifest", + "us_hours_worked_gate_from_summary", + "us_hours_worked_person_summary", "us_hours_worked_signal_gate", "us_hours_worked_summary", "us_hours_worked_stage_spec", "with_us_hours_worked_inputs", + "with_us_hours_worked_person", ] US_HOURS_WORKED_STAGE_NAME = "hours_worked" @@ -233,8 +236,36 @@ def with_us_hours_worked_inputs(frame: Frame, *, seed: int, time_period: int) -> if have_all and _weekly_hours_carry_signal(person): return frame + produced = with_us_hours_worked_person( + person, + weights=frame.resolve_weights("person"), + seed=seed, + time_period=time_period, + ) + tables = {entity: frame.table(entity).copy() for entity in frame.entities} + tables["person"] = produced + return Frame( + tables, + frame.schema, + {entity: frame.weights_for(entity) for entity in frame.weighted_entities}, + frame.strata, + mass_log=frame.mass_log, + metadata=frame.metadata, + ) + + +def with_us_hours_worked_person( + person: pd.DataFrame, *, weights: Weights, seed: int, time_period: int +) -> pd.DataFrame: + """Run the same manifest handler on a declared person view and typed weights.""" + if len(weights.values) != len(person): + raise ValueError("US hours-worked person weights are not row-aligned.") + if all(column in person for column in US_HOURS_WORKED_OUTPUT_COLUMNS) and ( + _weekly_hours_carry_signal(person) + ): + return person stage_person = person.copy(deep=True) - stage_person[_PERSON_WEIGHT_COLUMN] = frame.resolve_weights("person").values + stage_person[_PERSON_WEIGHT_COLUMN] = weights.values output = run_source_stage( us_hours_worked_stage_spec(), tables={"person": stage_person}, @@ -251,24 +282,27 @@ def with_us_hours_worked_inputs(frame: Frame, *, seed: int, time_period: int) -> f"{column!r}." ) - tables = {entity: frame.table(entity).copy() for entity in frame.entities} + produced = person.copy() for column in US_HOURS_WORKED_OUTPUT_COLUMNS: - tables["person"][column] = aligned[column].to_numpy(dtype=np.float64) - return Frame( - tables, - frame.schema, - {entity: frame.weights_for(entity) for entity in frame.weighted_entities}, - frame.strata, - mass_log=frame.mass_log, - metadata=frame.metadata, - ) + produced[column] = aligned[column].to_numpy(dtype=np.float64) + return produced def us_hours_worked_summary(frame: Frame) -> dict[str, object]: """Weighted hours-distribution summary for gates and release manifests.""" - person = frame.table("person") - weights = np.asarray(frame.resolve_weights("person").values, dtype=np.float64) + return us_hours_worked_person_summary( + frame.table("person"), weights=frame.resolve_weights("person") + ) + + +def us_hours_worked_person_summary( + person: pd.DataFrame, *, weights: Weights +) -> dict[str, object]: + """Observe the actual person view with canonically resolved, aligned weights.""" + if len(weights.values) != len(person): + raise ValueError("US hours-worked person weights are not row-aligned.") + weights = np.asarray(weights.values, dtype=np.float64) weekly = pd.to_numeric( person["weekly_hours_worked_before_lsr"], errors="coerce" ).fillna(0.0) @@ -306,7 +340,6 @@ def us_hours_worked_signal_gate(frame: Frame) -> GateResult: """ person = frame.table("person") - failures: list[str] = [] missing = [ column for column in US_HOURS_WORKED_OUTPUT_COLUMNS @@ -321,7 +354,40 @@ def us_hours_worked_signal_gate(frame: Frame) -> GateResult: ) summary = us_hours_worked_summary(frame) - for column, count in summary["unique_counts"].items(): + return us_hours_worked_gate_from_summary(summary) + + +def us_hours_worked_gate_from_summary(summary: dict[str, object]) -> GateResult: + """Apply the original signal checks to actual typed producer observations.""" + if ( + not isinstance(summary, dict) + or set(summary) + != { + "worked_share", + "mean_weekly_hours_workers", + "worked_share_band", + "mean_weekly_hours_band", + "unique_counts", + } + or summary["worked_share_band"] != list(_WORKED_SHARE_BAND) + or summary["mean_weekly_hours_band"] != list(_MEAN_WEEKLY_HOURS_BAND) + or not isinstance(summary["unique_counts"], dict) + or set(summary["unique_counts"]) != set(US_HOURS_WORKED_OUTPUT_COLUMNS) + or any( + type(value) is not int or value < 0 + for value in summary["unique_counts"].values() + ) + or any( + type(summary[name]) not in (int, float) or not np.isfinite(summary[name]) + for name in ("worked_share", "mean_weekly_hours_workers") + ) + ): + raise ValueError("US hours-worked gate requires the complete original summary.") + failures: list[str] = [] + # Canonical artifact JSON sorts mappings; failure order remains the + # original declared output order, including after a cache roundtrip. + for column in US_HOURS_WORKED_OUTPUT_COLUMNS: + count = summary["unique_counts"][column] if count < 2: failures.append( f"{column}: constant column (one observed value) — the hours " diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/housing_inputs.py b/packages/microcosm-build/src/microcosm/build/us_runtime/housing_inputs.py index 1f83bd82c..f6f1dbbba 100644 --- a/packages/microcosm-build/src/microcosm/build/us_runtime/housing_inputs.py +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/housing_inputs.py @@ -24,8 +24,11 @@ import hashlib import warnings +from collections.abc import Mapping +from dataclasses import dataclass from importlib.resources import files from pathlib import Path +from types import MappingProxyType from typing import Any import numpy as np @@ -63,6 +66,11 @@ "US_HOUSING_REQUIRED_PERSON_SOURCE_COLUMNS", "US_HOUSING_SPM_UNIT_OUTPUT_COLUMNS", "derive_us_housing_inputs", + "AcsRentDonorPreparation", + "AcsRentRecipientPreparation", + "prepare_acs_rent_donor", + "prepare_acs_rent_recipient", + "finalize_acs_rent", "impute_us_pre_subsidy_rent", "impute_us_housing_assistance_to_puf_support", "load_acs_2022_rent_donor", @@ -465,14 +473,17 @@ def _constant_source_by_unit( return values.to_numpy(dtype=np.float64) -def derive_us_housing_inputs(frame: Frame) -> Frame: - """Carry the three exact ASEC housing/tenure inputs onto their entities.""" +def derive_us_housing_tables( + person: pd.DataFrame, household: pd.DataFrame, spm_unit: pd.DataFrame +) -> tuple[pd.DataFrame, pd.DataFrame]: + """Return copied household/SPM tables with the exact measured housing leaves. + + Inputs are the actual linked tables (or declared views) at the existing + housing derivation boundary. SPM raw values align through person_spm_unit_id; + returned group row order, indices, other columns and input tables are kept. + This helper neither reconstructs other entities nor changes raw code meaning. + """ - if frame.schema != US_SCHEMA: - raise ValueError("US housing inputs require the US schema.") - person = frame.table("person") - household = frame.table("household") - spm_unit = frame.table("spm_unit") _required_columns( person, ("SPM_CAPHOUSESUB", "SPM_TENMORTSTATUS", "person_spm_unit_id"), @@ -521,15 +532,30 @@ def derive_us_housing_inputs(frame: Frame) -> Frame: f"{unknown_spm_codes}." ) - tables = {entity: frame.table(entity).copy() for entity in frame.entities} - tables["household"]["tenure_type"] = np.asarray( + household_result = household.copy() + spm_result = spm_unit.copy() + household_result["tenure_type"] = np.asarray( [_HOUSEHOLD_TENURE_MAP[code] for code in raw_household_codes], dtype=object ) - tables["spm_unit"]["receives_housing_assistance"] = subsidy > 0.0 - tables["spm_unit"]["takes_up_housing_assistance_if_eligible"] = subsidy > 0.0 - tables["spm_unit"]["spm_unit_tenure_type"] = np.asarray( + spm_result["receives_housing_assistance"] = subsidy > 0.0 + spm_result["takes_up_housing_assistance_if_eligible"] = subsidy > 0.0 + spm_result["spm_unit_tenure_type"] = np.asarray( [_SPM_TENURE_MAP[code] for code in raw_spm_codes], dtype=object ) + return household_result, spm_result + + +def derive_us_housing_inputs(frame: Frame) -> Frame: + """Carry the three exact ASEC housing/tenure inputs onto their entities.""" + + if frame.schema != US_SCHEMA: + raise ValueError("US housing inputs require the US schema.") + household, spm_unit = derive_us_housing_tables( + frame.table("person"), frame.table("household"), frame.table("spm_unit") + ) + tables = {entity: frame.table(entity).copy() for entity in frame.entities} + tables["household"] = household + tables["spm_unit"] = spm_unit return Frame( tables, frame.schema, @@ -703,38 +729,203 @@ def _encode_acs_predictors( ) -> tuple[pd.DataFrame, pd.DataFrame, tuple[str, ...]]: """Dummy-encode the retired QRF's string predictors without ordinality.""" + context = _rent_encoding_context(training) + return ( + _encode_rent_predictors(training, context=context), + _encode_rent_predictors(prediction, context=context), + context["predictors"], + ) + + +def _rent_encoding_context(training: pd.DataFrame) -> Mapping[str, object]: numeric_predictors = tuple( predictor for predictor in ACS_RENT_PREDICTORS if predictor not in _ACS_CATEGORICAL_PREDICTORS ) - encoded_training = training.loc[:, numeric_predictors].copy() - encoded_prediction = prediction.loc[:, numeric_predictors].copy() encoded_columns = list(numeric_predictors) + category_levels = {} for column in _ACS_CATEGORICAL_PREDICTORS: training_values = training[column].astype(str) - prediction_values = prediction[column].astype(str) levels = tuple(sorted(training_values.unique())) - unknown = sorted(set(prediction_values.unique()) - set(levels)) - if unknown: - raise ValueError( - f"ACS rent recipient {column!r} has donor-unsupported value(s): " - f"{unknown}." - ) if len(levels) < 2: raise ValueError( f"ACS rent donor categorical predictor {column!r} is constant." ) + category_levels[column] = levels + encoded_columns.extend(f"{column}__{level}" for level in levels[1:]) + return MappingProxyType( + { + "schema": "microcosm.us.acs_rent_encoding.v1", + "predictors": tuple(encoded_columns), + "category_levels": MappingProxyType(category_levels), + } + ) + + +def _validated_rent_encoding_context( + context: Mapping[str, object], +) -> Mapping[str, object]: + if ( + not isinstance(context, Mapping) + or set(context) != {"schema", "predictors", "category_levels"} + or context.get("schema") != "microcosm.us.acs_rent_encoding.v1" + ): + raise ValueError("Unsupported ACS rent donor encoding context.") + levels_by_column = context["category_levels"] + if not isinstance(levels_by_column, Mapping) or set(levels_by_column) != set( + _ACS_CATEGORICAL_PREDICTORS + ): + raise ValueError( + "ACS rent donor encoding must name its exact categorical columns." + ) + expected = [ + name for name in ACS_RENT_PREDICTORS if name not in _ACS_CATEGORICAL_PREDICTORS + ] + frozen_levels = {} + for column in _ACS_CATEGORICAL_PREDICTORS: + levels = levels_by_column[column] + if ( + not isinstance(levels, (tuple, list)) + or len(levels) < 2 + or any(not isinstance(value, str) for value in levels) + or list(levels) != sorted(set(levels)) + ): + raise ValueError( + "ACS rent donor encoding levels must be sorted unique strings." + ) + frozen_levels[column] = tuple(levels) + expected.extend(f"{column}__{level}" for level in levels[1:]) + if not isinstance(context["predictors"], (tuple, list)) or tuple( + context["predictors"] + ) != tuple(expected): + raise ValueError( + "ACS rent donor encoding predictor order differs from its levels." + ) + return MappingProxyType( + { + "schema": context["schema"], + "predictors": tuple(expected), + "category_levels": MappingProxyType(frozen_levels), + } + ) + + +def _encode_rent_predictors( + table: pd.DataFrame, *, context: Mapping[str, object] +) -> pd.DataFrame: + numeric = tuple( + name for name in ACS_RENT_PREDICTORS if name not in _ACS_CATEGORICAL_PREDICTORS + ) + encoded = table.loc[:, numeric].copy() + for column in _ACS_CATEGORICAL_PREDICTORS: + values = table[column].astype(str) + levels = context["category_levels"][column] + unknown = sorted(set(values.unique()) - set(levels)) + if unknown: + raise ValueError( + f"ACS rent recipient {column!r} has donor-unsupported value(s): {unknown}." + ) for level in levels[1:]: - dummy = f"{column}__{level}" - encoded_training[dummy] = training_values.eq(level).to_numpy( - dtype=np.float64 + encoded[f"{column}__{level}"] = values.eq(level).to_numpy(dtype=np.float64) + return encoded + + +@dataclass(frozen=True) +class AcsRentDonorPreparation: + """Actual sampled donor training rows and recipient-independent encoding.""" + + training: pd.DataFrame + predictors: tuple[str, ...] + context: Mapping[str, object] + + def __post_init__(self) -> None: + context = _validated_rent_encoding_context(self.context) + if ( + self.predictors != context["predictors"] + or not isinstance(self.training, pd.DataFrame) + or tuple(self.training.columns) + != (*self.predictors, "rent", _DONOR_WEIGHT_COLUMN) + ): + raise ValueError( + "ACS rent donor training columns differ from their encoding." ) - encoded_prediction[dummy] = prediction_values.eq(level).to_numpy( - dtype=np.float64 + object.__setattr__(self, "context", context) + + +@dataclass(frozen=True) +class AcsRentRecipientPreparation: + """Predictor rows and their household heads' original person positions. + + ``head_positions[i]`` belongs to household ``predictors.index[i]``; it + need not be sorted when person and household table orders differ. + """ + + predictors: pd.DataFrame + head_positions: np.ndarray + person_index: pd.Index + + def __post_init__(self) -> None: + positions = np.asarray(self.head_positions) + if ( + not isinstance(self.predictors, pd.DataFrame) + or not isinstance(self.person_index, pd.Index) + or positions.ndim != 1 + or not np.issubdtype(positions.dtype, np.integer) + or len(positions) != len(self.predictors) + or (positions < 0).any() + or (positions >= len(self.person_index)).any() + or len(np.unique(positions)) != len(positions) + ): + raise ValueError( + "ACS rent head positions must align to the original person rows." ) - encoded_columns.append(dummy) - return encoded_training, encoded_prediction, tuple(encoded_columns) + positions = positions.astype(np.int64, copy=True) + positions.flags.writeable = False + object.__setattr__(self, "head_positions", positions) + object.__setattr__(self, "person_index", self.person_index.copy(deep=True)) + + +def prepare_acs_rent_recipient( + frame: Frame, *, donor_context: Mapping[str, object] +) -> AcsRentRecipientPreparation: + """Encode actual head predictors against donor-owned categorical levels.""" + context = _validated_rent_encoding_context(donor_context) + features, head_mask = _recipient_head_features(frame) + head_positions = np.flatnonzero(head_mask) + head_households = pd.Index( + frame.person["person_household_id"].to_numpy()[head_positions] + ) + household_order = head_households.get_indexer(features.index) + if (household_order < 0).any(): + raise ValueError("ACS rent predictor household has no selected person head.") + return AcsRentRecipientPreparation( + _encode_rent_predictors(features, context=context), + head_positions[household_order], + frame.person.index, + ) + + +def finalize_acs_rent( + prepared: AcsRentRecipientPreparation, raw_rent: pd.Series | pd.DataFrame +) -> np.ndarray: + """Clip raw draws and scatter each household's value to its own head.""" + if not isinstance(prepared, AcsRentRecipientPreparation): + raise TypeError("ACS rent finalization requires recipient preparation.") + if isinstance(raw_rent, pd.DataFrame) and tuple(raw_rent.columns) == ("rent",): + raw_rent = raw_rent["rent"] + if not isinstance(raw_rent, pd.Series) or raw_rent.name != "rent": + raise ValueError("ACS rent finalization requires the named raw rent Series.") + if not raw_rent.index.equals(prepared.predictors.index): + raise ValueError( + "ACS rent raw prediction index differs from prepared household order." + ) + predicted = pd.to_numeric(raw_rent, errors="coerce").to_numpy(dtype=np.float64) + if not np.isfinite(predicted).all(): + raise ValueError("ACS rent QRF produced nonfinite predictions.") + person_rent = np.zeros(len(prepared.person_index), dtype=np.float64) + person_rent[prepared.head_positions] = np.maximum(predicted, 0.0) + return person_rent def _stable_string_hash(value: str) -> np.uint64: @@ -822,14 +1013,12 @@ def _archived_joint_training_sample( return sampled, sampled_masks -def impute_us_pre_subsidy_rent( - frame: Frame, - donor: pd.DataFrame, - *, - seed: int, - n_estimators: int = _DEFAULT_N_ESTIMATORS, -) -> np.ndarray: - """Draw annual ACS rent once per CPS household and place it on the head.""" +def prepare_acs_rent_donor(donor: pd.DataFrame) -> AcsRentDonorPreparation: + """Select and encode actual archived ACS rent support without recipients. + + The joint rent/real-estate-tax sampling, allocation masks, reset-index + order and household design weights retain their existing semantics. + """ required = ( *ACS_RENT_PREDICTORS, @@ -869,35 +1058,39 @@ def impute_us_pre_subsidy_rent( if float(fit_frame[_DONOR_WEIGHT_COLUMN].sum()) <= 0.0: raise ValueError("ACS rent donor sampled weights sum to zero.") + context = _rent_encoding_context(fit_frame) + encoded_training = _encode_rent_predictors(fit_frame, context=context) + encoded_training["rent"] = fit_frame["rent"].to_numpy(dtype=np.float64) + encoded_training[_DONOR_WEIGHT_COLUMN] = fit_frame[_DONOR_WEIGHT_COLUMN].to_numpy( + dtype=np.float64 + ) + return AcsRentDonorPreparation(encoded_training, context["predictors"], context) + + +def impute_us_pre_subsidy_rent( + frame: Frame, + donor: pd.DataFrame, + *, + seed: int, + n_estimators: int = _DEFAULT_N_ESTIMATORS, +) -> np.ndarray: + """Draw annual ACS rent once per CPS household and place it on the head.""" + + prepared_donor = prepare_acs_rent_donor(donor) + global QRF if QRF is None: from importlib import import_module QRF = import_module("microcosm.fit").QRF - features, head_mask = _recipient_head_features(frame) - encoded_training, encoded_features, encoded_predictors = _encode_acs_predictors( - fit_frame, - features, - ) - encoded_training["rent"] = fit_frame["rent"].to_numpy(dtype=np.float64) - encoded_training[_DONOR_WEIGHT_COLUMN] = fit_frame[_DONOR_WEIGHT_COLUMN].to_numpy( - dtype=np.float64 - ) + recipient = prepare_acs_rent_recipient(frame, donor_context=prepared_donor.context) fitted = QRF(n_estimators=int(n_estimators), seed=int(seed)).fit( - encoded_training, - predictors=list(encoded_predictors), + prepared_donor.training, + predictors=list(prepared_donor.predictors), targets=["rent"], weights=_DONOR_WEIGHT_COLUMN, ) - predicted = pd.to_numeric( - fitted.predict(encoded_features)["rent"], errors="coerce" - ).to_numpy(dtype=np.float64) - if not np.isfinite(predicted).all(): - raise ValueError("ACS rent QRF produced nonfinite predictions.") - predicted = np.maximum(predicted, 0.0) - person_rent = np.zeros(frame.n("person"), dtype=np.float64) - person_rent[head_mask] = predicted - return person_rent + return finalize_acs_rent(recipient, fitted.predict(recipient.predictors)) def _person_puf_predictors(frame: Frame) -> pd.DataFrame: diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/input_coverage_profile.py b/packages/microcosm-build/src/microcosm/build/us_runtime/input_coverage_profile.py new file mode 100644 index 000000000..21f7ad81f --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/input_coverage_profile.py @@ -0,0 +1,202 @@ +"""Explicit input-name profiles; neither a release gate nor source admission. + +The historical release gate retains its existing 163-input default. This +lightweight declaration adds the national/CD projection without importing its +engine-dependent anti-rot helpers or reading package resources. Its exact source +manifest pin documents the extraction, not authority over a receiving Frame. +Grain/applicability are deliberately not guessed from engine variable names. +""" + +from enum import StrEnum + + +class USInputProfile(StrEnum): + HISTORICAL = "us_release_163_v1" + NATIONAL_CD = "us_national_cd_161_v1" + + +MANIFEST_SHA256 = "d96d0e98078906a9b47d47662e173553b1937851c7a8dd576010b50705ca1b1d" +HISTORICAL_REQUIRED_INPUTS = ( + "age", + "alimony_expense", + "alimony_income", + "attends_eligible_educational_institution_for_american_opportunity_credit", + "auto_loan_balance", + "auto_loan_interest", + "bank_account_assets", + "block_geoid", + "bond_assets", + "business_is_sstb", + "casualty_loss", + "charitable_cash_donations", + "charitable_non_cash_donations", + "child_support_expense", + "child_support_received", + "congressional_district_geoid", + "county_fips", + "cps_race", + "detailed_occupation_recode", + "disability_benefits", + "domestic_production_ald", + "educational_assistance", + "educator_expense", + "employment_income_before_lsr", + "estate_income", + "estate_income_would_be_qualified", + "farm_income", + "farm_operations_income", + "farm_operations_income_would_be_qualified", + "farm_rent_income", + "farm_rent_income_would_be_qualified", + "first_home_mortgage_balance", + "first_home_mortgage_interest", + "first_home_mortgage_origination_year", + "fsla_overtime_premium", + "has_american_opportunity_credit_1098_t_or_exception", + "has_american_opportunity_credit_institution_ein", + "has_champva_health_coverage_at_interview", + "has_esi", + "has_indian_health_service_coverage_at_interview", + "has_marketplace_health_coverage_at_interview", + "has_medicaid_health_coverage_at_interview", + "has_never_worked", + "has_non_marketplace_direct_purchase_health_coverage_at_interview", + "has_other_means_tested_health_coverage_at_interview", + "has_tricare_health_coverage_at_interview", + "has_va_health_coverage_at_interview", + "health_insurance_premiums", + "health_insurance_premiums_without_medicare_part_b", + "health_savings_account_ald", + "home_mortgage_interest", + "hourly_wage", + "hours_worked_last_week", + "household_vehicles_owned", + "household_vehicles_value", + "household_weight", + "immigration_status_str", + "investment_income_elected_form_4952", + "investment_interest_expense", + "is_blind", + "is_computer_scientist", + "is_disabled", + "is_enrolled_at_least_half_time_for_american_opportunity_credit", + "is_executive_administrative_professional", + "is_farmer_fisher", + "is_female", + "is_full_time_college_student", + "is_hispanic", + "is_household_head", + "is_incapable_of_self_care", + "is_military", + "is_paid_hourly", + "is_pregnant", + "is_pursuing_credential_for_american_opportunity_credit", + "is_self_employed", + "is_separated", + "is_surviving_spouse", + "is_union_member_or_covered", + "keogh_distributions", + "long_term_capital_gains_before_response", + "long_term_capital_gains_on_collectibles", + "meets_ssi_disability_criteria", + "miscellaneous_income", + "net_worth", + "non_qualified_dividend_income", + "non_sch_d_capital_gains", + "other_health_insurance_premiums", + "other_medical_expenses", + "over_the_counter_health_expenses", + "own_children_in_household", + "partnership_s_corp_income_would_be_qualified", + "pre_subsidy_care_expenses", + "pre_subsidy_rent", + "previous_year_income_available", + "qualified_bdc_income", + "qualified_dividend_income", + "qualified_passenger_vehicle_loan_interest", + "qualified_reit_and_ptp_income", + "qualified_tuition_expenses", + "real_estate_taxes", + "receives_housing_assistance", + "rental_income", + "rental_income_would_be_qualified", + "roth_401k_contributions_desired", + "roth_ira_contributions_desired", + "salt_refund_income", + "schedule_d_capital_gain_distributions", + "selected_marketplace_plan_benchmark_ratio", + "self_employed_pension_contributions_desired", + "self_employment_income_before_lsr", + "self_employment_income_last_year", + "self_employment_income_would_be_qualified", + "short_term_capital_gains", + "social_security_dependents", + "social_security_disability", + "social_security_retirement", + "social_security_survivors", + "spm_unit_energy_subsidy", + "spm_unit_pre_subsidy_childcare_expenses", + "spm_unit_tenure_type", + "ssn_card_type", + "sstb_self_employment_income_before_lsr", + "sstb_self_employment_income_would_be_qualified", + "sstb_unadjusted_basis_qualified_property", + "sstb_w2_wages_from_qualified_business", + "state_fips", + "stock_assets", + "student_loan_interest", + "takes_up_aca_if_eligible", + "takes_up_eitc", + "takes_up_head_start_if_eligible", + "takes_up_housing_assistance_if_eligible", + "takes_up_medicaid_if_eligible", + "takes_up_medicare_if_eligible", + "takes_up_snap_if_eligible", + "takes_up_ssi_if_eligible", + "takes_up_tanf_if_eligible", + "takes_up_wic_if_eligible", + "tax_exempt_interest_income", + "tax_exempt_ira_distributions", + "tax_exempt_private_pension_income", + "taxable_401k_distributions", + "taxable_403b_distributions", + "taxable_interest_income", + "taxable_ira_distributions", + "taxable_private_pension_income", + "taxable_sep_distributions", + "tenure_type", + "tip_income", + "tract_geoid", + "traditional_401k_contributions_desired", + "traditional_ira_contributions_desired", + "treasury_tipped_occupation_code", + "unadjusted_basis_qualified_property", + "unemployment_compensation", + "unrecaptured_section_1250_gain", + "unreimbursed_business_employee_expenses", + "veterans_benefits", + "w2_wages_from_qualified_business", + "weekly_hours_worked_before_lsr", + "weeks_unemployed", + "workers_compensation", + "would_file_taxes_voluntarily", +) +NATIONAL_CD_ENGINE_OMISSIONS = ("block_geoid", "tract_geoid") +NATIONAL_CD_REQUIRED_INPUTS = tuple( + name + for name in HISTORICAL_REQUIRED_INPUTS + if name not in NATIONAL_CD_ENGINE_OMISSIONS +) +# Independent build state, even when not an engine input in the selected profile. +ASSIGNED_BLOCK_COLUMN = "census_block_geoid" + + +def required_us_inputs( + profile: USInputProfile = USInputProfile.HISTORICAL, +) -> tuple[str, ...]: + """Return a closed profile; no custom name list or silent string fallback.""" + if type(profile) is not USInputProfile: + raise TypeError("US_INPUT_PROFILE_TYPE") + if profile is USInputProfile.HISTORICAL: + return HISTORICAL_REQUIRED_INPUTS + return NATIONAL_CD_REQUIRED_INPUTS diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/multispine_pool.py b/packages/microcosm-build/src/microcosm/build/us_runtime/multispine_pool.py index 8f2ea06b9..1f4a86047 100644 --- a/packages/microcosm-build/src/microcosm/build/us_runtime/multispine_pool.py +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/multispine_pool.py @@ -15,7 +15,7 @@ import hashlib import json -from collections.abc import Callable, Mapping +from collections.abc import Callable, Iterable, Mapping from dataclasses import dataclass, field from functools import lru_cache from importlib.metadata import version @@ -38,12 +38,18 @@ from microcosm.build.us_runtime.adult_care import with_us_adult_care_inputs from microcosm.build.us_runtime.child_support import with_us_child_support_inputs from microcosm.build.us_runtime.childcare import with_us_childcare_inputs -from microcosm.build.us_runtime.cps_carried import derive_us_cps_carried_inputs +from microcosm.build.us_runtime.cps_carried import ( + CpsCarriedTables, + derive_us_cps_carried_inputs, + derive_us_cps_carried_tables, +) from microcosm.build.us_runtime.disability_benefits import ( with_us_disability_benefits, ) from microcosm.build.us_runtime.education_inputs import with_us_education_inputs from microcosm.build.us_runtime.eligibility_inputs import ( + prepare_us_eligibility_person, + us_eligibility_inputs_person_carries_signal, with_us_eligibility_inputs, ) from microcosm.build.us_runtime.energy_subsidy import ( @@ -51,8 +57,11 @@ ) from microcosm.build.us_runtime.hours_worked import ( US_HOURS_WORKED_POOL_EXCLUDED_COLUMNS, + us_hours_worked_gate_from_summary, + us_hours_worked_person_summary, us_hours_worked_signal_gate, with_us_hours_worked_inputs, + with_us_hours_worked_person, ) from microcosm.build.us_runtime.housing_inputs import ( US_HOUSING_ASSISTANCE_PUF_MAX_TRAIN_SAMPLES, @@ -70,6 +79,7 @@ ) from microcosm.build.us_runtime.pregnancy import with_us_pregnancy_inputs from microcosm.build.us_runtime.prior_year_income import ( + prepare_us_prior_year_person, with_us_prior_year_income_inputs, ) from microcosm.build.us_runtime.puf_qrf_chain import PRIMARY_QRF_TARGET_ORDER @@ -84,6 +94,8 @@ with_us_qbi_input_reconciliation, ) from microcosm.build.us_runtime.relationship_inputs import ( + prepare_us_relationship_person, + us_relationship_inputs_person_carries_signal, with_us_relationship_inputs, ) from microcosm.build.us_runtime.retirement_contributions import ( @@ -92,10 +104,6 @@ from microcosm.build.us_runtime.retirement_distributions import ( with_us_retirement_distribution_inputs, ) -from microcosm.build.us_runtime.spine_agreement import ( - default_spine_agreement_registry, - spine_agreement_gate, -) from microcosm.build.us_runtime.spine_assembly import assemble_spines from microcosm.build.us_runtime.support_provenance import ( SPINE_ASSEMBLY_MANIFEST_KEY, @@ -121,7 +129,7 @@ from microcosm.build.us_runtime.workers_compensation import ( with_us_workers_compensation, ) -from microcosm.frame import US_SCHEMA, Frame +from microcosm.frame import US_SCHEMA, Frame, Weights from microcosm.frame.adapters.policyengine_us import ( PolicyEngineUSVariableMetadataIndex, VariableDependencyClosure, @@ -163,6 +171,18 @@ "finalize_multispine_source_inputs", "materialize_multispine_agreement_outputs", "materialize_pool_deferred_transfer_inputs", + "multispine_hours_worked_boundary_columns", + "multispine_housing_boundary_columns", + "multispine_housing_implementation_contract", + "multispine_housing_output_family", + "multispine_housing_source_selection", + "assert_multispine_source_merge_labels_supported", + "merge_multispine_housing_source_outputs", + "validate_multispine_housing_person_boundary", + "observe_multispine_hours_person", + "observe_multispine_hours_worked_inputs", + "validate_multispine_hours_person_boundary", + "validate_multispine_hours_worked_boundary", "pool_input_surface", "pool_engine_input_projection_receipt", "pool_remaining_stage_input_manifest", @@ -861,9 +881,8 @@ def _resolve_take_up_program_bindings( for program in load_take_up_contract().programs ) for index, binding in enumerate(bindings): - if ( - len(binding) != 3 - or not all(isinstance(value, str) and value for value in binding) + if len(binding) != 3 or not all( + isinstance(value, str) and value for value in binding ): raise ValueError( "Take-up manifest program binding must contain three non-empty " @@ -1333,9 +1352,7 @@ def surface_provision(variable: str) -> str: variable, execution_scope="whole_pool", provision=provision, - available_by=( - "transferred" if variable in transfer_owned else "seeded" - ), + available_by=("transferred" if variable in transfer_owned else "seeded"), fallback=fallback, ) @@ -1715,6 +1732,14 @@ def materialize_pool_deferred_transfer_inputs(frame: Frame) -> PoolStageOutput: return PoolStageOutput(result, {"inputs": receipts}) +# The default agreement registry validates the engine ABI, whose fresh +# manifest reads the pool functions above. Define that surface before importing +# the registry so either module can be the first import in a fresh process. +from microcosm.build.us_runtime.spine_agreement import ( # noqa: E402 + default_spine_agreement_registry, + spine_agreement_gate, +) + POOL_SPINE_AGREEMENT_REGISTRY = default_spine_agreement_registry( pool_transfer_target_families() ) @@ -1792,32 +1817,872 @@ def prepare_multispine_source_inputs_for_clone( def _with_gated_us_hours_worked_inputs(frame: Frame) -> PoolStageOutput: """Run the shared hours kernel, then keep only pool-owned input leaves.""" + observed = _observe_us_hours_worked_inputs(frame) + gate = observed.receipt["hours_worked_signal_gate"] + if not gate["passed"]: + raise ValueError( + "Pool pre-clone hours-worked signal gate failed:\n " + + "\n ".join(gate["failures"]) + ) + return observed + + +def _observe_us_hours_worked_inputs(frame: Frame) -> PoolStageOutput: + """Compute actual values and gate evidence without certifying the surface.""" + produced = with_us_hours_worked_inputs( frame, seed=POOL_RANDOM_SEED, time_period=POOL_TIME_PERIOD, ) gate = us_hours_worked_signal_gate(produced) - if not gate.passed: - raise ValueError( - "Pool pre-clone hours-worked signal gate failed:\n " - + "\n ".join(gate.failures) - ) pool_surface, removed = _drop_source_output_columns( produced, {"person": US_HOURS_WORKED_POOL_EXCLUDED_COLUMNS}, ) return PoolStageOutput( pool_surface, + _hours_gate_receipt(gate, removed), + ) + + +def _hours_gate_receipt( + gate: GateResult, removed: Mapping[str, list[str]] +) -> dict[str, object]: + return { + "hours_worked_signal_gate": { + "name": gate.name, + "passed": gate.passed, + "failures": list(gate.failures), + "details": dict(gate.details), + }, + "pool_excluded_outputs_removed": dict(removed), + } + + +def multispine_hours_worked_boundary_columns() -> tuple[str, ...]: + """Declare source-boundary evidence without exposing role dispatch downstream.""" + return (_CPS_SOURCE_EVIDENCE_COLUMN, support_clone_index_column("person")) + + +def multispine_cps_carried_boundary_columns() -> tuple[str, ...]: + """Declare the registered pre-clone CPS evidence to its graph boundary.""" + return (_CPS_SOURCE_EVIDENCE_COLUMN, support_clone_index_column("person")) + + +def validate_multispine_cps_carried_person_boundary( + person: pd.DataFrame, *, metadata: Mapping[str, object] +) -> dict[str, object]: + """Retain source-role validation in the existing registered owner.""" + _assert_source_person_boundary( + person, metadata=metadata, person_entity="person", phase=_PRE_CLONE_PHASE + ) + mask = _cps_person_evidence_mask( + person, person_entity="person", phase=_PRE_CLONE_PHASE + ) + return {"phase": _PRE_CLONE_PHASE, "cps_person_rows": int(mask.sum())} + + +def multispine_cps_carried_implementation_contract() -> dict[str, object]: + """Bind live provider projections actually consumed by this source operator.""" + return { + "outputs": { + entity: sorted(columns) + for entity, columns in PRE_ASSEMBLY_OPERATOR_OUTPUT_FAMILIES[ + "cps_carried" + ].items() + }, + "transient_outputs": _transient_source_outputs( + ("derive_us_cps_carried_inputs",), PRE_ASSEMBLY_OPERATOR_OUTPUT_FAMILIES + ), + } + + +def observe_multispine_cps_carried_tables( + person: pd.DataFrame, + spm_unit: pd.DataFrame, + *, + weights: Weights, + metadata: Mapping[str, object], + pool_rows: Mapping[str, int], +) -> tuple[CpsCarriedTables, dict[str, object]]: + """Use actual person/SPM views for the original source projection and merge.""" + boundary = validate_multispine_cps_carried_person_boundary( + person, metadata=metadata + ) + if ( + set(pool_rows) != set(US_SCHEMA.entities) + or any(type(value) is not int or value < 0 for value in pool_rows.values()) + or pool_rows["person"] != len(person) + or pool_rows["spm_unit"] != len(spm_unit) + or len(weights) != len(person) + ): + raise ValueError("CPS-carried population counts or weights are misaligned.") + for group in US_SCHEMA.group_entities: + if pool_rows[group] != int( + person[US_SCHEMA.membership_column(group)].nunique() + ): + raise ValueError("CPS-carried population counts differ from memberships.") + if spm_unit.spm_unit_id.duplicated().any() or set(spm_unit.spm_unit_id) != set( + person.person_spm_unit_id + ): + raise ValueError("CPS-carried SPM rows differ from actual memberships.") + mask = _cps_person_evidence_mask( + person, person_entity="person", phase=_PRE_CLONE_PHASE + ) + selected_person = without_support_role_metadata(person.loc[mask], entity="person") + selected_spm = without_support_role_metadata( + spm_unit.loc[ + spm_unit.spm_unit_id.isin(selected_person.person_spm_unit_id) + ].reset_index(drop=True), + entity="spm_unit", + ) + # Match selected Frame weight validation without constructing a partial + # Frame carrying whole-population metadata or unobserved entity tables. + weights.with_values(weights.values[mask], kind=weights.kind) + outputs = PRE_ASSEMBLY_OPERATOR_OUTPUT_FAMILIES["cps_carried"] + selected = { + "person": selected_person.drop( + columns=_unavailable_output_columns(selected_person, outputs["person"]) + ), + "spm_unit": selected_spm.drop( + columns=_unavailable_output_columns(selected_spm, outputs["spm_unit"]) + ), + } + produced = derive_us_cps_carried_tables(selected["person"], selected["spm_unit"]) + merged = {} + merged_rows = {} + for entity, outcome in ( + ("person", produced.person), + ("spm_unit", produced.spm_unit), + ): + identity = US_SCHEMA.entity_id_column(entity) + _assert_source_table_identity( + selected[entity], + outcome, + entity_id=identity, + operator_name="derive_us_cps_carried_inputs", + ) + target = person if entity == "person" else spm_unit + merged[entity], merged_rows[entity] = _merge_source_person_or_group_outputs( + target.copy(), + outcome, + outputs[entity], + entity=entity, + entity_id=identity, + operator_name="derive_us_cps_carried_inputs", + ) + selected_rows = { + "person": len(selected_person), + **{ + group: int(selected_person[US_SCHEMA.membership_column(group)].nunique()) + for group in US_SCHEMA.group_entities + }, + } + receipt = _source_operator_receipt( + order_index=0, + operator_name="derive_us_cps_carried_inputs", + family="cps_carried", + phase=_PRE_CLONE_PHASE, + contract=POOL_OPERATOR_CONTRACTS["derive_us_cps_carried_inputs"], + before_rows=dict(pool_rows), + available_rows=selected_rows, + output_rows=selected_rows, + merged_rows=merged_rows, + declared_outputs=outputs, + formula_owned_removed={}, + kernel_receipt={}, + overlap_ownership=None, + ) + return CpsCarriedTables( + merged["person"], merged["spm_unit"] + ), _source_chain_receipt( + phase=_PRE_CLONE_PHASE, + operator_names=("derive_us_cps_carried_inputs",), + evidence_rows=boundary["cps_person_rows"], + receipts=[receipt], + output_families=PRE_ASSEMBLY_OPERATOR_OUTPUT_FAMILIES, + ) + + +def multispine_prior_year_boundary_columns() -> tuple[str, ...]: + """Declare the existing pre-clone evidence at its registered source owner.""" + return (_CPS_SOURCE_EVIDENCE_COLUMN, support_clone_index_column("person")) + + +def validate_multispine_prior_year_person_boundary( + person: pd.DataFrame, *, metadata: Mapping[str, object] +) -> dict[str, object]: + """Reject support populations before the source-only prior-year join.""" + _assert_source_person_boundary( + person, metadata=metadata, person_entity="person", phase=_PRE_CLONE_PHASE + ) + mask = _cps_person_evidence_mask( + person, person_entity="person", phase=_PRE_CLONE_PHASE + ) + return {"phase": _PRE_CLONE_PHASE, "cps_person_rows": int(mask.sum())} + + +def multispine_prior_year_implementation_contract() -> dict[str, object]: + """Bind the live output and transient projections consumed by this operator.""" + return { + "outputs": { + entity: sorted(columns) + for entity, columns in PRE_ASSEMBLY_OPERATOR_OUTPUT_FAMILIES[ + "prior_year_income" + ].items() + }, + "transient_outputs": _transient_source_outputs( + ("with_us_prior_year_income_inputs",), + PRE_ASSEMBLY_OPERATOR_OUTPUT_FAMILIES, + ), + "seed": POOL_RANDOM_SEED, + "time_period": POOL_TIME_PERIOD, + } + + +def observe_multispine_prior_year_person( + person: pd.DataFrame, + *, + weights: Weights, + metadata: Mapping[str, object], + pool_rows: Mapping[str, int], +) -> tuple[pd.DataFrame, dict[str, object]]: + """Run the real pre-clone join on its declared source person projection. + + Canonical person weights are selected in the original person order. Actual + memberships determine selected group counts; authenticated full counts + describe the unchanged pool. No incomplete Frame is constructed. Support + role fields are removed only from the validated ephemeral source view, so + the helper's support-QRF path cannot run at this boundary. + """ + boundary = validate_multispine_prior_year_person_boundary(person, metadata=metadata) + if ( + not isinstance(weights, Weights) + or set(pool_rows) != set(US_SCHEMA.entities) + or any(type(value) is not int or value < 0 for value in pool_rows.values()) + or pool_rows["person"] != len(person) + or len(weights) != len(person) + ): + raise ValueError("Prior-year population counts or weights are misaligned.") + for group in US_SCHEMA.group_entities: + if pool_rows[group] != int( + person[US_SCHEMA.membership_column(group)].nunique() + ): + raise ValueError("Prior-year population counts differ from memberships.") + mask = _cps_person_evidence_mask( + person, person_entity="person", phase=_PRE_CLONE_PHASE + ) + selected = without_support_role_metadata(person.loc[mask], entity="person") + outputs = PRE_ASSEMBLY_OPERATOR_OUTPUT_FAMILIES["prior_year_income"] + selected = selected.drop( + columns=_unavailable_output_columns(selected, outputs["person"]) + ) + selected_weights = weights.with_values(weights.values[mask], kind=weights.kind) + produced = prepare_us_prior_year_person( + selected, + weights=selected_weights, + seed=POOL_RANDOM_SEED, + time_period=POOL_TIME_PERIOD, + ) + _assert_source_table_identity( + selected, + produced, + entity_id="person_id", + operator_name="with_us_prior_year_income_inputs", + ) + merged, count = _merge_source_person_or_group_outputs( + person.copy(), + produced, + outputs["person"], + entity="person", + entity_id="person_id", + operator_name="with_us_prior_year_income_inputs", + ) + selected_rows = { + "person": len(selected), + **{ + group: int(selected[US_SCHEMA.membership_column(group)].nunique()) + for group in US_SCHEMA.group_entities + }, + } + receipt = _source_operator_receipt( + order_index=0, + operator_name="with_us_prior_year_income_inputs", + family="prior_year_income", + phase=_PRE_CLONE_PHASE, + contract=POOL_OPERATOR_CONTRACTS["with_us_prior_year_income_inputs"], + before_rows=dict(pool_rows), + available_rows=selected_rows, + output_rows=selected_rows, + merged_rows={"person": count}, + declared_outputs=outputs, + formula_owned_removed={}, + kernel_receipt={}, + overlap_ownership=None, + ) + return merged, _source_chain_receipt( + phase=_PRE_CLONE_PHASE, + operator_names=("with_us_prior_year_income_inputs",), + evidence_rows=boundary["cps_person_rows"], + receipts=[receipt], + output_families=PRE_ASSEMBLY_OPERATOR_OUTPUT_FAMILIES, + ) + + +def multispine_relationship_boundary_columns() -> tuple[str, ...]: + """Declare the registered pre-clone evidence for relationship preparation.""" + return (_CPS_SOURCE_EVIDENCE_COLUMN, support_clone_index_column("person")) + + +def multispine_eligibility_boundary_columns() -> tuple[str, ...]: + """Declare the registered pre-clone evidence for eligibility preparation.""" + return (_CPS_SOURCE_EVIDENCE_COLUMN, support_clone_index_column("person")) + + +def validate_multispine_relationship_person_boundary( + person: pd.DataFrame, *, metadata: Mapping[str, object] +) -> dict[str, object]: + """Reject support populations before the source-only relationship derivation.""" + return _validate_multispine_person_preparation_boundary(person, metadata=metadata) + + +def validate_multispine_eligibility_person_boundary( + person: pd.DataFrame, *, metadata: Mapping[str, object] +) -> dict[str, object]: + """Reject support populations before the source-only eligibility derivation.""" + return _validate_multispine_person_preparation_boundary(person, metadata=metadata) + + +def _validate_multispine_person_preparation_boundary( + person: pd.DataFrame, *, metadata: Mapping[str, object] +) -> dict[str, object]: + """Validate actual person evidence at the registered source boundary.""" + _assert_source_person_boundary( + person, metadata=metadata, person_entity="person", phase=_PRE_CLONE_PHASE + ) + mask = _cps_person_evidence_mask( + person, person_entity="person", phase=_PRE_CLONE_PHASE + ) + return {"phase": _PRE_CLONE_PHASE, "cps_person_rows": int(mask.sum())} + + +def multispine_relationship_implementation_contract() -> dict[str, object]: + """Bind the live projections and options this source operator executes.""" + return _multispine_person_preparation_contract( + "with_us_relationship_inputs", "relationship_inputs" + ) + + +def multispine_eligibility_implementation_contract() -> dict[str, object]: + """Bind the live projections and options this source operator executes.""" + return _multispine_person_preparation_contract( + "with_us_eligibility_inputs", "eligibility_inputs" + ) + + +def _multispine_person_preparation_contract( + operator_name: str, family: str +) -> dict[str, object]: + """Project the live output/transient rosters and the executed options. + + These values move through imported registries without an edit here, so a + graph identity that omits them would not describe what actually ran. + """ + return { + "outputs": { + entity: sorted(columns) + for entity, columns in PRE_ASSEMBLY_OPERATOR_OUTPUT_FAMILIES[family].items() + }, + "transient_outputs": _transient_source_outputs( + (operator_name,), PRE_ASSEMBLY_OPERATOR_OUTPUT_FAMILIES + ), + "seed": POOL_RANDOM_SEED, + "time_period": POOL_TIME_PERIOD, + } + + +def observe_multispine_relationship_person( + person: pd.DataFrame, + *, + weights: Weights, + metadata: Mapping[str, object], + pool_rows: Mapping[str, int], +) -> tuple[pd.DataFrame, dict[str, object]]: + """Run the real pre-clone relationship derivation on its person projection. + + Canonical person weights are selected in the original person order and the + authenticated full counts describe the unchanged pool. No incomplete Frame + is constructed. Support role fields are removed only from the validated + ephemeral source view. The public pass-through predicate decides whether + the always-recomputing helper runs, so the wrapper's order — signal check + before weight validation and before any raw-column requirement — holds. + """ + return _observe_multispine_person_preparation( + person, + weights=weights, + metadata=metadata, + pool_rows=pool_rows, + operator_name="with_us_relationship_inputs", + family="relationship_inputs", + label="Relationship-input", + carries_signal=lambda table: us_relationship_inputs_person_carries_signal( + table + ), + prepare=lambda table, values: prepare_us_relationship_person( + table, + values, + seed=POOL_RANDOM_SEED, + time_period=POOL_TIME_PERIOD, + ), + ) + + +def observe_multispine_eligibility_person( + person: pd.DataFrame, + *, + weights: Weights, + metadata: Mapping[str, object], + pool_rows: Mapping[str, int], +) -> tuple[pd.DataFrame, dict[str, object]]: + """Run the real pre-clone eligibility derivation on its person projection. + + The same registered boundary, canonical weight selection and full-pool ID + merge as the relationship observer. The legacy partial-null pass-through + rule — all five outputs present and observed disability varying — is the + public predicate's, not this helper's, so the graph cannot strengthen or + weaken it. + """ + return _observe_multispine_person_preparation( + person, + weights=weights, + metadata=metadata, + pool_rows=pool_rows, + operator_name="with_us_eligibility_inputs", + family="eligibility_inputs", + label="Eligibility-input", + carries_signal=lambda table: us_eligibility_inputs_person_carries_signal(table), + prepare=lambda table, values: prepare_us_eligibility_person( + table, + values, + seed=POOL_RANDOM_SEED, + time_period=POOL_TIME_PERIOD, + ), + ) + + +def _observe_multispine_person_preparation( + person: pd.DataFrame, + *, + weights: Weights, + metadata: Mapping[str, object], + pool_rows: Mapping[str, int], + operator_name: str, + family: str, + label: str, + carries_signal: Callable[[pd.DataFrame], bool], + prepare: Callable[[pd.DataFrame, np.ndarray], pd.DataFrame], +) -> tuple[pd.DataFrame, dict[str, object]]: + """The shared person-preparation source projection, run and full-pool merge.""" + boundary = _validate_multispine_person_preparation_boundary( + person, metadata=metadata + ) + if ( + not isinstance(weights, Weights) + or set(pool_rows) != set(US_SCHEMA.entities) + or any(type(value) is not int or value < 0 for value in pool_rows.values()) + or pool_rows["person"] != len(person) + or len(weights) != len(person) + ): + raise ValueError(f"{label} population counts or weights are misaligned.") + for group in US_SCHEMA.group_entities: + if pool_rows[group] != int( + person[US_SCHEMA.membership_column(group)].nunique() + ): + raise ValueError(f"{label} population counts differ from memberships.") + mask = _cps_person_evidence_mask( + person, person_entity="person", phase=_PRE_CLONE_PHASE + ) + selected = without_support_role_metadata(person.loc[mask], entity="person") + outputs = PRE_ASSEMBLY_OPERATOR_OUTPUT_FAMILIES[family] + selected = selected.drop( + columns=_unavailable_output_columns(selected, outputs["person"]) + ) + selected_weights = weights.with_values(weights.values[mask], kind=weights.kind) + # The wrapper returns its frame untouched on a signal-carrying surface, + # reading neither weights nor raw source columns. Preserve that order. + produced = ( + selected + if carries_signal(selected) + else prepare(selected, selected_weights.values) + ) + _assert_source_table_identity( + selected, + produced, + entity_id="person_id", + operator_name=operator_name, + ) + merged, count = _merge_source_person_or_group_outputs( + person.copy(), + produced, + outputs["person"], + entity="person", + entity_id="person_id", + operator_name=operator_name, + ) + selected_rows = { + "person": len(selected), + **{ + group: int(selected[US_SCHEMA.membership_column(group)].nunique()) + for group in US_SCHEMA.group_entities + }, + } + receipt = _source_operator_receipt( + order_index=0, + operator_name=operator_name, + family=family, + phase=_PRE_CLONE_PHASE, + contract=POOL_OPERATOR_CONTRACTS[operator_name], + before_rows=dict(pool_rows), + available_rows=selected_rows, + output_rows=selected_rows, + merged_rows={"person": count}, + declared_outputs=outputs, + formula_owned_removed={}, + kernel_receipt={}, + overlap_ownership=None, + ) + return merged, _source_chain_receipt( + phase=_PRE_CLONE_PHASE, + operator_names=(operator_name,), + evidence_rows=boundary["cps_person_rows"], + receipts=[receipt], + output_families=PRE_ASSEMBLY_OPERATOR_OUTPUT_FAMILIES, + ) + + +def multispine_hours_implementation_contract() -> dict[str, object]: + """Bind live provider rosters actually consumed by the hours projection. + + These values can change through imported registries without an edit to + this module. Keep their actual projection in graph implementation identity. + """ + return { + "outputs": { + entity: sorted(columns) + for entity, columns in PRE_ASSEMBLY_OPERATOR_OUTPUT_FAMILIES[ + "hours_worked" + ].items() + }, + "transient_outputs": _transient_source_outputs( + ("with_us_hours_worked_inputs",), + PRE_ASSEMBLY_OPERATOR_OUTPUT_FAMILIES, + ), + } + + +def validate_multispine_hours_worked_boundary(frame: Frame) -> dict[str, object]: + """Validate the existing pre-clone boundary before graph hours derivation.""" + return validate_multispine_hours_person_boundary( + frame.person, metadata=frame.metadata + ) + + +def validate_multispine_hours_person_boundary( + person: pd.DataFrame, *, metadata: Mapping[str, object] +) -> dict[str, object]: + """Validate actual person evidence at the registered source boundary.""" + _assert_source_person_boundary( + person, metadata=metadata, person_entity="person", phase=_PRE_CLONE_PHASE + ) + available = _cps_person_evidence_mask( + person, person_entity="person", phase=_PRE_CLONE_PHASE + ) + return {"phase": _PRE_CLONE_PHASE, "cps_person_rows": int(available.sum())} + + +def observe_multispine_hours_worked_inputs(frame: Frame) -> PoolStageOutput: + """Run one registered source operator and retain its uncertified gate evidence. + + The graph's mandatory gate must accept this evidence before any downstream + context is released. Projection, structure checks and ID merge are the + exact same source-boundary implementation used by the direct path. + """ + return _run_source_operator_chain( + frame, + phase=_PRE_CLONE_PHASE, + operator_names=("with_us_hours_worked_inputs",), + operators={"with_us_hours_worked_inputs": _observe_us_hours_worked_inputs}, + ) + + +def observe_multispine_hours_person( + person: pd.DataFrame, + *, + weights: Weights, + metadata: Mapping[str, object], + pool_rows: Mapping[str, int], +) -> tuple[pd.DataFrame, dict[str, object]]: + """Run the registered hours projection using only actual person columns. + + ``weights`` must be the pool's canonically resolved person vector. Full + entity counts come from the authenticated population descriptor; selected + group counts use actual membership IDs, as Frame.select prunes each group + to the referenced IDs. No partial Frame carries whole-pool metadata. + """ + boundary = validate_multispine_hours_person_boundary(person, metadata=metadata) + if ( + set(pool_rows) != set(US_SCHEMA.entities) + or any(type(value) is not int or value < 0 for value in pool_rows.values()) + or pool_rows["person"] != len(person) + or len(weights) != len(person) + ): + raise ValueError("Measured hours population counts or weights are misaligned.") + mask = _cps_person_evidence_mask( + person, person_entity="person", phase=_PRE_CLONE_PHASE + ) + selected = without_support_role_metadata(person.loc[mask], entity="person") + outputs = PRE_ASSEMBLY_OPERATOR_OUTPUT_FAMILIES["hours_worked"] + selected = selected.drop( + columns=_unavailable_output_columns(selected, outputs["person"]) + ) + selected_weights = weights.with_values(weights.values[mask], kind=weights.kind) + selected_rows = {"person": len(selected)} + selected_rows.update( { - "hours_worked_signal_gate": { - "name": gate.name, - "passed": True, - "failures": [], - "details": dict(gate.details), - }, - "pool_excluded_outputs_removed": removed, + group: int(selected[US_SCHEMA.membership_column(group)].nunique()) + for group in US_SCHEMA.group_entities + } + ) + if any(selected_rows[entity] > pool_rows[entity] for entity in pool_rows): + raise ValueError("Measured hours membership counts exceed the population.") + produced = with_us_hours_worked_person( + selected, + weights=selected_weights, + seed=POOL_RANDOM_SEED, + time_period=POOL_TIME_PERIOD, + ) + gate = us_hours_worked_gate_from_summary( + us_hours_worked_person_summary(produced, weights=selected_weights) + ) + removed = sorted(US_HOURS_WORKED_POOL_EXCLUDED_COLUMNS & set(produced.columns)) + produced = produced.drop(columns=removed) + _assert_source_table_identity( + selected, + produced, + entity_id="person_id", + operator_name="with_us_hours_worked_inputs", + ) + merged, count = _merge_source_person_or_group_outputs( + person.copy(), + produced, + outputs["person"], + entity="person", + entity_id="person_id", + operator_name="with_us_hours_worked_inputs", + ) + receipt = _source_operator_receipt( + order_index=0, + operator_name="with_us_hours_worked_inputs", + family="hours_worked", + phase=_PRE_CLONE_PHASE, + contract=POOL_OPERATOR_CONTRACTS["with_us_hours_worked_inputs"], + before_rows=dict(pool_rows), + available_rows=selected_rows, + output_rows=selected_rows, + merged_rows={"person": count}, + declared_outputs=outputs, + formula_owned_removed={}, + kernel_receipt=_hours_gate_receipt( + gate, {"person": removed} if removed else {} + ), + overlap_ownership=None, + ) + return merged, _source_chain_receipt( + phase=_PRE_CLONE_PHASE, + operator_names=("with_us_hours_worked_inputs",), + evidence_rows=boundary["cps_person_rows"], + receipts=[receipt], + output_families=PRE_ASSEMBLY_OPERATOR_OUTPUT_FAMILIES, + ) + + +def multispine_housing_boundary_columns() -> tuple[str, ...]: + """Declare the registered pre-clone housing evidence to its graph boundary.""" + return (_CPS_SOURCE_EVIDENCE_COLUMN, support_clone_index_column("person")) + + +def validate_multispine_housing_person_boundary( + person: pd.DataFrame, *, metadata: Mapping[str, object] +) -> dict[str, object]: + """Retain source-role validation in the existing registered owner.""" + _assert_source_person_boundary( + person, metadata=metadata, person_entity="person", phase=_PRE_CLONE_PHASE + ) + mask = _cps_person_evidence_mask( + person, person_entity="person", phase=_PRE_CLONE_PHASE + ) + return {"phase": _PRE_CLONE_PHASE, "cps_person_rows": int(mask.sum())} + + +def multispine_housing_source_selection( + person: pd.DataFrame, *, metadata: Mapping[str, object] +) -> tuple[pd.Series, dict[str, object]]: + """Return the registered CPS person mask and its boundary receipt. + + The mask is the same one the direct source-operator chain applies, so a + graph FILTER built from it selects exactly the historical projection rows. + """ + boundary = validate_multispine_housing_person_boundary(person, metadata=metadata) + mask = _cps_person_evidence_mask( + person, person_entity="person", phase=_PRE_CLONE_PHASE + ) + return mask, boundary + + +def multispine_housing_output_family() -> Mapping[str, frozenset[str]]: + """Return the live registered housing output roster.""" + return PRE_ASSEMBLY_OPERATOR_OUTPUT_FAMILIES["housing_inputs"] + + +def multispine_housing_implementation_contract() -> dict[str, object]: + """Bind live provider rosters actually consumed by this source operator.""" + contract = POOL_OPERATOR_CONTRACTS["with_us_housing_inputs"] + return { + "outputs": { + entity: sorted(columns) + for entity, columns in multispine_housing_output_family().items() + }, + "transient_outputs": _transient_source_outputs( + ("with_us_housing_inputs",), PRE_ASSEMBLY_OPERATOR_OUTPUT_FAMILIES + ), + "formula_owned_outputs": { + entity: sorted( + set(columns) & set(_FORMULA_OWNED_SOURCE_OUTPUTS.get(entity, ())) + ) + for entity, columns in multispine_housing_output_family().items() }, + "execution_scope": contract.execution_scope, + "phases": list(contract.phases), + "family": contract.family, + "seed": POOL_RANDOM_SEED, + "time_period": POOL_TIME_PERIOD, + } + + +def assert_multispine_source_merge_labels_supported( + target: pd.DataFrame, + columns: Iterable[str], + *, + entity: str, + operator_name: str, + source: pd.DataFrame | None = None, +) -> None: + """Refuse the inherited label-based merge seam instead of working around it. + + :func:`_merge_source_person_or_group_outputs` writes a non-boolean output + through ``target.loc[target.index[positions]]``. With repeated pandas index + labels that assignment reaches every row sharing a selected label, so the + combination of duplicated labels and that write is refused at this explicit + scope boundary. Fixing the shared merge is a separate approved scope; + unique labels retain exact original parity. + + The merge picks its write path from the **produced** values, so pass + ``source`` to decide from the same series it does. Without ``source`` this + falls back to the incumbent rather than the actual write dtype: it can + over-refuse a boolean-producing column whose incumbent is not boolean, and + it would under-refuse a non-boolean-producing column whose incumbent is. + """ + if not target.index.has_duplicates: + return + + def writes_by_label(column: str) -> bool: + decided = ( + target[column] + if source is None or column not in source + else (source[column]) + ) + return not _is_physical_boolean_series(decided) + + affected = sorted( + column for column in columns if column in target and writes_by_label(column) + ) + if affected: + raise ValueError( + f"Multispine source operator {operator_name!r} cannot merge " + f"{entity!r} output(s) {affected} onto repeated index labels; the " + "shared label-based merge would write every row sharing a label. " + "Fixing that inherited seam is a separate reviewed scope." + ) + + +def merge_multispine_housing_source_outputs( + tables: Mapping[str, pd.DataFrame], + produced: Mapping[str, pd.DataFrame], + *, + pool_rows: Mapping[str, int], + selected_rows: Mapping[str, int], + evidence_rows: int, +) -> tuple[dict[str, pd.DataFrame], dict[str, int], dict[str, object]]: + """Merge the five declared housing leaves by entity ID and receipt the order. + + ``tables`` are isolated full-pool copies and ``produced`` the projected + source rows carrying every declared output. Column values, the ID merge and + the ordered receipt come from the existing registered helpers; this wrapper + adds no new formula and changes no selection semantics. + """ + outputs = multispine_housing_output_family() + if set(produced) != set(outputs) or set(outputs) - set(tables): + raise ValueError( + "US housing source merge requires its three declared output tables." + ) + merged: dict[str, pd.DataFrame] = { + entity: table for entity, table in tables.items() + } + merged_rows: dict[str, int] = {} + for entity in sorted(outputs): + columns = outputs[entity] + identity = US_SCHEMA.entity_id_column(entity) + assert_multispine_source_merge_labels_supported( + merged[entity], + columns, + entity=entity, + operator_name="with_us_housing_inputs", + source=produced[entity], + ) + merged[entity], merged_rows[entity] = _merge_source_person_or_group_outputs( + merged[entity], + produced[entity], + columns, + entity=entity, + entity_id=identity, + operator_name="with_us_housing_inputs", + ) + receipt = _source_operator_receipt( + order_index=0, + operator_name="with_us_housing_inputs", + family="housing_inputs", + phase=_PRE_CLONE_PHASE, + contract=POOL_OPERATOR_CONTRACTS["with_us_housing_inputs"], + before_rows=dict(pool_rows), + available_rows=dict(selected_rows), + output_rows=dict(selected_rows), + merged_rows=merged_rows, + declared_outputs=dict(outputs), + formula_owned_removed={}, + kernel_receipt={}, + overlap_ownership=None, + ) + return ( + merged, + merged_rows, + _source_chain_receipt( + phase=_PRE_CLONE_PHASE, + operator_names=("with_us_housing_inputs",), + evidence_rows=evidence_rows, + receipts=[receipt], + output_families=PRE_ASSEMBLY_OPERATOR_OUTPUT_FAMILIES, + ), ) @@ -2265,43 +3130,21 @@ def _run_source_operator_chain( f"{operator_name!r}: input={before_rows}, output={after_rows}." ) receipts.append( - { - "order_index": order_index, - "operator": operator_name, - "family": family, - "phase": phase, - "execution_scope": contract.execution_scope, - "pool_input_rows": before_rows, - "operator_input_rows": available_rows, - "cps_available_rows": ( - available_rows - if contract.execution_scope == _CPS_SOURCE_EXECUTION_SCOPE - else None - ), - "operator_output_rows": output_rows, - "merged_rows": merged_rows, - "operator_projection": { - "selection": ( - _CPS_SOURCE_EVIDENCE_COLUMN - if contract.execution_scope == _CPS_SOURCE_EXECUTION_SCOPE - else _WHOLE_POOL_EXECUTION_SCOPE - ), - "lineage_state_persisted": ( - contract.execution_scope == _WHOLE_POOL_EXECUTION_SCOPE - ), - "support_role_metadata_exposed": phase == _POST_CLONE_PHASE, - }, - "output_columns": { - entity: sorted(columns) - for entity, columns in declared_outputs.items() - if columns - }, - "formula_owned_outputs_removed": formula_owned_removed, - "kernel_receipt": dict(kernel_receipt), - "overlap_ownership": ( - dict(overlap_ownership) if overlap_ownership is not None else None - ), - } + _source_operator_receipt( + order_index=order_index, + operator_name=operator_name, + family=family, + phase=phase, + contract=contract, + before_rows=before_rows, + available_rows=available_rows, + output_rows=output_rows, + merged_rows=merged_rows, + declared_outputs=declared_outputs, + formula_owned_removed=formula_owned_removed, + kernel_receipt=kernel_receipt, + overlap_ownership=overlap_ownership, + ) ) uses_cps_source = any( POOL_OPERATOR_CONTRACTS[name].execution_scope == _CPS_SOURCE_EXECUTION_SCOPE @@ -2309,38 +3152,126 @@ def _run_source_operator_chain( ) return PoolStageOutput( current, - { - "phase": phase, - "operator_order": list(operator_names), - "cps_source_evidence": ( - { - "column": _CPS_SOURCE_EVIDENCE_COLUMN, - "person_rows": int( - _cps_source_evidence_mask(frame, phase=phase).sum() - ), - } + _source_chain_receipt( + phase=phase, + operator_names=operator_names, + evidence_rows=( + int(_cps_source_evidence_mask(frame, phase=phase).sum()) if uses_cps_source else None ), - "transient_outputs_carried_through_clone": ( - _transient_source_outputs(operator_names, output_families) - if phase == _PRE_CLONE_PHASE - else {} + receipts=receipts, + output_families=output_families, + ), + ) + + +def _source_chain_receipt( + *, + phase: str, + operator_names: tuple[str, ...], + evidence_rows: int | None, + receipts: list[dict[str, object]], + output_families: Mapping[str, Mapping[str, frozenset[str]]], +) -> dict[str, object]: + return { + "phase": phase, + "operator_order": list(operator_names), + "cps_source_evidence": ( + { + "column": _CPS_SOURCE_EVIDENCE_COLUMN, + "person_rows": evidence_rows, + } + if evidence_rows is not None + else None + ), + "transient_outputs_carried_through_clone": ( + _transient_source_outputs(operator_names, output_families) + if phase == _PRE_CLONE_PHASE + else {} + ), + "suboperators": receipts, + } + + +def _source_operator_receipt( + *, + order_index: int, + operator_name: str, + family: str, + phase: str, + contract: SourceOperatorContract, + before_rows: Mapping[str, int], + available_rows: Mapping[str, int], + output_rows: Mapping[str, int], + merged_rows: Mapping[str, int], + declared_outputs: Mapping[str, frozenset[str]], + formula_owned_removed: Mapping[str, list[str]], + kernel_receipt: Mapping[str, object], + overlap_ownership: Mapping[str, object] | None, +) -> dict[str, object]: + return { + "order_index": order_index, + "operator": operator_name, + "family": family, + "phase": phase, + "execution_scope": contract.execution_scope, + "pool_input_rows": before_rows, + "operator_input_rows": available_rows, + "cps_available_rows": ( + available_rows + if contract.execution_scope == _CPS_SOURCE_EXECUTION_SCOPE + else None + ), + "operator_output_rows": output_rows, + "merged_rows": merged_rows, + "operator_projection": { + "selection": ( + _CPS_SOURCE_EVIDENCE_COLUMN + if contract.execution_scope == _CPS_SOURCE_EXECUTION_SCOPE + else _WHOLE_POOL_EXECUTION_SCOPE + ), + "lineage_state_persisted": ( + contract.execution_scope == _WHOLE_POOL_EXECUTION_SCOPE ), - "suboperators": receipts, + "support_role_metadata_exposed": phase == _POST_CLONE_PHASE, }, - ) + "output_columns": { + entity: sorted(columns) + for entity, columns in declared_outputs.items() + if columns + }, + "formula_owned_outputs_removed": formula_owned_removed, + "kernel_receipt": dict(kernel_receipt), + "overlap_ownership": ( + dict(overlap_ownership) if overlap_ownership is not None else None + ), + } def _assert_source_operator_boundary(frame: Frame, *, phase: str) -> None: - manifest = frame.metadata.get(SPINE_ASSEMBLY_MANIFEST_KEY) + _assert_source_person_boundary( + frame.table(frame.schema.person_entity), + metadata=frame.metadata, + person_entity=frame.schema.person_entity, + phase=phase, + ) + + +def _assert_source_person_boundary( + person: pd.DataFrame, + *, + metadata: Mapping[str, object], + person_entity: str, + phase: str, +) -> None: + manifest = metadata.get(SPINE_ASSEMBLY_MANIFEST_KEY) if not isinstance(manifest, Mapping): raise ValueError( "Multispine source operators require the immutable spine assembly " "manifest before any source derivation." ) - person = frame.table(frame.schema.person_entity) - clone_column = support_clone_index_column(frame.schema.person_entity) + clone_column = support_clone_index_column(person_entity) if clone_column not in person: raise ValueError( "Multispine source operators require post-assembly clone provenance; " @@ -2370,7 +3301,19 @@ def _assert_source_operator_boundary(frame: Frame, *, phase: str) -> None: def _cps_source_evidence_mask(frame: Frame, *, phase: str) -> pd.Series: """Select CPS lineage only from a raw column unavailable on ACS.""" - person = frame.table(frame.schema.person_entity) + return _cps_person_evidence_mask( + frame.table(frame.schema.person_entity), + person_entity=frame.schema.person_entity, + phase=phase, + ) + + +def _cps_person_evidence_mask( + person: pd.DataFrame, + *, + person_entity: str, + phase: str, +) -> pd.Series: if _CPS_SOURCE_EVIDENCE_COLUMN not in person: raise ValueError( "Multispine source operators require raw CPS evidence column " @@ -2385,7 +3328,7 @@ def _cps_source_evidence_mask(frame: Frame, *, phase: str) -> pd.Series: "Multispine source operators found no CPS-evidenced person rows in " f"{_CPS_SOURCE_EVIDENCE_COLUMN!r}." ) - clone_column = support_clone_index_column(frame.schema.person_entity) + clone_column = support_clone_index_column(person_entity) clone_index = pd.to_numeric(person[clone_column], errors="coerce") evidenced_clones = set(clone_index.loc[available].astype(int).tolist()) invalid_evidence = ( @@ -2679,15 +3622,39 @@ def _assert_source_operator_structure( ) for entity in before.entities: entity_id = before.schema.entity_id_column(entity) - before_ids = before.table(entity)[entity_id] - after_ids = after.table(entity)[entity_id] - if after_ids.duplicated().any() or set(after_ids.tolist()) != set( - before_ids.tolist() - ): - raise ValueError( - f"Multispine source operator {operator_name!r} changed structural " - f"{entity_id!r} values." - ) + _assert_source_table_identity( + before.table(entity), + after.table(entity), + entity_id=entity_id, + operator_name=operator_name, + ) + + +def _assert_source_table_identity( + before: pd.DataFrame, + after: pd.DataFrame, + *, + entity_id: str, + operator_name: str, +) -> None: + before_ids = before[entity_id] + after_ids = after[entity_id] + if after_ids.duplicated().any() or set(after_ids.tolist()) != set( + before_ids.tolist() + ): + raise ValueError( + f"Multispine source operator {operator_name!r} changed structural " + f"{entity_id!r} values." + ) + + +def _unavailable_output_columns( + table: pd.DataFrame, + columns: frozenset[str], +) -> list[str]: + return [ + column for column in columns if column in table and table[column].isna().all() + ] def _without_unavailable_output_columns( @@ -2701,11 +3668,7 @@ def _without_unavailable_output_columns( for entity, columns in outputs.items(): if entity not in tables: continue - unavailable = [ - column - for column in columns - if column in tables[entity] and tables[entity][column].isna().all() - ] + unavailable = _unavailable_output_columns(tables[entity], columns) if unavailable: tables[entity] = tables[entity].drop(columns=unavailable) dropped = True @@ -2743,83 +3706,14 @@ def _merge_source_operator_outputs( target = tables[entity] source = operated.table(entity) entity_id = pool.schema.entity_id_column(entity) - if entity_id not in target or entity_id not in source: - raise ValueError( - f"Multispine source operator {operator_name!r} cannot align " - f"{entity!r} without {entity_id!r}." - ) - if source[entity_id].duplicated().any(): - raise ValueError( - f"Multispine source operator {operator_name!r} returned duplicate " - f"{entity_id!r} values." - ) - missing_outputs = sorted(set(columns) - set(source.columns)) - if missing_outputs: - raise ValueError( - f"Multispine source operator {operator_name!r} did not emit its " - f"declared {entity!r} output(s): {missing_outputs}." - ) - source_by_id = source.set_index(entity_id) - target_ids = target[entity_id] - eligible = target_ids.isin(source_by_id.index) - if int(eligible.sum()) != len(source): - raise ValueError( - f"Multispine source operator {operator_name!r} output IDs do not " - f"align one-to-one with the {entity!r} pool." - ) - for column in sorted(columns): - source_values = source_by_id[column] - aligned = source_values.reindex(target_ids) - source_is_boolean = _is_physical_boolean_series(source_values) - if source_is_boolean: - positions = np.flatnonzero(eligible.to_numpy()) - aligned_boolean = pd.Series( - pd.array(aligned, dtype="boolean"), - index=target.index, - name=column, - ) - if column not in target: - target[column] = aligned_boolean - continue - incumbent = target[column] - invalid_incumbent = incumbent.dropna().map( - lambda value: not isinstance(value, (bool, np.bool_)) - ) - if invalid_incumbent.any(): - offending_types = sorted( - { - f"{type(value).__module__}.{type(value).__qualname__}" - for value in incumbent.dropna().loc[invalid_incumbent] - } - ) - raise TypeError( - f"Multispine source operator {operator_name!r} emitted " - f"physical booleans for {entity}.{column}, but the pool " - "materialized observed non-boolean values with " - f"dtype {incumbent.dtype!s}: {offending_types}." - ) - merged_boolean = pd.Series( - pd.array(incumbent, dtype="boolean"), - index=target.index, - name=column, - ) - merged_boolean.iloc[positions] = aligned_boolean.iloc[positions].array - target[column] = merged_boolean - continue - if column in target and pd.api.types.is_bool_dtype(target[column].dtype): - raise TypeError( - f"Multispine source operator {operator_name!r} emitted " - f"non-boolean values for boolean-materialized " - f"{entity}.{column}; source dtype={source_values.dtype!s}." - ) - if column not in target: - target[column] = aligned.to_numpy() - else: - positions = np.flatnonzero(eligible.to_numpy()) - target.loc[target.index[positions], column] = aligned.iloc[ - positions - ].to_numpy() - merged_rows[entity] = int(eligible.sum()) + tables[entity], merged_rows[entity] = _merge_source_person_or_group_outputs( + target, + source, + columns, + entity=entity, + entity_id=entity_id, + operator_name=operator_name, + ) merged = Frame( tables, @@ -2832,6 +3726,95 @@ def _merge_source_operator_outputs( return merged, merged_rows +def _merge_source_person_or_group_outputs( + target: pd.DataFrame, + source: pd.DataFrame, + columns: frozenset[str], + *, + entity: str, + entity_id: str, + operator_name: str, +) -> tuple[pd.DataFrame, int]: + """The shared source-boundary ID merge; callers supply an isolated target.""" + if entity_id not in target or entity_id not in source: + raise ValueError( + f"Multispine source operator {operator_name!r} cannot align " + f"{entity!r} without {entity_id!r}." + ) + if source[entity_id].duplicated().any(): + raise ValueError( + f"Multispine source operator {operator_name!r} returned duplicate " + f"{entity_id!r} values." + ) + missing_outputs = sorted(set(columns) - set(source.columns)) + if missing_outputs: + raise ValueError( + f"Multispine source operator {operator_name!r} did not emit its " + f"declared {entity!r} output(s): {missing_outputs}." + ) + source_by_id = source.set_index(entity_id) + target_ids = target[entity_id] + eligible = target_ids.isin(source_by_id.index) + if int(eligible.sum()) != len(source): + raise ValueError( + f"Multispine source operator {operator_name!r} output IDs do not " + f"align one-to-one with the {entity!r} pool." + ) + for column in sorted(columns): + source_values = source_by_id[column] + aligned = source_values.reindex(target_ids) + source_is_boolean = _is_physical_boolean_series(source_values) + if source_is_boolean: + positions = np.flatnonzero(eligible.to_numpy()) + aligned_boolean = pd.Series( + pd.array(aligned, dtype="boolean"), + index=target.index, + name=column, + ) + if column not in target: + target[column] = aligned_boolean + continue + incumbent = target[column] + invalid_incumbent = incumbent.dropna().map( + lambda value: not isinstance(value, (bool, np.bool_)) + ) + if invalid_incumbent.any(): + offending_types = sorted( + { + f"{type(value).__module__}.{type(value).__qualname__}" + for value in incumbent.dropna().loc[invalid_incumbent] + } + ) + raise TypeError( + f"Multispine source operator {operator_name!r} emitted " + f"physical booleans for {entity}.{column}, but the pool " + "materialized observed non-boolean values with " + f"dtype {incumbent.dtype!s}: {offending_types}." + ) + merged_boolean = pd.Series( + pd.array(incumbent, dtype="boolean"), + index=target.index, + name=column, + ) + merged_boolean.iloc[positions] = aligned_boolean.iloc[positions].array + target[column] = merged_boolean + continue + if column in target and pd.api.types.is_bool_dtype(target[column].dtype): + raise TypeError( + f"Multispine source operator {operator_name!r} emitted " + f"non-boolean values for boolean-materialized " + f"{entity}.{column}; source dtype={source_values.dtype!s}." + ) + if column not in target: + target[column] = aligned.to_numpy() + else: + positions = np.flatnonzero(eligible.to_numpy()) + target.loc[target.index[positions], column] = aligned.iloc[ + positions + ].to_numpy() + return target, int(eligible.sum()) + + def _is_physical_boolean_series(values: pd.Series) -> bool: """Recognize boolean values without treating numeric 0/1 as booleans.""" diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/national_age_activation.py b/packages/microcosm-build/src/microcosm/build/us_runtime/national_age_activation.py new file mode 100644 index 000000000..bf1a03535 --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/national_age_activation.py @@ -0,0 +1,721 @@ +"""Explicit activation of the direct-national ACS 2024 published age counts. + +A captured reference bundle is evidence, not permission. The accepted +`national-derived-inventory.json` says so about itself: `target_activation` is +false, `calibration_target_registry` is `not_created`, and every table carries +`targets_allowed: false`. This module never reads that flag as authority and +never edits the bundle. Authority comes from +:data:`NATIONAL_AGE_ACTIVATION` — an immutable declaration, checked into this +repository and reviewed as code, that names the source digest, the exact +publisher cells, the target role, the population universe, the target period +and the age convention. :func:`activate_national_age_targets` refuses unless +the pinned bytes reproduce that declaration, and it then builds a **new** +:class:`~microcosm.calibrate.registry.TargetRegistry` artifact. + +What is authenticated here: the inventory bytes against the declared digest; +each selected raw Census response against the digest recorded for it *and* +against its request descriptor; the publisher's own labels, group, concept and +integer predicate for every selected cell; the single national row's +`GEO_ID`/`us`/`NAME` identity; and a from-bytes re-derivation of every selected +cell through the same :func:`classify_acs_value` precedence the district +inventory uses. Cells are parsed from exactly the bytes that were hashed. + +What is *not* established here: that the resulting calibration is nationally or +congressional-district valid, that any modelled person was observed in 2024, or +that this registry may be released. B19001 and B25003 are reserved same-source +holdouts and are refused outright — this module cannot activate them, and it +cannot mark them fresh or unseen. + +Published margins of error are converted to standard errors with the Census +90 percent factor and travel on the specs. The current Adam loss does not +consume them. A cell whose margin is a controlled-estimate sentinel carries a +**null** standard error, never zero: "no published variance" is not "no +variance". +""" + +from __future__ import annotations + +import hashlib +import json +import math +import re +from dataclasses import asdict, dataclass +from pathlib import Path + +from microcosm.calibrate.hierarchy import ( + CalibrationHierarchy, + HierarchyCategory, + HierarchyDimension, + HierarchyGeography, + HierarchyNode, +) +from microcosm.calibrate.provider_labels import calibration_provider_label +from microcosm.calibrate.registry import TargetRegistry, TargetSpec +from microcosm.calibrate.variable_labels import calibration_variable_label + +from ..cd_benchmark.protocol import RESERVED_FAMILIES +from .cd_reference import AUTHORITIES, classify_acs_value +from .cd_reference_sources import strict_json + +__all__ = [ + "ACTIVATION_SCHEMA_VERSION", + "ActivationError", + "AgeBand", + "NationalAgeActivation", + "NATIONAL_AGE_ACTIVATION", + "activate_national_age_targets", + "activation_digest", + "band_columns", +] + +#: Revision of the activation declaration's canonical form. Bump it when the +#: meaning of a field changes; the digest already moves when a value changes. +ACTIVATION_SCHEMA_VERSION = 1 + +#: Census publishes ACS margins of error at 90 percent confidence; this is the +#: documented divisor that turns one into a standard error. The authority is +#: the same sample-size-and-data-quality note the district inventory cites. +_MOE_90_TO_SE = 1.645 +_MOE_AUTHORITY = AUTHORITIES["acs_confidence"] + +#: The published label prefix of the disjoint AGE distribution in S0101. Cells +#: under `SELECTED AGE CATEGORIES` overlap it and are refused by construction. +_AGE_LABEL_PREFIX = "Estimate!!Total!!Total population!!AGE!!" + +_HEX64 = re.compile(r"[0-9a-f]{64}") + + +class ActivationError(ValueError): + """The pinned evidence does not reproduce the activation declaration.""" + + +def _require(condition: object, reason: str) -> None: + if not condition: + raise ActivationError(reason) + + +def _canonical(value: object) -> bytes: + return json.dumps( + value, sort_keys=True, separators=(",", ":"), allow_nan=False + ).encode() + + +@dataclass(frozen=True, slots=True) +class AgeBand: + """One activated publisher cell and the household column that answers it. + + Attributes: + variable: The publisher cell, e.g. ``"S0101_C01_002"``. + label: The publisher's exact label for that cell's estimate. Binding + it here means an edited cell list cannot silently pull a + differently-defined published quantity. + low: Inclusive lower bound in completed years. + high: Inclusive upper bound, or ``None`` for the open top band. + column: The household count column the population operator owns. + """ + + variable: str + label: str + low: int + high: int | None + column: str + + def __post_init__(self) -> None: + _require( + isinstance(self.variable, str) + and isinstance(self.label, str) + and isinstance(self.column, str) + and self.variable + and self.label + and self.column, + "AGE_BAND_TEXT", + ) + _require( + type(self.low) is int + and self.low >= 0 + and ( + self.high is None or (type(self.high) is int and self.high >= self.low) + ), + f"AGE_BAND_BOUNDS:{self.variable}", + ) + + +#: Diagnostics schema 8 hierarchy vocabulary for activated national age cells. +#: The provider and category labels come from the reviewed US label catalogs; +#: the geography is the activation's own national publisher row; the single +#: dimension is the publisher cell whose published label the band binds. +HIERARCHY_PROVIDER = "census_acs" +HIERARCHY_DIMENSION_ID = "publisher_cell" +HIERARCHY_DIMENSION_LABEL = "Publisher cell" +NATIONAL_GEOGRAPHY_ID = "0100000US" +NATIONAL_GEOGRAPHY_LABEL = "United States" +NATIONAL_GEOGRAPHY_LEVEL = "country" + + +def _hierarchy_category_id(table: str) -> str: + if table == "S0101": + return "population_by_age" + if table == "B01001": + return "population_by_sex_and_age" + raise ActivationError(f"UNSUPPORTED_TABLE:{table}") + + +def expected_demographic_hierarchy( + table: str, *, variable: str, label: str, geography: str +) -> CalibrationHierarchy: + """The only hierarchy a national demographic count cell may carry. + + Everything but the published cell label is fixed by the table and the + national geography, so a registry validator can rebuild it from a spec and + refuse any other provider, category, geography, dimension, or target. + """ + _require(geography == NATIONAL_GEOGRAPHY_ID, f"UNSUPPORTED_GEOGRAPHY:{geography}") + category_id = _hierarchy_category_id(table) + provider_label = calibration_provider_label("us", HIERARCHY_PROVIDER) + category_label = calibration_variable_label("us", HIERARCHY_PROVIDER, category_id) + _require(bool(provider_label) and bool(category_label), "HIERARCHY_LABELS") + return CalibrationHierarchy( + provider=HierarchyNode(HIERARCHY_PROVIDER, provider_label), + category=HierarchyCategory( + f"{HIERARCHY_PROVIDER}.{category_id}", category_label, HIERARCHY_PROVIDER + ), + geography=HierarchyGeography( + NATIONAL_GEOGRAPHY_ID, NATIONAL_GEOGRAPHY_LABEL, NATIONAL_GEOGRAPHY_LEVEL + ), + dimensions=( + HierarchyDimension( + HIERARCHY_DIMENSION_ID, HIERARCHY_DIMENSION_LABEL, variable, label + ), + ), + target=HierarchyNode(variable, label), + ) + + +def demographic_target_hierarchy( + table: str, band: AgeBand, *, geography: str +) -> CalibrationHierarchy: + """The complete schema 8 hierarchy for one activated band's target spec.""" + _require(type(band) is AgeBand, "AGE_BAND_TYPE") + return expected_demographic_hierarchy( + table, variable=band.variable, label=band.label, geography=geography + ) + + +def _bands() -> tuple[AgeBand, ...]: + quinary = [(low, low + 4) for low in range(0, 85, 5)] + bands = [] + for index, (low, high) in enumerate(quinary): + if low == 0: + label = "Under 5 years" + else: + label = f"{low} to {high} years" + bands.append( + AgeBand( + variable=f"S0101_C01_{index + 2:03d}", + label=_AGE_LABEL_PREFIX + label, + low=low, + high=high, + column=f"people_age_{low}_{high}", + ) + ) + bands.append( + AgeBand( + variable="S0101_C01_019", + label=_AGE_LABEL_PREFIX + "85 years and over", + low=85, + high=None, + column="people_age_85_plus", + ) + ) + return tuple(bands) + + +@dataclass(frozen=True, slots=True) +class NationalAgeActivation: + """The immutable authority for one calibration connection. + + Every field is normative: :func:`activation_digest` covers all of them, and + the digest travels into each produced :class:`TargetSpec`, so a changed + declaration produces a different registry version and a different node key. + + Attributes: + inventory_sha256: Digest of the accepted derived inventory's bytes. + table: The activated publisher table. + dataset: The Census API dataset the selected responses come from. + acs_year: The publisher's data year. + acs_release: The publisher's release. + geography: The direct national geography id. + geography_query: The publisher query that produced that single row. + published_name: The publisher's own name for that row. + confidence_level: The confidence level of the published margins. + role: What the activated cells are for. + universe: The population universe the cells count, stated in full. + entity: The entity whose weights the targets constrain. + period: The target period tag. + age_convention_id: The closed token naming the convention. The + population operator implements exactly this one and refuses any + other, so the declaration and the operator cannot drift apart. + age_convention: How model ages are read against the published bands. + allowed_reservation_status: The only inventory reservation status this + activation may act on. The same-source holdouts carry a different + one and can never satisfy it. + bands: The activated cells, in publisher order. + """ + + inventory_sha256: str + table: str + dataset: str + acs_year: int + acs_release: str + geography: str + geography_query: str + published_name: str + confidence_level: float + role: str + universe: str + entity: str + period: str + age_convention_id: str + age_convention: str + allowed_reservation_status: str + bands: tuple[AgeBand, ...] + + def __post_init__(self) -> None: + _require( + _HEX64.fullmatch(self.inventory_sha256) is not None, + "ACTIVATION_INVENTORY_DIGEST", + ) + _require(self.table not in RESERVED_FAMILIES, f"RESERVED_TABLE:{self.table}") + _require(self.table == "S0101", f"UNSUPPORTED_TABLE:{self.table}") + _require(self.role == "calibration", f"UNSUPPORTED_ROLE:{self.role}") + _require(self.universe.startswith("population"), "UNSUPPORTED_UNIVERSE") + _require(self.entity == "household", f"UNSUPPORTED_ENTITY:{self.entity}") + _require(self.period == "2024", f"UNSUPPORTED_PERIOD:{self.period}") + _require( + self.age_convention_id == "observed_interview_age_completed_years", + f"UNSUPPORTED_AGE_CONVENTION:{self.age_convention_id}", + ) + _require(self.acs_year == 2024, "UNSUPPORTED_ACS_YEAR") + _require(self.geography == "0100000US", "UNSUPPORTED_GEOGRAPHY") + _require( + self.allowed_reservation_status == "inventory_only", + "UNSUPPORTED_RESERVATION_STATUS", + ) + _require( + isinstance(self.bands, tuple) + and all(isinstance(band, AgeBand) for band in self.bands), + "ACTIVATION_BANDS_TYPE", + ) + _require(len(self.bands) == 18, f"ACTIVATION_BAND_COUNT:{len(self.bands)}") + _require( + len({band.variable for band in self.bands}) == len(self.bands) + and len({band.column for band in self.bands}) == len(self.bands), + "ACTIVATION_BANDS_NOT_DISTINCT", + ) + expected_low = 0 + for index, band in enumerate(self.bands): + _require(band.low == expected_low, f"ACTIVATION_BANDS_GAP:{band.variable}") + _require( + band.label.startswith(_AGE_LABEL_PREFIX), + f"ACTIVATION_BAND_NOT_AGE_DISTRIBUTION:{band.variable}", + ) + last = index == len(self.bands) - 1 + _require( + (band.high is None) is last, + f"ACTIVATION_BAND_OPEN_INTERVAL:{band.variable}", + ) + if band.high is not None: + expected_low = band.high + 1 + # Publisher cell order is the band order; nothing may be reordered or + # substituted without changing the digest. + _require( + [band.variable for band in self.bands] + == [f"S0101_C01_{index:03d}" for index in range(2, 20)], + "ACTIVATION_CELL_SELECTOR", + ) + + +#: The reviewed authority. Root's first engineering rung: observed interview +#: ages from pooled source cohorts, calibrated to the 2024 published national +#: distribution. No birth dates, no invented aging, and no claim that any +#: modelled person was interviewed in 2024. +NATIONAL_AGE_ACTIVATION = NationalAgeActivation( + inventory_sha256=( + "750b624e89557944c442600befb5557847dd75be6ce439c20daa9dfea5a0baf7" + ), + table="S0101", + dataset="acs/acs1/subject", + acs_year=2024, + acs_release="1-year", + geography="0100000US", + geography_query="us:*", + published_name="United States", + confidence_level=0.9, + role="calibration", + universe=( + "population: every resident person the publisher counts, household " + "and group-quarters alike; no household-population subsetting" + ), + entity="household", + period="2024", + age_convention_id="observed_interview_age_completed_years", + age_convention=( + "age in completed years as observed at each person's own source " + "interview, used verbatim; source cohorts are pooled and their " + "observed age distribution is calibrated to the 2024 published " + "counts; no birth dates, no aging, and no assertion that any person " + "was observed in 2024" + ), + allowed_reservation_status="inventory_only", + bands=_bands(), +) + + +def activation_digest(declaration: NationalAgeActivation) -> str: + """The content digest of a declaration's canonical form.""" + + # asdict recurses into the frozen AgeBand rows, so every bound, label and + # column name is inside the digest. + return hashlib.sha256( + _canonical({"schema_version": ACTIVATION_SCHEMA_VERSION, **asdict(declaration)}) + ).hexdigest() + + +def band_columns( + declaration: NationalAgeActivation = NATIONAL_AGE_ACTIVATION, +) -> tuple[str, ...]: + """The household count columns the activation expects, in band order.""" + + return tuple(band.column for band in declaration.bands) + + +def _read_pinned(path: Path, digest: str, *, what: str) -> tuple[bytes, object]: + """Hash the bytes, then parse *those* bytes. Never re-read from disk.""" + + _require(_HEX64.fullmatch(digest) is not None, f"PINNED_DIGEST_SHAPE:{what}") + _require(path.is_file(), f"PINNED_FILE_MISSING:{what}") + payload = path.read_bytes() + actual = hashlib.sha256(payload).hexdigest() + _require(actual == digest, f"PINNED_DIGEST_MISMATCH:{what}:{actual}") + return payload, strict_json(payload) + + +def _selected_source(inventory: dict, declaration: NationalAgeActivation, kind: str): + sources = inventory.get("sources") + _require(isinstance(sources, dict), "INVENTORY_SOURCES") + census = sources.get("census") + _require(isinstance(census, list), "INVENTORY_CENSUS_SOURCES") + entries = [ + entry + for entry in census + if isinstance(entry, dict) + and entry.get("table") == declaration.table + and entry.get("kind") == kind + ] + _require(len(entries) == 1, f"SOURCE_ENTRY_NOT_UNIQUE:{kind}:{len(entries)}") + entry = entries[0] + digest = entry.get("sha256") + _require( + isinstance(digest, str) and _HEX64.fullmatch(digest) is not None, + f"SOURCE_DIGEST:{kind}", + ) + _require(entry.get("dataset") == declaration.dataset, f"SOURCE_DATASET:{kind}") + _require(entry.get("year") == declaration.acs_year, f"SOURCE_YEAR:{kind}") + _require( + entry.get("geography") == declaration.geography_query, + f"SOURCE_GEOGRAPHY:{kind}", + ) + url = entry.get("url") + _require(isinstance(url, str) and url.startswith("https://"), f"SOURCE_URL:{kind}") + _require( + f"/{declaration.acs_year}/{declaration.dataset}" in url, + f"SOURCE_URL_SCOPE:{kind}", + ) + _require( + entry.get("path") == f"raw/{entry.get('sha256')}.json", f"SOURCE_PATH:{kind}" + ) + return entry + + +def _authenticated_response( + root: Path, entry: dict, *, kind: str +) -> tuple[bytes, object]: + """Authenticate the request descriptor and the response it names.""" + + descriptor_path = ( + root / "requests" / f"{hashlib.sha256(entry['url'].encode()).hexdigest()}.json" + ) + _require(descriptor_path.is_file(), f"REQUEST_DESCRIPTOR_MISSING:{kind}") + descriptor = strict_json(descriptor_path.read_bytes()) + _require(descriptor == entry, f"REQUEST_DESCRIPTOR_MISMATCH:{kind}") + payload, document = _read_pinned( + root / entry["path"], entry["sha256"], what=f"response:{kind}" + ) + _require(len(payload) == entry.get("size_bytes"), f"RESPONSE_SIZE:{kind}") + return payload, document + + +def _national_row(document: object, declaration: NationalAgeActivation) -> dict: + _require( + isinstance(document, list) and len(document) == 2, + "RESPONSE_NOT_A_SINGLE_NATIONAL_ROW", + ) + header, values = document + _require( + isinstance(header, list) and all(isinstance(name, str) for name in header), + "RESPONSE_HEADER_SHAPE", + ) + _require(len(set(header)) == len(header), "RESPONSE_DUPLICATE_HEADER") + _require(isinstance(values, list) and len(values) == len(header), "RESPONSE_WIDTH") + row = dict(zip(header, values, strict=True)) + _require(row.get("GEO_ID") == declaration.geography, "RESPONSE_GEO_ID") + _require(row.get("us") == "1", "RESPONSE_NOT_THE_NATION") + _require(row.get("NAME") == declaration.published_name, "RESPONSE_PUBLISHED_NAME") + return row + + +def _published_variable(metadata: object, name: str, declaration) -> dict: + _require(isinstance(metadata, dict), "METADATA_SHAPE") + variables = metadata.get("variables") + _require(isinstance(variables, dict), "METADATA_VARIABLES") + entry = variables.get(name) + _require(isinstance(entry, dict), f"METADATA_VARIABLE_MISSING:{name}") + _require(entry.get("group") == declaration.table, f"METADATA_GROUP:{name}") + _require(entry.get("concept") == "Age and Sex", f"METADATA_CONCEPT:{name}") + return entry + + +def _count_estimate(cell: dict, variable: str) -> int: + estimate = cell["estimate"] + _require( + estimate["class"] == "numeric" and not estimate["annotation_conflict"], + f"NON_COUNT_ESTIMATE:{variable}:{estimate['class']}", + ) + value = estimate["numeric_value"] + _require( + value is not None + and math.isfinite(value) + and value >= 0 + and float(value).is_integer(), + f"ESTIMATE_NOT_A_NONNEGATIVE_COUNT:{variable}", + ) + return int(value) + + +def _standard_error( + cell: dict, variable: str, *, confidence_level: float +) -> float | None: + """A published margin becomes a standard error; a sentinel becomes null.""" + + moe = cell["moe"] + _require(not moe["annotation_conflict"], f"MOE_ANNOTATION_CONFLICT:{variable}") + if moe["class"] != "numeric": + # Controlled, suppressed and open-interval margins carry no published + # variance. Null says "unknown"; zero would claim certainty. + return None + _require(confidence_level == 0.9, "UNSUPPORTED_CONFIDENCE_LEVEL") + value = moe["numeric_value"] + _require( + value is not None and math.isfinite(value) and value >= 0, + f"MOE_NOT_NONNEGATIVE:{variable}", + ) + if value == 0: + # A zero margin is a statement about a controlled cell, not a claim + # that the count is certain. Null again, never a zero standard error. + return None + return float(value) / _MOE_90_TO_SE + + +def activate_national_age_targets( + bundle_dir: str | Path, + *, + declaration: NationalAgeActivation = NATIONAL_AGE_ACTIVATION, +) -> TargetRegistry: + """Authenticate the pinned bundle and mint a new age-count registry. + + Args: + bundle_dir: The captured reference bundle root, holding + ``national-derived-inventory.json``, ``raw/`` and ``requests/``. + declaration: The activation authority. The default is the reviewed + one; a caller-supplied declaration is validated identically and + has no privileges of its own. + + Returns: + A new :class:`TargetRegistry` of 18 disjoint national age counts. + + Raises: + ActivationError: On any digest, identity, universe, label, schema, + reservation, selector or value violation. The bundle is never + written to, and a reserved same-source holdout can never be + activated. + """ + + root = Path(bundle_dir) + _, inventory = _read_pinned( + root / "national-derived-inventory.json", + declaration.inventory_sha256, + what="inventory", + ) + _require(isinstance(inventory, dict), "INVENTORY_SHAPE") + _require( + inventory.get("kind") == "us_national_acs_reference_inventory", + "INVENTORY_KIND", + ) + # The bundle states it is not an activation. Assert that rather than + # letting an edited descriptive field pass for authority. + _require(inventory.get("target_activation") is False, "INVENTORY_CLAIMS_ACTIVATION") + _require( + inventory.get("calibration_target_registry") == "not_created", + "INVENTORY_CLAIMS_A_REGISTRY", + ) + scope = inventory.get("scope") + _require(isinstance(scope, dict), "INVENTORY_SCOPE") + _require(scope.get("country") == "us", "INVENTORY_COUNTRY") + _require(scope.get("acs_year") == declaration.acs_year, "INVENTORY_ACS_YEAR") + _require(scope.get("acs_release") == declaration.acs_release, "INVENTORY_RELEASE") + _require( + scope.get("geography") == f"nation (for={declaration.geography_query})", + "INVENTORY_GEOGRAPHY", + ) + + tables = inventory.get("acs") + _require(isinstance(tables, dict), "INVENTORY_TABLES") + for reserved in sorted(RESERVED_FAMILIES): + held = tables.get(reserved) + _require(isinstance(held, dict), f"RESERVED_TABLE_ABSENT:{reserved}") + held_reservation = held.get("reservation") + _require( + isinstance(held_reservation, dict), + f"RESERVED_RESERVATION_SHAPE:{reserved}", + ) + _require( + held_reservation.get("status") == "reserved_same_source_holdout", + f"RESERVED_TABLE_RELABELLED:{reserved}", + ) + + entry = tables.get(declaration.table) + _require(isinstance(entry, dict), f"TABLE_ABSENT:{declaration.table}") + _require(entry.get("lineage") == "published", "TABLE_LINEAGE") + _require(entry.get("district_derived") is False, "TABLE_IS_DISTRICT_DERIVED") + _require(entry.get("geography_level") == "nation", "TABLE_GEOGRAPHY_LEVEL") + _require(entry.get("geography_id") == declaration.geography, "TABLE_GEOGRAPHY_ID") + _require( + entry.get("source_geography_query") == declaration.geography_query, + "TABLE_GEOGRAPHY_QUERY", + ) + _require(entry.get("published_name") == declaration.published_name, "TABLE_NAME") + _require(entry.get("year") == declaration.acs_year, "TABLE_YEAR") + _require( + entry.get("confidence_level") == declaration.confidence_level, + "TABLE_CONFIDENCE_LEVEL", + ) + _require( + entry.get("grouping") == "subject_age_and_sex_published_groups", + "TABLE_GROUPING", + ) + reservation = entry.get("reservation") + _require(isinstance(reservation, dict), "TABLE_RESERVATION") + _require( + reservation.get("status") == declaration.allowed_reservation_status, + f"TABLE_RESERVATION_STATUS:{reservation.get('status')}", + ) + # The bundle's own permission flags stay false and stay unread as + # authority; an inventory edited to claim permission is refused, and the + # digest above would already have rejected it. + _require(reservation.get("targets_allowed") is False, "INVENTORY_GRANTS_TARGETS") + _require(reservation.get("tuning_allowed") is False, "INVENTORY_GRANTS_TUNING") + + metadata_entry = _selected_source(inventory, declaration, "metadata") + data_entry = _selected_source(inventory, declaration, "data") + _, metadata = _authenticated_response(root, metadata_entry, kind="metadata") + _, data = _authenticated_response(root, data_entry, kind="data") + row = _national_row(data, declaration) + + inventory_cells = entry.get("cells") + _require(isinstance(inventory_cells, list), "TABLE_CELLS") + by_variable: dict[str, dict] = {} + for cell in inventory_cells: + _require(isinstance(cell, dict), "CELL_SHAPE") + variable = cell.get("variable") + _require(isinstance(variable, str), "CELL_VARIABLE") + _require(variable not in by_variable, f"DUPLICATE_CELL:{variable}") + by_variable[variable] = cell + + def derived_cell(variable: str) -> dict: + """Re-derive one cell from the authenticated bytes and cross-check it. + + The same annotation-precedence reader the district inventory uses runs + again over exactly the response that was hashed; the accepted inventory + must then agree with it cell for cell. + """ + + for suffix in ("E", "M", "EA", "MA"): + _require( + f"{variable}{suffix}" in row, + f"RESPONSE_CELL_MISSING:{variable}{suffix}", + ) + derived = { + "variable": variable, + "estimate": classify_acs_value(row[f"{variable}E"], row[f"{variable}EA"]), + "moe": classify_acs_value(row[f"{variable}M"], row[f"{variable}MA"]), + } + recorded = by_variable.get(variable) + _require(recorded is not None, f"CELL_ABSENT_FROM_INVENTORY:{variable}") + _require(derived == recorded, f"CELL_DISAGREES_WITH_INVENTORY:{variable}") + return derived + + specs: list[TargetSpec] = [] + digest = activation_digest(declaration) + for band in declaration.bands: + variable = band.variable + published = _published_variable(metadata, f"{variable}E", declaration) + _require(published.get("label") == band.label, f"PUBLISHED_LABEL:{variable}") + _require( + published.get("predicateType") == "int", f"PUBLISHED_NOT_A_COUNT:{variable}" + ) + derived = derived_cell(variable) + value = _count_estimate(derived, variable) + specs.append( + TargetSpec( + name=variable, + entity=declaration.entity, + measure=band.column, + value=float(value), + period=declaration.period, + se=_standard_error( + derived, variable, confidence_level=declaration.confidence_level + ), + source=f"{data_entry['url']} ({declaration.table} {variable}E)", + family=f"acs.{declaration.table}", + hierarchy=demographic_target_hierarchy( + declaration.table, band, geography=declaration.geography + ), + notes=( + f"activation={digest}; band={band.low}-" + f"{'+' if band.high is None else band.high}; " + f"label={band.label}; " + f"age_convention={declaration.age_convention}; " + f"uncertainty=published 90% MOE converted at " + f"{_MOE_90_TO_SE} ({_MOE_AUTHORITY}), not consumed " + f"by the current loss" + ), + metadata={ + "table": declaration.table, + "reference_sha256": declaration.inventory_sha256, + "geography": declaration.geography, + "universe": "population", + "role": declaration.role, + "evidence_scope": "source_documented", + }, + ) + ) + + # Coherence with the publisher's own total, checked and then discarded: + # the total is deliberately not activated as a nineteenth loss term. + total_variable = f"{declaration.table}_C01_001" + total = _count_estimate(derived_cell(total_variable), total_variable) + _require( + sum(int(spec.value) for spec in specs) == total, + "ACTIVATED_BANDS_DO_NOT_PARTITION_THE_PUBLISHED_TOTAL", + ) + return TargetRegistry(specs, country="us") diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/native_household_origin.py b/packages/microcosm-build/src/microcosm/build/us_runtime/native_household_origin.py new file mode 100644 index 000000000..72f27cfd7 --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/native_household_origin.py @@ -0,0 +1,666 @@ +"""Private native household origins, authenticated from original source members. + +ACS uses the existing archive codec. ASEC requires a separately reviewed original +HOUSEHOLD CSV member registry: prepared household HDF and PERSON-member pins are +not substitutes. The three registered members have an independently reviewed +archive/member audit and a complete accepted-population identifier bridge. + +This module owns source lineage, not population treatments or benchmark scoring. +Source capsules can be issued only by a reader. Graph transport is distinct from +source authentication; its consumer must retain the actual producer artifact edge. +""" + +from __future__ import annotations + +import csv +import hashlib +import json +import re +import tempfile +from collections.abc import Mapping, Sequence +from dataclasses import InitVar, asdict, dataclass, field +from pathlib import Path + +import numpy as np + +from microcosm.build.cd_benchmark.origin import ( + AcsHouseholdOrigin, + AsecHouseholdOrigin, + SourceMember, + origin_key, +) +from microcosm.frame import US_SCHEMA, Frame +from microcosm.graph.canonical import canonical_json + +from . import acs_housing_universe_source as acs +from .asec_student_controls import _snapshot +from .education_assistance_source import ASEC_EDUCATION_ASSISTANCE_ARCHIVES +from .support_provenance import ( + support_channel_column, + support_clone_index_column, + support_source_id_column, +) + +_TOKEN = object() +_BOUND_TOKEN = object() +SOURCE_SCHEMA = "microcosm.us.native-household-origin-source.v1" +BINDING_SCHEMA = "microcosm.us.population-household-origins.v1" +MAX_SOURCE_BYTES = 512 * 1024**2 +_SHA = re.compile(r"[0-9a-f]{64}") + + +class NativeOriginError(ValueError): + """Bounded refusal code, without a private identifier or source token.""" + + +def _require(condition: bool, reason: str) -> None: + if not condition: + raise NativeOriginError(reason) + + +def _sha(payload: bytes) -> str: + return hashlib.sha256(payload).hexdigest() + + +def _integer(value: object) -> int: + _require( + not isinstance(value, (bool, np.bool_)) + and isinstance(value, (int, np.integer)), + "NATIVE_INTEGER", + ) + return int(value) + + +def _carried_native_integer(value: object) -> int: + # Union assembly may store a source-only integer column as canonical text + # beside the other arm's missing values. Accept only its lossless canonical + # decimal representation; source membership is still established against + # the independently authenticated integer H_SEQ roster, never this cast. + if isinstance(value, str): + _require( + re.fullmatch(r"[1-9][0-9]{0,17}", value) is not None, + "CARRIED_NATIVE_INTEGER", + ) + return int(value) + return _integer(value) + + +@dataclass(frozen=True) +class AsecNativeMemberPin: + """A reviewed original household member, not authority merely by construction. + + A root source audit must establish membership in the already-pinned Census + archive before adding this declaration to the closed registry below. + """ + + income_year: int + survey_year: int + canonical_member_id: str + member_name: str + archive_sha256: str + member_sha256: str + size_bytes: int + rows: int + + def __post_init__(self) -> None: + _require( + type(self.income_year) is int + and self.income_year in ASEC_EDUCATION_ASSISTANCE_ARCHIVES + and self.survey_year == self.income_year + 1, + "ASEC_MEMBER_COHORT", + ) + _require( + isinstance(self.canonical_member_id, str) + and bool(self.canonical_member_id) + and self.member_name == f"hhpub{str(self.survey_year)[-2:]}.csv", + "ASEC_MEMBER_IDENTITY", + ) + _require( + isinstance(self.member_sha256, str) + and _SHA.fullmatch(self.member_sha256) is not None + and self.archive_sha256 + == ASEC_EDUCATION_ASSISTANCE_ARCHIVES[self.income_year].zip_sha256, + "ASEC_MEMBER_ARCHIVE_BINDING", + ) + _require( + type(self.size_bytes) is int + and 0 < self.size_bytes <= MAX_SOURCE_BYTES + and type(self.rows) is int + and 0 < self.rows <= 1_000_000, + "ASEC_MEMBER_BOUNDS", + ) + + +# Root acceptance: us-asec-native-household-root-acceptance.json, SHA-256 +# f997bbdc3573ed346d0e84d29f90c7b7ded9a12720c036d2789a7d561a04d602. +# The original archive/member bytes and all accepted attachment/selected native +# household identifiers were independently verified. Tests use invented pins. +_ASEC_MEMBER_PINS: tuple[AsecNativeMemberPin, ...] = ( + AsecNativeMemberPin( + income_year=2022, + survey_year=2023, + canonical_member_id="census/cps/asec/2023/household/hhpub23.csv", + member_name="hhpub23.csv", + archive_sha256="d2e000250782adfbdd7f29c82b66d866591a30f0d330496698ec19f9c784ce11", + member_sha256="c88192cb3c963a90ea98022606112a01ade0eb8e2c17df4149052970a5d5246f", + size_bytes=30_259_450, + rows=88_978, + ), + AsecNativeMemberPin( + income_year=2023, + survey_year=2024, + canonical_member_id="census/cps/asec/2024/household/hhpub24.csv", + member_name="hhpub24.csv", + archive_sha256="cdb39cdac34bef99dd0940ab28e306f692404c2eea44d85dfd634214872a0a09", + member_sha256="b12c1078e02766a4033ba1a031abdd6b9ef9df546aee5886dc8439a35abaa2ac", + size_bytes=30_448_089, + rows=89_473, + ), + AsecNativeMemberPin( + income_year=2024, + survey_year=2025, + canonical_member_id="census/cps/asec/2025/household/hhpub25.csv", + member_name="hhpub25.csv", + archive_sha256="318845a2b5e0034eb2973898de1738f4df0025727de38499e7669cb9c0deef0b", + member_sha256="b5b7351d5d4e5d79ff189f1d90096b16b2d4749671c14328ec6474cfe83ce116", + size_bytes=33_298_479, + rows=88_932, + ), +) + + +def asec_member_registry() -> list[dict[str, object]]: + """Return the exact live registry for the separately scoped graph identity.""" + return [asdict(pin) for pin in _ASEC_MEMBER_PINS] + + +@dataclass(frozen=True) +class AuthenticatedNativeOriginSource: + """Owned source projection. Its constructor is not a public admission path.""" + + payload: bytes + _token: InitVar[object] = None + _issued_sha256: str = field(init=False, repr=False) + + def __post_init__(self, _token: object) -> None: + _require(_token is _TOKEN, "SOURCE_CONSTRUCTOR") + object.__setattr__(self, "_issued_sha256", _sha(self.payload)) + + @property + def document(self) -> dict: + _require(_sha(self.payload) == self._issued_sha256, "SOURCE_MUTATION") + return _source_document(self.payload) + + +def _source_document(payload: bytes) -> dict: + _require(type(payload) is bytes and len(payload) <= MAX_SOURCE_BYTES, "SOURCE_SIZE") + try: + value = json.loads(payload) + _require(canonical_json(value) == payload, "SOURCE_CANONICAL") + _require( + set(value) + == { + "schema", + "arm", + "members", + "records", + "source_receipt_sha256", + "release_eligible", + } + and value["schema"] == SOURCE_SCHEMA + and value["arm"] in ("acs", "asec") + and value["release_eligible"] is False + and _SHA.fullmatch(value["source_receipt_sha256"]) is not None, + "SOURCE_SHAPE", + ) + members = [SourceMember(**member) for member in value["members"]] + _require(bool(members) and len(set(members)) == len(members), "SOURCE_MEMBERS") + seen = set() + for period, native, member_index, key in value["records"]: + _require( + type(member_index) is int and 0 <= member_index < len(members), + "SOURCE_MEMBER_INDEX", + ) + origin = ( + AcsHouseholdOrigin(members[member_index], period, native) + if value["arm"] == "acs" + else AsecHouseholdOrigin(members[member_index], period, native) + ) + _require(origin_key(origin) == key, "SOURCE_ORIGIN_KEY") + identity = (period, native) + _require(identity not in seen, "SOURCE_NATIVE_KEY_AMBIGUOUS") + seen.add(identity) + _require(bool(seen), "SOURCE_EMPTY") + return value + except NativeOriginError: + raise + except (ValueError, TypeError, KeyError, IndexError, OverflowError): + raise NativeOriginError("SOURCE_SHAPE") from None + + +def _issue(arm, members, records, receipt_sha): + payload = canonical_json( + { + "schema": SOURCE_SCHEMA, + "arm": arm, + "members": members, + "records": records, + "source_receipt_sha256": receipt_sha, + "release_eligible": False, + } + ) + _source_document(payload) + return AuthenticatedNativeOriginSource(payload, _token=_TOKEN) + + +def produce_acs_native_origins(source_dir, *, snapshot_root, output_dir): + """Reuse the real ACS source reader; never accept caller-provided source rows.""" + source = acs.produce_acs_housing_source( + source_dir, + snapshot_root=snapshot_root, + output_dir=output_dir, + ) + receipt = json.loads(source.receipt_json) + table = source.households + members = [ + SourceMember( + # Original member identity is stable across staging aliases and + # recompression, and its original member bytes hash is explicit. + f"census/acs/pums/2024/household/{member['name']}", + member["sha256"], + ) + for member in receipt["members"]["household"] + if member["is_data"] + ] + names = [ + member["name"] + for member in receipt["members"]["household"] + if member["is_data"] + ] + indices = {name: i for i, name in enumerate(names)} + records = [] + for native, name in zip(table.SERIALNO, table.source_member, strict=True): + index = indices[name] + key = origin_key(AcsHouseholdOrigin(members[index], 2024, native)) + records.append([2024, native, index, key]) + return _issue( + "acs", + [member.as_payload() for member in members], + records, + _sha(source.receipt_json), + ) + + +def produce_asec_native_origins(member_paths: Mapping[int, Path]): + """Hash and parse registered original HOUSEHOLD CSV members only. + + No member name or digest is inferred from the person-side restoration. + Unregistered genuine sources refuse before any path is opened. + """ + pins = tuple(_ASEC_MEMBER_PINS) + _require(bool(pins), "ASEC_MEMBER_REGISTRY_UNAVAILABLE") + _require( + len({pin.income_year for pin in pins}) == len(pins) + and set(member_paths) == {pin.income_year for pin in pins}, + "ASEC_MEMBER_ROSTER", + ) + records, members = [], [] + with tempfile.TemporaryDirectory(prefix="native-asec-origin-") as tmp: + for index, pin in enumerate(pins): + # Revalidate the entire declared registry, including archive/cohort. + AsecNativeMemberPin(**asdict(pin)) + capture = Path(tmp) / f"{index}.csv" + _require( + _snapshot(member_paths[pin.income_year], capture, size=pin.size_bytes) + == pin.member_sha256, + "ASEC_MEMBER_SHA256", + ) + member = SourceMember(pin.canonical_member_id, pin.member_sha256) + members.append(member.as_payload()) + with capture.open("r", encoding="utf-8", newline="") as handle: + rows = csv.reader(handle, strict=True) + header = next(rows, []) + _require( + bool(header) + and len(header) == len(set(header)) + and "H_SEQ" in header, + "ASEC_MEMBER_HEADER", + ) + position = header.index("H_SEQ") + count = 0 + for row in rows: + _require(len(row) == len(header), "ASEC_MEMBER_ROW_WIDTH") + token = row[position] + _require( + re.fullmatch(r"[0-9]{1,18}", token) is not None + and int(token) > 0, + "ASEC_MEMBER_NATIVE_KEY", + ) + native = int(token) + records.append( + [ + pin.income_year, + native, + index, + origin_key( + AsecHouseholdOrigin(member, pin.income_year, native) + ), + ] + ) + count += 1 + _require(count <= pin.rows, "ASEC_MEMBER_ROWS") + _require(count == pin.rows, "ASEC_MEMBER_ROWS") + _require( + _sha(capture.read_bytes()) == pin.member_sha256, "ASEC_CAPTURE_CHANGED" + ) + return _issue( + "asec", members, records, _sha(canonical_json(asec_member_registry())) + ) + + +def _population_content(frame: Frame) -> str: + # Defined over all cells/axes, effective weights and strata. Graph context + # deliberately has no Frame.metadata, graph ledger or original design anchors. + try: + normalized = Frame( + { + entity: frame.table(entity).loc[:, sorted(frame.table(entity))] + for entity in frame.entities + }, + frame.schema, + {entity: frame.resolve_weights(entity) for entity in frame.entities}, + frame.strata, + ) + return acs.frame_content_sha256(normalized) + except (ValueError, TypeError, KeyError, OverflowError): + raise NativeOriginError("POPULATION_CONTENT") from None + + +@dataclass(frozen=True) +class PopulationOriginBinding: + """Private population-grain evidence; its summary carries no native keys.""" + + payload: bytes + _token: InitVar[object] = None + _issued_sha256: str = field(init=False, repr=False) + + def __post_init__(self, _token): + _require(_token is _BOUND_TOKEN, "POPULATION_BINDING_CONSTRUCTOR") + object.__setattr__(self, "_issued_sha256", _sha(self.payload)) + + @property + def document(self): + _require(_sha(self.payload) == self._issued_sha256, "BINDING_MUTATION") + return json.loads(self.payload) + + @property + def summary(self): + return self.document["summary"] + + +def verify_population_origin_binding(frame: Frame, binding: PopulationOriginBinding): + """Reject evidence from any other full population content/ordered axes.""" + _require(type(binding) is PopulationOriginBinding, "POPULATION_BINDING_TYPE") + _require( + binding.document["population_content_sha256"] == _population_content(frame), + "POPULATION_CONTENT", + ) + + +def _entity_rows(frame, household_origins): + person = frame.person + household_ids = person.person_household_id.to_numpy() + rows = {} + for entity in frame.entities: + table = frame.table(entity) + ids = table[US_SCHEMA.entity_id_column(entity)].to_numpy() + if entity == "person": + membership = dict(zip(ids.tolist(), household_ids.tolist(), strict=True)) + elif entity == "household": + membership = dict(zip(ids.tolist(), ids.tolist(), strict=True)) + else: + pairs = person[ + [US_SCHEMA.membership_column(entity), "person_household_id"] + ].drop_duplicates() + _require( + not pairs.iloc[:, 0].duplicated().any(), "GROUP_CROSSES_HOUSEHOLDS" + ) + membership = dict(pairs.itertuples(index=False, name=None)) + _require(set(ids) == set(membership), "ENTITY_HOUSEHOLD_COVERAGE") + records = [] + effective_weights = frame.resolve_weights(entity).values + for i, identity in enumerate(ids): + hh = household_origins[_integer(membership[identity])] + channel = table[support_channel_column(entity)].iloc[i] + source_id = _integer(table[support_source_id_column(entity)].iloc[i]) + role = _integer(table[support_clone_index_column(entity)].iloc[i]) + _require( + channel == hh["channel"] and role == hh["clone_index"], + "ENTITY_HOUSEHOLD_LINEAGE", + ) + records.append( + { + "entity_id": _integer(identity), + "entity_source_id": source_id, + "clone_index": role, + "channel": channel, + "household_source_id": hh["entity_source_id"], + "origin_key": hh["origin_key"], + "weight_hex": float(effective_weights[i]).hex(), + } + ) + _require( + len({(r["entity_source_id"], r["clone_index"]) for r in records}) + == len(records), + "ENTITY_LINEAGE_DUPLICATE", + ) + rows[entity] = records + groups = { + entity: {row["entity_id"]: row for row in rows[entity]} + for entity in US_SCHEMA.group_entities + } + for position, row in enumerate(rows["person"]): + memberships = {} + for entity, lookup in groups.items(): + member_id = _integer( + person[US_SCHEMA.membership_column(entity)].iloc[position] + ) + member = lookup[member_id] + _require( + all( + member[k] == row[k] + for k in ( + "clone_index", + "channel", + "household_source_id", + "origin_key", + ) + ), + "PERSON_GROUP_LINEAGE", + ) + memberships[entity] = member["entity_source_id"] + row["membership_source_ids"] = memberships + return rows + + +def _bind_population_origins( + frame: Frame, + *, + sources: Sequence[AuthenticatedNativeOriginSource], + parent: PopulationOriginBinding | None = None, +): + """Resolve every household and entity row, including all zero-weight records. + + Native columns identify original records; structural source IDs bind the + selection/clone ancestry and cannot substitute for that native identity. + """ + _require(frame.schema == US_SCHEMA and len(sources) == 2, "SOURCE_ARMS") + documents = [] + for source in sources: + _require(type(source) is AuthenticatedNativeOriginSource, "SOURCE_TYPE") + documents.append(source.document) + _require({d["arm"] for d in documents} == {"acs", "asec"}, "SOURCE_ARMS") + lookups = { + d["arm"]: {(p, n): key for p, n, _member, key in d["records"]} + for d in documents + } + household = frame.table("household") + person = frame.person + cohort_rows = person[["person_household_id", "source_year"]].drop_duplicates() + _require( + not cohort_rows.person_household_id.duplicated().any(), + "HOUSEHOLD_COHORT_AMBIGUOUS", + ) + cohorts = dict(cohort_rows.itertuples(index=False, name=None)) + weights = frame.resolve_weights("household").values + rows = [] + for i, hh in household.iterrows(): + identity = _integer(hh.household_id) + channel = hh[support_channel_column("household")] + _require(channel in lookups and identity in cohorts, "NATIVE_KEY_UNRESOLVED") + raw_period = cohorts[identity] + _require( + isinstance(raw_period, (str, int, np.integer)) + and re.fullmatch(r"[0-9]{4}", str(raw_period)) is not None, + "HOUSEHOLD_COHORT", + ) + period = int(raw_period) + native = ( + hh.SERIALNO if channel == "acs" else _carried_native_integer(hh.asec_H_SEQ) + ) + key = lookups[channel].get((period, native)) + _require(key is not None, "NATIVE_KEY_UNRESOLVED") + role = _integer(hh[support_clone_index_column("household")]) + _require(role in (0, 1), "CLONE_ROLE") + position = household.index.get_loc(i) + rows.append( + { + "household_id": identity, + "entity_source_id": _integer(hh[support_source_id_column("household")]), + "clone_index": role, + "channel": channel, + "origin_key": key, + "weight_hex": float(weights[position]).hex(), + } + ) + _require(len({r["household_id"] for r in rows}) == len(rows), "HOUSEHOLD_IDS") + entity_rows = _entity_rows(frame, {r["household_id"]: r for r in rows}) + cloned = any(r["clone_index"] for r in rows) + _require(not cloned or parent is not None, "CLONE_PARENT_REQUIRED") + pair_conserved = None + if parent is not None: + _require(type(parent) is PopulationOriginBinding, "PARENT_TYPE") + prior = parent.document + _require( + prior["source_payload_sha256"] == sorted(_sha(s.payload) for s in sources), + "PARENT_SOURCE_MISMATCH", + ) + _require( + not any(r["clone_index"] for r in prior["households"]), + "PARENT_MUST_BE_NATIVE", + ) + for entity, current in entity_rows.items(): + old = {r["entity_source_id"]: r for r in prior["entities"][entity]} + for row in current: + before = old.get(row["entity_source_id"]) + _require( + before is not None + and all( + row[k] == before[k] + for k in ("channel", "household_source_id", "origin_key") + ), + "PARENT_MEMBERSHIP_LINEAGE", + ) + if entity == "person": + _require( + row["membership_source_ids"] == before["membership_source_ids"], + "PARENT_MEMBERSHIP_LINEAGE", + ) + before_weight = float.fromhex(before["weight_hex"]) + if cloned: + half = before_weight * 0.5 + _require( + row["weight_hex"] == half.hex() and half * 2.0 == before_weight, + "CLONE_ENTITY_PAIR_WEIGHT", + ) + else: + _require( + row["weight_hex"] == before["weight_hex"], + "PARENT_ENTITY_WEIGHT", + ) + if cloned: + _require( + {(r["entity_source_id"], r["clone_index"]) for r in current} + == {(source_id, role) for source_id in old for role in (0, 1)}, + "CLONE_PAIR_COVERAGE", + ) + else: + retained_households = {r["entity_source_id"] for r in rows} + _require( + {r["entity_source_id"] for r in current} + == { + r["entity_source_id"] + for r in prior["entities"][entity] + if r["household_source_id"] in retained_households + }, + "PARENT_HOUSEHOLD_SELECTION", + ) + if cloned: + old = { + r["entity_source_id"]: float.fromhex(r["weight_hex"]) + for r in prior["households"] + } + _require( + all( + row["weight_hex"] == (old[row["entity_source_id"]] * 0.5).hex() + and (old[row["entity_source_id"]] * 0.5) * 2.0 + == old[row["entity_source_id"]] + for row in rows + ), + "CLONE_PAIR_WEIGHT", + ) + pair_conserved = True + content = _population_content(frame) + summary = { + "households": len(rows), + "original_household_origins": len({r["origin_key"] for r in rows}), + "zero_weight_households_with_lineage": sum( + float.fromhex(r["weight_hex"]) == 0 for r in rows + ), + "entity_rows": {entity: len(values) for entity, values in entity_rows.items()}, + "population_content_sha256": content, + "parent_pair_mass_conserved": pair_conserved, + "design_anchors": "not_in_frame_context_not_evaluated", + "release_eligible": False, + } + payload = canonical_json( + { + "schema": BINDING_SCHEMA, + "population_content_sha256": content, + "ordered_entity_ids_sha256": { + e: _sha(canonical_json([r["entity_id"] for r in records])) + for e, records in entity_rows.items() + }, + "source_payload_sha256": sorted(_sha(s.payload) for s in sources), + "parent_binding_sha256": None if parent is None else _sha(parent.payload), + "households": rows, + "entities": entity_rows, + "summary": summary, + } + ) + return PopulationOriginBinding(payload, _token=_BOUND_TOKEN) + + +def bind_population_origins( + frame: Frame, + *, + sources: Sequence[AuthenticatedNativeOriginSource], + parent: PopulationOriginBinding | None = None, +): + """Bind every current row; malformed inputs refuse without leaking native IDs.""" + try: + return _bind_population_origins(frame, sources=sources, parent=parent) + except NativeOriginError: + raise + except (ValueError, TypeError, KeyError, IndexError, OverflowError): + raise NativeOriginError("POPULATION_BINDING_CONTRACT") from None diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/native_origin_graph_inventory.json b/packages/microcosm-build/src/microcosm/build/us_runtime/native_origin_graph_inventory.json new file mode 100644 index 000000000..7232f1347 --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/native_origin_graph_inventory.json @@ -0,0 +1,89 @@ +{ + "base_stages": [ + "composed_asec_binding_v1", + "acs_housing_universe_2024" + ], + "contracts": { + "microcosm.build/cd_benchmark/canonical.py": { + "imports": [], + "resource_accesses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", + "unbound_uses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945" + }, + "microcosm.build/cd_benchmark/origin.py": { + "imports": [ + "microcosm.build.cd_benchmark.canonical" + ], + "resource_accesses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", + "unbound_uses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945" + }, + "microcosm.build/us_runtime/graph_native_household_origin.py": { + "imports": [ + "microcosm.build.us_runtime", + "microcosm.build.us_runtime.graph_native_origin_implementation", + "microcosm.frame", + "microcosm.graph", + "microcosm.graph.artifact_edges", + "microcosm.graph.canonical", + "microcosm.graph.keys" + ], + "resource_accesses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", + "unbound_uses_sha256": "6420544565b4a2b11dc7238e06eef099a429982afa3dfbcec3dbdfbba7a27bd1" + }, + "microcosm.build/us_runtime/graph_native_origin_implementation.py": { + "imports": [ + "microcosm.build.us_runtime", + "microcosm.build.us_runtime.graph_implementation" + ], + "resource_accesses_sha256": "ecc7b6bd345950216623b7de94c9aeb392504af36e398094481f04ad203914f9", + "unbound_uses_sha256": "51778c9d32f80dfeba9741570ff5fa19ef83b464d7b207bf2f438cb008ae3ae5" + }, + "microcosm.build/us_runtime/native_household_origin.py": { + "imports": [ + "microcosm.build.cd_benchmark.origin", + "microcosm.build.us_runtime", + "microcosm.build.us_runtime.asec_student_controls", + "microcosm.build.us_runtime.education_assistance_source", + "microcosm.build.us_runtime.support_provenance", + "microcosm.frame", + "microcosm.graph.canonical", + "numpy" + ], + "resource_accesses_sha256": "740699f0e8648d061a7057844a4aabf04bb5ce2df7118fa4256dc2d215820c93", + "unbound_uses_sha256": "d4cc434fc5fa5a17c6fa46592dbeb8019d061927ff218c618036288221e76503" + } + }, + "dependencies": [ + "numpy", + "pandas", + "microunit", + "tables", + "h5py", + "PyYAML", + "pyarrow" + ], + "extra_modules": [ + "microcosm.build/cd_benchmark/canonical.py", + "microcosm.build/cd_benchmark/origin.py", + "microcosm.build/us_runtime/native_household_origin.py", + "microcosm.build/us_runtime/graph_native_household_origin.py", + "microcosm.build/us_runtime/graph_native_origin_implementation.py" + ], + "import_classifications": { + "microcosm.build.cd_benchmark.canonical": "Frozen v4 canonical origin encoding helpers; extra module bytes attested.", + "microcosm.build.cd_benchmark.origin": "Frozen v4 typed original-household keys; extra module bytes attested, protocol unchanged.", + "microcosm.build.us_runtime": "Relative native source/ACS reader imports; exact imported-symbol use fences plus declared module closures.", + "microcosm.build.us_runtime.asec_student_controls": "Existing bounded immutable member snapshot helper, attested by the composition closure.", + "microcosm.build.us_runtime.education_assistance_source": "Existing original ASEC archive and cohort pins, attested by the composition closure.", + "microcosm.build.us_runtime.graph_implementation": "Unchanged upstream identity service and AST/resource dependency classifier.", + "microcosm.build.us_runtime.graph_native_origin_implementation": "This isolated extension identity; its own module bytes and inventory are attested.", + "microcosm.build.us_runtime.support_provenance": "Existing descendant source-ID, channel and role column names; no population treatment.", + "microcosm.frame": "Frame schema, linkage, effective typed weights and population content; existing complete Frame closure.", + "microcosm.graph": "Graph declarations, typed artifact outputs and kernel contracts; existing complete graph closure.", + "microcosm.graph.artifact_edges": "Actual graph typed-descriptor validation at full materialization boundary; existing graph closure.", + "microcosm.graph.canonical": "Graph canonical JSON encoding; existing complete graph closure.", + "microcosm.graph.keys": "Actual opaque producer/output artifact-key derivation; existing complete graph closure.", + "numpy": "Installed NumPy distribution version plus platform numeric scope; strict native integer handling." + }, + "schema": "microcosm.us.native-origin-extension-inventory.v1", + "stage": "native_household_origin_v1" +} diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/operator_boundary.py b/packages/microcosm-build/src/microcosm/build/us_runtime/operator_boundary.py index e3e18961e..310b9953c 100644 --- a/packages/microcosm-build/src/microcosm/build/us_runtime/operator_boundary.py +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/operator_boundary.py @@ -14,7 +14,6 @@ import pandas as pd -from microcosm.build.us_runtime.acs_transfer import ACS_DERIVED_TRANSFER_INPUTS from microcosm.build.us_runtime.adult_care import US_ADULT_CARE_OUTPUT_COLUMNS from microcosm.build.us_runtime.child_support import ( US_CHILD_SUPPORT_OUTPUT_COLUMNS, @@ -59,20 +58,6 @@ US_PRIOR_YEAR_INCOME_FORMULA_OWNED_OUTPUT_COLUMNS, US_PRIOR_YEAR_INCOME_OUTPUT_COLUMNS, ) -from microcosm.build.us_runtime.puf_capital_gains_tail import ( - PUF_CAPITAL_GAINS_TAIL_APPLIED_COLUMN, - PUF_CAPITAL_GAINS_TAIL_DONOR_AGI_BAND_COLUMN, - PUF_CAPITAL_GAINS_TAIL_DONOR_FILING_STATUS_COLUMN, - PUF_CAPITAL_GAINS_TAIL_DONOR_SOURCE_ID_COLUMN, - PUF_CAPITAL_GAINS_TAIL_DONOR_SYNTHETIC_COLUMN, - PUF_CAPITAL_GAINS_TAIL_PERSON_COLUMNS, - PUF_CAPITAL_GAINS_TAIL_TAX_UNIT_COLUMNS, - PUF_CAPITAL_GAINS_TAIL_TRANSFER_WEIGHT_COLUMN, -) -from microcosm.build.us_runtime.puf_support import ( - PUF_TAX_DETAIL_DEFAULT_PERSON_OUTPUTS, - PUF_TAX_DETAIL_DEFAULT_TAX_UNIT_OUTPUTS, -) from microcosm.build.us_runtime.qbi_inputs import US_QBI_RECONCILED_PERSON_COLUMNS from microcosm.build.us_runtime.relationship_inputs import ( US_RELATIONSHIP_INPUTS_OUTPUT_COLUMNS, @@ -96,6 +81,20 @@ ) from microcosm.frame import Frame +from .operator_column_contracts import ( + ACS_DERIVED_TRANSFER_INPUTS, + PUF_CAPITAL_GAINS_TAIL_APPLIED_COLUMN, + PUF_CAPITAL_GAINS_TAIL_DONOR_AGI_BAND_COLUMN, + PUF_CAPITAL_GAINS_TAIL_DONOR_FILING_STATUS_COLUMN, + PUF_CAPITAL_GAINS_TAIL_DONOR_SOURCE_ID_COLUMN, + PUF_CAPITAL_GAINS_TAIL_DONOR_SYNTHETIC_COLUMN, + PUF_CAPITAL_GAINS_TAIL_PERSON_COLUMNS, + PUF_CAPITAL_GAINS_TAIL_TAX_UNIT_COLUMNS, + PUF_CAPITAL_GAINS_TAIL_TRANSFER_WEIGHT_COLUMN, + PUF_TAX_DETAIL_DEFAULT_PERSON_OUTPUTS, + PUF_TAX_DETAIL_DEFAULT_TAX_UNIT_OUTPUTS, +) + __all__ = [ "FORMULA_OWNED_SOURCE_COLUMNS", "PRE_ASSEMBLY_OPERATOR_OUTPUT_FAMILIES", diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/operator_column_contracts.py b/packages/microcosm-build/src/microcosm/build/us_runtime/operator_column_contracts.py new file mode 100644 index 000000000..d5117016b --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/operator_column_contracts.py @@ -0,0 +1,157 @@ +"""Source-visible operator column declarations, without operation imports. + +The operation modules re-export these same objects. Source boundary checks can +inspect ownership without loading transfer code or its reference tables. +""" + +from __future__ import annotations + +# Declared by qbi_inputs.py; operations retain their historical imports. + +_GENERAL_QUALIFICATION_FLAGS: tuple[str, ...] = ( + "estate_income_would_be_qualified", + "farm_operations_income_would_be_qualified", + "farm_rent_income_would_be_qualified", + "partnership_s_corp_income_would_be_qualified", + "rental_income_would_be_qualified", + "self_employment_income_would_be_qualified", +) + +_SSTB_QUALIFICATION_FLAG = "sstb_self_employment_income_would_be_qualified" + +US_QBI_BOOLEAN_OUTPUT_COLUMNS: tuple[str, ...] = ( + *_GENERAL_QUALIFICATION_FLAGS, + _SSTB_QUALIFICATION_FLAG, + # Keep the classifier last so the chained QRF can condition the SSTB draw + # on the qualification flags it must agree with. + "business_is_sstb", +) + +US_QBI_NONNEGATIVE_OUTPUT_COLUMNS: tuple[str, ...] = ( + "qualified_bdc_income", + "qualified_reit_and_ptp_income", + "sstb_unadjusted_basis_qualified_property", + "sstb_w2_wages_from_qualified_business", + "unadjusted_basis_qualified_property", + "w2_wages_from_qualified_business", +) + +US_QBI_OUTPUT_COLUMNS: tuple[str, ...] = ( + *US_QBI_BOOLEAN_OUTPUT_COLUMNS, + "qualified_bdc_income", + "qualified_reit_and_ptp_income", + "sstb_self_employment_income_before_lsr", + "sstb_unadjusted_basis_qualified_property", + "sstb_w2_wages_from_qualified_business", + "unadjusted_basis_qualified_property", + "w2_wages_from_qualified_business", +) + + +# Declared by acs_transfer.py; operations retain their historical imports. + +#: Person columns the default transfer DERIVES deterministically after the +#: QRF fits (never fitted themselves). Coverage checks require them on the +#: recipient exactly like declared plan targets. +ACS_DERIVED_TRANSFER_INPUTS: tuple[str, ...] = ( + "schedule_d_capital_gain_distributions", +) + + +# Declared by puf_support.py; operations retain their historical imports. + +# Shared structural limit; importing it must not load the PUF donor operations. +PUF_SUPPORT_MAX_CLONE_SAFE_SOURCE_ID = 10**15 - 1 + +US_PUF_SUPPORT_STAGE_NAME = "puf_support_channel" + +PUF_TAX_DETAIL_DEFAULT_PERSON_OUTPUTS = ( + "employment_income_before_lsr", + "self_employment_income_before_lsr", + "taxable_interest_income", + "qualified_dividend_income", + "non_qualified_dividend_income", + "tax_exempt_interest_income", + "short_term_capital_gains", + "long_term_capital_gains_before_response", + "long_term_capital_gains_on_collectibles", + "non_sch_d_capital_gains", + "taxable_private_pension_income", + "taxable_ira_distributions", + "social_security_retirement", + "social_security_disability", + "social_security_dependents", + "social_security_survivors", + "alimony_income", + "alimony_expense", + "salt_refund_income", + "charitable_cash_donations", + "charitable_non_cash_donations", + "real_estate_taxes", + "home_mortgage_interest", + "investment_interest_expense", + "investment_income_elected_form_4952", + "student_loan_interest", + "educator_expense", + "qualified_tuition_expenses", + "casualty_loss", + "unreimbursed_business_employee_expenses", + # The engine owns the realized contribution amounts through the + # IRA-limit scale and self-employment caps; the persistable leaves are + # the desired contributions, equal to the PUF's observed deductions at + # baseline (issue #278). + "traditional_ira_contributions_desired", + "self_employed_pension_contributions_desired", + "rental_income", + "estate_income", + "farm_income", + "farm_operations_income", + "farm_rent_income", + "miscellaneous_income", + "partnership_income", + "s_corp_income", + "partnership_self_employment_net_earnings", + *US_QBI_OUTPUT_COLUMNS, +) + +PUF_TAX_DETAIL_DEFAULT_TAX_UNIT_OUTPUTS: tuple[str, ...] = ( + "domestic_production_ald", + "unrecaptured_section_1250_gain", + "first_home_mortgage_balance", + "second_home_mortgage_balance", + "first_home_mortgage_interest", + "second_home_mortgage_interest", + "first_home_mortgage_origination_year", + "second_home_mortgage_origination_year", + "health_savings_account_ald", +) + + +# Declared by puf_capital_gains_tail.py; operations retain their historical imports. + +PUF_CAPITAL_GAINS_TAIL_PERSON_COLUMNS = ( + "short_term_capital_gains", + "long_term_capital_gains_before_response", + "long_term_capital_gains_on_collectibles", + "non_sch_d_capital_gains", +) + +PUF_CAPITAL_GAINS_TAIL_TAX_UNIT_COLUMNS = ("unrecaptured_section_1250_gain",) + +PUF_CAPITAL_GAINS_TAIL_APPLIED_COLUMN = "puf_capital_gains_tail_transfer_applied" + +PUF_CAPITAL_GAINS_TAIL_DONOR_SOURCE_ID_COLUMN = "puf_capital_gains_tail_donor_source_id" + +PUF_CAPITAL_GAINS_TAIL_DONOR_SYNTHETIC_COLUMN = ( + "puf_capital_gains_tail_donor_is_synthetic" +) + +PUF_CAPITAL_GAINS_TAIL_DONOR_FILING_STATUS_COLUMN = ( + "puf_capital_gains_tail_donor_filing_status_code" +) + +PUF_CAPITAL_GAINS_TAIL_DONOR_AGI_BAND_COLUMN = ( + "puf_capital_gains_tail_donor_agi_band_index" +) + +PUF_CAPITAL_GAINS_TAIL_TRANSFER_WEIGHT_COLUMN = "puf_capital_gains_tail_transfer_weight" diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/population_input_coverage.py b/packages/microcosm-build/src/microcosm/build/us_runtime/population_input_coverage.py new file mode 100644 index 000000000..f02dce43f --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/population_input_coverage.py @@ -0,0 +1,535 @@ +"""Describe required inputs on one actual graph Population and attached manifest. + +This is a diagnostic, never an issuer, source qualifier, applicability model, +release gate or claim of non-default statistical signal. Call inside an actual +host's checked lifetime after its source/producer checks. Population and manifest +constructors are public: even coherent supplied objects cannot prove original +source ancestry. There is deliberately no supplied expected hash/list shortcut. + +The roster is a versioned coverage contract, not a claim that every name is +scientifically independent or consumed by every engine route. The current +source manifest does not declare grain or applicability. Present +columns report their actual grains; missing names retain unresolved grain. +Every row's applicability remains unresolved, including declared ABSENT cells. +Column ownership identifies the last writer, not the measurement source of all +rows: writer-mask counts and carried counts stay separate. Source names and +typed artifact ancestry below describe declarations/receipts, not fresh reads. + +Only the current group-linked survey schema is supported. Experimental link +tables refuse, rather than being dropped by the maintained replay comparator. +The single possible borrowed I/O is the attached manifest population lookup. +All detached output is built afterward, with final pure input seals. +""" + +from __future__ import annotations + +import hashlib +from dataclasses import asdict, dataclass + +import numpy as np +import pandas as pd + +from microcosm.graph import CompiledGraph, Population, RunManifest, compile_graph +from microcosm.graph.availability import execution_state +from microcosm.graph.canonical import canonical_json +from microcosm.graph.decl import Ownership, StructuralDelta +from microcosm.graph.population import _storage_parts, owned_ids +from microcosm.graph.serialize import graph_to_json +from microcosm.graph.store import ( + _axis_name_payload, + _encode_frame_metadata, + _encode_object_scalar, +) + +from .input_coverage_profile import ( + ASSIGNED_BLOCK_COLUMN, + MANIFEST_SHA256, + USInputProfile, + required_us_inputs, +) +from .support_provenance import ( + spine_source_id_column, + support_channel_column, + support_clone_index_column, +) +from .survey_population_replay import same_replayed_frame + + +@dataclass(frozen=True) +class CoverageGroup: + entity: str + origin: str + clone: int + id_dtype: str + ordered_ids: tuple[int | str, ...] + source_id_dtype: str + ordered_source_ids: tuple[int | str, ...] + + +@dataclass(frozen=True) +class CoverageCounts: + group: int + known: int + unknown: int + invalid: int + writer_rows: int + carried_rows: int + applicability_unresolved: int + # Neither nulls nor Ownership.ABSENT are evidence of non-applicability. + not_applicable: int = 0 + + +@dataclass(frozen=True) +class InputCoverage: + name: str + entity: str | None + dtype: str | None + owner: str | None + declaration: str + writer_mask: str | None + counts: tuple[CoverageCounts, ...] + + +@dataclass(frozen=True) +class CoverageProducer: + node: str + key: str + kernel_ref: str + implementation: str + declared_sources: tuple[tuple[str, str], ...] + predecessors: tuple[tuple[str, str], ...] + typed_artifacts_json: str + + +@dataclass(frozen=True) +class PopulationInputCoverage: + profile: USInputProfile + version: str + graph_sha256: str + manifest_key: str + population_storage_sha256: str + groups: tuple[CoverageGroup, ...] + inputs: tuple[InputCoverage, ...] + producers: tuple[CoverageProducer, ...] + missing_inputs: tuple[str, ...] + ambiguous_grains: tuple[str, ...] + assigned_block: InputCoverage + block_storage_issues: tuple[str, ...] + protocol: str = "microcosm.us.input_coverage_diagnostic.v1" + profile_reference_sha256: str = MANIFEST_SHA256 + profile_authority: str = "tracked_input_coverage_contract_v1" + source_ancestry_verified: bool = False + applicability_complete: bool = False + statistical_signal_verified: bool = False + release_eligible: bool = False + + def to_bytes(self) -> bytes: + return canonical_json(asdict(self)) + + +def _require(condition, reason): + if not condition: + raise ValueError("US_INPUT_COVERAGE_" + reason) + + +def _series_stamp(series): + digest = hashlib.sha256() + digest.update(canonical_json((str(series.dtype), _axis_name_payload(series.name)))) + if pd.api.types.is_object_dtype(series.dtype): + # Typed scalar encoding, never process-specific PyObject addresses. + for value in series: + part = _encode_object_scalar(value) + digest.update(len(part).to_bytes(8, "little")) + digest.update(part) + else: + for part in _storage_parts(series, np.ones(len(series), dtype=np.bool_)): + digest.update(len(part).to_bytes(8, "little")) + digest.update(part) + return digest.hexdigest() + + +def _axis_stamp(axis): + _require(not isinstance(axis, pd.MultiIndex), "MULTIINDEX") + return (type(axis).__name__, _series_stamp(pd.Series(axis.array, name=axis.name))) + + +def _frame_stamp(frame): + _require(frame.links == (), "LINK_TABLES_UNSUPPORTED") + tables = [] + for entity in frame.entities: + table = frame.table(entity) + tables.append( + ( + entity, + _axis_stamp(table.index), + _axis_stamp(table.columns), + table.flags.allows_duplicate_labels, + tuple((column, _series_stamp(table[column])) for column in table), + ) + ) + return hashlib.sha256( + canonical_json( + { + "schema": asdict(frame.schema), + "tables": tables, + "strata": ( + _axis_stamp(frame.strata.index), + _series_stamp(frame.strata), + ), + "weights": tuple( + ( + entity, + frame.weights_for(entity).kind.value, + frame.weights_for(entity).values.dtype.str, + frame.weights_for(entity).values.tobytes().hex(), + ) + for entity in frame.weighted_entities + ), + "metadata": _encode_frame_metadata(frame.metadata), + "mass_log": tuple(asdict(record) for record in frame.mass_log), + } + ) + ).hexdigest() + + +def _population_stamp(population): + return hashlib.sha256( + canonical_json( + { + "frame": _frame_stamp(population.frame), + "version": population.version, + "owners": sorted(population.owners.items()), + "kinds": sorted( + (k, v.value) for k, v in population.weight_kind.items() + ), + "ledger": tuple(asdict(record) for record in population.mass_ledger), + "design": tuple( + (k, v.dtype.str, v.shape, v.tobytes().hex()) + for k, v in sorted(population.design_weights.items()) + ), + } + ) + ).hexdigest() + + +def _ids(series): + _require(not series.isna().any(), "UNKNOWN_ID") + if pd.api.types.is_integer_dtype(series.dtype) and not pd.api.types.is_bool_dtype( + series.dtype + ): + return tuple(int(value) for value in series) + _require(isinstance(series.dtype, pd.StringDtype), "ID_DTYPE") + return tuple(str(value) for value in series) + + +def _groups(frame): + groups, selections, roles = [], [], {} + for entity in frame.entities: + table = frame.table(entity) + names = ( + support_channel_column(entity), + support_clone_index_column(entity), + spine_source_id_column(entity), + ) + _require(all(name in table for name in names), "ORIGIN_COLUMNS:" + entity) + channel, clone, native = (table[name] for name in names) + _require( + not channel.isna().any() and set(channel) <= {"acs", "asec"}, + "ORIGIN:" + entity, + ) + _require( + pd.api.types.is_integer_dtype(clone.dtype) + and not pd.api.types.is_bool_dtype(clone.dtype) + and not clone.isna().any() + and set(clone) <= {0, 1}, + "CLONE:" + entity, + ) + ids = _ids(table[frame.schema.entity_id_column(entity)]) + native_ids = _ids(native) + _require(len(set(ids)) == len(ids), "DUPLICATE_ID:" + entity) + row_roles = tuple((str(a), int(b)) for a, b in zip(channel, clone, strict=True)) + _require( + len(set(zip(row_roles, native_ids, strict=True))) == len(ids), + "DUPLICATE_SOURCE_ROLE:" + entity, + ) + roles[entity] = dict(zip(ids, row_roles, strict=True)) + for origin, role in sorted(set(row_roles)): + selected = np.array( + [v == (origin, role) for v in row_roles], dtype=np.bool_ + ) + groups.append( + CoverageGroup( + entity, + origin, + role, + str(table[frame.schema.entity_id_column(entity)].dtype), + tuple(v for v, use in zip(ids, selected, strict=True) if use), + str(native.dtype), + tuple( + v for v, use in zip(native_ids, selected, strict=True) if use + ), + ) + ) + selections.append(selected) + person = frame.table(frame.schema.person_entity) + person_ids = _ids(person[frame.schema.person_id_column]) + for entity in frame.schema.group_entities: + members = _ids(person[frame.schema.membership_column(entity)]) + _require( + all( + member in roles[entity] + and roles[entity][member] == roles[frame.schema.person_entity][pid] + for pid, member in zip(person_ids, members, strict=True) + ), + "MEMBERSHIP_ORIGIN_CLONE:" + entity, + ) + return tuple(groups), tuple(selections) + + +def _writer(population, compiled, entity, name): + owner = population.owners[entity, name] + node = compiled.graph.node(owner) + outputs = [o for o in node.outputs if (o.entity, o.column) == (entity, name)] + if not outputs: + _require(node.structural is not StructuralDelta.NONE, "OWNER_DECLARATION") + return ( + owner, + "structural_carrier", + None, + np.zeros(population.frame.n(entity), dtype=np.bool_), + ) + _require(len(outputs) == 1, "OWNER_DECLARATION") + output = outputs[0] + table = population.frame.table(entity) + selected = ( + table[population.frame.schema.entity_id_column(entity)] + .isin(owned_ids(population, output)) + .to_numpy(dtype=np.bool_) + ) + if output.ownership is Ownership.ABSENT: + _require(table[name].loc[selected].isna().all(), "ABSENT_VALUE") + return owner, output.ownership.value, output.rows, selected + + +def _input(population, compiled, groups, selections, name, entity): + if entity is None: + return InputCoverage( + name, None, None, None, "missing_unresolved_grain", None, () + ) + frame, table = population.frame, population.frame.table(entity) + if name == "household_weight": + series = pd.Series(frame.weights_for(entity).values) + owner, declaration, mask = population.version, "typed_weight_carrier", None + writer = np.zeros(len(series), dtype=np.bool_) + else: + series = table[name] + owner, declaration, mask, writer = _writer(population, compiled, entity, name) + unknown = series.isna().to_numpy(dtype=np.bool_) + invalid = np.zeros(len(series), dtype=np.bool_) + if pd.api.types.is_numeric_dtype(series.dtype): + numeric = series.to_numpy(dtype=np.float64, na_value=np.nan) + invalid = ~unknown & ~np.isfinite(numeric) + known = ~unknown & ~invalid + counts = tuple( + CoverageCounts( + i, + int(known[selected].sum()), + int(unknown[selected].sum()), + int(invalid[selected].sum()), + int(writer[selected].sum()), + int((~writer[selected]).sum()), + int(selected.sum()), + ) + for i, (group, selected) in enumerate(zip(groups, selections, strict=True)) + if group.entity == entity + ) + return InputCoverage( + name, entity, str(series.dtype), owner, declaration, mask, counts + ) + + +def diagnose_us_input_coverage( + population: Population, + *, + compiled: CompiledGraph, + manifest: RunManifest, + profile: USInputProfile = USInputProfile.NATIONAL_CD, +) -> PopulationInputCoverage: + """Bind descriptive coverage to actual tables and current graph ownership. + + This verifies supplied graph consistency, not actual-run/source issuance. + The final host must retain/requalify the real upstream owners separately. + No applicability exemption is accepted from caller metadata or a hash. + """ + names = required_us_inputs(profile) + _require( + type(population) is Population + and type(compiled) is CompiledGraph + and type(manifest) is RunManifest, + "TYPES", + ) + graph_json = graph_to_json(compiled.graph) + fresh = compile_graph(compiled.graph) + _require(compiled == fresh, "COMPILED_GRAPH") + _require( + manifest.country == compiled.graph.country == "us" + and tuple(manifest.nodes) == compiled.order, + "MANIFEST_ROSTER", + ) + _require(not manifest.known_failures, "MANIFEST_FAILURE") + for node_id in compiled.order: + record = manifest.node(node_id) + _require( + record.kernel_ref == compiled.graph.node(node_id).kernel + and execution_state(record.receipt) is None, + "PRODUCER_RECEIPT", + ) + frame = population.frame + frame.revalidate() + _require( + set(population.owners) + == { + (entity, str(column)) + for entity in frame.entities + for column in frame.table(entity) + }, + "PHYSICAL_OWNERS", + ) + _require( + compiled.versions.get(population.version) == population.version, + "POPULATION_VERSION", + ) + # CompiledGraph.owners contains explicit claims, not carried columns. + # Every structural step reowns its complete carrier; only writers in the + # current version replace those owners. Derive the roster from declarations + # along the base chain, never by accepting arbitrary physical columns. + expected_owners = {} + version = population.version + while True: + for (declared_version, entity, column), owner in compiled.owners.items(): + if declared_version == version: + expected_owners.setdefault( + (entity, column), + owner if version == population.version else population.version, + ) + holder = compiled.graph.node(version) + if holder.structural is StructuralDelta.CREATE: + break + version = holder.base + structural = { + (entity, frame.schema.entity_id_column(entity)) for entity in frame.entities + } + structural.update( + (frame.schema.person_entity, frame.schema.membership_column(entity)) + for entity in frame.schema.group_entities + ) + expected_owners.update(dict.fromkeys(structural, population.version)) + _require( + expected_owners and dict(population.owners) == expected_owners, "CURRENT_OWNERS" + ) + _require( + population.weight_kind + == { + entity: frame.weights_for(entity).kind for entity in frame.weighted_entities + }, + "WEIGHT_KIND", + ) + before, manifest_json = _population_stamp(population), manifest.to_json() + # A lazy manifest can perform store I/O here. Never construct detached output + # before this final borrow. Replay normalization is restricted to this edge. + materialized = manifest.population(population.version) + ledger = manifest.mass_ledger(population.version) + same_replayed_frame(frame, materialized) + _require( + canonical_json(tuple(asdict(r) for r in ledger)) + == canonical_json(tuple(asdict(r) for r in population.mass_ledger)), + "MASS_LEDGER", + ) + _require( + _population_stamp(population) == before and manifest.to_json() == manifest_json, + "BORROW_MUTATION", + ) + materialized_stamp = _frame_stamp(materialized) + groups, selections = _groups(frame) + inputs, missing, ambiguous = [], [], [] + for name in names: + entities = ( + ["household"] + if name == "household_weight" and "household" in frame.weighted_entities + else [entity for entity in frame.entities if name in frame.table(entity)] + ) + if name == "household_weight" and entities != ["household"]: + # Never accept a redundant ordinary column instead of typed weights. + entities = [] + if not entities: + missing.append(name) + inputs.append(_input(population, compiled, groups, selections, name, None)) + else: + if len(entities) > 1: + ambiguous.append(name) + inputs.extend( + _input(population, compiled, groups, selections, name, entity) + for entity in entities + ) + household = frame.table("household") + block = _input( + population, + compiled, + groups, + selections, + ASSIGNED_BLOCK_COLUMN, + "household" if ASSIGNED_BLOCK_COLUMN in household else None, + ) + block_issues = [] + if block.entity is None: + block_issues.append("missing_assigned_block") + else: + values = household[ASSIGNED_BLOCK_COLUMN] + if values.isna().any(): + block_issues.append("unknown_assigned_block") + if any( + not isinstance(v, str) or len(v) != 15 or not v.isascii() or not v.isdigit() + for v in values[values.notna()] + ): + block_issues.append("malformed_assigned_block") + sources = {source.name: source.codec for source in compiled.graph.sources} + producers = tuple( + CoverageProducer( + node_id, + manifest.node(node_id).key, + manifest.node(node_id).kernel_ref, + manifest.node(node_id).kernel_impl_hash, + tuple( + (name, sources[name]) for name in compiled.graph.node(node_id).sources + ), + tuple( + (name, manifest.node(name).key) + for name in compiled.predecessors[node_id] + ), + canonical_json(manifest.node(node_id).typed_artifacts).decode(), + ) + for node_id in compiled.order + ) + # Pure checks after decoding and grouping. No source/store/owner borrow follows. + _require( + _population_stamp(population) == before + and _frame_stamp(materialized) == materialized_stamp + and manifest.to_json() == manifest_json + and graph_to_json(compiled.graph) == graph_json + and compiled == fresh, + "FINAL_MUTATION", + ) + return PopulationInputCoverage( + profile, + population.version, + hashlib.sha256(graph_json.encode()).hexdigest(), + manifest.key, + before, + groups, + tuple(inputs), + producers, + tuple(missing), + tuple(ambiguous), + block, + tuple(block_issues), + ) diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/prior_year_income.py b/packages/microcosm-build/src/microcosm/build/us_runtime/prior_year_income.py index ab3e56d6f..7983db5bf 100644 --- a/packages/microcosm-build/src/microcosm/build/us_runtime/prior_year_income.py +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/prior_year_income.py @@ -45,7 +45,7 @@ support_role_series, without_support_role_metadata, ) -from microcosm.frame import Frame +from microcosm.frame import Frame, Weights from microcosm.frame.units import US_SCHEMA __all__ = [ @@ -63,6 +63,7 @@ "US_PRIOR_YEAR_INCOME_STAGE_NAME", "derive_us_prior_year_income_from_manifest", "impute_us_prior_year_income_to_puf_support_from_manifest", + "prepare_us_prior_year_person", "us_prior_year_income_signal_gate", "us_prior_year_income_source_reconciliation_gate", "us_prior_year_income_stage_spec", @@ -213,11 +214,7 @@ def _previous_year_availability_match_survival_factor(frame: Frame) -> float: f"spine manifest version {version!r}." ) factor = manifest.get("sample_fraction") - if ( - type(factor) is not float - or not np.isfinite(factor) - or not 0.0 < factor <= 1.0 - ): + if type(factor) is not float or not np.isfinite(factor) or not 0.0 < factor <= 1.0: raise ValueError( "US prior-year-income availability requires a finite production " "stacked sample_fraction in (0, 1]." @@ -695,34 +692,57 @@ def _replace_person_table(frame: Frame, person: pd.DataFrame) -> Frame: ) -def with_us_prior_year_income_inputs( - frame: Frame, +def _prior_year_person_is_complete(person: pd.DataFrame) -> bool: + return ( + not has_support_role_metadata(person, entity="person") + and all(column in person for column in US_PRIOR_YEAR_INCOME_OUTPUT_COLUMNS) + and not all( + column in person for column in US_PRIOR_YEAR_INCOME_REQUIRED_SOURCE_COLUMNS + ) + ) + + +def prepare_us_prior_year_person( + person: pd.DataFrame, *, + weights: Weights, seed: int, time_period: int, -) -> Frame: - """Materialize adjacent-year earnings and PUF-support replacements.""" + predictors: pd.DataFrame | None = None, +) -> pd.DataFrame: + """Run the actual prior-year stage on a real person table. - if frame.schema != US_SCHEMA: - raise ValueError("US prior-year income requires the US schema.") - person = frame.table("person") - has_support_roles = has_support_role_metadata(person, entity="person") - has_raw_sources = all( - column in person for column in US_PRIOR_YEAR_INCOME_REQUIRED_SOURCE_COLUMNS - ) - if ( - not has_support_roles - and all(column in person for column in US_PRIOR_YEAR_INCOME_OUTPUT_COLUMNS) - and not has_raw_sources - ): - return frame + ``weights`` must be resolved for this person order by the caller. The + source-only, pre-clone path requires no other entity table. The legacy + post-clone path supplies the actual person/tax-unit predictor projection; + it is never reconstructed from placeholder entities here. Monetary values + retain the existing source-period treatment; this helper does not uprate. + """ + if not isinstance(weights, Weights): + raise TypeError("US prior-year person weights require canonical Weights.") + if len(weights) != len(person): + raise ValueError("US prior-year weights must match the person row order.") + if _prior_year_person_is_complete(person): + if predictors is not None: + raise ValueError("Source-only prior-year inputs require no PUF predictors.") + return person + has_support_roles = has_support_role_metadata(person, entity="person") stage_person = person.copy(deep=True) - stage_person[_PERSON_WEIGHT_COLUMN] = frame.resolve_weights("person").values + stage_person[_PERSON_WEIGHT_COLUMN] = weights.values if has_support_roles: - predictors = _person_prior_year_income_predictors(frame) + if ( + not isinstance(predictors, pd.DataFrame) + or not predictors.index.equals(person.index) + or tuple(predictors.columns) != _PUF_PREDICTORS + ): + raise ValueError( + "Prior-year support inputs require the actual aligned PUF predictors." + ) for column in _PUF_PREDICTORS: stage_person[_PUF_PREDICTOR_PREFIX + column] = predictors[column].to_numpy() + elif predictors is not None: + raise ValueError("Source-only prior-year inputs require no PUF predictors.") output = run_source_stage( us_prior_year_income_stage_spec(), tables={"person": stage_person}, @@ -742,6 +762,34 @@ def with_us_prior_year_income_inputs( ) if has_support_roles: output = output.drop(columns=[_FORMULA_OWNED_OUTPUT]) + return output + + +def with_us_prior_year_income_inputs( + frame: Frame, + *, + seed: int, + time_period: int, +) -> Frame: + """Materialize adjacent-year earnings and PUF-support replacements.""" + if frame.schema != US_SCHEMA: + raise ValueError("US prior-year income requires the US schema.") + person = frame.table("person") + if _prior_year_person_is_complete(person): + return frame + weights = frame.resolve_weights("person") + predictors = ( + _person_prior_year_income_predictors(frame) + if has_support_role_metadata(person, entity="person") + else None + ) + output = prepare_us_prior_year_person( + person, + weights=weights, + seed=seed, + time_period=time_period, + predictors=predictors, + ) return _replace_person_table(frame, output) @@ -892,12 +940,10 @@ def us_prior_year_income_signal_gate(frame: Frame) -> GateResult: if match_survival_factor != 1.0: authored_lower, authored_upper = summary[availability_band_key] applied_floor = float(authored_lower) * match_survival_factor - availability_band_key = ( - "previous_year_income_available_applied_share_band" + availability_band_key = "previous_year_income_available_applied_share_band" + summary["previous_year_income_available_sampled_match_survival_factor"] = ( + match_survival_factor ) - summary[ - "previous_year_income_available_sampled_match_survival_factor" - ] = match_survival_factor summary["previous_year_income_available_applied_floor"] = applied_floor summary[availability_band_key] = [applied_floor, authored_upper] failures: list[str] = [] diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/puf55_canonical_donor.py b/packages/microcosm-build/src/microcosm/build/us_runtime/puf55_canonical_donor.py new file mode 100644 index 000000000..4eae37e78 --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/puf55_canonical_donor.py @@ -0,0 +1,141 @@ +"""Project a bound canonical59 artifact into the survey-SS PUF55 donor. + +This in-memory converter grants no source authority. The caller authenticates +the artifact upstream and supplies its expected identity. No file is opened, +person is invented, or recipient Social Security component is produced here. +""" + +from __future__ import annotations + +import hashlib +import re + +import numpy as np +import pandas as pd + +from . import full_puf_enrichment as enrichment +from .puf59_canonical_artifact import decode_canonical_puf59 + +_SS_CARRIER_METHOD = ( + "Component-neutral carrier: total SS in retirement, other component carriers " + "zero; requires recipient-profile SS reconciliation. Zeros are not " + "observations of absent benefits." +) + + +def canonical_puf55_donor_from_artifact( + payload: bytes, + *, + expected_artifact_sha256: str, + expected_growth_scheme: str = "family_observed", + profile=enrichment.PUF55_SURVEY_SS, +): + """Return the ordered donor and descriptive projection metadata. + + The canonical59 v2 producer puts grown E02400 into its retirement carrier + and zeroes the other three carriers. Require that exact modeled convention + before relabeling the total as a predictor. It is transported tax-return + income, not an observed current-year total or a beneficiary component. + + The explicit eight-predictor profile omits this total from conditioning; + both profiles retain the same 55 targets, source rows, weights and capacity. + Both still validate the complete canonical artifact and its carrier recipe. + + The returned table has the source RECID axis. Its mutable values and this + metadata do not constitute an authenticated graph edge or release receipt. + """ + if type(profile) is not enrichment.PufOutputProfile or profile not in ( + enrichment.PUF55_SURVEY_SS, + enrichment.PUF55_SURVEY_SS_NO_TOTAL, + ): + raise ValueError("PUF55_DONOR_PROFILE") + if ( + type(payload) is not bytes + or type(expected_artifact_sha256) is not str + or re.fullmatch(r"[0-9a-f]{64}", expected_artifact_sha256) is None + or hashlib.sha256(payload).hexdigest() != expected_artifact_sha256 + ): + raise ValueError("PUF55_DONOR_ARTIFACT_IDENTITY") + arrays, source_receipt = decode_canonical_puf59( + payload, expected_growth_scheme=expected_growth_scheme + ) + assumptions = source_receipt.get("model_assumptions") + if ( + not isinstance(assumptions, dict) + or assumptions.get("schema") != "microcosm.us.puf59_baseline_models/1" + or assumptions.get("ss_method") != _SS_CARRIER_METHOD + or any( + np.any(arrays[name] != 0) + for name in enrichment.SURVEY_SS_COMPONENTS + if name != "social_security_retirement" + ) + ): + raise ValueError("PUF55_DONOR_SS_CARRIER") + + index = pd.Index(arrays["RECID"], name="tax_unit_id") + tax_unit = pd.DataFrame( + { + "tax_unit_id": arrays["RECID"], + "weight": arrays["weight"], + "filing_status_code": arrays["puf_2015_filing_status_code"], + "puf_person_incidence_capacity": arrays["puf_person_incidence_capacity"], + **{name: arrays[name] for name in profile.targets}, + **{name: arrays[name] for name in enrichment.PUF59.source_predictors}, + **( + { + enrichment.SURVEY_SS_TOTAL_PREDICTOR: arrays[ + "social_security_retirement" + ] + } + if profile is enrichment.PUF55_SURVEY_SS + else {} + ), + }, + index=index, + ) + # Known here means present canonical cells, including modeled values. It + # does not reclassify the inherited source/model provenance as observed. + known = pd.DataFrame( + True, + index=index, + columns=( + "weight", + "filing_status_code", + "puf_person_incidence_capacity", + *profile.targets, + *profile.source_predictors, + ), + ) + donor = enrichment.canonical_full_puf_donor( + None, + tax_unit, + person_known=None, + tax_unit_known=known, + person_targets_at_tax_unit=profile.person_outputs, + profile=profile, + ) + return donor, { + "schema": "microcosm.us.puf55_canonical_donor_projection/1", + "artifact_sha256": expected_artifact_sha256, + "source_receipt_sha256": source_receipt["sha256"], + "source_statistical_year": 2015, + "money_year": 2024, + "rows": len(donor), + "profile": profile.value, + "ordered_columns": list(donor.columns), + "row_axis": "source RECID, tax-return grain", + "social_security_total": { + "predictor": ( + enrichment.SURVEY_SS_TOTAL_PREDICTOR + if profile is enrichment.PUF55_SURVEY_SS + else None + ), + "raw_field": "E02400", + "origin": "modeled_transport", + "growth_scheme": expected_growth_scheme, + "growth_recipe_sha256": source_receipt["growth"]["recipe_sha256"], + "component_interpretation": "No observed component allocation", + }, + "source_authority_granted": False, + "release_eligible": False, + } diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/puf55_route_finalization.py b/packages/microcosm-build/src/microcosm/build/us_runtime/puf55_route_finalization.py new file mode 100644 index 000000000..8cfd66785 --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/puf55_route_finalization.py @@ -0,0 +1,518 @@ +"""Validate and merge the two PUF55 raw chains before one finalization. + +This is a values-only boundary. The host must freshly qualify the recipient +matrices against its actual financial run, authenticate every typed graph edge, +and verify donor/model ownership before using the result. Caller bytes and this +receipt issue no source, Population, donor, fit or release authority. The public +raw merger loads no model. The private finalization bridge below requires an +upstream-trusted final model for each nonempty route and returns a candidate +Frame. It fits no model and issues no Population attachment. +""" + +from __future__ import annotations + +from dataclasses import dataclass + +import numpy as np +import pandas as pd + +from microcosm.fit import model_input + +from . import full_puf_enrichment as full +from . import graph_full_puf_enrichment as physical +from . import support_provenance as provenance + +PROTOCOL = "microcosm.us.puf55-route-raw-merge.v1" +FINALIZATION_PROTOCOL = "microcosm.us.puf55-route-finalization.v2" +PROFILES = (full.PUF55_SURVEY_SS, full.PUF55_SURVEY_SS_NO_TOTAL) + + +def _require(condition, reason): + if not condition: + raise ValueError("PUF55_ROUTE_" + reason) + + +@dataclass(frozen=True) +class Puf55RouteDraws: + """Descriptive typed-chain bytes, never an authenticated producer capsule.""" + + profile: full.PufOutputProfile + matrix: bytes + matrix_producer_key: str + raw_draws: tuple[tuple[str, bytes], ...] + apply_state: bytes + training_state: bytes + + +def _route_snapshot(route): + _require(type(route) is Puf55RouteDraws, "INPUT_TYPE") + # Frozen dataclasses can still be changed with object.__setattr__. Read each + # field once and use only these immutable retained values during decoding. + result = ( + route.profile, + route.matrix, + route.matrix_producer_key, + route.raw_draws, + route.apply_state, + route.training_state, + ) + profile, matrix, key, raw, apply_state, training_state = result + _require(type(profile) is full.PufOutputProfile and profile in PROFILES, "PROFILE") + _require( + all(type(value) is bytes for value in (matrix, apply_state, training_state)) + and type(key) is str + and full.codec._hash(key), + "ARTIFACT_TYPES", + ) + _require( + type(raw) is tuple + and all( + type(pair) is tuple + and len(pair) == 2 + and type(pair[0]) is str + and type(pair[1]) is bytes + for pair in raw + ) + and tuple(pair[0] for pair in raw) == profile.targets, + "RAW_ROSTER", + ) + return result + + +def _receiving_axis(frame): + table = frame.table("tax_unit") + column = provenance.support_clone_index_column("tax_unit") + _require( + table.columns.is_unique + and table.index.is_unique + and column in table + and table[column].dtype == np.dtype("int64") + and bool(table[column].isin((0, 1)).all()), + "CLONE_AXIS", + ) + all_ids = full._ids(table.tax_unit_id, "route_receiving") + _require(len(set(all_ids.tolist())) == len(all_ids), "RECEIVING_IDS") + mask = table[column].eq(1).to_numpy() + ids = all_ids[mask] + _require(len(ids) > 0, "NO_RECIPIENTS") + return ids, table.index[mask].copy() + + +def merge_puf55_route_draws(frame, *, recipient_matrices, route_draws, seed): + """Return all 55 raw targets in complete receiving pandas row order. + + ``recipient_matrices`` is the exact immutable matrix roster freshly derived + by the host's authenticated recipient qualifier. Empty routes are omitted + in nine/eight order. This function checks values, not that upstream duty. + Finalize the complete returned table once, after checking both donor/model + bindings; never finalize subsets or attach a route over another route. + """ + _require(type(seed) is int and seed >= 0, "SEED") + _require( + type(recipient_matrices) is tuple + and all( + type(pair) is tuple + and len(pair) == 2 + and type(pair[0]) is str + and type(pair[1]) is bytes + for pair in recipient_matrices + ), + "MATRIX_ROSTER", + ) + _require(type(route_draws) is tuple and 1 <= len(route_draws) <= 2, "ROUTE_ROSTER") + retained = tuple(_route_snapshot(route) for route in route_draws) + names = tuple(values[0].value for values in retained) + _require( + names == tuple(p.value for p in PROFILES if p.value in names) + and tuple(name for name, _ in recipient_matrices) == names, + "ROUTE_ORDER", + ) + ids, row_index = _receiving_axis(frame) + tables, seen, evidence, raw_seals = [], set(), [], [] + for values, (_, expected_matrix) in zip(retained, recipient_matrices, strict=True): + profile, matrix, key, raw, apply_state, training_state = values + _require(matrix == expected_matrix, "RECIPIENT_MATRIX_CHANGED") + prepared = model_input.decode_recipient_matrix(matrix) + route_ids = tuple(prepared.entity_ids.tolist()) + _require( + prepared.entity == "tax_unit" + and tuple(prepared.features.columns) == profile.predictors + and prepared.features.index.name == "tax_unit_id", + "MATRIX_PROFILE", + ) + _require(not seen.intersection(route_ids), "OVERLAPPING_IDS") + seen.update(route_ids) + decoded = full.decode_full_puf_draws( + matrix=matrix, + matrix_producer_key=key, + raw_draws=dict(raw), + apply_state=apply_state, + training_state=training_state, + seed=seed, + profile=profile, + ) + # Check the detached decoder result against the immutable input bytes; + # comparing two mutable returned tables would not anchor its values. + _require( + type(decoded) is pd.DataFrame + and tuple(decoded.columns) == profile.targets + and decoded.index.equals(pd.Index(route_ids, name="tax_unit_id")) + and all(dtype == np.dtype("float64") for dtype in decoded.dtypes) + and all( + full.codec.encode_raw_target( + decoded[target].to_numpy(), target=target, index=decoded.index + ) + == payload + for target, payload in raw + ), + "DECODED_RAW_CHANGED", + ) + tables.append(decoded.copy(deep=True)) + raw_seals.append((route_ids, raw)) + evidence.append( + { + "profile": profile.value, + "rows": len(route_ids), + "matrix_sha256": full.codec.sha(matrix), + "matrix_producer_key": key, + "apply_state_sha256": full.codec.sha(apply_state), + "training_state_sha256": full.codec.sha(training_state), + "raw_target_sha256": {target: full.codec.sha(p) for target, p in raw}, + } + ) + _require(seen == set(ids.tolist()), "INCOMPLETE_RECIPIENT_UNION") + combined = ( + pd.concat(tables, axis=0).loc[ids.tolist(), list(PROFILES[0].targets)].copy() + ) + # The IDs above establish the one explicit bridge to the existing pandas + # row index required by the maintained whole-cohort finalizer. + combined.index = row_index.copy() + fresh_ids, fresh_index = _receiving_axis(frame) + _require( + np.array_equal(ids, fresh_ids) and row_index.identical(fresh_index), + "RECEIVING_AXIS_CHANGED", + ) + receipt = full.codec.encode_json( + { + "protocol": PROTOCOL, + "routes": evidence, + "recipient_rows": len(ids), + "target_order": list(PROFILES[0].targets), + "ordered_recipient_ids_sha256": full.codec.sha(ids.tobytes()), + "merge": "disjoint complete clone-one union in receiving order", + "finalization_performed": False, + "donor_model_binding_verified_here": False, + "source_admission_issued": False, + "population_admission_issued": False, + "release_eligible": False, + } + ) + _require(len(receipt) <= 128 * 1024, "RECEIPT_SIZE") + # A later route decoder could have changed an earlier detached table. Check + # every final route slice against its original bytes after all decoding. + positions = {value: position for position, value in enumerate(ids.tolist())} + _require( + tuple(combined.columns) == PROFILES[0].targets + and combined.index.identical(row_index) + and all(dtype == np.dtype("float64") for dtype in combined.dtypes) + and all( + full.codec.encode_raw_target( + combined[target].to_numpy()[[positions[value] for value in route_ids]], + target=target, + index=pd.Index(route_ids, name="tax_unit_id"), + ) + == payload + for route_ids, raw in raw_seals + for target, payload in raw + ), + "MERGED_RAW_CHANGED", + ) + return combined, receipt + + +@dataclass(frozen=True) +class Puf55RouteFinalizationInput: + """Numerical inputs only, never evidence that model pickle is trustworthy. + + The graph host must authenticate the actual canonical donor construction, + receiving projection, complete typed fit/apply ancestry and phase before + passing these bytes to the private finalizer. A public dataclass, hash or + this module's descriptive receipt does not satisfy that precondition. + """ + + draws: Puf55RouteDraws + donor: pd.DataFrame + last_model: bytes + phase: str + + +def _donor_identity(donor, columns): + """Exact numerical projection seal, excluding descriptive DataFrame attrs.""" + ids = full._ids(donor.index, "route_donor_RECID") + _require( + donor.index.name == "tax_unit_id" and donor.index.is_unique, "DONOR_RECID_AXIS" + ) + for name in columns: + full._numeric(donor[name], label="route_donor." + name) + return full.codec.sha( + full.codec.encode_json( + { + "columns": list(columns), + "index_name": donor.index.name, + "dtypes": [str(donor[name].dtype) for name in columns], + "recid_sha256": full.codec.sha(ids.tobytes()), + "values_sha256": full.qrf_target._consumed_values_sha256( + donor, tuple(columns) + ), + } + ) + ) + + +def _model_donor_frame(model_donor): + """Convert already validated PUF55 donor values without recipient access. + + The caller first checks the closed profile and selected donor domains. This + fence preserves source RECID values under the canonical tax_unit_id pandas + axis, with separate maintained technical model IDs and exact design weights. + It issues no donor/source or Population authority. + """ + _require( + type(model_donor) is pd.DataFrame + and tuple(model_donor.columns)[-1:] == ("weight",), + "MODEL_DONOR_INPUT", + ) + seal = _donor_identity(model_donor, tuple(model_donor.columns)) + frame = full.support._tax_unit_model_frame(model_donor) + _require(type(frame) is full.Frame, "MODEL_DONOR_FRAME_TYPE") + table = frame.table("tax_unit") + ids = np.arange(1, len(model_donor) + 1, dtype=np.int64) + _require( + _donor_identity(model_donor, tuple(model_donor.columns)) == seal + and tuple(table.columns) == ("tax_unit_id", *model_donor.columns[:-1]) + and table.index.identical(model_donor.index) + and table.drop(columns="tax_unit_id").equals(model_donor.drop(columns="weight")) + and table.tax_unit_id.dtype == np.dtype("int64") + and np.array_equal(table.tax_unit_id.to_numpy(), ids) + and tuple(frame.person.columns) == ("person_id", "person_tax_unit_id") + and frame.person.index.identical(pd.RangeIndex(len(ids))) + and frame.person.person_id.dtype == np.dtype("int64") + and frame.person.person_tax_unit_id.dtype == np.dtype("int64") + and np.array_equal(frame.person.person_id.to_numpy(), ids) + and np.array_equal(frame.person.person_tax_unit_id.to_numpy(), ids) + and frame.weights_for("tax_unit").kind.value == "design" + and frame.weights_for("tax_unit").values.dtype == np.dtype("float64") + and frame.weights_for("tax_unit").values.tobytes() + == model_donor.weight.to_numpy(dtype=np.float64).tobytes(), + "MODEL_DONOR_CONVERSION", + ) + return frame + + +def _candidate_frame_sha256(frame): + """Complete physical candidate seal, not Population or source issuance.""" + _require(type(frame) is full.Frame, "FINALIZATION_RESULT_TYPE") + return physical._population_stamp( + physical.Population.from_frame(frame, "puf55_route_numerical_candidate") + ) + + +def _finalize_puf55_routes(frame, *, recipient_matrices, routes, seed): + """Validate two trusted raw/model chains, then finalize the whole cohort once. + + Mandatory caller precondition: authenticate donor source/construction and + every typed producer edge BEFORE this function can deserialize final-model + bytes. ``phase`` is checked as a value here; only the host can bind it to the + real graph node. Fresh source/recipient qualification and complete upstream + and materialized Population checks remain host duties after this function's + last I/O. This private numerical result grants no source or owner admission. + + Supply only nonempty routes, in nine/eight order. Each donor keeps its full + canonical RECID order, including zero design weights and incidence capacity. + No route-specific Frame/clone selection or full-nine matrix is constructed. + An entirely empty receiving clone-one cohort is refused, consistently with + the maintained finalizer's nonempty PUF detail requirement. + """ + _require(type(frame) is full.Frame, "FINALIZATION_FRAME") + _require(type(seed) is int and seed >= 0, "SEED") + _require(type(routes) is tuple and 1 <= len(routes) <= 2, "FINALIZATION_ROUTES") + _receiving_axis(frame) # Refuse an empty cohort before any trusted decoder. + retained, copies = [], [] + common_columns = ( + *PROFILES[1].predictors, + *PROFILES[1].targets, + "weight", + *PROFILES[1].donor_auxiliary_columns, + ) + for item in routes: + _require(type(item) is Puf55RouteFinalizationInput, "FINALIZATION_INPUT_TYPE") + draws, donor, last_model, phase = ( + item.draws, + item.donor, + item.last_model, + item.phase, + ) + values = _route_snapshot(draws) + profile = full.require_puf_output_profile(values[0]) + _require(type(last_model) is bytes and bool(last_model), "FINAL_MODEL_BYTES") + _require(type(phase) is str and phase == profile.phase, "PHASE") + _require(type(donor) is pd.DataFrame, "DONOR_TYPE") + model_donor = full._validated_model_donor(donor, profile=profile) + donor_seal = _donor_identity(donor, tuple(donor.columns)) + common_seal = _donor_identity(donor, common_columns) + _, training = full.codec.read_training(values[-1]) + _require(training.to_dict()["model_config"]["seed"] == seed, "MODEL_SEED") + # The maintained conversion retains the pandas RECID axis and inserts + # separate model entity IDs. It performs no source admission or fillna. + donor_frame = _model_donor_frame(model_donor) + model_seal = _donor_identity(model_donor, tuple(model_donor.columns)) + retained.append( + ( + item, + draws, + values, + donor, + donor_seal, + model_donor, + model_seal, + donor_frame, + last_model, + phase, + common_seal, + ) + ) + copies.append(Puf55RouteDraws(*values)) + _require(len({row[-1] for row in retained}) == 1, "DONOR_PARITY") + # The original merger validates every profile/matrix/raw predecessor before + # any trusted model decode and performs the single ID-to-pandas relabeling. + combined, merge_receipt = merge_puf55_route_draws( + frame, + recipient_matrices=recipient_matrices, + route_draws=tuple(copies), + seed=seed, + ) + raw_seal = full.qrf_target._consumed_values_sha256(combined, PROFILES[0].targets) + receiving_ids, receiving_rows = _receiving_axis(frame) + for row in retained: + _, _, values, _, _, _, _, donor_frame, last_model, _, _ = row + full._check_fitted_model_donor( + donor_frame, + training_state=values[-1], + last_model=last_model, + profile=values[0], + ) + + def check_inputs(): + for row in retained: + ( + item, + draws, + values, + donor, + donor_seal, + model_donor, + model_seal, + donor_frame, + last_model, + phase, + _, + ) = row + _require( + item.draws is draws + and item.donor is donor + and type(item.last_model) is bytes + and item.last_model == last_model + and type(item.phase) is str + and item.phase == phase + and _route_snapshot(draws) == values + and _donor_identity(donor, tuple(donor.columns)) == donor_seal + and _donor_identity(model_donor, tuple(model_donor.columns)) + == model_seal, + "FINALIZATION_INPUT_CHANGED", + ) + resolved = full.qrf._resolve_qrf_fit_input( + donor_frame, + list(values[0].predictors), + list(values[0].targets), + "design", + ) + _require( + resolved.table.index.identical(model_donor.index) + and resolved.table.loc[:, list(model_donor.columns[:-1])].equals( + model_donor.drop(columns="weight") + ) + and np.array_equal(resolved.weights, model_donor.weight.to_numpy()), + "MODEL_DONOR_FRAME_CHANGED", + ) + fresh_ids, fresh_rows = _receiving_axis(frame) + _require( + np.array_equal(fresh_ids, receiving_ids) + and fresh_rows.identical(receiving_rows) + and combined.index.identical(receiving_rows) + and tuple(combined.columns) == PROFILES[0].targets + and all(dtype == np.dtype("float64") for dtype in combined.dtypes) + and full.qrf_target._consumed_values_sha256(combined, PROFILES[0].targets) + == raw_seal, + "FINALIZATION_RAW_CHANGED", + ) + + check_inputs() + caps = [] + # Both projections have the same target/weight surface. The first nonempty + # route supplies it; its additional predictors are ignored by this finalizer. + candidate = full.support.finalize_us_puf_tax_detail_predictions( + frame, + retained[0][5], + combined.copy(deep=True), + person_outputs=PROFILES[0].person_outputs, + tax_unit_outputs=PROFILES[0].tax_unit_outputs, + tail_bound_diagnostics=caps, + absent_cells=full.support.PUF_ABSENT_CELLS_PRESERVE_NULLS, + ) + check_inputs() + _require(type(candidate) is full.Frame, "FINALIZATION_RESULT_TYPE") + # Construct detached descriptive output only after the last trusted-model + # and finalizer I/O plus the pure retained-input checks. + evidence = [ + { + "profile": row[2][0].value, + "declared_phase": row[9], + "donor_rows": len(row[3]), + "donor_numerical_sha256": row[4], + "common_donor_numerical_sha256": row[10], + "last_model_sha256": full.codec.sha(row[8]), + } + for row in retained + ] + # This immutable digest crosses the function-return boundary with the + # candidate. A caller must compare it before capturing its own baseline. + candidate_sha256 = _candidate_frame_sha256(candidate) + receipt = full.codec.encode_json( + { + "protocol": FINALIZATION_PROTOCOL, + "candidate_frame_sha256": candidate_sha256, + "routes": evidence, + "raw_merge_receipt_sha256": full.codec.sha(merge_receipt), + "common_donor_columns": list(common_columns), + "recipient_rows": len(receiving_ids), + "ordered_recipient_ids_sha256": full.codec.sha(receiving_ids.tobytes()), + "target_order": list(PROFILES[0].targets), + "tail_bounds": caps, + "finalizer_calls": 1, + "absent_cells": full.support.PUF_ABSENT_CELLS_PRESERVE_NULLS, + "model_consumed_donor_values_checked": True, + "source_and_typed_producer_authentication": "mandatory upstream host precondition", + "source_admission_issued": False, + "population_admission_issued": False, + "release_eligible": False, + } + ) + _require(type(receipt) is bytes and len(receipt) <= 128 * 1024, "RECEIPT_SIZE") + # Receipt encoding is a final callback boundary. Keep every prior retained + # input check and reject a changed candidate before handing it to the host. + check_inputs() + _require( + _candidate_frame_sha256(candidate) == candidate_sha256, + "FINALIZATION_CANDIDATE_CHANGED", + ) + return candidate, receipt diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/puf55_survey_recipients.py b/packages/microcosm-build/src/microcosm/build/us_runtime/puf55_survey_recipients.py new file mode 100644 index 000000000..7f3965be2 --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/puf55_survey_recipients.py @@ -0,0 +1,574 @@ +"""Disjoint PUF55 recipient values from an actual, authenticated financial run. + +Preparation only: no donor, fit, attachment or Population admission. Retained +modeled return roles are not observed filers or a current-money reconstruction. +The six existing money features retain their all-member aggregation contract. +Source SS reports/components/masks are detached projections on both clone arms; +the receiving Frame and its canonical benefit cells are never modified. +""" + +from __future__ import annotations + +import csv +import hashlib +import tempfile +from dataclasses import dataclass +from pathlib import Path + +import numpy as np +import pandas as pd + +from microcosm.fit import model_input +from microcosm.graph.codecs import RAW_BYTES_MAX_BYTES + +from . import graph_atomic_survey_financial as financial +from . import puf55_survey_ss_measurement as ss +from . import puf_full_source as puf_source +from . import support_provenance as provenance + +source, full, support = ss.source, ss.full, ss.full.support +PROTOCOL = "microcosm.us.puf55-survey-recipients.v1" +PROFILES = (full.PUF55_SURVEY_SS, full.PUF55_SURVEY_SS_NO_TOTAL) +MONEY_PREDICTORS = full.PUF59.predictors[2:] +MONEY_COLUMNS = ( + ("employment_income_before_lsr",), + ("self_employment_income_before_lsr",), + ("taxable_interest_income",), + ("non_qualified_dividend_income", "qualified_dividend_income"), + ("short_term_capital_gains",), + ("long_term_capital_gains_before_response",), +) +_STATUS = { + "SINGLE": 1, + "JOINT": 2, + "SEPARATE": 3, + "HEAD_OF_HOUSEHOLD": 4, + "SURVIVING_SPOUSE": 5, +} +MAX_RECEIPT_BYTES = 128 * 1024 + + +def _require(condition, reason): + if not condition: + raise ValueError("PUF55_SURVEY_RECIPIENTS_" + reason) + + +def _table_digest(table): + """Exact axes, storage domains, values and unknowns for our detached tables.""" + _require(type(table) is pd.DataFrame and table.columns.is_unique, "TABLE") + digest = hashlib.sha256() + digest.update( + source._encode( + [ + list(table.columns), + type(table.columns).__name__, + str(table.columns.dtype), + list(table.columns.names), + type(table.index).__name__, + str(table.index.dtype), + list(table.index.names), + [ + [ + str(d), + getattr(d, "storage", None), + str(getattr(d, "na_value", "")), + ] + for d in table.dtypes + ], + ] + ) + ) + for row in table.itertuples(index=True, name=None): + for value in row: + payload = source._frame_cell_encode(value) + digest.update(len(payload).to_bytes(8, "big")) + digest.update(payload) + for series in (pd.Series(table.index), *(table[c] for c in table)): + if series.dtype == np.dtype("object"): + # These projections use object storage only for non-null strings. + # Their bytes were encoded above; ndarray.tobytes() here would hash + # process-local Python pointers and make equivalent replay unstable. + _require( + series.map(lambda value: type(value) is str).all(), + "OBJECT_STRING_STORAGE", + ) + continue + for part in financial.reconstruction._storage_parts( + series, np.ones(len(series), dtype=np.bool_) + ): + digest.update(len(part).to_bytes(8, "big")) + digest.update(part) + return digest.hexdigest() + + +def _literal_snapshot(state): + """Capture only the selected literal rows against the actual closed pin. + + Immutable strings come from the independently hashed original CSV stream, + not from a detached qualifier result or carried reason/allocation columns. + The existing coverage capture owns file handling; this adds no source issuer. + """ + owner = ss.reports.coverage + native = state.native[1] + entry = source.asec_native._ISSUED.get(id(native)) + _require( + entry is not None and entry[0]() is native and entry[1] == native.payload, + "LITERAL_NATIVE_OWNER", + ) + pins = tuple(pin for pin in owner._MEMBER_PINS if pin[0] == 2024) + _require(len(pins) == 1, "LITERAL_PIN") + year, member, archive, digest, rows, size = pins[0] + retained = [ + row + for row in entry[2].coverage.receipt["sources"] + if row["source_year"] == year + ] + _require( + len(retained) == 1 + and all( + retained[0][key] == value + for key, value in ( + ("member", member), + ("archive_sha256", archive), + ("member_sha256", digest), + ("rows", rows), + ("member_bytes", size), + ) + ), + "LITERAL_NATIVE_PIN", + ) + people = state.frame.person + keys = set( + people.loc[ + people[provenance.support_channel_column("person")].eq("asec"), "PERIDNUM" + ] + ) + reader = ss.reports.source_csv_builtin.capture_csv_reader(csv) + _require(reader is not None, "LITERAL_CSV_READER") + captured_rows, count, total = {}, 0, 0 + hashed = hashlib.sha256() + with tempfile.TemporaryDirectory(prefix="microcosm-puf55-ss-literals-") as tmp: + path = Path(tmp) / member + owner._capture( + state.root / "asec" / member, + path, + size=size, + digest=digest, + budget=[owner._BODY_MAX], + ) + with path.open("rb") as stream: + + def lines(): + nonlocal total + while chunk := stream.readline(owner._ROW_MAX + 1): + _require( + len(chunk) <= owner._ROW_MAX and total + len(chunk) <= size, + "LITERAL_BYTES", + ) + first = total == 0 + total += len(chunk) + hashed.update(chunk) + yield chunk.decode("utf-8-sig" if first else "utf-8") + + records = reader(lines(), strict=True) + header = next(records, []) + _require( + len(header) == len(set(header)) + and set(ss.reports.READ_COLUMNS) <= set(header), + "LITERAL_HEADER", + ) + positions = [header.index(c) for c in ss.reports.READ_COLUMNS] + for record in records: + count += 1 + _require(count <= rows and len(record) == len(header), "LITERAL_ROWS") + selected = tuple(record[i] for i in positions) + if selected[0] in keys: + _require(selected[0] not in captured_rows, "LITERAL_DUPLICATE") + captured_rows[selected[0]] = selected + _require( + count == rows + and total == size + and hashed.hexdigest() == digest + and set(captured_rows) == keys, + "LITERAL_SOURCE_BYTES", + ) + return pins[0], tuple(captured_rows[key] for key in sorted(captured_rows)) + + +def _source_report(state, snapshot): + """Pure full projection from pinned literals and retained native source cells.""" + pin, records = snapshot + _require( + tuple(p for p in ss.reports.coverage._MEMBER_PINS if p[0] == 2024) == (pin,), + "LITERAL_PIN_CHANGED", + ) + people = state.frame.person + channel = people[provenance.support_channel_column("person")] + result = pd.DataFrame(index=pd.Index(people.person_id, name="person_id")) + result["native_person_id"] = people[ + provenance.spine_source_id_column("person") + ].to_numpy() + result["source"] = channel.to_numpy() + result["social_security_source_total"] = np.nan + result["source_reporting_universe"] = False + result["source_reporting_unit"] = "person_report_record" + components = ss.reports.basis_owner.COMPONENTS + for column in components: + result[column], result["allowed_" + column] = np.nan, True + result["basis_origin"] = "unresolved" + result["allocation_origin"] = "unresolved_allocation_provenance" + asec = people.loc[channel.eq("asec")] + literals = ( + pd.DataFrame(records, columns=ss.reports.READ_COLUMNS) + .set_index("PERIDNUM", drop=False) + .loc[asec.PERIDNUM] + ) + _require( + np.array_equal(literals.A_AGE.astype("int64"), asec.A_AGE) + and np.array_equal( + ss.reports._codes(literals.SS_VAL, range(100000)), + ss.reports._numeric(asec.SS_VAL), + ), + "LITERAL_SOURCE_IDENTITY", + ) + amount, basis, allowed, labels = ss.reports.basis_owner.asec_reporting_basis( + ss.reports._numeric(asec.SS_VAL), + ss.reports._numeric(asec.A_AGE), + ss.reports._codes(literals.SS_YN, {0, 1, 2}), + ss.reports._codes(literals.RESNSS1, range(9)), + ss.reports._codes(literals.RESNSS2, range(9)), + ) + ids = asec.person_id.to_numpy() + result.loc[ids, "social_security_source_total"] = amount + result.loc[ids, "source_reporting_universe"] = ss.reports._numeric(asec.A_AGE) >= 15 + result.loc[ids, "source_reporting_unit"] = ( + "person_report_may_combine_family_payments" + ) + for j, column in enumerate(components): + result.loc[ids, column], result.loc[ids, "allowed_" + column] = ( + basis[:, j], + allowed[:, j], + ) + result.loc[ids, "basis_origin"] = labels + result.loc[ids, "allocation_origin"] = ss.reports._allocation_labels( + literals + ).allocation_origin.to_numpy() + acs = people.loc[channel.eq("acs")] + eligible = ss.reports._numeric(acs.AGEP) >= 15 + total = ss.reports._numeric(acs.acs_social_security_income) + result.loc[acs.person_id, "social_security_source_total"] = total + result.loc[acs.person_id, "source_reporting_universe"] = eligible + zero = acs.person_id.to_numpy()[eligible & (total == 0)] + result.loc[zero, list(components)] = 0.0 + result.loc[zero, ["allowed_" + c for c in components]] = False + result.loc[zero, "basis_origin"] = "known_total_zero" + result.loc[acs.person_id.to_numpy()[eligible & (total > 0)], "basis_origin"] = ( + "acs_combined_positive_requires_model" + ) + result.loc[acs.person_id.to_numpy()[~eligible], "basis_origin"] = ( + "acs_below15_outside_reporting_universe" + ) + return result + + +def _aligned(source_frame, frame, entity): + original, receiving = source_frame.table(entity), frame.table(entity) + identity = entity + "_id" + key, arm = ( + provenance.support_source_id_column(entity), + provenance.support_clone_index_column(entity), + ) + for column in ( + original[identity], + receiving[identity], + receiving[key], + receiving[arm], + ): + ss._axis(column) + _require(original[identity].is_unique and receiving[identity].is_unique, "AXIS") + pairs = receiving[[key, arm]] + _require( + receiving[arm].isin((0, 1)).all() + and not pairs.duplicated().any() + and set(receiving[key]) == set(original[identity]) + and pairs.groupby(key, sort=False).size().eq(2).all(), + "CLONE_PAIRS", + ) + aligned = original.set_index(identity).loc[receiving[key]].copy(deep=True) + for column in ( + provenance.spine_source_id_column(entity), + provenance.support_channel_column(entity), + ): + _require( + np.array_equal(aligned[column].to_numpy(), receiving[column].to_numpy()), + "CLONE_SOURCE_IDENTITY", + ) + aligned.index = pd.Index(receiving[identity].to_numpy(copy=True), name=identity) + return aligned + + +def _current_money_surface(frame, recipient_mask): + """Aggregate already-qualified financial leaves; issue no source authority. + + The public caller rechecks the actual financial run before this arithmetic. + Its source owner already verifies ACS adjustment and age-universe zeros. + A raw stacked-spine universe gate would incorrectly require ACS columns on + an ASEC-only surface and is not the input boundary for these produced leaves. + """ + person, units = frame.person, frame.table("tax_unit") + plans = {} + for predictor, columns in zip(MONEY_PREDICTORS, MONEY_COLUMNS, strict=True): + plan = support._strict_predictor_source_plan( + predictor, tax_unit=units, person=person + ) + _require( + plan.entity == "person" and plan.columns == columns, + "CURRENT_MONEY_SOURCE_PLAN", + ) + plans[predictor] = plan + support._require_complete_recipient_predictor_sources( + frame, recipient_mask, MONEY_PREDICTORS, source_plans=plans + ) + features = support._tax_unit_feature_frame( + frame, MONEY_PREDICTORS, preserve_nulls=True, source_plans=plans + ) + support._require_complete_recipient_predictors( + features, recipient_mask, MONEY_PREDICTORS + ) + evidence = { + "input_boundary": "current_financial_person_leaves_after_upstream_qualification", + "aggregation": "sum_all_modeled_tax_unit_members", + "predictor_source_mapping": { + predictor: {"entity": plan.entity, "columns": list(plan.columns)} + for predictor, plan in plans.items() + }, + "recipient_tax_unit_rows": int(recipient_mask.sum()), + "recipient_feature_values_sha256": support._predictor_feature_values_sha256( + features.loc[recipient_mask, list(MONEY_PREDICTORS)], + units.loc[recipient_mask, "tax_unit_id"], + ), + "raw_acs_universe_requalified_here": False, + "source_admission_issued": False, + } + evidence["sha256"] = support._receipt_sha256(evidence) + return features, evidence + + +def _project(source_frame, frame, report, measured): + """Pure projection after live ownership checks; never an authority issuer.""" + people, units = frame.person, frame.table("tax_unit") + original_people = _aligned(source_frame, frame, "person") + original_units = _aligned(source_frame, frame, "tax_unit") + unit_source = units.set_index("tax_unit_id")[ + provenance.support_source_id_column("tax_unit") + ] + _require( + np.array_equal( + people.tax_unit_role_input.to_numpy(), + original_people.tax_unit_role_input.to_numpy(), + ) + and np.array_equal( + units.filing_status_input.to_numpy(), + original_units.filing_status_input.to_numpy(), + ) + and np.array_equal( + people.person_tax_unit_id.map(unit_source).to_numpy(), + original_people.person_tax_unit_id.to_numpy(), + ), + "MODELED_ROLE_INHERITANCE", + ) + person_ids = people[provenance.support_source_id_column("person")].to_numpy() + person_report = report.loc[person_ids].copy(deep=True) + person_report.index = pd.Index( + people.person_id.to_numpy(copy=True), name="person_id" + ) + # Recheck every receiving role structure, and independently reproduce the + # source measurement on both arms. Invalid roles never select route eight. + received_measurement, _ = ss._measure(people, units, person_report) + expected = measured.loc[ + units[provenance.support_source_id_column("tax_unit")] + ].copy(deep=True) + expected.index = received_measurement.index + _require( + ss._projection_digest(received_measurement) == ss._projection_digest(expected), + "CLONE_MEASUREMENT", + ) + dependent_count = ( + people.tax_unit_role_input.eq("DEPENDENT") + .groupby(people.person_tax_unit_id) + .sum() + ) + first = puf_source.puf_2015_receiver_size_measurement( + units.filing_status_input.map(_STATUS).to_numpy(dtype=np.int64), + dependent_count.reindex(units.tax_unit_id).to_numpy(dtype=np.int64), + ) + for name, values in first.items(): + received_measurement[name] = values + recipients = ( + units[provenance.support_clone_index_column("tax_unit")].eq(1).to_numpy() + ) + features, money_evidence = _current_money_surface(frame, recipients) + for name in MONEY_PREDICTORS: + received_measurement[name] = features[name].to_numpy( + dtype=np.float64, copy=True + ) + matrices, selected = [], [] + for profile in PROFILES: + mask = recipients & received_measurement[ss.ROUTE].eq(profile.value).to_numpy() + ids = units.loc[mask, "tax_unit_id"].to_numpy(copy=True) + selected.extend(ids.tolist()) + if not len(ids): + continue + _require( + len(ids) * (1 + len(profile.predictors)) * 8 <= RAW_BYTES_MAX_BYTES, + "MATRIX_SIZE", + ) + table = received_measurement.loc[ids, list(profile.predictors)].astype( + "float64" + ) + payload = model_input.encode_recipient_matrix( + table, entity="tax_unit", entity_ids=ids + ) + _require(len(payload) <= RAW_BYTES_MAX_BYTES, "MATRIX_SIZE") + matrices.append((profile.value, payload)) + _require( + len(selected) == len(set(selected)) + and set(selected) == set(units.loc[recipients, "tax_unit_id"]), + "RECIPIENT_PARTITION", + ) + return person_report, received_measurement, tuple(matrices), money_evidence + + +@dataclass(frozen=True) +class Puf55SurveyRecipients: + """Detached outputs with their actual upstream run; public copies are values.""" + + financial_run: financial.AtomicSurveyFinancialRunValues + person: pd.DataFrame + tax_unit: pd.DataFrame + matrices: tuple[tuple[str, bytes], ...] + receipt: bytes + + +def _result_stamp(result): + _require( + type(result) is Puf55SurveyRecipients + and type(result.receipt) is bytes + and type(result.matrices) is tuple + and all( + type(row) is tuple + and len(row) == 2 + and type(row[0]) is str + and type(row[1]) is bytes + for row in result.matrices + ), + "RESULT_TYPE", + ) + return ( + _table_digest(result.person), + _table_digest(result.tax_unit), + result.matrices, + result.receipt, + ) + + +def qualify_puf55_survey_recipients(financial_run): + """Require the actual financial run and requalify source observations. + + This admits numerical preparation under retained modeled roles only. It + cannot establish current-money tax-unit reconstruction or release validity. + Source receipt objects and caller-supplied role tables are not accepted. + """ + financial.check_atomic_survey_financial_run(financial_run) + entry = financial._run_entry(financial_run) + state = entry[2] + preparation = state.prefix.preparation + source_frame = state.preparation_entry[2].frame + frame = state.financial_population.frame + snapshot = _literal_snapshot(state.preparation_entry[2]) + report = ss.reports.qualify_current_social_security(preparation) + ss._source_identity(source_frame.person, report) + authoritative_report = _source_report(state.preparation_entry[2], snapshot) + _require( + _table_digest(report.person) == _table_digest(authoritative_report), + "COMPLETE_SOURCE_REPORT", + ) + expected_measurement, measurement_counts = ss._measure( + source_frame.person, source_frame.table("tax_unit"), authoritative_report + ) + person, units, matrices, money_evidence = _project( + source_frame, frame, authoritative_report, expected_measurement + ) + receipt = source._encode( + { + "protocol": PROTOCOL, + "financial_run_sha256": financial.codec.sha(entry[1]), + "preparation_sha256": financial.codec.sha(state.preparation_entry[1]), + "person_projection_sha256": _table_digest(person), + "tax_unit_projection_sha256": _table_digest(units), + "source_report_projection_sha256": _table_digest(authoritative_report), + "asec_current_literal_member_sha256": snapshot[0][3], + "asec_selected_literal_rows": len(snapshot[1]), + "asec_income_year": 2024, + "asec_interview_year": 2025, + "acs_income_window": "rolling prior12 months at2024 interview; ADJINC price basis", + "reporting_grain": "person report; family/child aggregation and repayment/Railroad alignment unresolved", + "ss_measurement": ss.MEASUREMENT, + "native_measurement_counts": measurement_counts, + "measurement_knownness": "all included HEAD/actual JOINT SPOUSE reports available; no dependent, component or partial-sum fallback", + "money_predictor_evidence": money_evidence, + "routes": [ + { + "profile": name, + "matrix_sha256": financial.codec.sha(payload), + "rows": len(model_input.decode_recipient_matrix(payload).features), + "predictors": list(full.PufOutputProfile(name).predictors), + "outputs": list(full.PufOutputProfile(name).targets), + } + for name, payload in matrices + ], + "recipient_tax_units": sum( + len(model_input.decode_recipient_matrix(p).features) + for _, p in matrices + ), + "receiving_tax_units": len(units), + "receiving_persons": len(person), + "partition": "all clone-one tax units exactly once; empty routes omitted in nine/eight order", + "role_boundary": "retained modeled HEAD/actual JOINT SPOUSE roles; not observed filer status", + "money_contract": "existing six features sum all modeled tax-unit members; current financial leaves only", + "current_money_tax_units_reconstructed_here": False, + "person_social_security_changed": False, + "population_changed": False, + "source_admission_issued": False, + "population_admission_issued": False, + "release_eligible": False, + }, + maximum=MAX_RECEIPT_BYTES, + ) + result = Puf55SurveyRecipients(financial_run, person, units, matrices, receipt) + stamp = _result_stamp(result) + financial.check_atomic_survey_financial_run(financial_run) + financial._pure_run(financial_run, entry) + fresh = _source_report(state.preparation_entry[2], snapshot) + fresh_measurement, _ = ss._measure( + source_frame.person, source_frame.table("tax_unit"), fresh + ) + final_person, final_units, final_matrices, final_money = _project( + source_frame, frame, fresh, fresh_measurement + ) + _require( + financial._run_entry(financial_run) is entry + and result.financial_run is financial_run + and _result_stamp(result) == stamp + and _table_digest(fresh) == _table_digest(authoritative_report) + and ( + _table_digest(final_person), + _table_digest(final_units), + final_matrices, + receipt, + ) + == stamp + and final_money == money_evidence, + "FINAL_PROJECTION", + ) + return result diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/puf55_survey_ss_measurement.py b/packages/microcosm-build/src/microcosm/build/us_runtime/puf55_survey_ss_measurement.py new file mode 100644 index 000000000..d5abe6afb --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/puf55_survey_ss_measurement.py @@ -0,0 +1,320 @@ +"""Filer/joint-spouse report sums for conditioning, never beneficiary amounts. + +The public entry point requires a live authenticated survey preparation. Its +retained modeled return roles are inputs, not roles inferred from the coarsened +PUF filing-class predictor. The private arithmetic is not a source issuer. +Graph hosts must retain/requalify the preparation and authenticate clone +inheritance, routing and typed model edges independently. +""" + +from __future__ import annotations + +import hashlib +from dataclasses import dataclass + +import numpy as np +import pandas as pd + +from . import current_social_security_source as reports +from . import full_puf_enrichment as full +from . import survey_population_preparation as source +from .support_provenance import spine_source_id_column, support_channel_column + +PROTOCOL = "microcosm.us.puf55-survey-ss-measurement.v1" +MEASUREMENT = "social_security_filer_joint_spouse_report_sum_proxy" +TOTAL = full.SURVEY_SS_TOTAL_PREDICTOR +KNOWN = "puf_conditioning_social_security_total_known" +ROUTE = "puf55_conditioning_profile" +COLUMNS = (TOTAL, KNOWN, ROUTE) +_PERSON_COLUMNS = ("person_id", "person_tax_unit_id", "tax_unit_role_input") +_UNIT_COLUMNS = ("tax_unit_id", "filing_status_input") +_REPORT_COLUMNS = ( + "social_security_source_total", + "source_reporting_universe", +) +_STATUSES = {"SINGLE", "JOINT", "SEPARATE", "HEAD_OF_HOUSEHOLD", "SURVIVING_SPOUSE"} + + +def _require(condition, reason): + if not condition: + raise ValueError("PUF55_SURVEY_SS_" + reason) + + +def _sha(payload): + return hashlib.sha256(payload).hexdigest() + + +def _axis(values): + _require( + values.dtype == np.dtype("int64") + and not values.isna().any() + and values.ge(0).all(), + "AXIS", + ) + + +def _measure(person, tax_unit, report): + """Pure numerical checks on already-qualified values; no admission issued.""" + for table, columns in ( + (person, _PERSON_COLUMNS), + (tax_unit, _UNIT_COLUMNS), + (report, _REPORT_COLUMNS), + ): + _require( + type(table) is pd.DataFrame + and table.columns.is_unique + and set(columns) <= set(table), + "COLUMNS", + ) + for column in (person.person_id, person.person_tax_unit_id, tax_unit.tax_unit_id): + _axis(column) + _require( + len(tax_unit) > 0 + and person.person_id.is_unique + and tax_unit.tax_unit_id.is_unique + and set(person.person_tax_unit_id) == set(tax_unit.tax_unit_id) + and report.index.name == "person_id" + and report.index.dtype == np.dtype("int64") + and report.index.is_unique + and set(report.index) == set(person.person_id), + "MEMBERSHIP", + ) + roles, statuses = person.tax_unit_role_input, tax_unit.filing_status_input + _require( + not roles.isna().any() + and roles.map(lambda v: type(v) is str).all() + and roles.isin(("HEAD", "SPOUSE", "DEPENDENT")).all() + and not statuses.isna().any() + and statuses.map(lambda v: type(v) is str).all() + and statuses.isin(_STATUSES).all(), + "ROLE_VALUES", + ) + # Validate every return's roles before deciding any missing-report route. + # SURVIVING_SPOUSE maps to PUF class 2 elsewhere but has no actual spouse. + groups = person.groupby("person_tax_unit_id", sort=False) + included = [] + for tid, status in tax_unit[list(_UNIT_COLUMNS)].itertuples(index=False, name=None): + members = groups.get_group(tid) + head = members.tax_unit_role_input.eq("HEAD") + spouse = members.tax_unit_role_input.eq("SPOUSE") + _require( + head.sum() == 1 and spouse.sum() == int(status == "JOINT"), + "ROLE_STRUCTURE", + ) + included.append(members.loc[head | spouse, "person_id"].to_numpy(copy=True)) + amount, universe = report[_REPORT_COLUMNS[0]], report[_REPORT_COLUMNS[1]] + _require( + pd.api.types.is_numeric_dtype(amount.dtype) + and not pd.api.types.is_bool_dtype(amount.dtype) + and not pd.api.types.is_complex_dtype(amount.dtype), + "REPORT_PHYSICAL_TYPE", + ) + _require(universe.dtype == np.dtype("bool"), "UNIVERSE_PHYSICAL_TYPE") + if pd.api.types.is_integer_dtype(amount.dtype): + _require(amount.dropna().between(-(2**53), 2**53).all(), "REPORT_PRECISION") + numeric = amount.to_numpy(dtype=np.float64, na_value=np.nan, copy=True) + available = np.isfinite(numeric) + _require( + (np.isnan(numeric) | available).all() and (numeric[available] >= 0).all(), + "REPORT_DOMAIN", + ) + _require( + np.isnan(numeric[~universe.to_numpy()]).all(), "OUTSIDE_UNIVERSE_OBSERVATION" + ) + values, known, routes = [], [], [] + missing_people, included_people = 0, 0 + for ids in included: + selected = report.loc[ids] + observed = selected[_REPORT_COLUMNS[0]].to_numpy( + dtype=np.float64, na_value=np.nan, copy=True + ) + present = selected[_REPORT_COLUMNS[1]].to_numpy() & np.isfinite(observed) + complete = bool(present.all()) + included_people += len(ids) + missing_people += int((~present).sum()) + # No pandas skipna, empty sum, dependent zero, component sum or partial + # report sum can manufacture a known measurement. + total = float(np.sum(observed)) if complete else np.nan + _require(not complete or np.isfinite(total), "SUM_OVERFLOW") + values.append(total) + known.append(complete) + routes.append( + (full.PUF55_SURVEY_SS if complete else full.PUF55_SURVEY_SS_NO_TOTAL).value + ) + result = pd.DataFrame( + { + TOTAL: np.asarray(values, dtype=np.float64), + KNOWN: np.asarray(known, dtype=bool), + ROUTE: pd.array(routes, dtype="string"), + }, + index=pd.Index(tax_unit.tax_unit_id.to_numpy(copy=True), name="tax_unit_id"), + ) + counts = { + "returns": len(result), + "included_reporters": included_people, + "excluded_dependents": len(person) - included_people, + "unavailable_included_reports": missing_people, + "nine_predictor_returns": int(result[KNOWN].sum()), + "eight_predictor_returns": int((~result[KNOWN]).sum()), + } + return result, counts + + +def _projection_digest(table): + _require( + type(table) is pd.DataFrame + and tuple(table) == COLUMNS + and table.index.name == "tax_unit_id" + and table.index.dtype == np.dtype("int64") + and table.index.is_unique + and table[TOTAL].dtype == np.dtype("float64") + and table[KNOWN].dtype == np.dtype("bool"), + "PROJECTION_STORAGE", + ) + digest = hashlib.sha256() + for tid, total, known, route in table.itertuples(index=True, name=None): + payload = source._encode( + [ + int(tid), + None if np.isnan(total) else float(total).hex(), + bool(known), + route, + ] + ) + digest.update(len(payload).to_bytes(4, "big")) + digest.update(payload) + return digest.hexdigest() + + +def _source_identity(people, projection): + """Check detached source output against the exact retained source cells. + + SS qualification already authenticates original current ASEC literals and + ACS SSP adjustment. This additional comparison prevents a return callback + from changing its detached amount/universe without changing the live owner. + """ + _require(type(projection) is reports.CurrentSocialSecurityProjection, "REPORT_TYPE") + report = projection.person + _require( + report.index.is_unique and set(report.index) == set(people.person_id), + "SOURCE_AXIS", + ) + report = report.reindex(people.person_id) + channels = people[support_channel_column("person")] + _require( + np.array_equal(report.source.to_numpy(), channels.to_numpy()) + and np.array_equal( + report.native_person_id.to_numpy(), + people[spine_source_id_column("person")].to_numpy(), + ), + "SOURCE_ORIGIN", + ) + for channel, age_column, amount_column in ( + ("asec", "A_AGE", "SS_VAL"), + ("acs", "AGEP", "acs_social_security_income"), + ): + mask = channels.eq(channel).to_numpy() + selected = people.loc[mask] + age = reports._numeric(selected[age_column]) + expected = reports._numeric(selected[amount_column]) + eligible = age >= 15 + expected[~eligible] = np.nan + observed = report.iloc[np.flatnonzero(mask)] + _require( + np.array_equal(observed.source_reporting_universe.to_numpy(), eligible) + and np.array_equal( + reports._numeric(observed.social_security_source_total), + expected, + equal_nan=True, + ), + "SOURCE_REPORT_IDENTITY", + ) + + +@dataclass(frozen=True) +class Puf55SurveySSMeasurement: + """Detached values and live upstream reference, not a reusable certificate.""" + + preparation: source.AuthenticatedSurveyPopulationPreparation + tax_unit: pd.DataFrame + evidence: bytes + + +def qualify_puf55_survey_ss_measurement(preparation): + """Project native stacked returns; clone inheritance remains the host's duty. + + Roles are the modeled roles retained by the exact preparation, not a claim + that this helper reran current-money tax-unit construction. Receipt bytes, + caller-made tables, and this result cannot replace the live upstream owner. + """ + _require( + type(preparation) is source.AuthenticatedSurveyPopulationPreparation, + "PREPARATION_TYPE", + ) + entry = preparation._checked() + state = entry[2] + projection = reports.qualify_current_social_security(preparation) + _source_identity(state.frame.person, projection) + report_evidence = source._encode(projection.evidence) + _require( + projection.evidence["preparation_sha256"] == _sha(entry[1]), + "REPORT_PREPARATION", + ) + values, counts = _measure( + state.frame.person, state.frame.table("tax_unit"), projection.person + ) + stamp = _projection_digest(values) + evidence = source._encode( + { + "protocol": PROTOCOL, + "measurement": MEASUREMENT, + "preparation_sha256": _sha(entry[1]), + "source_report_evidence_sha256": _sha(report_evidence), + "projection_sha256": stamp, + "counts": counts, + "columns": list(COLUMNS), + "membership": "HEAD plus SPOUSE only for literal JOINT; no dependents", + "role_boundary": "exact modeled return roles retained by live preparation", + "current_money_tax_units_reconstructed_here": False, + "knownness": "all included source reports available; not benefit ownership", + "fallback": full.PUF55_SURVEY_SS_NO_TOTAL.value, + "family_and_child_reports": "retained without allocation or deduplication", + "source_periods": { + "asec": "2024 calendar income; 2025 interview", + "acs": "rolling prior 12 months at 2024 interview; ADJINC price basis", + }, + "repayment_and_railroad_alignment": "unresolved; no adjustment applied", + "donor_E02400_editing": "unresolved; separate donor transport convention", + "person_social_security_changed": False, + "individual_beneficiary_assignment_claim": False, + "source_admission_issued": False, + "population_admission_issued": False, + "release_eligible": False, + }, + maximum=64 * 1024, + ) + result = Puf55SurveySSMeasurement(preparation, values, evidence) + # Finish live owner I/O, then only compare retained owners and computed + # values. The returned projection is never itself treated as authority. + final_entry = preparation._checked() + _require( + final_entry is entry + and source._ISSUED.get(id(preparation)) is entry + and preparation.payload == entry[1], + "FINAL_ISSUANCE", + ) + source._pure_final(state) + _source_identity(state.frame.person, projection) + fresh, fresh_counts = _measure( + state.frame.person, state.frame.table("tax_unit"), projection.person + ) + _require( + source._encode(projection.evidence) == report_evidence + and fresh_counts == counts + and _projection_digest(fresh) == stamp + and result.preparation is preparation + and result.evidence == evidence + and _projection_digest(result.tax_unit) == stamp, + "FINAL_PROJECTION", + ) + return result diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/puf59_canonical.py b/packages/microcosm-build/src/microcosm/build/us_runtime/puf59_canonical.py new file mode 100644 index 000000000..af1290c9f --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/puf59_canonical.py @@ -0,0 +1,307 @@ +"""Return-grain canonical PUF59 with explicit model/growth lineage. + +No tax engine, invented donor people, loan balances or loan vintages. The +observed statistical2015 source remains immutable beside modeled outputs. +""" + +from __future__ import annotations + +import hashlib +import json +from collections.abc import Mapping +from dataclasses import dataclass +from types import MappingProxyType + +import numpy as np + +from . import puf_full_source as source_owner +from . import puf_qbi_model as qbi +from . import puf_target2024_growth as growth + +VERSION = "microcosm.us.puf59_canonical_return/2" +INTEREST_ASSET_SHA256 = ( + "c3356ae216487f365cb0e0a7ab1ba46843a6c52950b75eae1bfab9b0b80a735a" +) + +INTEREST_BAND_FACTS_SHA256 = ( + "3ea894c474302dd818f347f5bfde3b331bf80468b0ac928ffe76ed1e6de62cbc" +) +PREFIX_NAMES = ( + "RECID", + "weight", + "puf_person_incidence_capacity", + "puf_2015_filing_status_code", + "puf_2015_capped_return_size", +) + + +def prefix_values_digest(arrays): + """Bind identities, source weights and source-specific features in row order.""" + digest = hashlib.sha256() + for name in PREFIX_NAMES: + digest.update(name.encode("ascii") + b"\0") + digest.update( + np.asarray( + arrays[name], dtype="= -(2**53)) & (a <= 2**53)).all()), + "PUF59_MODEL_INTEGER_PRECISION:" + name, + ) + a = a.astype(np.float64) + _require(bool(np.isfinite(a).all()), "PUF59_MODEL_NONFINITE:" + name) + return a + + +def baseline_modeled_columns(auxiliary, *, n, interest_bands, interest_asset_sha256): + """Eleven explicit baseline outputs absent as exact raw canonical leaves.""" + _require( + interest_asset_sha256 == INTEREST_ASSET_SHA256, "PUF59_INTEREST_ASSET_IDENTITY" + ) + _require(isinstance(auxiliary, Mapping), "PUF59_MODEL_AUXILIARY_MAPPING") + names = ( + "raw_adjusted_gross_income", + "raw_total_interest_deduction", + "raw_total_social_security", + "raw_realized_ira_deduction", + "raw_realized_keogh_deduction", + "raw_tuition_fees_deduction", + "raw_lifetime_learning_qualified_expenses", + "raw_miscellaneous_itemized_deductions", + "raw_partnership_nonpassive_net", + ) + _require(all(k in auxiliary for k in names), "PUF59_MODEL_MISSING_AUXILIARY") + a = {k: _money(auxiliary[k], k, n) for k in names} + for k in names: + if k not in ("raw_adjusted_gross_income", "raw_partnership_nonpassive_net"): + _require(bool((a[k] >= 0).all()), "PUF59_MODEL_NEGATIVE_SOURCE:" + k) + bands = tuple(interest_bands) + _require(len(bands) == 22, "PUF59_INTEREST_BAND_COUNT") + _require( + bands[0].lower_bound is None and bands[-1].upper_bound is None, + "PUF59_INTEREST_FULL_AGI", + ) + for left, right in zip(bands[:-1], bands[1:], strict=True): + _require(left.upper_bound == right.lower_bound, "PUF59_INTEREST_CONTIGUOUS") + total = a["raw_total_interest_deduction"] + agi = a["raw_adjusted_gross_income"] + investment = np.zeros(n) + covered = np.zeros(n, dtype=np.uint8) + band_facts = [] + fields = ( + "home_mortgage_interest_amount", + "deductible_points_amount", + "qualified_mortgage_insurance_premiums_amount", + "investment_interest_amount", + ) + for i, band in enumerate(bands): + _require(band.source_row == 11 + i, "PUF59_INTEREST_SOURCE_ROW") + lo, hi = band.lower_bound, band.upper_bound + _require( + (lo is None or type(lo) in (int, float) and np.isfinite(lo)) + and (hi is None or type(hi) in (int, float) and np.isfinite(hi)) + and (lo is None or hi is None or lo < hi), + "PUF59_INTEREST_BAND_BOUNDS", + ) + amounts = np.asarray([getattr(band, k) for k in fields], dtype=np.float64) + _require( + bool(np.isfinite(amounts).all() and (amounts >= 0).all()) + and amounts.sum() > 0, + "PUF59_INTEREST_COMPONENT_DOMAIN", + ) + _require( + type(band.total_interest_paid_amount) is int + and abs( + band.total_interest_paid_amount - sum(getattr(band, k) for k in fields) + ) + <= 1, + "PUF59_INTEREST_SOURCE_ROUNDING", + ) + mask = np.ones(n, dtype=bool) + if lo is not None: + mask &= agi >= lo + if hi is not None: + mask &= agi < hi + investment[mask] = total[mask] * (amounts[3] / amounts.sum()) + covered[mask] += 1 + band_facts.append( + { + "source_row": band.source_row, + "lower_bound": lo, + "upper_bound": hi, + "amounts_thousands_usd": amounts.tolist(), + } + ) + band_facts_sha256 = hashlib.sha256( + json.dumps(band_facts, sort_keys=True, separators=(",", ":")).encode() + ).hexdigest() + _require( + band_facts_sha256 == INTEREST_BAND_FACTS_SHA256, + "PUF59_INTEREST_BAND_FACTS_IDENTITY", + ) + _require(bool((covered == 1).all()), "PUF59_INTEREST_PARTITION") + home = total - investment + investment = total - home + _require(np.array_equal(home + investment, total), "PUF59_INTEREST_CONSERVATION") + columns = { + "home_mortgage_interest": home, + "investment_interest_expense": investment, + "social_security_retirement": a["raw_total_social_security"].copy(), + "social_security_disability": np.zeros(n), + "social_security_dependents": np.zeros(n), + "social_security_survivors": np.zeros(n), + "traditional_ira_contributions_desired": a["raw_realized_ira_deduction"].copy(), + "self_employed_pension_contributions_desired": a[ + "raw_realized_keogh_deduction" + ].copy(), + "qualified_tuition_expenses": np.maximum( + a["raw_tuition_fees_deduction"], + a["raw_lifetime_learning_qualified_expenses"], + ), + "unreimbursed_business_employee_expenses": a[ + "raw_miscellaneous_itemized_deductions" + ].copy(), + "partnership_self_employment_net_earnings": a[ + "raw_partnership_nonpassive_net" + ].copy(), + } + return columns, { + "schema": "microcosm.us.puf59_baseline_models/1", + "input_money_year": 2015, + "interest_asset_sha256": interest_asset_sha256, + "interest_band_facts_sha256": band_facts_sha256, + "interest_method": "Normalize all four published component amounts. Map mortgage+points+insurance to the available home interest leaf and investment only to investment expense; complementary rounding conserves E19200 exactly.", + "ss_method": "Component-neutral carrier: total SS in retirement, other component carriers zero; requires recipient-profile SS reconciliation. Zeros are not observations of absent benefits.", + "desired_deductions": "Realized IRA/Keogh deductions proxy for desired amounts; uncensored demand is not observed.", + "tuition": "Maximum of realized tuition deduction and Lifetime Learning qualified expenses; incomplete source expense proxy.", + "employee_expense": "Total miscellaneous itemized deductions proxy; broader than employee business expenses.", + "partnership_earnings": "Nonpassive partnership net earnings proxy; not reconstructed from capped ScheduleSE amounts.", + "mortgage_structure_generated": False, + "origin": "modeled_not_observed", + } + + +@dataclass(frozen=True) +class CanonicalPuf59Result: + columns: Mapping[str, np.ndarray] + source_predictors: Mapping[str, np.ndarray] + recids: np.ndarray + design_weight: np.ndarray + person_incidence_capacity: np.ndarray + money_year: int + qbi_calibration: qbi.QbiEmploymentCalibration + receipt: Mapping[str, object] + + +def construct_canonical_puf59( + source, + *, + interest_bands, + interest_asset_sha256, + selected_recids=None, + qbi_employment_calibration=None, + growth_scheme="family_observed", + seed=0, +): + """Construct canonical outputs from an owner-decoded ordinary source cohort. + + The caller supplies source-authenticated interest bands. Selection is owned + by the source decoder. No data source is opened or sampled in this function. + """ + observed, aux, status = source_owner.observed_and_derived_return_columns( + source, selected_recids=selected_recids + ) + n = len(status["RECID"]) + _require(n > 0, "PUF59_EMPTY_COHORT") + modeled, model_receipt = baseline_modeled_columns( + aux, + n=n, + interest_bands=interest_bands, + interest_asset_sha256=interest_asset_sha256, + ) + _require(not set(observed) & set(modeled), "PUF59_CANONICAL_COLLISION") + canonical = {**observed, **modeled} + known = {k: np.ones(n, dtype=bool) for k in canonical} + qresult = qbi.model_full_puf_qbi( + canonical, + status["RECID"], + known=known, + input_money_year=2015, + seed=seed, + employment_calibration=qbi_employment_calibration, + ) + canonical.update(qresult.columns) + _require(set(canonical) == set(growth.OUTPUTS), "PUF59_CANONICAL_COMPLETE_ROSTER") + known = {k: np.ones(n, dtype=bool) for k in canonical} + result = growth.grow_puf_2015_to_2024( + canonical, known=known, input_money_year=2015, scheme=growth_scheme + ) + weight = status["S006"].astype(np.float64) / 100 + _require( + bool((weight >= 0).all() and np.isfinite(weight).all() and weight.sum() > 0), + "PUF59_WEIGHT_DOMAIN", + ) + predictors = { + k: np.frombuffer(np.asarray(aux[k], dtype=np.int64).tobytes(), dtype=np.int64) + for k in ("puf_2015_filing_status_code", "puf_2015_capped_return_size") + } + prefix_arrays = { + "RECID": status["RECID"], + "weight": weight, + "puf_person_incidence_capacity": np.ones(n, dtype=np.int64), + **predictors, + } + receipt = { + "prefix_values_sha256": prefix_values_digest(prefix_arrays), + "schema": VERSION, + "source_definition_sha256": source.definition_sha256, + "source_sha256": dict(source.source_sha256), + "source_statistical_year": 2015, + "input_money_year": 2015, + "output_money_year": 2024, + "rows": n, + "model_assumptions": model_receipt, + "qbi": dict(qresult.receipt), + "growth": dict(result.receipt), + "return_size": "Source PUF2015 coarsened filing-status plus capped dependent count; no physical person claim.", + "person_incidence_capacity": "Constant1 for zero/one modeled return incidence; excluded from conditioning predictors.", + "weight": "Source S006 exact hundredths divided once by100; no weight growth.", + "recipient_conditioning_self_employment": "Must sum base and SSTB ScheduleC outcomes for the total source predictor.", + "source_observation_status": "33 direct/derived baseline columns; 11 explicit baseline models; 15 added QBI leaves and one replacement; all monetary transport is modeled.", + "mortgage_structure_generated": False, + "release_eligible": False, + } + receipt["sha256"] = hashlib.sha256( + json.dumps( + receipt, sort_keys=True, separators=(",", ":"), allow_nan=False + ).encode() + ).hexdigest() + + def readonly(a): + return np.frombuffer(a.tobytes(), dtype=a.dtype) + + return CanonicalPuf59Result( + result.columns, + MappingProxyType(predictors), + readonly(status["RECID"].astype(np.int64)), + readonly(weight), + readonly(np.ones(n, dtype=np.int64)), + 2024, + qresult.calibration, + MappingProxyType(receipt), + ) diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/puf59_canonical_artifact.py b/packages/microcosm-build/src/microcosm/build/us_runtime/puf59_canonical_artifact.py new file mode 100644 index 000000000..d0b5566c8 --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/puf59_canonical_artifact.py @@ -0,0 +1,237 @@ +"""Bounded deterministic private canonical59 array envelope; no source admission.""" + +from __future__ import annotations + +import hashlib +import json +import struct +from collections.abc import Mapping + +import numpy as np + +from .puf59_canonical import PREFIX_NAMES, VERSION, prefix_values_digest +from .puf_target2024_growth import INCIDENCE_FIELDS, OUTPUTS, RECIPE_SHA256, _digest + +MAGIC = b"MCPUF59\x02" +MAX_HEADER = 128 * 1024 +MAX_BODY = 128 * 1024 * 1024 +PREFIX = PREFIX_NAMES +NAMES = (*PREFIX, *OUTPUTS) +INTEGERS = set(PREFIX) - {"weight"} | set(INCIDENCE_FIELDS) + + +def _require(value, code): + if not value: + raise ValueError(code) + + +def _receipt(receipt, expected_growth_scheme): + _require( + expected_growth_scheme in ("family_observed", "cpi_only"), + "PUF59_ARTIFACT_EXPECTED_SCHEME", + ) + _require(isinstance(receipt, Mapping), "PUF59_ARTIFACT_RECEIPT_TYPE") + r = dict(receipt) + _require( + r.get("schema") == VERSION + and type(r.get("input_money_year")) is int + and r["input_money_year"] == 2015 + and type(r.get("output_money_year")) is int + and r["output_money_year"] == 2024 + and type(r.get("rows")) is int + and 0 < r["rows"] <= MAX_BODY // (len(NAMES) * 8) + and r.get("release_eligible") is False, + "PUF59_ARTIFACT_RECEIPT_CONTRACT", + ) + gr = r.get("growth") + _require( + isinstance(gr, Mapping) and gr.get("scheme") == expected_growth_scheme, + "PUF59_ARTIFACT_GROWTH_SCHEME", + ) + _require(gr.get("recipe_sha256") == RECIPE_SHA256, "PUF59_ARTIFACT_GROWTH_RECIPE") + h = r.pop("sha256", None) + _require( + h + == hashlib.sha256( + json.dumps( + r, sort_keys=True, separators=(",", ":"), allow_nan=False + ).encode() + ).hexdigest(), + "PUF59_ARTIFACT_RECEIPT_HASH", + ) + r["sha256"] = h + return r + + +def _validate(arrays, n): + _require( + type(n) is int and 0 < n <= MAX_BODY // (len(NAMES) * 8), "PUF59_ARTIFACT_ROWS" + ) + _require( + isinstance(arrays, Mapping) and set(arrays) == set(NAMES), + "PUF59_ARTIFACT_COLUMNS", + ) + for name in NAMES: + a = np.asarray(arrays[name]) + kind = "iu" if name in INTEGERS else "fiu" + _require( + a.shape == (n,) and a.dtype.kind in kind, + "PUF59_ARTIFACT_COLUMN_TYPE:" + name, + ) + _require(bool(np.isfinite(a).all()), "PUF59_ARTIFACT_NONFINITE:" + name) + if a.dtype.kind in "iu": + _require( + bool(((a >= -(2**53)) & (a <= 2**53)).all()), + "PUF59_ARTIFACT_INTEGER_RANGE:" + name, + ) + ids = arrays["RECID"] + w = arrays["weight"] + _require( + bool((ids > 0).all()) + and len(np.unique(ids)) == n + and not np.isin(ids, [999996, 999997, 999998, 999999]).any(), + "PUF59_ARTIFACT_ID_DOMAIN", + ) + _require( + bool((w >= 0).all()) and np.isfinite(w.sum()) and w.sum() > 0, + "PUF59_ARTIFACT_WEIGHT", + ) + _require( + bool((arrays["puf_person_incidence_capacity"] == 1).all()), + "PUF59_ARTIFACT_CAPACITY", + ) + _require( + bool(np.isin(arrays["puf_2015_filing_status_code"], [1, 2, 3, 4]).all()), + "PUF59_ARTIFACT_STATUS", + ) + size = arrays["puf_2015_capped_return_size"] + _require(bool(((size >= 1) & (size <= 5)).all()), "PUF59_ARTIFACT_RETURN_SIZE") + for name in INCIDENCE_FIELDS: + _require(bool(np.isin(arrays[name], [0, 1]).all()), "PUF59_ARTIFACT_INCIDENCE") + + +def _bindings(arrays, receipt): + _require( + prefix_values_digest(arrays) == receipt.get("prefix_values_sha256"), + "PUF59_ARTIFACT_PREFIX_BINDING", + ) + _require( + _digest(arrays) == receipt["growth"].get("output_values_sha256"), + "PUF59_ARTIFACT_GROWTH_BINDING", + ) + + +def _encode(arrays, receipt, expected_growth_scheme): + _require( + isinstance(arrays, Mapping) and set(arrays) == set(NAMES), + "PUF59_ARTIFACT_COLUMNS", + ) + n = len(arrays["RECID"]) + _validate(arrays, n) + r = _receipt(receipt, expected_growth_scheme) + _bindings(arrays, r) + _require(r["rows"] == n, "PUF59_ARTIFACT_RECEIPT_ROWS") + blocks = [ + np.asarray(arrays[name], dtype=" len(MAGIC) + 4 + length, + "PUF59_ARTIFACT_HEADER_SIZE", + ) + start = len(MAGIC) + 4 + + def pairs(items): + d = {} + for k, v in items: + _require(k not in d, "PUF59_ARTIFACT_DUPLICATE_JSON_KEY") + d[k] = v + return d + + h = json.loads( + payload[start : start + length], + object_pairs_hook=pairs, + parse_constant=lambda _: (_ for _ in ()).throw( + ValueError("PUF59_ARTIFACT_JSON_NONFINITE") + ), + ) + _require( + type(h) is dict + and set(h) == {"schema", "rows", "names", "dtypes", "body_sha256", "receipt"}, + "PUF59_ARTIFACT_HEADER_CONTRACT", + ) + _require( + json.dumps(h, sort_keys=True, separators=(",", ":"), allow_nan=False).encode( + "utf-8" + ) + == payload[start : start + length], + "PUF59_ARTIFACT_HEADER_CANONICAL", + ) + _require( + h["schema"] == "microcosm.us.puf59_canonical_artifact/2" + and h["names"] == list(NAMES) + and h["dtypes"] == [" None: + """Refuse using a stable reason, without reporting source row values.""" + if not condition: + raise ValueError(reason) + + +def fixture_sources(definition_text: str, projection_text: str): + """Validate actual fixture definitions; packaged source pins cannot enter.""" + # Source documents and fit packets use distinct JSON canonical conventions. Validate each document against its own producer's exact bytes. + definition = raw.fixture_definition(json.loads(definition_text)) + projection = agi.fixture_agi_projection_document( + json.loads(projection_text), definition + ) + require( + definition.params_text == definition_text + and projection.params_text == projection_text, + "FIXTURE_SOURCE_CANONICAL", + ) + require(tuple(sorted(projection.fields)) == MONEY_FIELDS, "FIXTURE_MONEY_FIELDS") + require( + all(c.fit_admission == SOURCE_ADMISSION for c in projection.columns), + "SOURCE_ADMISSION_CHANGED", + ) + return definition, projection + + +def decode_price_arrays( + payload: bytes, *, expected: Mapping, expected_sha256: str, max_rows=MAX_ROWS +): + """Read the exact reviewed local helper envelope, with external identity pins. + + This parser does not create a GrownPufTable, repeat growth, or admit a fit. + The returned arrays are read-only views of the authenticated body bytes. + """ + require(type(max_rows) is int and 0 < max_rows <= 250_000, "PRICE_ROW_BOUND") + require( + isinstance(payload, bytes) + and len(payload) <= MAX_BYTES + and payload.startswith(MAGIC), + "PRICE_ENVELOPE", + ) + require( + codec._hash(expected_sha256) and codec.sha(payload) == expected_sha256, + "PRICE_PAYLOAD_SHA", + ) + offset = len(MAGIC) + require(len(payload) >= offset + 8, "PRICE_HEADER_LENGTH") + size = int.from_bytes(payload[offset : offset + 8], "big") + start = offset + 8 + require(0 < size <= 65536 and len(payload) >= start + size, "PRICE_HEADER_BOUND") + header = codec.decode_json(payload[start : start + size]) + require( + set(expected) == HEADER_FIELDS and set(header) == HEADER_FIELDS | {"columns"}, + "PRICE_HEADER_FIELDS", + ) + require({k: header[k] for k in HEADER_FIELDS} == dict(expected), "PRICE_IDENTITY") + require( + header["schema"] == "local.puf.price_restatement.arrays.v1" + and header["release_eligible"] is False, + "PRICE_SCOPE", + ) + for field in ("input_binding_sha256", "contract_sha256"): + require(codec._hash(header[field]), "PRICE_HEADER_SHA") + for field in ("source_head", "helper_head"): + require( + isinstance(header[field], str) + and len(header[field]) == 40 + and all(c in "0123456789abcdef" for c in header[field]), + "PRICE_CODE_ID", + ) + columns = header["columns"] + require( + isinstance(columns, dict) and set(columns) == set(ARRAY_DTYPES), + "PRICE_COLUMN_ROSTER", + ) + cursor, rows, result = start + size, None, {} + for name in sorted(ARRAY_DTYPES): + entry = columns[name] + require( + isinstance(entry, dict) and set(entry) == {"dtype", "rows", "sha256"}, + "PRICE_COLUMN_FIELDS", + ) + require( + entry["dtype"] == ARRAY_DTYPES[name] + and type(entry["rows"]) is int + and 0 < entry["rows"] <= max_rows, + "PRICE_COLUMN_TYPE_ROWS", + ) + rows = entry["rows"] if rows is None else rows + require(entry["rows"] == rows, "PRICE_ROW_ALIGNMENT") + end = cursor + rows * np.dtype(entry["dtype"]).itemsize + require(end <= len(payload), "PRICE_BODY_TRUNCATED") + body = payload[cursor:end] + require(codec.sha(body) == entry["sha256"], "PRICE_BODY_SHA") + values = np.frombuffer(body, dtype=entry["dtype"]) + require(np.isfinite(values).all(), "PRICE_NONFINITE") + result[name] = values + cursor = end + require(cursor == len(payload), "PRICE_TRAILING_BYTES") + return result + + +def donor_frame(arrays: Mapping, status, decoded, projection) -> Frame: + """One computational person/return wrapper, with explicit raw S006 DESIGN weights.""" + require(projection.route == "test_fixture", "GENUINE_FIT_NOT_ADMITTED") + return _donor_frame(arrays, status, decoded, projection, scope=SCOPE) + + +def _donor_frame(arrays, status, decoded, projection, *, scope): + """Shared mechanical checks; the entry point must qualify its source/recipe.""" + known = np.asarray(decoded.amount_known, dtype=bool) + require( + np.array_equal(status.typed["RECID"], decoded.typed["RECID"]), + "DONOR_STATUS_JOIN", + ) + require( + np.array_equal(status.typed["disclosure_aggregate"], ~known), "DONOR_UNIVERSE" + ) + ids = status.typed["RECID"][known] + require( + len(ids) == len(np.unique(ids)) + and np.array_equal(ids, arrays["RECID"]) + and np.array_equal(ids, arrays[growth.PROVENANCE_RECID_COLUMN]), + "DONOR_RECID_ORDER", + ) + aggregate_ids = status.typed["RECID"][~known] + require( + len(aggregate_ids) == 4 + and len(np.unique(aggregate_ids)) == 4 + and set(aggregate_ids) == set(raw.PUF_AGGREGATE_RECIDS), + "DONOR_AGGREGATE_ROSTER", + ) + for field, delivered in ( + ("S006", "S006_delivered_int64"), + ("FLPDYR", "FLPDYR_delivered_int16"), + ("demographic_status", "demographic_status_delivered_int8"), + ): + original = status.typed[field][known] + require( + np.array_equal(original, arrays[delivered]) + and np.array_equal(original, arrays[field]), + "DONOR_CARRIED_STATUS", + ) + require( + np.array_equal( + decoded.typed["E00100"][known], arrays[growth.PROVENANCE_SOURCE_AGI_COLUMN] + ), + "DONOR_SOURCE_AGI", + ) + weights = arrays["S006_delivered_int64"].astype("float64") + require( + np.all(arrays["S006_delivered_int64"] >= 0) + and np.all(arrays["S006_delivered_int64"] <= 2**53) + and np.array_equal(weights.astype("int64"), arrays["S006_delivered_int64"]) + and weights.sum() > 0, + "DONOR_DESIGN_WEIGHTS", + ) + wage = arrays["E00200"] + require(np.isfinite(wage).all() and np.all(wage >= 0), "DONOR_WAGE_FEATURE") + mars = status.typed["MARS"][known] + require(np.isin(mars, (1, 2, 3, 4)).all(), "DONOR_MARS") + table = pd.DataFrame( + { + "tax_unit_id": ids.copy(), + FEATURES[0]: wage.copy(), + **{f"mars_{i}": (mars == i).astype("float64") for i in range(1, 5)}, + TARGET: arrays[TARGET].copy(), + } + ) + return Frame( + { + "tax_unit": table, + "person": pd.DataFrame( + {"person_id": ids.copy(), "person_tax_unit_id": ids.copy()} + ), + }, + EntitySchema(group_entities=("tax_unit",)), + {"tax_unit": Weights(weights, WeightKind.DESIGN)}, + metadata={ + "scope": scope, + "source_projection_sha256": projection.sha256, + "source_projection_document": projection.canonical.decode("utf-8"), + "carrier": "one technical person per return; not a taxpayer count", + "weight_units": "delivered_S006_integer_hundredths", + }, + ) + + +def population_content(frame: Frame) -> str: + """Full materialized cells/axes/effective weights/strata; no metadata shortcuts.""" + return origin._population_content(frame) + + +def recipient_matrix(frame: Frame, *, role_column=None, person_wages=None): + """Use every detail unit, including zero-weight rows, and actual filing roles.""" + require(frame.schema == US_SCHEMA, "HOST_SCHEMA") + # Build a manifest from the current native roster only for validation. The + # separate fixture source edge binds that roster's authority and full bytes. + tables = {e: frame.table(e) for e in frame.entities} + for entity, table in tables.items(): + role = table[provenance.support_clone_index_column(entity)] + require( + role.dtype == np.dtype("int64") and role.isin((0, 1)).all(), + "HOST_CLONE_TYPE", + ) + manifest = provenance.spine_assembly_manifest( + { + e: t.loc[t[provenance.support_clone_index_column(e)].eq(0)] + for e, t in tables.items() + }, + channels=("acs", "asec"), + ) + checked = Frame( + tables, + frame.schema, + {e: frame.weights_for(e) for e in frame.weighted_entities}, + frame.strata, + metadata=manifest, + ) + provenance.validate_assembly_provenance(checked, boundary="invented_puf_detail") + person, units = tables["person"], tables["tax_unit"] + if person_wages is not None: + require( + type(person_wages) is pd.Series + and person_wages.dtype == np.dtype("float64") + and person_wages.index.dtype == np.dtype("int64") + and person_wages.index.is_unique + and np.array_equal(person_wages.index.to_numpy(), person.person_id), + "HOST_WAGE_ROW_AXIS", + ) + clone_role = person[provenance.support_clone_index_column("person")] + wage_values = person_wages.to_numpy() + require( + np.array_equal( + wage_values[clone_role.eq(0)].view("uint64"), + wage_values[clone_role.eq(1)].view("uint64"), + ), + "HOST_WAGE_PAIR_BYTES", + ) + pairs = {} + for entity, table in tables.items(): + src, role = ( + provenance.support_source_id_column(entity), + provenance.support_clone_index_column(entity), + ) + require(table[src].dtype == np.dtype("int64"), "HOST_SOURCE_ID_TYPE") + require(not table.duplicated([src, role]).any(), "HOST_ROLE_PAIR_DUPLICATE") + native, detail = table.loc[table[role].eq(0)], table.loc[table[role].eq(1)] + require( + len(native) == len(detail) and np.array_equal(native[src], detail[src]), + "HOST_ROLE_PAIR_COVERAGE", + ) + structural = {US_SCHEMA.entity_id_column(entity), role} + if entity == "person": + structural.update( + US_SCHEMA.membership_column(e) for e in US_SCHEMA.group_entities + ) + for name in table.columns: + if name not in structural: + require( + native[name] + .reset_index(drop=True) + .equals(detail[name].reset_index(drop=True)), + "HOST_COPIED_SOURCE_CELL", + ) + weights = frame.resolve_weights(entity).values + require( + np.array_equal( + weights[table[role].eq(0)].view("uint64"), + weights[table[role].eq(1)].view("uint64"), + ), + "HOST_PAIR_WEIGHTS", + ) + pairs[entity] = dict( + zip( + table[US_SCHEMA.entity_id_column(entity)], + zip(table[src], table[role], strict=True), + strict=True, + ) + ) + for group in US_SCHEMA.group_entities: + member = US_SCHEMA.membership_column(group) + for pid, gid in zip(person.person_id, person[member], strict=True): + require(pairs["person"][pid][1] == pairs[group][gid][1], "HOST_MEMBER_ROLE") + by_source = {} + for pid, gid in zip(person.person_id, person[member], strict=True): + source, role = pairs["person"][pid] + group_source = pairs[group][gid][0] + if role == 0: + by_source[source] = group_source + else: + require(by_source.get(source) == group_source, "HOST_CLONE_MEMBERSHIP") + require( + set(units[provenance.support_channel_column("tax_unit")]) == {"acs", "asec"}, + "HOST_BOTH_ARMS", + ) + mask = units[provenance.support_clone_index_column("tax_unit")].eq(1).to_numpy() + selected = units.loc[mask] + values = [] + for row in selected.itertuples(index=False): + members = person.loc[person.person_tax_unit_id.eq(row.tax_unit_id)] + status = row.filing_status_input + require(status in MARS, "HOST_FILING_STATUS") + if role_column is None: + for name in ( + "is_tax_unit_head", + "is_tax_unit_spouse", + "is_tax_unit_dependent", + ): + require( + name in members + and members[name].dtype == np.dtype("bool") + and not members[name].isna().any(), + "HOST_FILING_ROLE", + ) + head, spouse, dependent = ( + members[name].to_numpy() + for name in ( + "is_tax_unit_head", + "is_tax_unit_spouse", + "is_tax_unit_dependent", + ) + ) + else: + require( + role_column == "tax_unit_role_input" and role_column in members, + "HOST_ROLE_INPUT_COLUMN", + ) + roles = members[role_column] + require( + pd.api.types.is_string_dtype(roles.dtype) + and not roles.isna().any() + and roles.isin(("HEAD", "SPOUSE", "DEPENDENT")).all(), + "HOST_ROLE_INPUT_VALUES", + ) + head, spouse, dependent = ( + roles.eq(name).to_numpy(dtype=bool) + for name in ("HEAD", "SPOUSE", "DEPENDENT") + ) + require( + np.all(head.astype(int) + spouse.astype(int) + dependent.astype(int) == 1) + and head.sum() == 1 + and spouse.sum() == int(status == "JOINT"), + "HOST_FILING_STRUCTURE", + ) + channel = getattr(row, provenance.support_channel_column("tax_unit")) + name = ( + "asec_reported_wage_income_2024_price" + if channel == "asec" + else "employment_income_before_lsr" + ) + earners = head | spouse + if person_wages is None: + require(name in members, "HOST_MISSING_FEATURE") + wage = members.loc[earners, name] + else: + wage = person_wages.loc[members.loc[earners, "person_id"]] + require( + pd.api.types.is_numeric_dtype(wage) + and not pd.api.types.is_bool_dtype(wage) + and not pd.api.types.is_complex_dtype(wage) + and not wage.isna().any(), + "HOST_MISSING_FEATURE", + ) + a = wage.to_numpy(dtype="float64") + require(np.isfinite(a).all() and (a >= 0).all(), "HOST_NONFINITE_FEATURE") + amount = float(a.sum()) + require(np.isfinite(amount), "HOST_FEATURE_OVERFLOW") + values.append([amount, *(float(MARS[status] == i) for i in range(1, 5))]) + ids = selected.tax_unit_id.to_numpy(copy=True) + features = pd.DataFrame( + values, index=pd.Index(ids), columns=FEATURES, dtype="float64" + ) + return encode_recipient_matrix(features, entity="tax_unit", entity_ids=ids), mask diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/puf_diagnostic_consumer.py b/packages/microcosm-build/src/microcosm/build/us_runtime/puf_diagnostic_consumer.py new file mode 100644 index 000000000..79b34e697 --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/puf_diagnostic_consumer.py @@ -0,0 +1,847 @@ +"""Qualified transport for one development-only PUF conditional diagnostic. + +Callers supply independently reviewed content pins. Receipt labels or fixture +definitions cannot replace those pins. No function opens a file or fits a model. +""" + +from __future__ import annotations + +import json +from collections.abc import Mapping + +import numpy as np +import pandas as pd + +from microcosm.fit import _graph_legacy_qrf as codec +from microcosm.fit import model_input + +from . import asec_current_money as current_money +from . import graph_asec_income as asec_income +from . import native_household_origin as origin +from . import puf_detail_transfer as detail +from . import puf_monetary_agi_projection as agi +from . import puf_raw_source as raw +from . import support_provenance as provenance +from .graph_context import _row_identity + +require = detail.require +RECIPE_ID = "us.puf.schedule_c_wage_mars_diagnostic.v1" +SCOPE = "development_conditional_support_not_tax_inputs" +SOURCE_HEAD = "c5c34a048fc38c01840b2da5454385e0a51d1c49" +HELPER_HEAD = "dea036278cc3f7c29c99da67bcadd968822c1d7a" +MAX_DONORS = 250_000 +ROLES = ( + "acceptance", + "proposal", + "config", + "completion", + "cold", + "required", + "definition", + "projection_definition", + "status", + "projection", + "arrays", +) + + +def recipe_document(): + """A diagnostic modeling choice, never a modification of source admission.""" + return { + "id": RECIPE_ID, + "scope": SCOPE, + "target": detail.TARGET, + "features": list(detail.FEATURES), + "output": detail.OUTPUT, + "trees": 2, + "seed": 578, + "zero_atol": 0.0, + "donor_weight": "delivered_S006_integer_hundredths_DESIGN", + "wage_proxy": "head_and_actual_JOINT_spouse_only_dependents_excluded", + "host_role_source": "microunit_tax_unit_role_input", + "filing_mapping": detail.MARS, + "missing_wage": "refuse_selected_earner_no_age_only_zero", + "prices": "2015_file_CPI_U_to_2024_development_prices", + "source_period": "unresolved_raw_FLPDYR", + "source_fit_admission": detail.SOURCE_ADMISSION, + "population_meaning": "potential_filing_unit_support_not_observed_filers", + "tax_input_admission": False, + "scientific_launch_admission": False, + "release_eligible": False, + } + + +def parse_document(payload): + """Strict JSON, allowing the reviewed helper's formatted receipt encoding.""" + require( + type(payload) is bytes and 0 < len(payload) <= 16_000_000, + "DIAGNOSTIC_DOCUMENT_BOUND", + ) + + def pairs(items): + result = {} + for key, value in items: + require(key not in result, "DIAGNOSTIC_DUPLICATE_KEY") + result[key] = value + return result + + def bad(_): + raise ValueError("DIAGNOSTIC_NONFINITE_JSON") + + value = json.loads(payload, object_pairs_hook=pairs, parse_constant=bad) + require(isinstance(value, dict), "DIAGNOSTIC_DOCUMENT_TYPE") + return value + + +def content_pin(payload): + return {"bytes": len(payload), "sha256": codec.sha(payload)} + + +def check_pin(payload, pin, reason): + require( + isinstance(pin, Mapping) + and type(pin.get("bytes")) is int + and codec._hash(pin.get("sha256")) + and content_pin(payload) == {k: pin[k] for k in ("bytes", "sha256")}, + reason, + ) + + +def qualify_price_transport(payloads, expected_pins, *, route, recipe): + """Validate DEA completion → config/source → grown-array transport. + + Expected pins are execution inputs established outside this parser. The + production route accepts only the packaged original source definitions; + tests use a visibly different acceptance/mode with actual fixture codecs. + """ + require( + codec.encode_json(recipe) == codec.encode_json(recipe_document()), + "DIAGNOSTIC_RECIPE", + ) + require(route in ("packaged", "test_fixture"), "DIAGNOSTIC_SOURCE_ROUTE") + require(set(payloads) == set(expected_pins) == set(ROLES), "PRICE_TRANSPORT_ROSTER") + for name in ROLES: + check_pin(payloads[name], expected_pins[name], "PRICE_TRANSPORT_PIN:" + name) + documents = {n: parse_document(payloads[n]) for n in ROLES[:6]} + acceptance, proposal, config, completed, cold, warm = ( + documents[n] for n in ROLES[:6] + ) + genuine = route == "packaged" + require( + acceptance["status"] + == ( + "ACCEPTED_GENUINE_DEVELOPMENTAL_PRICE_RESTATEMENT" + if genuine + else "ACCEPTED_INVENTED_DEVELOPMENTAL_PRICE_RESTATEMENT" + ) + and acceptance["release_eligible"] is False + and acceptance["fit_admission"] == detail.SOURCE_ADMISSION + and config["mode"] == ("genuine" if genuine else "bound_invented") + and completed["genuine_input_bound"] is genuine + and completed["status"] + == ( + "GENUINE_DEVELOPMENTAL_PRICE_RESTATEMENT_COMPLETED" + if genuine + else "INVENTED_PRICE_RESTATEMENT_PREPARATION_PASSED" + ), + "PRICE_TRANSPORT_AUTHORITY_SCOPE", + ) + for name in ( + "config", + "completion", + "cold", + "required", + "arrays", + "status", + "projection", + "definition", + "projection_definition", + ): + require( + any( + content_pin(payloads[name]) == {k: p[k] for k in ("bytes", "sha256")} + for p in acceptance["saved_file_pins_checked"] + ), + "PRICE_ACCEPTANCE_PIN:" + name, + ) + check_pin( + payloads["proposal"], acceptance["reviewed_proposal"], "PRICE_PROPOSAL_PIN" + ) + for name, receipt in (("cold", cold), ("required", warm)): + require( + completed[name]["sha256"] == codec.sha(payloads[name]), "PRICE_CHILD_PIN" + ) + require(receipt["phase"] == name, "PRICE_CHILD_PHASE") + check_pin(payloads["config"], completed["config_pin"], "PRICE_CONFIG_PIN") + require( + {k: v for k, v in config.items() if k != "execution"} == proposal["config"] + and proposal["price_execution_authorized"] is False + and proposal["release_eligible"] is False, + "PRICE_REVIEWED_CONFIGURATION", + ) + require( + config["inputs"] == cold["input_binding"] == warm["input_binding"], + "PRICE_SOURCE_INPUTS", + ) + require( + set(config["inputs"]) + == {"definition", "projection_definition", "status", "projection"}, + "PRICE_SOURCE_ROSTER", + ) + for name, pin in config["inputs"].items(): + check_pin(payloads[name], pin, "PRICE_SOURCE_PIN:" + name) + source = config["bound_source"] + require( + source["route"] == route and source["invented"] is (not genuine), + "PRICE_BOUND_ROUTE", + ) + source_sha = codec.sha(codec.encode_json(source)) + require( + source_sha + == config["binding_sha256"] + == cold["bound_source_identity"] + == warm["bound_source_identity"] + == completed["bound_source_identity"], + "PRICE_BOUND_SOURCE_SHA", + ) + runtime_keys = ( + "source_head", + "helper_head", + "python", + "executable", + "executable_sha256", + "versions", + ) + runtime = {k: cold["runtime"][k] for k in runtime_keys} + require( + all( + {k: item["runtime"][k] for k in runtime_keys} == runtime + for item in (warm, completed) + ), + "PRICE_RUNTIME_BINDING", + ) + require( + runtime["source_head"] == SOURCE_HEAD + and runtime["helper_head"] == HELPER_HEAD + and runtime["executable_sha256"] + == "bf4ea26231212e330b8e9b6f24969a30f894fec88665bf42754070d114ac7fc6" + and runtime["versions"] == {"numpy": "2.4.6", "pandas": "3.0.3"}, + "PRICE_REVIEWED_RUNTIME", + ) + for item in (cold, warm, completed): + isolation = item["runtime"]["bytecode_isolation"] + require( + isolation["isolated"] is True + and isolation["dont_write_bytecode"] is True + and isolation["absent_absolute_prefix"].startswith("/"), + "PRICE_BYTECODE_ISOLATION", + ) + require( + cold["runtime"]["bytecode_isolation"]["absent_absolute_prefix"] + != warm["runtime"]["bytecode_isolation"]["absent_absolute_prefix"], + "PRICE_FRESH_PROCESS_PREFIX", + ) + require( + cold["pid"] != warm["pid"] + and warm["all_hits"] + and warm["kernel_calls"] == [] + and warm["raising_replay_stubs"] is True + and all(v == 0 for v in warm["transform_calls"].values()), + "PRICE_REQUIRED_PROOF", + ) + for name in ( + "contract", + "array_identity", + "graph_artifacts", + "grown_artifact", + "excluded_lexical_artifact", + ): + require(cold[name] == warm[name], "PRICE_COLD_WARM_IDENTITY:" + name) + contract = cold["contract"] + require(source["contract"] == contract, "PRICE_SOURCE_CONTRACT") + require( + source["runtime"]["executable_sha256"] == runtime["executable_sha256"] + and source["runtime"]["package_versions"] == runtime["versions"], + "PRICE_SOURCE_RUNTIME", + ) + require( + contract["ratio"] == [313689, 237017] + and contract["ratio_bits"] == "0aa9e810012df53f" + and contract["release_eligible"] is False + and contract["per_record_preliminary_inflation"] is False + and contract["nominal_income_growth_claim"] is False, + "PRICE_BASIS", + ) + require( + acceptance["source_head"] == SOURCE_HEAD + and acceptance["helper_head"] == HELPER_HEAD + and config["helper_head"] == HELPER_HEAD, + "PRICE_REVIEWED_CODE", + ) + if genuine: + definition, projection = ( + raw.packaged_definition(), + agi.packaged_agi_projection(), + ) + else: + definition = raw.fixture_definition(parse_document(payloads["definition"])) + projection = agi.fixture_agi_projection_document( + parse_document(payloads["projection_definition"]), definition + ) + # Source files may use formatted JSON; their exact input pins were checked above. + require( + parse_document(payloads["definition"]) + == raw.definition_document_json(definition), + "PRICE_DEFINITION", + ) + require( + parse_document(payloads["projection_definition"]) + == json.loads(projection.canonical), + "PRICE_PROJECTION_DEFINITION", + ) + require( + cold["source_definition_sha256"] == definition.sha256 + and cold["projection_definition_sha256"] == projection.sha256, + "PRICE_DEFINITION_IDENTITY", + ) + require( + source["source"]["head"] == SOURCE_HEAD + and source["source"]["definition_canonical_sha256"] == definition.sha256 + and source["source"]["projection_canonical_sha256"] == projection.sha256, + "PRICE_DECLARED_SOURCE_IDENTITY", + ) + metadata = source["source_column_metadata"] + require( + set(metadata) == set(detail.MONEY_FIELDS) + and codec.sha(codec.encode_json(metadata)) == source["source_basis_sha256"] + and cold["source_column_metadata_preserved"] + == warm["source_column_metadata_preserved"] + == metadata, + "PRICE_SOURCE_METADATA", + ) + for value in metadata.values(): + require( + value["fit_admission"] == detail.SOURCE_ADMISSION + and value["period_semantics"] == "unresolved_raw_FLPDYR" + and value["amount_units"] == "whole_usd_source_year" + and value["grain"] == "source_return" + and value["projection_sha256"] == projection.sha256, + "PRICE_SOURCE_MEANING", + ) + rows = config["ordinary_rows"] + require( + type(rows) is int + and 0 < rows <= MAX_DONORS + and rows + == source["ordinary_rows"] + == acceptance["ordinary_rows"] + == completed["ordinary_rows"] + and acceptance["excluded_aggregate_rows"] == completed["aggregate_rows"] == 4, + "PRICE_ORDINARY_ROSTER", + ) + check_pin(payloads["arrays"], cold["grown_artifact"], "PRICE_EXPORT_PIN") + header = { + "schema": "local.puf.price_restatement.arrays.v1", + "input_binding_sha256": codec.sha(codec.encode_json(config["inputs"])), + "contract_sha256": contract["sha256"], + "source_head": SOURCE_HEAD, + "helper_head": HELPER_HEAD, + "release_eligible": False, + } + arrays = detail.decode_price_arrays( + payloads["arrays"], + expected=header, + expected_sha256=codec.sha(payloads["arrays"]), + max_rows=rows, + ) + require(len(arrays["RECID"]) == rows, "PRICE_EXACT_ROWS") + for name, values in arrays.items(): + require( + cold["array_identity"][name] + == { + "dtype": values.dtype.str, + "rows": len(values), + "sha256": codec.sha(values.tobytes()), + }, + "PRICE_ARRAY_RECEIPT", + ) + status = raw.decode_return_status(payloads["status"], definition) + decoded = agi.decode_agi_projection( + payloads["projection"], + projection, + expected_status_artifact_sha256=codec.sha(payloads["status"]), + ) + require( + metadata == agi._thawed(decoded.column_metadata), + "PRICE_ORIGINAL_METADATA_BINDING", + ) + frame = detail._donor_frame(arrays, status, decoded, projection, scope=SCOPE) + binding = { + "schema": "microcosm.us.puf_diagnostic_price_transport.v1", + "route": route, + "recipe": recipe, + "input_pins": {n: content_pin(payloads[n]) for n in ROLES}, + "source_column_metadata": metadata, + "source_projection_document": projection.canonical.decode(), + "original_source_admission": detail.SOURCE_ADMISSION, + "contract": contract, + "source_binding_sha256": source_sha, + "ordinary_rows": rows, + "donor_model_input_sha256": donor_model_input_sha256(frame), + "release_eligible": False, + } + return frame, codec.encode_json(binding) + + +def donor_model_input_sha256(frame): + """Exact donor IDs/features/target and DESIGN resampling weights.""" + table = frame.table("tax_unit").loc[:, [*detail.FEATURES, detail.TARGET]].copy() + table.index = pd.Index( + frame.table("tax_unit").tax_unit_id.to_numpy(dtype="int64"), name="tax_unit_id" + ) + payload = model_input.encode_recipient_matrix( + table.loc[:, [*detail.FEATURES, detail.TARGET]], + entity="tax_unit", + entity_ids=table.index.to_numpy(dtype="int64"), + ) + weight = frame.weights_for("tax_unit") + return codec.sha( + codec.encode_json( + { + "matrix_sha256": codec.sha(payload), + "weight_kind": weight.kind.value, + "weight_sha256": codec.sha(weight.values.astype("= 0) & (age <= 99) & (age == np.floor(age))).all() + and np.array_equal(age, _numeric(person.age, "ACS_WAGE_AGE")), + "ACS_WAGE_AGE", + ) + require(person.source_year.astype(str).eq("2024").all(), "ACS_WAGE_COHORT") + wage = _numeric(person.WAGP, "ACS_WAGE_RAW_TYPE", nullable=True) + adjustment = _numeric(person.ADJINC, "ACS_WAGE_ADJINC_TYPE", nullable=True) + carried = _numeric( + person.employment_income_before_lsr, "ACS_WAGE_CARRIED_TYPE", nullable=True + ) + observed = ~np.isnan(wage) + require( + np.isfinite(wage[observed]).all() + and (wage[observed] >= 0).all() + and np.isfinite(adjustment[observed]).all() + and (adjustment[observed] > 0).all(), + "ACS_WAGE_DOMAIN", + ) + require(not (observed & (age < 15)).any(), "ACS_WAGE_OUTSIDE_UNIVERSE_OBSERVED") + expected = wage * (adjustment / 1_000_000.0) + require( + np.array_equal(np.isnan(carried), ~observed) + and np.array_equal( + expected[observed].view("uint64"), carried[observed].view("uint64") + ), + "ACS_WAGE_ADJUSTED_BINDING", + ) + return { + "ids": person.person_id.to_numpy(dtype="int64").tolist(), + "age_sha256": codec.sha(age.astype("= 15).tolist(), + "missing_treatment": "preserved_no_zero_substitution", + "arithmetic": "WAGP*(ADJINC/1000000)", + "source_period_equivalence_claim": False, + } + + +def asec_wage_evidence(person, values): + """Reconstruct the accepted income artifact, then join its native person axis. + + ``values`` are executor-checked typed artifacts. H and its T1 descendant + have legitimate differing source ancestry; the accepted readers bind the + prepared receipt, current-money buffers, cohort and exact income payload. + """ + from . import graph_composed_asec_binding as asec_binding + from . import graph_composed_asec_measures as measures + + binding = measures._binding(values) + prepared = measures._prepared(values["prepared_receipt"]) + document = measures._document(values["frame_context"]) + measures._bound_population(document, binding) + selected = measures._selected_money(binding, prepared, values) + rows = asec_binding.bind_composed_asec_arm_rows( + values["arm_rows"].payload, binding_document=binding + ) + original = rows.array("person", "original_ids") + + def integer(name): + return asec_binding._int64_view(person[name], reason="ASEC_HOST:" + name) + + require( + np.array_equal(integer(provenance.spine_source_id_column("person")), original), + "ASEC_HOST_NATIVE_PERSON_ROSTER", + ) + composed = integer(provenance.support_source_id_column("person")) + require( + _row_identity(pd.DataFrame({"person_id": composed}), "person")[ + "ordered_ids_sha256" + ] + == binding["arm"]["person"]["composed_ordered_ids_sha256"], + "ASEC_HOST_COMPOSED_ROSTER", + ) + text_year = person.source_year.astype(str) + require(text_year.isin(("2022", "2023", "2024")).all(), "ASEC_HOST_COHORT") + years = text_year.to_numpy().astype("int64") + income = asec_income.bind_income_observations( + values["income_observations"].payload, prepared_receipt=prepared + ) + result = asec_income.read_reported_income( + values["reported_income"].payload, + selected, + income, + person_ids=original, + income_years=years, + person_positions_sha256=binding["arm"]["person"]["source_positions_sha256"], + prepared_receipt_sha256=codec.sha(values["prepared_receipt"].payload), + selection_sha256=codec.sha(values["asec_binding"].payload), + selected_money_sha256=codec.sha(values["selected_current_money"].payload), + income_payload_sha256=codec.sha(values["income_observations"].payload), + source_producer_key=values["prepared_receipt"].producer_key, + selection_producer_key=values["asec_binding"].producer_key, + frame_context_sha256=codec.sha(values["frame_context"].payload), + ) + require( + np.array_equal(integer("A_AGE"), result.array("A_AGE")) + and np.array_equal( + _numeric(person.age, "ASEC_HOST_AGE"), result.array("A_AGE") + ), + "ASEC_HOST_NATIVE_AGE", + ) + name = "asec_reported_wage_income_2024_price" + actual = _numeric(person[name], "ASEC_HOST_WAGE_TYPE") + require( + np.isfinite(actual).all() + and (actual >= 0).all() + and np.array_equal(actual.view("uint64"), result.array(name).view("uint64")), + "ASEC_HOST_WAGE_BINDING", + ) + require((result.array("WSAL_VAL.validity") == 1).all(), "ASEC_HOST_WAGE_VALIDITY") + return { + "ids": person.person_id.to_numpy(dtype="int64").tolist(), + "source_native_person_sha256": codec.sha(original.tobytes()), + "income_year_sha256": codec.sha(years.tobytes()), + "reported_income_sha256": codec.sha(values["reported_income"].payload), + "wage_sha256": codec.sha(actual.tobytes()), + "evidence_axes": { + n: result.array("WSAL_VAL." + n).tolist() + for n in ("status", "validity", "zero_origin") + }, + "monetary_basis": json.loads(result.header)["monetary_basis"], + "missing_treatment": "refuse_no_evidence_upgrade", + "source_period_equivalence_claim": False, + } + + +def recipient_matrix(frame): + """Read the actual microunit role carrier; do not calculate policy roles.""" + return detail.recipient_matrix(frame, role_column="tax_unit_role_input") + + +def qualify_host_population(frame, values): + """Bind the complete untouched clone pool before selecting model features. + + Reconstruct native membership from typed original-source projections and + native-parent binding. Exact content comparison also catches a caller's + omitted declared column; the materialized verifier remains mandatory. + """ + sources = [] + for name in ("acs", "asec"): + value = values[name] + require( + origin._source_document(value.payload)["arm"] == name, "HOST_ORIGIN_ARM" + ) + sources.append( + origin.AuthenticatedNativeOriginSource(value.payload, _token=origin._TOKEN) + ) + parent_doc = codec.decode_json(values["parent_binding"].payload) + require(parent_doc["schema"] == origin.BINDING_SCHEMA, "HOST_PARENT_SCHEMA") + require( + parent_doc["graph_parent_edges"] + == { + n: { + "producer_key": values[n].producer_key, + "artifact_key": values[n].key, + "payload_sha256": codec.sha(values[n].payload), + } + for n in ("acs", "asec") + }, + "HOST_NATIVE_ORIGIN_PRODUCERS", + ) + parent = origin.PopulationOriginBinding( + values["parent_binding"].payload, _token=origin._BOUND_TOKEN + ) + expected = origin.bind_population_origins( + frame, sources=sources, parent=parent + ).document + recorded = codec.decode_json(values["clone_binding"].payload) + require( + { + k: v + for k, v in recorded.items() + if k not in ("graph_parent_edges", "population_node") + } + == expected, + "HOST_FULL_ORIGIN_BINDING", + ) + require( + recorded["graph_parent_edges"] + == { + n: { + "producer_key": values[n].producer_key, + "artifact_key": values[n].key, + "payload_sha256": codec.sha(values[n].payload), + } + for n in ("acs", "asec", "parent_binding") + }, + "HOST_ORIGIN_PRODUCERS", + ) + # Exact pair checks include every carried cell, linked entity and weight, + # including zero-weight rows. This also refuses incomplete selected wages. + matrix, mask = recipient_matrix(frame) + native = frame.person.loc[ + frame.person[provenance.support_clone_index_column("person")].eq(0) + ] + channel = native[provenance.support_channel_column("person")] + acs_person = native.loc[channel.eq("acs")].reset_index(drop=True) + asec_person = native.loc[channel.eq("asec")].reset_index(drop=True) + require(len(acs_person) > 0 and len(asec_person) > 0, "HOST_BOTH_ARMS") + evidence = { + "acs": acs_wage_evidence(acs_person), + "asec": asec_wage_evidence(asec_person, values), + } + return { + "schema": "microcosm.us.puf_diagnostic_host_features.v1", + "recipe": recipe_document(), + "full_population_content_sha256": detail.population_content(frame), + "origin_binding_sha256": codec.sha(values["clone_binding"].payload), + "wage_evidence": evidence, + "matrix_sha256": codec.sha(matrix), + "recipients": int(mask.sum()), + "all_clone_rows_including_zero_weights": True, + "release_eligible": False, + } + + +CURRENT_SURVEY_ROUTE = "authenticated_current_survey_v1" +CURRENT_SURVEY_PROJECTION_PROTOCOL = "microcosm.us.survey-puf-host-projection.v1" +CURRENT_SURVEY_MAX_BYTES = 64 * 1024**2 + + +def current_survey_recipient_matrix(frame, projection): + """Pure typed-transport consumer; live owner verification remains mandatory. + + Both this parser and the existing matrix validate every clone pair. The + projection retains current ASEC evidence bytes. ACS arithmetic/missingness + remains the existing WAGP/ADJINC contract, including under-15 refusal for a + missing selected earner. No serialized field confers source admission. + """ + require( + type(projection) is bytes and 0 < len(projection) <= CURRENT_SURVEY_MAX_BYTES, + "SURVEY_HOST_PROJECTION_BOUND", + ) + document = codec.decode_json(projection) + require( + set(document) + == { + "protocol", + "route", + "preparation_sha256", + "allocation_sha256", + "clone_sha256", + "asec", + "release_eligible", + }, + "SURVEY_HOST_PROJECTION_FIELDS", + ) + require( + document["protocol"] == CURRENT_SURVEY_PROJECTION_PROTOCOL + and document["route"] == CURRENT_SURVEY_ROUTE + and document["release_eligible"] is False + and all( + codec._hash(document[k]) + for k in ("preparation_sha256", "allocation_sha256", "clone_sha256") + ), + "SURVEY_HOST_PROJECTION_CONTRACT", + ) + asec = document["asec"] + require( + type(asec) is dict + and set(asec) + == { + "preparation_sha256", + "asec_native_sha256", + "money_header", + "money_header_sha256", + "domain", + "zero_origin_code", + "columns", + "rows", + }, + "SURVEY_WAGE_FIELDS", + ) + require( + asec["preparation_sha256"] == document["preparation_sha256"] + and codec._hash(asec["asec_native_sha256"]) + and codec.sha(codec.encode_json(asec["money_header"])) + == asec["money_header_sha256"] + and asec["money_header"]["target_year"] == 2024 + and asec["money_header"]["semantic"] == "annual_current_money" + and asec["columns"] + == [ + "stacked_person_id", + "native_person_id", + "income_year", + "amount_f64le_hex", + "status", + "validity", + "zero_origin", + ], + "SURVEY_WAGE_CONTRACT", + ) + people = frame.person + native_role = people[provenance.support_clone_index_column("person")].eq(0) + native_acs = people.loc[ + native_role & people[provenance.support_channel_column("person")].eq("acs") + ] + native_asec = people.loc[ + native_role & people[provenance.support_channel_column("person")].eq("asec") + ] + acs_evidence = acs_wage_evidence(native_acs) + rows = asec["rows"] + require( + type(rows) is list and len(rows) == len(native_asec) > 0, "SURVEY_WAGE_ROSTER" + ) + require( + type(asec["domain"]) is dict + and set(asec["domain"]) + == {"name", "entity", "column", "grain", "minimum", "maximum", "zero_semantics"} + and asec["domain"]["name"] == "WSAL_VAL" + and asec["domain"]["entity"] == "person" + and type(asec["zero_origin_code"]) is int, + "SURVEY_WAGE_DOMAIN", + ) + amounts = {} + for row in rows: + require( + type(row) is list + and len(row) == 7 + and all(type(row[i]) is int for i in (0, 1, 2, 4, 5, 6)) + and row[2] == 2024 + and row[5] == 1 + and all(0 <= row[i] <= 255 for i in (4, 5, 6)) + and type(row[3]) is str + and len(row[3]) == 16 + and set(row[3]) <= set("0123456789abcdef"), + "SURVEY_WAGE_ROW", + ) + require(row[0] not in amounts, "SURVEY_WAGE_DUPLICATE") + amount = np.frombuffer(bytes.fromhex(row[3]), dtype="= 0, "SURVEY_WAGE_AMOUNT") + amounts[row[0]] = (row[1], amount) + # Reuse the maintained money owner's complete buffer/evidence/domain check. + # These value objects do not issue money/source authority. The live verifier + # compares the complete projection, including this source-owned domain. + current_money._validate_field( + current_money.MoneyField( + "WSAL_VAL", + b"".join(bytes.fromhex(row[3]) for row in rows), + bytes(row[4] for row in rows), + bytes(row[5] for row in rows), + bytes(row[6] for row in rows), + ), + current_money.MoneyDomain(**asec["domain"]), + len(rows), + nominal=True, + zero_origin_code=current_money.ZeroOrigin(asec["zero_origin_code"]), + ) + source_id = provenance.support_source_id_column("person") + original_id = provenance.spine_source_id_column("person") + require( + [r[0] for r in rows] == native_asec[source_id].tolist() + and [r[1] for r in rows] == native_asec[original_id].tolist() + and native_asec.source_year.astype(str).eq("2024").all(), + "SURVEY_WAGE_ORIGIN", + ) + wages = [] + columns = [ + provenance.support_channel_column("person"), + source_id, + original_id, + "employment_income_before_lsr", + ] + for channel, stacked, native, value in people[columns].itertuples( + index=False, name=None + ): + if channel == "asec": + require(stacked in amounts, "SURVEY_WAGE_COVERAGE") + original, amount = amounts[stacked] + require(original == native, "SURVEY_WAGE_ORIGIN") + wages.append(amount) + else: + require(channel == "acs", "SURVEY_WAGE_CHANNEL") + wages.append(np.nan if pd.isna(value) else value) + projected = pd.Series( + wages, index=pd.Index(people.person_id.to_numpy()), dtype="float64" + ) + matrix, mask = detail.recipient_matrix( + frame, role_column="tax_unit_role_input", person_wages=projected + ) + require(len(matrix) <= CURRENT_SURVEY_MAX_BYTES, "SURVEY_MATRIX_BOUND") + return ( + matrix, + mask, + { + "scope": SCOPE, + "host_route": CURRENT_SURVEY_ROUTE, + "projection_sha256": codec.sha(projection), + "acs_wage_evidence": acs_evidence, + "asec_money_header_sha256": asec["money_header_sha256"], + "selected_tax_units": int(mask.sum()), + "release_eligible": False, + "source_period_equivalence_claim": False, + }, + ) diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/puf_full_source.py b/packages/microcosm-build/src/microcosm/build/us_runtime/puf_full_source.py new file mode 100644 index 000000000..b666afa90 --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/puf_full_source.py @@ -0,0 +1,519 @@ +"""Additive full-return PUF source projection, before canonical modeling. + +This is not the older 13-column monetary envelope. It preserves every source +return, raw period, integer S006 and missing demographic state. Only ordinary +returns carry numeric amounts; disclosure aggregates retain sparse lexical +tokens and use an explicit int64 sentinel. The unchanged raw decoder owns CSV +layout, status-code and join validation. This owner authenticates both exact +delivery buffers before calling it and binds the subsequent projection to its +complete RECID sequence. No tax engine, growth, fit or donor admission occurs. +""" + +from __future__ import annotations + +import hashlib +import json +import re +import struct +from collections.abc import Mapping +from dataclasses import dataclass +from types import MappingProxyType + +import numpy as np + +from . import puf_raw_source as raw + +FULL_SOURCE_VERSION = "microcosm.us.puf_2015_full_return_source/3" +FULL_SOURCE_MAX_BYTES = 128 * 1024 * 1024 +OUTSIDE_AMOUNT_UNIVERSE = -(2**63) + +# Explicit publisher amount-field roster. No role is inferred from a prefix. +# E30400/E30500 deliberately excluded: capped Schedule SE amounts cannot +# reconstruct uncapped partnership earnings. +MONEY_COLUMNS = ( + "E00100", + "E00200", + "E00300", + "E00400", + "E00600", + "E00650", + "E00700", + "E00800", + "E00900", + "E01100", + "E01200", + "E01400", + "E01700", + "E02100", + "E02400", + "E03150", + "E03210", + "E03220", + "E03230", + "E03240", + "E03290", + "E03300", + "E03500", + "E18500", + "E19200", + "E19800", + "E20100", + "E20400", + "E20500", + "P22250", + "P23250", + "E24515", + "E24518", + "E25850", + "E25860", + "E25940", + "E25980", + "E25920", + "E25960", + "E26110", + "E26170", + "E26190", + "E26160", + "E26180", + "E26270", + "E26100", + "E26390", + "E26400", + "E27200", + "T27800", + "E58990", + "E87530", +) +COUNT_COLUMNS = ("XFPT", "XFST", "XOCAH", "XOCAWH", "XOODEP", "XOPAR", "XTOT") +DEPENDENT_COLUMNS = ("XOCAH", "XOCAWH", "XOODEP", "XOPAR") +PROJECTED_COLUMNS = (*MONEY_COLUMNS, *COUNT_COLUMNS) +_MONEY = re.compile(r"-?[0-9]{1,12}\Z", re.ASCII) +_COUNT = re.compile(r"[0-9]{1,2}\Z", re.ASCII) +# Operational preservation bound for four out-of-universe source rows. +# Empty, scientific notation and literal formatting are retained, not parsed. +AGGREGATE_LEXEME_MAX_CHARACTERS = 64 +_AGGREGATE = re.compile(r"[\x20-\x7e]{0,64}\Z", re.ASCII) + + +def _require(condition, code): + if not condition: + raise ValueError(code) + + +def _validate_count_fields(values, status): + """Publisher disclosure-coded counts; never a household/person roster.""" + ordinary = status["disclosure_aggregate"] == 0 + mars = status["MARS"][ordinary] + caps = np.select([mars == 1, mars == 2, mars == 3, mars == 4], [2, 3, 1, 3], -1) + _require((caps >= 0).all(), "FULL_COUNT_MARS") + for name in ("XFPT", "XFST"): + _require( + np.isin(values[name][ordinary], (0, 1)).all(), "FULL_EXEMPTION_FLAG:" + name + ) + cumulative = np.zeros(len(mars), dtype=np.int64) + for name in DEPENDENT_COLUMNS: + count = values[name][ordinary] + _require(((count >= 0) & (count <= 3)).all(), "FULL_DEPENDENT_DOMAIN:" + name) + cumulative += count + _require((cumulative <= caps).all(), "FULL_DEPENDENT_SEQUENTIAL_CAP:" + name) + xtot = values["XTOT"][ordinary] + _require(((xtot >= 0) & (xtot <= 5)).all(), "FULL_EXEMPTIONS_DOMAIN") + _require( + np.array_equal( + xtot, values["XFPT"][ordinary] + values["XFST"][ordinary] + cumulative + ), + "FULL_EXEMPTIONS_COMPONENT_IDENTITY", + ) + + +def puf_2015_receiver_size_measurement(filing_status_code, dependent_count): + """Match the PUF's coarse status and censored return-size predictor. + + Generic model status codes are single1/joint2/separate3/HOH4/widow5. + PUF code2 combines joint and surviving-spouse returns. Its baseline of two + is a predictor convention for that combined class, not an observed number + of living filers. Physical survey membership must remain unchanged. + """ + status = np.asarray(filing_status_code) + dependents = np.asarray(dependent_count) + _require( + status.ndim == 1 and dependents.shape == status.shape, + "FULL_RECEIVER_COUNT_SHAPE", + ) + _require( + status.dtype.kind in "iu" and dependents.dtype.kind in "iu", + "FULL_RECEIVER_COUNT_TYPE", + ) + _require(np.isin(status, (1, 2, 3, 4, 5)).all(), "FULL_RECEIVER_STATUS_DOMAIN") + _require( + ((dependents >= 0) & (dependents <= 1000)).all(), + "FULL_RECEIVER_DEPENDENT_BOUND", + ) + mars = np.where(status == 5, 2, status).astype(np.int64) + cap = np.select([mars == 1, mars == 2, mars == 3, mars == 4], [2, 3, 1, 3]) + return { + "puf_2015_filing_status_code": mars, + "puf_2015_capped_return_size": 1 + + (mars == 2).astype(np.int64) + + np.minimum(dependents, cap), + } + + +def _frozen_array(values): + values = np.asarray(values) + return np.frombuffer(values.tobytes(), dtype=values.dtype).reshape(values.shape) + + +@dataclass(frozen=True) +class FullPufSource: + """Read-only source values; available is not an admission or truth flag.""" + + definition_sha256: str + source_sha256: Mapping[str, str] + status: Mapping[str, np.ndarray] + values: Mapping[str, np.ndarray] + aggregate_tokens: Mapping[int, Mapping[str, str]] + status_payload: bytes + + @property + def ordinary(self): + return self.status["disclosure_aggregate"] == 0 + + +def decode_full_puf_source(main_bytes, demographic_bytes, definition): + """Authenticate, decode, and align a complete declared delivery in memory.""" + _require(isinstance(definition, raw.PufRawSourceDefinition), "FULL_DEFINITION") + for data, pin in ( + (main_bytes, definition.main), + (demographic_bytes, definition.demographic), + ): + _require(type(data) is bytes and len(data) == pin.bytes, "FULL_SOURCE_SIZE") + _require(hashlib.sha256(data).hexdigest() == pin.sha256, "FULL_SOURCE_SHA256") + status = raw.decode_puf_raw_source(main_bytes, demographic_bytes, definition) + rows = len(status.typed["RECID"]) + _require( + rows * len(PROJECTED_COLUMNS) * 8 <= FULL_SOURCE_MAX_BYTES, + "FULL_PROJECTION_BODY_LIMIT", + ) + profile = definition.document["csv_profile"] + reader = raw._reader(main_bytes, profile) + header = raw._check_header(reader, definition.main, profile=profile) + _require(set(PROJECTED_COLUMNS) <= set(header), "FULL_PROJECTED_HEADER") + positions = {name: header.index(name) for name in PROJECTED_COLUMNS} + recid_position = header.index("RECID") + values = { + name: np.full(rows, OUTSIDE_AMOUNT_UNIVERSE, dtype="= offset + 8, "FULL_HEADER_LENGTH") + size = struct.unpack(" 0, "FULL_HEADER_LIMIT") + offset += 8 + _require(offset + size <= len(payload), "FULL_TRUNCATED_HEADER") + header_bytes = payload[offset : offset + size] + try: + h = json.loads(header_bytes) + except (UnicodeError, ValueError) as error: + raise ValueError("FULL_HEADER_JSON") from error + _require( + type(h) is dict + and set(h) + == { + "definition_sha256", + "source_sha256", + "status_bytes", + "status_sha256", + "rows", + "columns", + "aggregate_tokens", + }, + "FULL_HEADER_KEYS", + ) + _require( + json.dumps( + h, sort_keys=True, separators=(",", ":"), ensure_ascii=True, allow_nan=False + ).encode("ascii") + == header_bytes, + "FULL_HEADER_CANONICAL", + ) + _require( + h["definition_sha256"] == definition.sha256 + and h["source_sha256"] + == { + "main": definition.main.sha256, + "demographic": definition.demographic.sha256, + }, + "FULL_ARTIFACT_SOURCE_BINDING", + ) + rows = h["rows"] + _require( + type(rows) is int and rows == definition.main.data_records, "FULL_ARTIFACT_ROWS" + ) + _require( + type(h["status_bytes"]) is int + and 0 < h["status_bytes"] <= FULL_SOURCE_MAX_BYTES, + "FULL_STATUS_LENGTH", + ) + offset += size + status_payload = payload[offset : offset + h["status_bytes"]] + _require( + len(status_payload) == h["status_bytes"] + and hashlib.sha256(status_payload).hexdigest() == h["status_sha256"], + "FULL_STATUS_SHA256", + ) + status = raw.decode_return_status(status_payload, definition=definition) + offset += h["status_bytes"] + _require( + type(h["columns"]) is list and len(h["columns"]) == len(PROJECTED_COLUMNS), + "FULL_ARTIFACT_COLUMNS", + ) + values = {} + for name, entry in zip(PROJECTED_COLUMNS, h["columns"], strict=True): + _require( + type(entry) is dict + and set(entry) == {"name", "bytes", "sha256"} + and entry["name"] == name + and type(entry["bytes"]) is int + and entry["bytes"] == rows * 8, + "FULL_COLUMN_DECLARATION", + ) + data = payload[offset : offset + entry["bytes"]] + _require( + len(data) == entry["bytes"] + and hashlib.sha256(data).hexdigest() == entry["sha256"], + "FULL_COLUMN_SHA256", + ) + values[name] = np.frombuffer(data, dtype="= -cap) & (value[~aggregate] <= cap)).all() + and (name not in COUNT_COLUMNS or (value[~aggregate] >= 0).all()), + "FULL_ORDINARY_TYPED_DOMAIN", + ) + _validate_count_fields(values, status.typed) + return FullPufSource( + definition.sha256, + MappingProxyType(h["source_sha256"]), + status.typed, + MappingProxyType(values), + MappingProxyType({int(k): MappingProxyType(v) for k, v in tokens.items()}), + status_payload, + ) + + +DIRECT_MAPPINGS = { + "employment_income_before_lsr": "E00200", + "self_employment_income_before_lsr": "E00900", + "taxable_interest_income": "E00300", + "qualified_dividend_income": "E00650", + "tax_exempt_interest_income": "E00400", + "short_term_capital_gains": "P22250", + "long_term_capital_gains_before_response": "P23250", + "long_term_capital_gains_on_collectibles": "E24518", + "non_sch_d_capital_gains": "E01100", + "taxable_private_pension_income": "E01700", + "taxable_ira_distributions": "E01400", + "alimony_income": "E00800", + "alimony_expense": "E03500", + "salt_refund_income": "E00700", + "charitable_cash_donations": "E19800", + "charitable_non_cash_donations": "E20100", + "real_estate_taxes": "E18500", + "investment_income_elected_form_4952": "E58990", + "student_loan_interest": "E03210", + "educator_expense": "E03220", + "casualty_loss": "E20500", + "farm_income": "T27800", + "farm_operations_income": "E02100", + "farm_rent_income": "E27200", + "miscellaneous_income": "E01200", + "domestic_production_ald": "E03240", + "unrecaptured_section_1250_gain": "E24515", + "health_savings_account_ald": "E03290", +} + + +def observed_and_derived_return_columns(source, *, selected_recids=None): + """Return canonical observed/derived quantities plus explicit model inputs. + + This stage intentionally retains total SS, total interest, realized IRA/ + Keogh deductions and total miscellaneous itemized deductions under honest + auxiliary names. The next declared model owner supplies component/proxy + outputs; their absence is never silently converted to a known zero here. + """ + _require(isinstance(source, FullPufSource), "FULL_SOURCE_TYPE") + mask = source.ordinary.copy() + if selected_recids is not None: + ids = tuple(selected_recids) + _require( + len(ids) > 0 + and all(type(i) is int for i in ids) + and len(set(ids)) == len(ids), + "FULL_SELECTION_IDS", + ) + _require( + set(ids) <= set(source.status["RECID"][mask].tolist()), + "FULL_SELECTION_UNIVERSE", + ) + mask &= np.isin(source.status["RECID"], ids) + a = {name: value[mask] for name, value in source.values.items()} + out = {name: a[field].copy() for name, field in DIRECT_MAPPINGS.items()} + out.update( + { + "non_qualified_dividend_income": a["E00600"] - a["E00650"], + "rental_income": a["E25850"] - a["E25860"], + "estate_income": a["E26390"] - a["E26400"], + "partnership_income": ( + a["E25940"] + a["E25980"] - a["E25920"] - a["E25960"] - a["E26110"] + ), + "s_corp_income": ( + a["E26170"] + a["E26190"] - a["E26160"] - a["E26180"] - a["E26100"] + ), + } + ) + auxiliary = { + "raw_adjusted_gross_income": a["E00100"], + "raw_total_social_security": a["E02400"], + "raw_total_interest_deduction": a["E19200"], + "raw_realized_ira_deduction": a["E03150"], + "raw_realized_keogh_deduction": a["E03300"], + "raw_tuition_fees_deduction": a["E03230"], + "raw_lifetime_learning_qualified_expenses": a["E87530"], + "raw_miscellaneous_itemized_deductions": a["E20400"], + "raw_partnership_nonpassive_net": a["E25980"] - a["E25960"], + "raw_partnership_s_corp_combined": a["E26270"], + "raw_exemptions_count": a["XTOT"], + **{"raw_" + name.lower(): a[name] for name in COUNT_COLUMNS[:-1]}, + "reported_dependent_count": sum(a[name] for name in DEPENDENT_COLUMNS), + "puf_2015_filing_status_code": source.status["MARS"][mask].copy(), + "puf_2015_capped_return_size": ( + 1 + + (source.status["MARS"][mask] == 2).astype(np.int64) + + sum(a[name] for name in DEPENDENT_COLUMNS) + ), + "raw_ordinary_dividend_income": a["E00600"], + } + auxiliary["partnership_component_reconciliation_residual"] = ( + out["partnership_income"] + out["s_corp_income"] - a["E26270"] + ) + # Canonical intermediates stay integer dollars until the explicit model + # boundary. S006 stays integer hundredths here too. + status = {name: value[mask].copy() for name, value in source.status.items()} + return out, auxiliary, status diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/puf_full_source_graph.py b/packages/microcosm-build/src/microcosm/build/us_runtime/puf_full_source_graph.py new file mode 100644 index 000000000..6f8f13044 --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/puf_full_source_graph.py @@ -0,0 +1,122 @@ +"""Graph owner of the additive complete PUF return-source artifact.""" + +from __future__ import annotations + +import hashlib +from importlib import import_module + +from microcosm.graph import ( + ArtifactOutput, + ArtifactType, + Capabilities, + Determinism, + KernelBase, + KernelResult, + Node, + Numeric, + StructuralDelta, + source_hash, +) +from microcosm.graph.canonical import canonical_json +from microcosm.graph.codecs import load_source_bytes + +from . import puf_full_source as full +from . import puf_raw_source as raw + +FULL_RETURN_SOURCE_TYPE = ArtifactType("microcosm.us.puf_2015_full_return_source", 3) + + +class FullPufReturnSourceKernel(KernelBase): + ref = "us.puf.full_return_source@3" + capabilities = Capabilities( + Determinism.DETERMINISTIC, numeric=Numeric.BITWISE, dependencies=("numpy",) + ) + + def __init__(self, *, definition=None, source_codecs=None): + self.definition = ( + raw.packaged_definition() if definition is None else definition + ) + self.source_codecs = ( + raw.puf_raw_source_codecs(self.definition) + if source_codecs is None + else source_codecs + ) + + def implementation_hash(self): + base = source_hash( + import_module(__name__), + full, + raw, + dependencies=self.capabilities.dependencies, + ) + return hashlib.sha256( + base.encode("ascii") + b"\0" + canonical_json(raw.csv_acceptance_profile()) + ).hexdigest() + + def run(self, context): + d = self.definition + if ( + context.node.kernel != self.ref + or context.node.structural is not StructuralDelta.NONE + or context.node.sources != (d.main.source_name, d.demographic.source_name) + or context.node.inputs + or context.node.outputs + or context.node.artifact_inputs + or context.node.artifact_outputs + != (ArtifactOutput("full_return_source", FULL_RETURN_SOURCE_TYPE),) + or dict(context.params) != {"definition": d.params_text} + ): + raise ValueError("FULL_SOURCE_NODE_DECLARATION") + main = load_source_bytes( + d.main.codec, + context.sources[d.main.source_name], + registry=self.source_codecs, + ) + demo = load_source_bytes( + d.demographic.codec, + context.sources[d.demographic.source_name], + registry=self.source_codecs, + ) + decoded = full.decode_full_puf_source(main, demo, d) + payload = full.encode_full_puf_source(decoded) + return KernelResult( + artifacts={"full_return_source": payload}, + receipt={ + "full_puf_return_source": { + "definition_sha256": d.sha256, + "definition_route": d.route, + "source_sha256": dict(decoded.source_sha256), + "artifact_sha256": hashlib.sha256(payload).hexdigest(), + "artifact_bytes": len(payload), + "rows": len(decoded.status["RECID"]), + "ordinary_rows": int(decoded.ordinary.sum()), + "projected_monetary_fields": len(full.MONEY_COLUMNS), + "projected_count_fields": len(full.COUNT_COLUMNS), + "amount_units": "source_whole_us_dollars", + "weight_units": "S006_integer_hundredths", + "period": "raw_FLPDYR_preserved", + "canonical_modeling": False, + "target_year_growth": False, + "donor_admission": False, + } + }, + ) + + +def full_puf_return_source_node( + *, population, definition=None, node_id="us_puf_full_source.return_source" +): + if not isinstance(population, str) or not population: + raise ValueError("FULL_SOURCE_POPULATION") + d = raw.packaged_definition() if definition is None else definition + return Node( + node_id, + FullPufReturnSourceKernel.ref, + population=population, + sources=(d.main.source_name, d.demographic.source_name), + params={"definition": d.params_text}, + artifact_outputs=( + ArtifactOutput("full_return_source", FULL_RETURN_SOURCE_TYPE), + ), + description="Authenticate and type 52 monetary fields, seven exemption/dependent count fields, and complete PUF source status; no model or growth.", + ) diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/puf_growth.py b/packages/microcosm-build/src/microcosm/build/us_runtime/puf_growth.py new file mode 100644 index 000000000..794486e8e --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/puf_growth.py @@ -0,0 +1,1713 @@ +"""Pure per-field growth contract and transform for a raw-source PUF successor. + +This is the first bounded slice of `microcosm#530 +`_ item 2: *pin the raw +PUF asset and implement PUF aging as a Microcosm stage, under the same uprating +discipline as the other sources*. Nothing here reads a file, acquires an +artifact, imports a country engine, or generates a factor. It compiles an +explicit contract and applies it to a table that the caller already holds. + +What the contract makes explicit, per consumed field +---------------------------------------------------- +Source reference year, target year, role (money / code / identifier / count / +design weight), growth kind (nominal price restatement versus real per-return +aging), sign branch, and the exact factor-series identity. A field with no +declared rule is refused; a field with two rules that could both select the +same row is refused; there is no implicit factor of one and no dispatch by +string substring. Every factor is resolved by naming a **variable row** and two +**year columns** of the factor table explicitly, and is applied to a given row +at most once. + +Why the row/column axes are named rather than iterated +------------------------------------------------------ +The archived producer looped ``for variable in uprating`` over a DataFrame +pivoted to year columns, so it iterated years while believing it iterated +variable names (:data:`ARCHIVED_PRODUCER_COMMIT`, ``datasets/puf/puf.py`` +against ``utils/uprating.py``). Its 2015→2021 step separately listed two of its +three sign-branched fields in a fallback group as well, so for those two a +sign-specific factor was followed by a second growth factor. Both failure modes are structural +here: a series name is looked up in a variable-keyed mapping, a year is looked +up in a year-keyed mapping, and the sign-branch cover is validated to be exactly +disjoint before any array is touched. + +Authority +--------- +A factor table declares its own authority, and only +:data:`RELEASE_ELIGIBLE_AUTHORITIES` — ``reviewed_resource`` alone — can stand +behind a release. ``invented_fixture_nonauthority`` proves the contract and the +transform on invented levels. ``developmental_public_resource`` is for a +bounded experiment on levels that really were read from a public publisher: it +must carry the same receipt a reviewed table owes, plus the digest and +retrieval time of the exact resource bytes, and it must declare its own divisor +and index/total shape. Labelling such a table an invented fixture would be a +misdescription, so it gets its own authority rather than a lie. + +What it does **not** get is a promotion path. Release eligibility is keyed on +``reviewed_resource`` and nothing else, and a developmental table whose digest +appears in :data:`REVIEWED_FACTOR_TABLE_DIGESTS` is refused outright, so an +allowlist entry cannot launder a developmental resource into a release recipe. +``reviewed_resource`` is itself refused until a reviewed table is generated, +pinned, and added to that deliberately empty tuple. No authority can be +promoted by passing a flag. + +Scope +----- +Money growth and design-weight growth are separate entry points, so a money +transform cannot move a weight. Codes, identifiers and counts are declared but +never written, and preservation is verified rather than assumed. The raw +``RECID`` and the source-year AGI used for SOI band assignment are copied into +separate provenance columns before any growth, so a later change of AGI basis +cannot destroy the band input. Nothing here decides whether the pinned +processed artifact is comparable to a rebuilt one; that artifact stays an +opaque legacy input, outside this contract. +""" + +from __future__ import annotations + +import datetime +import json +import struct +from collections.abc import Mapping, Sequence +from dataclasses import InitVar, dataclass +from enum import StrEnum +from hashlib import sha256 +from types import MappingProxyType + +import numpy as np +import pandas as pd + +from microcosm.build.monetary_targets import MonetaryBasis +from microcosm.build.us_runtime.puf_source_agi import PUF_SOURCE_YEAR + +__all__ = [ + "DESIGN_WEIGHT_FIELD", + "GROWN_MONEY_VALUATION", + "MAX_EXACT_AMOUNT", + "PROVENANCE_COLUMNS", + "PROVENANCE_RECID_COLUMN", + "PROVENANCE_SOURCE_AGI_COLUMN", + "ARCHIVED_PRODUCER_COMMIT", + "DEVELOPMENTAL_PROVENANCE_KEYS", + "DEVELOPMENTAL_PROVENANCE_LITERALS", + "PROPOSED_TY2015_FIELD_ROSTER", + "RELEASE_ELIGIBLE_AUTHORITIES", + "ROSTER_CITATION", + "ROSTER_SOURCE_REFERENCE_YEAR", + "REVIEWED_FACTOR_TABLE_DIGESTS", + "REVIEWED_PROVENANCE_KEYS", + "SOURCE_AGI_FIELD", + "SOURCE_RECID_FIELD", + "CompiledPufGrowth", + "FactorAuthority", + "FactorSeries", + "FieldRole", + "GrowthFactorTable", + "GrowthKind", + "GrowthRecipe", + "GrowthRule", + "GrownPufTable", + "PufGrowthRefusalError", + "ResolvedFactor", + "RosterEntry", + "UNDOCUMENTED_SOURCE_FIELDS", + "SignBranch", + "apply_puf_design_weight_growth", + "apply_puf_growth", + "compile_puf_growth", + "parse_growth_recipe", + "roster_role", + "puf_growth_monetary_metadata", + "require_transformable", +] + +#: Schema version of the recipe identity this module emits. +RECIPE_SCHEMA_VERSION = 1 + +#: Schema version of the factor-table document this module parses. +FACTOR_TABLE_SCHEMA_VERSION = 1 + +#: Raw IRS PUF field carrying the return identifier. +SOURCE_RECID_FIELD = "RECID" + +#: Raw IRS PUF field carrying adjusted gross income. +SOURCE_AGI_FIELD = "E00100" + +#: Raw IRS PUF field carrying the sample weight (stored times 100). +DESIGN_WEIGHT_FIELD = "S006" + +#: Copy of ``RECID``, written from the source before any growth. No rule may +#: target this column or take its name, but the column itself is an ordinary +#: pandas column: what protects it is the contract, and in the graph the fact +#: that no node declares it as a rewrite. +PROVENANCE_RECID_COLUMN = "puf_source_recid" + +#: Copy of source-year AGI on the same terms, kept for the TY2015 SOI band +#: assignment in :mod:`microcosm.build.us_runtime.puf_source_agi`. +PROVENANCE_SOURCE_AGI_COLUMN = "puf_source_year_agi" + +#: The provenance columns this transform adds, in output order. +PROVENANCE_COLUMNS = (PROVENANCE_RECID_COLUMN, PROVENANCE_SOURCE_AGI_COLUMN) + +#: SHA-256 digests of factor tables that a reviewed authority has accepted. +#: Empty: no factor table has been generated or reviewed for this lineage, so +#: ``reviewed_resource`` authority cannot compile. Adding a digest here is the +#: reviewed act; it is not something a caller can supply. +REVIEWED_FACTOR_TABLE_DIGESTS: tuple[str, ...] = () + +#: What a ``reviewed_resource`` factor table must state about itself. Nothing +#: can satisfy this yet, because :data:`REVIEWED_FACTOR_TABLE_DIGESTS` is +#: empty; the set is here so the receipt a reviewed table owes is written down +#: executably rather than only in prose. +REVIEWED_PROVENANCE_KEYS = frozenset( + { + "dependency_versions", + "generated_by", + "generator_code_sha256", + "index_or_total", + "parameter_paths", + "population_divisor", + "rounding", + "source_url", + } +) + +#: What a ``developmental_public_resource`` factor table must state about +#: itself: everything a reviewed table owes, plus the digest of the exact +#: resource bytes its levels were read from and when they were retrieved. A +#: public resource is a moving distribution, so "which bytes, read when" is +#: part of the claim rather than a footnote. +DEVELOPMENTAL_PROVENANCE_KEYS = frozenset( + REVIEWED_PROVENANCE_KEYS | {"resource_sha256", "retrieved_utc"} +) + +#: Provenance values a developmental table must spell out rather than leave to +#: a reader's inference. A price index has no return or population divisor and +#: is an index, not a total; a table that says otherwise is describing a +#: different quantity and is refused here rather than downstream. +DEVELOPMENTAL_PROVENANCE_LITERALS = MappingProxyType( + {"index_or_total": "index", "population_divisor": "none"} +) + +#: Largest integer amount float64 represents exactly. A larger integer money +#: value is refused rather than silently rounded by the working conversion. +MAX_EXACT_AMOUNT = 2**53 + +#: Largest table this transform will accept, so a mistake is a refusal rather +#: than an unbounded allocation. +MAX_ROWS = 2_000_000 + +#: Largest factor document this transform will parse. +MAX_DOCUMENT_BYTES = 512 * 1024 + +_INTEGER_KINDS = frozenset("iu") +_MONEY_KINDS_ALLOWED = frozenset("iuf") +_RESULT_TOKEN = object() + + +class PufGrowthRefusalError(ValueError): + """Sanitized field/reason refusal; never carries row values or identifiers.""" + + def __init__(self, reason: str, field: str = "contract") -> None: + self.reason = reason + self.field = field + super().__init__(f"{field}: {reason}") + + +def _require(condition: bool, reason: str, field: str = "contract") -> None: + if not condition: + raise PufGrowthRefusalError(reason, field) + + +def _sha(value: bytes) -> str: + return sha256(value).hexdigest() + + +def _json(value: object) -> bytes: + return json.dumps( + value, sort_keys=True, separators=(",", ":"), allow_nan=False, ensure_ascii=True + ).encode("ascii") + + +def _parse(document: bytes) -> dict: + """Parse a bounded JSON object, refusing duplicate keys and non-finite values.""" + _require( + type(document) is bytes and 0 < len(document) <= MAX_DOCUMENT_BYTES, + "DOCUMENT_SIZE_OR_TYPE", + ) + + def pairs(items): + result: dict = {} + for key, item in items: + _require(key not in result, "DUPLICATE_DOCUMENT_KEY") + result[key] = item + return result + + def constant(_): + raise PufGrowthRefusalError("NONFINITE_DOCUMENT") + + try: + parsed = json.loads(document, object_pairs_hook=pairs, parse_constant=constant) + except PufGrowthRefusalError: + raise + except (ValueError, RecursionError) as error: + raise PufGrowthRefusalError("MALFORMED_DOCUMENT") from error + _require(type(parsed) is dict, "DOCUMENT_OBJECT_REQUIRED") + return parsed + + +def _digest(value: object) -> bool: + return ( + type(value) is str + and len(value) == 64 + and all(character in "0123456789abcdef" for character in value) + ) + + +def _identifier(value: object, reason: str, field: str = "contract") -> str: + _require( + type(value) is str and 0 < len(value) <= 128 and value.strip() == value, + reason, + field, + ) + return value # type: ignore[return-value] + + +def _year(value: object, reason: str, field: str = "contract") -> int: + _require(type(value) is int and 1900 <= value <= 2200, reason, field) + return value # type: ignore[return-value] + + +class FieldRole(StrEnum): + """What a consumed raw field is, which fixes what may happen to it.""" + + MONEY = "money" + CODE = "code" + IDENTIFIER = "identifier" + COUNT = "count" + DESIGN_WEIGHT = "design_weight" + + +class GrowthKind(StrEnum): + """How a declared factor is meant, which is not recoverable from its value.""" + + #: Restates the same nominal amount into another year's purchasing power. + NOMINAL_PRICE_RESTATEMENT = "nominal_price_restatement" + #: Ages an amount per return: an aggregate total ratio divided by a return + #: count ratio, so it carries real change as well as price change. + REAL_PER_RETURN_AGING = "real_per_return_aging" + #: Grows the design weight itself. Never applied to money. + DESIGN_WEIGHT_GROWTH = "design_weight_growth" + #: Declared and deliberately untouched. + NONE = "none" + + +class SignBranch(StrEnum): + """Which rows of a money field a rule may select.""" + + ANY = "any" + POSITIVE = "positive" + NEGATIVE = "negative" + + +class FactorAuthority(StrEnum): + """What a factor table claims about itself.""" + + INVENTED_FIXTURE = "invented_fixture_nonauthority" + #: Levels really read from a named public publisher, for a bounded + #: developmental experiment. Never release-eligible; see + #: :data:`RELEASE_ELIGIBLE_AUTHORITIES`. + DEVELOPMENTAL_PUBLIC_RESOURCE = "developmental_public_resource" + REVIEWED_RESOURCE = "reviewed_resource" + + +#: The only authority a release recipe may stand behind. Written as a set so +#: the invariant is one readable membership test rather than a comparison +#: repeated at each site, and so a test can assert its contents directly. +RELEASE_ELIGIBLE_AUTHORITIES = frozenset({FactorAuthority.REVIEWED_RESOURCE}) + + +_MONEY_KINDS = frozenset( + {GrowthKind.NOMINAL_PRICE_RESTATEMENT, GrowthKind.REAL_PER_RETURN_AGING} +) +_PRESERVED_ROLES = frozenset({FieldRole.CODE, FieldRole.IDENTIFIER, FieldRole.COUNT}) + +#: How a grown money column's valuation is named, per growth kind. The string +#: goes into :class:`~microcosm.build.monetary_targets.MonetaryBasis`, so a +#: later fit-boundary check compares a restated amount against an aged amount +#: as the different things they are. +GROWN_MONEY_VALUATION = MappingProxyType( + { + GrowthKind.NOMINAL_PRICE_RESTATEMENT: "nominal_price_restated_from_{source}", + GrowthKind.REAL_PER_RETURN_AGING: "real_per_return_aged_from_{source}", + } +) + + +@dataclass(frozen=True) +class GrowthRule: + """One rule: which rows of which field grow by which factor series. + + Attributes: + field: The raw source column this rule governs. + role: What the field is. Only :attr:`FieldRole.MONEY` may carry a sign + branch, and only money and the design weight may grow at all. + source_reference_year: The year the observed amount belongs to. + target_year: The year the output amount is stated in. Preserved fields + retain ``source_reference_year``; growing roles may also use that year. + growth_kind: What the factor means. Fixed by the role for every role + except money, which must choose restatement or aging explicitly. + sign_branch: Which rows this rule selects. ``ANY`` must be the field's + only rule; ``POSITIVE`` and ``NEGATIVE`` must both be present. + factor_series: The exact variable-row name in the factor table. Empty + only for a declared no-growth field. + citation: Where the field's meaning was read. Descriptive. + """ + + field: str + role: FieldRole + source_reference_year: int + target_year: int + growth_kind: GrowthKind + sign_branch: SignBranch = SignBranch.ANY + factor_series: str = "" + citation: str = "" + + def __post_init__(self) -> None: + field = _identifier(self.field, "RULE_FIELD_NAME") + _require(type(self.role) is FieldRole, "RULE_ROLE", field) + _require(type(self.growth_kind) is GrowthKind, "RULE_GROWTH_KIND", field) + _require(type(self.sign_branch) is SignBranch, "RULE_SIGN_BRANCH", field) + _require(type(self.citation) is str, "RULE_CITATION", field) + source = _year(self.source_reference_year, "RULE_SOURCE_YEAR", field) + target = _year(self.target_year, "RULE_TARGET_YEAR", field) + if self.role is FieldRole.MONEY: + _require(self.growth_kind in _MONEY_KINDS, "MONEY_GROWTH_KIND", field) + elif self.role is FieldRole.DESIGN_WEIGHT: + _require( + self.growth_kind is GrowthKind.DESIGN_WEIGHT_GROWTH, + "WEIGHT_GROWTH_KIND", + field, + ) + _require(self.sign_branch is SignBranch.ANY, "WEIGHT_SIGN_BRANCH", field) + else: + _require( + self.growth_kind is GrowthKind.NONE, "PRESERVED_GROWTH_KIND", field + ) + _require(self.sign_branch is SignBranch.ANY, "PRESERVED_SIGN_BRANCH", field) + _require(source == target, "PRESERVED_YEAR_SPAN", field) + if self.growth_kind is GrowthKind.NONE: + _require(self.factor_series == "", "PRESERVED_FACTOR_SERIES", field) + else: + _identifier(self.factor_series, "RULE_FACTOR_SERIES", field) + _require(target >= source, "RULE_YEAR_ORDER", field) + + @property + def grows(self) -> bool: + """Whether this rule multiplies anything at all.""" + return self.growth_kind is not GrowthKind.NONE + + def as_document(self) -> dict: + """Canonical, hashable description of this rule.""" + return { + "field": self.field, + "role": str(self.role), + "source_reference_year": self.source_reference_year, + "target_year": self.target_year, + "growth_kind": str(self.growth_kind), + "sign_branch": str(self.sign_branch), + "factor_series": self.factor_series, + } + + +@dataclass(frozen=True) +class GrowthRecipe: + """A closed set of rules covering every consumed field exactly once. + + Coverage is validated here, before any table is seen: a money field is + covered either by one ``ANY`` rule or by exactly one ``POSITIVE`` and one + ``NEGATIVE`` rule, and never by both shapes. That is the structural refusal + of the archived overlap, where a sign-branched field also sat in a fallback + group and could receive a second factor. + """ + + recipe_id: str + rules: tuple[GrowthRule, ...] + + def __post_init__(self) -> None: + _identifier(self.recipe_id, "RECIPE_ID") + _require( + type(self.rules) is tuple + and bool(self.rules) + and all(type(rule) is GrowthRule for rule in self.rules), + "RECIPE_RULES", + ) + seen: set[tuple[str, SignBranch]] = set() + branches: dict[str, set[SignBranch]] = {} + shape: dict[str, tuple[FieldRole, int, int, GrowthKind]] = {} + for rule in self.rules: + key = (rule.field, rule.sign_branch) + _require(key not in seen, "DUPLICATE_FIELD_RULE", rule.field) + seen.add(key) + branches.setdefault(rule.field, set()).add(rule.sign_branch) + declaration = ( + rule.role, + rule.source_reference_year, + rule.target_year, + rule.growth_kind, + ) + if rule.field in shape: + _require( + shape[rule.field] == declaration, + "AMBIGUOUS_FIELD_DECLARATION", + rule.field, + ) + else: + shape[rule.field] = declaration + for name, present in branches.items(): + if present == {SignBranch.ANY}: + continue + _require(SignBranch.ANY not in present, "OVERLAPPING_SIGN_RULES", name) + _require( + present == {SignBranch.POSITIVE, SignBranch.NEGATIVE}, + "INCOMPLETE_SIGN_COVER", + name, + ) + for rule in self.rules: + _require( + rule.field not in PROVENANCE_COLUMNS, + "PROVENANCE_COLUMN_NAME", + rule.field, + ) + money = [rule for rule in self.rules if rule.role is FieldRole.MONEY] + _require(bool(money), "NO_MONEY_FIELD") + identifiers = [ + rule.field for rule in self.rules if rule.role is FieldRole.IDENTIFIER + ] + _require(len(set(identifiers)) == 1, "PROVENANCE_RECID_FIELD") + _require( + any(rule.field == SOURCE_AGI_FIELD for rule in money), + "PROVENANCE_AGI_FIELD", + SOURCE_AGI_FIELD, + ) + money_targets = {rule.target_year for rule in money} + _require(len(money_targets) == 1, "MIXED_TARGET_YEAR") + _require( + len({rule.source_reference_year for rule in money}) == 1, + "MIXED_SOURCE_REFERENCE_YEAR", + ) + weights = [rule for rule in self.rules if rule.role is FieldRole.DESIGN_WEIGHT] + _require(len(weights) <= 1, "MULTIPLE_DESIGN_WEIGHTS") + if weights and money_targets: + _require( + weights[0].target_year == next(iter(money_targets)), + "WEIGHT_TARGET_YEAR", + weights[0].field, + ) + object.__setattr__(self, "rules", tuple(sorted(self.rules, key=_rule_order))) + + @property + def fields(self) -> tuple[str, ...]: + """Every declared field name, once each, in the rules' sorted order. + + ``__post_init__`` sorts the rules by ``(field, sign_branch)``, so two + recipes that declare the same rules in a different order are the same + recipe and hash the same. + """ + ordered: list[str] = [] + for rule in self.rules: + if rule.field not in ordered: + ordered.append(rule.field) + return tuple(ordered) + + def rules_for(self, field: str) -> tuple[GrowthRule, ...]: + """Every rule governing ``field``.""" + return tuple(rule for rule in self.rules if rule.field == field) + + @property + def identity(self) -> bytes: + """Canonical bytes identifying this recipe.""" + return _json( + { + "schema_version": RECIPE_SCHEMA_VERSION, + "recipe_id": self.recipe_id, + "rules": [rule.as_document() for rule in self.rules], + } + ) + + +def parse_growth_recipe(document: bytes) -> GrowthRecipe: + """Rebuild a recipe from its own :attr:`GrowthRecipe.identity` bytes. + + Round-tripping through the identity document is how a recipe travels as a + graph node parameter without becoming an opaque blob: the bytes a node + carries are exactly the bytes the recipe hashes to. + + Args: + document: A recipe identity document. + + Returns: + The recipe those bytes describe. + + Raises: + PufGrowthRefusalError: If the document is malformed, is a different + schema version, or does not round-trip to itself. + """ + data = _parse(document) + _require( + set(data) == {"schema_version", "recipe_id", "rules"}, "RECIPE_DOCUMENT_SCHEMA" + ) + _require(data["schema_version"] == RECIPE_SCHEMA_VERSION, "RECIPE_SCHEMA_VERSION") + _require( + type(data["rules"]) is list and bool(data["rules"]), "RECIPE_DOCUMENT_RULES" + ) + rules = [] + for entry in data["rules"]: + _require( + type(entry) is dict + and set(entry) + == { + "field", + "role", + "source_reference_year", + "target_year", + "growth_kind", + "sign_branch", + "factor_series", + }, + "RECIPE_DOCUMENT_RULES", + ) + for key, values in ( + ("role", FieldRole), + ("growth_kind", GrowthKind), + ("sign_branch", SignBranch), + ): + _require( + entry[key] in tuple(str(value) for value in values), + "RECIPE_DOCUMENT_RULES", + entry["field"] if type(entry["field"]) is str else "contract", + ) + rules.append( + GrowthRule( + entry["field"], + FieldRole(entry["role"]), + entry["source_reference_year"], + entry["target_year"], + GrowthKind(entry["growth_kind"]), + SignBranch(entry["sign_branch"]), + entry["factor_series"], + ) + ) + recipe = GrowthRecipe(data["recipe_id"], tuple(rules)) + _require(recipe.identity == document, "RECIPE_DOCUMENT_ROUNDTRIP") + return recipe + + +def _rule_order(rule: GrowthRule) -> tuple[str, str]: + return (rule.field, str(rule.sign_branch)) + + +@dataclass(frozen=True) +class FactorSeries: + """One variable row of the factor table: a level per year column. + + Levels are index levels, not ratios. A factor is always a ratio of two + explicitly named year columns of one explicitly named row, so a table + cannot be read as if its year columns were variable names. + """ + + name: str + kind: GrowthKind + levels: tuple[tuple[int, str], ...] + + def __post_init__(self) -> None: + name = _identifier(self.name, "SERIES_NAME") + _require(type(self.kind) is GrowthKind, "SERIES_KIND", name) + _require(self.kind is not GrowthKind.NONE, "SERIES_KIND", name) + _require( + type(self.levels) is tuple and bool(self.levels), "SERIES_LEVELS", name + ) + _require( + all(type(pair) in (tuple, list) and len(pair) == 2 for pair in self.levels), + "SERIES_LEVELS", + name, + ) + # Own inner pairs too: frozen dataclasses do not freeze a caller's lists. + object.__setattr__(self, "levels", tuple(tuple(pair) for pair in self.levels)) + years = [year for year, _ in self.levels] + _require(len(set(years)) == len(years), "DUPLICATE_SERIES_YEAR", name) + _require(years == sorted(years), "UNORDERED_SERIES_YEARS", name) + for year, level in self.levels: + _year(year, "SERIES_YEAR", name) + _require(type(level) is str and level.strip() == level, "LEVEL_TEXT", name) + _require(_finite_positive(level), "LEVEL_VALUE", name) + + def level_text(self, year: int) -> str: + """The exact level text in the ``year`` column of this variable row.""" + for candidate, level in self.levels: + if candidate == year: + return level + raise PufGrowthRefusalError("MISSING_FACTOR_YEAR", self.name) + + @property + def years(self) -> tuple[int, ...]: + return tuple(year for year, _ in self.levels) + + +def _finite_positive(text: str) -> bool: + try: + value = float(text) + except ValueError: + return False + return bool(np.isfinite(value)) and value > 0.0 + + +def _freeze_provenance(value: object, depth: int = 0) -> object: + """Own a bounded JSON tree, including nested caller-supplied containers.""" + _require(depth <= 64, "TABLE_PROVENANCE_DEPTH") + if isinstance(value, Mapping): + _require(all(type(key) is str for key in value), "TABLE_PROVENANCE_KEY") + return MappingProxyType( + {key: _freeze_provenance(item, depth + 1) for key, item in value.items()} + ) + if type(value) in (tuple, list): + return tuple(_freeze_provenance(item, depth + 1) for item in value) + _require( + value is None or type(value) in (str, int, float, bool), + "TABLE_PROVENANCE_VALUE", + ) + if type(value) is float: + _require(bool(np.isfinite(value)), "TABLE_PROVENANCE_VALUE") + return value + + +def _provenance_document(value: object) -> object: + """Return detached JSON containers, never references into the owned tree.""" + if isinstance(value, Mapping): + return {key: _provenance_document(item) for key, item in value.items()} + if type(value) is tuple: + return [_provenance_document(item) for item in value] + return value + + +@dataclass(frozen=True) +class GrowthFactorTable: + """A parsed factor document: variable rows by year columns, plus authority. + + The document is retained verbatim so the table's identity is the bytes a + reviewer would read, not a re-serialization of this object's state. + """ + + document: bytes + table_id: str + authority: FactorAuthority + years: tuple[int, ...] + series: tuple[FactorSeries, ...] + provenance: Mapping[str, object] + + def __post_init__(self) -> None: + _require( + type(self.document) is bytes + and 0 < len(self.document) <= MAX_DOCUMENT_BYTES, + "DOCUMENT_SIZE_OR_TYPE", + ) + _identifier(self.table_id, "TABLE_ID") + _require(type(self.authority) is FactorAuthority, "TABLE_AUTHORITY") + _require( + type(self.series) is tuple + and bool(self.series) + and all(type(item) is FactorSeries for item in self.series), + "TABLE_SERIES", + ) + names = [item.name for item in self.series] + _require(len(set(names)) == len(names), "DUPLICATE_SERIES") + _require(type(self.years) is tuple and bool(self.years), "TABLE_YEARS") + for year in self.years: + _year(year, "TABLE_YEARS") + _require(list(self.years) == sorted(set(self.years)), "TABLE_YEARS") + for item in self.series: + _require(item.years == self.years, "SERIES_YEAR_COVERAGE", item.name) + _require(isinstance(self.provenance, Mapping), "TABLE_PROVENANCE") + object.__setattr__(self, "provenance", _freeze_provenance(self.provenance)) + # Re-serializing proves the retained bytes are the canonical document + # for exactly this parsed state, so no field can drift from the bytes. + _require(self.document == _json(self.as_document()), "TABLE_DOCUMENT") + + @classmethod + def from_bytes(cls, document: bytes) -> GrowthFactorTable: + """Parse a factor document, refusing anything it does not fully close.""" + data = _parse(document) + _require( + set(data) + == { + "authority", + "provenance", + "schema_version", + "series", + "table_id", + "years", + }, + "TABLE_SCHEMA", + ) + _require( + data["schema_version"] == FACTOR_TABLE_SCHEMA_VERSION, + "TABLE_SCHEMA_VERSION", + ) + _require( + data["authority"] in tuple(str(value) for value in FactorAuthority), + "TABLE_AUTHORITY", + ) + _require(type(data["years"]) is list and bool(data["years"]), "TABLE_YEARS") + years = tuple(_year(value, "TABLE_YEARS") for value in data["years"]) + _require(type(data["series"]) is dict and bool(data["series"]), "TABLE_SERIES") + _require(type(data["provenance"]) is dict, "TABLE_PROVENANCE") + series = [] + for name in sorted(data["series"]): + entry = data["series"][name] + _require( + type(entry) is dict and set(entry) == {"kind", "levels"}, + "SERIES_SCHEMA", + name, + ) + _require( + entry["kind"] in tuple(str(value) for value in GrowthKind), + "SERIES_KIND", + name, + ) + levels = entry["levels"] + _require(type(levels) is dict and bool(levels), "SERIES_LEVELS", name) + _require( + set(levels) == {str(year) for year in years}, + "SERIES_YEAR_COVERAGE", + name, + ) + series.append( + FactorSeries( + name, + GrowthKind(entry["kind"]), + tuple((year, levels[str(year)]) for year in years), + ) + ) + return cls( + document, + data["table_id"], + FactorAuthority(data["authority"]), + years, + tuple(series), + data["provenance"], + ) + + def as_document(self) -> dict: + """The canonical document for this table's parsed state.""" + return { + "authority": str(self.authority), + "provenance": _provenance_document(self.provenance), + "schema_version": FACTOR_TABLE_SCHEMA_VERSION, + "series": { + item.name: { + "kind": str(item.kind), + "levels": {str(year): level for year, level in item.levels}, + } + for item in self.series + }, + "table_id": self.table_id, + "years": list(self.years), + } + + def series_for(self, name: str) -> FactorSeries: + """The variable row called ``name``; refuses a year column or a miss.""" + for item in self.series: + if item.name == name: + return item + raise PufGrowthRefusalError("UNKNOWN_FACTOR_SERIES", name) + + @property + def sha256(self) -> str: + return _sha(self.document) + + +def _check_developmental_provenance(factors: GrowthFactorTable) -> None: + """Close what a ``developmental_public_resource`` table must state. + + Everything the reviewed shape owes, plus the digest and retrieval time of + the exact resource bytes, plus the two literals that keep a price index + from being read as a per-return total. The reviewed allowlist is checked + negatively here: a developmental digest listed there is a refusal, so the + one act that promotes a reviewed table cannot promote this one. + """ + _require( + set(factors.provenance) == DEVELOPMENTAL_PROVENANCE_KEYS, + "DEVELOPMENTAL_PROVENANCE_SHAPE", + ) + _require( + factors.sha256 not in REVIEWED_FACTOR_TABLE_DIGESTS, + "DEVELOPMENTAL_TABLE_IN_REVIEWED_ALLOWLIST", + ) + for key, expected in DEVELOPMENTAL_PROVENANCE_LITERALS.items(): + _require(factors.provenance[key] == expected, "DEVELOPMENTAL_PROVENANCE_VALUE") + _require( + _digest(factors.provenance["resource_sha256"]), "DEVELOPMENTAL_RESOURCE_DIGEST" + ) + url = factors.provenance["source_url"] + _require( + type(url) is str and url.startswith("https://") and len(url) > len("https://"), + "DEVELOPMENTAL_SOURCE_URL", + ) + retrieved = factors.provenance["retrieved_utc"] + _require(type(retrieved) is str, "DEVELOPMENTAL_RETRIEVED_UTC") + try: + stamp = datetime.datetime.fromisoformat(retrieved) + except ValueError as error: + raise PufGrowthRefusalError("DEVELOPMENTAL_RETRIEVED_UTC") from error + _require( + stamp.tzinfo is not None and stamp.utcoffset() == datetime.timedelta(0), + "DEVELOPMENTAL_RETRIEVED_UTC", + ) + + +@dataclass(frozen=True) +class ResolvedFactor: + """One rule's factor, resolved once from two named year columns.""" + + field: str + sign_branch: SignBranch + series: str + kind: GrowthKind + source_year: int + target_year: int + source_level: str + target_level: str + ratio_bits: str + + def __post_init__(self) -> None: + field = _identifier(self.field, "RESOLVED_FIELD") + _require(type(self.sign_branch) is SignBranch, "RESOLVED_SIGN_BRANCH", field) + _require( + type(self.kind) is GrowthKind and self.kind is not GrowthKind.NONE, + "RESOLVED_KIND", + field, + ) + _identifier(self.series, "RESOLVED_SERIES", field) + _year(self.source_year, "RESOLVED_SOURCE_YEAR", field) + _year(self.target_year, "RESOLVED_TARGET_YEAR", field) + # The stored bits are the factor. Recomputing them here means a + # hand-built ResolvedFactor cannot claim a factor its own levels do not + # produce, so the bits are evidence rather than an assertion. + expected = _factor_ratio_bits(self.source_level, self.target_level, field) + _require(self.ratio_bits == expected, "RESOLVED_RATIO_BITS", field) + + @property + def value(self) -> float: + """The exact float64 this factor is, recovered from its bit pattern.""" + return struct.unpack(" dict: + return { + "field": self.field, + "kind": str(self.kind), + "ratio_bits": self.ratio_bits, + "series": self.series, + "sign_branch": str(self.sign_branch), + "source_level": self.source_level, + "source_year": self.source_year, + "target_level": self.target_level, + "target_year": self.target_year, + } + + +@dataclass(frozen=True) +class CompiledPufGrowth: + """A recipe bound to a factor table, with every factor already resolved.""" + + recipe: GrowthRecipe + factors: GrowthFactorTable + resolved: tuple[ResolvedFactor, ...] + identity: bytes + + def __post_init__(self) -> None: + _require(type(self.recipe) is GrowthRecipe, "RECIPE_TYPE") + _require(type(self.factors) is GrowthFactorTable, "FACTOR_TABLE_TYPE") + # The authority gate lives here, not only in the compile function, so + # that no construction path can produce a release-eligible contract the + # compile chokepoint would have refused. + if self.factors.authority is FactorAuthority.REVIEWED_RESOURCE: + _require( + self.factors.sha256 in REVIEWED_FACTOR_TABLE_DIGESTS, + "UNREVIEWED_FACTOR_TABLE", + ) + _require( + set(self.factors.provenance) == REVIEWED_PROVENANCE_KEYS, + "REVIEWED_PROVENANCE_SHAPE", + ) + elif self.factors.authority is FactorAuthority.DEVELOPMENTAL_PUBLIC_RESOURCE: + _check_developmental_provenance(self.factors) + _require( + type(self.resolved) is tuple + and all(type(item) is ResolvedFactor for item in self.resolved), + "RESOLVED_TYPE", + ) + # Exactly one resolved factor per growing rule, agreeing with it, and + # none left over: a directly constructed contract cannot describe a + # different computation than its own recipe declares. + expected = tuple( + ( + rule.field, + rule.sign_branch, + rule.factor_series, + rule.growth_kind, + rule.source_reference_year, + rule.target_year, + ) + for rule in self.recipe.rules + if rule.grows + ) + actual = tuple( + ( + factor.field, + factor.sign_branch, + factor.series, + factor.kind, + factor.source_year, + factor.target_year, + ) + for factor in self.resolved + ) + _require(actual == expected, "RESOLVED_COVERAGE") + _require( + self.resolved == _resolve_factors(self.recipe, self.factors), + "RESOLVED_FACTOR_BINDING", + ) + _require( + type(self.identity) is bytes + and self.identity + == _contract_identity(self.recipe, self.factors, self.resolved), + "CONTRACT_BINDING", + ) + + @property + def sha256(self) -> str: + return _sha(self.identity) + + @property + def release_eligible(self) -> bool: + """Whether this contract may stand behind a release recipe. + + ``False`` for every invented fixture and every developmental public + resource, and there is no override: the answer is membership of + :data:`RELEASE_ELIGIBLE_AUTHORITIES`, which no caller, flag or + allowlist entry can widen. + """ + return self.factors.authority in RELEASE_ELIGIBLE_AUTHORITIES + + @property + def money_fields(self) -> tuple[str, ...]: + """Declared money fields, once each, in rule order.""" + return tuple( + dict.fromkeys( + rule.field for rule in self.recipe.rules if rule.role is FieldRole.MONEY + ) + ) + + @property + def money_view_fields(self) -> tuple[str, ...]: + """The columns :func:`apply_puf_growth` consumes: everything but weights. + + The design weight is excluded so a money transform neither reads nor + depends on it. In the graph that keeps the money node and the + design-weight node siblings rather than one ordering the other. + """ + weight = self.design_weight_rule + return tuple( + name + for name in self.recipe.fields + if weight is None or name != weight.field + ) + + @property + def preserved_fields(self) -> tuple[str, ...]: + """Declared code/identifier/count fields, which this contract never writes.""" + return tuple( + dict.fromkeys( + rule.field + for rule in self.recipe.rules + if rule.role in _PRESERVED_ROLES + ) + ) + + @property + def design_weight_rule(self) -> GrowthRule | None: + for rule in self.recipe.rules: + if rule.role is FieldRole.DESIGN_WEIGHT: + return rule + return None + + @property + def target_year(self) -> int: + for rule in self.recipe.rules: + if rule.role is FieldRole.MONEY: + return rule.target_year + raise PufGrowthRefusalError("NO_MONEY_FIELD") + + def factor_for(self, field: str, sign_branch: SignBranch) -> ResolvedFactor: + for factor in self.resolved: + if factor.field == field and factor.sign_branch is sign_branch: + return factor + raise PufGrowthRefusalError("NO_RESOLVED_FACTOR", field) + + +def compile_puf_growth( + recipe: GrowthRecipe, factors: GrowthFactorTable +) -> CompiledPufGrowth: + """Bind a recipe to a table using the same resolver as constructor validation. + + Args: + recipe: The closed rule set. Already coverage-validated. + factors: The factor table the rules name. + + Returns: + An immutable compiled contract carrying its own identity bytes. + + Raises: + PufGrowthRefusalError: If the table's authority is not compilable, if a + rule names a series the table does not have, if a named series + means a different growth kind than the rule declares, or if either + declared year is not a column of that series. + """ + _require(type(recipe) is GrowthRecipe, "RECIPE_TYPE") + _require(type(factors) is GrowthFactorTable, "FACTOR_TABLE_TYPE") + resolved = _resolve_factors(recipe, factors) + return CompiledPufGrowth( + recipe, factors, resolved, _contract_identity(recipe, factors, resolved) + ) + + +def _factor_ratio_bits(source_level: str, target_level: str, field: str) -> str: + """One finite positive float64 ratio for compiler and public constructors.""" + _require( + all( + type(level) is str and level.strip() == level and _finite_positive(level) + for level in (source_level, target_level) + ), + "RESOLVED_LEVEL", + field, + ) + ratio = float(target_level) / float(source_level) + _require(bool(np.isfinite(ratio)) and ratio > 0.0, "FACTOR_VALUE", field) + return struct.pack(" tuple[ResolvedFactor, ...]: + """Resolve the complete recipe from the table's immutable named series.""" + resolved = [] + for rule in recipe.rules: + if not rule.grows: + continue + series = factors.series_for(rule.factor_series) + _require(series.kind is rule.growth_kind, "FACTOR_KIND_MISMATCH", rule.field) + source_level = series.level_text(rule.source_reference_year) + target_level = series.level_text(rule.target_year) + resolved.append( + ResolvedFactor( + rule.field, + rule.sign_branch, + series.name, + rule.growth_kind, + rule.source_reference_year, + rule.target_year, + source_level, + target_level, + _factor_ratio_bits(source_level, target_level, rule.field), + ) + ) + return tuple(resolved) + + +def _contract_identity( + recipe: GrowthRecipe, + factors: GrowthFactorTable, + resolved: tuple[ResolvedFactor, ...], +) -> bytes: + """Canonical bytes identifying one recipe bound to one factor table.""" + return _json( + { + "schema_version": RECIPE_SCHEMA_VERSION, + "recipe": json.loads(recipe.identity), + "factor_table_sha256": factors.sha256, + "factor_table_id": factors.table_id, + "factor_authority": str(factors.authority), + "resolved": [factor.as_document() for factor in resolved], + } + ) + + +def puf_growth_monetary_metadata( + compiled: CompiledPufGrowth, +) -> Mapping[str, MonetaryBasis]: + """Declared output basis per money field, for a later fit-boundary check. + + The check itself is not in this slice. What is here is the metadata it + needs: currency, target period, and a valuation string that distinguishes a + nominal restatement from a real per-return aging, so a donor field and a + recipient predictor cannot be silently matched across different bases. + + Args: + compiled: A compiled contract. + + Returns: + Money field name to its declared output + :class:`~microcosm.build.monetary_targets.MonetaryBasis`. + """ + _require(type(compiled) is CompiledPufGrowth, "COMPILED_TYPE") + result: dict[str, MonetaryBasis] = {} + for rule in compiled.recipe.rules: + if rule.role is not FieldRole.MONEY or rule.field in result: + continue + result[rule.field] = MonetaryBasis( + currency="USD", + unit="base_currency", + period=str(rule.target_year), + temporal_basis="annual_flow", + sector="households", + perimeter="irs_individual_income_tax_returns", + valuation=GROWN_MONEY_VALUATION[rule.growth_kind].format( + source=rule.source_reference_year + ), + ) + return MappingProxyType(result) + + +@dataclass(frozen=True, eq=False) +class GrownPufTable: + """A grown table and its receipt; only :func:`apply_puf_growth` builds one.""" + + table: pd.DataFrame + contract_sha256: str + _receipt: bytes + _token: InitVar[object] = None + + def __post_init__(self, _token: object) -> None: + _require(_token is _RESULT_TOKEN, "RESULT_CONSTRUCTOR_UNAVAILABLE") + + @property + def receipt(self) -> dict: + """The parsed receipt describing exactly what was applied.""" + return _parse(self._receipt) + + @property + def receipt_bytes(self) -> bytes: + return self._receipt + + +def _checked_source_table( + table: pd.DataFrame, compiled: CompiledPufGrowth +) -> pd.DataFrame: + _require(type(table) is pd.DataFrame, "SOURCE_TABLE_TYPE") + rows = len(table) + _require(0 < rows <= MAX_ROWS, "SOURCE_TABLE_ROWS") + _require(all(type(name) is str for name in table.columns), "SOURCE_TABLE_COLUMNS") + _require(len(set(table.columns)) == len(table.columns), "DUPLICATE_SOURCE_COLUMN") + declared = set(compiled.money_view_fields) + present = set(table.columns) + weight = compiled.design_weight_rule + if weight is not None and weight.field in present: + raise PufGrowthRefusalError("DESIGN_WEIGHT_NOT_IN_MONEY_VIEW", weight.field) + missing = sorted(declared - present) + if missing: + raise PufGrowthRefusalError("MISSING_DECLARED_FIELD", missing[0]) + undeclared = sorted(present - declared) + if undeclared: + raise PufGrowthRefusalError("UNDECLARED_SOURCE_COLUMN", undeclared[0]) + return table + + +def _money_values(table: pd.DataFrame, field: str) -> np.ndarray: + """A money column as float64, refusing anything it would have to coerce.""" + series = table[field] + # An explicit kind allow-list rather than ``is_numeric_dtype``, which also + # admits complex, whose imaginary part a float64 cast would drop in silence. + _require( + series.dtype.kind in _MONEY_KINDS_ALLOWED + or ( + pd.api.types.is_extension_array_dtype(series.dtype) + and str(series.dtype).lower().startswith(("int", "uint", "float")) + ), + "FIELD_DTYPE", + field, + ) + _require(not series.isna().any(), "FIELD_MISSING_VALUE", field) + integral = series.dtype.kind in _INTEGER_KINDS or ( + pd.api.types.is_extension_array_dtype(series.dtype) + and str(series.dtype).lower().startswith(("int", "uint")) + ) + if integral: + # float64 is the working type, so an integer amount it cannot represent + # exactly is refused rather than silently rounded. + _require( + bool((np.abs(series.to_numpy(dtype=object)) <= MAX_EXACT_AMOUNT).all()), + "MONEY_MAGNITUDE", + field, + ) + values = series.to_numpy(dtype=np.float64, copy=True) + _require(bool(np.isfinite(values).all()), "FIELD_NONFINITE", field) + # A negative zero would pass every arithmetic guard untouched and then + # compare equal to zero while carrying a different bit pattern. + _require(not bool(np.signbit(values[values == 0.0]).any()), "NEGATIVE_ZERO", field) + return values + + +def _integral_values(table: pd.DataFrame, field: str, reason: str) -> np.ndarray: + series = table[field] + _require( + series.dtype.kind in _INTEGER_KINDS + or ( + pd.api.types.is_extension_array_dtype(series.dtype) + and str(series.dtype).lower().startswith(("int", "uint")) + ), + reason, + field, + ) + _require(not series.isna().any(), "FIELD_MISSING_VALUE", field) + return series.to_numpy(copy=True) + + +def _selection(values: np.ndarray, branch: SignBranch) -> np.ndarray: + """Rows this branch selects. Zero is never selected, so it never grows.""" + if branch is SignBranch.POSITIVE: + return values > 0.0 + if branch is SignBranch.NEGATIVE: + return values < 0.0 + return values != 0.0 + + +def require_transformable(compiled: CompiledPufGrowth) -> None: + """Check every precondition :func:`apply_puf_growth` needs, without a table. + + A caller that is about to declare work — a graph node factory, say — uses + this so a contract that could never run is refused before anything is + loaded, rather than at the moment the money transform reaches for a field + that is not there. + + Args: + compiled: The contract to check. + + Raises: + PufGrowthRefusalError: If the contract declares no money field, no + single identifier, or no source-year AGI field. + """ + _require(type(compiled) is CompiledPufGrowth, "COMPILED_TYPE") + _ = compiled.target_year + _source_reference_year(compiled) + _recid_field(compiled) + _agi_field(compiled) + + +def apply_puf_growth(table: pd.DataFrame, compiled: CompiledPufGrowth) -> GrownPufTable: + """Grow every declared money field once, preserving everything else. + + The input is never mutated: every refusal happens before any array is + written, and the output is a new frame built from copies. + + Args: + table: One row per raw source record, whose columns are exactly + :attr:`CompiledPufGrowth.money_view_fields` — every declared field + except the design weight. + compiled: The compiled contract. + + Row correspondence is positional, not by pandas label: the output carries a + fresh ``RangeIndex`` and row *i* of the output is row *i* of the input. A + caller that needs the input's labels back must reattach them. + + The input is never mutated and no partial result is ever returned. Most + refusals happen before any arithmetic; the product guards + (``GROWN_NONFINITE``, ``SIGN_CHANGED``, ``FACTOR_APPLIED_TWICE``) + necessarily run after a candidate array has been computed in local memory. + + Returns: + A :class:`GrownPufTable` whose frame carries the grown money columns, + the byte-identical code/identifier/count columns, and the two + provenance columns copied from the source before any growth. The design + weight is not part of this view at all; it grows only through + :func:`apply_puf_design_weight_growth`. + + Raises: + PufGrowthRefusalError: On any contract, dtype, value, or coverage + violation. Nothing is written when this is raised. + """ + _require(type(compiled) is CompiledPufGrowth, "COMPILED_TYPE") + source = _checked_source_table(table, compiled) + rows = len(source) + + # Validate every column and resolve every factor before writing anything. + grown: dict[str, np.ndarray] = {} + applications: dict[str, int] = {} + for name in compiled.money_view_fields: + rules = compiled.recipe.rules_for(name) + role = rules[0].role + if role is FieldRole.MONEY: + values = _money_values(source, name) + applied = np.zeros(rows, dtype=np.int64) + result = values.copy() + for rule in rules: + factor = compiled.factor_for(name, rule.sign_branch) + selected = _selection(values, rule.sign_branch) + applied += selected.astype(np.int64) + # The product is checked below; a warning here would only + # duplicate a refusal that is already explicit. + with np.errstate(over="ignore", under="ignore"): + result[selected] = values[selected] * factor.value + _require(bool((applied <= 1).all()), "FACTOR_APPLIED_TWICE", name) + _require(bool(np.isfinite(result).all()), "GROWN_NONFINITE", name) + _require( + bool((np.sign(result) == np.sign(values)).all()), "SIGN_CHANGED", name + ) + grown[name] = result + applications[name] = int(applied.sum()) + else: + _integral_values(source, name, "PRESERVED_FIELD_DTYPE") + + recid_field = _recid_field(compiled) + agi_field = _agi_field(compiled) + provenance_recid = _integral_values(source, recid_field, "PROVENANCE_RECID_DTYPE") + _require(len(np.unique(provenance_recid)) == rows, "DUPLICATE_RECID", recid_field) + provenance_agi = _money_values(source, agi_field) + + output = pd.DataFrame(index=pd.RangeIndex(rows)) + for name in source.columns: + output[name] = ( + pd.Series(grown[name], index=output.index, dtype="float64") + if name in grown + else source[name].reset_index(drop=True).copy(deep=True) + ) + output[PROVENANCE_RECID_COLUMN] = pd.Series( + provenance_recid, index=output.index, dtype=source[recid_field].dtype + ) + output[PROVENANCE_SOURCE_AGI_COLUMN] = pd.Series( + provenance_agi, index=output.index, dtype="float64" + ) + + receipt = _json( + { + "schema_version": RECIPE_SCHEMA_VERSION, + "artifact_kind": "microcosm.us_puf_growth", + "contract_sha256": _sha(compiled.identity), + "factor_authority": str(compiled.factors.authority), + "factor_table_sha256": compiled.factors.sha256, + "release_eligible": compiled.release_eligible, + "rows": rows, + "source_reference_year": _source_reference_year(compiled), + "target_year": compiled.target_year, + "grown_money_fields": sorted(grown), + "grown_row_counts": {name: applications[name] for name in sorted(grown)}, + "preserved_fields": sorted(compiled.preserved_fields), + "design_weight_field": ( + compiled.design_weight_rule.field + if compiled.design_weight_rule is not None + else None + ), + "design_weight_in_money_view": False, + "provenance_columns": list(PROVENANCE_COLUMNS), + "provenance_recid_source": recid_field, + "provenance_agi_source": agi_field, + } + ) + return GrownPufTable(output, _sha(compiled.identity), receipt, _token=_RESULT_TOKEN) + + +def apply_puf_design_weight_growth( + weights: Sequence[float] | np.ndarray, compiled: CompiledPufGrowth +) -> np.ndarray: + """Grow design weights by their own declared factor, and nothing else. + + Kept apart from :func:`apply_puf_growth` on purpose: money growth and + weight growth are different operations with different factor series, and a + single call that did both could silently move population with prices. + + Args: + weights: The source design weights, one per record. + compiled: The compiled contract, which must declare a design weight. + + Returns: + A new float64 array of grown weights. + + Raises: + PufGrowthRefusalError: If no design-weight rule is declared, or the + weights are not finite and positive. + """ + _require(type(compiled) is CompiledPufGrowth, "COMPILED_TYPE") + rule = compiled.design_weight_rule + _require(rule is not None, "NO_DESIGN_WEIGHT_RULE") + values = np.asarray(weights, dtype=np.float64) + _require(values.ndim == 1 and 0 < values.size <= MAX_ROWS, "WEIGHT_SHAPE") + _require(bool(np.isfinite(values).all()), "WEIGHT_NONFINITE") + _require(bool((values > 0.0).all()), "NONPOSITIVE_DESIGN_WEIGHT") + factor = compiled.factor_for(rule.field, SignBranch.ANY) + _require( + factor.kind is GrowthKind.DESIGN_WEIGHT_GROWTH, "WEIGHT_FACTOR_KIND", rule.field + ) + with np.errstate(over="ignore", under="ignore"): + grown = values * factor.value + _require(bool(np.isfinite(grown).all()), "GROWN_WEIGHT_NONFINITE", rule.field) + _require(bool((grown > 0.0).all()), "GROWN_WEIGHT_NONPOSITIVE", rule.field) + return grown + + +def _source_reference_year(compiled: CompiledPufGrowth) -> int: + years = { + rule.source_reference_year + for rule in compiled.recipe.rules + if rule.role is FieldRole.MONEY + } + _require(len(years) == 1, "MIXED_SOURCE_REFERENCE_YEAR") + return years.pop() + + +def _recid_field(compiled: CompiledPufGrowth) -> str: + """The one declared identifier. :class:`GrowthRecipe` already required it.""" + candidates = [ + rule.field + for rule in compiled.recipe.rules + if rule.role is FieldRole.IDENTIFIER + ] + _require(len(candidates) == 1, "PROVENANCE_RECID_FIELD") + return candidates[0] + + +def _agi_field(compiled: CompiledPufGrowth) -> str: + _require( + SOURCE_AGI_FIELD in compiled.recipe.fields, + "PROVENANCE_AGI_FIELD", + SOURCE_AGI_FIELD, + ) + return SOURCE_AGI_FIELD + + +@dataclass(frozen=True) +class RosterEntry: + """One raw TY2015 PUF field this lineage already names, and what it is. + + A roster entry states a **role** and a documented **meaning**, never a + factor. Which series a field should grow by is a producer decision that + belongs to a reviewed recipe, not to this file. + """ + + field: str + role: FieldRole + meaning: str + citation: str = "" + + def __post_init__(self) -> None: + _identifier(self.field, "ROSTER_FIELD") + _require(type(self.role) is FieldRole, "ROSTER_ROLE", self.field) + _require(bool(self.meaning), "ROSTER_MEANING", self.field) + + +#: The archived producer commit every historical claim in this module is read +#: from. The retired US data package is named by commit rather than by package +#: name because this repository's launch contract forbids naming it in the live +#: tree (``test_no_incumbent_data_package_references_in_live_tree``); paths in +#: the citations below are relative to that repository's Python package +#: directory. Read a blob with ``git show :/``. +ARCHIVED_PRODUCER_COMMIT = "42ed5d45c56df80d754fbe24cce21cfeb8d05cbe" + +#: Where the roster's field meanings come from. The archived file is a comment +#: block that itself cites an IRS booklet; that booklet was not read here, so a +#: meaning below is "the label the archived producer attached to this code", +#: not an independently authenticated IRS definition. +ROSTER_CITATION = ( + f"retired US data package @{ARCHIVED_PRODUCER_COMMIT}:" + "datasets/puf/aggregate_record_totals.yaml" + " (its own cited source: 2014 PUF General Description Booklet, pp. 33-36," + " June 2020)" +) + +_STRUCTURAL_CITATION = ( + f"retired US data package @{ARCHIVED_PRODUCER_COMMIT}:" + "datasets/puf/puf.py preprocess_puf" +) + +#: Field code to the meaning the archived codebook records for it. Money only. +_DOCUMENTED_MONEY_MEANINGS: tuple[tuple[str, str], ...] = ( + ("E00100", "AGI"), + ("E00200", "Wages and salaries"), + ("E00300", "Taxable interest"), + ("E00400", "Tax-exempt interest"), + ("E00600", "Ordinary dividends"), + ("E00650", "Qualified dividends"), + ("E00700", "State/local tax refund"), + ("E00800", "Alimony received"), + ("E00900", "Business income/loss"), + ("E01000", "Net capital gain/loss"), + ("E01100", "Capital gain distributions"), + ("E01200", "Other gains/losses"), + ("E01400", "Taxable IRA distributions"), + ("E01500", "Total pensions and annuities"), + ("E01700", "Taxable pensions and annuities"), + ("E02100", "Farm income/loss (Sch F)"), + ("E02300", "Unemployment compensation"), + ("E02400", "Social security benefits"), + ("E03150", "IRA deduction"), + ("E03210", "Student loan interest"), + ("E03220", "Educator expenses"), + ("E03230", "Tuition and fees"), + ("E03240", "Domestic production deduction"), + ("E03270", "Self-employed health insurance"), + ("E03290", "HSA deduction"), + ("E03300", "Self-employed SEP/SIMPLE"), + ("E03400", "Early withdrawal penalty"), + ("E03500", "Alimony paid"), + ("E07240", "Saver's credit"), + ("E07260", "Residential energy credit"), + ("E07300", "Foreign tax credit"), + ("E07400", "General business credit"), + ("E07600", "Prior year min tax credit"), + ("E09600", "AMT"), + ("E09700", "Recapture of investment credit"), + ("E09800", "Unreported SE tax"), + ("E09900", "Penalty on early withdrawal"), + ("E11200", "Excess FICA withheld"), + ("E17500", "Medical expenses"), + ("E18400", "State/local income tax"), + ("E18500", "Real estate taxes"), + ("E19200", "Interest paid"), + ("E19800", "Cash contributions"), + ("E20100", "Non-cash contributions"), + ("E20400", "Misc itemized deductions"), + ("E20500", "Casualty/theft loss"), + ("E22250", "Short-term capital gain/loss"), + ("E24515", "Unrecaptured Sec. 1250 gain"), + ("E24518", "Collectibles gain/loss"), + ("E25850", "Rental income (Sch E gross)"), + ("E25860", "Rental loss (Sch E gross)"), + ("E25940", "Partnership income (gross)"), + ("E25960", "Partnership loss (gross)"), + ("E25980", "Partnership net income"), + ("E26180", "S-corp loss (gross)"), + ("E26190", "S-corp net income"), + ("E26270", "Partnership/S-corp net"), + ("E26390", "Estate/trust income (gross)"), + ("E26400", "Estate/trust loss (gross)"), + ("E27200", "Farm rental income (Sch E)"), + ("E30400", "SE income (taxpayer)"), + ("E30500", "SE income (spouse)"), + ("E32800", "Child care credit expenses"), + ("E58990", "Investment interest (4952)"), + ("E62900", "AMT foreign tax credit"), + ("E87521", "American Opportunity Credit"), + ("P08000", "Other credits"), + ("P23250", "Long-term capital gain/loss"), + ("T27800", "Farm income (Sch J)"), +) + +_SOI_MEASURE_CITATION = ( + f"retired US data package @{ARCHIVED_PRODUCER_COMMIT}:utils/soi.py puf_measures" +) + +_SOI_RENAME_CITATION = ( + f"retired US data package @{ARCHIVED_PRODUCER_COMMIT}:" + "datasets/puf/uprate_puf.py SOI_TO_PUF_STRAIGHT_RENAMES" +) + +_SCREENED_FIELD_CITATION = ( + f"retired US data package @{ARCHIVED_PRODUCER_COMMIT}:" + "datasets/puf/aggregate_record_utils.py SCREENED_FIELDS" +) + +#: The raw TY2015 PUF fields this lineage names, with the role each one plays. +#: A **proposal**, not an authenticated raw-source manifest: every money code +#: the archived codebook describes, plus the four structural codes, the one +#: screened code, and five money codes the codebook omits that only the +#: archived SOI rename/measure tables name -- each cited to where its meaning +#: was actually read. It is not every code this repository reads, and not every +#: code the delivered file contains. It declares no factor for any field. +#: Adding or removing a field is a reviewed change, not something a recipe may +#: do implicitly. +PROPOSED_TY2015_FIELD_ROSTER: tuple[RosterEntry, ...] = ( + RosterEntry( + SOURCE_RECID_FIELD, + FieldRole.IDENTIFIER, + "Record id (the archived producer copies it to household_id)", + _STRUCTURAL_CITATION, + ), + RosterEntry( + DESIGN_WEIGHT_FIELD, + FieldRole.DESIGN_WEIGHT, + "Sample weight, stored times 100 (the archived producer divides by 100)", + _STRUCTURAL_CITATION, + ), + RosterEntry( + "MARS", + FieldRole.CODE, + "Filing status: 0 marks four aggregate disclosure records; " + "1 single, 2 joint, 3 separate, 4 head of household", + _STRUCTURAL_CITATION, + ), + RosterEntry( + "XTOT", + FieldRole.COUNT, + "Exemptions count", + _STRUCTURAL_CITATION, + ), + RosterEntry( + "P22250", + FieldRole.MONEY, + "Short-term capital gains", + _SCREENED_FIELD_CITATION, + ), + RosterEntry( + "E02500", + FieldRole.MONEY, + "Taxable social security (the archived SOI rename binds the SOI series" + " taxable_social_security to this code; the codebook has no entry)", + _SOI_RENAME_CITATION, + ), + # Four money codes the codebook omits and only the archived SOI measure + # table names. E06500 is bound to two different SOI series there, which is + # itself an open question a successor must settle rather than inherit. + RosterEntry("E04800", FieldRole.MONEY, "Taxable income", _SOI_MEASURE_CITATION), + RosterEntry( + "E06500", + FieldRole.MONEY, + "Income tax; the archived measure table binds it to both" + " total_income_tax and income_tax_before_credits", + _SOI_MEASURE_CITATION, + ), + RosterEntry( + "E08800", FieldRole.MONEY, "Income tax after credits", _SOI_MEASURE_CITATION + ), + RosterEntry( + "E19700", + FieldRole.MONEY, + "Charitable contributions deduction", + _SOI_MEASURE_CITATION, + ), + *( + RosterEntry(field, FieldRole.MONEY, meaning, ROSTER_CITATION) + for field, meaning in _DOCUMENTED_MONEY_MEANINGS + ), +) + +#: Codes the archived producer names in executable code at +#: :data:`ARCHIVED_PRODUCER_COMMIT` while nothing there says what they mean: +#: ``DSI`` and ``EIC`` appear only in a predictor list, ``E25920`` only as a +#: subtracted term, and ``E87530`` only inside two helpers whose names imply a +#: Lifetime-Learning qualified-tuition meaning that no comment states. They are +#: deliberately absent from the roster: a successor must resolve them against +#: real source documentation rather than inherit a guess. This is **not** a +#: complete inventory of undocumented codes -- it is what this slice tripped +#: over, and closing it is a documented remaining obligation. ``E22250`` is in +#: the roster because the codebook documents it, but no archived code path +#: reads it -- every path uses ``P22250`` -- so whether both codes exist in the +#: delivered file is open. +UNDOCUMENTED_SOURCE_FIELDS: tuple[str, ...] = ("DSI", "E25920", "E87530", "EIC") + +#: The source tax year those roster fields are observed in. +ROSTER_SOURCE_REFERENCE_YEAR = PUF_SOURCE_YEAR + + +def roster_role(field: str) -> FieldRole: + """The proposed role of ``field``; refuses a field the roster does not name.""" + for entry in PROPOSED_TY2015_FIELD_ROSTER: + if entry.field == field: + return entry.role + raise PufGrowthRefusalError("FIELD_NOT_IN_ROSTER", field) diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/puf_growth_graph.py b/packages/microcosm-build/src/microcosm/build/us_runtime/puf_growth_graph.py new file mode 100644 index 000000000..0c727a5aa --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/puf_growth_graph.py @@ -0,0 +1,375 @@ +"""Graph nodes that run the pure PUF growth transform inside a real population. + +Three nodes, in the shape the US build already uses for a source operation: + +``.boundary`` + A ``FILTER`` node that keeps every person. It exists because a column a + ``CREATE`` node loaded can only be rewritten once a structural node has + opened a new version. Each kernel reconstructs the declared contract from + its parameters before its computation. The boundary's receipt states the + contract digest, the factor authority, and whether that authority could + ever stand behind a release. + +``.money`` + Owns each declared money column as a rewrite, plus the two provenance + columns. Its whole computation is + :func:`~microcosm.build.us_runtime.puf_growth.apply_puf_growth`. + +``.design_weight`` + Owns only the declared design-weight column, through + :func:`~microcosm.build.us_runtime.puf_growth.apply_puf_design_weight_growth`. + A separate node so the graph itself records that money growth and weight + growth are different operations reading different factor series. + +The contract travels as node parameters — the recipe identity document and the +factor-table document, verbatim — so a changed contract changes the node key +and nothing is served from the store across a contract change. Codes, +identifiers and counts are read but never owned. Two executor mechanisms are +what actually keep them byte-identical: a kernel's returned coordinates must be +exactly the node's declared outputs, so it cannot write them at all; and the +projected input view is frozen and its digest compared before and after the +kernel runs, so it cannot mutate them in place either. + +The design weight this stage grows is the **raw source column**. Promoting it +into a frame's typed design weights is a separate, later obligation with its +own mass policy; nothing here touches ``Frame`` weights. The weight column is +owned as ``float64``, as are the money rewrites; a base version carrying any +of those columns as an integer would need a +declared dtype change, which is a reviewed decision rather than a cast here. + +**The closed read set is the transform's, not the frame's.** +:func:`~microcosm.build.us_runtime.puf_growth.apply_puf_growth` refuses a table +carrying a column no rule declares. These nodes cannot make that promise about +the *population*: a ``CREATE`` node upstream may have loaded raw money columns +this contract never mentions, and they pass through the stage ungrown, because a +node only sees the slices it declares. Closing that gap needs a source-stage +contract that declares the full raw column inventory and checks the population +against it; that is a documented remaining obligation, not something this stage +silently covers. +""" + +from __future__ import annotations + +import json +from collections.abc import Mapping, Sequence +from importlib import import_module + +import pandas as pd + +from microcosm.build.us_runtime.puf_growth import ( + PROVENANCE_RECID_COLUMN, + PROVENANCE_SOURCE_AGI_COLUMN, + CompiledPufGrowth, + GrowthFactorTable, + PufGrowthRefusalError, + apply_puf_design_weight_growth, + apply_puf_growth, + compile_puf_growth, + parse_growth_recipe, + require_transformable, +) +from microcosm.graph import ( + Capabilities, + Determinism, + KernelBase, + KernelContext, + KernelRegistry, + KernelResult, + Node, + Numeric, + Owned, + Slice, + StructuralDelta, + source_hash, +) + +__all__ = [ + "PUF_GROWTH_BOUNDARY_NODE", + "PUF_GROWTH_DESIGN_WEIGHT_NODE", + "PUF_GROWTH_MONEY_NODE", + "PUF_GROWTH_PROVENANCE_RECID_DTYPE", + "PUF_GROWTH_STAGE", + "PufGrowthBoundaryKernel", + "PufGrowthDesignWeightKernel", + "PufGrowthMoneyKernel", + "register_us_puf_growth_kernels", + "us_puf_growth_nodes", +] + +#: Stage name, and the node ids derived from it. +PUF_GROWTH_STAGE = "us_puf_growth" +PUF_GROWTH_BOUNDARY_NODE = f"{PUF_GROWTH_STAGE}.boundary" +PUF_GROWTH_MONEY_NODE = f"{PUF_GROWTH_STAGE}.money" +PUF_GROWTH_DESIGN_WEIGHT_NODE = f"{PUF_GROWTH_STAGE}.design_weight" + +#: The dtype the provenance record identifier is owned as. The raw identifier +#: is an integer record id; a float copy of it would not be a faithful key. +PUF_GROWTH_PROVENANCE_RECID_DTYPE = "int64" + +#: Modules whose bytes are attested by every kernel's implementation hash. The +#: pure contract is included because a change there changes what these kernels +#: compute even when this file is untouched. +_IMPLEMENTATION_MODULES = ( + "microcosm.build.monetary_targets", + "microcosm.build.us_runtime.puf_growth", + "microcosm.build.us_runtime.puf_growth_graph", + "microcosm.build.us_runtime.puf_source_agi", +) + +_PARAMS = ("entity", "factor_table", "recipe") + + +class _PufGrowthKernel(KernelBase): + """Shared parameter contract and code identity for the three kernels.""" + + def implementation_hash(self) -> str: + # Keep the declared dependencies in the hash the way KernelBase does: + # without them a declared distribution would bind only its name into + # the node key, never its installed version. + return source_hash( + *(import_module(name) for name in _IMPLEMENTATION_MODULES), + dependencies=self.capabilities.dependencies, + ) + + @staticmethod + def contract(params: Mapping[str, object]) -> tuple[str, CompiledPufGrowth]: + """Recompile the declared contract from the node's own parameters.""" + if set(params) != set(_PARAMS): + raise PufGrowthRefusalError("NODE_PARAMETER_CONTRACT") + entity = params["entity"] + recipe_document = params["recipe"] + factor_document = params["factor_table"] + if not ( + type(entity) is str + and type(recipe_document) is str + and type(factor_document) is str + ): + raise PufGrowthRefusalError("NODE_PARAMETER_CONTRACT") + recipe = parse_growth_recipe(recipe_document.encode("ascii")) + factors = GrowthFactorTable.from_bytes(factor_document.encode("ascii")) + return entity, compile_puf_growth(recipe, factors) + + +def _identifier_index(context: KernelContext, entity: str) -> pd.Index: + column = f"{entity}_id" + table = context.tables[entity] + if column not in table: + raise PufGrowthRefusalError("ENTITY_ID_COLUMN", entity) + return pd.Index(table[column], name=column) + + +def _declared_source(context: KernelContext, entity: str, fields: Sequence[str]): + table = context.tables[entity] + missing = [name for name in fields if name not in table.columns] + if missing: + raise PufGrowthRefusalError("MISSING_DECLARED_FIELD", missing[0]) + return table.loc[:, list(fields)] + + +def _contract_receipt(compiled: CompiledPufGrowth) -> dict[str, object]: + return { + "contract_sha256": compiled.sha256, + "factor_table_sha256": compiled.factors.sha256, + "factor_table_id": compiled.factors.table_id, + "factor_authority": str(compiled.factors.authority), + "release_eligible": compiled.release_eligible, + "target_year": compiled.target_year, + } + + +class PufGrowthBoundaryKernel(_PufGrowthKernel): + """Open a version for the rewrite, and publish the contract it will use.""" + + ref = "us.puf.growth.boundary@1" + capabilities = Capabilities( + Determinism.DETERMINISTIC, + structural=StructuralDelta.FILTER, + numeric=Numeric.BITWISE, + dependencies=("numpy", "pandas"), + ) + + def run(self, context: KernelContext) -> KernelResult: + _, compiled = self.contract(context.params) + person = context.tables["person"] + return KernelResult( + keep=pd.Series( + True, index=pd.Index(person.person_id, name="person_id"), dtype=bool + ), + receipt={"puf_growth_contract": _contract_receipt(compiled)}, + ) + + +class PufGrowthMoneyKernel(_PufGrowthKernel): + """Grow every declared money field once, and copy the provenance columns.""" + + ref = "us.puf.growth.money@1" + capabilities = Capabilities( + Determinism.DETERMINISTIC, + numeric=Numeric.BITWISE, + dependencies=("numpy", "pandas"), + ) + + def run(self, context: KernelContext) -> KernelResult: + entity, compiled = self.contract(context.params) + source = _declared_source(context, entity, compiled.money_view_fields) + result = apply_puf_growth(source, compiled) + ids = _identifier_index(context, entity) + grown = result.table + if grown[PROVENANCE_RECID_COLUMN].dtype != PUF_GROWTH_PROVENANCE_RECID_DTYPE: + raise PufGrowthRefusalError( + "PROVENANCE_RECID_DTYPE", PROVENANCE_RECID_COLUMN + ) + columns = { + (entity, name): pd.Series( + grown[name].to_numpy(copy=True), index=ids, dtype="float64" + ) + for name in compiled.money_fields + } + columns[(entity, PROVENANCE_RECID_COLUMN)] = pd.Series( + grown[PROVENANCE_RECID_COLUMN].to_numpy(copy=True), + index=ids, + dtype=PUF_GROWTH_PROVENANCE_RECID_DTYPE, + ) + columns[(entity, PROVENANCE_SOURCE_AGI_COLUMN)] = pd.Series( + grown[PROVENANCE_SOURCE_AGI_COLUMN].to_numpy(copy=True), + index=ids, + dtype="float64", + ) + return KernelResult( + columns=columns, + receipt={"puf_growth": json.loads(result.receipt_bytes)}, + ) + + +class PufGrowthDesignWeightKernel(_PufGrowthKernel): + """Grow the raw design-weight column, and nothing else.""" + + ref = "us.puf.growth.design_weight@1" + capabilities = Capabilities( + Determinism.DETERMINISTIC, + numeric=Numeric.BITWISE, + dependencies=("numpy", "pandas"), + ) + + def run(self, context: KernelContext) -> KernelResult: + entity, compiled = self.contract(context.params) + rule = compiled.design_weight_rule + if rule is None: + raise PufGrowthRefusalError("NO_DESIGN_WEIGHT_RULE") + source = _declared_source(context, entity, (rule.field,)) + grown = apply_puf_design_weight_growth( + source[rule.field].to_numpy(dtype="float64"), compiled + ) + factor = compiled.factor_for(rule.field, rule.sign_branch) + return KernelResult( + columns={ + (entity, rule.field): pd.Series( + grown, index=_identifier_index(context, entity), dtype="float64" + ) + }, + receipt={ + "puf_design_weight_growth": { + **_contract_receipt(compiled), + "field": rule.field, + "series": factor.series, + "source_year": factor.source_year, + "ratio_bits": factor.ratio_bits, + "rows": int(len(grown)), + } + }, + ) + + +def us_puf_growth_nodes( + compiled: CompiledPufGrowth, + *, + base: str, + entity: str, + person_boundary_columns: Sequence[str], + stage: str = PUF_GROWTH_STAGE, +) -> tuple[Node, ...]: + """Declare the boundary, money and design-weight nodes for one contract. + + Args: + compiled: The contract these nodes carry. Its recipe identity and its + factor document become node parameters verbatim, so the node keys + move whenever the contract moves. + base: Id of the population version to filter. + entity: Entity whose table carries the raw source fields. + person_boundary_columns: Person columns the boundary reads. A ``FILTER`` + node returns a person mask, so it must declare a person slice. + stage: Node-id prefix, for a graph that runs more than one contract. + + Returns: + The boundary node, the money node, and the design-weight node. + + Raises: + PufGrowthRefusalError: If the contract declares no design weight, names + no person boundary column, or could not run at all + (:func:`~microcosm.build.us_runtime.puf_growth.require_transformable`). + """ + if type(compiled) is not CompiledPufGrowth: + raise PufGrowthRefusalError("COMPILED_TYPE") + weight_rule = compiled.design_weight_rule + if weight_rule is None: + raise PufGrowthRefusalError("NO_DESIGN_WEIGHT_RULE") + if not person_boundary_columns: + raise PufGrowthRefusalError("PERSON_BOUNDARY_COLUMNS") + # Everything the money kernel needs is checked here, at declaration time, + # rather than after a graph run has already loaded a source and written the + # design-weight column. + require_transformable(compiled) + params = { + "entity": entity, + "factor_table": compiled.factors.document.decode("ascii"), + "recipe": compiled.recipe.identity.decode("ascii"), + } + money_view = Slice(entity, compiled.money_view_fields) + boundary = f"{stage}.boundary" + return ( + Node( + boundary, + PufGrowthBoundaryKernel.ref, + base=base, + structural=StructuralDelta.FILTER, + inputs=(Slice("person", tuple(person_boundary_columns)),), + params=params, + description="Open a version so the declared money columns can be rewritten.", + ), + Node( + f"{stage}.money", + PufGrowthMoneyKernel.ref, + population=boundary, + inputs=(money_view,), + outputs=( + *( + Owned(entity, name, "float64", rewrite=True) + for name in compiled.money_fields + ), + Owned( + entity, + PROVENANCE_RECID_COLUMN, + PUF_GROWTH_PROVENANCE_RECID_DTYPE, + ), + Owned(entity, PROVENANCE_SOURCE_AGI_COLUMN, "float64"), + ), + params=params, + description="Grow declared money once and carry source provenance.", + ), + Node( + f"{stage}.design_weight", + PufGrowthDesignWeightKernel.ref, + population=boundary, + inputs=(Slice(entity, (weight_rule.field,)),), + outputs=(Owned(entity, weight_rule.field, "float64", rewrite=True),), + params=params, + description="Grow the raw design weight by its own declared series.", + ), + ) + + +def register_us_puf_growth_kernels(registry: KernelRegistry) -> None: + """Register the three PUF growth kernels.""" + registry.register(PufGrowthBoundaryKernel()) + registry.register(PufGrowthMoneyKernel()) + registry.register(PufGrowthDesignWeightKernel()) diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/puf_monetary_agi_projection.py b/packages/microcosm-build/src/microcosm/build/us_runtime/puf_monetary_agi_projection.py new file mode 100644 index 000000000..73e16294b --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/puf_monetary_agi_projection.py @@ -0,0 +1,689 @@ +"""The thirteen-field 2015 PUF monetary source projection: the twelve accepted +columns plus the publisher's reported ``E00100``. + +The reviewed twelve-field increment +(:mod:`microcosm.build.us_runtime.puf_monetary_source`) decodes wages, the two +interest columns, the two dividend columns, the Schedule C and Schedule F net +results, the two pension columns and the three Schedule D quantities. The PUF +growth contract additionally requires ``E00100`` as a declared money field +(:class:`~microcosm.build.us_runtime.puf_growth.GrowthRecipe`), and the +provenance AGI column that contract writes is a *copy of its own input*, not an +independent source. So a growth run over this lineage needs a projection that +carries reported AGI, and this module is that projection. + +Why a separate module, and what it preserves +-------------------------------------------- +The twelve-field kernel hashes the bytes of ``puf_monetary_source.py`` and +``puf_raw_source.py`` into its implementation hash, and the executor folds that +hash into the artifact store key. Editing either module would move the accepted +twelve-field artifact's key. A repository test also pins the twelve-field +generator's own sha256 to the packaged document. **Nothing here edits any of +them.** This module imports the reviewed decoder, the reviewed envelope +helpers and the reviewed fact derivation, so the thirteenth column is decoded +by exactly the code that decodes the other twelve rather than by a second +implementation of the same contract. The private names it imports are imported +deliberately, and each one is a piece of the reviewed contract this edition +must not restate. + +What is the same +---------------- +Every property of the twelve-field contract holds here unchanged: whole-dollar +signed integer tokens on individual returns, the separately bounded +disclosure-aggregate lexical grammar, the minimum-int64 out-of-universe +sentinel, the ``amount_known == (disclosure_aggregate == 0)`` rule derived from +the authenticated flag on every access, all 207,696 delivered rows retained, +the three disjoint row classes, the binding to the reviewed status artifact, +and ``target_year_use: refused_pending_independent_growth_decision`` on every +column including ``E00100``. + +What is different +----------------- +One more column, and a different artifact identity. The type is +``microcosm.us.puf_2015_monetary_agi_source_projection`` version 1, with its +own magic line. A twelve-field payload fed to this reader is refused **by +name** rather than reinterpreted, because the two artifacts carry different +column sets and silently reading one as the other is exactly the failure the +twelve-field module's superseded-magic list exists to prevent. Nothing here +derives, reconstructs or reconciles AGI: ``E00100`` is the delivered reported +field, and the document records that the booklets state no component identity +between it and the twelve. +""" + +from __future__ import annotations + +import hashlib +import json +from collections.abc import Mapping +from importlib import resources +from types import MappingProxyType + +import numpy as np + +from microcosm.graph import ( + ArtifactInput, + ArtifactOutput, + ArtifactType, + Capabilities, + Determinism, + KernelBase, + KernelContext, + KernelRegistry, + KernelResult, + Node, + Numeric, + StructuralDelta, + source_hash, +) +from microcosm.graph.canonical import canonical_json +from microcosm.graph.codecs import SourceCodecRegistry, load_source_bytes + +# The reviewed twelve-field contract, imported rather than restated. The +# private names are deliberate: the thirteenth column is authenticated, +# encoded, read back and described by exactly the reviewed code, so a change +# there cannot leave a stale second copy of the same contract here. What this +# module owns is its own artifact identity, not its own parser. +from .puf_monetary_source import ( + _AGGREGATE_COLUMN_FACT_KEYS, + _ENVELOPE_FACT_KEYS, + _ENVELOPE_HEADER_KEYS, + _LEXICAL_KIND, + _RETURN_COLUMN_FACT_KEYS, + _STRUCTURAL_DTYPES, + _TYPED_KIND, + AGGREGATE_ROW_CLASS, + BODY_MAX_BYTES, + HEADER_MAX_BYTES, + MONETARY_PROJECTION_MAGIC, + ROW_CLASSES, + SUPERSEDED_MAGIC, + PufMonetaryProjection, + PufMonetarySourceProjection, + _check_envelope_sources, + _envelope_column_order, + _envelope_columns, + _envelope_int, + _exact_keys, + _facts_from_arrays, + _fixed_ascii, + _hex64, + _projection_from_document, + _read_fixed_ascii, + _refuse, + _thawed, + _verify_readback, + decode_puf_monetary_source, + packaged_projection, + payload_bound, +) +from .puf_raw_source import ( + PUF_RAW_SOURCE_NODE, + RETURN_STATUS_TYPE, + PufRawSourceDefinition, + csv_acceptance_profile, + decode_return_status, + puf_raw_source_codecs, +) +from .puf_raw_source import packaged_definition as packaged_raw_definition + +__all__ = [ + "AGI_FIELD", + "AGI_PROJECTION_EDITION", + "AGI_PROJECTION_MAGIC", + "AGI_PROJECTION_NODE", + "AGI_PROJECTION_STAGE", + "AGI_PROJECTION_TYPE", + "TWELVE_FIELD_MAGIC", + "USPufMonetaryAgiSourceKernel", + "decode_agi_projection", + "encode_agi_projection", + "fixture_agi_projection_document", + "packaged_agi_projection", + "register_us_puf_monetary_agi_source_kernels", + "us_puf_monetary_agi_source_node", +] + +#: The one column this edition adds to the accepted twelve. +AGI_FIELD = "E00100" + +#: The edition marker the packaged document must declare about itself. +AGI_PROJECTION_EDITION = "thirteen_field_reported_agi" + +AGI_PROJECTION_TYPE = ArtifactType( + "microcosm.us.puf_2015_monetary_agi_source_projection", 1 +) +AGI_PROJECTION_MAGIC = b"microcosm.us.puf_2015_monetary_agi_source_projection/1\n" + +#: Magic lines this reader refuses by name rather than reinterpreting. These +#: are the twelve-field artifact's own magics: those payloads carry a +#: different column set, and reading one here would report a twelve-column +#: artifact as a thirteen-column one. +TWELVE_FIELD_MAGIC = (MONETARY_PROJECTION_MAGIC, *SUPERSEDED_MAGIC) + +AGI_PROJECTION_STAGE = "us_puf_monetary_agi_source" +AGI_PROJECTION_NODE = f"{AGI_PROJECTION_STAGE}.projection" + +_AGI_RESOURCE = "puf_2015_monetary_agi_source_projection.json" +_AGI_PACKAGE = "microcosm.build.us_runtime" +_FIXTURE_AUTHORITY = "invented_fixture_nonauthority" + +_STATUS_ALIAS = "return_status" +_PROJECTION_ALIAS = "monetary_agi_projection" + + +# -------------------------------------------------------------------------- +# The packaged thirteen-field document +# -------------------------------------------------------------------------- + + +def _check_agi_edition( + document: Mapping[str, object], projection: PufMonetaryProjection +) -> None: + """Bind this edition to the accepted twelve-field document, at load time. + + The edition block is not decoration: it names the parent document by + digest and lists the parent's projected fields, so a thirteen-field + document that quietly dropped or renamed one of the accepted twelve — or + that was generated against a different parent — is refused here rather + than discovered downstream. + """ + + edition = document.get("projection_edition") + if not isinstance(edition, Mapping): + raise _refuse("AGI_PROJECTION_EDITION_MISSING") + if edition.get("edition") != AGI_PROJECTION_EDITION: + raise _refuse("AGI_PROJECTION_EDITION") + if list(edition.get("adds", ())) != [AGI_FIELD]: + raise _refuse("AGI_PROJECTION_EDITION_ADDS") + parent = packaged_projection() + if edition.get("parent_document_canonical_sha256") != parent.sha256: + raise _refuse("AGI_PROJECTION_PARENT_DOCUMENT") + if tuple(edition.get("parent_projected_fields", ())) != parent.fields: + raise _refuse("AGI_PROJECTION_PARENT_FIELDS") + fields = projection.fields + if AGI_FIELD not in fields: + raise _refuse("AGI_PROJECTION_FIELD_MISSING", AGI_FIELD) + if set(fields) != set(parent.fields) | {AGI_FIELD}: + raise _refuse("AGI_PROJECTION_FIELD_SET") + if len(fields) != len(parent.fields) + 1: + raise _refuse("AGI_PROJECTION_FIELD_COUNT") + # Every accepted column must still declare exactly what it declared + # upstream. This edition adds a column; it does not restate the twelve. + upstream = {column.field: column for column in parent.columns} + for column in projection.columns: + if column.field == AGI_FIELD: + continue + if column != upstream[column.field]: + raise _refuse("AGI_PROJECTION_PARENT_COLUMN", column.field) + + +def _packaged_agi_bytes() -> bytes: + return resources.files(_AGI_PACKAGE).joinpath(_AGI_RESOURCE).read_bytes() + + +_PACKAGED_AGI: PufMonetaryProjection | None = None + + +def packaged_agi_projection() -> PufMonetaryProjection: + """The one closed thirteen-field document a production run may use.""" + + global _PACKAGED_AGI + if _PACKAGED_AGI is None: + document = json.loads(_packaged_agi_bytes().decode("utf-8")) + projection = _projection_from_document( + document, route="packaged", definition=packaged_raw_definition() + ) + _check_agi_edition(document, projection) + _PACKAGED_AGI = projection + return _PACKAGED_AGI + + +def fixture_agi_projection_document( + document: Mapping[str, object], definition: PufRawSourceDefinition +) -> PufMonetaryProjection: + """Build a thirteen-field projection on the explicit non-production route. + + Same rules as the twelve-field fixture route: ``route="test_fixture"``, + the invented-fixture authority, and no restatement of a packaged source + pin. It must still name ``E00100``, because a fixture without it would be + testing the twelve-field contract under this module's name. + """ + + projection = _projection_from_document( + document, route="test_fixture", definition=definition + ) + if document.get("authority") != _FIXTURE_AUTHORITY: + raise _refuse("FIXTURE_AUTHORITY") + packaged_raw = packaged_raw_definition() + genuine = {packaged_raw.main.sha256, packaged_raw.demographic.sha256} + stated = { + document["sources"]["main"]["sha256"], + document["sources"]["demographic"]["sha256"], + } + if stated & genuine: + raise _refuse("FIXTURE_RESTATES_PACKAGED_PIN") + if AGI_FIELD not in projection.fields: + raise _refuse("AGI_PROJECTION_FIELD_MISSING", AGI_FIELD) + return projection + + +# -------------------------------------------------------------------------- +# The typed envelope +# -------------------------------------------------------------------------- + + +def encode_agi_projection( + decoded: PufMonetarySourceProjection, + projection: PufMonetaryProjection, + definition: PufRawSourceDefinition, + *, + status_artifact_sha256: str, +) -> bytes: + """Encode a thirteen-field projection into its own bounded envelope. + + Byte-for-byte the twelve-field layout — magic line, big-endian header + length, canonical JSON header, little-endian typed bodies then fixed-width + lexical bodies — under a different magic and type, with the column bodies + the shared order helper produces for this projection. + """ + + if AGI_FIELD not in projection.fields: + raise _refuse("AGI_PROJECTION_FIELD_MISSING", AGI_FIELD) + rows = decoded.rows + total, sizes = payload_bound(rows, projection) + if total > BODY_MAX_BYTES: + raise _refuse("BODY_OVER_BOUND", total) + dtypes = { + **_STRUCTURAL_DTYPES, + **{column.field: column.numpy_dtype for column in projection.columns}, + } + widths = {column.field: column.lexical_width for column in projection.columns} + bodies: list[bytes] = [] + columns: list[dict[str, object]] = [] + for name in _envelope_column_order(projection): + if name.endswith("_lexical"): + field = name.removesuffix("_lexical") + body = _fixed_ascii(decoded.lexical[field], widths[field], field) + entry: dict[str, object] = { + "name": name, + "kind": _LEXICAL_KIND, + "width": widths[field], + } + else: + array = np.ascontiguousarray(decoded.typed[name], dtype=dtypes[name]) + if array.shape != (rows,): + raise _refuse("COLUMN_SHAPE", name) + body = array.tobytes() + entry = {"name": name, "kind": _TYPED_KIND, "dtype": dtypes[name]} + if len(body) != sizes[name]: + raise _refuse("COLUMN_SIZE", name) + bodies.append(body) + entry["bytes"] = len(body) + entry["sha256"] = hashlib.sha256(body).hexdigest() + columns.append(entry) + header = { + "schema_version": AGI_PROJECTION_TYPE.schema_version, + "type": [AGI_PROJECTION_TYPE.name, AGI_PROJECTION_TYPE.schema_version], + "rows": rows, + "projection_sha256": projection.sha256, + "projection_route": projection.route, + "raw_source_definition_sha256": definition.sha256, + "status_artifact_sha256": _hex64( + status_artifact_sha256, "status_artifact_sha256" + ), + "sources": { + "main": { + "sha256": definition.main.sha256, + "git_blob_sha1": definition.main.git_blob_sha1, + "bytes": definition.main.bytes, + "header_record_canonical_sha256": ( + definition.main.header_record_canonical_sha256 + ), + }, + "demographic": { + "sha256": definition.demographic.sha256, + "git_blob_sha1": definition.demographic.git_blob_sha1, + "bytes": definition.demographic.bytes, + "header_record_canonical_sha256": ( + definition.demographic.header_record_canonical_sha256 + ), + }, + }, + "header_length_endianness": "big", + "body_endianness": "little", + "columns": columns, + "facts": dict(decoded.facts), + } + encoded = canonical_json(header) + if len(encoded) > HEADER_MAX_BYTES: + raise _refuse("HEADER_OVER_BOUND", len(encoded)) + return b"".join( + (AGI_PROJECTION_MAGIC, len(encoded).to_bytes(4, "big"), encoded, *bodies) + ) + + +def _resolve_agi_projection( + header: Mapping[str, object], projection: PufMonetaryProjection | None +) -> PufMonetaryProjection: + """The thirteen-field projection this payload may be checked against.""" + + route = header["projection_route"] + if route not in ("packaged", "test_fixture"): + raise _refuse("ENVELOPE_PROJECTION_ROUTE") + digest = _hex64(header["projection_sha256"], "envelope.projection_sha256") + if projection is not None: + if not isinstance(projection, PufMonetaryProjection): + raise _refuse("DECODE_PROJECTION_TYPE") + if projection.route != route or projection.sha256 != digest: + raise _refuse("ENVELOPE_PROJECTION_MISMATCH") + if AGI_FIELD not in projection.fields: + raise _refuse("AGI_PROJECTION_FIELD_MISSING", AGI_FIELD) + return projection + if route == "packaged": + packaged = packaged_agi_projection() + if packaged.sha256 != digest: + raise _refuse("ENVELOPE_PROJECTION_NOT_PACKAGED") + return packaged + # A fixture-route payload has no reconstructible projection document, so + # there is nothing to close its column set against. + raise _refuse("ENVELOPE_PROJECTION_REQUIRED") + + +def decode_agi_projection( + payload: bytes, + projection: PufMonetaryProjection | None = None, + *, + expected_status_artifact_sha256: str | None = None, +) -> PufMonetarySourceProjection: + """Decode a thirteen-field payload and re-prove it, never trusting it. + + Every reported fact is recomputed from the arrays actually read back by + the reviewed derivation and must agree exactly, and every row re-parses + under the universe its authenticated flag selects. + """ + + if not isinstance(payload, bytes): + raise _refuse("ENVELOPE_MAGIC") + if not payload.startswith(AGI_PROJECTION_MAGIC): + if any(payload.startswith(magic) for magic in TWELVE_FIELD_MAGIC): + raise _refuse("ENVELOPE_TWELVE_FIELD_MAGIC") + raise _refuse("ENVELOPE_MAGIC") + start = len(AGI_PROJECTION_MAGIC) + if len(payload) < start + 4: + raise _refuse("ENVELOPE_TRUNCATED") + length = int.from_bytes(payload[start : start + 4], "big") + if not 0 < length <= HEADER_MAX_BYTES or len(payload) < start + 4 + length: + raise _refuse("ENVELOPE_HEADER_LENGTH") + encoded = payload[start + 4 : start + 4 + length] + try: + header = json.loads(encoded.decode("utf-8")) + except (UnicodeError, ValueError, RecursionError) as error: + raise _refuse("ENVELOPE_HEADER_JSON") from error + try: + recanonical = canonical_json(header) + except (TypeError, ValueError, RecursionError) as error: + raise _refuse("ENVELOPE_HEADER_JSON") from error + if recanonical != encoded: + raise _refuse("ENVELOPE_HEADER_NOT_CANONICAL") + _exact_keys(header, _ENVELOPE_HEADER_KEYS, "header") + if header["schema_version"] != AGI_PROJECTION_TYPE.schema_version or header[ + "type" + ] != [AGI_PROJECTION_TYPE.name, AGI_PROJECTION_TYPE.schema_version]: + raise _refuse("ENVELOPE_TYPE") + if ( + header["header_length_endianness"] != "big" + or header["body_endianness"] != "little" + ): + raise _refuse("ENVELOPE_ENDIANNESS") + _hex64(header["raw_source_definition_sha256"], "raw_source_definition_sha256") + _hex64(header["status_artifact_sha256"], "status_artifact_sha256") + if expected_status_artifact_sha256 is not None: + _hex64(expected_status_artifact_sha256, "expected_status_artifact_sha256") + if header["status_artifact_sha256"] != expected_status_artifact_sha256: + raise _refuse("ENVELOPE_STATUS_ARTIFACT_MISMATCH") + rows = _envelope_int(header["rows"], "rows", minimum=1) + resolved = _resolve_agi_projection(header, projection) + if resolved.raw_definition_sha256 != header["raw_source_definition_sha256"]: + raise _refuse("ENVELOPE_RAW_DEFINITION_MISMATCH") + _check_envelope_sources(header, resolved) + columns = _envelope_columns(header, rows, resolved) + + cursor = start + 4 + length + typed: dict[str, np.ndarray] = {} + lexical: dict[str, tuple[str, ...]] = {} + for column in columns: + name = column["name"] + size = column["bytes"] + body = payload[cursor : cursor + size] + if len(body) != size: + raise _refuse("ENVELOPE_BODY_TRUNCATED", name) + if hashlib.sha256(body).hexdigest() != column["sha256"]: + raise _refuse("ENVELOPE_BODY_DIGEST", name) + cursor += size + if column["kind"] == _TYPED_KIND: + typed[name] = np.frombuffer(body, dtype=np.dtype(column["dtype"])) + else: + field = name.removesuffix("_lexical") + lexical[field] = _read_fixed_ascii(body, rows, column["width"], field) + if cursor != len(payload): + raise _refuse("ENVELOPE_TRAILING_BYTES") + + _verify_readback(typed, lexical, resolved) + facts = header["facts"] + _exact_keys(facts, _ENVELOPE_FACT_KEYS, "facts") + for field, classes in facts["column_facts"].items(): + if field not in resolved.fields: + raise _refuse("ENVELOPE_FACTS", "column_facts") + _exact_keys(classes, frozenset(ROW_CLASSES), f"column_facts.{field}") + for name, entry in classes.items(): + keys = ( + _AGGREGATE_COLUMN_FACT_KEYS + if name == AGGREGATE_ROW_CLASS + else _RETURN_COLUMN_FACT_KEYS + ) + _exact_keys(entry, keys, f"column_facts.{field}.{name}") + recomputed = _facts_from_arrays(typed, lexical, resolved) + if canonical_json(recomputed) != canonical_json(dict(facts)): + raise _refuse("ENVELOPE_FACTS_INCONSISTENT") + if recomputed["rows"] != rows: + raise _refuse("ENVELOPE_ROWS") + return PufMonetarySourceProjection( + typed=MappingProxyType(typed), + lexical=MappingProxyType(lexical), + facts=MappingProxyType(recomputed), + column_metadata=MappingProxyType(dict(recomputed["column_metadata"])), + ) + + +# -------------------------------------------------------------------------- +# The graph producer +# -------------------------------------------------------------------------- + +_IMPLEMENTATION_MODULES = ( + "microcosm.build.us_runtime.puf_raw_source", + "microcosm.build.us_runtime.puf_monetary_source", + "microcosm.build.us_runtime.puf_monetary_agi_projection", +) +_IMPLEMENTATION_IDENTITY_MAGIC = b"us.puf.monetary_agi_source/implementation/1\n" + + +class USPufMonetaryAgiSourceKernel(KernelBase): + """Read the pinned main delivery and produce the thirteen-field projection.""" + + ref = "us.puf.monetary_agi_source.projection@1" + capabilities = Capabilities( + Determinism.DETERMINISTIC, + numeric=Numeric.BITWISE, + dependencies=("numpy",), + ) + + def __init__( + self, + *, + projection: PufMonetaryProjection | None = None, + definition: PufRawSourceDefinition | None = None, + source_codecs: SourceCodecRegistry | None = None, + ) -> None: + self.definition = ( + packaged_raw_definition() if definition is None else definition + ) + self.projection = ( + packaged_agi_projection() if projection is None else projection + ) + if self.projection.route not in ("packaged", "test_fixture"): + raise _refuse("KERNEL_PROJECTION_ROUTE") + if self.projection.route != self.definition.route: + raise _refuse("KERNEL_ROUTE_DISAGREEMENT") + if self.projection.raw_definition_sha256 != self.definition.sha256: + raise _refuse("KERNEL_DEFINITION_MISMATCH") + if AGI_FIELD not in self.projection.fields: + raise _refuse("AGI_PROJECTION_FIELD_MISSING", AGI_FIELD) + if self.projection.route == "packaged": + packaged = packaged_agi_projection() + if ( + self.projection.canonical != packaged.canonical + or self.projection.sha256 != packaged.sha256 + or canonical_json(_thawed(self.projection.document)) + != self.projection.canonical + ): + raise _refuse("KERNEL_PROJECTION_NOT_PACKAGED") + self.source_codecs = ( + puf_raw_source_codecs(self.definition) + if source_codecs is None + else source_codecs + ) + + def implementation_hash(self) -> str: + from importlib import import_module + + base = source_hash( + *(import_module(name) for name in _IMPLEMENTATION_MODULES), + dependencies=self.capabilities.dependencies, + ) + digest = hashlib.sha256(_IMPLEMENTATION_IDENTITY_MAGIC) + digest.update(base.encode("ascii")) + digest.update(b"\0") + digest.update(canonical_json(csv_acceptance_profile())) + return digest.hexdigest() + + def run(self, context: KernelContext) -> KernelResult: + definition = self.definition + projection = self.projection + if ( + context.node.kernel != self.ref + or context.node.structural is not StructuralDelta.NONE + or context.node.sources != (definition.main.source_name,) + or context.node.inputs + or context.node.outputs + or len(context.node.artifact_inputs) != 1 + or context.node.artifact_inputs[0].type != RETURN_STATUS_TYPE + or context.node.artifact_outputs + != (ArtifactOutput(_PROJECTION_ALIAS, AGI_PROJECTION_TYPE),) + or set(context.params) != {"definition", "projection"} + ): + raise _refuse("NODE_DECLARATION") + if context.params["definition"] != definition.params_text: + raise _refuse("NODE_DEFINITION_PARAM") + if context.params["projection"] != projection.params_text: + raise _refuse("NODE_PROJECTION_PARAM") + alias = context.node.artifact_inputs[0].name + value = context.artifacts[alias] + if value.type != RETURN_STATUS_TYPE: + raise _refuse("STATUS_ARTIFACT_TYPE") + status = decode_return_status(value.payload, definition) + main_bytes = load_source_bytes( + definition.main.codec, + context.sources[definition.main.source_name], + registry=self.source_codecs, + ) + decoded = decode_puf_monetary_source(main_bytes, status, projection, definition) + payload = encode_agi_projection( + decoded, + projection, + definition, + status_artifact_sha256=hashlib.sha256(value.payload).hexdigest(), + ) + return KernelResult( + artifacts={_PROJECTION_ALIAS: payload}, + receipt={ + "puf_monetary_agi_source_projection": { + "projection_sha256": projection.sha256, + "projection_route": projection.route, + "projection_edition": AGI_PROJECTION_EDITION, + "definition_sha256": definition.sha256, + "artifact_sha256": hashlib.sha256(payload).hexdigest(), + "artifact_bytes": len(payload), + "status_artifact_sha256": hashlib.sha256(value.payload).hexdigest(), + "main_sha256": definition.main.sha256, + "csv_acceptance_profile": csv_acceptance_profile(), + "projected_fields": list(projection.fields), + "agi_field": AGI_FIELD, + "agi_is_derived": False, + **{ + key: value + for key, value in decoded.facts.items() + if key != "column_facts" + }, + } + }, + ) + + +def us_puf_monetary_agi_source_node( + *, + population: str, + producer: str = PUF_RAW_SOURCE_NODE, + producer_output: str = "return_status", + projection: PufMonetaryProjection | None = None, + definition: PufRawSourceDefinition | None = None, + stage: str = AGI_PROJECTION_STAGE, +) -> Node: + """Declare the thirteen-field producer inside an explicitly named host.""" + + if not isinstance(population, str) or not population: + raise _refuse("NODE_POPULATION_REQUIRED") + resolved_definition = ( + packaged_raw_definition() if definition is None else definition + ) + resolved = packaged_agi_projection() if projection is None else projection + if resolved.raw_definition_sha256 != resolved_definition.sha256: + raise _refuse("NODE_DEFINITION_MISMATCH") + if AGI_FIELD not in resolved.fields: + raise _refuse("AGI_PROJECTION_FIELD_MISSING", AGI_FIELD) + return Node( + f"{stage}.projection", + USPufMonetaryAgiSourceKernel.ref, + population=population, + sources=(resolved_definition.main.source_name,), + params={ + "definition": resolved_definition.params_text, + "projection": resolved.params_text, + }, + artifact_inputs=( + ArtifactInput(_STATUS_ALIAS, producer, producer_output, RETURN_STATUS_TYPE), + ), + artifact_outputs=(ArtifactOutput(_PROJECTION_ALIAS, AGI_PROJECTION_TYPE),), + description=( + "Decode the twelve accepted 2015 PUF monetary source columns plus " + "reported E00100 at return grain, bound to the reviewed status " + "artifact. Owns no population columns and admits no target year." + ), + ) + + +def register_us_puf_monetary_agi_source_kernels( + registry: KernelRegistry, + *, + projection: PufMonetaryProjection | None = None, + definition: PufRawSourceDefinition | None = None, + source_codecs: SourceCodecRegistry | None = None, +) -> KernelRegistry: + """Register the thirteen-field producer kernel into ``registry``.""" + + registry.register( + USPufMonetaryAgiSourceKernel( + projection=projection, + definition=definition, + source_codecs=source_codecs, + ) + ) + return registry diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/puf_monetary_source.py b/packages/microcosm-build/src/microcosm/build/us_runtime/puf_monetary_source.py new file mode 100644 index 000000000..5c91c5a55 --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/puf_monetary_source.py @@ -0,0 +1,2036 @@ +"""The first monetary amounts typed out of the 2015 PUF delivery bytes. + +The reviewed raw source slice +(:mod:`microcosm.build.us_runtime.puf_raw_source`) types six of the 222 +delivered main columns and the six demographic codes, reports +``amounts_decoded: False``, and drops the other 216 main columns after +checking their header name, order and record width. This module is the next +bounded increment: it authenticates the **whole** delivered layout against the +retained IRS booklets and decodes a small, explicitly enumerated set of +monetary columns at source-return grain. + +Two things it produces, and one it refuses. + +*The full layout is authenticated, not merely counted.* The packaged document +``puf_2015_monetary_source_projection.json`` locates every one of the 222 main +columns and 7 demographic columns in the September 2022 booklet by the +publisher's own position number and name, retains that record-layout row +verbatim with its exact character span and span digest, and corroborates each +against the February 2023 edition. Nothing about a column's role is inferred +from an ``E``, ``P``, ``S`` or ``T`` prefix or from membership of the +amount-field list: the publisher's own "Misc Codes" block sits inside that list +and holds ``RECID``, ``S006``, ``S008``, ``S009``, ``WSAMP`` and ``TXRT``, none +of which is money and one of which (``TXRT``) carries an implied decimal point. + +*Twelve monetary columns are decoded, each with its own declaration.* Wages, +taxable and tax-exempt interest, ordinary and qualified dividends, the +Schedule C and Schedule F net results, pensions received and pensions in AGI, +and the three Schedule D quantities the booklets define. Each declares exact +units, sign evidence, missingness policy, return grain, source-period status +and source pin identity, and each is refused for target-year use. + +*No amount is grown, restated or combined.* The row period is unresolved — the +cover says Tax Year 2015, the code definitions print FLPDYR 2011-2014, the +disclosure section removed pre-2012 returns, and the reviewed header audit +observed 2012-2015 — so every column carries +``target_year_use: refused_pending_independent_growth_decision``. Component +relationships the booklets do not state (qualified dividends inside ordinary +dividends, pensions in AGI inside pensions received, ``E01000`` out of +``P22250`` and ``P23250``) are recorded as unresolved and never computed. +``E22250`` is absent from the delivered layout and from both booklets and is +never aliased onto ``P22250``. An uncapped self-employment earnings derivation +from ``E30400``/``E30500`` is refused with the publisher text that refuses it. + +Three properties are load-bearing. + +*Every delivered main record is projected.* All 207,696 rows, including the +88,021 with no demographic row and the four disclosure aggregate records. The +aggregate rows are tagged and every derived fact is reported for three +disjoint row classes — aggregate, demographic-matched and +demographic-unmatched — so an aggregate can never be averaged into a +per-return statistic and an unmatched row is never a zero. + +*The individual-return amount universe is explicit.* A row is inside it if and +only if its authenticated disclosure-aggregate flag is 0, and the decoded +object exposes that rule as a read-only ``amount_known`` mask derived from the +flag on every access. Individual returns keep the unchanged integer, +whole-dollar, twelve-digit grammar, and a fractional token on one of them +still refuses. The four disclosure aggregates are outside that universe: their +typed slots hold the minimum-int64 out-of-universe sentinel — never a zero, +which is a delivered value in these columns — and their delivered tokens are +retained exactly under a separate, explicitly bounded lexical grammar that is +an operational input bound for this pinned file rather than a publisher +specification. Nothing converts an aggregate token to a number, rounds it, +grows it or drops its row. The aggregate row class carries only token-class +counts and a token width, so no header, receipt or summary can state an +aggregate amount, and the exact row/token pairs are available only through a +local accessor on the decoded object. + +*The projection is bound to the reviewed status artifact.* The node takes +``microcosm.us.puf_2015_raw_return_status`` as an artifact input and refuses +unless its own ``RECID`` sequence and aggregate flags are element-wise +identical to that artifact's. ``demographic_status`` is carried from it rather +than re-joined, so the two artifacts cannot disagree about which returns +matched. + +*The raw producer has an explicit packaging revision.* This module imports +:mod:`puf_raw_source` for its definition, codec and artifact decoder. Moving +the unchanged definition bytes into ``us_runtime`` changed the raw resource +loader and its implementation identity. New runs use that revised identity; +previously accepted raw artifacts retain their original keys and evidence. +The CSV reader helpers here remain local to this monetary projection. +""" + +from __future__ import annotations + +import csv +import hashlib +import io +import json +from collections.abc import Mapping, Sequence +from dataclasses import dataclass +from importlib import resources +from types import MappingProxyType + +import numpy as np + +from microcosm.graph import ( + ArtifactInput, + ArtifactOutput, + ArtifactType, + Capabilities, + Determinism, + KernelBase, + KernelContext, + KernelRegistry, + KernelResult, + Node, + Numeric, + StructuralDelta, + source_hash, +) +from microcosm.graph.canonical import canonical_json +from microcosm.graph.codecs import SourceCodecRegistry, load_source_bytes + +from .puf_raw_source import ( + PUF_AGGREGATE_RECIDS, + PUF_RAW_SOURCE_NODE, + RETURN_STATUS_TYPE, + PufRawSourceDefinition, + csv_acceptance_profile, + decode_return_status, + puf_raw_source_codecs, +) +from .puf_raw_source import packaged_definition as packaged_raw_definition + +__all__ = [ + "AGGREGATE_ROW_CLASS", + "AGGREGATE_SENTINEL", + "AMOUNT_KNOWN_RULE", + "BODY_MAX_BYTES", + "HEADER_MAX_BYTES", + "LEXICAL_WIDTH_MAX", + "MONETARY_PROJECTION_NODE", + "MONETARY_PROJECTION_STAGE", + "MONETARY_PROJECTION_SUMMARY_NODE", + "MONETARY_PROJECTION_TYPE", + "PufMonetaryProjection", + "PufMonetaryRefusalError", + "PufMonetarySourceDocument", + "ProjectedColumn", + "RETURN_ROW_CLASSES", + "SUPERSEDED_MAGIC", + "USPufMonetaryProjectionSummaryKernel", + "USPufMonetarySourceKernel", + "decode_monetary_projection", + "decode_puf_monetary_source", + "encode_monetary_projection", + "fixture_projection_document", + "packaged_projection", + "projection_document_json", + "register_us_puf_monetary_source_kernels", + "us_puf_monetary_projection_summary_node", + "us_puf_monetary_source_node", +] + +MONETARY_PROJECTION_TYPE = ArtifactType( + "microcosm.us.puf_2015_monetary_source_projection", 3 +) +MONETARY_PROJECTION_MAGIC = b"microcosm.us.puf_2015_monetary_source_projection/3\n" + +#: Magic lines this reader refuses by name instead of reinterpreting. No +#: genuine schema-1 or schema-2 monetary artifact was ever completed — the +#: first genuine run refused before writing one — but invented artifacts of +#: both do exist in test evidence, and the amount universe differs, so an old +#: payload gets its own refusal rather than falling through the generic magic +#: check as an unrecognised blob. +SUPERSEDED_MAGIC = ( + b"microcosm.us.puf_2015_monetary_source_projection/1\n", + b"microcosm.us.puf_2015_monetary_source_projection/2\n", +) + +MONETARY_PROJECTION_STAGE = "us_puf_monetary_source" +MONETARY_PROJECTION_NODE = f"{MONETARY_PROJECTION_STAGE}.projection" +MONETARY_PROJECTION_SUMMARY_NODE = f"{MONETARY_PROJECTION_STAGE}.projection_summary" + +#: Bound on the encoded artifact's column bodies, matching the raw slice's own +#: private bound. Twelve projected columns over the delivered 207,696 rows is +#: 262 bytes a row, so about 51.9 MiB; a thirteenth column of the same shape +#: would still fit and a much wider selection would not. +BODY_MAX_BYTES = 64 * 1024 * 1024 + +#: Bound on the canonical JSON header. The header carries per-column facts for +#: three row classes, so it is larger than the status artifact's. +HEADER_MAX_BYTES = 64 * 1024 + +#: The widest lexical allocation a projected column may declare. +LEXICAL_WIDTH_MAX = 64 + +_PROJECTION_RESOURCE = "puf_2015_monetary_source_projection.json" +_PROJECTION_PACKAGE = "microcosm.build.us_runtime" +_PROJECTION_SCHEMA = "microcosm.us.puf_2015_monetary_source_projection" +_FIXTURE_AUTHORITY = "invented_fixture_nonauthority" + +_STRUCTURAL_COLUMNS = ("RECID", "disclosure_aggregate", "demographic_status") +_STRUCTURAL_DTYPES = { + "RECID": " None: + super().__init__(" ".join((reason, *(str(part) for part in detail))).strip()) + self.reason = reason + + +def _refuse(reason: str, *detail: object) -> PufMonetaryRefusalError: + return PufMonetaryRefusalError(reason, *detail) + + +# -------------------------------------------------------------------------- +# The closed projection document +# -------------------------------------------------------------------------- + + +@dataclass(frozen=True) +class ProjectedColumn: + """One enumerated monetary column, with everything the parse needs.""" + + field: str + concept: str + concept_group: str + delivered_index: int + publisher_position: int + publisher_label: str + dtype: str + lexical_width: int + grammar: str + max_digits: int + publisher_signed_marker: bool + negative_values: str + fit_admission: str + unresolved: tuple[str, ...] + + @property + def numpy_dtype(self) -> str: + return _DECLARED_DTYPE[self.dtype] + + +@dataclass(frozen=True) +class PufMonetaryProjection: + """A closed layout authentication plus the enumerated projection.""" + + route: str + document: Mapping[str, object] + canonical: bytes + sha256: str + columns: tuple[ProjectedColumn, ...] + raw_definition_sha256: str + main_delivered_header: tuple[str, ...] + demographic_delivered_header: tuple[str, ...] + + @property + def params_text(self) -> str: + """The canonical JSON the node carries verbatim as a parameter.""" + + return self.canonical.decode("utf-8") + + @property + def fields(self) -> tuple[str, ...]: + return tuple(column.field for column in self.columns) + + +#: Kept as a separate public name so a caller can talk about the document +#: without holding a parsed projection. +PufMonetarySourceDocument = Mapping[str, object] + + +def _frozen(value: object) -> object: + """A deeply immutable view, as the raw source definition already does.""" + + if isinstance(value, Mapping): + return MappingProxyType({key: _frozen(item) for key, item in value.items()}) + if isinstance(value, (list, tuple)): + return tuple(_frozen(item) for item in value) + return value + + +def _thawed(value: object) -> object: + if isinstance(value, Mapping): + return {key: _thawed(item) for key, item in value.items()} + if isinstance(value, tuple): + return [_thawed(item) for item in value] + return value + + +def _hex64(value: object, label: str) -> str: + if ( + not isinstance(value, str) + or len(value) != 64 + or any(character not in "0123456789abcdef" for character in value) + ): + raise _refuse("PROJECTION_DIGEST", label) + return value + + +def _positive_int(value: object, label: str, *, minimum: int = 1) -> int: + if not isinstance(value, int) or isinstance(value, bool) or value < minimum: + raise _refuse("PROJECTION_INTEGER", label) + return value + + +def _layout_names( + entry: Mapping[str, object], label: str, reasons: frozenset[str] +) -> tuple[str, ...]: + columns = entry["columns"] + if not isinstance(columns, (list, tuple)) or not columns: + raise _refuse("PROJECTION_LAYOUT", label) + names: list[str] = [] + for position, column in enumerate(columns): + if not isinstance(column, Mapping): + raise _refuse("PROJECTION_LAYOUT", label) + name = column["name"] + if not isinstance(name, str) or not name: + raise _refuse("PROJECTION_LAYOUT_NAME", label) + if column["delivered_index"] != position: + raise _refuse("PROJECTION_LAYOUT_INDEX", name) + admission = column["admission"] + if admission not in _ADMISSIONS: + raise _refuse("PROJECTION_LAYOUT_ADMISSION", name) + if admission == "not_selected": + # An unselected column carries a reason from the document's own + # closed list, so nothing is dropped by omission. + if column.get("not_selected_reason") not in reasons: + raise _refuse("PROJECTION_LAYOUT_NOT_SELECTED_REASON", name) + elif "not_selected_reason" in column: + raise _refuse("PROJECTION_LAYOUT_NOT_SELECTED_REASON", name) + row = column["layout_row"] + if not isinstance(row, Mapping): + raise _refuse("PROJECTION_LAYOUT_EVIDENCE", name) + text = row["text"] + if not isinstance(text, str) or not text: + raise _refuse("PROJECTION_LAYOUT_EVIDENCE", name) + if hashlib.sha256(text.encode("utf-8")).hexdigest() != row["text_sha256"]: + raise _refuse("PROJECTION_LAYOUT_EVIDENCE_DIGEST", name) + if name not in text: + raise _refuse("PROJECTION_LAYOUT_EVIDENCE_NAME", name) + names.append(name) + if entry["delivered_header_width"] != len(names): + raise _refuse("PROJECTION_LAYOUT_WIDTH", label) + if len(set(names)) != len(names): + raise _refuse("PROJECTION_LAYOUT_DUPLICATE", label) + return tuple(names) + + +def _projected_column( + entry: Mapping[str, object], layout: Mapping[str, Mapping[str, object]] +) -> ProjectedColumn: + field = entry["field"] + if not isinstance(field, str) or field not in layout: + raise _refuse("PROJECTION_UNDELIVERED_FIELD", field) + for key in _FORBIDDEN_COLUMN_KEYS: + if key in entry: + raise _refuse("PROJECTION_DERIVATION_NOT_SUPPORTED", field) + column = layout[field] + if column["admission"] != "monetary_source_projection": + raise _refuse("PROJECTION_ADMISSION_DISAGREEMENT", field) + if entry["delivered_index"] != column["delivered_index"]: + raise _refuse("PROJECTION_INDEX_DISAGREEMENT", field) + if entry["publisher_position"] != column["publisher_position"]: + raise _refuse("PROJECTION_POSITION_DISAGREEMENT", field) + if entry["publisher_label"] != column["publisher_label"]: + raise _refuse("PROJECTION_LABEL_DISAGREEMENT", field) + if entry["dtype"] not in _DECLARED_DTYPE: + raise _refuse("PROJECTION_DTYPE", field) + if entry["grammar"] != _GRAMMAR: + raise _refuse("PROJECTION_GRAMMAR", field) + width = _positive_int(entry["lexical_width"], f"{field}.lexical_width") + if width > LEXICAL_WIDTH_MAX: + raise _refuse("PROJECTION_LEXICAL_WIDTH", field) + units = entry["units"] + digits = _positive_int(units["field_width_digits"], f"{field}.field_width_digits") + if digits > _MAX_AMOUNT_DIGITS or width < digits + 1: + raise _refuse("PROJECTION_WIDTH_BELOW_GRAMMAR", field) + if units["currency"] != "USD" or units["scale"] != "whole_dollars": + raise _refuse("PROJECTION_UNITS", field) + if units["implied_decimals"] != 0: + raise _refuse("PROJECTION_IMPLIED_DECIMALS", field) + sign = entry["sign"] + marker = sign["publisher_signed_marker"] + if not isinstance(marker, bool) or marker != column["publisher_signed_marker"]: + raise _refuse("PROJECTION_SIGN_MARKER", field) + negative = sign["negative_values"] + if negative not in _NEGATIVE_STATES: + raise _refuse("PROJECTION_SIGN_STATE", field) + if marker and negative != "documented_by_record_layout_marker": + raise _refuse("PROJECTION_SIGN_STATE", field) + if sign["negative_zero"] != "refused": + raise _refuse("PROJECTION_NEGATIVE_ZERO_POLICY", field) + missing = entry["missingness"] + if missing["empty_token"] != "refused": + raise _refuse("PROJECTION_EMPTY_TOKEN_POLICY", field) + if missing["zero_meaning"] != "delivered_zero_in_edited_public_file": + raise _refuse("PROJECTION_ZERO_POLICY", field) + if entry["return_grain"]["grain"] != _GRAIN: + raise _refuse("PROJECTION_GRAIN", field) + if entry["source_period_status"]["status"] != _PERIOD_SEMANTICS: + raise _refuse("PROJECTION_PERIOD_STATUS", field) + if entry["target_year_use"] != _TARGET_YEAR_USE: + raise _refuse("PROJECTION_TARGET_YEAR_USE", field) + if entry["derived_combinations"] != "refused": + raise _refuse("PROJECTION_DERIVATION_NOT_SUPPORTED", field) + admission = entry["fit_admission"] + if admission not in _FIT_ADMISSIONS: + raise _refuse("PROJECTION_FIT_ADMISSION", field) + unresolved = tuple(item["code"] for item in entry["unresolved"]) + return ProjectedColumn( + field=field, + concept=entry["concept"], + concept_group=entry["concept_group"], + delivered_index=entry["delivered_index"], + publisher_position=entry["publisher_position"], + publisher_label=entry["publisher_label"], + dtype=entry["dtype"], + lexical_width=width, + grammar=entry["grammar"], + max_digits=digits, + publisher_signed_marker=marker, + negative_values=negative, + fit_admission=admission, + unresolved=unresolved, + ) + + +def _projection_from_document( + document: Mapping[str, object], + *, + route: str, + definition: PufRawSourceDefinition, +) -> PufMonetaryProjection: + """Validate a projection document against a raw source definition.""" + + if not isinstance(definition, PufRawSourceDefinition): + raise _refuse("PROJECTION_DEFINITION_TYPE") + if document.get("schema") != _PROJECTION_SCHEMA: + raise _refuse("PROJECTION_SCHEMA") + if document.get("schema_version") != 1: + raise _refuse("PROJECTION_SCHEMA_VERSION") + if document.get("route") != route: + raise _refuse("PROJECTION_ROUTE") + + raw = document["raw_source_definition"] + raw_sha = _hex64(raw["sha256"], "raw_source_definition.sha256") + if raw_sha != definition.sha256: + raise _refuse("PROJECTION_RAW_DEFINITION_MISMATCH") + + layout = document["layout"] + reasons = layout["not_selected_reasons"] + if not isinstance(reasons, Mapping) or not reasons: + raise _refuse("PROJECTION_NOT_SELECTED_REASONS") + allowed = frozenset(reasons) + main_names = _layout_names(layout["main"], "main", allowed) + demographic_names = _layout_names(layout["demographic"], "demographic", allowed) + if main_names != definition.main.delivered_header: + raise _refuse("PROJECTION_MAIN_LAYOUT_HEADER") + if demographic_names != definition.demographic.delivered_header: + raise _refuse("PROJECTION_DEMOGRAPHIC_LAYOUT_HEADER") + + for name, entry in ( + ("main", definition.main), + ("demographic", definition.demographic), + ): + pin = document["sources"][name] + if ( + pin["sha256"] != entry.sha256 + or pin["git_blob_sha1"] != entry.git_blob_sha1 + or pin["bytes"] != entry.bytes + or pin["header_record_canonical_sha256"] + != entry.header_record_canonical_sha256 + or pin["delivered_header_width"] != len(entry.delivered_header) + or pin["data_records"] != entry.data_records + ): + raise _refuse("PROJECTION_SOURCE_PIN", name) + + by_name = {column["name"]: column for column in layout["main"]["columns"]} + for refused in document["refused_fields"]: + if refused in main_names or refused in demographic_names: + raise _refuse("PROJECTION_REFUSED_FIELD_DELIVERED", refused) + + projection = document["projection"] + if projection["grain"] != _GRAIN: + raise _refuse("PROJECTION_GRAIN") + if projection["period_semantics"] != _PERIOD_SEMANTICS: + raise _refuse("PROJECTION_PERIOD_STATUS") + if projection["target_year_use"] != _TARGET_YEAR_USE: + raise _refuse("PROJECTION_TARGET_YEAR_USE") + if projection["amounts_decoded"] is not True: + raise _refuse("PROJECTION_AMOUNTS_DECODED") + + entries = projection["columns"] + if not isinstance(entries, (list, tuple)) or not entries: + raise _refuse("PROJECTION_EMPTY") + columns = tuple(_projected_column(entry, by_name) for entry in entries) + fields = [column.field for column in columns] + if len(set(fields)) != len(fields): + raise _refuse("PROJECTION_DUPLICATE_FIELD") + for field in fields: + if field in document["refused_fields"]: + raise _refuse("PROJECTION_REFUSED_FIELD", field) + admitted = { + name + for name, column in by_name.items() + if column["admission"] == "monetary_source_projection" + } + if admitted != set(fields): + raise _refuse("PROJECTION_ADMISSION_SET") + if projection["column_count"] != len(columns): + raise _refuse("PROJECTION_COLUMN_COUNT") + if sorted(fields, key=lambda name: by_name[name]["delivered_index"]) != fields: + raise _refuse("PROJECTION_COLUMN_ORDER") + + if any(value is not False for value in document["non_claims"].values()): + raise _refuse("PROJECTION_NON_CLAIMS") + + canonical = canonical_json(_thawed(document)) + return PufMonetaryProjection( + route=route, + document=_frozen(json.loads(canonical.decode("utf-8"))), + canonical=canonical, + sha256=hashlib.sha256(canonical).hexdigest(), + columns=columns, + raw_definition_sha256=raw_sha, + main_delivered_header=main_names, + demographic_delivered_header=demographic_names, + ) + + +def _packaged_bytes() -> bytes: + return ( + resources.files(_PROJECTION_PACKAGE).joinpath(_PROJECTION_RESOURCE).read_bytes() + ) + + +_PACKAGED: PufMonetaryProjection | None = None + + +def packaged_projection() -> PufMonetaryProjection: + """The one closed projection document a production run may use.""" + + global _PACKAGED + if _PACKAGED is None: + document = json.loads(_packaged_bytes().decode("utf-8")) + _PACKAGED = _projection_from_document( + document, route="packaged", definition=packaged_raw_definition() + ) + return _PACKAGED + + +def projection_document_json( + projection: PufMonetaryProjection | None = None, +) -> dict[str, object]: + """A fresh, plain, mutable JSON copy of a projection document.""" + + resolved = packaged_projection() if projection is None else projection + return json.loads(resolved.canonical.decode("utf-8")) + + +def fixture_projection_document( + document: Mapping[str, object], definition: PufRawSourceDefinition +) -> PufMonetaryProjection: + """Build a projection on the explicit non-production test route. + + A fixture projection must declare ``route="test_fixture"`` and the + invented-fixture authority, and it may not restate either packaged source + pin. It still has to authenticate the whole delivered layout, because a + fixture that dropped the layout would be testing a different contract. + """ + + projection = _projection_from_document( + document, route="test_fixture", definition=definition + ) + if document.get("authority") != _FIXTURE_AUTHORITY: + raise _refuse("FIXTURE_AUTHORITY") + packaged_raw = packaged_raw_definition() + genuine = {packaged_raw.main.sha256, packaged_raw.demographic.sha256} + stated = { + document["sources"]["main"]["sha256"], + document["sources"]["demographic"]["sha256"], + } + if stated & genuine: + raise _refuse("FIXTURE_RESTATES_PACKAGED_PIN") + return projection + + +# -------------------------------------------------------------------------- +# The structural decoder +# -------------------------------------------------------------------------- + + +@dataclass(frozen=True) +class PufMonetarySourceProjection: + """Decoded source values with explicit, immutable fitting refusals.""" + + typed: Mapping[str, np.ndarray] + lexical: Mapping[str, tuple[str, ...]] + facts: Mapping[str, object] + column_metadata: Mapping[str, object] + + @property + def rows(self) -> int: + return int(self.facts["rows"]) + + @property + def amount_known(self) -> np.ndarray: + """Whether a row is inside the individual-return amount universe. + + Derived from the authenticated disclosure flag on every access and + handed back read-only, so there is no stored mask to tamper with and + nothing a caller holds can contradict the flags. A false entry means + *outside the universe*: the typed slot holds the sentinel and the + delivered token is the authority. + """ + + return _amount_known(self.typed["disclosure_aggregate"]) + + def aggregate_lexical_rows( + self, field: str | None = None + ) -> Mapping[str, tuple[tuple[int, str], ...]]: + """Exact ``(RECID, token)`` pairs for the disclosure-aggregate rows. + + A local accessor, for diagnosis at the console or in a test. Public + summaries, receipts and reports stay aggregate-only: nothing in the + kernels calls this. No token is parsed, totalled, ordered by value or + compared with another token. + """ + + if field is None: + fields = tuple(self.lexical) + elif field in self.lexical: + fields = (field,) + else: + raise _refuse("AGGREGATE_LEXICAL_FIELD", field) + aggregate = np.flatnonzero(~self.amount_known).tolist() + recids = self.typed["RECID"] + return MappingProxyType( + { + name: tuple( + (int(recids[position]), self.lexical[name][position]) + for position in aggregate + ) + for name in fields + } + ) + + +def _check_csv_profile(profile: Mapping[str, object]) -> None: + """Refuse when the effective acceptance state is not the pinned one.""" + + declared = profile["field_size_limit"] + if not isinstance(declared, int) or isinstance(declared, bool) or declared <= 0: + raise _refuse("DEFINITION_CSV_FIELD_SIZE_LIMIT") + if csv.field_size_limit() != declared: + raise _refuse("CSV_FIELD_SIZE_LIMIT") + if profile["quoting"] != "QUOTE_MINIMAL" or profile["strict"] is not True: + raise _refuse("DEFINITION_CSV_PROFILE") + if profile["encoding"] != "utf-8-strict" or profile["newline"] != "": + raise _refuse("DEFINITION_CSV_PROFILE") + + +def _reader(payload: bytes, profile: Mapping[str, object]): + if not isinstance(payload, bytes): + raise _refuse("DECODE_PAYLOAD_TYPE") + stream = io.TextIOWrapper( + io.BytesIO(payload), encoding="utf-8", errors="strict", newline="" + ) + return csv.reader( + stream, + delimiter=profile["delimiter"], + quotechar=profile["quotechar"], + doublequote=bool(profile["doublequote"]), + quoting=csv.QUOTE_MINIMAL, + strict=True, + ) + + +def _canonical_header_digest(header: Sequence[str]) -> str: + payload = json.dumps( + list(header), + sort_keys=True, + separators=(",", ":"), + ensure_ascii=True, + allow_nan=False, + ).encode("ascii") + return hashlib.sha256(payload).hexdigest() + + +def _key_token(value: str, *, width: int) -> str: + """The unsigned decimal ``RECID`` token, as the raw slice reads it.""" + + if not isinstance(value, str): + raise _refuse("KEY_TOKEN_TYPE") + if not value.isascii(): + raise _refuse("KEY_TOKEN_NON_ASCII") + if "\x00" in value: + raise _refuse("KEY_TOKEN_EMBEDDED_NUL") + if not value: + raise _refuse("KEY_TOKEN_EMPTY") + if len(value) > width: + raise _refuse("KEY_TOKEN_OVER_WIDTH") + if any(character not in _DIGITS for character in value): + raise _refuse("KEY_TOKEN_GRAMMAR") + number = int(value) + if number > _INT64_MAX: + raise _refuse("KEY_TOKEN_INT64_OVERFLOW") + return value + + +def _amount(value: str, column: ProjectedColumn) -> int: + """Parse one delivered amount token, or refuse. + + The grammar is exactly what the retained booklets support: an optional + leading minus, then decimal digits, no wider than the publisher's declared + amount-field width. There is no missing-value sentinel anywhere in either + booklet, so an empty token is a refusal to be preserved and reviewed + rather than a zero to be invented, and a plus sign, a decimal point, + surrounding whitespace or an exponent is a refusal rather than a silently + widened grammar. ``-0`` is refused outright: a signed zero would compare + equal to zero while spelling differently. + """ + + label = column.field + if not isinstance(value, str): + raise _refuse("AMOUNT_TOKEN_TYPE", label) + if not value.isascii(): + raise _refuse("AMOUNT_TOKEN_NON_ASCII", label) + if "\x00" in value: + raise _refuse("AMOUNT_TOKEN_EMBEDDED_NUL", label) + if not value: + raise _refuse("AMOUNT_TOKEN_EMPTY", label) + if len(value) > column.lexical_width: + raise _refuse("AMOUNT_TOKEN_OVER_WIDTH", label) + digits = value[1:] if value.startswith("-") else value + if not digits: + raise _refuse("AMOUNT_TOKEN_GRAMMAR", label) + if any(character not in _DIGITS for character in digits): + raise _refuse("AMOUNT_TOKEN_GRAMMAR", label) + if len(digits) > column.max_digits: + raise _refuse("AMOUNT_TOKEN_OVER_DIGITS", label) + number = -int(digits) if value.startswith("-") else int(digits) + if number == 0 and value.startswith("-"): + raise _refuse("AMOUNT_TOKEN_NEGATIVE_ZERO", label) + if not _INT64_MIN <= number <= _INT64_MAX: + raise _refuse("AMOUNT_TOKEN_INT64_OVERFLOW", label) + return number + + +def _aggregate_token(value: str, column: ProjectedColumn) -> str: + """Validate one disclosure-aggregate token, and return it unchanged. + + This is a **separate, explicitly bounded operational grammar** for the + delivered aggregate rows of this pinned file. It is not a newly discovered + publisher specification, and it is never applied to an individual return. + Optional leading minus, one or more digits, at most one point followed by + one or more digits: no plus sign, exponent, whitespace, embedded NUL, + empty token or negative zero in either spelling, at most the column's + declared digit bound and at most its declared width. + + The token is returned exactly as delivered. Nothing here converts it to a + number, rounds it, or compares it against another token. + """ + + label = column.field + if not isinstance(value, str): + raise _refuse("AGGREGATE_TOKEN_TYPE", label) + if not value.isascii(): + raise _refuse("AGGREGATE_TOKEN_NON_ASCII", label) + if "\x00" in value: + raise _refuse("AGGREGATE_TOKEN_EMBEDDED_NUL", label) + if not value: + raise _refuse("AGGREGATE_TOKEN_EMPTY", label) + if len(value) > column.lexical_width: + raise _refuse("AGGREGATE_TOKEN_OVER_WIDTH", label) + negative = value.startswith("-") + body = value[1:] if negative else value + if not body: + raise _refuse("AGGREGATE_TOKEN_GRAMMAR", label) + if body.count(".") > 1: + raise _refuse("AGGREGATE_TOKEN_GRAMMAR", label) + whole, point, fraction = body.partition(".") + if point and (not whole or not fraction): + raise _refuse("AGGREGATE_TOKEN_GRAMMAR", label) + digits = whole + fraction + if not digits or any(character not in _DIGITS for character in digits): + raise _refuse("AGGREGATE_TOKEN_GRAMMAR", label) + if len(digits) > column.max_digits: + raise _refuse("AGGREGATE_TOKEN_OVER_DIGITS", label) + if negative and set(digits) == {"0"}: + raise _refuse("AGGREGATE_TOKEN_NEGATIVE_ZERO", label) + return value + + +def _amount_known(flags: np.ndarray) -> np.ndarray: + """The individual-return amount universe, derived from the flag alone. + + Returned read-only and recomputed at every call, so no caller can hold a + mask that contradicts the disclosure flags it came from. + """ + + known = np.asarray(flags, dtype=_STRUCTURAL_DTYPES["disclosure_aggregate"]) == 0 + # A read-only flag on an owning ndarray can be reversed with setflags. + # An immutable bytes backing store makes that reversal impossible. + return np.frombuffer(known.tobytes(), dtype=np.bool_) + + +def _typed_column( + tokens: Sequence[str], known: np.ndarray, column: ProjectedColumn +) -> np.ndarray: + """Parse by universe: an amount where known, the sentinel where not.""" + + if len(tokens) != int(known.shape[0]): + raise _refuse("COLUMN_LEXICAL_ROWS", column.field) + values: list[int] = [] + for position, token in enumerate(tokens): + if bool(known[position]): + values.append(_amount(token, column)) + else: + _aggregate_token(token, column) + values.append(AGGREGATE_SENTINEL) + return np.asarray(values, dtype=column.numpy_dtype) + + +def _check_header( + reader, projection: PufMonetaryProjection, definition: PufRawSourceDefinition +) -> list[str]: + pin = definition.main + profile = definition.document["csv_profile"] + try: + header = next(reader) + except StopIteration as error: + raise _refuse("HEADER_MISSING") from error + except (csv.Error, UnicodeError) as error: + raise _refuse("HEADER_UNREADABLE") from error + if len(header) > profile["header_field_cap"]: + raise _refuse("HEADER_WIDTH_LIMIT") + if tuple(header) != pin.delivered_header: + raise _refuse("HEADER_MISMATCH") + if tuple(header) != projection.main_delivered_header: + raise _refuse("HEADER_NOT_AUTHENTICATED_LAYOUT") + if _canonical_header_digest(header) != pin.header_record_canonical_sha256: + raise _refuse("HEADER_DIGEST") + refused = set(projection.document["refused_fields"]) & set(header) + if refused: + raise _refuse("HEADER_REFUSED_FIELD", sorted(refused)) + return header + + +def _records(reader, definition: PufRawSourceDefinition): + profile = definition.document["csv_profile"] + width = len(definition.main.delivered_header) + cap = profile["logical_record_character_cap"] + limit = profile["record_cap_per_file"] + count = 0 + while True: + try: + record = next(reader) + except StopIteration: + return + except (csv.Error, UnicodeError) as error: + raise _refuse("RECORD_UNREADABLE") from error + count += 1 + if count > limit: + raise _refuse("RECORD_CAP") + if len(record) != width: + raise _refuse("RECORD_WIDTH") + if sum(len(cell) for cell in record) > cap: + raise _refuse("RECORD_CHARACTER_CAP") + yield record + + +def decode_puf_monetary_source( + main_bytes: bytes, + status: object, + projection: PufMonetaryProjection, + definition: PufRawSourceDefinition, +) -> PufMonetarySourceProjection: + """Decode the enumerated monetary columns, bound to the status artifact. + + ``status`` is the decoded ``puf_2015_raw_return_status``. Its ``RECID`` + sequence and aggregate flags must be element-wise identical to this + parse's, and its ``demographic_status`` is carried through rather than + re-joined, so the two artifacts cannot disagree about which returns + matched. Every delivered main record is projected: nothing is filtered, + and the four disclosure aggregates stay tagged rather than dropped. + + Amount tokens are collected before that alignment runs and parsed after + it, by universe: the unchanged integer grammar on individual returns, the + bounded aggregate lexical grammar plus the out-of-universe sentinel on + disclosure aggregates. A parse therefore cannot decide which grammar a row + answers to before the status artifact has proved which row it is. + """ + + if not isinstance(projection, PufMonetaryProjection): + raise _refuse("DECODE_PROJECTION_TYPE") + if projection.raw_definition_sha256 != definition.sha256: + raise _refuse("DECODE_DEFINITION_MISMATCH") + profile = definition.document["csv_profile"] + _check_csv_profile(profile) + key_width = next( + field["lexical_width"] + for field in definition.document["fields"] + if field["name"] == "RECID" + ) + + reader = _reader(main_bytes, profile) + header = _check_header(reader, projection, definition) + positions = { + column.field: header.index(column.field) for column in projection.columns + } + for column in projection.columns: + if positions[column.field] != column.delivered_index: + raise _refuse("DELIVERED_INDEX_DISAGREEMENT", column.field) + recid_position = header.index("RECID") + + keys: list[int] = [] + seen: set[str] = set() + aggregates = frozenset(PUF_AGGREGATE_RECIDS) + flags: list[int] = [] + tokens: dict[str, list[str]] = {column.field: [] for column in projection.columns} + + # Tokens are collected first and parsed afterwards. Which grammar a row's + # amount answers to is decided only once the status artifact has proved + # the RECID sequence and the aggregate flags, so "authenticate the + # alignment before interpreting a row's amount universe" is the literal + # order of operations rather than a claim about one. + for record in _records(reader, definition): + key = _key_token(record[recid_position], width=key_width) + if key in seen: + raise _refuse("MAIN_DUPLICATE_LEXICAL_KEY") + seen.add(key) + recid = int(key) + keys.append(recid) + flags.append(1 if recid in aggregates else 0) + for column in projection.columns: + tokens[column.field].append(record[positions[column.field]]) + rows = len(keys) + if rows == 0: + raise _refuse("MAIN_NO_RECORDS") + if rows != definition.main.data_records: + raise _refuse("MAIN_RECORD_COUNT") + + typed: dict[str, np.ndarray] = { + "RECID": np.asarray(keys, dtype=_STRUCTURAL_DTYPES["RECID"]), + "disclosure_aggregate": np.asarray( + flags, dtype=_STRUCTURAL_DTYPES["disclosure_aggregate"] + ), + } + if len(np.unique(typed["RECID"])) != rows: + raise _refuse("MAIN_NUMERIC_KEY_COLLISION") + typed["demographic_status"] = _status_alignment(status, typed, rows) + known = _amount_known(typed["disclosure_aggregate"]) + for column in projection.columns: + typed[column.field] = _typed_column(tokens[column.field], known, column) + + lexical = {name: tuple(items) for name, items in tokens.items()} + facts = _facts_from_arrays(typed, lexical, projection) + # Match envelope readback's immutable byte-backed arrays. In particular, + # disclosure flags cannot change after an amount-known mask is returned. + typed = { + name: np.frombuffer(array.tobytes(), dtype=array.dtype) + for name, array in typed.items() + } + return PufMonetarySourceProjection( + typed=MappingProxyType(typed), + lexical=MappingProxyType(lexical), + facts=MappingProxyType(facts), + column_metadata=_frozen(facts["column_metadata"]), + ) + + +def _status_alignment( + status: object, typed: Mapping[str, np.ndarray], rows: int +) -> np.ndarray: + """Bind this parse to the reviewed status artifact, element by element.""" + + try: + status_typed = status.typed + except AttributeError as error: + raise _refuse("STATUS_ARTIFACT_TYPE") from error + for name in ("RECID", "disclosure_aggregate", "demographic_status"): + if name not in status_typed: + raise _refuse("STATUS_ARTIFACT_COLUMN", name) + if int(status_typed["RECID"].shape[0]) != rows: + raise _refuse("STATUS_ROW_COUNT") + if not np.array_equal( + np.asarray(status_typed["RECID"], dtype=" dict[str, np.ndarray]: + """The three disjoint row classes, which must partition the rows exactly.""" + + aggregate = typed["disclosure_aggregate"] == 1 + status = typed["demographic_status"] == 1 + masks = { + "disclosure_aggregate": aggregate, + "demographic_matched": (~aggregate) & status, + "demographic_unmatched": (~aggregate) & (~status), + } + total = sum(int(np.count_nonzero(mask)) for mask in masks.values()) + if total != int(typed["RECID"].shape[0]): + raise _refuse("ROW_CLASS_PARTITION") + return masks + + +def _column_facts(values: np.ndarray, mask: np.ndarray) -> dict[str, object]: + selected = values[mask] + count = int(selected.shape[0]) + if count == 0: + return { + "rows": 0, + "zero": 0, + "negative": 0, + "positive": 0, + "minimum": None, + "maximum": None, + "sum": 0, + } + total = int(sum(int(item) for item in selected.tolist())) + if not _INT64_MIN <= total <= _INT64_MAX: + raise _refuse("COLUMN_SUM_OVERFLOW") + return { + "rows": count, + "zero": int(np.count_nonzero(selected == 0)), + "negative": int(np.count_nonzero(selected < 0)), + "positive": int(np.count_nonzero(selected > 0)), + "minimum": int(selected.min()), + "maximum": int(selected.max()), + "sum": total, + } + + +def _aggregate_column_facts( + tokens: Sequence[str], mask: np.ndarray +) -> dict[str, object]: + """Token classes for the aggregate rows, and nothing that reads as money. + + Counts of token shapes and one token width. There is no zero, sign census, + minimum, maximum or sum key here, because the delivered token is the + authority for these rows and this module never interprets it. A diagnostic + may count token classes; it may not state an aggregate amount. + """ + + selected = [ + token for token, taken in zip(tokens, mask.tolist(), strict=True) if taken + ] + if not selected: + return { + "rows": 0, + "integer_tokens": 0, + "fractional_tokens": 0, + "negative_tokens": 0, + "max_token_width": None, + } + return { + "rows": len(selected), + "integer_tokens": sum(1 for token in selected if "." not in token), + "fractional_tokens": sum(1 for token in selected if "." in token), + "negative_tokens": sum(1 for token in selected if token.startswith("-")), + "max_token_width": max(len(token) for token in selected), + } + + +def _facts_from_arrays( + typed: Mapping[str, np.ndarray], + lexical: Mapping[str, Sequence[str]], + projection: PufMonetaryProjection, +) -> dict[str, object]: + """Derive every reported fact from the arrays themselves. + + The producer builds its facts here and the envelope readback rebuilds them + here, so a header can never report a count, a sum or a sign census the + arrays do not carry. Layout counts and column semantics come from the + authenticated projection document, rather than being inferred from rows. + The lexical arrays are an input because the aggregate class is described + from its delivered tokens and from nothing else. + """ + + rows = int(typed["RECID"].shape[0]) + masks = _row_masks(typed) + known = _amount_known(typed["disclosure_aggregate"]) + for column in projection.columns: + if len(lexical[column.field]) != rows: + raise _refuse("FACTS_LEXICAL_ROWS", column.field) + return { + "rows": rows, + "main_records": rows, + "projected_columns": len(projection.columns), + "layout_columns_authenticated": len(projection.main_delivered_header), + "demographic_layout_columns": len(projection.demographic_delivered_header), + "aggregate_records": int(np.count_nonzero(masks["disclosure_aggregate"])), + "demographic_matched_rows": int( + np.count_nonzero(typed["demographic_status"] == 1) + ), + "demographic_unmatched_rows": int( + np.count_nonzero(typed["demographic_status"] == 0) + ), + "row_class_rows": { + name: int(np.count_nonzero(masks[name])) for name in ROW_CLASSES + }, + "column_facts": { + column.field: { + AGGREGATE_ROW_CLASS: _aggregate_column_facts( + lexical[column.field], masks[AGGREGATE_ROW_CLASS] + ), + **{ + name: _column_facts(typed[column.field], masks[name]) + for name in RETURN_ROW_CLASSES + }, + } + for column in projection.columns + }, + "column_metadata": { + column.field: { + "concept": column.concept, + "grain": _GRAIN, + # The packaged projection document's integer, whole-dollar, + # twelve-digit semantics describe individual returns. That + # document's bytes are unchanged; this metadata says who they + # apply to, and what happens to the rows they do not. + "amount_units": _AMOUNT_UNITS, + "amount_units_scope": _AMOUNT_UNITS_SCOPE, + "amount_universe": _AMOUNT_UNITS_SCOPE, + "amount_known_rule": AMOUNT_KNOWN_RULE, + "aggregate_amount_universe": _AGGREGATE_UNIVERSE, + "aggregate_row_treatment": _AGGREGATE_TREATMENT, + "aggregate_typed_sentinel": AGGREGATE_SENTINEL, + "aggregate_token_grammar": _AGGREGATE_GRAMMAR, + "aggregate_token_max_digits": column.max_digits, + "aggregate_token_max_width": column.lexical_width, + "aggregate_token_grammar_authority": _AGGREGATE_GRAMMAR_AUTHORITY, + "aggregate_lexical_authority": _AGGREGATE_AUTHORITY, + "aggregate_numeric_interpretation": _AGGREGATE_INTERPRETATION, + "period_semantics": _PERIOD_SEMANTICS, + "fit_admission": column.fit_admission, + "unresolved": list(column.unresolved), + "negative_values": column.negative_values, + "zero_meaning": "delivered_zero_in_edited_public_file", + "target_year_use": _TARGET_YEAR_USE, + "projection_sha256": projection.sha256, + } + for column in projection.columns + }, + "grain": _GRAIN, + "period_semantics": _PERIOD_SEMANTICS, + "amounts_decoded": True, + "amount_units": _AMOUNT_UNITS, + "amount_units_scope": _AMOUNT_UNITS_SCOPE, + "amount_known_rule": AMOUNT_KNOWN_RULE, + "amount_known_rows": int(np.count_nonzero(known)), + "aggregate_amount_universe": _AGGREGATE_UNIVERSE, + "aggregate_row_treatment": _AGGREGATE_TREATMENT, + "aggregate_typed_sentinel": AGGREGATE_SENTINEL, + "aggregate_token_grammar": _AGGREGATE_GRAMMAR, + "aggregate_lexical_authority": _AGGREGATE_AUTHORITY, + "aggregate_numeric_interpretation": _AGGREGATE_INTERPRETATION, + "target_year_use": _TARGET_YEAR_USE, + } + + +# -------------------------------------------------------------------------- +# The typed envelope +# -------------------------------------------------------------------------- + + +def _fixed_ascii(values: Sequence[str], width: int, label: str) -> bytes: + body = bytearray(len(values) * width) + for position, value in enumerate(values): + raw = value.encode("ascii") + if len(raw) > width: + raise _refuse("LEXICAL_OVER_WIDTH", label) + if b"\x00" in raw: + raise _refuse("LEXICAL_EMBEDDED_NUL", label) + start = position * width + body[start : start + len(raw)] = raw + return bytes(body) + + +def _read_fixed_ascii( + body: bytes, rows: int, width: int, label: str +) -> tuple[str, ...]: + values = [] + for position in range(rows): + cell = body[position * width : (position + 1) * width] + stripped = cell.rstrip(b"\x00") + if b"\x00" in stripped: + raise _refuse("LEXICAL_EMBEDDED_NUL", label) + try: + values.append(stripped.decode("ascii")) + except UnicodeDecodeError as error: + raise _refuse("LEXICAL_NON_ASCII", label) from error + return tuple(values) + + +def payload_bound( + rows: int, projection: PufMonetaryProjection +) -> tuple[int, dict[str, int]]: + """The exact body size this artifact allocates, and its per-column parts.""" + + sizes = { + name: rows * np.dtype(_STRUCTURAL_DTYPES[name]).itemsize + for name in _STRUCTURAL_COLUMNS + } + for column in projection.columns: + sizes[column.field] = rows * np.dtype(column.numpy_dtype).itemsize + sizes[f"{column.field}_lexical"] = rows * column.lexical_width + return sum(sizes.values()), sizes + + +def _envelope_column_order(projection: PufMonetaryProjection) -> list[str]: + return [ + *_STRUCTURAL_COLUMNS, + *(column.field for column in projection.columns), + *(f"{column.field}_lexical" for column in projection.columns), + ] + + +def encode_monetary_projection( + decoded: PufMonetarySourceProjection, + projection: PufMonetaryProjection, + definition: PufRawSourceDefinition, + *, + status_artifact_sha256: str, +) -> bytes: + """Encode the projection into the bounded, self-describing envelope.""" + + rows = decoded.rows + total, sizes = payload_bound(rows, projection) + if total > BODY_MAX_BYTES: + raise _refuse("BODY_OVER_BOUND", total) + dtypes = { + **_STRUCTURAL_DTYPES, + **{column.field: column.numpy_dtype for column in projection.columns}, + } + widths = {column.field: column.lexical_width for column in projection.columns} + bodies: list[bytes] = [] + columns: list[dict[str, object]] = [] + for name in _envelope_column_order(projection): + if name.endswith("_lexical"): + field = name.removesuffix("_lexical") + body = _fixed_ascii(decoded.lexical[field], widths[field], field) + entry: dict[str, object] = { + "name": name, + "kind": _LEXICAL_KIND, + "width": widths[field], + } + else: + array = np.ascontiguousarray(decoded.typed[name], dtype=dtypes[name]) + if array.shape != (rows,): + raise _refuse("COLUMN_SHAPE", name) + body = array.tobytes() + entry = {"name": name, "kind": _TYPED_KIND, "dtype": dtypes[name]} + if len(body) != sizes[name]: + raise _refuse("COLUMN_SIZE", name) + bodies.append(body) + entry["bytes"] = len(body) + entry["sha256"] = hashlib.sha256(body).hexdigest() + columns.append(entry) + header = { + "schema_version": MONETARY_PROJECTION_TYPE.schema_version, + "type": [ + MONETARY_PROJECTION_TYPE.name, + MONETARY_PROJECTION_TYPE.schema_version, + ], + "rows": rows, + "projection_sha256": projection.sha256, + "projection_route": projection.route, + "raw_source_definition_sha256": definition.sha256, + "status_artifact_sha256": _hex64( + status_artifact_sha256, "status_artifact_sha256" + ), + "sources": { + "main": { + "sha256": definition.main.sha256, + "git_blob_sha1": definition.main.git_blob_sha1, + "bytes": definition.main.bytes, + "header_record_canonical_sha256": ( + definition.main.header_record_canonical_sha256 + ), + }, + "demographic": { + "sha256": definition.demographic.sha256, + "git_blob_sha1": definition.demographic.git_blob_sha1, + "bytes": definition.demographic.bytes, + "header_record_canonical_sha256": ( + definition.demographic.header_record_canonical_sha256 + ), + }, + }, + "header_length_endianness": "big", + "body_endianness": "little", + "columns": columns, + "facts": dict(decoded.facts), + } + encoded = canonical_json(header) + if len(encoded) > HEADER_MAX_BYTES: + raise _refuse("HEADER_OVER_BOUND", len(encoded)) + return b"".join( + (MONETARY_PROJECTION_MAGIC, len(encoded).to_bytes(4, "big"), encoded, *bodies) + ) + + +def _exact_keys( + value: object, keys: frozenset[str], label: str +) -> Mapping[str, object]: + if not isinstance(value, dict) or set(value) != set(keys): + raise _refuse("ENVELOPE_SCHEMA", label) + return value + + +def _envelope_int(value: object, label: str, *, minimum: int = 0) -> int: + if not isinstance(value, int) or isinstance(value, bool) or value < minimum: + raise _refuse("ENVELOPE_INTEGER", label) + return value + + +def _resolve_envelope_projection( + header: Mapping[str, object], + projection: PufMonetaryProjection | None, +) -> PufMonetaryProjection | None: + """The projection this payload may be checked against, or ``None``.""" + + route = header["projection_route"] + if route not in ("packaged", "test_fixture"): + raise _refuse("ENVELOPE_PROJECTION_ROUTE") + digest = _hex64(header["projection_sha256"], "envelope.projection_sha256") + if projection is not None: + if not isinstance(projection, PufMonetaryProjection): + raise _refuse("DECODE_PROJECTION_TYPE") + if projection.route != route or projection.sha256 != digest: + raise _refuse("ENVELOPE_PROJECTION_MISMATCH") + return projection + if route == "packaged": + packaged = packaged_projection() + if packaged.sha256 != digest: + raise _refuse("ENVELOPE_PROJECTION_NOT_PACKAGED") + return packaged + return None + + +def _check_envelope_sources( + header: Mapping[str, object], projection: PufMonetaryProjection | None +) -> None: + sources = _exact_keys( + header["sources"], frozenset({"main", "demographic"}), "sources" + ) + for name in ("main", "demographic"): + entry = _exact_keys(sources[name], _ENVELOPE_SOURCE_KEYS, f"sources.{name}") + _hex64(entry["sha256"], f"sources.{name}.sha256") + _envelope_int(entry["bytes"], f"sources.{name}.bytes", minimum=1) + if projection is None: + continue + pin = projection.document["sources"][name] + if ( + entry["sha256"] != pin["sha256"] + or entry["git_blob_sha1"] != pin["git_blob_sha1"] + or entry["bytes"] != pin["bytes"] + or entry["header_record_canonical_sha256"] + != pin["header_record_canonical_sha256"] + ): + raise _refuse("ENVELOPE_SOURCE_PIN", name) + + +def _envelope_columns( + header: Mapping[str, object], rows: int, projection: PufMonetaryProjection +) -> list[Mapping[str, object]]: + """Close the column schema: exact order, kind, dtype, width and size.""" + + columns = header["columns"] + if not isinstance(columns, list): + raise _refuse("ENVELOPE_COLUMNS") + declared = _envelope_column_order(projection) + if len(columns) != len(declared): + raise _refuse("ENVELOPE_COLUMNS") + dtypes = { + **_STRUCTURAL_DTYPES, + **{column.field: column.numpy_dtype for column in projection.columns}, + } + widths = {column.field: column.lexical_width for column in projection.columns} + resolved: list[Mapping[str, object]] = [] + total = 0 + for column, name in zip(columns, declared, strict=True): + if not isinstance(column, dict) or column.get("name") != name: + raise _refuse("ENVELOPE_COLUMNS") + if name.endswith("_lexical"): + field = name.removesuffix("_lexical") + entry = _exact_keys(column, _ENVELOPE_LEXICAL_COLUMN_KEYS, name) + if entry["kind"] != _LEXICAL_KIND: + raise _refuse("ENVELOPE_COLUMN_KIND", name) + width = _envelope_int(entry["width"], f"{name}.width", minimum=1) + if width > LEXICAL_WIDTH_MAX or width != widths[field]: + raise _refuse("ENVELOPE_COLUMN_WIDTH", name) + expected = rows * width + else: + entry = _exact_keys(column, _ENVELOPE_TYPED_COLUMN_KEYS, name) + if entry["kind"] != _TYPED_KIND: + raise _refuse("ENVELOPE_COLUMN_KIND", name) + # An exact dtype string closes width, signedness and endianness in + # one comparison, so `` BODY_MAX_BYTES: + raise _refuse("ENVELOPE_BODY_OVER_BOUND", total) + resolved.append(entry) + return resolved + + +def _verify_readback( + typed: Mapping[str, np.ndarray], + lexical: Mapping[str, tuple[str, ...]], + projection: PufMonetaryProjection, +) -> None: + """Re-prove the projection's own invariants from the arrays read back.""" + + rows = int(typed["RECID"].shape[0]) + status = typed["demographic_status"] + aggregate = typed["disclosure_aggregate"] + if not np.isin(status, (0, 1)).all(): + raise _refuse("READBACK_DEMOGRAPHIC_STATUS") + if not np.isin(aggregate, (0, 1)).all(): + raise _refuse("READBACK_AGGREGATE_FLAG") + recids = typed["RECID"] + if len(np.unique(recids)) != rows: + raise _refuse("READBACK_NUMERIC_KEY_COLLISION") + expected = np.isin(recids, np.asarray(PUF_AGGREGATE_RECIDS, dtype=" PufMonetarySourceProjection: + """Decode an artifact payload and re-prove it, never trusting its header. + + The header is a closed schema; every reported fact is recomputed from the + arrays actually read back and must agree exactly, so a payload cannot + report a row count, a negative census, a class partition or a column sum + it does not carry. + """ + + if not isinstance(payload, bytes): + raise _refuse("ENVELOPE_MAGIC") + if not payload.startswith(MONETARY_PROJECTION_MAGIC): + # A superseded magic is named, not reinterpreted: those payloads + # declare a different amount universe, and silently reading one under + # this contract would report aggregates as individual amounts. + if any(payload.startswith(magic) for magic in SUPERSEDED_MAGIC): + raise _refuse("ENVELOPE_SUPERSEDED_MAGIC") + raise _refuse("ENVELOPE_MAGIC") + start = len(MONETARY_PROJECTION_MAGIC) + if len(payload) < start + 4: + raise _refuse("ENVELOPE_TRUNCATED") + length = int.from_bytes(payload[start : start + 4], "big") + if not 0 < length <= HEADER_MAX_BYTES or len(payload) < start + 4 + length: + raise _refuse("ENVELOPE_HEADER_LENGTH") + encoded = payload[start + 4 : start + 4 + length] + try: + header = json.loads(encoded.decode("utf-8")) + except (UnicodeError, ValueError, RecursionError) as error: + raise _refuse("ENVELOPE_HEADER_JSON") from error + try: + recanonical = canonical_json(header) + except (TypeError, ValueError, RecursionError) as error: + raise _refuse("ENVELOPE_HEADER_JSON") from error + if recanonical != encoded: + raise _refuse("ENVELOPE_HEADER_NOT_CANONICAL") + _exact_keys(header, _ENVELOPE_HEADER_KEYS, "header") + if header["schema_version"] != MONETARY_PROJECTION_TYPE.schema_version or header[ + "type" + ] != [ + MONETARY_PROJECTION_TYPE.name, + MONETARY_PROJECTION_TYPE.schema_version, + ]: + raise _refuse("ENVELOPE_TYPE") + if ( + header["header_length_endianness"] != "big" + or header["body_endianness"] != "little" + ): + raise _refuse("ENVELOPE_ENDIANNESS") + _hex64(header["raw_source_definition_sha256"], "raw_source_definition_sha256") + _hex64(header["status_artifact_sha256"], "status_artifact_sha256") + if expected_status_artifact_sha256 is not None: + _hex64(expected_status_artifact_sha256, "expected_status_artifact_sha256") + if header["status_artifact_sha256"] != expected_status_artifact_sha256: + raise _refuse("ENVELOPE_STATUS_ARTIFACT_MISMATCH") + rows = _envelope_int(header["rows"], "rows", minimum=1) + resolved = _resolve_envelope_projection(header, projection) + if resolved is None: + # A fixture-route payload has no reconstructible projection document, + # so there is nothing to close its column set against. That is a + # caller error here rather than a weaker structural pass: the + # projection is exactly what makes the columns meaningful. + raise _refuse("ENVELOPE_PROJECTION_REQUIRED") + if resolved.raw_definition_sha256 != header["raw_source_definition_sha256"]: + raise _refuse("ENVELOPE_RAW_DEFINITION_MISMATCH") + _check_envelope_sources(header, resolved) + columns = _envelope_columns(header, rows, resolved) + + cursor = start + 4 + length + typed: dict[str, np.ndarray] = {} + lexical: dict[str, tuple[str, ...]] = {} + for column in columns: + name = column["name"] + size = column["bytes"] + body = payload[cursor : cursor + size] + if len(body) != size: + raise _refuse("ENVELOPE_BODY_TRUNCATED", name) + if hashlib.sha256(body).hexdigest() != column["sha256"]: + raise _refuse("ENVELOPE_BODY_DIGEST", name) + cursor += size + if column["kind"] == _TYPED_KIND: + typed[name] = np.frombuffer(body, dtype=np.dtype(column["dtype"])) + else: + field = name.removesuffix("_lexical") + lexical[field] = _read_fixed_ascii(body, rows, column["width"], field) + if cursor != len(payload): + raise _refuse("ENVELOPE_TRAILING_BYTES") + + _verify_readback(typed, lexical, resolved) + facts = header["facts"] + _exact_keys(facts, _ENVELOPE_FACT_KEYS, "facts") + for field, classes in facts["column_facts"].items(): + if field not in resolved.fields: + raise _refuse("ENVELOPE_FACTS", "column_facts") + _exact_keys(classes, frozenset(ROW_CLASSES), f"column_facts.{field}") + for name, entry in classes.items(): + keys = ( + _AGGREGATE_COLUMN_FACT_KEYS + if name == AGGREGATE_ROW_CLASS + else _RETURN_COLUMN_FACT_KEYS + ) + _exact_keys(entry, keys, f"column_facts.{field}.{name}") + recomputed = _facts_from_arrays(typed, lexical, resolved) + if canonical_json(recomputed) != canonical_json(dict(facts)): + raise _refuse("ENVELOPE_FACTS_INCONSISTENT") + if recomputed["rows"] != rows: + raise _refuse("ENVELOPE_ROWS") + return PufMonetarySourceProjection( + typed=MappingProxyType(typed), + lexical=MappingProxyType(lexical), + facts=MappingProxyType(recomputed), + column_metadata=_frozen(recomputed["column_metadata"]), + ) + + +# -------------------------------------------------------------------------- +# The graph producer, consumer and registry +# -------------------------------------------------------------------------- + +_IMPLEMENTATION_MODULES = ( + "microcosm.build.us_runtime.puf_raw_source", + "microcosm.build.us_runtime.puf_monetary_source", +) +#: The implementation identity moves with the contract: a node written +#: against the version-2 amount universe must not bind to this kernel. +_IMPLEMENTATION_IDENTITY_MAGIC = b"us.puf.monetary_source/implementation/3\n" + +_STATUS_ALIAS = "return_status" +_PROJECTION_ALIAS = "monetary_projection" + + +class USPufMonetarySourceKernel(KernelBase): + """Read the pinned main delivery and produce the monetary projection.""" + + ref = "us.puf.monetary_source.projection@3" + capabilities = Capabilities( + Determinism.DETERMINISTIC, + numeric=Numeric.BITWISE, + dependencies=("numpy",), + ) + + def __init__( + self, + *, + projection: PufMonetaryProjection | None = None, + definition: PufRawSourceDefinition | None = None, + source_codecs: SourceCodecRegistry | None = None, + ) -> None: + self.definition = ( + packaged_raw_definition() if definition is None else definition + ) + self.projection = packaged_projection() if projection is None else projection + if self.projection.route not in ("packaged", "test_fixture"): + raise _refuse("KERNEL_PROJECTION_ROUTE") + if self.projection.route != self.definition.route: + raise _refuse("KERNEL_ROUTE_DISAGREEMENT") + if self.projection.raw_definition_sha256 != self.definition.sha256: + raise _refuse("KERNEL_DEFINITION_MISMATCH") + if self.projection.route == "packaged": + # The production wrapper, not the bare decoder, is the authority: + # the projection must be the packaged file's own canonical bytes, + # and the document it carries must still be those exact bytes. + packaged = packaged_projection() + if ( + self.projection.canonical != packaged.canonical + or self.projection.sha256 != packaged.sha256 + or canonical_json(_thawed(self.projection.document)) + != self.projection.canonical + ): + raise _refuse("KERNEL_PROJECTION_NOT_PACKAGED") + self.source_codecs = ( + puf_raw_source_codecs(self.definition) + if source_codecs is None + else source_codecs + ) + + def implementation_hash(self) -> str: + """Module source, dependency versions and the effective CSV profile.""" + + from importlib import import_module + + base = source_hash( + *(import_module(name) for name in _IMPLEMENTATION_MODULES), + dependencies=self.capabilities.dependencies, + ) + digest = hashlib.sha256(_IMPLEMENTATION_IDENTITY_MAGIC) + digest.update(base.encode("ascii")) + digest.update(b"\0") + digest.update(canonical_json(csv_acceptance_profile())) + return digest.hexdigest() + + def run(self, context: KernelContext) -> KernelResult: + definition = self.definition + projection = self.projection + if ( + context.node.kernel != self.ref + or context.node.structural is not StructuralDelta.NONE + or context.node.sources != (definition.main.source_name,) + or context.node.inputs + or context.node.outputs + or len(context.node.artifact_inputs) != 1 + or context.node.artifact_inputs[0].type != RETURN_STATUS_TYPE + or context.node.artifact_outputs + != (ArtifactOutput(_PROJECTION_ALIAS, MONETARY_PROJECTION_TYPE),) + or set(context.params) != {"definition", "projection"} + ): + raise _refuse("NODE_DECLARATION") + if context.params["definition"] != definition.params_text: + raise _refuse("NODE_DEFINITION_PARAM") + if context.params["projection"] != projection.params_text: + raise _refuse("NODE_PROJECTION_PARAM") + alias = context.node.artifact_inputs[0].name + value = context.artifacts[alias] + if value.type != RETURN_STATUS_TYPE: + raise _refuse("STATUS_ARTIFACT_TYPE") + status = decode_return_status(value.payload, definition) + main_bytes = load_source_bytes( + definition.main.codec, + context.sources[definition.main.source_name], + registry=self.source_codecs, + ) + decoded = decode_puf_monetary_source(main_bytes, status, projection, definition) + payload = encode_monetary_projection( + decoded, + projection, + definition, + status_artifact_sha256=hashlib.sha256(value.payload).hexdigest(), + ) + return KernelResult( + artifacts={_PROJECTION_ALIAS: payload}, + receipt={ + "puf_monetary_source_projection": { + "projection_sha256": projection.sha256, + "projection_route": projection.route, + "definition_sha256": definition.sha256, + "artifact_sha256": hashlib.sha256(payload).hexdigest(), + "artifact_bytes": len(payload), + "status_artifact_sha256": hashlib.sha256(value.payload).hexdigest(), + "main_sha256": definition.main.sha256, + "csv_acceptance_profile": csv_acceptance_profile(), + "projected_fields": list(projection.fields), + **{ + key: value + for key, value in decoded.facts.items() + if key != "column_facts" + }, + } + }, + ) + + +class USPufMonetaryProjectionSummaryKernel(KernelBase): + """A downstream consumer: decode the projection and report its census.""" + + ref = "us.puf.monetary_source.projection_summary@3" + capabilities = Capabilities( + Determinism.DETERMINISTIC, + numeric=Numeric.BITWISE, + dependencies=("numpy",), + ) + + def __init__(self, *, projection: PufMonetaryProjection | None = None) -> None: + self.projection = packaged_projection() if projection is None else projection + + def implementation_hash(self) -> str: + from importlib import import_module + + return source_hash( + *(import_module(name) for name in _IMPLEMENTATION_MODULES), + dependencies=self.capabilities.dependencies, + ) + + def run(self, context: KernelContext) -> KernelResult: + if ( + context.node.kernel != self.ref + or context.node.structural is not StructuralDelta.NONE + or context.node.sources + or context.node.inputs + or context.node.outputs + or context.node.artifact_outputs + or len(context.node.artifact_inputs) != 1 + or context.node.artifact_inputs[0].type != MONETARY_PROJECTION_TYPE + or context.params + ): + raise _refuse("SUMMARY_NODE_DECLARATION") + alias = context.node.artifact_inputs[0].name + value = context.artifacts[alias] + if value.type != MONETARY_PROJECTION_TYPE: + raise _refuse("SUMMARY_ARTIFACT_TYPE") + decoded = decode_monetary_projection(value.payload, self.projection) + facts = decoded.facts + # The per-return negative census covers the two return classes only. + # Summing it across all three would have counted a token class as a + # monetary sign, which is exactly the conflation this contract ends. + per_return_negatives = { + field: sum(int(classes[name]["negative"]) for name in RETURN_ROW_CLASSES) + for field, classes in facts["column_facts"].items() + } + # Aggregate-only: token-class counts and a width, never a token, a + # RECID or anything that could be read as an aggregate amount. The + # exact row/token pairs stay behind the local decoded accessor. + aggregate_token_classes = { + field: dict(classes[AGGREGATE_ROW_CLASS]) + for field, classes in facts["column_facts"].items() + } + return KernelResult( + receipt={ + "puf_monetary_projection_summary": { + "producer_key": value.producer_key, + "rows": decoded.rows, + "projected_columns": facts["projected_columns"], + "row_class_rows": dict(facts["row_class_rows"]), + "amount_known_rows": facts["amount_known_rows"], + "amount_known_rule": facts["amount_known_rule"], + "per_return_negative_values_by_field": per_return_negatives, + "aggregate_rows_are_lexical_only": True, + "aggregate_typed_sentinel": facts["aggregate_typed_sentinel"], + "aggregate_token_grammar": facts["aggregate_token_grammar"], + "aggregate_lexical_authority": facts["aggregate_lexical_authority"], + "aggregate_numeric_interpretation": facts[ + "aggregate_numeric_interpretation" + ], + "aggregate_lexical_token_classes": aggregate_token_classes, + "period_semantics": facts["period_semantics"], + "amount_units": facts["amount_units"], + "amount_units_scope": facts["amount_units_scope"], + "target_year_use": facts["target_year_use"], + "column_metadata": _thawed(decoded.column_metadata), + } + } + ) + + +def us_puf_monetary_source_node( + *, + population: str, + producer: str = PUF_RAW_SOURCE_NODE, + producer_output: str = "return_status", + projection: PufMonetaryProjection | None = None, + definition: PufRawSourceDefinition | None = None, + stage: str = MONETARY_PROJECTION_STAGE, +) -> Node: + """Declare the producer node inside an explicitly named host population.""" + + if not isinstance(population, str) or not population: + raise _refuse("NODE_POPULATION_REQUIRED") + resolved_definition = ( + packaged_raw_definition() if definition is None else definition + ) + resolved = packaged_projection() if projection is None else projection + if resolved.raw_definition_sha256 != resolved_definition.sha256: + raise _refuse("NODE_DEFINITION_MISMATCH") + return Node( + f"{stage}.projection", + USPufMonetarySourceKernel.ref, + population=population, + sources=(resolved_definition.main.source_name,), + params={ + "definition": resolved_definition.params_text, + "projection": resolved.params_text, + }, + artifact_inputs=( + ArtifactInput(_STATUS_ALIAS, producer, producer_output, RETURN_STATUS_TYPE), + ), + artifact_outputs=(ArtifactOutput(_PROJECTION_ALIAS, MONETARY_PROJECTION_TYPE),), + description=( + "Decode the enumerated 2015 PUF monetary source columns at return " + "grain, bound to the reviewed status artifact. Owns no population " + "columns and admits no target year." + ), + ) + + +def us_puf_monetary_projection_summary_node( + *, + population: str, + producer: str = MONETARY_PROJECTION_NODE, + stage: str = MONETARY_PROJECTION_STAGE, +) -> Node: + """Declare an ordinary consumer of the projection artifact.""" + + if not isinstance(population, str) or not population: + raise _refuse("NODE_POPULATION_REQUIRED") + return Node( + f"{stage}.projection_summary", + USPufMonetaryProjectionSummaryKernel.ref, + population=population, + artifact_inputs=( + ArtifactInput( + _PROJECTION_ALIAS, + producer, + _PROJECTION_ALIAS, + MONETARY_PROJECTION_TYPE, + ), + ), + description="Read the monetary projection artifact and report its census.", + ) + + +def register_us_puf_monetary_source_kernels( + registry: KernelRegistry, + *, + projection: PufMonetaryProjection | None = None, + definition: PufRawSourceDefinition | None = None, + source_codecs: SourceCodecRegistry | None = None, +) -> KernelRegistry: + """Register the producer and consumer kernels into ``registry``.""" + + registry.register( + USPufMonetarySourceKernel( + projection=projection, + definition=definition, + source_codecs=source_codecs, + ) + ) + registry.register(USPufMonetaryProjectionSummaryKernel(projection=projection)) + return registry diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/puf_price_baseline.py b/packages/microcosm-build/src/microcosm/build/us_runtime/puf_price_baseline.py new file mode 100644 index 000000000..ba565c369 --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/puf_price_baseline.py @@ -0,0 +1,1214 @@ +"""The developmental CPI-U price-restatement baseline for the 2015 PUF. + +One exact positive ratio of two published CPI-U all-items annual averages — +313689/237017, 2015 to 2024 — applied to every known money field of the +thirteen-field source projection. It restates 2015-file purchasing power at +2024 prices. It is **not** income aging: it carries no category-specific +nominal growth, no return-mass transition, and no claim that any field's real +level changed. ``S006`` is declared with an explicit identity series so the +no-mass-transition choice is recorded rather than implied. + +Three things this module is, and three it refuses +------------------------------------------------- +*It pins the resource.* The series is ``CUUR0000SA0``, U.S. city average, all +items, not seasonally adjusted, ``M13`` annual average, base 1982-84=100. The +levels are the publisher's own decimal text, copied verbatim, and the factor +table it builds carries the ``developmental_public_resource`` authority: the +retained bytes are checked against their named digest and their two records +against the declared level text. Retrieval location and time remain caller +metadata, not claims proved by hashing. The authority cannot become +release-eligible. + +*It declares the recipe.* One ``nominal_price_restatement`` series for all +thirteen money fields, one source reference year, one target year, ``RECID`` +as the identifier, ``S006`` as a design weight whose declared factor is +exactly one. + +*It adapts the artifact to the money view.* The projection is an artifact and +the growth transform consumes a table, so +:func:`adapt_projection_to_money_view` joins the projection to the reviewed +status artifact on ``RECID``, removes only the disclosure-aggregate rows, +records what it removed, and hands back ``FLPDYR``, the demographic status and +the unaltered integer ``S006`` separately. The growth itself is +:func:`~microcosm.build.us_runtime.puf_growth.apply_puf_growth`; nothing here +multiplies an amount. + +It refuses to interpret an aggregate token, to put the out-of-universe +sentinel anywhere near a numeric input, and to declare that this baseline +under- or over-states any 2024 amount. The public universes that would support +such a statement are not matched to this file. +""" + +from __future__ import annotations + +import hashlib +import json +import math +import struct +from collections.abc import Mapping, Sequence +from dataclasses import dataclass +from pathlib import Path +from types import MappingProxyType + +import numpy as np +import pandas as pd + +from .puf_growth import ( + DESIGN_WEIGHT_FIELD, + PROVENANCE_RECID_COLUMN, + PROVENANCE_SOURCE_AGI_COLUMN, + SOURCE_AGI_FIELD, + SOURCE_RECID_FIELD, + CompiledPufGrowth, + FactorAuthority, + FieldRole, + GrownPufTable, + GrowthFactorTable, + GrowthKind, + GrowthRecipe, + GrowthRule, + PufGrowthRefusalError, + SignBranch, + compile_puf_growth, +) +from .puf_monetary_agi_projection import AGI_FIELD +from .puf_monetary_source import AGGREGATE_SENTINEL, PufMonetarySourceProjection +from .puf_source_agi import PUF_SOURCE_YEAR + +__all__ = [ + "CPI_U_ANNUAL_AVERAGE_PERIOD", + "CPI_U_INDEX_BASE", + "CPI_U_SERIES_ID", + "CPI_U_SERIES_TITLE", + "DESIGN_WEIGHT_IDENTITY_SERIES", + "PRICE_BASELINE_LEVELS", + "PRICE_BASELINE_RATIO", + "PRICE_BASELINE_RECIPE_ID", + "PRICE_BASELINE_SERIES_NAME", + "PRICE_BASELINE_SOURCE_YEAR", + "PRICE_BASELINE_TABLE_ID", + "PRICE_BASELINE_TARGET_YEAR", + "PRICE_BASELINE_FACTOR_BITS", + "PAIRWISE_OBSERVATIONS", + "PufMoneyViewAdaptation", + "adapt_projection_to_money_view", + "compile_price_baseline", + "price_baseline_factor_document", + "price_baseline_invariants", + "price_baseline_recipe", + "require_price_baseline_contract", + "verify_cpi_resource_bytes", +] + +#: The primary BLS series this baseline is read from. +CPI_U_SERIES_ID = "CUUR0000SA0" +CPI_U_SERIES_TITLE = ( + "All items in U.S. city average, all urban consumers, not seasonally adjusted" +) +CPI_U_ANNUAL_AVERAGE_PERIOD = "M13" +CPI_U_INDEX_BASE = "1982-84=100" + +#: The source file's representative year, and the target the restatement +#: states amounts in. The source year is the growth slice's own PUF source +#: year, not a second opinion about it. +PRICE_BASELINE_SOURCE_YEAR = PUF_SOURCE_YEAR +PRICE_BASELINE_TARGET_YEAR = 2024 + +#: The publisher's own decimal text for the two annual averages. These are +#: level strings, never floats: the factor table stores text and the ratio is +#: computed once, in one place, by the growth contract. +PRICE_BASELINE_LEVELS = MappingProxyType( + {PRICE_BASELINE_SOURCE_YEAR: "237.017", PRICE_BASELINE_TARGET_YEAR: "313.689"} +) + +#: The exact rational factor, as ``(numerator, denominator)``. Both published +#: levels carry exactly three decimals, so the ratio of the thousandths-scaled +#: integers is the exact ratio of the published levels. +PRICE_BASELINE_RATIO = (313689, 237017) + +#: The one float64 the adopted factor is, as its little-endian bit pattern. +#: Derived from the exact rational here rather than written down, so it cannot +#: drift from :data:`PRICE_BASELINE_RATIO`; a test separately checks that the +#: division of the two published level *texts* lands on the same double. +#: :func:`require_price_baseline_contract` compares an incoming contract's +#: resolved bits against this, which is the check that makes every downstream +#: statement about "the factor" a statement about the contract in hand. +PRICE_BASELINE_FACTOR_BITS = struct.pack( + " None: + if not condition: + raise PufGrowthRefusalError(reason, field) + + +def _module_sha256() -> str: + """This module's own bytes, so the factor table names the code that built it.""" + return hashlib.sha256(Path(__file__).resolve().read_bytes()).hexdigest() + + +def _canonical(value: object) -> bytes: + return json.dumps( + value, sort_keys=True, separators=(",", ":"), allow_nan=False, ensure_ascii=True + ).encode("ascii") + + +def _cpi_annual_average_records( + resource_bytes: bytes, years: Sequence[int] +) -> dict[int, bytes]: + """The CPI-U all-items ``M13`` level text for each of *years*, or fewer. + + BLS pads ``series_id`` and ``value`` with spaces for monospace column + alignment; the character that actually delimits a field is the tab, so + every field is stripped before it is compared or returned — the same + convention this lane's own retrieval script used to extract and hash + these two lines. A row whose first field is this series but which does + not split into the publisher's five tab-delimited fields + (``series_id``, ``year``, ``period``, ``value``, ``footnote_codes``) is + refused rather than silently skipped, because a truncated or reshuffled + row is exactly the failure mode this check exists to catch. Rows for a + year not in *years* are ignored: this baseline does not require the file + to be otherwise well-formed, only that its own two records are. + """ + series = CPI_U_SERIES_ID.encode("ascii") + period = CPI_U_ANNUAL_AVERAGE_PERIOD.encode("ascii") + wanted = {str(year).encode("ascii") for year in years} + found: dict[int, bytes] = {} + for line in resource_bytes.split(b"\n"): + if not line.strip(): + continue + fields = line.split(b"\t") + if fields[0].strip() != series: + continue + _require(len(fields) == 5, "PRICE_BASELINE_RESOURCE_LINE_SHAPE") + _, year_field, period_field, value_field, _ = ( + field.strip() for field in fields + ) + if period_field != period or year_field not in wanted: + continue + year = int(year_field) + _require( + year not in found, "PRICE_BASELINE_RESOURCE_DUPLICATE_RECORD", str(year) + ) + found[year] = value_field + return found + + +def verify_cpi_resource_bytes( + resource_bytes: bytes, *, resource_sha256: str +) -> Mapping[str, object]: + """Refuse unless *resource_bytes* is this baseline's named public resource. + + The whole-file digest is bound first — the caller's ``resource_sha256`` + is no longer a string nobody checks. Then the bytes themselves are parsed + with the publisher's own tab-delimited, space-padded convention (see + :func:`_cpi_annual_average_records`) and must contain **exactly** the two + ``CUUR0000SA0`` ``M13`` records this baseline reads, at the two years + :data:`PRICE_BASELINE_LEVELS` names, each with exactly + the declared decimal text after removing BLS column padding. Different + numeric spellings and near-equal floats are refused: source lexical + identity is stronger than arithmetic agreement. + + Args: + resource_bytes: The exact bytes retrieved from ``source_url``. + resource_sha256: The digest the caller claims for those bytes. + + Returns: + A receipt naming the series, period, verified digest and the two + matched records. + + Raises: + PufGrowthRefusalError: On a type, size, digest, shape, duplicate, + missing-record or value mismatch. + """ + _require( + isinstance(resource_bytes, (bytes, bytearray)), "PRICE_BASELINE_RESOURCE_TYPE" + ) + _require(len(resource_bytes) > 0, "PRICE_BASELINE_RESOURCE_EMPTY") + _require( + len(resource_bytes) <= _CPI_RESOURCE_MAX_BYTES, + "PRICE_BASELINE_RESOURCE_OVERSIZE", + ) + body = bytes(resource_bytes) + actual_sha256 = hashlib.sha256(body).hexdigest() + _require(actual_sha256 == resource_sha256, "PRICE_BASELINE_RESOURCE_DIGEST") + + years = tuple(PRICE_BASELINE_LEVELS) + found = _cpi_annual_average_records(body, years) + records: dict[str, object] = {} + for year in years: + _require(year in found, "PRICE_BASELINE_RESOURCE_RECORD_MISSING", str(year)) + _require( + found[year] == PRICE_BASELINE_LEVELS[year].encode("ascii"), + "PRICE_BASELINE_RESOURCE_RECORD_VALUE", + str(year), + ) + # Decode only after exact ASCII equality; malformed values must refuse + # by the same domain reason rather than leaking a parser exception. + level_text = found[year].decode("ascii") + found_value = float(level_text) + records[str(year)] = MappingProxyType( + {"level_text": level_text, "level_float64": found_value} + ) + return MappingProxyType( + { + "series_id": CPI_U_SERIES_ID, + "period": CPI_U_ANNUAL_AVERAGE_PERIOD, + "resource_sha256": actual_sha256, + "resource_bytes": len(body), + "records": MappingProxyType(records), + } + ) + + +def price_baseline_factor_document( + *, source_url: str, resource_sha256: str, retrieved_utc: str, resource_bytes: bytes +) -> bytes: + """The developmental factor table, as the exact bytes it hashes to. + + Args: + source_url: The publisher endpoint the levels were read from. + resource_sha256: Digest of the exact resource bytes that were read. + retrieved_utc: When they were read, as an ISO 8601 UTC instant. + resource_bytes: The exact bytes retrieved from ``source_url``. Checked + against ``resource_sha256`` and parsed for the two annual-average + records this baseline names — see + :func:`verify_cpi_resource_bytes`. Without this the two prior + arguments were caller assertions nothing in this module verified. + + Returns: + A canonical factor-table document carrying the price series and the + identity design-weight series, with the + ``developmental_public_resource`` authority. + """ + verify_cpi_resource_bytes(resource_bytes, resource_sha256=resource_sha256) + return _canonical( + { + "authority": "developmental_public_resource", + "provenance": { + "dependency_versions": {}, + "generated_by": ( + "microcosm.build.us_runtime.puf_price_baseline." + "price_baseline_factor_document" + ), + "generator_code_sha256": _module_sha256(), + "index_or_total": "index", + "parameter_paths": [], + "population_divisor": "none", + "resource_sha256": resource_sha256, + "retrieved_utc": retrieved_utc, + "rounding": ( + "none; the two published levels are copied verbatim as " + "decimal text and the factor is one float64 division of them" + ), + "source_url": source_url, + }, + "schema_version": 1, + "series": { + DESIGN_WEIGHT_IDENTITY_SERIES: { + "kind": str(GrowthKind.DESIGN_WEIGHT_GROWTH), + "levels": { + str(PRICE_BASELINE_SOURCE_YEAR): "1", + str(PRICE_BASELINE_TARGET_YEAR): "1", + }, + }, + PRICE_BASELINE_SERIES_NAME: { + "kind": str(GrowthKind.NOMINAL_PRICE_RESTATEMENT), + "levels": { + str(year): level + for year, level in PRICE_BASELINE_LEVELS.items() + }, + }, + }, + "table_id": PRICE_BASELINE_TABLE_ID, + "years": [PRICE_BASELINE_SOURCE_YEAR, PRICE_BASELINE_TARGET_YEAR], + } + ) + + +def price_baseline_recipe( + money_fields: Sequence[str], *, recipe_id: str = PRICE_BASELINE_RECIPE_ID +) -> GrowthRecipe: + """One price series for every money field, plus the two structural rules. + + Args: + money_fields: The projection's money fields. Must contain + ``E00100``, because the growth contract requires it, and must not + contain ``RECID`` or ``S006``, which are not money. + recipe_id: Recipe identity, for a caller running more than one. + + Raises: + PufGrowthRefusalError: If the field set is not a money field set. + """ + _require(bool(money_fields), "PRICE_BASELINE_NO_MONEY_FIELD") + fields = tuple(money_fields) + _require(len(set(fields)) == len(fields), "PRICE_BASELINE_DUPLICATE_FIELD") + _require(SOURCE_AGI_FIELD in fields, "PRICE_BASELINE_AGI_FIELD", SOURCE_AGI_FIELD) + for reserved in (SOURCE_RECID_FIELD, DESIGN_WEIGHT_FIELD): + _require(reserved not in fields, "PRICE_BASELINE_NON_MONEY_FIELD", reserved) + rules = [ + GrowthRule( + field, + FieldRole.MONEY, + PRICE_BASELINE_SOURCE_YEAR, + PRICE_BASELINE_TARGET_YEAR, + GrowthKind.NOMINAL_PRICE_RESTATEMENT, + SignBranch.ANY, + PRICE_BASELINE_SERIES_NAME, + _CITATION, + ) + for field in fields + ] + rules.append( + GrowthRule( + SOURCE_RECID_FIELD, + FieldRole.IDENTIFIER, + PRICE_BASELINE_SOURCE_YEAR, + PRICE_BASELINE_SOURCE_YEAR, + GrowthKind.NONE, + ) + ) + rules.append( + GrowthRule( + DESIGN_WEIGHT_FIELD, + FieldRole.DESIGN_WEIGHT, + PRICE_BASELINE_SOURCE_YEAR, + PRICE_BASELINE_TARGET_YEAR, + GrowthKind.DESIGN_WEIGHT_GROWTH, + SignBranch.ANY, + DESIGN_WEIGHT_IDENTITY_SERIES, + "declared identity: this baseline transitions no return mass", + ) + ) + return GrowthRecipe(recipe_id, tuple(rules)) + + +def require_price_baseline_contract( + compiled: CompiledPufGrowth, *, resource_bytes: bytes +) -> None: + """Refuse any contract that is not the adopted price baseline. + + :func:`adapt_projection_to_money_view` and + :func:`price_baseline_invariants` both call this before they record + anything, because both go on to state the source year, the exact rational + factor, the ``RECID`` identity and "this baseline transitions no return + mass" as facts about the run. A contract that named a different pair of + years, mixed a second factor series, aged rather than restated, split a + field across sign branches, or grew the design weight would leave every + other check in this module passing while those recorded statements were + false. The false receipt is the failure this refuses. + + What is checked is exactly what those receipts assert: + + * the ``developmental_public_resource`` authority, and no release + eligibility; + * every money rule at ``2015 -> 2024``, ``nominal_price_restatement``, + ``SignBranch.ANY``, on the one named CPI series, resolving to the + published level texts and to the exact float64 of ``313689/237017``; + * exactly one identifier rule, on ``RECID``, at the baseline source year; + * exactly one design-weight rule, on ``S006``, at the same two years, on + the declared identity series, whose resolved factor is exactly ``1.0`` + and whose two level texts are the same text — not two texts whose + float64 quotient happens to round to one; + * no rule in any other role; + * the exact resource bytes against the digest the contract names, and the + two source records against their declared series, years, period and text. + + The *field set* is deliberately not fixed: a contract over a subset of the + projection's money fields is still this baseline, and the adapter checks + the subset relation against the artifact separately. + + Args: + compiled: The contract a caller is about to run. + resource_bytes: The retained BLS body; checked against the contract's + own resource digest even if it came from the generic growth compiler. + + Raises: + PufGrowthRefusalError: With a ``PRICE_BASELINE_*`` reason naming the + first field that does not match. + """ + _require(type(compiled) is CompiledPufGrowth, "COMPILED_TYPE") + _require( + compiled.factors.authority is FactorAuthority.DEVELOPMENTAL_PUBLIC_RESOURCE, + "PRICE_BASELINE_AUTHORITY", + ) + _require(compiled.release_eligible is False, "PRICE_BASELINE_RELEASE_ELIGIBLE") + verify_cpi_resource_bytes( + resource_bytes, + resource_sha256=compiled.factors.provenance["resource_sha256"], + ) + + rules = compiled.recipe.rules + money = tuple(rule for rule in rules if rule.role is FieldRole.MONEY) + identifiers = tuple(rule for rule in rules if rule.role is FieldRole.IDENTIFIER) + weights = tuple(rule for rule in rules if rule.role is FieldRole.DESIGN_WEIGHT) + for rule in rules: + _require( + rule.role + in (FieldRole.MONEY, FieldRole.IDENTIFIER, FieldRole.DESIGN_WEIGHT), + "PRICE_BASELINE_ROLE", + rule.field, + ) + _require(bool(money), "PRICE_BASELINE_NO_MONEY_FIELD") + _require( + SOURCE_AGI_FIELD in {rule.field for rule in money}, + "PRICE_BASELINE_AGI_FIELD", + SOURCE_AGI_FIELD, + ) + source_level = PRICE_BASELINE_LEVELS[PRICE_BASELINE_SOURCE_YEAR] + target_level = PRICE_BASELINE_LEVELS[PRICE_BASELINE_TARGET_YEAR] + for rule in money: + field = rule.field + _require( + rule.source_reference_year == PRICE_BASELINE_SOURCE_YEAR, + "PRICE_BASELINE_SOURCE_YEAR", + field, + ) + _require( + rule.target_year == PRICE_BASELINE_TARGET_YEAR, + "PRICE_BASELINE_TARGET_YEAR", + field, + ) + _require( + rule.growth_kind is GrowthKind.NOMINAL_PRICE_RESTATEMENT, + "PRICE_BASELINE_GROWTH_KIND", + field, + ) + # ``SignBranch.ANY`` is the field's only rule, which is what lets one + # positive factor be checked per row rather than per branch. + _require( + rule.sign_branch is SignBranch.ANY, "PRICE_BASELINE_SIGN_BRANCH", field + ) + _require( + rule.factor_series == PRICE_BASELINE_SERIES_NAME, + "PRICE_BASELINE_SERIES", + field, + ) + resolved = compiled.factor_for(field, SignBranch.ANY) + # The substantive claim first — this is the one float64 — and then the + # two level texts it must have come from. The order matters because + # more than one pair of texts divides to the same double, and this + # baseline is the published pair, not merely a pair that agrees. + _require( + resolved.ratio_bits == PRICE_BASELINE_FACTOR_BITS, + "PRICE_BASELINE_FACTOR", + field, + ) + _require( + resolved.source_level == source_level, "PRICE_BASELINE_SOURCE_LEVEL", field + ) + _require( + resolved.target_level == target_level, "PRICE_BASELINE_TARGET_LEVEL", field + ) + + # ``GrowthRecipe`` already requires exactly one identifier field, and + # ``GrowthRule`` already fixes a non-money, non-weight role to + # ``GrowthKind.NONE`` with equal years. What is open here is which field + # carries the role and which year it names. + _require( + len(identifiers) == 1 and identifiers[0].field == SOURCE_RECID_FIELD, + "PRICE_BASELINE_IDENTIFIER", + ) + identifier = identifiers[0] + _require( + identifier.source_reference_year == PRICE_BASELINE_SOURCE_YEAR, + "PRICE_BASELINE_IDENTIFIER_YEAR", + identifier.field, + ) + + # Likewise the design-weight role already fixes the kind and the sign + # branch. What is open is the field, the series, and whether the declared + # factor really is one. + _require( + len(weights) == 1 and weights[0].field == DESIGN_WEIGHT_FIELD, + "PRICE_BASELINE_DESIGN_WEIGHT", + ) + weight = weights[0] + _require( + weight.factor_series == DESIGN_WEIGHT_IDENTITY_SERIES, + "PRICE_BASELINE_DESIGN_WEIGHT_SERIES", + weight.field, + ) + _require( + weight.source_reference_year == PRICE_BASELINE_SOURCE_YEAR + and weight.target_year == PRICE_BASELINE_TARGET_YEAR, + "PRICE_BASELINE_DESIGN_WEIGHT_YEAR", + weight.field, + ) + resolved_weight = compiled.factor_for(weight.field, SignBranch.ANY) + # The substantive claim first: the declared weight factor is exactly one. + # Then the two level texts, which must be the same text and not merely two + # texts whose float64 quotient rounds to one. + _require( + resolved_weight.value == 1.0, + "PRICE_BASELINE_DESIGN_WEIGHT_FACTOR", + weight.field, + ) + _require( + resolved_weight.source_level == resolved_weight.target_level, + "PRICE_BASELINE_DESIGN_WEIGHT_LEVEL", + weight.field, + ) + + +def compile_price_baseline( + money_fields: Sequence[str], + *, + source_url: str, + resource_sha256: str, + retrieved_utc: str, + resource_bytes: bytes, + recipe_id: str = PRICE_BASELINE_RECIPE_ID, +) -> CompiledPufGrowth: + """Compile the price-baseline contract; never release-eligible. + + ``resource_bytes`` is required and is checked against ``resource_sha256`` + by :func:`price_baseline_factor_document`; this function cannot hand back + a contract pinned to resource metadata nobody verified. + """ + recipe = price_baseline_recipe(money_fields, recipe_id=recipe_id) + factors = GrowthFactorTable.from_bytes( + price_baseline_factor_document( + source_url=source_url, + resource_sha256=resource_sha256, + retrieved_utc=retrieved_utc, + resource_bytes=resource_bytes, + ) + ) + compiled = compile_puf_growth(recipe, factors) + # The authority gate already guarantees this. Asserting it here means the + # one function a caller reaches for cannot hand back an eligible contract + # even if the gate is later widened by mistake. + _require(compiled.release_eligible is False, "PRICE_BASELINE_RELEASE_ELIGIBLE") + # The same gate the two entry points apply, so this function cannot hand + # back a contract they would decline. + require_price_baseline_contract(compiled, resource_bytes=resource_bytes) + return compiled + + +# -------------------------------------------------------------------------- +# The artifact-to-money-view adapter +# -------------------------------------------------------------------------- + + +@dataclass(frozen=True) +class PufMoneyViewAdaptation: + """The money view the growth transform consumes, plus what stayed outside. + + Attributes: + money_view: One row per amount-known return, carrying ``RECID`` and + the declared money columns as int64. Never the design weight, and + never a disclosure-aggregate row. + recid: The kept rows' identifiers, in money-view order. + flpdyr: The kept rows' raw observation year, carried from the status + artifact and never restated. + demographic_status: The kept rows' demographic match flag. + design_weight: The kept rows' ``S006``, in the delivered integer + hundredths, unaltered and outside the money view. + excluded: The audit identity of the removed disclosure-aggregate rows. + receipt: What was joined, kept and removed. + """ + + money_view: pd.DataFrame + recid: np.ndarray + flpdyr: np.ndarray + demographic_status: np.ndarray + design_weight: np.ndarray + excluded: Mapping[str, object] + receipt: Mapping[str, object] + + +def _status_arrays(status: object, rows: int | None = None) -> Mapping[str, np.ndarray]: + """Every status column this adapter reads, present and the same length. + + The length check covers ``FLPDYR``, ``S006`` and the demographic status as + well as the two join keys, so a short or long side column is a refusal + rather than a silently misaligned slice later. + """ + try: + typed = status.typed + except AttributeError as error: + raise PufGrowthRefusalError("STATUS_ARTIFACT_TYPE") from error + # Presence first for every column, then length for every column, so a + # missing column is reported as missing rather than as a length mismatch. + for name in _STATUS_COLUMNS: + _require(name in typed, "STATUS_ARTIFACT_COLUMN", name) + if rows is not None: + for name in _STATUS_COLUMNS: + _require( + int(np.asarray(typed[name]).shape[0]) == rows, + "STATUS_COLUMN_ROWS", + name, + ) + return typed + + +def adapt_projection_to_money_view( + decoded: PufMonetarySourceProjection, + status: object, + compiled: CompiledPufGrowth, + *, + resource_bytes: bytes, +) -> PufMoneyViewAdaptation: + """Join the projection to the status artifact and drop only the aggregates. + + The join is by position **and** proved by value: the two artifacts' whole + ``RECID`` sequences and disclosure flags must be element-wise identical + before anything is selected, so a row's amounts and its status cannot come + from different returns. + + Exactly the rows outside the individual-return amount universe are + removed, and their identity is recorded. Every removed row must hold the + out-of-universe sentinel in every projected column, and no kept row may + hold it, so the sentinel cannot reach a numeric input in either direction. + + The contract is checked against the adopted baseline before anything is + recorded — see :func:`require_price_baseline_contract` — because the + receipt below states the source year, the design-weight identity and the + absence of a return-mass transition as facts about this run. + + Args: + decoded: A decoded thirteen-field projection. + status: The decoded ``puf_2015_raw_return_status`` artifact. + compiled: The compiled price-baseline contract. + resource_bytes: The retained BLS body authenticated by the entry gate. + + Returns: + The money view and everything deliberately kept out of it. + + Raises: + PufGrowthRefusalError: On any contract, join, universe or sentinel + violation. + """ + require_price_baseline_contract(compiled, resource_bytes=resource_bytes) + _require( + isinstance(decoded, PufMonetarySourceProjection), "PROJECTION_ARTIFACT_TYPE" + ) + projection_recid = np.asarray(decoded.typed[SOURCE_RECID_FIELD], dtype=" 0, "PROJECTION_NO_ROWS") + typed = _status_arrays(status, rows) + status_recid = np.asarray(typed[SOURCE_RECID_FIELD], dtype=" 0, "NO_AMOUNT_KNOWN_ROW") + _require( + removed == int(decoded.facts["row_class_rows"]["disclosure_aggregate"]), + "EXCLUDED_ROW_COUNT", + ) + + money_fields = compiled.money_fields + _require(set(money_fields) <= set(decoded.typed), "PROJECTION_MISSING_MONEY_FIELD") + _require(AGI_FIELD in money_fields, "PRICE_BASELINE_AGI_FIELD", AGI_FIELD) + columns: dict[str, np.ndarray] = {} + for field in money_fields: + values = np.asarray(decoded.typed[field], dtype=" int | None: + if greater not in frame.columns or lesser not in frame.columns: + return None + return int((frame[lesser] > frame[greater]).sum()) + + +def _bits(values: np.ndarray) -> np.ndarray: + """A float64 array's raw bit patterns, so ``-0.0`` is not ``0.0``.""" + return np.ascontiguousarray(values, dtype="float64").view(" Mapping[str, object]: + """Measure what one positive factor must leave true, and refuse otherwise. + + Three things happen here, in this order, and the order is deliberate. + + *The summaries.* Per column: dtype, finiteness, the sign set, the zero set, + no negative zero, and the exact integer sum of the input column times the + factor against the grown ``math.fsum``. Then the pairwise relations the + booklets leave open, counted before and after and required only to be + **equal**. These are the numbers a reader wants in the receipt, and each + names a specific symptom when it fires. + + *The exact row comparison.* Every kept row of every money column must be + bit-for-bit the contract's own resolved float64 factor applied to that + row's own delivered amount, and the two provenance columns must be + bit-for-bit their sources. This is the statement the summaries cannot + make: a coordinated rearrangement preserves every total, every sign, every + zero and every pairwise count, and is caught only here. It runs last so + that the narrower checks keep reporting the narrower diagnosis — and so + that each of them stays independently falsifiable. + + *The design weight.* Compared, if a caller supplies the weights it carried + forward, against the adaptation's own ``S006``. Not supplied, it is + reported as a declaration and **not** as a measurement. + + Args: + adaptation: The adaptation whose money view was grown. + grown: The result of :func:`~microcosm.build.us_runtime.puf_growth.apply_puf_growth` + on that money view, under this same contract. + compiled: The contract, checked against the adopted baseline. + resource_bytes: The retained BLS body authenticated by the entry gate. + design_weight_after: The post-growth design weights, if the caller ran + :func:`~microcosm.build.us_runtime.puf_growth.apply_puf_design_weight_growth`. + Omitted, ``design_weight_unchanged`` is ``None``, not ``True``. + relative_tolerance: Tolerance for the scaled-sum comparison only. + + Raises: + PufGrowthRefusalError: On any contract, identity, row, sum, pairwise or + weight violation. + """ + require_price_baseline_contract(compiled, resource_bytes=resource_bytes) + _require(type(grown) is GrownPufTable, "GROWN_TYPE") + _require(type(adaptation) is PufMoneyViewAdaptation, "ADAPTATION_TYPE") + # One contract, named the same way by all three objects. Without this the + # measurements below would describe a contract the table was not grown + # under, and every number in the receipt would be attributed to it. + contract_sha256 = compiled.sha256 + _require(grown.contract_sha256 == contract_sha256, "GROWN_CONTRACT_MISMATCH") + _require( + grown.receipt["contract_sha256"] == contract_sha256, + "GROWN_RECEIPT_CONTRACT_MISMATCH", + ) + _require( + adaptation.receipt["contract_sha256"] == contract_sha256, + "ADAPTATION_CONTRACT_MISMATCH", + ) + source = adaptation.money_view + table = grown.table + rows = len(source) + _require(len(table) == rows, "GROWN_ROWS") + _require( + bool( + np.array_equal( + table[SOURCE_RECID_FIELD].to_numpy(), + source[SOURCE_RECID_FIELD].to_numpy(), + ) + ), + "GROWN_RECID", + ) + factor = compiled.factor_for(compiled.money_fields[0], SignBranch.ANY).value + columns: dict[str, object] = {} + for field in compiled.money_fields: + resolved = compiled.factor_for(field, SignBranch.ANY) + _require( + resolved.ratio_bits + == compiled.factor_for(compiled.money_fields[0], SignBranch.ANY).ratio_bits, + "MIXED_FACTOR", + field, + ) + original = source[field].to_numpy() + result = table[field].to_numpy() + _require(result.dtype == np.dtype("float64"), "GROWN_DTYPE", field) + _require(bool(np.isfinite(result).all()), "GROWN_NONFINITE", field) + _require( + bool((np.sign(result) == np.sign(original.astype("float64"))).all()), + "SIGN_CHANGED", + field, + ) + zeros = result == 0.0 + _require(not bool(np.signbit(result[zeros]).any()), "NEGATIVE_ZERO", field) + _require(bool(np.array_equal(zeros, original == 0)), "ZERO_SET_CHANGED", field) + exact_sum = int(sum(int(value) for value in original.tolist())) + gross_sum = int(sum(abs(int(value)) for value in original.tolist())) + grown_sum = math.fsum(result.tolist()) + expected = exact_sum * factor + # Scaled by the column's **gross** magnitude, not its net total. The + # float error one positive factor can accumulate is bounded by the + # magnitude that passed through it; a signed column whose positives + # and negatives nearly cancel has a tiny net and the same gross, and + # scaling by the net alone would refuse a correct result for it. + net_scale = max(abs(expected), 1.0) + scale = max(abs(gross_sum * factor), net_scale) + _require( + abs(grown_sum - expected) <= relative_tolerance * scale, + "SCALED_SUM_DISAGREEMENT", + field, + ) + columns[field] = { + "rows": rows, + "exact_source_sum": exact_sum, + "gross_source_sum_absolute": gross_sum, + "grown_fsum": grown_sum, + "expected_grown_sum": expected, + "scale_basis": "gross_absolute_source_sum_times_factor", + "relative_difference": ( + abs(grown_sum - expected) / scale if scale else 0.0 + ), + "relative_difference_net_scaled": ( + abs(grown_sum - expected) / net_scale if net_scale else 0.0 + ), + "negative": int((original < 0).sum()), + "positive": int((original > 0).sum()), + "zero": int((original == 0).sum()), + "ratio_bits": resolved.ratio_bits, + "grown_minimum": float(result.min()), + "grown_maximum": float(result.max()), + } + pairwise = {} + for name, greater, lesser in PAIRWISE_OBSERVATIONS: + before = _violations(source, greater, lesser) + after = _violations(table, greater, lesser) + # A contract over a subset of the projection's fields can leave one + # side of a relation out of the money view entirely. ``None == None`` + # would then pass, and the entry would read as an observation that was + # never made, so the absence is named instead of being counted. + _require((before is None) == (after is None), "PAIRWISE_COLUMN_ASYMMETRY", name) + observable = before is not None + if observable: + _require(before == after, "PAIRWISE_VIOLATIONS_MOVED", name) + pairwise[name] = { + "greater": greater, + "lesser": lesser, + "violations_before": before, + "violations_after": after, + "status": ( + "observed_not_enforced" + if observable + else "not_observable_column_absent" + ), + "absent_columns": sorted( + column for column in (greater, lesser) if column not in source.columns + ), + } + + # The exact row comparison. ``apply_puf_growth`` selects ``values != 0.0`` + # for a SignBranch.ANY rule and writes ``values[selected] * factor``, so + # the expected array is reproduced here by the same two operations in the + # same order and compared on raw bits. Nothing is recomputed from the + # levels or the rational: the contract's own resolved float64 is the one + # multiplier, which is what makes this a check of the transform rather + # than a second opinion about the factor. + for field in compiled.money_fields: + resolved = compiled.factor_for(field, SignBranch.ANY) + original = source[field].to_numpy() + expected = original.astype("float64") + selected = expected != 0.0 + with np.errstate(over="ignore", under="ignore"): + expected[selected] = expected[selected] * resolved.value + result = table[field].to_numpy() + disagreeing = int(np.count_nonzero(_bits(result) != _bits(expected))) + _require(disagreeing == 0, "ROW_GROWTH_DISAGREEMENT", field) + + # The two columns ``apply_puf_growth`` copies rather than grows. A silent + # edit here would restate provenance while every money column still + # matched. + for column, origin in ( + (PROVENANCE_RECID_COLUMN, source[SOURCE_RECID_FIELD].to_numpy()), + ( + PROVENANCE_SOURCE_AGI_COLUMN, + source[SOURCE_AGI_FIELD].to_numpy().astype("float64"), + ), + ): + _require(column in table.columns, "PROVENANCE_COLUMN_MISSING", column) + carried = table[column].to_numpy() + if column == PROVENANCE_SOURCE_AGI_COLUMN: + _require( + bool((_bits(carried) == _bits(origin)).all()), + "PROVENANCE_DISAGREEMENT", + column, + ) + else: + _require( + bool(np.array_equal(carried, origin)), + "PROVENANCE_DISAGREEMENT", + column, + ) + + design_weight, design_weight_unchanged = _design_weight_observation( + adaptation, compiled, design_weight_after + ) + return MappingProxyType( + { + "rows": rows, + "factor_ratio_bits": compiled.factor_for( + compiled.money_fields[0], SignBranch.ANY + ).ratio_bits, + "factor_float64": factor, + "exact_rational": { + "numerator": PRICE_BASELINE_RATIO[0], + "denominator": PRICE_BASELINE_RATIO[1], + }, + "output_rounding": "none_unrounded_float64", + "columns": columns, + "pairwise_observations": pairwise, + "row_exact_growth": { + "rows": rows, + "fields": list(compiled.money_fields), + "comparison": ( + "bitwise float64 equality of every kept row against the " + "contract's own resolved factor applied to that row's " + "delivered amount, zeros left untouched" + ), + "disagreeing_rows": 0, + "provenance_columns_compared": list( + (PROVENANCE_RECID_COLUMN, PROVENANCE_SOURCE_AGI_COLUMN) + ), + }, + "contract_sha256": contract_sha256, + "contract_named_by": { + "compiled": contract_sha256, + "grown_result": grown.contract_sha256, + "grown_receipt": grown.receipt["contract_sha256"], + "adaptation_receipt": adaptation.receipt["contract_sha256"], + }, + "design_weight": design_weight, + # ``None`` when no post-growth weight was supplied. This is never + # ``True`` unless the arrays were actually compared. + "design_weight_unchanged": design_weight_unchanged, + "release_eligible": compiled.release_eligible, + } + ) + + +def _design_weight_observation( + adaptation: PufMoneyViewAdaptation, + compiled: CompiledPufGrowth, + design_weight_after: Sequence[float] | np.ndarray | None, +) -> tuple[dict[str, object], bool | None]: + """What the design weight did, separating the declaration from a measurement. + + The declaration is the contract's: an identity series whose two levels are + both one, so the declared factor is exactly ``1.0``. That is a statement + about the recipe, and it is reported as one. + + The measurement needs a second array. ``adaptation.design_weight`` is the + status artifact's own ``S006`` sliced to the kept rows — the before side, + with its digest recorded so the comparison is auditable. The after side is + whatever the caller carried forward. Without it there is no observation to + report, and this returns ``None`` rather than manufacturing one. + """ + rule = compiled.design_weight_rule + declared = compiled.factor_for(rule.field, SignBranch.ANY) + before = np.asarray(adaptation.design_weight) + _require(before.ndim == 1, "DESIGN_WEIGHT_SHAPE", rule.field) + record: dict[str, object] = { + "field": rule.field, + "rows": int(before.shape[0]), + "declared_series": rule.factor_series, + "declared_source_level": declared.source_level, + "declared_target_level": declared.target_level, + "declared_factor_float64": declared.value, + "declared_return_mass_transition": "none", + "before_source": ( + "adaptation.design_weight: the status artifact's own S006, in " + "delivered integer hundredths, sliced to the kept rows" + ), + "before_sha256": hashlib.sha256( + np.ascontiguousarray(before, dtype="= -(2**53)) & (values <= 2**53)).all()), + "QBI_INTEGER_PRECISION:" + name, + ) + values = values.astype(np.float64) + _require(bool(np.isfinite(values).all()), "QBI_NONFINITE:" + name) + return values + + +def _ids(recids): + values = np.asarray(recids) + _require( + values.ndim == 1 + and len(values) > 0 + and values.dtype.kind in "iu" + and bool(((values > 0) & (values <= np.iinfo(np.int64).max)).all()), + "QBI_RECID_PHYSICAL_TYPE", + ) + _require(len(np.unique(values)) == len(values), "QBI_DUPLICATE_RECID") + return values.astype(" 0 + if not positive.any(): + return 0.0 + term = slope * revenues[positive] + target_logit = np.log(target / (1 - target)) + low, high = target_logit - np.max(term) - 80, target_logit - np.min(term) + 80 + for _ in range(100): + mid = (low + high) / 2 + if _logistic(mid + term).mean() < target: + low = mid + else: + high = mid + return float((low + high) / 2) + + +@dataclass(frozen=True) +class QbiEmploymentCalibration: + """A reusable statistical fit; this numerical record is not source authority.""" + + version: str + parameters_sha256: str + seed: int + fitted_cohort_sha256: str + fitted_rows: int + positive_receipt_rows: int + intercept: float + + +@dataclass(frozen=True) +class QbiModelResult: + columns: Mapping[str, np.ndarray] + calibration: QbiEmploymentCalibration + receipt: Mapping[str, object] + + +def model_full_puf_qbi( + canonical, recids, *, known, input_money_year, seed=0, employment_calibration=None +): + """Model 15 QBI leaves and explicitly replace the ordinary Schedule C leaf. + + Knownness here means complete canonical input, possibly derived upstream; + it does not mean a modeled leaf was observed. Inputs and their RECIDs must + already belong to the ordinary-return cohort selected by the source owner. + Each RNG is keyed by RECID, seed, and the archived stream number. This is + intentionally not byte parity with the archive's row-position RNG. The + employee logit intercept is fit to the unweighted positive-receipt cohort + (the archive's 18% target); reuse the returned fit for stable subset/chunk + application. Fitting a different cohort is a different statistical model. + """ + ids = _ids(recids) + n = len(ids) + _require( + type(input_money_year) is int and input_money_year == 2015, + "QBI_INPUT_MONEY_YEAR", + ) + _require(type(seed) is int and 0 <= seed < 2**64, "QBI_SEED") + _require( + isinstance(canonical, Mapping) and isinstance(known, Mapping), + "QBI_INPUT_MAPPING", + ) + inputs = {} + for name in REQUIRED_INPUTS: + _require(name in canonical and name in known, "QBI_MISSING_INPUT:" + name) + mask = np.asarray(known[name]) + _require( + mask.shape == (n,) and mask.dtype.kind == "b", "QBI_KNOWNNESS_TYPE:" + name + ) + _require(bool(mask.all()), "QBI_UNKNOWN:" + name) + inputs[name] = _vector(canonical[name], name, n) + source = np.column_stack( + [inputs[name] for name in REQUIRED_INPUTS[:5]] + + [inputs["partnership_income"] + inputs["s_corp_income"]] + ) + _require(bool(np.isfinite(source).all()), "QBI_SOURCE_SUM_NONFINITE") + params = model_parameters() + qualified = np.zeros((n, 6), dtype=bool) + margins, labor = np.zeros((n, 6)), np.zeros((n, 6)) + employees_draw, capital_draw, normal_draw, sstb_draw = np.zeros((4, n)) + investment_draws = np.zeros((n, 3)) + # Generate each record independently; calculations below remain vectorized. + for row, recid in enumerate(ids): + + def rng(stream, recid=int(recid)): + return np.random.Generator( + np.random.PCG64(np.random.SeedSequence([seed, stream, recid])) + ) + + qualified[row] = rng(41).random(6) < params["qualification"] + business = rng(42) + for col, (a, b, scale, shift) in enumerate(params["profit_margin"]): + margins[row, col] = business.beta(a, b) * scale + shift + employees_draw[row] = business.random() + for col, (a, b, scale, shift) in enumerate(params["labor_ratio"]): + labor[row, col] = business.beta(a, b) * scale + shift + capital_draw[row] = business.random() + normal_draw[row] = business.standard_normal() + sstb_draw[row] = rng(64).random() + investment = rng(43) + for col, (_name, probability, a, b, scale) in enumerate( + params["reit_ptp"] + params["bdc"] + ): + receives = investment.random() < probability + share = np.clip(investment.beta(a, b) * scale, 0, 1) + investment_draws[row, col] = share if receives else 0.0 + components = source * qualified + positive = np.maximum(components, 0) + positive_total = positive.sum(axis=1) + _require(bool(np.isfinite(positive_total).all()), "QBI_POSITIVE_TOTAL_NONFINITE") + fractions = np.divide( + positive, + positive_total[:, None], + out=np.zeros_like(positive), + where=positive_total[:, None] > 0, + ) + qbi = components.sum(axis=1) + _require(bool(np.isfinite(qbi).all()), "QBI_NET_TOTAL_NONFINITE") + margin = (fractions * margins).sum(axis=1) + revenues = np.divide(np.maximum(qbi, 0), margin, out=np.zeros(n), where=margin > 0) + _require(bool(np.isfinite(revenues).all()), "QBI_REVENUE_NONFINITE") + cohort_sha = _digest_columns(ids, inputs) + logit = params["employee_logit"] + if employment_calibration is None: + # Sort by identity before reduction so input permutations replay exactly. + employment_calibration = QbiEmploymentCalibration( + VERSION, + PARAMETERS_SHA256, + seed, + cohort_sha, + n, + int((revenues > 0).sum()), + _employee_intercept( + revenues[np.argsort(ids)], + logit["slope_per_dollar"], + logit["target_share"], + ), + ) + _require( + isinstance(employment_calibration, QbiEmploymentCalibration) + and employment_calibration.version == VERSION + and employment_calibration.parameters_sha256 == PARAMETERS_SHA256 + and employment_calibration.seed == seed + and type(employment_calibration.intercept) is float + and np.isfinite(employment_calibration.intercept), + "QBI_EMPLOYMENT_CALIBRATION_IDENTITY", + ) + probability = np.where( + revenues > 0, + _logistic( + employment_calibration.intercept + logit["slope_per_dollar"] * revenues + ), + 0.0, + ) + wages = revenues * (fractions * labor).sum(axis=1) * (employees_draw < probability) + capital_probability = fractions @ params["ubia"]["capital_probability"] + mean_ubia = (fractions @ params["ubia"]["multiple"]) * np.maximum(qbi, 0) + ubia = np.where( + (capital_draw < capital_probability) & (mean_ubia > 0), + mean_ubia + * np.exp( + params["ubia"]["sigma"] * normal_draw - params["ubia"]["sigma"] ** 2 / 2 + ), + 0.0, + ) + # Canonical source names ensure the SSTB draw consumes qualification flags. + sstb_positions = [SOURCES.index(name) for name in params["sstb_sources"]] + sstb_sources = positive[:, sstb_positions] + largest = sstb_sources.argmax(axis=1) + sstb_probability = np.where( + sstb_sources.sum(axis=1) > 0, + np.asarray(params["sstb_probabilities"])[largest], + 0.0, + ) + is_sstb = sstb_draw < sstb_probability + output = { + name + "_would_be_qualified": qualified[:, i].copy() + for i, name in enumerate(SOURCES) + } + schedule_c = inputs["self_employment_income_before_lsr"] + output.update( + { + "business_is_sstb": is_sstb, + "self_employment_income_before_lsr": np.where(is_sstb, 0, schedule_c), + "sstb_self_employment_income_before_lsr": np.where(is_sstb, schedule_c, 0), + "self_employment_income_would_be_qualified": qualified[:, 0] & ~is_sstb, + "sstb_self_employment_income_would_be_qualified": qualified[:, 0] & is_sstb, + "w2_wages_from_qualified_business": wages, + "unadjusted_basis_qualified_property": ubia, + "sstb_w2_wages_from_qualified_business": np.where(is_sstb, wages, 0), + "sstb_unadjusted_basis_qualified_property": np.where(is_sstb, ubia, 0), + "qualified_reit_and_ptp_income": np.maximum( + inputs["non_qualified_dividend_income"], 0 + ) + * investment_draws[:, 0] + + np.maximum(source[:, 5], 0) * investment_draws[:, 1], + "qualified_bdc_income": np.maximum( + inputs["non_qualified_dividend_income"], 0 + ) + * investment_draws[:, 2], + } + ) + _require(set(output) == set(OUTPUTS), "QBI_OUTPUT_ROSTER") + for name, values in output.items(): + _require(bool(np.isfinite(values).all()), "QBI_OUTPUT_NONFINITE:" + name) + frozen = { + name: np.frombuffer( + np.asarray(output[name]).tobytes(), dtype=output[name].dtype + ) + for name in OUTPUTS + } + receipt = { + "version": VERSION, + "origin": "modeled_not_observed", + "release_eligible": False, + "archived_commit": ARCHIVED_COMMIT, + "archived_code_sha256": ARCHIVED_CODE_SHA256, + "archived_assumptions_sha256": ARCHIVED_ASSUMPTIONS_SHA256, + "parameters_sha256": PARAMETERS_SHA256, + "input_money_year": input_money_year, + "input_money_unit": "nominal USD", + "source_statistical_baseline": 2015, + "growth": "not applied; employee slope per dollar is bound to this basis", + "sources": list(REQUIRED_INPUTS), + "input_cohort_sha256": cohort_sha, + "rows": n, + "seed": seed, + "randomness": "PCG64 SeedSequence([seed, archived_stream, RECID]); streams 41/42/43/64", + "numpy_version": np.__version__, + "input_knownness": "complete_canonical_not_raw_observation", + "employment_fit": dict(employment_calibration.__dict__), + "applied_positive_receipt_rows": int((revenues > 0).sum()), + "applied_employee_expected_share": float(probability[revenues > 0].mean()) + if (revenues > 0).any() + else 0.0, + "employee_target_weighting": "unweighted positive simulated receipts, archived target 0.18", + "subset_replay": "reuse employment_calibration; refitting a subset changes the model", + "schedule_c": "all-or-nothing SSTB split preserving the signed input total", + "wages_and_ubia": "base leaves are total pools; SSTB leaves are conditional copies", + "output_columns": list(OUTPUTS), + "output_values_sha256": _digest_columns(ids, frozen), + } + receipt["sha256"] = _sha(_json(receipt).encode()) + return QbiModelResult( + MappingProxyType(frozen), employment_calibration, MappingProxyType(receipt) + ) diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/puf_raw_source.py b/packages/microcosm-build/src/microcosm/build/us_runtime/puf_raw_source.py new file mode 100644 index 000000000..c241ed011 --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/puf_raw_source.py @@ -0,0 +1,1726 @@ +"""The first typed artifact read straight from the 2015 PUF delivery bytes. + +One ordinary graph node reads the two accepted parent captures through a +dedicated pinned raw-byte codec and produces one opaque, typed artifact: +``microcosm.us.puf_2015_raw_return_status`` at source-return grain, 207,696 +rows in delivered main-file order. + +What it types is deliberately small. Six of the 222 main columns (``RECID``, +``FLPDYR``, ``FLPDMO``, ``MARS``, ``DSI``, ``S006``) and the six demographic +columns of the companion file. The other 216 main columns are checked for +header name, order and record width and then dropped: no amount is parsed, +no growth factor is admitted, no household or person is invented, and the +node owns no population columns at all. Its host population is whatever the +caller names. + +Three separations are load-bearing. + +*Structure is not provenance.* :func:`decode_puf_raw_source` validates CSV +structure against a definition it is handed. It authorizes nothing. The +production wrapper — :func:`puf_raw_source_codecs` and +:class:`USPufRawSourceKernel` — is what binds the closed packaged pins, and +it refuses any definition whose canonical bytes are not the packaged file's. + +*Fixtures take an explicit non-production route.* :func:`fixture_definition` +builds a definition marked ``route="test_fixture"``, which must carry the +invented-fixture authority and must not restate a packaged source pin. There +is no "expected hash omitted, so admit anything" path anywhere. + +*The private cap is private.* :data:`PUF_RAW_SOURCE_MAX_BYTES` is 128 MiB +because the pinned main delivery is 126,034,649 bytes. +:data:`microcosm.graph.codecs.RAW_BYTES_MAX_BYTES` stays at 64 MiB. + +Period semantics stay unresolved on purpose. The header audit observed +``FLPDYR`` in {2012, 2013, 2014, 2015} while the booklet prints 2011–2014; +the artifact records the raw code and the conflict, and resolves neither. +``S006`` is kept as exact integer hundredths and never divided. + +The four publisher aggregate records stay in the artifact, tagged +``disclosure_aggregate``, never disaggregated. The 88,021 returns with no +demographic record carry ``demographic_status = 0``, the ``-1`` sentinel and +empty lexical columns; absence is never spelled as a zero code. +""" + +from __future__ import annotations + +import csv +import hashlib +import io +import json +import os +import stat +from collections.abc import Mapping, Sequence +from dataclasses import dataclass +from functools import partial +from importlib import resources +from pathlib import Path +from types import MappingProxyType + +import numpy as np + +from microcosm.graph import ( + ArtifactInput, + ArtifactOutput, + ArtifactType, + Capabilities, + Determinism, + KernelBase, + KernelContext, + KernelRegistry, + KernelResult, + Node, + Numeric, + SourceRef, + StructuralDelta, + source_hash, +) +from microcosm.graph.canonical import canonical_json +from microcosm.graph.codecs import SourceCodecRegistry, load_source_bytes + +__all__ = [ + "BODY_MAX_BYTES", + "DEMOGRAPHIC_CODEC", + "DEMOGRAPHIC_SOURCE_NAME", + "HEADER_MAX_BYTES", + "MAIN_CODEC", + "MAIN_SOURCE_NAME", + "PUF_AGGREGATE_RECIDS", + "PUF_RAW_SOURCE_MAX_BYTES", + "PUF_RAW_SOURCE_NODE", + "PUF_RAW_SOURCE_STAGE", + "RETURN_STATUS_SUMMARY_NODE", + "RETURN_STATUS_TYPE", + "PufRawReturnStatus", + "PufRawSourceDefinition", + "PufRawSourceRefusalError", + "SourcePin", + "USPufRawReturnStatusSummaryKernel", + "USPufRawSourceKernel", + "csv_acceptance_profile", + "decode_puf_raw_source", + "definition_document_json", + "decode_return_status", + "encode_return_status", + "fixture_definition", + "load_pinned_puf_bytes", + "packaged_definition", + "puf_raw_source_codecs", + "register_us_puf_raw_source_codecs", + "register_us_puf_raw_source_kernels", + "us_puf_raw_return_status_summary_node", + "us_puf_raw_source_node", + "us_puf_raw_source_refs", +] + +#: The most this module reads from one PUF delivery file (128 MiB). Private: +#: the shared ``raw-bytes-v1`` cap is untouched at 64 MiB. +PUF_RAW_SOURCE_MAX_BYTES = 128 * 1024 * 1024 + +#: Bound on the encoded artifact's column bodies, below the 64 MiB a shared +#: opaque payload is expected to stay under. The full slice is ~35 MB. +BODY_MAX_BYTES = 64 * 1024 * 1024 + +#: Bound on the canonical JSON header, as in the ACS rent placement envelope. +HEADER_MAX_BYTES = 64 * 1024 + +RETURN_STATUS_TYPE = ArtifactType("microcosm.us.puf_2015_raw_return_status", 1) +RETURN_STATUS_MAGIC = b"microcosm.us.puf_2015_raw_return_status/1\n" + +MAIN_SOURCE_NAME = "puf_2015_main" +DEMOGRAPHIC_SOURCE_NAME = "puf_2015_demographic" +MAIN_CODEC = "us-puf-2015-main-raw-v1" +DEMOGRAPHIC_CODEC = "us-puf-2015-demographic-raw-v1" + +PUF_RAW_SOURCE_STAGE = "us_puf_raw_source" +PUF_RAW_SOURCE_NODE = f"{PUF_RAW_SOURCE_STAGE}.return_status" +RETURN_STATUS_SUMMARY_NODE = f"{PUF_RAW_SOURCE_STAGE}.return_status_summary" + +#: The publisher's four aggregate record identifiers. The literal values, not +#: an import of ``puf_aggregate_records``: that module pulls nine derivation +#: modules and the calibrate package behind it. +PUF_AGGREGATE_RECIDS = (999996, 999997, 999998, 999999) + +_DEFINITION_RESOURCE = "puf_2015_raw_source_definition.json" +_DEFINITION_PACKAGE = "microcosm.build.us_runtime" +_FIXTURE_AUTHORITY = "invented_fixture_nonauthority" + +_MAIN_FIELDS = ("RECID", "FLPDYR", "FLPDMO", "MARS", "DSI", "S006") +_DEMOGRAPHIC_FIELDS = ( + "AGEDP1", + "AGEDP2", + "AGEDP3", + "AGERANGE", + "EARNSPLIT", + "GENDER", +) +#: Typed column name to its explicit little-endian numpy dtype. Bodies are +#: written with these exact dtypes, so the payload is platform-independent. +_TYPED_DTYPES = { + "RECID": " None: + super().__init__(" ".join((reason, *(str(part) for part in detail))).strip()) + self.reason = reason + + +def _refuse(reason: str, *detail: object) -> PufRawSourceRefusalError: + return PufRawSourceRefusalError(reason, *detail) + + +# -------------------------------------------------------------------------- +# The closed definition +# -------------------------------------------------------------------------- + + +@dataclass(frozen=True) +class SourcePin: + """The closed content identity of one delivered PUF file.""" + + name: str + codec: str + source_name: str + bytes: int + sha256: str + git_blob_sha1: str + header_record_canonical_sha256: str + delivered_header: tuple[str, ...] + data_records: int + + +@dataclass(frozen=True) +class PufRawSourceDefinition: + """A closed source declaration plus the route that produced it.""" + + route: str + document: Mapping[str, object] + canonical: bytes + sha256: str + main: SourcePin + demographic: SourcePin + + @property + def params_text(self) -> str: + """The canonical JSON the node carries verbatim as a parameter.""" + + return self.canonical.decode("ascii") + + +def _hex64(value: object, label: str) -> str: + if ( + not isinstance(value, str) + or len(value) != 64 + or any(character not in "0123456789abcdef" for character in value) + ): + raise _refuse("DEFINITION_DIGEST", label) + return value + + +def _hex40(value: object, label: str) -> str: + if ( + not isinstance(value, str) + or len(value) != 40 + or any(character not in "0123456789abcdef" for character in value) + ): + raise _refuse("DEFINITION_DIGEST", label) + return value + + +def _pin(document: Mapping[str, object], name: str) -> SourcePin: + entry = document["sources"][name] + header = entry["delivered_header"] + if not isinstance(header, list) or not header: + raise _refuse("DEFINITION_HEADER", name) + if len(header) != entry["delivered_header_width"]: + raise _refuse("DEFINITION_HEADER_WIDTH", name) + if len(set(header)) != len(header): + raise _refuse("DEFINITION_HEADER_DUPLICATE", name) + if _canonical_header_digest(header) != entry["header_record_canonical_sha256"]: + raise _refuse("DEFINITION_HEADER_DIGEST", name) + size = entry["bytes"] + if not isinstance(size, int) or isinstance(size, bool) or size <= 0: + raise _refuse("DEFINITION_SOURCE_BYTES", name) + if size > PUF_RAW_SOURCE_MAX_BYTES: + raise _refuse("DEFINITION_SOURCE_OVER_CAP", name) + return SourcePin( + name=name, + codec=entry["codec"], + source_name=entry["source_name"], + bytes=size, + sha256=_hex64(entry["sha256"], f"{name}.sha256"), + git_blob_sha1=_hex40(entry["git_blob_sha1"], f"{name}.git_blob_sha1"), + header_record_canonical_sha256=_hex64( + entry["header_record_canonical_sha256"], f"{name}.header" + ), + delivered_header=tuple(header), + data_records=entry["data_records"], + ) + + +def _frozen(value: object) -> object: + """A deeply immutable view: mappings become proxies, sequences tuples. + + ``MappingProxyType`` only freezes the top level. ``definition.document`` + is reached through ordinary read-only attribute access, so handing out a + mutable nested container let a caller change a declared lexical width or + a CSV limit — changing what parses — while ``canonical``, ``sha256`` and + therefore ``params_text`` and the node key all stayed identical. Freezing + the whole tree closes that without any caller sandboxing: the document is + simply a value, as its canonical bytes always were. + """ + + if isinstance(value, Mapping): + return MappingProxyType({key: _frozen(item) for key, item in value.items()}) + if isinstance(value, (list, tuple)): + return tuple(_frozen(item) for item in value) + return value + + +def _canonical_header_digest(header: Sequence[str]) -> str: + """The audit's header identity: sha256 of the canonical JSON header list.""" + + payload = json.dumps( + list(header), + sort_keys=True, + separators=(",", ":"), + ensure_ascii=True, + allow_nan=False, + ).encode("ascii") + return hashlib.sha256(payload).hexdigest() + + +def _definition_from_document( + document: Mapping[str, object], *, route: str +) -> PufRawSourceDefinition: + if document.get("schema") != "microcosm.us.puf_2015_raw_source_definition": + raise _refuse("DEFINITION_SCHEMA") + if document.get("schema_version") != 1: + raise _refuse("DEFINITION_SCHEMA_VERSION") + if document.get("route") != route: + raise _refuse("DEFINITION_ROUTE") + fields = document["fields"] + declared = tuple(field["name"] for field in fields) + if declared != _MAIN_FIELDS + _DEMOGRAPHIC_FIELDS: + raise _refuse("DEFINITION_FIELDS") + for field in fields: + width = field["lexical_width"] + if not isinstance(width, int) or isinstance(width, bool) or not 0 < width <= 64: + raise _refuse("DEFINITION_LEXICAL_WIDTH", field["name"]) + if field["grammar"] != "ascii_decimal_unsigned": + raise _refuse("DEFINITION_GRAMMAR", field["name"]) + if tuple(document["aggregates"]["recids"]) != PUF_AGGREGATE_RECIDS: + raise _refuse("DEFINITION_AGGREGATE_RECIDS") + canonical = canonical_json(dict(document)) + # The document is re-derived from the canonical bytes and then deeply + # frozen, so the two can never disagree and neither can be edited. + return PufRawSourceDefinition( + route=route, + document=_frozen(json.loads(canonical.decode("ascii"))), + canonical=canonical, + sha256=hashlib.sha256(canonical).hexdigest(), + main=_pin(document, "main"), + demographic=_pin(document, "demographic"), + ) + + +def _thawed(value: object) -> object: + """The plain-container form of a frozen document, for canonical rendering.""" + + if isinstance(value, Mapping): + return {key: _thawed(item) for key, item in value.items()} + if isinstance(value, tuple): + return [_thawed(item) for item in value] + return value + + +def _packaged_bytes() -> bytes: + return ( + resources.files(_DEFINITION_PACKAGE).joinpath(_DEFINITION_RESOURCE).read_bytes() + ) + + +_PACKAGED: PufRawSourceDefinition | None = None + + +def packaged_definition() -> PufRawSourceDefinition: + """The one closed definition a production run may use.""" + + global _PACKAGED + if _PACKAGED is None: + document = json.loads(_packaged_bytes().decode("ascii")) + _PACKAGED = _definition_from_document(document, route="packaged") + return _PACKAGED + + +def definition_document_json( + definition: PufRawSourceDefinition, +) -> dict[str, object]: + """A fresh, plain, mutable JSON copy of a definition's document. + + The definition's own document is deeply frozen. A caller that wants to + build a *different* document from this one — a fixture, say — starts here + and gets a copy it owns, rather than a handle on the original. + """ + + return json.loads(definition.canonical.decode("ascii")) + + +def fixture_definition(document: Mapping[str, object]) -> PufRawSourceDefinition: + """Build a definition on the explicit non-production test route. + + A fixture definition must declare ``route="test_fixture"`` and the + invented-fixture authority, and it may not restate either packaged source + pin. There is no route where an omitted or optional expected hash admits + a file: a fixture carries its own real pins over its own invented bytes. + """ + + definition = _definition_from_document(document, route="test_fixture") + if document.get("authority") != _FIXTURE_AUTHORITY: + raise _refuse("FIXTURE_AUTHORITY") + packaged = packaged_definition() + genuine = {packaged.main.sha256, packaged.demographic.sha256} + if {definition.main.sha256, definition.demographic.sha256} & genuine: + raise _refuse("FIXTURE_RESTATES_PACKAGED_PIN") + return definition + + +# -------------------------------------------------------------------------- +# The pinned raw-byte codec +# -------------------------------------------------------------------------- + + +def _identity(status: os.stat_result) -> tuple[int, int, int, int, int]: + return ( + status.st_dev, + status.st_ino, + status.st_mode, + status.st_size, + status.st_mtime_ns, + ) + + +def _git_blob_sha1(payload: bytes) -> str: + header = b"blob " + str(len(payload)).encode("ascii") + b"\0" + return hashlib.sha1(header + payload).hexdigest() + + +def load_pinned_puf_bytes( + path: Path, *, store: object | None = None, pin: SourcePin +) -> bytes: + """Read one pinned PUF delivery file, or refuse. + + The codec authenticates the bytes itself: exact length, SHA-256 and git + blob SHA-1, all three from the descriptor it opened. It never consults a + capture receipt or run summary, so a forged receipt cannot admit changed + bytes. + + ``O_NOFOLLOW`` protects the path *this function receives*. The graph + executor resolves source paths strictly before codec lookup, so under the + executor the codec sees an already-resolved path and a user's symlink is + followed upstream; on a direct call a symlink is refused here. + ``O_NONBLOCK`` means a FIFO cannot make the codec wait. + """ + + del store # identity is content; no content store is consulted + if not isinstance(pin, SourcePin): + raise _refuse("CODEC_PIN_TYPE") + if pin.bytes > PUF_RAW_SOURCE_MAX_BYTES: + raise _refuse("CODEC_PIN_OVER_CAP", pin.name) + source = Path(path) + try: + descriptor = os.open(source, os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK) + except OSError as error: + raise _refuse("CODEC_OPEN", pin.name) from error + try: + before = os.fstat(descriptor) + if not stat.S_ISREG(before.st_mode): + raise _refuse("CODEC_NOT_REGULAR_FILE", pin.name) + if before.st_size != pin.bytes: + raise _refuse("CODEC_SIZE", pin.name) + if _identity(source.lstat()) != _identity(before): + raise _refuse("CODEC_PATH_IDENTITY", pin.name) + digest = hashlib.sha256() + blob = hashlib.sha1(b"blob " + str(pin.bytes).encode("ascii") + b"\0") + chunks: list[bytes] = [] + total = 0 + limit = pin.bytes + 1 + while True: + chunk = os.read(descriptor, 1024 * 1024) + if not chunk: + break + total += len(chunk) + if total > limit: + raise _refuse("CODEC_GREW", pin.name) + digest.update(chunk) + blob.update(chunk) + chunks.append(chunk) + if total != pin.bytes: + raise _refuse("CODEC_LENGTH", pin.name) + if _identity(os.fstat(descriptor)) != _identity(before): + raise _refuse("CODEC_CHANGED_DURING_READ", pin.name) + if digest.hexdigest() != pin.sha256: + raise _refuse("CODEC_SHA256", pin.name) + if blob.hexdigest() != pin.git_blob_sha1: + raise _refuse("CODEC_GIT_BLOB_SHA1", pin.name) + finally: + os.close(descriptor) + return b"".join(chunks) + + +def register_us_puf_raw_source_codecs( + registry: SourceCodecRegistry, *, definition: PufRawSourceDefinition | None = None +) -> SourceCodecRegistry: + """Bind the two pinned byte codecs into ``registry`` through ``register_bytes``.""" + + resolved = packaged_definition() if definition is None else definition + registry.register_bytes( + resolved.main.codec, partial(load_pinned_puf_bytes, pin=resolved.main) + ) + registry.register_bytes( + resolved.demographic.codec, + partial(load_pinned_puf_bytes, pin=resolved.demographic), + ) + return registry + + +def puf_raw_source_codecs( + definition: PufRawSourceDefinition | None = None, +) -> SourceCodecRegistry: + """A private registry holding only this slice's two pinned byte codecs.""" + + return register_us_puf_raw_source_codecs( + SourceCodecRegistry(), definition=definition + ) + + +# -------------------------------------------------------------------------- +# The structural decoder +# -------------------------------------------------------------------------- + + +@dataclass(frozen=True) +class PufRawReturnStatus: + """The decoded slice: typed arrays, lexical tokens and aggregate facts.""" + + typed: Mapping[str, np.ndarray] + lexical: Mapping[str, tuple[str, ...]] + facts: Mapping[str, object] + + @property + def rows(self) -> int: + return int(self.facts["rows"]) + + +def csv_acceptance_profile() -> dict[str, object]: + """The effective CSV acceptance state this decoder actually parses under. + + ``csv.field_size_limit`` is process-global state that silently changes + which deliveries parse: at a lowered limit a file that was admitted is + refused, and at a raised limit a file that was refused is admitted. It is + therefore part of the producer's identity, exactly as + :mod:`asec_income_observations` and :mod:`acs_housing_universe_source` + already bind it. This function only *reads* it — nothing in this module + ever sets it, so no other stage's acceptance is disturbed. + + The remaining entries are the reader semantics :func:`_reader` fixes + itself; the delimiter, quote character and doublequote flag come from the + closed definition and are checked against it at decode time. + """ + + return { + "engine": "stdlib_csv_reader", + "encoding": "utf-8-strict", + "newline": "", + "quoting": int(csv.QUOTE_MINIMAL), + "strict": True, + "csv.field_size_limit": csv.field_size_limit(), + } + + +def _check_csv_profile(profile: Mapping[str, object]) -> None: + """Refuse when the effective acceptance state is not the pinned one. + + The packaged definition states ``field_size_limit``. Before this check it + was a decorative fact: the decoder relied on the process default happening + to match. Now the pinned value is the value the parse actually ran under, + or the decode refuses — a lowered limit is a refusal, never a silent + reparse, and a raised limit is a refusal, never a silent widening. + """ + + declared = profile["field_size_limit"] + if not isinstance(declared, int) or isinstance(declared, bool) or declared <= 0: + raise _refuse("DEFINITION_CSV_FIELD_SIZE_LIMIT") + if csv.field_size_limit() != declared: + raise _refuse("CSV_FIELD_SIZE_LIMIT") + if profile["quoting"] != "QUOTE_MINIMAL" or profile["strict"] is not True: + raise _refuse("DEFINITION_CSV_PROFILE") + if profile["encoding"] != "utf-8-strict" or profile["newline"] != "": + raise _refuse("DEFINITION_CSV_PROFILE") + + +def _reader(payload: bytes, profile: Mapping[str, object]): + if not isinstance(payload, bytes): + raise _refuse("DECODE_PAYLOAD_TYPE") + stream = io.TextIOWrapper( + io.BytesIO(payload), encoding="utf-8", errors="strict", newline="" + ) + return csv.reader( + stream, + delimiter=profile["delimiter"], + quotechar=profile["quotechar"], + doublequote=bool(profile["doublequote"]), + quoting=csv.QUOTE_MINIMAL, + strict=True, + ) + + +def _token(value: str, *, width: int, label: str) -> str: + """An unsigned ASCII decimal token within its declared allocation.""" + + if not isinstance(value, str): + raise _refuse("TOKEN_TYPE", label) + raw = value.encode("ascii", errors="strict") if value.isascii() else None + if raw is None: + raise _refuse("TOKEN_NON_ASCII", label) + if b"\x00" in raw: + raise _refuse("TOKEN_EMBEDDED_NUL", label) + if not raw: + raise _refuse("TOKEN_EMPTY", label) + if len(raw) > width: + raise _refuse("TOKEN_OVER_WIDTH", label) + if any(byte not in _DIGITS for byte in raw): + raise _refuse("TOKEN_GRAMMAR", label) + return value + + +def _int64(token: str, label: str) -> int: + value = int(token) + if value > _INT64_MAX: + raise _refuse("TOKEN_INT64_OVERFLOW", label) + return value + + +def _widths(definition: PufRawSourceDefinition) -> dict[str, int]: + return { + field["name"]: field["lexical_width"] for field in definition.document["fields"] + } + + +def _domains(definition: PufRawSourceDefinition) -> dict[str, frozenset[int]]: + domains: dict[str, frozenset[int]] = {} + for field in definition.document["fields"]: + values = field.get("domain") + if values is not None: + domains[field["name"]] = frozenset(int(value) for value in values) + return domains + + +def _agerange_domains( + definition: PufRawSourceDefinition, +) -> dict[int, frozenset[int]]: + for field in definition.document["fields"]: + if field["name"] == "AGERANGE": + cases = field["conditional_domain"]["cases"] + return { + int(key): frozenset(int(value) for value in values) + for key, values in cases.items() + } + raise _refuse("DEFINITION_AGERANGE") + + +def _check_header( + reader, pin: SourcePin, *, profile: Mapping[str, object] +) -> list[str]: + try: + header = next(reader) + except StopIteration as error: + raise _refuse("HEADER_MISSING", pin.name) from error + except (csv.Error, UnicodeError) as error: + raise _refuse("HEADER_UNREADABLE", pin.name) from error + if len(header) > profile["header_field_cap"]: + raise _refuse("HEADER_WIDTH_LIMIT", pin.name) + if tuple(header) != pin.delivered_header: + raise _refuse("HEADER_MISMATCH", pin.name) + if _canonical_header_digest(header) != pin.header_record_canonical_sha256: + raise _refuse("HEADER_DIGEST", pin.name) + return header + + +def _records(reader, pin: SourcePin, *, profile: Mapping[str, object]): + width = len(pin.delivered_header) + cap = profile["logical_record_character_cap"] + limit = profile["record_cap_per_file"] + count = 0 + while True: + try: + record = next(reader) + except StopIteration: + return + except (csv.Error, UnicodeError) as error: + raise _refuse("RECORD_UNREADABLE", pin.name) from error + count += 1 + if count > limit: + raise _refuse("RECORD_CAP", pin.name) + if len(record) != width: + raise _refuse("RECORD_WIDTH", pin.name) + if sum(len(cell) for cell in record) > cap: + raise _refuse("RECORD_CHARACTER_CAP", pin.name) + yield record + + +def decode_puf_raw_source( + main_bytes: bytes, + demographic_bytes: bytes, + definition: PufRawSourceDefinition, +) -> PufRawReturnStatus: + """Decode the two delivery payloads into the typed status slice. + + This is structure only. It checks headers, widths, token grammar, closed + domains and the exact lexical join against the definition it is handed, + and it authorizes nothing about where the bytes came from. + """ + + if not isinstance(definition, PufRawSourceDefinition): + raise _refuse("DECODE_DEFINITION_TYPE") + profile = definition.document["csv_profile"] + _check_csv_profile(profile) + widths = _widths(definition) + domains = _domains(definition) + agerange = _agerange_domains(definition) + aggregates = frozenset(PUF_AGGREGATE_RECIDS) + + demographic = _decode_demographic( + demographic_bytes, definition, profile=profile, widths=widths + ) + main = _decode_main( + main_bytes, + definition, + profile=profile, + widths=widths, + domains=domains, + aggregates=aggregates, + ) + + rows = len(main["RECID_lexical"]) + orphans = set(demographic["by_key"]) - set(main["RECID_lexical"]) + if orphans: + raise _refuse("DEMOGRAPHIC_ORPHAN_KEYS", len(orphans)) + + typed: dict[str, np.ndarray] = { + name: np.empty(rows, dtype=_TYPED_DTYPES[name]) for name in _TYPED_COLUMNS + } + for name in _MAIN_FIELDS: + typed[name][:] = np.asarray(main[name], dtype=_TYPED_DTYPES[name]) + typed["disclosure_aggregate"][:] = np.asarray( + main["disclosure_aggregate"], dtype=_TYPED_DTYPES["disclosure_aggregate"] + ) + + lexical: dict[str, list[str]] = { + name: list( + main["RECID_lexical"] if name == "RECID" else main[f"{name}_lexical"] + ) + for name in _MAIN_FIELDS + } + for name in _DEMOGRAPHIC_FIELDS: + lexical[name] = [""] * rows + + status = np.zeros(rows, dtype=_TYPED_DTYPES["demographic_status"]) + for name in _DEMOGRAPHIC_FIELDS: + typed[name][:] = _ABSENT_SENTINEL + + aggregate_demographic_rows = 0 + for index, key in enumerate(main["RECID_lexical"]): + record = demographic["by_key"].get(key) + if record is None: + continue + status[index] = 1 + dsi = int(typed["DSI"][index]) + allowed = agerange.get(dsi) + if allowed is None: + raise _refuse("AGERANGE_CONDITION_UNKNOWN") + for name in _DEMOGRAPHIC_FIELDS: + token = record[name] + value = _int64(token, name) + permitted = allowed if name == "AGERANGE" else domains[name] + if value not in permitted: + raise _refuse("DEMOGRAPHIC_DOMAIN", name) + typed[name][index] = value + lexical[name][index] = token + if bool(typed["disclosure_aggregate"][index]): + aggregate_demographic_rows += 1 + typed["demographic_status"][:] = status + + del aggregate_demographic_rows # re-derived below from the arrays themselves + if demographic["records"] != int(status.sum()): + raise _refuse("DEMOGRAPHIC_RECORD_COUNT") + facts = _facts_from_arrays( + typed, + join_method=_join_method(definition), + untyped_main_columns=_untyped_main_columns(definition), + ) + _check_declared_expectations(facts, definition) + return PufRawReturnStatus( + typed=MappingProxyType({name: array for name, array in typed.items()}), + lexical=MappingProxyType( + {name: tuple(values) for name, values in lexical.items()} + ), + facts=MappingProxyType(facts), + ) + + +def _check_declared_expectations( + facts: Mapping[str, object], definition: PufRawSourceDefinition +) -> None: + """Cross-check the derived counts against the definition's declared ones. + + The packaged definition states how many records each delivery holds, how + many keys should match, how many main records should stay unmatched and + how many aggregate records to expect. Nothing compared them, so they were + documentation. The byte pins make them implied, but implied is not + checked: a delivery that hashed correctly and parsed to a different shape + would have gone unnoticed. + + This runs on the packaged route only. A fixture definition carries its own + invented counts over its own invented bytes, and inventing a fixture that + restates the genuine expectations would be the wrong kind of fidelity. + """ + + if definition.route != "packaged": + return + join = definition.document["join"] + for label, derived, declared in ( + ("rows", facts["rows"], definition.main.data_records), + ( + "demographic_records", + facts["demographic_records"], + definition.demographic.data_records, + ), + ("matched_keys", facts["matched_keys"], join["expected_matched_keys"]), + ( + "main_unmatched_records", + facts["main_unmatched_records"], + join["expected_main_unmatched_records"], + ), + ( + "aggregate_records", + facts["aggregate_records"], + definition.document["aggregates"]["expected_records"], + ), + ): + if derived != declared: + raise _refuse("DECLARED_EXPECTATION", label) + + +def _join_method(definition: PufRawSourceDefinition) -> str: + method = definition.document["join"]["method"] + if not isinstance(method, str) or not method: + raise _refuse("DEFINITION_JOIN_METHOD") + return method + + +def _untyped_main_columns(definition: PufRawSourceDefinition) -> int: + return len(definition.main.delivered_header) - len(_MAIN_FIELDS) + + +def _facts_from_arrays( + typed: Mapping[str, np.ndarray], + *, + join_method: str, + untyped_main_columns: int, +) -> dict[str, object]: + """Derive every reported fact from the typed arrays themselves. + + The producer builds its facts here and the envelope readback rebuilds them + here, so a header can never report a count the arrays do not carry. Only + ``join_method`` and ``untyped_main_columns`` come from outside: both are + properties of the definition, not of the decoded rows. + """ + + rows = int(typed["RECID"].shape[0]) + status = typed["demographic_status"] + aggregate = typed["disclosure_aggregate"] + matched = int(np.count_nonzero(status == 1)) + return { + "rows": rows, + "main_records": rows, + "demographic_records": matched, + "matched_keys": matched, + "main_unmatched_records": rows - matched, + "demographic_orphan_records": 0, + "aggregate_records": int(np.count_nonzero(aggregate == 1)), + "aggregate_demographic_records": int( + np.count_nonzero((aggregate == 1) & (status == 1)) + ), + "join_status": "one_to_one_subset_compatible", + "join_method": join_method, + "code_frequencies": _code_frequencies(typed), + "period_semantics": "unresolved_raw_FLPDYR", + "weight_units": "S006_hundredths", + "amounts_decoded": False, + "untyped_main_columns": untyped_main_columns, + } + + +def _code_frequencies(typed: Mapping[str, np.ndarray]) -> dict[str, dict[str, int]]: + counted = ("FLPDYR", "FLPDMO", "MARS", "DSI", "demographic_status") + frequencies: dict[str, dict[str, int]] = {} + for name in (*counted, *_DEMOGRAPHIC_FIELDS): + values, counts = np.unique(typed[name], return_counts=True) + frequencies[name] = { + str(int(value)): int(count) + for value, count in zip(values.tolist(), counts.tolist(), strict=True) + } + return frequencies + + +def _decode_main( + payload: bytes, + definition: PufRawSourceDefinition, + *, + profile, + widths, + domains, + aggregates, +) -> dict[str, object]: + pin = definition.main + reader = _reader(payload, profile) + header = _check_header(reader, pin, profile=profile) + index = {name: header.index(name) for name in _MAIN_FIELDS} + columns: dict[str, list] = {name: [] for name in _MAIN_FIELDS} + lexical: dict[str, list[str]] = {f"{name}_lexical": [] for name in _MAIN_FIELDS} + keys: list[str] = [] + numeric_keys: set[int] = set() + seen: set[str] = set() + aggregate_flags: list[int] = [] + for record in _records(reader, pin, profile=profile): + values: dict[str, int] = {} + for name in _MAIN_FIELDS: + token = _token(record[index[name]], width=widths[name], label=name) + lexical[f"{name}_lexical"].append(token) + values[name] = _int64(token, name) + key = lexical["RECID_lexical"][-1] + if key in seen: + raise _refuse("MAIN_DUPLICATE_LEXICAL_KEY") + seen.add(key) + recid = values["RECID"] + if recid in numeric_keys: + raise _refuse("MAIN_NUMERIC_KEY_COLLISION") + numeric_keys.add(recid) + keys.append(key) + for name in ("FLPDYR", "FLPDMO", "MARS", "DSI"): + if values[name] not in domains[name]: + raise _refuse("MAIN_DOMAIN", name) + aggregate = recid in aggregates + if aggregate != (values["MARS"] == 0): + raise _refuse("AGGREGATE_MARS_DISAGREEMENT") + aggregate_flags.append(1 if aggregate else 0) + for name in _MAIN_FIELDS: + columns[name].append(values[name]) + if not keys: + raise _refuse("MAIN_NO_RECORDS") + result: dict[str, object] = dict(columns) + result.update(lexical) + result["disclosure_aggregate"] = aggregate_flags + return result + + +def _decode_demographic( + payload: bytes, definition: PufRawSourceDefinition, *, profile, widths +) -> dict[str, object]: + pin = definition.demographic + reader = _reader(payload, profile) + header = _check_header(reader, pin, profile=profile) + index = {name: header.index(name) for name in ("RECID", *_DEMOGRAPHIC_FIELDS)} + by_key: dict[str, dict[str, str]] = {} + numeric_keys: set[int] = set() + records = 0 + for record in _records(reader, pin, profile=profile): + key = _token(record[index["RECID"]], width=widths["RECID"], label="RECID") + if key in by_key: + raise _refuse("DEMOGRAPHIC_DUPLICATE_LEXICAL_KEY") + recid = _int64(key, "RECID") + if recid in numeric_keys: + raise _refuse("DEMOGRAPHIC_NUMERIC_KEY_COLLISION") + numeric_keys.add(recid) + by_key[key] = { + name: _token(record[index[name]], width=widths[name], label=name) + for name in _DEMOGRAPHIC_FIELDS + } + records += 1 + return {"by_key": by_key, "records": records} + + +# -------------------------------------------------------------------------- +# The typed envelope +# -------------------------------------------------------------------------- + + +def _fixed_ascii(values: Sequence[str], width: int, label: str) -> bytes: + body = bytearray(len(values) * width) + for position, value in enumerate(values): + raw = value.encode("ascii") + if len(raw) > width: + raise _refuse("LEXICAL_OVER_WIDTH", label) + if b"\x00" in raw: + raise _refuse("LEXICAL_EMBEDDED_NUL", label) + start = position * width + body[start : start + len(raw)] = raw + return bytes(body) + + +def _read_fixed_ascii( + body: bytes, rows: int, width: int, label: str +) -> tuple[str, ...]: + values = [] + for position in range(rows): + cell = body[position * width : (position + 1) * width] + stripped = cell.rstrip(b"\x00") + if b"\x00" in stripped: + raise _refuse("LEXICAL_EMBEDDED_NUL", label) + try: + values.append(stripped.decode("ascii")) + except UnicodeDecodeError as error: + raise _refuse("LEXICAL_NON_ASCII", label) from error + return tuple(values) + + +def _payload_bound( + rows: int, definition: PufRawSourceDefinition +) -> tuple[int, dict[str, int]]: + """The exact body size this artifact will allocate, and its per-column parts.""" + + widths = _widths(definition) + sizes = { + name: rows * np.dtype(_TYPED_DTYPES[name]).itemsize for name in _TYPED_COLUMNS + } + sizes.update({f"{name}_lexical": rows * widths[name] for name in _LEXICAL_COLUMNS}) + return sum(sizes.values()), sizes + + +def encode_return_status( + status: PufRawReturnStatus, definition: PufRawSourceDefinition +) -> bytes: + """Encode the slice into the bounded, self-describing artifact envelope. + + Magic line, a 4-byte big-endian header length matching the ACS rent + placement precedent, a canonical JSON header, then explicit little-endian + integer bodies and fixed-width NUL-padded ASCII lexical bodies, in the + declared order. Every body carries its own SHA-256 in the header. + """ + + rows = status.rows + total, sizes = _payload_bound(rows, definition) + if total > BODY_MAX_BYTES: + raise _refuse("BODY_OVER_BOUND", total) + widths = _widths(definition) + bodies: list[bytes] = [] + columns: list[dict[str, object]] = [] + for name in _TYPED_COLUMNS: + array = np.ascontiguousarray(status.typed[name], dtype=_TYPED_DTYPES[name]) + if array.shape != (rows,): + raise _refuse("COLUMN_SHAPE", name) + body = array.tobytes() + if len(body) != sizes[name]: + raise _refuse("COLUMN_SIZE", name) + bodies.append(body) + columns.append( + { + "name": name, + "kind": "typed", + "dtype": _TYPED_DTYPES[name], + "bytes": len(body), + "sha256": hashlib.sha256(body).hexdigest(), + } + ) + for name in _LEXICAL_COLUMNS: + body = _fixed_ascii(status.lexical[name], widths[name], name) + if len(body) != sizes[f"{name}_lexical"]: + raise _refuse("COLUMN_SIZE", name) + bodies.append(body) + columns.append( + { + "name": f"{name}_lexical", + "kind": "lexical_fixed_ascii_nul_padded", + "width": widths[name], + "bytes": len(body), + "sha256": hashlib.sha256(body).hexdigest(), + } + ) + header = { + "schema_version": 1, + "type": [RETURN_STATUS_TYPE.name, RETURN_STATUS_TYPE.schema_version], + "rows": rows, + "definition_sha256": definition.sha256, + "definition_route": definition.route, + "sources": { + "main": { + "sha256": definition.main.sha256, + "git_blob_sha1": definition.main.git_blob_sha1, + "bytes": definition.main.bytes, + "header_record_canonical_sha256": ( + definition.main.header_record_canonical_sha256 + ), + }, + "demographic": { + "sha256": definition.demographic.sha256, + "git_blob_sha1": definition.demographic.git_blob_sha1, + "bytes": definition.demographic.bytes, + "header_record_canonical_sha256": ( + definition.demographic.header_record_canonical_sha256 + ), + }, + }, + "header_length_endianness": "big", + "body_endianness": "little", + "columns": columns, + "facts": dict(status.facts), + } + encoded = canonical_json(header) + if len(encoded) > HEADER_MAX_BYTES: + raise _refuse("HEADER_OVER_BOUND", len(encoded)) + return b"".join( + (RETURN_STATUS_MAGIC, len(encoded).to_bytes(4, "big"), encoded, *bodies) + ) + + +def _exact_keys( + value: object, keys: frozenset[str], label: str +) -> Mapping[str, object]: + """A mapping whose key set is exactly ``keys``. Nothing extra, nothing missing.""" + + if not isinstance(value, dict) or set(value) != set(keys): + raise _refuse("ENVELOPE_SCHEMA", label) + return value + + +def _envelope_int(value: object, label: str, *, minimum: int = 0) -> int: + if not isinstance(value, int) or isinstance(value, bool) or value < minimum: + raise _refuse("ENVELOPE_INTEGER", label) + return value + + +def _resolve_envelope_definition( + header: Mapping[str, object], definition: PufRawSourceDefinition | None +) -> PufRawSourceDefinition | None: + """The definition this payload may be checked against, or ``None``. + + A caller may hand one in. Otherwise a payload that declares the *packaged* + route is checked against this module's own packaged definition, because + that one is closed and importable. A fixture-route payload has no + reconstructible definition, so it is checked structurally only: the + production and fixture boundaries stay exactly where they are. + """ + + route = header["definition_route"] + if route not in ("packaged", "test_fixture"): + raise _refuse("ENVELOPE_DEFINITION_ROUTE") + digest = _hex64(header["definition_sha256"], "envelope.definition_sha256") + if definition is not None: + if not isinstance(definition, PufRawSourceDefinition): + raise _refuse("DECODE_DEFINITION_TYPE") + if definition.route != route or definition.sha256 != digest: + raise _refuse("ENVELOPE_DEFINITION_MISMATCH") + return definition + if route == "packaged": + packaged = packaged_definition() + if packaged.sha256 != digest: + raise _refuse("ENVELOPE_DEFINITION_NOT_PACKAGED") + return packaged + return None + + +def _check_envelope_sources( + header: Mapping[str, object], definition: PufRawSourceDefinition | None +) -> None: + sources = _exact_keys( + header["sources"], frozenset({"main", "demographic"}), "sources" + ) + for name in ("main", "demographic"): + entry = _exact_keys(sources[name], _ENVELOPE_SOURCE_KEYS, f"sources.{name}") + _hex64(entry["sha256"], f"sources.{name}.sha256") + _hex40(entry["git_blob_sha1"], f"sources.{name}.git_blob_sha1") + _hex64(entry["header_record_canonical_sha256"], f"sources.{name}.header") + size = _envelope_int(entry["bytes"], f"sources.{name}.bytes", minimum=1) + if size > PUF_RAW_SOURCE_MAX_BYTES: + raise _refuse("ENVELOPE_SOURCE_OVER_CAP", name) + if definition is None: + continue + pin = definition.main if name == "main" else definition.demographic + if ( + entry["sha256"] != pin.sha256 + or entry["git_blob_sha1"] != pin.git_blob_sha1 + or size != pin.bytes + or entry["header_record_canonical_sha256"] + != pin.header_record_canonical_sha256 + ): + raise _refuse("ENVELOPE_SOURCE_PIN", name) + + +def _envelope_columns( + header: Mapping[str, object], + rows: int, + definition: PufRawSourceDefinition | None, +) -> tuple[list[Mapping[str, object]], dict[str, int]]: + """Close the column schema: exact order, kind, dtype, width and size.""" + + columns = header["columns"] + if not isinstance(columns, list): + raise _refuse("ENVELOPE_COLUMNS") + declared = [ + *_TYPED_COLUMNS, + *(f"{name}_lexical" for name in _LEXICAL_COLUMNS), + ] + if len(columns) != len(declared): + raise _refuse("ENVELOPE_COLUMNS") + widths = _widths(definition) if definition is not None else None + resolved: list[Mapping[str, object]] = [] + sizes: dict[str, int] = {} + total = 0 + for column, name in zip(columns, declared, strict=True): + if not isinstance(column, dict) or column.get("name") != name: + raise _refuse("ENVELOPE_COLUMNS") + if name in _TYPED_DTYPES: + entry = _exact_keys(column, _ENVELOPE_TYPED_COLUMN_KEYS, name) + if entry["kind"] != _TYPED_KIND: + raise _refuse("ENVELOPE_COLUMN_KIND", name) + # Exact dtype string: this closes width, signedness and endianness + # at once, so `` _LEXICAL_WIDTH_MAX: + raise _refuse("ENVELOPE_COLUMN_WIDTH", name) + if widths is not None and width != widths[field]: + raise _refuse("ENVELOPE_COLUMN_WIDTH", name) + expected = rows * width + sizes[field] = width + _hex64(entry["sha256"], f"{name}.sha256") + size = _envelope_int(entry["bytes"], f"{name}.bytes") + if size != expected: + raise _refuse("ENVELOPE_COLUMN_SIZE", name) + total += size + if total > BODY_MAX_BYTES: + raise _refuse("ENVELOPE_BODY_OVER_BOUND", total) + resolved.append(entry) + return resolved, sizes + + +def _verify_readback( + typed: Mapping[str, np.ndarray], + lexical: Mapping[str, tuple[str, ...]], + widths: Mapping[str, int], + definition: PufRawSourceDefinition | None, +) -> None: + """Re-prove the slice's own invariants from the arrays that were read back. + + Lexical and numeric spellings must agree, absence must stay absence rather + than a zero code, the aggregate flag must follow from ``RECID`` and agree + with ``MARS``, and the join keys must still be unique both ways. + """ + + rows = int(typed["RECID"].shape[0]) + status = typed["demographic_status"] + aggregate = typed["disclosure_aggregate"] + if not np.isin(status, (0, 1)).all(): + raise _refuse("READBACK_DEMOGRAPHIC_STATUS") + if not np.isin(aggregate, (0, 1)).all(): + raise _refuse("READBACK_AGGREGATE_FLAG") + if int(typed["S006"].min(initial=0)) < 0: + raise _refuse("READBACK_S006_NEGATIVE") + + present = status == 1 + for name in _MAIN_FIELDS: + tokens = lexical[name] + if len(tokens) != rows: + raise _refuse("READBACK_LEXICAL_ROWS", name) + width = widths[name] + values = typed[name] + for position, token in enumerate(tokens): + _token(token, width=width, label=name) + if int(token) != int(values[position]): + raise _refuse("READBACK_LEXICAL_DISAGREEMENT", name) + for name in _DEMOGRAPHIC_FIELDS: + tokens = lexical[name] + if len(tokens) != rows: + raise _refuse("READBACK_LEXICAL_ROWS", name) + width = widths[name] + values = typed[name] + for position, token in enumerate(tokens): + if not present[position]: + # Absence is the sentinel and an empty spelling, never a zero. + if token or int(values[position]) != _ABSENT_SENTINEL: + raise _refuse("READBACK_ABSENCE_NOT_EMPTY", name) + continue + _token(token, width=width, label=name) + if int(token) != int(values[position]): + raise _refuse("READBACK_LEXICAL_DISAGREEMENT", name) + + keys = lexical["RECID"] + if len(set(keys)) != rows: + raise _refuse("READBACK_DUPLICATE_LEXICAL_KEY") + recids = typed["RECID"] + if len(np.unique(recids)) != rows: + raise _refuse("READBACK_NUMERIC_KEY_COLLISION") + expected_aggregate = np.isin(recids, np.asarray(PUF_AGGREGATE_RECIDS, dtype=" PufRawReturnStatus: + """Decode an artifact payload and re-prove it, never trusting its header. + + The header is a closed schema: exact key sets, an exact dtype string for + every typed column, an exact width and body size for every lexical column, + and a total body bound. Every reported fact is then **recomputed** from the + arrays actually read back and must agree exactly, so a payload cannot + report a row count, a match count or a code frequency it does not carry. + + This stays structural. It is not source authority: it proves that a payload + is internally consistent and, when a definition is available, that it + matches that definition's widths, domains and source pins. It authorizes + nothing about where the delivery bytes came from — the codec does that. + """ + + if not isinstance(payload, bytes) or not payload.startswith(RETURN_STATUS_MAGIC): + raise _refuse("ENVELOPE_MAGIC") + start = len(RETURN_STATUS_MAGIC) + if len(payload) < start + 4: + raise _refuse("ENVELOPE_TRUNCATED") + length = int.from_bytes(payload[start : start + 4], "big") + if not 0 < length <= HEADER_MAX_BYTES or len(payload) < start + 4 + length: + raise _refuse("ENVELOPE_HEADER_LENGTH") + encoded = payload[start + 4 : start + 4 + length] + try: + header = json.loads(encoded.decode("ascii")) + except (UnicodeError, ValueError, RecursionError) as error: + # A deeply nested or malformed header is a refusal, not a stray + # ``RecursionError`` or ``UnicodeDecodeError`` out of the JSON reader. + raise _refuse("ENVELOPE_HEADER_JSON") from error + try: + recanonical = canonical_json(header) + except (TypeError, ValueError, RecursionError) as error: + raise _refuse("ENVELOPE_HEADER_JSON") from error + if recanonical != encoded: + raise _refuse("ENVELOPE_HEADER_NOT_CANONICAL") + _exact_keys(header, _ENVELOPE_HEADER_KEYS, "header") + if header["schema_version"] != 1 or header["type"] != [ + RETURN_STATUS_TYPE.name, + RETURN_STATUS_TYPE.schema_version, + ]: + raise _refuse("ENVELOPE_TYPE") + if ( + header["header_length_endianness"] != "big" + or header["body_endianness"] != "little" + ): + raise _refuse("ENVELOPE_ENDIANNESS") + rows = _envelope_int(header["rows"], "rows", minimum=1) + resolved = _resolve_envelope_definition(header, definition) + _check_envelope_sources(header, resolved) + columns, widths = _envelope_columns(header, rows, resolved) + + cursor = start + 4 + length + typed: dict[str, np.ndarray] = {} + lexical: dict[str, tuple[str, ...]] = {} + for column in columns: + name = column["name"] + size = column["bytes"] + body = payload[cursor : cursor + size] + if len(body) != size: + raise _refuse("ENVELOPE_BODY_TRUNCATED", name) + if hashlib.sha256(body).hexdigest() != column["sha256"]: + raise _refuse("ENVELOPE_BODY_DIGEST", name) + cursor += size + if column["kind"] == _TYPED_KIND: + typed[name] = np.frombuffer(body, dtype=np.dtype(column["dtype"])) + else: + field = name.removesuffix("_lexical") + lexical[field] = _read_fixed_ascii(body, rows, column["width"], field) + if cursor != len(payload): + raise _refuse("ENVELOPE_TRAILING_BYTES") + + _verify_readback(typed, lexical, widths, resolved) + facts = header["facts"] + _exact_keys(facts, _ENVELOPE_FACT_KEYS, "facts") + if resolved is not None: + join_method = _join_method(resolved) + untyped_main_columns = _untyped_main_columns(resolved) + else: + join_method = facts["join_method"] + if not isinstance(join_method, str) or not join_method: + raise _refuse("ENVELOPE_FACTS", "join_method") + untyped_main_columns = _envelope_int( + facts["untyped_main_columns"], "facts.untyped_main_columns" + ) + recomputed = _facts_from_arrays( + typed, + join_method=join_method, + untyped_main_columns=untyped_main_columns, + ) + if canonical_json(recomputed) != canonical_json(dict(facts)): + raise _refuse("ENVELOPE_FACTS_INCONSISTENT") + if recomputed["rows"] != rows: + raise _refuse("ENVELOPE_ROWS") + if resolved is not None: + _check_declared_expectations(recomputed, resolved) + return PufRawReturnStatus( + typed=MappingProxyType(typed), + lexical=MappingProxyType(lexical), + facts=MappingProxyType(recomputed), + ) + + +# -------------------------------------------------------------------------- +# The graph producer, consumer and registry +# -------------------------------------------------------------------------- + +_IMPLEMENTATION_MODULES = ("microcosm.build.us_runtime.puf_raw_source",) +_IMPLEMENTATION_IDENTITY_MAGIC = b"us.puf.raw_source/implementation/2\n" + + +class USPufRawSourceKernel(KernelBase): + """Read the two pinned deliveries and produce one typed status artifact.""" + + ref = "us.puf.raw_source.return_status@1" + capabilities = Capabilities( + Determinism.DETERMINISTIC, + numeric=Numeric.BITWISE, + dependencies=("numpy",), + ) + + def __init__( + self, + *, + definition: PufRawSourceDefinition | None = None, + source_codecs: SourceCodecRegistry | None = None, + ) -> None: + self.definition = packaged_definition() if definition is None else definition + if self.definition.route not in ("packaged", "test_fixture"): + raise _refuse("KERNEL_DEFINITION_ROUTE") + if self.definition.route == "packaged": + # The production wrapper, not the bare decoder, is the authority: + # the definition must be the packaged file's own canonical bytes. + # Compare the bytes, not only their digest, and re-prove that the + # document this definition carries is still those exact bytes. + packaged = packaged_definition() + if ( + self.definition.canonical != packaged.canonical + or self.definition.sha256 != packaged.sha256 + or canonical_json(_thawed(self.definition.document)) + != self.definition.canonical + ): + raise _refuse("KERNEL_DEFINITION_NOT_PACKAGED") + self.source_codecs = ( + puf_raw_source_codecs(self.definition) + if source_codecs is None + else source_codecs + ) + + def implementation_hash(self) -> str: + """Module source, dependency versions **and** the effective CSV profile. + + The acceptance profile is process state the parse runs under, so it + belongs in the producer's identity: change it and this node's key + moves, which is a recompute or a ``require`` miss. It can no longer + reuse an admission made under a different acceptance. + """ + + from importlib import import_module + + base = source_hash( + *(import_module(name) for name in _IMPLEMENTATION_MODULES), + dependencies=self.capabilities.dependencies, + ) + digest = hashlib.sha256(_IMPLEMENTATION_IDENTITY_MAGIC) + digest.update(base.encode("ascii")) + digest.update(b"\0") + digest.update(canonical_json(csv_acceptance_profile())) + return digest.hexdigest() + + def run(self, context: KernelContext) -> KernelResult: + definition = self.definition + if ( + context.node.kernel != self.ref + or context.node.structural is not StructuralDelta.NONE + or context.node.sources + != (definition.main.source_name, definition.demographic.source_name) + or context.node.inputs + or context.node.outputs + or context.node.artifact_inputs + or context.node.artifact_outputs + != (ArtifactOutput("return_status", RETURN_STATUS_TYPE),) + or set(context.params) != {"definition"} + ): + raise _refuse("NODE_DECLARATION") + if context.params["definition"] != definition.params_text: + raise _refuse("NODE_DEFINITION_PARAM") + main_bytes = load_source_bytes( + definition.main.codec, + context.sources[definition.main.source_name], + registry=self.source_codecs, + ) + demographic_bytes = load_source_bytes( + definition.demographic.codec, + context.sources[definition.demographic.source_name], + registry=self.source_codecs, + ) + status = decode_puf_raw_source(main_bytes, demographic_bytes, definition) + payload = encode_return_status(status, definition) + return KernelResult( + artifacts={"return_status": payload}, + receipt={ + "puf_raw_return_status": { + "definition_sha256": definition.sha256, + "definition_route": definition.route, + "artifact_sha256": hashlib.sha256(payload).hexdigest(), + "artifact_bytes": len(payload), + "main_sha256": definition.main.sha256, + "demographic_sha256": definition.demographic.sha256, + "csv_acceptance_profile": csv_acceptance_profile(), + **{ + key: value + for key, value in status.facts.items() + if key != "code_frequencies" + }, + } + }, + ) + + +class USPufRawReturnStatusSummaryKernel(KernelBase): + """A downstream consumer: decode the typed artifact and report counts.""" + + ref = "us.puf.raw_source.return_status_summary@1" + capabilities = Capabilities( + Determinism.DETERMINISTIC, + numeric=Numeric.BITWISE, + dependencies=("numpy",), + ) + + def implementation_hash(self) -> str: + from importlib import import_module + + return source_hash( + *(import_module(name) for name in _IMPLEMENTATION_MODULES), + dependencies=self.capabilities.dependencies, + ) + + def run(self, context: KernelContext) -> KernelResult: + if ( + context.node.kernel != self.ref + or context.node.structural is not StructuralDelta.NONE + or context.node.sources + or context.node.inputs + or context.node.outputs + or context.node.artifact_outputs + or len(context.node.artifact_inputs) != 1 + or context.node.artifact_inputs[0].type != RETURN_STATUS_TYPE + or context.params + ): + raise _refuse("SUMMARY_NODE_DECLARATION") + alias = context.node.artifact_inputs[0].name + value = context.artifacts[alias] + if value.type != RETURN_STATUS_TYPE: + raise _refuse("SUMMARY_ARTIFACT_TYPE") + status = decode_return_status(value.payload) + return KernelResult( + receipt={ + "puf_raw_return_status_summary": { + "producer_key": value.producer_key, + "rows": status.rows, + "matched_keys": status.facts["matched_keys"], + "main_unmatched_records": status.facts["main_unmatched_records"], + "aggregate_records": status.facts["aggregate_records"], + "period_semantics": status.facts["period_semantics"], + "weight_units": status.facts["weight_units"], + "amounts_decoded": status.facts["amounts_decoded"], + } + } + ) + + +def us_puf_raw_source_refs( + definition: PufRawSourceDefinition | None = None, +) -> tuple[SourceRef, ...]: + """The two ``SourceRef`` declarations this slice's node reads.""" + + resolved = packaged_definition() if definition is None else definition + return ( + SourceRef( + resolved.main.source_name, + resolved.main.codec, + description="2015 PUF main delivery, pinned by content.", + ), + SourceRef( + resolved.demographic.source_name, + resolved.demographic.codec, + description="2015 PUF demographic delivery, pinned by content.", + ), + ) + + +def us_puf_raw_source_node( + *, + population: str, + definition: PufRawSourceDefinition | None = None, + stage: str = PUF_RAW_SOURCE_STAGE, +) -> Node: + """Declare the producer node inside an explicitly named host population.""" + + if not isinstance(population, str) or not population: + raise _refuse("NODE_POPULATION_REQUIRED") + resolved = packaged_definition() if definition is None else definition + return Node( + f"{stage}.return_status", + USPufRawSourceKernel.ref, + population=population, + sources=(resolved.main.source_name, resolved.demographic.source_name), + params={"definition": resolved.params_text}, + artifact_outputs=(ArtifactOutput("return_status", RETURN_STATUS_TYPE),), + description=( + "Type the 2015 PUF return status slice from the pinned delivery " + "bytes. Owns no population columns." + ), + ) + + +def us_puf_raw_return_status_summary_node( + *, + population: str, + producer: str = PUF_RAW_SOURCE_NODE, + stage: str = PUF_RAW_SOURCE_STAGE, +) -> Node: + """Declare an ordinary consumer of the typed artifact.""" + + if not isinstance(population, str) or not population: + raise _refuse("NODE_POPULATION_REQUIRED") + return Node( + f"{stage}.return_status_summary", + USPufRawReturnStatusSummaryKernel.ref, + population=population, + artifact_inputs=( + ArtifactInput( + "return_status", producer, "return_status", RETURN_STATUS_TYPE + ), + ), + description="Read the typed PUF status artifact and report its counts.", + ) + + +def register_us_puf_raw_source_kernels( + registry: KernelRegistry, + *, + definition: PufRawSourceDefinition | None = None, + source_codecs: SourceCodecRegistry | None = None, +) -> KernelRegistry: + """Register the producer and consumer kernels into ``registry``.""" + + registry.register( + USPufRawSourceKernel(definition=definition, source_codecs=source_codecs) + ) + registry.register(USPufRawReturnStatusSummaryKernel()) + return registry diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/puf_support.py b/packages/microcosm-build/src/microcosm/build/us_runtime/puf_support.py index d78c70a23..77178b006 100644 --- a/packages/microcosm-build/src/microcosm/build/us_runtime/puf_support.py +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/puf_support.py @@ -29,13 +29,9 @@ puf_capital_gains_joint_metrics, puf_processed_capital_gains_stage, ) -from microcosm.build.us_runtime.puf_interest_components import ( - split_us_puf_e19200_by_agi_band, -) from microcosm.build.us_runtime.qbi_inputs import ( US_QBI_BOOLEAN_OUTPUT_COLUMNS, US_QBI_NONNEGATIVE_OUTPUT_COLUMNS, - US_QBI_OUTPUT_COLUMNS, ) from microcosm.build.us_runtime.support_provenance import ( BASE_ASEC_SUPPORT_CHANNEL, @@ -54,6 +50,19 @@ from microcosm.frame import US_SCHEMA, Frame, WeightKind, Weights, wquantile from microcosm.frame.schema import EntitySchema +from .operator_column_contracts import ( + PUF_SUPPORT_MAX_CLONE_SAFE_SOURCE_ID as PUF_SUPPORT_MAX_CLONE_SAFE_SOURCE_ID, +) +from .operator_column_contracts import ( + PUF_TAX_DETAIL_DEFAULT_PERSON_OUTPUTS as PUF_TAX_DETAIL_DEFAULT_PERSON_OUTPUTS, +) +from .operator_column_contracts import ( + PUF_TAX_DETAIL_DEFAULT_TAX_UNIT_OUTPUTS as PUF_TAX_DETAIL_DEFAULT_TAX_UNIT_OUTPUTS, +) +from .operator_column_contracts import ( + US_PUF_SUPPORT_STAGE_NAME as US_PUF_SUPPORT_STAGE_NAME, +) + QRF: Any | None = None __all__ = [ @@ -91,7 +100,6 @@ "validate_puf_clone_attachment", ] -US_PUF_SUPPORT_STAGE_NAME = "puf_support_channel" #: Frozen receipt binding a seeded clone attachment (microcosm#578 revision #: item 3) to the live rows: fraction, seed, the floor-rule counts, and the @@ -207,54 +215,6 @@ class _PredictorSourcePlan: "puf_predictor_long_term_capital_gains", ) -PUF_TAX_DETAIL_DEFAULT_PERSON_OUTPUTS = ( - "employment_income_before_lsr", - "self_employment_income_before_lsr", - "taxable_interest_income", - "qualified_dividend_income", - "non_qualified_dividend_income", - "tax_exempt_interest_income", - "short_term_capital_gains", - "long_term_capital_gains_before_response", - "long_term_capital_gains_on_collectibles", - "non_sch_d_capital_gains", - "taxable_private_pension_income", - "taxable_ira_distributions", - "social_security_retirement", - "social_security_disability", - "social_security_dependents", - "social_security_survivors", - "alimony_income", - "alimony_expense", - "salt_refund_income", - "charitable_cash_donations", - "charitable_non_cash_donations", - "real_estate_taxes", - "home_mortgage_interest", - "investment_interest_expense", - "investment_income_elected_form_4952", - "student_loan_interest", - "educator_expense", - "qualified_tuition_expenses", - "casualty_loss", - "unreimbursed_business_employee_expenses", - # The engine owns the realized contribution amounts through the - # IRA-limit scale and self-employment caps; the persistable leaves are - # the desired contributions, equal to the PUF's observed deductions at - # baseline (issue #278). - "traditional_ira_contributions_desired", - "self_employed_pension_contributions_desired", - "rental_income", - "estate_income", - "farm_income", - "farm_operations_income", - "farm_rent_income", - "miscellaneous_income", - "partnership_income", - "s_corp_income", - "partnership_self_employment_net_earnings", - *US_QBI_OUTPUT_COLUMNS, -) PUF_TAX_DETAIL_SOCIAL_SECURITY_COMPONENT_OUTPUTS = ( "social_security_retirement", @@ -263,17 +223,6 @@ class _PredictorSourcePlan: "social_security_survivors", ) -PUF_TAX_DETAIL_DEFAULT_TAX_UNIT_OUTPUTS: tuple[str, ...] = ( - "domestic_production_ald", - "unrecaptured_section_1250_gain", - "first_home_mortgage_balance", - "second_home_mortgage_balance", - "first_home_mortgage_interest", - "second_home_mortgage_interest", - "first_home_mortgage_origination_year", - "second_home_mortgage_origination_year", - "health_savings_account_ald", -) _PUF_TAX_DETAIL_DISCRETE_TAX_UNIT_OUTPUTS = frozenset( { @@ -1505,6 +1454,16 @@ def _quarantine_us_puf_mortgage_fields( donor_build_summary["mortgage_field_quarantine"] = quarantine +def split_us_puf_e19200_by_agi_band( + total_interest_paid: Any, + adjusted_gross_income: Any, +) -> tuple[np.ndarray, np.ndarray]: + """Load the published component resource only for actual decomposition.""" + from .puf_interest_components import split_us_puf_e19200_by_agi_band as split + + return split(total_interest_paid, adjusted_gross_income) + + def _split_us_puf_e19200_components(donor: pd.DataFrame) -> None: """Split raw E19200 into mortgage and modeled non-mortgage components.""" @@ -2368,7 +2327,6 @@ def _id_multiplier_for_frame(frame: Frame) -> int: # indices up to 921 before int64 overflow — orders beyond any configured # clone count. Assembly enforces this bound; _remap_ids re-checks it so a # violation is a governed ValueError, never an OverflowError. -PUF_SUPPORT_MAX_CLONE_SAFE_SOURCE_ID = 10**15 - 1 _INT64_MAX = 2**63 - 1 @@ -2563,7 +2521,7 @@ def _formula_owned_engine() -> Any | None: except ImportError: return None try: - return PolicyEngineUSVariableMetadataIndex() + return PolicyEngineUSVariableMetadataIndex(include_consumers=False) except ImportError: return None diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/puf_target2024_growth.py b/packages/microcosm-build/src/microcosm/build/us_runtime/puf_target2024_growth.py new file mode 100644 index 000000000..f68c0e1e0 --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/puf_target2024_growth.py @@ -0,0 +1,164 @@ +"""Source-specific, explicit PUF2015 to target2024 monetary transport. + +This is a data model. It computes no tax, deduction limit, benefit eligibility +or recipient membership. Receipt knownness is separate from raw observation. +""" + +from __future__ import annotations + +import hashlib +import json +from collections.abc import Mapping +from dataclasses import dataclass +from types import MappingProxyType + +import numpy as np + +# Generated below from the reviewable, source-pinned GROWTH-RECIPE.json. +_RECIPE_JSON = '{\n "schema": "microcosm.us.puf2015_target2024_growth/1",\n "input_money_year": 2015,\n "output_money_year": 2024,\n "output_profile": "puf59_scf_mortgage_v1",\n "ordered_targets": [\n "employment_income_before_lsr",\n "self_employment_income_before_lsr",\n "taxable_interest_income",\n "qualified_dividend_income",\n "non_qualified_dividend_income",\n "tax_exempt_interest_income",\n "short_term_capital_gains",\n "long_term_capital_gains_before_response",\n "long_term_capital_gains_on_collectibles",\n "non_sch_d_capital_gains",\n "taxable_private_pension_income",\n "taxable_ira_distributions",\n "social_security_retirement",\n "social_security_disability",\n "social_security_dependents",\n "social_security_survivors",\n "alimony_income",\n "alimony_expense",\n "salt_refund_income",\n "charitable_cash_donations",\n "charitable_non_cash_donations",\n "real_estate_taxes",\n "home_mortgage_interest",\n "investment_interest_expense",\n "investment_income_elected_form_4952",\n "student_loan_interest",\n "educator_expense",\n "qualified_tuition_expenses",\n "casualty_loss",\n "unreimbursed_business_employee_expenses",\n "traditional_ira_contributions_desired",\n "self_employed_pension_contributions_desired",\n "rental_income",\n "estate_income",\n "farm_income",\n "farm_operations_income",\n "farm_rent_income",\n "miscellaneous_income",\n "partnership_income",\n "s_corp_income",\n "partnership_self_employment_net_earnings",\n "estate_income_would_be_qualified",\n "farm_operations_income_would_be_qualified",\n "farm_rent_income_would_be_qualified",\n "partnership_s_corp_income_would_be_qualified",\n "rental_income_would_be_qualified",\n "self_employment_income_would_be_qualified",\n "sstb_self_employment_income_would_be_qualified",\n "business_is_sstb",\n "qualified_bdc_income",\n "qualified_reit_and_ptp_income",\n "sstb_self_employment_income_before_lsr",\n "sstb_unadjusted_basis_qualified_property",\n "sstb_w2_wages_from_qualified_business",\n "unadjusted_basis_qualified_property",\n "w2_wages_from_qualified_business",\n "domestic_production_ald",\n "unrecaptured_section_1250_gain",\n "health_savings_account_ald"\n ],\n "money_fields": {\n "employment_income_before_lsr": {\n "positive_family": "awi",\n "negative_family": "awi",\n "positive_factor": "1.452153003110483604210764423019948801037",\n "negative_factor": "1.452153003110483604210764423019948801037",\n "assumption": "Average-worker wage index proxy; modeled QBI W2 amounts follow the same wage index."\n },\n "w2_wages_from_qualified_business": {\n "positive_family": "awi",\n "negative_family": "awi",\n "positive_factor": "1.452153003110483604210764423019948801037",\n "negative_factor": "1.452153003110483604210764423019948801037",\n "assumption": "Average-worker wage index proxy; modeled QBI W2 amounts follow the same wage index."\n },\n "sstb_w2_wages_from_qualified_business": {\n "positive_family": "awi",\n "negative_family": "awi",\n "positive_factor": "1.452153003110483604210764423019948801037",\n "negative_factor": "1.452153003110483604210764423019948801037",\n "assumption": "Average-worker wage index proxy; modeled QBI W2 amounts follow the same wage index."\n },\n "self_employment_income_before_lsr": {\n "positive_family": "business_profit",\n "negative_family": "business_loss",\n "positive_factor": "1.222517420922031096142334753675412822692",\n "negative_factor": "1.718898116890934040328182671660914217539",\n "assumption": "Both modeled Schedule C branches use the same sign-specific family factors, preserving the base/SSTB partition."\n },\n "sstb_self_employment_income_before_lsr": {\n "positive_family": "business_profit",\n "negative_family": "business_loss",\n "positive_factor": "1.222517420922031096142334753675412822692",\n "negative_factor": "1.718898116890934040328182671660914217539",\n "assumption": "Both modeled Schedule C branches use the same sign-specific family factors, preserving the base/SSTB partition."\n },\n "taxable_interest_income": {\n "positive_family": "taxable_interest",\n "negative_family": "taxable_interest",\n "positive_factor": "2.599750625636417399355437523686408804619",\n "negative_factor": "2.599750625636417399355437523686408804619",\n "assumption": "Reporting-return mean transport; 2023\\u21922024 stable-real-amount bridge; no unchanged reporting incidence claim."\n },\n "tax_exempt_interest_income": {\n "positive_family": "tax_exempt_interest",\n "negative_family": "tax_exempt_interest",\n "positive_factor": "0.9295395824924337172788943063597248917385",\n "negative_factor": "0.9295395824924337172788943063597248917385",\n "assumption": "Reporting-return mean transport; 2023\\u21922024 stable-real-amount bridge; no unchanged reporting incidence claim."\n },\n "qualified_dividend_income": {\n "positive_family": "qualified_dividends",\n "negative_family": "qualified_dividends",\n "positive_factor": "1.409891045424454325551259680558160347719",\n "negative_factor": "1.409891045424454325551259680558160347719",\n "assumption": "Reporting-return mean transport; 2023\\u21922024 stable-real-amount bridge; no unchanged reporting incidence claim."\n },\n "non_qualified_dividend_income": {\n "positive_family": "nonqualified_dividends",\n "negative_family": "nonqualified_dividends",\n "positive_factor": "2.483795459689869824933006427197633652403",\n "negative_factor": "2.483795459689869824933006427197633652403",\n "assumption": "Reporting-return mean transport; 2023\\u21922024 stable-real-amount bridge; no unchanged reporting incidence claim."\n },\n "non_sch_d_capital_gains": {\n "positive_family": "non_schedule_d_distributions",\n "negative_family": "non_schedule_d_distributions",\n "positive_factor": "1.120350061633613230084419890940102481805",\n "negative_factor": "1.120350061633613230084419890940102481805",\n "assumption": "Reporting-return mean transport; 2023\\u21922024 stable-real-amount bridge; no unchanged reporting incidence claim."\n },\n "taxable_private_pension_income": {\n "positive_family": "taxable_pensions",\n "negative_family": "taxable_pensions",\n "positive_factor": "1.327588034973940867230285994253866499704",\n "negative_factor": "1.327588034973940867230285994253866499704",\n "assumption": "Reporting-return mean transport; 2023\\u21922024 stable-real-amount bridge; no unchanged reporting incidence claim."\n },\n "taxable_ira_distributions": {\n "positive_family": "ira_distributions",\n "negative_family": "ira_distributions",\n "positive_factor": "1.510871073557979130794209158828995676924",\n "negative_factor": "1.510871073557979130794209158828995676924",\n "assumption": "Reporting-return mean transport; 2023\\u21922024 stable-real-amount bridge; no unchanged reporting incidence claim."\n },\n "alimony_income": {\n "positive_family": "alimony_income",\n "negative_family": "alimony_income",\n "positive_factor": "1.542033706590010885068081742929926580179",\n "negative_factor": "1.542033706590010885068081742929926580179",\n "assumption": "Reporting-return mean transport; 2023\\u21922024 stable-real-amount bridge; no unchanged reporting incidence claim."\n },\n "salt_refund_income": {\n "positive_family": "state_tax_refund",\n "negative_family": "state_tax_refund",\n "positive_factor": "0.9296526775720357466732616002244135476521",\n "negative_factor": "0.9296526775720357466732616002244135476521",\n "assumption": "Reporting-return mean transport; 2023\\u21922024 stable-real-amount bridge; no unchanged reporting incidence claim."\n },\n "short_term_capital_gains": {\n "positive_family": "short_current_gain",\n "negative_family": "short_current_loss",\n "positive_factor": "2.042865587775988756504710527637911842775",\n "negative_factor": "1.738139193325432816803529458382722438642",\n "assumption": "Current-year gross gains/loss components per all return proxy for signed net PUF amount; carried losses are excluded."\n },\n "long_term_capital_gains_before_response": {\n "positive_family": "long_current_gain",\n "negative_family": "long_current_loss",\n "positive_factor": "1.283710381837894882785950440755768785471",\n "negative_factor": "2.262384673681395979556694950087192296750",\n "assumption": "Current-year gross long-term components proxy for signed net amounts and narrower collectibles/1250 subfamilies; carried losses are excluded."\n },\n "long_term_capital_gains_on_collectibles": {\n "positive_family": "long_current_gain",\n "negative_family": "long_current_loss",\n "positive_factor": "1.283710381837894882785950440755768785471",\n "negative_factor": "2.262384673681395979556694950087192296750",\n "assumption": "Current-year gross long-term components proxy for signed net amounts and narrower collectibles/1250 subfamilies; carried losses are excluded."\n },\n "unrecaptured_section_1250_gain": {\n "positive_family": "long_current_gain",\n "negative_family": "long_current_loss",\n "positive_factor": "1.283710381837894882785950440755768785471",\n "negative_factor": "2.262384673681395979556694950087192296750",\n "assumption": "Current-year gross long-term components proxy for signed net amounts and narrower collectibles/1250 subfamilies; carried losses are excluded."\n },\n "social_security_retirement": {\n "positive_family": "cola",\n "negative_family": "cola",\n "positive_factor": "1.285886314567344447728640",\n "negative_factor": "1.285886314567344447728640",\n "assumption": "Cash-year COLA transport of fixed entitlement; carrier component convention remains modeled upstream."\n },\n "social_security_disability": {\n "positive_family": "cola",\n "negative_family": "cola",\n "positive_factor": "1.285886314567344447728640",\n "negative_factor": "1.285886314567344447728640",\n "assumption": "Cash-year COLA transport of fixed entitlement; carrier component convention remains modeled upstream."\n },\n "social_security_dependents": {\n "positive_family": "cola",\n "negative_family": "cola",\n "positive_factor": "1.285886314567344447728640",\n "negative_factor": "1.285886314567344447728640",\n "assumption": "Cash-year COLA transport of fixed entitlement; carrier component convention remains modeled upstream."\n },\n "social_security_survivors": {\n "positive_family": "cola",\n "negative_family": "cola",\n "positive_factor": "1.285886314567344447728640",\n "negative_factor": "1.285886314567344447728640",\n "assumption": "Cash-year COLA transport of fixed entitlement; carrier component convention remains modeled upstream."\n },\n "rental_income": {\n "positive_family": "rental_royalty_profit",\n "negative_family": "rental_royalty_loss",\n "positive_factor": "1.476663829685885394438774259073237903799",\n "negative_factor": "1.690793641559507564235995133214906035185",\n "assumption": "Related rental/royalty reporting-return mean proxy; published total family includes farm rent and does not exactly equal E25850/E25860 component definitions."\n },\n "estate_income": {\n "positive_family": "estate_profit",\n "negative_family": "estate_loss",\n "positive_factor": "1.476331638358104936021886589313386353228",\n "negative_factor": "1.582810027948011789708036602150526302352",\n "assumption": ""\n },\n "farm_operations_income": {\n "positive_family": "farm_profit",\n "negative_family": "farm_loss",\n "positive_factor": "1.444720208215047581256304812994467826979",\n "negative_factor": "1.728383897337776476443532509645504817056",\n "assumption": "Schedule F family; elected farm income T27800 is a selected subfamily proxy, not general farm income."\n },\n "farm_income": {\n "positive_family": "farm_profit",\n "negative_family": "farm_loss",\n "positive_factor": "1.444720208215047581256304812994467826979",\n "negative_factor": "1.728383897337776476443532509645504817056",\n "assumption": "Schedule F family; elected farm income T27800 is a selected subfamily proxy, not general farm income."\n },\n "farm_rent_income": {\n "positive_family": "farm_rent_profit",\n "negative_family": "farm_rent_loss",\n "positive_factor": "1.629452810459537374541656575493532799084",\n "negative_factor": "1.150378978298943195689490419607276484593",\n "assumption": ""\n },\n "miscellaneous_income": {\n "positive_family": "other_property_gain",\n "negative_family": "other_property_loss",\n "positive_factor": "1.738242164922618767195190248113271033066",\n "negative_factor": "1.411645744283618865226902643798237455207",\n "assumption": "Canonical source E01200 is other property gains/losses, not Table1.4 other income."\n },\n "partnership_income": {\n "positive_family": "partnership_s_corp_profit",\n "negative_family": "partnership_s_corp_loss",\n "positive_factor": "1.595257264233857504616939193376066035754",\n "negative_factor": "1.968942227400735507640732147170530268700",\n "assumption": "Combined component dollars per all return; reporting counts overlap and are never summed. Active partnership earnings use this same explicitly related-family proxy."\n },\n "s_corp_income": {\n "positive_family": "partnership_s_corp_profit",\n "negative_family": "partnership_s_corp_loss",\n "positive_factor": "1.595257264233857504616939193376066035754",\n "negative_factor": "1.968942227400735507640732147170530268700",\n "assumption": "Combined component dollars per all return; reporting counts overlap and are never summed. Active partnership earnings use this same explicitly related-family proxy."\n },\n "partnership_self_employment_net_earnings": {\n "positive_family": "partnership_s_corp_profit",\n "negative_family": "partnership_s_corp_loss",\n "positive_factor": "1.595257264233857504616939193376066035754",\n "negative_factor": "1.968942227400735507640732147170530268700",\n "assumption": "Combined component dollars per all return; reporting counts overlap and are never summed. Active partnership earnings use this same explicitly related-family proxy."\n },\n "qualified_bdc_income": {\n "positive_family": "nonqualified_dividends",\n "negative_family": "nonqualified_dividends",\n "positive_factor": "2.483795459689869824933006427197633652403",\n "negative_factor": "2.483795459689869824933006427197633652403",\n "assumption": "Modeled BDC component follows its parent nonqualified dividend pool; no independent source observation."\n },\n "qualified_reit_and_ptp_income": {\n "positive_family": "cpi",\n "negative_family": "cpi",\n "positive_factor": "1.323487344789614247079323424058189918867",\n "negative_factor": "1.323487344789614247079323424058189918867",\n "assumption": "Explicit constant-real proxy for the combined modeled REIT/PTP output, which mixes nonqualified-dividend and pass-through components. No claim that this is a pure dividend pool."\n },\n "alimony_expense": {\n "positive_family": "cpi",\n "negative_family": "cpi",\n "positive_factor": "1.323487344789614247079323424058189918867",\n "negative_factor": "1.323487344789614247079323424058189918867",\n "assumption": "Explicit constant-real-amount proxy. Realized deductions/desires and UBIA are upstream models; this is not a statutory limit, current eligibility, loan turnover, interest-rate, or asset-price model."\n },\n "charitable_cash_donations": {\n "positive_family": "cpi",\n "negative_family": "cpi",\n "positive_factor": "1.323487344789614247079323424058189918867",\n "negative_factor": "1.323487344789614247079323424058189918867",\n "assumption": "Explicit constant-real-amount proxy. Realized deductions/desires and UBIA are upstream models; this is not a statutory limit, current eligibility, loan turnover, interest-rate, or asset-price model."\n },\n "charitable_non_cash_donations": {\n "positive_family": "cpi",\n "negative_family": "cpi",\n "positive_factor": "1.323487344789614247079323424058189918867",\n "negative_factor": "1.323487344789614247079323424058189918867",\n "assumption": "Explicit constant-real-amount proxy. Realized deductions/desires and UBIA are upstream models; this is not a statutory limit, current eligibility, loan turnover, interest-rate, or asset-price model."\n },\n "real_estate_taxes": {\n "positive_family": "cpi",\n "negative_family": "cpi",\n "positive_factor": "1.323487344789614247079323424058189918867",\n "negative_factor": "1.323487344789614247079323424058189918867",\n "assumption": "Explicit constant-real-amount proxy. Realized deductions/desires and UBIA are upstream models; this is not a statutory limit, current eligibility, loan turnover, interest-rate, or asset-price model."\n },\n "home_mortgage_interest": {\n "positive_family": "cpi",\n "negative_family": "cpi",\n "positive_factor": "1.323487344789614247079323424058189918867",\n "negative_factor": "1.323487344789614247079323424058189918867",\n "assumption": "Explicit constant-real-amount proxy. Realized deductions/desires and UBIA are upstream models; this is not a statutory limit, current eligibility, loan turnover, interest-rate, or asset-price model."\n },\n "investment_interest_expense": {\n "positive_family": "cpi",\n "negative_family": "cpi",\n "positive_factor": "1.323487344789614247079323424058189918867",\n "negative_factor": "1.323487344789614247079323424058189918867",\n "assumption": "Explicit constant-real-amount proxy. Realized deductions/desires and UBIA are upstream models; this is not a statutory limit, current eligibility, loan turnover, interest-rate, or asset-price model."\n },\n "investment_income_elected_form_4952": {\n "positive_family": "cpi",\n "negative_family": "cpi",\n "positive_factor": "1.323487344789614247079323424058189918867",\n "negative_factor": "1.323487344789614247079323424058189918867",\n "assumption": "Explicit constant-real-amount proxy. Realized deductions/desires and UBIA are upstream models; this is not a statutory limit, current eligibility, loan turnover, interest-rate, or asset-price model."\n },\n "student_loan_interest": {\n "positive_family": "cpi",\n "negative_family": "cpi",\n "positive_factor": "1.323487344789614247079323424058189918867",\n "negative_factor": "1.323487344789614247079323424058189918867",\n "assumption": "Explicit constant-real-amount proxy. Realized deductions/desires and UBIA are upstream models; this is not a statutory limit, current eligibility, loan turnover, interest-rate, or asset-price model."\n },\n "educator_expense": {\n "positive_family": "cpi",\n "negative_family": "cpi",\n "positive_factor": "1.323487344789614247079323424058189918867",\n "negative_factor": "1.323487344789614247079323424058189918867",\n "assumption": "Explicit constant-real-amount proxy. Realized deductions/desires and UBIA are upstream models; this is not a statutory limit, current eligibility, loan turnover, interest-rate, or asset-price model."\n },\n "qualified_tuition_expenses": {\n "positive_family": "cpi",\n "negative_family": "cpi",\n "positive_factor": "1.323487344789614247079323424058189918867",\n "negative_factor": "1.323487344789614247079323424058189918867",\n "assumption": "Explicit constant-real-amount proxy. Realized deductions/desires and UBIA are upstream models; this is not a statutory limit, current eligibility, loan turnover, interest-rate, or asset-price model."\n },\n "casualty_loss": {\n "positive_family": "cpi",\n "negative_family": "cpi",\n "positive_factor": "1.323487344789614247079323424058189918867",\n "negative_factor": "1.323487344789614247079323424058189918867",\n "assumption": "Explicit constant-real-amount proxy. Realized deductions/desires and UBIA are upstream models; this is not a statutory limit, current eligibility, loan turnover, interest-rate, or asset-price model."\n },\n "unreimbursed_business_employee_expenses": {\n "positive_family": "cpi",\n "negative_family": "cpi",\n "positive_factor": "1.323487344789614247079323424058189918867",\n "negative_factor": "1.323487344789614247079323424058189918867",\n "assumption": "Explicit constant-real-amount proxy. Realized deductions/desires and UBIA are upstream models; this is not a statutory limit, current eligibility, loan turnover, interest-rate, or asset-price model."\n },\n "traditional_ira_contributions_desired": {\n "positive_family": "cpi",\n "negative_family": "cpi",\n "positive_factor": "1.323487344789614247079323424058189918867",\n "negative_factor": "1.323487344789614247079323424058189918867",\n "assumption": "Explicit constant-real-amount proxy. Realized deductions/desires and UBIA are upstream models; this is not a statutory limit, current eligibility, loan turnover, interest-rate, or asset-price model."\n },\n "self_employed_pension_contributions_desired": {\n "positive_family": "cpi",\n "negative_family": "cpi",\n "positive_factor": "1.323487344789614247079323424058189918867",\n "negative_factor": "1.323487344789614247079323424058189918867",\n "assumption": "Explicit constant-real-amount proxy. Realized deductions/desires and UBIA are upstream models; this is not a statutory limit, current eligibility, loan turnover, interest-rate, or asset-price model."\n },\n "domestic_production_ald": {\n "positive_family": "cpi",\n "negative_family": "cpi",\n "positive_factor": "1.323487344789614247079323424058189918867",\n "negative_factor": "1.323487344789614247079323424058189918867",\n "assumption": "Explicit constant-real-amount proxy. Realized deductions/desires and UBIA are upstream models; this is not a statutory limit, current eligibility, loan turnover, interest-rate, or asset-price model."\n },\n "health_savings_account_ald": {\n "positive_family": "cpi",\n "negative_family": "cpi",\n "positive_factor": "1.323487344789614247079323424058189918867",\n "negative_factor": "1.323487344789614247079323424058189918867",\n "assumption": "Explicit constant-real-amount proxy. Realized deductions/desires and UBIA are upstream models; this is not a statutory limit, current eligibility, loan turnover, interest-rate, or asset-price model."\n },\n "unadjusted_basis_qualified_property": {\n "positive_family": "cpi",\n "negative_family": "cpi",\n "positive_factor": "1.323487344789614247079323424058189918867",\n "negative_factor": "1.323487344789614247079323424058189918867",\n "assumption": "Explicit constant-real-amount proxy. Realized deductions/desires and UBIA are upstream models; this is not a statutory limit, current eligibility, loan turnover, interest-rate, or asset-price model."\n },\n "sstb_unadjusted_basis_qualified_property": {\n "positive_family": "cpi",\n "negative_family": "cpi",\n "positive_factor": "1.323487344789614247079323424058189918867",\n "negative_factor": "1.323487344789614247079323424058189918867",\n "assumption": "Explicit constant-real-amount proxy. Realized deductions/desires and UBIA are upstream models; this is not a statutory limit, current eligibility, loan turnover, interest-rate, or asset-price model."\n }\n },\n "return_incidence_fields": [\n "business_is_sstb",\n "estate_income_would_be_qualified",\n "farm_operations_income_would_be_qualified",\n "farm_rent_income_would_be_qualified",\n "partnership_s_corp_income_would_be_qualified",\n "rental_income_would_be_qualified",\n "self_employment_income_would_be_qualified",\n "sstb_self_employment_income_would_be_qualified"\n ],\n "source_pins": {\n "NATIONAL-GROWTH-EXTRACT.json": "775d89d3fe7a3b1eb9a09c50f6077d41bf838cdb80d9e4aaa9f0ab80ada4e0c8",\n "INDEX-VALUES.json": "221f3786a462fe105d3da15df3624de5ad3e3f921b4d0205e096067685fd59b3",\n "extract_national.py": "b6c3a8e4fea52653c01d801d6efe9620cdba4a489d6d7ae8c95cf00ee6dfbe1b",\n "ACQUISITION.json": "e9a2c34c2ca06a229b2a790ed2987f25d31996653490f342b4b54af37d5f098c"\n },\n "sensitivity_cpi_factor": "1.323487344789614247079323424058189918867",\n "provenance": "All growth applications are modeled distribution transport from observed national series; source observed/derived/modeled flags remain separate.",\n "operation_order": "Decode source statistical2015 \\u2192 source canonical transformations \\u2192 QBI2015 model (all16 outputs) \\u2192 this growth transform once \\u2192 2024 canonical donor. Monetary recipient predictors remain 2024.",\n "noninterference": "RECID, S006, raw FLPDYR/FLPDMO, raw money, count source fields and actual survey membership are separate inputs and are not accepted or mutated by this transform.",\n "sensitivities": "cpi_only uses exactly the same source cohort, zeros, signs and return incidence; no new policy evaluation or calibration authority."\n}\n' +RECIPE_SHA256 = "f08bcc3c37434730bccbf81643c96089383d8e943c60f76ac8337f57a163e1f4" +if hashlib.sha256(_RECIPE_JSON.encode("utf-8")).hexdigest() != RECIPE_SHA256: + raise ValueError("PUF_GROWTH_RECIPE_IDENTITY") +VERSION = "microcosm.us.puf2015_target2024_growth/1" + + +def growth_recipe(): + return json.loads(_RECIPE_JSON) + + +_RECIPE = growth_recipe() +OUTPUTS = tuple(_RECIPE["ordered_targets"]) +INCIDENCE_FIELDS = tuple(_RECIPE["return_incidence_fields"]) + + +def _require(value, code): + if not value: + raise ValueError(code) + + +def _digest(columns): + h = hashlib.sha256() + for name in OUTPUTS: + value = columns[name] + h.update(name.encode("ascii") + b"\0") + h.update(np.asarray(value, dtype=" 0, "PUF_GROWTH_NONEMPTY_VECTOR") + n = len(first) + clean = {} + for name in OUTPUTS: + mask = np.asarray(known[name]) + _require( + mask.shape == (n,) and mask.dtype.kind == "b", + "PUF_GROWTH_KNOWNNESS_TYPE:" + name, + ) + _require(bool(mask.all()), "PUF_GROWTH_UNKNOWN:" + name) + value = np.asarray(canonical[name]) + allowed = "biu" if name in INCIDENCE_FIELDS else "ifu" + _require( + value.shape == (n,) and value.dtype.kind in allowed, + "PUF_GROWTH_PHYSICAL_TYPE:" + name, + ) + if value.dtype.kind in "iu": + _require( + bool(((value >= -(2**53)) & (value <= 2**53)).all()), + "PUF_GROWTH_INTEGER_PRECISION:" + name, + ) + _require(bool(np.isfinite(value).all()), "PUF_GROWTH_NONFINITE:" + name) + if name in INCIDENCE_FIELDS: + _require( + bool(((value == 0) | (value == 1)).all()), + "PUF_GROWTH_INCIDENCE_DOMAIN:" + name, + ) + clean[name] = value.astype(np.int64) + else: + clean[name] = value.astype(np.float64) + outputs = {} + for name, value in clean.items(): + if name in INCIDENCE_FIELDS: + grown = value.copy() + else: + rule = _RECIPE["money_fields"][name] + positive = float(rule["positive_factor"]) + negative = float(rule["negative_factor"]) + if scheme == "cpi_only": + positive = negative = float(_RECIPE["sensitivity_cpi_factor"]) + _require( + np.isfinite(positive) + and np.isfinite(negative) + and min(positive, negative) > 0, + "PUF_GROWTH_FACTOR", + ) + with np.errstate(over="ignore", invalid="ignore"): + grown = value * np.where(value < 0, negative, positive) + _require( + bool(np.isfinite(grown).all()), "PUF_GROWTH_OUTPUT_NONFINITE:" + name + ) + _require( + bool(np.array_equal(value == 0, grown == 0)) + and bool(np.array_equal(np.sign(value), np.sign(grown))), + "PUF_GROWTH_SIGN_OR_ZERO:" + name, + ) + outputs[name] = np.frombuffer(grown.tobytes(), dtype=grown.dtype) + receipt = { + "version": VERSION, + "recipe_sha256": RECIPE_SHA256, + "scheme": scheme, + "input_money_year": 2015, + "output_money_year": 2024, + "money_unit": "nominal USD", + "source_statistical_year": 2015, + "rows": n, + "origin": "modeled_distribution_transport", + "input_knownness": "complete_canonical_after_observed_derived_or_modeled_owner", + "source_pins": _RECIPE["source_pins"], + "monetary_columns": 51, + "unchanged_return_incidence_columns": 8, + "output_profile": _RECIPE["output_profile"], + "input_values_sha256": _digest(clean), + "output_values_sha256": _digest(outputs), + "no_weight_growth": True, + "no_extra_raw_row_year_cpi": True, + "release_eligible": False, + } + receipt["sha256"] = hashlib.sha256( + json.dumps( + receipt, sort_keys=True, separators=(",", ":"), allow_nan=False + ).encode() + ).hexdigest() + return PufGrowthResult(MappingProxyType(outputs), 2024, MappingProxyType(receipt)) diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/puma_ladder.py b/packages/microcosm-build/src/microcosm/build/us_runtime/puma_ladder.py index bcbe95cdf..815081c95 100644 --- a/packages/microcosm-build/src/microcosm/build/us_runtime/puma_ladder.py +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/puma_ladder.py @@ -4,25 +4,17 @@ Microdata Area — the finest geography the ACS PUMS publishes. A household that already knows its PUMA (an ACS-spine record) keeps it; a household that knows only its state (an ASEC-spine record) is assigned a PUMA within that state, -sampled proportional to 2020 PUMA population. Every coarser layer of the -ladder then derives from the PUMA by a population-weighted draw over the -PUMA's overlap with that layer: - -- **congressional district (119th)** — sampled within the PUMA proportional to - the ``(PUMA, CD)`` block-population overlap (a 2020 PUMA can span several - districts and a district several PUMAs; they do not nest). -- **county** — sampled within the PUMA proportional to the ``(PUMA, county)`` - block-population overlap. -- **tract** (behind ``assign_tract``) — sampled within the PUMA proportional to - the ``(PUMA, tract)`` block-population overlap; when tract is assigned the - county derives structurally from it (``county = tract // 10**6``) so the two - never disagree. Congressional district, county and state are the launch - requirement; tract is the stretch rung. - -One national dataset, filter by geography, at any grain (microcosm #275; no -per-area files, the standing rule): the dense ACS-spine artifact is filtered to -state / congressional district / county for local analysis because every record -carries them. +sampled proportional to 2020 PUMA population. The derived geography is one +population-weighted draw from the actual ``(PUMA, tract, 119th CD)`` block +join. County is the selected tract's prefix, so county and CD always have +joint source support. The retained marginal tables validate this joint +population; assignment never multiplies their probabilities. + +The internal tract identifies a supported crosswalk cell, not an observed +residence. ``assign_tract`` controls whether that assigned tract is exported; +it does not change the PUMA, county or CD draw. National/CD analysis still +requires its separate calibration, vintage and coverage gates. An assigned +county or tract is not certification for analysis at that grain. Vintage discipline follows the country-spec geography-spine schema (``vintage_policy: "error"``): the artifact records one vintage per derived @@ -45,6 +37,7 @@ import json from collections.abc import Mapping from dataclasses import dataclass +from io import BytesIO from pathlib import Path from typing import Any @@ -59,6 +52,7 @@ from microcosm.build.us_runtime.puma_ladder_sources import ( COUNTY_FROM_TRACT_DIVISOR, PUMA_GEOID_STATE_DIVISOR, + US_PUMA_LADDER_SCHEMA_VERSION, ) from microcosm.frame import Frame @@ -81,7 +75,6 @@ #: The finer tract rung, written only when ``assign_tract=True``. US_PUMA_LADDER_TRACT_COLUMN = "tract_geoid" -US_PUMA_LADDER_SCHEMA_VERSION = 1 US_PUMA_LADDER_KIND = "us_puma_ladder" PUMA_LADDER_ARTIFACT_SHA256_ATTR = "populace_puma_ladder_artifact_sha256" @@ -108,14 +101,18 @@ _REQUIRED_ARRAY_KEYS = ( *_ANCHOR_KEYS, *(key for keys in _OVERLAP_KEYS.values() for key in keys), + "joint_overlap_puma", + "joint_overlap_tract", + "joint_overlap_cd", + "joint_overlap_population", ) @dataclass(frozen=True) class UsPumaLadder: """The national PUMA ladder: one anchor row per populated 2020 PUMA plus - three ``(PUMA, layer_value, population)`` overlap tables sorted by - ``(puma, layer_value)``. + three marginal overlap tables and the actual joint ``(PUMA, tract, CD, + population)`` table. Joint rows are unique and sorted by ``(puma, tract, CD)``. Attributes: puma: 2020 PUMA geoids (``state_fips * 10**5 + PUMA5CE``) as ``int64``, @@ -134,7 +131,10 @@ class UsPumaLadder: the ``(PUMA, tract)`` population overlap; tract geoids are the 11-digit ``state+county+tract`` integer. metadata: the artifact's embedded metadata, including one vintage per - derived layer. + derived layer. Schema v2 requires the joint source table. + joint_overlap_puma / joint_overlap_tract / joint_overlap_cd / + joint_overlap_population: the supported populated-block join, + aggregated to unique PUMA/tract/CD cells; county is a tract prefix. """ puma: np.ndarray @@ -149,6 +149,10 @@ class UsPumaLadder: tract_overlap_tract: np.ndarray tract_overlap_population: np.ndarray metadata: Mapping[str, Any] + joint_overlap_puma: np.ndarray + joint_overlap_tract: np.ndarray + joint_overlap_cd: np.ndarray + joint_overlap_population: np.ndarray def __len__(self) -> int: return len(self.puma) @@ -186,6 +190,17 @@ def load_us_puma_ladder(path: str | Path) -> UsPumaLadder: source = Path(path) if not source.exists(): raise FileNotFoundError(f"US PUMA ladder artifact not found: {source}") + return _decode_us_puma_ladder_archive(source) + + +def decode_us_puma_ladder(payload: bytes) -> UsPumaLadder: + """Validate the same NPZ contract from immutable graph artifact bytes.""" + if not isinstance(payload, bytes): + raise TypeError("US PUMA ladder payload must be immutable bytes.") + return _decode_us_puma_ladder_archive(BytesIO(payload)) + + +def _decode_us_puma_ladder_archive(source: Path | BytesIO) -> UsPumaLadder: with np.load(source, allow_pickle=False) as payload: missing = [key for key in _REQUIRED_ARRAY_KEYS if key not in payload.files] if missing: @@ -274,7 +289,7 @@ def load_us_puma_ladder(path: str | Path) -> UsPumaLadder: raise ValueError("tract_overlap_tract geoids must be unique (tracts nest).") _assert_conservation(puma, population, tract_puma, tract_pop, layer="tract") - return UsPumaLadder( + ladder = UsPumaLadder( puma=puma, puma_population=population, cd_overlap_puma=cd_puma, @@ -287,7 +302,13 @@ def load_us_puma_ladder(path: str | Path) -> UsPumaLadder: tract_overlap_tract=tract_value, tract_overlap_population=tract_pop, metadata=metadata, + joint_overlap_puma=arrays["joint_overlap_puma"], + joint_overlap_tract=arrays["joint_overlap_tract"], + joint_overlap_cd=arrays["joint_overlap_cd"], + joint_overlap_population=arrays["joint_overlap_population"], ) + _validated_joint_support(ladder) + return ladder def assign_us_puma_ladder( @@ -304,10 +325,11 @@ def assign_us_puma_ladder( Records that already carry a valid ``puma`` (the ACS spine) keep it; records that carry none (the ASEC spine) draw a PUMA within their ``state_fips`` - proportional to 2020 PUMA population. Every record then draws a congressional - district and a county within its PUMA proportional to the block-population - overlap, and — when ``assign_tract`` — a tract from which the county derives - structurally. Draws are consumed from one seeded generator in a fixed sorted + proportional to 2020 PUMA population. Every record then draws one supported + joint tract/CD cell within its PUMA proportional to block population. County + derives from that tract. The tract is written only when ``assign_tract`` is + true; changing that flag leaves the same PUMA/county/CD assignments. + Draws are consumed from one seeded generator in a fixed sorted order (states, then PUMAs), so the result is reproducible from ``seed``. A household ``puma`` absent from the ladder, or a household ``state_fips`` @@ -319,6 +341,9 @@ def assign_us_puma_ladder( f"household table must contain {state_fips_column!r} before PUMA-" "ladder assignment." ) + joint_puma, joint_tract, joint_cd, joint_population = _validated_joint_support( + ladder + ) if expected_congressional_district_vintage is not None: ladder_cd_vintage = ladder.layer_vintages["congressional_district"] if ladder_cd_vintage != expected_congressional_district_vintage: @@ -343,23 +368,17 @@ def assign_us_puma_ladder( if unknown_mask.any(): _draw_puma_within_state(row_puma, unknown_mask, state_ints, ladder, rng) - cd_puma, cd_value, cd_pop = ladder._overlap("congressional_district") - cd_values = _draw_layer_values( - row_puma, cd_puma, cd_value, cd_pop, rng, layer="congressional_district" + joint_rows = _draw_layer_values( + row_puma, + joint_puma, + np.arange(len(joint_puma), dtype=np.int64), + joint_population, + rng, + layer="joint tract/congressional_district", ) - - tract_values: np.ndarray | None = None - if assign_tract: - tract_puma, tract_value, tract_pop = ladder._overlap("tract") - tract_values = _draw_layer_values( - row_puma, tract_puma, tract_value, tract_pop, rng, layer="tract" - ) - county_values = tract_values // COUNTY_FROM_TRACT_DIVISOR - else: - county_puma, county_value, county_pop = ladder._overlap("county") - county_values = _draw_layer_values( - row_puma, county_puma, county_value, county_pop, rng, layer="county" - ) + cd_values = joint_cd[joint_rows] + county_values = joint_tract[joint_rows] // COUNTY_FROM_TRACT_DIVISOR + tract_values = joint_tract[joint_rows] if assign_tract else None _assert_row_state_consistency( state_ints, @@ -383,6 +402,213 @@ def assign_us_puma_ladder( return assigned +def _validated_joint_support( + ladder: UsPumaLadder, +) -> tuple[np.ndarray, np.ndarray, np.ndarray, np.ndarray]: + """Require real joint support and exact agreement with every marginal. + + A frozen dataclass does not make its arrays immutable. Check the current + arrays when assigning or verifying, including normally constructed ladders. + Nothing in this function manufactures joint rows from marginal weights. + """ + + _validate_ladder_metadata(ladder.metadata) + if ladder.metadata.get("puma_vintage") != "2020_puma": + raise ValueError( + "Joint PUMA assignment requires the observed 2020 PUMA vintage." + ) + if any( + ladder.layer_vintages[layer] != "2020_census" for layer in ("county", "tract") + ): + raise ValueError( + "Joint PUMA assignment requires 2020 Census county/tract vintages." + ) + anchor = _int64_array(ladder.puma, label="puma") + if ( + not len(anchor) + or (np.diff(anchor) <= 0).any() + or ( + (anchor // PUMA_GEOID_STATE_DIVISOR < 1) + | (anchor // PUMA_GEOID_STATE_DIVISOR > 56) + ).any() + ): + raise ValueError( + "Joint PUMA anchors must be unique, sorted and within states 01–56." + ) + columns = tuple( + _int64_array(getattr(ladder, name), label=name) + for name in ( + "joint_overlap_puma", + "joint_overlap_tract", + "joint_overlap_cd", + "joint_overlap_population", + ) + ) + puma, tract, cd, population = columns + if not len(puma) or any(len(value) != len(puma) for value in columns): + raise ValueError("Joint PUMA overlap arrays must be nonempty and aligned.") + if (population <= 0).any(): + raise ValueError( + "Joint PUMA overlap populations must be positive integer counts." + ) + rows = list(zip(puma.tolist(), tract.tolist(), cd.tolist(), strict=True)) + if rows != sorted(set(rows)): + raise ValueError( + "Joint PUMA overlap rows must be unique and sorted by PUMA/tract/CD." + ) + tract_owner: dict[int, int] = {} + for row_puma, row_tract, _ in rows: + if tract_owner.setdefault(row_tract, row_puma) != row_puma: + raise ValueError("Joint PUMA overlap assigns one tract to multiple PUMAs.") + if ((tract < 10**9) | (tract >= 10**11)).any() or ((cd < 100) | (cd > 9999)).any(): + raise ValueError("Joint PUMA overlap contains an invalid tract or CD geoid.") + _assert_layer_state_matches( + puma, tract // COUNTY_FROM_TRACT_DIVISOR // 1000, layer="joint tract" + ) + _assert_layer_state_matches(puma, cd // 100, layer="joint CD") + + def counts(keys: list[tuple[int, ...]], values: np.ndarray, label: str) -> dict: + values = np.asarray(values) + if values.ndim != 1 or len(values) != len(keys): + raise ValueError(f"{label} population must align with its keys.") + result: dict[tuple[int, ...], int] = {} + for key, value in zip(keys, values.tolist(), strict=True): + if ( + not isinstance(value, (int, float)) + or isinstance(value, bool) + or not np.isfinite(value) + or value <= 0 + or int(value) != value + ): + raise ValueError( + f"{label} population must contain positive integer counts." + ) + result[key] = result.get(key, 0) + int(value) + return result + + marginals = { + "anchor": [(int(key),) for key in puma], + "congressional_district": [ + (int(key), int(value)) for key, value in zip(puma, cd, strict=True) + ], + "county": [ + (int(key), int(value)) + for key, value in zip(puma, tract // COUNTY_FROM_TRACT_DIVISOR, strict=True) + ], + "tract": [ + (int(key), int(value)) for key, value in zip(puma, tract, strict=True) + ], + } + for layer, keys in marginals.items(): + if layer == "anchor": + reference_keys = [(int(key),) for key in anchor] + reference_pop = ladder.puma_population + else: + source_puma, source_value, reference_pop = ladder._overlap(layer) + source_puma = _int64_array(source_puma, label=layer + "_overlap_puma") + source_value = _int64_array(source_value, label=layer + "_overlap_value") + if len(source_puma) != len(source_value): + raise ValueError(f"{layer} overlap keys must align.") + reference_keys = list( + zip(source_puma.tolist(), source_value.tolist(), strict=True) + ) + if counts(keys, population, "joint") != counts( + reference_keys, reference_pop, layer + ): + raise ValueError( + f"Joint PUMA overlap does not exactly reproduce the {layer} marginal." + ) + return columns + + +def us_puma_ladder_joint_support_gate( + household: pd.DataFrame, + ladder: UsPumaLadder, + *, + assign_tract: bool = False, + expected_congressional_district_vintage: str | None = None, +) -> GateResult: + """Check current assigned rows against actual joint Census support. + + This structural support gate is independent of the national NYC mass gate + and of national/CD calibration; zero-weight clone rows must also be valid. + """ + + failures: list[str] = [] + details: dict[str, object] = {"household_rows": len(household)} + try: + puma, tract, cd, _ = _validated_joint_support(ladder) + if ( + expected_congressional_district_vintage is not None + and ladder.layer_vintages["congressional_district"] + != expected_congressional_district_vintage + ): + raise ValueError( + "Joint-support congressional-district vintage does not match the expected vintage." + ) + columns = ["puma", "county_fips", CONGRESSIONAL_DISTRICT_GEOID_COLUMN] + if assign_tract: + columns.append(US_PUMA_LADDER_TRACT_COLUMN) + missing = [name for name in columns if name not in household] + if missing: + raise ValueError(f"Joint support requires geography columns: {missing}.") + values = [] + for name in columns: + numeric = pd.to_numeric(household[name], errors="coerce").to_numpy( + dtype=np.float64, na_value=np.nan + ) + if not ( + np.isfinite(numeric) + & (numeric > 0) + & (numeric < 2**53) + & (numeric == np.floor(numeric)) + ).all(): + raise ValueError( + f"Joint support requires positive integral {name} geoids." + ) + values.append(numeric.astype(np.int64)) + support = set( + zip( + puma.tolist(), + (tract // COUNTY_FROM_TRACT_DIVISOR).tolist(), + cd.tolist(), + strict=True, + ) + ) + if assign_tract: + support = set( + zip( + puma.tolist(), + (tract // COUNTY_FROM_TRACT_DIVISOR).tolist(), + cd.tolist(), + tract.tolist(), + strict=True, + ) + ) + unsupported = [ + index + for index, row in enumerate( + zip(*(value.tolist() for value in values), strict=True) + ) + if row not in support + ] + details["unsupported_rows"] = len(unsupported) + if unsupported: + failures.append( + f"{len(unsupported)} household row(s) have no joint PUMA/county/CD support; row positions {unsupported[:5]}." + ) + details["joint_support_rows"] = len(puma) + details["layer_vintages"] = ladder.layer_vintages + except (TypeError, ValueError) as error: + failures.append(str(error)) + return GateResult( + name="us_puma_ladder_joint_support", + passed=not failures, + failures=tuple(failures), + details=details, + ) + + def with_household_us_puma_ladder( frame: Frame, ladder: UsPumaLadder, @@ -850,8 +1076,12 @@ def _assert_conservation( def _int64_array(values: np.ndarray, *, label: str) -> np.ndarray: array = np.asarray(values) + if array.ndim != 1: + raise ValueError(f"{label} must be a one-dimensional integer array.") if array.dtype.kind not in ("i", "u"): raise ValueError(f"{label} must be an integer array, got dtype {array.dtype}.") + if array.dtype.kind == "u" and (array > np.iinfo(np.int64).max).any(): + raise ValueError(f"{label} contains an integer outside signed int64.") return array.astype(np.int64, copy=False) @@ -898,5 +1128,6 @@ def _household_cd_state(values: Any) -> np.ndarray: "load_us_puma_ladder", "us_puma_ladder_assignment_summary", "us_puma_ladder_gate", + "us_puma_ladder_joint_support_gate", "with_household_us_puma_ladder", ] diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/puma_ladder_sources.py b/packages/microcosm-build/src/microcosm/build/us_runtime/puma_ladder_sources.py index bbf252e6b..1982f0b04 100644 --- a/packages/microcosm-build/src/microcosm/build/us_runtime/puma_ladder_sources.py +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/puma_ladder_sources.py @@ -17,8 +17,10 @@ - ``block -> 119th CD`` and ``block -> POP100`` reuse the block-ladder parsers :func:`parse_national_cd_bef` and :func:`parse_pl_geo_blocks` unchanged. -Summing block populations by ``(PUMA, CD)``, ``(PUMA, county)`` and -``(PUMA, tract)`` yields the three overlap tables; summing by PUMA yields the +Summing block populations by ``(PUMA, tract, CD)`` preserves the joint support +needed to draw a consistent county/CD pair (county is the tract prefix). The +three marginal overlap tables are retained for validation and diagnostics; +summing by PUMA yields the anchor population used for the ASEC state -> PUMA draw. Every populated block contributes to exactly one PUMA, one CD, one county and one tract, so each overlap table conserves its PUMA's population exactly — a defect (a populated @@ -45,6 +47,7 @@ TRACT_FROM_BLOCK_DIVISOR = 10**4 COUNTY_FROM_BLOCK_DIVISOR = 10**10 COUNTY_FROM_TRACT_DIVISOR = 10**6 +US_PUMA_LADDER_SCHEMA_VERSION = 2 _TRACT_TO_PUMA_HEADER = ("STATEFP", "COUNTYFP", "TRACTCE", "PUMA5CE") @@ -120,16 +123,21 @@ def assemble_us_puma_ladder( For every populated block the block's tract (a structural prefix) must map to a PUMA and the block must carry a congressional district; either gap is - a source defect, not a skippable row, so the three overlap tables each - conserve their PUMA's population exactly. Returns arrays for the anchor - (``puma`` / ``puma_population``) and the three overlap tables, each sorted - by ``(puma, layer_value)`` for a stable, searchsorted-friendly artifact. + a source defect, not a skippable row. Schema v2 retains the actual joint + PUMA/tract/CD population before aggregation is lost; county is the tract + prefix. It also emits the anchor and three marginal tables for exact + conservation checks. Joint rows are sorted by ``(puma, tract, CD)`` and + marginal rows by ``(puma, layer_value)``. """ + if metadata.get("schema_version") != US_PUMA_LADDER_SCHEMA_VERSION: + raise ValueError("Joint PUMA source assembly requires schema_version 2.") + puma_population: dict[int, int] = {} puma_cd_population: dict[tuple[int, int], int] = {} puma_county_population: dict[tuple[int, int], int] = {} puma_tract_population: dict[tuple[int, int], int] = {} + joint_population: dict[tuple[int, int, int], int] = {} missing_puma: list[int] = [] missing_cd: list[int] = [] @@ -157,6 +165,8 @@ def assemble_us_puma_ladder( puma_tract_population[tract_key] = ( puma_tract_population.get(tract_key, 0) + population ) + joint_key = (puma, tract, cd) + joint_population[joint_key] = joint_population.get(joint_key, 0) + population if missing_puma: examples = [f"{block:015d}" for block in sorted(missing_puma)[:5]] @@ -180,6 +190,7 @@ def assemble_us_puma_ladder( cd_puma, cd_value, cd_pop = _overlap_arrays(puma_cd_population) county_puma, county_value, county_pop = _overlap_arrays(puma_county_population) tract_puma, tract_value, tract_pop = _overlap_arrays(puma_tract_population) + joint_keys = sorted(joint_population) _assert_conserves( pumas, anchor_population, cd_puma, cd_pop, layer="congressional_district" @@ -199,6 +210,16 @@ def assemble_us_puma_ladder( "tract_overlap_puma": tract_puma, "tract_overlap_tract": tract_value, "tract_overlap_population": tract_pop, + "joint_overlap_puma": np.asarray( + [key[0] for key in joint_keys], dtype=np.int64 + ), + "joint_overlap_tract": np.asarray( + [key[1] for key in joint_keys], dtype=np.int64 + ), + "joint_overlap_cd": np.asarray([key[2] for key in joint_keys], dtype=np.int64), + "joint_overlap_population": np.asarray( + [joint_population[key] for key in joint_keys], dtype=np.int64 + ), "metadata_json": np.asarray(json.dumps(dict(metadata), sort_keys=True)), } diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/qbi_inputs.py b/packages/microcosm-build/src/microcosm/build/us_runtime/qbi_inputs.py index 13f5c35c5..ca5dc186b 100644 --- a/packages/microcosm-build/src/microcosm/build/us_runtime/qbi_inputs.py +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/qbi_inputs.py @@ -40,6 +40,22 @@ from microcosm.build.us_runtime.take_up_contract import load_take_up_contract from microcosm.frame import US_SCHEMA, Frame +from .operator_column_contracts import ( + _GENERAL_QUALIFICATION_FLAGS as _GENERAL_QUALIFICATION_FLAGS, +) +from .operator_column_contracts import ( + _SSTB_QUALIFICATION_FLAG as _SSTB_QUALIFICATION_FLAG, +) +from .operator_column_contracts import ( + US_QBI_BOOLEAN_OUTPUT_COLUMNS as US_QBI_BOOLEAN_OUTPUT_COLUMNS, +) +from .operator_column_contracts import ( + US_QBI_NONNEGATIVE_OUTPUT_COLUMNS as US_QBI_NONNEGATIVE_OUTPUT_COLUMNS, +) +from .operator_column_contracts import ( + US_QBI_OUTPUT_COLUMNS as US_QBI_OUTPUT_COLUMNS, +) + __all__ = [ "QBI_ARCHIVED_ASSUMPTIONS_URL", "QBI_ARCHIVED_CLONE_URL", @@ -87,40 +103,6 @@ US_QBI_STAGE_NAME = "puf_tax_detail" -_GENERAL_QUALIFICATION_FLAGS: tuple[str, ...] = ( - "estate_income_would_be_qualified", - "farm_operations_income_would_be_qualified", - "farm_rent_income_would_be_qualified", - "partnership_s_corp_income_would_be_qualified", - "rental_income_would_be_qualified", - "self_employment_income_would_be_qualified", -) -_SSTB_QUALIFICATION_FLAG = "sstb_self_employment_income_would_be_qualified" -US_QBI_BOOLEAN_OUTPUT_COLUMNS: tuple[str, ...] = ( - *_GENERAL_QUALIFICATION_FLAGS, - _SSTB_QUALIFICATION_FLAG, - # Keep the classifier last so the chained QRF can condition the SSTB draw - # on the qualification flags it must agree with. - "business_is_sstb", -) -US_QBI_NONNEGATIVE_OUTPUT_COLUMNS: tuple[str, ...] = ( - "qualified_bdc_income", - "qualified_reit_and_ptp_income", - "sstb_unadjusted_basis_qualified_property", - "sstb_w2_wages_from_qualified_business", - "unadjusted_basis_qualified_property", - "w2_wages_from_qualified_business", -) -US_QBI_OUTPUT_COLUMNS: tuple[str, ...] = ( - *US_QBI_BOOLEAN_OUTPUT_COLUMNS, - "qualified_bdc_income", - "qualified_reit_and_ptp_income", - "sstb_self_employment_income_before_lsr", - "sstb_unadjusted_basis_qualified_property", - "sstb_w2_wages_from_qualified_business", - "unadjusted_basis_qualified_property", - "w2_wages_from_qualified_business", -) US_QBI_NONCONSTANT_PERSON_COLUMNS = US_QBI_OUTPUT_COLUMNS _SELF_EMPLOYMENT_COLUMN = "self_employment_income_before_lsr" diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/relationship_inputs.py b/packages/microcosm-build/src/microcosm/build/us_runtime/relationship_inputs.py index 3695df002..f4f391509 100644 --- a/packages/microcosm-build/src/microcosm/build/us_runtime/relationship_inputs.py +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/relationship_inputs.py @@ -15,6 +15,7 @@ from __future__ import annotations +from collections.abc import Mapping from importlib.resources import files import numpy as np @@ -32,6 +33,9 @@ SourceRuntimeError, run_source_stage, ) +from microcosm.build.us_runtime._person_signal_summary import ( + validate_person_signal_summary, +) from microcosm.frame import Frame from microcosm.frame.units import US_SCHEMA @@ -41,6 +45,11 @@ "US_RELATIONSHIP_INPUTS_REQUIRED_SOURCE_COLUMNS", "US_RELATIONSHIP_INPUTS_STAGE_NAME", "derive_us_relationship_inputs_from_manifest", + "prepare_us_relationship_person", + "us_relationship_inputs_person_carries_signal", + "us_relationship_inputs_gate_from_summary", + "us_relationship_inputs_person_gate", + "us_relationship_inputs_person_summary", "us_relationship_inputs_signal_gate", "us_relationship_inputs_stage_spec", "us_relationship_inputs_summary", @@ -182,8 +191,13 @@ def derive_us_relationship_inputs_from_manifest( return result -def _relationship_surface_carries_signal(frame: Frame) -> bool: - person = frame.table("person") +def us_relationship_inputs_person_carries_signal(person: pd.DataFrame) -> bool: + """Return the exact incumbent pass-through decision from the real person table. + + All three outputs must be present and each must have multiple observed + values. Preserve the legacy null/constant rule; this does not validate raw + source columns or resolve weights, and is not a scientific signal gate. + """ if any(column not in person for column in US_RELATIONSHIP_INPUTS_OUTPUT_COLUMNS): return False return all( @@ -192,22 +206,70 @@ def _relationship_surface_carries_signal(frame: Frame) -> bool: ) -def with_us_relationship_inputs( - frame: Frame, +def _relationship_surface_carries_signal(frame: Frame) -> bool: + return us_relationship_inputs_person_carries_signal(frame.table("person")) + + +def _validated_relationship_weights( + person: pd.DataFrame, weights: np.ndarray +) -> np.ndarray: + """Return ``weights`` as a float64 array aligned 1:1 with ``person``.""" + + array = np.asarray(weights, dtype=np.float64) + if array.ndim != 1 or len(array) != len(person): + raise ValueError( + "US relationship-input weights must be a 1-D array aligned 1:1 " + f"with the person table ({len(person)} row(s)); got shape " + f"{array.shape}." + ) + if not np.isfinite(array).all(): + raise ValueError("US relationship-input weights must be finite.") + if (array < 0.0).any(): + raise ValueError("US relationship-input weights must be nonnegative.") + return array + + +def prepare_us_relationship_person( + person: pd.DataFrame, + weights: np.ndarray, *, seed: int, time_period: int, -) -> Frame: - """Materialize measured ASEC relationship inputs on a US frame.""" - - if frame.schema != US_SCHEMA: - raise ValueError("US relationship inputs require the US schema.") - if _relationship_surface_carries_signal(frame): - return frame - - person = frame.table("person") +) -> pd.DataFrame: + """Derive measured ASEC relationship inputs onto a copy of ``person``. + + The deterministic table-helper behind :func:`with_us_relationship_inputs`: + it always runs the ``relationship_inputs`` source-manifest stage over + ``person`` and ``weights`` and returns the aligned result. It does not + decide whether the surface already carries signal — that pass-through + decision belongs to the Frame wrapper. + + Args: + person: The actual person table, carrying the raw ASEC source + columns (``PH_SEQ``, ``P_SEQ``, ``A_MARITL``), in its own index + and row order. + weights: Person weights aligned 1:1 with ``person``'s rows (same + length and row order; need not be reindexed by ``person_id``). + seed: Build-wide imputation seed threaded to the source-stage + runtime (the derivation itself is deterministic). + time_period: The dataset's time period. + + Returns: + A copy of ``person`` with ``is_household_head``, ``is_separated``, + and ``is_surviving_spouse`` attached as ``bool`` columns, in + ``person``'s original index and row order. + + Raises: + ValueError: If ``weights`` does not align 1:1 with ``person``, is + not finite/nonnegative, or the stage output does not cover + every person. + SourceRuntimeError: If required raw ASEC column(s) are missing or + malformed. + """ + + weight_values = _validated_relationship_weights(person, weights) stage_person = person.copy(deep=True) - stage_person[_PERSON_WEIGHT_COLUMN] = frame.resolve_weights("person").values + stage_person[_PERSON_WEIGHT_COLUMN] = weight_values output = run_source_stage( us_relationship_inputs_stage_spec(), tables={"person": stage_person}, @@ -224,9 +286,34 @@ def with_us_relationship_inputs( f"for {column!r}." ) - tables = {entity: frame.table(entity).copy() for entity in frame.entities} + result = person.copy(deep=True) for column in US_RELATIONSHIP_INPUTS_OUTPUT_COLUMNS: - tables["person"][column] = aligned[column].to_numpy(dtype=bool) + result[column] = aligned[column].to_numpy(dtype=bool) + return result + + +def with_us_relationship_inputs( + frame: Frame, + *, + seed: int, + time_period: int, +) -> Frame: + """Materialize measured ASEC relationship inputs on a US frame.""" + + if frame.schema != US_SCHEMA: + raise ValueError("US relationship inputs require the US schema.") + if _relationship_surface_carries_signal(frame): + return frame + + person = frame.table("person") + new_person = prepare_us_relationship_person( + person, + frame.resolve_weights("person").values, + seed=seed, + time_period=time_period, + ) + tables = {entity: frame.table(entity).copy() for entity in frame.entities} + tables["person"] = new_person return Frame( tables, frame.schema, @@ -237,16 +324,35 @@ def with_us_relationship_inputs( ) -def us_relationship_inputs_summary(frame: Frame) -> dict[str, object]: - """Return weighted relationship shares and one-head invariants.""" +def us_relationship_inputs_person_summary( + person: pd.DataFrame, weights: np.ndarray +) -> dict[str, object]: + """Return weighted relationship shares and one-head invariants. - person = frame.table("person") - weights = np.asarray(frame.resolve_weights("person").values, dtype=np.float64) - total_weight = float(weights.sum()) + The real-person-table counterpart of :func:`us_relationship_inputs_summary`, + for callers (e.g. graph adapters) that hold a person table and an + explicit weight vector without a :class:`~microcosm.frame.Frame`. + + Args: + person: The actual person table, already carrying + ``is_household_head``, ``is_separated``, and + ``is_surviving_spouse``. + weights: Person weights aligned 1:1 with ``person``'s rows. + + Returns: + The same summary payload as :func:`us_relationship_inputs_summary`. + """ + + weight_values = _validated_relationship_weights(person, weights) + total_weight = float(weight_values.sum()) def _share(column: str) -> float: values = person[column].fillna(False).astype(bool).to_numpy() - return float(weights[values].sum()) / total_weight if total_weight > 0 else 0.0 + return ( + float(weight_values[values].sum()) / total_weight + if total_weight > 0 + else 0.0 + ) household_column = ( "person_household_id" if "person_household_id" in person else "PH_SEQ" @@ -276,24 +382,50 @@ def _share(column: str) -> float: } -def us_relationship_inputs_signal_gate(frame: Frame) -> GateResult: - """Require plausible signal and exactly one ASEC head per household.""" +def us_relationship_inputs_summary(frame: Frame) -> dict[str, object]: + """Return weighted relationship shares and one-head invariants.""" - person = frame.table("person") - missing = [ - column - for column in US_RELATIONSHIP_INPUTS_OUTPUT_COLUMNS - if column not in person - ] - if missing: - return GateResult( - name="relationship_inputs_signal", - passed=False, - failures=(f"person columns missing: {missing}.",), - details={"missing": missing}, - ) + return us_relationship_inputs_person_summary( + frame.table("person"), frame.resolve_weights("person").values + ) - summary = us_relationship_inputs_summary(frame) + +def us_relationship_inputs_gate_from_summary( + summary: Mapping[str, object], +) -> GateResult: + """Check relationship-input plausibility bands and invariants from a summary. + + The pure decision core of :func:`us_relationship_inputs_signal_gate`, + factored out so graph adapters can reuse the incumbent checks — same + bands, order, and meaning — against a summary computed off the real + person table (see :func:`us_relationship_inputs_person_summary`) + without a :class:`~microcosm.frame.Frame`. Assumes the caller has + already confirmed the three output columns are present; missing + columns are a separate failure mode (see + :func:`us_relationship_inputs_person_gate`). + + Raises: + ValueError: If required fields/counts are missing, measurements are + malformed, or supplied bands differ from the registered policy. + """ + + validate_person_signal_summary( + summary, + family="relationship", + outputs=US_RELATIONSHIP_INPUTS_OUTPUT_COLUMNS, + share_bands={ + "household_head_share": ( + "household_head_share_band", + _HOUSEHOLD_HEAD_SHARE_BAND, + ), + "separated_share": ("separated_share_band", _SEPARATED_SHARE_BAND), + "surviving_spouse_share": ( + "surviving_spouse_share_band", + _SURVIVING_SPOUSE_SHARE_BAND, + ), + }, + invariants=("households_without_exactly_one_head", "separated_and_surviving"), + ) failures: list[str] = [] for share_key, band_key, label in ( ( @@ -330,3 +462,51 @@ def us_relationship_inputs_signal_gate(frame: Frame) -> GateResult: failures=tuple(failures), details=summary, ) + + +def us_relationship_inputs_person_gate( + person: pd.DataFrame, weights: np.ndarray +) -> GateResult: + """Require plausible signal and exactly one ASEC head per household. + + The real-person-table counterpart of + :func:`us_relationship_inputs_signal_gate`, composing + :func:`us_relationship_inputs_person_summary` and + :func:`us_relationship_inputs_gate_from_summary` exactly as the Frame + wrapper does. + """ + + missing = [ + column + for column in US_RELATIONSHIP_INPUTS_OUTPUT_COLUMNS + if column not in person + ] + if missing: + return GateResult( + name="relationship_inputs_signal", + passed=False, + failures=(f"person columns missing: {missing}.",), + details={"missing": missing}, + ) + summary = us_relationship_inputs_person_summary(person, weights) + return us_relationship_inputs_gate_from_summary(summary) + + +def us_relationship_inputs_signal_gate(frame: Frame) -> GateResult: + """Require plausible signal and exactly one ASEC head per household.""" + + person = frame.table("person") + missing = [ + column + for column in US_RELATIONSHIP_INPUTS_OUTPUT_COLUMNS + if column not in person + ] + if missing: + return GateResult( + name="relationship_inputs_signal", + passed=False, + failures=(f"person columns missing: {missing}.",), + details={"missing": missing}, + ) + summary = us_relationship_inputs_summary(frame) + return us_relationship_inputs_gate_from_summary(summary) diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/reported_coverage_source.py b/packages/microcosm-build/src/microcosm/build/us_runtime/reported_coverage_source.py new file mode 100644 index 000000000..5992b7c1d --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/reported_coverage_source.py @@ -0,0 +1,448 @@ +"""Measured ASEC at-interview coverage recodes omitted by frozen H5 inputs. + +The pooled ``census_cps`` inputs carry ``NOW_GRP`` and ``NOW_MRK`` for every +income year, but their 2022 and 2023 person tables omit seven other recodes +consumed by :func:`derive_us_cps_carried_inputs`. Those omissions otherwise +turn measured coverage into structural false values for two pooled vintages. + +This module restores the seven fields from the same SHA-pinned official ASEC +person members already used for ``ED_VAL`` and ``PAW_TYP``. Restoration is an +exact ``(source_year, PERIDNUM)`` join with redundant Census identity checks; +it does not predict, infer, or substitute any coverage value. +""" + +from __future__ import annotations + +import hashlib +import zipfile +from collections.abc import Mapping +from pathlib import Path +from typing import BinaryIO + +import numpy as np +import pandas as pd + +from microcosm.build.us_runtime.education_assistance_source import ( + ASEC_EDUCATION_ASSISTANCE_ARCHIVES, + ASEC_EDUCATION_ASSISTANCE_INCOME_YEARS, + AsecEducationArchive, + fetch_asec_education_assistance_source, +) + +__all__ = [ + "ASEC_REPORTED_COVERAGE_INCOME_YEARS", + "ASEC_REPORTED_COVERAGE_RAW_COLUMNS", + "ASEC_REPORTED_COVERAGE_SOURCE_COLUMNS", + "fill_asec_reported_coverage_source", + "load_asec_reported_coverage_sources", +] + +# ``NOW_GRP`` and ``NOW_MRK`` are deliberately absent: they already survive +# in every frozen pooled source and remain native rather than sidecar-restored. +ASEC_REPORTED_COVERAGE_RAW_COLUMNS: tuple[str, ...] = ( + "NOW_NONM", + "NOW_MCAID", + "NOW_CHAMPVA", + "NOW_MIL", + "NOW_VACARE", + "NOW_OTHMT", + "NOW_IHSFLG", +) +ASEC_REPORTED_COVERAGE_SOURCE_COLUMNS: tuple[str, ...] = ( + "PH_SEQ", + "P_SEQ", + "A_LINENO", + "PERIDNUM", + *ASEC_REPORTED_COVERAGE_RAW_COLUMNS, +) +ASEC_REPORTED_COVERAGE_INCOME_YEARS = ASEC_EDUCATION_ASSISTANCE_INCOME_YEARS +_AUDIT_WEIGHT_COLUMN = "A_FNLWGT" +_VALID_CODES = frozenset({1, 2}) + + +def _sha256_stream(stream: BinaryIO, *, chunk_size: int) -> tuple[str, int]: + digest = hashlib.sha256() + size = 0 + for chunk in iter(lambda: stream.read(chunk_size), b""): + digest.update(chunk) + size += len(chunk) + return digest.hexdigest(), size + + +def _verify_member_path( + path: Path, + pins: AsecEducationArchive, + *, + chunk_size: int, +) -> None: + if not path.is_file(): + raise FileNotFoundError(path) + if path.stat().st_size != pins.member_size_bytes: + raise ValueError( + f"ASEC {pins.survey_year} person member byte length mismatch: " + f"expected {pins.member_size_bytes}, got {path.stat().st_size}." + ) + with path.open("rb") as stream: + digest, _ = _sha256_stream(stream, chunk_size=chunk_size) + if digest != pins.member_sha256: + raise ValueError( + f"ASEC {pins.survey_year} person member SHA-256 mismatch: " + f"expected {pins.member_sha256}, got {digest}." + ) + + +def _verified_zip_member( + archive: zipfile.ZipFile, + pins: AsecEducationArchive, +) -> zipfile.ZipInfo: + members = [info for info in archive.infolist() if info.filename == pins.member] + if len(members) != 1: + raise ValueError( + f"ASEC {pins.survey_year} archive must contain exactly one " + f"{pins.member!r} member; found {len(members)}." + ) + info = members[0] + if info.file_size != pins.member_size_bytes: + raise ValueError( + f"ASEC {pins.survey_year} person member byte length mismatch: " + f"expected {pins.member_size_bytes}, got {info.file_size}." + ) + actual_crc = f"{info.CRC:08x}" + if actual_crc != pins.member_crc32: + raise ValueError( + f"ASEC {pins.survey_year} person member CRC32 mismatch: expected " + f"{pins.member_crc32}, got {actual_crc}." + ) + return info + + +def _read_one_source( + path: Path, + pins: AsecEducationArchive, + *, + chunk_size: int, +) -> pd.DataFrame: + usecols = [*ASEC_REPORTED_COVERAGE_SOURCE_COLUMNS, _AUDIT_WEIGHT_COLUMN] + if zipfile.is_zipfile(path): + if path.stat().st_size != pins.zip_size_bytes: + raise ValueError( + f"ASEC {pins.survey_year} archive byte length mismatch: expected " + f"{pins.zip_size_bytes}, got {path.stat().st_size}." + ) + with path.open("rb") as stream: + archive_digest, _ = _sha256_stream(stream, chunk_size=chunk_size) + if archive_digest != pins.zip_sha256: + raise ValueError( + f"ASEC {pins.survey_year} archive SHA-256 mismatch: expected " + f"{pins.zip_sha256}, got {archive_digest}." + ) + with zipfile.ZipFile(path) as archive: + info = _verified_zip_member(archive, pins) + with archive.open(info) as member: + member_digest, member_size = _sha256_stream( + member, + chunk_size=chunk_size, + ) + if ( + member_size != pins.member_size_bytes + or member_digest != pins.member_sha256 + ): + raise ValueError( + f"ASEC {pins.survey_year} person member identity mismatch." + ) + with archive.open(info) as member: + try: + return pd.read_csv( + member, + usecols=usecols, + dtype={"PERIDNUM": "string"}, + low_memory=False, + ) + except ValueError as error: + raise ValueError( + f"ASEC {pins.survey_year} reported-coverage source is " + f"missing required column(s): {error}." + ) from error + + _verify_member_path(path, pins, chunk_size=chunk_size) + try: + return pd.read_csv( + path, + usecols=usecols, + dtype={"PERIDNUM": "string"}, + low_memory=False, + ) + except ValueError as error: + raise ValueError( + f"ASEC {pins.survey_year} reported-coverage source is missing " + f"required column(s): {error}." + ) from error + + +def _fixed_width_peridnum(values: pd.Series, *, label: str) -> pd.Series: + if values.isna().any(): + rows = values.index[values.isna()].tolist()[:5] + raise ValueError(f"{label} PERIDNUM is missing at row(s): {rows}.") + decoded = values.map( + lambda value: ( + value.decode() + if isinstance(value, (bytes, bytearray, np.bytes_)) + else value + ) + ) + valid = decoded.map(lambda value: isinstance(value, str)) + valid &= decoded.astype("string").str.fullmatch(r"[0-9]{22}", na=False) + if not valid.all(): + rows = decoded.index[~valid].tolist()[:5] + raise ValueError( + f"{label} PERIDNUM must be an exact 22-digit string at row(s): {rows}." + ) + return decoded.astype(str) + + +def _validated_source_year(values: pd.Series, *, label: str) -> pd.Series: + numeric = pd.to_numeric(values, errors="coerce") + valid = numeric.notna() & np.isfinite(numeric) & numeric.eq(np.floor(numeric)) + if not valid.all(): + rows = values.index[~valid].tolist()[:5] + raise ValueError(f"{label} source_year is invalid at row(s): {rows}.") + return numeric.astype(np.int64) + + +def _validated_codes(values: pd.Series, *, label: str) -> np.ndarray: + numeric = pd.to_numeric(values, errors="coerce").to_numpy(dtype=np.float64) + boolean = values.map(lambda value: isinstance(value, (bool, np.bool_))).to_numpy() + valid = np.isfinite(numeric) & np.isin(numeric, sorted(_VALID_CODES)) & ~boolean + if not valid.all(): + rows = np.flatnonzero(~valid)[:5].tolist() + raise ValueError( + f"{label} must be complete integers in {sorted(_VALID_CODES)} at " + f"row(s): {rows}." + ) + return numeric.astype(np.int64) + + +def load_asec_reported_coverage_sources( + paths: Mapping[int, str | Path] | None = None, + *, + income_years: tuple[int, ...] = ASEC_REPORTED_COVERAGE_INCOME_YEARS, + chunk_size: int = 8 * 1024 * 1024, +) -> pd.DataFrame: + """Load the seven measured coverage recodes from pinned ASEC members.""" + + unknown = sorted(set(paths or ()) - set(ASEC_EDUCATION_ASSISTANCE_ARCHIVES)) + if unknown: + raise ValueError( + "No pinned ASEC reported-coverage archive covers income year(s) " + f"{unknown}; pinned income years: " + f"{list(ASEC_REPORTED_COVERAGE_INCOME_YEARS)}." + ) + if not income_years: + empty = pd.DataFrame( + columns=["source_year", *ASEC_REPORTED_COVERAGE_SOURCE_COLUMNS] + ) + empty.attrs["source_audit"] = {} + return empty + + parts: list[pd.DataFrame] = [] + audits: dict[int, dict[str, object]] = {} + for income_year in income_years: + pins = ASEC_EDUCATION_ASSISTANCE_ARCHIVES.get(income_year) + if pins is None: + raise ValueError( + "No pinned ASEC reported-coverage archive covers income year " + f"{income_year}." + ) + provided = None if paths is None else paths.get(income_year) + path = ( + Path(provided).expanduser() + if provided is not None + else fetch_asec_education_assistance_source(income_year) + ) + raw = _read_one_source(path, pins, chunk_size=chunk_size) + if len(raw) != pins.rows: + raise ValueError( + f"ASEC {pins.survey_year} reported-coverage source row count " + f"mismatch: expected {pins.rows}, got {len(raw)}." + ) + raw["PERIDNUM"] = _fixed_width_peridnum( + raw["PERIDNUM"], + label=f"ASEC {pins.survey_year} reported-coverage source", + ) + if raw["PERIDNUM"].duplicated(keep=False).any(): + raise ValueError( + f"ASEC {pins.survey_year} reported-coverage source PERIDNUM " + "must be unique." + ) + weights = pd.to_numeric(raw[_AUDIT_WEIGHT_COLUMN], errors="coerce").to_numpy( + dtype=np.float64 + ) + if ( + not np.isfinite(weights).all() + or (weights < 0.0).any() + or float(weights.sum()) <= 0.0 + ): + raise ValueError( + f"ASEC {pins.survey_year} A_FNLWGT must be finite and " + "nonnegative with positive total mass." + ) + + column_audits: dict[str, dict[str, float | int]] = {} + scaled_weights = weights / 100.0 + for column in ASEC_REPORTED_COVERAGE_RAW_COLUMNS: + values = _validated_codes( + raw[column], + label=f"ASEC {pins.survey_year} {column}", + ) + raw[column] = values + yes = values == 1 + column_audits[column] = { + "no_rows": int(np.count_nonzero(values == 2)), + "yes_rows": int(np.count_nonzero(yes)), + "weighted_yes_share": float( + scaled_weights[yes].sum() / scaled_weights.sum() + ), + } + audits[income_year] = { + "rows": int(len(raw)), + "columns": column_audits, + } + part = raw.loc[:, list(ASEC_REPORTED_COVERAGE_SOURCE_COLUMNS)].copy() + part.insert(0, "source_year", np.int64(income_year)) + parts.append(part) + + result = pd.concat(parts, ignore_index=True) + result.attrs["source_audit"] = audits + return result + + +def fill_asec_reported_coverage_source( + person: pd.DataFrame, + source: pd.DataFrame, +) -> pd.DataFrame: + """Restore all seven recodes by exact ``(source_year, PERIDNUM)`` join.""" + + required_person = ("source_year", "PERIDNUM") + missing_person = [column for column in required_person if column not in person] + if missing_person: + raise ValueError( + "ASEC reported-coverage repair requires person column(s): " + f"{missing_person}." + ) + required_source = ("source_year", *ASEC_REPORTED_COVERAGE_SOURCE_COLUMNS) + missing_source = [column for column in required_source if column not in source] + if missing_source: + raise ValueError( + f"ASEC reported-coverage sidecar missing column(s): {missing_source}." + ) + + donor = source.copy(deep=True) + donor["source_year"] = _validated_source_year( + donor["source_year"], label="ASEC reported-coverage sidecar" + ) + donor["PERIDNUM"] = _fixed_width_peridnum( + donor["PERIDNUM"], label="ASEC reported-coverage sidecar" + ) + duplicate = donor.duplicated(["source_year", "PERIDNUM"], keep=False) + if duplicate.any(): + keys = donor.loc[duplicate, ["source_year", "PERIDNUM"]].head(5) + raise ValueError( + "ASEC reported-coverage sidecar (source_year, PERIDNUM) keys must " + f"be unique; duplicate key(s): {keys.to_dict(orient='records')}." + ) + for column in ASEC_REPORTED_COVERAGE_RAW_COLUMNS: + donor[column] = _validated_codes( + donor[column], label=f"ASEC reported-coverage sidecar {column}" + ) + + result = person.copy(deep=True) + person_years = _validated_source_year( + result["source_year"], label="ASEC reported-coverage person" + ) + result["PERIDNUM"] = _fixed_width_peridnum( + result["PERIDNUM"], label="ASEC reported-coverage frame" + ) + needed_years = sorted(int(year) for year in person_years.unique()) + covered_years = set(int(year) for year in donor["source_year"].unique()) + uncovered = [year for year in needed_years if year not in covered_years] + if uncovered: + raise ValueError( + "ASEC reported-coverage sidecar does not cover pooled income " + f"year(s): {uncovered}." + ) + + for year in needed_years: + year_mask = person_years.eq(year).to_numpy() + year_donor = donor.loc[donor["source_year"].eq(year)].set_index("PERIDNUM") + keys = result.loc[year_mask, "PERIDNUM"] + missing_keys = keys[~keys.isin(year_donor.index)].drop_duplicates() + if not missing_keys.empty: + raise ValueError( + "ASEC reported-coverage sidecar does not cover frame PERIDNUM " + f"key(s) for income year {year}: {missing_keys.tolist()[:5]}." + ) + aligned = year_donor.reindex(keys.to_numpy()) + aligned.index = result.index[year_mask] + + identity_pairs = ( + ( + "PH_SEQ", + "source_household_id" if "source_household_id" in result else "PH_SEQ", + ), + ("P_SEQ", "P_SEQ"), + ("A_LINENO", "A_LINENO"), + ) + for donor_column, frame_column in identity_pairs: + if frame_column not in result: + continue + observed = pd.to_numeric( + result.loc[year_mask, frame_column], errors="coerce" + ).to_numpy(dtype=np.float64) + expected = pd.to_numeric(aligned[donor_column], errors="coerce").to_numpy( + dtype=np.float64 + ) + mismatch = ( + ~np.isfinite(observed) | ~np.isfinite(expected) | (observed != expected) + ) + if mismatch.any(): + rows = result.index[year_mask].to_numpy()[mismatch][:5].tolist() + raise ValueError( + "ASEC reported-coverage redundant identity mismatch for " + f"{frame_column} against sidecar {donor_column} in income " + f"year {year} at row(s): {rows}." + ) + + for column in ASEC_REPORTED_COVERAGE_RAW_COLUMNS: + expected = aligned[column].to_numpy(dtype=np.int64) + if column in result: + current = result.loc[year_mask, column] + present = current.notna().to_numpy() + numeric = pd.to_numeric(current, errors="coerce").to_numpy( + dtype=np.float64 + ) + invalid = present & ( + ~np.isfinite(numeric) + | ~np.isin(numeric, sorted(_VALID_CODES)) + | current.map( + lambda value: isinstance(value, (bool, np.bool_)) + ).to_numpy() + ) + mismatch = present & ~invalid & (numeric != expected) + if invalid.any() or mismatch.any(): + bad = invalid | mismatch + rows = result.index[year_mask].to_numpy()[bad][:5].tolist() + raise ValueError( + f"ASEC reported-coverage existing {column} disagrees " + f"with the pinned sidecar in income year {year} at " + f"row(s): {rows}." + ) + else: + result[column] = np.nan + result.loc[year_mask, column] = expected + + for column in ASEC_REPORTED_COVERAGE_RAW_COLUMNS: + result[column] = pd.to_numeric(result[column], errors="raise").astype("int64") + result.attrs["reported_coverage_source_audit"] = source.attrs.get( + "source_audit", {} + ) + return result diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/source_csv_builtin.py b/packages/microcosm-build/src/microcosm/build/us_runtime/source_csv_builtin.py new file mode 100644 index 000000000..1f38f18b1 --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/source_csv_builtin.py @@ -0,0 +1,31 @@ +"""Bind source CSV readers to the real stdlib builtin, including caller aliases.""" + +import _csv +import csv +from types import BuiltinFunctionType + +_NATIVE_CSV = _csv +_CSV = csv +_READER = _csv.reader + + +def csv_reader_bound(module) -> bool: + """Refuse live rebinding, including wrappers installed before this import.""" + return ( + type(_READER) is BuiltinFunctionType + and _READER.__module__ == "_csv" + and _READER.__name__ == "reader" + and _READER.__self__ is _NATIVE_CSV + and module is _CSV + and getattr(module, "reader", None) is _READER + and getattr(_NATIVE_CSV, "reader", None) is _READER + ) + + +def capture_csv_reader(module): + """Return the checked builtin itself, never a later mutable alias lookup.""" + return _READER if csv_reader_bound(module) else None + + +if not csv_reader_bound(csv): + raise ValueError("SOURCE_CSV_READER_CHANGED") diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/spine_assembly.py b/packages/microcosm-build/src/microcosm/build/us_runtime/spine_assembly.py index ce6b6e296..4f39da34b 100644 --- a/packages/microcosm-build/src/microcosm/build/us_runtime/spine_assembly.py +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/spine_assembly.py @@ -8,14 +8,17 @@ from __future__ import annotations +import hashlib import re from collections.abc import Mapping +from dataclasses import asdict, dataclass +from types import MappingProxyType from typing import Any import numpy as np import pandas as pd -from microcosm.build.us_runtime.puf_support import ( +from microcosm.build.us_runtime.operator_column_contracts import ( PUF_SUPPORT_MAX_CLONE_SAFE_SOURCE_ID, ) from microcosm.build.us_runtime.support_provenance import ( @@ -36,7 +39,14 @@ Weights, ) -__all__ = ["assemble_spines"] +__all__ = [ + "SpinePreparation", + "SpineHarmonization", + "assemble_spines", + "prepare_spines", + "stack_survey_spines", + "harmonize_spine_weights", +] _CHANNEL_PATTERN = re.compile(r"^[a-z][a-z0-9_]*$") _SUPPORT_CLONE_INDEX = 0 @@ -80,11 +90,146 @@ def assemble_spines( provenance, columns, or ID spaces violate the assembly contract. """ + prepared = _stack_spine_tables( + spines, + household_mass_shares=household_mass_shares, + mass_anchor_channel=mass_anchor_channel, + ) + harmonized = _harmonize_spine_values( + prepared.tables["household"], prepared.values, prepared.context + ) + # These are newly composed per-source records, not a carried Frame log. + # Table provenance remains owned by the independent stacking result. + assembled_mass_log = harmonized.mass_log + result = Frame( + prepared.tables, + US_SCHEMA, + {"household": harmonized.weights}, + prepared.strata, + mass_log=assembled_mass_log, + metadata=prepared.metadata, + ) + validate_assembly_provenance(result, boundary="spine assembly output") + anchor_mass = prepared.context["incoming_masses"][mass_anchor_channel] + if not np.isclose( + result.weights_for("household").total, anchor_mass, rtol=_SHARE_RTOL, atol=0.0 + ): + raise RuntimeError("Spine assembly failed to conserve anchor household mass.") + return result + + +@dataclass(frozen=True) +class SpinePreparation: + """Actual DESIGN-weight tables plus an immutable assembly context. + + The context schema determines which subsequent weight operation can use it. + This numerical container does not authenticate source issuance or membership. + """ + + frame: Frame + context: Mapping[str, object] + + def __post_init__(self) -> None: + if not isinstance(self.frame, Frame): + raise TypeError("SpinePreparation.frame must be a Frame.") + if self.frame.weights_for("household").kind is not WeightKind.DESIGN: + raise ValueError("Spine preparation requires actual DESIGN weights.") + if not isinstance(self.context, Mapping): + raise ValueError("Spine preparation context must be a mapping.") + object.__setattr__(self, "context", _freeze_context(self.context)) + + +@dataclass(frozen=True) +class SpineHarmonization: + """Computed importance weights and exact ordered legacy operator history.""" + + weights: Weights + mass_log: tuple[MassChangeRecord, ...] + + +@dataclass(frozen=True) +class _StackedTables: + tables: Mapping[str, pd.DataFrame] + strata: pd.Series + values: np.ndarray + context: Mapping[str, object] + metadata: Mapping[str, object] + mass_log: tuple[MassChangeRecord, ...] + + +def _freeze_context(value: object) -> object: + if isinstance(value, Mapping): + return MappingProxyType( + {key: _freeze_context(item) for key, item in value.items()} + ) + if isinstance(value, (list, tuple)): + return tuple(_freeze_context(item) for item in value) + return value + + +def _numeric_digest(values: np.ndarray, dtype: str) -> str: + array = np.ascontiguousarray(values, dtype=dtype) + return hashlib.sha256(memoryview(array).cast("B")).hexdigest() + + +def _channel_digest(values: np.ndarray) -> str: + digest = hashlib.sha256() + for value in values: + if not isinstance(value, str): + raise ValueError("Prepared household channels must be strings.") + encoded = value.encode("utf-8") + digest.update(len(encoded).to_bytes(8, "little")) + digest.update(encoded) + return digest.hexdigest() + + +def _stack_spine_tables( + spines: Mapping[str, Frame], + *, + household_mass_shares: Mapping[str, float], + mass_anchor_channel: str, + require_design: bool = False, +) -> _StackedTables: ordered_channels = _validated_channels(spines, mass_anchor_channel) shares = _validated_shares(household_mass_shares, ordered_channels) frames = {channel: spines[channel] for channel in ordered_channels} + prepared = _stack_source_tables(frames, ordered_channels, require_design) + context = dict(prepared.context) + context.update( + schema="microcosm.us.spine-preparation.v1", + household_mass_shares=shares, + mass_anchor_channel=mass_anchor_channel, + incoming_masses={ + channel: float(frames[channel].weights_for("household").total) + for channel in ordered_channels + }, + ) + return _StackedTables( + prepared.tables, + prepared.strata, + prepared.values, + _freeze_context(context), + prepared.metadata, + prepared.mass_log, + ) + + +def _stack_source_tables( + spines: Mapping[str, Frame], + ordered_channels: tuple[str, ...], + require_design: bool, +) -> _StackedTables: + """Preserve source values while remapping only structural collisions.""" + frames = {channel: spines[channel] for channel in ordered_channels} for channel, frame in frames.items(): _validate_source_frame(frame, channel=channel) + if ( + require_design + and frame.weights_for("household").kind is not WeightKind.DESIGN + ): + raise ValueError( + f"Spine {channel!r} preparation requires actual DESIGN weights." + ) _validate_shared_column_dtypes(frames) offsets = _id_offsets(frames, ordered_channels) @@ -106,51 +251,286 @@ def assemble_spines( column_orders=column_orders, ) - anchor_mass = frames[mass_anchor_channel].weights_for("household").total - weights, mass_log = _assembled_household_weights( - frames, - ordered_channels=ordered_channels, - shares=shares, - anchor_mass=anchor_mass, + values = np.concatenate( + [ + frames[channel].weights_for("household").values + for channel in ordered_channels + ] ) - household_order = group_orders.get("household") - if household_order is not None: - weights = Weights(weights.values[household_order], weights.kind) - weights = _with_exact_total(weights, anchor_mass) - + order = group_orders.get("household") + if order is not None: + values = values[order] + household = tables["household"] + source_ids_digest = hashlib.sha256() + for channel in ordered_channels: + source_ids = np.ascontiguousarray( + prepared[channel]["household"]["household_id"].to_numpy(), dtype=" SpinePreparation: + """Stack actual DESIGN sources without performing importance allocation. + + Legacy ``assemble_spines`` continues to accept its existing mixed-kind + inputs. This new graph-facing seam never relabels such inputs as DESIGN. + """ + prepared = _stack_spine_tables( + spines, + household_mass_shares=household_mass_shares, + mass_anchor_channel=mass_anchor_channel, + require_design=True, + ) + return _design_preparation(prepared) + + +def stack_survey_spines(spines: Mapping[str, Frame]) -> SpinePreparation: + """Stack original DESIGN survey weights before domain allocation. + + Channels are ordered lexically, independent of mapping insertion order. + Every household weight, source age and measured value is carried unchanged; + only structural IDs are remapped and source provenance is added. No survey + total is an anchor, no shares are assigned, and no mass is normalized. + + This is the numerical stacking seam. A graph source owner must separately + authenticate the original issuances, complete membership and publisher + anchors. Sampling and population-domain allocation are subsequent declared + operations. The returned v1 survey-stack context is deliberately refused + by the legacy anchor-total harmonizer. + """ + channels = _validated_source_channels(spines) + prepared = _stack_source_tables(spines, channels, require_design=True) + return _design_preparation(prepared) + + +def _design_preparation(prepared: _StackedTables) -> SpinePreparation: + frame = Frame( + prepared.tables, + US_SCHEMA, + {"household": Weights(prepared.values, WeightKind.DESIGN)}, + prepared.strata, + mass_log=prepared.mass_log, + metadata=prepared.metadata, ) - if not np.isclose( - result.weights_for("household").total, - anchor_mass, - rtol=_SHARE_RTOL, - atol=0.0, + validate_assembly_provenance(frame, boundary="spine preparation output") + return SpinePreparation(frame, prepared.context) + + +def harmonize_spine_weights( + *, + household: pd.DataFrame, + weights: Weights, + context: Mapping[str, object], +) -> SpineHarmonization: + """Compute importance allocation from declared DESIGN views and context. + + Input digests bind exact normalized values and ordered IDs/channels. + The explicit permutation restores source-order sums and correction rows. + The result includes legacy per-source logs, not an executor graph ledger. + """ + if not isinstance(weights, Weights) or weights.kind is not WeightKind.DESIGN: + raise ValueError("Spine harmonization requires actual DESIGN weights.") + if ( + not isinstance(context, Mapping) + or context.get("schema") != "microcosm.us.spine-preparation.v1" ): - raise RuntimeError("Spine assembly failed to conserve anchor household mass.") - return result + raise ValueError("Unsupported spine preparation context.") + kinds = context.get("source_weight_kinds") + ordered = context.get("ordered_channels") + if ( + not isinstance(kinds, Mapping) + or not isinstance(ordered, (tuple, list)) + or set(kinds) != set(ordered) + or any(kind != WeightKind.DESIGN.value for kind in kinds.values()) + ): + raise ValueError("Spine preparation context does not declare DESIGN sources.") + return _harmonize_spine_values(household, weights.values, context) + + +def _validated_spine_inputs( + household: pd.DataFrame, values: np.ndarray, context: Mapping[str, object] +) -> tuple: + if ( + not isinstance(context, Mapping) + or context.get("schema") != "microcosm.us.spine-preparation.v1" + ): + raise ValueError("Unsupported spine preparation context.") + if not isinstance(household, pd.DataFrame) or not { + "household_id", + support_channel_column("household"), + }.issubset(household): + raise ValueError("Harmonization requires household IDs and source channels.") + ids = household["household_id"].to_numpy() + channels = household[support_channel_column("household")].to_numpy() + if not np.issubdtype(ids.dtype, np.integer) or len(values) != len(ids): + raise ValueError("Prepared household IDs/weights are malformed.") + bindings = { + "household_ids_sha256": _numeric_digest(ids, " SpineHarmonization: + ordered, shares, counts, source_values, source_channels, order, anchor_mass = ( + _validated_spine_inputs(household, values, context) + ) + result, logs, allocated, start = [], [], 0.0, 0 + for index, channel in enumerate(ordered): + count = counts[channel] + existing_values = source_values[start : start + count] + if not np.all(source_channels[start : start + count] == channel): + raise ValueError( + "Prepared permutation does not restore source channel order." + ) + start += count + existing_mass = float(existing_values.sum()) + if existing_mass != float(context["incoming_masses"][channel]): + raise ValueError("Prepared source-order mass changed.") + target = ( + anchor_mass - allocated + if index == len(ordered) - 1 + else anchor_mass * shares[channel] + ) + scaled = _values_to_total(existing_values, target) + result.append(scaled) + allocated += float(scaled.sum()) + logs.extend( + MassChangeRecord(**dict(record)) + for record in context["source_mass_logs"][channel] + ) + logs.append( + MassChangeRecord( + entity="household", + old_total=existing_mass, + new_total=float(scaled.sum()), + declared_factor=target / existing_mass, + reason=f"allocated {channel!r} source mass in pre-operator spine assembly", + ) + ) + combined = np.concatenate(result) + if order is not None: + combined = combined[order] + return SpineHarmonization( + _with_exact_total(Weights(combined, WeightKind.IMPORTANCE), anchor_mass), + tuple(logs), + ) def _validated_channels( spines: Mapping[str, Frame], mass_anchor_channel: str, ) -> tuple[str, ...]: + channels = _validated_source_channels(spines) + if ( + not isinstance(mass_anchor_channel, str) + or _CHANNEL_PATTERN.fullmatch(mass_anchor_channel) is None + ): + raise ValueError( + "mass_anchor_channel must be a stable lower-snake-case identifier." + ) + if mass_anchor_channel not in channels: + raise ValueError( + f"mass_anchor_channel {mass_anchor_channel!r} is absent from spines." + ) + return ( + mass_anchor_channel, + *(channel for channel in channels if channel != mass_anchor_channel), + ) + + +def _validated_source_channels(spines: Mapping[str, Frame]) -> tuple[str, ...]: if not isinstance(spines, Mapping): raise TypeError(f"spines must be a mapping, got {type(spines).__name__}.") if len(spines) < 2: @@ -171,21 +551,7 @@ def _validated_channels( f"{PUF_TAX_DETAIL_SUPPORT_CHANNEL!r} is a clone operator channel, " "not a peer household spine." ) - if ( - not isinstance(mass_anchor_channel, str) - or _CHANNEL_PATTERN.fullmatch(mass_anchor_channel) is None - ): - raise ValueError( - "mass_anchor_channel must be a stable lower-snake-case identifier." - ) - if mass_anchor_channel not in spines: - raise ValueError( - f"mass_anchor_channel {mass_anchor_channel!r} is absent from spines." - ) - return ( - mass_anchor_channel, - *sorted(channel for channel in channels if channel != mass_anchor_channel), - ) + return tuple(sorted(channels)) def _validated_shares( @@ -394,6 +760,11 @@ def _id_offsets( ) offsets[channel][entity] = offset remapped = ids if offset == 0 else ids + offset + if int(remapped.max()) > PUF_SUPPORT_MAX_CLONE_SAFE_SOURCE_ID: + raise ValueError( + f"Spine {channel!r} {entity!r} collision remapping exceeds " + f"the clone-safe bound {PUF_SUPPORT_MAX_CLONE_SAFE_SOURCE_ID}." + ) accumulated[entity] = np.concatenate([used, remapped]) return offsets @@ -493,45 +864,6 @@ def _combined_tables( return tables, group_orders -def _assembled_household_weights( - frames: Mapping[str, Frame], - *, - ordered_channels: tuple[str, ...], - shares: Mapping[str, float], - anchor_mass: float, -) -> tuple[Weights, tuple[MassChangeRecord, ...]]: - values: list[np.ndarray] = [] - mass_log: list[MassChangeRecord] = [] - allocated = 0.0 - for index, channel in enumerate(ordered_channels): - existing = frames[channel].weights_for("household") - target = ( - anchor_mass - allocated - if index == len(ordered_channels) - 1 - else anchor_mass * shares[channel] - ) - scaled = _values_to_total(existing.values, target) - values.append(scaled) - allocated += float(scaled.sum()) - factor = target / existing.total - mass_log.extend(frames[channel].mass_log) - mass_log.append( - MassChangeRecord( - entity="household", - old_total=existing.total, - new_total=float(scaled.sum()), - declared_factor=factor, - reason=( - f"allocated {channel!r} source mass in pre-operator spine assembly" - ), - ) - ) - return ( - Weights(np.concatenate(values), WeightKind.IMPORTANCE), - tuple(mass_log), - ) - - def _values_to_total(values: np.ndarray, target: float) -> np.ndarray: result = np.asarray(values, dtype=np.float64) * ( target / float(np.asarray(values, dtype=np.float64).sum()) diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/stacked_spine.py b/packages/microcosm-build/src/microcosm/build/us_runtime/stacked_spine.py index d71320311..1e62b8c2d 100644 --- a/packages/microcosm-build/src/microcosm/build/us_runtime/stacked_spine.py +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/stacked_spine.py @@ -26,10 +26,11 @@ ``w_i' = w_i * share_s * M_anchor / M_s`` where ``M_s`` is arm ``s``'s incoming household mass and ``M_anchor`` is the -mass-anchor arm's incoming mass. For the seeded ACS sample, -``M_acs_sample ~= fraction * M_acs_full``, so the allocation factor contains -the inverse-sampling upweighting ``1 / fraction`` automatically; the realized -per-arm scale factors are receipted rather than assumed. +mass-anchor arm's incoming mass. Production sampling first normalizes each +source by its full-source mass divided by its realized sampled mass. For +unequal source weights, that ratio is not generally the inverse realized +household inclusion count ``N / n``. Legacy pilot controls allocate directly +from the sampled ACS mass. Both paths receipt their actual scale factors. """ from __future__ import annotations @@ -40,7 +41,6 @@ import math import os import pickle -import struct import sys from collections import Counter from collections.abc import Callable, Mapping, Sequence @@ -80,6 +80,12 @@ ) from microcosm.build.serialization_dtypes import canonicalize_table_string_dtypes from microcosm.build.source_manifest import load_source_manifest +from microcosm.build.table_identity import ( + TABLE_VALUES_DIGEST_CODEC as _LATE_TABLE_DIGEST_CODEC, +) +from microcosm.build.table_identity import ( + table_values_sha256 as _late_table_values_sha256, +) from microcosm.build.us_runtime import ( post_transfer_calibration as post_transfer_calibration_runtime, ) @@ -183,7 +189,13 @@ puf_tax_detail_tail_bound_quantiles_identity, validate_puf_clone_attachment, ) -from microcosm.build.us_runtime.spine_assembly import assemble_spines +from microcosm.build.us_runtime.spine_assembly import ( + _freeze_context, + _validated_spine_inputs, + assemble_spines, + harmonize_spine_weights, + prepare_spines, +) from microcosm.build.us_runtime.support_provenance import ( BASE_ASEC_SUPPORT_CHANNEL, PUF_TAX_DETAIL_CLONE_INDEX, @@ -220,7 +232,7 @@ us_late_producer_schedule_receipt, ) from microcosm.fit import Regime -from microcosm.frame import US_SCHEMA, Frame, WeightKind +from microcosm.frame import US_SCHEMA, Frame, MassChangeRecord, WeightKind, Weights __all__ = [ "ACS_STACKED_SUPPORT_CHANNEL", @@ -249,6 +261,11 @@ "StackedLateProducerResult", "StackedPostPufTransferResult", "StackedSpineResult", + "StackedSpinePreparation", + "StackedSpineHarmonization", + "prepare_stacked_spine", + "harmonize_stacked_spine_weights", + "finish_stacked_spine", "assemble_stacked_spine", "assert_stacked_tail_cells_preserved", "by_origin_battery", @@ -553,7 +570,47 @@ def _acs_native_group_quarters_receipt( } -def assemble_stacked_spine( +@dataclass(frozen=True) +class _SampledStack: + spines: Mapping[str, Frame] + shares: Mapping[str, float] + incoming_masses: Mapping[str, float] + sampling: Mapping[str, object] + manifest_version: int + mass_anchor_channel: str + + +@dataclass(frozen=True) +class StackedSpinePreparation: + """DESIGN preparation and a distinct pre-harmonization receipt.""" + + frame: Frame + receipt: Mapping[str, object] + + def __post_init__(self) -> None: + if not isinstance(self.frame, Frame) or not isinstance(self.receipt, Mapping): + raise TypeError( + "StackedSpinePreparation requires a Frame and receipt mapping." + ) + object.__setattr__(self, "receipt", _freeze_context(self.receipt)) + _validate_stacked_preparation(self.frame, self.receipt) + + +@dataclass(frozen=True) +class StackedSpineHarmonization: + """Actual weights and completed context, with ordered legacy history.""" + + weights: Weights + metadata: Mapping[str, object] + receipt: Mapping[str, object] + legacy_mass_log: tuple[MassChangeRecord, ...] + + def __post_init__(self) -> None: + object.__setattr__(self, "metadata", _freeze_context(self.metadata)) + object.__setattr__(self, "receipt", _freeze_context(self.receipt)) + + +def _sample_stacked_inputs( asec: Frame, acs: Frame, *, @@ -563,27 +620,8 @@ def assemble_stacked_spine( sample_seed: int | None = None, household_mass_shares: Mapping[str, float] | None = None, mass_anchor_channel: str = BASE_ASEC_SUPPORT_CHANNEL, -) -> StackedSpineResult: - """Assemble uniformly sampled ASEC and ACS survey arms into one spine. - - Production callers provide the single ``sample_fraction`` and - ``sample_seed`` scale-ladder controls. The exact same fraction is applied - independently to both survey arms at whole-household grain; each sampled - arm is then normalized back to its full-source household mass before the - reviewed :func:`assemble_spines` harmonization. This preserves each arm's - composition and prevents the anchor population from shrinking with the - rung. PUF donors are not accepted here and therefore remain unsampled. - - ``acs_sample_fraction``/``acs_sample_seed`` retain the reviewed version-1 - pilot contract for reproducibility only: ASEC remains full and ACS alone - is sampled. Supplying pilot and production controls together fails - closed. - - Returns: - A validated :class:`StackedSpineResult` whose receipt mirrors the - frozen manifest as a JSON-ready mapping. - """ - +) -> _SampledStack: + """Run the unchanged pilot or production sampling/normalization controls.""" shares = ( dict(DEFAULT_STACKED_HOUSEHOLD_MASS_SHARES) if household_mass_shares is None @@ -688,6 +726,287 @@ def assemble_stacked_spine( }, } + return _SampledStack( + { + BASE_ASEC_SUPPORT_CHANNEL: sampled_asec, + ACS_STACKED_SUPPORT_CHANNEL: sampled_acs, + }, + shares, + incoming_masses, + sampling_manifest, + manifest_version, + mass_anchor_channel, + ) + + +def assemble_stacked_spine( + asec: Frame, + acs: Frame, + *, + acs_sample_fraction: float | None = None, + acs_sample_seed: int | None = None, + sample_fraction: float | None = None, + sample_seed: int | None = None, + household_mass_shares: Mapping[str, float] | None = None, + mass_anchor_channel: str = BASE_ASEC_SUPPORT_CHANNEL, +) -> StackedSpineResult: + """Compose actual sampling, stacking and importance harmonization. + + Production controls normalize each sampled source by M_full/M_sample. + This ratio is not generally inverse realized household inclusion N/n. + Legacy pilot controls and accepted source weight kinds remain unchanged. + """ + sampled = _sample_stacked_inputs( + asec, + acs, + acs_sample_fraction=acs_sample_fraction, + acs_sample_seed=acs_sample_seed, + sample_fraction=sample_fraction, + sample_seed=sample_seed, + household_mass_shares=household_mass_shares, + mass_anchor_channel=mass_anchor_channel, + ) + if all( + frame.weights_for("household").kind is WeightKind.DESIGN + for frame in sampled.spines.values() + ): + return finish_stacked_spine(_prepare_sampled_stack(sampled)) + # Existing direct callers may supply calibrated or mixed source kinds. + # They keep their historical path, never relabeled as a DESIGN anchor. + return _finish_legacy_sampled_stack(sampled) + + +def prepare_stacked_spine( + asec: Frame, + acs: Frame, + *, + acs_sample_fraction: float | None = None, + acs_sample_seed: int | None = None, + sample_fraction: float | None = None, + sample_seed: int | None = None, + household_mass_shares: Mapping[str, float] | None = None, + mass_anchor_channel: str = BASE_ASEC_SUPPORT_CHANNEL, +) -> StackedSpinePreparation: + """Sample/normalize actual DESIGN sources and stack them before allocation. + + Returns a dedicated preparation receipt, never an incomplete completed + stacked-spine manifest. No harmonized weights are precomputed. + """ + sampled = _sample_stacked_inputs( + asec, + acs, + acs_sample_fraction=acs_sample_fraction, + acs_sample_seed=acs_sample_seed, + sample_fraction=sample_fraction, + sample_seed=sample_seed, + household_mass_shares=household_mass_shares, + mass_anchor_channel=mass_anchor_channel, + ) + return _prepare_sampled_stack(sampled) + + +def _prepare_sampled_stack(sampled: _SampledStack) -> StackedSpinePreparation: + prepared = prepare_spines( + sampled.spines, + household_mass_shares=sampled.shares, + mass_anchor_channel=sampled.mass_anchor_channel, + ) + samples = sampled.sampling.get( + "survey_samples", + {ACS_STACKED_SUPPORT_CHANNEL: sampled.sampling.get("acs_sample")}, + ) + full_masses = { + channel: float(samples[channel]["incoming_household_mass"]) + if channel in samples + else float(sampled.incoming_masses[channel]) + for channel in sampled.spines + } + receipt = { + "schema": "microcosm.us.stacked-spine-preparation.v1", + "stacked_manifest_version": sampled.manifest_version, + "sampling": sampled.sampling, + "full_source_masses": full_masses, + "household_mass_shares": sampled.shares, + "mass_anchor_channel": sampled.mass_anchor_channel, + "assembly_context": prepared.context, + "assembly_metadata": prepared.frame.metadata, + "acs_native_group_quarters": _acs_native_group_quarters_receipt( + sampled.spines[ACS_STACKED_SUPPORT_CHANNEL], prepared.frame + ), + } + return StackedSpinePreparation(prepared.frame, receipt) + + +def _validate_stacked_preparation(frame: Frame, receipt: Mapping[str, object]) -> None: + boundary = "stacked spine preparation" + if frame.weights_for("household").kind is not WeightKind.DESIGN: + raise ValueError("Stacked spine preparation requires actual DESIGN weights.") + if STACKED_SPINE_MANIFEST_KEY in frame.metadata: + raise ValueError( + "A preparation must not carry a completed stacked spine manifest." + ) + validate_assembly_provenance(frame, boundary=boundary) + _validate_preparation_receipt(receipt) + if _json_ready(receipt["assembly_metadata"]) != _json_ready(frame.metadata): + raise ValueError("Prepared assembly metadata differs from the actual frame.") + _validated_spine_inputs( + frame.table("household"), + frame.weights_for("household").values, + receipt["assembly_context"], + ) + version, sampling = receipt["stacked_manifest_version"], receipt["sampling"] + production = version == _STACKED_SPINE_MANIFEST_VERSION + fraction = sampling["sample_fraction" if production else "acs_sample_fraction"] + seed = sampling["sample_seed" if production else "acs_sample_seed"] + _validate_fraction(fraction) + _validate_seed(seed) + samples = ( + sampling["survey_samples"] + if production + else {ACS_STACKED_SUPPORT_CHANNEL: sampling["acs_sample"]} + ) + expected = ( + {BASE_ASEC_SUPPORT_CHANNEL, ACS_STACKED_SUPPORT_CHANNEL} + if production + else {ACS_STACKED_SUPPORT_CHANNEL} + ) + if set(samples) != expected: + raise ValueError( + "Prepared sampling receipts do not cover their declared sources." + ) + for channel, sample in samples.items(): + _validate_survey_sample_receipt( + frame, + channel=channel, + fraction=fraction, + seed=seed, + sample=sample, + boundary=boundary, + require_normalization=production, + ) + if float(receipt["full_source_masses"][channel]) != float( + sample["incoming_household_mass"] + ): + raise ValueError( + "Prepared full-source mass differs from its sampling receipt." + ) + _validate_stacked_clone_role_lifecycle(frame, boundary=boundary) + _validated_acs_native_group_quarters_masks(frame, receipt, boundary=boundary) + + +def _validate_preparation_receipt(preparation: Mapping[str, object]) -> None: + if ( + not isinstance(preparation, Mapping) + or preparation.get("schema") != "microcosm.us.stacked-spine-preparation.v1" + ): + raise ValueError("Unsupported stacked-spine preparation receipt.") + if ( + preparation.get("stacked_manifest_version") + not in _SUPPORTED_STACKED_SPINE_MANIFEST_VERSIONS + ): + raise ValueError("Unsupported prepared stacked-spine manifest version.") + for field_name in ( + "sampling", + "full_source_masses", + "household_mass_shares", + "assembly_context", + "assembly_metadata", + "acs_native_group_quarters", + ): + if not isinstance(preparation.get(field_name), Mapping): + raise ValueError( + f"Prepared stacked-spine field {field_name!r} must be a mapping." + ) + context = preparation["assembly_context"] + if preparation.get("mass_anchor_channel") != context.get( + "mass_anchor_channel" + ) or preparation["household_mass_shares"] != context.get("household_mass_shares"): + raise ValueError("Prepared stack controls disagree with the assembly context.") + if set(preparation["full_source_masses"]) != { + BASE_ASEC_SUPPORT_CHANNEL, + ACS_STACKED_SUPPORT_CHANNEL, + }: + raise ValueError("Prepared full-source masses must name ASEC and ACS.") + + +def harmonize_stacked_spine_weights( + *, + household: pd.DataFrame, + weights: Weights, + preparation: Mapping[str, object], +) -> StackedSpineHarmonization: + """Compute importance weights and completed metadata from declared views. + + The preparation producer validates full source/person lineage. This helper + binds its household views and performs allocation without restacking or + reading a precomputed final-weight vector. The returned legacy mass log is + distinct from a graph executor's structural transition history. + """ + _validate_preparation_receipt(preparation) + result = harmonize_spine_weights( + household=household, weights=weights, context=preparation["assembly_context"] + ) + shares = preparation["household_mass_shares"] + incoming = preparation["assembly_context"]["incoming_masses"] + anchor = preparation["mass_anchor_channel"] + harmonization = _harmonization_receipt_from_views( + household, + result.weights, + shares=shares, + anchor_mass=incoming[anchor], + incoming_masses=incoming, + ) + manifest = { + "version": preparation["stacked_manifest_version"], + **preparation["sampling"], + "household_mass_shares": { + channel: float(share) for channel, share in shares.items() + }, + "mass_anchor_channel": anchor, + "weight_harmonization": harmonization, + "acs_native_group_quarters": preparation["acs_native_group_quarters"], + } + metadata = { + **preparation["assembly_metadata"], + STACKED_SPINE_MANIFEST_KEY: manifest, + } + return StackedSpineHarmonization( + result.weights, metadata, _json_ready(manifest), result.mass_log + ) + + +def finish_stacked_spine(prepared: StackedSpinePreparation) -> StackedSpineResult: + """Complete a validated preparation, preserving exact legacy Frame history.""" + if not isinstance(prepared, StackedSpinePreparation): + raise TypeError("finish_stacked_spine requires a StackedSpinePreparation.") + _validate_stacked_preparation(prepared.frame, prepared.receipt) + result = harmonize_stacked_spine_weights( + household=prepared.frame.table("household"), + weights=prepared.frame.weights_for("household"), + preparation=prepared.receipt, + ) + frame = Frame( + {entity: prepared.frame.table(entity) for entity in prepared.frame.entities}, + prepared.frame.schema, + {"household": result.weights}, + prepared.frame.strata, + mass_log=result.legacy_mass_log, + metadata=result.metadata, + ) + validated = validate_stacked_spine_frame( + frame, boundary="stacked spine assembly output" + ) + return StackedSpineResult(frame, _json_ready(validated)) + + +def _finish_legacy_sampled_stack(sampled: _SampledStack) -> StackedSpineResult: + sampled_asec, sampled_acs = ( + sampled.spines[BASE_ASEC_SUPPORT_CHANNEL], + sampled.spines[ACS_STACKED_SUPPORT_CHANNEL], + ) + shares, incoming_masses = sampled.shares, sampled.incoming_masses + sampling_manifest, manifest_version = sampled.sampling, sampled.manifest_version + mass_anchor_channel = sampled.mass_anchor_channel assembled = assemble_spines( { BASE_ASEC_SUPPORT_CHANNEL: sampled_asec, @@ -1632,9 +1951,25 @@ def _harmonization_receipt( anchor_mass: float, incoming_masses: Mapping[str, float], ) -> dict[str, dict[str, float]]: - household = assembled.table("household") + return _harmonization_receipt_from_views( + assembled.table("household"), + assembled.weights_for("household"), + shares=shares, + anchor_mass=anchor_mass, + incoming_masses=incoming_masses, + ) + + +def _harmonization_receipt_from_views( + household: pd.DataFrame, + household_weights: Weights, + *, + shares: Mapping[str, float], + anchor_mass: float, + incoming_masses: Mapping[str, float], +) -> dict[str, dict[str, float]]: channel_values = household[support_channel_column("household")].astype(str) - weights = np.asarray(assembled.weights_for("household").values, dtype=np.float64) + weights = np.asarray(household_weights.values, dtype=np.float64) receipt: dict[str, dict[str, float]] = {} for channel, share in shares.items(): incoming = float(incoming_masses[channel]) @@ -5104,7 +5439,6 @@ def validate_stacked_post_puf_transfer_receipt( ) -_LATE_TABLE_DIGEST_CODEC = "canonical_scalar_v1" _LATE_PRIMARY_QRF_INPUT_BINDING_ARTIFACT_KIND = ( "populace_us_stacked_late_primary_qrf_input_binding" ) @@ -5112,259 +5446,6 @@ def validate_stacked_post_puf_transfer_receipt( _LATE_RESOURCE_SEMANTICS_ARTIFACT_KIND = ( "populace_us_stacked_late_producer_resource_semantics" ) -_LATE_TABLE_DIGEST_CHUNK_ROWS = 65_536 - - -def _late_digest_part( - digest, - *, - domain: str, - payload: bytes | bytearray | memoryview | np.ndarray, -) -> None: - """Append one length-framed, domain-separated byte field to a digest.""" - - domain_bytes = domain.encode("utf-8") - payload_view = memoryview(payload) - if payload_view.format != "B" or payload_view.ndim != 1: - payload_view = payload_view.cast("B") - digest.update(struct.pack(" np.ndarray: - """Return a contiguous, explicitly little-endian numeric byte source.""" - - array = np.asarray(values) - array = array.astype(array.dtype.newbyteorder("<"), copy=False) - return np.ascontiguousarray(array) - - -def _late_scalar_bytes(value: object) -> bytes: - """Encode one supported object scalar without lossy intermediary hashes.""" - - missing = pd.isna(value) - if isinstance(missing, (bool, np.bool_)) and bool(missing): - return b"null" - if isinstance(value, (bool, np.bool_)): - return b"bool\x01" if bool(value) else b"bool\x00" - if isinstance(value, (int, np.integer)): - return b"integer\x00" + str(int(value)).encode("ascii") - if isinstance(value, (float, np.floating)): - if isinstance(value, np.floating) and value.dtype.itemsize > 8: - raise TypeError( - "US late-producer content digest does not support object " - f"floating scalar {value.dtype!s}." - ) - return b"float64\x00" + struct.pack(" 16: - raise TypeError( - "US late-producer content digest does not support object " - f"complex scalar {value.dtype!s}." - ) - numeric = complex(value) - return b"complex128\x00" + struct.pack(" None: - """Stream framed string or object scalars in bounded-memory chunks.""" - - for chunk_index, start in enumerate( - range(0, len(values), _LATE_TABLE_DIGEST_CHUNK_ROWS) - ): - stop = min(start + _LATE_TABLE_DIGEST_CHUNK_ROWS, len(values)) - lengths = np.zeros(stop - start, dtype=" None: - """Hash one ordered logical Series with explicit dtype and null domains.""" - - dtype = series.dtype - missing = series.isna().to_numpy(dtype=bool) - _late_digest_part( - digest, - domain=f"{domain}/dtype", - payload=str(dtype).encode("utf-8"), - ) - _late_digest_part( - digest, - domain=f"{domain}/row_count", - payload=struct.pack(" str: - """Hash ordered table scalars directly with typed, null-aware framing.""" - - values = ( - canonicalize_table_string_dtypes( - table, - boundary="late-producer content digest", - table_name="declared_surface", - ) - if normalize_strings - else table - ) - if isinstance(values.index, pd.MultiIndex): - index_levels = [ - pd.Series(values.index.get_level_values(level), copy=False) - for level in range(values.index.nlevels) - ] - else: - index_levels = [pd.Series(values.index, copy=False)] - header = { - "codec": _LATE_TABLE_DIGEST_CODEC, - "columns": [str(column) for column in values.columns], - "dtypes": [ - str(values.iloc[:, index].dtype) for index in range(values.shape[1]) - ], - "index_type": type(values.index).__name__, - "index_dtype": str(values.index.dtype), - "index_level_dtypes": [str(level.dtype) for level in index_levels], - "index_names": [ - None if name is None else str(name) for name in values.index.names - ], - } - digest = hashlib.sha256() - _late_digest_part( - digest, - domain="late_table_digest_codec", - payload=_LATE_TABLE_DIGEST_CODEC.encode("ascii"), - ) - _late_digest_part( - digest, - domain="late_table_header_json", - payload=json.dumps( - header, - sort_keys=True, - separators=(",", ":"), - allow_nan=False, - ).encode("utf-8"), - ) - for level_index, level in enumerate(index_levels): - _late_digest_series_values( - digest, - level, - domain=f"index_level/{level_index}", - ) - for column_index in range(values.shape[1]): - _late_digest_series_values( - digest, - values.iloc[:, column_index], - domain=f"column/{column_index}", - ) - return digest.hexdigest() def _late_virtual_resource_kind(column: str) -> str: diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/survey_age_activation.py b/packages/microcosm-build/src/microcosm/build/us_runtime/survey_age_activation.py new file mode 100644 index 000000000..deffa57ee --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/survey_age_activation.py @@ -0,0 +1,220 @@ +"""Explicit S0101-only activation for the selected ACS/ASEC development path. + +A reviewed declaration pins two source responses. The caller's review and +guarded acquisition supply trust; this decoder is not independent attestation. +There is deliberately no default genuine declaration or mixed-inventory reader. +""" + +from __future__ import annotations + +import hashlib +from dataclasses import dataclass, replace +from pathlib import Path + +from microcosm.calibrate.registry import TargetRegistry, TargetSpec +from microcosm.graph.canonical import canonical_json + +from . import demographic_calibration_graph as demographic +from . import national_age_activation as age +from .survey_age_sources import _digest, read_survey_age_response, survey_age_requests + +PROFILE = "selected_survey_s0101_development_v1" +_CONVENTION = ( + "ACS 2024 observed interview ages and ASEC 2025 interview ages with income " + "year 2024; completed years used verbatim without temporal aging; calibration " + "to the published 2024 all-resident distribution does not establish that " + "the selected survey population covers that complete universe" +) + + +def _require(condition: object, reason: str) -> None: + if not condition: + raise ValueError("SURVEY_AGE_ACTIVATION_" + reason) + + +@dataclass(frozen=True, slots=True) +class SurveyAgeActivation: + """Exact source pins whose approval is a separate caller responsibility.""" + + metadata_sha256: str + data_sha256: str + + def __post_init__(self) -> None: + _digest(self.metadata_sha256) + _digest(self.data_sha256) + + +def activation_binding(declaration: SurveyAgeActivation) -> dict: + """Return the complete, detached semantic declaration, without source I/O.""" + _require(type(declaration) is SurveyAgeActivation, "DECLARATION") + declaration.__post_init__() + return { + "profile": PROFILE, + "requests": list(survey_age_requests()), + "metadata_sha256": declaration.metadata_sha256, + "data_sha256": declaration.data_sha256, + "period": "2024", + "entity": "household", + "universe": age.NATIONAL_AGE_ACTIVATION.universe, + "age_convention_id": "observed_interview_age_completed_years", + "age_convention": _CONVENTION, + "bands": [ + { + "variable": b.variable, + "label": b.label, + "low": b.low, + "high": b.high, + "column": b.column, + } + for b in age.NATIONAL_AGE_ACTIVATION.bands + ], + "published_total": "S0101_C01_001_consistency_only", + "consumes_se": False, + "release_eligible": False, + } + + +def declaration_from_binding(binding: object) -> SurveyAgeActivation: + """Validate value-level profile semantics; grant no source authorization.""" + _require(type(binding) is dict, "DECLARATION") + _require("metadata_sha256" in binding and "data_sha256" in binding, "DECLARATION") + declaration = SurveyAgeActivation( + binding["metadata_sha256"], binding["data_sha256"] + ) + _require( + canonical_json(binding) == canonical_json(activation_binding(declaration)), + "DECLARATION", + ) + return declaration + + +def activation_digest(declaration: SurveyAgeActivation) -> str: + """Bind source pins and all selected-survey conventions in one identifier.""" + return hashlib.sha256(canonical_json(activation_binding(declaration))).hexdigest() + + +def validate_survey_age_registry( + registry: TargetRegistry, binding: dict +) -> TargetRegistry: + """Check numerical profile metadata, not the truth of caller-supplied values.""" + declaration = declaration_from_binding(binding) + frozen = demographic._registry_from_json(demographic._registry_json(registry)) + bands = age.NATIONAL_AGE_ACTIVATION.bands + _require(len(frozen) == len(bands), "REGISTRY") + digest = activation_digest(declaration) + for spec, band in zip(frozen, bands, strict=True): + _require( + spec.name == band.variable + and spec.measure == band.column + and spec.metadata["table"] == "S0101" + and spec.metadata["reference_sha256"] == digest + and spec.metadata["evidence_scope"] == "source_documented", + "REGISTRY", + ) + return frozen + + +def activate_survey_age_targets( + source_dir: str | Path, *, declaration: SurveyAgeActivation +) -> TargetRegistry: + """Derive exactly 18 targets from the two explicitly pinned local responses.""" + binding = activation_binding(declaration) + _, metadata = read_survey_age_response( + source_dir, "metadata", declaration.metadata_sha256 + ) + _, data = read_survey_age_response(source_dir, "data", declaration.data_sha256) + # Reuse the established publisher/count/MOE interpretation without invoking + # its old inventory loader. This object supplies only pure field semantics. + semantics = replace(age.NATIONAL_AGE_ACTIVATION, age_convention=_CONVENTION) + row = age._national_row(data, semantics) + _require( + all( + name in {"GEO_ID", "NAME", "us"} or name.startswith("S0101_") + for name in row + ), + "RESPONSE_SCOPE", + ) + _require( + type(metadata) is dict and type(metadata.get("variables")) is dict, "METADATA" + ) + _require( + all( + type(name) is str + and (name in {"GEO_ID", "NAME", "us"} or name.startswith("S0101_")) + for name in metadata["variables"] + ), + "METADATA_SCOPE", + ) + + def cell(variable): + _require( + all(variable + suffix in row for suffix in ("E", "EA", "M", "MA")), "CELL" + ) + return { + "estimate": age.classify_acs_value( + row[variable + "E"], row[variable + "EA"] + ), + "moe": age.classify_acs_value(row[variable + "M"], row[variable + "MA"]), + } + + digest = activation_digest(declaration) + specs = [] + data_url = survey_age_requests()[1]["url"] + for band in semantics.bands: + published = age._published_variable(metadata, band.variable + "E", semantics) + _require( + published.get("label") == band.label + and published.get("predicateType") == "int", + "PUBLISHED_DEFINITION", + ) + derived = cell(band.variable) + specs.append( + TargetSpec( + name=band.variable, + entity="household", + measure=band.column, + value=float(age._count_estimate(derived, band.variable)), + period="2024", + se=age._standard_error(derived, band.variable, confidence_level=0.9), + source=f"{data_url} (S0101 {band.variable}E)", + family="acs.S0101", + hierarchy=age.demographic_target_hierarchy( + "S0101", band, geography="0100000US" + ), + notes=f"activation={digest}; label={band.label}; age_convention={_CONVENTION}; published 90% MOE divided by {age._MOE_90_TO_SE}, not consumed by current loss", + metadata={ + "table": "S0101", + "reference_sha256": digest, + "geography": "0100000US", + "universe": "population", + "role": "calibration", + "evidence_scope": "source_documented", + }, + ) + ) + total = age._published_variable(metadata, "S0101_C01_001E", semantics) + _require( + total.get("label") == "Estimate!!Total!!Total population" + and total.get("predicateType") == "int", + "PUBLISHED_TOTAL_DEFINITION", + ) + _require( + sum(int(s.value) for s in specs) + == age._count_estimate(cell("S0101_C01_001"), "S0101_C01_001"), + "PARTITION", + ) + return validate_survey_age_registry(TargetRegistry(specs, country="us"), binding) + + +def verify_survey_age_targets( + source_dir: str | Path, + *, + declaration: SurveyAgeActivation, + registry: TargetRegistry, +) -> None: + """Rederive the pinned target values; no stored registry substitutes for this.""" + actual = activate_survey_age_targets(source_dir, declaration=declaration) + _require( + demographic._registry_json(actual) == demographic._registry_json(registry), + "REGISTRY", + ) diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/survey_age_calibration.py b/packages/microcosm-build/src/microcosm/build/us_runtime/survey_age_calibration.py new file mode 100644 index 000000000..c2051861b --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/survey_age_calibration.py @@ -0,0 +1,952 @@ +"""Country admission for survey age profiles with an explicit source prefix. + +Every call reconstructs the accepted source preparation. The numerical kernels +are values-only; source, full-population, group/row-cap and cache checks live +here. The original entry point remains invented-only; the named development +entry point derives its targets from an explicitly pinned S0101-only capture. +Both permit only the first IMPORTANCE-to-CALIBRATED transition and no release. +An optional atomic-geography recipe enriches the accepted allocation before +cloning; the sampling budget continues to retain the original raw allocation. +""" + +from __future__ import annotations + +import hashlib +import json +import os +import stat +from dataclasses import dataclass, replace +from types import SimpleNamespace + +import numpy as np + +from microcosm.frame import Frame +from microcosm.graph import ( + CompiledGraph, + KernelRegistry, + StructuralDelta, + compile_graph, + run_graph, +) +from microcosm.graph.artifact_edges import typed_contracts +from microcosm.graph.canonical import canonical_json +from microcosm.graph.executor import ( + _all_node_keys, + _input_writers, + _project_context, + _source_paths_and_keys, + _tolerance_writer_payload, +) +from microcosm.graph.keys import ( + _capabilities_projection, + artifact_key, + frame_key, + opaque_artifact_key, + seed, + weights_key, +) +from microcosm.graph.manifest import _freeze_json +from microcosm.graph.population import ( + Population, + mass_record_receipt, + weight_cap_receipt, +) +from microcosm.graph.store import _verified_meta + +from . import graph_combined_clone as clone +from . import graph_survey_age_artifact as ages +from . import graph_survey_budget as transport +from . import graph_survey_calibration as numerical +from . import graph_survey_population as source_graph +from . import survey_age_activation as age_activation +from . import survey_calibration_diagnostics as diagnostic_check +from . import survey_origin_budget as budgets +from . import survey_population_replay as replay + +COUNT_NODE = "survey.age_count_matrix" + + +def _require(condition, reason): + if not condition: + raise ValueError("SURVEY_AGE_RUN_" + reason) + + +def _sha(payload): + return hashlib.sha256(payload).hexdigest() + + +def _node_identity(node): + """Immutable snapshot excluding execution time and cache-hit bookkeeping.""" + return canonical_json( + { + "key": node.key, + "ref": node.kernel_ref, + "implementation": node.kernel_impl_hash, + "capabilities": _capabilities_projection(node.capabilities), + "typed_artifacts": node.typed_artifacts, + "seed": node.seed, + "frame_key": node.frame_key, + "weight_key": node.weight_key, + "artifacts": [ + [entity, column, key] + for (entity, column), key in sorted(node.artifacts.items()) + ], + "opaque_artifacts": node.opaque_artifacts, + "receipt": node.receipt, + "legacy_capabilities": node.legacy_capabilities, + } + ) + + +def _measurement(population): + node = ages.survey_age_count_artifact_node( + population=population.version, node_id=COUNT_NODE + ) + context = _project_context( + node, + population, + key="0" * 64, + sources={}, + tolerances={}, + numerics={}, + ) + measured = ages.SurveyAgeCountArtifactKernel().run(context) + payload = measured.artifacts["counts"] + values = ages.decode_survey_age_counts(payload) + _require( + tuple(values.household_ids) + == tuple(population.frame.table("household").household_id), + "MEASUREMENT_HOUSEHOLD_ORDER", + ) + return payload, json.loads(canonical_json(measured.receipt)) + + +def _measure(population): + return _measurement(population)[0] + + +def _load_diagnostics(store, key): + """Bound the diagnostic payload buffer before reading it. + + The existing store verifier still authenticates metadata and streams its + payload hashes. This does not claim a bound on that verifier's metadata + decoding or total I/O. Only the selected diagnostic payload is buffered here. + """ + path = store.object_path(key) + payload_path = path / "payload.bin" + flags = os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK + fd = os.open(payload_path, flags) + try: + before = os.fstat(fd) + limit = diagnostic_check.MAX_DIAGNOSTIC_BYTES + _require( + stat.S_ISREG(before.st_mode) and 0 < before.st_size <= limit, + "DIAGNOSTIC_STORE_LIMIT", + ) + metadata = _verified_meta(path, expected_kind="bytes") + _require( + set(metadata["payloads"]) == {"payload.bin"}, "DIAGNOSTIC_STORE_ROSTER" + ) + record = metadata["payloads"]["payload.bin"] + _require(record["size"] == before.st_size, "DIAGNOSTIC_STORE_SIZE") + with os.fdopen(fd, "rb", closefd=False) as stream: + payload = stream.read(limit + 1) + after = os.fstat(fd) + current = os.stat(payload_path, follow_symlinks=False) + + def seal(s): + return (s.st_dev, s.st_ino, s.st_size, s.st_mtime_ns, s.st_ctime_ns) + + _require( + seal(before) == seal(after) == seal(current) + and stat.S_ISREG(current.st_mode) + and len(payload) == record["size"] <= limit + and _sha(payload) == record["sha256"], + "DIAGNOSTIC_STORE_CHANGED", + ) + return payload + finally: + os.close(fd) + + +def _expected_nodes( + compiled: CompiledGraph, + kernels: KernelRegistry, + keys: dict[str, str], + implementations: dict[str, str], + prefix_identities: dict[str, bytes], + frame: Frame, +): + """Detach declaration expectations before any graph kernel/store execution.""" + states = {} + cells = tuple((e, str(c)) for e in frame.entities for c in frame.table(e)) + for node in compiled.graph.nodes: + if node.id in prefix_identities: + state = json.loads(prefix_identities[node.id]) + state["artifacts"] = {(e, c): k for e, c, k in state["artifacts"]} + # The accepted prefix runner already independently sealed this + # receipt. No receipt from the receiving run is trusted here. + states[node.id] = state + continue + key = keys[node.id] + structural = node.structural is not StructuralDelta.NONE + outputs = ( + cells if structural else tuple((o.entity, o.column) for o in node.outputs) + ) + states[node.id] = { + "key": key, + "ref": node.kernel, + "implementation": implementations[node.id], + "capabilities": json.loads( + canonical_json( + _capabilities_projection(kernels.get(node.kernel).capabilities) + ) + ), + "typed_artifacts": json.loads( + canonical_json(typed_contracts(compiled, node, keys, kernels)) + ), + "seed": seed(key), + "frame_key": frame_key(key) if structural else None, + "weight_key": weights_key(key, node.weights.entity) + if node.weights + else None, + "artifacts": {(e, c): artifact_key(key, e, c) for e, c in outputs}, + "opaque_artifacts": { + o.name: opaque_artifact_key(key, o.name) for o in node.artifact_outputs + }, + } + return states + + +def _complete_receipts( + compiled: CompiledGraph, + states: dict[str, dict[str, object]], + authored: dict[str, dict[str, object]], + observed: dict[str, Population], +): + """Normalize independently authored receipts through the executor's rules.""" + previous = {} + for node_id in compiled.order: + state = states[node_id] + if node_id in authored: + node = compiled.graph.node(node_id) + receipt = json.loads(canonical_json(authored[node_id])) + receipt["capabilities"] = dict(state["capabilities"]) + writers = _tolerance_writer_payload( + _input_writers(compiled, node_id, receipts=previous) + ) + if writers: + receipt["capabilities"]["tolerance_writers"] = writers + if node.structural not in {StructuralDelta.NONE, StructuralDelta.CREATE}: + receipt["mass"] = mass_record_receipt(observed[node_id].mass_ledger[-1]) + receipt.update(weight_cap_receipt(observed[node_id], node)) + state["receipt"] = json.loads(canonical_json(receipt)) + previous[node_id] = SimpleNamespace(receipt=state["receipt"]) + + +def _check_manifest(manifest, compiled, states): + _require( + manifest.country == compiled.graph.country and manifest.decisions == (), + "MANIFEST_COUNTRY_DECISIONS", + ) + _require( + set(manifest.populations) == set(compiled.versions.values()) + and set(manifest.mass_ledgers) == set(compiled.versions.values()), + "MANIFEST_POPULATION_VERSIONS", + ) + # Detached expectations use JSON arrays. NodeReceipt freezes those arrays + # as tuples, including nested capabilities and typed artifact descriptors. + # Match its representation without weakening the canonical value check. + frozen_states = { + node_id: { + **state, + "receipt": _freeze_json(state["receipt"]), + "typed_artifacts": _freeze_json(state["typed_artifacts"]), + } + for node_id, state in states.items() + } + source_graph._check_node_states(manifest, frozen_states) + for node_id, state in states.items(): + expected = {**state, "legacy_capabilities": False} + expected["artifacts"] = [ + [e, c, key] for (e, c), key in sorted(state["artifacts"].items()) + ] + _require( + _node_identity(manifest.node(node_id)) == canonical_json(expected), + "MANIFEST_CANONICAL_VALUES", + ) + + +@dataclass(frozen=True) +class SurveyAgeCalibrationRun: + """Values validated at return; retained source handles still require rechecks.""" + + manifest: object + compiled: object + budget: object + successor: object + diagnostics: dict + counts_sha256: str + numeric_bounds_sha256: str + release_eligible: bool = False + + +def run_survey_age_calibration( + source_dir, + *, + snapshot_root, + store_root, + fraction, + seed_value, + target_registry, + epochs, + learning_rate, + resume="auto", + geography_config=None, +): + """Reconstruct sources and admit a complete invented calibration graph. + + The source and store paths are the caller's explicitly approved local + inputs. Cache reuse never bypasses fresh source preparation or the complete + receiving-population and sampling-reference checks. This v1 does not admit + genuine targets, donor-detail descendants, pruning or repeated calibration. + """ + # Refuse unsupported targets/options before any source I/O. + calibration = numerical.survey_age_calibration_node( + target_registry, + base=clone.COMBINED_CLONE_NODE, + budget_node=transport.BUDGET_NODE, + count_node=COUNT_NODE, + epochs=epochs, + learning_rate=learning_rate, + ) + return _run_survey_age_calibration( + source_dir, + snapshot_root=snapshot_root, + store_root=store_root, + fraction=fraction, + seed_value=seed_value, + epochs=epochs, + learning_rate=learning_rate, + resume=resume, + calibration=calibration, + geography_config=geography_config, + ) + + +def run_survey_age_development( + source_dir, + *, + age_source_dir, + activation, + snapshot_root, + store_root, + fraction, + seed_value, + epochs, + learning_rate, + resume="auto", + geography_config=None, +): + """Run development calibration after deriving targets from exact source pins. + + The caller separately reviews the declaration and explicitly admits these + local source paths. There is no default capture, caller-supplied registry, + mixed-inventory fallback, production attestation or release permission. + """ + _require(type(resume) is str and resume in {"auto", "require"}, "RESUME") + _require(type(epochs) is int and 1 <= epochs <= 1000, "EPOCHS") + _require( + type(learning_rate) in (int, float) + and np.isfinite(learning_rate) + and 0 < learning_rate <= 1, + "LEARNING_RATE", + ) + binding = age_activation.activation_binding(activation) + registry = age_activation.activate_survey_age_targets( + age_source_dir, declaration=activation + ) + calibration = numerical.survey_age_development_node( + registry, + activation_binding=binding, + base=clone.COMBINED_CLONE_NODE, + budget_node=transport.BUDGET_NODE, + count_node=COUNT_NODE, + epochs=epochs, + learning_rate=learning_rate, + ) + return _run_survey_age_calibration( + source_dir, + snapshot_root=snapshot_root, + store_root=store_root, + fraction=fraction, + seed_value=seed_value, + epochs=epochs, + learning_rate=learning_rate, + resume=resume, + calibration=calibration, + age_source_dir=age_source_dir, + activation=activation, + geography_config=geography_config, + ) + + +def _run_survey_age_calibration( + source_dir, + *, + snapshot_root, + store_root, + fraction, + seed_value, + epochs, + learning_rate, + resume, + calibration, + age_source_dir=None, + activation=None, + geography_config=None, +): + """Shared population, budget, replay, diagnostic and final-return checks.""" + _require(type(resume) is str and resume in {"auto", "require"}, "RESUME") + activation_identity = None + calibration_binding = numerical.calibration_activation_binding(calibration.params) + _require( + (calibration_binding is None) == (activation is None), "ACTIVATION_REQUIRED" + ) + if activation is not None: + activation_identity = canonical_json( + age_activation.activation_binding(activation) + ) + _require( + canonical_json(calibration_binding) == activation_identity, + "ACTIVATION_BINDING", + ) + geography = None + geography_identity = None + geography_config_payload = None + prefix_populations = {} + if geography_config is None: + prefix = source_graph.run_authenticated_survey_population( + source_dir, + snapshot_root=snapshot_root, + store_root=store_root, + fraction=fraction, + seed=seed_value, + resume=resume, + clones=True, + return_values=True, + ) + else: + from . import graph_atomic_survey_population as atomic_source + + geography_config_payload = ( + atomic_source.reconstruction.AtomicSurveyReconstruction.to_bytes( + geography_config + ) + ) + prefix = atomic_source.run_atomic_survey_population( + source_dir, + snapshot_root=snapshot_root, + store_root=store_root, + fraction=fraction, + seed=seed_value, + resume=resume, + geography_config=geography_config, + return_values=True, + ) + geography = atomic_source.reconstruction.reconstruct_atomic_survey_geography( + prefix.preparation, prefix.allocated_population, geography_config + ) + replay.same_replayed_population( + geography.population, prefix.geography_population + ) + geography_identity = budgets._population_identity(prefix.geography_population) + prefix_populations = { + stage.node.id: stage.population for stage in geography.stages + } + source_owner, source_view = source_graph._checked_preparation(prefix.preparation) + preparation_entry = source_owner._ISSUED.get(id(prefix.preparation)) + instructions = source_graph.allocation_instructions( + source_view.selection_plan, source_view.receipt["origins"]["households"] + ) + _, allocated_context, allocation_payload, _, _ = source_graph._allocation_output( + source_view.frame, source_view.context, instructions, _sha(source_view.payload) + ) + prefix_artifacts = ( + ( + source_graph.CREATE_NODE, + "preparation", + source_graph.PREPARATION_TYPE, + source_view.payload, + source_graph.SurveyPopulationCreateKernel.capabilities, + ), + ( + source_graph.CREATE_NODE, + "frame_context", + source_graph.US_FRAME_CONTEXT_TYPE, + source_view.context, + source_graph.SurveyPopulationCreateKernel.capabilities, + ), + ( + source_graph.ALLOCATION_NODE, + "allocation", + source_graph.ALLOCATION_TYPE, + allocation_payload, + source_graph.SurveyPopulationAllocationKernel.capabilities, + ), + ( + source_graph.ALLOCATION_NODE, + "frame_context", + source_graph.US_FRAME_CONTEXT_TYPE, + allocated_context, + source_graph.SurveyPopulationAllocationKernel.capabilities, + ), + ) + if geography is not None: + prefix_artifacts += tuple( + ( + node.id, + "support", + atomic_source.ATOMIC_SUPPORT_TYPE, + geography.support_payload, + atomic_source.AtomicSupportImportKernel.capabilities, + ) + for node in geography.nodes + if node.kernel == atomic_source.AtomicSupportImportKernel.ref + ) + gate = geography.stages[-1] + prefix_artifacts += ( + ( + gate.node.id, + "validation", + atomic_source.reconstruction.atomic_graph.ATOMIC_GEOGRAPHY_VALIDATION_TYPE, + gate.receipt, + atomic_source.reconstruction.atomic_graph.AtomicGeographyGateKernel.capabilities, + ), + ) + initial = prefix.clone_population + _require(initial is not None, "COMPLETE_CLONE_REQUIRED") + budget = budgets.freeze_survey_origin_budget( + prefix.preparation, + allocated_population=prefix.allocated_population, + clone_population=initial, + geography_config=geography_config, + ) + budget_view = budget.checked_view() + budget_payload = budget_view.payload + numeric_payload = transport.numeric_survey_budget_payload(budget_payload) + numeric_bounds = numerical.decode_numeric_survey_bounds(numeric_payload) + count_payload, count_receipt = _measurement(initial) + _require( + numeric_bounds.grouped.household_ids + == tuple(initial.frame.table("household").household_id), + "BUDGET_HOUSEHOLD_ORDER", + ) + prefix_identities = { + name: _node_identity(row) for name, row in prefix.manifest.nodes.items() + } + initial_identity = budgets._population_identity(initial) + allocation_identity = budgets._population_identity(prefix.allocated_population) + prefix_population_identities = { + name: budgets._population_identity(population) + for name, population in prefix_populations.items() + } + prefix_budget = budget + prefix_budget_entry = budgets._entry(prefix_budget, budgets.SamplingOriginBudget) + kernels = prefix.kernels + kernels.register(transport.SurveySamplingBudgetKernel(budget_payload)) + kernels.register(ages.SurveyAgeCountArtifactKernel()) + kernels.register(numerical.SurveyAgeCalibrationKernel()) + age_nodes = ( + transport.survey_sampling_budget_node(budget_sha256=_sha(budget_payload)), + ages.survey_age_count_artifact_node( + population=initial.version, node_id=COUNT_NODE + ), + calibration, + ) + graph = replace( + prefix.compiled.graph, + nodes=(*prefix.compiled.graph.nodes, *age_nodes), + ) + compiled = compile_graph(graph) + prefix_roster = set(prefix.compiled.order) + age_roster = {transport.BUDGET_NODE, COUNT_NODE, numerical.CALIBRATION_NODE} + _require( + set(prefix_identities) == prefix_roster + and not prefix_roster & age_roster + and tuple(node.id for node in age_nodes) + == (transport.BUDGET_NODE, COUNT_NODE, numerical.CALIBRATION_NODE) + and set(compiled.order) == prefix_roster | age_roster + and len(compiled.order) == len(prefix.compiled.order) + 3 + and tuple(name for name in compiled.order if name in prefix_roster) + == prefix.compiled.order, + "EXACT_GRAPH", + ) + _require( + clone.COMBINED_CLONE_CLAIM_NODE in compiled.predecessors[transport.BUDGET_NODE], + "OWNERSHIP_CLAIM_EDGE", + ) + _paths, source_keys = _source_paths_and_keys(compiled, prefix.sources, prefix.store) + keys, implementations = _all_node_keys(compiled, kernels, source_keys) + expected_nodes = _expected_nodes( + compiled, kernels, keys, implementations, prefix_identities, initial.frame + ) + observed, snapshots = {}, {} + admitted = None + successor_entry = None + successor_payload = None + receiving_initial = None + receiving_budget_entry = None + receiving_node = ( + clone.COMBINED_CLONE_CLAIM_NODE + if geography is None + else geography.stages[-1].node.id + ) + + def observe(node_id, population): + nonlocal admitted, budget, receiving_initial, receiving_budget_entry + nonlocal successor_entry, successor_payload + _require(node_id not in observed, "DUPLICATE_OBSERVATION") + if node_id == numerical.CALIBRATION_NODE: + _require(receiving_initial is not None, "RECEIVING_BUDGET_REQUIRED") + # Independent of the numerical kernel, on both execution and load. + numerical.check_numeric_survey_weights( + numeric_bounds, population.frame.weights_for("household").values + ) + admitted = budgets.admit_survey_weight_only_population( + budget, + previous=receiving_initial, + current=population, + ) + successor_entry = budgets._entry(admitted, budgets.SamplingOriginSuccessor) + successor_payload = admitted.payload + elif node_id in {transport.BUDGET_NODE, COUNT_NODE}: + # Read-only count/transport may precede the geography gate. Compare + # their complete current version, including whichever prefix writes + # have actually occurred. Calibration waits for every base member. + if geography is None: + replay.same_replayed_frame(initial.frame, population.frame) + else: + preceding = next( + name + for name in reversed(observed) + if name in prefix_populations + and compiled.versions[name] == initial.version + ) + replay.same_replayed_population( + prefix_populations[preceding], population + ) + _require( + population.frame.weights_for("household").values.tobytes() + == numeric_bounds.incoming.tobytes(), + "RECEIVING_INCOMING_WEIGHTS", + ) + if node_id == transport.BUDGET_NODE and geography is None: + _require(receiving_initial is not None, "RECEIVING_BUDGET_REQUIRED") + _require( + budgets._population_identity(population) + == snapshots[clone.COMBINED_CLONE_CLAIM_NODE], + "COMPLETE_CLAIMED_CLONE", + ) + elif node_id == COUNT_NODE: + _require(_measure(population) == count_payload, "REMEASURED_COUNTS") + elif node_id == source_graph.ALLOCATION_NODE: + replay.same_replayed_population(prefix.allocated_population, population) + elif node_id == receiving_node: + replay.same_replayed_population(initial, population) + _require(source_graph.ALLOCATION_NODE in observed, "ALLOCATION_REQUIRED") + # Actual source reconstruction rebinds an issued budget to this run's + # receiving objects. Candidate bytes grant no authority by themselves. + budget = budgets.freeze_survey_origin_budget( + prefix.preparation, + allocated_population=observed[source_graph.ALLOCATION_NODE], + clone_population=population, + candidate=budget_payload, + geography_config=geography_config, + ) + receiving_budget_entry = budgets._entry( + budget, budgets.SamplingOriginBudget + ) + receiving_initial = population + elif node_id == source_graph.CREATE_NODE: + replay.same_replayed_frame( + prefix.manifest.population(node_id), population.frame + ) + elif node_id == clone.COMBINED_CLONE_NODE: + actual_allocation = observed[ + source_graph.ALLOCATION_NODE + if geography is None + else atomic_source.projection.NODE + ] + source_graph._verify_cloned_frame( + actual_allocation.frame, + population.frame, + actual_allocation.design_weights["household"], + ) + if geography is None: + replay.same_replayed_frame(initial.frame, population.frame) + else: + replay.same_replayed_population(prefix_populations[node_id], population) + elif node_id in prefix_populations: + replay.same_replayed_population(prefix_populations[node_id], population) + else: + _require(False, "UNEXPECTED_NODE") + observed[node_id] = population + snapshots[node_id] = budgets._population_identity(population) + + manifest = run_graph( + compiled, + sources=prefix.sources, + store=prefix.store, + kernels=kernels, + resume=resume, + _population_observer=observe, + ) + _require( + tuple(observed) == compiled.order and admitted is not None, + "OBSERVATION_COVERAGE", + ) + _require(set(manifest.nodes) == set(compiled.order), "MANIFEST_NODES") + for node in graph.nodes: + actual = manifest.node(node.id) + _require( + actual.key == keys[node.id] + and actual.kernel_ref == node.kernel + and actual.kernel_impl_hash == implementations[node.id] + and actual.typed_artifacts == expected_nodes[node.id]["typed_artifacts"] + and actual.seed == seed(keys[node.id]) + and actual.legacy_capabilities is False, + "MANIFEST_NODE_IDENTITY", + ) + if node.id in prefix_identities: + _require( + _node_identity(actual) == prefix_identities[node.id], + "PREFIX_RECEIPT_REPLAY", + ) + for node_id, name, type_, payload, capabilities in ( + *prefix_artifacts, + ( + transport.BUDGET_NODE, + "budget", + budgets.BUDGET_TYPE, + budget_payload, + transport.SurveySamplingBudgetKernel.capabilities, + ), + ( + transport.BUDGET_NODE, + "numeric_bounds", + numerical.BOUNDS_TYPE, + numeric_payload, + transport.SurveySamplingBudgetKernel.capabilities, + ), + ( + COUNT_NODE, + "counts", + ages.COUNTS_TYPE, + count_payload, + ages.SurveyAgeCountArtifactKernel.capabilities, + ), + ): + source_graph._final_artifact( + manifest, + prefix.store, + node_id=node_id, + name=name, + type_=type_, + payload=payload, + capabilities=capabilities, + ) + receipt = manifest.node(numerical.CALIBRATION_NODE) + diagnostic_payload = _load_diagnostics( + prefix.store, + expected_nodes[numerical.CALIBRATION_NODE]["opaque_artifacts"]["diagnostics"], + ) + current = observed[numerical.CALIBRATION_NODE] + anchors = { + "budget_sha256": _sha(budget_payload), + "numeric_bounds_sha256": _sha(numeric_payload), + "counts_sha256": _sha(count_payload), + "accepted_weight_sha256": _sha( + current.frame.weights_for("household").values.tobytes() + ), + "constraint_digest": numeric_bounds.grouped.digest, + "weight_anchor": calibration.params["weight_anchor"], + "cap_enforcement": calibration.params["cap_enforcement"], + "fixed_zero_rows": int(np.count_nonzero(numeric_bounds.incoming == 0)), + } + _require( + _sha(diagnostic_payload) == receipt.receipt.get("diagnostics_sha256"), + "DIAGNOSTICS_DIGEST", + ) + _require( + all(receipt.receipt.get(k) == v for k, v in anchors.items()), "RECEIPT_ANCHORS" + ) + registry = numerical.demographic._registry_from_json(calibration.params["registry"]) + diagnostics = diagnostic_check.validate_survey_calibration_diagnostics( + diagnostic_payload, + counts_payload=count_payload, + bounds_payload=numeric_payload, + weights=current.frame.weights_for("household").values, + registry=registry, + epochs=epochs, + learning_rate=learning_rate, + anchors=anchors, + activation_binding=calibration_binding, + ) + diagnostic_identity = canonical_json(diagnostics) + _complete_receipts( + compiled, + expected_nodes, + { + transport.BUDGET_NODE: { + "budget_sha256": _sha(budget_payload), + "numeric_bounds_sha256": _sha(numeric_payload), + "constraint_digest": numeric_bounds.grouped.digest, + "group_count": numeric_bounds.grouped.group_count, + "release_eligible": False, + "source_admission": "required_from_country_runner", + }, + COUNT_NODE: count_receipt, + numerical.CALIBRATION_NODE: { + **anchors, + "diagnostics_sha256": _sha(diagnostic_payload), + "scope": numerical.calibration_receipt_scope(calibration.params), + "release_eligible": False, + "source_admission": "required_from_country_runner", + }, + }, + observed, + ) + _check_manifest(manifest, compiled, expected_nodes) + result = SurveyAgeCalibrationRun( + manifest, + compiled, + budget, + admitted, + diagnostics, + _sha(count_payload), + _sha(numeric_payload), + ) + final_numbers = numerical.decode_numeric_survey_bounds(numeric_payload) + # A population version can be shared by several nonstructural nodes. Check + # the actual last receiving state for every version, including the prefix. + terminal_nodes = tuple({compiled.versions[n]: n for n in compiled.order}.values()) + terminal_frames = { + node_id: manifest.population(compiled.versions[node_id]) + for node_id in terminal_nodes + } + terminal_ledgers = { + node_id: manifest.mass_ledger(compiled.versions[node_id]) + for node_id in terminal_nodes + } + # Finish storage and key/implementation I/O before final source/target + # revalidation and complete-population checks. No store read follows. + final_keys, final_implementations = _all_node_keys(compiled, kernels, source_keys) + _require( + final_keys == keys and final_implementations == implementations, + "FINAL_IMPLEMENTATIONS", + ) + _require(_measure(initial) == count_payload, "FINAL_REMEASUREMENT") + _require( + _measure(receiving_initial) == count_payload, "FINAL_RECEIVING_MEASUREMENT" + ) + # Retain both authorities; the original source/planned budget is not waived + # when a candidate-identical handle is issued for the actual receiving run. + budgets.verify_survey_origin_budget(prefix_budget) + budgets.verify_survey_weight_only_successor(admitted) + if activation is not None: + # Recheck target evidence before the optional final support read; + # complete population, configuration and owner seals follow both. + age_activation.verify_survey_age_targets( + age_source_dir, declaration=activation, registry=registry + ) + if geography is not None: + _require( + atomic_source.reconstruction.AtomicSurveyReconstruction.to_bytes( + geography_config + ) + == geography_config_payload, + "FINAL_GEOGRAPHY_CONFIG", + ) + final_support, _support_identity = atomic_source.reconstruction._read_support( + geography_config + ) + _require(final_support == geography.support_payload, "FINAL_GEOGRAPHY_SUPPORT") + _require( + source_owner._ISSUED.get(id(prefix.preparation)) is preparation_entry + and preparation_entry is not None + and prefix.preparation.payload == preparation_entry[1], + "FINAL_SOURCE_ISSUANCE", + ) + source_owner._pure_final(preparation_entry[2]) + if geography is not None: + _require( + atomic_source.reconstruction.AtomicSurveyReconstruction.to_bytes( + geography_config + ) + == geography_config_payload + and budgets._population_identity(prefix.geography_population) + == geography_identity + and all( + budgets._population_identity(population) + == prefix_population_identities[name] + for name, population in prefix_populations.items() + ), + "FINAL_GEOGRAPHY_PREFIX", + ) + _require(budget.payload == budget_payload, "FINAL_BUDGET_BYTES") + _require( + budgets._population_identity(initial) == initial_identity + and budgets._population_identity(prefix.allocated_population) + == allocation_identity, + "FINAL_PREFIX_POPULATIONS", + ) + for node_id, population in observed.items(): + _require( + budgets._population_identity(population) == snapshots[node_id], + "FINAL_POPULATION_STATE", + ) + # A nonstructural node can precede the claim within the same version; + # its own snapshot is checked above. Terminal version state is checked + # on the declared weight transition and complete ownership claim. + if node_id in terminal_nodes: + replay.same_replayed_frame(population.frame, terminal_frames[node_id]) + _require( + terminal_ledgers[node_id] == population.mass_ledger, + "FINAL_MASS_LEDGER", + ) + numerical.check_numeric_survey_weights( + final_numbers, + current.frame.weights_for("household").values, + ) + if activation is not None: + _require( + canonical_json(age_activation.activation_binding(activation)) + == activation_identity + and calibration.params.get("activation") == activation_identity.decode(), + "FINAL_ACTIVATION_BINDING", + ) + # No decoding, store access, source borrow or implementation helper follows + # this seal. It covers the complete selected prefix and age trio, including + # scope and release flags, using expectations detached before execution. + _check_manifest(manifest, compiled, expected_nodes) + _require( + canonical_json(result.diagnostics) == diagnostic_identity, + "FINAL_DIAGNOSTICS_VALUES", + ) + # The verifications above perform I/O. Finish with the retained issuance + # entries, so the second verification cannot invalidate the first handle. + _require( + budgets._entry(prefix_budget, budgets.SamplingOriginBudget) + is prefix_budget_entry + and budgets._entry(budget, budgets.SamplingOriginBudget) + is receiving_budget_entry + and prefix_budget.payload == budget.payload == budget_payload, + "FINAL_BUDGET_ISSUANCE", + ) + _require( + budgets._entry(admitted, budgets.SamplingOriginSuccessor) is successor_entry + and admitted.payload == successor_payload, + "FINAL_SUCCESSOR_ISSUANCE", + ) + return result diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/survey_age_sources.py b/packages/microcosm-build/src/microcosm/build/us_runtime/survey_age_sources.py new file mode 100644 index 000000000..d4314af93 --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/survey_age_sources.py @@ -0,0 +1,181 @@ +"""Two explicit S0101-only source captures; no network or target activation. + +Callers supply already acquired response bytes. Reading an explicit capture +never discovers files or falls back to a mixed national reference inventory. +""" + +from __future__ import annotations + +import hashlib +import os +import stat +from pathlib import Path +from urllib.parse import urlencode + +from microcosm.graph.canonical import canonical_json + +from .cd_reference_sources import strict_json + +MAX_RESPONSE_BYTES = 8 * 1024**2 +MAX_DESCRIPTOR_BYTES = 4096 +_DESCRIPTOR_FIELDS = frozenset( + { + "table", + "year", + "dataset", + "geography", + "kind", + "url", + "sha256", + "size_bytes", + "path", + } +) + + +def _require(condition: object, reason: str) -> None: + if not condition: + raise ValueError("SURVEY_AGE_SOURCE_" + reason) + + +def survey_age_requests() -> tuple[dict[str, object], dict[str, object]]: + """Return detached descriptors of exactly two supported national requests.""" + base = "https://api.census.gov/data/2024/acs/acs1/subject" + common = { + "table": "S0101", + "year": 2024, + "dataset": "acs/acs1/subject", + "geography": "us:*", + } + return ( + {**common, "kind": "metadata", "url": f"{base}/groups/S0101.json"}, + { + **common, + "kind": "data", + "url": base + "?" + urlencode({"get": "group(S0101)", "for": "us:*"}), + }, + ) + + +def _request(value: object) -> dict[str, object]: + _require(type(value) is dict, "REQUEST") + for request in survey_age_requests(): + if set(value) == set(request) and all( + type(value[k]) is type(v) and value[k] == v for k, v in request.items() + ): + return request + _require(False, "REQUEST") + + +def _digest(value: object) -> str: + _require( + type(value) is str + and len(value) == 64 + and all(c in "0123456789abcdef" for c in value), + "PIN", + ) + return value + + +def _path(value: str | Path) -> Path: + path = Path(value) + _require(".." not in path.parts, "PATH") + path = path.absolute() + for member in (*reversed(path.parents), path): + _require(not member.is_symlink(), "SYMLINK") + return path + + +def _read(path: Path, limit: int) -> bytes: + path = _path(path) + try: + # A FIFO must reach the regular-file check instead of waiting for a + # writer before fstat can reject it. Nonblocking leaves regular reads unchanged. + fd = os.open(path, os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK) + with os.fdopen(fd, "rb") as stream: + before = os.fstat(stream.fileno()) + _require(stat.S_ISREG(before.st_mode), "REGULAR_FILE") + _require(0 < before.st_size <= limit, "SIZE") + payload = stream.read(limit + 1) + after = os.fstat(stream.fileno()) + _require( + len(payload) == before.st_size + and (before.st_size, before.st_mtime_ns, before.st_ctime_ns) + == (after.st_size, after.st_mtime_ns, after.st_ctime_ns), + "CHANGED_FILE", + ) + return payload + except OSError: + raise ValueError("SURVEY_AGE_SOURCE_FILE") from None + + +def _write(path: Path, payload: bytes, limit: int) -> None: + path = _path(path) + if path.exists(): + _require(_read(path, limit) == payload, "IMMUTABLE") + return + path.parent.mkdir(parents=True, exist_ok=True) + _path(path) + with path.open("xb") as stream: + stream.write(payload) + + +def capture_survey_age_response( + root: str | Path, request: dict, payload: bytes +) -> dict: + """Store one caller-acquired response immutably; grant no target permission.""" + request = _request(request) + _require(type(payload) is bytes and 0 < len(payload) <= MAX_RESPONSE_BYTES, "SIZE") + strict_json(payload) + root = _path(root) + digest = hashlib.sha256(payload).hexdigest() + descriptor = { + **request, + "sha256": digest, + "size_bytes": len(payload), + "path": f"raw/{digest}.json", + } + key = hashlib.sha256(request["url"].encode()).hexdigest() + descriptor_path = root / "requests" / f"{key}.json" + encoded = canonical_json(descriptor) + # Refuse replacement before writing even an unreferenced new raw file. + if _path(descriptor_path).exists(): + _require(_read(descriptor_path, MAX_DESCRIPTOR_BYTES) == encoded, "IMMUTABLE") + _write(root / descriptor["path"], payload, MAX_RESPONSE_BYTES) + _write(descriptor_path, encoded, MAX_DESCRIPTOR_BYTES) + return descriptor + + +def read_survey_age_response( + root: str | Path, kind: str, expected_sha256: str +) -> tuple[bytes, object]: + """Read the fixed named response against a separately reviewed content pin.""" + _require(type(kind) is str and kind in {"metadata", "data"}, "KIND") + expected_sha256 = _digest(expected_sha256) + request = next(row for row in survey_age_requests() if row["kind"] == kind) + root = _path(root) + key = hashlib.sha256(request["url"].encode()).hexdigest() + descriptor = strict_json( + _read(root / "requests" / f"{key}.json", MAX_DESCRIPTOR_BYTES) + ) + _require( + type(descriptor) is dict and set(descriptor) == _DESCRIPTOR_FIELDS, "DESCRIPTOR" + ) + _request({key: descriptor[key] for key in request}) + _require( + descriptor["sha256"] == expected_sha256 + and descriptor["path"] == f"raw/{expected_sha256}.json", + "PIN", + ) + _require( + type(descriptor["size_bytes"]) is int + and 0 < descriptor["size_bytes"] <= MAX_RESPONSE_BYTES, + "SIZE", + ) + payload = _read(root / f"raw/{expected_sha256}.json", MAX_RESPONSE_BYTES) + _require( + len(payload) == descriptor["size_bytes"] + and hashlib.sha256(payload).hexdigest() == expected_sha256, + "RESPONSE_PIN", + ) + return payload, strict_json(payload) diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/survey_atomic_geography.py b/packages/microcosm-build/src/microcosm/build/us_runtime/survey_atomic_geography.py new file mode 100644 index 000000000..63e6efcea --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/survey_atomic_geography.py @@ -0,0 +1,638 @@ +"""Reconstruct observed constraints, initial clones and geography from live sources. + +The immutable recipe names normalized support bytes; their digest proves byte +integrity, not publisher provenance. A native source adapter must establish the +latter separately. This helper executes the maintained pure geography operators, +not graph kernels, and issues no source, Population, or release authority. +""" + +from __future__ import annotations + +import hashlib +import json +import math +import os +import re +import stat +from dataclasses import asdict, dataclass, replace +from io import BytesIO +from pathlib import Path +from zipfile import ZipFile + +import numpy as np +import pandas as pd + +from microcosm.build import atomic_geography as atomic +from microcosm.build import graph_atomic_geography as atomic_graph +from microcosm.frame import US_SCHEMA, Frame, WeightKind, Weights +from microcosm.graph import ( + Graph, + KernelResult, + Node, + SourceRef, + StructuralDelta, + compile_graph, +) +from microcosm.graph.canonical import canonical_json +from microcosm.graph.codecs import RAW_BYTES_MAX_BYTES +from microcosm.graph.executor import ( + _expand_declared_payload, + _expand_rewrite_coordinates, +) +from microcosm.graph.population import ( + Population, + _mass_record, + _storage_parts, + expand_lineage_receipt, + expand_writes_receipt, + patch, +) + +from . import atomic_block_support as blocks +from . import current_survey_geography as observed +from . import graph_atomic_survey_clone as composition +from . import graph_combined_clone as clone +from . import graph_current_survey_geography as observed_graph +from . import graph_survey_population as graph +from . import puf_support +from . import survey_population_preparation as source +from . import survey_population_replay as replay +from .graph_sources import frame_column_declarations + +PROTOCOL = "microcosm.us.atomic-survey-reconstruction.v2" +MAX_SUPPORT_BYTES = RAW_BYTES_MAX_BYTES +MAX_SUPPORT_EXPANDED_BYTES = 2 * 1024**3 +MAX_SUPPORT_MEMBERS = 32 + + +def _require(condition, reason): + if not condition: + raise ValueError("ATOMIC_SURVEY_RECONSTRUCTION_" + reason) + + +def _sha(payload): + return hashlib.sha256(payload).hexdigest() + + +@dataclass(frozen=True, slots=True) +class AtomicSurveyReconstruction: + """Exact immutable inputs; construction itself authenticates no source.""" + + support_path: str + support_sha256: str + source_ids: tuple[tuple[str, str], ...] + seed: int + + def __post_init__(self): + _config_bytes(self) + + def to_bytes(self): + """Revalidate and detach the recipe, including the literal source path.""" + return _config_bytes(self) + + +def _config_bytes(config): + _require(type(config) is AtomicSurveyReconstruction, "CONFIG_TYPE") + _require( + type(config.support_path) is str + and 0 < len(config.support_path) <= 4096 + and "\0" not in config.support_path + and Path(config.support_path).is_absolute() + and ".." not in Path(config.support_path).parts, + "SUPPORT_PATH", + ) + _require( + type(config.support_sha256) is str + and re.fullmatch(r"[0-9a-f]{64}", config.support_sha256) is not None, + "SUPPORT_DIGEST", + ) + _require( + type(config.source_ids) is tuple + and len(config.source_ids) == 3 + and all( + type(pair) is tuple + and len(pair) == 2 + and all(type(value) is str for value in pair) + and 0 < len(pair[1]) <= 4096 + and pair[1].strip() == pair[1] + for pair in config.source_ids + ) + and tuple(pair[0] for pair in config.source_ids) + == ("district", "population", "puma"), + "SOURCE_IDENTITIES", + ) + _require(type(config.seed) is int and 0 <= config.seed < 2**63, "SEED") + return canonical_json( + { + "protocol": PROTOCOL, + "support_path": config.support_path, + "support_sha256": config.support_sha256, + "source_ids": dict(config.source_ids), + "seed": config.seed, + } + ) + + +def _file_identity(value): + return tuple( + getattr(value, name) + for name in ("st_dev", "st_ino", "st_size", "st_mtime_ns", "st_ctime_ns") + ) + + +def _read_support(config): + """Read one bounded regular file and verify its exact pinned bytes.""" + descriptor = os.open( + config.support_path, + os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0) | getattr(os, "O_NONBLOCK", 0), + ) + with os.fdopen(descriptor, "rb") as stream: + before = os.fstat(stream.fileno()) + _require( + stat.S_ISREG(before.st_mode) and 0 < before.st_size <= MAX_SUPPORT_BYTES, + "SUPPORT_SIZE", + ) + payload = stream.read(before.st_size + 1) + after = os.fstat(stream.fileno()) + current = os.stat(config.support_path, follow_symlinks=False) + _require( + stat.S_ISREG(current.st_mode) + and _file_identity(before) == _file_identity(after) == _file_identity(current) + and len(payload) == before.st_size + and _sha(payload) == config.support_sha256, + "SUPPORT_CHANGED", + ) + return payload, _file_identity(current) + + +def _decode_support(payload): + # Bound declared decompressed bytes before numpy opens any array. The shared + # decoder still owns every schema, value, mapping and sampling-weight check. + with ZipFile(BytesIO(payload)) as archive: + members = archive.infolist() + _require( + 0 < len(members) <= MAX_SUPPORT_MEMBERS + and sum(member.file_size for member in members) + <= MAX_SUPPORT_EXPANDED_BYTES, + "SUPPORT_EXPANDED_SIZE", + ) + for member in members: + with archive.open(member) as stream: + version = np.lib.format.read_magic(stream) + if version == (1, 0): + shape, _fortran, dtype = np.lib.format.read_array_header_1_0(stream) + elif version == (2, 0): + shape, _fortran, dtype = np.lib.format.read_array_header_2_0(stream) + else: + raise ValueError("ATOMIC_SURVEY_RECONSTRUCTION_SUPPORT_NPY_VERSION") + # A small ZIP member must not claim a huge allocation in its + # NPY header. Validate the physical payload length before load. + _require( + not dtype.hasobject + and all(type(size) is int and size >= 0 for size in shape) + and math.prod(shape) * dtype.itemsize + == member.file_size - stream.tell(), + "SUPPORT_ARRAY_SIZE", + ) + return atomic.decode_atomic_support(payload) + + +def _copy_population(population): + frame = population.frame + return replace( + population, + frame=Frame( + {entity: frame.table(entity).copy(deep=True) for entity in frame.entities}, + frame.schema, + { + entity: Weights( + frame.weights_for(entity).values.copy(), + frame.weights_for(entity).kind, + ) + for entity in frame.weighted_entities + }, + frame.strata.copy(deep=True), + # Frame recursively freezes fresh metadata containers. The admitted + # mass records contain scalars and immutable tuples; replacing each + # record avoids deepcopy's reflective dataclass class-cache writes. + metadata=frame.metadata, + mass_log=tuple(replace(record) for record in frame.mass_log), + ), + owners=dict(population.owners), + weight_kind=dict(population.weight_kind), + design_weights={ + name: values.copy() for name, values in population.design_weights.items() + }, + mass_ledger=tuple(replace(record) for record in population.mass_ledger), + ) + + +def _population_stamp(population): + """Pure in-process seal, including storage beneath nullable masks. + + Never persist it or compare it across reconstructions: it folds physical + storage parts (masked ``_data`` under nulls, and before microcosm#907 + object-dtype pointer bytes), so equal content rebuilt elsewhere need not + match. A cross-call pin carries ``source._frame_identity`` plus the + version, owners, weight kinds, mass ledger and design weights instead. + """ + _require(type(population) is Population, "POPULATION_TYPE") + digest = hashlib.sha256() + frame = population.frame + digest.update( + canonical_json( + { + "frame": source._frame_identity(frame), + "version": population.version, + "owners": sorted(population.owners.items()), + "weight_kind": list(population.weight_kind.items()), + "mass_ledger": [asdict(record) for record in population.mass_ledger], + } + ) + ) + for series in ( + *( + frame.table(entity)[column] + for entity in frame.entities + for column in frame.table(entity) + ), + frame.strata, + ): + for part in _storage_parts(series, np.ones(len(series), dtype=np.bool_)): + digest.update(len(part).to_bytes(8, "little")) + digest.update(part) + for entity, values in population.design_weights.items(): + digest.update(canonical_json((entity, str(values.dtype), values.shape))) + digest.update(values.tobytes()) + return digest.hexdigest() + + +def _raw_allocation(view, population): + """Reconstruct the original allocation, retaining all existing checks.""" + _require(type(population) is Population, "RAW_POPULATION_TYPE") + instructions = graph.allocation_instructions( + view.selection_plan, view.receipt["origins"]["households"] + ) + _weights, _context, _allocation, receipt, expected = graph._allocation_output( + view.frame, view.context, instructions, _sha(view.payload) + ) + columns = frame_column_declarations(view.frame) + nodes = graph.survey_population_nodes( + columns, + preparation_sha256=_sha(view.payload), + fraction=view.selection_plan.fraction, + seed=view.selection_plan.seed, + ) + graph._same_frame(expected, population.frame) + replay.same_replayed_frame(expected, population.frame) + design = view.frame.weights_for("household").values + graph._check_design_anchors(population, design) + cells = ( + (entity, str(column)) + for entity in view.frame.entities + for column in view.frame.table(entity) + ) + ledger = ( + _mass_record( + view.frame, expected, nodes[1], KernelResult(receipt=receipt), "declared" + ), + ) + graph._check_population_state( + population, + version=graph.ALLOCATION_NODE, + owners=dict.fromkeys(cells, graph.ALLOCATION_NODE), + kind=WeightKind.IMPORTANCE, + ledger=ledger, + ) + return columns, nodes + + +def _clone_expectations(before, nodes, compiled): + """Independently reconstruct the complete clone, including inherited cells.""" + expanded = puf_support.clone_us_frame_for_puf_support( + before.frame, + clone_attachment_fraction=1.0, + clone_attachment_seed=0, + ) + design = graph._verify_cloned_frame( + before.frame, expanded, before.design_weights["household"] + ) + expand = next(n for n in nodes if n.structural is StructuralDelta.EXPAND) + claim = next(n for n in nodes if n.id == clone.COMBINED_CLONE_CLAIM_NODE) + ledger = ( + *before.mass_ledger, + _mass_record(before.frame, expanded, expand, KernelResult(), "conserve"), + ) + owners = { + (e, str(c)): expand.id for e in expanded.entities for c in expanded.table(e) + } + expected, receipts = {}, {} + lineage, facts = {}, {} + for entity in US_SCHEMA.entities: + lineage[entity], facts[entity] = clone._entity_lineage( + before.frame, expanded, entity + ) + authority = puf_support.validate_puf_clone_attachment( + expanded, boundary=graph.PHASE, expected_fraction=1.0, expected_seed=0 + ) + receipt = clone.USCombinedSurveyCloneExpandKernel._receipt( + before.frame, expanded, ("acs", "asec"), authority, facts + ) + receipt["expand"] = expand_lineage_receipt(lineage) + receipt["expand_declared"] = _expand_declared_payload(expand) + receipt["expand_writes"] = expand_writes_receipt( + before.frame, + expanded, + expand, + receipt, + rewrite_coordinates=_expand_rewrite_coordinates(compiled, expand), + ) + receipts[expand.id] = receipt + receipts[claim.id] = { + "phase": clone.COMBINED_CLONE_PHASE, + "claimed_cells": sorted(f"{o.entity}.{o.column}" for o in claim.outputs), + } + # Preserve exact structural and claim ownership before assignment. + for node_id in compiled.order: + if node_id not in receipts: + continue + if node_id == claim.id: + owners.update({(o.entity, o.column): claim.id for o in claim.outputs}) + expected[node_id] = Population.from_frame( + expanded, + expand.id, + owners=owners, + mass_ledger=ledger, + design_weights={"household": np.array(design, copy=True)}, + ) + return expected, receipts + + +@dataclass(frozen=True, slots=True) +class AtomicSurveyGeographyStage: + """Expected complete Population and pure receipt, never executor evidence.""" + + node: Node + population: Population + receipt: bytes + artifacts: tuple[tuple[str, bytes], ...] = () + + +@dataclass(frozen=True, slots=True) +class AtomicSurveyGeographyReconstruction: + """Detached reconstruction values; consumers must independently requalify.""" + + population: Population + observed_population: Population + expanded_population: Population + stages: tuple[AtomicSurveyGeographyStage, ...] + nodes: tuple[Node, ...] + definition: bytes + projection_receipt: bytes + support_payload: bytes + sources: tuple[tuple[str, str], ...] + config_sha256: str + receipt: bytes + + +def reconstruct_atomic_survey_geography(preparation, allocated_population, config): + """Qualify sources, complete initial clones, then assign on the clone version. + + Stage values follow the actual compiled order, including the structural + expansion. Support import and clone ownership may have either relative order. + Callers compare these complete expected Populations to executor observations + on cold and required-warm runs; these values do not replace those checks. + """ + config_bytes = _config_bytes(config) + _require( + type(preparation) is source.AuthenticatedSurveyPopulationPreparation, + "PREPARATION_TYPE", + ) + entry = preparation._checked() + payload, state = entry[1], entry[2] + view = source.CheckedSurveyPopulationView( + payload, state.context, state.frame, state.plan, json.loads(payload) + ) + columns, source_nodes = _raw_allocation(view, allocated_population) + original_stamp = _population_stamp(allocated_population) + current = _copy_population(allocated_population) + support_payload, support_identity = _read_support(config) + support = _decode_support(support_payload) + supports = {blocks.SYSTEM: support} + definition = blocks.assignment_definition( + identity=composition.ASSIGNMENT_IDENTITY, + state_column=observed.COLUMNS[1], + puma_column=observed.COLUMNS[2], + source_ids=dict(config.source_ids), + seed=config.seed, + ) + definition_bytes = canonical_json(definition) + projection = observed.qualify_current_survey_geography(preparation) + projection_receipt = projection.receipt + observed_graph._check_projection( + observed, projection, payload=payload, receipt_sha256=_sha(projection_receipt) + ) + _require( + projection.household.index.tolist() + == current.frame.table("household").household_id.tolist(), + "PROJECTION_ORDER", + ) + observation = observed_graph.current_survey_geography_node( + preparation_sha256=_sha(payload), + projection_receipt_sha256=_sha(projection_receipt), + population=graph.ALLOCATION_NODE, + ) + additions = composition.atomic_survey_clone_nodes( + definition, (*columns, *observation.outputs), base=graph.ALLOCATION_NODE + ) + nodes = (observation, *additions) + compiled = compile_graph( + Graph( + "us", + ( + SourceRef(graph.SOURCE_NAME, graph.SOURCE_CODEC), + SourceRef(blocks.SOURCE, "raw-bytes-v1"), + ), + (*source_nodes, *nodes), + ) + ) + wanted = {node.id for node in nodes} + ordered = tuple( + compiled.graph.node(name) for name in compiled.order if name in wanted + ) + stages = [] + cloned, clone_receipts = {}, {} + for node in ordered: + values, artifacts = {}, () + structural_or_claim = False + if node.id == observation.id: + values = { + ("household", name): projection.household[name].copy(deep=True) + for name in observed.COLUMNS + } + receipt = { + "phase": observed_graph.PHASE, + "preparation_sha256": _sha(payload), + "projection_receipt_sha256": _sha(projection_receipt), + "source_projection": json.loads(projection_receipt), + "population_admission_issued": False, + "release_eligible": False, + } + elif node.structural is StructuralDelta.EXPAND: + cloned, clone_receipts = _clone_expectations(current, nodes, compiled) + current = _copy_population(cloned[node.id]) + receipt = clone_receipts[node.id] + structural_or_claim = True + elif node.id == clone.COMBINED_CLONE_CLAIM_NODE: + _require(node.id in cloned, "CLONE_BEFORE_CLAIM") + current = _copy_population(cloned[node.id]) + receipt = clone_receipts[node.id] + structural_or_claim = True + elif node.kernel == atomic_graph.AtomicSupportImportKernel.ref: + _require( + support.metadata["system"] == node.params["system"], "SUPPORT_SYSTEM" + ) + artifacts = (("support", support_payload),) + receipt = { + "support_sha256": support.sha256, + "areas": len(support.arrays["area"]), + "metadata": support.metadata, + } + else: + households = current.frame.table("household") + if node.kernel == atomic_graph.AtomicAssignKernel.ref: + output = atomic.assign_atomic(households, definition, supports) + receipt = { + "scope": "atomic_area_assignment", + "households": len(output), + "support_sha256": { + name: value.sha256 for name, value in supports.items() + }, + } + elif node.kernel == atomic_graph.AtomicDeriveKernel.ref: + output = atomic.derive_geography(households, definition, supports) + receipt = { + "scope": "atomic_area_functional_lookup", + "layers": { + system["id"]: system["layers"] + for system in definition["systems"] + }, + } + else: + _require( + node.kernel == atomic_graph.AtomicGeographyGateKernel.ref, + "STAGE_ROSTER", + ) + output = None + receipt = atomic.validate_geography(households, definition, supports) + artifacts = (("validation", canonical_json(receipt)),) + if output is not None: + ids = pd.Index(households.household_id.to_numpy(), name="household_id") + values = { + ("household", name): pd.Series(output[name].array.copy(), index=ids) + for name in output + } + if not structural_or_claim: + current = _copy_population( + patch(current, node, KernelResult(columns=values)) + ) + stages.append( + AtomicSurveyGeographyStage( + node, current, canonical_json(receipt), artifacts + ) + ) + stage_tuple = tuple(stages) + by_id = {stage.node.id: stage for stage in stage_tuple} + observed_population = by_id[observation.id].population + expanded_population = by_id[clone.COMBINED_CLONE_CLAIM_NODE].population + validation_payload = by_id["geography.gate"].artifacts[0][1] + receipt = canonical_json( + { + "protocol": PROTOCOL, + "config_sha256": _sha(config_bytes), + "preparation_sha256": _sha(payload), + "projection_receipt_sha256": _sha(projection_receipt), + "support_sha256": _sha(support_payload), + "definition_sha256": _sha(definition_bytes), + "validation_receipt_sha256": _sha(validation_payload), + "assignment_identity": list(composition.ASSIGNMENT_IDENTITY), + "stages": [stage.node.id for stage in stage_tuple], + "frame_sha256": source._frame_identity(current.frame), + "population_sha256": _population_stamp(current), + "observed_population_sha256": _population_stamp(observed_population), + "expanded_population_sha256": _population_stamp(expanded_population), + "publisher_provenance_established": False, + "source_admission_issued": False, + "population_admission_issued": False, + "release_eligible": False, + } + ) + result = AtomicSurveyGeographyReconstruction( + population=_copy_population(current), + observed_population=observed_population, + expanded_population=expanded_population, + stages=stage_tuple, + nodes=nodes, + definition=definition_bytes, + projection_receipt=projection_receipt, + support_payload=support_payload, + sources=((blocks.SOURCE, config.support_path),), + config_sha256=_sha(config_bytes), + receipt=receipt, + ) + result_stamps = tuple( + _population_stamp(stage.population) for stage in result.stages + ) + stage_values = tuple( + (stage.node, stage.receipt, stage.artifacts) for stage in result.stages + ) + final_stamp = _population_stamp(result.population) + # Finish support and source-owner I/O before pure rechecks of every retained + # input and every detached expected Population. No callback is an authority. + _require( + _read_support(config) == (support_payload, support_identity), "FINAL_SUPPORT" + ) + final_entry = preparation._checked() + support_after = os.stat(config.support_path, follow_symlinks=False) + _require( + final_entry is entry + and source._ISSUED.get(id(preparation)) is entry + and preparation.payload == payload + and _config_bytes(config) == config_bytes, + "FINAL_ISSUANCE", + ) + _require( + stat.S_ISREG(support_after.st_mode) + and _file_identity(support_after) == support_identity, + "FINAL_SUPPORT_STAT", + ) + source._pure_final(state) + _raw_allocation(view, allocated_population) + _require( + _population_stamp(allocated_population) == original_stamp + and tuple(_population_stamp(stage.population) for stage in result.stages) + == result_stamps + and _population_stamp(result.population) == final_stamp, + "FINAL_POPULATIONS", + ) + observed_graph._check_projection( + observed, projection, payload=payload, receipt_sha256=_sha(projection_receipt) + ) + _require( + result.nodes == nodes + and result.observed_population is observed_population + and result.expanded_population is expanded_population + and tuple(stage.node for stage in result.stages) == ordered + and tuple( + (stage.node, stage.receipt, stage.artifacts) for stage in result.stages + ) + == stage_values + and result.definition == definition_bytes + and result.projection_receipt == projection_receipt + and result.support_payload == support_payload + and result.sources == ((blocks.SOURCE, config.support_path),) + and result.config_sha256 == _sha(config_bytes) + and result.receipt == receipt, + "FINAL_RESULT", + ) + return result diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/survey_calibration_diagnostics.py b/packages/microcosm-build/src/microcosm/build/us_runtime/survey_calibration_diagnostics.py new file mode 100644 index 000000000..033322cc3 --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/survey_calibration_diagnostics.py @@ -0,0 +1,207 @@ +"""Recompute supported age diagnostics without rerunning an optimizer. + +This is a value checker, not source authority. The caller supplies independently +verified ordered counts, sampling bounds and receiving weights. Optimizer history +has a bounded schema but cannot be reconstructed from those final values. +""" + +from __future__ import annotations + +import hashlib +import json +from dataclasses import replace +from types import SimpleNamespace + +import numpy as np + +from microcosm.calibrate import diagnostics_payload +from microcosm.calibrate.score import score_targets +from microcosm.frame import WeightKind, Weights +from microcosm.graph.canonical import canonical_json + +from . import graph_survey_calibration as numeric + +MAX_DIAGNOSTIC_BYTES = 8 * 1024**2 +HISTORY_FIELDS = ( + "initial_loss", + "loss_trajectory", + "options.matrix_format", + "options.grouped_upper_bounds.last_corrected_group_count", +) + + +def _require(condition, reason): + if not condition: + raise ValueError("SURVEY_DIAGNOSTICS_" + reason) + + +def _history(document, epochs, group_count): + options = document.get("options") + _require(type(options) is dict, "OPTIONS") + grouped = options.get("grouped_upper_bounds") + _require(type(grouped) is dict, "GROUPED_OPTIONS") + corrected = grouped.get("last_corrected_group_count") + _require(type(corrected) is int and 0 <= corrected <= group_count, "HISTORY_COUNT") + layout = options.get("matrix_format") + _require( + type(layout) is str and layout in {"dense", "sparse_csr"}, "HISTORY_LAYOUT" + ) + trajectory = document.get("loss_trajectory") + _require(type(trajectory) is list and len(trajectory) == epochs, "HISTORY_LENGTH") + _require( + all(type(v) is float and np.isfinite(v) and 0 <= v <= 10 for v in trajectory), + "HISTORY_LOSS", + ) + _require( + type(document.get("initial_loss")) is float + and document["initial_loss"] == trajectory[0], + "HISTORY_INITIAL", + ) + return corrected, layout, trajectory + + +def validate_survey_calibration_diagnostics( + payload, + *, + counts_payload, + bounds_payload, + weights, + registry, + epochs, + learning_rate, + anchors, + activation_binding=None, +): + """Return defensive schema-8 values plus an explicit verification annotation. + + Every field except the four named history fields is reconstructed using + score_targets and the package's diagnostics encoder. Only warning-free + schema 8 (every registry-backed target carrying its calibration hierarchy) + with target-loss attribution is supported in this first profile. + No convergence, optimality, target eligibility or release claim is granted. + """ + _require( + type(payload) is bytes and 0 < len(payload) <= MAX_DIAGNOSTIC_BYTES, + "PAYLOAD_LIMIT", + ) + try: + document = json.loads(payload) + except (ValueError, UnicodeError, RecursionError): + raise ValueError("SURVEY_DIAGNOSTICS_JSON") from None + _require(type(document) is dict, "DOCUMENT") + _require(canonical_json(document) == payload, "CANONICAL") + # Reuse the actual declaration's option/registry restrictions. This is a + # declaration and numeric work Frame, never a population/source substitute. + factory = numeric.survey_age_calibration_node + profile_options = {} + if activation_binding is not None: + factory = numeric.survey_age_development_node + profile_options["activation_binding"] = activation_binding + factory( + registry, + **profile_options, + base="diagnostic-values", + budget_node="bounds", + count_node="counts", + epochs=epochs, + learning_rate=learning_rate, + ) + bounds = numeric.decode_numeric_survey_bounds(bounds_payload) + counts = numeric.ages.decode_survey_age_counts(counts_payload) + _require( + counts.population == bounds.population + and tuple(counts.household_ids) == bounds.grouped.household_ids, + "ORDERED_POPULATION", + ) + numeric.check_numeric_survey_weights(bounds, weights) + accepted_weights = weights.tobytes() + corrected, layout, trajectory = _history( + document, epochs, bounds.grouped.group_count + ) + table = counts.counts.copy(deep=True) + table.insert(0, "household_id", counts.household_ids.copy()) + work = numeric.demographic.kernels_module._frame_from_context( + SimpleNamespace( + tables={"household": table}, + weights={ + "household": Weights(bounds.incoming.copy(), WeightKind.IMPORTANCE) + }, + ), + "household", + ) + targets = registry.to_target_set() + initial = score_targets( + work, targets, weights=bounds.incoming, target_loss_cap=10.0 + ) + final = score_targets(work, targets, weights=weights, target_loss_cap=10.0) + _require(not initial.skipped and not final.skipped, "SKIPPED_TARGETS") + options = { + # The closed grouped profile supplies no gates or record-budget search. + # Reconstruct these solver fields; never inherit them from the payload. + "gate_initialization_supplied": False, + "budget_basis": "nonzero_count", + "feasible_draw_pi_hi": None, + "budget_search": None, + "grouped_preserve_zeros": { + "enabled": True, + "fixed_zero_count": int(np.count_nonzero(bounds.incoming == 0)), + "ordered_zero_mask_sha256": hashlib.sha256( + np.asarray(bounds.incoming == 0, dtype=np.uint8).tobytes() + ).hexdigest(), + }, + "grouped_upper_bounds": bounds.grouped.diagnostics(weights, corrected), + "method": "adam", + "epochs": epochs, + "learning_rate": learning_rate, + # This profile always uses grouped bounds, whose accepted weights are + # the closing state. Ungrouped Adam's best-iterate history cannot apply. + "iterate_selection": "closing_state", + "iterate_selection_receipt": {}, + "mass": "free", + "mass_reason": None, + "max_weight_ratio": None, + "target_records": None, + "l1_lambda": 0.0, + "l1_penalty": "mean_initial_weight_ratio_abs", + "l2_lambda": 0.0, + "l2_anchor": "initial", + "l2_anchor_weights_supplied": False, + "l2_penalty": "mean_initial_pre_gate_weight_ratio_squared", + "seed": 0, + "target_loss_weights": final.options["target_loss_weights"], + "target_loss_scales": final.options["target_loss_scales"], + "warm_start_weights": {"enabled": False, "kind": None}, + "matrix_format": layout, + } + combined = replace( + final, + diagnostics=tuple( + replace(after, initial_estimate=before.final_estimate) + for before, after in zip( + initial.diagnostics, final.diagnostics, strict=True + ) + ), + options=options, + loss_trajectory=np.asarray(trajectory, dtype=np.float64), + ) + expected = diagnostics_payload(combined, target_registry=registry, build=anchors) + _require( + expected.get("schema_version") == 8 + and expected.get("diagnostic_warnings") == [] + and "target_loss_basis" in expected, + "UNSUPPORTED_DIAGNOSTIC_SCHEMA", + ) + _require(canonical_json(expected) == payload, "RECOMPUTED_VALUES") + # Make the distinction visible to downstream callers without altering the + # stored schema-8 evidence or its digest. + document["verification"] = { + "protocol": "microcosm.us.survey-age-diagnostics-verification.v1", + "artifact_sha256": hashlib.sha256(payload).hexdigest(), + "recomputed": "all artifact fields except the listed optimizer history", + "optimizer_history_schema_checked_only": list(HISTORY_FIELDS), + "optimizer_rerun": False, + "release_eligible": False, + } + numeric.check_numeric_survey_weights(bounds, weights) + _require(weights.tobytes() == accepted_weights, "FINAL_WEIGHT_BYTES") + return document diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/survey_catalogue_selection.py b/packages/microcosm-build/src/microcosm/build/us_runtime/survey_catalogue_selection.py new file mode 100644 index 000000000..933e86851 --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/survey_catalogue_selection.py @@ -0,0 +1,259 @@ +"""Plan one draw over complete supplied survey records, before native Frames. + +This is declaration/arithmetic only. Closed source owners must authenticate the +complete inputs and bind the result to the selected population. This module +reads no source, creates no Frame and never totals publisher weights. Original +anchors remain separate from the one-time share/inclusion multiplier. +""" + +from __future__ import annotations + +from dataclasses import dataclass +from fractions import Fraction + +from microcosm.build.survey_domain_sample import select_domain_households + +from . import survey_population_domains as domains + +PROTOCOL = "microcosm.us.survey-catalogue-selection.v1" +_BATCH_HOUSEHOLDS = 10_000 +_BATCH_PEOPLE = 100_000 + + +class CatalogueSelectionError(ValueError): + """Static refusal, without raw source identities or observations.""" + + +def _require(condition, code): + if not condition: + raise CatalogueSelectionError(code) + + +@dataclass(frozen=True, slots=True) +class SelectedHousehold: + key: domains.HouseholdKey + domain: domains.Domain + statistical_unit: str + original_design_weight: Fraction + share: Fraction + inclusion_probability: Fraction + + @property + def importance_multiplier(self) -> Fraction: + return self.share / self.inclusion_probability + + +@dataclass(frozen=True, slots=True) +class ExcludedHousehold: + key: domains.HouseholdKey + reason: str + + +@dataclass(frozen=True, slots=True) +class SelectionCell: + source: domains.Source + domain: domains.Domain + statistical_unit: str + share: Fraction + eligible_households: int + selected_households: int + inclusion_probability: Fraction | None + + +@dataclass(frozen=True, slots=True) +class CatalogueSelectionPlan: + selected: tuple[SelectedHousehold, ...] + excluded: tuple[ExcludedHousehold, ...] + cells: tuple[SelectionCell, ...] + supplied_households: int + fraction: Fraction + seed: int + + @property + def source_authenticated(self) -> bool: + return False + + @property + def population_binding_authenticated(self) -> bool: + return False + + @property + def release_eligible(self) -> bool: + return False + + +def _sampling_key(key): + # Keep the original H_SEQ token in the result, but order ASEC keys + # numerically. All validated values fit five digits; padded strings allow + # both survey channels to use the shared selector's one literal-ID axis. + return ( + key.native_id + if key.source is domains.Source.ACS + else f"{int(key.native_id):05d}" + ) + + +def _ordered_key(key): + return key.source.value, _sampling_key(key) + + +def _original_anchor(decision): + row = decision.original + if row.key.source is domains.Source.ASEC: + return Fraction(int(row.hsup_wgt), 100) + if decision.statistical_unit == "person": + return Fraction(int(row.persons[0].pwgtp)) + return Fraction(int(row.wgtp)) + + +def plan_catalogue_selection(*, acs_households, asec_households, fraction, seed): + """Classify all supplied records and draw once in each positive-share cell. + + ``N`` counts complete eligible households in the supplied catalogue, never + selected Frame rows. Unknown classifications refuse before any draw, even + if a tentative share is zero. Known structural exclusions have a separate + ledger. Original zero ASEC anchors remain eligible; an all-zero selected + cell refuses without redrawing. No common-total normalization is performed. + + Full source completeness, immutable source authority, and original files + are obligations of the calling source orchestration, not this value API. + """ + _require( + type(acs_households) is tuple and type(asec_households) is tuple, + "CATALOGUE_TUPLES", + ) + _require(type(fraction) is Fraction and 0 < fraction <= 1, "FRACTION") + _require(type(seed) is int and 0 <= seed < 2**64, "SEED") + _require( + type(_BATCH_HOUSEHOLDS) is int + and 0 < _BATCH_HOUSEHOLDS <= min(10_000, domains.MAX_HOUSEHOLDS) + and type(_BATCH_PEOPLE) is int + and 0 < _BATCH_PEOPLE <= min(100_000, domains.MAX_TOTAL_MEMBERS), + "BATCH_LIMITS", + ) + rows, exclusions = [], [] + household_keys, person_keys = set(), set() + batch, batch_people = [], 0 + + def consume(): + for decision in domains.classify_households(tuple(batch)): + _require( + decision.status is not domains.Status.REVIEW_REQUIRED, + "SOURCE_CLASSIFICATION_REVIEW_REQUIRED", + ) + if decision.status is domains.Status.EXCLUDED: + _require(decision.share == 0, "EXCLUSION_SHARE") + exclusions.append( + ExcludedHousehold(decision.original.key, decision.reason) + ) + else: + _require( + decision.status is domains.Status.ELIGIBLE + and decision.share is not None + and decision.share > 0 + and decision.domain is not None, + "ELIGIBLE_CLASSIFICATION", + ) + # Retain only the compact household decision, not another full + # population of per-person decisions and observation objects. + rows.append( + ( + decision.original.key, + decision.domain, + decision.statistical_unit, + _original_anchor(decision), + decision.share, + ) + ) + + for source, expected, catalogue in ( + (domains.Source.ACS, domains.AcsHousehold, acs_households), + (domains.Source.ASEC, domains.AsecHousehold, asec_households), + ): + for row in catalogue: + _require(type(row) is expected, "CATALOGUE_SOURCE_TYPE") + canonical = domains._key(row.key) + _require( + canonical[0] is source + and row.key.source_year == 2024 + and row.key.survey_year + == (2024 if source is domains.Source.ACS else 2025), + "CATALOGUE_PERIOD_SOURCE", + ) + _require(canonical not in household_keys, "GLOBAL_HOUSEHOLD_COLLISION") + household_keys.add(canonical) + _require(type(row.persons) is tuple, "MEMBER_TUPLE") + _require( + len(row.persons) <= min(domains.MAX_MEMBERS, _BATCH_PEOPLE), + "HOUSEHOLD_MEMBER_BOUND", + ) + if source is domains.Source.ASEC: + for person in row.persons: + _require(type(person) is domains.AsecPerson, "PERSON_TYPE") + identity = (2024, person.peridnum) + _require(identity not in person_keys, "GLOBAL_PERSON_COLLISION") + person_keys.add(identity) + if batch and ( + len(batch) == _BATCH_HOUSEHOLDS + or batch_people + len(row.persons) > _BATCH_PEOPLE + ): + consume() + batch, batch_people = [], 0 + batch.append(row) + batch_people += len(row.persons) + if batch: + consume() + + declarations = { + (item.domain.value, source.value): (item.statistical_unit, share) + for item in domains.declaration() + for source, share in ( + (domains.Source.ACS, item.acs_share), + (domains.Source.ASEC, item.asec_share), + ) + if share > 0 + } + selection = select_domain_households( + row_ids=tuple(_sampling_key(row[0]) for row in rows), + source_channels=tuple(row[0].source.value for row in rows), + domain_keys=tuple(row[1].value for row in rows), + cells=tuple(declarations), + fraction=fraction, + seed=seed, + ) + chosen, cells = [], [] + for cell in selection: + unit, share = declarations[cell.domain, cell.source] + selected = [rows[position] for position in cell.positions] + _require( + not selected or any(row[3] > 0 for row in selected), + "SELECTED_CELL_HAS_NO_POSITIVE_ANCHOR", + ) + for key, domain, observed_unit, anchor, observed_share in selected: + _require( + (observed_unit, observed_share) == (unit, share), "DECLARATION_DRIFT" + ) + chosen.append( + SelectedHousehold( + key, domain, unit, anchor, share, cell.inclusion_probability + ) + ) + cells.append( + SelectionCell( + domains.Source(cell.source), + domains.Domain(cell.domain), + unit, + share, + cell.eligible_households, + len(selected), + cell.inclusion_probability, + ) + ) + return CatalogueSelectionPlan( + tuple(sorted(chosen, key=lambda row: _ordered_key(row.key))), + tuple(sorted(exclusions, key=lambda row: _ordered_key(row.key))), + tuple(cells), + len(household_keys), + fraction, + seed, + ) diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/survey_financial_successor.py b/packages/microcosm-build/src/microcosm/build/us_runtime/survey_financial_successor.py new file mode 100644 index 000000000..c689b746f --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/survey_financial_successor.py @@ -0,0 +1,254 @@ +"""Admit a checked financial graph result under its original sampling budget. + +This narrow IMPORTANCE-to-IMPORTANCE binding changes only the seven current +financial leaves authenticated by the actual twenty-node runner. It neither +changes the original budget's initial population nor admits arbitrary Frames. + +Age composition still needs an explicit dependency on financial ATTACH and a +checked binding to the actual receiving graph's Population before calibration +admission. This module does not supply those graph edges or a new age runner. +""" + +from __future__ import annotations + +import json +import weakref +from dataclasses import dataclass + +import numpy as np + +from microcosm.graph import ArtifactType + +PROTOCOL = "microcosm.us.sampling-origin-financial-successor.v1" +FINANCIAL_SUCCESSOR_TYPE = ArtifactType( + "microcosm.us.sampling_origin_financial_successor", 1 +) +MAX_PAYLOAD_BYTES = 256 * 1024 +_ISSUED = {} + + +class SurveyFinancialSuccessorError(ValueError): + """A financial result lacks its exact original budget/run ancestry.""" + + +def _require(condition, reason): + if not condition: + raise SurveyFinancialSuccessorError("SURVEY_FINANCIAL_SUCCESSOR_" + reason) + + +def _entry(value): + entry = _ISSUED.get(id(value)) + _require( + type(value) is SamplingOriginFinancialSuccessor + and entry is not None + and entry[0]() is value + and type(value.payload) is bytes + and value.payload == entry[1], + "UNISSUED_OR_CHANGED", + ) + return entry + + +@dataclass(frozen=True) +class _State: + budget: object + budget_entry: tuple + financial_run: object + run_entry: tuple + previous: object + current: object + previous_identity: tuple + current_identity: tuple + + +def _document(state, budget_view, run_view): + # Local imports avoid a budget -> financial owner -> runner -> host cycle. + from . import graph_atomic_survey_financial as runner + from . import survey_origin_budget as budget + + run, prefix = state.financial_run, state.financial_run.prefix + original = state.budget_entry[2] + _require( + budget._entry(state.budget, budget.SamplingOriginBudget) is state.budget_entry + and runner._run_entry(run) is state.run_entry + and original.preparation is prefix.preparation + and original.allocated is prefix.allocated_population + and original.expanded is prefix.clone_population + and original.geography_config is prefix.geography_config + and budget_view.preparation is prefix.preparation + and budget_view.allocated_population is prefix.allocated_population + and budget_view.initial_population is state.previous is prefix.clone_population + and budget_view.payload == state.budget_entry[1] + and budget_view.digest == budget._sha(state.budget_entry[1]) + and run_view.population is state.current is run.financial_population + and run_view.payload == state.run_entry[1] + and run_view.digest == budget._sha(state.run_entry[1]), + "ORIGINAL_BUDGET_RUN_ANCESTRY", + ) + previous, current = state.previous, state.current + _require( + previous.version == current.version == runner.atomic.clone.COMBINED_CLONE_NODE + and current.mass_ledger == previous.mass_ledger + and current.weight_kind == previous.weight_kind + and np.array_equal( + current.frame.weights_for("household").values, + previous.frame.weights_for("household").values, + ), + "UNCHANGED_SAMPLING_MASS", + ) + budget.graph._check_design_anchors(current, previous.design_weights["household"]) + owners = { + **previous.owners, + **{ + ("person", column): runner.financial.ATTACH_NODE + for column in runner.values.OUTPUTS + }, + } + budget.graph._check_population_state( + current, + version=previous.version, + owners=owners, + kind=budget.WeightKind.IMPORTANCE, + ledger=previous.mass_ledger, + ) + # The checked run freshly compares the complete result to source-derived + # current values and its pinned raw draws; no caller column table is used. + return budget._json( + { + "protocol": PROTOCOL, + "budget_sha256": budget._sha(state.budget_entry[1]), + "financial_run_sha256": budget._sha(state.run_entry[1]), + "financial_run": json.loads(state.run_entry[1]), + "previous_version": previous.version, + "current_version": current.version, + "previous_frame_sha256": budget.source._frame_identity(previous.frame), + "current_frame_sha256": budget.source._frame_identity(current.frame), + "owned_columns": list(runner.values.OUTPUTS), + "sampling_bounds_changed": False, + "source_admission_issued": False, + "release_eligible": False, + } + ) + + +def _pure_state(state): + """Final retained-owner checks; never opens an artifact or source file.""" + from . import graph_atomic_survey_financial as runner + from . import survey_origin_budget as budget + + _require( + budget._entry(state.budget, budget.SamplingOriginBudget) is state.budget_entry, + "FINAL_BUDGET_ISSUANCE", + ) + runner._pure_run(state.financial_run, state.run_entry) + _require( + budget._population_identity(state.previous) == state.previous_identity + and budget._population_identity(state.current) == state.current_identity, + "FINAL_POPULATION_SEAL", + ) + _require(budget._live() == budget._LIVE, "FINAL_PRODUCER_SEAL") + + +def _final_state(state): + from . import survey_origin_budget as budget + + # Complete the last budget/support I/O before the financial owner's pure + # seal, so that borrow cannot invalidate the already-checked financial run. + budget._final_budget_state(state.budget_entry[2]) + _pure_state(state) + + +@dataclass(frozen=True) +class SamplingOriginFinancialSuccessor: + payload: bytes + + def __post_init__(self): + raise SurveyFinancialSuccessorError("NO_PUBLIC_FINANCIAL_SUCCESSOR_CONSTRUCTOR") + + def checked_view(self): + from . import graph_atomic_survey_financial as runner + + entry = _entry(self) + state = entry[2] + _pure_state(state) + budget_view = state.budget.checked_view() + run_view = runner.check_atomic_survey_financial_run(state.financial_run) + _require(_document(state, budget_view, run_view) == entry[1], "RECONSTRUCTION") + _final_state(state) + _require(_entry(self) is entry, "FINAL_ISSUANCE") + # Do not expose a detached view to the final support-borrow callback. + # All owner/file I/O and retained-state seals are complete above. + return CheckedSamplingOriginFinancialSuccessor( + entry[1], + runner.codec.sha(entry[1]), + state.budget, + state.financial_run, + state.previous, + state.current, + ) + + def to_bytes(self): + return self.checked_view().payload + + +@dataclass(frozen=True) +class CheckedSamplingOriginFinancialSuccessor: + """Values borrowed from a checked handle, never decoded authority.""" + + payload: bytes + digest: str + budget: object + financial_run: object + previous: object + current: object + + +def admit_survey_financial_population(budget, *, financial_run, candidate=None): + """Admit only a live completed graph run under its own original budget.""" + from . import graph_atomic_survey_financial as runner + from . import survey_origin_budget as budgets + + _require( + candidate is None + or (type(candidate) is bytes and len(candidate) <= MAX_PAYLOAD_BYTES), + "CANDIDATE_TYPE_OR_BOUND", + ) + # An unissued descriptive dataclass must fail before any owner/file borrow. + run_entry = runner._run_entry(financial_run) + budget_entry = budgets._entry(budget, budgets.SamplingOriginBudget) + budget_view = budget.checked_view() + run_view = runner.check_atomic_survey_financial_run(financial_run) + state = _State( + budget, + budget_entry, + financial_run, + run_entry, + budget_entry[2].expanded, + financial_run.financial_population, + budgets._population_identity(budget_entry[2].expanded), + budgets._population_identity(financial_run.financial_population), + ) + payload = _document(state, budget_view, run_view) + _require(len(payload) <= MAX_PAYLOAD_BYTES, "PAYLOAD_BOUND") + _require(candidate is None or candidate == payload, "CANDIDATE_RECONSTRUCTION") + result = object.__new__(SamplingOriginFinancialSuccessor) + object.__setattr__(result, "payload", payload) + identifier = id(result) + + def forget(reference): + entry = _ISSUED.get(identifier) + if entry is not None and entry[0] is reference: + _ISSUED.pop(identifier, None) + + reference = weakref.ref(result, forget) + _ISSUED[identifier] = (reference, payload, state) + entry = _entry(result) + _final_state(state) + _require(_entry(result) is entry, "FINAL_ISSUANCE") + return result + + +def verify_survey_financial_successor(binding): + _entry(binding) + binding.checked_view() + return binding diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/survey_observed_age.py b/packages/microcosm-build/src/microcosm/build/us_runtime/survey_observed_age.py new file mode 100644 index 000000000..fde1388e8 --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/survey_observed_age.py @@ -0,0 +1,96 @@ +"""Pure observed-age alias; no source, population or release authority. + +The caller authenticates the integer A_AGE observations. This rule changes no +observation date, top code or raw value and reads no source/weight/domain label. +""" + +from __future__ import annotations + +import numpy as np +import pandas as pd + +RULE = "microcosm.us.observed-age-normalization.v1" +AGE_CONVENTION = "observed_interview_age_completed_years" +MAX_ROWS = 2_000_000 +MAX_EXACT_FLOAT64_INTEGER = 2**53 + + +def rule_document() -> dict: + return { + "protocol": RULE, + "input": "A_AGE", + "output": "age", + "dtype": "float64", + "relation": "numeric_identity", + "age_convention": AGE_CONVENTION, + "existing_nonmissing": "must_equal_observed_age", + "temporal_adjustment": False, + "top_code_replacement": False, + } + + +def _require(condition, reason): + if not condition: + raise ValueError("SURVEY_OBSERVED_AGE_" + reason) + + +def normalize_observed_age( + raw_age: pd.Series, incumbent_age: pd.Series | None = None +) -> pd.Series: + """Return a defensive common age, accepting only verified numeric identity. + + Missing common cells may acquire the already observed value; missing or + malformed raw ages refuse. The float bound is a representation limit, not + a scientifically permitted maximum age. Measurement keeps its own range. + """ + _require(isinstance(raw_age, pd.Series), "RAW_SERIES") + _require(0 < len(raw_age) <= MAX_ROWS, "ROW_BOUND") + _require( + pd.api.types.is_integer_dtype(raw_age.dtype) + and not pd.api.types.is_bool_dtype(raw_age.dtype) + and not raw_age.isna().any(), + "RAW_INTEGER_REQUIRED", + ) + _require( + ((raw_age >= 0) & (raw_age <= MAX_EXACT_FLOAT64_INTEGER)).all(), + "RAW_REPRESENTATION", + ) + values = raw_age.to_numpy(dtype=np.float64, copy=True) + if incumbent_age is not None: + _require(isinstance(incumbent_age, pd.Series), "INCUMBENT_SERIES") + _require( + incumbent_age.index.identical(raw_age.index) + and len(incumbent_age) == len(raw_age), + "ORDERED_INDEX", + ) + present = incumbent_age.notna().to_numpy(dtype=bool) + if present.any(): + _require( + pd.api.types.is_numeric_dtype(incumbent_age.dtype) + and not pd.api.types.is_bool_dtype(incumbent_age.dtype) + and not pd.api.types.is_complex_dtype(incumbent_age.dtype), + "INCUMBENT_NUMERIC_REQUIRED", + ) + known = incumbent_age[present] + _require( + ((known >= 0) & (known <= MAX_EXACT_FLOAT64_INTEGER)).all(), + "INCUMBENT_REPRESENTATION", + ) + _require((known == raw_age[present]).all(), "INCUMBENT_CONFLICT") + if pd.api.types.is_integer_dtype(incumbent_age.dtype): + _require( + np.array_equal( + known.to_numpy(dtype=np.int64), + raw_age[present].to_numpy(dtype=np.int64), + ), + "INCUMBENT_CONFLICT", + ) + incumbent = incumbent_age.to_numpy(dtype=np.float64, na_value=np.nan) + _require( + np.isfinite(incumbent[present]).all() + and np.array_equal(incumbent[present], values[present]), + "INCUMBENT_CONFLICT", + ) + # Preserve the original float bits of every known incumbent cell. + values[present] = incumbent[present] + return pd.Series(values, index=raw_age.index.copy(deep=True), name="age") diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/survey_origin_budget.py b/packages/microcosm-build/src/microcosm/build/us_runtime/survey_origin_budget.py new file mode 100644 index 000000000..6a72b771d --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/survey_origin_budget.py @@ -0,0 +1,1345 @@ +"""Source-qualified, sampling-aware budgets for the complete first survey clone. + +Original publisher d is never replaced. The sampling reference b=d/p and +incoming a=s*b are different quantities. These development coefficients are +provisional; this module grants neither graph execution nor release authority. +Decoded bytes cannot issue a budget or a weight-only successor. +An optional immutable geography recipe independently reconstructs the complete +postclone geography Frame; the retained allocation remains the raw source allocation. +""" + +from __future__ import annotations + +import hashlib +import json +import math +import sys +import weakref +from dataclasses import asdict, dataclass +from fractions import Fraction +from pathlib import Path +from types import FunctionType + +import numpy as np +import pandas as pd + +from microcosm.calibrate import group_bounds +from microcosm.frame import Frame, WeightKind +from microcosm.graph import ( + ArtifactType, + KernelResult, + Node, + StructuralDelta, + WeightTransition, +) +from microcosm.graph.population import ( + Population, + _mass_record, + _storage_parts, + storage_equal, +) + +from . import graph_combined_clone as clone +from . import graph_survey_population as graph +from . import survey_atomic_geography as geography +from . import survey_financial_successor as financial_successor +from . import survey_population_preparation as source +from .graph_sources import frame_column_declarations +from .support_provenance import support_clone_index_column, support_source_id_column + +BUDGET_PROTOCOL = "microcosm.us.sampling-origin-budget.v1" +SUCCESSOR_PROTOCOL = "microcosm.us.sampling-origin-weight-only-successor.v1" +BUDGET_TYPE = ArtifactType("microcosm.us.sampling_origin_budget", 1) +SUCCESSOR_TYPE = ArtifactType("microcosm.us.sampling_origin_weight_only_successor", 1) +MAX_PAYLOAD_BYTES = 64 * 1024**2 +MAX_GROUPS = 1_000_000 +MAX_SCALAR_CHARS = 4096 +PRESCRIPTION = ( + "development-provisional-8b-4a-v1", + "allocation=float(exact d*s/p)", + "reference=float(exact d/p)", + "bounds=min(8.0*reference,4.0*actual_incoming)", +) +_ISSUED = {} + + +class SurveyOriginBudgetError(ValueError): + """Static refusal without source observations or identifiers.""" + + +def _require(condition, reason): + if not condition: + raise SurveyOriginBudgetError(reason) + + +def _sha(value): + return hashlib.sha256(value).hexdigest() + + +def _json(value): + return graph._bounded_json(value, MAX_PAYLOAD_BYTES) + + +def _pair(value): + _require(type(value) is Fraction, "EXACT_FRACTION_REQUIRED") + _require( + value.numerator.bit_length() <= 128 and value.denominator.bit_length() <= 128, + "FRACTION_BOUND", + ) + return [value.numerator, value.denominator] + + +def _reference(d, p, s, actual_incoming): + """Pure exact arithmetic, not source authority; no float-product shortcut.""" + for value in (d, p, s): + _pair(value) + _require( + d >= 0 and 0 < p <= 1 and s in (Fraction(1), Fraction(1, 2)), "REFERENCE_DOMAIN" + ) + _require(type(actual_incoming) is float, "INCOMING_FLOAT64_REQUIRED") + b, a = d / p, d * s / p + expected_a, reference = float(a), float(b) + _require( + math.isfinite(actual_incoming) + and actual_incoming.hex() == expected_a.hex() + and math.isfinite(reference), + "ALLOCATION_OPERATION_ORDER", + ) + design_bound, incoming_bound = 8.0 * reference, 4.0 * actual_incoming + _require( + all(math.isfinite(v) and v >= 0 for v in (design_bound, incoming_bound)), + "REFERENCE_OVERFLOW", + ) + upper = min(design_bound, incoming_bound) + _require(upper == incoming_bound, "CURRENT_SHARE_BOUND_RELATION") + return { + "d": _pair(d), + "p": _pair(p), + "s": _pair(s), + "b": _pair(b), + "a": _pair(a), + "original_float64_hex": float(d).hex(), + "reference_float64_hex": reference.hex(), + "actual_incoming_float64_hex": actual_incoming.hex(), + "design_bound_float64_hex": design_bound.hex(), + "incoming_bound_float64_hex": incoming_bound.hex(), + "upper_float64_hex": upper.hex(), + } + + +def _modules(): + modules = ( + sys.modules[__name__], + graph, + clone, + group_bounds, + sys.modules[Population.__module__], + sys.modules[Frame.__module__], + sys.modules[frame_column_declarations.__module__], + sys.modules[support_source_id_column.__module__], + sys.modules[clone.clone_us_frame_for_puf_support.__module__], + geography, + geography.replay, + financial_successor, + geography.blocks, + geography.atomic, + geography.atomic_graph, + geography.composition, + geography.observed, + geography.observed_graph, + geography.observed.demographics, + geography.observed.demographics.demographic, + geography.observed.demographics.household, + geography.observed.demographics.source_csv_builtin, + sys.modules[geography.observed.demographics.demographic._snapshot.__module__], + sys.modules[geography.atomic.keyed_uniform.__module__], + sys.modules[geography.canonical_json.__module__], + sys.modules[geography.compile_graph.__module__], + sys.modules[geography._expand_declared_payload.__module__], + ) + return tuple({module.__name__: module for module in modules}.values()) + + +def _live(): + result = {} + for module in _modules(): + for name, value in vars(module).items(): + if isinstance(value, FunctionType): + result[module.__name__, name] = source._function_seal(value) + elif isinstance(value, type) and value.__module__ == module.__name__: + result[module.__name__, name] = value + for method, function in vars(value).items(): + if isinstance(function, (staticmethod, classmethod)): + function = function.__func__ + if isinstance(function, property): + function = function.fget + if isinstance(function, FunctionType): + result[module.__name__, name, method] = source._function_seal( + function + ) + result["contract"] = ( + BUDGET_PROTOCOL, + SUCCESSOR_PROTOCOL, + (BUDGET_TYPE.name, BUDGET_TYPE.schema_version), + (SUCCESSOR_TYPE.name, SUCCESSOR_TYPE.schema_version), + MAX_PAYLOAD_BYTES, + MAX_GROUPS, + MAX_SCALAR_CHARS, + PRESCRIPTION, + np.__version__, + np.ndarray, + np.float64, + ) + result["geography_contract"] = source._runtime_marker( + ( + geography.PROTOCOL, + geography.MAX_SUPPORT_BYTES, + geography.MAX_SUPPORT_EXPANDED_BYTES, + geography.MAX_SUPPORT_MEMBERS, + geography.RAW_BYTES_MAX_BYTES, + geography.blocks.SYSTEM, + geography.blocks.SOURCE, + tuple(sorted(geography.blocks._STATES)), + tuple(sorted(geography.atomic.RELATIONS)), + geography.atomic._U53, + geography.observed.PROTOCOL, + geography.observed.COLUMNS, + geography.composition.ASSIGNMENT_IDENTITY, + geography.observed.MAX_HOUSEHOLDS, + geography.observed.MAX_RECEIPT_BYTES, + geography.observed_graph.NODE, + geography.observed_graph.REF, + geography.observed_graph.PHASE, + geography.observed_graph.OUTPUT_COLUMNS, + geography.observed.demographics.PROTOCOL, + geography.observed.demographics.STATE_COLUMNS, + geography.observed.demographics.STATE_CONTRACT, + ( + geography.atomic_graph.ATOMIC_SUPPORT_TYPE.name, + geography.atomic_graph.ATOMIC_SUPPORT_TYPE.schema_version, + ), + ( + geography.atomic_graph.ATOMIC_GEOGRAPHY_VALIDATION_TYPE.name, + geography.atomic_graph.ATOMIC_GEOGRAPHY_VALIDATION_TYPE.schema_version, + ), + geography.atomic_graph._DEPENDENCIES, + tuple( + (kernel.ref, asdict(kernel.capabilities)) + for kernel in ( + geography.atomic_graph.AtomicSupportImportKernel, + geography.atomic_graph.AtomicAssignKernel, + geography.atomic_graph.AtomicDeriveKernel, + geography.atomic_graph.AtomicGeographyGateKernel, + ) + ), + ) + ) + result["financial_successor_contract"] = ( + financial_successor.PROTOCOL, + financial_successor.FINANCIAL_SUCCESSOR_TYPE.name, + financial_successor.FINANCIAL_SUCCESSOR_TYPE.schema_version, + financial_successor.MAX_PAYLOAD_BYTES, + ) + return result + + +def _code_bytes(): + return {m.__name__: _sha(Path(m.__file__).read_bytes()) for m in _modules()} + + +def _producer(): + _require(_live() == _LIVE and _code_bytes() == _BYTES, "PRODUCER_CHANGED") + return _BYTES + + +def _population_identity(population): + _require(type(population) is Population, "LIVE_POPULATION_REQUIRED") + return ( + source._frame_identity(population.frame), + _physical_nonweight_identity(population.frame), + population.version, + tuple(sorted(population.owners.items())), + tuple(population.weight_kind.items()), + _json([asdict(record) for record in population.mass_ledger]), + tuple( + (e, str(v.dtype), v.shape, v.tobytes()) + for e, v in population.design_weights.items() + ), + ) + + +def _physical_nonweight_identity(frame): + """Detach actual storage, including masked numeric slots, one column at a time. + + Axis/schema/dtype identity is bound separately by the full Frame identity. + This is an in-process population seal, not a cache canonicalization boundary. + """ + digest = hashlib.sha256() + for series in ( + *( + frame.table(entity)[column] + for entity in frame.entities + for column in frame.table(entity) + ), + frame.strata, + ): + selected = np.ones(len(series), dtype=np.bool_) + for part in _storage_parts(series, selected): + digest.update(len(part).to_bytes(8, "little")) + digest.update(part) + return digest.hexdigest() + + +def _config_payload(config): + return ( + None + if config is None + else geography.AtomicSurveyReconstruction.to_bytes(config) + ) + + +def _geography_binding(value, config_payload, preparation_payload): + """Seal fresh reconstruction values; decoded receipts never issue authority.""" + _require( + type(value) is geography.AtomicSurveyGeographyReconstruction, + "GEOGRAPHY_RECONSTRUCTION_TYPE", + ) + document = json.loads(value.receipt) + population_sha256 = geography._population_stamp(value.population) + stages = {stage.node.id: stage for stage in value.stages} + _require( + value.observed_population is stages[geography.observed_graph.NODE].population + and value.expanded_population + is stages[clone.COMBINED_CLONE_CLAIM_NODE].population, + "GEOGRAPHY_STAGE_BINDINGS", + ) + validation = stages["geography.gate"].receipt + _require( + stages["geography.gate"].artifacts == (("validation", validation),), + "GEOGRAPHY_VALIDATION_ARTIFACT", + ) + _require( + value.config_sha256 == _sha(config_payload) + and document["protocol"] == geography.PROTOCOL + and document["preparation_sha256"] == _sha(preparation_payload) + and document["config_sha256"] == value.config_sha256 + and document["frame_sha256"] == source._frame_identity(value.population.frame) + and document["population_sha256"] == population_sha256 + and document["observed_population_sha256"] + == geography._population_stamp(value.observed_population) + and document["expanded_population_sha256"] + == geography._population_stamp(value.expanded_population) + and document["validation_receipt_sha256"] == _sha(validation) + and document["assignment_identity"] + == list(geography.composition.ASSIGNMENT_IDENTITY) + and document["projection_receipt_sha256"] == _sha(value.projection_receipt) + and document["definition_sha256"] == _sha(value.definition) + and document["support_sha256"] == _sha(value.support_payload) + and document["publisher_provenance_established"] is False + and document["source_admission_issued"] is False + and document["population_admission_issued"] is False + and document["release_eligible"] is False, + "GEOGRAPHY_RECONSTRUCTION_SEAL", + ) + # The full receipt above seals this detached object exactly. Frame-store + # replay can canonicalize storage beneath masked nulls, so its physical + # stamp cannot identify an independently reconstructed object across runs. + physical_fields = ( + "population_sha256", + "observed_population_sha256", + "expanded_population_sha256", + ) + semantic_receipt = { + key: item for key, item in document.items() if key not in physical_fields + } + population = value.population + semantic_population = { + "frame_sha256": document["frame_sha256"], + "version": population.version, + "owners": sorted(population.owners.items()), + "weight_kind": [ + (entity, kind.value) for entity, kind in population.weight_kind.items() + ], + "mass_ledger": [asdict(record) for record in population.mass_ledger], + "design_weights": [ + (entity, str(values.dtype), values.shape, values.tobytes().hex()) + for entity, values in population.design_weights.items() + ], + } + return _json( + { + "config_sha256": value.config_sha256, + "reconstruction_semantic_sha256": _sha(_json(semantic_receipt)), + "projection_receipt_sha256": _sha(value.projection_receipt), + "definition_sha256": _sha(value.definition), + "support_sha256": _sha(value.support_payload), + "postclone_geography_population_semantic_sha256": _sha( + _json(semantic_population) + ), + "validation_receipt": json.loads(validation), + "identity_scope": { + "reconstruction_omitted_fields": list(physical_fields), + "population_fields": list(semantic_population), + "null_backing": "retained_in_in_process_physical_seals_only", + }, + "publisher_provenance_established": False, + } + ) + + +def _initial( + view, + allocated, + expanded, + *, + preparation=None, + geography_config=None, + _with_geography_binding=False, +): + _require(type(_with_geography_binding) is bool, "GEOGRAPHY_BINDING_FLAG") + _require( + type(allocated) is Population and type(expanded) is Population, + "LIVE_POPULATION_REQUIRED", + ) + instructions = graph.allocation_instructions( + view.selection_plan, view.receipt["origins"]["households"] + ) + _require(0 < len(instructions) <= MAX_GROUPS, "GROUP_COUNT_BOUND") + design = view.frame.weights_for("household").values.copy() + _weights, _context, allocation, receipt, expected = graph._allocation_output( + view.frame, view.context, instructions, _sha(view.payload) + ) + columns = frame_column_declarations(view.frame) + nodes = graph.survey_population_nodes( + columns, + preparation_sha256=_sha(view.payload), + fraction=view.selection_plan.fraction, + seed=view.selection_plan.seed, + ) + graph._same_frame(expected, allocated.frame) + graph._check_design_anchors(allocated, design) + allocation_ledger = ( + _mass_record( + view.frame, expected, nodes[1], KernelResult(receipt=receipt), "declared" + ), + ) + cells = tuple((e, str(c)) for e in view.frame.entities for c in view.frame.table(e)) + graph._check_population_state( + allocated, + version=graph.ALLOCATION_NODE, + owners=dict.fromkeys(cells, graph.ALLOCATION_NODE), + kind=WeightKind.IMPORTANCE, + ledger=allocation_ledger, + ) + geography_binding = None + if geography_config is not None: + config_payload = geography.AtomicSurveyReconstruction.to_bytes(geography_config) + reconstructed = geography.reconstruct_atomic_survey_geography( + preparation, allocated, geography_config + ) + geography_binding = _geography_binding( + reconstructed, config_payload, view.payload + ) + # Verify the exact clone before assignment, then the complete geography + # population. Geography additions never relax the clone equality rule. + graph._verify_cloned_frame( + reconstructed.observed_population.frame, + reconstructed.expanded_population.frame, + design, + ) + geography.replay.same_replayed_population(reconstructed.population, expanded) + else: + clone_nodes = clone.us_combined_survey_clone_nodes( + columns, base=graph.ALLOCATION_NODE, source_channels=("acs", "asec") + ) + copied_design = graph._verify_cloned_frame(expected, expanded.frame, design) + graph._check_design_anchors(expanded, copied_design) + owners = { + (entity, str(column)): clone.COMBINED_CLONE_NODE + for entity in expected.entities + for column in expected.table(entity) + } + owners.update( + { + (o.entity, o.column): clone.COMBINED_CLONE_CLAIM_NODE + for o in clone_nodes[1].outputs + } + ) + graph._check_population_state( + expanded, + version=clone.COMBINED_CLONE_NODE, + owners=owners, + kind=WeightKind.IMPORTANCE, + ledger=( + *allocation_ledger, + _mass_record( + expected, expanded.frame, clone_nodes[0], KernelResult(), "conserve" + ), + ), + ) + if geography_config is not None: + _require( + geography.AtomicSurveyReconstruction.to_bytes(geography_config) + == config_payload + and _geography_binding(reconstructed, config_payload, view.payload) + == geography_binding, + "FINAL_GEOGRAPHY_RECONSTRUCTION_SEAL", + ) + # Current survey financial and diagnostic hosts consume the historical pair. + # Only the budget owner explicitly requests its additional reconstruction seal. + if _with_geography_binding: + return instructions, allocation, geography_binding + return instructions, allocation + + +def _document( + view, + allocated, + expanded, + instructions, + allocation, + producer, + geography_binding=None, +): + before, after = ( + allocated.frame.table("household"), + expanded.frame.table("household"), + ) + source_column = support_source_id_column("household") + role_column = support_clone_index_column("household") + positions = {int(value): i for i, value in enumerate(before[source_column])} + _require(len(positions) == len(instructions), "ROOT_SOURCE_ID_COLLISION") + members = [[] for _ in instructions] + ids, groups = [], [] + for row in after[["household_id", source_column, role_column]].itertuples( + index=False, name=None + ): + hh_id, source_id, role = (int(value) for value in row) + _require(source_id in positions and role in (0, 1), "CLONE_ORIGIN_ROLE") + group = positions[source_id] + members[group].append((hh_id, role)) + ids.append(hh_id) + groups.append(group) + _require( + len(ids) == 2 * len(instructions) and len(set(ids)) == len(ids), + "CLONE_CARDINALITY", + ) + incoming = allocated.frame.weights_for("household").values + clone_weights = expanded.frame.weights_for("household").values + by_id = {value: i for i, value in enumerate(ids)} + bounds = [] + selected_by_key = {row.key: row for row in view.selection_plan.selected} + + def records(): + for i, (instruction, group_members) in enumerate( + zip(instructions, members, strict=True) + ): + _require( + len(group_members) == 2 and {m[1] for m in group_members} == {0, 1}, + "COMPLETE_CLONE_ROLES", + ) + actual = float(incoming[i]) + _require( + math.fsum(float(clone_weights[by_id[m[0]]]) for m in group_members) + == actual, + "CLONE_INCOMING_CONSERVATION", + ) + reference = _reference( + instruction.original_anchor, + instruction.inclusion_probability, + instruction.share, + actual, + ) + bounds.append(float.fromhex(reference["upper_float64_hex"])) + key = instruction.key + _require( + type(key.native_id) is str and len(key.native_id) <= MAX_SCALAR_CHARS, + "NATIVE_KEY_BOUND", + ) + yield { + "source": key.source.value, + "source_year": key.source_year, + "survey_year": key.survey_year, + "raw_native_id": key.native_id, + "selected_receiving_household_id": instruction.selected_receiving_household_id, + "combined_household_id": instruction.household_id, + "statistical_unit": selected_by_key[instruction.key].statistical_unit, + "original_design_float64_bytes": allocated.design_weights["household"][ + i : i + 1 + ] + .tobytes() + .hex(), + **reference, + "members": [[hh_id, role] for hh_id, role in group_members], + "incoming_clone_float64_bytes": [ + clone_weights[by_id[hh_id] : by_id[hh_id] + 1].tobytes().hex() + for hh_id, _role in group_members + ], + } + + header = { + "protocol": BUDGET_PROTOCOL, + "prescription": PRESCRIPTION, + "coefficient_status": "provisional_development_not_scientific_approval", + "current_share_simplification": "min(8b,4a)=4a; 8b is redundant for current shares", + "preparation_sha256": _sha(view.payload), + "source_producer": view.receipt["producer"], + "source_native": view.receipt["native"], + "source_catalogues": view.receipt["catalogues"], + "selection_sha256": _sha(_json(view.receipt["selection"])), + "allocation_sha256": _sha(allocation), + "producer": producer, + "allocated_frame_sha256": source._frame_identity(allocated.frame), + "clone_frame_sha256": source._frame_identity(expanded.frame), + "household_ids": ids, + "group_indices": groups, + "group_count": len(instructions), + "release_eligible": False, + } + if geography_binding is not None: + header["atomic_geography"] = json.loads(geography_binding) + # Stream one bounded origin record at a time; never materialize an unbounded + # list of origin dictionaries before the transport cap is checked. + payload = bytearray() + + def append(piece): + _require(len(piece) <= MAX_PAYLOAD_BYTES - len(payload), "TRANSPORT_LIMIT") + payload.extend(piece) + + append(b"{") + keys = sorted((*header, "origins")) + for index, key in enumerate(keys): + if index: + append(b",") + append(_json(key) + b":") + if key == "origins": + append(b"[") + for position, record in enumerate(records()): + if position: + append(b",") + append(_json(record)) + append(b"]") + else: + append(_json(header[key])) + append(b"}") + constraint = group_bounds.GroupedUpperBounds(ids, groups, bounds) + constraint.check(clone_weights, positive=False) + return bytes(payload), constraint + + +@dataclass(frozen=True) +class _BudgetState: + preparation: object + preparation_entry: tuple + initial_view: source.CheckedSurveyPopulationView + allocated: Population + expanded: Population + preparation_payload: bytes + allocated_identity: tuple + expanded_identity: tuple + geography_config: geography.AtomicSurveyReconstruction | None + geography_config_payload: bytes | None + geography_binding: bytes | None + + +def _preparation_entry(preparation, payload, expected=None): + """Check retained issuance identity without repeating source-file reads.""" + entry = source._ISSUED.get(id(preparation)) + _require( + type(preparation) is source.AuthenticatedSurveyPopulationPreparation + and entry is not None + and (expected is None or entry is expected) + and entry[0]() is preparation + and type(preparation.payload) is bytes + and preparation.payload == entry[1] == payload, + "FINAL_PREPARATION_ISSUANCE", + ) + return entry + + +def _final_budget_state(state): + """Seal retained owners after helper/file I/O, with no recursive borrow. + + Optional support bytes are checked before the source owner's pure final + check. That pure check hashes retained Frames without rereading survey + files, issuing replacement capsules or granting decoded authority. + """ + _producer() + _require( + _config_payload(state.geography_config) == state.geography_config_payload, + "FINAL_GEOGRAPHY_CONFIG_SEAL", + ) + if state.geography_config is not None: + # Finish support I/O before the same pure owner/Population final seals. + # Re-reading exact pinned bytes grants no publisher provenance. + geography._read_support(state.geography_config) + _pure_budget_state(state) + + +def _pure_budget_state(state): + """Check retained source/Population state without a new file borrow.""" + entry = _preparation_entry( + state.preparation, state.preparation_payload, state.preparation_entry + ) + source._pure_final(entry[2]) + _require( + _population_identity(state.allocated) == state.allocated_identity + and _population_identity(state.expanded) == state.expanded_identity, + "FINAL_POPULATION_SEAL", + ) + _require( + ( + state.geography_config is None + and state.geography_config_payload is None + and state.geography_binding is None + ) + or ( + state.geography_config is not None + and geography.AtomicSurveyReconstruction.to_bytes(state.geography_config) + == state.geography_config_payload + ), + "FINAL_GEOGRAPHY_CONFIG_SEAL", + ) + _require(_live() == _LIVE, "FINAL_PRODUCER_SEAL") + _preparation_entry( + state.preparation, state.preparation_payload, state.preparation_entry + ) + + +def _validate_budget(state, payload): + _producer() + _require( + _config_payload(state.geography_config) == state.geography_config_payload, + "INITIAL_GEOGRAPHY_CONFIG_CHANGED", + ) + _require( + _population_identity(state.allocated) == state.allocated_identity + and _population_identity(state.expanded) == state.expanded_identity, + "INITIAL_POPULATION_CHANGED", + ) + # This private retained view is only a reconstruction input. The fresh + # source borrow below authenticates it after the first reconstruction. + view = state.initial_view + _require(view.payload == state.preparation_payload, "SOURCE_PREPARATION_CHANGED") + instructions, allocation, geography_binding = _initial( + view, + state.allocated, + state.expanded, + preparation=state.preparation, + geography_config=state.geography_config, + _with_geography_binding=True, + ) + _require(geography_binding == state.geography_binding, "GEOGRAPHY_RECONSTRUCTION") + expected, constraint = _document( + view, + state.allocated, + state.expanded, + instructions, + allocation, + _producer(), + geography_binding, + ) + _require(expected == payload, "BUDGET_RECONSTRUCTION") + final = source.AuthenticatedSurveyPopulationPreparation.checked_view( + state.preparation + ) + _require( + final.payload == view.payload + and final.frame is view.frame + and final.selection_plan is view.selection_plan, + "FINAL_SOURCE_SEAL", + ) + _instructions, _allocation, final_geography = _initial( + final, + state.allocated, + state.expanded, + preparation=state.preparation, + geography_config=state.geography_config, + _with_geography_binding=True, + ) + _require(final_geography == state.geography_binding, "FINAL_GEOGRAPHY_SEAL") + return constraint + + +def _entry(value, cls): + entry = _ISSUED.get(id(value)) + _require( + type(value) is cls + and entry is not None + and entry[0]() is value + and type(value.payload) is bytes + and value.payload == entry[1], + "UNISSUED_OR_CHANGED", + ) + return entry + + +def _issue(cls, payload, state): + value = object.__new__(cls) + object.__setattr__(value, "payload", payload) + identifier = id(value) + + def forget(reference): + current = _ISSUED.get(identifier) + if current is not None and current[0] is reference: + _ISSUED.pop(identifier, None) + + reference = weakref.ref(value, forget) + _ISSUED[identifier] = (reference, payload, state) + return value + + +def _check_view_constraint(constraint, document): + """Bind every derived numerical field to an already requalified issued view.""" + reason = "FINAL_BUDGET_VIEW_CONSTRAINT" + _require(type(constraint) is group_bounds.GroupedUpperBounds, reason) + ids = tuple(document["household_ids"]) + groups = document["group_indices"] + bounds_hex = [row["upper_float64_hex"] for row in document["origins"]] + # This is the maintained constraint's stable-ID reduction order. These + # expected buckets come from the issued document, not constructor output. + buckets = [[] for _ in bounds_hex] + for index in sorted( + range(len(ids)), key=lambda i: (isinstance(ids[i], str), ids[i]) + ): + buckets[groups[index]].append(index) + + def immutable_array(value, expected, dtype): + payload = np.asarray(expected, dtype=dtype).tobytes() + return ( + type(value) is np.ndarray + and value.dtype.str == dtype + and value.shape == (len(expected),) + and value.strides == (8,) + and not value.flags.writeable + and not value.flags.owndata + and type(value.base) is bytes + and value.base == payload + and value.tobytes() == payload + ) + + digest_payload = { + "contract": "stable-household-fsum-group-upper-v1", + "household_ids": ids, + "group_indices": groups, + "absolute_bounds_hex": bounds_hex, + } + expected_digest = _sha( + json.dumps( + digest_payload, sort_keys=True, separators=(",", ":"), ensure_ascii=True + ).encode() + ) + _require( + type(constraint.household_ids) is tuple + and len(constraint.household_ids) == len(ids) + and all( + type(actual) is type(expected) and actual == expected + for actual, expected in zip(constraint.household_ids, ids, strict=True) + ) + and immutable_array(constraint.group_indices, groups, " bool: + return False + + @property + def population_binding_authenticated(self) -> bool: + return False + + @property + def release_eligible(self) -> bool: + return False + + @property + def required_bindings(self) -> tuple[str, ...]: + return ( + "original_source_bytes", + "complete_original_membership", + "original_publisher_weight_authority", + "source_period_and_residence_scope", + "full_receiving_population_and_producer_edges", + "actual_selection_probability", + ) + + @property + def analysis_year(self) -> int: + return 2024 + + @property + def income_reference(self) -> str: + if self.original.key.source is Source.ASEC: + return f"calendar_year_{self.original.key.source_year}" + return "previous_12_months_at_each_2024_interview" + + +def _require(condition: bool, code: str) -> None: + if not condition: + raise DomainInputError(code) + + +def _token(value: str) -> str: + _require(type(value) is str and len(value) <= MAX_TOKEN_CHARS, "LITERAL_TYPE_SIZE") + return value + + +def _unsigned(raw: str, digits: int, maximum: int) -> int | None: + _token(raw) + if re.fullmatch(rf"[0-9]{{1,{digits}}}", raw) is None: + return None + value = int(raw) + return value if value <= maximum else None + + +def _key(key: HouseholdKey) -> tuple[Source, int, int, str | int]: + _require(type(key) is HouseholdKey and type(key.source) is Source, "HOUSEHOLD_KEY") + _require( + type(key.source_year) is int and type(key.survey_year) is int, "COHORT_TYPE" + ) + _token(key.native_id) + if key.source is Source.ACS: + _require((key.source_year, key.survey_year) == (2024, 2024), "COHORT_IDENTITY") + _require( + re.fullmatch(r"2024(?:HU|GQ)[0-9]{7}", key.native_id) is not None + and int(key.native_id[-7:]) > 0, + "HOUSEHOLD_KEY", + ) + native = key.native_id + else: + _require( + key.source_year in (2022, 2023, 2024) + and key.survey_year == key.source_year + 1, + "COHORT_IDENTITY", + ) + native = _unsigned(key.native_id, MAX_TOKEN_CHARS, 99999) + _require(native is not None and native > 0, "HOUSEHOLD_KEY") + return key.source, key.source_year, key.survey_year, native + + +def declaration() -> tuple[DomainShare, ...]: + """The prospectively fixed coefficients, without applying them to weights.""" + return ( + DomainShare( + Domain.SHARED_HOUSING, + "occupied_housing_unit", + Fraction(1, 2), + Fraction(1, 2), + ), + DomainShare( + Domain.RESIDUAL_HOUSING, "occupied_housing_unit", Fraction(1), Fraction(0) + ), + DomainShare(Domain.INSTITUTIONAL_GQ, "person", Fraction(1), Fraction(0)), + DomainShare(Domain.NONINSTITUTIONAL_GQ, "person", Fraction(1), Fraction(0)), + ) + + +def reduce_qualifiers(values: tuple[Qualifier, ...]) -> Qualifier: + """Existential reduction over known roster members, not membership proof.""" + _require( + type(values) is tuple and 0 < len(values) <= MAX_MEMBERS, "QUALIFIER_ROSTER" + ) + _require(all(type(v) is Qualifier for v in values), "QUALIFIER_TYPE") + if Qualifier.TRUE in values: + return Qualifier.TRUE + if Qualifier.UNKNOWN in values: + return Qualifier.UNKNOWN + return Qualifier.FALSE + + +def _acs_state(age: str, raw: str, minimum: int, codes: tuple[str, ...]) -> str: + # Exact vocabulary of acs_person_coverage_columns._field_state. No owner import. + value = _unsigned(age, 2, 99) + if value is None: + return "age_unresolved" + if value < minimum: + return "outside_age_universe" if raw == "" else "value_below_age_universe" + if raw == "": + return "missing_in_universe" + return "observed_code" if raw in codes else "unlabelled_code" + + +def _asec_state(raw: str) -> str: + # Exact vocabulary of asec_person_coverage_source._state. + if raw in ("1", "2", "3"): + return "observed_code" + if raw == "": + return "blank_unresolved" + if raw in ("-4", "-3", "-2", "-1", "0"): + return "unlabelled_in_range" + if re.fullmatch(r"-?(?:0|[1-9][0-9]*)", raw) and raw != "-0": + return "out_of_range" + return "malformed_token" + + +def _person(person: AcsPerson | AsecPerson) -> PersonDecision: + is_acs = type(person) is AcsPerson + age = _unsigned(person.age, 2 if is_acs else 19, 99 if is_acs else 2**63 - 1) + observations = [ + Observation( + "AGEP" if is_acs else "A_AGE", + person.age, + "observed_integer" if age is not None else "age_unresolved", + ) + ] + diagnostics = [] + if is_acs: + _token(person.pwgtp) + for raw, state in ( + (person.esr, person.esr_state), + (person.mil, person.mil_state), + ): + _token(raw) + _token(state) + _require( + person.esr_state + == _acs_state(person.age, person.esr, 16, ("1", "2", "3", "4", "5", "6")), + "FIELD_STATE_MISMATCH", + ) + _require( + person.mil_state + == _acs_state(person.age, person.mil, 17, ("1", "2", "3", "4")), + "FIELD_STATE_MISMATCH", + ) + observations.extend( + ( + Observation("ESR", person.esr, person.esr_state), + Observation("MIL", person.mil, person.mil_state), + ) + ) + true = person.esr in ("1", "2", "3", "6") + false = person.esr in ("4", "5") + if ( + person.mil_state == "observed_code" + and person.esr_state == "observed_code" + and ( + (person.mil == "1" and true) + or (person.mil in ("2", "3", "4") and false) + ) + ): + diagnostics.append("mil_esr_disagreement") + if person.mil_state != "observed_code": + diagnostics.append("mil_" + person.mil_state) + contradiction = False + else: + _token(person.prpertyp) + _token(person.prpertyp_state) + _require( + person.prpertyp_state == _asec_state(person.prpertyp), + "FIELD_STATE_MISMATCH", + ) + observations.append( + Observation("PRPERTYP", person.prpertyp, person.prpertyp_state) + ) + true, false = person.prpertyp == "2", person.prpertyp == "3" + contradiction = age is not None and age >= 16 and person.prpertyp == "1" + if age is None: + qualifier, reason = Qualifier.UNKNOWN, "source_age_unresolved" + elif age < 16: + qualifier, reason = Qualifier.FALSE, "source_age_below_16" + elif contradiction: + qualifier, reason = Qualifier.UNKNOWN, "adult_age_with_child_type" + elif true: + qualifier, reason = Qualifier.TRUE, "source_civilian_proxy" + elif false: + qualifier, reason = Qualifier.FALSE, "source_armed_forces_proxy" + else: + qualifier, reason = Qualifier.UNKNOWN, "required_person_status_unresolved" + return PersonDecision( + person, qualifier, reason, tuple(observations), tuple(diagnostics) + ) + + +def _members(row: AcsHousehold | AsecHousehold) -> tuple[PersonDecision, ...]: + _require( + type(row.persons) is tuple and len(row.persons) <= MAX_MEMBERS, + "MEMBER_TUPLE_BOUND", + ) + household = _key(row.key) + seen_keys, seen_lines = set(), set() + expected = AcsPerson if type(row) is AcsHousehold else AsecPerson + for person in row.persons: + _require(type(person) is expected, "MEMBER_TYPE") + _require(_key(person.household_key) == household, "MEMBER_HOUSEHOLD_KEY") + raw_line = person.sporder if expected is AcsPerson else person.a_lineno + line = _unsigned( + raw_line, + 2 if expected is AcsPerson else 19, + 20 if expected is AcsPerson else 2**63 - 1, + ) + _require( + line is not None and line > 0 and line not in seen_lines, "MEMBER_LINE_KEY" + ) + seen_lines.add(line) + person_key = line if expected is AcsPerson else _token(person.peridnum) + if expected is AsecPerson: + _require( + re.fullmatch(r"[0-9]{22}", person_key) is not None, "MEMBER_PERSON_KEY" + ) + _require(person_key not in seen_keys, "MEMBER_DUPLICATE_KEY") + seen_keys.add(person_key) + return tuple(_person(person) for person in row.persons) + + +def _field( + name: str, raw: str, digits: int, maximum: int, minimum: int = 0 +) -> tuple[int | None, Observation]: + value = _unsigned(raw, digits, maximum) + known = value is not None and value >= minimum + state = ( + "valid" + if known + else "missing" + if raw == "" + else "malformed" + if re.fullmatch(rf"[0-9]{{1,{digits}}}", raw) is None + else "unlabelled" + ) + return value if known else None, Observation(name, raw, state) + + +def classify_household(row: AcsHousehold | AsecHousehold) -> HouseholdDecision: + """Classify supplied complete rows; unresolved results authorize no allocation. + + Key/tuple/count inconsistencies raise. Unknown required observations retain + their literals and produce REVIEW_REQUIRED, never residual by default. + Source and population authority cannot be supplied as a boolean argument. + """ + _require(type(row) in (AcsHousehold, AsecHousehold), "HOUSEHOLD_TYPE") + key = _key(row.key) + is_acs = type(row) is AcsHousehold + _require(key[0] is (Source.ACS if is_acs else Source.ASEC), "HOUSEHOLD_SOURCE") + people = _members(row) + observations = [] + zero = None + + def field(name, raw, digits, maximum, minimum=0): + value, observation = _field(name, raw, digits, maximum, minimum) + observations.append(observation) + return value + + def result(status, reason, domain=None, share=None, unit=None, housing_units=None): + return HouseholdDecision( + row, + status, + domain, + share, + reason, + unit, + housing_units, + zero, + people, + tuple(observations), + ( + Annotation( + DECLARATION, + "source_age_16_ESR_only_or_PRPERTYP", + "joint_MIL_ESR_or_other_age_threshold", + "prespecified_development_choice", + ), + Annotation( + DECLARATION, + "fixed_domain_shares_and_structural_exclusions", + "estimated_overlap_or_optimized_shares", + "not_empirically_validated", + ), + Annotation( + DECLARATION, + reason, + "no_silent_residual_or_donor_admission", + status.value, + ), + ), + ) + + count = field( + "NP" if is_acs else "H_NUMPER", + row.np if is_acs else row.h_numper, + 2, + 20 if is_acs else 16, + ) + if count is not None: + _require(count == len(row.persons), "REPORTED_MEMBERSHIP_COUNT") + if is_acs: + housing = field("TYPEHUGQ", row.typehugq, 1, 3, 1) + weight = field("WGTP", row.wgtp, 4, 9999) + if count is None or housing is None: + return result(Status.REVIEW_REQUIRED, "acs_household_condition_unresolved") + if row.key.native_id[4:6] != ("HU" if housing == 1 else "GQ"): + return result(Status.REVIEW_REQUIRED, "acs_serialno_housing_conflict") + if housing == 1 and count == 0: + return result( + Status.EXCLUDED, "acs_vacancy", share=Fraction(0), housing_units=0 + ) + if housing in (2, 3): + if count != 1 or weight != 0: + return result(Status.REVIEW_REQUIRED, "acs_gq_placeholder_conflict") + person_weight = field("PWGTP", row.persons[0].pwgtp, 4, 9999, 1) + if person_weight is None: + return result(Status.REVIEW_REQUIRED, "publisher_weight_unresolved") + zero = False + return result( + Status.ELIGIBLE, + "acs_gq_allocation", + Domain.INSTITUTIONAL_GQ if housing == 2 else Domain.NONINSTITUTIONAL_GQ, + Fraction(1), + "person", + 0, + ) + if weight is None or weight <= 0: + return result(Status.REVIEW_REQUIRED, "publisher_weight_unresolved") + zero = False + else: + interview = field("H_HHTYPE", row.h_hhtype, 1, 3, 1) + household_type = field("HRHTYPE", row.hrhtype, 2, 10) + living = field("H_LIVQRT", row.h_livqrt, 2, 12, 1) + weight = field("HSUP_WGT", row.hsup_wgt, MAX_TOKEN_CHARS, 999999999) + if count is None: + return result(Status.REVIEW_REQUIRED, "asec_membership_count_unresolved") + if row.key.source_year != 2024: + return result(Status.EXCLUDED, "asec_older_cohort", share=Fraction(0)) + if interview in (2, 3): + return result(Status.EXCLUDED, "asec_noninterview", share=Fraction(0)) + if interview is None or living is None or household_type is None: + return result(Status.REVIEW_REQUIRED, "asec_household_condition_unresolved") + if living == 11: + return result( + Status.REVIEW_REQUIRED, + "asec_student_quarters_source_review", + share=Fraction(0), + housing_units=0, + ) + if living >= 8: + return result( + Status.EXCLUDED, "asec_nonhousing", share=Fraction(0), housing_units=0 + ) + if household_type not in range(1, 9) or count == 0: + return result(Status.REVIEW_REQUIRED, "asec_housing_membership_conflict") + if weight is None: + return result(Status.REVIEW_REQUIRED, "publisher_weight_unresolved") + zero = weight == 0 + predicate = reduce_qualifiers(tuple(person.qualifier for person in people)) + if predicate is Qualifier.UNKNOWN: + return result(Status.REVIEW_REQUIRED, "household_qualifier_unresolved") + if predicate is Qualifier.TRUE: + return result( + Status.ELIGIBLE, + "chosen_shared_housing", + Domain.SHARED_HOUSING, + Fraction(1, 2), + "occupied_housing_unit", + 1, + ) + return result( + Status.ELIGIBLE if is_acs else Status.EXCLUDED, + "acs_known_residual_housing" if is_acs else "asec_residual_zero_allocation", + Domain.RESIDUAL_HOUSING, + Fraction(1) if is_acs else Fraction(0), + "occupied_housing_unit", + 1, + ) + + +def classify_households( + rows: tuple[AcsHousehold | AsecHousehold, ...], +) -> tuple[HouseholdDecision, ...]: + """Classify a bounded batch, rejecting household and cohort-person collisions.""" + _require(type(rows) is tuple and 0 < len(rows) <= MAX_HOUSEHOLDS, "HOUSEHOLD_BATCH") + keys, members = set(), 0 + for row in rows: + _require(type(row) in (AcsHousehold, AsecHousehold), "HOUSEHOLD_TYPE") + key = _key(row.key) + _require(key not in keys, "DUPLICATE_HOUSEHOLD_KEY") + keys.add(key) + _require( + type(row.persons) is tuple and len(row.persons) <= MAX_MEMBERS, + "MEMBER_TUPLE_BOUND", + ) + members += len(row.persons) + _require(members <= MAX_TOTAL_MEMBERS, "BATCH_MEMBER_BOUND") + person_keys = set() + for row in rows: + if type(row) is AsecHousehold: + for person in row.persons: + _require(type(person) is AsecPerson, "MEMBER_TYPE") + literal = _token(person.peridnum) + _require( + re.fullmatch(r"[0-9]{22}", literal) is not None, + "MEMBER_PERSON_KEY", + ) + person_key = (row.key.source_year, literal) + _require(person_key not in person_keys, "DUPLICATE_COHORT_PERSON_KEY") + person_keys.add(person_key) + return tuple(classify_household(row) for row in rows) diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/survey_population_preparation.py b/packages/microcosm-build/src/microcosm/build/us_runtime/survey_population_preparation.py new file mode 100644 index 000000000..72428c0d1 --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/survey_population_preparation.py @@ -0,0 +1,1521 @@ +"""Authenticate complete catalogues, select once, and stack original anchors. + +The selection is a development declaration. Source issuers retain their own +exact-key/unknown-inclusion receipts; this successor binds the separately +demonstrated catalogue sampling relation. No allocation, calibration or donor +operation runs here. Source catalogues/parents still have full-source costs. +""" + +from __future__ import annotations + +import _csv +import csv +import hashlib +import json +import math +import os +import stat +import sys +import weakref +from contextlib import contextmanager +from dataclasses import InitVar, dataclass, fields +from enum import Enum +from fractions import Fraction +from pathlib import Path +from types import FunctionType + +import numpy as np +import pandas as pd + +from microcosm.build import survey_domain_sample +from microcosm.frame import US_SCHEMA, Frame, WeightKind, Weights +from microcosm.graph import population as graph_population +from microcosm.graph.population import dtype_for_token, storage_equal + +from . import acs_native_coverage_binding as acs_native +from . import acs_population_catalogue as acs_catalogue +from . import asec_2024_native_population as asec_native +from . import asec_current_money as current_money +from . import asec_population_catalogue as asec_catalogue +from . import graph_context, graph_sources, spine_assembly +from . import survey_catalogue_selection as selection +from . import survey_observed_age as observed_age +from . import survey_population_domains as domains +from .support_provenance import spine_source_id_column, support_channel_column + +PROTOCOL = "microcosm.us.survey-population-preparation.v2" +REQUEST_PROTOCOL = "microcosm.us.survey-population-request.v1" +SOURCE_CODEC = "us-survey-population-source-v1" +MAX_REQUEST_BYTES = 4096 +MAX_PAYLOAD_BYTES = 64 * 1024**2 +_MAX_SCALAR_BYTES = 1024**2 +_FRAME_FAST_STRING_CHARS = 4096 +_SOURCE_ROSTER = ( + "selection-request.json", + "acs/csv_hus.zip", + "acs/csv_pus.zip", + "asec/parent.h5", + "asec/household-attachment.h5", + "asec/person-income-attachment.h5", + "asec/pppub23.csv", + "asec/pppub24.csv", + "asec/pppub25.csv", + "asec/hhpub25.csv", +) +_TOKEN = object() +_ISSUED = {} + + +class SurveyPopulationPreparationError(ValueError): + """Static refusal; messages contain no source observations or paths.""" + + +def _require(condition, code): + if not condition: + raise SurveyPopulationPreparationError(code) + + +def _sha(value): + return hashlib.sha256(value).hexdigest() + + +def _check_scalars(value, depth=0): + _require(depth <= 64, "VALUE_DEPTH") + if type(value) is str: + _require(len(value) <= _MAX_SCALAR_BYTES, "SCALAR_LIMIT") + elif type(value) in (tuple, list): + for item in value: + _check_scalars(item, depth + 1) + elif type(value) is dict: + for key, item in value.items(): + _check_scalars(key, depth + 1) + _check_scalars(item, depth + 1) + + +def _chunks(value): + # A single primitive is bounded before JSON quoting. The encoder never + # builds an unbounded complete JSON string before enforcing the transport. + _check_scalars(value) + yield from json.JSONEncoder( + sort_keys=True, separators=(",", ":"), ensure_ascii=False, allow_nan=False + ).iterencode(value) + + +def _encode(value, maximum=MAX_PAYLOAD_BYTES): + result = bytearray() + for chunk in _chunks(value): + encoded = chunk.encode("utf-8") + _require(len(result) + len(encoded) <= maximum, "PAYLOAD_LIMIT") + result.extend(encoded) + return bytes(result) + + +def _digest(value): + digest = hashlib.sha256() + for chunk in _chunks(value): + digest.update(chunk.encode("utf-8")) + return digest.hexdigest() + + +def _catalogue_fast_path(value): + """Recognize immutable, bounded raw records without changing admission.""" + if type(value) is not tuple or len(value) != 2: + return False + for group in value: + if type(group) is not tuple: + return False + for record in group: + if type(record) is not tuple or len(record) != 7: + return False + people = record[6] + if type(people) is not tuple or len(people) > 20: + return False + if any(type(person) is not tuple or len(person) != 9 for person in people): + return False + characters = 0 + for row in (record[:6], *people): + for item in row: + if type(item) is str: + if not item.isascii(): + return False + characters += len(item) + if characters > 65_536: + return False + elif type(item) is int: + if not -(2**63) <= item < 2**63: + return False + else: + return False + return True + + +def _catalogue_chunks(value): + # Decide for the whole value before encoding. Unexpected shapes retain + # the generic encoder's full-depth scalar pass and error precedence. + if not _catalogue_fast_path(value): + yield from _chunks(value) + return + _check_scalars(value) + encoder = json.JSONEncoder( + sort_keys=True, separators=(",", ":"), ensure_ascii=False, allow_nan=False + ) + yield "[" + for group_index, group in enumerate(value): + if group_index: + yield "," + yield "[" + for record_index, record in enumerate(group): + if record_index: + yield "," + # encode uses the ordinary one-shot C provider when available. + # At most 186 leaves and 65,536 ASCII characters bound this + # individual JSON record below 400 KiB, including escaping. + yield encoder.encode(record) + yield "]" + yield "]" + + +def _catalogue_digest(value): + digest = hashlib.sha256() + for chunk in _catalogue_chunks(value): + digest.update(chunk.encode("utf-8")) + return digest.hexdigest() + + +def _catalogue_memo(value, expected_digest): + """Retain one proved immutable tree, never an unchecked caller digest. + + The exact-tuple/int/ASCII-str eligibility test also excludes mutable leaves + and subclasses. Strong references prevent identity reuse. Rehash once while + creating the memo to bind those exact immutable roots to the earlier seal. + A replacement tree or an ineligible shape keeps the original digest path. + """ + if not _catalogue_fast_path(value): + return None + if _catalogue_digest(value) != expected_digest: + return None + return (value[0], value[1], expected_digest) + + +def _memoized_catalogue_digest(value, memo=None, *, expected_digest=None): + """Reuse only the issued preparation's unchanged immutable record roots. + + This private memo is not source authority. The owning preparation still + performs every source/producer borrow and full physical Frame seal. Its + original nested digest is checked independently of the memo's digest. + """ + if ( + type(value) is tuple + and len(value) == 2 + and type(memo) is tuple + and len(memo) == 3 + and value[0] is memo[0] + and value[1] is memo[1] + and type(memo[2]) is str + and memo[2] == expected_digest + ): + return memo[2] + return _catalogue_digest(value) + + +def _value(value): + if isinstance(value, Enum): + return value.value + if type(value) is Fraction: + return [value.numerator, value.denominator] + if hasattr(type(value), "__dataclass_fields__"): + return { + field.name: _value(getattr(value, field.name)) for field in fields(value) + } + if type(value) in (tuple, list): + return [_value(item) for item in value] + if type(value) is dict: + return {key: _value(item) for key, item in value.items()} + if isinstance(value, np.generic): + return value.item() + _require(value is None or type(value) in (str, int, bool, float), "VALUE_TYPE") + return value + + +def _plan_document(plan): + _require(type(plan) is selection.CatalogueSelectionPlan, "SELECTION_TYPE") + result = { + "fraction": _value(plan.fraction), + "seed": plan.seed, + "supplied_households": plan.supplied_households, + } + budget = [0] + for name in ("selected", "excluded", "cells"): + result[name] = [] + for row in getattr(plan, name): + _bounded_append(result[name], _value(row), budget) + return result + + +def _root(value, *, allow_missing=False): + _require(isinstance(value, (str, Path)), "PATH_TYPE") + path = Path(value).absolute() + for component in (path, *path.parents): + if allow_missing: + try: + component.lstat() + except FileNotFoundError: + continue + _require(not stat.S_ISLNK(component.lstat().st_mode), "SOURCE_SYMLINK") + return path + + +def _stat_identity(info): + return (info.st_dev, info.st_ino, info.st_size, info.st_mtime_ns, info.st_ctime_ns) + + +@contextmanager +def _regular_reader(path, maximum): + """Never follow a replaced final symlink or block on a substituted FIFO.""" + _root(path.parent) + before = path.lstat() + _require( + stat.S_ISREG(before.st_mode) and 0 <= before.st_size <= maximum, + "SOURCE_REGULAR_FILE", + ) + descriptor = os.open(path, os.O_RDONLY | os.O_NONBLOCK | os.O_NOFOLLOW) + try: + opened = os.fstat(descriptor) + _require( + stat.S_ISREG(opened.st_mode) + and _stat_identity(opened) == _stat_identity(before), + "SOURCE_DESCRIPTOR_CHANGED", + ) + with os.fdopen(descriptor, "rb", closefd=False) as stream: + yield stream, opened + after = os.fstat(descriptor) + current = path.lstat() + _require( + stat.S_ISREG(current.st_mode) + and _stat_identity(opened) + == _stat_identity(after) + == _stat_identity(current), + "SOURCE_CHANGED", + ) + finally: + os.close(descriptor) + + +def _request(source_dir): + root = _root(source_dir) + path = root / "selection-request.json" + info = path.lstat() + _require( + stat.S_ISREG(info.st_mode) and info.st_size <= MAX_REQUEST_BYTES, "REQUEST_FILE" + ) + with _regular_reader(path, MAX_REQUEST_BYTES) as (stream, _opened): + payload = stream.read(MAX_REQUEST_BYTES + 1) + _require(len(payload) <= MAX_REQUEST_BYTES, "REQUEST_LIMIT") + document = json.loads(payload) + _require( + type(document) is dict + and set(document) == {"protocol", "declaration", "fraction", "seed"}, + "REQUEST_FIELDS", + ) + _require( + document["protocol"] == REQUEST_PROTOCOL + and document["declaration"] == domains.DECLARATION, + "REQUEST_DECLARATION", + ) + pair = document["fraction"] + _require( + type(pair) is list + and len(pair) == 2 + and all(type(v) is int for v in pair) + and pair[1] > 0, + "REQUEST_FRACTION", + ) + fraction = Fraction(*pair) + _require( + 0 < fraction <= 1 and [fraction.numerator, fraction.denominator] == pair, + "REQUEST_FRACTION", + ) + seed = document["seed"] + _require(type(seed) is int and 0 <= seed < 2**64, "REQUEST_SEED") + _require(_encode(document, MAX_REQUEST_BYTES) == payload, "REQUEST_CANONICAL") + return root, document, payload, fraction, seed + + +def read_survey_population_request(source_dir): + """Decode bounded request arguments, without granting source authority.""" + try: + _path, _document, _payload, fraction, seed = _request(source_dir) + return fraction, seed + except SurveyPopulationPreparationError: + raise + except Exception: + raise SurveyPopulationPreparationError("REQUEST_REFUSED") from None + + +def _file_limits(): + # Limits bound only transport. Actual closed owners independently verify + # their pinned source bytes and stricter member/row/receipt contracts. + return { + **{name: asec_native._MAX_CHECKPOINT_BYTES for name in _SOURCE_ROSTER}, + **{ + f"asec/pppub{year}.csv": asec_native.coverage_owner.literal.MAX_MEMBER_BYTES + for year in (23, 24, 25) + }, + "asec/hhpub25.csv": asec_native.anchor_owner._MAX_MEMBER_BYTES, + "selection-request.json": MAX_REQUEST_BYTES, + "acs/csv_hus.zip": acs_catalogue._ARCHIVE_BYTES, + "acs/csv_pus.zip": acs_catalogue._ARCHIVE_BYTES, + } + + +def _source_files(root): + _root(root) + _require(stat.S_ISDIR(root.lstat().st_mode), "SOURCE_ROOT") + found = [] + for directory, expected in ( + (root, {"acs", "asec", "selection-request.json"}), + (root / "acs", {"csv_hus.zip", "csv_pus.zip"}), + ( + root / "asec", + {Path(n).name for n in _SOURCE_ROSTER if n.startswith("asec/")}, + ), + ): + _require(stat.S_ISDIR(directory.lstat().st_mode), "SOURCE_DIRECTORY") + children = tuple(directory.iterdir()) + _require( + {p.name for p in children} == expected and len(children) == len(expected), + "SOURCE_ROSTER", + ) + for path in children: + mode = path.lstat().st_mode + _require(not stat.S_ISLNK(mode), "SOURCE_SYMLINK") + if path.name in {"acs", "asec"} and path.parent == root: + _require(stat.S_ISDIR(mode), "SOURCE_DIRECTORY") + else: + _require(stat.S_ISREG(mode), "SOURCE_REGULAR_FILE") + found.append(path.relative_to(root).as_posix()) + _require(set(found) == set(_SOURCE_ROSTER), "SOURCE_ROSTER") + limits = _file_limits() + _require( + sum( + (root / name).lstat().st_size + for name in ("acs/csv_hus.zip", "acs/csv_pus.zip") + ) + <= acs_catalogue._ARCHIVE_BYTES, + "ARCHIVE_LIMIT", + ) + result = [] + for name in _SOURCE_ROSTER: + path = root / name + digest, size = hashlib.sha256(), 0 + with _regular_reader(path, limits[name]) as (stream, opened): + while block := stream.read(1024**2): + size += len(block) + _require(size <= limits[name], "SOURCE_FILE_LIMIT") + digest.update(block) + _require(size == opened.st_size, "SOURCE_CHANGED") + result.append([name, size, digest.hexdigest()]) + return tuple(tuple(row) for row in result) + + +def _file_stats(root): + _root(root) + result = [] + # Directory ctime/mtime also bind the exact admitted roster across the last + # producer I/O, including a newly added unlisted entry in either source arm. + for directory in (root, root / "acs", root / "asec"): + info = directory.lstat() + _require(stat.S_ISDIR(info.st_mode), "SOURCE_DIRECTORY") + result.append(_stat_identity(info)) + for name in _SOURCE_ROSTER: + path = root / name + _root(path.parent) + info = path.lstat() + _require(stat.S_ISREG(info.st_mode), "SOURCE_REGULAR_FILE") + result.append(_stat_identity(info)) + return tuple(result) + + +def _modules(): + modules = ( + sys.modules[__name__], + domains, + selection, + survey_domain_sample, + spine_assembly, + graph_sources, + graph_context, + graph_population, + observed_age, + acs_catalogue, + asec_catalogue, + acs_native, + acs_native.housing, + acs_native.coverage, + acs_native.coverage.literal, + sys.modules[acs_native.housing.build_acs_pums_unit_frame.__module__], + sys.modules[acs_native.housing.map_acs_native_inputs.__module__], + *asec_native._modules(), + ) + return tuple({module.__name__: module for module in modules}.values()) + + +def _runtime_marker(value, depth=0): + """Detach simple callable configuration, retaining opaque dependency identity.""" + _require(depth <= 16, "PRODUCER_CONFIGURATION_DEPTH") + if type(value) in (type(None), bool, int, float, str, bytes): + return type(value), value + if type(value) in (tuple, list): + return type(value), tuple(_runtime_marker(v, depth + 1) for v in value) + if type(value) is dict: + return tuple( + (_runtime_marker(k, depth + 1), _runtime_marker(v, depth + 1)) + for k, v in value.items() + ) + if isinstance(value, FunctionType): + return value, value.__code__ + return type(value), id(value) + + +def _function_seal(function, depth=0): + # contextmanager keeps executable code in both __wrapped__ and a closure; + # the public wrapper's __code__ alone cannot bind that implementation. + _require(depth <= 16, "PRODUCER_WRAPPER_DEPTH") + wrapped = getattr(function, "__wrapped__", None) + return ( + function, + function.__code__, + _runtime_marker(function.__defaults__), + _runtime_marker(function.__kwdefaults__), + tuple(_runtime_marker(c.cell_contents) for c in function.__closure__ or ()), + _function_seal(wrapped, depth + 1) + if isinstance(wrapped, FunctionType) + else _runtime_marker(wrapped), + ) + + +def _live(): + result = {} + for module in _modules(): + for name, value in vars(module).items(): + if isinstance(value, FunctionType): + result[(module.__name__, name)] = _function_seal(value) + elif isinstance(value, type) and value.__module__ == module.__name__: + result[(module.__name__, name)] = value + for method, function in vars(value).items(): + if isinstance(function, (staticmethod, classmethod)): + function = function.__func__ + if isinstance(function, property): + function = function.fget + if isinstance(function, FunctionType): + result[(module.__name__, name, method)] = _function_seal( + function + ) + result["rng"] = (np.random.Generator, np.random.PCG64, np.random.SeedSequence) + result["csv"] = (csv.reader, _csv.reader) + # The bounded catalogue path uses JSONEncoder.encode and its C provider; + # frame cells also use the same encode_basestring provider as _chunks. + # Bind their actual live identities through this existing seal. + result["json_catalogue"] = ( + json.JSONEncoder, + json.encoder, + tuple( + _function_seal(function) + if isinstance(function, FunctionType) + else _runtime_marker(function) + for function in ( + getattr(json.JSONEncoder, name, None) + for name in ("__init__", "encode", "iterencode") + ) + ), + getattr(json.encoder, "c_make_encoder", None), + getattr(json.encoder, "encode_basestring", None), + ) + result["contract"] = ( + PROTOCOL, + REQUEST_PROTOCOL, + SOURCE_CODEC, + MAX_REQUEST_BYTES, + MAX_PAYLOAD_BYTES, + _MAX_SCALAR_BYTES, + _FRAME_FAST_STRING_CHARS, + _SOURCE_ROSTER, + domains.DECLARATION, + domains.MAX_MEMBERS, + domains.MAX_HOUSEHOLDS, + domains.MAX_TOTAL_MEMBERS, + selection._BATCH_HOUSEHOLDS, + selection._BATCH_PEOPLE, + observed_age.RULE, + observed_age.AGE_CONVENTION, + observed_age.MAX_ROWS, + observed_age.MAX_EXACT_FLOAT64_INTEGER, + ) + return result + + +def _code_bytes(): + return { + module.__name__: _sha(Path(module.__file__).read_bytes()) + for module in _modules() + } + + +def _producer(): + _require(_live() == _LIVE and _code_bytes() == _BYTES, "PRODUCER_CHANGED") + return { + "implementation": _BYTES, + "acs": acs_catalogue._producer(), + "asec": _sha(_encode(asec_catalogue._implementation())), + "numpy": np.__version__, + "rng": "numpy.random.PCG64/SeedSequence", + "declaration": domains.DECLARATION, + } + + +def _authority(): + return _encode( + { + "asec": asec_catalogue._source_authority().hex(), + "acs": acs_native.housing._ARCHIVE_PINS, + } + ) + + +def _cell(value): + if ( + value is None + or value is pd.NA + or (isinstance(value, (float, np.floating)) and math.isnan(value)) + ): + return None + if isinstance(value, np.generic): + value = value.item() + _require(type(value) in (str, int, bool, float), "FRAME_CELL_TYPE") + if type(value) is float: + _require(math.isfinite(value), "FRAME_NONFINITE") + return ["float", value.hex()] + return value + + +def _frame_cell_encode(value, maximum=MAX_PAYLOAD_BYTES): + """Encode one normalized cell with the exact _encode preimage and limits. + + _cell admits only None, exact bool/int/str, or its own freshly constructed + ["float", finite_float.hex()] list. Keep normalization and the scalar walk + before encoding; large strings/integers retain the original slow path. + No Frame, Series, index, or normalized value is retained between calls. + """ + value = _cell(value) + kind = type(value) + if not ( + value is None + or kind is bool + or (kind is int and -(2**63) <= value < 2**64) + or (kind is str and len(value) <= _FRAME_FAST_STRING_CHARS) + or kind is list + ): + return _encode(value, maximum) + + _check_scalars(value) + if value is None: + encoded = b"null" + elif kind is bool: + encoded = b"true" if value else b"false" + elif kind is int: + # Match JSON's exact-int formatter, including its decimal digit policy. + encoded = int.__repr__(value).encode("ascii") + elif kind is str: + # _chunks selects this very provider with ensure_ascii=False. UTF-8 + # conversion still precedes the payload check, including surrogates. + # At most 6 * 4096 + 2 encoded bytes, even for all control characters. + encoded = json.encoder.encode_basestring(value).encode("utf-8") + else: + # Only _cell can construct this list: both strings are ASCII without + # JSON escapes. Keep hex spelling (especially -0.0), never decimalize. + encoded = b'["float","' + value[1].encode("ascii") + b'"]' + _require(len(encoded) <= maximum, "PAYLOAD_LIMIT") + return encoded + + +def _frame_identity(frame): + _require( + isinstance(frame, Frame) and frame.schema == US_SCHEMA and not frame.links, + "FRAME_TYPE", + ) + digest = hashlib.sha256() + + def update(value): + digest.update(_encode(value)) + digest.update(b"\n") + + def update_cell(value): + digest.update(_frame_cell_encode(value)) + digest.update(b"\n") + + update([list(frame.entities), list(frame.weighted_entities)]) + for entity in frame.entities: + table = frame.table(entity) + _require( + type(table) is pd.DataFrame and not table.columns.has_duplicates, + "FRAME_TABLE", + ) + update( + [ + entity, + list(table.columns), + type(table.columns).__name__, + str(table.columns.dtype), + list(table.columns.names), + type(table.index).__name__, + str(table.index.dtype), + list(table.index.names), + ] + ) + for value in table.index: + update_cell(value) + for column in table: + series = table[column] + dtype = series.dtype + update( + [ + column, + str(dtype), + getattr(dtype, "storage", None), + str(getattr(dtype, "na_value", "")), + ] + ) + for value in series: + update_cell(value) + update( + [ + str(frame.strata.dtype), + type(frame.strata.index).__name__, + str(frame.strata.index.dtype), + list(frame.strata.index.names), + getattr(frame.strata.dtype, "storage", None), + frame.strata.name, + ] + ) + for value in frame.strata.index: + update_cell(value) + for value in frame.strata: + update_cell(value) + for entity in frame.weighted_entities: + weights = frame.weights_for(entity) + update( + [ + entity, + weights.kind.value, + str(weights.values.dtype), + list(weights.values.shape), + ] + ) + digest.update(weights.values.tobytes()) + digest.update(graph_context.encode_us_frame_context(frame)) + return digest.hexdigest() + + +def _copy_source(frame): + result = Frame( + {e: frame.table(e).copy(deep=True) for e in frame.entities}, + frame.schema, + { + e: Weights(frame.weights_for(e).values.copy(), frame.weights_for(e).kind) + for e in frame.weighted_entities + }, + frame.strata.copy(deep=True), + mass_log=frame.mass_log, + metadata=frame.metadata, + ) + # Only physical string storage changes on owned copies, before the stack's + # exact shared-dtype check. Actual values and missing masks stay identical. + for entity in result.entities: + for column in result.table(entity): + original = result.table(entity)[column] + if isinstance(original.dtype, pd.StringDtype): + changed = original.astype(dtype_for_token("string")) + _same_values(original, changed) + result.table(entity)[column] = changed + return result + + +def _same_values(left, right): + _require(len(left) == len(right), "ROW_COUNT") + _require(list(map(_cell, left)) == list(map(_cell, right)), "SOURCE_VALUE_CHANGED") + + +def _normalized_source_copy(frame: Frame) -> Frame: + result = _copy_source(frame) + _require("A_AGE" in frame.person, "OBSERVED_AGE_REQUIRED") + result.person["age"] = observed_age.normalize_observed_age( + frame.person["A_AGE"], frame.person.get("age") + ) + _verify_normalized_copy(frame, result) + return result + + +def _verify_normalized_copy(original: Frame, normalized: Frame): + """Only the recorded age alias and existing string storage may differ.""" + _require( + original.schema == normalized.schema + and original.entities == normalized.entities + and original.links == normalized.links == () + and original.metadata == normalized.metadata + and original.mass_log == normalized.mass_log + and original.weighted_entities == normalized.weighted_entities, + "NORMALIZED_FRAME_CONTEXT", + ) + for entity in original.entities: + before, after = original.table(entity), normalized.table(entity) + expected_columns = list(before) + original_axis = after.columns + if entity == "person" and "age" not in before: + expected_columns.append("age") + original_axis = after.columns[:-1] + _require( + list(after) == expected_columns + and before.columns.identical(original_axis) + and before.index.identical(after.index), + "NORMALIZED_ORDERED_COLUMNS", + ) + for column in before: + if entity == "person" and column == "age": + continue # Independently reconstructed from raw observations below. + left, right = before[column], after[column] + if isinstance(left.dtype, pd.StringDtype): + _require( + right.dtype == dtype_for_token("string"), + "NORMALIZED_STRING_DTYPE", + ) + _same_values(left, right) + else: + _require(storage_equal(left, right), "NORMALIZED_SOURCE_STORAGE") + _require( + original.strata.index.identical(normalized.strata.index) + and original.strata.name == normalized.strata.name + and storage_equal(original.strata, normalized.strata), + "NORMALIZED_STRATA", + ) + for entity in original.weighted_entities: + left, right = original.weights_for(entity), normalized.weights_for(entity) + _require( + left.kind is right.kind + and left.values.dtype == right.values.dtype + and left.values.tobytes() == right.values.tobytes(), + "NORMALIZED_WEIGHTS", + ) + expected = observed_age.normalize_observed_age( + original.person["A_AGE"], original.person.get("age") + ) + _require( + storage_equal(expected, normalized.person["age"]), "NORMALIZED_AGE_IDENTITY" + ) + + +def _bounded_append(rows, row, budget): + size = len(_encode(row)) + 1 + _require(budget[0] + size <= MAX_PAYLOAD_BYTES, "ORIGIN_LIMIT") + budget[0] += size + rows.append(row) + + +def _origins(frame, sources, selected, native_receipts): + selected_by_key = {(r.key.source.value, r.key.native_id): r for r in selected} + native_hh, native_people = {}, {} + for row in native_receipts["asec"]["households"]: + native_hh[("asec", row["household_id"])] = row["source"]["H_SEQ"] + asec_rows = {row["person_id"]: row for row in native_receipts["asec"]["persons"]} + for channel, source in sources.items(): + for row in source.table("household").itertuples(index=False): + if channel == "acs": + native_hh[(channel, int(row.household_id))] = row.SERIALNO + for row in source.person.itertuples(index=False): + if channel == "acs": + person_key, line = str(row.SPORDER), str(row.SPORDER) + else: + original = asec_rows[int(row.person_id)] + person_key, line = original["PERIDNUM"], str(original["A_LINENO"]) + native_people[(channel, int(row.person_id))] = ( + person_key, + line, + int(row.person_household_id), + ) + maps, entities, households, people, budget = {}, {}, [], [], [0] + household_positions = {} + for channel, source in sources.items(): + ids = source.table("household").household_id + positions = {int(value): index for index, value in enumerate(ids)} + _require(len(positions) == len(ids), "SOURCE_ID_COLLISION") + household_positions[channel] = positions + for entity in frame.entities: + table = frame.table(entity) + ids = US_SCHEMA.entity_id_column(entity) + channels, previous = ( + support_channel_column(entity), + spine_source_id_column(entity), + ) + provenance = set(spine_assembly._support_metadata_columns(entity)) + expected_columns = provenance | { + column for source in sources.values() for column in source.table(entity) + } + _require(set(table) == expected_columns, "UNDECLARED_STACK_COLUMN") + records, mapping = [], {} + for new_id, channel, source_id in table[[ids, channels, previous]].itertuples( + index=False, name=None + ): + key = (channel, int(source_id)) + _require(key not in mapping, "SOURCE_ID_COLLISION") + mapping[key] = int(new_id) + _bounded_append(records, [int(new_id), channel, int(source_id)], budget) + expected = { + (channel, int(i)) + for channel, source in sources.items() + for i in source.table(entity)[ids] + } + _require(set(mapping) == expected, "COMPLETE_ENTITY_ORIGIN") + maps[entity], entities[entity] = mapping, records + for entity in frame.entities: + table = frame.table(entity) + id_column = US_SCHEMA.entity_id_column(entity) + for channel, source in sources.items(): + arm = table.loc[table[support_channel_column(entity)].eq(channel)] + old_ids = arm[spine_source_id_column(entity)].to_numpy() + original = ( + source.table(entity).set_index(id_column, drop=False).loc[old_ids] + ) + for column in original: + if column == id_column: + expected = [ + maps[entity][(channel, int(v))] for v in original[column] + ] + elif entity == "person" and column in { + US_SCHEMA.membership_column(e) for e in US_SCHEMA.group_entities + }: + group = next( + e + for e in US_SCHEMA.group_entities + if US_SCHEMA.membership_column(e) == column + ) + expected = [ + maps[group][(channel, int(v))] for v in original[column] + ] + else: + expected = original[column] + _same_values(expected, arm[column]) + for column in ( + set(arm) + - set(original) + - set(spine_assembly._support_metadata_columns(entity)) + ): + _require(arm[column].isna().all(), "ABSENT_SOURCE_VALUE_INVENTED") + if entity == "person": + source_positions = { + int(value): index + for index, value in enumerate(source.person.person_id) + } + expected_strata = source.strata.iloc[ + [source_positions[int(value)] for value in old_ids] + ] + receiving_positions = np.flatnonzero( + table[support_channel_column(entity)].eq(channel).to_numpy() + ) + _same_values(expected_strata, frame.strata.iloc[receiving_positions]) + household_table = frame.table("household") + for position, (new_id, channel, source_id) in enumerate(entities["household"]): + raw = native_hh[(channel, source_id)] + chosen = selected_by_key.pop((channel, raw), None) + _require(chosen is not None, "SELECTED_SOURCE_IDENTITY") + original = chosen.original_design_weight + source = sources[channel] + source_position = household_positions[channel][source_id] + _require( + source.weights_for("household").kind is WeightKind.DESIGN + and frame.weights_for("household").kind is WeightKind.DESIGN, + "ORIGINAL_WEIGHT_KIND", + ) + expected = np.array([float(original)], dtype=np.float64).tobytes() + _require( + source.weights_for("household") + .values[source_position : source_position + 1] + .tobytes() + == expected + == frame.weights_for("household").values[position : position + 1].tobytes(), + "ORIGINAL_ANCHOR_CHANGED", + ) + _bounded_append( + households, + { + "household_id": new_id, + "source": channel, + "source_year": 2024, + "survey_year": 2024 if channel == "acs" else 2025, + "raw_native_id": raw, + "selected_receiving_household_id": source_id, + "original_anchor": _value(original), + }, + budget, + ) + _require( + not selected_by_key and len(households) == len(household_table), + "SELECTED_HOUSEHOLD_ROSTER", + ) + for new_id, channel, source_id in entities["person"]: + person_key, line, household_id = native_people[(channel, source_id)] + _bounded_append( + people, + [ + new_id, + channel, + 2024, + 2024 if channel == "acs" else 2025, + native_hh[(channel, household_id)], + person_key, + line, + source_id, + household_id, + maps["household"][(channel, household_id)], + ], + budget, + ) + return { + "households": households, + "entities": entities, + "persons": { + "columns": [ + "person_id", + "source", + "source_year", + "survey_year", + "raw_native_household_id", + "raw_native_person_id", + "native_line_numeric_original", + "selected_receiving_person_id", + "selected_receiving_household_id", + "household_id", + ], + "rows": people, + }, + } + + +@dataclass(frozen=True) +class _State: + root: Path + files: tuple + file_stats: tuple + producer: bytes + authority: bytes + catalogues: tuple + native: tuple + attached: tuple + source_frames: tuple + source_frame_seals: tuple + frame: Frame + identity: str + context: bytes + plan: selection.CatalogueSelectionPlan + plan_sha256: str + nested: tuple + acs_catalogue_memo: tuple | None = None + + +def _attached(state): + return tuple(value.payload for value in (*state.catalogues, *state.native)) + + +def _nested_seals( + catalogues, native, *, catalogue_memo=None, expected_catalogue_digest=None +): + """Pure final borrowed-object seals after all source/producer file checks. + + These inspect the actual retained issued owners, never construct issuer + authority from decoded records. The ancestor Frames still impose their + existing full-parent verification cost. + """ + acs = acs_native._owned(native[0]) + acs_cat = acs_catalogue._lookup(catalogues[0]) + values = [ + id(acs), + id(acs.prepared), + id(acs.prepared.source), + id(acs.literal), + acs.prepared.receipt_json, + acs.prepared.source.receipt_json, + acs.prepared.source.projection_json, + acs.literal.payload, + _encode( + [ + [str(path) for _, path in sorted(paths.items())] + for paths in acs.snapshots + ] + ), + _memoized_catalogue_digest( + (acs_cat.records, acs_cat.vacancies), + catalogue_memo, + expected_digest=expected_catalogue_digest, + ), + ] + for module, value in ((asec_catalogue, catalogues[1]), (asec_native, native[1])): + entry = module._ISSUED.get(id(value)) + _require( + entry is not None and entry[0]() is value and entry[1] == value.payload, + "ANCESTOR_ISSUANCE_CHANGED", + ) + state = entry[2] + attached = asec_native._attached_evidence( + state.parent, state.coverage, state.anchors, state.fields + ) + values.extend( + ( + id(state), + id(state.parent), + id(attached[0]), + *attached[1:], + asec_native._frame_identity(state.parent.frame), + ) + ) + if module is asec_catalogue: + values.append(asec_catalogue._current_records_identity(state)) + return tuple(values) + + +def _pure_final(state): + _require( + _live() == _LIVE and _authority() == state.authority, "FINAL_AUTHORITY_CHANGED" + ) + _require(_attached(state) == state.attached, "ATTACHED_EVIDENCE_CHANGED") + _require( + _nested_seals( + state.catalogues, + state.native, + catalogue_memo=state.acs_catalogue_memo, + expected_catalogue_digest=state.nested[9], + ) + == state.nested, + "NESTED_EVIDENCE_CHANGED", + ) + _require(_digest(_plan_document(state.plan)) == state.plan_sha256, "PLAN_CHANGED") + _require( + _frame_identity(state.frame) == state.identity + and graph_context.encode_us_frame_context(state.frame) == state.context, + "PREPARED_FRAME_CHANGED", + ) + _require( + tuple( + ( + acs_native._frame_sha256(frame) + if i == 0 + else asec_native._frame_identity(frame) + ) + for i, frame in enumerate(state.source_frames) + ) + == state.source_frame_seals, + "SOURCE_FRAME_CHANGED", + ) + + +def _validate(state): + _pure_final(state) + acs_catalogue.verify_acs_source_catalogue(state.catalogues[0]) + asec_catalogue.verify_asec_source_catalogue(state.catalogues[1]) + acs_native.verify_acs_native_coverage(state.native[0], state.source_frames[0]) + state.native[1].validate() + _require(_source_files(state.root) == state.files, "SOURCE_CHANGED") + _require(_encode(_producer()) == state.producer, "PRODUCER_CHANGED") + _require(_file_stats(state.root) == state.file_stats, "SOURCE_STAT_CHANGED") + _pure_final(state) + + +@dataclass(frozen=True, slots=True, weakref_slot=True, eq=False) +class AuthenticatedSurveyPopulationPreparation: + payload: bytes + _token: InitVar[object] = None + + def __post_init__(self, _token): + _require(_token is _TOKEN, "ISSUANCE_CONSTRUCTOR") + + def _checked(self): + try: + entry = _ISSUED.get(id(self)) + _require( + type(self) is AuthenticatedSurveyPopulationPreparation + and entry is not None + and entry[0]() is self + and type(self.payload) is bytes + and self.payload == entry[1], + "UNISSUED_OR_CHANGED", + ) + _validate(entry[2]) + _require( + _ISSUED.get(id(self)) is entry + and type(self.payload) is bytes + and self.payload == entry[1], + "UNISSUED_OR_CHANGED", + ) + return entry + except SurveyPopulationPreparationError: + raise + except Exception: + raise SurveyPopulationPreparationError( + "PREPARATION_VERIFICATION_REFUSED" + ) from None + + def validate(self): + self._checked() + + def checked_view(self): + """Borrow one checked bundle; the view itself grants no authority.""" + _reference, payload, state = self._checked() + return CheckedSurveyPopulationView( + payload, state.context, state.frame, state.plan, json.loads(payload) + ) + + @property + def frame(self): + return self._checked()[2].frame + + @property + def context(self): + return self._checked()[2].context + + @property + def selection_plan(self): + return self._checked()[2].plan + + @property + def receipt(self): + return json.loads(self._checked()[1]) + + def to_bytes(self): + return self._checked()[1] + + +@dataclass(frozen=True, slots=True) +class CheckedSurveyPopulationView: + """A checked borrow's values, not an issued source or reusable certificate.""" + + payload: bytes + context: bytes + frame: Frame + selection_plan: selection.CatalogueSelectionPlan + receipt: dict + + +def verify_survey_population_preparation(value): + _require( + type(value) is AuthenticatedSurveyPopulationPreparation, "PREPARATION_TYPE" + ) + value._checked() + return value + + +def verify_materialized_survey_population(preparation, frame): + _require( + type(preparation) is AuthenticatedSurveyPopulationPreparation, + "PREPARATION_TYPE", + ) + entry = preparation._checked() + _require(_frame_identity(frame) == entry[2].identity, "MATERIALIZED_FRAME_CHANGED") + _validate(entry[2]) + _require(_frame_identity(frame) == entry[2].identity, "MATERIALIZED_FRAME_CHANGED") + _require(preparation.payload == entry[1], "UNISSUED_OR_CHANGED") + + +def prepare_authenticated_survey_population( + source_dir, *, snapshot_root, fraction, seed, candidate=None +): + """Reconstruct actual source authority before accepting candidate bytes.""" + try: + _require(type(candidate) is bytes or candidate is None, "CANDIDATE_TYPE") + _require( + candidate is None or len(candidate) <= MAX_PAYLOAD_BYTES, "CANDIDATE_LIMIT" + ) + root, request, request_bytes, requested_fraction, requested_seed = _request( + source_dir + ) + _require( + type(fraction) is Fraction + and fraction == requested_fraction + and type(seed) is int + and seed == requested_seed, + "REQUEST_ARGUMENTS", + ) + snapshots = _root(snapshot_root, allow_missing=True) + _require( + not snapshots.is_relative_to(root) and not root.is_relative_to(snapshots), + "SNAPSHOT_LOCATION", + ) + snapshots.mkdir(parents=True, exist_ok=True) + authority, files_before, producer = ( + _authority(), + _source_files(root), + _encode(_producer()), + ) + _require( + files_before[0] + == ("selection-request.json", len(request_bytes), _sha(request_bytes)), + "REQUEST_CHANGED", + ) + file_stats = _file_stats(root) + kwargs = dict( + parent_path=root / "asec/parent.h5", + household_attachment_path=root / "asec/household-attachment.h5", + person_income_attachment_path=root / "asec/person-income-attachment.h5", + person_member_paths={ + 2022: root / "asec/pppub23.csv", + 2023: root / "asec/pppub24.csv", + 2024: root / "asec/pppub25.csv", + }, + household_member_path=root / "asec/hhpub25.csv", + ) + acs = acs_catalogue.issue_acs_source_catalogue( + root / "acs", snapshot_root=snapshots + ) + asec = asec_catalogue.issue_asec_source_catalogue(**kwargs) + acs_rows, asec_rows = acs.households, asec.households + plan = selection.plan_catalogue_selection( + acs_households=acs_rows, + asec_households=asec_rows, + fraction=fraction, + seed=seed, + ) + acs_keys = tuple( + r.key.native_id for r in plan.selected if r.key.source is domains.Source.ACS + ) + asec_keys = tuple( + (2024, int(r.key.native_id)) + for r in plan.selected + if r.key.source is domains.Source.ASEC + ) + _require( + acs_keys and asec_keys and len(set(asec_keys)) == len(asec_keys), + "SELECTED_SOURCE_SUPPORT", + ) + actual_acs = acs_native.issue_acs_native_coverage( + root / "acs", snapshot_root=snapshots, serialnos=acs_keys + ) + actual_asec = asec_native.load_authenticated_asec_2024_native_population( + **kwargs, selected_households=asec_keys + ) + native_frames = actual_acs.frame, actual_asec.frame + source_seals = ( + acs_native._frame_sha256(native_frames[0]), + asec_native._frame_identity(native_frames[1]), + ) + source_copies = { + channel: _normalized_source_copy(frame) + for channel, frame in zip(("acs", "asec"), native_frames, strict=True) + } + stacked = spine_assembly.stack_survey_spines(source_copies) + frame = stacked.frame + transitions = graph_sources._canonical_assembly( + frame, tuple(source_copies.values()) + ) + context = graph_context.encode_us_frame_context(frame) + native_receipts = {"acs": actual_acs.receipt, "asec": actual_asec.receipt} + for channel, original in zip(("acs", "asec"), native_frames, strict=True): + _verify_normalized_copy(original, source_copies[channel]) + origins = _origins(frame, source_copies, plan.selected, native_receipts) + origins["storage_transitions"] = list(transitions) + origins["observed_age_normalization"] = { + "rule": observed_age.rule_document(), + "sources": { + channel: { + "native_frame_sha256": source_seals[i], + "normalized_frame_sha256": _frame_identity(source_copies[channel]), + "common_age_preexisting": "age" in native_frames[i].person, + } + for i, channel in enumerate(("acs", "asec")) + }, + } + documents = {"acs": acs.receipt, "asec": asec.receipt} + catalogues = { + "acs": { + "receipt_sha256": _sha(acs.to_bytes()), + "records_sha256": documents["acs"]["counts"]["canonical_record_sha256"], + "counts": documents["acs"]["counts"], + }, + "asec": { + "receipt_sha256": _sha(asec.to_bytes()), + "records_sha256": documents["asec"]["records"]["sha256"], + "counts": documents["asec"]["counts"], + }, + } + identity = _frame_identity(frame) + payload = _encode( + { + "protocol": PROTOCOL, + "request": request, + "request_sha256": _sha(request_bytes), + "source_files": files_before, + "producer": json.loads(producer), + "catalogues": catalogues, + "native": { + channel: { + "receipt_sha256": _sha(value.payload), + "frame_sha256": source_seals[i], + "households": native_frames[i].n("household"), + "persons": native_frames[i].n("person"), + } + for i, (channel, value) in enumerate( + (("acs", actual_acs), ("asec", actual_asec)) + ) + }, + "selection": _plan_document(plan), + "origins": origins, + "frame_sha256": identity, + "context_sha256": _sha(context), + "release_eligible": False, + } + ) + nested = _nested_seals((acs, asec), (actual_acs, actual_asec)) + acs_owned = acs_catalogue._lookup(acs) + catalogue_memo = _catalogue_memo( + (acs_owned.records, acs_owned.vacancies), nested[9] + ) + state = _State( + root, + files_before, + file_stats, + producer, + authority, + (acs, asec), + (actual_acs, actual_asec), + tuple(value.payload for value in (acs, asec, actual_acs, actual_asec)), + native_frames, + source_seals, + frame, + identity, + context, + plan, + _digest(_plan_document(plan)), + nested, + catalogue_memo, + ) + _validate(state) + _require(candidate is None or candidate == payload, "CANDIDATE_MISMATCH") + result = AuthenticatedSurveyPopulationPreparation(payload, _token=_TOKEN) + key = id(result) + + def cleanup(reference): + entry = _ISSUED.get(key) + if entry is not None and entry[0] is reference: + del _ISSUED[key] + + _ISSUED[key] = (weakref.ref(result, cleanup), payload, state) + _pure_final(state) + _require( + type(result.payload) is bytes + and result.payload == payload + and _ISSUED[key][0]() is result, + "UNISSUED_OR_CHANGED", + ) + return result + except SurveyPopulationPreparationError: + raise + except Exception: + raise SurveyPopulationPreparationError("PREPARATION_ISSUANCE_REFUSED") from None + + +_BYTES = _code_bytes() + + +def _current_survey_wage_projection(preparation, entry): + """Project one just-checked retained owner; bytes grant no source authority. + + The caller must have obtained ``entry`` from ``preparation._checked()`` in + the current operation, and must seal its receiving Populations afterwards. + The one ready() call retains the existing all-field/full-parent admission. + No old reported-income contract or prior-wage column is manufactured. + """ + _require( + type(preparation) is AuthenticatedSurveyPopulationPreparation + and _ISSUED.get(id(preparation)) is entry + and entry[0]() is preparation + and type(preparation.payload) is bytes + and preparation.payload == entry[1], + "WAGE_PREPARATION_ISSUANCE", + ) + state = entry[2] + native_entry = asec_native._ISSUED.get(id(state.native[1])) + _require( + native_entry is not None + and native_entry[0]() is state.native[1] + and state.native[1].payload == native_entry[1], + "WAGE_NATIVE_ISSUANCE", + ) + parent = native_entry[2].parent + ready = parent.ready() # Exactly once, outside both row and feature loops. + field = ready.field("WSAL_VAL") + domain = next(d for d in parent.spec.fields if d.name == "WSAL_VAL") + scope = parent.scope + positions = {pid: i for i, pid in enumerate(scope.person_ids)} + _require(len(positions) == len(scope.person_ids), "WAGE_PARENT_ROSTER") + native_ids = set(state.source_frames[1].person.person_id) + rows, budget = [], [0] + people = state.frame.person + selected = people.loc[people[support_channel_column("person")].eq("asec")] + _require( + set(selected[spine_source_id_column("person")]) == native_ids, + "WAGE_SELECTED_ROSTER", + ) + for stacked, native in selected[ + ["person_id", spine_source_id_column("person")] + ].itertuples(index=False, name=None): + _require(int(native) in positions, "WAGE_PARENT_MEMBER") + i = positions[int(native)] + _require(scope.person_years[i] == 2024, "WAGE_CURRENT_COHORT") + amount = field.amount_bytes[8 * i : 8 * (i + 1)] + number = np.frombuffer(amount, dtype="= 0, + "WAGE_CURRENT_FEATURE", + ) + _bounded_append( + rows, + [ + int(stacked), + int(native), + 2024, + amount.hex(), + field.status_bytes[i], + field.validity_bytes[i], + field.zero_origin_bytes[i], + ], + budget, + ) + document = { + "preparation_sha256": _sha(entry[1]), + "asec_native_sha256": _sha(native_entry[1]), + "money_header": json.loads(ready.header), + "money_header_sha256": _sha(ready.header), + "domain": {f.name: getattr(domain, f.name) for f in fields(domain)}, + "zero_origin_code": int(current_money._origin_code(parent.spec, "WSAL_VAL")), + "columns": [ + "stacked_person_id", + "native_person_id", + "income_year", + "amount_f64le_hex", + "status", + "validity", + "zero_origin", + ], + "rows": rows, + } + payload = _encode(document) + # ready() may perform I/O. Recheck the actual retained entries and all pure + # owner seals after it; this does not reread the national catalogues. + _pure_final(state) + _require( + _ISSUED.get(id(preparation)) is entry + and preparation.payload == entry[1] + and asec_native._ISSUED.get(id(state.native[1])) is native_entry + and state.native[1].payload == native_entry[1] + and native_entry[2].parent is parent, + "WAGE_FINAL_ISSUANCE", + ) + return payload + + +_LIVE = _live() diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/survey_population_replay.py b/packages/microcosm-build/src/microcosm/build/us_runtime/survey_population_replay.py new file mode 100644 index 000000000..dd55a5e65 --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/survey_population_replay.py @@ -0,0 +1,208 @@ +"""Directional ContentStore replay checks, never source or population authority. + +The caller must separately retain the original source/population seals. These +comparisons permit the store's missing-buffer normalization and scalar object +encoding only at a named materialization boundary. They are not mutation seals. +""" + +from __future__ import annotations + +from dataclasses import asdict + +import numpy as np +import pandas as pd + +from microcosm.frame import Frame, MassChangeRecord, WeightKind +from microcosm.graph.canonical import canonical_json +from microcosm.graph.population import MassRecord, Population +from microcosm.graph.store import ( + _axis_name_payload, + _encode_frame_metadata, + _encode_object_scalar, +) + + +def _require(condition, reason): + if not condition: + raise ValueError("SURVEY_POPULATION_REPLAY_" + reason) + + +def _array_bytes_equal(left, right): + return ( + type(left) is np.ndarray + and type(right) is np.ndarray + and left.dtype == right.dtype + and not left.dtype.hasobject + and left.shape == right.shape + and left.tobytes() == right.tobytes() + ) + + +def _object_bytes(value): + try: + return _encode_object_scalar(value) + except (TypeError, ValueError, OverflowError, UnicodeError): + raise ValueError("SURVEY_POPULATION_REPLAY_UNSUPPORTED_OBJECT") from None + + +def _series(expected, actual): + _require( + type(expected) is pd.Series + and type(actual) is pd.Series + and type(expected.dtype) is type(actual.dtype) + and expected.dtype == actual.dtype + and len(expected) == len(actual), + "SERIES_DTYPE_OR_LENGTH", + ) + dtype = expected.dtype + left, right = expected.array, actual.array + if isinstance(dtype, pd.BooleanDtype) or ( + isinstance(dtype, pd.api.extensions.ExtensionDtype) + and pd.api.types.is_integer_dtype(dtype) + ): + # The v1 country profile supports pandas' actual masked integer/bool + # storage. Other extension-array implementations must be reviewed first. + ld, rd = getattr(left, "_data", None), getattr(right, "_data", None) + lm, rm = getattr(left, "_mask", None), getattr(right, "_mask", None) + _require( + _array_bytes_equal(lm, rm) + and lm.dtype == np.dtype(np.bool_) + and lm.shape == (len(expected),) + and type(ld) is np.ndarray + and type(rd) is np.ndarray + and ld.dtype == rd.dtype + and not ld.dtype.hasobject + and ld.shape == rd.shape == lm.shape, + "MASKED_STORAGE", + ) + _require(_array_bytes_equal(ld[~lm], rd[~rm]), "PRESENT_BITS") + _require( + _array_bytes_equal(ld, rd) or bool(np.all(rd[rm] == 0)), + "NONCANONICAL_NULL_BACKING", + ) + return + if isinstance(dtype, pd.StringDtype): + _require( + dtype.storage == actual.dtype.storage + and (dtype.na_value is pd.NA) == (actual.dtype.na_value is pd.NA), + "STRING_POLICY", + ) + lm = expected.isna().to_numpy(dtype=np.bool_, copy=False) + rm = actual.isna().to_numpy(dtype=np.bool_, copy=False) + _require(_array_bytes_equal(lm, rm), "STRING_MASK") + for missing, a, b in zip(lm, expected, actual, strict=True): + if not missing: + _require(type(a) is str and type(b) is str and a == b, "STRING_VALUE") + return + if pd.api.types.is_object_dtype(dtype): + # Reuse the store's typed scalar bytes. None/pd.NA/NaT, bool/int, + # float signed zero and NaN payloads remain distinguished by that codec. + # Object pointers and np scalar wrappers are not portable identities. + for a, b in zip(expected, actual, strict=True): + _require(_object_bytes(a) == _object_bytes(b), "OBJECT_VALUE") + return + _require( + not isinstance(dtype, pd.api.extensions.ExtensionDtype), + "UNSUPPORTED_EXTENSION_DTYPE", + ) + _require( + _array_bytes_equal(expected.to_numpy(copy=False), actual.to_numpy(copy=False)), + "NATIVE_BITS", + ) + + +def _name_bytes(value): + try: + return canonical_json(_axis_name_payload(value)) + except (TypeError, ValueError, OverflowError, UnicodeError): + raise ValueError("SURVEY_POPULATION_REPLAY_UNSUPPORTED_AXIS_NAME") from None + + +def _axis(expected, actual): + _require( + type(expected) is type(actual) + and expected.identical(actual) + and not isinstance(expected, pd.MultiIndex), + "AXIS", + ) + _require(_name_bytes(expected.name) == _name_bytes(actual.name), "AXIS_NAME") + _series(pd.Series(expected.array, copy=False), pd.Series(actual.array, copy=False)) + + +def same_replayed_frame(expected: Frame, actual: Frame) -> None: + """Compare actual Frames, allowing only the named store representation rule. + + No I/O or allocation of another Frame. Temporary buffers are proportional + to individual columns and context metadata, without a total input RAM cap. + """ + _require(isinstance(expected, Frame) and isinstance(actual, Frame), "FRAME_TYPE") + _require( + expected.schema == actual.schema + and expected.entities == actual.entities + and expected.links == actual.links == () + and canonical_json(_encode_frame_metadata(expected.metadata)) + == canonical_json(_encode_frame_metadata(actual.metadata)) + and all( + type(r) is MassChangeRecord for r in (*expected.mass_log, *actual.mass_log) + ) + and canonical_json([asdict(r) for r in expected.mass_log]) + == canonical_json([asdict(r) for r in actual.mass_log]) + and expected.weighted_entities == actual.weighted_entities, + "FRAME_CONTEXT", + ) + for entity in expected.entities: + left, right = expected.table(entity), actual.table(entity) + _require( + type(left) is pd.DataFrame + and type(right) is pd.DataFrame + and left.flags == right.flags, + "TABLE_TYPE_OR_FLAGS", + ) + _axis(left.index, right.index) + _axis(left.columns, right.columns) + for column in left: + _series(left[column], right[column]) + _axis(expected.strata.index, actual.strata.index) + _require( + _name_bytes(expected.strata.name) == _name_bytes(actual.strata.name), + "STRATA_NAME", + ) + _series(expected.strata, actual.strata) + for entity in expected.weighted_entities: + left, right = expected.weights_for(entity), actual.weights_for(entity) + _require( + left.kind is right.kind and _array_bytes_equal(left.values, right.values), + "WEIGHT_BYTES", + ) + + +def same_replayed_population(expected: Population, actual: Population) -> None: + """Require complete receiving state as well as the directional Frame check.""" + _require( + type(expected) is Population and type(actual) is Population, "POPULATION_TYPE" + ) + same_replayed_frame(expected.frame, actual.frame) + _require( + type(expected.version) is type(actual.version) is str + and expected.version == actual.version + and tuple(sorted(expected.owners.items())) + == tuple(sorted(actual.owners.items())) + and all(type(v) is str for v in actual.owners.values()) + and tuple(expected.weight_kind.items()) == tuple(actual.weight_kind.items()) + and all(type(v) is WeightKind for v in actual.weight_kind.values()) + and type(expected.mass_ledger) is type(actual.mass_ledger) is tuple + and all( + type(r) is MassRecord for r in (*expected.mass_ledger, *actual.mass_ledger) + ) + and canonical_json([asdict(r) for r in expected.mass_ledger]) + == canonical_json([asdict(r) for r in actual.mass_ledger]) + and tuple(expected.design_weights) == tuple(actual.design_weights), + "POPULATION_CONTEXT", + ) + for entity in expected.design_weights: + _require( + _array_bytes_equal( + expected.design_weights[entity], actual.design_weights[entity] + ), + "DESIGN_BYTES", + ) diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/survey_social_security.py b/packages/microcosm-build/src/microcosm/build/us_runtime/survey_social_security.py new file mode 100644 index 000000000..adf73f3ab --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/survey_social_security.py @@ -0,0 +1,176 @@ +"""Explicit source-report Social Security component basis. + +These are numerical owners, not source admission APIs. A graph host must bind +the live source projection and any actual fitted completion artifact before +using the returned values. Missing components never become zero. Report-record totals are not individual +beneficiary incidence. This module neither replaces totals with PUF draws nor +allocates combined family payments to individual beneficiaries. +""" + +from __future__ import annotations + +import numpy as np + +COMPONENTS = ( + "social_security_retirement", + "social_security_disability", + "social_security_dependents", + "social_security_survivors", +) +PROTOCOL = "microcosm.us.survey-social-security-basis.v1" +REASON_COMPONENTS = { + 1: (0,), + 2: (1,), + 3: (3,), + 4: (2,), + 5: (3,), + 6: (2,), + # The published code mixes surviving, dependent and disabled children. + 7: (1, 2, 3), + 8: (0, 1, 2, 3), +} + + +def require(condition, reason): + if not condition: + raise ValueError("SURVEY_SOCIAL_SECURITY_" + reason) + + +def _float64(values, *, shape, nullable=False): + require(type(values) is np.ndarray, "ARRAY_TYPE") + require(values.dtype == np.dtype("float64") and values.shape == shape, "ARRAY_AXIS") + require(not np.isinf(values).any(), "NONFINITE") + require(nullable or not np.isnan(values).any(), "UNKNOWN") + require((values[~np.isnan(values)] >= 0).all(), "NEGATIVE") + return values.copy() + + +def asec_reason_basis(total, reason_1, reason_2): + """Map a source total and reasons without inventing component amounts. + + Callers qualify the actual money and literal reason observations. Codes + 3/5 share survivors and 4/6 share dependents. Multiple distinct components, + code 7, code 8, or missing reasons leave the positive amount unallocated. + A single component receives the total as an explicit mapping judgment. + No age fallback or priority between reported reasons is applied. + """ + require(type(total) is np.ndarray and total.ndim == 1, "TOTAL_AXIS") + amount = _float64(total, shape=total.shape) + reasons = [] + for values in (reason_1, reason_2): + arr = _float64(values, shape=total.shape, nullable=True) + require( + ( + (arr[np.isfinite(arr)] == np.floor(arr[np.isfinite(arr)])) + & (arr[np.isfinite(arr)] <= 8) + ).all(), + "REASON_DOMAIN", + ) + reasons.append(arr) + out = np.full((len(amount), 4), np.nan, dtype=np.float64) + allowed = np.ones((len(amount), 4), dtype=np.bool_) + labels = [] + for i, value in enumerate(amount): + if value == 0: + out[i] = 0 + allowed[i] = False + labels.append("known_total_zero") + continue + codes = [arr[i] for arr in reasons] + if any(np.isnan(code) for code in codes): + labels.append("unresolved_missing_reason") + continue + nonzero = [int(code) for code in codes if code != 0] + if not nonzero: + labels.append("unresolved_niu_reason_with_positive_total") + continue + possible = set().union(*(REASON_COMPONENTS[code] for code in nonzero)) + allowed[i] = False + allowed[i, list(possible)] = True + if len(possible) == 1: + out[i] = 0 + out[i, next(iter(possible))] = value + labels.append("source_reason_total_allocation") + else: + labels.append("unresolved_component_split") + return out, allowed, tuple(labels) + + +def asec_reporting_basis(total, age, recipiency, reason_1, reason_2): + """Respect the age-15 reporting universe without inferring beneficiaries. + + SS_VAL=0 outside the question universe is a source sentinel, not observed + absence of benefits. Retain that literal upstream and return unknown total + and shares here. A reporting adult may receive combined family payments; + the returned component basis remains at that source report's grain. + """ + require(type(total) is np.ndarray and total.ndim == 1, "TOTAL_AXIS") + amount = _float64(total, shape=total.shape) + ages = _float64(age, shape=total.shape) + receipt = _float64(recipiency, shape=total.shape, nullable=True) + require(((ages == np.floor(ages)) & (ages <= 99)).all(), "AGE_DOMAIN") + require( + np.isin(receipt[np.isfinite(receipt)], [0, 1, 2]).all(), "RECIPIENCY_DOMAIN" + ) + eligible = ages >= 15 + require( + ((amount[~eligible] == 0) & (receipt[~eligible] == 0)).all(), + "OUTSIDE_REPORTING_UNIVERSE_OBSERVATION", + ) + require(np.isin(receipt[eligible], [1, 2]).all(), "IN_UNIVERSE_RECIPIENCY_UNKNOWN") + require(not ((amount > 0) & (receipt != 1)).any(), "RECIPIENCY_CONTRADICTION") + basis, allowed, labels = asec_reason_basis(amount, reason_1, reason_2) + amount[~eligible] = np.nan + basis[~eligible] = np.nan + allowed[~eligible] = True + labels = tuple( + label if inside else "asec_below15_outside_reporting_universe" + for label, inside in zip(labels, eligible, strict=True) + ) + return amount, basis, allowed, labels + + +def complete_positive_basis(total, source_basis, allowed, modeled_scores): + """Convert actual model scores to shares only for unresolved positive rows. + + The graph owner must authenticate the fitted artifact. This numerical API + cannot do so. Existing complete rows must have no modeled scores; eligible + rows require a complete nonnegative four-vector with positive allowed mass. + Source totals and excluded component possibilities are retained exactly. + A missing/zero model vector refuses instead of creating equal shares. + """ + require(type(total) is np.ndarray and total.ndim == 1, "TOTAL_AXIS") + amount = _float64(total, shape=total.shape) + shape = (len(amount), 4) + basis = _float64(source_basis, shape=shape, nullable=True) + scores = _float64(modeled_scores, shape=shape, nullable=True) + require( + type(allowed) is np.ndarray + and allowed.dtype == np.dtype("bool") + and allowed.shape == shape, + "ALLOWED_AXIS", + ) + complete = np.isfinite(basis).all(axis=1) + require((complete | np.isnan(basis).all(axis=1)).all(), "PARTIAL_COMPONENT_BASIS") + require(np.isnan(scores[complete]).all(), "MODELED_SOURCE_OVERWRITE") + require( + np.array_equal(basis[complete].sum(axis=1), amount[complete]), + "SOURCE_TOTAL_IDENTITY", + ) + require((basis[complete][~allowed[complete]] == 0).all(), "SOURCE_SUPPORT") + missing = ~complete + require((amount[missing] > 0).all(), "UNRESOLVED_ZERO_TOTAL") + require(np.isfinite(scores[missing]).all(), "COMPLETION_UNKNOWN") + selected = scores[missing].copy() + selected[~allowed[missing]] = 0 + mass = selected.sum(axis=1) + require(np.isfinite(mass).all() and (mass > 0).all(), "COMPLETION_NO_ALLOWED_MASS") + filled = selected / mass[:, None] * amount[missing, None] + # Put the floating residual on the largest supported component. This keeps + # excluded components exactly zero and avoids systematic lost pennies. + if len(filled): + j = filled.argmax(axis=1) + filled[np.arange(len(filled)), j] += amount[missing] - filled.sum(axis=1) + require(np.isfinite(filled).all() and (filled >= 0).all(), "COMPLETION_ARITHMETIC") + basis[missing] = filled + return basis diff --git a/packages/microcosm-build/tests/test_atomic_geography.py b/packages/microcosm-build/tests/test_atomic_geography.py new file mode 100644 index 000000000..ccc4b172c --- /dev/null +++ b/packages/microcosm-build/tests/test_atomic_geography.py @@ -0,0 +1,511 @@ +"""Invented support controls; no survey, Census or administrative files are read.""" + +from copy import deepcopy + +import numpy as np +import pandas as pd +import pytest + +from microcosm.build import atomic_geography as geo +from microcosm.build.graph_atomic_geography import ( + ATOMIC_GEOGRAPHY_VALIDATION_TYPE, + atomic_geography_nodes, + register_atomic_geography_kernels, +) +from microcosm.frame import EntitySchema, Frame, WeightKind, Weights +from microcosm.graph import ( + ArtifactInput, + ArtifactType, + Capabilities, + ContentStore, + Determinism, + Graph, + KernelBase, + KernelRegistry, + KernelResult, + Node, + Owned, + SourceRef, + StructuralDelta, + compile_graph, + run_graph, +) +from microcosm.graph.canonical import canonical_json +from microcosm.graph.keys import opaque_artifact_key + + +def support_parts(system="invented_block", *, district=True): + columns = { + name: { + "kind": "code", + "source": "invented-lookup-v1", + "vintage": "invented-2020", + "relation": "official_tabulation" if name == "district" else "exact", + } + for name in ("area", "region", "puma", *(("district",) if district else ())) + } + columns.update( + { + name: {"kind": "weight", "source": "invented-counts-v1", "basis": basis} + for name, basis in (("population", "persons"), ("households", "households")) + } + ) + arrays = { + "area": np.asarray(["0001", "0002", "0003", "0004"]), + "region": np.asarray(["R", "R", "R", "S"]), + "puma": np.asarray(["A", "A", "B", "C"]), + "population": np.asarray([0, 2, 6, 1]), + "households": np.asarray([1, 3, 4, 1]), + } + if district: + arrays["district"] = np.asarray(["01", "02", "02", "03"]) + return { + "version": 1, + "system": system, + "level": "block", + "code_system": "invented", + "vintage": "invented-2020", + "columns": columns, + }, arrays + + +def fixture(system="invented_block", *, district=True): + metadata, arrays = support_parts(system, district=district) + payload = geo.encode_atomic_support(metadata, arrays) + support = geo.decode_atomic_support(payload) + spec = { + "version": 1, + "identity": ["source_record"], + "stream": ["sha256-u53-v1", "invented-geography", 0, 21], + "outputs": {"area": "block", "system": "area_system", "basis": "area_basis"}, + "systems": [ + { + "id": system, + "level": "block", + "code_system": "invented", + "vintage": "invented-2020", + "source": system + "_source", + "selector": {}, + "constraints": [ + {"input": "observed_region", "support": "region", "required": True}, + {"input": "observed_puma", "support": "puma", "required": False}, + ], + "observed_area": None, + "stages": [{"level": "area", "weight": "population"}], + "layers": ( + [ + { + "input": "district", + "output": "district_code", + "vintage": "invented-2020", + "relation": "official_tabulation", + "source": "invented-lookup-v1", + } + ] + if district + else [] + ), + } + ], + } + households = pd.DataFrame( + { + "household_id": np.arange(1, 9, dtype=np.int64), + "source_record": pd.array( + ["source-" + str(i) for i in range(8)], dtype="string" + ), + "observed_region": pd.array(["R"] * 7 + ["S"], dtype="string"), + "observed_puma": pd.array( + [None, "A", "B", None, "A", None, "B", "C"], dtype="string" + ), + } + ) + return households, spec, {system: support}, payload + + +def complete(households, spec, supports): + assigned = pd.concat( + [households, geo.assign_atomic(households, spec, supports)], axis=1 + ) + return pd.concat([assigned, geo.derive_geography(assigned, spec, supports)], axis=1) + + +def test_codec_is_deterministic_and_decoded_values_are_immutable(): + metadata, arrays = support_parts() + first = geo.encode_atomic_support(metadata, arrays) + assert first == geo.encode_atomic_support( + metadata, dict(reversed(list(arrays.items()))) + ) + support = geo.decode_atomic_support(first) + assert support.arrays["area"].tolist() == ["0001", "0002", "0003", "0004"] + with pytest.raises(ValueError): + support.arrays["area"].setflags(write=True) + with pytest.raises(TypeError): + support.metadata["columns"]["area"]["source"] = "changed" + + +def test_integer_inverse_cdf_respects_zero_mass_and_boundaries(): + _, _, supports, _ = fixture() + index = geo._SupportIndex(supports["invented_block"]) + stage = {"level": "area", "weight": "population"} + cell = (("region", "R"),) + assert index.pick(cell, stage, 0.0) == "0002" + assert index.pick(cell, stage, 0.25 - 1 / 2**53) == "0002" + assert index.pick(cell, stage, 0.25) == "0003" + assert index.pick(cell, stage, 1 - 1 / 2**53) == "0003" + + +def test_draws_are_stable_under_order_subset_extra_columns_and_unrelated_rows(): + households, spec, supports, _ = fixture() + before = households.copy(deep=True) + expected = geo.assign_atomic(households, spec, supports) + reordered = households.iloc[[7, 2, 0, 5]] + actual = geo.assign_atomic(reordered.assign(unrelated=42), spec, supports) + pd.testing.assert_frame_equal(actual, expected.loc[reordered.index]) + extra = households.iloc[[0]].copy() + extra.index = [100] + extra["source_record"] = "unrelated-record" + grown = geo.assign_atomic(pd.concat([households, extra]), spec, supports) + pd.testing.assert_frame_equal(grown.loc[households.index], expected) + pd.testing.assert_frame_equal(households, before) + + +def test_stage_law_uses_declared_weights_at_each_level(): + households, spec, supports, _ = fixture() + spec["systems"][0]["stages"] = [ + {"level": "puma", "weight": "households"}, + {"level": "area", "weight": "population"}, + ] + result = complete(households, spec, supports) + assert result.loc[1, "block"] == "0002" + assert result.loc[2, "block"] == "0003" + assert geo.validate_geography(result, spec, supports)["outcome"] == "pass" + # A/B household mass is 4/4, but population mass is 2/6. + index = geo._SupportIndex(supports["invented_block"]) + assert index.pick((("region", "R"),), spec["systems"][0]["stages"][0], 0.49) == "A" + assert index.pick((("region", "R"),), spec["systems"][0]["stages"][0], 0.5) == "B" + + +def test_qualified_observed_area_is_retained_even_with_zero_sampling_mass(): + households, spec, supports, _ = fixture() + households["observed_block"] = pd.array(["0001"] + [None] * 7, dtype="string") + spec["systems"][0]["observed_area"] = "observed_block" + result = complete(households, spec, supports) + assert result.loc[0, "block"] == "0001" + assert result.loc[0, "area_basis"] == "observed" + assert result.loc[0, "district_code"] == "01" + assert geo.validate_geography(result, spec, supports)["outcome"] == "pass" + + +@pytest.mark.parametrize( + "defect", + [ + "missing_region", + "wrong_region", + "inconsistent_puma", + "duplicate_identity", + "null_identity", + "existing_output", + "unknown_observed_area", + "conflicting_observed_area", + ], +) +def test_assignment_refuses_invalid_or_conflicting_households(defect): + households, spec, supports, _ = fixture() + if defect == "missing_region": + households.loc[0, "observed_region"] = pd.NA + elif defect == "wrong_region": + households.loc[0, "observed_region"] = "missing" + elif defect == "inconsistent_puma": + households.loc[0, "observed_puma"] = "C" + elif defect == "duplicate_identity": + households.loc[0, "source_record"] = households.loc[1, "source_record"] + elif defect == "null_identity": + households.loc[0, "source_record"] = pd.NA + elif defect == "existing_output": + households["block"] = "untouched" + else: + households["observed_block"] = pd.array( + ["unknown" if defect == "unknown_observed_area" else "0004"] + [None] * 7, + dtype="string", + ) + spec["systems"][0]["observed_area"] = "observed_block" + with pytest.raises(ValueError): + geo.assign_atomic(households, spec, supports) + + +@pytest.mark.parametrize( + "defect", + [ + "duplicate_area", + "negative_weight", + "float_weight", + "bool_weight", + "overflow_weight", + "all_zero_weight", + "unknown_relation", + "wrong_area_vintage", + "empty_code", + "wrong_length", + ], +) +def test_support_refuses_ambiguous_or_invalid_artifacts(defect): + metadata, arrays = support_parts() + if defect == "duplicate_area": + arrays["area"][1] = arrays["area"][0] + elif defect == "negative_weight": + arrays["population"][0] = -1 + elif defect == "float_weight": + arrays["population"] = arrays["population"].astype(float) + elif defect == "bool_weight": + arrays["population"] = arrays["population"].astype(bool) + elif defect == "overflow_weight": + arrays["population"] = np.asarray([2**63] * 4, dtype=np.uint64) + elif defect == "all_zero_weight": + arrays["population"] *= 0 + elif defect == "unknown_relation": + metadata["columns"]["district"]["relation"] = "exact_enough" + elif defect == "wrong_area_vintage": + metadata["columns"]["area"]["vintage"] = "different" + elif defect == "empty_code": + arrays["area"][0] = "" + elif defect == "wrong_length": + arrays["region"] = arrays["region"][:-1] + with pytest.raises(ValueError): + geo.encode_atomic_support(metadata, arrays) + + +@pytest.mark.parametrize("field", ["source", "relation", "vintage"]) +def test_layer_metadata_must_match_the_pinned_support(field): + households, spec, supports, _ = fixture() + spec["systems"][0]["layers"][0][field] = ( + "exact" if field == "relation" else "different" + ) + with pytest.raises(ValueError): + geo.assign_atomic(households, spec, supports) + + +def test_three_nation_systems_use_the_same_operations_and_mark_absent_layers(): + households, spec, supports, _ = fixture() + households["nation"] = pd.array( + ["one"] * 3 + ["two"] * 3 + ["three"] * 2, dtype="string" + ) + spec["systems"][0]["selector"] = {"nation": ["one"]} + for name, selector, district in ( + ("invented_oa", "two", True), + ("invented_dz", "three", False), + ): + _, other, added, _ = fixture(name, district=district) + system = other["systems"][0] + system["selector"] = {"nation": [selector]} + spec["systems"].append(system) + supports.update(added) + result = complete(households, spec, supports) + assert ( + result["area_system"].tolist() + == ["invented_block"] * 3 + ["invented_oa"] * 3 + ["invented_dz"] * 2 + ) + assert result.loc[[6, 7], "district_code"].isna().all() + assert geo.validate_geography(result, spec, supports)["outcome"] == "pass" + spec["systems"][1]["selector"] = {"nation": ["one", "two"]} + with pytest.raises(ValueError, match="exactly one"): + geo.assign_atomic(households, spec, supports) + + +def test_clone_inheritance_and_pruned_mapping_checks_do_not_redraw(): + households, spec, supports, _ = fixture() + original = complete(households, spec, supports) + clone = original.iloc[[2, 0, 5]].copy() + clone["household_id"] += 100 + clone["source_record"] += "-clone" + assert geo.validate_geography(clone, spec, supports)["households"] == 3 + clone.iloc[0, clone.columns.get_loc("district_code")] = "wrong" + with pytest.raises(ValueError, match="derived geography differs"): + geo.validate_geography(clone, spec, supports) + + +@pytest.mark.parametrize("defect", ["atomic", "basis", "observed", "system"]) +def test_final_integrity_gate_refuses_changed_geography(defect): + households, spec, supports, _ = fixture() + result = complete(households, spec, supports) + if defect == "atomic": + result.loc[0, "block"] = "missing" + elif defect == "basis": + result.loc[0, "area_basis"] = "observed" + elif defect == "observed": + result.loc[0, "observed_region"] = "wrong" + elif defect == "system": + result.loc[0, "area_system"] = pd.NA + with pytest.raises(ValueError): + geo.validate_geography(result, spec, supports) + + +class _InventedSpine(KernelBase): + ref = "invented.atomic_spine@1" + capabilities = Capabilities( + determinism=Determinism.DETERMINISTIC, structural=StructuralDelta.CREATE + ) + + def run(self, context): + households, _, _, _ = fixture() + person = pd.DataFrame( + { + "person_id": np.arange(1, 9, dtype=np.int64), + "person_household_id": households["household_id"], + "age": np.arange(21, 29, dtype=np.int64), + } + ) + return KernelResult( + frame=Frame( + {"person": person, "household": households}, + EntitySchema(group_entities=("household",)), + {"household": Weights(np.ones(8), WeightKind.DESIGN)}, + pd.Series(["invented"] * 8, name="stratum"), + ) + ) + + +@pytest.mark.parametrize("emit_validation_artifact", (False, True)) +def test_real_executor_cold_warm_and_changed_source_identity( + tmp_path, emit_validation_artifact +): + households, spec, supports, payload = fixture() + columns = ( + Owned("person", "age", "int64"), + *( + Owned("household", name, "string") + for name in ("source_record", "observed_region", "observed_puma") + ), + ) + base = Node( + "spine", + _InventedSpine.ref, + sources=("invented_spine",), + structural=StructuralDelta.CREATE, + outputs=columns, + ) + nodes = atomic_geography_nodes( + spec, columns, base=base.id, emit_validation_artifact=emit_validation_artifact + ) + graph = compile_graph( + Graph( + "invented", + ( + SourceRef("invented_spine", "raw-bytes-v1"), + SourceRef("invented_block_source", "raw-bytes-v1"), + ), + (base, *nodes), + ) + ) + registry = KernelRegistry() + registry.register(_InventedSpine()) + register_atomic_geography_kernels(registry) + raw_spine = tmp_path / "invented-spine.bin" + raw_spine.write_bytes(b"invented test source declaration") + raw_support = tmp_path / "invented-support.npz" + raw_support.write_bytes(payload) + sources = {"invented_spine": raw_spine, "invented_block_source": raw_support} + store = ContentStore(tmp_path / "store") + cold = run_graph(graph, sources=sources, store=store, kernels=registry) + warm = run_graph( + graph, sources=sources, store=store, kernels=registry, resume="require" + ) + actual = cold.population(base.id).table("household") + expected = complete(households, spec, supports) + for column in expected: + left, right = actual[column], expected[column] + if isinstance(expected[column].dtype, pd.StringDtype): + # Source checkpoints and emitted columns may use different string + # backends. Check the nullable-string contract, then compare every + # value and missing cell using one explicit physical representation. + assert isinstance(left.dtype, pd.StringDtype) + assert left.dtype.na_value is pd.NA + left = left.astype(pd.StringDtype(storage="python")) + right = right.astype(pd.StringDtype(storage="python")) + pd.testing.assert_series_equal(left, right, check_index_type=False) + assert cold.nodes["geography.assign"].key == warm.nodes["geography.assign"].key + assert cold.nodes["geography.gate"].receipt["outcome"] == "pass" + gate = cold.nodes["geography.gate"] + if emit_validation_artifact: + output = graph.graph.node("geography.gate").artifact_outputs + assert len(output) == 1 and output[0].type == ATOMIC_GEOGRAPHY_VALIDATION_TYPE + key = gate.opaque_artifacts["validation"] + assert key == opaque_artifact_key(gate.key, "validation") + assert store.load_bytes(key) == canonical_json( + geo.validate_geography(expected, spec, supports) + ) + assert warm.nodes["geography.gate"].opaque_artifacts["validation"] == key + else: + assert not gate.opaque_artifacts + + np.testing.assert_array_equal( + cold.population(base.id).weights_for("household").values, np.ones(8) + ) + metadata, arrays = support_parts() + arrays["population"][1] += 1 + raw_support.write_bytes(geo.encode_atomic_support(metadata, arrays)) + changed = run_graph(graph, sources=sources, store=store, kernels=registry) + assert changed.nodes["geography.assign"].key != cold.nodes["geography.assign"].key + + +def test_graph_builder_refuses_input_overwrites_and_duplicate_inventory(): + _, spec, _, _ = fixture() + columns = tuple( + Owned("household", c, "string") + for c in ("source_record", "observed_region", "observed_puma") + ) + with pytest.raises(ValueError, match="overwrite"): + atomic_geography_nodes( + spec, (*columns, Owned("household", "block", "string")), base="spine" + ) + with pytest.raises(ValueError, match="repeats"): + atomic_geography_nodes(spec, (*columns, columns[0]), base="spine") + with pytest.raises(ValueError, match="missing"): + atomic_geography_nodes(spec, columns[1:], base="spine") + altered = deepcopy(spec) + altered["outputs"]["area"] = "observed_region" + with pytest.raises(ValueError, match="overwritten"): + atomic_geography_nodes(altered, columns, base="spine") + + +@pytest.mark.parametrize("defect", ("missing", "type")) +def test_typed_geography_gate_edge_refuses_invalid_producer_contract(defect): + _, spec, _, _ = fixture() + columns = tuple( + Owned("household", name, "string") + for name in ("source_record", "observed_region", "observed_puma") + ) + base = Node( + "spine", + _InventedSpine.ref, + sources=("invented_spine",), + structural=StructuralDelta.CREATE, + outputs=columns, + ) + nodes = atomic_geography_nodes( + spec, columns, base=base.id, emit_validation_artifact=(defect != "missing") + ) + edge_type = ( + ArtifactType("invented.wrong_gate", 1) + if defect == "type" + else ATOMIC_GEOGRAPHY_VALIDATION_TYPE + ) + consumer = Node( + "consumer", + "invented.consumer@1", + population=base.id, + artifact_inputs=( + ArtifactInput("validation", "geography.gate", "validation", edge_type), + ), + ) + with pytest.raises(ValueError, match="no declared artifact|type does not match"): + compile_graph( + Graph( + "invented", + ( + SourceRef("invented_spine", "raw-bytes-v1"), + SourceRef("invented_block_source", "raw-bytes-v1"), + ), + (base, *nodes, consumer), + ) + ) diff --git a/packages/microcosm-build/tests/test_frame_checkpoint.py b/packages/microcosm-build/tests/test_frame_checkpoint.py index 7c79ba3f8..d5e052a40 100644 --- a/packages/microcosm-build/tests/test_frame_checkpoint.py +++ b/packages/microcosm-build/tests/test_frame_checkpoint.py @@ -576,6 +576,205 @@ def test_schema_3_checkpoint_requires_nullable_boolean_encoding( load_frame_checkpoint(path) +@pytest.mark.parametrize( + "dtype", ["Int8", "Int16", "Int32", "Int64", "UInt8", "UInt16", "UInt32", "UInt64"] +) +@pytest.mark.parametrize("has_missing", [False, True]) +def test_nullable_integer_round_trip_preserves_width_values_and_null_mask( + tmp_path: Path, dtype: str, has_missing: bool +) -> None: + frame = _nullable_boolean_checkpoint_frame() + numpy_dtype = pd.api.types.pandas_dtype(dtype).numpy_dtype + bounds = np.iinfo(numpy_dtype) + frame.person["nullable_integer"] = pd.Series( + [int(bounds.min), pd.NA if has_missing else 0, int(bounds.max)], + index=frame.person.index, + dtype=dtype, + ) + first_path = tmp_path / "integer-first.h5" + second_path = tmp_path / "integer-second.h5" + + write_frame_checkpoint(first_path, frame) + loaded = load_frame_checkpoint(first_path) + write_frame_checkpoint(second_path, loaded.frame) + + assert first_path.read_bytes() == second_path.read_bytes() + pd.testing.assert_frame_equal(loaded.frame.person, frame.person, check_exact=True) + pd.testing.assert_frame_equal(loaded.frame.link("jobs"), frame.link("jobs")) + h5py = pytest.importorskip("h5py") + with h5py.File(first_path, mode="r") as h5: + metadata, spec, group = _checkpoint_series_group( + h5["_populace_frame_checkpoint"], table="person", column="nullable_integer" + ) + assert metadata["schema_version"] == 4 + assert spec == { + "name": "nullable_integer", + "dtype": dtype, + "encoding": "nullable_integer_v1", + "has_null_mask": has_missing, + } + values = np.asarray(group["values"]) + assert values.dtype == numpy_dtype + assert values.tolist() == [int(bounds.min), 0, int(bounds.max)] + if has_missing: + mask = np.asarray(group["null_mask"]) + assert mask.dtype == np.dtype(np.uint8) + assert mask.tolist() == [0, 1, 0] + else: + assert "null_mask" not in group + + +def _nullable_integer_checkpoint_frame() -> Frame: + frame = _checkpoint_frame() + frame.person["nullable_integer"] = pd.Series( + [1, pd.NA, 3], index=frame.person.index, dtype="Int64" + ) + return frame + + +def test_nullable_integer_masked_storage_is_canonical(tmp_path: Path) -> None: + paths = [tmp_path / "hidden-zero.h5", tmp_path / "hidden-nonzero.h5"] + for hidden, path in zip((0, 97), paths, strict=True): + frame = _nullable_integer_checkpoint_frame() + frame.person["nullable_integer"] = pd.Series( + pd.arrays.IntegerArray( + np.asarray([1, hidden, 3], dtype=np.int64), + np.asarray([False, True, False], dtype=np.bool_), + ), + index=frame.person.index, + ) + write_frame_checkpoint(path, frame) + assert paths[0].read_bytes() == paths[1].read_bytes() + + +@pytest.mark.parametrize( + "damage", + ["missing", "all_zero", "nonbinary", "wrong_length", "wrong_dtype", "wrong_rank"], +) +def test_nullable_integer_checkpoint_rejects_malformed_null_mask( + tmp_path: Path, damage: str +) -> None: + path = tmp_path / "malformed-integer-mask.h5" + write_frame_checkpoint(path, _nullable_integer_checkpoint_frame()) + h5py = pytest.importorskip("h5py") + with h5py.File(path, mode="r+") as h5: + _metadata, _spec, group = _checkpoint_series_group( + h5["_populace_frame_checkpoint"], table="person", column="nullable_integer" + ) + del group["null_mask"] + if damage != "missing": + masks = { + "all_zero": np.asarray([0, 0, 0], dtype=np.uint8), + "nonbinary": np.asarray([0, 2, 0], dtype=np.uint8), + "wrong_length": np.asarray([0, 1], dtype=np.uint8), + "wrong_dtype": np.asarray([0, 1, 0], dtype=np.int16), + "wrong_rank": np.asarray([[0, 1, 0]], dtype=np.uint8), + } + group.create_dataset("null_mask", data=masks[damage], track_times=False) + with pytest.raises(ValueError, match="null mask"): + load_frame_checkpoint(path) + + +@pytest.mark.parametrize("damage", ["wrong_dtype", "wrong_rank", "hidden_nonzero"]) +def test_nullable_integer_checkpoint_rejects_malformed_values( + tmp_path: Path, damage: str +) -> None: + path = tmp_path / "malformed-integer-values.h5" + write_frame_checkpoint(path, _nullable_integer_checkpoint_frame()) + h5py = pytest.importorskip("h5py") + with h5py.File(path, mode="r+") as h5: + _metadata, _spec, group = _checkpoint_series_group( + h5["_populace_frame_checkpoint"], table="person", column="nullable_integer" + ) + del group["values"] + values = { + "wrong_dtype": np.asarray([1, 0, 3], dtype=np.uint64), + "wrong_rank": np.asarray([[1, 0, 3]], dtype=np.int64), + "hidden_nonzero": np.asarray([1, 2, 3], dtype=np.int64), + } + group.create_dataset("values", data=values[damage], track_times=False) + message = ( + "noncanonical nonzero" + if damage == "hidden_nonzero" + else "nullable integer values" + ) + with pytest.raises(ValueError, match=message): + load_frame_checkpoint(path) + + +@pytest.mark.parametrize("version", [2, 3]) +def test_legacy_checkpoint_cannot_smuggle_nullable_integer_encoding( + tmp_path: Path, version: int +) -> None: + path = tmp_path / "forged-legacy-integer.h5" + write_frame_checkpoint(path, _nullable_integer_checkpoint_frame()) + h5py = pytest.importorskip("h5py") + with h5py.File(path, mode="r+") as h5: + root = h5["_populace_frame_checkpoint"] + metadata, _spec, _group = _checkpoint_series_group( + root, table="person", column="nullable_integer" + ) + metadata["schema_version"] = version + _replace_checkpoint_metadata(root, metadata) + with pytest.raises(ValueError, match=f"schema version {version}.*nullable integer"): + load_frame_checkpoint(path) + + +@pytest.mark.parametrize( + ("field", "value", "message"), + [ + ("has_null_mask", "yes", "has_null_mask"), + ("dtype", "int64", "must pair a supported pandas integer dtype"), + ("encoding", "numpy", "must pair a supported pandas integer dtype"), + ], +) +def test_nullable_integer_checkpoint_rejects_malformed_spec( + tmp_path: Path, field: str, value: object, message: str +) -> None: + path = tmp_path / "malformed-integer-spec.h5" + write_frame_checkpoint(path, _nullable_integer_checkpoint_frame()) + h5py = pytest.importorskip("h5py") + with h5py.File(path, mode="r+") as h5: + root = h5["_populace_frame_checkpoint"] + metadata, spec, _group = _checkpoint_series_group( + root, table="person", column="nullable_integer" + ) + spec[field] = value + _replace_checkpoint_metadata(root, metadata) + with pytest.raises(ValueError, match=message): + load_frame_checkpoint(path) + + +def test_maskless_nullable_integer_rejects_unexpected_null_mask(tmp_path: Path) -> None: + frame = _nullable_integer_checkpoint_frame() + frame.person["nullable_integer"] = frame.person["nullable_integer"].fillna(0) + path = tmp_path / "unexpected-integer-mask.h5" + write_frame_checkpoint(path, frame) + h5py = pytest.importorskip("h5py") + with h5py.File(path, mode="r+") as h5: + _metadata, _spec, group = _checkpoint_series_group( + h5["_populace_frame_checkpoint"], table="person", column="nullable_integer" + ) + group.create_dataset( + "null_mask", data=np.zeros(3, dtype=np.uint8), track_times=False + ) + with pytest.raises(ValueError, match="unexpected null mask"): + load_frame_checkpoint(path) + + +def test_schema_4_checkpoint_requires_nullable_integer_encoding(tmp_path: Path) -> None: + path = tmp_path / "forged-schema-4.h5" + write_frame_checkpoint(path, _nullable_boolean_checkpoint_frame()) + h5py = pytest.importorskip("h5py") + with h5py.File(path, mode="r+") as h5: + root = h5["_populace_frame_checkpoint"] + metadata = json.loads(np.asarray(root["metadata_json"]).tobytes()) + metadata["schema_version"] = 4 + _replace_checkpoint_metadata(root, metadata) + with pytest.raises(ValueError, match="schema version 4.*nullable integer"): + load_frame_checkpoint(path) + + def test_frame_checkpoint_fsyncs_parent_directory_after_rename( monkeypatch: pytest.MonkeyPatch, tmp_path: Path, @@ -656,10 +855,9 @@ def test_frame_checkpoint_preserves_range_indexes_and_column_axis_name( @pytest.mark.parametrize( "unsupported", [ - pd.Series([1, pd.NA, 3], dtype="Int64"), pd.Series(["a", "b", "a"], dtype="category"), ], - ids=["nullable_integer", "categorical"], + ids=["categorical"], ) def test_frame_checkpoint_rejects_unsupported_dtype_without_replacing_destination( tmp_path: Path, diff --git a/packages/microcosm-build/tests/test_spec_engine_country_bundles.py b/packages/microcosm-build/tests/test_spec_engine_country_bundles.py index fecadaba5..ed52cc979 100644 --- a/packages/microcosm-build/tests/test_spec_engine_country_bundles.py +++ b/packages/microcosm-build/tests/test_spec_engine_country_bundles.py @@ -25,7 +25,7 @@ "spine", "vintages", } -AM_SPEC_SHA256 = "c4163f9ab3577cd5b6d509c6fe091d0f95ae7892a4746562a57a62a3f8b3be8e" +AM_SPEC_SHA256 = "c15bd12508731eb11fe061df810d9047b4c62c2887645cb19d3c54c7a104aa1c" @pytest.mark.parametrize( @@ -45,7 +45,7 @@ ), ( "be", - "fe25bbc48c785801bc7f518380eff9f528a3ceab4fa8501e1b527362c9c579b6", + "5285b530974a9b8f7c21497fb140fdbd6060c3f1cc983987f35e9f244e71ee62", { "household.household_id", "person.person_id", @@ -55,7 +55,7 @@ ), ( "uk", - "3ea1b53abb0ceef57b6ddef6d49251579e633bfe7d5ae7ec5a2152c0858445af", + "1ac642230eb3166dd9ea2a113536985e35bf2898f914e289f5564ac8fde6e0b4", { "benunit.benunit_id", "household.household_id", diff --git a/packages/microcosm-build/tests/test_spec_engine_loader.py b/packages/microcosm-build/tests/test_spec_engine_loader.py index 240228a0b..90b0c458e 100644 --- a/packages/microcosm-build/tests/test_spec_engine_loader.py +++ b/packages/microcosm-build/tests/test_spec_engine_loader.py @@ -236,7 +236,7 @@ def test_semantic_hash_has_golden_vector_and_surface_separation(tmp_path) -> Non # Pin the domain separator, normalization rules, schema-set receipt, and # exact normative projection as one reviewable golden vector. assert first.spec_sha256 == ( - "b465989064184f03ecc296c13bec8ffc5a70d5dd4cf30d42487b399633d32a62" + "77f043a13853bbc109b6aa8cb42635ab1b2ebea69a1997b5bdb4c78a3726aeb3" ) second_root = _rich_minimal(tmp_path / "xy", note="second", store="local:b") diff --git a/packages/microcosm-build/tests/test_spec_seed_diagnostic_refusal_context.py b/packages/microcosm-build/tests/test_spec_seed_diagnostic_refusal_context.py new file mode 100644 index 000000000..16e62dd45 --- /dev/null +++ b/packages/microcosm-build/tests/test_spec_seed_diagnostic_refusal_context.py @@ -0,0 +1,130 @@ +"""Source-only context formatting: no audit install or denied file operation.""" + +import importlib.util +import json +import sys +from pathlib import Path + +import pytest + + +@pytest.fixture(scope="module") +def diagnostic(): + root = Path(__file__).resolve().parents[3] + spec = importlib.util.spec_from_file_location( + "spec_seed_diagnostic_context_test", + root / "tools/spec_seed_identity_diagnostics.py", + ) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +@pytest.mark.parametrize( + ("value", "scope", "shown"), + ( + ("/repo/tools/example.py", "repository", "tools/example.py"), + ( + "/repo/packages/example/schema.json", + "repository", + "packages/example/schema.json", + ), + ("/repo/data/invented.csv", "repository", None), + ("/repo/.env", "repository", None), + ("/repo/credentials.json", "repository", None), + ("/repo/example.err.log", "repository", None), + ("/unlisted/identity/example.py", "unlisted", "example.py"), + ("/opt/python/lib/example.py", "system_opt", "python/lib/example.py"), + ), +) +def test_path_classification_is_bounded_and_redacts_data_and_credentials( + diagnostic, value, scope, shown +): + result = diagnostic.refusal_path_context( + value, (("repository", Path("/repo")), ("system_opt", Path("/opt"))) + ) + assert result["scope"] == scope + assert result["path"] == shown + assert result["name_redacted"] is (shown is None) + + +def test_requested_and_resolved_code_scopes_remain_distinct_without_io(diagnostic): + # Literal examples only: no symlink or path is created, resolved or opened. + payload = diagnostic.read_refusal_context( + "READ_SCOPE", + "open", + Path("/opt/python/lib/loader.py"), + "/environment/lib/loader.py", + (("environment", Path("/environment")), ("system_opt", Path("/opt"))), + ) + value = json.loads(payload) + assert value["requested"]["scope"] == "environment" + assert value["resolved"]["scope"] == "system_opt" + assert value["paths_are_metadata_only"] is True + assert len(payload) <= 4096 and len(value["frames"]) <= 8 + + +def test_context_never_serializes_locals_or_exception_text(diagnostic): + private_local = "invented-secret-value-that-must-never-be-exported" + error = RuntimeError(private_local) + payload = diagnostic.read_refusal_context( + "READ_SCOPE", + "open", + Path("/repo/credentials.json"), + "/repo/credentials.json", + (("repository", Path("/repo")),), + ) + assert str(error).encode() not in payload + value = json.loads(payload) + assert value["requested"]["path"] is None + assert value["resolved"]["path"] is None + assert value["code"] == "READ_SCOPE" and value["event"] == "open" + assert all(set(frame) == {"file", "function", "line"} for frame in value["frames"]) + assert len(payload) <= 4096 + + +@pytest.mark.parametrize( + ("path", "label"), + ( + ("/etc/os-release", "etc_os_release"), + ("/usr/lib/os-release", "usr_lib_os_release"), + ("/lib/os-release", "lib_os_release"), + ("/etc/localtime", "etc_localtime"), + ( + f"/runtime/lib/python{sys.version_info.major}{sys.version_info.minor}.zip", + "python_stdlib_zip_candidate", + ), + ("/runtime/lib/invented-data.zip", None), + ), +) +def test_fixed_os_metadata_labels_do_not_expose_data_path_text(diagnostic, path, label): + value = diagnostic.refusal_path_context(path, (("runtime", Path("/runtime")),)) + assert value["known_path_label"] == label + assert value["path"] is None and value["name_redacted"] is True + + +def test_terminal_refusal_retains_own_context_after_an_earlier_probe(diagnostic): + first = diagnostic.encoded({"code": "DATA_FILE", "event": "open"}) + last = diagnostic.read_refusal_context( + "READ_SCOPE", + "open", + Path("/usr/lib/os-release"), + "/etc/os-release", + (("system_usr", Path("/usr")), ("system_etc", Path("/etc"))), + ) + swallowed = diagnostic.RefusalError("DATA_FILE", boundary_context=first) + terminal = diagnostic.RefusalError("READ_SCOPE", boundary_context=last) + assert str(swallowed) == "DATA_FILE" and str(terminal) == "READ_SCOPE" + assert type(terminal.boundary_context) is bytes + assert terminal.boundary_context == last and swallowed.boundary_context == first + context = json.loads(terminal.boundary_context) + assert context["requested"]["known_path_label"] == "etc_os_release" + assert context["resolved"]["known_path_label"] == "usr_lib_os_release" + assert context["code"] != json.loads(swallowed.boundary_context)["code"] + assert diagnostic.RefusalError("WALL").boundary_context is None + + +@pytest.mark.parametrize("value", (bytearray(b"{}"), b"x" * 4097)) +def test_exception_context_requires_immutable_bounded_bytes(diagnostic, value): + with pytest.raises(TypeError, match="REFUSAL_CONTEXT"): + diagnostic.RefusalError("READ_SCOPE", boundary_context=value) diff --git a/packages/microcosm-build/tests/test_spec_seed_identity_cpu_bootstrap.py b/packages/microcosm-build/tests/test_spec_seed_identity_cpu_bootstrap.py new file mode 100644 index 000000000..9c31d579c --- /dev/null +++ b/packages/microcosm-build/tests/test_spec_seed_identity_cpu_bootstrap.py @@ -0,0 +1,226 @@ +"""Invented bootstrap controls; no Torch, Microcosm, engines or data imported.""" + +from __future__ import annotations + +import importlib +import importlib.machinery +import importlib.util +import sys +from pathlib import Path +from types import SimpleNamespace + +import pytest + + +@pytest.fixture(scope="module") +def diagnostic(): + # Normal stdlib-only module loading; main/audit/derive are never called. + source = ( + Path(__file__).resolve().parents[3] / "tools/spec_seed_identity_diagnostics.py" + ) + spec = importlib.util.spec_from_file_location("cpu_bootstrap_controls", source) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +@pytest.fixture +def fresh_import_state(monkeypatch): + # Keep the process import roster isolated without importing any dependency. + for name in tuple(sys.modules): + if ( + name == "torch" + or name == "cuda.bindings" + or name.startswith("cuda.bindings.") + ): + monkeypatch.delitem(sys.modules, name) + monkeypatch.setattr(sys, "meta_path", list(sys.meta_path)) + + +@pytest.mark.parametrize( + "name", ["json", "torch", "cuda", "cuda.pathfinder", "cuda.bindings_other"] +) +def test_non_cuda_bindings_imports_pass_through(diagnostic, name): + finder = diagnostic._CpuOnlyCudaBindingsFinder() + assert finder.find_spec(name, ["invented"], object()) is None + assert finder.attempts == 0 + + +def test_only_exact_optional_bindings_are_excluded(diagnostic): + finder = diagnostic._CpuOnlyCudaBindingsFinder() + with pytest.raises(ModuleNotFoundError) as error: + finder.find_spec("cuda.bindings") + assert error.value.name == "cuda.bindings" + assert finder.attempts == 1 + + +@pytest.mark.parametrize("fails", [False, True]) +def test_finder_removed_on_success_and_failure(diagnostic, fresh_import_state, fails): + before = tuple(sys.meta_path) + other = object() + evidence = {} + + class InventedFailureError(Exception): + pass + + try: + with diagnostic.cpu_only_torch_import(evidence): + finder = sys.meta_path[0] + assert type(finder) is diagnostic._CpuOnlyCudaBindingsFinder + assert tuple(sys.meta_path[1:]) == before + sys.meta_path.append(other) + with pytest.raises(ModuleNotFoundError): + finder.find_spec("cuda.bindings") + if fails: + raise InventedFailureError + except InventedFailureError: + assert fails + assert tuple(sys.meta_path) == (*before, other) + assert evidence == { + "cuda_bindings_import_attempts": 1, + "temporary_finder_removed": True, + } + + +@pytest.mark.parametrize("name", ["torch", "cuda.bindings", "cuda.bindings.runtime"]) +def test_preloaded_optional_path_refuses_before_finder( + diagnostic, fresh_import_state, monkeypatch, name +): + monkeypatch.setitem(sys.modules, name, SimpleNamespace()) + before = tuple(sys.meta_path) + with pytest.raises(diagnostic.RefusalError, match="^TORCH_BOOTSTRAP_PRELOADED$"): + with diagnostic.cpu_only_torch_import({}): + pytest.fail("preloaded dependency was accepted") + assert tuple(sys.meta_path) == before + + +def test_actual_fallback_contract_is_required( + diagnostic, fresh_import_state, monkeypatch +): + evidence = { + "temporary_finder_removed": True, + "cuda_bindings_import_attempts": 1, + } + monkeypatch.setitem( + sys.modules, + "torch.cuda._utils", + SimpleNamespace(_HAS_CUDA_BINDINGS=False, _cuda_bindings_runtime=None), + ) + diagnostic.verify_cpu_only_torch_import(evidence) + assert evidence["torch_optional_bindings_fallback_verified"] is True + + +@pytest.mark.parametrize("bindings_present", [False, True]) +def test_unconfirmed_or_loaded_bindings_refuse( + diagnostic, fresh_import_state, monkeypatch, bindings_present +): + evidence = { + "temporary_finder_removed": True, + "cuda_bindings_import_attempts": 1 if bindings_present else 0, + } + monkeypatch.setitem( + sys.modules, + "torch.cuda._utils", + SimpleNamespace(_HAS_CUDA_BINDINGS=False, _cuda_bindings_runtime=None), + ) + if bindings_present: + monkeypatch.setitem(sys.modules, "cuda.bindings", SimpleNamespace()) + with pytest.raises(diagnostic.RefusalError, match="^TORCH_CPU_FALLBACK$"): + diagnostic.verify_cpu_only_torch_import(evidence) + + +def _candidate(monkeypatch): + monkeypatch.setattr(sys, "base_prefix", "/invented-bootstrap-prefix") + return str( + Path(sys.base_prefix) + / "lib" + / f"python{sys.version_info.major}{sys.version_info.minor}.zip" + ) + + +def test_only_exact_confirmed_absent_zip_entry_is_removed(diagnostic, monkeypatch): + candidate = _candidate(monkeypatch) + others = ["", "/invented/code", "/invented/unrelated.zip", candidate + "-other"] + monkeypatch.setattr(sys, "path", [candidate, *others, candidate]) + checked = [] + + def absent(path): + checked.append(path) + raise FileNotFoundError(path) + + monkeypatch.setattr(diagnostic.os, "lstat", absent) + assert diagnostic.omit_absent_stdlib_zip() is True + assert checked == [candidate] + assert sys.path == others + + +@pytest.mark.parametrize("kind", ["regular", "broken_symlink"]) +def test_existing_zip_or_symlink_remains_subject_to_original_guard( + diagnostic, monkeypatch, kind +): + candidate = _candidate(monkeypatch) + monkeypatch.setattr(sys, "path", [candidate, "other"]) + # Any successful lstat, including a broken symlink, prevents omission. + monkeypatch.setattr(diagnostic.os, "lstat", lambda path: SimpleNamespace(kind=kind)) + assert diagnostic.omit_absent_stdlib_zip() is False + assert sys.path == [candidate, "other"] + + +def test_other_stat_errors_do_not_justify_omission(diagnostic, monkeypatch): + candidate = _candidate(monkeypatch) + monkeypatch.setattr(sys, "path", [candidate]) + + def denied(path): + raise PermissionError(path) + + monkeypatch.setattr(diagnostic.os, "lstat", denied) + with pytest.raises(PermissionError): + diagnostic.omit_absent_stdlib_zip() + assert sys.path == [candidate] + + +def test_unlisted_stdlib_candidate_is_not_probed(diagnostic, monkeypatch): + _candidate(monkeypatch) + monkeypatch.setattr(sys, "path", ["other"]) + monkeypatch.setattr( + diagnostic.os, "lstat", lambda path: pytest.fail("unexpected metadata probe") + ) + assert diagnostic.omit_absent_stdlib_zip() is False + assert sys.path == ["other"] + + +def test_real_non_cuda_import_keeps_original_loader( + diagnostic, fresh_import_state, monkeypatch +): + name = "invented_non_cuda_bootstrap_control" + monkeypatch.delitem(sys.modules, name, raising=False) + loaded = [] + + class Loader: + def create_module(self, spec): + return None + + def exec_module(self, module): + loaded.append(module.__name__) + module.marker = "ordinary_loader" + + loader = Loader() + + class Finder: + def find_spec(self, fullname, path=None, target=None): + if fullname == name: + return importlib.machinery.ModuleSpec(fullname, loader) + return None + + finder = Finder() + sys.meta_path.insert(0, finder) + before = tuple(sys.meta_path) + try: + with diagnostic.cpu_only_torch_import({}): + module = importlib.import_module(name) + assert module.marker == "ordinary_loader" + assert module.__loader__ is loader + assert loaded == [name] + assert tuple(sys.meta_path) == before + finally: + sys.modules.pop(name, None) diff --git a/packages/microcosm-build/tests/test_spec_seed_identity_engine_parameter_files.py b/packages/microcosm-build/tests/test_spec_seed_identity_engine_parameter_files.py new file mode 100644 index 000000000..2771d8988 --- /dev/null +++ b/packages/microcosm-build/tests/test_spec_seed_identity_engine_parameter_files.py @@ -0,0 +1,186 @@ +"""Pinned engine parameter files and the urllib3 probe: invented hooks only. + +The derive phase imports policyengine-us, which reads eight public parameter +CSVs at module import. The boundary may open exactly those files, only when +their bytes hash to the pins verified before the hook is armed; every other +data-suffixed open still refuses. urllib3's import-time IPv6 probe creates one +socket and binds it to loopback from ``_has_ipv6``; nothing else on a socket is +accepted. +""" + +import base64 +import hashlib +import importlib.util +import os +import types +from importlib import metadata +from pathlib import Path + +import pytest + + +@pytest.fixture(scope="module") +def diagnostic(): + source = ( + Path(__file__).resolve().parents[3] / "tools/spec_seed_identity_diagnostics.py" + ) + spec = importlib.util.spec_from_file_location("engine_parameter_controls", source) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def _boundary(diagnostic, monkeypatch, allowed): + hooks, first, distinct, contexts = [], [], [], {} + monkeypatch.setattr(diagnostic.sys, "addaudithook", hooks.append) + monkeypatch.setattr(Path, "resolve", lambda path: path) + monkeypatch.setattr(Path, "cwd", classmethod(lambda cls: Path("/invented/repo"))) + diagnostic.install_boundary( + Path("/invented/repo"), + Path("/invented/owned"), + Path("/invented/output"), + first_refusal=first, + distinct_refusals=distinct, + code_contexts=contexts, + allowed_data_files=allowed, + ) + assert len(hooks) == 1 + return hooks[0], distinct + + +@pytest.mark.requires_us +def test_pins_match_the_installed_distribution_record(diagnostic): + """Every pin is a regular file in the distribution whose RECORD digest agrees.""" + + dist = metadata.distribution(diagnostic.ENGINE_PUBLIC_PARAMETER_DISTRIBUTION) + root = Path(dist.locate_file("")).resolve() + record = {} + for entry in dist.files or (): + if entry.hash is not None and entry.hash.mode == "sha256": + padded = entry.hash.value + "=" * (-len(entry.hash.value) % 4) + record[str(entry)] = base64.urlsafe_b64decode(padded).hex() + assert len(diagnostic.ENGINE_PUBLIC_PARAMETER_FILES) == 8 + for relative, expected, size in diagnostic.ENGINE_PUBLIC_PARAMETER_FILES: + path = root / relative + assert path.is_file() and not path.is_symlink(), relative + raw = path.read_bytes() + assert len(raw) == size and hashlib.sha256(raw).hexdigest() == expected + assert record[relative] == expected + report, allowed = diagnostic.pinned_engine_parameter_files() + assert report["policy"] == "pinned_engine_public_parameter_files_v1" + assert report["count"] == 8 and report["version"] == dist.version + assert allowed == frozenset( + (root / relative).resolve() + for relative, _digest, _size in diagnostic.ENGINE_PUBLIC_PARAMETER_FILES + ) + + +def test_a_pin_whose_bytes_differ_refuses_before_arming(diagnostic, tmp_path): + """Verification fails closed on bytes, size, RECORD, or a missing file.""" + + relative = "invented_engine/parameters/table.csv" + root = tmp_path / "site-packages" + (root / "invented_engine/parameters").mkdir(parents=True) + raw = b"invented,public,parameter\n1,2,3\n" + (root / relative).write_bytes(raw) + good = hashlib.sha256(raw).hexdigest() + encoded = base64.urlsafe_b64encode(hashlib.sha256(raw).digest()).rstrip(b"=") + + class _Hash: + mode = "sha256" + + def __init__(self, value: str) -> None: + self.value = value + + class _Entry(str): + hash = _Hash(encoded.decode()) + + class _Distribution: + version = "0.0.0-invented" + files = [_Entry(relative)] + + def locate_file(self, _name): + return root + + def distribution(name): + assert name == "invented-engine" + return _Distribution() + + real = metadata.distribution + try: + metadata.distribution = distribution + report, allowed = diagnostic.pinned_engine_parameter_files( + distribution="invented-engine", pins=((relative, good, len(raw)),) + ) + assert report["count"] == 1 and allowed == {(root / relative).resolve()} + for pins in ( + ((relative, "0" * 64, len(raw)),), + ((relative, good, len(raw) + 1),), + (("invented_engine/parameters/absent.csv", good, len(raw)),), + (("../outside.csv", good, len(raw)),), + ): + with pytest.raises(diagnostic.RefusalError, match="^ENGINE_PARAMETER_PIN$"): + diagnostic.pinned_engine_parameter_files( + distribution="invented-engine", pins=pins + ) + _Entry.hash = _Hash("AAAA") # RECORD disagrees with the file bytes. + with pytest.raises(diagnostic.RefusalError, match="^ENGINE_PARAMETER_PIN$"): + diagnostic.pinned_engine_parameter_files( + distribution="invented-engine", pins=((relative, good, len(raw)),) + ) + finally: + metadata.distribution = real + + +def test_hook_opens_only_the_allowed_data_files(diagnostic, monkeypatch): + allowed = frozenset({Path("/invented/site-packages/engine/parameters/pinned.csv")}) + hook, distinct = _boundary(diagnostic, monkeypatch, allowed) + hook("open", (str(next(iter(allowed))), "r", os.O_RDONLY)) + assert distinct == [] + for path in ( + "/invented/site-packages/engine/parameters/other.csv", + "/invented/site-packages/engine/parameters/pinned.csv.bak.csv", + "/invented/repo/pinned.csv", + ): + with pytest.raises(diagnostic.RefusalError, match="^DATA_FILE$"): + hook("open", (path, "r", os.O_RDONLY)) + assert distinct == ["DATA_FILE"] + + +def test_proc_stat_is_readable_metadata(diagnostic, monkeypatch): + hook, distinct = _boundary(diagnostic, monkeypatch, frozenset()) + hook("open", ("/proc/stat", "r", os.O_RDONLY)) + assert distinct == [] + with pytest.raises(diagnostic.RefusalError, match="^READ_SCOPE$"): + hook("open", ("/proc/version", "r", os.O_RDONLY)) + + +def _call_as_has_ipv6(diagnostic, event, args): + """Invoke the hook from a frame named like urllib3's probe function.""" + + code = compile( + "def _has_ipv6(hook, event, args):\n return hook(event, args)\n", + "/invented/site-packages/urllib3/util/connection.py", + "exec", + ) + namespace: dict[str, object] = {} + exec(code, namespace) + return namespace["_has_ipv6"](diagnostic, event, args) + + +def test_urllib3_ipv6_probe_is_the_only_accepted_socket_use(diagnostic, monkeypatch): + hook, distinct = _boundary(diagnostic, monkeypatch, frozenset()) + sock = types.SimpleNamespace() + _call_as_has_ipv6(hook, "socket.__new__", (sock, 10, 1, 0)) + _call_as_has_ipv6(hook, "socket.bind", (sock, ("::1", 0))) + _call_as_has_ipv6(hook, "socket.bind", (sock, ("127.0.0.1", 0))) + assert distinct == [] + with pytest.raises(diagnostic.RefusalError, match="^NETWORK_OR_CHILD$"): + _call_as_has_ipv6(hook, "socket.bind", (sock, ("0.0.0.0", 0))) + with pytest.raises(diagnostic.RefusalError, match="^NETWORK_OR_CHILD$"): + _call_as_has_ipv6(hook, "socket.connect", (sock, ("::1", 80))) + with pytest.raises(diagnostic.RefusalError, match="^NETWORK_OR_CHILD$"): + hook("socket.__new__", (sock, 10, 1, 0)) + with pytest.raises(diagnostic.RefusalError, match="^NETWORK_OR_CHILD$"): + hook("socket.bind", (sock, ("::1", 0))) + assert distinct == ["NETWORK_OR_CHILD"] diff --git a/packages/microcosm-build/tests/test_spec_seed_identity_outer_failure.py b/packages/microcosm-build/tests/test_spec_seed_identity_outer_failure.py new file mode 100644 index 000000000..a24788b54 --- /dev/null +++ b/packages/microcosm-build/tests/test_spec_seed_identity_outer_failure.py @@ -0,0 +1,107 @@ +"""Formatting-only outer-failure controls; no main/audit/dependency execution.""" + +import importlib.util +import json +from pathlib import Path + +import pytest + + +@pytest.fixture(scope="module") +def diagnostic(): + source = ( + Path(__file__).resolve().parents[3] / "tools/spec_seed_identity_diagnostics.py" + ) + spec = importlib.util.spec_from_file_location("outer_failure_controls", source) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +@pytest.mark.parametrize("kind", [TypeError, PermissionError]) +def test_outer_error_keeps_fixed_type_and_code_coordinates_without_message( + diagnostic, kind +): + try: + raise kind("secret-password.csv user-input-payload") + except kind as error: + payload = diagnostic.outer_failure_context(error) + value = json.loads(payload) + assert value["exception_type"] == kind.__name__ + assert value["code"] == "OUTER_FAILURE" + assert value["completed"] is value["coverage_pass"] is False + assert value["artifact_status_not_modified"] is True + assert b"secret-password" not in payload + assert b"user-input-payload" not in payload + assert len(payload) <= 8192 + + +def test_actual_short_named_traceback_frame_is_retained(diagnostic): + def cleanup_probe(): + raise TypeError("not emitted") + + try: + cleanup_probe() + except TypeError as error: + result = json.loads(diagnostic.outer_failure_context(error)) + assert result["traceback_code_frames"][-1]["function"] == "cleanup_probe" + assert result["traceback_code_frames"][-1]["line"] > 0 + + +def test_fixed_refusal_retains_attached_sanitized_context(diagnostic): + context = {"code": "READ_SCOPE", "event": "open", "requested": {"path": None}} + error = diagnostic.RefusalError( + "READ_SCOPE", boundary_context=diagnostic.encoded(context) + ) + result = json.loads(diagnostic.outer_failure_context(error)) + assert result["exception_type"] == "RefusalError" + assert result["code"] == "READ_SCOPE" + assert result["boundary_refusal"] == context + + +def test_unknown_refusal_and_exception_names_are_fixed_labels(diagnostic): + class SecretNamedError(Exception): + pass + + errors = ( + diagnostic.RefusalError("arbitrary-secret-code"), + SecretNamedError("secret-message"), + ) + for error in errors: + payload = diagnostic.outer_failure_context(error) + assert b"secret" not in payload.lower() + assert b"SecretNamedError" not in payload + result = json.loads(payload) + assert result["code"] in {"OUTER_FAILURE", "OTHER_FIXED_REFUSAL"} + + +def test_traceback_scan_and_output_are_bounded(diagnostic): + def recurse(depth): + if depth: + recurse(depth - 1) + else: + raise ValueError("unprinted") + + try: + recurse(40) + except ValueError as error: + payload = diagnostic.outer_failure_context(error) + result = json.loads(payload) + assert result["traceback_truncated"] is True + assert len(result["traceback_code_frames"]) <= 8 + assert len(payload) <= 8192 + + +def test_exception_only_metadata_does_not_open_or_resolve_files( + diagnostic, monkeypatch +): + def forbidden(*args, **kwargs): + pytest.fail("outer report attempted file I/O or resolution") + + monkeypatch.setattr(Path, "read_bytes", forbidden) + monkeypatch.setattr(Path, "read_text", forbidden) + monkeypatch.setattr(Path, "resolve", forbidden) + monkeypatch.setattr(Path, "stat", forbidden) + monkeypatch.setattr(Path, "open", forbidden) + payload = diagnostic.outer_failure_context(TypeError("no traceback")) + assert json.loads(payload)["traceback_code_frames"] == [] diff --git a/packages/microcosm-build/tests/test_spec_seed_identity_owned_temp.py b/packages/microcosm-build/tests/test_spec_seed_identity_owned_temp.py new file mode 100644 index 000000000..2fe9d1739 --- /dev/null +++ b/packages/microcosm-build/tests/test_spec_seed_identity_owned_temp.py @@ -0,0 +1,60 @@ +"""Owned temporary-directory controls; no audit, main, Torch or Microcosm run.""" + +import importlib.util +import stat +from pathlib import Path + +import pytest + + +@pytest.fixture(scope="module") +def diagnostic(): + source = ( + Path(__file__).resolve().parents[3] / "tools/spec_seed_identity_diagnostics.py" + ) + spec = importlib.util.spec_from_file_location("owned_temp_controls", source) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +@pytest.mark.parametrize("refuses", [False, True]) +def test_owned_fixture_survives_success_and_exact_refusal( + diagnostic, tmp_path, refuses +): + context = b'{"code":"READ_SCOPE","event":"open"}\n' + error = diagnostic.RefusalError("READ_SCOPE", boundary_context=context) + caught = None + try: + with diagnostic.retained_owned_directory(tmp_path) as owned: + assert owned.parent == tmp_path + assert owned.name.startswith("spec-seed-owned-") + assert stat.S_IMODE(owned.stat().st_mode) == 0o700 + fixture = owned / "invented-spec.txt" + fixture.write_text("invented fixture only") + if refuses: + raise error + except diagnostic.RefusalError as actual: + caught = actual + assert caught is (error if refuses else None) + if caught is not None: + assert caught.boundary_context is context + assert fixture.read_text() == "invented fixture only" + assert owned.is_dir() + + +def test_each_workspace_is_distinct_and_has_no_implicit_cleanup( + diagnostic, tmp_path, monkeypatch +): + def forbidden(*args, **kwargs): + pytest.fail("implicit TemporaryDirectory cleanup was registered") + + monkeypatch.setattr(diagnostic.tempfile, "TemporaryDirectory", forbidden) + with diagnostic.retained_owned_directory(tmp_path) as first: + (first / "invented.txt").write_text("first") + with diagnostic.retained_owned_directory(tmp_path) as second: + (second / "invented.txt").write_text("second") + assert first != second + assert first.parent == second.parent == tmp_path + assert (first / "invented.txt").read_text() == "first" + assert (second / "invented.txt").read_text() == "second" diff --git a/packages/microcosm-build/tests/test_spec_seed_identity_per_code_context.py b/packages/microcosm-build/tests/test_spec_seed_identity_per_code_context.py new file mode 100644 index 000000000..0ee55f9a9 --- /dev/null +++ b/packages/microcosm-build/tests/test_spec_seed_identity_per_code_context.py @@ -0,0 +1,228 @@ +"""Invented audit calls only; no real hook, source read, child or derivation.""" + +import importlib.util +import json +import os +from pathlib import Path + +import pytest + + +@pytest.fixture(scope="module") +def diagnostic(): + source = ( + Path(__file__).resolve().parents[3] / "tools/spec_seed_identity_diagnostics.py" + ) + spec = importlib.util.spec_from_file_location("per_code_context_controls", source) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +@pytest.fixture +def audited(diagnostic, monkeypatch): + hooks, first, distinct, contexts = [], [], [], {} + monkeypatch.setattr(diagnostic.sys, "addaudithook", hooks.append) + monkeypatch.setattr(Path, "resolve", lambda path: path) + monkeypatch.setattr(Path, "cwd", classmethod(lambda cls: Path("/invented/repo"))) + refusals = diagnostic.install_boundary( + Path("/invented/repo"), + Path("/invented/owned"), + Path("/invented/output"), + first_refusal=first, + distinct_refusals=distinct, + code_contexts=contexts, + ) + assert len(hooks) == 1 + return hooks[0], refusals, first, distinct, contexts + + +def test_caught_middle_child_denial_keeps_own_context(diagnostic, audited): + hook, refusals, first, distinct, contexts = audited + terminal = None + secret = "invented-secret-never-retain" + for event, args, code in ( + ("open", ("/proc/version", "r", os.O_RDONLY), "READ_SCOPE"), + ( + "subprocess.Popen", + (secret, [secret], secret, {secret: secret}), + "NETWORK_OR_CHILD", + ), + ("open", ("/invented/repo/never-open.csv", "r", os.O_RDONLY), "DATA_FILE"), + ): + with pytest.raises(diagnostic.RefusalError, match="^" + code + "$") as error: + hook(event, args) + terminal = error.value + # The historical refusal counter remains first-code-only. Every operation + # still raises; retaining context must not turn a swallowed denial into success. + assert refusals == ["READ_SCOPE"] + assert distinct == ["READ_SCOPE", "NETWORK_OR_CHILD", "DATA_FILE"] + assert list(contexts) == distinct + assert first == [contexts["READ_SCOPE"]] + assert terminal.boundary_context == contexts["DATA_FILE"] + child = json.loads(contexts["NETWORK_OR_CHILD"]) + assert child["code"] == "NETWORK_OR_CHILD" and child["event"] == "subprocess.Popen" + assert child["frames"] and len(child["frames"]) <= 8 + assert child["requested"]["path"] is None and child["resolved"]["path"] is None + for payload in contexts.values(): + assert type(payload) is bytes and len(payload) <= 4096 + assert secret.encode() not in payload and b"never-open.csv" not in payload + + +def test_repeated_code_retains_first_record_but_exception_has_fresh_context( + diagnostic, audited +): + hook, refusals, first, distinct, contexts = audited + + def original_caller(): + hook("subprocess.Popen", ("invented", [], None, None)) + + def later_caller(): + hook("subprocess.Popen", ("invented", [], None, None)) + + with pytest.raises(diagnostic.RefusalError) as original: + original_caller() + retained = contexts["NETWORK_OR_CHILD"] + with pytest.raises(diagnostic.RefusalError) as later: + later_caller() + assert contexts == {"NETWORK_OR_CHILD": retained} + assert first == [retained] and refusals == distinct == ["NETWORK_OR_CHILD"] + assert original.value.boundary_context == retained + assert later.value.boundary_context != retained + assert "original_caller" in [f["function"] for f in json.loads(retained)["frames"]] + assert "later_caller" in [ + f["function"] for f in json.loads(later.value.boundary_context)["frames"] + ] + + +def test_eight_record_capacity_never_discards_or_grows(diagnostic, audited): + hook, refusals, first, distinct, contexts = audited + # A full invented collector exercises the bound even though the current + # boundary emits only seven different fixed codes. + codes = ( + "WRITE_SCOPE", + "DATA_FILE", + "FILESYSTEM_PATH", + "FILESYSTEM_DESCRIPTOR", + "FILESYSTEM_LINK", + "NETWORK_OR_CHILD", + "WALL", + "OUTPUT_CHANGED", + ) + contexts.update({code: diagnostic.encoded({"code": code}) for code in codes}) + retained = tuple(contexts.items()) + with pytest.raises(diagnostic.RefusalError, match="^READ_SCOPE$") as error: + hook("open", ("/proc/version", "r", os.O_RDONLY)) + assert tuple(contexts.items()) == retained and len(contexts) == 8 + assert refusals == distinct == ["READ_SCOPE"] + assert first == [error.value.boundary_context] + + +@pytest.mark.parametrize( + ("event", "label"), + ( + ("os.posix_spawn", "os.posix_spawn"), + ("socket.connect", "network_or_child_event"), + ("subprocess.invented-secret-event", "network_or_child_event"), + ), +) +def test_network_context_event_is_fixed_and_never_serializes_arguments( + diagnostic, audited, event, label +): + hook, _, _, _, contexts = audited + secret = "invented-secret-arguments" + with pytest.raises(diagnostic.RefusalError, match="^NETWORK_OR_CHILD$"): + hook(event, (secret, {secret: secret})) + payload = contexts["NETWORK_OR_CHILD"] + assert json.loads(payload)["event"] == label + assert secret.encode() not in payload and b"invented-secret-event" not in payload + + +def test_context_capture_reads_no_source_files_or_locals( + diagnostic, audited, monkeypatch +): + hook, _, _, _, contexts = audited + private_local = "invented-private-local-not-evidence" + + def forbidden(*args, **kwargs): + pytest.fail("context formatting attempted a source or filesystem read") + + with monkeypatch.context() as patch: + for name in ("open", "read_bytes", "read_text", "stat", "lstat"): + patch.setattr(Path, name, forbidden) + with pytest.raises(diagnostic.RefusalError, match="^NETWORK_OR_CHILD$"): + hook("subprocess.Popen", (private_local,)) + payload = contexts["NETWORK_OR_CHILD"] + assert private_local.encode() not in payload + assert all( + set(frame) == {"file", "function", "line"} + for frame in json.loads(payload)["frames"] + ) + + +def test_context_capture_failure_does_not_replace_denial_with_exception_text( + diagnostic, audited, monkeypatch +): + hook, refusals, first, _, contexts = audited + + def unavailable(*args): + raise RuntimeError("invented-sensitive-exception-text") + + monkeypatch.setattr(diagnostic, "read_refusal_context", unavailable) + with pytest.raises(diagnostic.RefusalError, match="^NETWORK_OR_CHILD$") as error: + hook("subprocess.Popen", ("invented",)) + expected = diagnostic.encoded( + {"code": "NETWORK_OR_CHILD", "context": "unavailable"} + ) + assert refusals == ["NETWORK_OR_CHILD"] + assert first == [expected] and contexts == {"NETWORK_OR_CHILD": expected} + assert error.value.boundary_context == expected + + +def test_all_maximum_records_fit_status_cap_and_preserve_aggregate_cap(diagnostic): + base = diagnostic.encoded({"code": "READ_SCOPE", "context": ""}) + maximum = diagnostic.encoded( + {"code": "READ_SCOPE", "context": "x" * (4096 - len(base))} + ) + assert len(maximum) == 4096 + status = diagnostic.encoded( + { + "completed": False, + "coverage_pass": False, + "first_boundary_refusal": json.loads(maximum), + "terminal_boundary_refusal": json.loads(maximum), + "boundary_refusal_contexts": [json.loads(maximum)] * 8, + "boundary_refusal_context_limit": {"codes": 8, "bytes_per_code": 4096}, + } + ) + assert 16 * 1024 < len(status) < 64 * 1024 + assert diagnostic.CAPS == { + "candidate-digests.json": 64 * 1024, + "seed-protocol.json": 256 * 1024, + "seed-map.json": 256 * 1024, + "seed-bindings.json": 128 * 1024, + "environment-and-source.json": 128 * 1024, + "diagnostic-status.json": 64 * 1024, + } + assert diagnostic.MAX_TOTAL == 1024 * 1024 + payloads = {name: b"{}" for name in diagnostic.CAPS} + payloads["diagnostic-status.json"] = status + diagnostic.validate_payloads(payloads) + + +@pytest.mark.parametrize( + "name", + ( + "candidate-digests.json", + "seed-protocol.json", + "seed-map.json", + "seed-bindings.json", + "environment-and-source.json", + "diagnostic-status.json", + ), +) +def test_each_existing_output_cap_still_refuses_before_publication(diagnostic, name): + payloads = {key: b"{}" for key in diagnostic.CAPS} + payloads[name] = b"x" * (diagnostic.CAPS[name] + 1) + with pytest.raises(diagnostic.RefusalError, match="^OUTPUT_CAP$"): + diagnostic.validate_payloads(payloads) diff --git a/packages/microcosm-build/tests/test_spec_seed_identity_system_metadata.py b/packages/microcosm-build/tests/test_spec_seed_identity_system_metadata.py new file mode 100644 index 000000000..8b4c89b98 --- /dev/null +++ b/packages/microcosm-build/tests/test_spec_seed_identity_system_metadata.py @@ -0,0 +1,353 @@ +"""Invented metadata controls; no main, engine, data, child or real audit hook.""" + +import importlib.util +import os +import platform +import subprocess +from pathlib import Path +from types import SimpleNamespace + +import pytest + + +@pytest.fixture(scope="module") +def diagnostic(): + source = ( + Path(__file__).resolve().parents[3] / "tools/spec_seed_identity_diagnostics.py" + ) + spec = importlib.util.spec_from_file_location("system_metadata_controls", source) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +@pytest.fixture +def invented_uname(diagnostic, monkeypatch): + observed = SimpleNamespace( + sysname="Linux", + nodename="invented-host", + release="invented-release", + version="invented-version", + machine="invented_arch", + ) + info = platform.uname_result( + observed.sysname, + observed.nodename, + observed.release, + observed.version, + observed.machine, + ) + monkeypatch.setattr(diagnostic.sys, "platform", "linux") + monkeypatch.setattr(diagnostic.os, "uname", lambda: observed) + monkeypatch.setattr(diagnostic.platform, "uname", lambda: info) + + def forbidden(*args, **kwargs): + pytest.fail("processor metadata attempted a subprocess") + + monkeypatch.setattr(subprocess, "check_output", forbidden) + return info, observed + + +def test_real_cached_property_uses_machine_without_replacing_stdlib( + diagnostic, invented_uname +): + info, observed = invented_uname + before = ( + platform.uname, + platform.processor, + platform.uname_result, + vars(type(info))["processor"], + ) + bootstrap = {} + state = diagnostic.prime_processor_metadata(bootstrap) + assert platform.processor() == observed.machine + assert tuple(info) == ( + observed.sysname, + observed.nodename, + observed.release, + observed.version, + observed.machine, + observed.machine, + ) + assert bootstrap["system_metadata"]["uname_p_parity_claimed"] is False + assert bootstrap["system_metadata"]["source"] == "os.uname.machine" + assert before == ( + platform.uname, + platform.processor, + platform.uname_result, + vars(type(info))["processor"], + ) + diagnostic.verify_processor_metadata(state, bootstrap) + # An already-cached identical machine value is not silently altered. + again = diagnostic.prime_processor_metadata(bootstrap) + assert again == state + + +def test_conflicting_observed_processor_cache_is_not_overwritten( + diagnostic, invented_uname +): + info, _ = invented_uname + info.processor = "different_observed_processor" + bootstrap = {} + with pytest.raises(diagnostic.RefusalError, match="^PROCESSOR_CACHE_CONFLICT$"): + diagnostic.prime_processor_metadata(bootstrap) + assert info.processor == "different_observed_processor" + assert bootstrap == {} + + +@pytest.mark.parametrize("change", ("cached_value", "os_machine", "evidence")) +def test_metadata_change_refuses(diagnostic, invented_uname, change): + info, observed = invented_uname + bootstrap = {} + state = diagnostic.prime_processor_metadata(bootstrap) + if change == "cached_value": + info.processor = "changed" + elif change == "os_machine": + observed.machine = "changed" + else: + bootstrap["system_metadata"]["uname_p_parity_claimed"] = True + with pytest.raises(diagnostic.RefusalError, match="^PROCESSOR_METADATA_CHANGED$"): + diagnostic.verify_processor_metadata(state, bootstrap) + + +@pytest.fixture +def audited(diagnostic, monkeypatch): + hooks = [] + monkeypatch.setattr(diagnostic.sys, "addaudithook", hooks.append) + monkeypatch.setattr(diagnostic.os, "getpid", lambda: 321) + monkeypatch.setattr(diagnostic.os, "readlink", lambda _: "pipe:[invented]") + monkeypatch.setattr(Path, "cwd", classmethod(lambda cls: Path("/invented/repo"))) + + def resolve(path): + return Path("/proc/321/maps") if path == Path("/proc/self/maps") else path + + monkeypatch.setattr(Path, "resolve", resolve) + refusals = diagnostic.install_boundary( + Path("/invented/repo"), Path("/invented/owned"), Path("/invented/output") + ) + assert len(hooks) == 1 + return hooks[0], refusals + + +@pytest.mark.parametrize("path", ("/proc/self/maps", "/proc/321/maps")) +def test_only_logical_and_resolved_own_maps_are_admitted(audited, path): + hook, refusals = audited + hook("open", (path, "r", os.O_RDONLY)) + assert refusals == [] + + +@pytest.mark.parametrize( + "kind", ("other_process", "other_file", "relative", "descriptor", "child") +) +def test_maps_alias_does_not_admit_other_paths_descriptors_or_children( + diagnostic, audited, kind +): + hook, refusals = audited + path = { + "other_process": "/proc/322/maps", + "other_file": "/proc/321/mem", + "relative": "maps", + "descriptor": 55, + "child": None, + }[kind] + code = "NETWORK_OR_CHILD" if kind == "child" else "READ_SCOPE" + with pytest.raises(diagnostic.RefusalError, match="^" + code + "$"): + if kind == "child": + hook("subprocess.Popen", ("uname", ["uname", "-p"], None, None)) + else: + hook("open", (path, "r", os.O_RDONLY)) + assert refusals == [code] + + +@pytest.fixture +def coordinate_environment(diagnostic, monkeypatch): + """Exercise coordinate validation against an isolated invented environment.""" + values = { + "DIAG_CHECKOUT_SHA": "1" * 40, + "GITHUB_SHA": "1" * 40, + "DIAG_WORKFLOW_SHA": "2" * 40, + "DIAG_PR_HEAD_SHA": "3" * 40, + "DIAG_PR_BASE_SHA": "4" * 40, + "DIAG_MERGE_SHA": "5" * 40, + "GITHUB_EVENT_NAME": "pull_request", + "GITHUB_RUN_ID": "123456", + "GITHUB_RUN_ATTEMPT": "2", + "GITHUB_REPOSITORY": "PolicyEngine/microcosm", + } + # coordinates() consumes only environ. Keep the real process environment, + # filesystem, dependency imports and bootstrap paths outside these cases. + monkeypatch.setattr(diagnostic, "os", SimpleNamespace(environ=values)) + return values + + +@pytest.mark.parametrize("event", ("pull_request", "push")) +def test_coordinates_preserve_all_distinct_supplied_identities( + diagnostic, coordinate_environment, event +): + values = coordinate_environment + values["GITHUB_EVENT_NAME"] = event + original = dict(values) + assert diagnostic.coordinates() == { + "checkout_sha": "1" * 40, + "event_sha": "1" * 40, + "workflow_sha": "2" * 40, + "pr_head_sha": "3" * 40, + "pr_base_sha": "4" * 40, + "event_merge_sha": "5" * 40, + "github_run_id": "123456", + "github_run_attempt": "2", + "event": event, + "repository": "PolicyEngine/microcosm", + } + assert values == original + + +@pytest.mark.parametrize("missing", (False, True)) +def test_pull_request_coordinates_keep_absent_merge_metadata_empty( + diagnostic, coordinate_environment, missing +): + values = coordinate_environment + if missing: + del values["DIAG_MERGE_SHA"] + else: + values["DIAG_MERGE_SHA"] = "" + result = diagnostic.coordinates() + assert result["event"] == "pull_request" + assert result["event_merge_sha"] == "" + assert result["checkout_sha"] == result["event_sha"] == "1" * 40 + assert result["workflow_sha"] == "2" * 40 + assert result["pr_head_sha"] == "3" * 40 + assert result["pr_base_sha"] == "4" * 40 + assert values.get("DIAG_MERGE_SHA", "") == "" + + +@pytest.mark.parametrize("missing", (False, True)) +def test_push_coordinates_allow_absent_pull_request_metadata( + diagnostic, coordinate_environment, missing +): + values = coordinate_environment + values["GITHUB_EVENT_NAME"] = "push" + for name in ("DIAG_PR_HEAD_SHA", "DIAG_PR_BASE_SHA", "DIAG_MERGE_SHA"): + if missing: + del values[name] + else: + values[name] = "" + result = diagnostic.coordinates() + assert result["event"] == "push" + assert result["pr_head_sha"] == result["pr_base_sha"] == "" + assert result["event_merge_sha"] == "" + assert result["checkout_sha"] == result["event_sha"] == "1" * 40 + assert result["workflow_sha"] == "2" * 40 + + +@pytest.mark.parametrize( + ("event", "name"), + ( + ("pull_request", "DIAG_CHECKOUT_SHA"), + ("pull_request", "GITHUB_SHA"), + ("pull_request", "DIAG_WORKFLOW_SHA"), + ("pull_request", "DIAG_PR_HEAD_SHA"), + ("pull_request", "DIAG_PR_BASE_SHA"), + ("push", "DIAG_CHECKOUT_SHA"), + ("push", "GITHUB_SHA"), + ("push", "DIAG_WORKFLOW_SHA"), + ), +) +def test_required_coordinate_hashes_still_refuse_when_absent( + diagnostic, coordinate_environment, event, name +): + values = coordinate_environment + values["GITHUB_EVENT_NAME"] = event + del values[name] + with pytest.raises(diagnostic.RefusalError, match="^COORDINATES$"): + diagnostic.coordinates() + + +@pytest.mark.parametrize( + ("event", "name"), + ( + ("pull_request", "DIAG_CHECKOUT_SHA"), + ("pull_request", "GITHUB_SHA"), + ("pull_request", "DIAG_WORKFLOW_SHA"), + ("pull_request", "DIAG_PR_HEAD_SHA"), + ("pull_request", "DIAG_PR_BASE_SHA"), + ("pull_request", "DIAG_MERGE_SHA"), + ("push", "DIAG_PR_HEAD_SHA"), + ("push", "DIAG_PR_BASE_SHA"), + ("push", "DIAG_MERGE_SHA"), + ), +) +def test_nonempty_coordinate_hashes_still_require_valid_format( + diagnostic, coordinate_environment, event, name +): + values = coordinate_environment + values["GITHUB_EVENT_NAME"] = event + values[name] = "not-a-sha" + with pytest.raises(diagnostic.RefusalError, match="^COORDINATES$"): + diagnostic.coordinates() + + +@pytest.mark.parametrize( + "value", ("null", "A" * 40, "f" * 39, "f" * 41, "f" * 40 + "\n") +) +def test_missing_merge_exception_does_not_accept_malformed_present_hashes( + diagnostic, coordinate_environment, value +): + coordinate_environment["DIAG_MERGE_SHA"] = value + with pytest.raises(diagnostic.RefusalError, match="^COORDINATES$"): + diagnostic.coordinates() + + +@pytest.mark.parametrize("event", ("pull_request", "push")) +def test_checkout_must_match_event_even_without_merge_metadata( + diagnostic, coordinate_environment, event +): + values = coordinate_environment + values["GITHUB_EVENT_NAME"] = event + values["DIAG_MERGE_SHA"] = "" + values["DIAG_CHECKOUT_SHA"] = "6" * 40 + with pytest.raises(diagnostic.RefusalError, match="^CHECKOUT$"): + diagnostic.coordinates() + + +@pytest.mark.parametrize( + ("name", "value"), + ( + ("GITHUB_RUN_ID", None), + ("GITHUB_RUN_ID", ""), + ("GITHUB_RUN_ID", "not-a-run"), + ("GITHUB_RUN_ATTEMPT", None), + ("GITHUB_RUN_ATTEMPT", ""), + ("GITHUB_RUN_ATTEMPT", "not-an-attempt"), + ), +) +def test_coordinate_run_identity_remains_required( + diagnostic, coordinate_environment, name, value +): + if value is None: + del coordinate_environment[name] + else: + coordinate_environment[name] = value + with pytest.raises(diagnostic.RefusalError, match="^RUN_COORDINATES$"): + diagnostic.coordinates() + + +@pytest.mark.parametrize( + ("name", "value", "code"), + ( + ("GITHUB_EVENT_NAME", None, "EVENT"), + ("GITHUB_EVENT_NAME", "", "EVENT"), + ("GITHUB_EVENT_NAME", "workflow_dispatch", "EVENT"), + ("GITHUB_REPOSITORY", None, "REPOSITORY"), + ("GITHUB_REPOSITORY", "invented/other", "REPOSITORY"), + ), +) +def test_coordinate_event_and_repository_boundaries_remain_required( + diagnostic, coordinate_environment, name, value, code +): + if value is None: + del coordinate_environment[name] + else: + coordinate_environment[name] = value + with pytest.raises(diagnostic.RefusalError, match="^" + code + "$"): + diagnostic.coordinates() diff --git a/packages/microcosm-build/tests/test_target_materialization.py b/packages/microcosm-build/tests/test_target_materialization.py index 4c81e6173..7b1592e3b 100644 --- a/packages/microcosm-build/tests/test_target_materialization.py +++ b/packages/microcosm-build/tests/test_target_materialization.py @@ -158,6 +158,31 @@ def test_prepared_column_path_materializes_filtered_values(): ] +@pytest.mark.parametrize( + "left,right,expected", + [ + ([True, False, True], [True, False, False], [2.0, 0.0, 1.0]), + ([True, False, True], [2, 3, 4], [3.0, 3.0, 5.0]), + ([2, 3, 4], [True, False, True], [3.0, 3.0, 5.0]), + ([1, 2, 3], [4, 5, 6], [5.0, 7.0, 9.0]), + ([0.25, -0.5, 2.0], [0.5, 1.5, -1.0], [0.75, 1.0, 1.0]), + ], +) +def test_shared_expression_uses_arithmetic_for_boolean_terms(left, right, expected): + adapter = StubAdapter() + adapter.tables["person"]["left"] = np.asarray(left) + adapter.tables["person"]["right"] = np.asarray(right) + registry = _resolution_registry("sum") + result = materialize_target_bindings( + adapter, + registry, + {"sum": {"bindings": {"policyengine": {"value_expression": "left + right"}}}}, + period=2025, + ) + assert result.skipped == () + np.testing.assert_array_equal(adapter.tables["person"]["sum_measure"], expected) + + @pytest.mark.parametrize("fact_period", [2024, 2025]) @pytest.mark.parametrize("require_matching_fact_period", [None, False, True]) def test_existing_measure_respects_fact_guard_at_default_measurement_period( diff --git a/packages/microcosm-build/tests/test_uk_atomic_area_support.py b/packages/microcosm-build/tests/test_uk_atomic_area_support.py new file mode 100644 index 000000000..b25887a87 --- /dev/null +++ b/packages/microcosm-build/tests/test_uk_atomic_area_support.py @@ -0,0 +1,482 @@ +"""Invented array/identity controls only; no publisher files or country engine.""" + +import numpy as np +import pandas as pd +import pytest + +from microcosm.build import atomic_geography as geo +from microcosm.build.uk_runtime.atomic_area_support import ( + IDENTITY_COLUMN, + SOURCES, + SYSTEMS, + assemble_uk_atomic_area_support, + uk_atomic_assignment_definition, +) +from microcosm.build.uk_runtime.atomic_household_identity import household_draw_key +from microcosm.build.uk_runtime.rowwise_geography import FRS_REGION_TO_REGION_CODE + + +def parts(system): + index = SYSTEMS.index(system) + regions = ( + [ + r + for r in FRS_REGION_TO_REGION_CODE + if r not in {"SCOTLAND", "NORTHERN_IRELAND"} + ] + if index == 0 + else ["SCOTLAND"] + if index == 1 + else ["NORTHERN_IRELAND"] + ) + rows = [] + region_order = list(FRS_REGION_TO_REGION_CODE) + for region in regions: + prefix = FRS_REGION_TO_REGION_CODE[region][0] + for offset in range(3): + serial = region_order.index(region) * 3 + offset + 1 + area = prefix + ("20" if index == 2 else "00") + f"{serial:06d}" + rows.append( + { + "oa_code": area, + "population": [2.0, 6.0, 1000.0][offset], + "households": [0.0, 30.0, 10.0][offset], + "constituency_code": prefix + + "14" + + f"{serial - offset + (offset == 2):06d}", + "region_code": FRS_REGION_TO_REGION_CODE[region], + "lsoa_code": area + if index == 2 + else prefix + "01" + f"{serial:06d}", + "msoa_code": prefix + "02" + f"{serial:06d}", + "local_authority_code": prefix + "06" + f"{serial:06d}", + "ward_code": prefix + "05" + f"{serial:06d}", + "itl3_code": "TL" + + "CDEFGHIJKLMN"[region_order.index(region)] + + "01", + } + ) + arrays = {column: np.asarray([r[column] for r in rows]) for column in rows[0]} + descriptions = {} + for column in arrays: + if column in {"population", "households"}: + descriptions[column] = { + "kind": "weight", + "source": "invented-" + system + "-" + column, + "basis": "invented persons" + if column == "population" + else "invented occupied households", + } + else: + descriptions[column] = { + "kind": "code", + "source": "invented-" + system + "-" + column, + "vintage": "2022_census" if index == 1 else "2021_census", + "relation": "exact" + if column in {"oa_code", "lsoa_code", "msoa_code", "region_code"} + else "best_fit", + } + descriptions["constituency_code"]["vintage"] = "2024_pcon" + if index == 2: + descriptions["constituency_code"]["relation"] = "official_tabulation" + return arrays, descriptions + + +def payloads(): + return { + system: assemble_uk_atomic_area_support( + system=system, arrays=arrays, column_metadata=metadata + ) + for system in SYSTEMS + for arrays, metadata in [parts(system)] + } + + +def key(source_id=101, path=()): + return household_draw_key( + source="invented-frs", + source_vintage="invented-2024-25", + source_household_id=source_id, + clone_path=path, + ) + + +def complete(households, payload): + spec = uk_atomic_assignment_definition(payload, seed=43) + supports = { + system: geo.decode_atomic_support(data) for system, data in payload.items() + } + result = pd.concat( + [households, geo.assign_atomic(households, spec, supports)], axis=1 + ) + result = pd.concat([result, geo.derive_geography(result, spec, supports)], axis=1) + return result, spec, supports + + +def test_normalization_preserves_all_rows_counts_mappings_and_source_labels(): + for system in SYSTEMS: + arrays, metadata = parts(system) + original = {k: v.copy() for k, v in arrays.items()} + data = assemble_uk_atomic_area_support( + system=system, arrays=arrays, column_metadata=metadata + ) + support = geo.decode_atomic_support(data) + order = np.argsort(arrays["oa_code"], kind="stable") + for column in arrays: + normalized = "area" if column == "oa_code" else column + np.testing.assert_array_equal( + support.arrays[normalized], arrays[column][order] + ) + assert dict(support.metadata["columns"][normalized]) == metadata[column] + np.testing.assert_array_equal(arrays[column], original[column]) + assert support.arrays["population"].dtype == np.dtype("int64") + assert support.arrays["households"].dtype == np.dtype("int64") + assert support.arrays["area"].flags.writeable is False + assert support.sha256 == geo.decode_atomic_support(data).sha256 + + +def test_normalized_bytes_are_stable_under_input_row_order_and_integer_count_storage(): + arrays, metadata = parts(SYSTEMS[0]) + expected = assemble_uk_atomic_area_support( + system=SYSTEMS[0], arrays=arrays, column_metadata=metadata + ) + reversed_arrays = {k: v[::-1].copy() for k, v in arrays.items()} + for column in ("population", "households"): + reversed_arrays[column] = reversed_arrays[column].astype(np.int64) + assert ( + assemble_uk_atomic_area_support( + system=SYSTEMS[0], arrays=reversed_arrays, column_metadata=metadata + ) + == expected + ) + + +@pytest.mark.parametrize( + "defect", + [ + "nonintegral", + "float32", + "bool", + "negative", + "nan", + "overflow_total", + "zero_population", + "zero_region_households", + "duplicate_area", + "length", + "missing_column", + "object_code", + "foreign_nation", + "cross_ew_nation", + "blank_code", + "missing_region", + "bad_atomic_vintage", + "missing_source", + ], +) +def test_converter_refuses_bad_counts_codes_coverage_and_metadata(defect): + arrays, metadata = parts(SYSTEMS[0]) + if defect == "nonintegral": + arrays["households"][0] = 0.5 + elif defect == "float32": + arrays["households"] = arrays["households"].astype(np.float32) + elif defect == "bool": + arrays["households"] = arrays["households"] > 0 + elif defect == "negative": + arrays["households"][0] = -1 + elif defect == "nan": + arrays["population"][0] = np.nan + elif defect == "overflow_total": + arrays["population"][0] = float(2**53) + elif defect == "zero_population": + arrays["population"][0] = 0 + elif defect == "zero_region_households": + arrays["households"][:3] = 0 + elif defect == "duplicate_area": + arrays["oa_code"][1] = arrays["oa_code"][0] + elif defect == "length": + arrays["msoa_code"] = arrays["msoa_code"][:-1] + elif defect == "missing_column": + del arrays["population"] + elif defect == "object_code": + arrays["oa_code"] = arrays["oa_code"].astype(object) + elif defect == "foreign_nation": + arrays["ward_code"][0] = "S05000001" + elif defect == "cross_ew_nation": + arrays["ward_code"][0] = "W05000001" + elif defect == "blank_code": + arrays["ward_code"][0] = "" + elif defect == "missing_region": + arrays = {k: v[3:].copy() for k, v in arrays.items()} + elif defect == "bad_atomic_vintage": + metadata["oa_code"]["vintage"] = "2011_census" + elif defect == "missing_source": + metadata["population"]["source"] = "" + with pytest.raises(ValueError, match="UK atomic support:"): + assemble_uk_atomic_area_support( + system=SYSTEMS[0], arrays=arrays, column_metadata=metadata + ) + + +@pytest.mark.parametrize("defect", ["inferred_ni_constituency", "ni_lsoa_not_dz"]) +def test_ni_source_convention_refusals(defect): + arrays, metadata = parts(SYSTEMS[2]) + if defect == "inferred_ni_constituency": + metadata["constituency_code"]["relation"] = "inferred_modal" + else: + arrays["lsoa_code"][0] = "N01000001" + with pytest.raises(ValueError, match="UK atomic support: NI"): + assemble_uk_atomic_area_support( + system=SYSTEMS[2], arrays=arrays, column_metadata=metadata + ) + + +def test_three_system_assignment_keeps_observed_region_and_native_alias_roles(): + households = pd.DataFrame( + { + "household_id": np.arange(1, 5, dtype=np.int64), + "region": pd.array( + ["LONDON", "WALES", "SCOTLAND", "NORTHERN_IRELAND"], dtype="string" + ), + IDENTITY_COLUMN: pd.array( + [key(i) for i in (101, 102, 103, 104)], dtype="string" + ), + } + ) + result, spec, supports = complete(households, payloads()) + pd.testing.assert_series_equal(result["region"], households["region"]) + assert result["atomic_area_system"].tolist() == [ + SYSTEMS[0], + SYSTEMS[0], + SYSTEMS[1], + SYSTEMS[2], + ] + assert result["atomic_area_basis"].tolist() == ["assigned"] * 4 + assert result.loc[:2, "output_area_code"].equals(result.loc[:2, "atomic_area_code"]) + assert pd.isna(result.loc[3, "output_area_code"]) + assert result.loc[2, "data_zone_code"] == result.loc[2, "lsoa_code"] + assert result.loc[2, "intermediate_zone_code"] == result.loc[2, "msoa_code"] + assert result.loc[3, "data_zone_code"] == result.loc[3, "atomic_area_code"] + assert result.loc[3, "super_data_zone_code"] == result.loc[3, "msoa_code"] + assert result.loc[3, "district_electoral_area_code"] == result.loc[3, "ward_code"] + assert pd.isna(result.loc[0, "super_data_zone_code"]) + assert geo.validate_geography(result, spec, supports)["outcome"] == "pass" + assert {s["source"] for s in spec["systems"]} == set(SOURCES.values()) + + +def test_source_metadata_changes_are_identified_and_declaration_refuses_forged_derived_field(): + payload = payloads() + original = payload[SYSTEMS[0]] + arrays, metadata = parts(SYSTEMS[0]) + metadata["population"]["source"] += "-revision-2" + changed = assemble_uk_atomic_area_support( + system=SYSTEMS[0], arrays=arrays, column_metadata=metadata + ) + assert changed != original + support = geo.decode_atomic_support(original) + forged_arrays = {k: v.copy() for k, v in support.arrays.items()} + forged_arrays["itl1_code"][0] = "TLN" + forged = geo.encode_atomic_support( + {k: dict(v) if k == "columns" else v for k, v in support.metadata.items()}, + forged_arrays, + ) + payload[SYSTEMS[0]] = forged + with pytest.raises(ValueError, match="canonical UK adapter output"): + uk_atomic_assignment_definition(payload, seed=43) + + +@pytest.mark.parametrize("defect", ["missing_system", "wrong_system", "bad_seed"]) +def test_declaration_refuses_mismatched_inputs(defect): + payload = payloads() + options = {"seed": 43} + if defect == "missing_system": + del payload[SYSTEMS[2]] + elif defect == "wrong_system": + payload[SYSTEMS[2]] = payload[SYSTEMS[1]] + elif defect == "bad_seed": + options["seed"] = True + with pytest.raises(ValueError): + uk_atomic_assignment_definition(payload, **options) + + +def test_declaration_cannot_substitute_an_income_or_counter_identity_column(): + with pytest.raises(TypeError, match="identity_column"): + uk_atomic_assignment_definition( + payloads(), seed=43, identity_column="household_weight" + ) + + +def test_distinct_post_clone_keys_assign_then_remain_stable_under_order_subset_and_growth(): + paths = [ + (("spi_support_channel", 0), ("cgt_incidence_clone", 0)), + (("spi_support_channel", 1), ("cgt_incidence_clone", 0)), + (("spi_support_channel", 1), ("cgt_incidence_clone", 1)), + ( + ("spi_support_channel", 1), + ("cgt_incidence_clone", 1), + ("cgt_band_donors", 1), + ), + ] + original = pd.DataFrame( + { + "household_id": np.asarray([11, 22, 33, 44], dtype=np.int64), + "region": pd.array(["LONDON"] * 4, dtype="string"), + IDENTITY_COLUMN: pd.array( + [key(101, path) for path in paths], dtype="string" + ), + } + ) + assert original[IDENTITY_COLUMN].nunique() == 4 + payload = payloads() + expected, spec, supports = complete(original, payload) + changed = original.iloc[[3, 1, 0]].copy() + changed["household_weight"] = [0.0, 2.0, 8.0] + extra = pd.DataFrame( + { + "household_id": [55], + "region": pd.array(["LONDON"], dtype="string"), + IDENTITY_COLUMN: pd.array( + [key(202, (("geographic_support", 1),))], dtype="string" + ), + "household_weight": [1.0], + }, + index=[8], + ) + actual, _, _ = complete(pd.concat([changed, extra]), payload) + for column in expected: + if column != "region": + pd.testing.assert_series_equal( + actual.loc[changed.index, column], expected.loc[changed.index, column] + ) + assert geo.validate_geography(actual, spec, supports)["outcome"] == "pass" + assigned_columns = ["atomic_area_code", "oa_code", "constituency_code"] + pruned = expected.iloc[[3, 1]].copy() + assert geo.validate_geography(pruned, spec, supports)["outcome"] == "pass" + pd.testing.assert_frame_equal( + pruned[assigned_columns], expected.loc[pruned.index, assigned_columns] + ) + with pytest.raises(ValueError, match="assignment output already exists"): + geo.assign_atomic(expected, spec, supports) + + +def test_country_declaration_retains_the_two_stage_sampling_law(): + households = pd.DataFrame( + { + "household_id": np.arange(1, 2001, dtype=np.int64), + "region": pd.array(["LONDON"] * 2000, dtype="string"), + IDENTITY_COLUMN: pd.array([key(i) for i in range(1, 2001)], dtype="string"), + } + ) + result, spec, _ = complete(households, payloads()) + assert all( + s["stages"] + == [ + {"level": "constituency_code", "weight": "households"}, + {"level": "area", "weight": "population"}, + ] + for s in spec["systems"] + ) + arrays, _ = parts(SYSTEMS[0]) + london = arrays["region_code"] == FRS_REGION_TO_REGION_CODE["LONDON"] + areas = arrays["oa_code"][london] + shares = result["atomic_area_code"].value_counts(normalize=True) + # Constituent household mass 30:10, then within-first-area population 2:6; + # the third area's population 1000 must not dominate the first-stage draw. + for area, expected in zip(areas, (0.1875, 0.5625, 0.25), strict=True): + assert abs(shares.get(area, 0) - expected) < 0.04 + + +def test_exact_source_ids_and_ordered_branches_are_preserved(): + first = key(2**53 + 1, (("spi_support_channel", 1), ("cgt_incidence_clone", 0))) + second = key(2**53 + 2, (("spi_support_channel", 1), ("cgt_incidence_clone", 0))) + assert first != second + assert str(2**53 + 1) in first + assert first == key( + np.int64(2**53 + 1), + (("spi_support_channel", np.int64(1)), ("cgt_incidence_clone", 0)), + ) + grown = [key(101, (("geographic_support", i),)) for i in range(6)] + assert grown[:3] == [key(101, (("geographic_support", i),)) for i in range(3)] + + +@pytest.mark.parametrize( + "defect", + [ + "float_id", + "bool_id", + "zero_id", + "oversize_id", + "no_vintage", + "mutable_path", + "counter_branch", + "income_branch", + "max_id_branch", + "out_of_order", + "duplicate_branch", + "float_ordinal", + "negative_ordinal", + "oversize_ordinal", + ], +) +def test_identity_refuses_inferred_or_inexact_conventions(defect): + args = { + "source": "invented-frs", + "source_vintage": "invented-2024-25", + "source_household_id": 101, + "clone_path": (), + } + if defect == "float_id": + args["source_household_id"] = 101.0 + elif defect == "bool_id": + args["source_household_id"] = True + elif defect == "zero_id": + args["source_household_id"] = 0 + elif defect == "oversize_id": + args["source_household_id"] = 2**63 + elif defect == "no_vintage": + args["source_vintage"] = "" + elif defect == "mutable_path": + args["clone_path"] = [] + elif defect in {"counter_branch", "income_branch", "max_id_branch"}: + args["clone_path"] = ( + ( + { + "counter_branch": "row_counter", + "income_branch": "income", + "max_id_branch": "sample_max_id", + }[defect], + 1, + ), + ) + elif defect == "out_of_order": + args["clone_path"] = (("cgt_incidence_clone", 1), ("spi_support_channel", 1)) + elif defect == "duplicate_branch": + args["clone_path"] = (("spi_support_channel", 0), ("spi_support_channel", 1)) + else: + ordinal = { + "float_ordinal": 1.5, + "negative_ordinal": -1, + "oversize_ordinal": 2**32, + }[defect] + args["clone_path"] = (("spi_support_channel", ordinal),) + with pytest.raises(ValueError, match="UK geography identity:"): + household_draw_key(**args) + + +def test_duplicate_or_unknown_household_identity_is_refused_by_shared_assignment(): + payload = payloads() + spec = uk_atomic_assignment_definition(payload, seed=43) + supports = {s: geo.decode_atomic_support(p) for s, p in payload.items()} + households = pd.DataFrame( + { + "household_id": [11, 22], + "region": ["LONDON", "LONDON"], + IDENTITY_COLUMN: [key(101), key(101)], + } + ) + with pytest.raises(ValueError, match="duplicate draw identity"): + geo.assign_atomic(households, spec, supports) + households.loc[1, IDENTITY_COLUMN] = key(102) + households.loc[1, "region"] = "UNKNOWN" + with pytest.raises(ValueError, match="exactly one"): + geo.assign_atomic(households, spec, supports) diff --git a/packages/microcosm-build/tests/test_uk_runtime_lazy_exports.py b/packages/microcosm-build/tests/test_uk_runtime_lazy_exports.py new file mode 100644 index 000000000..90f57fdf8 --- /dev/null +++ b/packages/microcosm-build/tests/test_uk_runtime_lazy_exports.py @@ -0,0 +1,138 @@ +"""Normal-import contracts for the real UK runtime public package. + +A fresh no-country-resource guard supplies the cold-import check. In a larger +suite, already imported country modules are allowed; these tests never remove, +stub or replace modules to manufacture an isolated import environment. +""" + +import hashlib +import importlib +import json +import sys + +import pytest + +_PACKAGE = "microcosm.build.uk_runtime" +_UNRELATED_MODULES = ( + f"{_PACKAGE}.calibration_run", + f"{_PACKAGE}.national_calibration", + f"{_PACKAGE}.firm_generation", +) +# SHA256 of compact JSON for all 456 ordered entries from ec3b306f3f9742ff... +# This includes the existing duplicate ladder_vs_chronicle_household_dispersion. +_ORDERED_ALL_SHA256 = "282baaa8103b6a68ec916669dc9b96909d6faf0a9beecdbb8359df12f128f66c" + + +def _unrelated_modules() -> set[str]: + return { + name + for name in sys.modules + if any( + name == prefix or name.startswith(prefix + ".") + for prefix in _UNRELATED_MODULES + ) + } + + +def test_normal_package_and_pure_helper_imports_do_not_load_unrelated_modules(): + before = _unrelated_modules() + package = importlib.import_module(_PACKAGE) + helper = importlib.import_module(f"{_PACKAGE}.rowwise_geography") + + assert sys.modules[_PACKAGE] is package + assert package.rowwise_geography is helper + assert callable(helper.assign_household_geography) + assert _unrelated_modules() == before + + +def test_public_from_import_returns_and_caches_the_real_defining_object(): + from microcosm.build.uk_runtime import assign_household_geography + from microcosm.build.uk_runtime.rowwise_geography import ( + assign_household_geography as direct, + ) + + package = importlib.import_module(_PACKAGE) + assert assign_household_geography is direct + assert package.assign_household_geography is direct + assert vars(package)["assign_household_geography"] is direct + assert package.assign_household_geography is assign_household_geography + + +def test_direct_submodule_import_and_from_import_preserve_module_identity(): + # In the fresh guard this exercises Python's fallback before any direct + # content_identity import, without removing an already loaded module in CI. + from microcosm.build.uk_runtime import content_identity + + assert content_identity is sys.modules[f"{_PACKAGE}.content_identity"] + + import microcosm.build.uk_runtime.content_identity as content_direct + import microcosm.build.uk_runtime.rowwise_geography as direct + from microcosm.build.uk_runtime import rowwise_geography + + package = importlib.import_module(_PACKAGE) + assert content_identity is content_direct + assert content_direct is sys.modules[f"{_PACKAGE}.content_identity"] + assert package.content_identity is content_direct + assert "content_identity" not in package.__all__ + assert rowwise_geography is direct + assert direct is sys.modules[f"{_PACKAGE}.rowwise_geography"] + assert package.rowwise_geography is direct + assert "rowwise_geography" not in package.__all__ + + +def test_discovery_preserves_the_frozen_ordered_export_contract_without_resolution(): + package = importlib.import_module(_PACKAGE) + imports_before = _unrelated_modules() + aliases_before = { + name: vars(package)[name] for name in package.__all__ if name in vars(package) + } + + discovered = dir(package) + + assert len(package.__all__) == 456 + assert len(set(package.__all__)) == 455 + assert package.__all__.count("ladder_vs_chronicle_household_dispersion") == 2 + encoded = json.dumps(package.__all__, separators=(",", ":")).encode() + assert hashlib.sha256(encoded).hexdigest() == _ORDERED_ALL_SHA256 + assert set(package.__all__).issubset(discovered) + assert discovered == sorted(set(discovered)) + aliases_after = { + name: vars(package)[name] for name in package.__all__ if name in vars(package) + } + assert aliases_after.keys() == aliases_before.keys() + assert all(aliases_after[name] is value for name, value in aliases_before.items()) + assert _unrelated_modules() == imports_before + with pytest.raises(TypeError): + package._EXPORTS["invented_missing_export"] = ("invented", "missing") + + +def test_unknown_attribute_and_unknown_from_import_keep_normal_failures(): + package = importlib.import_module(_PACKAGE) + with pytest.raises( + AttributeError, match="has no attribute 'invented_missing_export'" + ): + _ = package.invented_missing_export + with pytest.raises( + ImportError, match="cannot import name 'invented_missing_export'" + ): + from microcosm.build.uk_runtime import invented_missing_export # noqa: F401 + + assert "invented_missing_export" not in vars(package) + + +def test_reload_clears_public_aliases_and_rebinds_the_same_defining_objects(): + package = importlib.import_module(_PACKAGE) + helper = importlib.import_module(f"{_PACKAGE}.rowwise_geography") + real_function = helper.assign_household_geography + assert package.assign_household_geography is real_function + assert "assign_household_geography" in vars(package) + imports_before = _unrelated_modules() + + assert importlib.reload(package) is package + + assert "assign_household_geography" not in vars(package) + assert package.rowwise_geography is helper + assert sys.modules[f"{_PACKAGE}.rowwise_geography"] is helper + assert package.assign_household_geography is real_function + assert vars(package)["assign_household_geography"] is real_function + assert _unrelated_modules() == imports_before diff --git a/packages/microcosm-build/tests/test_us_acs_housing_source.py b/packages/microcosm-build/tests/test_us_acs_housing_source.py new file mode 100644 index 000000000..eb5a461c9 --- /dev/null +++ b/packages/microcosm-build/tests/test_us_acs_housing_source.py @@ -0,0 +1,2634 @@ +"""Independent invented-source tests for the approved ACS housing-unit source bridge. + +Written against the approved v2 plan contract, not against the production +module: no byte of ``acs_housing_universe_source.py`` or of the packaged +``acs_2024_housing_universe.json`` was read while authoring this file, so a +failure here is evidence about the contract rather than a restatement of the +implementation. + +Every archive, member, header, identifier and row below is wholly invented. No +genuine ACS/Census archive, candidate, target or dictionary file is opened, +statted or referenced. Nothing here authenticates real data, and nothing here +makes a scientific claim: the fixtures exercise the closed source rules only. + +Scope is the source producer/readback pair. ``prepare_acs_housing_population`` +builds a Frame and is deliberately left to the production author's own +Frame/graph tests. + +Two reconciliation seams are collected in one place each, so integrating this +file with the finished module is a rename exercise and never a semantic one: + +* ``NAMES`` - column, constant, filename and definition-name spellings. +* ``EXPECTED_REASONS`` - exact refusal reason codes, asserted only once + ``REASON_CODES_AGREED`` is flipped to True after agreeing them with the + production author. Until then every refusal test still asserts the exception + type, message sanitisation, and that semantically distinct causes carry + distinct codes, which is the part that does not need agreement. +""" + +import csv +import hashlib +import io +import json +import os +import re +import signal +import stat +import struct +import tempfile +import zipfile +from contextlib import contextmanager +from dataclasses import dataclass, field +from pathlib import Path +from types import SimpleNamespace + +import numpy as np +import pandas as pd +import pytest + +# The production import must surface as a loud collection error naming the +# module while it is absent. It must never skip: no engine is involved here. +from microcosm.build.us_runtime import acs_housing_universe_source as source +from microcosm.build.us_runtime.acs_housing_universe import ( + classify_acs_housing_universe, +) + +# -------------------------------------------------------------------------- +# Reconciliation seam 1: identifier spellings. +# -------------------------------------------------------------------------- + +NAMES = { + # Frame columns carrying the archive-member/row lineage of each row. + "member_column": "source_member", + "ordinal_column": "source_row_ordinal", + # The normalised two-digit state column exposed on the household frame. + # ``_state_series`` falls back to the raw ST/STATE header aliases, so only + # the dedicated naming test depends on this spelling. + "state_column": "ST", + # Files written into the source artifact directory. + "projection_filename": "projection.json", + "receipt_filename": "receipt.json", + # Named source definitions bound by the artifact receipt. + "vacancy_definition": "acs_2024_source_vacancy_v1", + "gq_definition": "acs_2024_gq_person_placeholder_design_v1", + # Module byte caps, monkeypatched tiny so no test allocates real GiBs. + "source_max_bytes": "ACS_HU_SOURCE_MAX_BYTES", + "receipt_max_bytes": "ACS_HU_RECEIPT_MAX_BYTES", + # The private archive pin tuple; entries are (role, filename, sha256, size). + "archive_pins": "_ARCHIVE_PINS", + "household_role": "household", + "person_role": "person", +} + +HOUSEHOLD_ARCHIVE = "csv_hus.zip" +PERSON_ARCHIVE = "csv_pus.zip" + +# -------------------------------------------------------------------------- +# Reconciliation seam 2: refusal reason codes. +# -------------------------------------------------------------------------- + +REASON_CODES_AGREED = True + +EXPECTED_REASONS: dict[str, str] = { + "artifact_forgery": "RECONSTRUCTION_MISMATCH", + "capture_digest": "SOURCE_SHA256", + "csv_record": "CSV_ROW_WIDTH", + "header_contract": "CSV_STATE_REQUIRED", + "join_integrity": "ORPHAN_PERSON", + "output_precondition": "OUTPUT_EXISTS", + "selection": "SELECTION_UNKNOWN", + "value_domain": "TEN", + "zip_preflight": "ZIP_DUPLICATE_MEMBER", +} + +# Coarse refusal families that must not collapse onto one reason code. +DISTINCT_FAMILIES = ( + "capture_digest", + "zip_preflight", + "csv_record", + "header_contract", + "value_domain", + "join_integrity", + "selection", + "artifact_forgery", + "output_precondition", +) + +# -------------------------------------------------------------------------- +# Invented source bytes. +# -------------------------------------------------------------------------- + +HOUSEHOLD_HEADER = ( + "RT", + "SERIALNO", + "DIVISION", + "PUMA", + "REGION", + "ST", + "ADJHSG", + "ADJINC", + "WGTP", + "NP", + "TYPEHUGQ", + "ACR", + "BDSP", + "TEN", + "VALP", + "HINCP", +) + +PERSON_HEADER = ( + "RT", + "SERIALNO", + "SPORDER", + "PUMA", + "ST", + "ADJINC", + "PWGTP", + "AGEP", + "SEX", + "WAGP", +) + +HOUSEHOLD_DEFAULTS = { + "RT": "H", + "DIVISION": "9", + "PUMA": "00100", + "REGION": "4", + "ST": "06", + "ADJHSG": "1000000", + "ADJINC": "1010000", + "ACR": "1", + "BDSP": "3", + "VALP": "410000", + "HINCP": "82000", +} + +PERSON_DEFAULTS = { + "RT": "P", + "PUMA": "00100", + "ST": "06", + "ADJINC": "1010000", + "AGEP": "41", + "SEX": "1", + "WAGP": "52000", +} + +# Projected observation rosters, per the closed plan. +HOUSEHOLD_ROSTER = ("SERIALNO", "TYPEHUGQ", "NP", "TEN", "WGTP", "PUMA") +PERSON_ROSTER = ("SERIALNO", "SPORDER", "PWGTP") + +# Fixed classifier output order, read from the pinned pure module. +CODE_COLUMNS = ( + "interview_scope", + "physical_unit", + "household_kind", + "tenure_subtype", + "occupied_hu", + "hu_tenure_class", + "unresolved_reasons", + "TEN_valid", +) + + +def household_row(serialno, typehugq, persons, ten, wgtp, **overrides): + """One invented household record as an ordered tuple of lexical tokens.""" + values = dict(HOUSEHOLD_DEFAULTS) + values.update(SERIALNO=serialno, TYPEHUGQ=typehugq, NP=persons, TEN=ten, WGTP=wgtp) + values.update(overrides) + return tuple(values[name] for name in HOUSEHOLD_HEADER) + + +def person_row(serialno, sporder, pwgtp, **overrides): + """One invented person record as an ordered tuple of lexical tokens.""" + values = dict(PERSON_DEFAULTS) + values.update(SERIALNO=serialno, SPORDER=sporder, PWGTP=pwgtp) + values.update(overrides) + return tuple(values[name] for name in PERSON_HEADER) + + +# Eight invented households covering every accepted source class exactly once, +# plus both WGTP boundaries and the whole observed TEN domain. +BASE_HOUSEHOLDS = ( + # occupied owner with a mortgage, two people + household_row("2024HU0000001", "1", "2", "1", "118", PUMA="00100", ST="06"), + # occupied renter paying rent, one person + household_row("2024HU0000002", "1", "1", "3", "76", PUMA="03701", ST="06"), + # occupied housing unit whose tenure is not available, three people + household_row("2024HU0000003", "1", "3", "", "1", PUMA="81003", ST="36"), + # vacant housing unit: no people, blank tenure, positive weight + household_row("2024HU0000004", "1", "0", "", "55", PUMA="00100", ST="11"), + # institutional group quarters placeholder + household_row("2024GQ0000005", "2", "1", "", "0", PUMA="03701", ST="36"), + # noninstitutional group quarters placeholder + household_row("2024GQ0000006", "3", "1", "", "0", PUMA="00100", ST="01"), + # occupied owner outright + household_row("2024HU0000007", "1", "1", "2", "200", PUMA="00100", ST="01"), + # occupied renter paying no rent, maximum housing weight + household_row("2024HU0000008", "1", "1", "4", "9999", PUMA="81003", ST="11"), +) + +BASE_PERSONS = ( + person_row("2024HU0000001", "01", "121", WAGP="52000"), + person_row("2024HU0000001", "02", "97", WAGP="0"), + person_row("2024HU0000002", "1", "80", WAGP="31000"), + person_row("2024HU0000003", "1", "12", WAGP="17000"), + person_row("2024HU0000003", "2", "11", WAGP="0"), + person_row("2024HU0000003", "3", "9", WAGP="0"), + person_row("2024GQ0000005", "01", "64", WAGP="0"), + person_row("2024GQ0000006", "1", "9999", WAGP="7000"), + person_row("2024HU0000007", "01", "205", WAGP="99000"), + person_row("2024HU0000008", "1", "1", WAGP="4000"), +) + +# The base population is split across two members per role so that member and +# ordinal lineage is exercised by every ordinary test rather than by one. +BASE_HOUSEHOLD_MEMBERS = ( + ("psam_husa.csv", BASE_HOUSEHOLDS[:5]), + ("psam_husb.csv", BASE_HOUSEHOLDS[5:]), +) +BASE_PERSON_MEMBERS = ( + ("psam_pusa.csv", BASE_PERSONS[:6]), + ("psam_pusb.csv", BASE_PERSONS[6:]), +) + +BASE_SERIALNOS = tuple( + row[HOUSEHOLD_HEADER.index("SERIALNO")] for row in BASE_HOUSEHOLDS +) + +# The plan fixes what each row contains, never the order rows are emitted in. +# Content assertions below are therefore keyed by identifier, and the ordering +# decision lives in exactly one place: this constant and the two tests that +# read it. "sorted_serialno" is the order the production module emits; +# "source" would be member-then-row-ordinal order. See the author report. +CANONICAL_ROW_ORDER = "sorted_serialno" + + +def expected_row_order(serialnos): + if CANONICAL_ROW_ORDER == "sorted_serialno": + return sorted(serialnos) + if CANONICAL_ROW_ORDER == "source": + return list(serialnos) + raise AssertionError(f"unknown row-order contract {CANONICAL_ROW_ORDER!r}") + + +# (occupied_hu, hu_tenure_class, tenure_subtype, unresolved_reasons, +# physical_unit, TEN_valid) recomputed by hand from the pinned classifier. +BASE_EXPECTED_CODES = { + "interview_scope": [0, 0, 0, 0, 0, 0, 0, 0], + "physical_unit": [1, 1, 1, 1, 2, 2, 1, 1], + "household_kind": [0, 0, 0, 0, 0, 0, 0, 0], + "tenure_subtype": [1, 3, 0, 0, 0, 0, 2, 4], + "occupied_hu": [1, 1, 1, 0, 2, 2, 1, 1], + "hu_tenure_class": [1, 2, 3, 0, 0, 0, 1, 2], + "unresolved_reasons": [0, 0, 2, 1, 0, 0, 0, 0], + "TEN_valid": [1, 1, 0, 0, 0, 0, 1, 1], +} + + +# -------------------------------------------------------------------------- +# Building invented archives. +# -------------------------------------------------------------------------- + + +EXPECTED_HOUSEHOLDS = { + row[HOUSEHOLD_HEADER.index("SERIALNO")]: { + name: row[HOUSEHOLD_HEADER.index(name)] for name in HOUSEHOLD_ROSTER + ("ST",) + } + for row in BASE_HOUSEHOLDS +} + +EXPECTED_PERSONS = { + ( + row[PERSON_HEADER.index("SERIALNO")], + row[PERSON_HEADER.index("SPORDER")], + ): row[PERSON_HEADER.index("PWGTP")] + for row in BASE_PERSONS +} + +EXPECTED_CODES = { + serialno: tuple(BASE_EXPECTED_CODES[name][index] for name in CODE_COLUMNS) + for index, serialno in enumerate(BASE_SERIALNOS) +} + +EXPECTED_HOUSEHOLD_MEMBER = { + row[HOUSEHOLD_HEADER.index("SERIALNO")]: member + for member, rows in BASE_HOUSEHOLD_MEMBERS + for row in rows +} + +EXPECTED_PERSON_MEMBER = { + ( + row[PERSON_HEADER.index("SERIALNO")], + row[PERSON_HEADER.index("SPORDER")], + ): member + for member, rows in BASE_PERSON_MEMBERS + for row in rows +} + + +def csv_bytes(header, rows, *, newline="\n", trailing_newline=True): + """Serialise invented tokens without any quoting the fixtures do not ask for.""" + buffer = io.StringIO(newline="") + writer = csv.writer(buffer, lineterminator=newline, quoting=csv.QUOTE_MINIMAL) + writer.writerow(header) + writer.writerows(rows) + text = buffer.getvalue() + if not trailing_newline and text.endswith(newline): + text = text[: -len(newline)] + return text.encode("utf-8") + + +@dataclass(frozen=True) +class Member: + """One planned archive entry, data or auxiliary.""" + + name: str + data: bytes + compress_type: int = zipfile.ZIP_DEFLATED + external_attr: int | None = None + create_system: int = 3 + + +def data_members(header, members, *, newline="\n", **kwargs): + return tuple( + Member(name, csv_bytes(header, rows, newline=newline, **kwargs)) + for name, rows in members + ) + + +def write_archive(path, members): + """Write invented members exactly as planned, preserving order and duplicates.""" + with zipfile.ZipFile(path, "w") as archive: + for member in members: + info = zipfile.ZipInfo(member.name, date_time=(2026, 1, 1, 0, 0, 0)) + info.compress_type = member.compress_type + info.create_system = member.create_system + info.external_attr = ( + member.external_attr + if member.external_attr is not None + else (0o644 << 16) + ) + archive.writestr(info, member.data) + return path + + +def flip_archive_byte(path, offset=None): + """Change one compressed byte in place: same size, different digest. + + A deflated archive holds no plaintext, so a token replacement here would + silently do nothing and the test would stop testing what it claims. + """ + payload = Path(path).read_bytes() + index = len(payload) // 2 if offset is None else offset + mutated = payload[:index] + bytes([payload[index] ^ 0xFF]) + payload[index + 1 :] + assert len(mutated) == len(payload) and mutated != payload + Path(path).write_bytes(mutated) + return payload + + +def digest_and_size(path): + payload = Path(path).read_bytes() + return hashlib.sha256(payload).hexdigest(), len(payload) + + +@dataclass +class Fixture: + """An invented source directory plus everything a test needs to drive it.""" + + tmp_path: Path + source_dir: Path + snapshot_root: Path + monkeypatch: pytest.MonkeyPatch + households: tuple = () + persons: tuple = () + secrets: list = field(default_factory=list) + _output_index: int = 0 + + @property + def household_zip(self): + return self.source_dir / HOUSEHOLD_ARCHIVE + + @property + def person_zip(self): + return self.source_dir / PERSON_ARCHIVE + + def pin(self): + """Point the private pin tuple at the invented archives as they are now.""" + pins = [] + for role, name in ( + (NAMES["household_role"], HOUSEHOLD_ARCHIVE), + (NAMES["person_role"], PERSON_ARCHIVE), + ): + path = self.source_dir / name + sha, size = digest_and_size(path) + pins.append((role, name, sha, size)) + self.monkeypatch.setattr(source, NAMES["archive_pins"], tuple(pins)) + return tuple(pins) + + def output_dir(self, label="artifact"): + self._output_index += 1 + return self.tmp_path / f"out-{self._output_index}-{label}" + + def produce(self, *, output_dir=None, serialnos=None, source_dir=None): + return source.produce_acs_housing_source( + self.source_dir if source_dir is None else source_dir, + snapshot_root=self.snapshot_root, + output_dir=self.output_dir() if output_dir is None else output_dir, + serialnos=serialnos, + ) + + def load(self, output_dir, *, serialnos=None, source_dir=None): + return source.load_acs_housing_source( + self.source_dir if source_dir is None else source_dir, + output_dir, + snapshot_root=self.snapshot_root, + serialnos=serialnos, + ) + + def produce_and_load(self, *, serialnos=None): + output_dir = self.output_dir() + produced = self.produce(output_dir=output_dir, serialnos=serialnos) + return produced, self.load(output_dir, serialnos=serialnos), output_dir + + +def build_fixture( + tmp_path, + monkeypatch, + *, + households=BASE_HOUSEHOLD_MEMBERS, + persons=BASE_PERSON_MEMBERS, + household_members=None, + person_members=None, + household_header=HOUSEHOLD_HEADER, + person_header=PERSON_HEADER, + newline="\n", + pin=True, +): + """Assemble an invented source directory; explicit members override the rows.""" + # Resolve first: on macOS pytest's tmp_path sits under /var, a symlink to + # /private/var, and the capture refuses symlinked path components. The + # deliberate symlink refusals below build their own links explicitly. + tmp_path = Path(tmp_path).resolve() + source_dir = Path(tmp_path) / "acs-source" + source_dir.mkdir(parents=True, exist_ok=True) + snapshot_root = Path(tmp_path) / "snapshots" + snapshot_root.mkdir(parents=True, exist_ok=True) + + if household_members is None: + household_members = data_members(household_header, households, newline=newline) + if person_members is None: + person_members = data_members(person_header, persons, newline=newline) + + write_archive(source_dir / HOUSEHOLD_ARCHIVE, household_members) + write_archive(source_dir / PERSON_ARCHIVE, person_members) + + fixture = Fixture( + tmp_path=Path(tmp_path), + source_dir=source_dir, + snapshot_root=snapshot_root, + monkeypatch=monkeypatch, + households=households, + persons=persons, + ) + fixture.secrets = sanitisation_secrets(fixture) + if pin: + fixture.pin() + return fixture + + +def sanitisation_secrets(fixture): + """Strings a sanitised refusal must never echo back to the caller.""" + secrets = [ + str(fixture.tmp_path), + str(fixture.source_dir), + str(fixture.snapshot_root), + ] + secrets.extend(BASE_SERIALNOS) + # Unprojected invented money tokens: no legitimate reason code names these, + # unlike the domain bounds (1..9999, 100..81003) a code may legitimately cite. + secrets.extend(["82000", "52000", "410000", "31000", "99000", "17000"]) + return secrets + + +@pytest.fixture(autouse=True) +def _invented_disk_budget(monkeypatch): + """Eight invented rows must not require a full-source disk reserve in CI.""" + monkeypatch.setattr( + source.shutil, "disk_usage", lambda _path: SimpleNamespace(free=64 * 1024**3) + ) + + +@pytest.fixture +def acs(tmp_path, monkeypatch): + """The plain accepted invented population, pinned and ready to produce.""" + return build_fixture(tmp_path, monkeypatch) + + +# -------------------------------------------------------------------------- +# Refusal helpers. +# -------------------------------------------------------------------------- + + +def refuses(fixture, call, *args, **kwargs): + """Assert the exact refusal type, capture the reason, and check sanitisation.""" + with pytest.raises(source.ACSHousingSourceError) as caught: + call(*args, **kwargs) + reason = str(caught.value) + assert type(caught.value) is source.ACSHousingSourceError + assert reason, "a refusal must carry a reason" + assert len(reason) <= 120, "a reason code must not be prose" + assert "\n" not in reason + for secret in fixture.secrets: + assert secret not in reason, "refusal leaked invented source content" + return reason + + +def flat_scalars(payload): + """Every scalar reachable in a decoded receipt, for shape-tolerant assertions.""" + found = [] + stack = [payload] + while stack: + item = stack.pop() + if isinstance(item, dict): + stack.extend(item.keys()) + stack.extend(item.values()) + elif isinstance(item, (list, tuple)): + stack.extend(item) + else: + found.append(item) + return found + + +def flat_strings(payload): + return {item for item in flat_scalars(payload) if isinstance(item, str)} + + +def flat_numbers(payload): + return { + item + for item in flat_scalars(payload) + if isinstance(item, int) and not isinstance(item, bool) + } + + +def state_series(households): + """Read the state column under whichever spelling the module exposes.""" + for name in (NAMES["state_column"], "ST", "STATE"): + if name in households.columns: + return households[name] + raise AssertionError("household frame exposes no state column") + + +def household_keys(households): + keys = lexical(households["SERIALNO"]) + assert len(set(keys)) == len(keys), "household identifiers must be unique" + return keys + + +def household_map(households, names=HOUSEHOLD_ROSTER + ("ST",)): + """Every projected household token, keyed by identifier rather than by row.""" + keys = household_keys(households) + columns = { + name: lexical(state_series(households) if name == "ST" else households[name]) + for name in names + } + return { + key: {name: columns[name][row] for name in names} + for row, key in enumerate(keys) + } + + +def person_map(persons): + """Person weight keyed by identifier and the raw SPORDER token.""" + entries = list( + zip( + lexical(persons["SERIALNO"]), + lexical(persons["SPORDER"]), + lexical(persons["PWGTP"]), + strict=True, + ) + ) + keyed = {(serialno, sporder): weight for serialno, sporder, weight in entries} + assert len(keyed) == len(entries), "person keys must be unique" + return keyed + + +def codes_map(result): + """The eight native codes as a tuple per household identifier.""" + keys = household_keys(result.households) + codes = result.codes + return { + key: tuple(int(codes[name].iloc[row]) for name in CODE_COLUMNS) + for row, key in enumerate(keys) + } + + +def lexical(series): + """A projected column must be lexical text with no NA conversion.""" + values = series.tolist() + assert all(isinstance(value, str) for value in values), ( + f"{series.name} is not lexical: {[type(v).__name__ for v in values]}" + ) + return values + + +def strict_arrays(households): + """Rebuild the classifier inputs from the projected lexical household tokens.""" + tenure_tokens = lexical(households["TEN"]) + tenure_valid = np.asarray([token != "" for token in tenure_tokens], dtype=bool) + frame = pd.DataFrame( + { + "TYPEHUGQ": np.asarray( + [int(token) for token in lexical(households["TYPEHUGQ"])], dtype="int64" + ), + "NP": np.asarray( + [int(token) for token in lexical(households["NP"])], dtype="int64" + ), + "TEN": np.asarray( + [int(token) if token != "" else 0 for token in tenure_tokens], + dtype="int64", + ), + }, + index=households.index, + ) + return frame, tenure_valid + + +def assert_lineage(frame, keys, expected_member): + """Find member/ordinal lineage without depending on either column spelling. + + Order-independent by construction: the member column must agree row by row + with the invented layout, and each member's ordinals must form one + consecutive run, whatever order the rows are emitted in. + """ + members = [expected_member[key] for key in keys] + member_column = next( + ( + name + for name in frame.columns + if [value if isinstance(value, str) else None for value in frame[name]] + == members + ), + None, + ) + assert member_column is not None, ( + "no column carries the archive member of each row; expected " + f"{sorted(set(members))}" + ) + + for name in frame.columns: + if name == member_column: + continue + try: + values = [int(value) for value in frame[name].tolist()] + except (TypeError, ValueError): + continue + if len(values) != len(members): + continue + runs = {} + for member, value in zip(members, values, strict=True): + runs.setdefault(member, []).append(value) + bases = {min(run) for run in runs.values()} + if bases <= {0} or bases <= {1}: + if all( + sorted(run) == list(range(min(run), min(run) + len(run))) + for run in runs.values() + ): + return member_column, name + raise AssertionError( + "no column carries a within-member row ordinal restarting at each member" + ) + + +# -------------------------------------------------------------------------- +# Accepted population: projection, classification and roundtrip. +# -------------------------------------------------------------------------- + + +def test_projected_households_are_lexical_and_complete(acs): + result = acs.produce() + households = result.households + assert len(households) == len(BASE_HOUSEHOLDS) + assert households.columns.is_unique + assert set(HOUSEHOLD_ROSTER) <= set(households.columns) + for name in HOUSEHOLD_ROSTER: + lexical(households[name]) + # Every projected token survives byte for byte: blank tenure stays an empty + # string rather than NaN, None or the dictionary's display character, and + # leading zeros on PUMA and ST are preserved. + assert household_map(households) == EXPECTED_HOUSEHOLDS + + +def test_projected_persons_keep_raw_sporder_tokens(acs): + persons = acs.produce().persons + assert len(persons) == len(BASE_PERSONS) + assert set(PERSON_ROSTER) <= set(persons.columns) + # "01" and "1" are both accepted and both retained exactly as written. + assert person_map(persons) == EXPECTED_PERSONS + + +def test_native_codes_are_uint8_in_the_pinned_order(acs): + result = acs.produce() + codes = result.codes + assert list(codes.columns) == list(CODE_COLUMNS) + for name in CODE_COLUMNS: + assert codes[name].dtype == np.dtype("uint8") + assert codes.index.equals(result.households.index) + assert codes_map(result) == EXPECTED_CODES + + +def test_codes_equal_the_pinned_pure_classifier_on_the_strict_arrays(acs): + """The source result must not invent its own semantics for the same rows.""" + result = acs.produce() + frame, tenure_valid = strict_arrays(result.households) + expected, _header = classify_acs_housing_universe(frame, tenure_valid=tenure_valid) + pd.testing.assert_frame_equal(result.codes, expected, check_like=False) + + +def test_np0_housing_unit_stays_occupancy_unknown_in_the_native_codes(acs): + """Source vacancy evidence must not be back-written into the pure classes.""" + codes = dict( + zip(CODE_COLUMNS, codes_map(acs.produce())["2024HU0000004"], strict=True) + ) + assert codes["occupied_hu"] == 0 + assert codes["hu_tenure_class"] == 0 + assert codes["unresolved_reasons"] == 1 + assert codes["physical_unit"] == 1 + assert codes["TEN_valid"] == 0 + + +def test_occupied_unit_with_unavailable_tenure_is_retained_not_recoded(acs): + codes = dict( + zip(CODE_COLUMNS, codes_map(acs.produce())["2024HU0000003"], strict=True) + ) + assert codes["occupied_hu"] == 1 + assert codes["hu_tenure_class"] == 3 + assert codes["tenure_subtype"] == 0 + assert codes["unresolved_reasons"] == 2 + + +def test_member_and_row_ordinal_lineage_is_accessible(acs): + result = acs.produce() + assert_lineage( + result.households, + household_keys(result.households), + EXPECTED_HOUSEHOLD_MEMBER, + ) + assert_lineage( + result.persons, + list(person_map(result.persons)), + EXPECTED_PERSON_MEMBER, + ) + + +def test_rows_are_emitted_in_the_agreed_canonical_order(acs): + """The plan leaves row order open; this is the single place it is decided.""" + assert household_keys(acs.produce().households) == expected_row_order( + BASE_SERIALNOS + ) + + +def test_row_order_does_not_depend_on_the_member_split(tmp_path, monkeypatch): + """Splitting the same invented rows differently must not reorder them.""" + one = single_member(tmp_path / "one", monkeypatch).produce() + three = build_fixture( + tmp_path / "three", + monkeypatch, + household_members=( + Member("psam_husa.csv", csv_bytes(HOUSEHOLD_HEADER, BASE_HOUSEHOLDS[6:])), + Member("psam_husb.csv", csv_bytes(HOUSEHOLD_HEADER, BASE_HOUSEHOLDS[:3])), + Member("psam_husc.csv", csv_bytes(HOUSEHOLD_HEADER, BASE_HOUSEHOLDS[3:6])), + ), + person_members=(Member("psam_pusa.csv", BASE_PERSON_CSV),), + ).produce() + assert household_keys(three.households) == household_keys(one.households) + assert household_map(three.households) == household_map(one.households) + + +def test_source_artifact_holds_exactly_the_projection_and_receipt(acs): + output_dir = acs.output_dir() + result = acs.produce(output_dir=output_dir) + assert sorted(path.name for path in output_dir.iterdir()) == sorted( + (NAMES["projection_filename"], NAMES["receipt_filename"]) + ) + # The house convention appends a trailing newline to published JSON. + assert (output_dir / NAMES["projection_filename"]).read_bytes().rstrip( + b"\n" + ) == result.projection_json.rstrip(b"\n") + assert (output_dir / NAMES["receipt_filename"]).read_bytes().rstrip( + b"\n" + ) == result.receipt_json.rstrip(b"\n") + json.loads(result.projection_json) + json.loads(result.receipt_json) + + +def test_readback_returns_the_same_bytes_and_the_same_population(acs): + produced, loaded, _output_dir = acs.produce_and_load() + assert loaded.projection_json == produced.projection_json + assert loaded.receipt_json == produced.receipt_json + pd.testing.assert_frame_equal(loaded.households, produced.households) + pd.testing.assert_frame_equal(loaded.persons, produced.persons) + pd.testing.assert_frame_equal(loaded.codes, produced.codes) + + +def test_two_produces_from_identical_inputs_are_byte_identical(acs): + """Readback compares exact bytes, so both artifacts must be deterministic.""" + first = acs.produce(output_dir=acs.output_dir("first")) + second = acs.produce(output_dir=acs.output_dir("second")) + assert first.projection_json == second.projection_json + assert first.receipt_json == second.receipt_json + + +@pytest.mark.parametrize("attribute", ["households", "persons", "codes"]) +def test_returned_frames_are_owned_copies(acs, attribute): + result = acs.produce() + first = getattr(result, attribute) + assert getattr(result, attribute) is not first + before = first.copy(deep=True) + projection = result.projection_json + first.iloc[0, 0] = first.iloc[-1, 0] + first.rename(columns={first.columns[0]: "clobbered"}, inplace=True) + first.index = pd.RangeIndex(1000, 1000 + len(first)) + after = getattr(result, attribute) + pd.testing.assert_frame_equal(after, before) + assert result.projection_json == projection + + +def test_receipt_records_auditable_member_and_definition_evidence(acs): + result = acs.produce() + receipt = json.loads(result.receipt_json) + strings = flat_strings(receipt) + numbers = flat_numbers(receipt) + + assert NAMES["vacancy_definition"] in strings + assert NAMES["gq_definition"] in strings + assert {HOUSEHOLD_ARCHIVE, PERSON_ARCHIVE} <= strings + assert {"psam_husa.csv", "psam_husb.csv", "psam_pusa.csv", "psam_pusb.csv"} <= ( + strings + ) + + for archive, names in ( + (acs.household_zip, ("psam_husa.csv", "psam_husb.csv")), + (acs.person_zip, ("psam_pusa.csv", "psam_pusb.csv")), + ): + with zipfile.ZipFile(archive) as opened: + for name in names: + info = opened.getinfo(name) + payload = opened.read(name) + assert hashlib.sha256(payload).hexdigest() in strings, ( + f"receipt omits the decompressed digest of {name}" + ) + assert info.CRC in numbers, f"receipt omits the CRC of {name}" + assert info.file_size in numbers + + # Counts an auditor needs: the whole population and the vacancy evidence. + assert len(BASE_HOUSEHOLDS) in numbers + assert len(BASE_PERSONS) in numbers + + +def test_receipt_binds_the_pinned_archive_digests(acs): + pins = acs.pin() + strings = flat_strings(json.loads(acs.produce().receipt_json)) + for _role, name, sha, _size in pins: + assert sha in strings, f"receipt omits the pinned digest of {name}" + + +# -------------------------------------------------------------------------- +# Population variants: one deliberate change against the accepted invented rows. +# -------------------------------------------------------------------------- + + +def with_household(index, **fields): + rows = list(BASE_HOUSEHOLDS) + values = dict(zip(HOUSEHOLD_HEADER, rows[index], strict=True)) + values.update(fields) + rows[index] = tuple(values[name] for name in HOUSEHOLD_HEADER) + return tuple(rows) + + +def with_person(index, **fields): + rows = list(BASE_PERSONS) + values = dict(zip(PERSON_HEADER, rows[index], strict=True)) + values.update(fields) + rows[index] = tuple(values[name] for name in PERSON_HEADER) + return tuple(rows) + + +def rename_serialno(old, new): + """Rename a key on both sides so the only change is the identifier token.""" + key = HOUSEHOLD_HEADER.index("SERIALNO") + households = tuple( + row[:key] + (new,) + row[key + 1 :] if row[key] == old else row + for row in BASE_HOUSEHOLDS + ) + key = PERSON_HEADER.index("SERIALNO") + persons = tuple( + row[:key] + (new,) + row[key + 1 :] if row[key] == old else row + for row in BASE_PERSONS + ) + return households, persons + + +def single_member(tmp_path, monkeypatch, *, households=None, persons=None, **kwargs): + """One member per role, so a variant's only difference is the row content.""" + return build_fixture( + tmp_path, + monkeypatch, + households=( + ("psam_husa.csv", BASE_HOUSEHOLDS if households is None else households), + ), + persons=(("psam_pusa.csv", BASE_PERSONS if persons is None else persons),), + **kwargs, + ) + + +# -------------------------------------------------------------------------- +# Key selection. +# -------------------------------------------------------------------------- + + +def test_selection_none_retains_the_whole_invented_source(acs): + result = acs.produce(serialnos=None) + assert set(household_keys(result.households)) == set(BASE_SERIALNOS) + assert len(result.persons) == len(BASE_PERSONS) + + +def test_selection_keeps_whole_households_and_no_other_person(acs): + result = acs.produce(serialnos=("2024HU0000001", "2024GQ0000005")) + assert set(household_keys(result.households)) == { + "2024HU0000001", + "2024GQ0000005", + } + assert person_map(result.persons) == { + ("2024HU0000001", "01"): "121", + ("2024HU0000001", "02"): "97", + ("2024GQ0000005", "01"): "64", + } + assert codes_map(result) == { + key: EXPECTED_CODES[key] for key in ("2024HU0000001", "2024GQ0000005") + } + + +def test_a_vacancy_only_selection_is_a_valid_source_projection(acs): + """Source-only preparation may validly retain no people at all.""" + result = acs.produce(serialnos=("2024HU0000004",)) + assert household_keys(result.households) == ["2024HU0000004"] + assert len(result.persons) == 0 + assert result.codes["occupied_hu"].tolist() == [0] + assert result.codes["unresolved_reasons"].tolist() == [1] + + +def test_selected_rows_ignore_the_caller_tuple_order(acs): + """The request is a set of keys; canonical order is the module's own.""" + forward = acs.produce( + output_dir=acs.output_dir("forward"), + serialnos=("2024HU0000002", "2024HU0000008"), + ) + reversed_request = acs.produce( + output_dir=acs.output_dir("reversed"), + serialnos=("2024HU0000008", "2024HU0000002"), + ) + assert forward.projection_json == reversed_request.projection_json + assert household_keys(forward.households) == expected_row_order( + ("2024HU0000002", "2024HU0000008") + ) + + +def test_selection_changes_the_projection_bytes(acs): + whole = acs.produce() + selected = acs.produce(serialnos=("2024HU0000001",)) + assert selected.projection_json != whole.projection_json + + +def test_readback_under_a_different_selection_refuses(acs): + output_dir = acs.output_dir() + acs.produce(output_dir=output_dir, serialnos=("2024HU0000001",)) + refuses(acs, acs.load, output_dir, serialnos=("2024HU0000002",)) + refuses(acs, acs.load, output_dir, serialnos=None) + + +@pytest.mark.parametrize( + "serialnos", + [ + pytest.param(("2024HU0000009",), id="unknown_key"), + pytest.param(("2024HU0000001", "2024HU0000009"), id="one_unknown_key"), + pytest.param(("2024HU0000001", "2024HU0000001"), id="duplicate_key"), + pytest.param((), id="empty_selection"), + pytest.param(("",), id="blank_key"), + pytest.param((" 2024HU0000001",), id="whitespace_padded_key"), + pytest.param(("2024hu0000001",), id="case_folded_key"), + ], +) +def test_invalid_selection_refuses(acs, serialnos): + refuses(acs, acs.produce, serialnos=serialnos) + + +# -------------------------------------------------------------------------- +# Complete-join integrity, checked before any exclusion or selection. +# -------------------------------------------------------------------------- + +JOIN_VARIANTS = [ + pytest.param( + BASE_HOUSEHOLDS, + BASE_PERSONS + (person_row("2024HU0000004", "1", "40"),), + id="person_attached_to_a_vacant_household", + ), + pytest.param( + BASE_HOUSEHOLDS, + BASE_PERSONS + (person_row("2024HU0000009", "1", "40"),), + id="orphan_person_without_a_household", + ), + pytest.param( + BASE_HOUSEHOLDS + (BASE_HOUSEHOLDS[0],), + BASE_PERSONS, + id="duplicate_household_key", + ), + pytest.param( + BASE_HOUSEHOLDS, + BASE_PERSONS + (person_row("2024HU0000002", "1", "44"),), + id="duplicate_person_key", + ), + pytest.param( + BASE_HOUSEHOLDS, + BASE_PERSONS + (person_row("2024HU0000002", "01", "44"),), + id="duplicate_person_key_via_leading_zero_alias", + ), + pytest.param( + with_household(0, NP="3"), + BASE_PERSONS, + id="household_np_above_linked_person_count", + ), + pytest.param( + with_household(0, NP="1"), + BASE_PERSONS, + id="household_np_below_linked_person_count", + ), + pytest.param( + BASE_HOUSEHOLDS, + BASE_PERSONS[:6] + BASE_PERSONS[7:], + id="group_quarters_placeholder_without_its_person", + ), + pytest.param( + with_household(4, NP="2"), + BASE_PERSONS + (person_row("2024GQ0000005", "02", "51"),), + id="group_quarters_placeholder_with_two_people", + ), + pytest.param( + with_household(4, NP="0"), + BASE_PERSONS[:6] + BASE_PERSONS[7:], + id="group_quarters_placeholder_with_no_people", + ), +] + + +@pytest.mark.parametrize("households,persons", JOIN_VARIANTS) +def test_join_integrity_refusals(tmp_path, monkeypatch, households, persons): + fixture = single_member( + tmp_path, monkeypatch, households=households, persons=persons + ) + refuses(fixture, fixture.produce) + + +@pytest.mark.parametrize( + "serialnos", + [ + pytest.param(None, id="whole_source"), + pytest.param(("2024HU0000001",), id="unrelated_single_key"), + ], +) +def test_vacant_household_person_refuses_before_exclusion_or_selection( + tmp_path, monkeypatch, serialnos +): + """An NP0 household with a person must refuse even when nothing selects it. + + An implementation that dropped NP0 rows, or applied the key selection, + before validating would silently accept this invented contradiction. + """ + fixture = single_member( + tmp_path, + monkeypatch, + persons=BASE_PERSONS + (person_row("2024HU0000004", "1", "40"),), + ) + refuses(fixture, fixture.produce, serialnos=serialnos) + + +def test_orphan_person_refuses_even_when_selection_excludes_it(tmp_path, monkeypatch): + fixture = single_member( + tmp_path, + monkeypatch, + persons=BASE_PERSONS + (person_row("2024HU0000009", "1", "40"),), + ) + refuses(fixture, fixture.produce, serialnos=("2024HU0000002",)) + + +# -------------------------------------------------------------------------- +# Closed source-value domains. +# -------------------------------------------------------------------------- + +DOMAIN_VARIANTS = [ + # SERIALNO: exactly 13 ASCII characters, 2024HU/2024GQ plus seven digits, + # numeric suffix 1..9999999, prefix agreeing with TYPEHUGQ, no whitespace, + # no empty identifier and no normalisation of the accepted bytes. + pytest.param( + *rename_serialno("2024HU0000001", " 2024HU000001"), id="serialno_leading_space" + ), + pytest.param( + *rename_serialno("2024HU0000001", "2024HU000001 "), id="serialno_trailing_space" + ), + pytest.param( + *rename_serialno("2024HU0000001", "2024HU 000001"), id="serialno_interior_space" + ), + pytest.param(*rename_serialno("2024HU0000001", ""), id="serialno_empty"), + pytest.param( + *rename_serialno("2024HU0000001", "2024HU000001"), id="serialno_too_short" + ), + pytest.param( + *rename_serialno("2024HU0000001", "2024HU00000010"), id="serialno_too_long" + ), + pytest.param( + *rename_serialno("2024HU0000001", "2023HU0000001"), id="serialno_wrong_vintage" + ), + pytest.param( + *rename_serialno("2024HU0000001", "2024XX0000001"), + id="serialno_unknown_type_token", + ), + pytest.param( + *rename_serialno("2024HU0000001", "2024hu0000001"), + id="serialno_lowercase_type_token", + ), + pytest.param( + *rename_serialno("2024HU0000001", "2024HU0000000"), id="serialno_zero_suffix" + ), + pytest.param( + *rename_serialno("2024HU0000001", "2024HU000000A"), + id="serialno_nondigit_suffix", + ), + pytest.param( + *rename_serialno("2024HU0000001", "2024GQ0000001"), + id="serialno_gq_prefix_on_housing_unit", + ), + pytest.param( + *rename_serialno("2024GQ0000005", "2024HU0000005"), + id="serialno_hu_prefix_on_group_quarters", + ), + # SPORDER: one or two ASCII digits, value 1..20, no sign/exponent/point. + pytest.param(BASE_HOUSEHOLDS, with_person(2, SPORDER="0"), id="sporder_zero"), + pytest.param( + BASE_HOUSEHOLDS, with_person(2, SPORDER="21"), id="sporder_above_domain" + ), + pytest.param( + BASE_HOUSEHOLDS, with_person(2, SPORDER="001"), id="sporder_three_digits" + ), + pytest.param( + BASE_HOUSEHOLDS, with_person(2, SPORDER=" 1"), id="sporder_leading_space" + ), + pytest.param( + BASE_HOUSEHOLDS, with_person(2, SPORDER="1 "), id="sporder_trailing_space" + ), + pytest.param(BASE_HOUSEHOLDS, with_person(2, SPORDER="+1"), id="sporder_signed"), + pytest.param(BASE_HOUSEHOLDS, with_person(2, SPORDER="-1"), id="sporder_negative"), + pytest.param( + BASE_HOUSEHOLDS, with_person(2, SPORDER="1.0"), id="sporder_decimal_point" + ), + pytest.param(BASE_HOUSEHOLDS, with_person(2, SPORDER="1e0"), id="sporder_exponent"), + pytest.param(BASE_HOUSEHOLDS, with_person(2, SPORDER=""), id="sporder_empty"), + # TYPEHUGQ: 1, 2 or 3 only. + pytest.param(with_household(0, TYPEHUGQ="0"), BASE_PERSONS, id="typehugq_zero"), + pytest.param( + with_household(0, TYPEHUGQ="4"), BASE_PERSONS, id="typehugq_above_domain" + ), + pytest.param(with_household(0, TYPEHUGQ=""), BASE_PERSONS, id="typehugq_empty"), + pytest.param( + with_household(0, TYPEHUGQ="H"), BASE_PERSONS, id="typehugq_nonnumeric" + ), + # NP: published integer domain 0..20. + pytest.param(with_household(0, NP="21"), BASE_PERSONS, id="np_above_domain"), + pytest.param(with_household(0, NP="-1"), BASE_PERSONS, id="np_negative"), + pytest.param(with_household(0, NP=""), BASE_PERSONS, id="np_empty"), + pytest.param(with_household(0, NP=" 2"), BASE_PERSONS, id="np_leading_space"), + pytest.param(with_household(0, NP="2.0"), BASE_PERSONS, id="np_decimal_point"), + # WGTP: 1..9999 for housing units including vacant ones; zero only for the + # group-quarters placeholders. + pytest.param( + with_household(0, WGTP="0"), BASE_PERSONS, id="wgtp_zero_on_occupied_unit" + ), + pytest.param( + with_household(3, WGTP="0"), BASE_PERSONS, id="wgtp_zero_on_vacant_unit" + ), + pytest.param(with_household(0, WGTP="10000"), BASE_PERSONS, id="wgtp_above_domain"), + pytest.param(with_household(0, WGTP="-5"), BASE_PERSONS, id="wgtp_negative"), + pytest.param(with_household(0, WGTP=""), BASE_PERSONS, id="wgtp_empty"), + # PWGTP: 1..9999 for every person, group quarters included. + pytest.param(BASE_HOUSEHOLDS, with_person(2, PWGTP="0"), id="pwgtp_zero"), + pytest.param( + BASE_HOUSEHOLDS, + with_person(6, PWGTP="0"), + id="pwgtp_zero_on_group_quarters_person", + ), + pytest.param( + BASE_HOUSEHOLDS, with_person(2, PWGTP="10000"), id="pwgtp_above_domain" + ), + pytest.param(BASE_HOUSEHOLDS, with_person(2, PWGTP="-1"), id="pwgtp_negative"), + pytest.param(BASE_HOUSEHOLDS, with_person(2, PWGTP=""), id="pwgtp_empty"), + # TEN: blank means missing; observed values are 1..4; group quarters and + # vacancy require the blank token. + pytest.param(with_household(0, TEN="5"), BASE_PERSONS, id="ten_above_domain"), + pytest.param(with_household(0, TEN="0"), BASE_PERSONS, id="ten_zero"), + pytest.param(with_household(0, TEN="b"), BASE_PERSONS, id="ten_display_character"), + pytest.param(with_household(0, TEN=" "), BASE_PERSONS, id="ten_single_space"), + pytest.param( + with_household(3, TEN="1"), BASE_PERSONS, id="ten_present_on_vacant_unit" + ), + pytest.param( + with_household(4, TEN="1"), BASE_PERSONS, id="ten_present_on_group_quarters" + ), + # PUMA: exactly five ASCII digits with integer value 100..81003. + pytest.param(with_household(0, PUMA="0010"), BASE_PERSONS, id="puma_four_digits"), + pytest.param(with_household(0, PUMA="000100"), BASE_PERSONS, id="puma_six_digits"), + pytest.param(with_household(0, PUMA="00099"), BASE_PERSONS, id="puma_below_domain"), + pytest.param(with_household(0, PUMA="81004"), BASE_PERSONS, id="puma_above_domain"), + pytest.param(with_household(0, PUMA="0010A"), BASE_PERSONS, id="puma_nondigit"), + pytest.param(with_household(0, PUMA=""), BASE_PERSONS, id="puma_empty"), + # ST: exactly two ASCII digits from the 50-state and DC subset. + pytest.param(with_household(0, ST="72"), BASE_PERSONS, id="state_puerto_rico"), + pytest.param(with_household(0, ST="99"), BASE_PERSONS, id="state_unassigned_code"), + pytest.param(with_household(0, ST="6"), BASE_PERSONS, id="state_single_digit"), + pytest.param(with_household(0, ST="006"), BASE_PERSONS, id="state_three_digits"), + pytest.param( + with_household(0, ST="CA"), BASE_PERSONS, id="state_postal_abbreviation" + ), + pytest.param(with_household(0, ST=""), BASE_PERSONS, id="state_empty"), +] + + +@pytest.mark.parametrize("households,persons", DOMAIN_VARIANTS) +def test_source_value_domain_refusals(tmp_path, monkeypatch, households, persons): + fixture = single_member( + tmp_path, monkeypatch, households=households, persons=persons + ) + refuses(fixture, fixture.produce) + + +def test_sporder_leading_zero_is_accepted_and_the_raw_token_is_retained( + tmp_path, monkeypatch +): + """Leading-zero SPORDER normalises for the join without losing its bytes.""" + fixture = single_member(tmp_path, monkeypatch, persons=with_person(2, SPORDER="01")) + assert ("2024HU0000002", "01") in person_map(fixture.produce().persons) + + +def test_maximum_and_minimum_accepted_weights_are_retained(acs): + weights = household_map(acs.produce().households) + assert weights["2024HU0000003"]["WGTP"] == "1" + assert weights["2024HU0000008"]["WGTP"] == "9999" + + +# -------------------------------------------------------------------------- +# Strict lexical parsing and the ordered-header contract. +# -------------------------------------------------------------------------- + +BASE_HOUSEHOLD_CSV = csv_bytes(HOUSEHOLD_HEADER, BASE_HOUSEHOLDS) +BASE_PERSON_CSV = csv_bytes(PERSON_HEADER, BASE_PERSONS) + + +def household_bytes_fixture(tmp_path, monkeypatch, payload, *, name="psam_husa.csv"): + return build_fixture( + tmp_path, + monkeypatch, + household_members=(Member(name, payload),), + person_members=(Member("psam_pusa.csv", BASE_PERSON_CSV),), + ) + + +def person_bytes_fixture(tmp_path, monkeypatch, payload, *, name="psam_pusa.csv"): + return build_fixture( + tmp_path, + monkeypatch, + household_members=(Member("psam_husa.csv", BASE_HOUSEHOLD_CSV),), + person_members=(Member(name, payload),), + ) + + +def splice_line(payload, index, line): + """Insert an invented physical line at a record boundary.""" + lines = payload.split(b"\n") + lines.insert(index, line) + return b"\n".join(lines) + + +def drop_column(header, rows, name): + position = header.index(name) + trimmed = header[:position] + header[position + 1 :] + return trimmed, tuple(row[:position] + row[position + 1 :] for row in rows) + + +def rename_column(header, old, new): + return tuple(new if name == old else name for name in header) + + +WIDE_SUFFIX = tuple(f"AUX{index:02d}" for index in range(24)) +WIDE_HEADER = HOUSEHOLD_HEADER + WIDE_SUFFIX +WIDE_ROWS = tuple(row + ("0",) * len(WIDE_SUFFIX) for row in BASE_HOUSEHOLDS) + + +def oversized_record_csv(): + """One physical record above 1 MiB whose every field stays under 64 KiB.""" + padded = ("4" * 50_000,) * len(WIDE_SUFFIX) + rows = (WIDE_ROWS[0][: len(HOUSEHOLD_HEADER)] + padded,) + WIDE_ROWS[1:] + return csv_bytes(WIDE_HEADER, rows) + + +def oversized_field_csv(): + """One field token above 64 KiB in a column that is never projected.""" + return csv_bytes(HOUSEHOLD_HEADER, with_household(0, HINCP="7" * 65_537)) + + +HOUSEHOLD_PARSE_VARIANTS = [ + pytest.param(b"\xef\xbb\xbf" + BASE_HOUSEHOLD_CSV, id="utf8_byte_order_mark"), + pytest.param( + BASE_HOUSEHOLD_CSV.replace(b"2024HU0000002", b"2024HU\xff000002"), + id="invalid_utf8_byte", + ), + pytest.param( + BASE_HOUSEHOLD_CSV.replace(b"82000", b"820\x0000"), id="embedded_nul_in_field" + ), + pytest.param(splice_line(BASE_HOUSEHOLD_CSV, 3, b""), id="interior_blank_record"), + pytest.param(BASE_HOUSEHOLD_CSV + b"\n", id="trailing_blank_record"), + pytest.param( + csv_bytes(HOUSEHOLD_HEADER, BASE_HOUSEHOLDS[1:]) + + ",".join(BASE_HOUSEHOLDS[0][:-1]).encode() + + b"\n", + id="short_record", + ), + pytest.param( + csv_bytes(HOUSEHOLD_HEADER, BASE_HOUSEHOLDS[1:]) + + ",".join(BASE_HOUSEHOLDS[0] + ("extra",)).encode() + + b"\n", + id="long_record", + ), + pytest.param( + csv_bytes(rename_column(HOUSEHOLD_HEADER, "REGION", "ST"), BASE_HOUSEHOLDS), + id="duplicate_header_name", + ), + pytest.param( + csv_bytes(rename_column(HOUSEHOLD_HEADER, "REGION", ""), BASE_HOUSEHOLDS), + id="blank_header_name", + ), + pytest.param( + csv_bytes( + rename_column(HOUSEHOLD_HEADER, "SERIALNO", " SERIALNO"), BASE_HOUSEHOLDS + ), + id="whitespace_padded_header_name", + ), + pytest.param( + csv_bytes(*drop_column(HOUSEHOLD_HEADER, BASE_HOUSEHOLDS, "NP")), + id="missing_np_column", + ), + pytest.param( + csv_bytes(*drop_column(HOUSEHOLD_HEADER, BASE_HOUSEHOLDS, "SERIALNO")), + id="missing_serialno_column", + ), + pytest.param( + csv_bytes(*drop_column(HOUSEHOLD_HEADER, BASE_HOUSEHOLDS, "TEN")), + id="missing_ten_column", + ), + pytest.param( + csv_bytes(*drop_column(HOUSEHOLD_HEADER, BASE_HOUSEHOLDS, "WGTP")), + id="missing_wgtp_column", + ), + pytest.param( + csv_bytes(*drop_column(HOUSEHOLD_HEADER, BASE_HOUSEHOLDS, "PUMA")), + id="missing_puma_column", + ), + pytest.param( + csv_bytes(*drop_column(HOUSEHOLD_HEADER, BASE_HOUSEHOLDS, "ST")), + id="neither_st_nor_state_column", + ), + pytest.param( + csv_bytes(HOUSEHOLD_HEADER, with_household(0, HINCP="82000\r82000")), + id="embedded_carriage_return_in_field", + ), + pytest.param( + csv_bytes(HOUSEHOLD_HEADER, with_household(0, HINCP="82000\n82000")), + id="embedded_line_feed_in_field", + ), + pytest.param(oversized_field_csv(), id="field_token_above_64_kib"), + pytest.param(oversized_record_csv(), id="physical_record_above_1_mib"), + pytest.param(BASE_HOUSEHOLD_CSV.replace(b",", b";"), id="semicolon_delimiter"), + pytest.param(b"", id="empty_member_without_a_header"), +] + + +@pytest.mark.parametrize("payload", HOUSEHOLD_PARSE_VARIANTS) +def test_household_parser_refusals(tmp_path, monkeypatch, payload): + fixture = household_bytes_fixture(tmp_path, monkeypatch, payload) + refuses(fixture, fixture.produce) + + +PERSON_PARSE_VARIANTS = [ + pytest.param( + csv_bytes(*drop_column(PERSON_HEADER, BASE_PERSONS, "SPORDER")), + id="missing_sporder_column", + ), + pytest.param( + csv_bytes(*drop_column(PERSON_HEADER, BASE_PERSONS, "PWGTP")), + id="missing_pwgtp_column", + ), + pytest.param( + csv_bytes(*drop_column(PERSON_HEADER, BASE_PERSONS, "SERIALNO")), + id="missing_person_serialno_column", + ), + pytest.param(splice_line(BASE_PERSON_CSV, 2, b""), id="person_blank_record"), + pytest.param( + csv_bytes(PERSON_HEADER, BASE_PERSONS[1:]) + + ",".join(BASE_PERSONS[0][:-2]).encode() + + b"\n", + id="person_short_record", + ), +] + + +@pytest.mark.parametrize("payload", PERSON_PARSE_VARIANTS) +def test_person_parser_refusals(tmp_path, monkeypatch, payload): + fixture = person_bytes_fixture(tmp_path, monkeypatch, payload) + refuses(fixture, fixture.produce) + + +def test_cross_member_reordered_header_refuses(tmp_path, monkeypatch): + """Identical column sets in a different order are still a header mismatch.""" + reordered = list(HOUSEHOLD_HEADER) + first, second = reordered.index("REGION"), reordered.index("ADJHSG") + reordered[first], reordered[second] = reordered[second], reordered[first] + rows = tuple( + tuple(dict(zip(HOUSEHOLD_HEADER, row, strict=True))[name] for name in reordered) + for row in BASE_HOUSEHOLDS[5:] + ) + fixture = build_fixture( + tmp_path, + monkeypatch, + household_members=( + Member("psam_husa.csv", csv_bytes(HOUSEHOLD_HEADER, BASE_HOUSEHOLDS[:5])), + Member("psam_husb.csv", csv_bytes(tuple(reordered), rows)), + ), + person_members=(Member("psam_pusa.csv", BASE_PERSON_CSV),), + ) + refuses(fixture, fixture.produce) + + +def test_cross_member_extra_column_refuses(tmp_path, monkeypatch): + extended = HOUSEHOLD_HEADER + ("FINCP",) + fixture = build_fixture( + tmp_path, + monkeypatch, + household_members=( + Member("psam_husa.csv", csv_bytes(HOUSEHOLD_HEADER, BASE_HOUSEHOLDS[:5])), + Member( + "psam_husb.csv", + csv_bytes(extended, tuple(row + ("0",) for row in BASE_HOUSEHOLDS[5:])), + ), + ), + person_members=(Member("psam_pusa.csv", BASE_PERSON_CSV),), + ) + refuses(fixture, fixture.produce) + + +def test_header_only_member_is_allowed_and_contributes_no_rows(tmp_path, monkeypatch): + fixture = build_fixture( + tmp_path, + monkeypatch, + household_members=( + Member("psam_husa.csv", BASE_HOUSEHOLD_CSV), + Member("psam_husb.csv", csv_bytes(HOUSEHOLD_HEADER, ())), + ), + person_members=( + Member("psam_pusa.csv", csv_bytes(PERSON_HEADER, ())), + Member("psam_pusb.csv", BASE_PERSON_CSV), + ), + ) + result = fixture.produce() + assert set(household_keys(result.households)) == set(BASE_SERIALNOS) + assert person_map(result.persons) == EXPECTED_PERSONS + assert_lineage( + result.households, + household_keys(result.households), + dict.fromkeys(BASE_SERIALNOS, "psam_husa.csv"), + ) + assert_lineage( + result.persons, + list(person_map(result.persons)), + dict.fromkeys(EXPECTED_PERSONS, "psam_pusb.csv"), + ) + # The empty members are still inventoried even though they carry no rows. + strings = flat_strings(json.loads(result.receipt_json)) + assert {"psam_husb.csv", "psam_pusa.csv"} <= strings + + +def test_an_entirely_header_only_source_is_not_a_population(tmp_path, monkeypatch): + fixture = build_fixture( + tmp_path, + monkeypatch, + household_members=(Member("psam_husa.csv", csv_bytes(HOUSEHOLD_HEADER, ())),), + person_members=(Member("psam_pusa.csv", csv_bytes(PERSON_HEADER, ())),), + ) + refuses(fixture, fixture.produce) + + +def test_state_only_household_header_is_supported(tmp_path, monkeypatch): + header = rename_column(HOUSEHOLD_HEADER, "ST", "STATE") + fixture = household_bytes_fixture( + tmp_path, monkeypatch, csv_bytes(header, BASE_HOUSEHOLDS) + ) + result = fixture.produce() + assert household_map(result.households)["2024HU0000001"]["ST"] == "06" + # The header alias itself is preserved in the source evidence. + assert "STATE" in flat_strings(json.loads(result.receipt_json)) + + +def test_agreeing_st_and_state_columns_are_accepted(tmp_path, monkeypatch): + header = HOUSEHOLD_HEADER + ("STATE",) + state = HOUSEHOLD_HEADER.index("ST") + rows = tuple(row + (row[state],) for row in BASE_HOUSEHOLDS) + fixture = household_bytes_fixture(tmp_path, monkeypatch, csv_bytes(header, rows)) + result = fixture.produce() + assert household_map(result.households)["2024HU0000003"]["ST"] == "36" + + +def test_disagreeing_st_and_state_columns_refuse(tmp_path, monkeypatch): + header = HOUSEHOLD_HEADER + ("STATE",) + state = HOUSEHOLD_HEADER.index("ST") + rows = tuple( + row + ("36" if index == 0 else row[state],) + for index, row in enumerate(BASE_HOUSEHOLDS) + ) + fixture = household_bytes_fixture(tmp_path, monkeypatch, csv_bytes(header, rows)) + refuses(fixture, fixture.produce) + + +@pytest.mark.parametrize("newline", ["\n", "\r\n"]) +def test_both_physical_line_terminators_project_identically( + tmp_path, monkeypatch, newline +): + """A genuine archive's terminator choice must not change the projection.""" + fixture = build_fixture(tmp_path, monkeypatch, newline=newline) + result = fixture.produce() + assert household_map(result.households) == EXPECTED_HOUSEHOLDS + assert person_map(result.persons) == EXPECTED_PERSONS + + +# -------------------------------------------------------------------------- +# Archive preflight, run on the central directory before any by-name lookup. +# -------------------------------------------------------------------------- + +SYMLINK_ATTR = (stat.S_IFLNK | 0o777) << 16 +DIRECTORY_ATTR = (stat.S_IFDIR | 0o755) << 16 | 0x10 + + +def household_archive_fixture(tmp_path, monkeypatch, members): + return build_fixture( + tmp_path, + monkeypatch, + household_members=members, + person_members=(Member("psam_pusa.csv", BASE_PERSON_CSV),), + ) + + +def aux_members(count, *, prefix="ACS2024_note"): + return tuple( + Member(f"{prefix}{index:03d}.txt", b"invented auxiliary text\n") + for index in range(count) + ) + + +ARCHIVE_VARIANTS = [ + pytest.param( + ( + Member("psam_husa.csv", BASE_HOUSEHOLD_CSV), + Member("psam_husa.csv", BASE_HOUSEHOLD_CSV), + ), + id="duplicate_member_name", + ), + pytest.param( + ( + Member("psam_husa.csv", csv_bytes(HOUSEHOLD_HEADER, BASE_HOUSEHOLDS[:5])), + Member("PSAM_HUSA.CSV", csv_bytes(HOUSEHOLD_HEADER, BASE_HOUSEHOLDS[5:])), + ), + id="case_folded_member_alias", + ), + pytest.param( + ( + Member("psam_husa.csv", BASE_HOUSEHOLD_CSV), + Member("psam_hus/", b"", external_attr=DIRECTORY_ATTR), + ), + id="directory_entry", + ), + pytest.param( + (Member("pums/psam_husa.csv", BASE_HOUSEHOLD_CSV),), + id="nested_path_member", + ), + pytest.param( + (Member("/psam_husa.csv", BASE_HOUSEHOLD_CSV),), + id="absolute_path_member", + ), + pytest.param( + (Member("../psam_husa.csv", BASE_HOUSEHOLD_CSV),), + id="parent_traversal_member", + ), + pytest.param( + ( + Member("psam_husa.csv", BASE_HOUSEHOLD_CSV), + Member("psam_husb.txt", BASE_HOUSEHOLD_CSV), + ), + id="prefix_matching_non_csv_member", + ), + pytest.param( + ( + Member("psam_husa.csv", BASE_HOUSEHOLD_CSV), + Member( + "psam_husb.csv", BASE_HOUSEHOLD_CSV, compress_type=zipfile.ZIP_BZIP2 + ), + ), + id="bzip2_compressed_member", + ), + pytest.param( + ( + Member("psam_husa.csv", BASE_HOUSEHOLD_CSV), + Member("psam_link.csv", b"psam_husa.csv", external_attr=SYMLINK_ATTR), + ), + id="symlink_member", + ), + pytest.param( + (Member("psam_pusa.csv", BASE_PERSON_CSV),), + id="wrong_role_members_only", + ), + pytest.param( + (Member("ACS2024_readme.txt", b"invented auxiliary text\n"),), + id="no_applicable_csv_member", + ), + pytest.param((), id="empty_archive"), + pytest.param( + (Member("psam_husa.csv", BASE_HOUSEHOLD_CSV),) + aux_members(64), + id="member_count_above_cap", + ), +] + + +@pytest.mark.parametrize("members", ARCHIVE_VARIANTS) +def test_archive_preflight_refusals(tmp_path, monkeypatch, members): + fixture = household_archive_fixture(tmp_path, monkeypatch, members) + refuses(fixture, fixture.produce) + + +def test_member_count_at_the_cap_is_accepted(tmp_path, monkeypatch): + fixture = household_archive_fixture( + tmp_path, + monkeypatch, + (Member("psam_husa.csv", BASE_HOUSEHOLD_CSV),) + aux_members(63), + ) + assert len(fixture.produce().households) == len(BASE_HOUSEHOLDS) + + +def test_ordinary_auxiliary_members_are_inventory_only(tmp_path, monkeypatch): + """A non-matching ordinary file is recorded but never read as data.""" + fixture = household_archive_fixture( + tmp_path, + monkeypatch, + ( + Member("psam_husa.csv", BASE_HOUSEHOLD_CSV), + Member("ACS2024_PUMS_README.txt", b"invented auxiliary text\n"), + ), + ) + result = fixture.produce() + assert set(household_keys(result.households)) == set(BASE_SERIALNOS) + assert_lineage( + result.households, + household_keys(result.households), + dict.fromkeys(BASE_SERIALNOS, "psam_husa.csv"), + ) + assert "ACS2024_PUMS_README.txt" in flat_strings(json.loads(result.receipt_json)) + + +def test_a_non_matching_csv_member_is_auxiliary_not_data(tmp_path, monkeypatch): + """The role prefix, not the extension, decides what counts as data.""" + fixture = household_archive_fixture( + tmp_path, + monkeypatch, + ( + Member("psam_husa.csv", BASE_HOUSEHOLD_CSV), + Member("ACS2024_shells.csv", csv_bytes(("A", "B"), (("1", "2"),))), + ), + ) + result = fixture.produce() + assert len(result.households) == len(BASE_HOUSEHOLDS) + assert "ACS2024_shells.csv" in flat_strings(json.loads(result.receipt_json)) + + +def test_stored_members_are_accepted(tmp_path, monkeypatch): + fixture = household_archive_fixture( + tmp_path, + monkeypatch, + ( + Member( + "psam_husa.csv", BASE_HOUSEHOLD_CSV, compress_type=zipfile.ZIP_STORED + ), + ), + ) + assert len(fixture.produce().households) == len(BASE_HOUSEHOLDS) + + +def test_several_members_per_role_join_in_name_order(tmp_path, monkeypatch): + fixture = build_fixture( + tmp_path, + monkeypatch, + household_members=( + Member("psam_husa.csv", csv_bytes(HOUSEHOLD_HEADER, BASE_HOUSEHOLDS[:3])), + Member("psam_husb.csv", csv_bytes(HOUSEHOLD_HEADER, BASE_HOUSEHOLDS[3:6])), + Member("psam_husc.csv", csv_bytes(HOUSEHOLD_HEADER, BASE_HOUSEHOLDS[6:])), + ), + person_members=( + Member("psam_pusa.csv", csv_bytes(PERSON_HEADER, BASE_PERSONS[:4])), + Member("psam_pusb.csv", csv_bytes(PERSON_HEADER, BASE_PERSONS[4:])), + ), + ) + result = fixture.produce() + assert set(household_keys(result.households)) == set(BASE_SERIALNOS) + members = { + row[HOUSEHOLD_HEADER.index("SERIALNO")]: name + for name, rows in ( + ("psam_husa.csv", BASE_HOUSEHOLDS[:3]), + ("psam_husb.csv", BASE_HOUSEHOLDS[3:6]), + ("psam_husc.csv", BASE_HOUSEHOLDS[6:]), + ) + for row in rows + } + assert_lineage(result.households, household_keys(result.households), members) + # 2024HU0000003's three people straddle the two person members. + assert lexical(result.persons["SERIALNO"]).count("2024HU0000003") == 3 + assert person_map(result.persons) == EXPECTED_PERSONS + person_members = { + ( + row[PERSON_HEADER.index("SERIALNO")], + row[PERSON_HEADER.index("SPORDER")], + ): name + for name, rows in ( + ("psam_pusa.csv", BASE_PERSONS[:4]), + ("psam_pusb.csv", BASE_PERSONS[4:]), + ) + for row in rows + } + assert_lineage(result.persons, list(person_map(result.persons)), person_members) + + +@pytest.mark.parametrize("archive", [HOUSEHOLD_ARCHIVE, PERSON_ARCHIVE]) +def test_missing_role_archive_refuses(acs, archive): + (acs.source_dir / archive).unlink() + refuses(acs, acs.produce) + + +def test_corrupted_member_payload_fails_the_recorded_crc(tmp_path, monkeypatch): + """A stored member whose bytes were swapped keeps its declared size and CRC.""" + fixture = household_archive_fixture( + tmp_path, + monkeypatch, + ( + Member( + "psam_husa.csv", BASE_HOUSEHOLD_CSV, compress_type=zipfile.ZIP_STORED + ), + ), + ) + payload = fixture.household_zip.read_bytes() + assert b"82000" in payload, "the stored member is not plain in the archive" + fixture.household_zip.write_bytes(payload.replace(b"82000", b"82001", 1)) + fixture.pin() + refuses(fixture, fixture.produce) + + +# -------------------------------------------------------------------------- +# Private capture: paths, pins, originals and byte bounds. +# -------------------------------------------------------------------------- + + +@contextmanager +def bounded_time(seconds=30): + """Fail rather than hang if a non-regular original is opened blocking.""" + + def expire(signum, frame): + raise TimeoutError("the call did not return within the bounded test window") + + previous = signal.signal(signal.SIGALRM, expire) + signal.setitimer(signal.ITIMER_REAL, seconds) + try: + yield + finally: + signal.setitimer(signal.ITIMER_REAL, 0) + signal.signal(signal.SIGALRM, previous) + + +def test_missing_snapshot_root_refuses(acs): + refuses( + acs, + source.produce_acs_housing_source, + acs.source_dir, + snapshot_root=acs.tmp_path / "absent-root", + output_dir=acs.output_dir(), + ) + + +def test_snapshot_root_that_is_a_file_refuses(acs): + root = acs.tmp_path / "root-file" + root.write_bytes(b"invented\n") + refuses( + acs, + source.produce_acs_housing_source, + acs.source_dir, + snapshot_root=root, + output_dir=acs.output_dir(), + ) + + +def test_symlinked_snapshot_root_refuses(acs): + link = acs.tmp_path / "root-link" + link.symlink_to(acs.snapshot_root, target_is_directory=True) + refuses( + acs, + source.produce_acs_housing_source, + acs.source_dir, + snapshot_root=link, + output_dir=acs.output_dir(), + ) + + +def test_existing_output_directory_refuses(acs): + output_dir = acs.output_dir() + output_dir.mkdir() + refuses(acs, acs.produce, output_dir=output_dir) + + +def test_existing_output_file_refuses(acs): + output_dir = acs.output_dir() + output_dir.write_bytes(b"invented\n") + refuses(acs, acs.produce, output_dir=output_dir) + + +def test_output_directory_without_a_parent_refuses(acs): + refuses(acs, acs.produce, output_dir=acs.tmp_path / "absent" / "artifact") + + +def test_missing_source_directory_refuses(acs): + refuses(acs, acs.produce, source_dir=acs.tmp_path / "absent-source") + + +def test_source_directory_that_is_a_file_refuses(acs): + path = acs.tmp_path / "source-file" + path.write_bytes(b"invented\n") + refuses(acs, acs.produce, source_dir=path) + + +def test_symlinked_source_directory_refuses(acs): + link = acs.tmp_path / "source-link" + link.symlink_to(acs.source_dir, target_is_directory=True) + refuses(acs, acs.produce, source_dir=link) + + +def test_symlinked_original_archive_refuses(acs): + real = acs.tmp_path / "real-csv_hus.zip" + real.write_bytes(acs.household_zip.read_bytes()) + acs.household_zip.unlink() + acs.household_zip.symlink_to(real) + refuses(acs, acs.produce) + + +def test_fifo_original_archive_refuses(acs): + acs.household_zip.unlink() + os.mkfifo(acs.household_zip) + with bounded_time(): + refuses(acs, acs.produce) + + +def test_directory_in_place_of_an_original_archive_refuses(acs): + acs.household_zip.unlink() + acs.household_zip.mkdir() + refuses(acs, acs.produce) + + +def test_pinned_digest_mismatch_at_the_same_size_refuses(acs): + """Authentication is over the whole archive bytes, not over the projection.""" + pins = acs.pin() + flip_archive_byte(acs.household_zip) + acs.monkeypatch.setattr(source, NAMES["archive_pins"], pins) + refuses(acs, acs.produce) + + +def test_pinned_size_mismatch_refuses(acs): + pins = acs.pin() + with acs.household_zip.open("ab") as handle: + handle.write(b"\n") + acs.monkeypatch.setattr(source, NAMES["archive_pins"], pins) + refuses(acs, acs.produce) + + +def test_original_archives_are_never_modified(acs): + before = { + path: (path.read_bytes(), path.stat().st_mtime_ns) + for path in (acs.household_zip, acs.person_zip) + } + acs.produce() + for path, (payload, mtime) in before.items(): + assert path.read_bytes() == payload + assert path.stat().st_mtime_ns == mtime + + +def test_capture_writes_only_under_the_supplied_snapshot_root(acs): + assert not any(acs.snapshot_root.iterdir()) + output_dir = acs.output_dir() + acs.produce(output_dir=output_dir) + children = list(acs.snapshot_root.iterdir()) + assert children, "the private capture did not land under the snapshot root" + for child in children: + assert child.is_dir() and not child.is_symlink() + assert stat.S_IMODE(child.stat().st_mode) == 0o700 + assert sorted(path.name for path in output_dir.iterdir()) == sorted( + (NAMES["projection_filename"], NAMES["receipt_filename"]) + ) + + +def test_capture_never_falls_back_to_system_temporary_storage(acs, monkeypatch): + absent = acs.tmp_path / "no-such-tmpdir" + monkeypatch.setenv("TMPDIR", str(absent)) + monkeypatch.setattr(tempfile, "tempdir", str(absent)) + result = acs.produce() + assert len(result.households) == len(BASE_HOUSEHOLDS) + assert not absent.exists() + + +def test_two_produces_share_one_snapshot_root_without_collision(acs): + first = acs.produce(output_dir=acs.output_dir("first")) + second = acs.produce(output_dir=acs.output_dir("second")) + assert first.projection_json == second.projection_json + assert len(list(acs.snapshot_root.iterdir())) >= 2 + + +def test_a_refused_produce_leaves_no_receipt_behind(tmp_path, monkeypatch): + fixture = single_member( + tmp_path, monkeypatch, households=with_household(0, TEN="5") + ) + output_dir = fixture.output_dir() + refuses(fixture, fixture.produce, output_dir=output_dir) + assert not (output_dir / NAMES["receipt_filename"]).exists() + + +def byte_cap_names(): + return tuple(sorted(name for name in dir(source) if name.endswith("_MAX_BYTES"))) + + +def test_module_declares_its_own_byte_caps(): + assert byte_cap_names(), "the module declares no explicit byte caps" + + +@pytest.mark.parametrize("cap", [0, 1, 64]) +def test_tiny_byte_caps_refuse_before_any_payload_is_returned(acs, cap): + """Exercise the streaming bounds with mocked caps, never with real GiBs.""" + for name in byte_cap_names(): + acs.monkeypatch.setattr(source, name, cap) + refuses(acs, acs.produce) + + +def test_declared_cap_values_match_the_approved_plan(): + """Reconciliation: the plan fixes both ACS source caps by name and value.""" + assert getattr(source, NAMES["source_max_bytes"]) == 8 * 1024**3 + assert getattr(source, NAMES["receipt_max_bytes"]) == 1024**2 + + +# -------------------------------------------------------------------------- +# Artifact custody: readback reconstructs, it never trusts what it decoded. +# -------------------------------------------------------------------------- + + +def make_writable(path): + """The producer publishes read-only files; tampering needs write access.""" + path = Path(path) + path.chmod(stat.S_IMODE(path.stat().st_mode) | 0o600) + return path + + +def forge(output_dir, old, new): + """Rewrite the artifact so its own internal digests stay self-consistent.""" + projection = (output_dir / NAMES["projection_filename"]).read_bytes() + receipt = (output_dir / NAMES["receipt_filename"]).read_bytes() + assert old in projection, "the invented forgery target is not in the projection" + forged = projection.replace(old, new) + assert len(forged) == len(projection), "a forgery must not change the length" + old_digest = hashlib.sha256(projection).hexdigest().encode() + new_digest = hashlib.sha256(forged).hexdigest().encode() + make_writable(output_dir / NAMES["projection_filename"]).write_bytes(forged) + make_writable(output_dir / NAMES["receipt_filename"]).write_bytes( + receipt.replace(old_digest, new_digest) + ) + return old_digest in receipt + + +def test_rehashed_projection_forgery_refuses(acs): + output_dir = acs.output_dir() + acs.produce(output_dir=output_dir) + forge(output_dir, b"2024HU0000002", b"2024HU0000012") + refuses(acs, acs.load, output_dir) + + +def test_receipt_binds_the_projection_digest(acs): + """Otherwise a forged projection would be self-consistent on its face.""" + output_dir = acs.output_dir() + acs.produce(output_dir=output_dir) + assert forge(output_dir, b"2024HU0000002", b"2024HU0000012") + + +def test_forging_a_weight_to_another_valid_value_refuses(acs): + """Custody is byte reconstruction, so an in-domain forgery refuses too.""" + output_dir = acs.output_dir() + acs.produce(output_dir=output_dir) + forge(output_dir, b"118", b"119") + refuses(acs, acs.load, output_dir) + + +def test_rewritten_receipt_refuses(acs): + output_dir = acs.output_dir() + acs.produce(output_dir=output_dir) + receipt = json.loads((output_dir / NAMES["receipt_filename"]).read_bytes()) + forged = json.dumps(receipt, sort_keys=True).encode() + make_writable(output_dir / NAMES["receipt_filename"]).write_bytes(forged) + refuses(acs, acs.load, output_dir) + + +@pytest.mark.parametrize( + "filename", [NAMES["projection_filename"], NAMES["receipt_filename"]] +) +def test_truncated_or_missing_artifact_files_refuse(acs, filename): + output_dir = acs.output_dir() + acs.produce(output_dir=output_dir) + make_writable(output_dir / filename).write_bytes( + (output_dir / filename).read_bytes()[:20] + ) + refuses(acs, acs.load, output_dir) + + output_dir = acs.output_dir() + acs.produce(output_dir=output_dir) + (output_dir / filename).unlink() + refuses(acs, acs.load, output_dir) + + +def test_missing_artifact_directory_refuses(acs): + refuses(acs, acs.load, acs.tmp_path / "absent-artifact") + + +def test_extra_file_in_the_artifact_directory_refuses(acs): + output_dir = acs.output_dir() + acs.produce(output_dir=output_dir) + (output_dir / "notes.json").write_bytes(b"{}\n") + refuses(acs, acs.load, output_dir) + + +def test_parent_change_between_produce_and_readback_refuses(acs): + output_dir = acs.output_dir() + acs.produce(output_dir=output_dir) + flip_archive_byte(acs.household_zip) + refuses(acs, acs.load, output_dir) + + +def test_reparented_readback_refuses_even_after_repinning(acs): + """Fresh authentic parents that no longer reconstruct the artifact refuse.""" + output_dir = acs.output_dir() + acs.produce(output_dir=output_dir) + rows = with_household(1, TEN="4") + write_archive( + acs.household_zip, + ( + Member("psam_husa.csv", csv_bytes(HOUSEHOLD_HEADER, rows[:5])), + Member("psam_husb.csv", csv_bytes(HOUSEHOLD_HEADER, rows[5:])), + ), + ) + acs.pin() + refuses(acs, acs.load, output_dir) + + +def test_readback_against_swapped_role_archives_refuses(acs): + output_dir = acs.output_dir() + acs.produce(output_dir=output_dir) + household = acs.household_zip.read_bytes() + acs.household_zip.write_bytes(acs.person_zip.read_bytes()) + acs.person_zip.write_bytes(household) + acs.pin() + refuses(acs, acs.load, output_dir) + + +def test_readback_from_an_unrelated_artifact_refuses(tmp_path, monkeypatch): + other = build_fixture( + tmp_path / "other", + monkeypatch, + households=(("psam_husa.csv", BASE_HOUSEHOLDS[:4]),), + persons=(("psam_pusa.csv", BASE_PERSONS[:6]),), + ) + output_dir = other.output_dir() + other.produce(output_dir=output_dir) + + mine = build_fixture(tmp_path / "mine", monkeypatch) + refuses(mine, mine.load, output_dir) + + +@pytest.mark.parametrize("attribute", ["households", "persons", "codes"]) +def test_readback_frames_are_owned_copies(acs, attribute): + _produced, loaded, _output_dir = acs.produce_and_load() + first = getattr(loaded, attribute) + assert getattr(loaded, attribute) is not first + before = first.copy(deep=True) + first.iloc[0, 0] = first.iloc[-1, 0] + pd.testing.assert_frame_equal(getattr(loaded, attribute), before) + + +# -------------------------------------------------------------------------- +# Refusal semantics: sanitised, and one code per cause. +# -------------------------------------------------------------------------- + + +def collect_family_reasons(tmp_path, monkeypatch): + """One refusal per coarse family, each from its own invented source.""" + reasons = {} + + def record(family, build, call): + fixture = build(tmp_path / family, monkeypatch) + fixture.pin() + reasons[family] = refuses(fixture, call, fixture) + + def digest_change(fixture): + pins = fixture.pin() + flip_archive_byte(fixture.household_zip) + fixture.monkeypatch.setattr(source, NAMES["archive_pins"], pins) + fixture.produce() + + def forged_artifact(fixture): + output_dir = fixture.output_dir() + fixture.produce(output_dir=output_dir) + forge(output_dir, b"2024HU0000002", b"2024HU0000012") + fixture.load(output_dir) + + def existing_output(fixture): + output_dir = fixture.output_dir() + output_dir.mkdir() + fixture.produce(output_dir=output_dir) + + plain = build_fixture + + record("capture_digest", plain, digest_change) + record("artifact_forgery", plain, forged_artifact) + record("output_precondition", plain, existing_output) + record("selection", plain, lambda f: f.produce(serialnos=("2024HU0000009",))) + record( + "zip_preflight", + lambda base, patch: household_archive_fixture( + base, + patch, + ( + Member("psam_husa.csv", BASE_HOUSEHOLD_CSV), + Member("psam_husa.csv", BASE_HOUSEHOLD_CSV), + ), + ), + lambda f: f.produce(), + ) + record( + "csv_record", + lambda base, patch: household_bytes_fixture( + base, + patch, + csv_bytes(HOUSEHOLD_HEADER, BASE_HOUSEHOLDS[1:]) + + ",".join(BASE_HOUSEHOLDS[0][:-1]).encode() + + b"\n", + ), + lambda f: f.produce(), + ) + record( + "header_contract", + lambda base, patch: household_bytes_fixture( + base, + patch, + csv_bytes(*drop_column(HOUSEHOLD_HEADER, BASE_HOUSEHOLDS, "ST")), + ), + lambda f: f.produce(), + ) + record( + "value_domain", + lambda base, patch: single_member( + base, patch, households=with_household(0, TEN="5") + ), + lambda f: f.produce(), + ) + record( + "join_integrity", + lambda base, patch: single_member( + base, + patch, + persons=BASE_PERSONS + (person_row("2024HU0000009", "1", "40"),), + ), + lambda f: f.produce(), + ) + return reasons + + +@pytest.fixture +def family_reasons(tmp_path, monkeypatch): + return collect_family_reasons(tmp_path, monkeypatch) + + +def test_refusal_type_is_a_sanitised_value_error(): + assert issubclass(source.ACSHousingSourceError, ValueError) + + +def test_every_refusal_family_is_exercised(family_reasons): + assert set(family_reasons) == set(DISTINCT_FAMILIES) + + +def test_distinct_causes_carry_distinct_reason_codes(family_reasons): + """A single catch-all code would make every refusal above unfalsifiable.""" + collisions = {} + for family, reason in family_reasons.items(): + collisions.setdefault(reason, []).append(family) + shared = {reason: names for reason, names in collisions.items() if len(names) > 1} + assert not shared, f"distinct causes share a reason code: {shared}" + + +def test_no_refusal_reason_echoes_invented_source_content(family_reasons): + """refuses() checks each call; this states the whole-sweep contract once.""" + forbidden = set(BASE_SERIALNOS) | {"psam_husa.csv", "82000", "52000"} + for family, reason in family_reasons.items(): + for secret in forbidden: + assert secret not in reason, f"{family} leaked {secret!r}" + + +def test_reason_codes_match_the_agreed_table(family_reasons): + """Shape always; exact spellings once EXPECTED_REASONS is filled in. + + The house idiom in the committed sibling source modules is a bare + UPPER_SNAKE token, so that much is asserted unconditionally and nothing is + parked behind a skip. Only the agreement on which token names which cause + waits on the production author. + """ + for family, reason in family_reasons.items(): + assert re.fullmatch(r"[A-Z][A-Z0-9_]*", reason), ( + f"{family} reason {reason!r} is not a bare reason code" + ) + if REASON_CODES_AGREED: + assert family_reasons == EXPECTED_REASONS + else: + assert not EXPECTED_REASONS, ( + "EXPECTED_REASONS is populated; set REASON_CODES_AGREED = True" + ) + + +# -------------------------------------------------------------------------- +# Naming reconciliation: everything this file had to guess, in one place. +# -------------------------------------------------------------------------- + + +def test_module_exposes_the_agreed_public_surface(): + for name in ( + "ACSHousingSourceError", + "produce_acs_housing_source", + "load_acs_housing_source", + "prepare_acs_housing_population", + ): + assert hasattr(source, name), f"missing agreed API name {name}" + + +def test_archive_pins_are_consulted_at_call_time(acs): + """The whole suite rests on this seam, so assert it directly. + + Every fixture authenticates its invented archives by pointing the private + pin tuple at them. That only works if the module reads the attribute when + it captures, rather than copying it at import. + """ + real = acs.pin() + assert acs.produce().households is not None + + wrong = tuple((role, name, "f" * 64, size) for role, name, _sha, size in real) + acs.monkeypatch.setattr(source, NAMES["archive_pins"], wrong) + refuses(acs, acs.produce) + + +def test_private_archive_pins_are_a_sentinel_or_the_two_fixed_archives(): + """Naming seam, reconciled with the production author, not a defect. + + The brief described `_ARCHIVE_PINS` as a populated tuple of + (role, filename, sha256, size). The module instead defaults it to None and + loads the packaged manifest lazily, while tests still patch the same + private name - which is what + ``test_archive_pins_are_consulted_at_call_time`` proves still works. Both + readings are admitted here; the shape is checked whenever it is populated, + so a wrong shape cannot hide behind the sentinel. + """ + pins = getattr(source, NAMES["archive_pins"]) + if pins is None: + return + assert type(pins) is tuple and len(pins) == 2 + assert [entry[0] for entry in pins] == [ + NAMES["household_role"], + NAMES["person_role"], + ] + assert [entry[1] for entry in pins] == [HOUSEHOLD_ARCHIVE, PERSON_ARCHIVE] + for _role, _name, sha, size in pins: + assert type(sha) is str and len(sha) == 64 and sha == sha.lower() + assert int(sha, 16) >= 0 + assert type(size) is int and size > 0 + + +def test_lineage_and_state_columns_use_the_agreed_names(acs): + households = acs.produce().households + assert NAMES["member_column"] in households.columns + assert NAMES["ordinal_column"] in households.columns + assert NAMES["state_column"] in households.columns + + +# -------------------------------------------------------------------------- +# Tail coverage: cases an obvious pass over the five dimensions omits. +# -------------------------------------------------------------------------- + + +def set_encrypted_flag(path): + """Mark every entry encrypted in place; zipfile itself will not write this.""" + raw = bytearray(Path(path).read_bytes()) + marked = 0 + for signature, offset in ((b"PK\x03\x04", 6), (b"PK\x01\x02", 8)): + index = 0 + while True: + index = raw.find(signature, index) + if index < 0: + break + flag = struct.unpack_from("= 2 + Path(path).write_bytes(bytes(raw)) + + +def raw_household_record(**overrides): + """A physical household line written by hand, bypassing the csv writer.""" + values = dict(zip(HOUSEHOLD_HEADER, BASE_HOUSEHOLDS[0], strict=True)) + values.update(overrides) + return ",".join(values[name] for name in HOUSEHOLD_HEADER).encode() + + +def appended_record(record): + return csv_bytes(HOUSEHOLD_HEADER, BASE_HOUSEHOLDS[1:]) + record + b"\n" + + +def test_encrypted_member_refuses(tmp_path, monkeypatch): + fixture = household_archive_fixture( + tmp_path, monkeypatch, (Member("psam_husa.csv", BASE_HOUSEHOLD_CSV),) + ) + set_encrypted_flag(fixture.household_zip) + fixture.pin() + refuses(fixture, fixture.produce) + + +def test_lzma_member_refuses(tmp_path, monkeypatch): + fixture = household_archive_fixture( + tmp_path, + monkeypatch, + ( + Member("psam_husa.csv", BASE_HOUSEHOLD_CSV), + Member("psam_husb.csv", BASE_HOUSEHOLD_CSV, compress_type=zipfile.ZIP_LZMA), + ), + ) + refuses(fixture, fixture.produce) + + +def test_bytes_that_are_not_an_archive_refuse(acs): + acs.household_zip.write_bytes(b"this is invented, and it is not a zip archive\n") + acs.pin() + refuses(acs, acs.produce) + + +def test_person_archive_holding_household_members_refuses(tmp_path, monkeypatch): + fixture = build_fixture( + tmp_path, + monkeypatch, + household_members=(Member("psam_husa.csv", BASE_HOUSEHOLD_CSV),), + person_members=(Member("psam_husa.csv", BASE_HOUSEHOLD_CSV),), + ) + refuses(fixture, fixture.produce) + + +UTF16_MEMBER = BASE_HOUSEHOLD_CSV.decode().encode("utf-16") + +TAIL_PARSE_VARIANTS = [ + pytest.param(UTF16_MEMBER, id="utf16_encoded_member"), + pytest.param( + appended_record(raw_household_record(ST='"06"x')), + id="character_after_a_closing_quote", + ), + pytest.param( + appended_record(raw_household_record(HINCP='"82000')), + id="unterminated_quote", + ), + pytest.param( + splice_line(BASE_HOUSEHOLD_CSV, 3, b" "), id="whitespace_only_record" + ), + pytest.param( + csv_bytes(HOUSEHOLD_HEADER, with_household(0, TEN="NA")), + id="na_token_is_not_a_missing_tenure", + ), + pytest.param( + csv_bytes( + rename_column(HOUSEHOLD_HEADER, "SERIALNO", "serialno"), BASE_HOUSEHOLDS + ), + id="lowercase_roster_column_name", + ), +] + + +@pytest.mark.parametrize("payload", TAIL_PARSE_VARIANTS) +def test_tail_parser_refusals(tmp_path, monkeypatch, payload): + fixture = household_bytes_fixture(tmp_path, monkeypatch, payload) + refuses(fixture, fixture.produce) + + +def test_household_rows_absent_while_person_rows_remain_refuses(tmp_path, monkeypatch): + fixture = build_fixture( + tmp_path, + monkeypatch, + household_members=(Member("psam_husa.csv", csv_bytes(HOUSEHOLD_HEADER, ())),), + person_members=(Member("psam_pusa.csv", BASE_PERSON_CSV),), + ) + refuses(fixture, fixture.produce) + + +def test_a_final_record_without_a_trailing_newline_is_accepted(tmp_path, monkeypatch): + fixture = household_bytes_fixture( + tmp_path, + monkeypatch, + csv_bytes(HOUSEHOLD_HEADER, BASE_HOUSEHOLDS, trailing_newline=False), + ) + assert household_map(fixture.produce().households) == EXPECTED_HOUSEHOLDS + + +def test_many_unprojected_columns_are_accepted(tmp_path, monkeypatch): + """Genuine PUMS members carry hundreds of columns this projection ignores.""" + fixture = household_bytes_fixture( + tmp_path, monkeypatch, csv_bytes(WIDE_HEADER, WIDE_ROWS) + ) + assert household_map(fixture.produce().households) == EXPECTED_HOUSEHOLDS + + +def test_roster_columns_in_any_header_position_are_accepted(tmp_path, monkeypatch): + """The plan pins header order across members of a role, not within one.""" + reversed_header = tuple(reversed(HOUSEHOLD_HEADER)) + rows = tuple(tuple(reversed(row)) for row in BASE_HOUSEHOLDS) + fixture = household_bytes_fixture( + tmp_path, monkeypatch, csv_bytes(reversed_header, rows) + ) + assert household_map(fixture.produce().households) == EXPECTED_HOUSEHOLDS + + +def test_maximum_serialno_suffix_is_accepted(tmp_path, monkeypatch): + households, persons = rename_serialno("2024HU0000001", "2024HU9999999") + fixture = single_member( + tmp_path, monkeypatch, households=households, persons=persons + ) + keys = household_keys(fixture.produce().households) + assert "2024HU9999999" in keys and "2024HU0000001" not in keys + + +def test_np_at_the_published_upper_bound_is_accepted(tmp_path, monkeypatch): + households = (household_row("2024HU0000020", "1", "20", "1", "300"),) + persons = tuple( + person_row("2024HU0000020", str(order), str(order + 10)) + for order in range(1, 21) + ) + fixture = single_member( + tmp_path, monkeypatch, households=households, persons=persons + ) + result = fixture.produce() + assert lexical(result.households["NP"]) == ["20"] + assert len(result.persons) == 20 + assert result.codes["occupied_hu"].tolist() == [1] + + +def test_an_all_group_quarters_source_is_retained(tmp_path, monkeypatch): + fixture = single_member( + tmp_path, + monkeypatch, + households=BASE_HOUSEHOLDS[4:6], + persons=BASE_PERSONS[6:8], + ) + result = fixture.produce() + assert codes_map(result) == { + key: EXPECTED_CODES[key] for key in ("2024GQ0000005", "2024GQ0000006") + } + + +def test_a_vacancy_only_source_is_retained(tmp_path, monkeypatch): + fixture = build_fixture( + tmp_path, + monkeypatch, + household_members=( + Member("psam_husa.csv", csv_bytes(HOUSEHOLD_HEADER, BASE_HOUSEHOLDS[3:4])), + ), + person_members=(Member("psam_pusa.csv", csv_bytes(PERSON_HEADER, ())),), + ) + result = fixture.produce() + assert household_keys(result.households) == ["2024HU0000004"] + assert len(result.persons) == 0 + assert codes_map(result) == {"2024HU0000004": EXPECTED_CODES["2024HU0000004"]} + + +@pytest.mark.parametrize( + "serialnos", + [ + pytest.param((1,), id="integer_element"), + pytest.param((None,), id="none_element"), + pytest.param((b"2024HU0000001",), id="bytes_element"), + ], +) +def test_non_string_selection_elements_refuse(acs, serialnos): + refuses(acs, acs.produce, serialnos=serialnos) + + +def test_an_explicit_full_selection_matches_the_none_selection(acs): + whole = acs.produce(output_dir=acs.output_dir("whole")) + explicit = acs.produce( + output_dir=acs.output_dir("explicit"), serialnos=BASE_SERIALNOS + ) + assert explicit.projection_json == whole.projection_json + + +def test_a_published_artifact_survives_a_later_refused_produce(acs): + output_dir = acs.output_dir("kept") + produced = acs.produce(output_dir=output_dir) + before = {path.name: path.read_bytes() for path in sorted(output_dir.iterdir())} + acs.household_zip.write_bytes(b"not an archive\n") + acs.pin() + refuses(acs, acs.produce, output_dir=acs.output_dir("doomed")) + assert {path.name: path.read_bytes() for path in sorted(output_dir.iterdir())} == ( + before + ) + assert before[NAMES["projection_filename"]].rstrip(b"\n") == ( + produced.projection_json.rstrip(b"\n") + ) + + +def test_readback_with_a_missing_snapshot_root_refuses(acs): + output_dir = acs.output_dir() + acs.produce(output_dir=output_dir) + refuses( + acs, + source.load_acs_housing_source, + acs.source_dir, + output_dir, + snapshot_root=acs.tmp_path / "absent-root", + ) + + +def test_wrong_size_candidate_refuses_as_a_candidate_not_as_a_parent(acs): + """Readback size failures must name the candidate, not the pinned archives.""" + for filename, mutate in ( + (NAMES["projection_filename"], lambda raw: raw + b" "), + (NAMES["projection_filename"], lambda raw: raw[:20]), + (NAMES["receipt_filename"], lambda raw: raw + b" "), + (NAMES["receipt_filename"], lambda raw: raw[:20]), + ): + output_dir = acs.output_dir() + acs.produce(output_dir=output_dir) + path = output_dir / filename + make_writable(path).write_bytes(mutate(path.read_bytes())) + assert refuses(acs, acs.load, output_dir) == "RECONSTRUCTION_SIZE" + + +def test_parent_archive_size_failures_keep_their_own_reason_codes(acs): + """The candidate code must not be borrowed by, or borrow from, the parents.""" + output_dir = acs.output_dir() + acs.produce(output_dir=output_dir) + intact = acs.household_zip.read_bytes() + acs.household_zip.write_bytes(intact[:-1]) + assert refuses(acs, acs.load, output_dir) == "SOURCE_SIZE" + acs.household_zip.write_bytes(intact + b"\x00") + assert refuses(acs, acs.load, output_dir) == "FILE_TOO_LARGE" + acs.household_zip.write_bytes(intact) + assert acs.load(output_dir) is not None + + +def test_failure_record_write_failure_does_not_mask_the_original_refusal( + acs, monkeypatch +): + """A failed failure record must never replace the refusal it was recording.""" + actual_write = source._write + attempted = [] + + def refuse_the_failure_record(path, data, cap): + if Path(path).name == "failure.json": + attempted.append(Path(path).name) + raise OSError(28, "No space left on device") + return actual_write(path, data, cap) + + monkeypatch.setattr(source, "_write", refuse_the_failure_record) + corrupted = bytearray(acs.household_zip.read_bytes()) + corrupted[-1] ^= 0xFF + acs.household_zip.write_bytes(bytes(corrupted)) + assert refuses(acs, acs.produce) == "SOURCE_SHA256" + assert attempted == ["failure.json"] diff --git a/packages/microcosm-build/tests/test_us_acs_multispine.py b/packages/microcosm-build/tests/test_us_acs_multispine.py index afc38b72d..fbaa393b7 100644 --- a/packages/microcosm-build/tests/test_us_acs_multispine.py +++ b/packages/microcosm-build/tests/test_us_acs_multispine.py @@ -356,15 +356,25 @@ def _test_puma_ladder() -> UsPumaLadder: tract_overlap_puma=puma.copy(), tract_overlap_tract=np.asarray([6_001_000_100, 36_001_000_100], dtype=np.int64), tract_overlap_population=population.copy(), + joint_overlap_puma=puma.copy(), + joint_overlap_tract=np.asarray([6_001_000_100, 36_001_000_100], dtype=np.int64), + joint_overlap_cd=np.asarray([601, 3_601], dtype=np.int64), + joint_overlap_population=np.asarray([100, 200], dtype=np.int64), metadata={ - "schema_version": 1, + "schema_version": 2, "kind": "us_puma_ladder", "puma_vintage": "2020_puma", "sampling_basis": "population", "layers": { - "congressional_district": {"vintage": "119th_congress"}, - "county": {"vintage": "2020_census"}, - "tract": {"vintage": "2020_census"}, + "congressional_district": { + "vintage": "119th_congress", + "source": "invented joint fixture", + }, + "county": { + "vintage": "2020_census", + "source": "invented joint fixture", + }, + "tract": {"vintage": "2020_census", "source": "invented joint fixture"}, }, }, ) diff --git a/packages/microcosm-build/tests/test_us_acs_multispine_legacy_builder.py b/packages/microcosm-build/tests/test_us_acs_multispine_legacy_builder.py index 31742ad61..47e423dcb 100644 --- a/packages/microcosm-build/tests/test_us_acs_multispine_legacy_builder.py +++ b/packages/microcosm-build/tests/test_us_acs_multispine_legacy_builder.py @@ -193,15 +193,25 @@ def test_main_wires_verified_sources_transfer_audit_export_and_summary( tract_overlap_puma=np.asarray([100_100], dtype=np.int64), tract_overlap_tract=np.asarray([1_001_000_100], dtype=np.int64), tract_overlap_population=np.asarray([100.0]), + joint_overlap_puma=np.asarray([100_100], dtype=np.int64), + joint_overlap_tract=np.asarray([1_001_000_100], dtype=np.int64), + joint_overlap_cd=np.asarray([101], dtype=np.int64), + joint_overlap_population=np.asarray([100], dtype=np.int64), metadata={ - "schema_version": 1, + "schema_version": 2, "kind": "us_puma_ladder", "puma_vintage": "2020_puma", "sampling_basis": "population", "layers": { - "congressional_district": {"vintage": "119th_congress"}, - "county": {"vintage": "2020_census"}, - "tract": {"vintage": "2020_census"}, + "congressional_district": { + "vintage": "119th_congress", + "source": "invented joint fixture", + }, + "county": { + "vintage": "2020_census", + "source": "invented joint fixture", + }, + "tract": {"vintage": "2020_census", "source": "invented joint fixture"}, }, }, ) diff --git a/packages/microcosm-build/tests/test_us_acs_person_coverage_authentication.py b/packages/microcosm-build/tests/test_us_acs_person_coverage_authentication.py new file mode 100644 index 000000000..ea76a97ab --- /dev/null +++ b/packages/microcosm-build/tests/test_us_acs_person_coverage_authentication.py @@ -0,0 +1,588 @@ +"""Invented-only source authority and explicitly ungranted native consistency.""" + +import csv +import hashlib +import importlib.util +import io +import json +import os +import sys +from dataclasses import FrozenInstanceError, replace +from pathlib import Path +from types import SimpleNamespace + +import pandas as pd +import pytest + +from microcosm.build.us_runtime import acs_housing_universe_source as custody +from microcosm.build.us_runtime import acs_person_coverage_authentication as owner +from microcosm.build.us_runtime import acs_person_coverage_columns as literal +from microcosm.build.us_runtime.acs_inputs import map_acs_native_inputs +from microcosm.build.us_runtime.acs_pums import AcsPumsSource, build_acs_pums_unit_frame + + +def _helper(filename): + spec = importlib.util.spec_from_file_location( + "coverage_helpers_" + filename, Path(__file__).with_name(filename + ".py") + ) + module = importlib.util.module_from_spec(spec) + sys.modules[spec.name] = module + spec.loader.exec_module(module) + return module + + +_authority = _helper("test_us_acs_housing_source") +Member = _authority.Member +build_fixture = _authority.build_fixture +flip_archive_byte = _authority.flip_archive_byte +write_archive = _authority.write_archive +_native = _helper("test_us_acs_pums") +_household, _person = _native._household, _native._person + + +def _csv(rows, *, bom=False, ending="\n"): + stream = io.StringIO(newline="") + writer = csv.DictWriter( + stream, fieldnames=list(rows[0]), quoting=csv.QUOTE_ALL, lineterminator=ending + ) + writer.writeheader() + writer.writerows(rows) + return (("\ufeff" if bom else "") + stream.getvalue()).encode() + + +@pytest.fixture +def invented(tmp_path, monkeypatch): + monkeypatch.setattr( + custody.shutil, "disk_usage", lambda _p: SimpleNamespace(free=64 * 1024**3) + ) + h = [_household("2024HU0000001", NP=2), _household("2024HU0000002", NP=1)] + p = [ + _person("2024HU0000001", 1, 20, AGEP=30, MIL="1", ESR="4"), + _person("2024HU0000001", 2, 25, AGEP=15, MIL="", ESR=""), + _person("2024HU0000002", 1, 20, AGEP=80, MIL="4", ESR="6"), + ] + fixture = build_fixture( + tmp_path, + monkeypatch, + household_members=(Member("psam_husa.csv", _csv(h)),), + person_members=( + Member("psam_pusa.csv", _csv(p[:1])), + Member("psam_pusb.csv", _csv(p[1:])), + ), + ) + frame = map_acs_native_inputs( + build_acs_pums_unit_frame( + AcsPumsSource(fixture.household_zip, fixture.person_zip) + )[0] + ).frame + return fixture, frame, h, p + + +def _load(invented, **kwargs): + fixture, frame, _h, _p = invented + return owner.load_authenticated_acs_person_coverage( + fixture.source_dir, snapshot_root=fixture.snapshot_root, frame=frame, **kwargs + ) + + +def _rewrite( + invented, *, persons=None, households=None, members=None, bom=False, ending="\n" +): + fixture, _frame, h, p = invented + if households is not None: + write_archive( + fixture.household_zip, (Member("psam_husa.csv", _csv(households)),) + ) + if persons is not None or members is not None: + p = persons if persons is not None else p + write_archive( + fixture.person_zip, + members + if members is not None + else ( + Member("psam_pusa.csv", _csv(p[:1], bom=bom, ending=ending)), + Member("psam_pusb.csv", _csv(p[1:], bom=bom, ending=ending)), + ), + ) + fixture.pin() + + +def _refuses(code): + return pytest.raises(owner.ACSCoverageAuthenticationError, match="^" + code + "$") + + +def test_source_authentication_preserves_low_receipt_and_parent_without_population_grant( + invented, +): + fixture, frame, _h, _p = invented + before = custody.frame_content_sha256(frame) + original_tables = {e: frame.table(e).copy(deep=True) for e in frame.entities} + result = _load(invented) + table, receipt = literal.read_acs_person_coverage_columns( + AcsPumsSource(fixture.household_zip, fixture.person_zip), + person_keys=owner._native_roster(frame)[0], + chunksize=3, + ) + pd.testing.assert_frame_equal(result.table, table) + assert result.receipt["original_literal_receipt"] == receipt + assert receipt["source_authenticated"] is False + assert result.receipt["source_authenticated"] is True + assert result.native_binding.receipt["population_binding_authenticated"] is False + assert ( + result.native_binding.receipt["missing_authority"] + == "closed_prepared_acs_native_population" + ) + assert ( + result.native_binding.receipt["original_age"]["relation"] == "numeric_identity" + ) + for flag in ( + "domain_assignment_authenticated", + "period_harmonized", + "release_eligible", + "cross_survey_coverage_equivalence_established", + "population_binding_authenticated", + ): + assert result.receipt[flag] is False + assert custody.frame_content_sha256(frame) == before + for entity, original in original_tables.items(): + pd.testing.assert_frame_equal(frame.table(entity), original) + assert ( + owner.verify_acs_coverage_native_consistency(result, frame) + == result.native_binding + ) + assert [m["rows"] for m in result.receipt["members"]["person"]] == [1, 2] + assert all(len(m["sha256"]) == 64 for m in result.receipt["members"]["person"]) + + +def test_first_frame_hash_precedes_the_roster_snapshot(invented, monkeypatch): + """The returned binding must describe one state, including at hash entry.""" + _fixture, frame, _h, _p = invented + original_hash = custody.frame_content_sha256 + first = True + + def at_hash_entry(value): + nonlocal first + if first: + first = False + value.person.loc[:, "source_row_id"] += 100 + return original_hash(value) + + monkeypatch.setattr(custody, "frame_content_sha256", at_hash_entry) + result = _load(invented) + assert ( + owner.verify_acs_coverage_native_consistency(result, frame) + == result.native_binding + ) + + +def test_mutation_during_roster_snapshot_refuses_issuance(invented, monkeypatch): + original_roster = owner._native_roster + first = True + + def after_roster_read(value): + nonlocal first + result = original_roster(value) + if first: + first = False + value.person.loc[:, "source_row_id"] += 100 + return result + + monkeypatch.setattr(owner, "_native_roster", after_roster_read) + with _refuses("NATIVE_FRAME_CHANGED"): + _load(invented) + + +def test_closed_default_manifest_has_no_caller_pin_or_prepared_authority( + invented, monkeypatch +): + fixture, frame, _h, _p = invented + pins = custody._ARCHIVE_PINS + monkeypatch.setattr(custody, "_ARCHIVE_PINS", None) + calls = [] + + def default_manifest(*args, **kwargs): + assert not args and not kwargs + calls.append(True) + return SimpleNamespace( + artifacts=tuple( + SimpleNamespace(role=r, filename=n, sha256=d, size_bytes=s) + for r, n, d, s in pins + ) + ) + + monkeypatch.setattr(custody, "load_acs_source_manifest", default_manifest) + result = _load(invented) + assert len(calls) == 2 + with pytest.raises(TypeError): + _load(invented, manifest={"sha256": "0" * 64}) + for constructor, payload in ( + (owner.AuthenticatedACSPersonCoverage, result.payload), + (owner.UngrantedACSNativeBinding, result.native_binding.receipt_json), + ): + with _refuses("SOURCE_CONSTRUCTOR|BINDING_CONSTRUCTOR"): + constructor(payload) + for forged in ( + frame.person, + {"frame": frame, "frame_sha256": custody.frame_content_sha256(frame)}, + "0" * 64, + ): + with _refuses("NATIVE_FRAME_REQUIRED"): + owner.load_authenticated_acs_person_coverage( + fixture.source_dir, snapshot_root=fixture.snapshot_root, frame=forged + ) + + +@pytest.mark.parametrize( + "field,value,code", + [ + ("SPORDER", 3, "NATIVE_ALIASES"), + ("A_LINENO", 3, "NATIVE_ALIASES"), + ("source_person_id", "2", "NATIVE_ALIASES"), + ("source_household_id", 2, "NATIVE_ALIASES"), + ("source_year", 2023, "NATIVE_VINTAGE"), + ("person_household_id", 2, "NATIVE_ALIASES"), + ("source_row_id", 1, "NATIVE_IDS"), + ], +) +def test_native_alias_and_membership_changes_with_same_person_ids_refuse( + invented, field, value, code +): + invented[1].person.loc[0, field] = value + with _refuses(code): + _load(invented) + + +@pytest.mark.parametrize( + "change", ["serial", "age", "membership", "weight", "source_row"] +) +def test_live_binding_refuses_same_ids_changed_native_parent(invented, change): + _fixture, frame, _h, _p = invented + result = _load(invented) + if change == "serial": + h = frame.table("household") + h["SERIALNO"] = h.SERIALNO.iloc[::-1].to_numpy() + elif change == "age": + frame.person.loc[2, "age"] = 82 + elif change == "membership": + for c in ("person_household_id", "source_household_id"): + frame.person.loc[[1, 2], c] = frame.person.loc[[2, 1], c].to_numpy() + elif change == "weight": + frame.table("household")["extra_cell"] = ( + 123 # all parent cells bind, including additions + ) + else: + frame.person.loc[0, "source_row_id"] = 100 + with _refuses( + "NATIVE_FRAME_CHANGED|NATIVE_HOUSEHOLD_COUNT|NATIVE_CONSISTENCY_REFUSED" + ): + owner.verify_acs_coverage_native_consistency(result, frame) + + +@pytest.mark.parametrize( + "change", + [ + "missing", + "extra", + "duplicate", + "household_np", + "household_missing", + "wrong_key", + "wrong_vintage", + ], +) +def test_source_complete_household_mismatch_refuses(invented, change): + _fixture, _frame, h, p = invented + if change == "missing": + p = p[:2] + elif change == "extra": + p += [{**p[1], "SPORDER": 3}] + elif change == "duplicate": + p[1] = p[0] + elif change == "household_np": + h[0]["NP"] = 1 + elif change == "household_missing": + h = h[:1] + elif change == "wrong_vintage": + p[0]["SERIALNO"] = "2023HU0000001" + else: + p[0]["SPORDER"] = 3 + _rewrite(invented, persons=p, households=h) + with _refuses("SOURCE_PERSON_ROSTER|SOURCE_HOUSEHOLD_ROSTER|SOURCE_DUPLICATE_KEY"): + _load(invented) + + +@pytest.mark.parametrize( + "age,mil,esr", + [ + ("", "", ""), + ("NA", "NA", "1.0"), + ("30 ", " 1", "4 "), + ("30", '1,"quoted"\r\n\t', "4\r\n\t"), + ("30\t", "1", "4"), + ("100", "0", "7"), + ], +) +@pytest.mark.parametrize("ending", ["\n", "\r", "\r\n"]) +def test_bom_multiline_tabs_and_malformed_literals_survive_envelope( + invented, age, mil, esr, ending +): + _fixture, frame, _h, p = invented + p[0].update(AGEP=age, MIL=mil, ESR=esr) + _rewrite(invented, persons=p, bom=True, ending=ending) + result = _load(invented) + assert result.table.loc[0, ["AGEP", "MIL", "ESR"]].tolist() == [age, mil, esr] + assert not result.table.isna().any().any() + assert frame.person.loc[0, "age"] == 30 + if age != "30": + assert result.table.loc[0, "MIL_state"] == "age_unresolved" + assert result.native_binding.receipt["original_age"]["relation"] == "unproven" + + +@pytest.mark.parametrize( + "column,value", [("AGEP", 82), ("A_AGE", 82), ("age", 82), ("age", float("nan"))] +) +def test_model_age_never_fills_or_authenticates_original_relation( + invented, column, value +): + frame = invented[1] + if isinstance(value, float): + frame.person[column] = frame.person[column].astype("float64") + frame.person.loc[2, column] = value + result = _load(invented) + assert result.table.loc[2, "AGEP"] == "80" + assert result.native_binding.receipt["original_age"]["mismatching_rows"] == 1 + assert result.native_binding.receipt["population_binding_authenticated"] is False + + +def test_candidate_reconstructed_before_comparison_and_self_rehash_cannot_authorize( + invented, monkeypatch +): + result = _load(invented) + candidate = invented[0].tmp_path / "candidate.coverage" + candidate.write_bytes(result.payload) + assert _load(invented, candidate_path=candidate).payload == result.payload + header_raw, body = result._parts() + header = json.loads(header_raw) + tampered = body.replace(b'"30","1","4"', b'"30","2","4"', 1) + assert tampered != body + header["body_sha256"] = hashlib.sha256(tampered).hexdigest() + raw = owner._json(header) + candidate.write_bytes(owner._MAGIC + len(raw).to_bytes(4, "big") + raw + tampered) + with _refuses("CANDIDATE_MISMATCH"): + _load(invented, candidate_path=candidate) + flip_archive_byte(invented[0].person_zip) + original_copy = custody._copy + + def guarded_copy(source, destination, *args, **kwargs): + assert source != candidate, ( + "candidate was accessed before source reconstruction" + ) + return original_copy(source, destination, *args, **kwargs) + + monkeypatch.setattr(custody, "_copy", guarded_copy) + with _refuses("SOURCE_RECONSTRUCTION_REFUSED"): + _load(invented, candidate_path=candidate) + + +@pytest.mark.parametrize("replacement", [None, "", "different"]) +def test_candidate_null_empty_and_field_contract_tampering_refuse( + invented, replacement +): + result = _load(invented) + header, body = result._parts() + rows = [json.loads(row) for row in body.splitlines()] + rows[1][3] = replacement + if replacement == "": + metadata = json.loads(header) + metadata["field_contract"]["fields"]["MIL"]["minimum_age"] = 16 + else: + metadata = json.loads(header) + changed_body = b"".join(owner._json(row) + b"\n" for row in rows) + metadata.update( + body_sha256=hashlib.sha256(changed_body).hexdigest(), + body_bytes=len(changed_body), + ) + changed_header = owner._json(metadata) + candidate = invented[0].tmp_path / "candidate.coverage" + candidate.write_bytes( + owner._MAGIC + + len(changed_header).to_bytes(4, "big") + + changed_header + + changed_body + ) + with _refuses("CANDIDATE_MISMATCH|SOURCE_RECONSTRUCTION_REFUSED"): + _load(invented, candidate_path=candidate) + + +def test_views_and_closed_objects_cannot_mutate_owned_evidence(invented): + result = _load(invented) + before = result.payload + view = result.table + view.loc[0, "MIL"] = "2" + result.receipt["field_contract"].clear() + result.native_binding.receipt["population_binding_authenticated"] = True + assert result.payload == before + assert result.table.loc[0, "MIL"] == "1" + assert result.native_binding.receipt["population_binding_authenticated"] is False + with pytest.raises(FrozenInstanceError): + result.payload = b"forged" + with _refuses("SOURCE_CONSTRUCTOR"): + replace(result, payload=b"forged") + + +@pytest.mark.parametrize( + "kind", ["duplicate", "case_duplicate", "unsafe", "symlink", "prefix_non_csv"] +) +def test_entire_zip_directory_is_checked_before_literal_reader( + invented, monkeypatch, kind +): + p = invented[3] + extra = { + "duplicate": Member("psam_pusa.csv", _csv(p)), + "case_duplicate": Member("PSAM_PUSA.CSV", _csv(p)), + "unsafe": Member("../unrelated.txt", b"ignored"), + "symlink": Member("unrelated.txt", b"target", external_attr=0o120777 << 16), + "prefix_non_csv": Member("psam_pus.txt", b"ignored"), + }[kind] + if kind == "duplicate": + with pytest.warns(UserWarning, match="Duplicate name"): + _rewrite(invented, members=(Member("psam_pusa.csv", _csv(p)), extra)) + else: + _rewrite(invented, members=(Member("psam_pusa.csv", _csv(p)), extra)) + monkeypatch.setattr( + literal, + "read_acs_person_coverage_columns", + lambda *a, **k: pytest.fail("reader entered"), + ) + with _refuses( + "ZIP_DUPLICATE_MEMBER|ZIP_MEMBER_PATH|ZIP_MEMBER_TYPE|ZIP_PREFIX_NONCSV" + ): + _load(invented) + + +@pytest.mark.parametrize( + "target,constant,value,code", + [ + (custody, "_MEMBER_COUNT_MAX", 1, "ZIP_MEMBER_COUNT"), + (custody, "_MEMBER_MAX", 100, "ZIP_MEMBER_SIZE"), + (custody, "_EXPANDED_MAX", 100, "ZIP_EXPANDED_SIZE"), + (owner, "MAX_CSV_HEADER_BYTES", 30, "CSV_RECORD_BYTES"), + (owner, "MAX_RECORD_BYTES", 64, "CSV_RECORD_BYTES"), + (owner, "MAX_TOKEN_BYTES", 10, "CSV_TOKEN_BYTES"), + (owner, "MAX_BODY_BYTES", 100, "SELECTED_BODY_BUDGET"), + (owner, "MAX_HEADER_BYTES", 100, "CANONICAL_SIZE"), + (literal, "MAX_ROWS", 1, "SOURCE_ROWS"), + (literal, "MAX_SELECTED_ROWS", 2, "NATIVE_ROWS"), + ], +) +def test_small_limits_refuse_before_relevant_allocation( + invented, monkeypatch, target, constant, value, code +): + monkeypatch.setattr(target, constant, value) + with _refuses(code): + _load(invented) + + +@pytest.mark.parametrize("kind", ["corrupt", "growth", "symlink", "fifo"]) +def test_exact_capture_refusals_are_static_and_hide_cause_chains(invented, kind): + path = invented[0].person_zip + if kind == "corrupt": + flip_archive_byte(path) + elif kind == "growth": + with path.open("ab") as stream: + stream.write(b"extra") + else: + saved = path.with_suffix(".saved") + path.rename(saved) + if kind == "symlink": + path.symlink_to(saved) + else: + os.mkfifo(path) + # Keep directory roster exact; saved bytes live outside the source dir. + saved.rename(invented[0].tmp_path / "saved.zip") + with _refuses("SOURCE_RECONSTRUCTION_REFUSED") as exc: + _load(invented) + assert exc.value.__cause__ is None + assert exc.value.__suppress_context__ + + +@pytest.mark.parametrize( + "race", ["private_bytes", "original_replace", "pins", "producer", "frame"] +) +def test_races_refuse_before_issuance(invented, monkeypatch, race): + original = literal.read_acs_person_coverage_columns + + def raced(source, **kwargs): + result = original(source, **kwargs) + if race == "private_bytes": + source.person_zip.chmod(0o600) + flip_archive_byte(source.person_zip) + elif race == "original_replace": + # Replace during _copy instead, below. + pass + elif race == "pins": + monkeypatch.setattr(custody, "_ARCHIVE_PINS", ()) + elif race == "producer": + monkeypatch.setattr(owner, "MAX_TOKEN_BYTES", owner.MAX_TOKEN_BYTES - 1) + else: + invented[1].person.loc[0, "age"] = 31 + return result + + if race == "original_replace": + old_identity = custody._identity + changed = False + + def replaced_identity(value): + nonlocal changed + if not changed: + changed = True + path = invented[0].household_zip + raw = path.read_bytes() + path.unlink() + path.write_bytes(raw) + return old_identity(value) + + monkeypatch.setattr(custody, "_identity", replaced_identity) + monkeypatch.setattr(literal, "read_acs_person_coverage_columns", raced) + with _refuses( + "SNAPSHOT_CHANGED|SOURCE_RECONSTRUCTION_REFUSED|PRODUCER_CHANGED|NATIVE_FRAME_CHANGED" + ): + _load(invented) + + +@pytest.mark.parametrize("ending", [b"\n", b"\r", b"\r\n", b""]) +def test_raw_record_byte_boundary_precedes_decoding_and_preserves_endings( + monkeypatch, ending +): + header = b"A,B\n" + body = b'"x\r\n\ty",z' + ending + monkeypatch.setattr(owner, "MAX_RECORD_BYTES", len(body)) + assert list(owner._records(io.BytesIO(header + body))) == [header, body] + monkeypatch.setattr(owner, "MAX_RECORD_BYTES", len(body) - 1) + with _refuses("CSV_RECORD_BYTES"): + list(owner._records(io.BytesIO(header + body))) + + +@pytest.mark.parametrize( + "value", + [ + "", + None, + "\r\n\t", + '😀"\\\b\f', + ["", None, "é"], + {"b": True, "a": [1, -2, False]}, + ], +) +def test_canonical_size_is_checked_before_string_encoding(monkeypatch, value): + expected = json.dumps( + value, sort_keys=True, separators=(",", ":"), ensure_ascii=True + ).encode() + assert owner._json(value, len(expected)) == expected + + def forbidden_encoder(*a, **k): + pytest.fail("JSON encoder allocated before preflight refusal") + + monkeypatch.setattr(owner.json, "JSONEncoder", forbidden_encoder) + with _refuses("CANONICAL_SIZE"): + owner._json(value, len(expected) - 1) diff --git a/packages/microcosm-build/tests/test_us_acs_person_coverage_columns.py b/packages/microcosm-build/tests/test_us_acs_person_coverage_columns.py new file mode 100644 index 000000000..5ac09804c --- /dev/null +++ b/packages/microcosm-build/tests/test_us_acs_person_coverage_columns.py @@ -0,0 +1,496 @@ +"""Literal coverage projections from invented, multi-member ACS archives.""" + +import csv +import hashlib +import io +import json +from zipfile import ZipFile + +import pandas as pd +import pytest + +from microcosm.build.us_runtime import acs_person_coverage_columns as module +from microcosm.build.us_runtime import acs_pums + + +def _row(serial="2024HU0000001", order="1", age="30", mil="4", esr="6"): + return dict(zip(module.READ_COLUMNS, (serial, order, age, mil, esr), strict=True)) + + +def _archive(tmp_path, members): + path = tmp_path / "persons.zip" + with ZipFile(path, "w") as archive: + for name, rows in members.items(): + stream = io.StringIO(newline="") + writer = csv.DictWriter( + stream, fieldnames=module.READ_COLUMNS, lineterminator="\n" + ) + writer.writeheader() + writer.writerows(rows) + archive.writestr(name, stream.getvalue()) + return acs_pums.AcsPumsSource( + tmp_path / "unused-households.zip", path, max_households=1 + ) + + +def _keys(rows): + return pd.DataFrame(rows).loc[:, list(module.KEYS)] + + +def test_private_scanner_exhausts_members_despite_consumer_return_value(tmp_path): + first = _row(order="02", age="16", mil="", esr="1") + second = _row(order="01", age="17", mil="4", esr=" 6") + third = _row("2024GQ0000001", age="", mil='x,"\t', esr="") + source = _archive( + tmp_path, + {"psam_pusb.csv": [second, third], "psam_pusa.csv": [first]}, + ) + received = [] + + def consume(cells): + received.append(cells) + return True # No return value can turn a prefix into a successful scan. + + members, rows = module._scan_acs_person_coverage(source, consume) + assert members == ["psam_pusa.csv", "psam_pusb.csv"] + assert rows == 3 + assert received == [ + ("2024HU0000001", "02", "16", "", "1"), + ("2024HU0000001", "01", "17", "4", " 6"), + ("2024GQ0000001", "1", "", 'x,"\t', ""), + ] + + +@pytest.mark.parametrize( + "tail,match", + [ + ("2024HU0000009,1,30,4,6,EXTRA\n", "record width"), + ("2024HU0000009,1,30,4,6\x00\n", "forbidden control"), + ('2024HU0000009,1,30,"unfinished,6\n', "invalid literal CSV"), + ], +) +def test_private_scanner_refuses_invalid_tail_after_delivering_expected_rows( + tmp_path, tail, match +): + source = _raw_archive( + tmp_path, + "SERIALNO,SPORDER,AGEP,MIL,ESR\n2024HU0000001,1,30,4,6\n" + tail, + ) + received = [] + with pytest.raises(ValueError, match=match): + module._scan_acs_person_coverage(source, received.append) + assert received == [("2024HU0000001", "1", "30", "4", "6")] + + +def test_exact_household_roster_across_members_preserves_blanks_and_request_order( + tmp_path, +): + rows = [ + _row(mil="1", esr="4"), + _row(order="2", age="10", mil="", esr=""), + _row("2024GQ0000001", age="16", mil="", esr="1"), + ] + unrelated = _row("2024HU0000009") + source = _archive( + tmp_path, + { + "nested/psam_pusb.csv": [rows[0], unrelated], + "psam_pusa.csv": [rows[2], rows[1]], + }, + ) + requested = _keys([rows[1], rows[2], rows[0]]) + requested["SPORDER"] = requested.SPORDER.astype("int64") + result, receipt = module.read_acs_person_coverage_columns( + source, person_keys=requested, chunksize=1 + ) + assert result.MIL.tolist() == ["", "", "1"] + assert result.ESR.tolist() == ["", "1", "4"] + assert result.MIL_state.tolist() == [ + "outside_age_universe", + "outside_age_universe", + "observed_code", + ] + assert result.ESR_state.tolist() == [ + "outside_age_universe", + "observed_code", + "observed_code", + ] + assert result.SERIALNO.tolist() == requested.SERIALNO.tolist() + assert result.SPORDER.tolist() == requested.SPORDER.tolist() + assert not result.isna().any().any() + assert receipt["source_rows_streamed"] == 4 + assert receipt["selected_person_rows"] == 3 + assert receipt["source_authenticated"] is False + assert receipt["release_eligible"] is False + assert receipt["legacy_max_households_applied"] is False + assert receipt["coverage_status"] == "literal_source_fields_only" + second, other = module.read_acs_person_coverage_columns( + source, person_keys=requested, chunksize=3 + ) + pd.testing.assert_frame_equal(result, second) + assert other == receipt + + +@pytest.mark.parametrize( + "age,mil,esr,mil_state,esr_state", + [ + ("17", "2", "2", "observed_code", "observed_code"), + ("45", "3", "5", "observed_code", "observed_code"), + ("30", "", "", "missing_in_universe", "missing_in_universe"), + ("30", "0", "7", "unlabelled_code", "unlabelled_code"), + ("30", "NA", "1.0", "unlabelled_code", "unlabelled_code"), + ("30", " 1", "4 ", "unlabelled_code", "unlabelled_code"), + ("15", "1", "4", "value_below_age_universe", "value_below_age_universe"), + ("", "1", "4", "age_unresolved", "age_unresolved"), + ("100", "4", "6", "age_unresolved", "age_unresolved"), + ], +) +def test_unresolved_states_preserve_literal_source_tokens( + tmp_path, age, mil, esr, mil_state, esr_state +): + row = _row(age=age, mil=mil, esr=esr) + source = _archive(tmp_path, {"psam_pusa.csv": [row]}) + table, _ = module.read_acs_person_coverage_columns(source, person_keys=_keys([row])) + assert table.loc[0, ["AGEP", "MIL", "ESR"]].tolist() == [age, mil, esr] + assert table.MIL_state.tolist() == [mil_state] + assert table.ESR_state.tolist() == [esr_state] + + +@pytest.mark.parametrize( + "mutation,match", + [ + ("extra", "extra selected-household"), + ("missing", "native roster differ"), + ("duplicate", "keys repeat"), + ("wrong_order", "native roster differ"), + ], +) +def test_partial_household_or_conflicting_native_roster_refuses( + tmp_path, mutation, match +): + rows = [_row(), _row(order="2")] + observed = rows.copy() + if mutation == "extra": + observed += [_row(order="3")] + elif mutation == "missing": + observed.pop() + elif mutation == "duplicate": + observed[1] = rows[0] + else: + observed[1] = _row(order="3") + source = _archive(tmp_path, {"psam_pusa.csv": observed}) + with pytest.raises(ValueError, match=match): + module.read_acs_person_coverage_columns( + source, person_keys=_keys(rows), chunksize=1 + ) + + +@pytest.mark.parametrize( + "serial,order", + [ + ("2023HU0000001", "1"), + ("2024HU0000001", "0"), + ("2024HU0000001", "21"), + ("2024HU0000001", "1.0"), + ("2024HU0000001\x00other", "1"), + ("2024HU0000001", "1\x00other"), + (None, "1"), + ("2024HU0000001", True), + ], +) +def test_invalid_native_keys_refuse_before_file_read(tmp_path, serial, order): + source = acs_pums.AcsPumsSource( + tmp_path / "absent-h.zip", tmp_path / "absent-p.zip" + ) + with pytest.raises(ValueError, match="native"): + module.read_acs_person_coverage_columns( + source, person_keys=pd.DataFrame({"SERIALNO": [serial], "SPORDER": [order]}) + ) + + +@pytest.mark.parametrize( + "mutation", ["missing_column", "duplicate_column", "duplicate_member"] +) +def test_ambiguous_source_schema_refuses(tmp_path, mutation): + source = _archive(tmp_path, {"psam_pusa.csv": [_row()]}) + with ZipFile(source.person_zip, "r") as archive: + raw = archive.read("psam_pusa.csv") + if mutation == "missing_column": + raw = raw.replace(b"MIL", b"OTHER") + elif mutation == "duplicate_column": + raw = raw.replace(b"ESR", b"MIL") + with ZipFile(source.person_zip, "w") as archive: + archive.writestr("psam_pusa.csv", raw) + if mutation == "duplicate_member": + with pytest.warns(UserWarning, match="Duplicate name"): + archive.writestr("psam_pusa.csv", raw) + with pytest.raises(ValueError, match="missing or duplicated"): + module.read_acs_person_coverage_columns(source, person_keys=_keys([_row()])) + + +def test_explicit_row_bound_refuses_without_truncation(tmp_path, monkeypatch): + row = _row() + source = _archive(tmp_path, {"psam_pusa.csv": [row, _row("2024HU0000002")]}) + monkeypatch.setattr(module, "MAX_ROWS", 1) + with pytest.raises(ValueError, match="row bound"): + module.read_acs_person_coverage_columns( + source, person_keys=_keys([row]), chunksize=1 + ) + + +def test_selected_row_ceiling_refuses_before_opening_the_source(tmp_path, monkeypatch): + source = acs_pums.AcsPumsSource( + tmp_path / "absent-h.zip", tmp_path / "absent-p.zip" + ) + monkeypatch.setattr(module, "MAX_SELECTED_ROWS", 1) + with pytest.raises(ValueError, match="selected person count is outside the bound"): + module.read_acs_person_coverage_columns( + source, person_keys=_keys([_row(), _row(order="2")]) + ) + + +def test_contract_is_owned_and_separate_from_legacy_native_roster(): + contract = module.coverage_field_contract() + assert contract["fields"]["MIL"]["minimum_age"] == 17 + assert contract["fields"]["ESR"]["minimum_age"] == 16 + contract["fields"]["MIL"]["codes"].clear() + assert len(module.coverage_field_contract()["fields"]["MIL"]["codes"]) == 4 + assert not {"MIL", "ESR"}.intersection(acs_pums._PERSON_REQUIRED) + + +def _raw_archive(tmp_path, raw): + source = acs_pums.AcsPumsSource( + tmp_path / "unused-households.zip", tmp_path / "literal-persons.zip" + ) + with ZipFile(source.person_zip, "w") as archive: + archive.writestr("psam_pusa.csv", raw) + return source + + +@pytest.mark.parametrize("field", module.READ_COLUMNS) +def test_nul_suffix_refuses_before_truncating_value_or_identity(tmp_path, field): + row = _row(mil="1") + row[field] += "\x00garbage" + source = _archive(tmp_path, {"psam_pusa.csv": [row]}) + with pytest.raises(ValueError, match="control"): + module.read_acs_person_coverage_columns(source, person_keys=_keys([_row()])) + + +@pytest.mark.parametrize( + "control", + [chr(code) for code in range(32) if code not in (9, 10, 13)] + + [chr(code) for code in range(127, 160)], +) +def test_non_csv_control_characters_refuse(tmp_path, control): + source = _archive(tmp_path, {"psam_pusa.csv": [_row(mil="1" + control)]}) + with pytest.raises(ValueError, match="control"): + module.read_acs_person_coverage_columns(source, person_keys=_keys([_row()])) + + +@pytest.mark.parametrize("location", ["header", "ignored_column", "unselected_row"]) +def test_control_validation_precedes_column_and_household_selection(tmp_path, location): + header = "SERIALNO,SPORDER,AGEP,MIL,ESR,OTHER\n" + body = "2024HU0000001,1,30,1,6,ok\n" + if location == "header": + header = header.replace("OTHER", "OTHER\x00suffix") + elif location == "ignored_column": + body = body.replace("ok", "ok\x00suffix") + else: + body += "2024HU0000009,1,30,1\x00suffix,6,ok\n" + source = _raw_archive(tmp_path, header + body) + with pytest.raises(ValueError, match="control"): + module.read_acs_person_coverage_columns(source, person_keys=_keys([_row()])) + + +@pytest.mark.parametrize("serial", ["2024HU0000001", "2024HU0000009"]) +@pytest.mark.parametrize( + "tail", + ["1,30,4", "1,30,4,6", "1,30,4,6,ok,extra", "1,30,4,6,ok,"], + ids=["missing_required", "missing_unused", "too_many", "trailing_delimiter"], +) +def test_full_record_width_refuses_even_outside_projection(tmp_path, serial, tail): + source = _raw_archive( + tmp_path, + "SERIALNO,SPORDER,AGEP,MIL,ESR,OTHER\n" + serial + "," + tail + "\n", + ) + with pytest.raises(ValueError, match="record width"): + module.read_acs_person_coverage_columns(source, person_keys=_keys([_row()])) + + +@pytest.mark.parametrize( + "body", + ["\n", '2024HU0000001,1,30,4,"unterminated\n', '2024HU0000001,1,30,"1"garbage,6\n'], +) +def test_blank_records_and_malformed_quoting_refuse(tmp_path, body): + source = _raw_archive(tmp_path, "SERIALNO,SPORDER,AGEP,MIL,ESR\n" + body) + with pytest.raises(ValueError, match="CSV|record width"): + module.read_acs_person_coverage_columns(source, person_keys=_keys([_row()])) + + +@pytest.mark.parametrize("line_end", ["\n", "\r", "\r\n"]) +def test_quoted_delimiters_newlines_tabs_and_digest_preserve_tokens(tmp_path, line_end): + stream = io.StringIO(newline="") + writer = csv.writer(stream, quoting=csv.QUOTE_ALL, lineterminator=line_end) + writer.writerow(["OTHER", "ESR", "MIL", "AGEP", "SPORDER", "SERIALNO"]) + writer.writerow( + [ + 'ignored, "quoted"\r\nfield', + "4\r\n\t", + '1,"quoted"\n\r', + "30", + "01", + "2024HU0000001", + ] + ) + source = _raw_archive(tmp_path, "\ufeff" + stream.getvalue()) + table, receipt = module.read_acs_person_coverage_columns( + source, person_keys=_keys([_row()]), chunksize=1 + ) + expected = [ + { + "SERIALNO": "2024HU0000001", + "SPORDER": 1, + "AGEP": "30", + "MIL": '1,"quoted"\n\r', + "ESR": "4\r\n\t", + "MIL_state": "unlabelled_code", + "ESR_state": "unlabelled_code", + } + ] + assert table.to_dict("records") == expected + assert receipt["source_rows_streamed"] == 1 + + def digest(records): + return hashlib.sha256( + json.dumps( + records, sort_keys=True, separators=(",", ":"), allow_nan=False + ).encode() + ).hexdigest() + + assert receipt["projection_sha256"] == digest(expected) + for token in ("4\n\t", "4\r\t", r"4\r\n\t", "4"): + assert receipt["projection_sha256"] != digest([{**expected[0], "ESR": token}]) + + +@pytest.mark.parametrize("field", ["AGEP", "MIL", "ESR"]) +@pytest.mark.parametrize("control", ["\t", "\r", "\n", "\r\n"]) +def test_accepted_csv_whitespace_is_literal_and_never_an_observed_code( + tmp_path, field, control +): + row = _row(mil="1", esr="4") + row[field] += control + stream = io.StringIO(newline="") + writer = csv.DictWriter( + stream, fieldnames=module.READ_COLUMNS, quoting=csv.QUOTE_ALL + ) + writer.writeheader() + writer.writerow(row) + source = _raw_archive(tmp_path, stream.getvalue()) + table, _ = module.read_acs_person_coverage_columns( + source, person_keys=_keys([_row()]) + ) + assert table.loc[0, field] == row[field] + if field == "AGEP": + assert ( + table.MIL_state.tolist() == table.ESR_state.tolist() == ["age_unresolved"] + ) + else: + assert table[field + "_state"].tolist() == ["unlabelled_code"] + + +@pytest.mark.parametrize("field", module.KEYS) +@pytest.mark.parametrize("suffix", ["\t", "\n", "\r\n", " other", ",other"]) +def test_literal_identity_suffix_cannot_join_canonical_requested_key( + tmp_path, field, suffix +): + row = _row() + row[field] += suffix + source = _archive(tmp_path, {"psam_pusa.csv": [row]}) + with pytest.raises(ValueError, match="missing|native"): + module.read_acs_person_coverage_columns(source, person_keys=_keys([_row()])) + + +def test_duplicate_canonical_person_order_in_later_member_refuses(tmp_path): + source = _archive( + tmp_path, + {"psam_pusa.csv": [_row(order="01")], "psam_pusb.csv": [_row(order="1")]}, + ) + with pytest.raises(ValueError, match="keys repeat"): + module.read_acs_person_coverage_columns( + source, person_keys=_keys([_row(), _row(order="2")]), chunksize=1 + ) + + +def test_invalid_utf8_refuses(tmp_path): + source = _raw_archive( + tmp_path, b"SERIALNO,SPORDER,AGEP,MIL,ESR\n2024HU0000001,1,30,1\xff,6\n" + ) + with pytest.raises(ValueError): + module.read_acs_person_coverage_columns(source, person_keys=_keys([_row()])) + + +@pytest.mark.parametrize("multiline", [False, True]) +def test_record_character_bound_includes_all_quoted_physical_lines( + tmp_path, monkeypatch, multiline +): + monkeypatch.setattr(module, "MAX_CSV_RECORD_CHARS", 100, raising=False) + suffix = ("x\n" if multiline else "xx") * 60 + source = _archive(tmp_path, {"psam_pusa.csv": [_row(mil=suffix)]}) + with pytest.raises(ValueError, match="record character bound"): + module.read_acs_person_coverage_columns(source, person_keys=_keys([_row()])) + + +@pytest.mark.parametrize("line_end", ["", "\n", "\r\n"]) +def test_record_character_bound_accepts_exact_limit_and_resets_per_record( + tmp_path, monkeypatch, line_end +): + monkeypatch.setattr(module, "MAX_CSV_RECORD_CHARS", 100) + prefix, suffix = '2024HU0000001,1,30,"', '",6' + line_end + token = "x\n" + "x" * (100 - len(prefix) - len(suffix) - 2) + body = prefix + token + suffix + assert len(body) == 100 + source = _raw_archive(tmp_path, "SERIALNO,SPORDER,AGEP,MIL,ESR\n" + body) + table, receipt = module.read_acs_person_coverage_columns( + source, person_keys=_keys([_row()]) + ) + assert table.MIL.tolist() == [token] + assert receipt["source_rows_streamed"] == 1 + + +def test_stream_opens_member_once_and_stops_at_refusal_without_full_validation( + tmp_path, monkeypatch +): + source = _raw_archive( + tmp_path, + "SERIALNO,SPORDER,AGEP,MIL,ESR\n" + + "2024HU0000001,1,30,4,6\n" * 2 + + "2024HU0000009,1,30,4,6\n" * 20_000, + ) + opened, consumed = [], [] + + class GuardedZipFile(ZipFile): + def open(self, name, *args, **kwargs): + opened.append(name) + member = super().open(name, *args, **kwargs) + original_read, original_read1 = member.read, member.read1 + + def bounded(method, size=-1): + assert size >= 0, "unbounded archive read" + result = method(size) + consumed.append(len(result)) + return result + + member.read = lambda size=-1: bounded(original_read, size) + member.read1 = lambda size=-1: bounded(original_read1, size) + return member + + def read(self, *args, **kwargs): + pytest.fail("whole-member validation read") + + monkeypatch.setattr(module, "ZipFile", GuardedZipFile) + with pytest.raises(ValueError, match="extra selected-household"): + module.read_acs_person_coverage_columns( + source, person_keys=_keys([_row()]), chunksize=1 + ) + assert opened == ["psam_pusa.csv"] + assert sum(consumed) < 32_768 diff --git a/packages/microcosm-build/tests/test_us_acs_population_catalogue.py b/packages/microcosm-build/tests/test_us_acs_population_catalogue.py new file mode 100644 index 000000000..ab98b77dc --- /dev/null +++ b/packages/microcosm-build/tests/test_us_acs_population_catalogue.py @@ -0,0 +1,966 @@ +"""Actual pinned invented archives; never a native Frame or fake source issuer.""" + +import _csv +import ast +import copy +import csv +import gc +import hashlib +import inspect +import io +import json +import pickle +import sys +import weakref +import zipfile +from pathlib import Path +from types import CodeType, FunctionType, SimpleNamespace + +import pytest + +from microcosm.build.us_runtime import acs_population_catalogue as owner +from microcosm.frame import Frame + + +def csv_bytes(rows): + stream = io.StringIO(newline="") + writer = csv.DictWriter(stream, list(rows[0]), lineterminator="\n") + writer.writeheader() + writer.writerows(rows) + return stream.getvalue().encode() + + +@pytest.fixture +def invented(tmp_path, monkeypatch): + source, snapshots = tmp_path / "source", tmp_path / "snapshots" + source.mkdir() + snapshots.mkdir() + households = [ + dict( + SERIALNO="2024HU0000001", + TYPEHUGQ="1", + NP="2", + WGTP="0010", + TEN="1", + ST="01", + PUMA="00100", + ), + dict( + SERIALNO="2024GQ0000001", + TYPEHUGQ="2", + NP="1", + WGTP="0", + TEN="", + ST="01", + PUMA="00100", + ), + dict( + SERIALNO="2024HU0000002", + TYPEHUGQ="1", + NP="0", + WGTP="20", + TEN="", + ST="01", + PUMA="00100", + ), + dict( + SERIALNO="2024GQ0000002", + TYPEHUGQ="3", + NP="1", + WGTP="0", + TEN="", + ST="01", + PUMA="00100", + ), + dict( + SERIALNO="2024HU0000003", + TYPEHUGQ="1", + NP="1", + WGTP="30", + TEN="3", + ST="01", + PUMA="00100", + ), + ] + people = [ + dict( + SERIALNO="2024HU0000001", + SPORDER="01", + PWGTP="0011", + AGEP="30", + MIL="1", + ESR="4", + ), + dict( + SERIALNO="2024GQ0000001", + SPORDER="01", + PWGTP="0077", + AGEP="40", + MIL="4", + ESR="6", + ), + dict( + SERIALNO="2024HU0000001", + SPORDER="02", + PWGTP="12", + AGEP="15", + MIL="", + ESR="", + ), + dict( + SERIALNO="2024GQ0000002", + SPORDER="1", + PWGTP="78", + AGEP="50", + MIL="4", + ESR="6", + ), + dict( + SERIALNO="2024HU0000003", SPORDER="1", PWGTP="31", AGEP="99", MIL="", ESR="" + ), + ] + + def write(): + for _role, filename, prefix, rows in ( + ("household", "csv_hus.zip", "psam_hus", households), + ("person", "csv_pus.zip", "psam_pus", people), + ): + with zipfile.ZipFile(source / filename, "w") as archive: + empty = ( + b"SERIALNO,SPORDER,PWGTP,AGEP,MIL,ESR\n" + if _role == "person" + else b"SERIALNO,TYPEHUGQ,NP,WGTP,TEN,ST,PUMA\n" + ) + archive.writestr( + prefix + "a.csv", csv_bytes(rows[:2]) if rows[:2] else empty + ) + archive.writestr( + prefix + "b.csv", csv_bytes(rows[2:]) if rows[2:] else empty + ) + pins = tuple( + ( + role, + name, + hashlib.sha256((source / name).read_bytes()).hexdigest(), + (source / name).stat().st_size, + ) + for role, name in (("household", "csv_hus.zip"), ("person", "csv_pus.zip")) + ) + monkeypatch.setattr(owner.housing, "_ARCHIVE_PINS", pins) + + monkeypatch.setattr( + owner.housing.shutil, + "disk_usage", + lambda _path: SimpleNamespace(free=64 * 1024**3), + ) + monkeypatch.setattr(owner, "_BATCH_PEOPLE", 2) + write() + return SimpleNamespace( + source=source, + snapshots=snapshots, + households=households, + people=people, + write=write, + root=tmp_path, + ) + + +def issue(fixture, **kwargs): + return owner.issue_acs_source_catalogue( + fixture.source, snapshot_root=fixture.snapshots, **kwargs + ) + + +def test_real_complete_catalogue_without_population_or_classification(invented): + forbidden = [] + + def trace(frame, event, arg): + module = frame.f_globals.get("__name__", "") + if event == "call" and ( + frame.f_code is Frame.__init__.__code__ + or module.startswith("microunit") + or frame.f_code.co_name + in { + "build_acs_pums_unit_frame", + "prepare_acs_housing_population", + "classify_household", + "classify_households", + } + ): + forbidden.append(frame.f_code.co_name) + raise AssertionError("Population construction/classification occurred") + + previous = sys.getprofile() + sys.setprofile(trace) + try: + result = issue(invented) + rows = result.households + receipt = result.receipt + finally: + sys.setprofile(previous) + assert not forbidden + assert result.validate() is result + assert owner.verify_acs_source_catalogue(result) is result + assert len(rows) == 4 and len(result.exclusion_ledger) == 1 + assert {r.key.native_id for r in rows} == { + r["SERIALNO"] for r in invented.households if r["NP"] != "0" + } + row = rows[0] + assert row.wgtp == "0010" + assert [(p.sporder, p.age, p.esr, p.mil, p.pwgtp) for p in row.persons] == [ + ("01", "30", "4", "1", "0011"), + ("02", "15", "", "", "12"), + ] + assert row.persons[1].esr_state == "outside_age_universe" + assert rows[-1].persons[0].esr_state == "missing_in_universe" + assert rows[1].persons[0].pwgtp == "0077" + assert result.lineage[row.key.native_id]["persons"] == ( + ("01", "psam_pusa.csv", 1), + ("02", "psam_pusb.csv", 1), + ) + assert receipt["counts"]["people"] == 5 + assert receipt["counts"]["literal_batches"] == 3 + assert ( + receipt["source_authenticated"] + and not receipt["population_binding_authenticated"] + ) + assert ( + not receipt["domain_assignment_authenticated"] + and not receipt["selection_performed"] + ) + assert not receipt["execution"]["full_person_scan_per_literal_batch"] + assert receipt["execution"]["literal_full_scans"] == 1 + assert issue(invented, candidate=result.to_bytes()).to_bytes() == result.to_bytes() + (invented.root / "catalogue.receipt.json").write_bytes(result.to_bytes()) + held = owner._lookup(result) + (invented.root / "invented-records.json").write_bytes( + owner.native.coverage._json( + {"households": held.records, "vacancy_ledger": held.vacancies} + ) + ) + + +def test_independent_views_cannot_rewrite_issued_rows(invented): + result = issue(invented) + original = result.to_bytes() + row = result.households[0] + object.__setattr__(row, "wgtp", "9999") + object.__setattr__(row.persons[0], "esr", "6") + result.receipt["source_authenticated"] = False + result.lineage.clear() + assert result.households[0].wgtp == "0010" + assert result.households[0].persons[0].esr == "4" + assert result.to_bytes() == original + + +def expected_records(): + """Explicit invented source-order records, independent of collector logic.""" + return ( + ( + "2024HU0000001", + "1", + "2", + "0010", + "psam_husa.csv", + 1, + ( + ( + "01", + "30", + "4", + "observed_code", + "1", + "observed_code", + "0011", + "psam_pusa.csv", + 1, + ), + ( + "02", + "15", + "", + "outside_age_universe", + "", + "outside_age_universe", + "12", + "psam_pusb.csv", + 1, + ), + ), + ), + ( + "2024GQ0000001", + "2", + "1", + "0", + "psam_husa.csv", + 2, + ( + ( + "01", + "40", + "6", + "observed_code", + "4", + "observed_code", + "0077", + "psam_pusa.csv", + 2, + ), + ), + ), + ("2024HU0000002", "1", "0", "20", "psam_husb.csv", 1, ()), + ( + "2024GQ0000002", + "3", + "1", + "0", + "psam_husb.csv", + 2, + ( + ( + "1", + "50", + "6", + "observed_code", + "4", + "observed_code", + "78", + "psam_pusb.csv", + 2, + ), + ), + ), + ( + "2024HU0000003", + "1", + "1", + "30", + "psam_husb.csv", + 3, + ( + ( + "1", + "99", + "", + "missing_in_universe", + "", + "missing_in_universe", + "31", + "psam_pusb.csv", + 3, + ), + ), + ), + ) + + +@pytest.mark.parametrize("batch_limit", [2, 3, 5]) +def test_explicit_canonical_records_and_bytes_ignore_batch_boundaries( + invented, monkeypatch, batch_limit +): + monkeypatch.setattr(owner, "_BATCH_PEOPLE", batch_limit) + result = issue(invented) + held, expected = owner._lookup(result), expected_records() + assert held.records == (expected[0], expected[1], expected[3], expected[4]) + assert held.vacancies == (expected[2],) + raw = b"".join( + json.dumps(row, sort_keys=True, separators=(",", ":"), allow_nan=False).encode() + + b"\n" + for row in expected + ) + assert result.receipt["counts"]["canonical_record_bytes"] == len(raw) + assert ( + result.receipt["counts"]["canonical_record_sha256"] + == hashlib.sha256(raw).hexdigest() + ) + + +@pytest.mark.parametrize("batch_limit", [2, 5]) +def test_catalogue_literal_member_scan_count_is_independent_of_batches( + invented, monkeypatch, batch_limit +): + monkeypatch.setattr(owner, "_BATCH_PEOPLE", batch_limit) + scanner = owner.literal._scan_acs_person_coverage.__code__ + active, scans, literal_opens, all_person_opens = set(), [], [], [] + + def trace(frame, event, arg): + if frame.f_code is scanner: + if event == "call": + scans.append(True) + active.add(id(frame)) + elif event == "return": + active.remove(id(frame)) + if event == "call" and frame.f_code is zipfile.ZipFile.open.__code__: + name = frame.f_locals["name"] + name = name.filename if isinstance(name, zipfile.ZipInfo) else name + if name.startswith("psam_pus"): + all_person_opens.append(name) + if active: + literal_opens.append(name) + + previous = sys.getprofile() + sys.setprofile(trace) + try: + result = issue(invented) + finally: + sys.setprofile(previous) + assert scans == [True] and not active + assert literal_opens == ["psam_pusa.csv", "psam_pusb.csv"] + assert all_person_opens == literal_opens * 3 + assert result.receipt["counts"]["literal_batches"] == (3 if batch_limit == 2 else 1) + + +@pytest.mark.parametrize( + "mutation,code", + [ + ("duplicate", "DUPLICATE_LITERAL_PERSON"), + ("extra", "UNEXPECTED_LITERAL_PERSON"), + ("orphan", "UNEXPECTED_LITERAL_PERSON"), + ("missing", "GLOBAL_PERSON_KEYS"), + ("key_suffix", "LITERAL_PERSON_KEY"), + ], +) +def test_collector_reconciles_every_literal_slot_after_housing_validation( + invented, mutation, code +): + # Exercise this private reconciliation separately from the unchanged + # housing/preflight checks. A mismatched helper input issues no authority. + original = issue(invented) + projection = json.loads(owner._lookup(original).projection_path.read_bytes()) + if mutation == "duplicate": + invented.people.append(dict(invented.people[0], SPORDER="1")) + elif mutation == "extra": + invented.people.append(dict(invented.people[0], SPORDER="3")) + elif mutation == "orphan": + invented.people[-1]["SERIALNO"] = "2024HU0000099" + elif mutation == "missing": + invented.people.pop() + else: + invented.people[-1]["SPORDER"] = "1 " + invented.write() + paths = { + "household": invented.source / "csv_hus.zip", + "person": invented.source / "csv_pus.zip", + } + with pytest.raises(owner.ACSSourceCatalogueError, match=f"^{code}$"): + owner._collect(projection, paths) + + +def test_prospective_byte_charge_refuses_before_retaining_the_next_person( + invented, monkeypatch +): + original = issue(invented) + held = owner._lookup(original) + projection = json.loads(held.projection_path.read_bytes()) + expected = expected_records() + + def encoded(value): + return json.dumps(value, separators=(",", ":"), allow_nan=False).encode() + + overhead = sum(len(encoded((*row[:6], ()))) + 1 for row in expected) + monkeypatch.setattr( + owner, "_CATALOGUE_BYTES", overhead + len(encoded(expected[0][6][0])) + ) + retained = [] + + def trace(frame, event, arg): + if ( + event == "return" + and frame.f_globals.get("__name__") == owner.__name__ + and frame.f_code.co_qualname == "_collect..consume_row" + ): + retained.append( + sum(person is not None for person in frame.f_locals["people"]) + ) + + previous = sys.getprofile() + sys.setprofile(trace) + try: + with pytest.raises( + owner.native.coverage.ACSCoverageAuthenticationError, + match="^CANONICAL_SIZE$", + ): + owner._collect(projection, dict(held.paths)) + finally: + sys.setprofile(previous) + assert retained == [1, 1] + + +def test_source_order_and_original_person_order_survive_reverse_zip_insertion(invented): + # Keep serials unsorted and make the split household's raw person order + # descending; canonical order remains source-member then source-row order. + invented.people[0]["SPORDER"], invented.people[2]["SPORDER"] = "02", "01" + invented.write() + first = issue(invented) + before = owner._lookup(first) + for name in ("csv_hus.zip", "csv_pus.zip"): + path = invented.source / name + with zipfile.ZipFile(path) as archive: + members = [ + (member.filename, archive.read(member)) for member in archive.infolist() + ] + with zipfile.ZipFile(path, "w") as archive: + for member, raw in reversed(members): + archive.writestr(member, raw) + pins = tuple( + ( + role, + name, + hashlib.sha256((invented.source / name).read_bytes()).hexdigest(), + (invented.source / name).stat().st_size, + ) + for role, name in (("household", "csv_hus.zip"), ("person", "csv_pus.zip")) + ) + # New physical ZIPs have their own invented pins; compare source records, + # not archive/producer receipts from different physical artifacts. + owner.housing._ARCHIVE_PINS = pins + second = issue(invented) + after = owner._lookup(second) + assert after.records == before.records and after.vacancies == before.vacancies + assert [p[0] for p in after.records[0][6]] == ["02", "01"] + + +@pytest.mark.parametrize("kind", ["new", "copy", "deepcopy", "pickle", "payload"]) +def test_foreign_changed_or_copied_authority_refuses(invented, kind): + result = issue(invented) + if kind == "new": + other = object.__new__(owner.AuthenticatedACSSourceCatalogue) + object.__setattr__(other, "payload", result.payload) + elif kind == "payload": + other = result + raw = json.loads(result.payload) + raw["counts"]["people"] = 16 + object.__setattr__(other, "payload", owner.native.coverage._json(raw)) + else: + try: + other = { + "copy": copy.copy, + "deepcopy": copy.deepcopy, + "pickle": lambda x: pickle.loads(pickle.dumps(x)), + }[kind](result) + except (TypeError, owner.ACSSourceCatalogueError): + return + with pytest.raises( + owner.ACSSourceCatalogueError, match="^ISSUANCE_(NOT_OWNED|CHANGED)$" + ): + other.validate() + + +def test_independent_reissue_and_weak_cleanup(invented): + one, two = issue(invented), issue(invented) + assert one.to_bytes() == two.to_bytes() and one is not two + identity, reference = id(one), weakref.ref(one) + del one + gc.collect() + assert reference() is None and identity not in owner._ISSUED + assert two.validate() is two + + +@pytest.mark.parametrize("kind", ["duplicate", "orphan", "missing", "late_extra", "np"]) +def test_global_complete_roster_refuses_across_members_and_batches(invented, kind): + if kind == "duplicate": + invented.people[-1] = dict(invented.people[0]) + elif kind == "orphan": + invented.people[-1]["SERIALNO"] = "2024HU0000099" + elif kind == "missing": + invented.people.pop() + elif kind == "late_extra": + extra = dict(invented.people[0], SPORDER="3") + invented.people.append(extra) + else: + invented.households[0]["NP"] = "3" + invented.write() + with pytest.raises(owner.ACSSourceCatalogueError): + issue(invented) + + +@pytest.mark.parametrize( + "cap,value", + [ + ("_ARCHIVE_BYTES", 1), + ("_EXPANDED_BYTES", 1), + ("_MEMBERS", 1), + ("_SOURCE_ROWS", 1), + ("_BATCH_PEOPLE", 1), + ("_CATALOGUE_BYTES", 1), + ("_RECEIPT_BYTES", 1), + ], +) +def test_prospective_small_caps_refuse(invented, monkeypatch, cap, value): + monkeypatch.setattr(owner, cap, value) + with pytest.raises(owner.ACSSourceCatalogueError): + issue(invented) + + +@pytest.mark.parametrize("escaped_literals", [False, True]) +def test_byte_ceiling_exact_boundary_is_record_based( + invented, monkeypatch, escaped_literals +): + if escaped_literals: + invented.people[0]["MIL"] = 'x,"\\\t' + invented.people[0]["ESR"] = "é" + invented.write() + size = issue(invented).receipt["counts"]["canonical_record_bytes"] + monkeypatch.setattr(owner, "_CATALOGUE_BYTES", size) + assert issue(invented).receipt["counts"]["canonical_record_bytes"] == size + monkeypatch.setattr(owner, "_CATALOGUE_BYTES", size - 1) + with pytest.raises(owner.ACSSourceCatalogueError): + issue(invented) + + +def test_candidate_is_reconstructed_and_cannot_authorize_changed_values(invented): + result = issue(invented) + candidate = result.to_bytes() + invented.people[0]["ESR"] = "6" + invented.write() + with pytest.raises(owner.ACSSourceCatalogueError, match="^CANDIDATE_MISMATCH$"): + issue(invented, candidate=candidate) + assert issue(invented).households[0].persons[0].esr == "6" + with pytest.raises( + owner.ACSSourceCatalogueError, match="^SOURCE_AUTHORITY_CHANGED$" + ): + result.validate() + + +@pytest.mark.parametrize("target", ["original", "projection", "producer"]) +def test_live_source_projection_and_producer_changes_refuse( + invented, monkeypatch, target +): + result = issue(invented) + if target == "original": + path = invented.source / "csv_pus.zip" + raw = path.read_bytes() + path.write_bytes(bytes([raw[0] ^ 1]) + raw[1:]) + elif target == "projection": + owned = owner._lookup(result) + raw = owned.projection_path.read_bytes() + owned.projection_path.chmod(0o600) + owned.projection_path.write_bytes(b" " + raw[1:]) + else: + monkeypatch.setattr(owner, "_BATCH_PEOPLE", 3) + with pytest.raises(owner.ACSSourceCatalogueError): + result.validate() + + +def test_new_owner_static_surface_excludes_population_selection_and_classification(): + tree = ast.parse(Path(owner.__file__).read_text()) + calls = { + node.func.id if isinstance(node.func, ast.Name) else node.func.attr + for node in ast.walk(tree) + if isinstance(node, ast.Call) + and isinstance(node.func, (ast.Name, ast.Attribute)) + } + assert not calls & { + "Frame", + "build_acs_pums_unit_frame", + "prepare_acs_housing_population", + "assign_us_unit_structure", + "classify_household", + "classify_households", + "issue_acs_native_coverage", + "sample", + "choice", + } + assert {"_reconstruct", "_scan_acs_person_coverage", "_live_code"} <= calls + assert "read_acs_person_coverage_columns" not in calls + + +@pytest.mark.parametrize("mutation", ["source", "limit"]) +def test_mutations_at_literal_return_refuse(invented, mutation): + target = owner.literal._scan_acs_person_coverage + fired = [] + + def timing(frame, event, arg): + if frame.f_code is target.__code__ and event == "return" and not fired: + fired.append(True) + if mutation == "source": + path = invented.source / "csv_pus.zip" + raw = path.read_bytes() + path.write_bytes(bytes([raw[0] ^ 1]) + raw[1:]) + else: + owner._BATCH_PEOPLE = 3 + + before, previous = owner._BATCH_PEOPLE, sys.getprofile() + sys.setprofile(timing) + try: + with pytest.raises(owner.ACSSourceCatalogueError): + issue(invented) + finally: + sys.setprofile(previous) + owner._BATCH_PEOPLE = before + assert fired + + +def test_returned_housing_capsule_cannot_replace_actual_projected_anchors(invented): + target, fired = owner.housing._reconstruct, [] + + def timing(frame, event, arg): + if frame.f_code is target.__code__ and event == "return" and not fired: + fired.append(True) + changed = json.loads(arg.projection_json) + changed["households"][0][changed["household_columns"].index("WGTP")] = ( + "9999" + ) + object.__setattr__(arg, "projection_json", owner.housing._json(changed)) + + previous = sys.getprofile() + sys.setprofile(timing) + try: + result = issue(invented) + finally: + sys.setprofile(previous) + assert fired and result.households[0].wgtp == "0010" + + +@pytest.mark.parametrize("aliases", ["csv", "native", "both"]) +@pytest.mark.parametrize("moment", ["entry", "before_call"]) +def test_actual_csv_alias_timing_cannot_supply_catalogue_values( + invented, aliases, moment +): + target = owner.literal._literal_csv_records + source, first = inspect.getsourcelines(target) + call_line = next( + first + i for i, line in enumerate(source) if "reader = csv_reader(" in line + ) + original, native, fired, calls = csv.reader, _csv.reader, [], [] + + def restore(): + csv.reader, _csv.reader = original, native + + def replacement(*args, **kwargs): + calls.append(True) + restore() + return original(*args, **kwargs) + + def timing(frame, event, arg): + if frame.f_code is target.__code__: + trigger = (moment == "entry" and event == "call") or ( + moment == "before_call" + and event == "line" + and frame.f_lineno == call_line + ) + if trigger and not fired: + fired.append(True) + if aliases in ("csv", "both"): + csv.reader = replacement + if aliases in ("native", "both"): + _csv.reader = replacement + if event == "return": + restore() + return timing + + previous = sys.gettrace() + sys.settrace(timing) + try: + if moment == "entry": + with pytest.raises(owner.ACSSourceCatalogueError): + issue(invented) + else: + result = issue(invented) + finally: + sys.settrace(previous) + restore() + assert fired and not calls + if moment == "before_call": + assert result.households[0].persons[0].esr == "4" + + +def test_public_constructor_has_no_authority(invented): + raw = issue(invented).to_bytes() + with pytest.raises(owner.ACSSourceCatalogueError, match="^ISSUANCE_CONSTRUCTOR$"): + owner.AuthenticatedACSSourceCatalogue(raw) + + +def test_full_source_can_exceed_unchanged_selected_reader_cap(invented, monkeypatch): + monkeypatch.setattr(owner.literal, "MAX_SELECTED_ROWS", 2) + result = issue(invented) + assert result.receipt["counts"]["people"] == 5 + assert result.receipt["counts"]["literal_batches"] == 3 + + +def test_vacancy_only_catalogue_has_no_literal_batch(invented): + invented.households[:] = [dict(invented.households[2])] + invented.people.clear() + invented.write() + result = issue(invented) + assert result.households == () + assert len(result.exclusion_ledger) == 1 + assert result.exclusion_ledger[0].persons == () + assert result.receipt["counts"]["literal_batches"] == 0 + assert result.receipt["execution"]["literal_full_scans"] == 0 + + +def test_reordered_literal_column_declaration_refuses_before_capture( + invented, monkeypatch +): + monkeypatch.setattr( + owner.literal, "READ_COLUMNS", ("SERIALNO", "SPORDER", "ESR", "MIL", "AGEP") + ) + with pytest.raises( + owner.ACSSourceCatalogueError, match="^LITERAL_CONTRACT_CHANGED$" + ): + issue(invented) + assert list(invented.snapshots.iterdir()) == [] + + +@pytest.mark.parametrize( + "interface", + ["validate", "to_bytes", "receipt", "households", "exclusion_ledger", "lineage"], +) +@pytest.mark.parametrize("moment", ["source_checks", "final_producer"]) +def test_final_source_registry_seal_all_borrows( + invented, monkeypatch, interface, moment +): + result = issue(invented) + pins = owner.housing._ARCHIVE_PINS + changed = tuple((r, n, "0" * 64, s) for r, n, d, s in pins) + fired, producers = [], [] + + def timing(frame, event, arg): + if event != "return": + return + if frame.f_code is owner._producer.__code__: + producers.append(True) + if not fired and ( + moment == "source_checks" + and frame.f_code is owner._source_checks.__code__ + or moment == "final_producer" + and frame.f_code is owner._producer.__code__ + and len(producers) == 2 + ): + fired.append(True) + monkeypatch.setattr(owner.housing, "_ARCHIVE_PINS", changed) + + previous = sys.getprofile() + try: + sys.setprofile(timing) + with pytest.raises( + owner.ACSSourceCatalogueError, match="SOURCE_AUTHORITY_CHANGED" + ): + value = getattr(result, interface) + if callable(value): + value() + finally: + sys.setprofile(previous) + assert fired + + +@pytest.mark.parametrize("semantic", [False, True]) +def test_import_time_bytes_and_matching_live_code_remain_authority( + invented, monkeypatch, semantic +): + source = Path(owner.__file__).read_text() + if semantic: + before = " weight,\n tuple(" + assert source.count(before) == 1 + source = source.replace(before, ' "9999",\n tuple(') + else: + source += "\n# invented drift after the real module import\n" + replacement = invented.root / "changed-catalogue.py" + replacement.write_text(source) + monkeypatch.setattr(owner, "__file__", str(replacement)) + if semantic: + code = next( + c + for c in compile( + source, str(replacement), "exec", dont_inherit=True + ).co_consts + if isinstance(c, CodeType) and c.co_name == "_household" + ) + monkeypatch.setattr(owner._household, "__code__", code) + with pytest.raises(owner.ACSSourceCatalogueError, match="PRODUCER_CODE_CHANGED"): + issue(invented) + + +@pytest.mark.parametrize("change", ["bytes", "generated_constructor"]) +def test_raw_domain_import_authority_precedes_first_issuance( + invented, monkeypatch, change +): + if change == "bytes": + replacement = invented.root / "changed-raw-types.py" + replacement.write_bytes( + Path(owner.domains.__file__).read_bytes() + b"\n# drift\n" + ) + monkeypatch.setattr(owner.domains, "__file__", str(replacement)) + else: + # Native live-code traversal explicitly skips generated code; + # the local import-time seal must still bind those actual constructors. + code = next( + c + for c in compile( + "def __init__(self, *args, **kwargs): pass", "", "exec" + ).co_consts + if isinstance(c, CodeType) + ) + monkeypatch.setattr( + owner.domains.AcsHousehold, + "__init__", + FunctionType(code, vars(owner.domains)), + ) + with pytest.raises(owner.ACSSourceCatalogueError, match="PRODUCER_CODE_CHANGED"): + issue(invented) + + +@pytest.mark.parametrize("issuance", [False, True]) +@pytest.mark.parametrize("change", ["pins", "live_code"]) +def test_final_producer_return_seal_issuance_and_borrow( + invented, monkeypatch, issuance, change +): + result = None if issuance else issue(invented) + fired, producers = [], [] + original = owner._household + + def changed(record): + return original(record) + + def timing(frame, event, arg): + if event == "return" and frame.f_code is owner._producer.__code__: + producers.append(True) + if len(producers) == 2: + fired.append(True) + if change == "pins": + monkeypatch.setattr( + owner.housing, + "_ARCHIVE_PINS", + tuple( + (r, n, "0" * 64, s) + for r, n, d, s in owner.housing._ARCHIVE_PINS + ), + ) + else: + monkeypatch.setattr(owner, "_household", changed) + + previous = sys.getprofile() + try: + sys.setprofile(timing) + with pytest.raises(owner.ACSSourceCatalogueError): + issue(invented) if issuance else result.households + finally: + sys.setprofile(previous) + assert fired + + +@pytest.mark.parametrize( + "shape", ["outer_list", "inner_list", "string_subclass", "bool_size"] +) +def test_source_authority_requires_deeply_immutable_primitive_pins( + invented, monkeypatch, shape +): + original = owner.housing._ARCHIVE_PINS + if shape == "outer_list": + changed = list(original) + elif shape == "inner_list": + changed = tuple(list(pin) for pin in original) + elif shape == "string_subclass": + + class MutableString(str): + pass + + changed = ((MutableString(original[0][0]), *original[0][1:]), original[1]) + else: + changed = ((*original[0][:3], True), original[1]) + monkeypatch.setattr(owner.housing, "_ARCHIVE_PINS", changed) + with pytest.raises(owner.ACSSourceCatalogueError, match="SOURCE_AUTHORITY_TYPE"): + issue(invented) + assert list(invented.snapshots.iterdir()) == [] diff --git a/packages/microcosm-build/tests/test_us_acs_pums.py b/packages/microcosm-build/tests/test_us_acs_pums.py index 48b656d96..d663d21ac 100644 --- a/packages/microcosm-build/tests/test_us_acs_pums.py +++ b/packages/microcosm-build/tests/test_us_acs_pums.py @@ -8,13 +8,13 @@ import pytest from microcosm.build.serialization_dtypes import CANONICAL_STRING_DTYPE +from microcosm.build.us_runtime import acs_pums from microcosm.build.us_runtime.acs_pums import ( ACS_2024_1YR_SPINE, AcsPumsSource, build_acs_pums_unit_frame, load_acs_pums_tables, ) -from microcosm.build.us_runtime.spine_assembly import assemble_spines from microcosm.frame import US_SCHEMA, Frame, WeightKind, Weights @@ -126,6 +126,15 @@ def _source(tmp_path: Path) -> AcsPumsSource: return AcsPumsSource(household_zip=household_zip, person_zip=person_zip) +def _axis_household(kinds: list[int]) -> pd.DataFrame: + return pd.DataFrame( + { + "SERIALNO": [f"2024HU000000{index}" for index in range(len(kinds))], + "TYPEHUGQ": kinds, + } + ) + + def _asec_shaped_frame() -> Frame: """Return one ASEC-like row sharing only structural and lineage fields.""" @@ -264,6 +273,8 @@ def test_built_acs_lineage_assembles_with_asec_without_measured_coercion( measured_wages = acs.table("person")["WAGP"].copy() raw_serials = acs.table("household")["SERIALNO"].copy() + from microcosm.build.us_runtime.spine_assembly import assemble_spines + assembled = assemble_spines( {"asec": _asec_shaped_frame(), "acs": acs}, household_mass_shares={"asec": 0.5, "acs": 0.5}, @@ -595,3 +606,167 @@ def test_load_acs_pums_tables_requires_native_mapping_columns( load_acs_pums_tables( AcsPumsSource(household_zip=household_zip, person_zip=person_zip) ) + + +def test_load_acs_pums_tables_rejects_group_quarters_with_positive_weight( + tmp_path: Path, +) -> None: + source = _source(tmp_path) + _write_csv_zip( + source.household_zip, + { + "psam_husa.csv": [ + _household( + "2024GQ0000001", + WGTP=7, + TYPEHUGQ=2, + TEN=None, + TAXAMT=None, + ) + ] + }, + ) + + with pytest.raises(ValueError, match="TYPEHUGQ/WGTP disagree"): + load_acs_pums_tables(source, chunksize=1) + + +def test_load_acs_pums_tables_rejects_housing_unit_with_zero_weight( + tmp_path: Path, +) -> None: + source = _source(tmp_path) + _write_csv_zip( + source.household_zip, + {"psam_husa.csv": [_household("2024HU0000001", WGTP=0)]}, + ) + + with pytest.raises(ValueError, match="TYPEHUGQ/WGTP disagree"): + load_acs_pums_tables(source, chunksize=1) + + +def test_build_acs_pums_unit_frame_reports_household_axis_composition( + tmp_path: Path, +) -> None: + pytest.importorskip("microunit") # sanctioned tax-unit constructor (us extra) + household_zip = tmp_path / "csv_hus.zip" + person_zip = tmp_path / "csv_pus.zip" + _write_csv_zip( + household_zip, + { + "psam_husa.csv": [ + _household("2024HU0000001", WGTP=10, NP=2), + _household("2024HU0000002", WGTP=20), + _household( + "2024GQ0000001", + WGTP=0, + TYPEHUGQ=2, + TEN=None, + TAXAMT=None, + ), + _household( + "2024GQ0000002", + WGTP=0, + TYPEHUGQ=3, + TEN=None, + TAXAMT=None, + ), + _household("2024HU0000003", WGTP=40, NP=0), + ] + }, + ) + _write_csv_zip( + person_zip, + { + "psam_pusa.csv": [ + _person("2024HU0000001", 1, 20, PWGTP=11), + _person("2024HU0000001", 2, 25, MAR=5, PWGTP=12), + _person("2024HU0000002", 1, 20, PWGTP=13), + _person("2024GQ0000001", 1, 37, MAR=5, PWGTP=99), + _person("2024GQ0000002", 1, 38, MAR=5, PWGTP=55), + ] + }, + ) + + frame, metadata = build_acs_pums_unit_frame( + AcsPumsSource(household_zip=household_zip, person_zip=person_zip) + ) + + composition = metadata["household_axis_composition"] + assert composition == { + "occupied_housing_unit_rows": 2, + "occupied_housing_unit_design_weight_total": 30.0, + "institutional_gq_person_rows": 1, + "institutional_gq_person_design_weight_total": 99.0, + "noninstitutional_gq_person_rows": 1, + "noninstitutional_gq_person_design_weight_total": 55.0, + } + # The dropped vacant housing unit is outside the described axis entirely. + assert metadata["vacant_household_rows_dropped"] == 1 + # Units conserve, and design mass conserves separately from units. + assert ( + composition["occupied_housing_unit_rows"] + + composition["institutional_gq_person_rows"] + + composition["noninstitutional_gq_person_rows"] + == metadata["household_rows"] + == frame.n("household") + == 4 + ) + assert composition["occupied_housing_unit_design_weight_total"] + composition[ + "institutional_gq_person_design_weight_total" + ] + composition["noninstitutional_gq_person_design_weight_total"] == pytest.approx( + metadata["weighted_household_population"] + ) + + +# Reached through the module, not imported by name, so this file still imports +# against a build without the helper: the four checks below then fail on their +# own instead of erroring the whole module at collection. +def test_household_axis_composition_separates_units_and_design_mass() -> None: + household = _axis_household([1, 1, 2, 3]) + weights = Weights( + np.asarray([10.0, 20.0, 99.0, 55.0]), + WeightKind.DESIGN, + ) + + composition = acs_pums._household_axis_composition(household, weights) + + assert composition == { + "occupied_housing_unit_rows": 2, + "occupied_housing_unit_design_weight_total": 30.0, + "institutional_gq_person_rows": 1, + "institutional_gq_person_design_weight_total": 99.0, + "noninstitutional_gq_person_rows": 1, + "noninstitutional_gq_person_design_weight_total": 55.0, + } + assert sum( + value for key, value in composition.items() if key.endswith("_rows") + ) == len(household) + assert sum( + value + for key, value in composition.items() + if key.endswith("_design_weight_total") + ) == pytest.approx(weights.total) + + +def test_household_axis_composition_requires_typehugq() -> None: + household = _axis_household([1]).drop(columns=["TYPEHUGQ"]) + weights = Weights(np.asarray([10.0]), WeightKind.DESIGN) + + with pytest.raises(ValueError, match="requires the TYPEHUGQ column"): + acs_pums._household_axis_composition(household, weights) + + +def test_household_axis_composition_refuses_unpartitioned_rows() -> None: + household = _axis_household([1, 4]) + weights = Weights(np.asarray([10.0, 20.0]), WeightKind.DESIGN) + + with pytest.raises(ValueError, match="does not partition the loaded rows"): + acs_pums._household_axis_composition(household, weights) + + +def test_household_axis_composition_refuses_misaligned_weights() -> None: + household = _axis_household([1, 2]) + weights = Weights(np.asarray([10.0, 99.0, 20.0]), WeightKind.DESIGN) + + with pytest.raises(ValueError, match="one DESIGN weight per loaded"): + acs_pums._household_axis_composition(household, weights) diff --git a/packages/microcosm-build/tests/test_us_acs_source_compile_cache.py b/packages/microcosm-build/tests/test_us_acs_source_compile_cache.py new file mode 100644 index 000000000..6d108fe7a --- /dev/null +++ b/packages/microcosm-build/tests/test_us_acs_source_compile_cache.py @@ -0,0 +1,457 @@ +"""Invented controls for the real ACS owner's source-only compilation cache.""" + +from __future__ import annotations +import __future__ + +import ast +import builtins +import sys +from contextlib import contextmanager +from pathlib import Path +from types import CodeType, FunctionType, ModuleType + +import pytest + +from microcosm.build.us_runtime import acs_native_coverage_binding as native + + +@pytest.fixture(autouse=True) +def empty_compile_cache(): + native._clear_compile_cache() + yield + native._clear_compile_cache() + + +@contextmanager +def _observed_calls(): + """Observe real compile/read calls without replacing attested functions.""" + assert sys.getprofile() is None + counts = {"compile": 0, "read": 0} + helper_code = native._compile_source.__code__ + read_code = Path.read_bytes.__code__ + + def observe(frame, event, arg): + if ( + event == "c_call" + and frame.f_code is helper_code + and arg is builtins.compile + ): + counts["compile"] += 1 + elif event == "call" and frame.f_code is read_code: + counts["read"] += 1 + + sys.setprofile(observe) + try: + yield counts + finally: + sys.setprofile(None) + + +def _module(tmp_path, monkeypatch, source, *, name="invented_acs_compile_cache"): + path = tmp_path / f"{name}.py" + path.write_bytes(source) + module = ModuleType(name) + module.__file__ = str(path) + module.__package__ = "" + monkeypatch.setitem(sys.modules, name, module) + exec(builtins.compile(source, str(path), "exec", dont_inherit=True), vars(module)) + return module, path + + +def _index(code): + result = {} + + def visit(item): + result[item.co_qualname] = item + for value in item.co_consts: + if isinstance(value, CodeType): + visit(value) + + visit(code) + return result + + +@pytest.mark.parametrize("optimize", [0, 1, 2]) +def test_plain_helper_preserves_compiled_code_and_qualified_name_order(optimize): + source = """# coding: utf-8 +from __future__ import annotations +label = "café" +def outer(value: Missing): + "docstring" + assert value is not None + def nested(): + return [item for item in range(value)] + def nested(): + return lambda: {item: item for item in range(value)} + return nested +class Example: + @property + def item(self): + return label + @item.setter + def item(self, value): + self._item = value +""".encode() + filename = "invented/café/../module.py" + expected = builtins.compile( + source, filename, "exec", dont_inherit=True, optimize=optimize + ) + with _observed_calls() as calls: + first = native._compile_source(source, filename, optimize=optimize) + second = native._compile_source(source, filename, optimize=optimize) + assert type(native._compile_source) is FunctionType + assert first is second + assert first == expected + assert first.co_filename == filename + assert _index(first) == _index(expected) + assert calls["compile"] == 1 + + +def test_complete_inputs_separate_cache_entries(): + source = b"42" + options = [ + ("literal/../source.py", "exec", 0, 0), + ("source.py", "exec", 0, 0), + ("source.py", "eval", 0, 0), + ("source.py", "exec", __future__.annotations.compiler_flag, 0), + ("source.py", "exec", 0, 1), + ("source.py", "exec", 0, 2), + ] + with _observed_calls() as calls: + for filename, mode, flags, optimize in options: + expected = builtins.compile( + source, + filename, + mode, + flags=flags, + dont_inherit=True, + optimize=optimize, + ) + first = native._compile_source( + source, filename, mode, flags=flags, optimize=optimize + ) + assert first == expected + assert first.co_filename == filename + assert ( + native._compile_source( + source, filename, mode, flags=flags, optimize=optimize + ) + is first + ) + assert calls["compile"] == len(options) + assert len(native._COMPILE_CACHE) == len(options) + with _observed_calls() as calls: + effective = native._compile_source( + source, "source.py", optimize=sys.flags.optimize + ) + assert native._compile_source(source, "source.py", optimize=-1) is effective + assert calls["compile"] == 0 + + +def test_same_literal_path_tracks_a_b_a_source_bytes(): + first_source, second_source = b"value = 1", b"value = 2" + assert len(first_source) == len(second_source) + with _observed_calls() as calls: + first = native._compile_source(first_source, "unchanged.py") + second = native._compile_source(second_source, "unchanged.py") + again = native._compile_source(first_source, "unchanged.py") + assert first is again + assert second != first + assert calls["compile"] == 2 + + +def test_changed_compiler_is_called_each_time_and_original_entry_remains(monkeypatch): + source = b"value = 1" + original = native._compile_source(source, "compiler.py") + replacements = [] + + def replacement(raw, filename, mode, **options): + replacements.append(raw) + return builtins.compile( + f"value = {len(replacements) + 1}".encode(), filename, mode, **options + ) + + with monkeypatch.context() as context: + context.setattr(native, "compile", replacement, raising=False) + first = native._compile_source(source, "compiler.py") + second = native._compile_source(source, "compiler.py") + assert replacements == [source, source] + assert first != second + assert first != original + assert len(native._COMPILE_CACHE) == 1 + assert native._compile_source(source, "compiler.py") is original + + +@pytest.mark.parametrize( + "options", [{"dont_inherit": False}, {"flags": ast.PyCF_ONLY_AST}] +) +def test_context_dependent_or_mutable_compiler_outputs_are_not_retained(options): + with _observed_calls() as calls: + first = native._compile_source(b"value = 1", "bypass.py", **options) + second = native._compile_source(b"value = 1", "bypass.py", **options) + assert first is not second + assert calls["compile"] == 2 + assert not native._COMPILE_CACHE + + +def test_failures_are_not_memoized(): + with _observed_calls() as calls: + for _ in range(2): + with pytest.raises(SyntaxError): + native._compile_source(b"def invalid(:", "repair.py") + assert not native._COMPILE_CACHE + fixed = native._compile_source(b"value = 1", "repair.py") + assert calls["compile"] == 3 + assert type(fixed) is CodeType + assert len(native._COMPILE_CACHE) == 1 + + +def test_fifo_entry_limit_and_clear(monkeypatch): + monkeypatch.setattr(native, "_COMPILE_CACHE_MAX_ENTRIES", 2) + source = b"value = 1" + first = native._compile_source(source, "first.py") + second = native._compile_source(source, "second.py") + assert native._compile_source(source, "first.py") is first + native._compile_source(source, "third.py") + assert len(native._COMPILE_CACHE) == 2 + assert native._compile_source(source, "second.py") is second + with _observed_calls() as calls: + assert native._compile_source(source, "first.py") is not first + assert calls["compile"] == 1 + native._clear_compile_cache() + assert not native._COMPILE_CACHE + with _observed_calls() as calls: + native._compile_source(source, "first.py") + assert calls["compile"] == 1 + + +def test_source_byte_budget_counts_distinct_filename_keys(monkeypatch): + source = b"value = 1" + monkeypatch.setattr(native, "_COMPILE_CACHE_MAX_SOURCE_BYTES", len(source) * 2) + first = native._compile_source(source, "first.py") + native._compile_source(source, "second.py") + native._compile_source(source, "third.py") + assert sum(len(key[0]) for key in native._COMPILE_CACHE) == len(source) * 2 + with _observed_calls() as calls: + assert native._compile_source(source, "first.py") is not first + assert calls["compile"] == 1 + + +@pytest.mark.parametrize( + "limit_name", ["_COMPILE_CACHE_MAX_ENTRY_BYTES", "_COMPILE_CACHE_MAX_SOURCE_BYTES"] +) +def test_oversize_source_is_compiled_without_cache(limit_name, monkeypatch): + source = b"value = 1" + monkeypatch.setattr(native, limit_name, len(source) - 1) + with _observed_calls() as calls: + first = native._compile_source(source, "oversize.py") + second = native._compile_source(source, "oversize.py") + assert first == second + assert first is not second + assert calls["compile"] == 2 + assert not native._COMPILE_CACHE + + +def test_live_check_reads_source_twice_even_after_warming(tmp_path, monkeypatch): + module, _ = _module(tmp_path, monkeypatch, b"def value():\n return 1\n") + with _observed_calls() as cold: + first_index = {} + native._live_code(module, first_index) + with _observed_calls() as warm: + second_index = {} + native._live_code(module, second_index) + assert cold == {"compile": 1, "read": 2} + assert warm == {"compile": 0, "read": 2} + assert first_index == second_index + assert first_index is not second_index + first_index[module.__file__]["value"] = None + native._live_code(module, {}) + assert second_index[module.__file__]["value"] is not None + + +def test_real_producer_keeps_read_counts_and_full_evidence_on_warm_cache(): + # Settle unrelated import/manifest memoization, then compare only compile + # cache cold versus warm with the real producer and unchanged source files. + native._producer() + native._clear_compile_cache() + with _observed_calls() as cold: + first = native._producer() + with _observed_calls() as warm: + second = native._producer() + assert cold["compile"] > 0 + assert warm["compile"] == 0 + assert cold["read"] > 0 + assert warm["read"] == cold["read"] + assert first == second + assert len(native._COMPILE_CACHE) <= native._COMPILE_CACHE_MAX_ENTRIES + assert ( + sum(len(key[0]) for key in native._COMPILE_CACHE) + <= native._COMPILE_CACHE_MAX_SOURCE_BYTES + ) + + +@pytest.mark.parametrize("drift", ["code", "globals", "declaration"]) +def test_warm_cache_does_not_hide_loaded_function_drift(drift, tmp_path, monkeypatch): + module, _ = _module(tmp_path, monkeypatch, b"def value():\n return 1\n") + native._live_code(module, {}) + if drift == "code": + module.value.__code__ = module.value.__code__.replace(co_consts=(None, 2)) + elif drift == "globals": + module.value = FunctionType(module.value.__code__, dict(vars(module))) + else: + del module.value + with pytest.raises(native.ACSNativeCoverageBindingError, match="^LOADED_PRODUCER$"): + native._live_code(module, {}) + + +def test_warm_cache_preserves_imported_alias_identity(tmp_path, monkeypatch): + origin, _ = _module( + tmp_path, + monkeypatch, + b"def value():\n return 1\n", + name="invented_acs_origin", + ) + consumer, _ = _module( + tmp_path, + monkeypatch, + b"from invented_acs_origin import value\n", + name="invented_acs_consumer", + ) + native._live_code(consumer, {}) + consumer.value = FunctionType(origin.value.__code__, vars(origin)) + assert consumer.value.__code__ is origin.value.__code__ + with pytest.raises(native.ACSNativeCoverageBindingError, match="^LOADED_PRODUCER$"): + native._live_code(consumer, {}) + + +def test_warm_cache_checks_distinct_functions_with_shared_code_and_closures( + tmp_path, monkeypatch +): + source = b"""def make(): + def target(): + return 1 + def entry(): + return target() + return entry +first = make() +second = make() +""" + module, _ = _module(tmp_path, monkeypatch, source) + native._live_code(module, {}) + assert module.first.__code__ is module.second.__code__ + first_target = module.first.__closure__[0].cell_contents + second_target = module.second.__closure__[0].cell_contents + assert first_target is not second_target + second_target.__code__ = second_target.__code__.replace(co_consts=(None, 2)) + with pytest.raises(native.ACSNativeCoverageBindingError, match="^LOADED_PRODUCER$"): + native._live_code(module, {}) + + +def test_second_source_read_can_invalidate_a_warm_entry(tmp_path, monkeypatch): + original = b"def value():\n return 1\n" + changed = b"def value():\n return 2\n" + module, path = _module(tmp_path, monkeypatch, original) + native._live_code(module, {}) + read_bytes = Path.read_bytes + calls = 0 + + def change_on_second_read(self): + nonlocal calls + if self == path: + calls += 1 + if calls == 2: + path.write_bytes(changed) + return read_bytes(self) + + monkeypatch.setattr(Path, "read_bytes", change_on_second_read) + with pytest.raises(native.ACSNativeCoverageBindingError, match="^LOADED_PRODUCER$"): + native._live_code(module, {}) + assert calls == 2 + + +def test_malformed_cache_entry_is_recompiled_and_replaced(): + source = b"value = 1" + first = native._compile_source(source, "malformed.py") + key = next(iter(native._COMPILE_CACHE)) + native._COMPILE_CACHE[key] = None + with _observed_calls() as calls: + repaired = native._compile_source(source, "malformed.py") + again = native._compile_source(source, "malformed.py") + assert repaired == first + assert repaired is not first + assert again is repaired + assert calls["compile"] == 1 + + +def test_nested_compilation_keeps_current_fifo_bound(monkeypatch): + monkeypatch.setattr(native, "_COMPILE_CACHE_MAX_ENTRIES", 1) + assert sys.getprofile() is None + nested = [] + helper_code = native._compile_source.__code__ + + def after_compile(frame, event, arg): + if ( + not nested + and event == "c_return" + and frame.f_code is helper_code + and arg is builtins.compile + ): + nested.append(True) + native._compile_source(b"nested = 2", "nested.py") + + sys.setprofile(after_compile) + try: + outer = native._compile_source(b"outer = 1", "outer.py") + finally: + sys.setprofile(None) + assert nested == [True] + assert outer == builtins.compile( + b"outer = 1", "outer.py", "exec", dont_inherit=True + ) + assert len(native._COMPILE_CACHE) == 1 + assert next(iter(native._COMPILE_CACHE))[1] == "outer.py" + + +def test_swapped_valid_entries_are_recompiled_for_their_complete_inputs(): + first_source, second_source = b"value = 1", b"value = 2" + first = native._compile_source(first_source, "first.py") + second = native._compile_source(second_source, "second.py") + first_key, second_key = tuple(native._COMPILE_CACHE) + native._COMPILE_CACHE[first_key], native._COMPILE_CACHE[second_key] = ( + native._COMPILE_CACHE[second_key], + native._COMPILE_CACHE[first_key], + ) + with _observed_calls() as calls: + repaired_first = native._compile_source(first_source, "first.py") + repaired_second = native._compile_source(second_source, "second.py") + assert native._compile_source(first_source, "first.py") is repaired_first + assert native._compile_source(second_source, "second.py") is repaired_second + assert calls["compile"] == 2 + assert repaired_first == first and repaired_first is not first + assert repaired_second == second and repaired_second is not second + assert repaired_first.co_filename == "first.py" + assert repaired_second.co_filename == "second.py" + + +def test_python_compiler_captured_as_initial_compiler_still_bypasses(monkeypatch): + # This is the exact captured/current state after a pre-import replacement; + # leave all issued-owner modules loaded and exercise the real cache helper. + calls = [] + + def replacement(source, filename, mode, **options): + calls.append(source) + return builtins.compile( + f"value = {len(calls)}".encode(), filename, mode, **options + ) + + # Textual labels alone must not classify a Python replacement as builtin. + replacement.__module__ = "builtins" + replacement.__name__ = "compile" + replacement.__self__ = builtins + monkeypatch.setattr(native, "_COMPILE_CACHE_COMPILER", replacement) + monkeypatch.setattr(native, "compile", replacement, raising=False) + first = native._compile_source(b"value = 0", "captured.py") + second = native._compile_source(b"value = 0", "captured.py") + assert calls == [b"value = 0", b"value = 0"] + assert first != second + assert not native._COMPILE_CACHE diff --git a/packages/microcosm-build/tests/test_us_acs_transfer.py b/packages/microcosm-build/tests/test_us_acs_transfer.py index e9e615bec..42d03205a 100644 --- a/packages/microcosm-build/tests/test_us_acs_transfer.py +++ b/packages/microcosm-build/tests/test_us_acs_transfer.py @@ -1467,9 +1467,8 @@ def test_pregnancy_draws_once_per_eligible_source_person_and_fans_to_clones( ) person = result.frame.person - eligible = ( - person["is_female"].astype(bool) - & person["age"].between(15, 44, inclusive="both") + eligible = person["is_female"].astype(bool) & person["age"].between( + 15, 44, inclusive="both" ) assert person.loc[eligible, "is_pregnant"].all() assert not person.loc[~eligible, "is_pregnant"].any() @@ -1490,7 +1489,9 @@ def test_pregnancy_draws_once_per_eligible_source_person_and_fans_to_clones( .first() .sum() ) - record = next(item for item in result.imputed_inputs if item.column == "is_pregnant") + record = next( + item for item in result.imputed_inputs if item.column == "is_pregnant" + ) receipt = record.structural_receipt assert receipt is not None assert sum(pattern.recipient_rows for pattern in record.patterns) == ( @@ -1576,12 +1577,8 @@ def test_pregnancy_partial_clone_fanout_receipt_categories_are_disjoint( record = result.imputed_inputs[0] receipt = record.structural_receipt assert receipt is not None - assert receipt["preexisting_value_fanout_rows"] == int( - (missing & eligible).sum() - ) - assert receipt["ineligible_rows_assigned_false"] == int( - (missing & ~eligible).sum() - ) + assert receipt["preexisting_value_fanout_rows"] == int((missing & eligible).sum()) + assert receipt["ineligible_rows_assigned_false"] == int((missing & ~eligible).sum()) assert ( receipt["preexisting_value_fanout_rows"] + receipt["ineligible_rows_assigned_false"] @@ -2468,7 +2465,12 @@ def test_strict_leaf_audit_reports_missing_us_extra( from microcosm.frame.adapters import policyengine_us as adapter_module class _MissingMetadataIndex: - def __init__(self) -> None: + # Mirrors PolicyEngineUSVariableMetadataIndex's keyword-only surface. + # The audit path constructs it twice: puf_support._formula_owned_engine + # passes include_consumers=False before the strict-leaf branch + # constructs it bare, and both must raise the absent-extra ImportError + # rather than a signature TypeError. + def __init__(self, *, include_consumers: bool = True) -> None: raise ImportError("policyengine-us is absent") monkeypatch.setattr( diff --git a/packages/microcosm-build/tests/test_us_asec_2024_native_population.py b/packages/microcosm-build/tests/test_us_asec_2024_native_population.py new file mode 100644 index 000000000..79b14b6ea --- /dev/null +++ b/packages/microcosm-build/tests/test_us_asec_2024_native_population.py @@ -0,0 +1,692 @@ +"""Real closed loaders over invented source bytes; no genuine source admission.""" + +import csv +import hashlib +import importlib.util +import json +import sys +from dataclasses import replace +from pathlib import Path + +import numpy as np +import pandas as pd +import pytest + +from microcosm.build.frame_checkpoint import ( + load_frame_checkpoint, + write_frame_checkpoint, +) +from microcosm.build.outer_stage_runtime import frame_identity +from microcosm.build.us_runtime import asec_2024_native_population as native +from microcosm.build.us_runtime import asec_coverage_authentication as coverage +from microcosm.build.us_runtime import asec_current_money_source as money +from microcosm.build.us_runtime import asec_household_observations as household_owner +from microcosm.build.us_runtime import asec_person_income_source as restoration +from microcosm.build.us_runtime import native_household_origin as origin +from microcosm.frame import Frame, WeightKind, Weights + + +def _helpers(): + spec = importlib.util.spec_from_file_location( + "native_2024_invented_fixture", + Path(__file__).with_name("test_us_asec_coverage_authentication.py"), + ) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def _rebuild_observations( + tmp_path, + monkeypatch, + persons, + *, + extra_household, + legacy_weights, + missing_money=None, + current_predictor_money=None, +): + """Build larger invented real checkpoints and the actual observation attachment.""" + parent_path = tmp_path / "invented-original-v4.h5" + household_path = tmp_path / "invented-household-attachment.h5" + loaded = load_frame_checkpoint(parent_path) + tables = {e: loaded.frame.table(e).copy(deep=True) for e in loaded.frame.entities} + strata = loaded.frame.strata.copy(deep=True) + if extra_household: + extra = tables["person"].iloc[-2:].copy(deep=True) + extra["person_id"] = np.array([107, 108], dtype=np.int64) + extra["PERIDNUM"] = pd.array([str(i).zfill(22) for i in (7, 8)], dtype="string") + extra["source_household_id"] = 8 + extra["PH_SEQ"] = 4 + for entity in loaded.frame.schema.group_entities: + extra[f"person_{entity}_id"] = 4 + group = tables[entity].iloc[-1:].copy(deep=True) + group[f"{entity}_id"] = 4 + tables[entity] = pd.concat([tables[entity], group], ignore_index=True) + tables["person"] = pd.concat([tables["person"], extra]) + strata = pd.concat([strata, strata.iloc[-2:]]) + cohort_path = tmp_path / "invented-cohort-2024.h5" + cohort = pd.read_hdf(cohort_path, key="household") + extra_cohort = cohort.iloc[-1:].copy(deep=True) + extra_cohort["H_SEQ"] = 8 + pd.concat([cohort, extra_cohort], ignore_index=True).to_hdf( + cohort_path, key="household", format="fixed", mode="w" + ) + + def add_people(rows): + extra_rows = [r.copy() for r in rows[1:]] + for row in extra_rows: + row[0] = str(int(row[0]) + 2).zfill(22) + row[1] = "8" + rows.extend(extra_rows) + + _helpers()._rewrite(persons[2024], add_people) + if missing_money is not None: + year, field = missing_money + assert (year, field) in ((2024, "ANN_VAL"), (2022, "WSAL_VAL")) + person = tables["person"] + position = np.flatnonzero(person.source_year.to_numpy() == year)[0] + # Alter only invented source construction, before identities, attachments + # and normal issuance. Never mutate an already authenticated parent. + person.iloc[position, person.columns.get_loc(field)] = np.nan + if current_predictor_money is not None: + # Alter only invented original source construction before checkpoint + # identities, observation attachments, private pins and fresh issuance. + person = tables["person"] + positions = np.flatnonzero(person.source_year.to_numpy() == 2024) + assert set(current_predictor_money) == { + "WSAL_VAL", + "SEMP_VAL", + "INT_VAL", + "DIV_VAL", + "CAP_VAL", + } + for field, observations in current_predictor_money.items(): + observations = np.asarray(observations) + assert observations.dtype == np.dtype("float64") + assert ( + observations.shape == (len(positions),) + and np.isfinite(observations).all() + ) + person.iloc[positions, person.columns.get_loc(field)] = observations + values = ( + legacy_weights + if legacy_weights is not None + else ([1, 2, 3, 9] if extra_household else [1, 2, 3]) + ) + frame = Frame( + tables, + loaded.frame.schema, + {"household": Weights(np.array(values), WeightKind.DESIGN)}, + strata, + ) + metadata = loaded.metadata + for key in ("identity", "source_construction_identity"): + metadata[key] = frame_identity(frame).to_payload() + sources = [] + for year in (2022, 2023, 2024): + path = tmp_path / f"invented-cohort-{year}.h5" + sources.append( + household_owner.AsecHouseholdObservationSource( + year, path, hashlib.sha256(path.read_bytes()).hexdigest() + ) + ) + metadata["source_receipt"]["sources"] = [ + {"year": source.year, "path": str(source.path), "sha256": source.sha256} + for source in sources + ] + write_frame_checkpoint(parent_path, frame, metadata=metadata) + parent_sha = hashlib.sha256(parent_path.read_bytes()).hexdigest() + attachment = household_owner.with_asec_household_observations( + frame, + checkpoint_metadata=metadata, + checkpoint_sha256=parent_sha, + sources=sources, + ) + write_frame_checkpoint( + household_path, + attachment.frame, + metadata={ + "schema_version": 1, + "artifact_kind": "microcosm.asec_household_observations_source", + "parent_checkpoint_sha256": parent_sha, + "household_observations": attachment.receipt, + }, + ) + monkeypatch.setattr( + money, + "_SOURCE_PINS", + ( + parent_sha, + hashlib.sha256(household_path.read_bytes()).hexdigest(), + tuple((s.year, s.sha256) for s in sources), + ), + ) + _helpers()._pin(persons, monkeypatch) + + +def _fixture( + tmp_path, + monkeypatch, + *, + household_rows=None, + extra_household=False, + legacy_weights=None, + missing_money=None, + current_predictor_money=None, + **changes, +): + helpers = _helpers() + _, persons = helpers._fixtures(tmp_path, monkeypatch, **changes) + if ( + extra_household + or legacy_weights is not None + or missing_money is not None + or current_predictor_money is not None + ): + _rebuild_observations( + tmp_path, + monkeypatch, + persons, + extra_household=extra_household, + legacy_weights=legacy_weights, + missing_money=missing_money, + current_predictor_money=current_predictor_money, + ) + # Both real owners must pin the same final invented original member bytes. + monkeypatch.setattr(restoration, "_MEMBER_PINS", coverage._MEMBER_PINS) + parent = tmp_path / "invented-original-v4.h5" + household = tmp_path / "invented-household-attachment.h5" + output = tmp_path / "restored" + restoration.restore_asec_person_income_source( + parent, household, member_paths=persons, output_dir=output + ) + member = tmp_path / "hhpub25.csv" + rows = ( + household_rows + if household_rows is not None + else [ + ["00007", "1", "000255212", "6", "1", "2"], + ["00008", "2", "", "0", "0", "0"], + ] + ) + if extra_household and household_rows is None: + rows[1] = ["00008", "1", "00000000", "6", "1", "2"] + with member.open("w", newline="") as handle: + writer = csv.writer(handle) + writer.writerow( + ["H_SEQ", "H_HHTYPE", "HSUP_WGT", "HRHTYPE", "H_LIVQRT", "H_NUMPER"] + ) + writer.writerows(rows) + data = member.read_bytes() + monkeypatch.setattr( + origin, + "_ASEC_MEMBER_PINS", + ( + origin.AsecNativeMemberPin( + income_year=2024, + survey_year=2025, + canonical_member_id="invented/2025/household/hhpub25.csv", + member_name="hhpub25.csv", + archive_sha256=origin.ASEC_EDUCATION_ASSISTANCE_ARCHIVES[ + 2024 + ].zip_sha256, + member_sha256=hashlib.sha256(data).hexdigest(), + size_bytes=len(data), + rows=len(rows), + ), + ), + ) + return dict( + parent_path=parent, + household_attachment_path=household, + person_income_attachment_path=output / restoration.CHECKPOINT_FILENAME, + person_member_paths=persons, + household_member_path=member, + ) + + +def test_real_three_cohort_loader_selects_2024_original_anchors(tmp_path, monkeypatch): + paths = _fixture(tmp_path, monkeypatch) + result = native.load_authenticated_asec_2024_native_population(**paths) + frame, receipt = result.frame, result.receipt + assert frame.person.source_year.tolist() == [2024, 2024] + assert frame.person.person_id.tolist() == [105, 106] + assert frame.person.A_AGE.tolist() == [55, 14] + assert frame.person.asec_PTOTVAL.tolist() == [-9999, 0] + assert frame.weights_for("household").kind is WeightKind.DESIGN + assert frame.weights_for("household").values.tolist() == [2552.12] + assert dict(frame.metadata) == {} and frame.mass_log == () + assert json.loads(result.context)["entities"]["person"]["rows"] == 2 + assert receipt["parent_custody"]["cohorts"] == [2022, 2023, 2024] + assert receipt["parent_custody"]["population_cohorts"] == [2024] + assert receipt["households"][0]["HSUP_WGT"] == "000255212" + assert receipt["households"][0]["fraction"] == [63803, 25] + assert receipt["households"][0]["native_key"] == [2024, 7] + assert not receipt["release_eligible"] + assert not receipt["selection"]["inclusion_probabilities_known"] + assert {r["A_LINENO"] for r in receipt["persons"]} == {1, 2} + for entity in frame.schema.group_entities: + assert len(frame.table(entity)) == 1 + # Real parent remains separate, complete and sealed after all child borrows. + state = native._ISSUED[id(result)][2] + assert state.parent.frame.person.source_year.tolist() == [ + 2022, + 2022, + 2023, + 2023, + 2024, + 2024, + ] + assert state.parent.frame.weights_for("household").values.tolist() == [1, 2, 3] + state.parent.validate() + + +def test_exact_selection_candidate_reconstruction_and_value_equal_copy_refusal( + tmp_path, monkeypatch +): + paths = _fixture(tmp_path, monkeypatch) + result = native.load_authenticated_asec_2024_native_population( + **paths, selected_households=((2024, 7),) + ) + candidate = result.to_bytes() + reconstructed = native.load_authenticated_asec_2024_native_population( + **paths, selected_households=((2024, 7),), candidate=candidate + ) + assert reconstructed == result and reconstructed is not result + reconstructed.validate() + for copy in ( + replace(result), + native.AuthenticatedAsec2024NativePopulation(candidate), + ): + assert copy == result + with pytest.raises(native.AsecNativePopulationError, match="UNISSUED"): + copy.validate() + with pytest.raises(native.AsecNativePopulationError, match="CANDIDATE_MISMATCH"): + native.load_authenticated_asec_2024_native_population(**paths, candidate=b"{}") + + +@pytest.mark.parametrize( + "selection", + [ + (), + ((2023, 7),), + ((2024, 7), (2024, 7)), + ((2024, True),), + [(2024, 7)], + ((2024, 8),), + ], +) +def test_non_exact_or_absent_selection_refuses(tmp_path, monkeypatch, selection): + paths = _fixture(tmp_path, monkeypatch) + with pytest.raises(native.AsecNativePopulationError, match="SELECTION"): + native.load_authenticated_asec_2024_native_population( + **paths, selected_households=selection + ) + + +@pytest.mark.parametrize( + "field,value", + [ + ("H_NUMPER", "1"), + ("H_NUMPER", ""), + ("H_SEQ", "9"), + ("HSUP_WGT", "2552.12"), + ("HSUP_WGT", ""), + ("HSUP_WGT", "-1"), + ("HSUP_WGT", "0"), + ("H_HHTYPE", "2"), + ], +) +def test_source_household_roster_or_unusable_anchor_refuses( + tmp_path, monkeypatch, field, value +): + names = ["H_SEQ", "H_HHTYPE", "HSUP_WGT", "HRHTYPE", "H_LIVQRT", "H_NUMPER"] + row = ["00007", "1", "255212", "6", "1", "2"] + row[names.index(field)] = value + paths = _fixture(tmp_path, monkeypatch, household_rows=[row]) + with pytest.raises(native.AsecNativePopulationError): + native.load_authenticated_asec_2024_native_population(**paths) + + +def test_absent_interview_household_not_silently_lost(tmp_path, monkeypatch): + paths = _fixture( + tmp_path, + monkeypatch, + household_rows=[ + ["7", "1", "100", "6", "1", "2"], + ["8", "1", "200", "6", "1", "1"], + ], + ) + with pytest.raises( + native.AsecNativePopulationError, match="HOUSEHOLD_MEMBER_COUNT" + ): + native.load_authenticated_asec_2024_native_population( + **paths, selected_households=((2024, 7),) + ) + + +@pytest.mark.parametrize( + "mutation", + [ + "age", + "unrelated", + "group", + "weight", + "strata", + "strata_flags", + "table_flags", + "attrs", + "columns", + "metadata", + "membership", + ], +) +def test_full_receiving_frame_mutation_refuses(tmp_path, monkeypatch, mutation): + result = native.load_authenticated_asec_2024_native_population( + **_fixture(tmp_path, monkeypatch) + ) + frame = result.frame + if mutation == "age": + frame.person.loc[:, "A_AGE"] += 1 + elif mutation == "unrelated": + frame.person.loc[:, "unrelated_raw_observation"] += 1 + elif mutation == "group": + frame.table("household").loc[:, "state_fips"] += 1 + elif mutation == "weight": + vector = frame.weights_for("household").values + vector.setflags(write=True) + vector[0] += 1 + vector.setflags(write=False) + elif mutation == "strata": + frame.strata.iloc[0] = "other" + elif mutation == "strata_flags": + frame.strata.flags.allows_duplicate_labels = False + elif mutation == "table_flags": + frame.table("household").flags.allows_duplicate_labels = False + elif mutation == "attrs": + frame.person.attrs["changed"] = True + elif mutation == "columns": + frame.person.columns.name = "changed" + elif mutation == "metadata": + frame._metadata = {"changed": True} + elif mutation == "membership": + frame.person.iloc[0, frame.person.columns.get_loc("person_tax_unit_id")] = 999 + with pytest.raises(native.AsecNativePopulationError): + result.validate() + + +@pytest.mark.parametrize( + "which", + [ + "parent_path", + "household_attachment_path", + "person_income_attachment_path", + "household_member_path", + "older_person", + ], +) +def test_changed_original_source_refuses_existing_borrow(tmp_path, monkeypatch, which): + paths = _fixture(tmp_path, monkeypatch) + result = native.load_authenticated_asec_2024_native_population(**paths) + path = ( + paths["person_member_paths"][2022] if which == "older_person" else paths[which] + ) + with path.open("ab") as handle: + handle.write(b"changed") + with pytest.raises(native.AsecNativePopulationError, match="SOURCE_FILE"): + result.validate() + + +def test_capsule_payload_rehash_and_live_owner_changes_refuse(tmp_path, monkeypatch): + paths = _fixture(tmp_path, monkeypatch) + result = native.load_authenticated_asec_2024_native_population(**paths) + state = native._ISSUED[id(result)][2] + payload = state.anchors.payload + b" " + object.__setattr__(state.anchors, "payload", payload) + object.__setattr__( + state.anchors, "_issued_sha256", hashlib.sha256(payload).hexdigest() + ) + with pytest.raises(native.AsecNativePopulationError): + result.validate() + original = restoration._implementation + monkeypatch.setattr(restoration, "_implementation", lambda: original()) + with pytest.raises(native.AsecNativePopulationError, match="PRODUCER_CODE_CHANGED"): + native.load_authenticated_asec_2024_native_population(**paths) + + +@pytest.mark.parametrize("legacy", [[100, 200, 17], [0.001, 10_000, 0.0001]]) +def test_original_anchor_ignores_legacy_pooled_weights(tmp_path, monkeypatch, legacy): + paths = _fixture(tmp_path, monkeypatch, legacy_weights=legacy) + result = native.load_authenticated_asec_2024_native_population(**paths) + assert result.frame.weights_for("household").values.tolist() == [2552.12] + assert result.receipt["households"][0]["fraction"] == [63803, 25] + assert ( + native._ISSUED[id(result)][2] + .parent.frame.weights_for("household") + .values.tolist() + == legacy + ) + + +def test_mixed_zero_complete_households_and_selected_zero_refusal( + tmp_path, monkeypatch +): + paths = _fixture(tmp_path, monkeypatch, extra_household=True) + result = native.load_authenticated_asec_2024_native_population(**paths) + assert result.frame.person.person_id.tolist() == [105, 106, 107, 108] + assert result.frame.person.A_AGE.tolist() == [55, 14, 55, 14] + assert result.frame.weights_for("household").values.tolist() == [2552.12, 0] + assert [r["fraction"] for r in result.receipt["households"]] == [ + [63803, 25], + [0, 1], + ] + selected = native.load_authenticated_asec_2024_native_population( + **paths, selected_households=((2024, 7),) + ) + assert selected.frame.person.person_id.tolist() == [105, 106] + for entity in selected.frame.schema.group_entities: + assert selected.frame.table(entity)[f"{entity}_id"].tolist() == [3] + with pytest.raises(native.AsecNativePopulationError): + native.load_authenticated_asec_2024_native_population( + **paths, selected_households=((2024, 8),) + ) + + +def test_late_mutation_during_final_file_verification_refuses(tmp_path, monkeypatch): + result = native.load_authenticated_asec_2024_native_population( + **_fixture(tmp_path, monkeypatch) + ) + frame = result.frame + fired = [] + + def trace(stack, event, argument): + if ( + event == "return" + and stack.f_code is native._file_identity.__code__ + and not fired + ): + fired.append(True) + frame.person.loc[:, "A_AGE"] += 1 + return trace + + sys.settrace(trace) + try: + with pytest.raises(native.AsecNativePopulationError, match="FRAME_CHANGED"): + result.validate() + finally: + sys.settrace(None) + assert fired + + +def test_real_loaders_run_without_prepared_population_or_money_execution( + tmp_path, monkeypatch +): + paths = _fixture(tmp_path, monkeypatch) + called = set() + + def trace(stack, event, argument): + if event == "call": + module = stack.f_globals.get("__name__", "") + name = stack.f_code.co_name + assert module not in ( + "microcosm.build.us_runtime.asec_prepared_source", + "microcosm.build.us_runtime.engine", + ) + assert name not in ( + "classify_asec_money", + "prepare_asec_current_money_population", + ) + if module.startswith("microcosm.build.us_runtime."): + called.add(name) + + sys.setprofile(trace) + try: + native.load_authenticated_asec_2024_native_population(**paths) + finally: + sys.setprofile(None) + assert { + "load_authenticated_restored_current_money_source", + "authenticate_asec_coverage", + "load_authenticated_asec_household_weights", + "load_authenticated_asec_household_coverage_fields", + } <= called + + +@pytest.mark.parametrize( + "mutation", ["missing", "duplicate", "substitute", "split", "age"] +) +def test_original_person_mutation_refuses_reconstruction( + tmp_path, monkeypatch, mutation +): + paths = _fixture(tmp_path, monkeypatch) + + def change(rows): + if mutation == "missing": + rows.pop() + elif mutation == "duplicate": + rows[2] = rows[1].copy() + elif mutation == "substitute": + rows[1][0] = "9" * 22 + elif mutation == "split": + rows[1][1] = "8" + elif mutation == "age": + rows[1][3] = "54" + + _helpers()._rewrite(paths["person_member_paths"][2024], change) + with pytest.raises(native.AsecNativePopulationError): + native.load_authenticated_asec_2024_native_population(**paths) + + +def test_external_state_and_rehashed_population_cannot_be_rewritten( + tmp_path, monkeypatch +): + result = native.load_authenticated_asec_2024_native_population( + **_fixture(tmp_path, monkeypatch) + ) + state = native._ISSUED[id(result)][2] + with pytest.raises(AttributeError): + object.__setattr__(state, "frame_identity", "a" * 64) + changed = json.loads(result.payload) + changed["frame_sha256"] = "a" * 64 + object.__setattr__(result, "payload", native._encode(changed)) + with pytest.raises(native.AsecNativePopulationError, match="UNISSUED_OR_CHANGED"): + result.validate() + + +@pytest.mark.parametrize("owner", ["anchors", "fields", "coverage"]) +def test_equal_unissued_source_evidence_refuses(tmp_path, monkeypatch, owner): + result = native.load_authenticated_asec_2024_native_population( + **_fixture(tmp_path, monkeypatch) + ) + state = native._ISSUED[id(result)][2] + evidence = getattr(state, owner) + copy = object.__new__(type(evidence)) + # Copy/deserialization cannot inherit the reader's external issuance seal. + for name in evidence.__slots__: + if name != "__weakref__": + object.__setattr__(copy, name, getattr(evidence, name)) + if owner != "coverage": + assert copy == evidence + verifier = { + "anchors": native.anchor_owner.verify_asec_household_weights_source, + "fields": native.field_owner.verify_asec_household_coverage_fields, + "coverage": lambda value: native.coverage_owner.verify_asec_coverage_parent( + value, state.parent + ), + }[owner] + with pytest.raises(ValueError): + verifier(copy) + result.validate() + + +@pytest.mark.parametrize( + "interface", ["validate", "frame", "context", "receipt", "to_bytes"] +) +def test_late_population_payload_mutation_refuses_every_interface( + tmp_path, monkeypatch, interface +): + result = native.load_authenticated_asec_2024_native_population( + **_fixture(tmp_path, monkeypatch) + ) + fired = [] + + def trace(stack, event, argument): + if ( + event == "return" + and stack.f_code is native._file_identity.__code__ + and not fired + ): + fired.append(True) + object.__setattr__(result, "payload", b'{"release_eligible":true}') + return trace + + sys.settrace(trace) + try: + with pytest.raises(native.AsecNativePopulationError): + value = getattr(result, interface) + if callable(value): + value() + finally: + sys.settrace(None) + assert fired + + +@pytest.mark.parametrize("owner", ["anchors", "fields", "coverage", "parent"]) +def test_late_attached_evidence_mutation_refuses(tmp_path, monkeypatch, owner): + result = native.load_authenticated_asec_2024_native_population( + **_fixture(tmp_path, monkeypatch) + ) + state = native._ISSUED[id(result)][2] + fired = [] + + def trace(stack, event, argument): + if ( + event == "return" + and stack.f_code is native._file_identity.__code__ + and not fired + ): + fired.append(True) + if owner == "parent": + object.__setattr__( + state.parent.source, "evidence", state.parent.source.evidence + b" " + ) + elif owner == "coverage": + object.__setattr__(state.coverage, "_body", state.coverage._body + b"x") + else: + source = getattr(state, owner) + object.__setattr__(source, "payload", source.payload + b" ") + + return_trace = trace + + def tracing(stack, event, argument): + return_trace(stack, event, argument) + return tracing + + sys.settrace(tracing) + try: + with pytest.raises(native.AsecNativePopulationError): + result.to_bytes() + finally: + sys.settrace(None) + assert fired diff --git a/packages/microcosm-build/tests/test_us_asec_catalogue_records_memo.py b/packages/microcosm-build/tests/test_us_asec_catalogue_records_memo.py new file mode 100644 index 000000000..d22eed814 --- /dev/null +++ b/packages/microcosm-build/tests/test_us_asec_catalogue_records_memo.py @@ -0,0 +1,352 @@ +"""Immutable-leaf reuse of the ASEC records identity plus actual issuer controls. + +Supplied records exercise private value helpers only; they are never treated +as an authenticated source catalogue. The last controls use real issuance over +the maintained invented preparation fixture. +""" + +import sys + +import pytest +from test_us_survey_population_preparation import fixture + +from microcosm.build.us_runtime import asec_population_catalogue as owner +from microcosm.build.us_runtime import survey_population_domains as domains +from microcosm.build.us_runtime import survey_population_preparation as preparation + + +def _key(native_id="1"): + return domains.HouseholdKey(domains.Source.ASEC, 2024, 2025, native_id) + + +def _records(): + key = _key() + person = domains.AsecPerson("2024000000000000000001", "1", "37", "1", "1", key) + household = domains.AsecHousehold(key, "1", "1", "1", "1", "1234567", (person,)) + ledger = owner.UnrepresentedAsecHousehold(_key("2"), "2", "9", "1", "0", "7654321") + return (household,), (ledger,) + + +def _calls(action): + calls = [] + + def trace(frame, event, arg): + if event == "call" and frame.f_code is owner._records_identity.__code__: + calls.append(True) + + previous = sys.getprofile() + sys.setprofile(trace) + try: + result = action() + finally: + sys.setprofile(previous) + return result, len(calls) + + +def _issued(households, ledger): + (identity, memo), calls = _calls( + lambda: owner._records_identity(households, ledger, memo=True) + ) + assert calls == 1 + return identity, memo + + +def _memoized(households, ledger, memo, identity): + return _calls( + lambda: owner._memoized_records_identity( + households, ledger, memo, expected_identity=identity + ) + ) + + +def test_same_issued_leaves_reuse_exact_identity_without_another_encode(): + households, ledger = _records() + identity, memo = _issued(households, ledger) + assert identity == owner._records_identity(households, ledger) + assert type(memo) is owner._RecordsMemo + assert memo.households is households and memo.ledger is ledger + assert memo.identity == identity + assert memo.household_leaves == (owner._household_leaves(households[0]),) + assert memo.ledger_leaves == (owner._ledger_leaves(ledger[0]),) + result, calls = _calls( + lambda: [ + owner._memoized_records_identity( + households, ledger, memo, expected_identity=identity + ) + for _ in range(3) + ] + ) + assert result == [identity] * 3 and calls == 0 + + +def test_equal_replacement_roots_miss_without_changing_identity_or_memo(): + households, ledger = _records() + identity, memo = _issued(households, ledger) + replacement = (tuple(list(households)), tuple(list(ledger))) + assert replacement == (households, ledger) + assert replacement[0] is not households and replacement[1] is not ledger + result, calls = _memoized(*replacement, memo, identity) + assert result == identity and calls == 1 + assert memo.households is households and memo.ledger is ledger + + +@pytest.mark.parametrize( + "target", + ["household", "person", "household_key", "person_key", "ledger", "reason"], +) +def test_in_place_leaf_change_misses_and_changes_identity(target): + households, ledger = _records() + identity, memo = _issued(households, ledger) + household, person = households[0], households[0].persons[0] + if target == "household": + object.__setattr__(household, "hsup_wgt", "0000000") + elif target == "person": + object.__setattr__(person, "age", "38") + elif target == "household_key": + object.__setattr__(household.key, "native_id", "9") + elif target == "person_key": + object.__setattr__(person, "household_key", _key("9")) + elif target == "ledger": + object.__setattr__(ledger[0], "h_numper", "1") + else: + object.__setattr__(ledger[0], "reason", "other") + result, calls = _memoized(households, ledger, memo, identity) + assert calls == 1 and result != identity + assert result == owner._records_identity(households, ledger) + + +@pytest.mark.parametrize("target", ["persons", "key"]) +def test_replaced_container_misses_even_when_equal(target): + households, ledger = _records() + identity, memo = _issued(households, ledger) + household = households[0] + original = getattr(household, target) + replacement = ( + tuple(list(original)) if target == "persons" else _key(original.native_id) + ) + assert replacement == original and replacement is not original + object.__setattr__(household, target, replacement) + result, calls = _memoized(households, ledger, memo, identity) + assert result == identity and calls == 1 + + +def test_persons_replaced_by_a_list_takes_the_refusing_path(): + households, ledger = _records() + identity, memo = _issued(households, ledger) + object.__setattr__(households[0], "persons", list(households[0].persons)) + with pytest.raises(owner.AsecSourceCatalogueError, match="^RECORD_TYPE$"): + owner._memoized_records_identity( + households, ledger, memo, expected_identity=identity + ) + + +def test_deleted_leaf_misses_instead_of_raising_inside_the_memo(): + households, ledger = _records() + identity, memo = _issued(households, ledger) + object.__delattr__(households[0].persons[0], "age") + with pytest.raises(AttributeError): + owner._memoized_records_identity( + households, ledger, memo, expected_identity=identity + ) + + +@pytest.mark.parametrize( + "kind", + [ + "str_subclass_literal", + "str_subclass_person", + "str_subclass_reason", + "int_subclass_year", + "str_subclass_native_id", + ], +) +def test_subclass_leaves_yield_no_memo_and_keep_the_full_identity_path(kind): + class StrSubclass(str): + pass + + class IntSubclass(int): + pass + + households, ledger = _records() + household, person = households[0], households[0].persons[0] + if kind == "str_subclass_literal": + object.__setattr__(household, "h_hhtype", StrSubclass("1")) + elif kind == "str_subclass_person": + object.__setattr__(person, "prpertyp", StrSubclass("1")) + elif kind == "str_subclass_reason": + object.__setattr__(ledger[0], "reason", StrSubclass("x")) + elif kind == "int_subclass_year": + object.__setattr__(household.key, "source_year", IntSubclass(2024)) + else: + object.__setattr__(ledger[0].key, "native_id", StrSubclass("2")) + identity, memo = _issued(households, ledger) + assert memo is None + assert identity == owner._records_identity(households, ledger) + result, calls = _memoized(households, ledger, memo, identity) + assert result == identity and calls == 1 + + +def test_memo_requires_agreement_with_the_owner_identity(): + households, ledger = _records() + identity, memo = _issued(households, ledger) + wrong = (*identity[:3], identity[3] + 1) + result, calls = _memoized(households, ledger, memo, wrong) + assert result == identity and calls == 1 + drifted = memo._replace(identity=wrong) + result, calls = _memoized(households, ledger, drifted, identity) + assert result == identity and calls == 1 + plain = tuple(memo) + assert plain == memo + result, calls = _memoized(households, ledger, plain, identity) + assert result == identity and calls == 1 + + +def test_swapped_memo_entry_misses(): + households, ledger = _records() + identity, _ = _issued(households, ledger) + other_households, other_ledger = _records() + object.__setattr__(other_households[0], "hsup_wgt", "0000001") + other_identity, other_memo = _issued(other_households, other_ledger) + assert other_identity != identity + result, calls = _memoized(households, ledger, other_memo, identity) + assert result == identity and calls == 1 + result, calls = _memoized(households, ledger, other_memo, other_identity) + assert result == identity and calls == 1 + + +def test_crafted_memo_over_lists_still_refuses(): + households, ledger = _records() + identity, memo = _issued(households, ledger) + rows = list(households) + crafted = owner._RecordsMemo( + rows, ledger, memo.household_leaves, memo.ledger_leaves, identity + ) + with pytest.raises(owner.AsecSourceCatalogueError, match="^RECORD_TYPE$"): + owner._memoized_records_identity( + rows, ledger, crafted, expected_identity=identity + ) + + +def test_wrong_source_member_or_foreign_types_yield_no_memo(): + households, ledger = _records() + foreign = domains.HouseholdKey(domains.Source.ACS, 2024, 2025, "1") + object.__setattr__(ledger[0], "key", foreign) + with pytest.raises(owner.AsecSourceCatalogueError, match="^RECORD_SOURCE$"): + owner._records_identity(households, ledger, memo=True) + assert owner._eligible_leaves(ledger[0], "unrepresented") is None + assert owner._eligible_leaves(households[0], "unrepresented") is None + assert owner._eligible_leaves(ledger[0], "household") is None + + +def test_source_member_value_tampering_misses_and_the_full_path_reflects_it(): + households, ledger = _records() + identity, memo = _issued(households, ledger) + member = domains.Source.ASEC + original = member._value_ + object.__setattr__(member, "_value_", "tampered") + try: + assert member.value == "tampered" + result, calls = _memoized(households, ledger, memo, identity) + assert calls == 1 and result != identity + assert result == owner._records_identity(households, ledger) + finally: + object.__setattr__(member, "_value_", original) + assert member.value == original + result, calls = _memoized(households, ledger, memo, identity) + assert result == identity and calls == 0 + + +@pytest.mark.parametrize("target", ["household", "person", "key", "ledger"]) +def test_class_reassignment_misses_and_the_full_path_refuses(target): + class SwappedHousehold(domains.AsecHousehold): + __slots__ = () + + class SwappedPerson(domains.AsecPerson): + __slots__ = () + + class SwappedKey(domains.HouseholdKey): + __slots__ = () + + class SwappedLedger(owner.UnrepresentedAsecHousehold): + __slots__ = () + + households, ledger = _records() + identity, memo = _issued(households, ledger) + victim, swapped = { + "household": (households[0], SwappedHousehold), + "person": (households[0].persons[0], SwappedPerson), + "key": (households[0].key, SwappedKey), + "ledger": (ledger[0], SwappedLedger), + }[target] + original = type(victim) + object.__setattr__(victim, "__class__", swapped) + try: + assert type(victim) is swapped + with pytest.raises(owner.AsecSourceCatalogueError, match="^RECORD_TYPE$"): + owner._memoized_records_identity( + households, ledger, memo, expected_identity=identity + ) + finally: + object.__setattr__(victim, "__class__", original) + result, calls = _memoized(households, ledger, memo, identity) + assert result == identity and calls == 0 + + +@pytest.fixture(scope="module") +def actual_preparation(tmp_path_factory): + with pytest.MonkeyPatch.context() as monkeypatch: + arguments = fixture(tmp_path_factory.mktemp("asec-records-memo"), monkeypatch) + result = preparation.prepare_authenticated_survey_population(**arguments) + state = preparation._ISSUED[id(result)][2] + catalogue = state.catalogues[1] + catalogue_state = owner._ISSUED[id(catalogue)][2] + assert type(catalogue_state.records_memo) is owner._RecordsMemo + assert catalogue_state.records_memo.identity == catalogue_state.records_identity + yield result, catalogue, catalogue_state + result.validate() + catalogue.validate() + + +def test_actual_borrows_reuse_the_issued_identity(actual_preparation): + result, catalogue, _ = actual_preparation + _, calls = _calls(catalogue.validate) + assert calls == 0 + _, calls = _calls(result.checked_view) + assert calls == 0 + + +def test_actual_household_leaf_change_refuses_every_borrow(actual_preparation): + result, catalogue, catalogue_state = actual_preparation + row = catalogue_state.households[0] + original = row.hsup_wgt + object.__setattr__(row, "hsup_wgt", original + "0") + try: + with pytest.raises(owner.AsecSourceCatalogueError, match="RECORDS_CHANGED"): + catalogue.validate() + with pytest.raises( + preparation.SurveyPopulationPreparationError, + match="NESTED_EVIDENCE_CHANGED", + ): + result.checked_view() + finally: + object.__setattr__(row, "hsup_wgt", original) + _, calls = _calls(catalogue.validate) + assert calls == 0 + + +def test_actual_person_key_change_refuses_every_borrow(actual_preparation): + result, catalogue, catalogue_state = actual_preparation + person = catalogue_state.households[0].persons[0] + original = person.household_key + object.__setattr__(person, "household_key", _key(original.native_id + "0")) + try: + with pytest.raises(owner.AsecSourceCatalogueError, match="RECORDS_CHANGED"): + catalogue.validate() + with pytest.raises( + preparation.SurveyPopulationPreparationError, + match="NESTED_EVIDENCE_CHANGED", + ): + result.checked_view() + finally: + object.__setattr__(person, "household_key", original) + result.validate() diff --git a/packages/microcosm-build/tests/test_us_asec_checkpoint.py b/packages/microcosm-build/tests/test_us_asec_checkpoint.py index aa2a1338a..3496eb470 100644 --- a/packages/microcosm-build/tests/test_us_asec_checkpoint.py +++ b/packages/microcosm-build/tests/test_us_asec_checkpoint.py @@ -7,6 +7,7 @@ import pandas as pd import pytest +import microcosm.build.us_runtime.asec_checkpoint as checkpoint_module from microcosm.build.frame_checkpoint import write_frame_checkpoint from microcosm.build.outer_stage_runtime import ( OUTER_STAGE_CONTEXT_SCHEMA_VERSION, @@ -26,6 +27,396 @@ ) from microcosm.frame import US_SCHEMA, EntitySchema, Frame, WeightKind, Weights + +def _v4_frame() -> Frame: + frame = _raw_us_frame() + for column in checkpoint_module.ASEC_REPORTED_COVERAGE_RAW_COLUMNS: + frame.table("person")[column] = np.asarray([1, 2], dtype=np.int64) + return frame + + +def _v4_binding(frame: Frame) -> dict: + metadata = _raw_binding(frame) + metadata["schema_version"] = 4 + metadata["source_receipt"]["sources"].append( + dict(metadata["source_receipt"]["sources"][0], year=2023) + ) + pins = checkpoint_module.ASEC_EDUCATION_ASSISTANCE_ARCHIVES + for column in checkpoint_module.ASEC_REPORTED_COVERAGE_RAW_COLUMNS: + metadata["raw_source_mappings"][column] = { + "column": column, + "entity": "person", + "operation": "exact_source_join", + "join_keys": ["source_year", "PERIDNUM"], + "source_pins": [ + { + "income_year": year, + "locator": pins[year].zip_url, + "member": pins[year].member, + "sha256": pins[year].zip_sha256, + "member_sha256": pins[year].member_sha256, + } + for year in (2022, 2023) + ], + # Audit describes the full official member, not this two-row subset. + "audit": { + str(year): { + "rows": pins[year].rows, + "yes_rows": 1, + "no_rows": pins[year].rows - 1, + "weighted_yes_share": 0.25, + } + for year in (2022, 2023) + }, + } + return metadata + + +def test_v4_roundtrip_requires_explicit_codec(tmp_path: Path) -> None: + frame = _v4_frame() + path = tmp_path / "v4.h5" + _write_checkpoint(path, frame, metadata=_v4_binding(frame)) + loaded, binding = checkpoint_module.load_asec_raw_stage_checkpoint_v4(path) + assert binding["schema_version"] == 4 + assert loaded.table("person")["NOW_MCAID"].tolist() == [1, 2] + with pytest.raises(ValueError, match="unsupported raw-stage schema version"): + load_asec_raw_stage_checkpoint(path) + path3 = tmp_path / "v3.h5" + legacy = _raw_us_frame() + _write_checkpoint(path3, legacy, metadata=_raw_binding(legacy)) + with pytest.raises(ValueError, match="unsupported raw-stage schema version"): + checkpoint_module.load_asec_raw_stage_checkpoint_v4(path3) + + +def test_restored_raw_observations_carry_to_input_leaves_without_engine() -> None: + from microcosm.build.us_runtime.cps_carried import derive_us_cps_carried_inputs + + frame = _raw_us_frame() + person = frame.table("person") + person["NOW_GRP"] = [1, 2] + person["NOW_MRK"] = [2, 1] + person["OI_VAL"] = [0.0, 0.0] + person["OI_OFF"] = [0, 0] + sidecar = person[["source_year", "PERIDNUM"]].copy() + sidecar["PH_SEQ"] = [101, 102] + sidecar["P_SEQ"] = [1, 1] + sidecar["A_LINENO"] = [1, 1] + for column in checkpoint_module.ASEC_REPORTED_COVERAGE_RAW_COLUMNS: + sidecar[column] = [1, 2] + result = derive_us_cps_carried_inputs(frame, reported_coverage_source=sidecar) + assert "NOW_MCAID" not in frame.table("person") + assert result.table("person")[ + "has_medicaid_health_coverage_at_interview" + ].tolist() == [True, False] + assert result.table("person")["has_esi"].tolist() == [True, False] + assert result.table("person")[ + "has_marketplace_health_coverage_at_interview" + ].tolist() == [False, True] + assert result.weights_for("household").kind is WeightKind.DESIGN + pd.testing.assert_frame_equal( + result.table("person"), derive_us_cps_carried_inputs(result).table("person") + ) + + +@pytest.mark.parametrize( + "with_mass_history,forward_metadata", + [(False, False), (True, False), (True, True)], + ids=["empty_log", "declared_mass_change", "metadata_forwarding"], +) +def test_v4_restoration_producer_authenticates_source_and_writes_new_bundle( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + with_mass_history: bool, + forward_metadata: bool, +) -> None: + import hashlib + from dataclasses import replace + + from microcosm.build.us_runtime import asec_raw_stage_v4 as restoration + from microcosm.build.us_runtime import reported_coverage_source + from microcosm.build.us_runtime.asec_raw_stage_v4 import restore_asec_raw_stage_v4 + from microcosm.frame import MassChange + + legacy = _raw_us_frame() + if with_mass_history: + original = legacy + weights = original.weights_for("household") + legacy = original.with_weights( + "household", + Weights(weights.values * 2.0, weights.kind), + mass=MassChange( + factor=2.0, reason="Invented raw-source design-weight rescaling" + ), + ) + assert original.mass_log == () + assert len(legacy.mass_log) == 1 + record = legacy.mass_log[0] + assert record.entity == "household" + assert record.old_total == float(weights.values.sum()) + assert record.new_total == 2.0 * record.old_total + assert record.declared_factor == 2.0 + assert record.reason == "Invented raw-source design-weight rescaling" + expected_mass_log = legacy.mass_log + assert bool(expected_mass_log) is with_mass_history + binding = _raw_binding(legacy) + binding["source_receipt"]["sources"].append( + dict(binding["source_receipt"]["sources"][0], year=2023) + ) + input_path = tmp_path / "v3.h5" + _write_checkpoint(input_path, legacy, metadata=binding) + loaded_input, _ = checkpoint_module.load_asec_raw_stage_checkpoint(input_path) + assert loaded_input.mass_log == expected_mass_log + np.testing.assert_array_equal( + loaded_input.weights_for("household").values, + legacy.weights_for("household").values, + ) + input_sha = hashlib.sha256(input_path.read_bytes()).hexdigest() + pins = {} + paths = {} + for year, key in ( + (2022, "0000000000000000000001"), + (2023, "0000000000000000000002"), + ): + path = tmp_path / f"person-{year}.csv" + row = { + "PERIDNUM": key, + "PH_SEQ": 1, + "P_SEQ": 1, + "A_LINENO": 1, + "A_FNLWGT": 100.0, + } + row.update( + { + column: 1 if year == 2022 else 2 + for column in checkpoint_module.ASEC_REPORTED_COVERAGE_RAW_COLUMNS + } + ) + pd.DataFrame([row]).to_csv(path, index=False) + pins[year] = replace( + checkpoint_module.ASEC_EDUCATION_ASSISTANCE_ARCHIVES[year], + rows=1, + member=path.name, + member_size_bytes=path.stat().st_size, + member_sha256=hashlib.sha256(path.read_bytes()).hexdigest(), + ) + paths[year] = path + monkeypatch.setattr(checkpoint_module, "ASEC_EDUCATION_ASSISTANCE_ARCHIVES", pins) + monkeypatch.setattr( + reported_coverage_source, "ASEC_EDUCATION_ASSISTANCE_ARCHIVES", pins + ) + forwarded = [] + if forward_metadata: + # The checkpoint codec does not persist Frame.metadata. Supply a + # descriptive value only after the real v3 loader has validated its + # input, then observe the actual converter at the real writer boundary. + supplied_metadata = {"forwarding_probe": {"purpose": "invented observation"}} + real_load = checkpoint_module.load_asec_raw_stage_checkpoint + real_write = restoration.write_frame_checkpoint + + def load_with_descriptive_metadata(path): + loaded, binding = real_load(path) + assert not loaded.metadata + return ( + Frame( + {entity: loaded.table(entity) for entity in loaded.entities}, + loaded.schema, + { + entity: loaded.weights_for(entity) + for entity in loaded.weighted_entities + }, + loaded.strata, + mass_log=loaded.mass_log, + metadata=supplied_metadata, + ), + binding, + ) + + def observe_real_write(path, frame, **kwargs): + assert set(frame.metadata) == {"forwarding_probe"} + assert ( + dict(frame.metadata["forwarding_probe"]) + == supplied_metadata["forwarding_probe"] + ) + assert frame.mass_log == expected_mass_log + forwarded.append(True) + return real_write(path, frame, **kwargs) + + monkeypatch.setattr( + checkpoint_module, + "load_asec_raw_stage_checkpoint", + load_with_descriptive_metadata, + ) + monkeypatch.setattr(restoration, "write_frame_checkpoint", observe_real_write) + output_dir = tmp_path / "restored" + receipt = restore_asec_raw_stage_v4( + input_path, + expected_sha256=input_sha, + coverage_paths=paths, + output_dir=output_dir, + ) + assert forwarded == ([True] if forward_metadata else []) + assert receipt["input_sha256"] == input_sha + output = output_dir / "asec_raw_stage.checkpoint.h5" + assert receipt["output_sha256"] == hashlib.sha256(output.read_bytes()).hexdigest() + assert hashlib.sha256(input_path.read_bytes()).hexdigest() == input_sha + restored, metadata = checkpoint_module.load_asec_raw_stage_checkpoint_v4(output) + assert restored.mass_log == expected_mass_log + # The codec does not reconstruct the separately supplied Frame metadata. + assert not restored.metadata + assert restored.weights_for("household").kind is WeightKind.DESIGN + np.testing.assert_array_equal( + restored.weights_for("household").values, + loaded_input.weights_for("household").values, + ) + assert restored.table("person")["NOW_MCAID"].tolist() == [1, 2] + assert metadata["source_receipt"] == binding["source_receipt"] + assert json.loads((output_dir / "restoration.receipt.json").read_text()) == receipt + with pytest.raises(FileExistsError): + restore_asec_raw_stage_v4( + input_path, + expected_sha256=input_sha, + coverage_paths=paths, + output_dir=output_dir, + ) + with pytest.raises(ValueError, match="SHA-256"): + restore_asec_raw_stage_v4( + input_path, + expected_sha256="f" * 64, + coverage_paths=paths, + output_dir=tmp_path / "bad", + ) + with pytest.raises(ValueError, match="local coverage paths"): + restore_asec_raw_stage_v4( + input_path, + expected_sha256=input_sha, + coverage_paths={2022: paths[2022]}, + output_dir=tmp_path / "missing", + ) + paths[2022].write_text("tampered") + with pytest.raises(ValueError, match="byte length mismatch"): + restore_asec_raw_stage_v4( + input_path, + expected_sha256=input_sha, + coverage_paths=paths, + output_dir=tmp_path / "tampered", + ) + assert not (tmp_path / "bad").exists() + assert not (tmp_path / "missing").exists() + assert not (tmp_path / "tampered").exists() + + +@pytest.mark.parametrize( + "mutation", + ( + "pin_sha", + "pin_member", + "pin_year_missing", + "pin_year_duplicate", + "audit_year", + "audit_rows", + "audit_counts", + "audit_nan", + "audit_bool", + "receipt_year", + "missing_mapping", + "float_version", + ), +) +def test_v4_refuses_inconsistent_attestation(tmp_path: Path, mutation: str) -> None: + frame = _v4_frame() + metadata = _v4_binding(frame) + mapping = metadata["raw_source_mappings"]["NOW_MCAID"] + if mutation == "pin_sha": + mapping["source_pins"][0]["sha256"] = "f" * 64 + elif mutation == "pin_member": + mapping["source_pins"][0]["member"] = "other.csv" + elif mutation == "pin_year_missing": + mapping["source_pins"].pop() + elif mutation == "pin_year_duplicate": + mapping["source_pins"].append(dict(mapping["source_pins"][0])) + elif mutation == "audit_year": + del mapping["audit"]["2023"] + elif mutation == "audit_rows": + mapping["audit"]["2022"]["rows"] = 2 + elif mutation == "audit_counts": + mapping["audit"]["2022"]["no_rows"] = 0 + elif mutation == "audit_nan": + mapping["audit"]["2022"]["weighted_yes_share"] = float("nan") + elif mutation == "audit_bool": + mapping["audit"]["2022"]["yes_rows"] = True + elif mutation == "receipt_year": + metadata["source_receipt"]["sources"].pop() + elif mutation == "missing_mapping": + del metadata["raw_source_mappings"]["NOW_MCAID"] + elif mutation == "float_version": + metadata["schema_version"] = 4.0 + path = tmp_path / "invalid.h5" + # Exercise the same metadata validators directly for NaN because canonical + # checkpoint serialization correctly refuses it before the loader does. + if mutation == "audit_nan": + with pytest.raises(ValueError, match="audit counts/share"): + checkpoint_module._validate_coverage_v4_binding(frame, metadata, path=path) + else: + _write_checkpoint(path, frame, metadata=metadata) + with pytest.raises(ValueError): + checkpoint_module.load_asec_raw_stage_checkpoint_v4(path) + + +@pytest.mark.parametrize("value", (0, 3, 1.5, float("inf"), float("nan"), True)) +def test_v4_refuses_bad_measured_coverage(tmp_path: Path, value: object) -> None: + frame = _v4_frame() + frame.table("person")["NOW_MCAID"] = [value, value] + path = tmp_path / "bad-code.h5" + _write_checkpoint(path, frame, metadata=_v4_binding(frame)) + with pytest.raises(ValueError, match="NOW_MCAID must be complete integer recodes"): + checkpoint_module.load_asec_raw_stage_checkpoint_v4(path) + + +@pytest.mark.parametrize( + "mutation", ("missing_column", "peridnum", "duplicate", "weights") +) +def test_v4_refuses_wrong_source_boundary(tmp_path: Path, mutation: str) -> None: + frame = _v4_frame() + person = frame.table("person") + if mutation == "missing_column": + person.drop(columns="NOW_IHSFLG", inplace=True) + elif mutation == "peridnum": + person["PERIDNUM"] = ["1", "2"] + elif mutation == "duplicate": + person["source_year"] = [2022, 2022] + person["PERIDNUM"] = ["0" * 22, "0" * 22] + elif mutation == "weights": + frame = Frame( + {entity: frame.table(entity) for entity in frame.entities}, + frame.schema, + {"household": Weights(np.asarray([2.0, 3.0]), WeightKind.CALIBRATED)}, + frame.strata, + ) + metadata = _v4_binding(frame) + if mutation == "duplicate": + metadata["source_receipt"]["sources"].pop() + path = tmp_path / "boundary.h5" + _write_checkpoint(path, frame, metadata=metadata) + with pytest.raises(ValueError): + checkpoint_module.load_asec_raw_stage_checkpoint_v4(path) + + +def test_v4_duplicate_guard_compares_normalized_identity() -> None: + frame = _v4_frame() + person = frame.table("person") + person["source_year"] = pd.Series([2022, "2022"], dtype=object) + person["PERIDNUM"] = pd.Series(["0" * 22, b"0" * 22], dtype=object) + metadata = _v4_binding(frame) + metadata["source_receipt"]["sources"].pop() + for column in checkpoint_module.ASEC_REPORTED_COVERAGE_RAW_COLUMNS: + metadata["raw_source_mappings"][column]["source_pins"].pop() + del metadata["raw_source_mappings"][column]["audit"]["2023"] + with pytest.raises(ValueError, match="repeats a source-year/PERIDNUM"): + checkpoint_module._validate_coverage_v4_binding( + frame, metadata, path=Path("fixture.h5") + ) + + _OUTER_STAGE_ARTIFACT_KIND = "populace_outer_stage_frame" diff --git a/packages/microcosm-build/tests/test_us_asec_coverage_authentication.py b/packages/microcosm-build/tests/test_us_asec_coverage_authentication.py new file mode 100644 index 000000000..22ee44b3c --- /dev/null +++ b/packages/microcosm-build/tests/test_us_asec_coverage_authentication.py @@ -0,0 +1,944 @@ +"""Invented closed source-parent fixtures only; no genuine preparation.""" + +import _csv +import csv +import hashlib +import importlib.util +import json +import os +import struct +import subprocess +import sys +import textwrap +import traceback +from dataclasses import FrozenInstanceError, replace +from pathlib import Path +from types import SimpleNamespace + +import numpy as np +import pandas as pd +import pytest + +from microcosm.build.frame_checkpoint import ( + load_frame_checkpoint, + write_frame_checkpoint, +) +from microcosm.build.outer_stage_runtime import frame_identity +from microcosm.build.us_runtime import asec_coverage_authentication as coverage +from microcosm.build.us_runtime import asec_current_money_source as money +from microcosm.build.us_runtime import asec_person_coverage_source as literal + + +def _helper(name): + spec = importlib.util.spec_from_file_location( + "coverage_invented_" + name, Path(__file__).with_name(name + ".py") + ) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def _pin(paths, monkeypatch): + pins = [] + for year, member, archive, _, _, _ in coverage._MEMBER_PINS: + value = paths[year].read_bytes() + with paths[year].open(newline="") as handle: + rows = list(csv.reader(handle)) + pins.append( + (year, member, archive, coverage._sha(value), len(rows) - 1, len(value)) + ) + monkeypatch.setattr(coverage, "_MEMBER_PINS", tuple(pins)) + + +def _rewrite(path, change): + with path.open(newline="") as handle: + rows = list(csv.reader(handle)) + change(rows) + with path.open("w", newline="") as handle: + csv.writer(handle).writerows(rows) + + +def _changed_parent(parent, attachment, monkeypatch, changes): + for path in (parent, attachment): + loaded = load_frame_checkpoint(path) + for column, values in changes.items(): + loaded.frame.person[column] = values + identity = frame_identity(loaded.frame) + for key in ("identity", "source_construction_identity"): + if key in loaded.metadata: + loaded.metadata[key] = identity.to_payload() + if path == parent: + parent_identity = identity + else: + digest = coverage._sha(parent.read_bytes()) + loaded.metadata["parent_checkpoint_sha256"] = digest + loaded.metadata["household_observations"]["input_checkpoint_sha256"] = ( + digest + ) + loaded.metadata["household_observations"][ + "input_structural_identity_sha256" + ] = parent_identity.sha256 + write_frame_checkpoint(path, loaded.frame, metadata=loaded.metadata) + monkeypatch.setattr( + money, + "_SOURCE_PINS", + ( + coverage._sha(parent.read_bytes()), + coverage._sha(attachment.read_bytes()), + money._SOURCE_PINS[2], + ), + ) + + +def _fixtures( + tmp_path, + monkeypatch, + tokens=("1", "2", "3", "", "-4", "NA"), + *, + native_ids=None, + person_changes=None, + raw_ages=None, +): + parent, attachment, paths = _helper( + "test_us_asec_person_income_source" + ).invented_sources(tmp_path, monkeypatch, missing=False) + if native_ids is not None: + _helper("test_us_asec_demographic_source")._repoint_published_households( + parent, attachment, monkeypatch, native_ids + ) + if person_changes: + _changed_parent(parent, attachment, monkeypatch, person_changes) + source = money.load_authenticated_current_money_source(parent, attachment) + for path in paths.values(): + + def change(rows): + rows[0].append("PRPERTYP") + for row in rows[1:]: + position = int(row[0]) - 1 + row.append(tokens[position % len(tokens)]) + if native_ids is not None: + row[1] = str(native_ids[position]) + if raw_ages is not None: + row[3] = str(raw_ages[position]) + if person_changes and "PERIDNUM" in person_changes: + row[0] = person_changes["PERIDNUM"][position] + + _rewrite(path, change) + _pin(paths, monkeypatch) + return source, paths + + +def _read(source, paths, **kwargs): + return coverage.authenticate_asec_coverage(source, member_paths=paths, **kwargs) + + +def test_closed_complete_source_exact_parent_and_defensive_views(tmp_path, monkeypatch): + source, paths = _fixtures(tmp_path, monkeypatch) + before = money._frame_signature(source.frame) + tables = {e: source.frame.table(e).copy(deep=True) for e in source.frame.entities} + result = _read(source, paths) + coverage.verify_asec_coverage_parent(result, source) + view = result.table() + assert view.PRPERTYP.tolist() == ["1", "2", "3", "", "-4", "NA"] + assert view.PRPERTYP_state.tolist() == ["observed_code"] * 3 + [ + "blank_unresolved", + "unlabelled_in_range", + "malformed_token", + ] + assert view.PERIDNUM.tolist() == list(source.scope.person_native_keys) + assert view.person_household_id.tolist() == [1, 1, 2, 2, 3, 3] + raw, original_receipt = literal.read_asec_person_coverage_source( + paths, person_roster=source.frame.person + ) + receipt = result.receipt + assert receipt["literal_reader_receipt"] == original_receipt + # Native key cells retain their exact strings; the artifact view declares + # one non-nullable vocabulary independently of the parent's string storage. + pd.testing.assert_frame_equal(view[list(raw)], raw.astype({"PERIDNUM": "string"})) + assert receipt["source_authenticated"] is True + assert receipt["named_parent_binding_authenticated"] is True + for name in ( + "release_eligible", + "domain_authority", + "period_harmonized", + "cross_survey_coverage_equivalence_established", + "original_design_weight_semantics_established", + "archive_bytes_read", + ): + assert receipt[name] is False + assert receipt["literal_reader_receipt"]["source_authenticated"] is False + assert ( + receipt["literal_reader_receipt"]["population_binding_authenticated"] is False + ) + assert receipt["age_relation"]["compared_rows"] == 6 + assert ( + receipt["age_relation"]["relation"] == "original_csv_A_AGE_equals_parent_A_AGE" + ) + assert receipt["household_partitions"]["households"] == 3 + assert [p["rows"] for p in receipt["sources"]] == [2, 2, 2] + assert str(tmp_path) not in json.dumps(receipt) + assert "0000000000000000000001" not in json.dumps(receipt) + pin = result.content_sha256 + view.loc[0, "PRPERTYP"] = "changed" + view.index = pd.Index([99, 1, 2, 3, 4, 5]) + receipt["parent"]["frame_sha256"] = "changed" + assert result.content_sha256 == pin and result.table().PRPERTYP.iloc[0] == "1" + assert money._frame_signature(source.frame) == before + for entity, expected in tables.items(): + pd.testing.assert_frame_equal( + source.frame.table(entity), expected, check_exact=True + ) + with pytest.raises(FrozenInstanceError): + result._body = b"changed" + with pytest.raises(coverage.AsecCoverageAuthenticationError, match="CONSTRUCTOR"): + coverage.AuthenticatedAsecCoverage(result._header, result._body) + with pytest.raises(coverage.AsecCoverageAuthenticationError, match="CONSTRUCTOR"): + replace(result) + + +@pytest.mark.parametrize( + "token,state", + [ + ("", "blank_unresolved"), + ("\0", "malformed_token"), + ("1\0x", "malformed_token"), + (" 1", "malformed_token"), + ("1.0", "malformed_token"), + ("NA", "malformed_token"), + ('a"b', "malformed_token"), + ("a,b\r\n\t😀", "malformed_token"), + ("-4", "unlabelled_in_range"), + ("4", "out_of_range"), + ("3", "observed_code"), + ], +) +def test_literal_utf8_roundtrip_without_coercion(tmp_path, monkeypatch, token, state): + source, paths = _fixtures(tmp_path, monkeypatch, (token,)) + result = _read(source, paths) + assert result.table().PRPERTYP.tolist() == [token] * 6 + assert result.table().PRPERTYP_state.tolist() == [state] * 6 + assert not result.table().PRPERTYP.isna().any() + + +@pytest.mark.parametrize("kind", ["frame", "receipt", "hash"]) +def test_no_plain_parent_can_issue_authority(tmp_path, monkeypatch, kind): + source, paths = _fixtures(tmp_path, monkeypatch) + candidate = { + "frame": source.frame, + "receipt": json.loads(source.source.identity), + "hash": money._frame_signature(source.frame), + }[kind] + with pytest.raises( + coverage.AsecCoverageAuthenticationError, match="AUTHENTICATED_PARENT" + ): + _read(candidate, paths) + with pytest.raises(TypeError): + coverage.authenticate_asec_coverage(source, member_paths=paths, member_pins=()) + + +@pytest.mark.parametrize( + "column,value", + [ + ("A_AGE", "82"), + ("A_AGE", ""), + ("A_AGE", "55.0"), + ("A_AGE", "\0"), + ("PH_SEQ", "8"), + ("A_LINENO", "8"), + ("PERIDNUM", "9" * 22), + ], +) +def test_authenticated_raw_coordinate_conflicts_refuse( + tmp_path, monkeypatch, column, value +): + source, paths = _fixtures(tmp_path, monkeypatch) + _rewrite( + paths[2022], lambda rows: rows[1].__setitem__(rows[0].index(column), value) + ) + _pin(paths, monkeypatch) + with pytest.raises( + coverage.AsecCoverageAuthenticationError, match="SOURCE_REFUSAL" + ): + _read(source, paths) + + +@pytest.mark.parametrize( + "column", + [ + "A_AGE", + "PERIDNUM", + "person_household_id", + "unrelated_raw_observation", + "WSAL_VAL", + ], +) +def test_same_ids_mutated_parent_cannot_reuse_binding(tmp_path, monkeypatch, column): + source, paths = _fixtures(tmp_path, monkeypatch) + result = _read(source, paths) + source.frame.person.loc[:, column] = "9" * 22 if column == "PERIDNUM" else 1 + with pytest.raises( + coverage.AsecCoverageAuthenticationError, match="PARENT_REFUSAL" + ): + coverage.verify_asec_coverage_parent(result, source) + + +def test_distinct_authenticated_parent_with_same_ids_cannot_reuse_artifact( + tmp_path, monkeypatch +): + first = tmp_path / "first" + second = tmp_path / "second" + first.mkdir() + second.mkdir() + source, paths = _fixtures(first, monkeypatch) + result = _read(source, paths) + other, other_paths = _fixtures( + second, + monkeypatch, + person_changes={"unrelated_raw_observation": np.arange(6, dtype=np.int64) + 1}, + ) + assert other.scope.person_ids == source.scope.person_ids + # Restore the producer's fixture pins so this specifically checks parent + # identity rather than the independent source-pin implementation guard. + _pin(paths, monkeypatch) + with pytest.raises( + coverage.AsecCoverageAuthenticationError, match="PARENT_CHANGED" + ): + coverage.verify_asec_coverage_parent(result, other) + assert set(other_paths) == {2022, 2023, 2024} + + +def test_merged_native_households_refuse_even_with_matching_per_row_coordinates( + tmp_path, monkeypatch +): + # The current-money parent already refuses this shape, before coverage + # issuance. Keep that layering and also test the wrapper's own fence. + with pytest.raises(ValueError, match="SOURCE_CONTRACT_REFUSAL"): + _fixtures(tmp_path, monkeypatch, native_ids=[7, 8, 7, 7, 7, 7]) + person = pd.DataFrame( + { + "source_year": [2022, 2022], + "source_household_id": [7, 8], + "person_household_id": [1, 1], + }, + dtype="int64", + ) + with pytest.raises( + coverage.AsecCoverageAuthenticationError, match="HOUSEHOLD_MERGE" + ): + coverage._partitions(person) + + +@pytest.mark.parametrize("parent_age,raw_age", [(80, 80), (82, 80), (0, "")]) +def test_raw_age_identity_never_substitutes_model_age( + tmp_path, monkeypatch, parent_age, raw_age +): + source, paths = _fixtures( + tmp_path, + monkeypatch, + person_changes={ + "A_AGE": np.array([parent_age, 14] * 3, dtype=np.int64), + }, + raw_ages=[raw_age, 14] * 3, + ) + before = money._frame_signature(source.frame) + if parent_age == raw_age: + result = _read(source, paths) + assert result.table().A_AGE.tolist() == [80, 14] * 3 + assert result.receipt["age_relation"]["model_age_used"] is False + else: + with pytest.raises( + coverage.AsecCoverageAuthenticationError, match="SOURCE_REFUSAL" + ): + _read(source, paths) + assert money._frame_signature(source.frame) == before + + +def test_parent_authority_refuses_model_age_before_source_issuance( + tmp_path, monkeypatch +): + # Canonical model age is already forbidden by the raw parent boundary. + # Keep that refusal; do not loosen the parent merely to exercise coverage. + with pytest.raises(ValueError, match="SOURCE_CONTRACT_REFUSAL"): + _fixtures( + tmp_path, + monkeypatch, + person_changes={ + "A_AGE": np.array([80, 14] * 3, dtype=np.int64), + "age": [82.0, 14.0] * 3, + }, + raw_ages=[80, 14] * 3, + ) + + +def test_same_native_person_keys_across_cohorts_remain_distinct(tmp_path, monkeypatch): + keys = [str(i).zfill(22) for i in (1, 2)] * 3 + source, paths = _fixtures(tmp_path, monkeypatch, person_changes={"PERIDNUM": keys}) + result = _read(source, paths) + assert result.table().PERIDNUM.tolist() == keys + assert result.table().PRPERTYP.tolist() == ["1", "2", "3", "", "-4", "NA"] + + +def test_partition_bijection_refuses_splits_and_cross_cohort_merges(): + person = pd.DataFrame( + { + "source_year": [2022, 2022], + "source_household_id": [7, 7], + "person_household_id": [1, 2], + }, + dtype="int64", + ) + with pytest.raises( + coverage.AsecCoverageAuthenticationError, match="HOUSEHOLD_SPLIT" + ): + coverage._partitions(person) + person.source_year = [2022, 2023] + person.person_household_id = [1, 1] + with pytest.raises( + coverage.AsecCoverageAuthenticationError, match="HOUSEHOLD_MERGE" + ): + coverage._partitions(person) + + +@pytest.mark.parametrize( + "kind", + [ + "missing_cohort", + "duplicate_cohort", + "duplicate_row", + "missing_row", + "swapped_cohort", + ], +) +def test_complete_cohort_authority_refuses_incomplete_or_duplicate_inputs( + tmp_path, monkeypatch, kind +): + source, paths = _fixtures(tmp_path, monkeypatch) + if kind == "missing_cohort": + paths.pop(2023) + elif kind == "duplicate_cohort": + pins = coverage._MEMBER_PINS + monkeypatch.setattr(coverage, "_MEMBER_PINS", (pins[0], pins[0], pins[2])) + elif kind == "swapped_cohort": + paths[2022], paths[2023] = paths[2023], paths[2022] + else: + _rewrite( + paths[2022], + lambda rows: ( + rows.__setitem__(1, rows[2].copy()) + if kind == "duplicate_row" + else rows.pop() + ), + ) + _pin(paths, monkeypatch) + with pytest.raises(coverage.AsecCoverageAuthenticationError): + _read(source, paths) + + +def test_candidate_self_rehash_is_not_authority_and_reconstruction_runs_first( + tmp_path, monkeypatch +): + source, paths = _fixtures(tmp_path, monkeypatch) + result = _read(source, paths) + candidate = tmp_path / "candidate.bin" + candidate.write_bytes(result.to_bytes()) + assert ( + _read(source, paths, candidate_path=candidate).to_bytes() == result.to_bytes() + ) + body = result._body.replace(b"observed_code", b"invented_code", 1) + header = json.loads(result._header) + header["body_sha256"] = coverage._sha(body) + header = coverage._json(header) + payload = coverage.MAGIC + struct.pack(" UsPumaLadder: tract_overlap_puma=puma.copy(), tract_overlap_tract=np.asarray([1_001_000_100, 1_003_000_100], dtype=np.int64), tract_overlap_population=population.copy(), + joint_overlap_puma=puma.copy(), + joint_overlap_tract=np.asarray([1_001_000_100, 1_003_000_100], dtype=np.int64), + joint_overlap_cd=np.asarray([101, 102], dtype=np.int64), + joint_overlap_population=np.asarray([40, 60], dtype=np.int64), metadata={ - "schema_version": 1, + "schema_version": 2, "kind": "us_puma_ladder", "puma_vintage": "2020_puma", "sampling_basis": "population", "layers": { - "congressional_district": {"vintage": "119th_congress"}, - "county": {"vintage": "2020_census"}, - "tract": {"vintage": "2020_census"}, + "congressional_district": { + "vintage": "119th_congress", + "source": "invented joint fixture", + }, + "county": { + "vintage": "2020_census", + "source": "invented joint fixture", + }, + "tract": {"vintage": "2020_census", "source": "invented joint fixture"}, }, }, ) diff --git a/packages/microcosm-build/tests/test_us_common_frame_export_contract.py b/packages/microcosm-build/tests/test_us_common_frame_export_contract.py new file mode 100644 index 000000000..349e78527 --- /dev/null +++ b/packages/microcosm-build/tests/test_us_common_frame_export_contract.py @@ -0,0 +1,405 @@ +"""Invented supplied-parent comparisons; no calibration or source issuer imitation.""" + +import json + +import numpy as np +import pandas as pd +import pytest + +from microcosm.build import frame_checkpoint +from microcosm.build.us_runtime import common_frame_export_contract as contract +from microcosm.frame import EntitySchema, Frame, WeightKind, Weights + + +def _parent(): + person = pd.DataFrame( + { + "person_id": np.arange(1, 7, dtype=np.int64), + "person_household_id": np.array([10, 10, 20, 20, 30, 30], dtype=np.int64), + "person_tax_unit_id": np.array( + [100, 100, 200, 201, 300, 300], dtype=np.int64 + ), + "money": np.array([-0.0, 2.0, 3.0, 4.0, 5.0, 6.0]), + "hours": pd.Series([10, pd.NA, 30, 40, pd.NA, 60], dtype="Int32"), + "reported": pd.Series( + [True, pd.NA, False, True, pd.NA, False], dtype="boolean" + ), + "source_person_id": np.arange(101, 107, dtype=np.int64), + } + ) + household = pd.DataFrame( + { + "household_id": np.array([10, 20, 30], dtype=np.int64), + "origin": pd.Series(["asec", "acs", "acs"], dtype="string"), + "assigned_block": pd.Series(["001", "002", "003"], dtype="string"), + "county": pd.Series(["A", "A", "B"], dtype="string"), + "clone": np.array([0, 1, 0], dtype=np.int8), + } + ) + tax = pd.DataFrame( + { + "tax_unit_id": np.array([100, 200, 201, 300], dtype=np.int64), + "deduction": np.array([1.0, 2.0, 3.0, 4.0]), + } + ) + return Frame( + {"person": person, "household": household, "tax_unit": tax}, + EntitySchema(group_entities=("household", "tax_unit")), + {"household": Weights(np.array([1.0, 2.0, 3.0]), WeightKind.IMPORTANCE)}, + pd.Series(["a", "a", "b", "b", "b", "b"], name="stratum"), + metadata={"construction": "one-common-parent"}, + ) + + +def _weighted(parent, person_mask, weights): + selected = parent.select(np.asarray(person_mask, dtype=np.bool_)) + return Frame( + {e: selected.table(e).copy(deep=True) for e in selected.entities}, + selected.schema, + { + "household": Weights( + np.asarray(weights, dtype=np.float64), WeightKind.CALIBRATED + ) + }, + selected.strata.copy(), + metadata=selected.metadata, + ) + + +def _arguments(parent): + return { + "parent_reference": "supplied-parent-example-not-an-issued-graph-key", + "ordered_household_ids": parent.table("household") + .household_id.to_numpy() + .copy(), + "calibrated_weights": np.array([0.0, 9.0, 10.0]), + "calibration_specification": b'{"targets":"invented","scope":"national"}', + } + + +@pytest.mark.parametrize("view", ["full", "pruned", "local"]) +def test_one_parent_full_pruned_and_local_preserve_all_inputs(view): + parent = _parent() + args = _arguments(parent) + if view == "full": + candidate = _weighted(parent, [True] * 6, [0, 9, 10]) + args["prune_zero_weight"] = False + elif view == "pruned": + candidate = _weighted(parent, [False, False, True, True, True, True], [9, 10]) + else: + candidate = _weighted(parent, [False, False, False, False, True, True], [10]) + args["scope_household_ids"] = np.array([30], dtype=np.int64) + binding = contract.verify_retained_frame_export(parent, candidate, **args) + report = json.loads(binding) + assert report["actual_graph_calibration_ancestry_verified"] is False + assert report["release_eligible"] is False + assert report["retained_entity_ids"]["person"]["rows"] == candidate.n("person") + if view != "full": + # A complete survey origin may legitimately have zero calibrated weight. + assert candidate.table("household").origin.tolist() == ["acs"] * candidate.n( + "household" + ) + assert ( + contract.verify_retained_frame_export( + parent, candidate, expected_binding=binding, **args + ) + == binding + ) + + +def test_actual_checkpoint_readback_compares_retained_values_and_masks(tmp_path): + parent = _parent() + candidate = _weighted(parent, [False, False, True, True, True, True], [9, 10]) + args = _arguments(parent) + before = contract.verify_retained_frame_export(parent, candidate, **args) + path = tmp_path / "invented-export.h5" + frame_checkpoint.write_frame_checkpoint( + path, candidate, metadata={"comparison": before.decode()} + ) + loaded = frame_checkpoint.load_frame_checkpoint( + path, frame_metadata=parent.metadata + ) + assert loaded.metadata["comparison"] == before.decode() + assert ( + contract.verify_retained_frame_export( + parent, + loaded.frame, + comparison="frame-checkpoint-readback", + expected_binding=before, + **args, + ) + == before + ) + loaded.frame.person.loc[loaded.frame.person.index[0], "money"] += 1 + with pytest.raises( + contract.RetainedFrameExportError, match="RETAINED_INPUT:person.money" + ): + contract.verify_retained_frame_export( + parent, + loaded.frame, + comparison="frame-checkpoint-readback", + expected_binding=before, + **args, + ) + + +@pytest.mark.parametrize( + "change", + [ + "value", + "mask", + "membership", + "id", + "geography", + "lineage", + "weight", + "strata", + "column", + ], +) +def test_retained_input_changes_refuse(change): + parent = _parent() + candidate = _weighted(parent, [False, False, True, True, True, True], [9, 10]) + if change == "value": + candidate.person.loc[2, "money"] += 1 + elif change == "mask": + candidate.person.loc[4, "hours"] = 0 + elif change == "membership": + candidate.person.loc[2, "person_household_id"] = 30 + elif change == "id": + candidate.person.loc[2, "person_id"] = 99 + elif change == "geography": + candidate.table("household").loc[0, "assigned_block"] = "004" + elif change == "lineage": + candidate.person.loc[2, "source_person_id"] = 999 + elif change == "weight": + candidate = _weighted(parent, [False, False, True, True, True, True], [8, 10]) + elif change == "strata": + candidate.strata.iloc[0] = "changed" + else: + candidate.person.drop(columns="money", inplace=True) + with pytest.raises( + contract.RetainedFrameExportError, match="RETAINED_|COLUMN_ROSTER" + ): + contract.verify_retained_frame_export(parent, candidate, **_arguments(parent)) + + +def test_partial_household_selection_refuses_even_with_valid_frame_links(): + parent = _parent() + candidate = _weighted(parent, [False, False, True, False, True, True], [9, 10]) + with pytest.raises( + contract.RetainedFrameExportError, match="RETAINED_ENTITY_IDS:person" + ): + contract.verify_retained_frame_export(parent, candidate, **_arguments(parent)) + + +@pytest.mark.parametrize( + "change", + ["order", "dtype", "nan", "negative", "shape", "scope_order", "scope_unknown"], +) +def test_weight_and_scope_axes_are_explicit(change): + parent = _parent() + candidate = _weighted(parent, [False, False, True, True, True, True], [9, 10]) + args = _arguments(parent) + if change == "order": + args["ordered_household_ids"] = args["ordered_household_ids"][::-1] + elif change == "dtype": + args["ordered_household_ids"] = args["ordered_household_ids"].astype(float) + elif change == "nan": + args["calibrated_weights"][0] = np.nan + elif change == "negative": + args["calibrated_weights"][0] = -1 + elif change == "shape": + args["calibrated_weights"] = args["calibrated_weights"][:2] + elif change == "scope_order": + args["scope_household_ids"] = np.array([30, 20], dtype=np.int64) + else: + args["scope_household_ids"] = np.array([99], dtype=np.int64) + with pytest.raises(contract.RetainedFrameExportError): + contract.verify_retained_frame_export(parent, candidate, **args) + + +@pytest.mark.parametrize( + "change", ["specification", "parent_reference", "excluded_weight", "scope", "prune"] +) +def test_readback_binding_covers_even_inputs_outside_retained_rows(change): + parent = _parent() + candidate = _weighted(parent, [False, False, True, True, True, True], [9, 10]) + args = _arguments(parent) + args["scope_household_ids"] = np.array([20, 30], dtype=np.int64) + before = contract.verify_retained_frame_export(parent, candidate, **args) + if change == "specification": + args["calibration_specification"] = b'{"targets":"different"}' + elif change == "parent_reference": + args["parent_reference"] = "another-unverified-reference" + elif change == "excluded_weight": + args["calibrated_weights"][0] = 100 + elif change == "scope": + args["scope_household_ids"] = None + else: + args["prune_zero_weight"] = False + with pytest.raises(contract.RetainedFrameExportError, match="EXPORT_BINDING"): + contract.verify_retained_frame_export( + parent, candidate, expected_binding=before, **args + ) + + +@pytest.mark.parametrize("scope", ["empty", "all_zero"]) +def test_empty_analysis_is_an_explicit_unsupported_frame_contract(scope): + parent = _parent() + candidate = _weighted(parent, [True] * 6, [0, 9, 10]) + args = _arguments(parent) + if scope == "empty": + args["scope_household_ids"] = np.array([], dtype=np.int64) + else: + args["calibrated_weights"] = np.zeros(3, dtype=np.float64) + with pytest.raises(contract.RetainedFrameExportError, match="EMPTY_EXPORT_SUPPORT"): + contract.verify_retained_frame_export(parent, candidate, **args) + + +def test_uint64_stable_ids_above_int64_are_not_coerced(): + ids = np.array([2**63 + 1, 2**63 + 7], dtype=np.uint64) + parent = Frame( + { + "person": pd.DataFrame( + { + "person_id": np.array([1, 2], dtype=np.int32), + "person_household_id": ids, + } + ), + "household": pd.DataFrame({"household_id": ids}), + }, + EntitySchema(group_entities=("household",)), + {"household": Weights(np.ones(2), WeightKind.IMPORTANCE)}, + ) + candidate = _weighted(parent, [True, True], [2, 3]) + args = { + "parent_reference": "large-ids", + "ordered_household_ids": ids, + "calibrated_weights": np.array([2.0, 3.0]), + "calibration_specification": b"{}", + } + report = json.loads( + contract.verify_retained_frame_export(parent, candidate, **args) + ) + assert report["complete_ordered_household_ids"]["dtype"] == ids.dtype.str + with pytest.raises(contract.RetainedFrameExportError, match="HOUSEHOLD_ID_VECTOR"): + contract.verify_retained_frame_export( + parent, candidate, **{**args, "ordered_household_ids": ids.astype(float)} + ) + + +def test_readback_normalization_is_closed_and_never_changes_knownness(): + parent = _parent() + candidate = _weighted(parent, [False, False, True, True, True, True], [9, 10]) + args = _arguments(parent) + before = contract.verify_retained_frame_export(parent, candidate, **args) + values = candidate.person["hours"].array + assert values._mask[2] + values._data[2] ^= np.int32(1) + with pytest.raises( + contract.RetainedFrameExportError, match="RETAINED_INPUT:person.hours" + ): + contract.verify_retained_frame_export( + parent, candidate, expected_binding=before, **args + ) + assert ( + contract.verify_retained_frame_export( + parent, + candidate, + comparison="frame-checkpoint-readback", + expected_binding=before, + **args, + ) + == before + ) + with pytest.raises(contract.RetainedFrameExportError, match="COMPARISON_MODE"): + contract.verify_retained_frame_export( + parent, candidate, comparison="approximate", **args + ) + values._mask[2] = False + with pytest.raises( + contract.RetainedFrameExportError, match="RETAINED_INPUT:person.hours" + ): + contract.verify_retained_frame_export( + parent, candidate, comparison="frame-checkpoint-readback", **args + ) + + +@pytest.mark.parametrize("prune_zero_weight", [True, False]) +@pytest.mark.parametrize("scope", ["all", "zero_weight_household"]) +def test_zero_weight_export_support_refuses(prune_zero_weight, scope): + parent = _parent() + # Keep a valid candidate so the export contract itself checks the supplied + # weight/scope inputs before comparing retained rows or weight storage. + candidate = _weighted(parent, [True] * 6, [1, 1, 1]) + args = _arguments(parent) + args["calibrated_weights"] = np.zeros(3, dtype=np.float64) + if scope == "zero_weight_household": + args["calibrated_weights"][1:] = [9, 10] + args["scope_household_ids"] = args["ordered_household_ids"][:1].copy() + with pytest.raises(contract.RetainedFrameExportError, match="EMPTY_EXPORT_SUPPORT"): + contract.verify_retained_frame_export( + parent, candidate, prune_zero_weight=prune_zero_weight, **args + ) + + +@pytest.mark.parametrize("prune_zero_weight", [True, False]) +def test_positive_calibrated_scope_can_have_zero_parent_weight(prune_zero_weight): + baseline = _parent() + parent = Frame( + {e: baseline.table(e).copy(deep=True) for e in baseline.entities}, + baseline.schema, + { + "household": Weights(np.array([0.0, 2.0, 3.0]), WeightKind.IMPORTANCE), + "tax_unit": Weights(np.array([7.0, 8.0, 9.0, 10.0]), WeightKind.DESIGN), + }, + baseline.strata.copy(), + metadata=baseline.metadata, + ) + before_tables = {e: parent.table(e).copy(deep=True) for e in parent.entities} + before_strata = parent.strata.copy(deep=True) + before_metadata, before_mass_log = parent.metadata, parent.mass_log + before_weights = { + e: (parent.weights_for(e).values.copy(), parent.weights_for(e).kind) + for e in parent.weighted_entities + } + retained = baseline.select(np.array([True, True, False, False, False, False])) + + def candidate(tax_weight): + return Frame( + {e: retained.table(e).copy(deep=True) for e in retained.entities}, + retained.schema, + { + "household": Weights(np.array([4.0]), WeightKind.CALIBRATED), + "tax_unit": Weights(np.array([tax_weight]), WeightKind.DESIGN), + }, + retained.strata.copy(), + metadata=retained.metadata, + ) + + args = _arguments(parent) + args["calibrated_weights"] = np.array([4.0, 0.0, 0.0]) + args["scope_household_ids"] = np.array([10], dtype=np.int64) + report = json.loads( + contract.verify_retained_frame_export( + parent, candidate(7.0), prune_zero_weight=prune_zero_weight, **args + ) + ) + assert report["actual_graph_calibration_ancestry_verified"] is False + assert report["release_eligible"] is False + assert report["complete_ordered_household_ids"]["rows"] == 3 + assert report["retained_entity_ids"]["household"]["rows"] == 1 + with pytest.raises( + contract.RetainedFrameExportError, match="RETAINED_WEIGHTS:tax_unit" + ): + contract.verify_retained_frame_export( + parent, candidate(8.0), prune_zero_weight=prune_zero_weight, **args + ) + for entity, table in before_tables.items(): + pd.testing.assert_frame_equal(parent.table(entity), table, check_exact=True) + pd.testing.assert_series_equal(parent.strata, before_strata, check_exact=True) + assert parent.metadata == before_metadata and parent.mass_log == before_mass_log + for entity, (values, kind) in before_weights.items(): + assert parent.weights_for(entity).kind is kind + assert parent.weights_for(entity).values.tobytes() == values.tobytes() diff --git a/packages/microcosm-build/tests/test_us_current_acs_income_anchor_source.py b/packages/microcosm-build/tests/test_us_current_acs_income_anchor_source.py new file mode 100644 index 000000000..00d1c2269 --- /dev/null +++ b/packages/microcosm-build/tests/test_us_current_acs_income_anchor_source.py @@ -0,0 +1,310 @@ +"""Actual retained ACS source owners over privately pinned invented bytes.""" + +import copy +import io + +import numpy as np +import pandas as pd +import pytest + +from microcosm.build.us_runtime import current_acs_income_anchor_source as owner + + +@pytest.mark.parametrize( + "field,token,age,status,known", + [ + ("INTP", "-10000", "30", "observed", True), + ("INTP", "-4", "30", "observed", True), + ("INTP", "0", "30", "observed", True), + ("RETP", "0", "30", "observed", True), + ("RETP", "999999", "30", "observed", True), + ("INTP", "-3", "30", "outside_published_domain", False), + ("RETP", "-4", "30", "outside_published_domain", False), + ("INTP", "1000000", "30", "outside_published_domain", False), + ("INTP", "", "14", "outside_universe_blank", False), + ("INTP", "", "15", "missing_source_amount", False), + ("RETP", "4", "14", "outside_universe_observation", False), + ("INTP", "4.0", "30", "malformed_source_amount", False), + ("INTP", " 4", "30", "malformed_source_amount", False), + ("INTP", "NA", "30", "malformed_source_amount", False), + ], +) +def test_published_anchor_domain_is_distinct_from_storage_and_universe( + field, token, age, status, known +): + value = owner.parse_anchor( + token, age=age, adjustment="1000133", allocation="1", field=field + ) + assert value["status"] == status and value["known"] is known + assert value["allocation_status"] == "allocated" + if known: + expected = np.float64(int(token)) * (np.float64(1000133) / 1_000_000.0) + assert np.float64(value["amount"]).view("uint64") == expected.view("uint64") + else: + assert value["amount"] is None + + +@pytest.mark.parametrize( + "flag,status", + [ + ("0", "not_allocated"), + ("1", "allocated"), + ("", "allocation_missing"), + ("x", "allocation_unrecognized"), + ], +) +def test_allocation_does_not_decide_amount_knownness(flag, status): + value = owner.parse_anchor( + "4", age="15", adjustment="1000133", allocation=flag, field="RETP" + ) + assert value["known"] and value["allocation_status"] == status + + +@pytest.mark.parametrize("adjustment", ["", "0", "-1", "1.1", "inf"]) +def test_unknown_or_invalid_adjustment_never_supplies_an_amount(adjustment): + value = owner.parse_anchor( + "4", age="15", adjustment=adjustment, allocation="0", field="INTP" + ) + assert not value["known"] and value["amount"] is None + assert value["status"] == "invalid_adjustment" + + +def test_column_numeric_identity_keeps_scalar_mapper_bits_and_unknowns(): + tokens = ["-800", "0", "bad", "", "4.0", "-0.0", "NA", "999999", "1000133"] + expected = np.array( + [ + pd.to_numeric(pd.Series([v], dtype=object), errors="coerce").iloc[0] + for v in tokens + ], + dtype=np.float64, + ) + actual = owner._native_numbers(tokens) + assert np.array_equal(np.isnan(actual), np.isnan(expected)) + assert np.array_equal( + actual[~np.isnan(actual)].view("uint64"), + expected[~np.isnan(expected)].view("uint64"), + ) + + +def _literal_rows(): + return [ + { + "SERIALNO": "2024HU0000001", + "SPORDER": "1", + "INTP": "-800", + "RETP": "444", + "ADJINC": "1000133", + "AGEP": "30", + "FINTP": "0", + "FRETP": "1", + }, + { + "SERIALNO": "2024HU0000001", + "SPORDER": "2", + "INTP": "0", + "RETP": "", + "ADJINC": "1000133", + "AGEP": "15", + "FINTP": "1", + "FRETP": "", + }, + { + "SERIALNO": "2024HU0000002", + "SPORDER": "1", + "INTP": "bad", + "RETP": "4", + "ADJINC": "1000133", + "AGEP": "80", + "FINTP": "", + "FRETP": "x", + }, + *( + { + "SERIALNO": serial, + "SPORDER": "1", + "INTP": "0", + "RETP": "0", + "ADJINC": "1000000", + "AGEP": age, + "FINTP": "0", + "FRETP": "0", + } + for serial, age in (("2024GQ0000001", "40"), ("2024GQ0000002", "50")) + ), + ] + + +def _scan(rows, *, wanted=None): + import csv + + text = io.StringIO(newline="") + writer = csv.DictWriter(text, fieldnames=owner.COLUMNS) + writer.writeheader() + writer.writerows(rows) + selected = {} + wanted = ( + wanted + if wanted is not None + else {(r["SERIALNO"], int(r["SPORDER"])) for r in rows} + ) + count = owner._scan( + io.BytesIO(text.getvalue().encode()), wanted, selected, maximum=100 + ) + return count, selected + + +def test_literal_scan_is_keyed_and_keeps_every_original_token(): + count, values = _scan(_literal_rows()[::-1]) + assert count == 5 + assert values["2024HU0000001", 2]["RETP"] == "" + assert values["2024HU0000002", 1]["INTP"] == "bad" + + +@pytest.mark.parametrize("defect", ["duplicate", "foreign_bad_key", "short_row"]) +def test_literal_reader_refuses_duplicate_selected_and_invalid_tail(defect): + rows = _literal_rows() + if defect == "duplicate": + rows.append(rows[0]) + elif defect == "foreign_bad_key": + rows.append({**rows[0], "SERIALNO": "bad"}) + else: + stream = io.BytesIO((",".join(owner.COLUMNS) + "\n1,2\n").encode()) + with pytest.raises(ValueError, match="ACS_INCOME_ANCHOR_"): + owner._scan(stream, set(), {}, maximum=100) + return + with pytest.raises(ValueError, match="ACS_INCOME_ANCHOR_"): + _scan(rows) + + +def _arguments(tmp_path, monkeypatch): + import test_us_survey_population_preparation as fixture + + original = fixture._person + lookup = {(r["SERIALNO"], int(r["SPORDER"])): r for r in _literal_rows()} + + def person(*args, **kwargs): + row = original(*args, **kwargs) + key = (row["SERIALNO"], int(row["SPORDER"])) + row.update( + { + k: v + for k, v in lookup.get(key, {}).items() + if k not in ("SERIALNO", "SPORDER") + } + ) + row.setdefault("FINTP", "0") + row.setdefault("FRETP", "0") + return row + + monkeypatch.setattr(fixture, "_person", person) + return fixture.fixture(tmp_path, monkeypatch, zero=False) + + +def test_actual_qualifier_binds_originals_and_adjusted_bits(tmp_path, monkeypatch): + prepared = owner.preparation.prepare_authenticated_survey_population( + **_arguments(tmp_path, monkeypatch) + ) + original = prepared.checked_view().frame + before = owner.preparation._frame_identity(original) + qualified = owner.qualify_current_acs_income_anchors(prepared) + assert qualified.evidence["source_admission_issued"] is False + assert qualified.evidence["decomposition_performed"] is False + assert len(qualified.anchors) == 5 + first = qualified.anchors.set_index(["SERIALNO", "SPORDER"]) + assert first.loc[("2024HU0000001", "1"), "property_income_amount"] < 0 + assert first.loc[("2024HU0000001", "2"), "property_income_amount"] == 0 + assert ( + first.loc[("2024HU0000001", "2"), "retirement_income_status"] + == "missing_source_amount" + ) + assert ( + first.loc[("2024HU0000002", "1"), "property_income_status"] + == "malformed_source_amount" + ) + assert pd.isna(first.loc[("2024HU0000002", "1"), "property_income_amount"]) + assert owner.preparation._frame_identity(original) == before + fresh = owner.qualify_current_acs_income_anchors(prepared) + assert owner.income_anchor_seal(fresh) == owner.income_anchor_seal(qualified) + with pytest.raises(ValueError): + owner.qualify_current_acs_income_anchors(copy.copy(prepared)) + + +@pytest.mark.parametrize( + "defect", ["row_order", "person_id", "raw_amount", "adjusted_bit"] +) +def test_exact_retained_comparison_refuses_source_or_amount_mutation( + tmp_path, monkeypatch, defect +): + prepared = owner.preparation.prepare_authenticated_survey_population( + **_arguments(tmp_path, monkeypatch) + ) + entry = prepared._checked() + document = owner.json.loads(entry[1]) + origins = owner._origins(entry[2], document) + _, selected = _scan(_literal_rows()) + raw = owner._raw_table(origins, selected) + native = owner.preparation._copy_source(entry[2].source_frames[0]) + if defect == "row_order": + origins = origins.iloc[::-1].copy() + elif defect == "person_id": + native.person.loc[native.person.index[0], "person_id"] += 100 + elif defect == "raw_amount": + native.person.loc[native.person.index[0], "RETP"] = 888 + else: + column = "acs_interest_dividend_rental_income" + native.person.loc[native.person.index[0], column] = np.nextafter( + native.person[column].iloc[0], np.inf + ) + with pytest.raises(ValueError, match="ACS_INCOME_ANCHOR_"): + owner._compare_retained(raw, origins, native) + + +def test_final_source_io_mutation_cannot_return_qualified_value(tmp_path, monkeypatch): + prepared = owner.preparation.prepare_authenticated_survey_population( + **_arguments(tmp_path, monkeypatch) + ) + original = prepared.checked_view().frame + capture = owner._capture_person + mutated = [] + + def mutate_after_io(*args): + value = capture(*args) + original.person.loc[original.person.index[0], "age"] += 1 + mutated.append(True) + return value + + monkeypatch.setattr(owner, "_capture_person", mutate_after_io) + with pytest.raises(ValueError): + owner.qualify_current_acs_income_anchors(prepared) + assert mutated == [True] + + +def test_final_source_fence_seals_exact_returned_amount_bits(tmp_path, monkeypatch): + prepared = owner.preparation.prepare_authenticated_survey_population( + **_arguments(tmp_path, monkeypatch) + ) + original_seal = owner.income_anchor_seal + changed = [] + + def change_after_seal(qualified): + seal = original_seal(qualified) + if not changed: + changed.append(True) + table = qualified.anchors + index = table.index[ + table.SERIALNO.eq("2024HU0000001") & table.SPORDER.eq("1") + ][0] + table.loc[index, "property_income_amount"] = np.nextafter( + table.loc[index, "property_income_amount"], np.inf + ) + # Portable JSON's precision is not an exact physical mutation seal. + assert ( + qualified.projection + == table.reset_index().to_json(orient="table", index=False).encode() + ) + return seal + + monkeypatch.setattr(owner, "income_anchor_seal", change_after_seal) + with pytest.raises(ValueError, match="FINAL_VALUES_CHANGED"): + owner.qualify_current_acs_income_anchors(prepared) + assert changed == [True] diff --git a/packages/microcosm-build/tests/test_us_current_asec_demographics.py b/packages/microcosm-build/tests/test_us_current_asec_demographics.py new file mode 100644 index 000000000..c61c62b15 --- /dev/null +++ b/packages/microcosm-build/tests/test_us_current_asec_demographics.py @@ -0,0 +1,191 @@ +"""Actual source readers over constructed, privately pinned invented source bytes.""" + +import csv +import hashlib +import json +import shutil +from dataclasses import replace +from types import SimpleNamespace + +import pandas as pd +import pytest +from test_us_survey_population_preparation import fixture + +from microcosm.build.us_runtime import asec_coverage_authentication as coverage +from microcosm.build.us_runtime import asec_person_income_source as restoration +from microcosm.build.us_runtime import current_asec_demographics as owner +from microcosm.build.us_runtime import native_household_origin as origin + + +def _csv(path, headers, rows): + with path.open("w", newline="") as stream: + writer = csv.writer(stream) + writer.writerow(headers) + writer.writerows(rows) + data = path.read_bytes() + return SimpleNamespace( + size_bytes=len(data), + rows=len(rows), + member_sha256=hashlib.sha256(data).hexdigest(), + ) + + +@pytest.mark.parametrize( + "literal,code,status", + ( + ("06", 6, "in_printed_range"), + ("36", 36, "in_printed_range"), + ("", None, "missing"), + ("NA", None, "malformed"), + (" 6", None, "malformed"), + ("6.0", None, "malformed"), + ("57", None, "outside_printed_range"), + ("00", None, "outside_printed_range"), + ), +) +def test_household_state_reader_preserves_literal_and_knownness( + tmp_path, literal, code, status +): + path = tmp_path / "invented.csv" + pin = _csv(path, ("H_SEQ", "GESTFIPS", "unread"), [("00007", literal, "anything")]) + value = owner._read_state_capture(path, pin)[7] + assert value == { + "H_SEQ": "00007", + "GESTFIPS": literal, + "state_code": code, + "status": status, + "member_row_1based": 1, + } + + +@pytest.mark.parametrize("defect", ("duplicate", "digest", "size", "rows", "header")) +def test_state_capture_structure_and_exact_bytes_refuse(tmp_path, defect): + path = tmp_path / "invented.csv" + headers, rows = ("H_SEQ", "GESTFIPS"), [("00007", "06")] + if defect == "duplicate": + rows.append(("7", "36")) + if defect == "header": + headers = ("H_SEQ", "other") + pin = _csv(path, headers, rows) + if defect == "digest": + pin.member_sha256 = "f" * 64 + elif defect == "size": + pin.size_bytes += 1 + elif defect == "rows": + pin.rows += 1 + with pytest.raises(ValueError, match="CURRENT_ASEC_DEMOGRAPHICS_"): + owner._read_state_capture(path, pin) + + +def _demographic_arguments(tmp_path, monkeypatch, *, unknown=False, zero=True): + """Extend invented source members before actual preparation/issuance. + + Rebuild the real person-income attachment after member bytes change. Only + fixture registry pins are changed; no source issuer or ready() is replaced. + """ + arguments = fixture(tmp_path, monkeypatch, zero=zero) + source = arguments["source_dir"] / "asec" + members, pins = {}, [] + for year, member, archive, *_ in coverage._MEMBER_PINS: + path = source / f"pppub{year - 1999}.csv" + table = pd.read_csv(path, dtype=str, keep_default_na=False) + lines = table.A_LINENO.map(int) + table["A_SEX"] = lines.map({1: "1", 2: "2"}) + table["AXSEX"] = lines.map({1: "0", 2: "4"}) + table["A_EXPRRP"] = lines.map({1: "1", 2: "5"}) + table["P_SEQ"] = table.A_LINENO + if unknown and year == 2024: + table.loc[table.index[0], "AXSEX"] = "1" + assert table.notna().all().all() + # Reverse the actual source order, so receiving-row alignment cannot pass. + table.iloc[::-1].to_csv(path, index=False) + data = path.read_bytes() + pins.append( + ( + year, + member, + archive, + hashlib.sha256(data).hexdigest(), + len(table), + len(data), + ) + ) + members[year] = path + for module in (coverage, restoration, owner.demographic): + monkeypatch.setattr(module, "_MEMBER_PINS", tuple(pins)) + output = tmp_path / "demographic-restored-money" + restoration.restore_asec_person_income_source( + source / "parent.h5", + source / "household-attachment.h5", + member_paths=members, + output_dir=output, + ) + shutil.copyfile( + output / restoration.CHECKPOINT_FILENAME, source / "person-income-attachment.h5" + ) + path = source / "hhpub25.csv" + table = pd.read_csv(path, dtype=str, keep_default_na=False) + table["GESTFIPS"] = table.H_SEQ.map({"00007": "06", "00008": "36"}) + if unknown: + table.loc[table.H_SEQ.eq("00008"), "GESTFIPS"] = "" + assert table.notna().all().all() + table.iloc[::-1].to_csv(path, index=False) + data = path.read_bytes() + monkeypatch.setattr( + origin, + "_ASEC_MEMBER_PINS", + tuple( + replace( + pin, + member_sha256=hashlib.sha256(data).hexdigest(), + size_bytes=len(data), + ) + for pin in origin._ASEC_MEMBER_PINS + ), + ) + return arguments + + +@pytest.mark.parametrize("unknown", (False, True)) +def test_actual_current_native_parent_binds_demographics_by_origin( + tmp_path, monkeypatch, unknown +): + arguments = _demographic_arguments(tmp_path, monkeypatch, unknown=unknown) + prepared = owner.preparation_owner.prepare_authenticated_survey_population( + **arguments + ) + original = prepared.checked_view().frame + retained = {e: original.table(e).copy(deep=True) for e in original.entities} + result = owner.qualify_current_asec_demographics(prepared) + receipt = json.loads(result.receipt) + assert receipt["sex_observation_year"] == receipt["state_observation_year"] == 2025 + assert receipt["income_year"] == 2024 + assert not receipt["state_is_income_year_residence_claim"] + assert not receipt["source_admission_issued"] and not receipt["release_eligible"] + h = result.household.set_index("H_SEQ_integer") + assert h.loc[7, "GESTFIPS"] == "06" and h.loc[7, "state_fips"] == 6 + if unknown: + assert h.loc[8, "GESTFIPS"] == "" and pd.isna(h.loc[8, "state_fips"]) + assert receipt["state_unknown_households"] == 1 + assert receipt["sex_unknown_persons"] == 1 + unbound = result.person.asec_AXSEX.eq(1) + assert result.person.loc[unbound, "is_female"].isna().all() + assert not result.person.loc[unbound, "sex_known"].any() + else: + assert h.loc[8, "state_fips"] == 36 + assert ( + receipt["state_unknown_households"] == receipt["sex_unknown_persons"] == 0 + ) + allocated = result.person.asec_AXSEX.eq(4) + assert allocated.any() and result.person.loc[allocated, "is_female"].all() + assert result.person.loc[allocated, "sex_origin"].eq("census_allocated").all() + for entity, expected in retained.items(): + pd.testing.assert_frame_equal( + expected, original.table(entity), check_exact=True + ) + assert owner.qualify_current_asec_demographics(prepared).receipt == result.receipt + # Value transport is deliberately not an issued authority. Mutating it must + # not affect a fresh live projection or its source-bound receipt. + result.household.loc[result.household.index[0], "state_fips"] = 99 + fresh = owner.qualify_current_asec_demographics(prepared) + assert not fresh.household.state_fips.eq(99).any() diff --git a/packages/microcosm-build/tests/test_us_current_survey_amounts.py b/packages/microcosm-build/tests/test_us_current_survey_amounts.py new file mode 100644 index 000000000..89f92aecb --- /dev/null +++ b/packages/microcosm-build/tests/test_us_current_survey_amounts.py @@ -0,0 +1,316 @@ +"""Invented reporting bases and clone joins; no native data or country engine.""" + +import numpy as np +import pandas as pd +import pytest + +from microcosm.build.us_runtime import current_asec_unemployment_source as uc +from microcosm.build.us_runtime import current_survey_amounts as values +from microcosm.fit import model_input +from microcosm.frame import US_SCHEMA, Frame, WeightKind, Weights + + +@pytest.mark.parametrize( + "amount,age,token,label,canonical", + [ + (100, 40, "1", "known_receipt", 100), + (0, 40, "2", "known_nonreceipt", 0), + (0, 40, "1", "ambiguous_recipient_zero", None), + (0, 40, "0", "niu", None), + (0, 14, "0", "outside_reporting_universe", None), + (0, 14, "2", "contradictory_outside_reporting_universe", None), + (100, 14, "1", "contradictory_outside_reporting_universe", None), + (100, 40, "2", "contradictory_no_positive", None), + (100, 40, "0", "contradictory_niu_positive", None), + (0, 40, "", "missing_receipt_literal", None), + (100, 40, "9", "unrecognized_receipt_literal", None), + (100, 40, "yes", "unrecognized_receipt_literal", None), + (100, 40, "01", "unrecognized_receipt_literal", None), + (np.nan, 40, "1", "missing_amount", None), + ], +) +def test_uc_raw_amount_and_reporting_knownness_are_separate( + amount, age, token, label, canonical +): + basis = uc.reporting_basis( + np.array([amount], dtype="float64"), np.array([age], dtype="float64"), [token] + ) + row = basis.iloc[0] + assert row.reporting_status == label + assert row.receipt_literal == token + assert bool(row.source_reporting_universe) == (age >= 15) + assert bool(row.canonical_amount_known) == (canonical is not None) + if canonical is None: + assert np.isnan(row.canonical_amount) + else: + assert row.canonical_amount == canonical + assert ( + np.isnan(row.source_amount) if np.isnan(amount) else row.source_amount == amount + ) + + +@pytest.mark.parametrize( + "amount,age", [(-1, 40), (100000, 40), (np.inf, 40), (0, -1), (0, 40.5)] +) +def test_uc_refuses_out_of_dictionary_numeric_domain(amount, age): + with pytest.raises(ValueError): + uc.reporting_basis( + np.array([amount], dtype="float64"), np.array([age], dtype="float64"), ["2"] + ) + + +def test_uc_bounded_literal_reader_and_duplicate_join_refusal(tmp_path): + path = tmp_path / "invented.csv" + header = ",".join(uc.READ_COLUMNS) + "\n" + row = "0000000000000000000001,1,1,40,0,2\n" + path.write_text(header + row) + result = uc._read_capture(path, rows=1) + assert result.iloc[0].UC_YN == "2" + path.write_text(header + row + row) + with pytest.raises(ValueError, match="DUPLICATE"): + uc._read_capture(path, rows=2) + path.write_text(header + row.replace(",0,2", ",0," + "9" * 65)) + with pytest.raises(ValueError, match="TOKEN_BOUND"): + uc._read_capture(path, rows=1) + + +def test_money_literal_join_retains_selected_and_unselected_missingness(): + field = uc.money.MoneyField( + "UC_VAL", + np.array([0.0, 0.0, 40.0], dtype=" 0 + assert type(receipt) is MappingProxyType + with pytest.raises(TypeError): + receipt["minimum_age"] = 0 + assert receipt["minimum_age"] == 15 + probe.person.loc[acs_index, "age"] = 15 + probe.person.loc[acs_index, "AGEP"] = 15 + with pytest.raises(ValueError, match="ACS_ELIGIBLE_EARNINGS_UNKNOWN"): + values._acs_earnings(probe) + matrix = graph.model_input.decode_recipient_matrix(qualified.matrix) + invented_draw = pd.DataFrame( + { + target: np.arange(1.0, 1.0 + len(matrix.entity_ids)) * (i + 1.0) + for i, target in enumerate(values.TARGETS) + }, + index=matrix.features.index, + ) + original_join = values.complete_predictor_columns( + qualified, clone.frame, invented_draw + ) + reversed_join = values.complete_predictor_columns( + replace( + qualified, + native_money=qualified.native_money.iloc[::-1], + origins=qualified.origins.iloc[::-1], + ), + clone.frame, + invented_draw, + ) + for name in original_join: + pd.testing.assert_series_equal(original_join[name], reversed_join[name]) + completed_interest = qualified.native_money["INT_VAL"].fillna( + invented_draw[values.TARGETS[0]] + ) + expected_interest = completed_interest.reindex( + clone.frame.person[values.provenance.support_source_id_column("person")] + ).to_numpy() + np.testing.assert_allclose( + original_join["person", "taxable_interest_income"] + + original_join["person", "tax_exempt_interest_income"], + expected_interest, + rtol=1e-15, + atol=0, + ) + assert allocated.frame.resolve_weights("person").kind is WeightKind.IMPORTANCE + if nonconstant: + origins = qualified.origins + ids = origins.index[origins.source.eq("asec")] + # Join the current-money projection by native identity, never row position. + ordered = origins.loc[ids].sort_values("native_person_id").index + assert origins.loc[ordered, "native_person_id"].tolist() == [105, 106, 107, 108] + for field, expected in observations.items(): + np.testing.assert_array_equal( + qualified.native_money.loc[ordered, field], expected + ) + for target in values.TARGETS: + assert qualified.donor_columns[target].nunique() == 3 + weights = qualified.donor_frame.resolve_weights("person") + assert weights.kind is WeightKind.DESIGN and (weights.values > 0).all() + np.testing.assert_array_equal(weights.values, [2552.12, 2552.12, 100.0, 100.0]) + else: + assert qualified.donor_columns.loc[:, list(values.TARGETS)].eq(0).all().all() + pins = {} + for edge in graph.host.current_survey_host_edges(): + receipt = live.manifest.node(edge.producer) + key = receipt.opaque_artifacts[edge.artifact] + pins[edge.name] = { + "producer_key": receipt.key, + "artifact_key": key, + "payload_sha256": graph.codec.sha(live.store.load_bytes(key)), + } + nodes = graph.current_survey_predictor_nodes( + qualified, clone.frame, host_pins=pins, n_estimators=2 + ) + assert nodes[1].base == survey.CREATE_NODE + assert nodes[0].population == graph.DONOR_NODE + assert tuple(edge.name for edge in nodes[1].artifact_inputs) == ("preparation",) + assert all("state" not in c and "female" not in c for c in values.FEATURES) + compiled = compile_graph( + replace(live.compiled.graph, nodes=(*live.compiled.graph.nodes, *nodes)) + ) + # Actual compiler dependencies retain allocation evidence without an + # ordinary node in CREATE depending on its own downstream allocation. + assert graph.PROJECTION_NODE not in compiled.predecessors[survey.ALLOCATION_NODE] + assert graph.DONOR_NODE in compiled.predecessors[graph.PROJECTION_NODE] + assert survey.ALLOCATION_NODE in compiled.predecessors[graph.PROJECTION_NODE] + assert graph.PROJECTION_NODE not in compiled.predecessors[graph.DONOR_NODE] + for cls in ( + graph.CurrentSurveyPredictorProjectionKernel, + graph.CurrentSurveyPredictorDonorFilterKernel, + graph.CurrentSurveyPredictorDonorColumnsKernel, + graph.CurrentSurveyPredictorAttachKernel, + ): + live.kernels.register( + cls(preparation, allocated, clone, host_pins=pins, n_estimators=2) + ) + live.kernels.register(LegacyQRFTrainKernel()) + live.kernels.register(LegacyQRFApplyMatrixKernel()) + artifacts = [ + ( + graph.PROJECTION_NODE, + "projection", + graph.PROJECTION_TYPE, + graph.CurrentSurveyPredictorProjectionKernel, + ), + ( + graph.PROJECTION_NODE, + "matrix", + graph.model_input.RECIPIENT_MATRIX_TYPE, + graph.CurrentSurveyPredictorProjectionKernel, + ), + ] + for i in range(3): + artifacts.extend( + [ + ( + f"{graph.FIT_PREFIX}.{i:03d}", + "model", + qrf_target.LEGACY_QRF_TARGET_TYPE, + LegacyQRFTrainKernel, + ), + ( + f"{graph.APPLY_PREFIX}.{i:03d}", + "raw_draw", + graph.codec.RAW_TARGET_TYPE, + LegacyQRFApplyMatrixKernel, + ), + ( + f"{graph.APPLY_PREFIX}.{i:03d}", + "apply_state", + graph.MATRIX_APPLY_STATE_TYPE, + LegacyQRFApplyMatrixKernel, + ), + ] + ) + results = [] + for resume in ("auto", "require"): + observed = {} + manifest = run_graph( + compiled, + store=live.store, + kernels=live.kernels, + sources=live.sources, + resume=resume, + _population_observer=lambda name, population, observed=observed: ( + observed.__setitem__(name, population) + ), + ) + donor_weights = observed[graph.DONOR_NODE].frame.resolve_weights("person") + assert donor_weights.kind is WeightKind.DESIGN + np.testing.assert_array_equal( + donor_weights.values, + qualified.donor_frame.resolve_weights("person").values, + ) + loaded = {} + for node_id, name, type_, cls in artifacts: + receipt = manifest.node(node_id) + key = opaque_artifact_key(receipt.key, name) + assert receipt.opaque_artifacts[name] == key + payload = live.store.load_bytes(key) + survey._final_artifact( + manifest, + live.store, + node_id=node_id, + name=name, + type_=type_, + payload=payload, + capabilities=cls.capabilities, + ) + loaded[node_id, name] = payload + for i, target in enumerate(values.TARGETS): + fit_id = f"{graph.FIT_PREFIX}.{i:03d}" + payload = loaded[fit_id, "model"] + # Actual typed producer/store verification above precedes pickle loading. + artifact = qrf_target.LegacyQRFTargetArtifact.from_trusted_bytes( + payload, expected_sha256=graph.codec.sha(payload) + ) + assert artifact.target == target + assert artifact._target_model.columns == ( + *values.FEATURES, + *values.TARGETS[:i], + ) + expected_regime = ( + qrf.Regime.ZERO_INFLATED_POSITIVE + if nonconstant + else qrf.Regime.DEGENERATE_ZERO + ) + assert artifact.regime == expected_regime + if nonconstant: + assert artifact._target_model.gate is not None + assert artifact._target_model.positive is not None + if resume == "require": + assert all(record.hit for record in manifest.nodes.values()) + raw = tuple( + loaded[f"{graph.APPLY_PREFIX}.{i:03d}", "raw_draw"] for i in range(3) + ) + states = tuple( + loaded[f"{graph.APPLY_PREFIX}.{i:03d}", "apply_state"] for i in range(3) + ) + matrix_key = manifest.node(graph.PROJECTION_NODE).key + actual = observed[graph.ATTACH_NODE] + verification = dict( + population=actual, + projection=loaded[graph.PROJECTION_NODE, "projection"], + matrix=loaded[graph.PROJECTION_NODE, "matrix"], + matrix_producer_key=matrix_key, + raw_draws=raw, + apply_states=states, + host_pins=pins, + n_estimators=2, + ) + result = graph.verify_materialized_current_survey_predictors( + preparation, allocated, clone, **verification + ) + assert ( + result["all_output_cells_available"] is True + and result["release_eligible"] is False + ) + assert not actual.frame.person.loc[:, list(values.OUTPUTS)].isna().any().any() + actual_person = actual.frame.person + for _, rows in actual_person.groupby( + values.provenance.support_source_id_column("person") + ): + assert len(rows) == 2 + np.testing.assert_array_equal( + rows.loc[:, list(values.OUTPUTS)].iloc[0], + rows.loc[:, list(values.OUTPUTS)].iloc[1], + ) + for raw_name in ("WAGP", "SEMP", "ADJINC"): + pd.testing.assert_series_equal( + actual_person[raw_name], clone.frame.person[raw_name] + ) + puf_mask = support.puf_tax_detail_clone_mask( + actual.frame.table("tax_unit"), entity="tax_unit" + ) + features, _ = support._strict_recipient_predictor_surface( + actual.frame, + puf_mask, + support.PUF_TAX_DETAIL_DEFAULT_PREDICTORS, + person_outputs=(), + ) + assert features.shape[1] == 8 and np.isfinite(features.to_numpy()).all() + changed = _copy_population(actual) + changed.frame.person.loc[changed.frame.person.index[0], values.OUTPUTS[2]] += ( + 1.0 + ) + with pytest.raises(ValueError): + graph.verify_materialized_current_survey_predictors( + preparation, allocated, clone, **{**verification, "population": changed} + ) + with pytest.raises(ValueError, match="DRAW_MATRIX_BINDING"): + graph.read_current_survey_draws(qualified.matrix, "f" * 64, raw, states) + results.append(actual) + graph.replay.same_replayed_population(*results) diff --git a/packages/microcosm-build/tests/test_us_current_survey_puf_host.py b/packages/microcosm-build/tests/test_us_current_survey_puf_host.py new file mode 100644 index 000000000..50454a91d --- /dev/null +++ b/packages/microcosm-build/tests/test_us_current_survey_puf_host.py @@ -0,0 +1,444 @@ +"""Invented host-only bridge; ordinary pytest, no donor/fit/engine/target access. + +The actual source test deliberately retains normal readiness and inventory gates. +Its proposal has not been executed. Fixture private file pins follow the existing +invented source helpers; no source/issuer function or money.ready is replaced. +""" + +import copy +from dataclasses import replace + +import numpy as np +import pandas as pd +import pytest +from test_us_graph_survey_population import authenticated_arguments +from test_us_puf_detail_transfer import host as legacy_host +from test_us_survey_population_preparation import fixture as source_fixture + +from microcosm.build.us_runtime import asec_current_money as money +from microcosm.build.us_runtime import graph_combined_clone as clone +from microcosm.build.us_runtime import graph_puf_diagnostic_consumer as bridge +from microcosm.build.us_runtime import graph_survey_population as survey +from microcosm.build.us_runtime import puf_detail_transfer as detail +from microcosm.build.us_runtime import puf_diagnostic_consumer as consumer +from microcosm.build.us_runtime import support_provenance as provenance +from microcosm.build.us_runtime import survey_population_preparation as owner +from microcosm.build.us_runtime import survey_population_replay as replay +from microcosm.build.us_runtime.graph_sources import frame_column_declarations +from microcosm.fit import _graph_legacy_qrf as codec +from microcosm.fit.model_input import decode_recipient_matrix +from microcosm.frame import Frame, Weights +from microcosm.graph import NodeRejected, compile_graph, executor, run_graph +from microcosm.graph.keys import opaque_artifact_key + + +def _copy_frame(frame): + return Frame( + {e: frame.table(e).copy(deep=True) for e in frame.entities}, + frame.schema, + {e: frame.weights_for(e) for e in frame.weighted_entities}, + frame.strata.copy(deep=True), + metadata=frame.metadata, + mass_log=frame.mass_log, + ) + + +def _legacy_clone(): + return clone.clone_us_frame_for_puf_support(legacy_host()) + + +def test_existing_matrix_default_and_explicit_projection_have_identical_bytes(): + frame = _legacy_clone() + old, old_mask = detail.recipient_matrix(frame) + people = frame.person + values = [ + row["asec_reported_wage_income_2024_price"] + if row[provenance.support_channel_column("person")] == "asec" + else row["employment_income_before_lsr"] + for _, row in people.iterrows() + ] + wages = pd.Series( + values, index=pd.Index(people.person_id.to_numpy()), dtype="float64" + ) + new, new_mask = detail.recipient_matrix(frame, person_wages=wages) + assert new == old + np.testing.assert_array_equal(new_mask, old_mask) + # Independent hand expectations: actual JOINT spouse included, dependent + # wages 999/800 excluded, zero-weight single unit retained. + assert sorted( + decode_recipient_matrix(new).features.reported_wage_proxy.tolist() + ) == [10.0, 20.0, 30.0, 170.0] + with pytest.raises(ValueError, match="HOST_WAGE_ROW_AXIS"): + detail.recipient_matrix(frame, person_wages=wages.iloc[::-1]) + wrong = wages.copy() + role = people[provenance.support_clone_index_column("person")].to_numpy() + wrong.iloc[np.flatnonzero(role == 1)[0]] += 1 + with pytest.raises(ValueError, match="HOST_WAGE_PAIR_BYTES"): + detail.recipient_matrix(frame, person_wages=wrong) + missing = wages.copy() + first = people.iloc[0][provenance.support_source_id_column("person")] + pair = people[provenance.support_source_id_column("person")].eq(first).to_numpy() + missing.iloc[np.flatnonzero(pair)] = np.nan + with pytest.raises(ValueError, match="HOST_MISSING_FEATURE"): + detail.recipient_matrix(frame, person_wages=missing) + with pytest.raises(ValueError, match="HOST_WAGE_ROW_AXIS"): + detail.recipient_matrix(frame, person_wages=wages.astype("object")) + + +def test_current_route_is_explicit_and_legacy_declarations_are_separate(): + columns = frame_column_declarations(_legacy_clone()) + current = bridge.current_survey_host_nodes( + columns, + preparation_sha256="1" * 64, + allocation_sha256="2" * 64, + clone_sha256="3" * 64, + host_pins={ + e.name: { + "producer_key": "4" * 64, + "artifact_key": "5" * 64, + "payload_sha256": "6" * 64, + } + for e in bridge.current_survey_host_edges() + }, + ) + assert len(current) == 2 + assert all(not n.outputs and not n.sources for n in current) + assert [e.name for e in current[0].artifact_inputs] == [ + "preparation", + "allocation", + "frame_context", + ] + assert [e.name for e in current[1].artifact_inputs] == ["projection"] + assert current[0].params["host_route"] == consumer.CURRENT_SURVEY_ROUTE + assert len(bridge.host_edges(native_binding="native", clone_binding="clones")) == 11 + assert not any( + word in n.kernel + for n in current + for word in ("donor", "fit", "attach", "apply") + ) + + +def test_unissued_byte_equal_preparation_refuses_before_projection(): + fake = object.__new__(owner.AuthenticatedSurveyPopulationPreparation) + object.__setattr__(fake, "payload", b"{}") + with pytest.raises( + owner.SurveyPopulationPreparationError, match="UNISSUED_OR_CHANGED" + ): + bridge.qualify_current_survey_host(fake, None, None) + + +@pytest.mark.parametrize("year,field", [(2024, "ANN_VAL"), (2022, "WSAL_VAL")]) +def test_current_host_preserves_full_parent_readiness( + tmp_path, monkeypatch, year, field +): + arguments = source_fixture(tmp_path, monkeypatch, missing_asec_money=(year, field)) + live = survey.run_authenticated_survey_population( + **arguments, + store_root=tmp_path / "graph-store", + clones=True, + return_values=True, + ) + # Reaching this point establishes actual preparation and native issuance; + # an earlier SOURCE_CHANGED error is not the readiness result under test. + preparation_entry = owner._ISSUED[id(live.preparation)] + native = preparation_entry[2].native[1] + parent = owner.asec_native._ISSUED[id(native)][2].parent + decoded = money.classify_asec_money(parent.views(), parent.spec, parent.scope) + years = np.asarray(parent.scope.person_years) + absent = ~decoded.field(field).validity.astype(bool) + assert int(absent.sum()) == 1 + assert years[absent].tolist() == [year] + assert decoded.field("WSAL_VAL").validity[years == 2024].all() + with pytest.raises(money.MoneyRefusalError) as caught: + bridge.qualify_current_survey_host( + live.preparation, live.allocated_population, live.clone_population + ) + assert caught.value.reason == "MISSING_REQUIRED_AMOUNT" + assert caught.value.field == field + + +def test_actual_current_sources_cold_and_replayed_host(tmp_path, monkeypatch): + arguments = authenticated_arguments(tmp_path, monkeypatch) + live = survey.run_authenticated_survey_population( + **arguments, clones=True, return_values=True + ) + preparation, allocated, expanded = ( + live.preparation, + live.allocated_population, + live.clone_population, + ) + before = owner._frame_identity(expanded.frame) + projection, matrix, mask, evidence = bridge.qualify_current_survey_host( + preparation, allocated, expanded + ) + document = codec.decode_json(projection) + assert evidence["release_eligible"] is False + assert evidence["source_period_equivalence_claim"] is False + assert len(document["asec"]["rows"]) == 4 + assert [r[2] for r in document["asec"]["rows"]] == [2024] * 4 + assert [ + np.frombuffer(bytes.fromhex(r[3]), dtype=" 1 + for changed_slice in ( + replace(first, columns=first.columns[:-1]), + replace(first, columns=first.columns[::-1]), + replace(first, columns=(*first.columns, "__extra_input__")), + # A declared column lets Node construction succeed; this direct + # contract check must reject non-ALL rows before any mask evaluation. + replace(first, rows=first.columns[0]), + ): + bad_node = replace(nodes[0], inputs=(changed_slice, *nodes[0].inputs[1:])) + with pytest.raises(ValueError, match="^SURVEY_HOST_INPUTS$"): + bridge._current_context_frame( + replace(context, node=bad_node), expanded.frame + ) + for field, values, reason in ( + ("tables", context.tables, "SURVEY_HOST_TABLES"), + ("weights", context.weights, "SURVEY_HOST_WEIGHTS"), + ): + missing = dict(values) + del missing[next(iter(missing))] + extra = {**values, "__extra_entity__": next(iter(values.values()))} + for changed_values in (missing, extra): + with pytest.raises(ValueError, match="^" + reason + "$"): + bridge._current_context_frame( + replace(context, **{field: changed_values}), expanded.frame + ) + + person = context.tables[expanded.frame.schema.person_entity] + changed_tables = dict(context.tables) + changed_tables[expanded.frame.schema.person_entity] = person.iloc[:, ::-1] + with pytest.raises(ValueError, match="^SURVEY_POPULATION_REPLAY_AXIS$"): + bridge._current_context_frame( + replace(context, tables=changed_tables), expanded.frame + ) + changed_tables[expanded.frame.schema.person_entity] = person.rename_axis( + "__changed_columns__", axis="columns" + ) + with pytest.raises(ValueError, match="^SURVEY_POPULATION_REPLAY_AXIS$"): + bridge._current_context_frame( + replace(context, tables=changed_tables), expanded.frame + ) + changed_person = person.copy(deep=True) + changed_person.iloc[0, changed_person.columns.get_loc("age")] += 1 + changed_tables[expanded.frame.schema.person_entity] = changed_person + with pytest.raises(ValueError, match="^SURVEY_POPULATION_REPLAY_NATIVE_BITS$"): + bridge._current_context_frame( + replace(context, tables=changed_tables), expanded.frame + ) + + # Positive and negative zero are both valid weights; only their exact bits + # differ. This must reach the replay comparison rather than Frame refusal. + entity = next(e for e, w in context.weights.items() if np.any(w.values == 0)) + value = context.weights[entity] + changed_values = value.values.copy() + zero = int(np.flatnonzero(changed_values == 0)[0]) + changed_values[zero] = 0.0 if np.signbit(changed_values[zero]) else -0.0 + changed_weights = {**context.weights, entity: Weights(changed_values, value.kind)} + with pytest.raises(ValueError, match="^SURVEY_POPULATION_REPLAY_WEIGHT_BYTES$"): + bridge._current_context_frame( + replace(context, weights=changed_weights), expanded.frame + ) + with pytest.raises(ValueError, match="^SURVEY_HOST_STRATA_NAME$"): + bridge._current_context_frame( + replace(context, strata=context.strata.rename("__changed_name__")), + expanded.frame, + ) + with pytest.raises(ValueError, match="^SURVEY_POPULATION_REPLAY_AXIS$"): + bridge._current_context_frame( + replace(context, strata=context.strata.rename_axis("__changed_index__")), + expanded.frame, + ) + assert owner._frame_identity(expanded.frame) == before + + graph = replace(live.compiled.graph, nodes=(*live.compiled.graph.nodes, *nodes)) + compiled = compile_graph(graph) + assert len(compiled.graph.nodes) == 6 + live.kernels.register( + bridge.CurrentSurveyHostProjectionKernel(preparation, allocated, expanded) + ) + live.kernels.register(bridge.CurrentSurveyMatrixKernel()) + outputs = [] + for resume in ("auto", "require"): + observed = {} + manifest = run_graph( + compiled, + store=live.store, + kernels=live.kernels, + sources=live.sources, + resume=resume, + _population_observer=lambda name, population, observed=observed: ( + observed.__setitem__(name, population) + ), + ) + actual = [] + for node_id, name, type_, expected, kernel in ( + ( + bridge.CURRENT_PROJECTION_NODE, + "projection", + bridge.CURRENT_PROJECTION_TYPE, + projection, + bridge.CurrentSurveyHostProjectionKernel, + ), + ( + bridge.CURRENT_MATRIX_NODE, + "matrix", + bridge.model_input.RECIPIENT_MATRIX_TYPE, + matrix, + bridge.CurrentSurveyMatrixKernel, + ), + ): + survey._final_artifact( + manifest, + live.store, + node_id=node_id, + name=name, + type_=type_, + payload=expected, + capabilities=kernel.capabilities, + ) + receipt = manifest.node(node_id) + assert receipt.opaque_artifacts[name] == opaque_artifact_key( + receipt.key, name + ) + actual.append(live.store.load_bytes(receipt.opaque_artifacts[name])) + replay.same_replayed_population(expanded, observed[node_id]) + checked = bridge.verify_materialized_current_survey_host( + preparation, + allocated, + expanded, + population=observed[bridge.CURRENT_MATRIX_NODE], + projection=actual[0], + matrix=actual[1], + ) + assert checked == evidence + outputs.append(tuple(actual)) + assert outputs[0] == outputs[1] == (projection, matrix) + assert owner._frame_identity(expanded.frame) == before + wrong_pins = copy.deepcopy(pins) + wrong_pins["preparation"]["producer_key"] = "f" * 64 + bad_projection_node = replace( + nodes[0], + params={ + **dict(nodes[0].params), + "host_edges": codec.encode_json(wrong_pins).decode(), + }, + ) + bad_graph = replace( + graph, nodes=(*live.compiled.graph.nodes, bad_projection_node, nodes[1]) + ) + with pytest.raises(NodeRejected, match="SURVEY_HOST_EDGE_PIN"): + run_graph( + compile_graph(bad_graph), + store=live.store, + kernels=live.kernels, + sources=live.sources, + resume="auto", + ) + + # A well-formed amount mutation can pass the pure value parser; the live + # materialized verifier must reject it against the retained current owner. + changed = copy.deepcopy(document) + changed["asec"]["rows"][0][3] = np.asarray([60001.0], dtype=" 0) + stage.check_fiscal_calibration_weights(bounds, output.weights.values) + assert json.loads(output.artifacts["origin_diagnostics"])["fixed_zero_rows"] == 1 + with pytest.raises(ValueError, match="DESIGN_ANCHOR"): + stage.verify_fiscal_calibration_bounds( + c.artifacts["bounds"].payload, + household_ids=ids, + incoming_weights=c.weights["household"], + original_design_weights=Weights( + np.array([8.0, 8.0, 10.0]), WeightKind.DESIGN + ), + ) + + +@pytest.mark.parametrize( + "field,value", + [ + ("epochs", 0), + ("epochs", True), + ("epochs", 1.5), + ("learning_rate", float("nan")), + ("learning_rate", float("inf")), + ("learning_rate", 0.0), + ("learning_rate", True), + ], +) +def test_solver_settings_refuse_nonfinite_or_undeclared_modes(field, value): + _, mc, _ = setup() + with pytest.raises(ValueError): + stage.fiscal_dense_calibration_node( + measurement_node=mc.node, bounds_node="bounds", **{field: value} + ) + + +def test_misaligned_measurement_bounds_and_calibrated_parent_refuse_before_solve( + monkeypatch, +): + _, _, c = setup() + + def forbidden(*a, **kw): + pytest.fail("solver ran before admission") + + monkeypatch.setattr(stage, "calibrate", forbidden) + with pytest.raises(ValueError, match="IMPORTANCE"): + stage.FiscalDenseCalibrationKernel().run( + replace( + c, + weights={ + "household": Weights( + np.array([1.0, 2.0, 3.0]), WeightKind.CALIBRATED + ) + }, + ) + ) + changed = dict(c.tables) + changed["person"] = c.tables["person"].copy() + changed["person"]["amount"] += 1 + with pytest.raises(ValueError, match="PROJECTION_BINDING"): + stage.FiscalDenseCalibrationKernel().run(replace(c, tables=changed)) + raw = json.loads(c.artifacts["bounds"].payload) + raw["household_ids"] = measurement._array(np.array([30, 20, 10], dtype=np.int64)) + artifacts = dict(c.artifacts) + artifacts["bounds"] = replace( + artifacts["bounds"], payload=measurement.canonical_json(raw) + ) + with pytest.raises(ValueError, match="HOUSEHOLD_ALIGNMENT"): + stage.FiscalDenseCalibrationKernel().run(replace(c, artifacts=artifacts)) + + +def test_group_and_row_caps_refuse_without_clipping(): + _, _, c = setup(row_upper=[1.0, 6.0, 5.0]) + with pytest.raises(ValueError, match="ROW_REFERENCE_CAP"): + stage.FiscalDenseCalibrationKernel().run(c) + bounds = stage.decode_fiscal_calibration_bounds(c.artifacts["bounds"].payload) + with pytest.raises(ValueError, match="frozen absolute bounds"): + stage.check_fiscal_calibration_weights(bounds, np.array([5.0, 2.0, 3.0])) + + +@pytest.mark.parametrize( + "change,reason", + [ + ("names", "TARGET_ALIGNMENT"), + ("matrix", "MATRIX_IDENTITY"), + ("diagnostic", "DIAGNOSTIC_ESTIMATES"), + ], +) +def test_solver_return_with_wrong_matrix_or_target_order_is_refused( + monkeypatch, change, reason +): + _, _, c = setup() + actual = stage.calibrate + + def changed(*a, **kw): + r = actual(*a, **kw) + if change == "names": + return replace( + r, problem=replace(r.problem, names=tuple(reversed(r.problem.names))) + ) + if change == "matrix": + matrix = r.problem.matrix.copy() + matrix.data[0] += 1 + return replace(r, problem=replace(r.problem, matrix=matrix)) + return replace( + r, + diagnostics=( + replace(r.diagnostics[0], final_estimate=999.0), + *r.diagnostics[1:], + ), + ) + + monkeypatch.setattr(stage, "calibrate", changed) + with pytest.raises(ValueError, match=reason): + stage.FiscalDenseCalibrationKernel().run(c) + + +class InventedImportance(KernelBase): + ref = "test.fiscal_dense_importance@1" + capabilities = Capabilities( + Determinism.DETERMINISTIC, structural=StructuralDelta.REWEIGHT + ) + + def run(self, context): + return KernelResult( + weights=Weights(np.array(context.params["incoming"]), WeightKind.IMPORTANCE) + ) + + +class InventedBounds(KernelBase): + ref = "test.fiscal_dense_bounds@1" + capabilities = Capabilities(Determinism.DETERMINISTIC) + + def run(self, context): + payload = context.sources["bounds"].read_bytes() + stage.decode_fiscal_calibration_bounds(payload) + return KernelResult(artifacts={"numeric_bounds": payload}) + + +def graph_fixture(tmp_path): + value, mc, c = setup(incoming=[1.0, 0.0, 3.0]) + value = Frame( + {e: value.table(e) for e in value.entities}, + value.schema, + {"household": Weights(np.array([8.0, 9.0, 10.0]), WeightKind.DESIGN)}, + ) + create = Node( + "invented.design", + InventedSource.ref, + sources=("fixture",), + structural=StructuralDelta.CREATE, + outputs=( + Owned("person", "amount", "float64"), + Owned("person", "keep", "bool"), + Owned("tax_unit", "tax_unit_amount", "float64"), + Owned("household", "state_fips", "int64"), + Owned("household", "congressional_district_geoid", "int64"), + ), + ) + importance = Node( + "invented", + InventedImportance.ref, + base=create.id, + structural=StructuralDelta.REWEIGHT, + inputs=(Slice("household", ("state_fips",)),), + weights=WeightTransition("household", "importance", mass="free"), + mass="free", + params={"incoming": (1.0, 0.0, 3.0)}, + ) + bounds = Node( + "invented.bounds", + InventedBounds.ref, + population=importance.id, + sources=("bounds",), + artifact_outputs=(ArtifactOutput("numeric_bounds", stage.BOUNDS_TYPE),), + ) + nodes = (create, importance, bounds, mc.node, c.node) + sources = (SourceRef("fixture", "frame-store"), SourceRef("bounds", "raw-bytes-v1")) + kernels = KernelRegistry() + for kernel in ( + InventedSource(), + InventedImportance(), + InventedBounds(), + measurement.FiscalMeasurementKernel(), + stage.FiscalDenseCalibrationKernel(), + ): + kernels.register(kernel) + store = ContentStore(tmp_path / "store") + path = store.put_frame( + hashlib.sha256(b"invented fiscal dense fixture").hexdigest(), value + ) + bound_path = tmp_path / "bounds.json" + bound_path.write_bytes(c.artifacts["bounds"].payload) + arguments = { + "store": store, + "kernels": kernels, + "sources": {"fixture": path, "bounds": bound_path}, + } + return value, sources, nodes, arguments + + +def test_real_graph_cold_required_replay_preserves_parent_and_original_anchors( + tmp_path, +): + original, sources, nodes, arguments = graph_fixture(tmp_path) + compiled = compile_graph(Graph("us", sources, nodes)) + anchors = {} + + def observe(node_id, population): + anchors[node_id] = population.design_weights["household"].tobytes() + + cold = run_graph(compiled, **arguments, _population_observer=observe) + cold_anchors = dict(anchors) + warm = run_graph( + compiled, **arguments, resume="require", _population_observer=observe + ) + assert cold.key == warm.key and all(n.store_hit for n in warm.nodes.values()) + before = warm.population("invented") + after = warm.population(nodes[-1].id) + # The graph performs precisely DESIGN -> IMPORTANCE -> CALIBRATED. + assert after.weights_for("household").kind is WeightKind.CALIBRATED + assert before.weights_for("household").kind is WeightKind.IMPORTANCE + for entity in original.entities: + pd.testing.assert_frame_equal(before.table(entity), original.table(entity)) + pd.testing.assert_frame_equal(after.table(entity), original.table(entity)) + assert after.weights_for("household").values[1] == 0 + assert ( + cold.population(nodes[-1].id).weights_for("household").values.tobytes() + == after.weights_for("household").values.tobytes() + ) + # Detached observer snapshots expose original anchors without owner authority. + assert anchors == cold_anchors + assert anchors[nodes[-1].id] == np.array([8.0, 9.0, 10.0]).tobytes() + payload = arguments["store"].load_bytes( + warm.node(nodes[-1].id).opaque_artifacts["origin_diagnostics"] + ) + assert json.loads(payload)["fixed_zero_rows"] == 1 + # A repeated CALIBRATED -> CALIBRATED stage is refused before another solve. + new_measure = replace(nodes[-2], id="repeat.measurement", population=nodes[-1].id) + repeated = stage.fiscal_dense_calibration_node( + measurement_node=new_measure, + bounds_node=nodes[2].id, + node_id="repeat.calibration", + ) + compiled_repeat = compile_graph( + Graph("us", sources, (*nodes, new_measure, repeated)) + ) + with pytest.raises( + NodeRejectedError, match="(?i)(importance|calibrated|transition|forward)" + ): + run_graph(compiled_repeat, **arguments) + + +def test_real_graph_replay_refuses_changed_bounds_and_target_declaration(tmp_path): + _, sources, nodes, arguments = graph_fixture(tmp_path) + compiled = compile_graph(Graph("us", sources, nodes)) + run_graph(compiled, **arguments) + changed_measure = measurement.fiscal_measurement_node(**declaration()) + changed_solve = stage.fiscal_dense_calibration_node( + measurement_node=changed_measure, bounds_node=nodes[2].id + ) + changed = compile_graph( + Graph("us", sources, (*nodes[:3], changed_measure, changed_solve)) + ) + with pytest.raises( + (NodeRejectedError, StoreMissError), match="(?i)(require|cache|missing)" + ): + run_graph(changed, **arguments, resume="require") + path = arguments["sources"]["bounds"] + raw = json.loads(path.read_bytes()) + raw["group_upper"] = measurement._array(np.array([5.5, 5.0])) + path.write_bytes(measurement.canonical_json(raw)) + with pytest.raises( + (NodeRejectedError, StoreMissError), match="(?i)(require|cache|missing)" + ): + run_graph(compiled, **arguments, resume="require") + + +def test_bound_codec_refuses_noncanonical_values_and_wrong_types(): + _, _, c = setup() + payload = c.artifacts["bounds"].payload + for field, value in ( + ("incoming", np.array([1.0, float("nan"), 3.0])), + ("group_indices", np.array([0, 0, 9], dtype=np.int64)), + ("original_design", np.array([8.0, 9.0])), + ("household_ids", np.array([10, 10, 30], dtype=np.int64)), + ): + raw = json.loads(payload) + raw[field] = measurement._array(value) + with pytest.raises(ValueError): + stage.decode_fiscal_calibration_bounds(measurement.canonical_json(raw)) + with pytest.raises(ValueError, match="CANONICAL"): + stage.decode_fiscal_calibration_bounds(payload + b"\n") + + +def test_implementation_identity_binds_existing_solver_and_target_math( + monkeypatch, tmp_path +): + # Point source identity at changed synthetic bytes without editing package source. + baseline = stage.FiscalDenseCalibrationKernel().implementation_hash() + changed = tmp_path / "changed_target.py" + changed.write_text("# invented changed target arithmetic\n") + monkeypatch.setattr(stage.target_module, "__file__", str(changed)) + assert stage.FiscalDenseCalibrationKernel().implementation_hash() != baseline diff --git a/packages/microcosm-build/tests/test_us_graph_fiscal_measurement.py b/packages/microcosm-build/tests/test_us_graph_fiscal_measurement.py new file mode 100644 index 000000000..ad80a0f09 --- /dev/null +++ b/packages/microcosm-build/tests/test_us_graph_fiscal_measurement.py @@ -0,0 +1,552 @@ +"""Invented fiscal measures through the actual interpreter, CSR and graph store.""" + +import hashlib +import json +from dataclasses import replace +from pathlib import Path + +import numpy as np +import pandas as pd +import pytest + +from microcosm.build.us_runtime import graph_fiscal_measurement as stage +from microcosm.calibrate import TargetRegistry, TargetSpec +from microcosm.calibrate import target as target_math +from microcosm.calibrate.hierarchy import ( + CalibrationHierarchy, + HierarchyCategory, + HierarchyGeography, + HierarchyNode, +) +from microcosm.frame import US_SCHEMA, Frame, WeightKind, Weights +from microcosm.frame import materialize as frame_materialize +from microcosm.graph import ( + Capabilities, + ContentStore, + Determinism, + Graph, + KernelBase, + KernelContext, + KernelRegistry, + KernelResult, + Node, + NodeRejectedError, + Owned, + SourceRef, + StoreMissError, + StructuralDelta, + compile_graph, + load_source, + run_graph, + source_hash, +) +from microcosm.graph.canonical import canonical_json + + +def frame(): + tables = { + "person": pd.DataFrame( + { + "person_id": np.array([400, 100, 300, 200], dtype=np.int64), + **{ + f"person_{e}_id": np.array([30, 10, 20, 10], dtype=np.int64) + for e in US_SCHEMA.group_entities + }, + "amount": [40.0, 10.0, 30.0, 20.0], + "keep": [True, True, False, True], + } + ), + **{ + e: pd.DataFrame({f"{e}_id": np.array([10, 20, 30], dtype=np.int64)}) + for e in US_SCHEMA.group_entities + }, + } + tables["household"]["state_fips"] = np.array([6, 6, 36], dtype=np.int64) + tables["household"]["congressional_district_geoid"] = np.array( + [601, 602, 3601], dtype=np.int64 + ) + tables["tax_unit"]["tax_unit_amount"] = [100.0, 200.0, 300.0] + return Frame( + tables, + US_SCHEMA, + {"household": Weights(np.array([1.0, 2.0, 3.0]), WeightKind.DESIGN)}, + ) + + +def declaration(*, value_variable="person_count", entity="person", model_outputs=()): + definitions = ( + ("national", "0100000US", None, None), + ("state", "0400000US06", "state_fips", 6), + ("state", "0400000US36", "state_fips", 36), + ( + "congressional_district", + "5001900US0601", + "congressional_district_geoid", + 601, + ), + ( + "congressional_district", + "5001900US0602", + "congressional_district_geoid", + 602, + ), + ( + "congressional_district", + "5001900US3601", + "congressional_district_geoid", + 3601, + ), + ) + scopes = tuple( + stage.FiscalGeographyScope(HierarchyGeography(id_, id_, level), column, value) + for level, id_, column, value in definitions + ) + specs, bindings = [], {} + for i, scope in enumerate(scopes): + name = f"invented_{i}" + specs.append( + TargetSpec( + name, + entity, + 1.0, + f"measure_{i}", + period=2024, + source="Invented test; not an administrative observation", + family="invented", + metadata={"contract_target_id": name}, + hierarchy=CalibrationHierarchy( + HierarchyNode("test", "Test"), + HierarchyCategory("count", "Count", "test"), + scope.geography, + (), + HierarchyNode(name, name), + ), + ) + ) + bindings[name] = { + "bindings": {"policyengine": {"value_variable": value_variable}} + } + return { + "registry": TargetRegistry(specs, country="us"), + "population": "invented", + "input_columns": { + "person": ("amount", "keep"), + "tax_unit": ("tax_unit_amount",), + "household": ("state_fips", "congressional_district_geoid"), + }, + "contract_targets": bindings, + "advertised_scopes": scopes, + "geography_vintage": "invented CD119 mapping", + "period": 2024, + "model_outputs": model_outputs, + } + + +def context(arguments=None, value=None): + value = frame() if value is None else value + arguments = declaration() if arguments is None else arguments + node = stage.fiscal_measurement_node(**arguments) + tables = {} + for slice_ in node.inputs: + entity = slice_.entity + structural = [f"{entity}_id"] + if entity == "person": + structural += [f"person_{e}_id" for e in US_SCHEMA.group_entities] + tables[entity] = ( + value.table(entity) + .loc[:, list(dict.fromkeys([*structural, *slice_.columns]))] + .copy() + ) + return KernelContext( + node, + tables, + {"household": value.weights_for("household")}, + value.strata, + node.params, + np.random.default_rng(0), + ) + + +def test_actual_measurement_maps_person_counts_and_tax_units_to_households(): + original = frame() + c = context(value=original) + result = stage.FiscalMeasurementKernel().run(c) + payload = result.artifacts["measurement"] + measured = stage.verify_fiscal_measurement( + payload, context=c, expected_sha256=hashlib.sha256(payload).hexdigest() + ) + np.testing.assert_array_equal(measured.household_ids, [10, 20, 30]) + np.testing.assert_array_equal( + measured.matrix.toarray(), + [[2, 1, 1], [2, 1, 0], [0, 0, 1], [2, 0, 0], [0, 1, 0], [0, 0, 1]], + ) + np.testing.assert_array_equal( + measured.matrix.toarray()[3:].sum(axis=0), measured.matrix.toarray()[0] + ) + assert result.frame is None and result.weights is None and not result.columns + assert result.receipt["fit_performed"] is False + assert result.receipt["release_eligible"] is False + for entity in original.entities: + pd.testing.assert_frame_equal(original.table(entity), frame().table(entity)) + c = context(declaration(value_variable="tax_unit_amount", entity="tax_unit")) + values = stage.decode_fiscal_measurement( + stage.FiscalMeasurementKernel().run(c).artifacts["measurement"] + ) + np.testing.assert_array_equal(values.matrix.toarray()[0], [100, 200, 300]) + + +def test_actual_shared_interpreter_sum_and_filter(): + arguments = declaration(value_variable="amount") + for target in arguments["contract_targets"].values(): + target["bindings"]["policyengine"] = { + "value_expression": "amount + amount", + "filters": [{"variable": "amount", "operator": ">", "value": 5}], + } + measured = stage.decode_fiscal_measurement( + stage.FiscalMeasurementKernel().run(context(arguments)).artifacts["measurement"] + ) + np.testing.assert_array_equal(measured.matrix.toarray()[0], [60, 60, 80]) + + +def test_boolean_indicator_expression_is_an_arithmetic_sum(): + arguments = declaration() + for target in arguments["contract_targets"].values(): + target["bindings"]["policyengine"] = {"value_expression": "keep + keep"} + value = frame() + value.person["keep"] = [True, False, True, True] + measured = stage.decode_fiscal_measurement( + stage.FiscalMeasurementKernel() + .run(context(arguments, value)) + .artifacts["measurement"] + ) + np.testing.assert_array_equal(measured.matrix.toarray()[0], [2, 2, 2]) + + +@pytest.mark.parametrize("kind", ["float", "nullable_boolean"]) +@pytest.mark.parametrize("location", ["binding", "target_filter"]) +def test_missing_consumed_predicate_is_refused(kind, location): + arguments, value = declaration(), frame() + variable = "amount" if kind == "float" else "keep" + if kind == "float": + value.person.loc[1, variable] = np.nan + predicate = {"variable": variable, "operator": ">", "value": 5} + else: + value.person[variable] = pd.Series([True, pd.NA, True, True], dtype="boolean") + predicate = {"variable": variable, "equals": True} + if location == "binding": + for binding in arguments["contract_targets"].values(): + binding["bindings"]["policyengine"]["filters"] = [predicate] + else: + arguments["registry"] = TargetRegistry( + [replace(s, filter=variable) for s in arguments["registry"]], country="us" + ) + with pytest.raises( + ValueError, match="MISSING_PREDICATE_INPUT:person\\." + variable + ): + stage.FiscalMeasurementKernel().run(context(arguments, value)) + + +@pytest.mark.parametrize("location", ["binding", "target_filter"]) +def test_known_false_predicates_and_unconsumed_missing_inputs_are_preserved(location): + arguments, value = declaration(), frame() + value.person["keep"] = [True, False, True, True] + value.person.loc[0, "amount"] = np.nan # This measure does not read amount. + if location == "binding": + for binding in arguments["contract_targets"].values(): + binding["bindings"]["policyengine"]["filters"] = [ + {"variable": "keep", "equals": True} + ] + else: + arguments["registry"] = TargetRegistry( + [replace(s, filter="keep") for s in arguments["registry"]], country="us" + ) + measured = stage.decode_fiscal_measurement( + stage.FiscalMeasurementKernel() + .run(context(arguments, value)) + .artifacts["measurement"] + ) + np.testing.assert_array_equal(measured.matrix.toarray()[0], [1, 1, 1]) + + +def test_implementation_identity_includes_target_arithmetic_source( + tmp_path, monkeypatch +): + before = stage.FiscalMeasurementKernel().implementation_hash() + changed = tmp_path / "changed_target.py" + source = Path(target_math.__file__).read_text() + original = " return values * filter_mask\n" + assert source.count(original) == 1 + changed.write_text( + source.replace(original, " return 2.0 * values * filter_mask\n") + ) + monkeypatch.setattr(target_math, "__file__", str(changed)) + assert stage.FiscalMeasurementKernel().implementation_hash() != before + + +def test_implementation_identity_includes_engine_table_transform(tmp_path, monkeypatch): + assert stage.policyengine_us.engine_tables is frame_materialize.engine_tables + before = stage.FiscalMeasurementKernel().implementation_hash() + changed = tmp_path / "changed_engine_tables.py" + source = Path(frame_materialize.__file__).read_text() + original = " ).values\n" + assert source.count(original) == 1 + changed.write_text(source.replace(original, " ).values * 2.0\n")) + monkeypatch.setattr(frame_materialize, "__file__", str(changed)) + assert stage.FiscalMeasurementKernel().implementation_hash() != before + + +@pytest.mark.parametrize( + "change,reason", + [ + ("binding", "UNSUPPORTED_BINDING"), + ("coverage", "ADVERTISED_LEVELS"), + ("period", "TARGET_PERIOD_HIERARCHY"), + ("collision", "INPUT_MEASURE_COLLISION"), + ("unknown", "UNMATERIALIZED_TARGETS"), + ("empty_area", "EMPTY_ADVERTISED_GEOGRAPHY"), + ("zero_support", "UNSUPPORTED_TARGET"), + ("zero_weight", "UNSUPPORTED_TARGET"), + ], +) +def test_declared_surface_and_advertised_support_fail_closed(change, reason): + arguments, value = declaration(), frame() + if change == "binding": + arguments["contract_targets"]["invented_0"]["bindings"]["policyengine"][ + "kind" + ] = "input_substitution_counterfactual" + elif change == "coverage": + arguments["advertised_scopes"] = arguments["advertised_scopes"][:3] + elif change == "period": + arguments["period"] = 2025 + elif change == "collision": + arguments["input_columns"]["person"] += ("measure_0",) + elif change == "unknown": + arguments["contract_targets"]["invented_0"]["bindings"]["policyengine"][ + "value_variable" + ] = "not_produced" + elif change == "empty_area": + value.table("household").loc[2, "congressional_district_geoid"] = 3699 + elif change == "zero_support": + arguments["contract_targets"]["invented_5"]["bindings"]["policyengine"][ + "filters" + ] = [{"variable": "amount", "operator": ">", "value": 999}] + elif change == "zero_weight": + value = Frame( + {e: value.table(e) for e in value.entities}, + US_SCHEMA, + {"household": Weights(np.array([1.0, 2.0, 0.0]), WeightKind.DESIGN)}, + ) + with pytest.raises(ValueError, match=reason): + stage.FiscalMeasurementKernel().run(context(arguments, value)) + + +def test_artifact_digest_projection_and_order_tamper_refusals(): + c = context() + payload = stage.FiscalMeasurementKernel().run(c).artifacts["measurement"] + digest = hashlib.sha256(payload).hexdigest() + changed = json.loads(payload) + changed["data"]["hex"] = np.full(9, 99.0, dtype="