From ff7e4dd9c3316685eaf24d349370fb1b43bd6d6a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mar=C3=ADa=20Juaristi?= <127882282+juaristi22@users.noreply.github.com> Date: Thu, 10 Sep 2026 18:13:24 +0200 Subject: [PATCH 1/8] Add reusable graph evidence and calibration artifact contracts --- changelog.d/uk-full-graph-contracts.added.md | 4 + docs/graph-acceptance.md | 20 + .../src/microcosm/build/artifact_files.py | 162 +++++ .../src/microcosm/build/gate_battery.py | 98 ++- .../src/microcosm/build/stage_evidence.py | 86 +++ .../tests/test_artifact_files.py | 91 +++ .../tests/test_gate_battery_replay.py | 76 +++ .../tests/test_stage_evidence.py | 53 ++ packages/microcosm-calibrate/README.md | 19 + .../src/microcosm/calibrate/artifacts.py | 566 ++++++++++++++++++ .../microcosm/calibrate/target_selection.py | 123 ++++ .../tests/test_ordered_artifacts.py | 140 +++++ .../tests/test_target_selection.py | 72 +++ packages/microcosm-graph/README.md | 12 + .../src/microcosm/graph/__init__.py | 4 + .../src/microcosm/graph/decl.py | 34 +- .../src/microcosm/graph/executor.py | 15 + .../src/microcosm/graph/kernel.py | 33 +- .../src/microcosm/graph/population.py | 16 + .../src/microcosm/graph/serialize.py | 20 +- .../src/microcosm/graph/weight_update.py | 29 + .../parity/kernels/calibrate/pins.json | 2 +- .../fixtures/parity/kernels/fit.qrf/pins.json | 2 +- .../parity/kernels/simulate/pins.json | 2 +- .../tests/test_acceptance_b_ownership.py | 4 + .../tests/test_graph_kernel_contract.py | 20 +- .../tests/test_graph_population.py | 93 +++ .../tests/test_weight_update.py | 212 +++++++ 28 files changed, 1998 insertions(+), 10 deletions(-) create mode 100644 changelog.d/uk-full-graph-contracts.added.md create mode 100644 packages/microcosm-build/src/microcosm/build/artifact_files.py create mode 100644 packages/microcosm-build/src/microcosm/build/stage_evidence.py create mode 100644 packages/microcosm-build/tests/test_artifact_files.py create mode 100644 packages/microcosm-build/tests/test_gate_battery_replay.py create mode 100644 packages/microcosm-build/tests/test_stage_evidence.py create mode 100644 packages/microcosm-calibrate/src/microcosm/calibrate/artifacts.py create mode 100644 packages/microcosm-calibrate/src/microcosm/calibrate/target_selection.py create mode 100644 packages/microcosm-calibrate/tests/test_ordered_artifacts.py create mode 100644 packages/microcosm-calibrate/tests/test_target_selection.py create mode 100644 packages/microcosm-graph/src/microcosm/graph/weight_update.py create mode 100644 packages/microcosm-graph/tests/test_weight_update.py diff --git a/changelog.d/uk-full-graph-contracts.added.md b/changelog.d/uk-full-graph-contracts.added.md new file mode 100644 index 000000000..188412444 --- /dev/null +++ b/changelog.d/uk-full-graph-contracts.added.md @@ -0,0 +1,4 @@ +Add explicit graph weight normalization, immutable Frame evidence projections, +ordered target-selection receipts and portable calibration problems/results. +Register UK dense calibration, informed size search, exact-count draws and +refits as separate resumable nodes over the original pool. diff --git a/docs/graph-acceptance.md b/docs/graph-acceptance.md index 4f602d4a7..f99091773 100644 --- a/docs/graph-acceptance.md +++ b/docs/graph-acceptance.md @@ -296,6 +296,26 @@ Adding a normative field with a default changes the canonical projection of every node that carries it, so node keys moved with amendments 11 and 13's sibling field `entrants`; no released artifact pins a graph key yet. +### UK full-build registration extensions + +The UK registration adds two explicit contracts to the frozen declarations: + +- `WeightUpdate(entity, kind, reason, mass)` declares normalization without + changing the weight kind. It requires `conserve` or `declared` mass and an + ordered entity-ID receipt; it preserves design-weight ancestry. + `WeightTransition` retains its existing forward-only kind check. +- `KernelContext.frame_metadata` and `frame_mass_log` expose immutable + population metadata and legacy mass records alongside declared table + slices. `frame_column_order` gives the original order of projected columns + only. The mutation guard covers these fields. A consumer that needs a + completed legacy ledger must depend on a structural population checkpoint + or explicit predecessor evidence; incidental execution order does not + establish that dependency. Graph mass accounting remains executor-authored. + +`test_weight_update.py` covers aligned identity, kind checks, mass validation, +design ancestry, immutable context and replay. These extensions do not grant +kernels access to undeclared population columns. + ## Ownership Max's ruling (2026-09-01): the agents build all of it. The "implementer ≠ diff --git a/packages/microcosm-build/src/microcosm/build/artifact_files.py b/packages/microcosm-build/src/microcosm/build/artifact_files.py new file mode 100644 index 000000000..ff14fd7af --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/artifact_files.py @@ -0,0 +1,162 @@ +"""Atomic filesystem materialization of declared, portable build artifacts.""" + +from __future__ import annotations + +import os +import tempfile +from collections.abc import Mapping +from pathlib import Path + +from .trace import sha256_file + + +def file_artifact(path: str | Path) -> dict[str, object]: + """Bind one regular file without loading its entire payload into memory.""" + source = Path(path) + if not source.is_file(): + raise ValueError(f"Artifact is not a regular file: {source}.") + before = source.stat() + digest = sha256_file(source) + after = source.stat() + if (before.st_size, before.st_mtime_ns, before.st_ino) != ( + after.st_size, + after.st_mtime_ns, + after.st_ino, + ): + raise ValueError(f"Artifact changed while binding its identity: {source}.") + return {"filename": source.name, "sha256": digest, "size_bytes": after.st_size} + + +def materialize_bytes(payload: bytes, path: str | Path) -> dict[str, object]: + """Write deterministic bytes atomically, including after a graph cache hit.""" + if not isinstance(payload, bytes): + raise TypeError("Artifact payload must be immutable bytes.") + destination = Path(path) + destination.parent.mkdir(parents=True, exist_ok=True) + fd, temporary = tempfile.mkstemp( + prefix=f".{destination.name}.", dir=destination.parent + ) + try: + with os.fdopen(fd, "wb") as stream: + stream.write(payload) + stream.flush() + os.fsync(stream.fileno()) + Path(temporary).replace(destination) + finally: + Path(temporary).unlink(missing_ok=True) + return file_artifact(destination) + + +def validate_file_inventory( + inventory: Mapping[str, Mapping[str, object]], *, root: str | Path +) -> None: + """Check exact bytes and sizes for each named file in a bundle directory.""" + directory = Path(root).resolve() + for role, expected in inventory.items(): + name = expected.get("filename") + if ( + not isinstance(name, str) + or not name + or Path(name).name != name + or name in {".", ".."} + ): + raise ValueError(f"Artifact {role!r} has an invalid bundle filename.") + path = directory / name + if path.resolve().parent != directory: + raise ValueError( + f"Artifact {role!r} filename escapes its bundle directory." + ) + if file_artifact(path) != dict(expected): + raise ValueError(f"Artifact {role!r} identity differs from its inventory.") + + +def publish_staged_bundle( + staged: Mapping[str, str | Path], + destinations: Mapping[str, str | Path], + *, + completion_role: str = "manifest", + expected: Mapping[str, Mapping[str, object]] | None = None, +) -> dict[str, dict[str, object]]: + """Publish a validated bundle with rollback and the completion marker last. + + The marker is absent while files change. Handled failures, including + KeyboardInterrupt, restore the prior bundle before restoring its marker. + This is a transaction over individual atomic renames, not a claim of + multi-file atomicity across process kill or power loss. + """ + import shutil + + if set(staged) != set(destinations) or completion_role not in staged: + raise ValueError( + "Staged bundle roles must match destinations and include a completion marker." + ) + source = {role: Path(path) for role, path in staged.items()} + target = {role: Path(path) for role, path in destinations.items()} + directories = {path.parent.resolve() for path in target.values()} + if len(directories) != 1 or len(set(target.values())) != len(target): + raise ValueError("Bundle destinations must be distinct files in one directory.") + directory = next(iter(directories)) + inventory = {} + for role in source: + if source[role].is_symlink() or target[role].is_symlink(): + raise ValueError("Bundle publication does not accept symlink files.") + if source[role].resolve() == target[role].resolve(): + raise ValueError("Bundle sources must be separate staging files.") + if source[role].name != target[role].name: + raise ValueError("Staged bundle filenames must match their destination.") + record = file_artifact(source[role]) + if expected is not None and ( + role not in expected + or any(expected[role].get(key) != value for key, value in record.items()) + ): + raise ValueError( + f"Staged artifact {role!r} differs from its declared identity." + ) + inventory[role] = record + created = not directory.exists() + directory.mkdir(parents=True, exist_ok=True) + backup = Path(tempfile.mkdtemp(prefix=".bundle-backup-", dir=directory.parent)) + order = tuple(role for role in staged if role != completion_role) + ( + completion_role, + ) + saved, published = [], [] + succeeded = False + try: + # Remove the old completion marker before replacing any old payload. + for role in ( + completion_role, + *[role for role in order if role != completion_role], + ): + if target[role].exists(): + if not target[role].is_file(): + raise ValueError( + f"Bundle destination is not a regular file: {target[role]}." + ) + target[role].replace(backup / target[role].name) + saved.append(role) + for role in order: + # Recheck just before moving; publication never binds mixed bytes. + if file_artifact(source[role]) != inventory[role]: + raise ValueError( + f"Staged artifact {role!r} changed during publication." + ) + source[role].replace(target[role]) + published.append(role) + succeeded = True + return inventory + finally: + if not succeeded: + for role in reversed(published): + target[role].unlink(missing_ok=True) + # The previous marker returns only after all previous payloads. + for role in ( + *[role for role in saved if role != completion_role], + *([completion_role] if completion_role in saved else []), + ): + (backup / target[role].name).replace(target[role]) + shutil.rmtree(backup) + if created and not succeeded: + try: + directory.rmdir() + except OSError: + pass diff --git a/packages/microcosm-build/src/microcosm/build/gate_battery.py b/packages/microcosm-build/src/microcosm/build/gate_battery.py index 401879e9a..2ea6918e7 100644 --- a/packages/microcosm-build/src/microcosm/build/gate_battery.py +++ b/packages/microcosm-build/src/microcosm/build/gate_battery.py @@ -80,6 +80,8 @@ "GatePhaseReport", "GateStatus", "evaluate_phase", + "gate_phase_report_from_payload", + "gate_phase_report_payload", "gate_signing_key_env", ] @@ -541,6 +543,83 @@ def failures(self) -> tuple[str, ...]: # --------------------------------------------------------------------------- +def gate_phase_report_payload( + report: GatePhaseReport, *, gates: GatesManifest +) -> dict[str, object]: + """Portable numerical verdicts, bound to their exact declared policy. + + Release IDs, signing keys and filesystem paths belong to materialization, + and therefore do not enter a cached evaluation artifact. + """ + expected = tuple(entry for entry in gates.gates if entry.phase == report.phase) + if ( + report.phase not in gates.phases + or tuple(o.entry for o in report.outcomes) != expected + ): + raise ValueError("Gate phase outcomes do not match the declared manifest.") + return { + "schema_version": 1, + "gates_manifest_sha256": _canonical_sha256(_gates_manifest_payload(gates)), + "phase": report.phase, + "outcomes": [ + { + "id": outcome.entry.id, + **outcome.to_payload(), + "result_name": outcome.result.name + if outcome.result is not None + else None, + "evidence_sha256": outcome.evidence_sha256, + } + for outcome in report.outcomes + ], + } + + +def gate_phase_report_from_payload( + payload: Mapping[str, object], *, gates: GatesManifest +) -> GatePhaseReport: + """Validate stored outcomes against current entries before report replay.""" + if payload.get("schema_version") != 1: + raise ValueError("Unsupported gate phase report schema.") + if payload.get("gates_manifest_sha256") != _canonical_sha256( + _gates_manifest_payload(gates) + ): + raise ValueError("Stored gate report has a different gate manifest.") + phase = payload.get("phase") + if phase not in gates.phases: + raise ValueError("Stored gate report has an undeclared phase.") + entries = tuple(entry for entry in gates.gates if entry.phase == phase) + rows = payload.get("outcomes") + if not isinstance(rows, list) or len(rows) != len(entries): + raise ValueError("Stored gate report outcomes do not cover its phase.") + outcomes = [] + for entry, row in zip(entries, rows, strict=True): + if not isinstance(row, Mapping) or any( + row.get(key) != getattr(entry, key) + for key in ("id", "gate", "phase", "criticality") + ): + raise ValueError("Stored gate report outcomes differ from its manifest.") + status = GateStatus(row["status"]) + result = None + if status in (GateStatus.PASSED, GateStatus.FAILED): + result = GateResult( + name=str(row["result_name"]), + passed=status is GateStatus.PASSED, + failures=tuple(row["failures"]), + details=dict(row["details"]), + ) + outcomes.append( + GateOutcome( + entry=entry, + status=status, + result=result, + reason=row.get("reason"), + evidence_sha256=row.get("evidence_sha256"), + ) + ) + return GatePhaseReport(phase=str(phase), outcomes=tuple(outcomes)) + + def _evaluate_gate(name: str, evaluator: Callable[[], GateResult]) -> GateResult: """Run one evaluator, failing closed on any misbehaviour. @@ -886,10 +965,25 @@ def run_phase(self, phase: str, context: EvidenceContext) -> GatePhaseReport: f"(declared order {list(self._gates.phases)})." ) report = evaluate_phase(self._gates, phase, context, registry=self._registry) - self._phase_reports[phase] = report - self._write_report() + self.record_phase(report) return report + def record_phase(self, report: GatePhaseReport) -> None: + """Persist an evaluated or verified cached phase before enforcement. + + This is the same ordered write-then-block boundary as ``run_phase``; + it never reruns evaluators and never accepts a different gate policy. + """ + if self._blocked_at_phase is not None: + raise ValueError(f"battery blocked at phase {self._blocked_at_phase!r}.") + if report.phase != self._next_phase(): + raise ValueError( + f"phase {report.phase!r} is out of order; expected {self._next_phase()!r}." + ) + gate_phase_report_payload(report, gates=self._gates) + self._phase_reports[report.phase] = report + self._write_report() + def enforce(self, phase: str, *, mode: BlockingMode) -> bool: """Apply the phase's blocking verdict, strictly after persistence. diff --git a/packages/microcosm-build/src/microcosm/build/stage_evidence.py b/packages/microcosm-build/src/microcosm/build/stage_evidence.py new file mode 100644 index 000000000..f0373a078 --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/stage_evidence.py @@ -0,0 +1,86 @@ +"""Portable evidence transport for existing stateful stage adapters. + +Capture only their declared checkpoint/evidence hooks after computation. Model +objects, closures and execution timestamps never become numerical artifacts. +Consumers need only this data contract, not a live fitting instance. +""" + +from __future__ import annotations + +import json +from collections.abc import Mapping + +from microcosm.graph import ArtifactType + +STAGE_EVIDENCE_TYPE = ArtifactType("microcosm.stage-evidence", 1) + + +def snapshot_stage_evidence(stage: str, transform: object | None) -> dict[str, object]: + """Snapshot checkpoint, fit-weight and sampling evidence without rerunning.""" + + checkpoint = None + hook = getattr(transform, "checkpoint_metadata", None) + if callable(hook): + checkpoint = dict(hook()) + evidence = checkpoint.get("evidence", checkpoint) + else: + result = getattr(transform, "last_result", None) + evidence_hook = getattr(result, "evidence", None) + evidence = ( + evidence_hook() + if callable(evidence_hook) + else result + if isinstance(result, Mapping) + else None + ) + payload: dict[str, object] = { + "schema_version": STAGE_EVIDENCE_TYPE.schema_version, + "stage": stage, + "checkpoint_metadata": checkpoint, + "evidence": evidence, + "sampling": getattr(transform, "sampling", None), + } + # Do not evaluate a raising property merely to detect whether it exists: + # missing/unreadable fitting evidence must stay visible to weight audits. + exposes_records = getattr(type(transform), "fit_weight_records", None) is not None + exposes_records |= "fit_weight_records" in getattr(transform, "__dict__", {}) + if exposes_records: + try: + records = tuple(transform.fit_weight_records or ()) + payload["fit_weight_records"] = [ + { + "fit_name": str(record.fit_name), + "weight_kind": str(record.weight_kind), + } + for record in records + ] + payload["fit_weight_records_status"] = "present" if records else "empty" + except Exception: # noqa: BLE001 - preserve the existing fail-visible audit + payload["fit_weight_records"] = [] + payload["fit_weight_records_status"] = "unreadable" + elif checkpoint is not None and "fit_weight_records" in checkpoint: + payload["fit_weight_records"] = checkpoint["fit_weight_records"] + payload["fit_weight_records_status"] = ( + "present" if checkpoint["fit_weight_records"] else "empty" + ) + return payload + + +def encode_stage_evidence(payload: Mapping[str, object]) -> bytes: + """Encode finite JSON; reject nonportable state rather than pickling it.""" + + return json.dumps( + dict(payload), sort_keys=True, separators=(",", ":"), allow_nan=False + ).encode("utf-8") + + +def decode_stage_evidence(payload: bytes, *, stage: str) -> dict[str, object]: + document = json.loads(payload) + if not isinstance(document, dict) or document.get("schema_version") != 1: + raise ValueError("Unsupported stage evidence schema.") + if document.get("stage") != stage: + raise ValueError("Stored stage evidence has a different stage identity.") + for key in ("checkpoint_metadata", "evidence", "sampling"): + if key not in document: + raise ValueError(f"Stage evidence is missing {key!r}.") + return document diff --git a/packages/microcosm-build/tests/test_artifact_files.py b/packages/microcosm-build/tests/test_artifact_files.py new file mode 100644 index 000000000..0abe02002 --- /dev/null +++ b/packages/microcosm-build/tests/test_artifact_files.py @@ -0,0 +1,91 @@ +"""Declared file artifacts are atomically materialized and checked by bytes.""" + +import pytest + +from microcosm.build.artifact_files import materialize_bytes, validate_file_inventory + + +def test_atomic_artifact_materialization_can_recreate_a_missing_output(tmp_path): + path = tmp_path / "evidence.json" + record = materialize_bytes(b'{"result":1}\n', path) + path.unlink() + assert materialize_bytes(b'{"result":1}\n', path) == record + validate_file_inventory({"evidence": record}, root=tmp_path) + path.write_bytes(b"changed") + with pytest.raises(ValueError, match="identity"): + validate_file_inventory({"evidence": record}, root=tmp_path) + + +def test_artifact_inventory_rejects_paths_outside_bundle(tmp_path): + with pytest.raises(ValueError, match="filename"): + validate_file_inventory({"escape": {"filename": "../other"}}, root=tmp_path) + + +def test_bundle_publication_replaces_existing_payloads_and_marker_last( + tmp_path, monkeypatch +): + from pathlib import Path + + from microcosm.build.artifact_files import publish_staged_bundle + + stage = tmp_path / "stage" + output = tmp_path / "output" + stage.mkdir() + output.mkdir() + sources = { + role: stage / name + for role, name in (("dataset", "full.h5"), ("manifest", "full.build.json")) + } + targets = {role: output / path.name for role, path in sources.items()} + for role in sources: + sources[role].write_bytes(("new-" + role).encode()) + targets[role].write_bytes(("old-" + role).encode()) + moves = [] + original = Path.replace + + def record(path, destination): + if Path(destination).parent == output: + moves.append(Path(destination).name) + return original(path, destination) + + monkeypatch.setattr(Path, "replace", record) + inventory = publish_staged_bundle(sources, targets) + assert moves == ["full.h5", "full.build.json"] + assert targets["manifest"].read_bytes() == b"new-manifest" + validate_file_inventory(inventory, root=output) + + +@pytest.mark.parametrize("interrupt", [RuntimeError, KeyboardInterrupt]) +def test_bundle_publication_rolls_back_old_complete_bundle( + tmp_path, monkeypatch, interrupt +): + from pathlib import Path + + from microcosm.build.artifact_files import publish_staged_bundle + + stage = tmp_path / "stage" + output = tmp_path / "output" + stage.mkdir() + output.mkdir() + sources = { + role: stage / name + for role, name in (("dataset", "full.h5"), ("manifest", "full.build.json")) + } + targets = {role: output / path.name for role, path in sources.items()} + for role in sources: + sources[role].write_bytes(("new-" + role).encode()) + targets[role].write_bytes(("old-" + role).encode()) + original = Path.replace + + def fail(path, destination): + if path == sources["manifest"]: + assert not targets["manifest"].exists() + raise interrupt("synthetic publication interruption") + return original(path, destination) + + monkeypatch.setattr(Path, "replace", fail) + with pytest.raises(interrupt): + publish_staged_bundle(sources, targets) + for role in targets: + assert targets[role].read_bytes() == ("old-" + role).encode() + assert not list(tmp_path.glob(".bundle-backup-*")) diff --git a/packages/microcosm-build/tests/test_gate_battery_replay.py b/packages/microcosm-build/tests/test_gate_battery_replay.py new file mode 100644 index 000000000..ce4a813e3 --- /dev/null +++ b/packages/microcosm-build/tests/test_gate_battery_replay.py @@ -0,0 +1,76 @@ +"""Replayed numerical gate outcomes retain the same enforcement policy.""" + +from dataclasses import replace + +import pytest + +from microcosm.build.country_spec import GateSelectionSpec, GatesManifest +from microcosm.build.gate_battery import ( + BlockingMode, + EvidenceContext, + GateBatteryBlockedError, + GateBatteryRun, + evaluate_phase, + gate_phase_report_from_payload, + gate_phase_report_payload, +) + + +def _manifest(): + return GatesManifest( + country="xx", + version=1, + policy="test", + phases=("terminal",), + gates=( + GateSelectionSpec( + id="mass", + gate="input_mass_parity", + phase="terminal", + criticality="release_blocking", + parameters={"relative_tolerance": 0.01}, + ), + ), + ) + + +def test_replayed_gate_failure_is_persisted_before_it_blocks(tmp_path): + gates = _manifest() + report = evaluate_phase( + gates, + "terminal", + EvidenceContext( + artifacts={ + "candidate_input_mass_totals": {"income": 80.0}, + "reference_input_mass_totals": {"income": 100.0}, + } + ), + ) + payload = gate_phase_report_payload(report, gates=gates) + restored = gate_phase_report_from_payload(payload, gates=gates) + run = GateBatteryRun( + gates, + release_id="replay", + report_path=tmp_path / "gates.json", + release_candidate=False, + ) + run.record_phase(restored) + with pytest.raises(GateBatteryBlockedError): + run.enforce("terminal", mode=BlockingMode.BLOCKS_ARTIFACT) + assert run.report_path.is_file() + assert run.report_payload()["blocked_at_phase"] == "terminal" + assert run.phase_report("terminal").failures == report.failures + + +def test_replay_rejects_changed_policy_and_missing_outcomes(): + gates = _manifest() + report = evaluate_phase(gates, "terminal", EvidenceContext()) + payload = gate_phase_report_payload(report, gates=gates) + changed = replace( + gates, gates=(replace(gates.gates[0], parameters={"relative_tolerance": 1.0}),) + ) + with pytest.raises(ValueError, match="manifest"): + gate_phase_report_from_payload(payload, gates=changed) + payload["outcomes"] = [] + with pytest.raises(ValueError, match="outcomes"): + gate_phase_report_from_payload(payload, gates=gates) diff --git a/packages/microcosm-build/tests/test_stage_evidence.py b/packages/microcosm-build/tests/test_stage_evidence.py new file mode 100644 index 000000000..7832b0a4e --- /dev/null +++ b/packages/microcosm-build/tests/test_stage_evidence.py @@ -0,0 +1,53 @@ +"""Stored evidence is independent of the transform instance that produced it.""" + +import json +from types import SimpleNamespace + +import pytest + +from microcosm.build.stage_evidence import ( + decode_stage_evidence, + encode_stage_evidence, + snapshot_stage_evidence, +) + + +def test_snapshot_preserves_checkpoint_replay_and_fit_weight_records(): + class Transform: + sampling = {"fraction": 0.1, "seed": 42} + fit_weight_records = (SimpleNamespace(fit_name="wealth", weight_kind="design"),) + + def checkpoint_metadata(self): + return {"evidence": {"rows": 4}, "replay_payload": {"target": [1, 2]}} + + payload = encode_stage_evidence(snapshot_stage_evidence("wealth", Transform())) + recovered = decode_stage_evidence(payload, stage="wealth") + assert recovered["evidence"] == {"rows": 4} + assert recovered["checkpoint_metadata"]["replay_payload"] == {"target": [1, 2]} + assert recovered["fit_weight_records"] == [ + {"fit_name": "wealth", "weight_kind": "design"} + ] + assert recovered["sampling"] == {"fraction": 0.1, "seed": 42} + + +def test_unreadable_fit_records_remain_an_explicit_failed_audit_input(): + class Transform: + @property + def fit_weight_records(self): + raise RuntimeError("fit did not expose its records") + + payload = snapshot_stage_evidence("wealth", Transform()) + assert payload["fit_weight_records"] == [] + assert payload["fit_weight_records_status"] == "unreadable" + + +def test_stage_evidence_refuses_wrong_identity_and_nonportable_data(): + payload = encode_stage_evidence(snapshot_stage_evidence("one", object())) + with pytest.raises(ValueError, match="stage identity"): + decode_stage_evidence(payload, stage="two") + mutated = json.loads(payload) + mutated["schema_version"] = 2 + with pytest.raises(ValueError, match="schema"): + decode_stage_evidence(json.dumps(mutated).encode(), stage="one") + with pytest.raises(ValueError): + encode_stage_evidence({"value": float("nan")}) diff --git a/packages/microcosm-calibrate/README.md b/packages/microcosm-calibrate/README.md index 233f24847..39b1c4cc7 100644 --- a/packages/microcosm-calibrate/README.md +++ b/packages/microcosm-calibrate/README.md @@ -94,6 +94,25 @@ calibrated_frame = result.frame # CALIBRATED weights result.fraction_within_10pct # representation quality ``` +## Portable graph inputs and results + +`microcosm.calibrate.target_selection.select_targets` selects declared +geography levels from a `TargetRegistry`, keeping all targets by default. +Country adapters normalize each target's geography metadata. The receipt +records ordered target IDs and periods, included and excluded rows, sources +and selection reasons. It does not replace support checks or validation gates. + +`microcosm.calibrate.artifacts` encodes ordered CSR problems, solutions and +completed calibration results as versioned bytes without pickles. Problems +bind target rows and periods, entity IDs, original typed weights and caller +metadata. Solutions verify their problem digest and ordered entity axis. +Completed results retain loss trajectories, loss weights and scales, solver +options and optional gate probabilities. Decoding uses the public result +rebuild operation without running optimization. Compiled contribution rows +stay sparse until the solver consumes each row; source measure closures are +never serialized. Grouped-constraint results require a separate protocol and +are explicitly refused by this codec. + ## Why a shard `microcosm-calibrate` pulls **torch** and sparse/L0 solvers; an analyst doing diff --git a/packages/microcosm-calibrate/src/microcosm/calibrate/artifacts.py b/packages/microcosm-calibrate/src/microcosm/calibrate/artifacts.py new file mode 100644 index 000000000..b7cd2783b --- /dev/null +++ b/packages/microcosm-calibrate/src/microcosm/calibrate/artifacts.py @@ -0,0 +1,566 @@ +"""Portable ordered calibration problems and solutions, without pickles. + +The authoritative numerical input is CSR plus aligned target and entity axes. +Country adapters supply declarative row metadata and identity bindings. Python +measure closures are not serialized: ``to_target_set`` can reconstruct exact +compiled contributions, checking the consuming Frame's ordered entity IDs. +""" + +from __future__ import annotations + +import hashlib +import json +from collections.abc import Mapping, Sequence +from dataclasses import dataclass, replace +from io import BytesIO +from numbers import Integral +from zipfile import ZIP_DEFLATED, ZipFile, ZipInfo + +import numpy as np +from scipy import sparse + +from microcosm.frame import Frame, WeightKind, Weights +from microcosm.graph import ArtifactType +from microcosm.graph.canonical import canonical_json + +from .matrix import CalibrationProblem, SkippedTarget +from .target import Target, TargetSet + +PROBLEM_TYPE = ArtifactType("microcosm.calibrate.ordered-problem", 1) +SOLUTION_TYPE = ArtifactType("microcosm.calibrate.ordered-solution", 1) +RESULT_TYPE = ArtifactType("microcosm.calibrate.calibration-result", 1) + + +def _ids(values: Sequence[int | str]) -> tuple[int | str, ...]: + if isinstance(values, str | bytes): + raise ValueError("Entity axis must be a sequence of IDs.") + result = [] + for value in values: + if isinstance(value, Integral) and not isinstance(value, bool): + result.append(int(value)) + elif isinstance(value, str) and value: + result.append(value) + else: + raise ValueError("Entity IDs must be integers or nonempty strings.") + if not result or len(set(result)) != len(result): + raise ValueError("Entity IDs must be nonempty and unique.") + return tuple(result) + + +def _sha(value: object) -> str: + if ( + not isinstance(value, str) + or len(value) != 64 + or any(c not in "0123456789abcdef" for c in value) + ): + raise ValueError("A problem binding must be a lowercase SHA-256 digest.") + return value + + +def _freeze(array: np.ndarray) -> np.ndarray: + return np.frombuffer(array.tobytes(), dtype=array.dtype).reshape(array.shape) + + +def _pack(metadata: Mapping, arrays: Mapping[str, np.ndarray]) -> bytes: + output = BytesIO() + with ZipFile(output, "w", compression=ZIP_DEFLATED) as archive: + members = { + **arrays, + "metadata": np.frombuffer(canonical_json(metadata), dtype=np.uint8), + } + for name, array in sorted(members.items()): + stream = BytesIO() + np.lib.format.write_array(stream, np.asarray(array), allow_pickle=False) + info = ZipInfo(name + ".npy", date_time=(1980, 1, 1, 0, 0, 0)) + info.compress_type = ZIP_DEFLATED + archive.writestr(info, stream.getvalue()) + return output.getvalue() + + +def _unique(pairs): + result = {} + for key, value in pairs: + if key in result: + raise ValueError("Duplicate calibration metadata key.") + result[key] = value + return result + + +def _unpack(payload: bytes, *, schema: str, members: set[str]) -> tuple[dict, dict]: + if type(payload) is not bytes: + raise TypeError("Calibration artifacts require immutable bytes.") + try: + with np.load(BytesIO(payload), allow_pickle=False) as archive: + if len(archive.files) != len(set(archive.files)) or set( + archive.files + ) != members | {"metadata"}: + raise ValueError("Calibration artifact members differ from its schema.") + raw = archive["metadata"] + if raw.dtype != np.uint8 or raw.ndim != 1: + raise ValueError("Calibration artifact metadata must be UTF-8 bytes.") + metadata = json.loads(raw.tobytes(), object_pairs_hook=_unique) + if ( + canonical_json(metadata) != raw.tobytes() + or metadata.get("schema") != schema + ): + raise ValueError( + "Calibration artifact schema/canonical metadata differs." + ) + arrays = {key: _freeze(archive[key]) for key in members} + return metadata, arrays + except (OSError, KeyError, TypeError, UnicodeError) as error: + raise ValueError("Malformed calibration artifact.") from error + + +def _target(target: Target) -> dict[str, object]: + return { + "name": target.name, + "entity": target.entity, + "value": target.value, + "period": target.period, + "source": target.source, + "tolerance": target.tolerance, + "metadata": dict(target.metadata), + "measure": target.measure if isinstance(target.measure, str) else None, + "filter": target.filter if isinstance(target.filter, str) else None, + "compiled_measure": callable(target.measure), + "compiled_filter": callable(target.filter), + } + + +def _read_target(raw: Mapping) -> Target: + expected = { + "name", + "entity", + "value", + "period", + "source", + "tolerance", + "metadata", + "measure", + "filter", + "compiled_measure", + "compiled_filter", + } + if not isinstance(raw, Mapping) or set(raw) != expected: + raise ValueError("Malformed calibration target descriptor.") + if ( + type(raw["compiled_measure"]) is not bool + or type(raw["compiled_filter"]) is not bool + ): + raise ValueError("Malformed compiled target flags.") + if raw["compiled_measure"] != (raw["measure"] is None): + raise ValueError("Target measure descriptor is inconsistent.") + if raw["compiled_filter"] and raw["filter"] is not None: + raise ValueError("Target filter descriptor is inconsistent.") + return Target( + **{ + key: raw[key] + for key in ( + "name", + "entity", + "value", + "period", + "source", + "tolerance", + "metadata", + ) + }, + measure=raw["measure"] or "__compiled_contribution__", + filter=raw["filter"], + ) + + +@dataclass(frozen=True) +class _CompiledRow: + entity: str + entity_ids: tuple[int | str, ...] + values: sparse.csr_array + + def __call__(self, frame: Frame) -> np.ndarray: + column = frame.schema.entity_id_column(self.entity) + if tuple(frame.table(self.entity)[column]) != self.entity_ids: + raise ValueError( + "Compiled contribution requires its exact ordered entity axis." + ) + # Keep target definitions sparse; compilation materializes one row + # at a time rather than retaining a targets × households dense array. + return self.values.toarray().ravel() + + +@dataclass(frozen=True) +class OrderedProblem: + """An identified numerical problem with explicit ordered entity IDs.""" + + problem: CalibrationProblem + entity_ids: tuple[int | str, ...] + target_metadata: tuple[Mapping, ...] + bindings: Mapping + sha256: str + + def to_target_set(self) -> TargetSet: + """Use the compiled rows in the public solver without original closures. + + Contributions already include original filters and entity aggregation. + No raw measure is recomputed; consuming a different row axis refuses. + """ + return TargetSet( + [ + replace( + target, + entity=self.problem.weight_entity, + filter=None, + measure=_CompiledRow( + self.problem.weight_entity, + self.entity_ids, + self.problem.matrix[index : index + 1], + ), + ) + for index, target in enumerate(self.problem.targets) + ] + ) + + +def encode_problem( + problem: CalibrationProblem, + *, + entity_ids: Sequence[int | str], + target_metadata: Sequence[Mapping] | None = None, + bindings: Mapping | None = None, +) -> bytes: + """Encode numerical rows, axes, skipped facts and declarative bindings.""" + ids = _ids(entity_ids) + if not isinstance(problem, CalibrationProblem): + raise TypeError("encode_problem requires CalibrationProblem.") + if len(ids) != problem.n_weights: + raise ValueError("Problem entity axis differs from the weight columns.") + matrix = sparse.csr_array(problem.matrix, dtype=np.float64, copy=True) + rows = ( + tuple({} for _ in problem.targets) + if target_metadata is None + else tuple(target_metadata) + ) + if len(rows) != problem.n_targets or any( + not isinstance(row, Mapping) for row in rows + ): + raise ValueError("Target metadata must match the ordered target axis.") + metadata = { + "schema": PROBLEM_TYPE.name + ".v1", + "shape": list(matrix.shape), + "entity_ids": ids, + "weight_entity": problem.weight_entity, + "weight_kind": problem.initial_weights.kind.value, + "names": problem.names, + "targets": [_target(target) for target in problem.targets], + "skipped": [ + {"target": _target(item.target), "reason": item.reason} + for item in problem.skipped + ], + "target_metadata": rows, + "bindings": {} if bindings is None else bindings, + } + payload = _pack( + metadata, + { + "data": matrix.data.astype(" OrderedProblem: + """Decode and validate a portable problem, preserving CSR row order.""" + metadata, arrays = _unpack( + payload, + schema=PROBLEM_TYPE.name + ".v1", + members={"data", "indices", "indptr", "targets", "weights"}, + ) + if set(metadata) != { + "schema", + "shape", + "entity_ids", + "weight_entity", + "weight_kind", + "names", + "targets", + "skipped", + "target_metadata", + "bindings", + }: + raise ValueError("Problem metadata fields differ.") + ids = _ids(metadata["entity_ids"]) + shape = metadata["shape"] + if ( + not isinstance(shape, list) + or len(shape) != 2 + or any(type(n) is not int or n < 0 for n in shape) + ): + raise ValueError("Malformed problem shape.") + if shape[1] != len(ids): + raise ValueError("Problem entity axis differs from the weight columns.") + for key, values in arrays.items(): + dtype = np.dtype("= shape[1]).any() + ): + raise ValueError("Malformed CSR index arrays.") + matrix = sparse.csr_array((data, indices, indptr), shape=tuple(shape)) + targets = tuple(_read_target(raw) for raw in metadata["targets"]) + if len({target.key for target in targets}) != len(targets) or tuple( + metadata["names"] + ) != tuple(target.row_name for target in targets): + raise ValueError("Problem target IDs/names must form a unique ordered axis.") + if arrays["targets"].tolist() != [target.value for target in targets]: + raise ValueError("Problem target values disagree with their descriptors.") + skipped = [] + for item in metadata["skipped"]: + if ( + set(item) != {"target", "reason"} + or not isinstance(item["reason"], str) + or not item["reason"] + ): + raise ValueError("Malformed skipped-target evidence.") + skipped.append(SkippedTarget(_read_target(item["target"]), item["reason"])) + rows = metadata["target_metadata"] + if ( + len(rows) != len(targets) + or any(not isinstance(row, dict) for row in rows) + or not isinstance(metadata["bindings"], dict) + ): + raise ValueError("Problem row metadata/bindings differ from their contract.") + problem = CalibrationProblem( + matrix, + arrays["targets"], + tuple(metadata["names"]), + Weights(arrays["weights"], WeightKind(metadata["weight_kind"])), + metadata["weight_entity"], + targets, + tuple(skipped), + ) + return OrderedProblem( + problem, + ids, + tuple(rows), + metadata["bindings"], + hashlib.sha256(payload).hexdigest(), + ) + + +@dataclass(frozen=True) +class OrderedSolution: + """Aligned calibrated weights bound to an exact problem artifact.""" + + weights: np.ndarray + entity_ids: tuple[int | str, ...] + problem_sha256: str + diagnostics: Mapping + sha256: str + + +def encode_solution( + weights: Sequence[float] | np.ndarray, + *, + entity_ids: Sequence[int | str], + problem_sha256: str, + diagnostics: Mapping | None = None, +) -> bytes: + """Encode a solution once; later population installation need not solve.""" + ids = _ids(entity_ids) + payload = _pack( + { + "schema": SOLUTION_TYPE.name + ".v1", + "entity_ids": ids, + "problem_sha256": _sha(problem_sha256), + "diagnostics": {} if diagnostics is None else diagnostics, + }, + {"weights": np.asarray(weights, dtype=" OrderedSolution: + """Validate solution axes, finite nonnegative weights and optional binding.""" + metadata, arrays = _unpack( + payload, schema=SOLUTION_TYPE.name + ".v1", members={"weights"} + ) + if set(metadata) != { + "schema", + "entity_ids", + "problem_sha256", + "diagnostics", + } or not isinstance(metadata["diagnostics"], dict): + raise ValueError("Solution metadata fields differ.") + ids = _ids(metadata["entity_ids"]) + digest = _sha(metadata["problem_sha256"]) + weights = arrays["weights"] + if ( + weights.dtype != np.dtype(" bytes: + """Persist the complete numerical state needed by dense/search continuation. + + A result is bound to a separately encoded ordered problem. Optimizer state + is not claimed: continuation rebuilds completed results, never mid-epoch + training. Grouped solves require a separate constraints-aware protocol. + """ + ids = _ids(entity_ids) + if len(ids) != len(result.weights): + raise ValueError("Calibration result differs from the ordered entity axis.") + if ( + "grouped_upper_bounds" in result.options + or "grouped_preserve_zeros" in result.options + ): + raise ValueError( + "Grouped calibration results require their original constraints." + ) + probabilities = result.gate_open_probabilities + return _pack( + { + "schema": RESULT_TYPE.name + ".v1", + "entity_ids": ids, + "problem_sha256": _sha(problem_sha256), + "l0_lambda": float(result.l0_lambda), + "n_nonzero": int(result.n_nonzero), + "target_loss_cap": float(result.target_loss_cap), + "closing_loss": float(result.final_loss), + "options": dict(result.options), + "has_probabilities": probabilities is not None, + }, + { + "weights": np.asarray(result.weights, dtype=" dict[str, object]: + """Return a fresh receipt; caller mutations cannot alter its identity.""" + return json.loads(self._receipt_json) + + @property + def sha256(self) -> str: + return hashlib.sha256(self.to_bytes()).hexdigest() + + def to_bytes(self) -> bytes: + """Return the canonical selection receipt bytes.""" + return self._receipt_json.encode("utf-8") + + +def _level(spec: TargetSpec) -> str: + return spec.metadata.get("geography_level", "") + + +def select_targets( + registry: TargetRegistry, + *, + geography_levels: Sequence[str] | None = None, + geography_resolver: Callable[[TargetSpec], str] | None = None, +) -> TargetSelection: + """Keep all targets by default, or explicitly select normalized levels. + + Ordering and ``(name, period)`` identity are retained. Unknown level names, + unclassified facts and an empty selected problem refuse. A resolver must + validate country aliases/ambiguities; it is never serialized as a callback. + The receipt stores its resolved result for every original fact instead. + """ + if not isinstance(registry, TargetRegistry): + raise TypeError("Target selection requires TargetRegistry.") + levels = None + if geography_levels is not None: + if isinstance(geography_levels, str | bytes): + raise ValueError("geography_levels must be a nonempty sequence of levels.") + levels = tuple(geography_levels) + if not levels or any( + not isinstance(level, str) or not level or level != level.strip() + for level in levels + ): + raise ValueError("geography_levels must contain nonempty literal names.") + if len(set(levels)) != len(levels): + raise ValueError("geography_levels must not repeat levels.") + levels = tuple(sorted(levels)) + resolve = _level if geography_resolver is None else geography_resolver + resolved = [] + for spec in registry.specs: + level = resolve(spec) + if not isinstance(level, str) or not level or level != level.strip(): + raise ValueError( + f"Target {spec.key!r} lacks normalized geography metadata." + ) + resolved.append(level) + if levels is not None and (unknown := set(levels) - set(resolved)): + raise ValueError( + f"Target selector contains unknown geography levels {sorted(unknown)}." + ) + by_key = { + spec.key: level for spec, level in zip(registry.specs, resolved, strict=True) + } + selected = registry.select( + predicate=lambda spec: levels is None or by_key[spec.key] in levels + ) + if not selected.specs: + raise ValueError("Target selection requires a nonempty selected problem.") + included, excluded = [], [] + for spec, level in zip(registry.specs, resolved, strict=True): + row = { + "name": spec.name, + "period": spec.period, + "geography_level": level, + "family": spec.family, + "source": spec.source, + } + if levels is None or level in levels: + included.append( + { + **row, + "reason": "all_geographies" + if levels is None + else "geography_selected", + } + ) + else: + excluded.append({**row, "reason": "geography_not_selected"}) + receipt = { + "schema": "microcosm.calibrate.target-selection.v1", + "source_registry_version": registry.version, + "selected_registry_version": selected.version, + "selector": {"geography_levels": levels, "explicit": levels is not None}, + "included": included, + "excluded": excluded, + } + return TargetSelection( + selected, + json.dumps(receipt, sort_keys=True, separators=(",", ":"), allow_nan=False), + ) diff --git a/packages/microcosm-calibrate/tests/test_ordered_artifacts.py b/packages/microcosm-calibrate/tests/test_ordered_artifacts.py new file mode 100644 index 000000000..285940abf --- /dev/null +++ b/packages/microcosm-calibrate/tests/test_ordered_artifacts.py @@ -0,0 +1,140 @@ +"""Portable matrix/solution artifacts retain axes and reject corrupt inputs.""" + +import numpy as np +import pandas as pd +import pytest +from scipy import sparse + +from microcosm.calibrate import CalibrationProblem, Target, build_constraint_matrix +from microcosm.calibrate.artifacts import ( + decode_calibration_result, + decode_problem, + decode_solution, + encode_calibration_result, + encode_problem, + encode_solution, +) +from microcosm.frame import EntitySchema, Frame, WeightKind, Weights + + +def problem(): + targets = ( + Target("count", "household", lambda f: np.ones(2), value=10, period=2024), + Target("money", "household", "money", value=21, period="2025"), + ) + return CalibrationProblem( + sparse.csr_array([[1.0, 1.0], [2.0, 5.0]]), + np.array([10.0, 21.0]), + tuple(t.row_name for t in targets), + Weights(np.array([2.0, 3.0]), WeightKind.IMPORTANCE), + "household", + targets, + ) + + +def test_problem_roundtrip_is_deterministic_and_recompiles_bound_rows(): + payload = encode_problem( + problem(), entity_ids=(10, 20), bindings={"selector": "all"} + ) + assert ( + encode_problem(problem(), entity_ids=(10, 20), bindings={"selector": "all"}) + == payload + ) + restored = decode_problem(payload) + assert restored.entity_ids == (10, 20) + assert restored.problem.names == problem().names + assert restored.bindings == {"selector": "all"} + np.testing.assert_array_equal( + restored.problem.matrix.toarray(), problem().matrix.toarray() + ) + frame = Frame( + { + "person": pd.DataFrame( + {"person_id": [1, 2], "person_household_id": [10, 20]} + ), + "household": pd.DataFrame({"household_id": [10, 20]}), + }, + EntitySchema(group_entities=("household",)), + {"household": problem().initial_weights}, + pd.Series(["a", "a"]), + ) + recompiled = build_constraint_matrix( + frame, restored.to_target_set(), weight_entity="household" + ) + np.testing.assert_array_equal( + recompiled.matrix.toarray(), problem().matrix.toarray() + ) + + +def test_artifact_axes_and_solution_binding_are_strict(): + with pytest.raises(ValueError, match="unique"): + encode_problem(problem(), entity_ids=(10, 10)) + with pytest.raises(ValueError, match="axis"): + encode_problem(problem(), entity_ids=(10,)) + bound = decode_problem(encode_problem(problem(), entity_ids=(10, 20))) + payload = encode_solution( + [4.0, 6.0], + entity_ids=(10, 20), + problem_sha256=bound.sha256, + diagnostics={"converged": True}, + ) + result = decode_solution(payload, problem_sha256=bound.sha256, entity_ids=(10, 20)) + np.testing.assert_array_equal(result.weights, [4.0, 6.0]) + with pytest.raises(ValueError, match="axis"): + decode_solution(payload, entity_ids=(20, 10)) + with pytest.raises(ValueError, match="problem"): + decode_solution(payload, problem_sha256="0" * 64) + with pytest.raises(ValueError): + decode_problem(payload) + + +def test_problem_target_definitions_do_not_densify_the_whole_sparse_system(monkeypatch): + bound = decode_problem(encode_problem(problem(), entity_ids=(10, 20))) + # Country matrices can have millions of households and thousands of rows. + # Creating definitions must not materialize any dense contribution row. + monkeypatch.setattr( + sparse.csr_array, "toarray", lambda *a, **kw: pytest.fail("eager densification") + ) + assert len(bound.to_target_set().targets) == 2 + + +def test_complete_result_rebuild_preserves_dense_and_search_state(monkeypatch): + import microcosm.calibrate.solve as solve + from microcosm.calibrate import calibrate + + frame = Frame( + { + "person": pd.DataFrame( + {"person_id": [1, 2], "person_household_id": [10, 20]} + ), + "household": pd.DataFrame({"household_id": [10, 20]}), + }, + EntitySchema(group_entities=("household",)), + {"household": problem().initial_weights}, + pd.Series(["a", "a"]), + ) + bound = decode_problem(encode_problem(problem(), entity_ids=(10, 20))) + for penalty in (0.0, 0.01): + result = calibrate( + frame, + bound.to_target_set(), + epochs=3, + seed=17, + mass="free", + l0_lambda=penalty, + ) + payload = encode_calibration_result( + result, entity_ids=(10, 20), problem_sha256=bound.sha256 + ) + with monkeypatch.context() as context: + context.setattr( + solve, "_optimize", lambda *a, **kw: pytest.fail("replay optimized") + ) + restored = decode_calibration_result(payload, frame=frame, problem=bound) + np.testing.assert_array_equal(restored.weights, result.weights) + np.testing.assert_array_equal(restored.loss_trajectory, result.loss_trajectory) + assert restored.options == result.options + if penalty: + np.testing.assert_array_equal( + restored.gate_open_probabilities, result.gate_open_probabilities + ) diff --git a/packages/microcosm-calibrate/tests/test_target_selection.py b/packages/microcosm-calibrate/tests/test_target_selection.py new file mode 100644 index 000000000..74c40fddd --- /dev/null +++ b/packages/microcosm-calibrate/tests/test_target_selection.py @@ -0,0 +1,72 @@ +"""Target scope is explicit and preserves ordered fact identities.""" + +import pytest + +from microcosm.calibrate import TargetRegistry, TargetSpec +from microcosm.calibrate.target_selection import select_targets + + +def registry(): + return TargetRegistry( + [ + TargetSpec( + "population", + "person", + 20, + "one", + period=2024, + source="census", + metadata={"geography_level": "country"}, + ), + TargetSpec( + "population", + "person", + 22, + "one", + period=2025, + source="census", + metadata={"geography_level": "region"}, + ), + TargetSpec( + "local", + "person", + 5, + "one", + period=2025, + source="census", + metadata={"geography_level": "district"}, + ), + ], + country="test", + ) + + +def test_default_all_and_explicit_country_are_distinct(): + all_targets = select_targets(registry()) + assert all_targets.registry.specs == registry().specs + assert all_targets.receipt["selector"] == { + "geography_levels": None, + "explicit": False, + } + selected = select_targets(registry(), geography_levels=("country",)) + assert [spec.key for spec in selected.registry.specs] == [("population", 2024)] + assert [row["period"] for row in selected.receipt["excluded"]] == [2025, 2025] + assert all( + row["reason"] == "geography_not_selected" + for row in selected.receipt["excluded"] + ) + assert selected.receipt["source_registry_version"] == registry().version + + +def test_missing_unknown_empty_and_resolved_geographies(): + with pytest.raises(ValueError, match="unknown"): + select_targets(registry(), geography_levels=("missing",)) + with pytest.raises(ValueError, match="nonempty"): + select_targets(registry(), geography_levels=()) + missing = TargetRegistry( + [TargetSpec("x", "person", 1, "one", source="test")], country="test" + ) + with pytest.raises(ValueError, match="geography"): + select_targets(missing) + resolved = select_targets(missing, geography_resolver=lambda spec: "country") + assert resolved.receipt["included"][0]["geography_level"] == "country" diff --git a/packages/microcosm-graph/README.md b/packages/microcosm-graph/README.md index cb0d46142..cef87cfd8 100644 --- a/packages/microcosm-graph/README.md +++ b/packages/microcosm-graph/README.md @@ -32,3 +32,15 @@ Module map: The shard depends on `microcosm-frame` only. Kernels that wrap fit, calibrate, or a rules engine live in those shards and register here. + +Same-kind normalization uses `WeightUpdate`, with a reason, a conserved or +declared mass policy, and `weight_update_receipt(ordered_entity_ids)` in the +kernel receipt. The runtime checks the exact row axis and preserves the +original design-weight ancestry. `WeightTransition` continues to require a +forward change of kind. + +Kernels receive immutable Frame metadata, legacy mass records and the +original order of their projected columns through `KernelContext`. Complete +legacy evidence requires an explicit structural checkpoint or artifact +dependency. The executor's population mass ledger remains the authority for +graph mass checks. diff --git a/packages/microcosm-graph/src/microcosm/graph/__init__.py b/packages/microcosm-graph/src/microcosm/graph/__init__.py index bbcbc5ea7..219fb1912 100644 --- a/packages/microcosm-graph/src/microcosm/graph/__init__.py +++ b/packages/microcosm-graph/src/microcosm/graph/__init__.py @@ -30,6 +30,7 @@ SourceRef, StructuralDelta, WeightTransition, + WeightUpdate, compile_graph, ) from .errors import ( @@ -57,12 +58,15 @@ ) from .keys import platform_fingerprint from .randomness import keyed_uniform +from .weight_update import weight_update_receipt __all__ = [ "ArtifactInput", "ArtifactOutput", "ArtifactType", "ArtifactValue", + "WeightUpdate", + "weight_update_receipt", "keyed_uniform", "platform_fingerprint", "DESCRIPTIVE_FIELDS", diff --git a/packages/microcosm-graph/src/microcosm/graph/decl.py b/packages/microcosm-graph/src/microcosm/graph/decl.py index b6134c210..9cf5ec4f7 100644 --- a/packages/microcosm-graph/src/microcosm/graph/decl.py +++ b/packages/microcosm-graph/src/microcosm/graph/decl.py @@ -70,6 +70,7 @@ "SourceRef", "StructuralDelta", "WeightTransition", + "WeightUpdate", "compile_graph", ] @@ -317,6 +318,35 @@ def __post_init__(self) -> None: ) +@dataclass(frozen=True) +class WeightUpdate: + """An explicit same-kind numerical update, never a kind transition. + + The kernel must bind its ordered entity axis using ``weight_update_receipt``. + A nonempty reason is normative. Mass must be conserved or declared; an + unconstrained free-mass update is intentionally not part of this contract. + Original design-weight ancestry is carried unchanged by the executor. + """ + + entity: str + kind: str + reason: str + mass: str = "declared" + + def __post_init__(self) -> None: + _name("WeightUpdate.entity", self.entity) + _nonempty("WeightUpdate.reason", self.reason) + if self.kind not in WEIGHT_KINDS: + raise GraphError("WeightUpdate.kind must name an existing weight kind.") + if self.mass not in {"conserve", "declared"}: + raise GraphError("WeightUpdate requires conserved or declared mass.") + + @property + def to_kind(self) -> str: + """The unchanged kind, for shared structural-weight accounting.""" + return self.kind + + @dataclass(frozen=True) class Node: """One unit of computation and cell ownership. @@ -363,7 +393,7 @@ class Node: structural: StructuralDelta = StructuralDelta.NONE base: str | None = None sources: tuple[str, ...] = () - weights: WeightTransition | None = None + weights: WeightTransition | WeightUpdate | None = None mass: str = "conserve" description: str = "" citation: str = "" @@ -451,6 +481,8 @@ def __post_init__(self) -> None: raise GraphError( f"Node {self.id!r}: a REWEIGHT node declares its WeightTransition." ) + if not isinstance(self.weights, WeightTransition | WeightUpdate): + raise GraphError("REWEIGHT requires WeightTransition or WeightUpdate.") if self.mass != self.weights.mass: raise GraphError( f"Node {self.id!r}: mass policy {self.mass!r} disagrees with its " diff --git a/packages/microcosm-graph/src/microcosm/graph/executor.py b/packages/microcosm-graph/src/microcosm/graph/executor.py index acb0a77a9..0a003c85a 100644 --- a/packages/microcosm-graph/src/microcosm/graph/executor.py +++ b/packages/microcosm-graph/src/microcosm/graph/executor.py @@ -7,6 +7,7 @@ import socket import time from collections.abc import Callable, Mapping +from dataclasses import asdict from datetime import UTC, datetime from pathlib import Path from types import MappingProxyType @@ -76,6 +77,7 @@ StoreCorrupt, StoreMiss, StoreUnavailable, + _encode_frame_metadata, ) __all__ = ["NodeRejected", "NodeRejectedError", "run_graph"] @@ -409,6 +411,9 @@ def _context_digest(context: KernelContext) -> bytes: digest.update(weights.kind.value.encode("ascii") + b"\0") _update_array(digest, weights.values) _update_series(digest, context.strata) + digest.update(canonical_json(_encode_frame_metadata(context.frame_metadata))) + digest.update(canonical_json([asdict(record) for record in context.frame_mass_log])) + digest.update(canonical_json(dict(context.frame_column_order))) for name, value in sorted(context.artifacts.items()): digest.update( canonical_json( @@ -605,6 +610,16 @@ def _project_context( tolerances=tolerances, numerics=numerics, artifacts={} if artifacts is None else artifacts, + frame_metadata=frame.metadata, + frame_mass_log=frame.mass_log, + frame_column_order={ + entity: tuple( + column + for column in frame.table(entity).columns + if column in table.columns + ) + for entity, table in tables.items() + }, ) diff --git a/packages/microcosm-graph/src/microcosm/graph/kernel.py b/packages/microcosm-graph/src/microcosm/graph/kernel.py index 0846a3eaa..e6dcc63d3 100644 --- a/packages/microcosm-graph/src/microcosm/graph/kernel.py +++ b/packages/microcosm-graph/src/microcosm/graph/kernel.py @@ -57,7 +57,8 @@ import numpy as np import pandas as pd -from microcosm.frame import Frame, Weights +from microcosm.frame import Frame, MassChangeRecord, Weights +from microcosm.frame.bundle import _freeze_metadata from .decl import ArtifactType, Node, Param, StructuralDelta @@ -321,6 +322,12 @@ class KernelContext: artifacts: Immutable typed bytes for declared artifact aliases only. Consumers validate versioned payloads before using them; nominal types do not themselves verify arbitrary serialized data. + frame_column_order: Original order restricted to projected columns; + undeclared column names are never exposed. + frame_metadata: Immutable metadata from the population version. + frame_mass_log: Immutable legacy Frame mass records. Consumers requiring + a completed stage log must run after its structural boundary or read + explicit predecessor evidence; incidental node order is not authority. sources: Source name to a content-verified path, for declared sources only. tolerances: ``(entity, column)`` of each declared input column to @@ -343,8 +350,32 @@ class KernelContext: tolerances: Mapping[tuple[str, str], Tolerance | None] = field(default_factory=dict) numerics: Mapping[tuple[str, str], NumericScope] = field(default_factory=dict) artifacts: Mapping[str, ArtifactValue] = field(default_factory=dict) + frame_metadata: Mapping[str, object] = field(default_factory=dict) + frame_mass_log: tuple[MassChangeRecord, ...] = () + frame_column_order: Mapping[str, tuple[str, ...]] = field(default_factory=dict) def __post_init__(self) -> None: + object.__setattr__( + self, "frame_metadata", _freeze_metadata(self.frame_metadata) + ) + column_order = dict(self.frame_column_order) + for entity, columns in column_order.items(): + if ( + entity not in self.tables + or not isinstance(columns, tuple) + or len(columns) != len(set(columns)) + or set(columns) != set(self.tables[entity].columns) + ): + raise TypeError( + "KernelContext.frame_column_order must order exactly the projected columns." + ) + object.__setattr__(self, "frame_column_order", MappingProxyType(column_order)) + if not isinstance(self.frame_mass_log, tuple) or any( + not isinstance(record, MassChangeRecord) for record in self.frame_mass_log + ): + raise TypeError( + "KernelContext.frame_mass_log must contain immutable mass records." + ) values = dict(self.artifacts) if any( not isinstance(name, str) diff --git a/packages/microcosm-graph/src/microcosm/graph/population.py b/packages/microcosm-graph/src/microcosm/graph/population.py index 1a52baf3f..60e991de6 100644 --- a/packages/microcosm-graph/src/microcosm/graph/population.py +++ b/packages/microcosm-graph/src/microcosm/graph/population.py @@ -24,8 +24,10 @@ Owned, Ownership, StructuralDelta, + WeightUpdate, ) from .kernel import KernelResult +from .weight_update import weight_update_receipt __all__ = [ "MassRecord", @@ -1800,6 +1802,10 @@ def _patch_columns( continue else: incumbent = _empty_column(len(table), owned.dtype, owned_mask) + # Frame sampling can preserve non-contiguous pandas row labels. + # The temporary column is positional; label-alignment against a + # fresh RangeIndex would insert nulls and promote dense dtypes. + incumbent.index = table.index table[owned.column] = incumbent positions = pd.Series( @@ -1914,6 +1920,16 @@ def _apply_weight_transition( ) old = population.frame.weights_for(transition.entity) declared_kind = WeightKind(transition.to_kind) + if isinstance(transition, WeightUpdate): + if declared_kind is not old.kind or result.weights.kind is not old.kind: + raise PopulationError("WeightUpdate must preserve the same kind.") + id_column = population.frame.schema.entity_id_column(transition.entity) + axis = population.frame.table(transition.entity)[id_column] + if result.receipt.get("weight_update") != weight_update_receipt(axis): + raise PopulationError("WeightUpdate receipt differs from the ordered axis.") + if len(result.weights) != len(axis): + raise PopulationError("WeightUpdate values must match the ordered axis.") + return _replace_weights(frame, transition.entity, result.weights) # Forward moves only, matching the Frame kernel's own rule: design may go # straight to calibrated (the UK pipeline calibrates design weights), and # nothing moves backwards or stays in place. diff --git a/packages/microcosm-graph/src/microcosm/graph/serialize.py b/packages/microcosm-graph/src/microcosm/graph/serialize.py index 85c71b4de..9120ed2d2 100644 --- a/packages/microcosm-graph/src/microcosm/graph/serialize.py +++ b/packages/microcosm-graph/src/microcosm/graph/serialize.py @@ -19,6 +19,7 @@ SourceRef, StructuralDelta, WeightTransition, + WeightUpdate, ) __all__ = ["graph_from_json", "graph_to_json"] @@ -160,6 +161,11 @@ def _node_payload(node: Node) -> dict[str, object]: "entity": node.weights.entity, "to_kind": node.weights.to_kind, "mass": node.weights.mass, + **( + {"update": True, "reason": node.weights.reason} + if isinstance(node.weights, WeightUpdate) + else {} + ), } ), "mass": node.mass, @@ -292,10 +298,22 @@ def _owned_from_payload(value: object, label: str) -> Owned: ) -def _weights_from_payload(value: object, label: str) -> WeightTransition | None: +def _weights_from_payload( + value: object, label: str +) -> WeightTransition | WeightUpdate | None: if value is None: return None payload = _mapping(value, label) + if "update" in payload: + _exact_fields(payload, {"entity", "to_kind", "mass", "update", "reason"}, label) + if payload["update"] is not True: + raise ValueError("WeightUpdate requires update=true.") + return WeightUpdate( + entity=_string(payload["entity"], f"{label}.entity"), + kind=_string(payload["to_kind"], f"{label}.to_kind"), + reason=_string(payload["reason"], f"{label}.reason"), + mass=_string(payload["mass"], f"{label}.mass"), + ) _exact_fields(payload, {"entity", "to_kind", "mass"}, label) return WeightTransition( entity=_string(payload["entity"], f"{label}.entity"), diff --git a/packages/microcosm-graph/src/microcosm/graph/weight_update.py b/packages/microcosm-graph/src/microcosm/graph/weight_update.py new file mode 100644 index 000000000..ff22b0c8d --- /dev/null +++ b/packages/microcosm-graph/src/microcosm/graph/weight_update.py @@ -0,0 +1,29 @@ +"""Ordered entity-axis evidence for explicit same-kind weight updates.""" + +from collections.abc import Sequence +from numbers import Integral + +from .canonical import canonical_json, sha256_domain + + +def weight_update_receipt(entity_ids: Sequence[int | str]) -> dict[str, object]: + """Bind positional replacement weights to unique, ordered integer/string IDs. + + Place this result under ``KernelResult.receipt['weight_update']``. The + executor recomputes it against the actual incumbent axis, including replay. + """ + ids = [] + for value in entity_ids: + if isinstance(value, Integral) and not isinstance(value, bool): + ids.append(int(value)) + elif isinstance(value, str) and value: + ids.append(value) + else: + raise ValueError("Weight update IDs must be integers or nonempty strings.") + if len(set(ids)) != len(ids): + raise ValueError("Weight update IDs must be unique.") + return { + "schema": "microcosm.graph.weight-update-axis.v1", + "count": len(ids), + "entity_ids_sha256": sha256_domain("weight-update-axis", canonical_json(ids)), + } diff --git a/packages/microcosm-graph/tests/fixtures/parity/kernels/calibrate/pins.json b/packages/microcosm-graph/tests/fixtures/parity/kernels/calibrate/pins.json index d985a22d8..00fb906a5 100644 --- a/packages/microcosm-graph/tests/fixtures/parity/kernels/calibrate/pins.json +++ b/packages/microcosm-graph/tests/fixtures/parity/kernels/calibrate/pins.json @@ -1 +1 @@ -{"dependencies":{"numpy":"2.4.6","pandas":"3.0.3","scipy":"1.17.1","torch":"2.12.0"},"implementation_hash":"5f92148c5e438baaf7d89fb00cc9c5a690fe1b629bddf89c7afd439064ccfc40","kernel":"calibrate.adam@1","node":"calibrate","node_key":"d7191dc141b702b3a23049bf48ccf370d72cd2822864cb8551453f6dfcd5b055","numeric":"bitwise","platform":"arm64/darwin/py3.14","platforms":{"arm64/darwin/py3.14":{"direct":"direct.csv","node_key":"d7191dc141b702b3a23049bf48ccf370d72cd2822864cb8551453f6dfcd5b055"}},"seed":0} +{"dependencies":{"numpy":"2.4.6","pandas":"3.0.3","scipy":"1.17.1","torch":"2.12.0"},"implementation_hash":"8f4583ffdd254cbe9bbbf30de5dc85e67dbd9c2d8e5394bf715cbdf5e7cd5137","kernel":"calibrate.adam@1","node":"calibrate","node_key":"61e05c896df532fee0e530b9a9584136b526611f394dfc3634d4afdf614d71ac","numeric":"bitwise","platform":"arm64/darwin/py3.14","platforms":{"arm64/darwin/py3.14":{"direct":"direct.csv","node_key":"61e05c896df532fee0e530b9a9584136b526611f394dfc3634d4afdf614d71ac"}},"seed":0} diff --git a/packages/microcosm-graph/tests/fixtures/parity/kernels/fit.qrf/pins.json b/packages/microcosm-graph/tests/fixtures/parity/kernels/fit.qrf/pins.json index 1bfbebe22..3b61d6fc1 100644 --- a/packages/microcosm-graph/tests/fixtures/parity/kernels/fit.qrf/pins.json +++ b/packages/microcosm-graph/tests/fixtures/parity/kernels/fit.qrf/pins.json @@ -1 +1 @@ -{"dependencies":{"numpy":"2.4.6","pandas":"3.0.3","quantile-forest":"1.4.2","scikit-learn":"1.8.0"},"implementation_hash":"02db8f5c849d876be20a95152b5302a5cacc0a7c77c58d8b436a3a00f57b4c92","kernel":"fit.qrf@1","node":"fit_qrf","node_key":"8878352db3439439f412f26c8762ff5871b94fe8e5fc8d3469dd1c45d7ef7da4","numeric":"platform_bitwise","platform":"arm64/darwin/py3.14","platforms":{"arm64/darwin/py3.14":{"direct":"direct.csv","node_key":"8878352db3439439f412f26c8762ff5871b94fe8e5fc8d3469dd1c45d7ef7da4"},"x86_64/linux/py3.13":{"direct":"platforms/x86_64-linux-py3_13/direct.csv","node_key":"f6984280e1ef0f156bd24345f7d3677d573c650ac706f9a3f949627c0d76d2d4"},"x86_64/linux/py3.14":{"direct":"platforms/x86_64-linux-py3_14/direct.csv","node_key":"9e80ee3ac5c30f99725c4dd932535983dabde7913c653e79819b1df3a289720c"}},"seed":947} +{"dependencies":{"numpy":"2.4.6","pandas":"3.0.3","quantile-forest":"1.4.2","scikit-learn":"1.8.0"},"implementation_hash":"d1f8b1929e6452aa507b0d9c64ba42a59851dc31c71021303c25f060e34c075b","kernel":"fit.qrf@1","node":"fit_qrf","node_key":"35af6a452bd32ca39d313d78255236df01877d684c2d847b1bd5a6016a68e237","numeric":"platform_bitwise","platform":"arm64/darwin/py3.14","platforms":{"arm64/darwin/py3.14":{"direct":"direct.csv","node_key":"35af6a452bd32ca39d313d78255236df01877d684c2d847b1bd5a6016a68e237"},"x86_64/linux/py3.13":{"direct":"platforms/x86_64-linux-py3_13/direct.csv","node_key":"6c43edcb3edf2bdef20d915b1be16503d37583c678aced78f1442aca1139e1ae"},"x86_64/linux/py3.14":{"direct":"platforms/x86_64-linux-py3_14/direct.csv","node_key":"ecb6b20c02b0bc442c4baf3fd7def6d6aec06e6b9567910294f945d66c43bbf8"}},"seed":947} diff --git a/packages/microcosm-graph/tests/fixtures/parity/kernels/simulate/pins.json b/packages/microcosm-graph/tests/fixtures/parity/kernels/simulate/pins.json index 15f5d879b..57ec48a0f 100644 --- a/packages/microcosm-graph/tests/fixtures/parity/kernels/simulate/pins.json +++ b/packages/microcosm-graph/tests/fixtures/parity/kernels/simulate/pins.json @@ -1 +1 @@ -{"dependencies":{},"implementation_hash":"eed54eaf27b53faf446068aa833ec16ab398840faf870445dbdcd6a4604c566c","kernel":"simulate.rules@1","node":"simulate","node_key":"a643736e821e841fa74996f0768d61110fe4d6e590d8db4286fd241f7d75b123","numeric":"bitwise","platform":"arm64/darwin/py3.14","platforms":{"arm64/darwin/py3.14":{"direct":"direct.csv","node_key":"a643736e821e841fa74996f0768d61110fe4d6e590d8db4286fd241f7d75b123"}},"seed":null} +{"dependencies":{},"implementation_hash":"9a6937b103dfdc5b1e8d0f69d4a27792400b1ee3c0bbea85f63e4d8c02458384","kernel":"simulate.rules@1","node":"simulate","node_key":"85c622d86b4fc358e94472187144c627c5ff3ff6a44614c48a38c7b92e112cd9","numeric":"bitwise","platform":"arm64/darwin/py3.14","platforms":{"arm64/darwin/py3.14":{"direct":"direct.csv","node_key":"85c622d86b4fc358e94472187144c627c5ff3ff6a44614c48a38c7b92e112cd9"}},"seed":null} diff --git a/packages/microcosm-graph/tests/test_acceptance_b_ownership.py b/packages/microcosm-graph/tests/test_acceptance_b_ownership.py index 2ce41a7f9..6ab5cbce1 100644 --- a/packages/microcosm-graph/tests/test_acceptance_b_ownership.py +++ b/packages/microcosm-graph/tests/test_acceptance_b_ownership.py @@ -118,6 +118,10 @@ def test_b2_executor_enforces_ownership(tmp_path: Path) -> None: "sources", "tolerances", # amendment 13: declared tolerances of the inputs' owners "numerics", # amendment 17: per-coordinate numeric class, bound, platform + "artifacts", # declared typed dependencies, no undeclared Frame access + "frame_metadata", # immutable evidence, bound by the population identity + "frame_mass_log", + "frame_column_order", # ordering only, restricted to projected columns } graph = toy.small_graph( diff --git a/packages/microcosm-graph/tests/test_graph_kernel_contract.py b/packages/microcosm-graph/tests/test_graph_kernel_contract.py index 057bec79f..8d6f6fc79 100644 --- a/packages/microcosm-graph/tests/test_graph_kernel_contract.py +++ b/packages/microcosm-graph/tests/test_graph_kernel_contract.py @@ -191,9 +191,25 @@ def test_numeric_scope_validates_class_tolerance_and_platform() -> None: def test_context_numerics_default_empty_and_carry_scopes() -> None: - """Amendment 17: ``numerics`` defaults empty and rides at the end of the context.""" + """New evidence fields retain the established positional context prefix.""" fields = [f.name for f in dataclasses.fields(KernelContext)] - assert fields[-2:] == ["tolerances", "numerics"] + assert fields[:9] == [ + "node", + "tables", + "weights", + "strata", + "params", + "rng", + "sources", + "tolerances", + "numerics", + ] + assert fields[9:] == [ + "artifacts", + "frame_metadata", + "frame_mass_log", + "frame_column_order", + ] scope = NumericScope( numeric=Numeric.PLATFORM_BITWISE, platform="arm64/darwin/py3.13" ) diff --git a/packages/microcosm-graph/tests/test_graph_population.py b/packages/microcosm-graph/tests/test_graph_population.py index a96562d88..825151156 100644 --- a/packages/microcosm-graph/tests/test_graph_population.py +++ b/packages/microcosm-graph/tests/test_graph_population.py @@ -1434,3 +1434,96 @@ def test_reweight_can_synthesize_frame_but_must_not_change_ids() -> None: ) with pytest.raises(PopulationError, match="changed 'person' ids"): patch(population, node, KernelResult(frame=reordered)) + + +@pytest.mark.parametrize( + "dtype,values", + [ + ("int64", [7, 8, 9, 10]), + ("bool", [True, False, True, False]), + ("string", ["a", "b", "c", "d"]), + ("float64", [0.1, 0.2, 0.3, 0.4]), + ], +) +def test_new_columns_preserve_noncontiguous_frame_labels(dtype, values): + original = _frame() + person = original.table("person").copy() + person.index = pd.Index([11, 21, 44, 105]) + strata = original.strata.copy() + strata.index = person.index + frame = _replace_person_table(original, person, strata) + population = Population.from_frame(frame, "source") + node = Node("new_column", "test@1", outputs=(Owned("person", "new", dtype),)) + incoming = pd.Series(list(reversed(values)), index=[4, 3, 2, 1], dtype=dtype) + updated = patch( + population, node, KernelResult(columns={("person", "new"): incoming}) + ) + actual = updated.frame.table("person") + assert actual.index.equals(person.index) + pd.testing.assert_series_equal( + actual["new"], + pd.Series(values, index=person.index, dtype=dtype_for_token(dtype), name="new"), + ) + pd.testing.assert_frame_equal(actual.drop(columns="new"), person) + assert "new" not in population.frame.table("person") + + +def test_new_masked_nullable_column_preserves_unowned_rows_with_noncontiguous_labels(): + original = _frame() + person = original.table("person").copy() + person.index = pd.Index([11, 21, 44, 105]) + strata = original.strata.copy() + strata.index = person.index + population = Population.from_frame( + _replace_person_table(original, person, strata), "source" + ) + node = Node( + "new_masked", + "test@1", + inputs=(Slice("person", ("owned",)),), + outputs=(Owned("person", "new", "Int64", rows="owned"),), + ) + result = patch( + population, + node, + KernelResult( + columns={("person", "new"): pd.Series([8, 10], index=[2, 4], dtype="Int64")} + ), + ) + pd.testing.assert_series_equal( + result.frame.table("person")["new"], + pd.Series([pd.NA, 8, pd.NA, 10], index=person.index, dtype="Int64", name="new"), + ) + + +@pytest.mark.parametrize( + "dtype,values", + [ + ("float32", [8.0, 10.0]), + ("float64", [8.0, 10.0]), + ("Int64", [8, 10]), + ("string", ["x", "y"]), + ], +) +def test_masked_constant_fillers_have_same_values_for_sampled_row_labels(dtype, values): + original = _frame() + person = original.table("person").copy() + person.index = pd.Index([11, 21, 44, 105]) + strata = original.strata.copy() + strata.index = person.index + selected = _replace_person_table(original, person, strata) + node = Node( + "masked_new", + "test@1", + inputs=(Slice("person", ("owned",)),), + outputs=(Owned("person", "new", dtype, rows="owned"),), + ) + incoming = pd.Series(values, index=[2, 4], dtype=dtype_for_token(dtype)) + result = KernelResult(columns={("person", "new"): incoming}) + ordinary = patch(Population.from_frame(original, "source"), node, result) + sampled = patch(Population.from_frame(selected, "source"), node, result) + pd.testing.assert_series_equal( + ordinary.frame.table("person")["new"], + sampled.frame.table("person")["new"].reset_index(drop=True), + ) + assert sampled.frame.table("person")["new"].iloc[[0, 2]].isna().all() diff --git a/packages/microcosm-graph/tests/test_weight_update.py b/packages/microcosm-graph/tests/test_weight_update.py new file mode 100644 index 000000000..de08370e9 --- /dev/null +++ b/packages/microcosm-graph/tests/test_weight_update.py @@ -0,0 +1,212 @@ +"""Same-kind normalization remains explicit, aligned and replayable.""" + +import hashlib + +import numpy as np +import pandas as pd +import pytest + +from microcosm.frame import EntitySchema, Frame, MassChangeRecord, WeightKind, Weights +from microcosm.graph import ( + Capabilities, + ContentStore, + Determinism, + Graph, + KernelBase, + KernelRegistry, + KernelResult, + Node, + Numeric, + Owned, + Slice, + SourceRef, + StructuralDelta, + WeightTransition, + WeightUpdate, + compile_graph, + run_graph, + weight_update_receipt, +) +from microcosm.graph.population import Population, PopulationError, patch +from microcosm.graph.serialize import graph_from_json, graph_to_json + + +def frame(): + return Frame( + { + "person": pd.DataFrame( + {"person_id": [1, 2], "person_household_id": [10, 20]} + ), + "household": pd.DataFrame({"household_id": [10, 20], "value": [2.0, 3.0]}), + }, + EntitySchema(group_entities=("household",)), + {"household": Weights(np.array([2.0, 3.0]), WeightKind.DESIGN)}, + pd.Series(["a", "b"], name="stratum"), + metadata={"time_period": "2024", "source": {"years": [2023, 2024]}}, + mass_log=(MassChangeRecord("household", 5.0, 5.0, 1.0, "source"),), + ) + + +def update_node(): + return Node( + "normalize", + "normalize@1", + base="source", + structural=StructuralDelta.REWEIGHT, + inputs=(Slice("household", ("value",)),), + mass="declared", + weights=WeightUpdate("household", "design", "restore sampled mass"), + ) + + +def update_result(ids=(10, 20), after=10.0): + return KernelResult( + weights=Weights(np.array([4.0, 6.0]), WeightKind.DESIGN), + receipt={ + "weight_update": weight_update_receipt(ids), + "mass": { + "policy": "declared", + "before": 5.0, + "after": after, + "stratum_before": {"a": 2.0, "b": 3.0}, + "stratum_after": {"a": 4.0, "b": 6.0}, + }, + }, + ) + + +def test_normalization_keeps_design_ancestry_and_refuses_misaligned_ids(): + original = Population.from_frame(frame(), "source") + updated = patch(original, update_node(), update_result()) + np.testing.assert_array_equal( + updated.frame.weights_for("household").values, [4.0, 6.0] + ) + np.testing.assert_array_equal(updated.design_weights["household"], [2.0, 3.0]) + assert updated.mass_ledger[-1].after_total == 10.0 + assert updated.frame.weights_for("household").kind is WeightKind.DESIGN + with pytest.raises(PopulationError, match="axis"): + patch(original, update_node(), update_result((20, 10))) + with pytest.raises(PopulationError, match="computed value"): + patch(original, update_node(), update_result(after=11.0)) + + +def test_forward_transition_and_same_kind_contracts_are_distinct(): + original = Population.from_frame(frame(), "source") + ordinary = Node( + "bad", + "bad@1", + base="source", + structural=StructuralDelta.REWEIGHT, + weights=WeightTransition("household", "design", "free"), + mass="free", + ) + with pytest.raises(PopulationError, match="forward"): + patch(original, ordinary, update_result()) + mismatch = Node( + "bad", + "bad@1", + base="source", + structural=StructuralDelta.REWEIGHT, + weights=WeightUpdate("household", "importance", "normalize"), + mass="declared", + ) + with pytest.raises(PopulationError, match="same kind"): + patch(original, mismatch, update_result()) + + +class Source(KernelBase): + ref = "source@1" + capabilities = Capabilities( + Determinism.DETERMINISTIC, + numeric=Numeric.BITWISE, + structural=StructuralDelta.CREATE, + ) + + def implementation_hash(self): + return hashlib.sha256(self.ref.encode()).hexdigest() + + def run(self, context): + return KernelResult(frame=frame()) + + +class Normalize(Source): + ref = "normalize@1" + capabilities = Capabilities( + Determinism.DETERMINISTIC, + numeric=Numeric.BITWISE, + structural=StructuralDelta.REWEIGHT, + ) + + def run(self, context): + assert context.frame_metadata["time_period"] == "2024" + assert context.frame_column_order["household"] == ("household_id", "value") + with pytest.raises(TypeError): + context.frame_column_order["household"] = () + assert context.frame_mass_log[0].reason == "source" + with pytest.raises(TypeError): + context.frame_metadata["source"]["years"] = () + return update_result(tuple(context.tables["household"]["household_id"])) + + +def test_context_exposes_original_order_of_only_the_declared_columns(): + from microcosm.graph.executor import _project_context + + original = frame() + household = original.table("household") + household.insert(0, "secret", [100, 200]) + household.insert(0, "another", [3, 4]) + context = _project_context( + Node( + "reader", + "reader@1", + population="source", + inputs=(Slice("household", ("value", "another")),), + ), + Population.from_frame(original, "source"), + key="0" * 64, + sources={}, + tolerances={}, + numerics={}, + ) + assert context.frame_column_order["household"] == ( + "another", + "household_id", + "value", + ) + assert "secret" not in context.frame_column_order["household"] + + +def test_normalization_roundtrip_and_required_replay(tmp_path): + source = tmp_path / "source.csv" + source.write_text("value\n1\n") + graph = Graph( + "test", + (SourceRef("source", "raw-bytes-v1"),), + ( + Node( + "source", + "source@1", + structural=StructuralDelta.CREATE, + sources=("source",), + outputs=(Owned("household", "value", "float64"),), + ), + update_node(), + ), + ) + assert graph_from_json(graph_to_json(graph)) == graph + registry = KernelRegistry() + registry.register(Source()) + registry.register(Normalize()) + store = ContentStore(tmp_path / "store") + first = run_graph( + compile_graph(graph), kernels=registry, store=store, sources={"source": source} + ) + second = run_graph( + compile_graph(graph), + kernels=registry, + store=store, + sources={"source": source}, + resume="require", + ) + assert all(receipt.hit for receipt in second.receipts.values()) + assert first.mass_ledger("normalize") == second.mass_ledger("normalize") From df34ec76865d24af654b297d902a0d121270bb33 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mar=C3=ADa=20Juaristi?= <127882282+juaristi22@users.noreply.github.com> Date: Thu, 10 Sep 2026 18:17:22 +0200 Subject: [PATCH 2/8] Register the canonical UK full build as graph stages --- .github/workflows/test.yml | 2 +- changelog.d/uk-full-build.changed.md | 7 + docs/uk-dataset-size-plan-355.md | 2 +- docs/uk-dense-release-assembly-runbook-762.md | 147 +- docs/uk-full-build-graph.md | 78 + docs/uk-national-calibration-runbook-623.md | 95 +- ...k-national-release-assembly-runbook-806.md | 161 +- packages/microcosm-build/README.md | 33 +- packages/microcosm-build/pyproject.toml | 3 +- .../microcosm/build/spec_engine/resolver.py | 2 +- .../microcosm/build/uk/country_package.json | 5 - .../build/uk/efrs_parity_known_gaps.json | 4 +- .../build/uk/hmrc_income_source_stages.json | 735 ---- .../uk/release_input_coverage_manifest.json | 68 +- .../src/microcosm/build/uk/source_stages.json | 737 ---- .../src/microcosm/build/uk/spec/bundle.yaml | 7 +- .../src/microcosm/build/uk/spec/catalogs.yaml | 8 +- .../src/microcosm/build/uk/spec/sources.yaml | 704 +--- .../src/microcosm/build/uk/spec/spine.yaml | 16 +- .../src/microcosm/build/uk/spec/vintages.yaml | 12 +- .../microcosm/build/uk_runtime/__init__.py | 12 +- .../build/uk_runtime/calibration_run.py | 587 +-- .../build/uk_runtime/country_adapter.py | 134 + .../build/uk_runtime/dataset_size.py | 124 +- .../build/uk_runtime/frs_hmrc_leaves.py | 893 ----- .../build/uk_runtime/frs_hmrc_source.py | 311 ++ .../build/uk_runtime/full_build_cli.py | 798 +++++ .../build/uk_runtime/full_certification.py | 524 +++ .../microcosm/build/uk_runtime/full_gates.py | 339 ++ .../build/uk_runtime/full_measure.py | 283 ++ .../build/uk_runtime/full_problem.py | 205 ++ .../build/uk_runtime/full_targets.py | 151 + .../build/uk_runtime/geography_ladder.py | 99 +- .../src/microcosm/build/uk_runtime/graph.py | 125 +- .../microcosm/build/uk_runtime/graph_build.py | 333 ++ .../build/uk_runtime/graph_calibration.py | 851 +++++ .../build/uk_runtime/graph_evidence.py | 385 ++ .../build/uk_runtime/graph_kernels.py | 61 +- .../build/uk_runtime/graph_population.py | 541 +++ .../build/uk_runtime/graph_targets.py | 617 ++++ .../build/uk_runtime/graph_terminal.py | 1113 ++++++ .../build/uk_runtime/hmrc_source_contract.py | 272 +- .../incumbent_surface_evaluation.py | 35 + .../build/uk_runtime/local_rowwise.py | 307 +- .../build/uk_runtime/national_calibration.py | 326 +- .../build/uk_runtime/release_certification.py | 103 +- .../uk_runtime/release_input_coverage.py | 95 +- .../build/uk_runtime/rowwise_dataset.py | 150 +- .../build/uk_runtime/size_checkpoint.py | 22 +- .../build/uk_runtime/source_runtime.py | 19 +- .../microcosm/build/uk_runtime/spi_income.py | 2 +- .../microcosm/build/uk_runtime/spi_spine.py | 6 +- .../microcosm/build/uk_runtime/spine_build.py | 1548 ++++++++ .../tests/test_country_spec.py | 2 - .../tests/test_gate_battery_contract_pins.py | 16 +- .../tests/test_spec_engine_country_bundles.py | 8 +- .../tests/test_uk_battery_bindings.py | 18 +- .../tests/test_uk_calibration_run.py | 914 +---- .../tests/test_uk_calibration_seam_driver.py | 300 +- .../tests/test_uk_cgt_observation_period.py | 97 +- .../tests/test_uk_cgt_source_manifest.py | 35 +- .../tests/test_uk_country_adapter.py | 52 + .../tests/test_uk_frs_hmrc_leaves.py | 507 +-- .../tests/test_uk_frs_spine.py | 27 +- .../tests/test_uk_full_build_cli.py | 402 +++ .../tests/test_uk_full_build_preparation.py | 110 + .../tests/test_uk_full_calibration_graph.py | 487 +++ .../tests/test_uk_full_certification.py | 288 ++ .../tests/test_uk_full_gates.py | 317 ++ .../tests/test_uk_full_measure.py | 272 ++ .../tests/test_uk_full_population_graph.py | 160 + .../tests/test_uk_full_solve_scope.py | 82 + .../tests/test_uk_full_target_graph.py | 447 +++ .../tests/test_uk_full_targets.py | 147 + .../microcosm-build/tests/test_uk_graph.py | 96 +- .../tests/test_uk_graph_evidence.py | 234 ++ .../tests/test_uk_graph_terminal.py | 724 ++++ .../test_uk_hmrc_income_source_manifest.py | 773 +--- .../tests/test_uk_hmrc_replay_artifacts.py | 27 +- .../test_uk_incumbent_surface_evaluation.py | 34 + .../tests/test_uk_ladder_rowwise_clone.py | 587 +-- .../tests/test_uk_national_calibration.py | 600 +--- .../tests/test_uk_national_sampling.py | 25 - .../tests/test_uk_release_certification.py | 41 - .../tests/test_uk_release_input_coverage.py | 65 +- ...test_uk_release_input_coverage_manifest.py | 82 +- .../tests/test_uk_rowwise_build_driver.py | 716 +--- .../tests/test_uk_rowwise_candidate.py | 2239 +----------- .../tests/test_uk_rowwise_dry_run.py | 455 +-- .../tests/test_uk_source_runtime.py | 17 +- .../tests/test_uk_source_stages.py | 141 +- .../tests/test_uk_spi_income.py | 2 +- .../tests/test_uk_spine_acceptance_receipt.py | 12 +- .../microcosm-build/tests/test_us_plan.py | 1 - .../fixtures/parity/uk_spine/uk_spine.json | 2 +- tools/build_uk_frs_spine.py | 1491 +------- tools/build_uk_full.py | 6 + ...uild_uk_release_input_coverage_manifest.py | 246 +- tools/build_uk_rowwise_candidate.py | 3190 +---------------- tools/build_uk_rowwise_dataset.py | 1903 +--------- tools/calibrate_uk_national_dataset.py | 321 +- tools/certify_uk_release_cut.py | 322 +- tools/evaluate_uk_incumbent_surface.py | 21 +- tools/graph_uk_spine_fixture.py | 4 +- uv.lock | 2 + 105 files changed, 13928 insertions(+), 19336 deletions(-) create mode 100644 changelog.d/uk-full-build.changed.md create mode 100644 docs/uk-full-build-graph.md delete mode 100644 packages/microcosm-build/src/microcosm/build/uk/hmrc_income_source_stages.json create mode 100644 packages/microcosm-build/src/microcosm/build/uk_runtime/country_adapter.py delete mode 100644 packages/microcosm-build/src/microcosm/build/uk_runtime/frs_hmrc_leaves.py create mode 100644 packages/microcosm-build/src/microcosm/build/uk_runtime/frs_hmrc_source.py create mode 100644 packages/microcosm-build/src/microcosm/build/uk_runtime/full_build_cli.py create mode 100644 packages/microcosm-build/src/microcosm/build/uk_runtime/full_certification.py create mode 100644 packages/microcosm-build/src/microcosm/build/uk_runtime/full_gates.py create mode 100644 packages/microcosm-build/src/microcosm/build/uk_runtime/full_measure.py create mode 100644 packages/microcosm-build/src/microcosm/build/uk_runtime/full_problem.py create mode 100644 packages/microcosm-build/src/microcosm/build/uk_runtime/full_targets.py create mode 100644 packages/microcosm-build/src/microcosm/build/uk_runtime/graph_build.py create mode 100644 packages/microcosm-build/src/microcosm/build/uk_runtime/graph_calibration.py create mode 100644 packages/microcosm-build/src/microcosm/build/uk_runtime/graph_evidence.py create mode 100644 packages/microcosm-build/src/microcosm/build/uk_runtime/graph_population.py create mode 100644 packages/microcosm-build/src/microcosm/build/uk_runtime/graph_targets.py create mode 100644 packages/microcosm-build/src/microcosm/build/uk_runtime/graph_terminal.py create mode 100644 packages/microcosm-build/src/microcosm/build/uk_runtime/spine_build.py create mode 100644 packages/microcosm-build/tests/test_uk_country_adapter.py create mode 100644 packages/microcosm-build/tests/test_uk_full_build_cli.py create mode 100644 packages/microcosm-build/tests/test_uk_full_build_preparation.py create mode 100644 packages/microcosm-build/tests/test_uk_full_calibration_graph.py create mode 100644 packages/microcosm-build/tests/test_uk_full_certification.py create mode 100644 packages/microcosm-build/tests/test_uk_full_gates.py create mode 100644 packages/microcosm-build/tests/test_uk_full_measure.py create mode 100644 packages/microcosm-build/tests/test_uk_full_population_graph.py create mode 100644 packages/microcosm-build/tests/test_uk_full_solve_scope.py create mode 100644 packages/microcosm-build/tests/test_uk_full_target_graph.py create mode 100644 packages/microcosm-build/tests/test_uk_full_targets.py create mode 100644 packages/microcosm-build/tests/test_uk_graph_evidence.py create mode 100644 packages/microcosm-build/tests/test_uk_graph_terminal.py create mode 100644 tools/build_uk_full.py diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 9bcc4bc65..a9883d361 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -3,8 +3,8 @@ name: Tests on: push: branches: [main] + # Stacked pull requests need the same acceptance checks as main-bound work. pull_request: - branches: [main] # Cancel outdated PR runs; give each main push its own independent group. concurrency: diff --git a/changelog.d/uk-full-build.changed.md b/changelog.d/uk-full-build.changed.md new file mode 100644 index 000000000..a9a882287 --- /dev/null +++ b/changelog.d/uk-full-build.changed.md @@ -0,0 +1,7 @@ +Consolidate UK source construction, geographic cloning, target selection, +calibration, exact-count sizing and acceptance evidence into one executable +graph. Calibrate all applicable geographies by default, with country-only as +an explicit filter and independent pool/output sizing. Retire obsolete H5 +migration stages and independent calibration entrypoints. Restore bound stage, +gate and diagnostic artifacts on replay, validate exported bytes, and report +unsigned certification readiness from the full graph. diff --git a/docs/uk-dataset-size-plan-355.md b/docs/uk-dataset-size-plan-355.md index 4b2ce4b21..86863d825 100644 --- a/docs/uk-dataset-size-plan-355.md +++ b/docs/uk-dataset-size-plan-355.md @@ -76,7 +76,7 @@ Use the inputs and environment from the existing Pass the same pinned source arguments to the existing driver and add: ```bash -uv run python tools/build_uk_rowwise_candidate.py \ +uv run python tools/build_uk_full.py \ --input-h5 "$UK_SPINE_H5" --input-sha256 "$UK_SPINE_SHA256" \ --ladder "$UK_LADDER_NPZ" --ladder-sha256 "$UK_LADDER_SHA256" \ --ledger-facts "$UK_LEDGER_FACTS" \ diff --git a/docs/uk-dense-release-assembly-runbook-762.md b/docs/uk-dense-release-assembly-runbook-762.md index f923829f3..cdcfe0cfd 100644 --- a/docs/uk-dense-release-assembly-runbook-762.md +++ b/docs/uk-dense-release-assembly-runbook-762.md @@ -1,146 +1,5 @@ -# UK dense line release assembly runbook (#762) +# UK dense release assembly -The dense line `microcosm-uk-2024-25-dense` is the spine cloned K=15 times -through the OA geography ladder and calibrated to the national and local -target surfaces in one solve (`tools/build_uk_rowwise_candidate.py`). It ships -on the **inspect lane only**: a constant release id, an immutable per-cut tag, -`dataset_role: non_default_local_area`, an empty `default_datasets` map, and -`--no-latest` at publication, so it can never displace the default artifact. -It is registered as `("uk", 2025, "dense")` in the private repo -`policyengine/populace-uk-private`. Publication is a separate human step. +Use the [UK full-build graph runbook](uk-full-build-graph.md). The dense reference is the output of the same full build when `--dataset-households` is omitted; exact-count builds use its informed search, selection and refit branch. -The R16/R17 release verdicts recorded in the historical receipts used the -previous gate policy. They do not satisfy the current contract: four quality -gates now block release, and the incumbent-surface evaluation is mandatory. -No historical run was re-signed or recalibrated by the PR #870 review fixes. - -## Prerequisites - -- The four pins the run stood on (`spine`, `ladder`, `facts`, `manifest`) and - the signing key in `MICROCOSM_UK_TERMINAL_GATE_SIGNING_KEY` (base64, 32 bytes). -- The Ledger consumer artifact, the spine H5 with its sidecar, the OA ladder. -- The incumbent extraction (`tools/extract_uk_local_incumbent_surface.py`) for - the head-to-head score. - -## 1. Pre-flight the environment - -```bash -uv run --no-sync python tools/preflight_uk_local_release_candidate.py --env \ - --pins --spine-h5 --ladder build/uk/uk_oa_ladder_2021.npz \ - --ledger-facts -``` - -Fails closed and by name on a missing or malformed key, a missing pin, a -digest mismatch, or doctrine constants that are not the ruled ones. - -## 2. Run the release candidate - -```bash -uv run --no-sync python tools/build_uk_rowwise_candidate.py --release-candidate \ - --input-h5 --input-sha256 \ - --ladder build/uk/uk_oa_ladder_2021.npz --ladder-sha256 \ - --ledger-facts --ledger-facts-sha256 \ - --ledger-manifest-sha256 --seed 42 \ - --logbook-prev-row-digest --out -``` - -`--release-candidate` pins the doctrine (bound 10, `grain_equal`, K=15, 1500 -epochs), resolves the engine in a single block, and runs the rotated holdout. -Expect about 3.5 hours and 10 GB at K=15. - -## 3. Pre-flight the finished run, then score it - -```bash -uv run --no-sync python tools/preflight_uk_local_release_candidate.py --candidate-dir -uv run --no-sync python tools/score_uk_local_candidate.py ... --output-json /score_vs_incumbent.json -``` - -The pre-flight checks the manifest and the signed gate report for everything -the contract will demand: release posture attested, shippable, every -release-blocking gate passed, single-block engine, the doctrine values, the -A15/A17 uprating, the measure exclusions and their windows, the holdout, the -Logbook row, the artifact digest. - -Measure the full pinned incumbent surface before assembly: - -```bash -uv run --no-sync python tools/evaluate_uk_incumbent_surface.py \ - --candidate-h5 /microcosm_uk_2025_local.h5 \ - --candidate-manifest /rowwise_candidate_manifest.json \ - --ledger-facts --ledger-facts-sha256 \ - --ledger-manifest-sha256 --engine-blocks 1 \ - --incumbent-manifest /incumbent_local_surface_manifest.json \ - --incumbent-metrics-csv /household_metrics.csv \ - --incumbent-weights-csv /wide_weights.csv \ - --out-json /incumbent_surface_evaluation.json \ - --out-md /incumbent_surface_evaluation.md -``` - -Use the actual metrics and weights filenames from the extraction manifest. -The evaluator remains diagnostic: missing optional incumbent inputs and poor -fit produce a failed assessment, not permission to publish. Only one engine -block is accepted. Assembly requires the complete authenticated evaluation, -including finite candidate measurements on every national and local row and -finite realized incumbent estimates on every local row. National comparisons -use the pinned incumbent targets; they do not claim realized incumbent fit. -Signed deferrals stay in this evaluation. A missing or unmeasurable row blocks -release until its measurement is supplied. - -The same existing absolute quality limits apply to this surface: every row -within 25%, and at least half each family's rows within 25% when the family -has at least five rows. The within-10% family share remains diagnostic. -The candidate's fitted score uses uniform rows on its active local surface. -Its holdout uses the separately recorded weighting rule over held local -grains. Their shared cap does not make the losses directly comparable; no -ranking of fitted versus holdout losses is reported. - -## 4. Assemble the release directory - -```bash -uv run --no-sync python tools/assemble_uk_dense_release_dir.py \ - --candidate-dir --spine-h5 \ - --incumbent-manifest /incumbent_local_surface_manifest.json \ - --out-dir releases -``` - -Assembly verifies the hash join (every manifest output against its bytes, the -spine against its pin, the gate report against the Logbook build id), re-runs -the candidate pre-flight, mints the cut tag -`microcosm-uk-2024-25-dense--` from the run's attempt -id, clones the H5 beside itself as `microcosm_uk_2025_dense.h5`, stages -`build_manifest.json`, `release_manifest.json`, `calibration_diagnostics.json`, -`gate_summary.json`, `uk_source_coverage.json`, the signed `uk_local_gates.json`, -`score_vs_incumbent.json`, `incumbent_surface_evaluation.json`, the original -`rowwise_candidate_manifest.json`, `source_calibration_diagnostics.json`, -`incumbent_manifest.json`, and `sha256sums.txt`, validates the directory with -`microcosm.data.contract.validate_release_dir`, and only then renames it into -`releases/microcosm-uk-2024-25-dense/`. Re-assembling requires removing the -previous directory first. The JSON summary prints the publication command. - -Assembly and every later directory validation require measured clean code -(`code.git_dirty` exactly `false`) and full measure-exclusion provenance. -Approval and expiry dates must be valid ISO dates and in force on the current -validation date; expiry-day validation is allowed, the following day is not. -The upload path invokes this validator again before uploading bytes. Separate support -and binding adjudications keep their own policies. The gate thresholds and -existing approvals have not been widened or renewed. - -## 5. Publish for inspection (human step) - -Run the printed command. Its shape is: - -```bash -uv run python -m microcosm.data.publish_cli releases/microcosm-uk-2024-25-dense \ - --repo-id policyengine/populace-uk-private --artifact-root \ - --no-latest --tag-name microcosm-uk-2024-25-dense-- -``` - -`--no-latest` is mandatory and enforced: publication refuses to move -`latest.json` for a non-default role. The artifact is reachable by its tag and -by the registry key `("uk", 2025, "dense")` only. - -## Promotion is a separate change - -Making the dense line (or a sparse successor via the L0 penalty, #762 I10) a -default dataset is a registry and contract change with its own review; nothing -in this runbook promotes anything. +The standard target scope includes all applicable geographies. Strict release-candidate checks retain the maintained source pins, solve doctrine, full sample, single engine block and applicable holdouts. Exporting a candidate does not authorize publication or establish native or matched-size certification. diff --git a/docs/uk-full-build-graph.md b/docs/uk-full-build-graph.md new file mode 100644 index 000000000..0e861c73c --- /dev/null +++ b/docs/uk-full-build-graph.md @@ -0,0 +1,78 @@ +# UK full-build graph + +The UK has one calibration build. It constructs the canonical FRS spine, samples the pool when requested, expands linked entities into K geographic copies, assigns locations, constructs the selected contribution matrix, calibrates, optionally selects exactly k households and refits, evaluates gates and diagnostics, and packages a checked H5. + +**The default is to calibrate all applicable geographies together.** An omitted selector and `--target-geographies all` have the same target scope. `--target-geographies country` explicitly selects country-level rows in the same graph; regional rows are not country-level rows. No failure, size request or performance setting changes the selector implicitly. + +## Run the build + +From a canonical spine checkpoint with its `.build.json` and `.spine_gates.json` sidecars: + +```bash +uv run --no-sync python tools/build_uk_full.py \ + --input-h5 /data/uk/spine.h5 \ + --ladder /data/uk/ladder.npz \ + --ledger-facts /data/chronicle/uk-artifact \ + --out /data/uk/full-build +``` + +The checkpoint must bind the exact frame content, current spine stage roster and gate-report bytes. Historical candidate H5 files and reviewed-bypass sidecars are not alternate build sources. Chronicle facts and manifest must match the committed feed pins; filtering targets does not relax source validation. + +To include raw spine construction in the same execution, pass `--spine-request /data/uk/spine-request.json`. This file is a JSON array of the raw-source arguments accepted by the maintained spine preparation API: + +```json +[ + "--frs-raw-dir", "/data/frs/2024-25", + "--spi-tab", "/data/spi/put2223uk.tab", + "--hmrc-ods", "/data/hmrc/collated.ods", + "--cgt-ods", "/data/hmrc/cgt.ods", + "--was-tab", "/data/was/household.tab", + "--lcfs-hh-tab", "/data/lcfs/household.tab", + "--lcfs-person-tab", "/data/lcfs/person.tab", + "--etb-tab", "/data/etb/household.tab" +] +``` + +The request declares existing source adapters and lazy transforms. It does not launch the separate spine command first. A spine-only command remains available to materialize an execution checkpoint. + +For smaller outputs, add `--dataset-households 100000` to use the common informed L0 search, exact-count draw and refit. `--n-clones K` sets the number of geographic copies in the pool. K and k are independent, and neither narrows target scope. A country-only run may explicitly request a smaller K, but this is never inferred. + +`--sample-fraction` samples before geographic cloning. Raw-spine sampling in the JSON request instead occurs at the original ingest boundary, before enrichment. An already sampled spine cannot be sampled a second time. The effective sample fraction controls development gate and target-admission policy. + +## Graph owners and shared contracts + +The existing source-stage roster supplies the spine graph. The superseded `frs_hmrc_retained_leaves` and `hmrc_spi_income` executable stages are removed. The active replacements are `frs_hmrc_spine_leaves`, `spi_support_channel` and `hmrc_spi_income_spine`; source extraction helpers remain under source-focused owners. + +| Operation | Graph owner | +| --- | --- | +| Raw FRS and donor preparation, enrichment, support channels | Existing UK spine stage nodes and their declared composite operations | +| Assembled and transferred spine gates | `spine.gates.assembled`, `spine.gates.transferred` | +| Bound checkpoint admission | `uk.full.spine_checkpoint`, when resuming a saved spine | +| Pool sample and mass normalization | `uk.full.sample`, `uk.full.normalize` | +| Linked entity expansion and ancestry | `uk.full.expand`, `uk.full.expand.owned` | +| Location draw, mapping and integrity | `uk.full.locations`, `uk.full.geography_mapping`, `uk.full.geography_gate` | +| Full pinned source/register compilation | `uk.full.target_compilation` | +| Explicit target selection and inclusion/exclusion receipt | `uk.full.target_selection` | +| Engine measures and ordered contribution problem | `uk.full.measures`, `uk.full.problem` | +| Complete original-pool checkpoint | `uk.full.pool` | +| Source/reference preflight and dense reference | `uk.full.gates.preflight`, `uk.full.dense` | +| Optional informed search, exact draw and refit | `uk.full.size_search`, `uk.full.size_draw`, `uk.full.size_refit` | +| Selected population and installed calibrated weights | `uk.full.selected`, `uk.full.calibrated` | +| Rotated local holdout and final gates/diagnostics | `uk.full.holdout`, `uk.full.gates.calibrated` | +| Export contract, H5 readback and package inventory | `uk.full.export.prepare`, `uk.full.export.readback`, `uk.full.package` | + +`operations.json` is generated from the compiled graph, including actual dependencies and artifact owners. It is the execution inventory, rather than a second manually maintained pipeline roster. Composite source/model stages preserve their existing numerical boundary; for example, WAS retains its joint donor/recipient encoding dependency. This registration does not change imputation order, RNG consumption, clone IDs or geography methodology. + +Shared machinery includes graph execution/storage/replay, typed artifacts, explicit same-kind weight updates, target selection receipts, ordered sparse calibration problem/solution/result codecs, exact-count selection, atomic artifact materialization and bundle publication. UK adapters retain source interpretation, entity relationships, geography mappings, measure bindings and gate prescriptions. + +The full target compiler preserves the unreduced band-edge register, reference-period compilations, approved exclusions and frozen-register completeness checks before target selection. A nonempty country-only problem can have zero local rows. Local holdout is then inapplicable, while source, identity, mass and geographic integrity checks remain applicable. Omitted rows are never reported as fitted constraints. + +## Replay and outputs + +The shared store defaults to `/.graph-store`. `--graph-store` can reuse another store. `--resume require` requires completed numerical nodes and evidence to be available; output materialization and byte readback still verify the recreated files. `--resume-size-checkpoint` imports a legacy size-search checkpoint only after validating invocation identity, ordered target/household axes, initial weights and recomputed losses. It skips the saved dense solve and search. New runs persist their intermediates as graph artifacts before drawing. + +The output bundle includes `microcosm_uk_.h5`, selected targets, target diagnostics, area support, rotated holdout, stored source/stage/gate evidence, graph manifests and `build.json`. Physical output bytes are checked against their declared artifacts. Bundle publication writes the completion marker last and rolls back handled failures or interrupts. A process kill or power loss can leave an absent completion marker; a directory without a valid bound marker is not a completed release. + +Structural failures stop export. The maintained local statistical failure policy may still export an unreleasable diagnostic candidate with a nonzero process status. Missing evidence remains explicit. `--release-candidate` applies the maintained strictness and solve settings; it does not publish, sign or authorize a release. Fixture acceptance proves graph behavior, not native data certification. Exact-count promotion additionally requires matched-size comparison evidence. + +The old `build_uk_rowwise_candidate.py` and `calibrate_uk_national_dataset.py` command names temporarily forward to this exact CLI. Both default to all geographies. They contain no separate scientific execution and should be removed after downstream command invocations migrate. diff --git a/docs/uk-national-calibration-runbook-623.md b/docs/uk-national-calibration-runbook-623.md index d7907f1c7..531629ddd 100644 --- a/docs/uk-national-calibration-runbook-623.md +++ b/docs/uk-national-calibration-runbook-623.md @@ -1,94 +1,5 @@ -# UK national calibration runbook (#623) +# UK calibration runbook -This runbook documents the held licensed run for the first Ledger-calibrated UK -national candidate. It is not an instruction to run it in PR CI. +The separate national calibration driver is retired. Use the [UK full-build graph runbook](uk-full-build-graph.md) for current commands and evidence requirements. -## Unblock conditions - -Run only after one of these is true: - -- WS-E spine is complete through E8 (#684) and E10 (#686). -- Maria explicitly names a different base spine for this run. - -The input posture is a non-certified staging candidate at pre-clone grain, -matching the incumbent published national surface. The seam records that -posture itself and marks its artifact `shippable: false`: release certification -is the release-cut producer's job, not calibration's. - -## Command shape - -Calibration runs through `tools/calibrate_uk_national_dataset.py`. That driver -is the only one that builds the measure resolver and applies the committed -measure-exclusion register, and 187 of the activated references bind model -outputs that no frame carries — so it is the only path on which this target -surface materializes. The June builder -(`tools/build_uk_national_dataset.py`) constructs the calibration stage without -either and aborts on the first unmaterializable reference; it also rebuilds SPI -income onto its input, which a spine artifact already carries. - -```bash -uv run --no-sync python tools/calibrate_uk_national_dataset.py \ - --input-h5 data/ukds/acceptance/623-first-calibrated-candidate/input-spine.h5 \ - --input-sha256 \ - --ledger-facts \ - --ledger-facts-sha256 \ - --ledger-manifest-sha256 \ - --staging-h5 data/ukds/acceptance/623-first-calibrated-candidate/populace_uk_2023.h5 \ - --diagnostics-json data/ukds/acceptance/623-first-calibrated-candidate/calibration_diagnostics.json \ - --build-record-json data/ukds/acceptance/623-first-calibrated-candidate/build_record.json \ - --terminal-gate-json data/ukds/acceptance/623-first-calibrated-candidate/terminal_gates.json \ - --release-id dev-623-first-calibrated-candidate -``` - -The diagnostics digest is measured, not declared: the seam writes the -diagnostics file, hashes its actual bytes, and only then constructs and signs -the terminal gate evidence. There is no `--calibration-diagnostics-sha256` to -supply, and no way for the receipt to claim an identity the file does not have. - -Solve parameters are per-run overrides, not defaults. The campaign settings are -`--epochs 1500 --target-weight-rule family_equal`; each override is validated -through the doctrine dataclass and echoed as an explicit deviation in the -manifest, diagnostics and build record. `--release-candidate` refuses every -override flag, and refuses `--measure-exclusions`, so a release candidate is -always solved under declared doctrine against the committed target surface. - -Signing the terminal gate report needs -`MICROCOSM_UK_TERMINAL_GATE_SIGNING_KEY` in the environment. - -## Scoring - -Rule 1 is decided by a separate tool run against the staged artifact, so run -identity never depends on the incumbent's bytes: - -```bash -uv run --no-sync python tools/score_uk_national_candidate.py \ - --candidate-h5 data/ukds/acceptance/623-first-calibrated-candidate/populace_uk_2023.h5 \ - --candidate-sha256 \ - --incumbent-h5 \ - --incumbent-sha256 \ - --registry-json data/ukds/acceptance/623-first-calibrated-candidate/frozen-register.json \ - --output-json data/ukds/acceptance/623-first-calibrated-candidate/score_vs_enhanced_frs.json -``` - -Both artifacts are verified against the supplied digests before they are read, -and both sides are scored on the same frozen register. - -## Evidence directory - -`data/ukds/acceptance/623-first-calibrated-candidate/` should contain: - -- `build_record.json` -- `terminal_gates.json` -- `calibration_diagnostics.json` -- `score_vs_enhanced_frs.json` -- `logbook-spool/` (one row for the attempt, whatever its disposition) - -Acceptance follows #578: the candidate must not regress incumbent battery -observables, and the score block decides rule 1. A rule-1 loss is evidence for -#686/#736, not a threshold-edit instruction. - -The calibration battery is scoped to the calibration-relevant gates; the -spine-construction and imputation gates are out of scope here and are listed in -the report as scope exclusions with their rationale. A publishable -certification combines this with the spine build's own battery, which is -release-cut work (#757). +The standard build calibrates all applicable geographies together. A country-only request uses `tools/build_uk_full.py --target-geographies country` with the same source, pool, solver, sizing and packaging machinery. This filter excludes regional and local target rows; it does not bypass source validation or geography integrity. K geographic copies and k output households remain separate explicit settings. diff --git a/docs/uk-national-release-assembly-runbook-806.md b/docs/uk-national-release-assembly-runbook-806.md index 8a7ba5db7..d6c9e0393 100644 --- a/docs/uk-national-release-assembly-runbook-806.md +++ b/docs/uk-national-release-assembly-runbook-806.md @@ -1,160 +1,5 @@ -# UK national release assembly runbook (#806) +# UK release assembly -This runbook turns a green UK national calibration candidate into an -inspectable release without promoting it to `latest.json`. The release id is -constant across cuts (`microcosm-uk-2024-25-national`); each cut gets an -immutable tag derived from the calibration attempt id. +Independent national-lane assembly is retired. Use the [UK full-build graph runbook](uk-full-build-graph.md). Standard builds select all geographies, and one graph supplies the target registry, national and local validation evidence, diagnostics and exact export-byte bindings. -Do not run this sequence in PR CI. Calibration, certification, assembly, and -publication consume licensed data and operator credentials. - -## Prerequisites - -Before starting, export: - -- `MICROCOSM_UK_TERMINAL_GATE_SIGNING_KEY`, containing the stable release key - as base64-encoded 32 bytes. Certification signs with it and assembly uses it - to re-verify the copied certification. -- `HF_TOKEN`, authenticated for write access to - `policyengine/populace-uk-private`. -- `SLACK_WEBHOOK_POPULACE_UK`, for the eventual promoted-release alert. The - inspect publication below uses `--no-latest`, so it does not announce a new - latest release. - -Keep the spine H5, its sibling `.build.json` and `.spine_gates.json`, the -Ledger consumer artifact and manifest digest, and the licensed input-mass -reference together through the run. - -## 1. Calibrate the national candidate - -Use the national calibration driver and record the input digest rather than -relying on a mutable path: - -```bash -uv run --no-sync python tools/calibrate_uk_national_dataset.py \ - --input-h5 \ - --input-sha256 \ - --ledger-facts \ - --ledger-facts-sha256 \ - --ledger-manifest-sha256 \ - --staging-h5 /microcosm_uk_2024.h5 \ - --diagnostics-json /calibration_diagnostics.json \ - --build-record-json /build_record.json \ - --terminal-gate-json /terminal_gates.json \ - --release-id dev-uk-national-calibration -``` - -Use only the campaign doctrine overrides that were separately adjudicated. -The build record's id has the form -`uk-frs-calibration-attempt--`; assembly derives the -per-cut tag from that suffix. - -## 2. Score and certify the cut - -First create the rule-1 score receipt against the pinned incumbent, following -the scoring section of -`docs/uk-national-calibration-runbook-623.md`. Then run the release-cut battery -and compose the signed certification: - -```bash -uv run --no-sync python tools/certify_uk_release_cut.py \ - --candidate-h5 /microcosm_uk_2024.h5 \ - --candidate-sha256 \ - --candidate-name microcosm_uk_2024 \ - --spine-h5 \ - --diagnostics-json /calibration_diagnostics.json \ - --build-record-json /build_record.json \ - --seam-gate-report /terminal_gates.json \ - --ledger-facts \ - --ledger-facts-sha256 \ - --ledger-manifest-sha256 \ - --input-mass-reference \ - --score-receipt /score_vs_enhanced_frs.json \ - --release-id microcosm-uk-2024-25-national -``` - -With the default paths, this writes -`microcosm_uk_2024.release_cut_gates.json` and -`microcosm_uk_2024.release_certification.json` next to the candidate. Continue -only when the certification says `shippable: true`. - -## 3. Assemble the release directory - -Assembly verifies the complete hash join before writing, mints the calibration -NPZ from the candidate and spine weights, copies signed evidence byte-for-byte, -and validates the finished directory: - -```bash -uv run --no-sync python tools/assemble_uk_release_dir.py \ - --candidate-h5 /microcosm_uk_2024.h5 \ - --spine-h5 \ - --certification-json /microcosm_uk_2024.release_certification.json \ - --build-record-json /build_record.json \ - --diagnostics-json /calibration_diagnostics.json \ - --seam-gate-report /terminal_gates.json \ - --release-cut-gate-json /microcosm_uk_2024.release_cut_gates.json \ - --score-receipt /score_vs_enhanced_frs.json \ - --out-dir releases -``` - -The output is -`releases/microcosm-uk-2024-25-national/`. The JSON summary records every -digest, the derived cut tag, and the exact publication command. Use -`--cut-tag microcosm-uk-2024-25-national--` only to -override the derived tag deliberately; the override must keep that grammar, -which the contract validates on every artifact revision. - -Assembly stages into a private directory, validates there, and atomically -renames into empty destinations: re-assembling a cut requires removing the -previous `releases/microcosm-uk-2024-25-national/` directory and the -previously minted calibration NPZ first. Release identity — the attempt id, -spine digest, and every runtime pin — comes only from the signed diagnostics -build block; `--runtime-version PACKAGE=VERSION` may re-assert a signed value -as an operator cross-check but refuses to replace one. - -## 4. Publish for inspection - -Run the command printed by the assembler. Its shape is: - -```bash -uv run python -m microcosm.data.publish_cli \ - releases/microcosm-uk-2024-25-national \ - --repo-id policyengine/populace-uk-private \ - --artifact-root \ - --no-latest \ - --tag-name microcosm-uk-2024-25-national-- -``` - -Do not omit `--tag-name`, and do not pass `--no-create-tag`: every artifact in -the manifest is pinned to that immutable per-cut tag. `--no-latest` is -mandatory for this inspect lane — and enforced: publication refuses to move -`latest.json` for any tag that is not the release id itself, so omitting the -flag fails closed instead of promoting an inspect cut. - -If tag creation returns HTTP 409 after the staging commit, publication can -leave the constant branch -`release-staging/microcosm-uk-2024-25-national` behind. Delete that branch -manually in the private Hugging Face repository before retrying the same cut. -Do not delete the immutable cut tag. - -## 5. Inspect on the dashboard - -Open the calibration-diagnostics dashboard with: - -```text -?country=uk&release=microcosm-uk-2024-25-national -``` - -Adjudicate the release using the copied certification, scoped gate reports, -calibration diagnostics, and score receipt. The release remains inspect-only -until that review is complete. - -## Promotion is a separate change - -Do not write `latest.json` for this line yet. The current pointer cannot name a -per-cut tag, while the certified loader expects the artifact revision to equal -the release id and fetches the manifest from a tag named by that id. Promotion -needs the loader/pointer design tracked in microcosm#823 before a reviewed cut -can become the default; publication enforces this by refusing a pointer move -for any per-cut tag. Until then, publish every national cut with `--no-latest` -and its explicit per-cut tag. +Historical signed release records remain historical evidence. They do not select a current build path or substitute for a completed full-build graph and its scoped certification evidence. diff --git a/packages/microcosm-build/README.md b/packages/microcosm-build/README.md index d371f1b71..498fa2487 100644 --- a/packages/microcosm-build/README.md +++ b/packages/microcosm-build/README.md @@ -53,9 +53,18 @@ Country namespaces under `microcosm.build.us` and `microcosm.build.uk` are resource packages only. They may contain specs and data artifacts, but no Python modules; guard tests enforce this so country content stays declarative. -## UK local-geography path - -`microcosm.build.uk_runtime.local_rowwise` is the UK local-solve surface: one +## UK full-build graph + +The canonical command is `microcosm-build-uk` (or `tools/build_uk_full.py`). +It builds or resumes the canonical FRS spine and calibrates all applicable +geographies together by default. Country-only calibration is an explicit +`--target-geographies country` filter in the same graph. Pool copies K and +exact exported household count k remain independent. See the +[full-build runbook](../../docs/uk-full-build-graph.md) for raw inputs, checkpoint +binding, exact-count sizing, replay, diagnostics and certification readiness. + +`microcosm.build.uk_runtime.local_rowwise` supplies the local contribution and +solve helpers used by the full graph: one weight per cloned household, each household assigned to exactly one area by the OA geography ladder, so an area's target rows draw support only from the households assigned there. The matrix builder fails closed when an assigned @@ -104,24 +113,6 @@ input to generation; production runs should provide an Axiom RuleSpec artifact through `AxiomVATRuleEvaluator`. The processed-table reader remains only for paper-repository migration comparisons. -Build the row-wise local-geography H5 from a compact Microcosm UK H5 with: - -```bash -uv run --project packages/microcosm-build --extra uk python \ - tools/build_uk_rowwise_dataset.py \ - --input-h5 /path/to/populace_uk_2023.h5 \ - --out /tmp/populace-uk-rowwise \ - --n-clones 2 \ - --constituency-codes /path/to/constituencies_2024.csv \ - --la-codes /path/to/local_authorities_2021.csv -``` - -If `--crosswalk` is omitted, the driver builds -`uk_official_geography_crosswalk.csv.gz` from public ONS, NRS, NISRA, and -postcode sources. It writes the cloned row-wise H5, a geography coverage CSV, -and `rowwise_build_manifest.json` with input/output hashes, row counts, target -coverage, weight preservation, and weakest local-support diagnostics. - ## US plan status `microcosm.build.us_runtime` declares the US build: stage order, donor graph with diff --git a/packages/microcosm-build/pyproject.toml b/packages/microcosm-build/pyproject.toml index 7521108f8..b45689438 100644 --- a/packages/microcosm-build/pyproject.toml +++ b/packages/microcosm-build/pyproject.toml @@ -39,9 +39,10 @@ us = [ # The UK extra adds the rules engine for local metric generation from a # Microcosm UK H5. Target tables remain explicit inputs, and the base package # still does not import policyengine-uk at import time. -uk = ["policyengine-uk>=2.97.0", "h5py>=3", "tables>=3"] +uk = ["policyengine-uk>=2.97.0", "microcosm-data>=0.1,<0.2", "h5py>=3", "tables>=3"] [project.scripts] +microcosm-build-uk = "microcosm.build.uk_runtime.full_build_cli:main" microcosm-export-us-l0-refit-h5 = "microcosm.build.us_runtime.l0_refit_export:main" [project.urls] diff --git a/packages/microcosm-build/src/microcosm/build/spec_engine/resolver.py b/packages/microcosm-build/src/microcosm/build/spec_engine/resolver.py index eba46efc8..3d02586ac 100644 --- a/packages/microcosm-build/src/microcosm/build/spec_engine/resolver.py +++ b/packages/microcosm-build/src/microcosm/build/spec_engine/resolver.py @@ -165,7 +165,7 @@ def contract_only_ids(self) -> frozenset[str]: "clone_assign_communities", "clone_assign_communes", "load_populace_us_support_pool", - "load_uk_national_frame", + "build_uk_frs_spine", "silc_load", "uk_geography_ladder_gate", } diff --git a/packages/microcosm-build/src/microcosm/build/uk/country_package.json b/packages/microcosm-build/src/microcosm/build/uk/country_package.json index 6805e32df..8bd0431c2 100644 --- a/packages/microcosm-build/src/microcosm/build/uk/country_package.json +++ b/packages/microcosm-build/src/microcosm/build/uk/country_package.json @@ -117,11 +117,6 @@ "kind": "legacy_json", "schema_id": "legacy_json" }, - { - "path": "hmrc_income_source_stages.json", - "kind": "legacy_json", - "schema_id": "legacy_json" - }, { "path": "need_energy_targets.json", "kind": "legacy_json", diff --git a/packages/microcosm-build/src/microcosm/build/uk/efrs_parity_known_gaps.json b/packages/microcosm-build/src/microcosm/build/uk/efrs_parity_known_gaps.json index 9fe5e1d56..1db07a511 100644 --- a/packages/microcosm-build/src/microcosm/build/uk/efrs_parity_known_gaps.json +++ b/packages/microcosm-build/src/microcosm/build/uk/efrs_parity_known_gaps.json @@ -632,12 +632,13 @@ }, "description": "Canonical UK enhanced-FRS parity debt ledger. A reference-populated loader input appears in known_gaps when neither the sha-pinned certified Microcosm UK candidate nor a pinned post-candidate source-family restoration carries non-default signal on the reviewed minimum share of effective population mass.", "exclusion_policy": { - "reason": "not yet ported from enhanced FRS pipeline — pending review", + "reason": "not yet ported from enhanced FRS pipeline \u2014 pending review", "tracking_note": "Tracked in UK_COVERAGE_PROGRESS.md; assign this column to a named source-family restoration milestone before promoting it to required." }, "known_gaps": {}, "restored_required_columns": { "charitable_investment_gifts": { + "current_producer_stage": "hmrc_spi_income_spine", "effective_signal_mass_share": 0.00028055329260683216, "minimum_nondefault_mass_share": 1e-06, "positive_mass_signal_rows": 294, @@ -647,6 +648,7 @@ "support_channel": "spi" }, "gift_aid": { + "current_producer_stage": "hmrc_spi_income_spine", "effective_signal_mass_share": 0.01330315665904484, "minimum_nondefault_mass_share": 1e-06, "positive_mass_signal_rows": 12894, diff --git a/packages/microcosm-build/src/microcosm/build/uk/hmrc_income_source_stages.json b/packages/microcosm-build/src/microcosm/build/uk/hmrc_income_source_stages.json deleted file mode 100644 index 8e8635ae4..000000000 --- a/packages/microcosm-build/src/microcosm/build/uk/hmrc_income_source_stages.json +++ /dev/null @@ -1,735 +0,0 @@ -{ - "version": 1, - "country": "uk", - "policy": "The UK HMRC/SPI income family is source-manifest-defined. Private donor data must be supplied locally, every artifact must be SHA-256 verified at runtime, retained FRS constituents and published bands fail closed, and the current replay keeps importance-kind weights because all 208 banded facts require an unavailable full FRS total-income measure.", - "stages": [ - { - "stage": "hmrc_spi_income", - "survey": "Survey of Personal Incomes Public Use Tape 2022-23 and HMRC Personal Incomes Tables 3.6/3.7 2023-24", - "source": "https://assets.publishing.service.gov.uk/media/69f1f12d2fae53a03709682f/Collated_Tables_3_1_to_3_11_2324.ods", - "grain": "person", - "base_candidate": { - "filename": "populace_uk_2023.h5", - "tier": "frs", - "revision": "populace-uk-2023-dd68c73-4aa4b14-20260619T023711Z", - "sha256": "f17306ccb2aad7ff0130be3589b560afb2e2a12a943570911cd0c77f07934833", - "size_bytes": 1315880118, - "runtime_sha256_required": true - }, - "artifacts": [ - { - "role": "qrf_donor", - "kind": "private_microdata", - "format": "tab_delimited", - "survey": "Survey of Personal Incomes Public Use Tape 2022-23", - "vintage": "2022-23", - "tax_year_start": 2022, - "ukds_study_number": "SN 9422", - "doi": "10.5255/UKDA-SN-9422-1", - "filename": "put2223uk.tab", - "sha256": "5ef829461060c91a2a47be59ad541d9b519fc3976d66ca80d4920f711bb96f66", - "size_bytes": 141323762, - "reviewed_source": "PolicyEngine licensed copy from policyengine/policyengine-uk-data-private on Hugging Face, spi_2022_23.zip", - "access": "private_local_input", - "locator": "caller-supplied local input", - "runtime_sha256_required": true - }, - { - "role": "published_fact_surface", - "kind": "administrative_table", - "format": "ods", - "survey": "HMRC Personal Incomes Tables 3.6 and 3.7", - "publication": "https://www.gov.uk/government/statistics/personal-incomes-statistics-for-the-tax-year-2023-to-2024", - "vintage": "2023-24", - "tax_year_start": 2023, - "locator": "https://assets.publishing.service.gov.uk/media/69f1f12d2fae53a03709682f/Collated_Tables_3_1_to_3_11_2324.ods", - "sha256": "ad063b06b2bdeef8600dbbb09d48153337a4966f8c7eea50df7a2e0304ebd73e", - "size_bytes": 166693, - "mime_type": "application/vnd.oasis.opendocument.spreadsheet", - "sheets": [ - "Table_3_6", - "Table_3_7" - ], - "mapped_build_period": 2023, - "period_mapping": "tax_year_start", - "runtime_sha256_required": true - } - ], - "operations": [ - { - "kind": "verify_certified_candidate", - "artifact": "base_candidate", - "runtime_sha256_required": true, - "fail_on_mismatch": true - }, - { - "kind": "retain_adjudicated_frs_hmrc_leaves", - "population": "certified_microcosm_uk_candidate_base_channel", - "source_vintage": "2023-24", - "mapped_build_period": 2023, - "annualization": "weekly raw FRS amounts * (365.25 / 7)", - "status": "adjudicated_partial_replay", - "retained_full_constituents": { - "hmrc_spi_pay": { - "spi_concept": "PAY", - "scope": "full", - "raw_sources": [ - "ADULT.INEARNS" - ], - "formula": "max(0, ADULT.INEARNS) * (365.25 / 7)" - }, - "hmrc_spi_unemployment_benefit_income": { - "spi_concept": "UBISJA", - "scope": "full", - "raw_sources": [ - "BENEFITS.BENEFIT=14:BENAMT", - "BENEFITS.BENEFIT=19:BENAMT" - ], - "formula": "sum(BENAMT where BENEFIT in {14, 19}) * (365.25 / 7)" - }, - "hmrc_spi_incapacity_benefit_income": { - "spi_concept": "INCPBEN", - "scope": "full", - "raw_sources": [ - "BENEFITS.BENEFIT=17:BENAMT" - ], - "formula": "sum(BENAMT where BENEFIT == 17) * (365.25 / 7)", - "observed_support": "structural zero in the audited 2023-24 FRS; retained so future vintages flow" - } - }, - "retained_named_subsets": { - "ossben_identifiable_subset": { - "spi_concept": "OSSBEN", - "raw_sources": [ - "BENEFITS.BENEFIT=13:BENAMT", - "BENEFITS.BENEFIT=16,VAR2 in {1,3}:BENAMT" - ], - "formula": "sum(BENAMT where BENEFIT == 13 or (BENEFIT == 16 and VAR2 in {1, 3})) * (365.25 / 7)", - "scope": "identifiable_subset" - }, - "srp_regular_code5": { - "spi_concept": "SRP", - "raw_sources": [ - "BENEFITS.BENEFIT=5:BENAMT" - ], - "formula": "sum(BENAMT where BENEFIT == 5) * (365.25 / 7)", - "scope": "regular_code5_subset" - } - }, - "source_absent_full_constituents": [ - "EPB", - "EXPS", - "TAXTERM", - "MOTHINC", - "OTHERINC" - ], - "full_concepts_forbidden_on_frs": [ - "hmrc_spi_employment_benefits", - "hmrc_spi_employment_expenses", - "hmrc_spi_taxable_termination_pay", - "hmrc_spi_miscellaneous_employment_income", - "hmrc_spi_other_income", - "hmrc_spi_other_social_security_income", - "hmrc_spi_state_pension_income" - ], - "forbid_proxy_substitution": [ - "employment_income", - "miscellaneous_income" - ], - "fail_on_missing_retained_constituent": true, - "fail_on_full_concept_alias": true - }, - { - "kind": "verify_pinned_hmrc_source_pair", - "artifact_roles": [ - "qrf_donor", - "published_fact_surface" - ], - "require_before_source_read": true, - "runtime_sha256_required": true, - "fail_on_mismatch": true - }, - { - "kind": "replace_zero_weight_spi_support", - "existing_channel": "spi", - "require_existing_weight": 0, - "replacement_strata": [ - "clone_index", - "household_is_capital_gains_clone", - "region" - ], - "spi_prior_national_household_mass_share": 0.5, - "output_weight_kind": "importance", - "preserve_total_household_mass": true, - "require_mass_change_record": true, - "mass_change_reason": "Allocate 50% of certified UK national household prior mass to the rebuilt 2022-23 SPI support channel; total national mass is conserved.", - "fail_on_live_existing_spi_mass": true - }, - { - "kind": "strict_read_private_table", - "artifact_role": "qrf_donor", - "filename": "put2223uk.tab", - "delimiter": "\t", - "weight": "FACT", - "required_columns": [ - "AGERANGE", - "GORCODE", - "SEX", - "FACT", - "PAY", - "EPB", - "EXPS", - "TAXTERM", - "INCPBEN", - "OSSBEN", - "UBISJA", - "MOTHINC", - "OTHERINC", - "PROFITS", - "CAPALL", - "LOSSBF", - "SRP", - "INCBBS", - "DIVIDENDS", - "PENSION", - "INCPROP", - "OTHERINV", - "GIFTAID", - "GIFTINV", - "TEI", - "TII", - "TI" - ], - "runtime_sha256_required": true, - "fail_on_missing_file": true, - "fail_on_missing_columns": true, - "fail_on_invalid_weight": true - }, - { - "kind": "fit_weighted_qrf_stage1", - "training_artifact_role": "qrf_donor", - "predictors": [ - "age", - "gender", - "region" - ], - "categorical_predictors": [ - "gender", - "region" - ], - "source_sampling_weight": "FACT", - "sample_size": 100000, - "sample_with_replacement": true, - "post_sample_fit_weight": "uniform", - "fit_weight_kind": "design", - "double_apply_source_weight": false, - "source_columns": { - "self_employment_income": [ - "PROFITS", - "CAPALL", - "LOSSBF" - ], - "savings_interest_income": [ - "INCBBS" - ], - "dividend_income": [ - "DIVIDENDS" - ], - "private_pension_income": [ - "PENSION" - ], - "property_income": [ - "INCPROP" - ], - "other_investment_income": [ - "OTHERINV" - ], - "gift_aid": [ - "GIFTAID" - ], - "charitable_investment_gifts": [ - "GIFTINV" - ], - "hmrc_spi_pay": [ - "PAY" - ], - "hmrc_spi_employment_benefits": [ - "EPB" - ], - "hmrc_spi_employment_expenses": [ - "EXPS" - ], - "hmrc_spi_incapacity_benefit_income": [ - "INCPBEN" - ], - "hmrc_spi_other_social_security_income": [ - "OSSBEN" - ], - "hmrc_spi_taxable_termination_pay": [ - "TAXTERM" - ], - "hmrc_spi_unemployment_benefit_income": [ - "UBISJA" - ], - "hmrc_spi_miscellaneous_employment_income": [ - "MOTHINC" - ], - "hmrc_spi_other_income": [ - "OTHERINC" - ], - "hmrc_spi_state_pension_income": [ - "SRP" - ] - }, - "derived_policyengine_outputs": { - "employment_income": { - "source_columns": [ - "PAY", - "EPB", - "TAXTERM" - ], - "formula": "hmrc_spi_pay + hmrc_spi_employment_benefits + hmrc_spi_taxable_termination_pay", - "derive_after_draw": true - } - }, - "outputs": [ - "self_employment_income", - "savings_interest_income", - "dividend_income", - "private_pension_income", - "property_income", - "other_investment_income", - "gift_aid", - "charitable_investment_gifts", - "hmrc_spi_pay", - "hmrc_spi_employment_benefits", - "hmrc_spi_employment_expenses", - "hmrc_spi_incapacity_benefit_income", - "hmrc_spi_other_social_security_income", - "hmrc_spi_taxable_termination_pay", - "hmrc_spi_unemployment_benefit_income", - "hmrc_spi_miscellaneous_employment_income", - "hmrc_spi_other_income", - "hmrc_spi_state_pension_income" - ], - "joint_draw": true, - "savings_interest_source_semantics": "INCBBS is taxable bank/building-society interest before reconstruction to the PolicyEngine gross input", - "employment_income_source_semantics": "PolicyEngine input = PAY + EPB + TAXTERM, matching the pinned enhanced-FRS pipeline; it is not the Table 3.6 measure", - "hmrc_employed_income_source_semantics": "Derived after each draw as max(0, PAY + EPB - EXPS) + INCPBEN + OSSBEN + TAXTERM + UBISJA + MOTHINC, using normalized leaves identically on FRS and SPI channels", - "self_employment_income_source_semantics": "max(0, PROFITS - CAPALL - LOSSBF)", - "assessable_income_source_semantics": "QRF draws leaves only; TEI, TII, and TI are deterministic post-draw accounting aggregates and TI equals TEI + TII exactly", - "source_ti_identity_fields": [ - "TI", - "TEI", - "TII" - ], - "source_leaf_reconciliation": { - "documentation_url": "https://doc.ukdataservice.ac.uk/doc/9422/mrdoc/pdf/9422_put_2223_full_documentation.pdf", - "composite_indicator": "AGERANGE == -1", - "formulas": { - "TEI": "max(0, PAY + EPB - EXPS) + INCPBEN + OSSBEN + TAXTERM + UBISJA + MOTHINC + OTHERINC + SRP + PENSION + max(0, PROFITS - CAPALL - LOSSBF)", - "TII": "OTHERINV + DIVIDENDS + INCPROP + INCBBS", - "TI": "TEI + TII" - }, - "maximum_absolute_difference_gbp": { - "ordinary": { - "TEI": 15, - "TII": 10, - "TI": 20 - }, - "composite": { - "TEI": 180, - "TII": 10, - "TI": 180 - } - }, - "rationale": "The official PUT rounds source fields, averages documented composite records, then rounds remaining income fields to GBP 5. These are the observed envelopes in the exact sha-pinned donor; post-draw synthetic identities remain exact." - }, - "ti_identity_absolute_tolerance_gbp": 5, - "stochastic_aggregates_forbidden": [ - "hmrc_spi_employed_income", - "hmrc_spi_total_earned_income", - "hmrc_spi_total_investment_income", - "hmrc_spi_assessable_income" - ], - "require_all_predictors": true, - "require_all_outputs": true - }, - { - "kind": "fit_weighted_qrf_stage2", - "training_population": "certified_microcosm_uk_candidate_base_channel", - "target_population": "rebuilt_spi_support_channel", - "predictors": [ - "age", - "gender", - "region", - "employment_income", - "self_employment_income", - "savings_interest_income", - "dividend_income", - "private_pension_income", - "property_income" - ], - "reviewed_absent_predictors": { - "other_investment_income": "This remains a stage-1 SPI draw and an official HMRC fact component, but it is not an FRS-only stage-2 predictor: policyengine-uk-data frs_only.py defines exactly six income predictors and the certified Microcosm UK base candidate has no other_investment_income column." - }, - "categorical_predictors": [ - "gender", - "region" - ], - "weight": "household_weight", - "weight_mapping": "household_to_person", - "outputs": [ - "employee_pension_contributions", - "employer_pension_contributions", - "personal_pension_contributions", - "pension_contributions_via_salary_sacrifice", - "tax_free_savings_income", - "universal_credit_reported", - "pension_credit_reported", - "child_benefit_reported", - "housing_benefit_reported", - "income_support_reported", - "working_tax_credit_reported", - "child_tax_credit_reported", - "attendance_allowance_reported", - "state_pension_reported", - "dla_sc_reported", - "dla_m_reported", - "pip_m_reported", - "pip_dl_reported", - "sda_reported", - "carers_allowance_reported", - "iidb_reported", - "afcs_reported", - "bsp_reported", - "winter_fuel_allowance_reported", - "council_tax_benefit_reported", - "jsa_contrib_reported", - "jsa_income_reported", - "esa_contrib_reported", - "esa_income_reported" - ], - "reviewed_absent_outputs": { - "incapacity_benefit_reported": "Absent/all-default on the pinned enhanced-FRS export and certified Microcosm UK base; not a populated loader layer.", - "maternity_allowance_reported": "Absent from the pinned enhanced-FRS export and certified Microcosm UK base; no training source can be materialized for this stage." - }, - "postprocess": { - "gross_savings_interest_income": "stage1 INCBBS draw + stage2 tax_free_savings_income", - "refresh_disability_categories": [ - "aa_category", - "dla_sc_category", - "dla_m_category", - "pip_m_category", - "pip_dl_category" - ], - "refresh_disability_flags": [ - "is_disabled_for_benefits", - "is_enhanced_disabled_for_benefits", - "is_severely_disabled_for_benefits" - ] - }, - "joint_draw": true, - "require_all_predictors": true, - "require_all_materializable_outputs": true, - "require_all_outputs": false - }, - { - "kind": "materialize_hmrc_income_bands_fail_closed", - "artifact_role": "published_fact_surface", - "mapped_build_period": 2023, - "period_mapping": "tax_year_start", - "column_index_base": 0, - "data_row_start_index": 5, - "stop_label": "All ranges", - "count_unit_multiplier": 1000, - "amount_unit_multiplier": 1000000, - "component_columns": { - "employment_income": { - "sheet": "Table_3_6", - "count_column_index": 4, - "amount_column_index": 5 - }, - "self_employment_income": { - "sheet": "Table_3_6", - "count_column_index": 1, - "amount_column_index": 2 - }, - "state_pension": { - "sheet": "Table_3_6", - "count_column_index": 7, - "amount_column_index": 8 - }, - "private_pension_income": { - "sheet": "Table_3_6", - "count_column_index": 10, - "amount_column_index": 11 - }, - "property_income": { - "sheet": "Table_3_7", - "count_column_index": 1, - "amount_column_index": 2 - }, - "savings_interest_income": { - "sheet": "Table_3_7", - "count_column_index": 4, - "amount_column_index": 5 - }, - "dividend_income": { - "sheet": "Table_3_7", - "count_column_index": 7, - "amount_column_index": 8 - }, - "other_investment_income": { - "sheet": "Table_3_7", - "count_column_index": 10, - "amount_column_index": 11 - } - }, - "required_band_lower_bounds_gbp": [ - 12570, - 15000, - 20000, - 30000, - 40000, - 50000, - 70000, - 100000, - 150000, - 200000, - 300000, - 500000, - 1000000 - ], - "required_measures": [ - "count", - "amount" - ], - "fail_on_missing_sheet": true, - "fail_on_missing_component": true, - "fail_on_missing_band": true, - "fail_on_non_numeric_value": true - }, - { - "kind": "classify_hmrc_income_facts_with_reviewed_fences", - "target_operation": "materialize_hmrc_income_bands_fail_closed", - "components": [ - "employment_income", - "self_employment_income", - "state_pension", - "private_pension_income", - "property_income", - "savings_interest_income", - "dividend_income", - "other_investment_income" - ], - "breakdown_dependency": "hmrc_spi_assessable_income", - "frs_breakdown_status": "unavailable_full_measure", - "input_weight_kind": "importance", - "output_weight_kind": "importance", - "calibration_permitted": false, - "required_fact_count": 208, - "outcome_counts": { - "exact_pass": 0, - "exact_fail": 0, - "directional_pass": 0, - "directional_fail": 0, - "excluded_with_fence": 208 - }, - "classification_rationale": "Every published fact uses non-overlapping total-income bands. The FRS channel cannot materialize full TEI, and omitted income can move a person between bands, so neither an exact fact nor a per-band directional bound is valid.", - "reviewed_fences": [ - { - "fence_id": "frs_epb_source_absent", - "constituents": [ - "EPB" - ], - "raw_sources_searched": [ - "JOB.EXPBEN01-EXPBEN13", - "JOB.CARVAL", - "JOB.CARAMT", - "JOB.FUELAMT", - "JOB.VCHAMT", - "JOB.CHVAMT" - ], - "finding": "Missing. EXPBEN* are receipt flags, and the amount fields cover only selected benefits; they cannot produce complete taxable expenses payments and benefits.", - "mass_implication": "12.9485464% of certified-candidate FRS effective person mass has at least one receipt flag, but this is not monetary support.", - "rationale": "Receipt flags and selected benefit amounts cannot be promoted to the SPI EPB monetary concept without an imputation or proxy.", - "dependent_fence_ids": [] - }, - { - "fence_id": "frs_exps_source_absent", - "constituents": [ - "EXPS" - ], - "raw_sources_searched": [ - "JOB.EXPBEN04/EXPBEN05", - "JOB.MILEAMT/JOB.MOTAMT", - "JOB.UMILEAMT/JOB.UMOTAMT", - "JOB.DEDUC1-DEDUC9", - "JOB.UDEDUC1-UDEDUC9" - ], - "finding": "Missing. These fields describe reimbursements or payroll deductions, not the complete tax-deductible employment-expense amount required by SPI.", - "mass_implication": "5.1302528% of certified-candidate FRS effective person mass has an adjacent reimbursement flag; the true EXPS mass is not estimable.", - "rationale": "The nearby fields do not measure the required deductible amount, and EXPS enters the employed-income identity with a negative sign.", - "dependent_fence_ids": [] - }, - { - "fence_id": "frs_taxterm_source_absent", - "constituents": [ - "TAXTERM" - ], - "raw_sources_searched": [ - "ADULT.REDAMT", - "ADULT and JOB taxable-termination split search" - ], - "finding": "Missing. REDAMT is gross redundancy pay and has neither the taxable amount nor non-redundancy termination pay.", - "mass_implication": "0.3746084% of certified-candidate FRS effective person mass has positive gross redundancy pay; taxable mass is unknown.", - "rationale": "Gross redundancy pay cannot be relabeled as taxable termination pay.", - "dependent_fence_ids": [] - }, - { - "fence_id": "frs_mothinc_source_absent", - "constituents": [ - "MOTHINC" - ], - "raw_sources_searched": [ - "ODDJOB.OJAMT/ODDJOB.OJNOW", - "ADULT.ALLPAY2", - "ADULT.ROYYR2-ROYYR4", - "JOB.OWNOTHER" - ], - "finding": "Missing. The fields are heterogeneous and belong to distinct income concepts; assigning their union to SPI miscellaneous employment income would be a proxy.", - "mass_implication": "Odd-job-only effective person mass is 0.1724207%; the broader unresolved miscellaneous pool is 1.4650566%.", - "rationale": "The FRS instrument cannot separate the SPI miscellaneous-employment concept source-faithfully.", - "dependent_fence_ids": [] - }, - { - "fence_id": "frs_otherinc_source_absent", - "constituents": [ - "OTHERINC" - ], - "raw_sources_searched": [ - "ADULT, ODDJOB, and JOB miscellaneous fields", - "PENSION", - "ACCOUNTS", - "ASSETS", - "BENEFITS" - ], - "finding": "Missing. No person-level raw FRS variable has SPI OTHERINC semantics, and the miscellaneous pool cannot be split between MOTHINC and OTHERINC from source evidence.", - "mass_implication": "No separable mass estimate exists; the unresolved miscellaneous pool is 1.4650566% of certified-candidate FRS effective person mass.", - "rationale": "A union of heterogeneous residual fields would be a new proxy, not a retained source constituent.", - "dependent_fence_ids": [] - }, - { - "fence_id": "frs_ossben_identifiable_subset", - "constituents": [ - "OSSBEN", - "ossben_identifiable_subset" - ], - "raw_sources_searched": [ - "BENEFITS.BENAMT", - "BENEFITS.BENEFIT", - "BENEFITS.VAR2", - "BENEFITS codes 13, 16, 6, and 30" - ], - "finding": "Incomplete. Carer's Allowance and contribution-based ESA form an identifiable subset, but code 6 mixes tax treatments and code 30 is an undifferentiated catch-all, so the complete taxable family cannot be emitted.", - "mass_implication": "1.8045088% of certified-candidate FRS effective person mass carries the identifiable lower-bound subset; it is not full OSSBEN support.", - "rationale": "The retained column must remain explicitly named as a subset and cannot satisfy the full SPI concept.", - "dependent_fence_ids": [] - }, - { - "fence_id": "frs_srp_regular_code5_subset", - "constituents": [ - "SRP", - "srp_regular_code5" - ], - "raw_sources_searched": [ - "BENEFITS.BENAMT where BENEFIT == 5", - "BENEFITS codes 6 and 9" - ], - "finding": "Incomplete. Code 5 supplies regular State Pension, but the FRS source does not identify the full SPI combination of State Pension lump sums and widow's pension; code 6 mixes benefits and code 9 is tax-free War Widow's Pension.", - "mass_implication": "18.1567916% of certified-candidate FRS effective person mass carries regular code-5 State Pension; it is not complete SRP support.", - "rationale": "The retained column must remain explicitly named as a subset and cannot be reported as the full published state-pension measure.", - "dependent_fence_ids": [] - }, - { - "fence_id": "full_frs_tei_band_unavailable", - "constituents": [ - "EPB", - "EXPS", - "TAXTERM", - "MOTHINC", - "OTHERINC", - "OSSBEN", - "SRP" - ], - "raw_sources_searched": [], - "finding": "The complete FRS TEI measure cannot be materialized from retained source constituents, so exact HMRC total-income band assignment is unavailable on the FRS channel.", - "mass_implication": "Every one of the 208 published facts is banded by total income and therefore depends on this unavailable like-for-like measure.", - "rationale": "A component-level subset does not imply a per-band lower bound: omitted income can move a taxpayer into or out of any non-overlapping published band. Biased partial bands are not emitted as estimates.", - "dependent_fence_ids": [ - "frs_epb_source_absent", - "frs_exps_source_absent", - "frs_taxterm_source_absent", - "frs_mothinc_source_absent", - "frs_otherinc_source_absent", - "frs_ossben_identifiable_subset", - "frs_srp_regular_code5_subset" - ] - } - ], - "fact_fence_id": "full_frs_tei_band_unavailable", - "blocked_dependency": "hmrc_spi_assessable_income", - "fail_on_unfenced_exclusion": true, - "fail_on_fact_count_mismatch": true, - "forbid_biased_estimate_or_delta": true - }, - { - "kind": "gate_distributional_effective_mass", - "columns": [ - "gift_aid", - "charitable_investment_gifts" - ], - "weight": "household_weight", - "weight_mapping": "household_to_person", - "support_channel_column": "person_support_channel", - "required_support_channel": "spi", - "mass_share_denominator": "all_person_effective_mass", - "minimum_nondefault_mass_share": 0.000001, - "fail_below_floor": true - } - ], - "official_table_components": [ - "employment_income", - "self_employment_income", - "state_pension", - "private_pension_income", - "property_income", - "savings_interest_income", - "dividend_income", - "other_investment_income" - ], - "donor_relief_outputs": [ - "gift_aid", - "charitable_investment_gifts" - ], - "outputs": [ - "employment_income", - "self_employment_income", - "state_pension", - "private_pension_income", - "property_income", - "savings_interest_income", - "dividend_income", - "other_investment_income", - "gift_aid", - "charitable_investment_gifts", - "hmrc_spi_employed_income", - "hmrc_spi_total_earned_income", - "hmrc_spi_total_investment_income", - "hmrc_spi_assessable_income" - ], - "notes": "Current-source adjudicated replay contract: the private 2022-23 SPI donor and public 2023-24 HMRC ODS are pinned by reviewed SHA-256 and size and verified together before either is opened. The QRF draws source leaves; HMRC employed income, TEI, TII, and TI are deterministic post-draw aggregates on the SPI channel, with TI exactly equal to TEI + TII. PolicyEngine employment_income remains the narrow PAY + EPB + TAXTERM input on SPI rows. Stage 2 mirrors policyengine-uk-data frs_only.py exactly: its income predictors are employment, self-employment, savings interest, dividends, private pension, and property income. Other investment income remains a stage-1 SPI draw and official HMRC fact component, but is excluded from stage 2 because the certified FRS candidate does not carry it. The FRS channel retains source-faithful full PAY, UBISJA, and INCPBEN plus explicitly named ossben_identifiable_subset and srp_regular_code5; EPB, EXPS, TAXTERM, MOTHINC, OTHERINC, full OSSBEN, and full SRP remain forbidden. Because the missing legs prevent a complete FRS TEI measure, none of the 208 non-overlapping total-income-band facts is exact or directional. Every fact is an excluded-with-fence record, no calibration is performed, and weights remain importance-kind. Gift Aid restoration still requires the rebuilt positive-mass SPI channel to clear the reviewed 1ppm effective-mass floor." - } - ] -} diff --git a/packages/microcosm-build/src/microcosm/build/uk/release_input_coverage_manifest.json b/packages/microcosm-build/src/microcosm/build/uk/release_input_coverage_manifest.json index 3afdaedda..3e2be6bc2 100644 --- a/packages/microcosm-build/src/microcosm/build/uk/release_input_coverage_manifest.json +++ b/packages/microcosm-build/src/microcosm/build/uk/release_input_coverage_manifest.json @@ -473,7 +473,7 @@ "capital_gains" ], "source_manifest": "source_stages.json", - "source_manifest_sha256": "f1bb134b5456be67c10f2e09b5dc9999c75b1bf3c47165092e260cbd79a54732", + "source_manifest_sha256": "021ac72ced9c4dc4c239b2b68d8e91750cc1140959ccc23f49c46a9fa7717ea4", "source_vintages": { "source": "HMRC Capital Gains Tax statistics, July 2025, Table 2.1a", "survey": "HMRC Capital Gains Tax statistics Table 2.1a and Advani-Summers capital-gains incidence" @@ -496,7 +496,7 @@ "capital_gains" ], "source_manifest": "source_stages.json", - "source_manifest_sha256": "f1bb134b5456be67c10f2e09b5dc9999c75b1bf3c47165092e260cbd79a54732", + "source_manifest_sha256": "021ac72ced9c4dc4c239b2b68d8e91750cc1140959ccc23f49c46a9fa7717ea4", "source_vintages": { "source": "Advani and Summers (2020), Capital Gains and UK Inequality, CAGE Working Paper 465", "survey": "Family Resources Survey 2024-25, SPI synthetic support, and Advani-Summers capital-gains incidence" @@ -525,7 +525,7 @@ "required_mass_change_reason": "E5 source-stage transform preserves household rows and typed household weights; total household mass is conserved.", "rewrites": [], "source_manifest": "source_stages.json", - "source_manifest_sha256": "f1bb134b5456be67c10f2e09b5dc9999c75b1bf3c47165092e260cbd79a54732", + "source_manifest_sha256": "021ac72ced9c4dc4c239b2b68d8e91750cc1140959ccc23f49c46a9fa7717ea4", "source_vintages": { "source": "UK Data Service SN 8856 Effects of Taxes and Benefits household tab, DfT rail fare index, and public NHS activity/cost table.", "survey": "Effects of Taxes and Benefits 1977-2024 and NHS age-gender public table" @@ -545,7 +545,7 @@ "required_mass_change_reason": "E5 source-stage transform preserves household rows and typed household weights; total household mass is conserved.", "rewrites": [], "source_manifest": "source_stages.json", - "source_manifest_sha256": "f1bb134b5456be67c10f2e09b5dc9999c75b1bf3c47165092e260cbd79a54732", + "source_manifest_sha256": "021ac72ced9c4dc4c239b2b68d8e91750cc1140959ccc23f49c46a9fa7717ea4", "source_vintages": { "source": "UK Data Service SN 8856 Effects of Taxes and Benefits household tab and cited VAT anchor resource.", "survey": "Effects of Taxes and Benefits 1977-2024" @@ -553,33 +553,6 @@ "stage": "etb_vat", "status": "required_at_build" }, - "hmrc_cgt_gains": { - "base_candidate_sha256": "f17306ccb2aad7ff0130be3589b560afb2e2a12a943570911cd0c77f07934833", - "base_candidate_tier": "frs", - "calibration_permitted": false, - "effective_mass_requirements": {}, - "fact_fence_id": "cgt_band_facts_policy_endogenous_proxy_conditioned", - "fenced_fact_count": 76, - "output_weight_kind": "importance", - "outputs": [ - "capital_gains" - ], - "required_mass_change_reason": "Amounts-only capital gains redraw: household weights pass through unchanged and total household mass is conserved.", - "source_manifest": "cgt_source_stages.json", - "source_manifest_sha256": "71104111b4b9f2da00ce49ad5abec54a35d300196032c9742d62e02aa1730774", - "source_vintages": { - "hmrc_surface": "2023-24", - "mapped_build_period": "2024" - }, - "stage": "hmrc_cgt_gains", - "status": "required_at_build", - "superseded_by": { - "reason": "The FRS spine build executes hmrc_cgt_gains_spine, which applies the same HMRC Table 3 amounts redraw directly in source_stages.json before calibration.", - "source_manifest": "source_stages.json", - "source_manifest_sha256": "f1bb134b5456be67c10f2e09b5dc9999c75b1bf3c47165092e260cbd79a54732", - "stage": "hmrc_cgt_gains_spine" - } - }, "hmrc_cgt_gains_spine": { "base_candidate_sha256": "f17306ccb2aad7ff0130be3589b560afb2e2a12a943570911cd0c77f07934833", "base_candidate_tier": "frs", @@ -596,7 +569,7 @@ "capital_gains" ], "source_manifest": "source_stages.json", - "source_manifest_sha256": "f1bb134b5456be67c10f2e09b5dc9999c75b1bf3c47165092e260cbd79a54732", + "source_manifest_sha256": "021ac72ced9c4dc4c239b2b68d8e91750cc1140959ccc23f49c46a9fa7717ea4", "source_vintages": { "hmrc_surface": "2023-24", "mapped_build_period": "2024" @@ -606,11 +579,8 @@ }, "hmrc_spi_income": { "band_measure": "hmrc_spi_assessable_income", - "base_candidate_sha256": "f17306ccb2aad7ff0130be3589b560afb2e2a12a943570911cd0c77f07934833", "base_candidate_tier": "frs", "calibration_permitted": false, - "canonical_source_manifest": "source_stages.json", - "canonical_source_manifest_sha256": "f1bb134b5456be67c10f2e09b5dc9999c75b1bf3c47165092e260cbd79a54732", "effective_mass_requirements": { "charitable_investment_gifts": { "mass_share_denominator": "all_person_effective_mass", @@ -649,6 +619,10 @@ "other_investment_income" ], "required_mass_change_reason": "Allocate 50% of certified UK national household prior mass to the rebuilt 2022-23 SPI support channel; total national mass is conserved.", + "required_predecessor_stages": [ + "frs_hmrc_spine_leaves", + "spi_support_channel" + ], "required_target_count": 208, "restoration_status": "adjudicated_partial_replay", "retained_frs_constituents": { @@ -679,8 +653,8 @@ "frs_srp_regular_code5_subset", "full_frs_tei_band_unavailable" ], - "source_manifest": "hmrc_income_source_stages.json", - "source_manifest_sha256": "c0341af7166ae3a85a3c1164e7d9e880c4b4aec122f1a8fa90c73b46c596e1ea", + "source_manifest": "source_stages.json", + "source_manifest_sha256": "021ac72ced9c4dc4c239b2b68d8e91750cc1140959ccc23f49c46a9fa7717ea4", "source_vintages": { "hmrc_surface": "2023-24", "mapped_build_period": "2024", @@ -688,14 +662,8 @@ "spi_donor": "2022-23" }, "spi_prior_national_household_mass_share": 0.5, - "stage": "hmrc_spi_income", - "status": "required_at_build", - "superseded_by": { - "reason": "The FRS spine build executes hmrc_spi_income_spine, which supersedes the June retained-leaves/hmrc_spi_income pair inside source_stages.json.", - "source_manifest": "source_stages.json", - "source_manifest_sha256": "f1bb134b5456be67c10f2e09b5dc9999c75b1bf3c47165092e260cbd79a54732", - "stage": "hmrc_spi_income_spine" - } + "stage": "hmrc_spi_income_spine", + "status": "required_at_build" }, "lcfs_consumption": { "base_candidate_sha256": "f17306ccb2aad7ff0130be3589b560afb2e2a12a943570911cd0c77f07934833", @@ -727,7 +695,7 @@ "required_mass_change_reason": "E5 source-stage transform preserves household rows and typed household weights; total household mass is conserved.", "rewrites": [], "source_manifest": "source_stages.json", - "source_manifest_sha256": "f1bb134b5456be67c10f2e09b5dc9999c75b1bf3c47165092e260cbd79a54732", + "source_manifest_sha256": "021ac72ced9c4dc4c239b2b68d8e91750cc1140959ccc23f49c46a9fa7717ea4", "source_vintages": { "source": "UK Data Service SN 9468 Living Costs and Food Survey 2023-24 household/person tabs, NEED 2023 headline energy tables, Ofgem Q2 2026 unit rates, and WAS round-8 bridge donor.", "survey": "Living Costs and Food Survey 2023-24" @@ -748,7 +716,7 @@ "property_wealth" ], "source_manifest": "source_stages.json", - "source_manifest_sha256": "f1bb134b5456be67c10f2e09b5dc9999c75b1bf3c47165092e260cbd79a54732", + "source_manifest_sha256": "021ac72ced9c4dc4c239b2b68d8e91750cc1140959ccc23f49c46a9fa7717ea4", "source_vintages": { "source": "MHCLG dwellings and ONS UK House Price Index December 2025 regional average prices.", "survey": "Public regional property reference" @@ -772,7 +740,7 @@ "employee_pension_contributions" ], "source_manifest": "source_stages.json", - "source_manifest_sha256": "f1bb134b5456be67c10f2e09b5dc9999c75b1bf3c47165092e260cbd79a54732", + "source_manifest_sha256": "021ac72ced9c4dc4c239b2b68d8e91750cc1140959ccc23f49c46a9fa7717ea4", "source_vintages": { "source": "HMRC, Salary sacrifice reform for pension contributions effective from 6 April 2029", "survey": "Family Resources Survey 2024-25 salary-sacrifice respondents and HMRC salary-sacrifice reform analysis" @@ -794,7 +762,7 @@ "student_loan_plan" ], "source_manifest": "source_stages.json", - "source_manifest_sha256": "f1bb134b5456be67c10f2e09b5dc9999c75b1bf3c47165092e260cbd79a54732", + "source_manifest_sha256": "021ac72ced9c4dc4c239b2b68d8e91750cc1140959ccc23f49c46a9fa7717ea4", "source_vintages": { "source": "Explore Education Statistics Table 6a, Higher education total", "survey": "Family Resources Survey 2024-25 and Student Loans Company borrower forecasts for England" @@ -829,7 +797,7 @@ "required_mass_change_reason": "E5 source-stage transform preserves household rows and typed household weights; total household mass is conserved.", "rewrites": [], "source_manifest": "source_stages.json", - "source_manifest_sha256": "f1bb134b5456be67c10f2e09b5dc9999c75b1bf3c47165092e260cbd79a54732", + "source_manifest_sha256": "021ac72ced9c4dc4c239b2b68d8e91750cc1140959ccc23f49c46a9fa7717ea4", "source_vintages": { "source": "Office for National Statistics Wealth and Assets Survey, UK Data Service SN 7215, DOI 10.5255/UKDA-SN-7215-20; local licensed 2006-22 household tab.", "survey": "Wealth and Assets Survey round 8" diff --git a/packages/microcosm-build/src/microcosm/build/uk/source_stages.json b/packages/microcosm-build/src/microcosm/build/uk/source_stages.json index 52fe5be61..91eaa7d79 100644 --- a/packages/microcosm-build/src/microcosm/build/uk/source_stages.json +++ b/packages/microcosm-build/src/microcosm/build/uk/source_stages.json @@ -3103,743 +3103,6 @@ "student_loan_plan" ], "notes": "Reported PAYE repayers are classified without a country gate. England tertiary cohorts are then topped up PLAN_5 first and PLAN_2 second to the pinned liable stocks at the FRS release calibration year; PLAN_4 is never imputed." - }, - { - "stage": "frs_hmrc_retained_leaves", - "survey": "Family Resources Survey 2024-25", - "source": "Department for Work and Pensions Family Resources Survey 2024-25 raw adult.tab and benefits.tab, caller-supplied local input", - "grain": "person", - "artifacts": [], - "operations": [ - { - "kind": "verify_certified_candidate", - "artifact": "base_candidate", - "runtime_sha256_required": true, - "fail_on_mismatch": true - }, - { - "kind": "retain_adjudicated_frs_hmrc_leaves", - "population": "certified_microcosm_uk_candidate_base_channel", - "source_vintage": "2024-25", - "mapped_build_period": 2024, - "annualization": "weekly raw FRS amounts * (365.25 / 7)", - "status": "adjudicated_partial_replay", - "retained_full_constituents": { - "hmrc_spi_pay": { - "spi_concept": "PAY", - "scope": "full", - "raw_sources": [ - "ADULT.INEARNS" - ], - "formula": "max(0, ADULT.INEARNS) * (365.25 / 7)" - }, - "hmrc_spi_unemployment_benefit_income": { - "spi_concept": "UBISJA", - "scope": "full", - "raw_sources": [ - "BENEFITS.BENEFIT=14:BENAMT", - "BENEFITS.BENEFIT=19:BENAMT" - ], - "formula": "sum(BENAMT where BENEFIT in {14, 19}) * (365.25 / 7)" - }, - "hmrc_spi_incapacity_benefit_income": { - "spi_concept": "INCPBEN", - "scope": "full", - "raw_sources": [ - "BENEFITS.BENEFIT=17:BENAMT" - ], - "formula": "sum(BENAMT where BENEFIT == 17) * (365.25 / 7)", - "observed_support": "structural zero in the audited 2023-24 FRS; retained so future vintages flow" - } - }, - "retained_named_subsets": { - "ossben_identifiable_subset": { - "spi_concept": "OSSBEN", - "raw_sources": [ - "BENEFITS.BENEFIT=13:BENAMT", - "BENEFITS.BENEFIT=16,VAR2 in {1,3}:BENAMT" - ], - "formula": "sum(BENAMT where BENEFIT == 13 or (BENEFIT == 16 and VAR2 in {1, 3})) * (365.25 / 7)", - "scope": "identifiable_subset" - }, - "srp_regular_code5": { - "spi_concept": "SRP", - "raw_sources": [ - "BENEFITS.BENEFIT=5:BENAMT" - ], - "formula": "sum(BENAMT where BENEFIT == 5) * (365.25 / 7)", - "scope": "regular_code5_subset" - } - }, - "source_absent_full_constituents": [ - "EPB", - "EXPS", - "TAXTERM", - "MOTHINC", - "OTHERINC" - ], - "full_concepts_forbidden_on_frs": [ - "hmrc_spi_employment_benefits", - "hmrc_spi_employment_expenses", - "hmrc_spi_taxable_termination_pay", - "hmrc_spi_miscellaneous_employment_income", - "hmrc_spi_other_income", - "hmrc_spi_other_social_security_income", - "hmrc_spi_state_pension_income" - ], - "forbid_proxy_substitution": [ - "employment_income", - "miscellaneous_income" - ], - "fail_on_missing_retained_constituent": true, - "fail_on_full_concept_alias": true - } - ], - "outputs": [ - "hmrc_spi_pay", - "hmrc_spi_unemployment_benefit_income", - "hmrc_spi_incapacity_benefit_income", - "ossben_identifiable_subset", - "srp_regular_code5" - ], - "notes": "Retains the adjudicated source-faithful FRS HMRC leaf columns before the SPI income rebuild: full PAY, UBISJA, and INCPBEN, plus explicitly named OSSBEN and SRP subsets. The runtime verifies the certified candidate before retaining these leaves." - }, - { - "stage": "hmrc_spi_income", - "survey": "Survey of Personal Incomes Public Use Tape 2022-23 and HMRC Personal Incomes Tables 3.6/3.7 2023-24", - "source": "https://assets.publishing.service.gov.uk/media/69f1f12d2fae53a03709682f/Collated_Tables_3_1_to_3_11_2324.ods", - "grain": "person", - "artifacts": [ - { - "role": "qrf_donor", - "kind": "private_microdata", - "format": "tab_delimited", - "survey": "Survey of Personal Incomes Public Use Tape 2022-23", - "vintage": "2022-23", - "tax_year_start": 2022, - "ukds_study_number": "SN 9422", - "doi": "10.5255/UKDA-SN-9422-1", - "filename": "put2223uk.tab", - "sha256": "5ef829461060c91a2a47be59ad541d9b519fc3976d66ca80d4920f711bb96f66", - "size_bytes": 141323762, - "reviewed_source": "PolicyEngine licensed UKDS mirror (private Hugging Face repository), spi_2022_23.zip", - "access": "private_local_input", - "locator": "caller-supplied local input", - "runtime_sha256_required": true - }, - { - "role": "published_fact_surface", - "kind": "administrative_table", - "format": "ods", - "survey": "HMRC Personal Incomes Tables 3.6 and 3.7", - "publication": "https://www.gov.uk/government/statistics/personal-incomes-statistics-for-the-tax-year-2023-to-2024", - "vintage": "2023-24", - "tax_year_start": 2023, - "locator": "https://assets.publishing.service.gov.uk/media/69f1f12d2fae53a03709682f/Collated_Tables_3_1_to_3_11_2324.ods", - "sha256": "ad063b06b2bdeef8600dbbb09d48153337a4966f8c7eea50df7a2e0304ebd73e", - "size_bytes": 166693, - "mime_type": "application/vnd.oasis.opendocument.spreadsheet", - "sheets": [ - "Table_3_6", - "Table_3_7" - ], - "mapped_build_period": 2024, - "period_mapping": "latest_published_tax_year", - "runtime_sha256_required": true - } - ], - "operations": [ - { - "kind": "verify_pinned_hmrc_source_pair", - "artifact_roles": [ - "qrf_donor", - "published_fact_surface" - ], - "require_before_source_read": true, - "runtime_sha256_required": true, - "fail_on_mismatch": true - }, - { - "kind": "replace_zero_weight_spi_support", - "existing_channel": "spi", - "require_existing_weight": 0, - "replacement_strata": [ - "clone_index", - "household_is_capital_gains_clone", - "region" - ], - "spi_prior_national_household_mass_share": 0.5, - "output_weight_kind": "importance", - "preserve_total_household_mass": true, - "require_mass_change_record": true, - "mass_change_reason": "Allocate 50% of certified UK national household prior mass to the rebuilt 2022-23 SPI support channel; total national mass is conserved.", - "fail_on_live_existing_spi_mass": true - }, - { - "kind": "strict_read_private_table", - "artifact_role": "qrf_donor", - "filename": "put2223uk.tab", - "delimiter": "\t", - "weight": "FACT", - "required_columns": [ - "AGERANGE", - "GORCODE", - "SEX", - "FACT", - "PAY", - "EPB", - "EXPS", - "TAXTERM", - "INCPBEN", - "OSSBEN", - "UBISJA", - "MOTHINC", - "OTHERINC", - "PROFITS", - "CAPALL", - "LOSSBF", - "SRP", - "INCBBS", - "DIVIDENDS", - "PENSION", - "INCPROP", - "OTHERINV", - "GIFTAID", - "GIFTINV", - "TEI", - "TII", - "TI" - ], - "runtime_sha256_required": true, - "fail_on_missing_file": true, - "fail_on_missing_columns": true, - "fail_on_invalid_weight": true - }, - { - "kind": "fit_weighted_qrf_stage1", - "training_artifact_role": "qrf_donor", - "predictors": [ - "age", - "gender", - "region" - ], - "categorical_predictors": [ - "gender", - "region" - ], - "source_sampling_weight": "FACT", - "sample_size": 100000, - "sample_with_replacement": true, - "post_sample_fit_weight": "uniform", - "fit_weight_kind": "design", - "double_apply_source_weight": false, - "source_columns": { - "self_employment_income": [ - "PROFITS", - "CAPALL", - "LOSSBF" - ], - "savings_interest_income": [ - "INCBBS" - ], - "dividend_income": [ - "DIVIDENDS" - ], - "private_pension_income": [ - "PENSION" - ], - "property_income": [ - "INCPROP" - ], - "other_investment_income": [ - "OTHERINV" - ], - "gift_aid": [ - "GIFTAID" - ], - "charitable_investment_gifts": [ - "GIFTINV" - ], - "hmrc_spi_pay": [ - "PAY" - ], - "hmrc_spi_employment_benefits": [ - "EPB" - ], - "hmrc_spi_employment_expenses": [ - "EXPS" - ], - "hmrc_spi_incapacity_benefit_income": [ - "INCPBEN" - ], - "hmrc_spi_other_social_security_income": [ - "OSSBEN" - ], - "hmrc_spi_taxable_termination_pay": [ - "TAXTERM" - ], - "hmrc_spi_unemployment_benefit_income": [ - "UBISJA" - ], - "hmrc_spi_miscellaneous_employment_income": [ - "MOTHINC" - ], - "hmrc_spi_other_income": [ - "OTHERINC" - ], - "hmrc_spi_state_pension_income": [ - "SRP" - ] - }, - "derived_policyengine_outputs": { - "employment_income": { - "source_columns": [ - "PAY", - "EPB", - "TAXTERM" - ], - "formula": "hmrc_spi_pay + hmrc_spi_employment_benefits + hmrc_spi_taxable_termination_pay", - "derive_after_draw": true - } - }, - "outputs": [ - "self_employment_income", - "savings_interest_income", - "dividend_income", - "private_pension_income", - "property_income", - "other_investment_income", - "gift_aid", - "charitable_investment_gifts", - "hmrc_spi_pay", - "hmrc_spi_employment_benefits", - "hmrc_spi_employment_expenses", - "hmrc_spi_incapacity_benefit_income", - "hmrc_spi_other_social_security_income", - "hmrc_spi_taxable_termination_pay", - "hmrc_spi_unemployment_benefit_income", - "hmrc_spi_miscellaneous_employment_income", - "hmrc_spi_other_income", - "hmrc_spi_state_pension_income" - ], - "joint_draw": true, - "savings_interest_source_semantics": "INCBBS is taxable bank/building-society interest before reconstruction to the PolicyEngine gross input", - "employment_income_source_semantics": "PolicyEngine input = PAY + EPB + TAXTERM, matching the pinned enhanced-FRS pipeline; it is not the Table 3.6 measure", - "hmrc_employed_income_source_semantics": "Derived after each draw as max(0, PAY + EPB - EXPS) + INCPBEN + OSSBEN + TAXTERM + UBISJA + MOTHINC, using normalized leaves identically on FRS and SPI channels", - "self_employment_income_source_semantics": "max(0, PROFITS - CAPALL - LOSSBF)", - "assessable_income_source_semantics": "QRF draws leaves only; TEI, TII, and TI are deterministic post-draw accounting aggregates and TI equals TEI + TII exactly", - "source_ti_identity_fields": [ - "TI", - "TEI", - "TII" - ], - "source_leaf_reconciliation": { - "documentation_url": "https://doc.ukdataservice.ac.uk/doc/9422/mrdoc/pdf/9422_put_2223_full_documentation.pdf", - "composite_indicator": "AGERANGE == -1", - "formulas": { - "TEI": "max(0, PAY + EPB - EXPS) + INCPBEN + OSSBEN + TAXTERM + UBISJA + MOTHINC + OTHERINC + SRP + PENSION + max(0, PROFITS - CAPALL - LOSSBF)", - "TII": "OTHERINV + DIVIDENDS + INCPROP + INCBBS", - "TI": "TEI + TII" - }, - "maximum_absolute_difference_gbp": { - "ordinary": { - "TEI": 15, - "TII": 10, - "TI": 20 - }, - "composite": { - "TEI": 180, - "TII": 10, - "TI": 180 - } - }, - "rationale": "The official PUT rounds source fields, averages documented composite records, then rounds remaining income fields to GBP 5. These are the observed envelopes in the exact sha-pinned donor; post-draw synthetic identities remain exact." - }, - "ti_identity_absolute_tolerance_gbp": 5, - "stochastic_aggregates_forbidden": [ - "hmrc_spi_employed_income", - "hmrc_spi_total_earned_income", - "hmrc_spi_total_investment_income", - "hmrc_spi_assessable_income" - ], - "require_all_predictors": true, - "require_all_outputs": true - }, - { - "kind": "fit_weighted_qrf_stage2", - "training_population": "certified_microcosm_uk_candidate_base_channel", - "target_population": "rebuilt_spi_support_channel", - "predictors": [ - "age", - "gender", - "region", - "employment_income", - "self_employment_income", - "savings_interest_income", - "dividend_income", - "private_pension_income", - "property_income" - ], - "reviewed_absent_predictors": { - "other_investment_income": "This remains a stage-1 SPI draw and an official HMRC fact component, but it is not an FRS-only stage-2 predictor: the incumbent UK data build's frs_only.py defines exactly six income predictors and the certified Microcosm UK base candidate has no other_investment_income column." - }, - "categorical_predictors": [ - "gender", - "region" - ], - "weight": "household_weight", - "weight_mapping": "household_to_person", - "outputs": [ - "employee_pension_contributions", - "employer_pension_contributions", - "personal_pension_contributions", - "pension_contributions_via_salary_sacrifice", - "tax_free_savings_income", - "universal_credit_reported", - "pension_credit_reported", - "child_benefit_reported", - "housing_benefit_reported", - "income_support_reported", - "working_tax_credit_reported", - "child_tax_credit_reported", - "attendance_allowance_reported", - "state_pension_reported", - "dla_sc_reported", - "dla_m_reported", - "pip_m_reported", - "pip_dl_reported", - "sda_reported", - "carers_allowance_reported", - "iidb_reported", - "afcs_reported", - "bsp_reported", - "winter_fuel_allowance_reported", - "council_tax_benefit_reported", - "jsa_contrib_reported", - "jsa_income_reported", - "esa_contrib_reported", - "esa_income_reported" - ], - "reviewed_absent_outputs": { - "incapacity_benefit_reported": "Absent/all-default on the pinned enhanced-FRS export and certified Microcosm UK base; not a populated loader layer.", - "maternity_allowance_reported": "Absent from the pinned enhanced-FRS export and certified Microcosm UK base; no training source can be materialized for this stage." - }, - "postprocess": { - "gross_savings_interest_income": "stage1 INCBBS draw + stage2 tax_free_savings_income", - "refresh_disability_categories": [ - "aa_category", - "dla_sc_category", - "dla_m_category", - "pip_m_category", - "pip_dl_category" - ], - "refresh_disability_flags": [ - "is_disabled_for_benefits", - "is_enhanced_disabled_for_benefits", - "is_severely_disabled_for_benefits" - ] - }, - "joint_draw": true, - "require_all_predictors": true, - "require_all_materializable_outputs": true, - "require_all_outputs": false - }, - { - "kind": "materialize_hmrc_income_bands_fail_closed", - "artifact_role": "published_fact_surface", - "mapped_build_period": 2024, - "period_mapping": "latest_published_tax_year", - "column_index_base": 0, - "data_row_start_index": 5, - "stop_label": "All ranges", - "count_unit_multiplier": 1000, - "amount_unit_multiplier": 1000000, - "component_columns": { - "employment_income": { - "sheet": "Table_3_6", - "count_column_index": 4, - "amount_column_index": 5 - }, - "self_employment_income": { - "sheet": "Table_3_6", - "count_column_index": 1, - "amount_column_index": 2 - }, - "state_pension": { - "sheet": "Table_3_6", - "count_column_index": 7, - "amount_column_index": 8 - }, - "private_pension_income": { - "sheet": "Table_3_6", - "count_column_index": 10, - "amount_column_index": 11 - }, - "property_income": { - "sheet": "Table_3_7", - "count_column_index": 1, - "amount_column_index": 2 - }, - "savings_interest_income": { - "sheet": "Table_3_7", - "count_column_index": 4, - "amount_column_index": 5 - }, - "dividend_income": { - "sheet": "Table_3_7", - "count_column_index": 7, - "amount_column_index": 8 - }, - "other_investment_income": { - "sheet": "Table_3_7", - "count_column_index": 10, - "amount_column_index": 11 - } - }, - "required_band_lower_bounds_gbp": [ - 12570, - 15000, - 20000, - 30000, - 40000, - 50000, - 70000, - 100000, - 150000, - 200000, - 300000, - 500000, - 1000000 - ], - "required_measures": [ - "count", - "amount" - ], - "fail_on_missing_sheet": true, - "fail_on_missing_component": true, - "fail_on_missing_band": true, - "fail_on_non_numeric_value": true - }, - { - "kind": "classify_hmrc_income_facts_with_reviewed_fences", - "target_operation": "materialize_hmrc_income_bands_fail_closed", - "components": [ - "employment_income", - "self_employment_income", - "state_pension", - "private_pension_income", - "property_income", - "savings_interest_income", - "dividend_income", - "other_investment_income" - ], - "breakdown_dependency": "hmrc_spi_assessable_income", - "frs_breakdown_status": "unavailable_full_measure", - "input_weight_kind": "importance", - "output_weight_kind": "importance", - "calibration_permitted": false, - "required_fact_count": 208, - "outcome_counts": { - "exact_pass": 0, - "exact_fail": 0, - "directional_pass": 0, - "directional_fail": 0, - "excluded_with_fence": 208 - }, - "classification_rationale": "Every published fact uses non-overlapping total-income bands. The FRS channel cannot materialize full TEI, and omitted income can move a person between bands, so neither an exact fact nor a per-band directional bound is valid.", - "reviewed_fences": [ - { - "fence_id": "frs_epb_source_absent", - "constituents": [ - "EPB" - ], - "raw_sources_searched": [ - "JOB.EXPBEN01-EXPBEN13", - "JOB.CARVAL", - "JOB.CARAMT", - "JOB.FUELAMT", - "JOB.VCHAMT", - "JOB.CHVAMT" - ], - "finding": "Missing. EXPBEN* are receipt flags, and the amount fields cover only selected benefits; they cannot produce complete taxable expenses payments and benefits.", - "mass_implication": "12.9485464% of certified-candidate FRS effective person mass has at least one receipt flag, but this is not monetary support.", - "rationale": "Receipt flags and selected benefit amounts cannot be promoted to the SPI EPB monetary concept without an imputation or proxy.", - "dependent_fence_ids": [] - }, - { - "fence_id": "frs_exps_source_absent", - "constituents": [ - "EXPS" - ], - "raw_sources_searched": [ - "JOB.EXPBEN04/EXPBEN05", - "JOB.MILEAMT/JOB.MOTAMT", - "JOB.UMILEAMT/JOB.UMOTAMT", - "JOB.DEDUC1-DEDUC9", - "JOB.UDEDUC1-UDEDUC9" - ], - "finding": "Missing. These fields describe reimbursements or payroll deductions, not the complete tax-deductible employment-expense amount required by SPI.", - "mass_implication": "5.1302528% of certified-candidate FRS effective person mass has an adjacent reimbursement flag; the true EXPS mass is not estimable.", - "rationale": "The nearby fields do not measure the required deductible amount, and EXPS enters the employed-income identity with a negative sign.", - "dependent_fence_ids": [] - }, - { - "fence_id": "frs_taxterm_source_absent", - "constituents": [ - "TAXTERM" - ], - "raw_sources_searched": [ - "ADULT.REDAMT", - "ADULT and JOB taxable-termination split search" - ], - "finding": "Missing. REDAMT is gross redundancy pay and has neither the taxable amount nor non-redundancy termination pay.", - "mass_implication": "0.3746084% of certified-candidate FRS effective person mass has positive gross redundancy pay; taxable mass is unknown.", - "rationale": "Gross redundancy pay cannot be relabeled as taxable termination pay.", - "dependent_fence_ids": [] - }, - { - "fence_id": "frs_mothinc_source_absent", - "constituents": [ - "MOTHINC" - ], - "raw_sources_searched": [ - "ODDJOB.OJAMT/ODDJOB.OJNOW", - "ADULT.ALLPAY2", - "ADULT.ROYYR2-ROYYR4", - "JOB.OWNOTHER" - ], - "finding": "Missing. The fields are heterogeneous and belong to distinct income concepts; assigning their union to SPI miscellaneous employment income would be a proxy.", - "mass_implication": "Odd-job-only effective person mass is 0.1724207%; the broader unresolved miscellaneous pool is 1.4650566%.", - "rationale": "The FRS instrument cannot separate the SPI miscellaneous-employment concept source-faithfully.", - "dependent_fence_ids": [] - }, - { - "fence_id": "frs_otherinc_source_absent", - "constituents": [ - "OTHERINC" - ], - "raw_sources_searched": [ - "ADULT, ODDJOB, and JOB miscellaneous fields", - "PENSION", - "ACCOUNTS", - "ASSETS", - "BENEFITS" - ], - "finding": "Missing. No person-level raw FRS variable has SPI OTHERINC semantics, and the miscellaneous pool cannot be split between MOTHINC and OTHERINC from source evidence.", - "mass_implication": "No separable mass estimate exists; the unresolved miscellaneous pool is 1.4650566% of certified-candidate FRS effective person mass.", - "rationale": "A union of heterogeneous residual fields would be a new proxy, not a retained source constituent.", - "dependent_fence_ids": [] - }, - { - "fence_id": "frs_ossben_identifiable_subset", - "constituents": [ - "OSSBEN", - "ossben_identifiable_subset" - ], - "raw_sources_searched": [ - "BENEFITS.BENAMT", - "BENEFITS.BENEFIT", - "BENEFITS.VAR2", - "BENEFITS codes 13, 16, 6, and 30" - ], - "finding": "Incomplete. Carer's Allowance and contribution-based ESA form an identifiable subset, but code 6 mixes tax treatments and code 30 is an undifferentiated catch-all, so the complete taxable family cannot be emitted.", - "mass_implication": "1.8045088% of certified-candidate FRS effective person mass carries the identifiable lower-bound subset; it is not full OSSBEN support.", - "rationale": "The retained column must remain explicitly named as a subset and cannot satisfy the full SPI concept.", - "dependent_fence_ids": [] - }, - { - "fence_id": "frs_srp_regular_code5_subset", - "constituents": [ - "SRP", - "srp_regular_code5" - ], - "raw_sources_searched": [ - "BENEFITS.BENAMT where BENEFIT == 5", - "BENEFITS codes 6 and 9" - ], - "finding": "Incomplete. Code 5 supplies regular State Pension, but the FRS source does not identify the full SPI combination of State Pension lump sums and widow's pension; code 6 mixes benefits and code 9 is tax-free War Widow's Pension.", - "mass_implication": "18.1567916% of certified-candidate FRS effective person mass carries regular code-5 State Pension; it is not complete SRP support.", - "rationale": "The retained column must remain explicitly named as a subset and cannot be reported as the full published state-pension measure.", - "dependent_fence_ids": [] - }, - { - "fence_id": "full_frs_tei_band_unavailable", - "constituents": [ - "EPB", - "EXPS", - "TAXTERM", - "MOTHINC", - "OTHERINC", - "OSSBEN", - "SRP" - ], - "raw_sources_searched": [], - "finding": "The complete FRS TEI measure cannot be materialized from retained source constituents, so exact HMRC total-income band assignment is unavailable on the FRS channel.", - "mass_implication": "Every one of the 208 published facts is banded by total income and therefore depends on this unavailable like-for-like measure.", - "rationale": "A component-level subset does not imply a per-band lower bound: omitted income can move a taxpayer into or out of any non-overlapping published band. Biased partial bands are not emitted as estimates.", - "dependent_fence_ids": [ - "frs_epb_source_absent", - "frs_exps_source_absent", - "frs_taxterm_source_absent", - "frs_mothinc_source_absent", - "frs_otherinc_source_absent", - "frs_ossben_identifiable_subset", - "frs_srp_regular_code5_subset" - ] - } - ], - "fact_fence_id": "full_frs_tei_band_unavailable", - "blocked_dependency": "hmrc_spi_assessable_income", - "fail_on_unfenced_exclusion": true, - "fail_on_fact_count_mismatch": true, - "forbid_biased_estimate_or_delta": true - }, - { - "kind": "gate_distributional_effective_mass", - "columns": [ - "gift_aid", - "charitable_investment_gifts" - ], - "weight": "household_weight", - "weight_mapping": "household_to_person", - "support_channel_column": "person_support_channel", - "required_support_channel": "spi", - "mass_share_denominator": "all_person_effective_mass", - "minimum_nondefault_mass_share": 1e-06, - "fail_below_floor": true - } - ], - "official_table_components": [ - "employment_income", - "self_employment_income", - "state_pension", - "private_pension_income", - "property_income", - "savings_interest_income", - "dividend_income", - "other_investment_income" - ], - "donor_relief_outputs": [ - "gift_aid", - "charitable_investment_gifts" - ], - "outputs": [ - "employment_income", - "self_employment_income", - "hmrc_spi_state_pension_income", - "private_pension_income", - "property_income", - "savings_interest_income", - "dividend_income", - "other_investment_income", - "gift_aid", - "charitable_investment_gifts", - "hmrc_spi_employed_income", - "hmrc_spi_total_earned_income", - "hmrc_spi_total_investment_income", - "hmrc_spi_assessable_income" - ], - "notes": "Current-source adjudicated replay contract: the private 2022-23 SPI donor and public 2023-24 HMRC ODS are pinned by reviewed SHA-256 and size and verified together before either is opened. The QRF draws source leaves; HMRC employed income, TEI, TII, and TI are deterministic post-draw aggregates on the SPI channel, with TI exactly equal to TEI + TII. PolicyEngine employment_income remains the narrow PAY + EPB + TAXTERM input on SPI rows. Stage 2 mirrors the incumbent UK data build's frs_only.py exactly: its income predictors are employment, self-employment, savings interest, dividends, private pension, and property income. Other investment income remains a stage-1 SPI draw and official HMRC fact component, but is excluded from stage 2 because the certified FRS candidate does not carry it. The FRS channel retains source-faithful full PAY, UBISJA, and INCPBEN plus explicitly named ossben_identifiable_subset and srp_regular_code5; EPB, EXPS, TAXTERM, MOTHINC, OTHERINC, full OSSBEN, and full SRP remain forbidden. Because the missing legs prevent a complete FRS TEI measure, none of the 208 non-overlapping total-income-band facts is exact or directional. Every fact is an excluded-with-fence record, no calibration is performed, and weights remain importance-kind. Gift Aid restoration still requires the rebuilt positive-mass SPI channel to clear the reviewed 1ppm effective-mass floor." } ] } diff --git a/packages/microcosm-build/src/microcosm/build/uk/spec/bundle.yaml b/packages/microcosm-build/src/microcosm/build/uk/spec/bundle.yaml index 8ce5fec5f..f6425656a 100644 --- a/packages/microcosm-build/src/microcosm/build/uk/spec/bundle.yaml +++ b/packages/microcosm-build/src/microcosm/build/uk/spec/bundle.yaml @@ -2,7 +2,8 @@ country: uk identity_generation: 1 seed_protocol: legacy-v1 dataset_run: - target_period: 2023 + target_period: 2025 status: >- - Compiler walking skeleton over the pinned UK national candidate. It does not - replace the generation-0 national, HMRC, calibration, or release drivers. + Raw FRS and canonical source-stage declarations consumed by the UK full-build + country adapter. The shared F0 IR remains a static schema/identity projection; + executable population and calibration stages use uk_full_graph. diff --git a/packages/microcosm-build/src/microcosm/build/uk/spec/catalogs.yaml b/packages/microcosm-build/src/microcosm/build/uk/spec/catalogs.yaml index 247da2415..0a69d09fa 100644 --- a/packages/microcosm-build/src/microcosm/build/uk/spec/catalogs.yaml +++ b/packages/microcosm-build/src/microcosm/build/uk/spec/catalogs.yaml @@ -5,7 +5,7 @@ columns: dtype: int64 unit: count definition_period: eternity - vintage: vintage:uk_target_2023 + vintage: vintage:uk_frs_2024_25 nullable: false domain: frame_identity public_stability: internal @@ -15,7 +15,7 @@ columns: dtype: int64 unit: count definition_period: eternity - vintage: vintage:uk_target_2023 + vintage: vintage:uk_frs_2024_25 nullable: false domain: frame_identity public_stability: internal @@ -25,7 +25,7 @@ columns: dtype: category unit: categorical definition_period: year - vintage: vintage:uk_target_2023 + vintage: vintage:uk_frs_2024_25 nullable: false domain: observed_geography public_stability: internal @@ -35,7 +35,7 @@ columns: dtype: int64 unit: count definition_period: eternity - vintage: vintage:uk_target_2023 + vintage: vintage:uk_frs_2024_25 nullable: false domain: frame_identity public_stability: internal diff --git a/packages/microcosm-build/src/microcosm/build/uk/spec/sources.yaml b/packages/microcosm-build/src/microcosm/build/uk/spec/sources.yaml index 7a7308ff5..297077e47 100644 --- a/packages/microcosm-build/src/microcosm/build/uk/spec/sources.yaml +++ b/packages/microcosm-build/src/microcosm/build/uk/spec/sources.yaml @@ -1,15 +1,106 @@ sources: -- id: uk_national_candidate_2023 - role: uk_national_candidate - sha256: f17306ccb2aad7ff0130be3589b560afb2e2a12a943570911cd0c77f07934833 - byte_size: 1315880118 - loader: kernel:load_uk_national_frame +- id: frs_accounts + role: frs_raw_table + sha256: fa7871eb45cad0db5fd05ede454ced60405d2f9c598651ea5acea5c91a6ff52f + byte_size: 1812923 + loader: kernel:build_uk_frs_spine vintages: - - vintage:uk_candidate_2023 + - vintage:uk_frs_2024_25 +- id: frs_adult + role: frs_raw_table + sha256: 4eaea0809a7ccca0fddeb98e358771e4a6e5ebb81b21c4fac4070fc9d227658d + byte_size: 34885825 + loader: kernel:build_uk_frs_spine + vintages: + - vintage:uk_frs_2024_25 vintage_authorities: - - id: uk_candidate_2023 + - id: uk_frs_2024_25 kind: survey_period - value: 2023 + value: 2024 +- id: frs_benefits + role: frs_raw_table + sha256: f6ad22b408a13e2239c04b0d076a36418dcf5dd89a8c60daa792c4d735b911d3 + byte_size: 2362329 + loader: kernel:build_uk_frs_spine + vintages: + - vintage:uk_frs_2024_25 +- id: frs_benunit + role: frs_raw_table + sha256: 66b894624498316d19b6259e287a607e98ed3daacc9be3d3e9067d32b8e09a5a + byte_size: 13986782 + loader: kernel:build_uk_frs_spine + vintages: + - vintage:uk_frs_2024_25 +- id: frs_child + role: frs_raw_table + sha256: 88ec53fc52eea4374864bbc74219d551f4b4c5f54a220bf9607c7a6289719aa5 + byte_size: 2753961 + loader: kernel:build_uk_frs_spine + vintages: + - vintage:uk_frs_2024_25 +- id: frs_chldcare + role: frs_raw_table + sha256: 7ccd3f92f299a1f49b24063188177cdb8a958d8bcd753fc3d74dadda6ad04023 + byte_size: 275878 + loader: kernel:build_uk_frs_spine + vintages: + - vintage:uk_frs_2024_25 +- id: frs_extchild + role: frs_raw_table + sha256: c661379a4aa5079ce482b1f98f0bfb9157ad9b3ba4eb10739b61846f9c9548e4 + byte_size: 15150 + loader: kernel:build_uk_frs_spine + vintages: + - vintage:uk_frs_2024_25 +- id: frs_househol + role: frs_raw_table + sha256: 2b93b6aed49e1591d6f5360736b4aee3a11ef506b276435f4d2c011a8afbb6a5 + byte_size: 12108606 + loader: kernel:build_uk_frs_spine + vintages: + - vintage:uk_frs_2024_25 +- id: frs_job + role: frs_raw_table + sha256: eb7faf7ada3a3851cb2afb83e2983f8907ffeec897cfbe01e56cb0dfefa853e2 + byte_size: 10518760 + loader: kernel:build_uk_frs_spine + vintages: + - vintage:uk_frs_2024_25 +- id: frs_maint + role: frs_raw_table + sha256: e7a8d6f47cab7bf9db9bfd7b3ad5ebe5830ec75245d065dcf8654c7c20b97a7d + byte_size: 13993 + loader: kernel:build_uk_frs_spine + vintages: + - vintage:uk_frs_2024_25 +- id: frs_mortgage + role: frs_raw_table + sha256: 6a08f6846970dfdc544a7efc8a93fed4f3210d872cd2d160dfb14ca8d92d5ed0 + byte_size: 600552 + loader: kernel:build_uk_frs_spine + vintages: + - vintage:uk_frs_2024_25 +- id: frs_oddjob + role: frs_raw_table + sha256: dfff1baf71a3de05f3a2fcf0c01a3995df5657f242cd7846aa61f6cc27a1cead + byte_size: 5339 + loader: kernel:build_uk_frs_spine + vintages: + - vintage:uk_frs_2024_25 +- id: frs_penprov + role: frs_raw_table + sha256: 9e53de0dc969baec000b3cd68387f0f2dfb3f678732e408de175e0a1d6e3fdc1 + byte_size: 513614 + loader: kernel:build_uk_frs_spine + vintages: + - vintage:uk_frs_2024_25 +- id: frs_pension + role: frs_raw_table + sha256: 2b9be1eb6583cc8916fc06294be27e6217f2aea73da24b97b3226293f6a6ec24 + byte_size: 1232411 + loader: kernel:build_uk_frs_spine + vintages: + - vintage:uk_frs_2024_25 stage_manifest: version: 1 country: uk @@ -2462,600 +2553,3 @@ stages: rewrites: - student_loan_plan notes: Reported PAYE repayers are classified without a country gate. England tertiary cohorts are then topped up PLAN_5 first and PLAN_2 second to the pinned liable stocks at the FRS release calibration year; PLAN_4 is never imputed. -- stage: frs_hmrc_retained_leaves - survey: Family Resources Survey 2024-25 - source: Department for Work and Pensions Family Resources Survey 2024-25 raw adult.tab and benefits.tab, caller-supplied local input - grain: person - artifacts: [] - operations: - - kind: verify_certified_candidate - artifact: base_candidate - runtime_sha256_required: true - fail_on_mismatch: true - - kind: retain_adjudicated_frs_hmrc_leaves - population: certified_microcosm_uk_candidate_base_channel - source_vintage: 2024-25 - mapped_build_period: 2024 - annualization: weekly raw FRS amounts * (365.25 / 7) - status: adjudicated_partial_replay - retained_full_constituents: - hmrc_spi_pay: - spi_concept: PAY - scope: full - raw_sources: - - ADULT.INEARNS - formula: max(0, ADULT.INEARNS) * (365.25 / 7) - hmrc_spi_unemployment_benefit_income: - spi_concept: UBISJA - scope: full - raw_sources: - - BENEFITS.BENEFIT=14:BENAMT - - BENEFITS.BENEFIT=19:BENAMT - formula: sum(BENAMT where BENEFIT in {14, 19}) * (365.25 / 7) - hmrc_spi_incapacity_benefit_income: - spi_concept: INCPBEN - scope: full - raw_sources: - - BENEFITS.BENEFIT=17:BENAMT - formula: sum(BENAMT where BENEFIT == 17) * (365.25 / 7) - observed_support: structural zero in the audited 2023-24 FRS; retained so future vintages flow - retained_named_subsets: - ossben_identifiable_subset: - spi_concept: OSSBEN - raw_sources: - - BENEFITS.BENEFIT=13:BENAMT - - BENEFITS.BENEFIT=16,VAR2 in {1,3}:BENAMT - formula: sum(BENAMT where BENEFIT == 13 or (BENEFIT == 16 and VAR2 in {1, 3})) * (365.25 / 7) - scope: identifiable_subset - srp_regular_code5: - spi_concept: SRP - raw_sources: - - BENEFITS.BENEFIT=5:BENAMT - formula: sum(BENAMT where BENEFIT == 5) * (365.25 / 7) - scope: regular_code5_subset - source_absent_full_constituents: - - EPB - - EXPS - - TAXTERM - - MOTHINC - - OTHERINC - full_concepts_forbidden_on_frs: - - hmrc_spi_employment_benefits - - hmrc_spi_employment_expenses - - hmrc_spi_taxable_termination_pay - - hmrc_spi_miscellaneous_employment_income - - hmrc_spi_other_income - - hmrc_spi_other_social_security_income - - hmrc_spi_state_pension_income - forbid_proxy_substitution: - - employment_income - - miscellaneous_income - fail_on_missing_retained_constituent: true - fail_on_full_concept_alias: true - outputs: - - hmrc_spi_pay - - hmrc_spi_unemployment_benefit_income - - hmrc_spi_incapacity_benefit_income - - ossben_identifiable_subset - - srp_regular_code5 - notes: 'Retains the adjudicated source-faithful FRS HMRC leaf columns before the SPI income rebuild: full PAY, UBISJA, and INCPBEN, plus explicitly named OSSBEN and SRP subsets. The runtime verifies the certified candidate before retaining these leaves.' -- stage: hmrc_spi_income - survey: Survey of Personal Incomes Public Use Tape 2022-23 and HMRC Personal Incomes Tables 3.6/3.7 2023-24 - source: https://assets.publishing.service.gov.uk/media/69f1f12d2fae53a03709682f/Collated_Tables_3_1_to_3_11_2324.ods - grain: person - artifacts: - - role: qrf_donor - kind: private_microdata - format: tab_delimited - survey: Survey of Personal Incomes Public Use Tape 2022-23 - vintage: 2022-23 - tax_year_start: 2022 - ukds_study_number: SN 9422 - doi: 10.5255/UKDA-SN-9422-1 - filename: put2223uk.tab - sha256: 5ef829461060c91a2a47be59ad541d9b519fc3976d66ca80d4920f711bb96f66 - size_bytes: 141323762 - reviewed_source: PolicyEngine licensed UKDS mirror (private Hugging Face repository), spi_2022_23.zip - access: private_local_input - locator: caller-supplied local input - runtime_sha256_required: true - - role: published_fact_surface - kind: administrative_table - format: ods - survey: HMRC Personal Incomes Tables 3.6 and 3.7 - publication: https://www.gov.uk/government/statistics/personal-incomes-statistics-for-the-tax-year-2023-to-2024 - vintage: 2023-24 - tax_year_start: 2023 - locator: https://assets.publishing.service.gov.uk/media/69f1f12d2fae53a03709682f/Collated_Tables_3_1_to_3_11_2324.ods - sha256: ad063b06b2bdeef8600dbbb09d48153337a4966f8c7eea50df7a2e0304ebd73e - size_bytes: 166693 - mime_type: application/vnd.oasis.opendocument.spreadsheet - sheets: - - Table_3_6 - - Table_3_7 - mapped_build_period: 2024 - period_mapping: latest_published_tax_year - runtime_sha256_required: true - operations: - - kind: verify_pinned_hmrc_source_pair - artifact_roles: - - qrf_donor - - published_fact_surface - require_before_source_read: true - runtime_sha256_required: true - fail_on_mismatch: true - - kind: replace_zero_weight_spi_support - existing_channel: spi - require_existing_weight: 0 - replacement_strata: - - clone_index - - household_is_capital_gains_clone - - region - spi_prior_national_household_mass_share: 0.5 - output_weight_kind: importance - preserve_total_household_mass: true - require_mass_change_record: true - mass_change_reason: Allocate 50% of certified UK national household prior mass to the rebuilt 2022-23 SPI support channel; total national mass is conserved. - fail_on_live_existing_spi_mass: true - - kind: strict_read_private_table - artifact_role: qrf_donor - filename: put2223uk.tab - delimiter: "\t" - weight: FACT - required_columns: - - AGERANGE - - GORCODE - - SEX - - FACT - - PAY - - EPB - - EXPS - - TAXTERM - - INCPBEN - - OSSBEN - - UBISJA - - MOTHINC - - OTHERINC - - PROFITS - - CAPALL - - LOSSBF - - SRP - - INCBBS - - DIVIDENDS - - PENSION - - INCPROP - - OTHERINV - - GIFTAID - - GIFTINV - - TEI - - TII - - TI - runtime_sha256_required: true - fail_on_missing_file: true - fail_on_missing_columns: true - fail_on_invalid_weight: true - - kind: fit_weighted_qrf_stage1 - training_artifact_role: qrf_donor - predictors: - - age - - gender - - region - categorical_predictors: - - gender - - region - source_sampling_weight: FACT - sample_size: 100000 - sample_with_replacement: true - post_sample_fit_weight: uniform - fit_weight_kind: design - double_apply_source_weight: false - source_columns: - self_employment_income: - - PROFITS - - CAPALL - - LOSSBF - savings_interest_income: - - INCBBS - dividend_income: - - DIVIDENDS - private_pension_income: - - PENSION - property_income: - - INCPROP - other_investment_income: - - OTHERINV - gift_aid: - - GIFTAID - charitable_investment_gifts: - - GIFTINV - hmrc_spi_pay: - - PAY - hmrc_spi_employment_benefits: - - EPB - hmrc_spi_employment_expenses: - - EXPS - hmrc_spi_incapacity_benefit_income: - - INCPBEN - hmrc_spi_other_social_security_income: - - OSSBEN - hmrc_spi_taxable_termination_pay: - - TAXTERM - hmrc_spi_unemployment_benefit_income: - - UBISJA - hmrc_spi_miscellaneous_employment_income: - - MOTHINC - hmrc_spi_other_income: - - OTHERINC - hmrc_spi_state_pension_income: - - SRP - derived_policyengine_outputs: - employment_income: - source_columns: - - PAY - - EPB - - TAXTERM - formula: hmrc_spi_pay + hmrc_spi_employment_benefits + hmrc_spi_taxable_termination_pay - derive_after_draw: true - outputs: - - self_employment_income - - savings_interest_income - - dividend_income - - private_pension_income - - property_income - - other_investment_income - - gift_aid - - charitable_investment_gifts - - hmrc_spi_pay - - hmrc_spi_employment_benefits - - hmrc_spi_employment_expenses - - hmrc_spi_incapacity_benefit_income - - hmrc_spi_other_social_security_income - - hmrc_spi_taxable_termination_pay - - hmrc_spi_unemployment_benefit_income - - hmrc_spi_miscellaneous_employment_income - - hmrc_spi_other_income - - hmrc_spi_state_pension_income - joint_draw: true - savings_interest_source_semantics: INCBBS is taxable bank/building-society interest before reconstruction to the PolicyEngine gross input - employment_income_source_semantics: PolicyEngine input = PAY + EPB + TAXTERM, matching the pinned enhanced-FRS pipeline; it is not the Table 3.6 measure - hmrc_employed_income_source_semantics: Derived after each draw as max(0, PAY + EPB - EXPS) + INCPBEN + OSSBEN + TAXTERM + UBISJA + MOTHINC, using normalized leaves identically on FRS and SPI channels - self_employment_income_source_semantics: max(0, PROFITS - CAPALL - LOSSBF) - assessable_income_source_semantics: QRF draws leaves only; TEI, TII, and TI are deterministic post-draw accounting aggregates and TI equals TEI + TII exactly - source_ti_identity_fields: - - TI - - TEI - - TII - source_leaf_reconciliation: - documentation_url: https://doc.ukdataservice.ac.uk/doc/9422/mrdoc/pdf/9422_put_2223_full_documentation.pdf - composite_indicator: AGERANGE == -1 - formulas: - TEI: max(0, PAY + EPB - EXPS) + INCPBEN + OSSBEN + TAXTERM + UBISJA + MOTHINC + OTHERINC + SRP + PENSION + max(0, PROFITS - CAPALL - LOSSBF) - TII: OTHERINV + DIVIDENDS + INCPROP + INCBBS - TI: TEI + TII - maximum_absolute_difference_gbp: - ordinary: - TEI: 15 - TII: 10 - TI: 20 - composite: - TEI: 180 - TII: 10 - TI: 180 - rationale: The official PUT rounds source fields, averages documented composite records, then rounds remaining income fields to GBP 5. These are the observed envelopes in the exact sha-pinned donor; post-draw synthetic identities remain exact. - ti_identity_absolute_tolerance_gbp: 5 - stochastic_aggregates_forbidden: - - hmrc_spi_employed_income - - hmrc_spi_total_earned_income - - hmrc_spi_total_investment_income - - hmrc_spi_assessable_income - require_all_predictors: true - require_all_outputs: true - - kind: fit_weighted_qrf_stage2 - training_population: certified_microcosm_uk_candidate_base_channel - target_population: rebuilt_spi_support_channel - predictors: - - age - - gender - - region - - employment_income - - self_employment_income - - savings_interest_income - - dividend_income - - private_pension_income - - property_income - reviewed_absent_predictors: - other_investment_income: 'This remains a stage-1 SPI draw and an official HMRC fact component, but it is not an FRS-only stage-2 predictor: the incumbent UK data build''s frs_only.py defines exactly six income predictors and the certified Microcosm UK base candidate has no other_investment_income column.' - categorical_predictors: - - gender - - region - weight: household_weight - weight_mapping: household_to_person - outputs: - - employee_pension_contributions - - employer_pension_contributions - - personal_pension_contributions - - pension_contributions_via_salary_sacrifice - - tax_free_savings_income - - universal_credit_reported - - pension_credit_reported - - child_benefit_reported - - housing_benefit_reported - - income_support_reported - - working_tax_credit_reported - - child_tax_credit_reported - - attendance_allowance_reported - - state_pension_reported - - dla_sc_reported - - dla_m_reported - - pip_m_reported - - pip_dl_reported - - sda_reported - - carers_allowance_reported - - iidb_reported - - afcs_reported - - bsp_reported - - winter_fuel_allowance_reported - - council_tax_benefit_reported - - jsa_contrib_reported - - jsa_income_reported - - esa_contrib_reported - - esa_income_reported - reviewed_absent_outputs: - incapacity_benefit_reported: Absent/all-default on the pinned enhanced-FRS export and certified Microcosm UK base; not a populated loader layer. - maternity_allowance_reported: Absent from the pinned enhanced-FRS export and certified Microcosm UK base; no training source can be materialized for this stage. - postprocess: - gross_savings_interest_income: stage1 INCBBS draw + stage2 tax_free_savings_income - refresh_disability_categories: - - aa_category - - dla_sc_category - - dla_m_category - - pip_m_category - - pip_dl_category - refresh_disability_flags: - - is_disabled_for_benefits - - is_enhanced_disabled_for_benefits - - is_severely_disabled_for_benefits - joint_draw: true - require_all_predictors: true - require_all_materializable_outputs: true - require_all_outputs: false - - kind: materialize_hmrc_income_bands_fail_closed - artifact_role: published_fact_surface - mapped_build_period: 2024 - period_mapping: latest_published_tax_year - column_index_base: 0 - data_row_start_index: 5 - stop_label: All ranges - count_unit_multiplier: 1000 - amount_unit_multiplier: 1000000 - component_columns: - employment_income: - sheet: Table_3_6 - count_column_index: 4 - amount_column_index: 5 - self_employment_income: - sheet: Table_3_6 - count_column_index: 1 - amount_column_index: 2 - state_pension: - sheet: Table_3_6 - count_column_index: 7 - amount_column_index: 8 - private_pension_income: - sheet: Table_3_6 - count_column_index: 10 - amount_column_index: 11 - property_income: - sheet: Table_3_7 - count_column_index: 1 - amount_column_index: 2 - savings_interest_income: - sheet: Table_3_7 - count_column_index: 4 - amount_column_index: 5 - dividend_income: - sheet: Table_3_7 - count_column_index: 7 - amount_column_index: 8 - other_investment_income: - sheet: Table_3_7 - count_column_index: 10 - amount_column_index: 11 - required_band_lower_bounds_gbp: - - 12570 - - 15000 - - 20000 - - 30000 - - 40000 - - 50000 - - 70000 - - 100000 - - 150000 - - 200000 - - 300000 - - 500000 - - 1000000 - required_measures: - - count - - amount - fail_on_missing_sheet: true - fail_on_missing_component: true - fail_on_missing_band: true - fail_on_non_numeric_value: true - - kind: classify_hmrc_income_facts_with_reviewed_fences - target_operation: materialize_hmrc_income_bands_fail_closed - components: - - employment_income - - self_employment_income - - state_pension - - private_pension_income - - property_income - - savings_interest_income - - dividend_income - - other_investment_income - breakdown_dependency: hmrc_spi_assessable_income - frs_breakdown_status: unavailable_full_measure - input_weight_kind: importance - output_weight_kind: importance - calibration_permitted: false - required_fact_count: 208 - outcome_counts: - exact_pass: 0 - exact_fail: 0 - directional_pass: 0 - directional_fail: 0 - excluded_with_fence: 208 - classification_rationale: Every published fact uses non-overlapping total-income bands. The FRS channel cannot materialize full TEI, and omitted income can move a person between bands, so neither an exact fact nor a per-band directional bound is valid. - reviewed_fences: - - fence_id: frs_epb_source_absent - constituents: - - EPB - raw_sources_searched: - - JOB.EXPBEN01-EXPBEN13 - - JOB.CARVAL - - JOB.CARAMT - - JOB.FUELAMT - - JOB.VCHAMT - - JOB.CHVAMT - finding: Missing. EXPBEN* are receipt flags, and the amount fields cover only selected benefits; they cannot produce complete taxable expenses payments and benefits. - mass_implication: 12.9485464% of certified-candidate FRS effective person mass has at least one receipt flag, but this is not monetary support. - rationale: Receipt flags and selected benefit amounts cannot be promoted to the SPI EPB monetary concept without an imputation or proxy. - dependent_fence_ids: [] - - fence_id: frs_exps_source_absent - constituents: - - EXPS - raw_sources_searched: - - JOB.EXPBEN04/EXPBEN05 - - JOB.MILEAMT/JOB.MOTAMT - - JOB.UMILEAMT/JOB.UMOTAMT - - JOB.DEDUC1-DEDUC9 - - JOB.UDEDUC1-UDEDUC9 - finding: Missing. These fields describe reimbursements or payroll deductions, not the complete tax-deductible employment-expense amount required by SPI. - mass_implication: 5.1302528% of certified-candidate FRS effective person mass has an adjacent reimbursement flag; the true EXPS mass is not estimable. - rationale: The nearby fields do not measure the required deductible amount, and EXPS enters the employed-income identity with a negative sign. - dependent_fence_ids: [] - - fence_id: frs_taxterm_source_absent - constituents: - - TAXTERM - raw_sources_searched: - - ADULT.REDAMT - - ADULT and JOB taxable-termination split search - finding: Missing. REDAMT is gross redundancy pay and has neither the taxable amount nor non-redundancy termination pay. - mass_implication: 0.3746084% of certified-candidate FRS effective person mass has positive gross redundancy pay; taxable mass is unknown. - rationale: Gross redundancy pay cannot be relabeled as taxable termination pay. - dependent_fence_ids: [] - - fence_id: frs_mothinc_source_absent - constituents: - - MOTHINC - raw_sources_searched: - - ODDJOB.OJAMT/ODDJOB.OJNOW - - ADULT.ALLPAY2 - - ADULT.ROYYR2-ROYYR4 - - JOB.OWNOTHER - finding: Missing. The fields are heterogeneous and belong to distinct income concepts; assigning their union to SPI miscellaneous employment income would be a proxy. - mass_implication: Odd-job-only effective person mass is 0.1724207%; the broader unresolved miscellaneous pool is 1.4650566%. - rationale: The FRS instrument cannot separate the SPI miscellaneous-employment concept source-faithfully. - dependent_fence_ids: [] - - fence_id: frs_otherinc_source_absent - constituents: - - OTHERINC - raw_sources_searched: - - ADULT, ODDJOB, and JOB miscellaneous fields - - PENSION - - ACCOUNTS - - ASSETS - - BENEFITS - finding: Missing. No person-level raw FRS variable has SPI OTHERINC semantics, and the miscellaneous pool cannot be split between MOTHINC and OTHERINC from source evidence. - mass_implication: No separable mass estimate exists; the unresolved miscellaneous pool is 1.4650566% of certified-candidate FRS effective person mass. - rationale: A union of heterogeneous residual fields would be a new proxy, not a retained source constituent. - dependent_fence_ids: [] - - fence_id: frs_ossben_identifiable_subset - constituents: - - OSSBEN - - ossben_identifiable_subset - raw_sources_searched: - - BENEFITS.BENAMT - - BENEFITS.BENEFIT - - BENEFITS.VAR2 - - BENEFITS codes 13, 16, 6, and 30 - finding: Incomplete. Carer's Allowance and contribution-based ESA form an identifiable subset, but code 6 mixes tax treatments and code 30 is an undifferentiated catch-all, so the complete taxable family cannot be emitted. - mass_implication: 1.8045088% of certified-candidate FRS effective person mass carries the identifiable lower-bound subset; it is not full OSSBEN support. - rationale: The retained column must remain explicitly named as a subset and cannot satisfy the full SPI concept. - dependent_fence_ids: [] - - fence_id: frs_srp_regular_code5_subset - constituents: - - SRP - - srp_regular_code5 - raw_sources_searched: - - BENEFITS.BENAMT where BENEFIT == 5 - - BENEFITS codes 6 and 9 - finding: Incomplete. Code 5 supplies regular State Pension, but the FRS source does not identify the full SPI combination of State Pension lump sums and widow's pension; code 6 mixes benefits and code 9 is tax-free War Widow's Pension. - mass_implication: 18.1567916% of certified-candidate FRS effective person mass carries regular code-5 State Pension; it is not complete SRP support. - rationale: The retained column must remain explicitly named as a subset and cannot be reported as the full published state-pension measure. - dependent_fence_ids: [] - - fence_id: full_frs_tei_band_unavailable - constituents: - - EPB - - EXPS - - TAXTERM - - MOTHINC - - OTHERINC - - OSSBEN - - SRP - raw_sources_searched: [] - finding: The complete FRS TEI measure cannot be materialized from retained source constituents, so exact HMRC total-income band assignment is unavailable on the FRS channel. - mass_implication: Every one of the 208 published facts is banded by total income and therefore depends on this unavailable like-for-like measure. - rationale: 'A component-level subset does not imply a per-band lower bound: omitted income can move a taxpayer into or out of any non-overlapping published band. Biased partial bands are not emitted as estimates.' - dependent_fence_ids: - - frs_epb_source_absent - - frs_exps_source_absent - - frs_taxterm_source_absent - - frs_mothinc_source_absent - - frs_otherinc_source_absent - - frs_ossben_identifiable_subset - - frs_srp_regular_code5_subset - fact_fence_id: full_frs_tei_band_unavailable - blocked_dependency: hmrc_spi_assessable_income - fail_on_unfenced_exclusion: true - fail_on_fact_count_mismatch: true - forbid_biased_estimate_or_delta: true - - kind: gate_distributional_effective_mass - columns: - - gift_aid - - charitable_investment_gifts - weight: household_weight - weight_mapping: household_to_person - support_channel_column: person_support_channel - required_support_channel: spi - mass_share_denominator: all_person_effective_mass - minimum_nondefault_mass_share: 1.0e-06 - fail_below_floor: true - official_table_components: - - employment_income - - self_employment_income - - state_pension - - private_pension_income - - property_income - - savings_interest_income - - dividend_income - - other_investment_income - donor_relief_outputs: - - gift_aid - - charitable_investment_gifts - outputs: - - employment_income - - self_employment_income - - hmrc_spi_state_pension_income - - private_pension_income - - property_income - - savings_interest_income - - dividend_income - - other_investment_income - - gift_aid - - charitable_investment_gifts - - hmrc_spi_employed_income - - hmrc_spi_total_earned_income - - hmrc_spi_total_investment_income - - hmrc_spi_assessable_income - notes: 'Current-source adjudicated replay contract: the private 2022-23 SPI donor and public 2023-24 HMRC ODS are pinned by reviewed SHA-256 and size and verified together before either is opened. The QRF draws source leaves; HMRC employed income, TEI, TII, and TI are deterministic post-draw aggregates on the SPI channel, with TI exactly equal to TEI + TII. PolicyEngine employment_income remains the narrow PAY + EPB + TAXTERM input on SPI rows. Stage 2 mirrors the incumbent UK data build''s frs_only.py exactly: its income predictors are employment, self-employment, savings interest, dividends, private pension, and property income. Other investment income remains a stage-1 SPI draw and official HMRC fact component, but is excluded from stage 2 because the certified FRS candidate does not carry it. The FRS channel retains source-faithful full PAY, UBISJA, and INCPBEN plus explicitly named ossben_identifiable_subset and srp_regular_code5; EPB, EXPS, TAXTERM, MOTHINC, OTHERINC, full OSSBEN, - and full SRP remain forbidden. Because the missing legs prevent a complete FRS TEI measure, none of the 208 non-overlapping total-income-band facts is exact or directional. Every fact is an excluded-with-fence record, no calibration is performed, and weights remain importance-kind. Gift Aid restoration still requires the rebuilt positive-mass SPI channel to clear the reviewed 1ppm effective-mass floor.' diff --git a/packages/microcosm-build/src/microcosm/build/uk/spec/spine.yaml b/packages/microcosm-build/src/microcosm/build/uk/spec/spine.yaml index 930073bda..f42eda37b 100644 --- a/packages/microcosm-build/src/microcosm/build/uk/spec/spine.yaml +++ b/packages/microcosm-build/src/microcosm/build/uk/spec/spine.yaml @@ -1,6 +1,20 @@ channels: - id: frs - source: uk_national_candidate_2023 + source: + - frs_accounts + - frs_adult + - frs_benefits + - frs_benunit + - frs_child + - frs_chldcare + - frs_extchild + - frs_househol + - frs_job + - frs_maint + - frs_mortgage + - frs_oddjob + - frs_penprov + - frs_pension observed_geography: region assembly: mass_anchor_channel: frs diff --git a/packages/microcosm-build/src/microcosm/build/uk/spec/vintages.yaml b/packages/microcosm-build/src/microcosm/build/uk/spec/vintages.yaml index 28d16a395..5431701d5 100644 --- a/packages/microcosm-build/src/microcosm/build/uk/spec/vintages.yaml +++ b/packages/microcosm-build/src/microcosm/build/uk/spec/vintages.yaml @@ -1,16 +1,16 @@ records: -- id: uk_candidate_2023 +- id: uk_frs_2024_25 kind: survey_period_ref authority_ref: kind: source_record - source: source:uk_national_candidate_2023 - authority: uk_candidate_2023 + source: source:frs_adult + authority: uk_frs_2024_25 compatible_with: - - vintage:uk_target_2023 -- id: uk_target_2023 + - vintage:uk_target_2025 +- id: uk_target_2025 kind: target_period_ref authority_ref: kind: dataset_run pointer: /dataset_run/target_period compatible_with: - - vintage:uk_candidate_2023 + - vintage:uk_frs_2024_25 diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/__init__.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/__init__.py index 07d05bec4..a634730cd 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/__init__.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/__init__.py @@ -122,17 +122,13 @@ add_frs_employment, derive_frs_employment, ) -from microcosm.build.uk_runtime.frs_hmrc_leaves import ( +from microcosm.build.uk_runtime.frs_hmrc_source import ( FRS_HMRC_INCPBEN_COLUMN, FRS_HMRC_OSSBEN_IDENTIFIABLE_SUBSET_COLUMN, FRS_HMRC_PAY_COLUMN, FRS_HMRC_RETAINED_LEAF_COLUMNS, - FRS_HMRC_RETAINED_LEAVES_STAGE_NAME, FRS_HMRC_SRP_REGULAR_CODE5_COLUMN, FRS_HMRC_UBISJA_COLUMN, - UKFRSHMRCRetainedLeavesResult, - UKFRSHMRCRetainedLeavesStageTransform, - retain_uk_frs_hmrc_leaves, ) from microcosm.build.uk_runtime.frs_legacy_proxies import ( FRS_LEGACY_PROXY_OUTPUT_COLUMNS, @@ -257,7 +253,6 @@ ) from microcosm.build.uk_runtime.hmrc_source_contract import ( HMRC_DISTRIBUTIONAL_INPUTS, - UK_HMRC_INCOME_SOURCE_STAGES_RESOURCE, assert_uk_hmrc_income_source_contract_current, ) from microcosm.build.uk_runtime.ladder_targets import ( @@ -642,7 +637,6 @@ "FRS_HMRC_OSSBEN_IDENTIFIABLE_SUBSET_COLUMN", "FRS_HMRC_PAY_COLUMN", "FRS_HMRC_RETAINED_LEAF_COLUMNS", - "FRS_HMRC_RETAINED_LEAVES_STAGE_NAME", "FRS_HMRC_SRP_REGULAR_CODE5_COLUMN", "FRS_HMRC_UBISJA_COLUMN", "FRS_REGION_TO_COUNTRY", @@ -708,7 +702,6 @@ "SPI_SYNTHETIC_SUPPORT_CHANNEL", "UK_ENGLAND_WALES_REGION_CODES", "UK_GEOGRAPHY_LADDER_COLUMNS", - "UK_HMRC_INCOME_SOURCE_STAGES_RESOURCE", "UK_LONDON_REGION_CODE", "UK_LOADER_INPUT_ALIASES", "UK_OA_LADDER_DERIVED_LAYERS", @@ -729,8 +722,6 @@ "UKFirmTargetLayout", "UKFirmVATRuleEvaluator", "UKFirmValidationReport", - "UKFRSHMRCRetainedLeavesResult", - "UKFRSHMRCRetainedLeavesStageTransform", "UKLadderRowwiseDatasetResult", "UKLocalSolveDoctrine", "UKRowwiseLocalMatrix", @@ -832,7 +823,6 @@ "frozen_vs_recomputed", "impute_uk_spi_income_support", "replace_uk_spi_support_tables", - "retain_uk_frs_hmrc_leaves", "add_frs_council_tax", "add_frs_disability", "add_frs_education", diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/calibration_run.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/calibration_run.py index 7898ddc59..b51ec3cda 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/calibration_run.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/calibration_run.py @@ -1,4 +1,4 @@ -"""Production UK national calibration seam orchestration.""" +"""UK shared gate scopes, provenance and bound-spine validation helpers.""" from __future__ import annotations @@ -9,55 +9,25 @@ import json import os import platform -import time -import uuid from collections.abc import Mapping -from dataclasses import dataclass -from datetime import UTC, datetime from importlib import metadata from pathlib import Path -from types import SimpleNamespace from typing import Any import numpy as np from microcosm.build.country_spec import GatesManifest, load_country_spec -from microcosm.build.gate_battery import ( - BlockingMode, - EvidenceContext, - GateBatteryRun, - gate_signing_key_env, -) from microcosm.build.gate_battery import ( _canonical_json_bytes as gate_battery_canonical_json_bytes, ) -from microcosm.build.logbook import canonical_json_bytes -from microcosm.build.logbook_adoption import ( - AttemptState, - append_phase, - apply_error_verdict, - error_receipt_path, - git_code_pin, - local_artifact_reference, - record_terminal_attempt, - resolve_predecessor, - role_pins_digest, - write_error_receipt, -) -from microcosm.build.target_materialization import assert_calibration_input_finite -from microcosm.build.uk_runtime.battery_bindings import UK_GATE_REGISTRY -from microcosm.build.uk_runtime.diagnostics import ( - uk_target_geography_levels, - write_uk_calibration_diagnostics, +from microcosm.build.gate_battery import ( + gate_signing_key_env, ) from microcosm.build.uk_runtime.etb_services import ( UK_NHS_SPENDING_COMPONENT_COLUMNS, ) -from microcosm.build.uk_runtime.national_calibration import UKNationalCalibrationStage from microcosm.build.uk_runtime.national_frame import ( - load_uk_national_frame, uk_household_weight_kind, - write_uk_national_frame, ) from microcosm.calibrate import TargetRegistry from microcosm.frame import Frame @@ -66,28 +36,6 @@ # The FRS line's spine, staging, imputation and calibration stages share one # hash chain (logbook/README.md): the dataset token names the base data, not # the build mechanism, so calibration derives the ratified `uk/frs` scope. -_PIPELINE = "uk-frs-calibration" - - -@dataclass(frozen=True) -class UKCalibrationRunPaths: - input_h5: Path - staging_h5: Path - diagnostics_json: Path - build_record_json: Path - terminal_gate_json: Path - - -@dataclass(frozen=True) -class UKCalibrationRunResult: - frame: Frame - diagnostics_sha256: str - staging_sha256: str - build_record_sha256: str - terminal_gate_sha256: str - logbook_spool: Path - gate_report: Mapping[str, object] - build_record: Mapping[str, object] UK_CALIBRATION_GATE_SCOPE = ( @@ -245,114 +193,6 @@ def uk_local_gate_scope_exclusions() -> dict[str, str]: return exclusions -def run_uk_calibration( - *, - paths: UKCalibrationRunPaths, - input_sha256: str, - ledger_artifact: Any, - register_registry: TargetRegistry, - band_edge_registry: TargetRegistry, - calibration_year: int, - exclusion_receipt: Mapping[str, Mapping[str, str]], - doctrine: Any, - doctrine_overrides: Mapping[str, Mapping[str, object]], - measure_resolver: object | None, - source_pins: Mapping[str, Mapping[str, object]], - run_config_extra: Mapping[str, object], - release_id: str, - logbook_prev_row_digest: str | None = None, -) -> UKCalibrationRunResult: - """Run the UK national calibration seam and write its sidecars.""" - - started_at = time.perf_counter() - started_ts = datetime.now(UTC) - # Pure-argument validation precedes every environment probe: an - # incoherent register/receipt/band-edge triple must refuse identically - # whether or not a git checkout or Logbook chain is reachable. - _validate_band_edge_registry( - register_registry=register_registry, - band_edge_registry=band_edge_registry, - exclusion_receipt=exclusion_receipt, - ) - edge_registry = band_edge_registry - code_pin = git_code_pin(_REPOSITORY) - # Predecessor configuration is validated before anything is written: a - # disagreeing chain must refuse with no artifact on disk, not after a - # staged H5, diagnostics and a signed gate report already exist. - predecessor = resolve_predecessor(logbook_prev_row_digest) - run_config = { - "pipeline": _PIPELINE, - "release_id": release_id, - "register_sha256": register_registry.version, - "calibration_year": int(calibration_year), - "doctrine": _doctrine_payload(doctrine), - "doctrine_overrides": dict(doctrine_overrides), - # The caller verifies the feed's facts and manifest digests; sealing - # the verified identity into run_config carries it through the - # identity digest, the build record and the Logbook row, so the run - # says which Ledger artifact it was measured against. - "ledger": _ledger_provenance(ledger_artifact), - **dict(run_config_extra), - } - run_config["band_edge_register_sha256"] = edge_registry.version - state = AttemptState( - # Attempts are distinct rows even when they re-run one release: both - # the local chain and the store refuse a repeated build id. - build_id=_new_calibration_attempt_id(timestamp=started_ts), - identity_digest=hashlib.sha256(canonical_json_bytes(run_config)).hexdigest(), - input_pins_digest=role_pins_digest(source_pins), - phases_reached=["attempt_started"], - gate_verdicts={}, - ) - spool_dir = paths.staging_h5.parent / "logbook-spool" - try: - return _run_uk_calibration_attempt( - paths=paths, - input_sha256=input_sha256, - ledger_artifact=ledger_artifact, - register_registry=register_registry, - band_edge_registry=edge_registry, - calibration_year=calibration_year, - exclusion_receipt=exclusion_receipt, - doctrine=doctrine, - doctrine_overrides=doctrine_overrides, - measure_resolver=measure_resolver, - source_pins=source_pins, - release_id=release_id, - state=state, - run_config=run_config, - code_pin=code_pin, - started_at=started_at, - started_ts=started_ts, - predecessor=predecessor, - spool_dir=spool_dir, - ) - except BaseException as error: - # Every terminal disposition records a row — successful, failed, or - # refused (logbook/README.md). A refusal that left no row would be a - # silent gap in the chain the run is supposed to evidence. - _record_failed_attempt( - error=error, - state=state, - started_at=started_at, - started_ts=started_ts, - seed=getattr(doctrine, "seed", None), - code_pin=code_pin, - predecessor=predecessor, - receipt_base_dir=paths.staging_h5.parent, - spool_dir=spool_dir, - ) - raise - - -def _new_calibration_attempt_id(*, timestamp: datetime) -> str: - instant = timestamp.astimezone(UTC) - return ( - "uk-frs-calibration-attempt-" - f"{instant.strftime('%Y%m%dT%H%M%SZ')}-{uuid.uuid4().hex[:8]}" - ) - - def _validate_band_edge_registry( *, register_registry: TargetRegistry, @@ -392,281 +232,133 @@ def _registry_spec_names(registry: TargetRegistry) -> set[str]: return {str(spec.name) for spec in registry.specs} -def _record_failed_attempt( - *, - error: BaseException, - state: AttemptState, - started_at: float, - started_ts: datetime, - seed: int | None, - code_pin: str, - predecessor: str | None, - receipt_base_dir: Path, - spool_dir: Path, -) -> None: - if state.spool_path is not None: - return - error_path = write_error_receipt( - error_receipt_path(receipt_base_dir, build_id=state.build_id), - state=state, - pipeline=_PIPELINE, - error=error, - ) - apply_error_verdict( - state, - f"{local_artifact_reference(error_path, repository_hint=_REPOSITORY)}" - "#/error_type", - ) - record_terminal_attempt( - state=state, - started_at=started_at, - started_ts=started_ts, - pipeline=_PIPELINE, - rung="f100", - seed=seed, - code_pin=code_pin, - # An operator interrupt is a discarded attempt, not a failed one; the - # row says which so the chain reads honestly. - disposition=("discarded" if isinstance(error, KeyboardInterrupt) else "failed"), - predecessor=predecessor, - spool_dir=spool_dir, - ) - - -def _run_uk_calibration_attempt( - *, - paths: UKCalibrationRunPaths, - input_sha256: str, - ledger_artifact: Any, - register_registry: TargetRegistry, - band_edge_registry: TargetRegistry, - calibration_year: int, - exclusion_receipt: Mapping[str, Mapping[str, str]], - doctrine: Any, - doctrine_overrides: Mapping[str, Mapping[str, object]], - measure_resolver: object | None, - source_pins: Mapping[str, Mapping[str, object]], - release_id: str, - state: AttemptState, - run_config: Mapping[str, object], - code_pin: str, - started_at: float, - started_ts: datetime, - predecessor: str | None, - spool_dir: Path, -) -> UKCalibrationRunResult: - measured_input_sha = _sha256_file(paths.input_h5) - if measured_input_sha != input_sha256: - raise ValueError( - "input H5 sha mismatch: " - f"measured {measured_input_sha}, pinned {input_sha256}" - ) - append_phase(state, "input_sha_verified") - frame, _provenance = load_uk_national_frame(paths.input_h5) - append_phase(state, "input_loaded") - spine_sidecar_path = paths.input_h5.with_suffix(".build.json") - spine_sidecar = load_bound_spine_sidecar(spine_sidecar_path, frame) - append_phase(state, "input_sidecar_bound") - assert_calibration_input_finite(frame) - append_phase(state, "input_finite") - - stage = UKNationalCalibrationStage( - register_registry, - # The declared calibration year the register was compiled at — the - # stage validates it; there is deliberately no fallback to the input - # frame's base-year time_period or any ambient default. - period=calibration_year, - doctrine=doctrine, - measure_resolver=measure_resolver, - band_edge_registry=band_edge_registry, - ) - calibrated = stage(frame) - append_phase(state, "national_calibration_solved") - - build_block = { - "build_id": state.build_id, - "code_pin": code_pin, - # Captured at solve time and signed with the diagnostics bytes, so a - # release assembler can only ever pin the environment that actually - # calibrated the candidate — never an invented one. - "runtime": _runtime_provenance(), - "source_pins": dict(source_pins), - "ledger": run_config["ledger"], - "input_posture": { - "tier": "staging_candidate", - "sha256": measured_input_sha, - "size_bytes": paths.input_h5.stat().st_size, - }, - "doctrine": _doctrine_payload(doctrine), - "doctrine_overrides": dict(doctrine_overrides), - "measure_exclusions": dict(exclusion_receipt), - "measure_resolution": ( - stage.manifest.get("measure_resolution") - if isinstance(stage.manifest, Mapping) - else None - ), - "register": _register_census(register_registry, exclusion_receipt), - "spine_provenance": spine_provenance_from_sidecar( - spine_sidecar_path, - spine_sidecar, - ), - "score_vs_enhanced_frs": None, - } - write_uk_calibration_diagnostics( - stage.solve_result, - paths.diagnostics_json, - calibrated, - target_geography_levels=uk_target_geography_levels(stage.registry), - target_registry=stage.registry, - build=build_block, - ) - diagnostics_sha = _sha256_file(paths.diagnostics_json) - append_phase(state, "diagnostics_written") - - gate_report = _run_calibration_gate_battery( - calibrated, - stage, - paths.terminal_gate_json, - release_id=release_id, - diagnostics_sha256=diagnostics_sha, - ) - append_phase(state, "calibration_gates_evaluated") - for gate_id, payload in gate_report["gates"].items(): - state.gate_verdicts[gate_id] = { - "verdict": payload["status"], - "receipt": f"local://{paths.terminal_gate_json.name}#/gates/{gate_id}", - } - - write_uk_national_frame(calibrated, paths.staging_h5) - staging_sha = _sha256_file(paths.staging_h5) - append_phase(state, "staging_h5_written") - - record = { - "schema_version": 1, - "pipeline": _PIPELINE, - "build_id": state.build_id, - "run_config": run_config, - "source_pins": dict(source_pins), - "role_pins_digest": role_pins_digest(source_pins), - "input_posture": build_block["input_posture"], - "spine_provenance": build_block["spine_provenance"], - "register": build_block["register"], - "calibration": stage.manifest, - "gate_summary": _gate_summary(gate_report), - # No shippability claim lives here: the calibration-scoped battery - # covers 6 of the declared gate entries. The release verdict is the - # release-cut certification's, produced over this record. - "certification": { - "expected_artifact": str( - paths.staging_h5.with_suffix(".release_certification.json") - ), - "producer": "tools/certify_uk_release_cut.py", - }, - "artifacts": { - "staging_h5": {"path": str(paths.staging_h5), "sha256": staging_sha}, - "diagnostics_json": { - "path": str(paths.diagnostics_json), - "sha256": diagnostics_sha, - }, - "terminal_gate_json": { - "path": str(paths.terminal_gate_json), - "sha256": _sha256_file(paths.terminal_gate_json), - }, - }, - } - _write_json(paths.build_record_json, record) - build_record_sha = _sha256_file(paths.build_record_json) - append_phase(state, "build_record_written") - state.artifact_location = local_artifact_reference( - paths.staging_h5, repository_hint=_REPOSITORY - ) - spool = record_terminal_attempt( - state=state, - started_at=started_at, - started_ts=started_ts, - pipeline=_PIPELINE, - rung="f100", - seed=getattr(doctrine, "seed", None), - code_pin=code_pin, - disposition="iterating", - predecessor=predecessor, - spool_dir=spool_dir, - ) - return UKCalibrationRunResult( - frame=calibrated, - diagnostics_sha256=diagnostics_sha, - staging_sha256=staging_sha, - build_record_sha256=build_record_sha, - terminal_gate_sha256=_sha256_file(paths.terminal_gate_json), - logbook_spool=spool, - gate_report=gate_report, - build_record=record, - ) +def load_bound_spine_sidecar(path: Path, frame: Frame) -> dict[str, object]: + if not path.is_file(): + raise ValueError(f"input H5 build sidecar absent: {path}") + try: + sidecar = json.loads(path.read_text(encoding="utf-8")) + except json.JSONDecodeError as exc: + raise ValueError(f"input H5 build sidecar is invalid JSON: {path}") from exc + if not isinstance(sidecar, dict): + raise ValueError(f"input H5 build sidecar must be a JSON object: {path}") + _assert_spine_sidecar_binds_frame(sidecar, frame) + _assert_spine_gate_report_passed(_spine_gate_report_path(path), sidecar) + return sidecar -def _run_calibration_gate_battery( - frame: Frame, - stage: UKNationalCalibrationStage, +def load_bound_spine_checkpoint( path: Path, + frame: Frame, *, - release_id: str, - diagnostics_sha256: str, + gate_report_path: Path | None = None, ) -> dict[str, object]: - manifest = _calibration_gate_manifest() - admin_totals, admin_receipt = uk_aggregate_admin_totals(frame, manifest) - artifacts = { - "national_calibration": stage.manifest, - "parity_evidence": SimpleNamespace( - target_relative_errors={ - str(row["name"]): float(row["relative_error"]) - for row in stage.diagnostics - } - ), - "aggregate_admin": admin_totals, - # The target-fit deferral register is evaluated against the run - # clock (schema-2 approval windows); the seam supplies today's date - # exactly as the rowwise candidate build supplies its start date. - "exclusions_evaluated_on": datetime.now(UTC).date(), - } - battery = GateBatteryRun( - manifest, - release_id=release_id, - # The seam never runs release-candidate posture: its scoped battery - # covers 6 of the declared entries and must never sign a - # shippability claim (the #757 release-cut audit). Shippability - # comes only from the release-cut certification. - report_path=path, - release_candidate=False, - registry=UK_GATE_REGISTRY, - release_evidence={"calibration_diagnostics_sha256": diagnostics_sha256}, - ) - battery.run_phase("terminal", EvidenceContext(frame=frame, artifacts=artifacts)) - battery.enforce("terminal", mode=BlockingMode.BLOCKS_ARTIFACT) - payload = battery.report_payload() - finalize_uk_scoped_gate_report( - payload, - posture="calibration_seam", - scope_exclusions=dict(UK_CALIBRATION_GATE_SCOPE_EXCLUSIONS), - aggregate_admin_measurement=admin_receipt, - ) - _write_json(path, payload) - return payload + """Authenticate a canonical graph checkpoint, without historical bypasses.""" + from microcosm.build.uk_runtime.content_identity import uk_frame_content_identity - -def load_bound_spine_sidecar(path: Path, frame: Frame) -> dict[str, object]: + path = Path(path) if not path.is_file(): raise ValueError(f"input H5 build sidecar absent: {path}") try: - sidecar = json.loads(path.read_text(encoding="utf-8")) - except json.JSONDecodeError as exc: + sidecar = json.loads(path.read_bytes()) + except (json.JSONDecodeError, UnicodeError) as exc: raise ValueError(f"input H5 build sidecar is invalid JSON: {path}") from exc if not isinstance(sidecar, dict): raise ValueError(f"input H5 build sidecar must be a JSON object: {path}") _assert_spine_sidecar_binds_frame(sidecar, frame) - _assert_spine_gate_report_passed(_spine_gate_report_path(path), sidecar) + identity = sidecar.get("uk_frame_content_identity") + if not isinstance(identity, str) or not identity: + raise ValueError("Unbound spine checkpoint: no uk_frame_content_identity.") + if identity != uk_frame_content_identity(frame): + raise ValueError("Spine checkpoint uk_frame_content_identity mismatch.") + _strict_spine_gate_report(path, sidecar, gate_report_path=gate_report_path) return sidecar +def _strict_spine_gate_report( + path: Path, + sidecar: Mapping[str, object], + *, + gate_report_path: Path | None = None, +) -> tuple[Path, dict[str, object]]: + if sidecar.get("spine_gate_bypass") is not None: + raise ValueError("Canonical spine checkpoints do not accept spine_gate_bypass.") + report_path = ( + _spine_gate_report_path(Path(path)) + if gate_report_path is None + else Path(gate_report_path) + ) + binding = sidecar.get("spine_gate_report") + if not isinstance(binding, Mapping) or not binding.get("sha256"): + raise ValueError("Unbound spine checkpoint: no spine gate report SHA-256.") + if not report_path.is_file(): + raise ValueError(f"input H5 spine gate report absent: {report_path}") + report_bytes = report_path.read_bytes() + if hashlib.sha256(report_bytes).hexdigest() != binding["sha256"]: + raise ValueError("Spine checkpoint gate report SHA-256 mismatch.") + _assert_spine_gate_report_passed(report_path, sidecar) + report = json.loads(report_bytes) + for field, expected in uk_spine_checkpoint_gate_digests().items(): + if report.get(field) != expected: + raise ValueError( + f"Spine checkpoint gate report {field} differs from current declarations." + ) + gates = report["gates"] + expected = set(UK_SPINE_GATE_SCOPE) + if set(gates) != expected or any( + not isinstance(gates[gate_id], Mapping) for gate_id in gates + ): + raise ValueError( + "Spine checkpoint gate report differs from the declared spine scope." + ) + declared = { + entry.id: entry + for entry in load_country_spec("uk").gates.gates + if entry.id in expected + } + for gate_id, entry in declared.items(): + outcome = gates[gate_id] + if outcome.get("criticality") != entry.criticality: + raise ValueError(f"Spine checkpoint gate {gate_id} criticality mismatch.") + if ( + entry.criticality == "release_blocking" + and outcome.get("status") != "passed" + ): + raise ValueError(f"Spine checkpoint gate {gate_id} did not pass.") + return report_path, report + + +def uk_spine_checkpoint_gate_digests() -> dict[str, str]: + """Declare the current spine gate policy as part of checkpoint identity.""" + from microcosm.build.uk_runtime.release_certification import _scoped_digests + + return _scoped_digests( + frozenset(UK_SPINE_GATE_SCOPE), + phases=("assembled", "transferred"), + policy_suffix="spine_build_scope", + ) + + +def strict_spine_provenance_from_sidecar( + path: Path, + sidecar: Mapping[str, object], + *, + gate_report_path: Path | None = None, +) -> dict[str, object]: + """Retain exact gate bytes and fit records after strict checkpoint loading.""" + report_path, report = _strict_spine_gate_report( + path, sidecar, gate_report_path=gate_report_path + ) + provenance = spine_provenance_from_sidecar(path, sidecar) + provenance["uk_frame_content_identity"] = sidecar["uk_frame_content_identity"] + provenance["fit_weight_records"] = dict(sidecar.get("fit_weight_records", {})) + provenance["spine_gate_report"] = { + "path": str(report_path), + "sha256": sidecar["spine_gate_report"]["sha256"], + "payload": report, + } + return provenance + + def _assert_spine_sidecar_binds_frame( sidecar: Mapping[str, object], frame: Frame, @@ -1024,47 +716,6 @@ def runtime_provenance() -> dict[str, str]: _runtime_provenance = runtime_provenance -def _register_census( - registry: TargetRegistry, exclusions: Mapping[str, Mapping[str, str]] -) -> dict[str, object]: - return { - "country": registry.country, - "version": registry.version, - "compiled_count": len(registry.specs) + len(exclusions), - "excluded_count": len(exclusions), - "calibrated_count": len(registry.specs), - } - - -def _doctrine_payload(doctrine: Any) -> dict[str, object]: - return { - key: getattr(doctrine, key) - for key in ( - "epochs", - "learning_rate", - "max_weight_ratio", - "seed", - "target_loss_cap", - "scale_rule", - "target_weight_rule", - "mass_rule", - "l0_lambda", - ) - if hasattr(doctrine, key) - } - - -def _gate_summary(report: Mapping[str, object]) -> dict[str, object]: - gates = report.get("gates", {}) - if not isinstance(gates, Mapping): - return {} - return { - gate_id: payload.get("status") - for gate_id, payload in gates.items() - if isinstance(payload, Mapping) - } - - def finalize_uk_scoped_gate_report( payload: dict[str, object], *, @@ -1124,13 +775,3 @@ def _sha256_file(path: Path) -> str: for chunk in iter(lambda: stream.read(1 << 20), b""): digest.update(chunk) return digest.hexdigest() - - -def _write_json(path: Path, payload: Mapping[str, object]) -> None: - path.parent.mkdir(parents=True, exist_ok=True) - temporary = path.with_name(path.name + ".tmp") - temporary.write_text( - json.dumps(payload, indent=2, sort_keys=True, allow_nan=False) + "\n", - encoding="utf-8", - ) - temporary.replace(path) diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/country_adapter.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/country_adapter.py new file mode 100644 index 000000000..19c657cdf --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/country_adapter.py @@ -0,0 +1,134 @@ +"""Bridge the UK country declarations to the canonical executable full graph. + +The F0 compiler remains a static schema/identity projection. This adapter +validates its source projection, then uses the existing UK graph composition; +it never loads a historical candidate H5 or creates a second execution graph. +""" + +from __future__ import annotations + +import hashlib +from dataclasses import replace +from datetime import date +from importlib import metadata +from typing import TYPE_CHECKING + +from microcosm.build.country_spec import CountrySpec, load_country_spec +from microcosm.build.uk_runtime.frs_release import load_uk_frs_release + +if TYPE_CHECKING: + from .graph_build import UKFullBuildConfig, UKFullGraph + + +def validate_uk_country_source_projection(spec: CountrySpec) -> None: + """Keep raw input declarations and their executable stage pins identical.""" + if spec.country != "uk" or spec.resolved_spec is None: + raise ValueError( + "The UK full-build adapter requires a resolved UK country spec." + ) + resolved = spec.resolved_spec + sources = resolved.resource("sources").domain.to_wire() + spine = resolved.resource("spine").domain.to_wire() + bundle = resolved.resource("bundle").domain.to_wire() + release = load_uk_frs_release() + root = next(stage for stage in spec.sources.stages if stage.stage == "frs_spine") + vintage = f"uk_frs_{release.vintage}" + expected = [ + { + "id": f"frs_{artifact['table']}", + "role": "frs_raw_table", + "sha256": artifact["sha256"], + "byte_size": artifact["size_bytes"], + "loader": "kernel:build_uk_frs_spine", + "vintages": [f"vintage:{vintage}"], + **( + { + "vintage_authorities": [ + { + "id": vintage, + "kind": "survey_period", + "value": release.survey_year, + } + ] + } + if artifact["table"] == "adult" + else {} + ), + } + for artifact in root.artifacts + if artifact["role"] == "frs_table" + ] + if sources["sources"] != expected: + raise ValueError( + "UK country raw-source pins differ from the canonical FRS spine stage." + ) + if spine["channels"] != [ + { + "id": "frs", + "source": [row["id"] for row in expected], + "observed_geography": "region", + } + ]: + raise ValueError("UK FRS channel must consume the declared raw FRS tables.") + if bundle["dataset_run"]["target_period"] != release.calibration_year: + raise ValueError( + "UK country target period differs from the FRS release calibration year." + ) + + +def build_uk_country_graph( + config: UKFullBuildConfig | None = None, + *, + spec: CountrySpec | None = None, + engine_identity: str | None = None, + review_date: date | None = None, + release_candidate: bool = False, + skip_holdout: bool = False, +) -> UKFullGraph: + """Compile the canonical full graph; default target scope is all geographies.""" + from microcosm.graph import compile_graph + from microcosm.graph.canonical import canonical_json + + from .graph import uk_spine_graph + from .graph_build import UKFullBuildConfig, uk_full_graph + from .graph_evidence import add_uk_spine_gate_nodes + from .graph_terminal import append_uk_full_gate_nodes + + spec = load_country_spec("uk") if spec is None else spec + validate_uk_country_source_projection(spec) + release = load_uk_frs_release() + if config is None: + config = UKFullBuildConfig( + calibration_year=release.calibration_year, + time_period=release.time_period, + source_year=release.survey_year, + ) + if engine_identity is None: + engine_identity = hashlib.sha256( + canonical_json( + { + "package": "policyengine-uk", + "version": metadata.version("policyengine-uk"), + } + ) + ).hexdigest() + review_date = date.today() if review_date is None else review_date + spine = add_uk_spine_gate_nodes( + uk_spine_graph(spec, source_mode="split"), + spec=spec, + engine_identity=engine_identity, + release_candidate=release_candidate, + ) + full = uk_full_graph(config, spine=spine, review_date=review_date.isoformat()) + graph = append_uk_full_gate_nodes( + full.graph, + calibration=full.calibration, + spine_stage_names=tuple(stage.stage for stage in spec.sources.stages), + engine_identity=engine_identity, + review_date=review_date, + sample_fraction=config.effective_sample_fraction, + release_candidate=release_candidate, + skip_holdout=skip_holdout, + ) + compile_graph(graph) + return replace(full, graph=graph) diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/dataset_size.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/dataset_size.py index a358d5fd8..4cca8be91 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/dataset_size.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/dataset_size.py @@ -1,5 +1,6 @@ """Exact household-count UK candidates on a fixed, materialized target surface.""" +import hashlib import json from collections.abc import Callable from dataclasses import dataclass, replace @@ -53,6 +54,78 @@ class UKSizeSelection: search_pi_hi: float +@dataclass(frozen=True) +class UKSizeDraw: + """An executed exact-count draw, independently reusable by the refit.""" + + support: np.ndarray + sampling: dict[str, object] + inclusion_probabilities: np.ndarray + feasibility: dict[str, object] + seed: int + pi_hi: float + probabilities_sha256: str + + +def _probability_digest(probabilities: np.ndarray) -> str: + return hashlib.sha256(np.asarray(probabilities, dtype=" UKSizeDraw: + """Execute only the existing exact-count draw, without search or refit.""" + n = _check_size_inputs(frame, dense, households) + pi_hi = _check_pi_hi(pi_hi) + probabilities = selection.selection.gate_open_probabilities + if ( + selection.households != households + or selection.seed != seed + or probabilities is None + or len(probabilities) != n + or len(selection.protected) != n + ): + raise ValueError("Exact-count draw requires its aligned selection and seed.") + search_result = selection.selection + feasibility = selection_feasibility( + probabilities, + households, + protected=selection.protected, + n_nonzero=int(search_result.n_nonzero), + l0_lambda=float(search_result.l0_lambda), + requested_pi_hi=pi_hi, + budget_search=search_result.options.get("budget_search"), + search_pi_hi=selection.search_pi_hi, + ) + try: + support, sampling, q = select_exact_k( + probabilities, households, pi_hi=pi_hi, seed=seed + ) + except ValueError as error: + raise ValueError( + f"{error} Selection feasibility (requested pi_hi={pi_hi:g}): " + f"{json.dumps(feasibility, sort_keys=True)}" + ) from error + support = assert_exact_k_support(support, households, pool_size=n) + if not np.isin(np.flatnonzero(selection.protected), support).all(): + raise RuntimeError("exact-count selection lost a protected carrier.") + return UKSizeDraw( + support, + sampling, + q, + feasibility, + seed, + pi_hi, + _probability_digest(probabilities), + ) + + ProgressCallback = Callable[[dict[str, object]], None] @@ -204,6 +277,7 @@ def refit_uk_dataset_size( seed: int, pi_hi: float = 1.0, selection: UKSizeSelection | None = None, + draw: UKSizeDraw | None = None, progress_callback: ProgressCallback | None = None, ) -> UKDatasetSize: """Run informed L0, a fixed-size draw, and refit under the dense doctrine. @@ -224,6 +298,8 @@ def refit_uk_dataset_size( :class:`UKSizeSelection` (a checkpoint restored by :mod:`microcosm.build.uk_runtime.size_checkpoint`); it must have been searched for the same size, epochs, learning rate and seed on this pool. + ``draw`` additionally reuses an authenticated completed draw without + consuming its random stream again; the probability binding must match. """ n = _check_size_inputs(frame, dense, households) pi_hi = _check_pi_hi(pi_hi) @@ -287,28 +363,34 @@ def refit_uk_dataset_size( probabilities = selection.selection.gate_open_probabilities assert probabilities is not None search_result = selection.selection - feasibility = selection_feasibility( - probabilities, - households, - protected=init_protected, - n_nonzero=int(search_result.n_nonzero), - l0_lambda=float(search_result.l0_lambda), - requested_pi_hi=pi_hi, - budget_search=search_result.options.get("budget_search"), - search_pi_hi=selection.search_pi_hi, - ) - try: - support, sampling, q = select_exact_k( - probabilities, households, pi_hi=pi_hi, seed=seed + if draw is None: + draw = draw_uk_dataset_size( + frame, + dense, + selection=selection, + households=households, + seed=seed, + pi_hi=pi_hi, ) - except ValueError as error: - # The draw refuses rather than clamps; carry the measured gate mass - # with the refusal so the ruling it needs can be made from the receipt. - raise ValueError( - f"{error} Selection feasibility (requested pi_hi={pi_hi:g}): " - f"{json.dumps(feasibility, sort_keys=True)}" - ) from error - support = assert_exact_k_support(support, households, pool_size=n) + if ( + draw.seed != seed + or draw.pi_hi != pi_hi + or draw.probabilities_sha256 != _probability_digest(probabilities) + ): + raise ValueError("Reused exact-count draw differs from its selection or seed.") + support = assert_exact_k_support(draw.support, households, pool_size=n) + sampling, q, feasibility = ( + draw.sampling, + draw.inclusion_probabilities, + draw.feasibility, + ) + if ( + np.asarray(q).shape != (households,) + or not np.isfinite(q).all() + or (q <= 0).any() + or (q > 1).any() + ): + raise ValueError("Reused exact-count draw has invalid inclusion probabilities.") if not np.isin(np.flatnonzero(init_protected), support).all(): raise RuntimeError("exact-count selection lost a protected carrier.") frozen = _frozen_targets(frame, dense, support) diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/frs_hmrc_leaves.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/frs_hmrc_leaves.py deleted file mode 100644 index 2154f2848..000000000 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/frs_hmrc_leaves.py +++ /dev/null @@ -1,893 +0,0 @@ -"""Retain adjudicated raw-FRS leaves for the UK HMRC income surface. - -The certified UK candidate was built from the 2023-24 FRS, but it does not -retain the raw constituents needed to compare its income measure with the -published HMRC tables. This stage reopens only ``adult.tab`` and -``benefits.tab`` and carries the source-faithful, adjudicated constituents -through the candidate's SPI, capital-gains, and geography-clone descendants. - -The two partial concepts deliberately keep subset names. They must never be -mistaken for the full SPI ``OSSBEN`` or ``SRP`` concepts. -""" - -from __future__ import annotations - -import hashlib -from collections.abc import Mapping -from dataclasses import dataclass, field -from pathlib import Path - -import numpy as np -import pandas as pd - -from microcosm.build.uk_runtime.content_identity import uk_frame_content_identity -from microcosm.build.uk_runtime.national_frame import ( - uk_household_weight_kind, - uk_national_frame, - uk_time_period, - validate_uk_national_frame, -) -from microcosm.build.uk_runtime.spi_support import ( - HOUSEHOLD_IS_SPI_SYNTHETIC_COLUMN, - SPI_HMRC_INCAPACITY_BENEFIT_INCOME_COLUMN, - SPI_HMRC_PAY_COLUMN, - SPI_HMRC_UNEMPLOYMENT_BENEFIT_INCOME_COLUMN, -) -from microcosm.frame import Frame - -__all__ = [ - "FRS_HMRC_INCPBEN_COLUMN", - "FRS_HMRC_OSSBEN_IDENTIFIABLE_SUBSET_COLUMN", - "FRS_HMRC_PAY_COLUMN", - "FRS_HMRC_RETAINED_LEAF_COLUMNS", - "FRS_HMRC_RETAINED_LEAF_SOURCE_EVIDENCE", - "FRS_HMRC_SRP_REGULAR_CODE5_COLUMN", - "FRS_HMRC_UBISJA_COLUMN", - "FRS_HMRC_RETAINED_LEAVES_STAGE_NAME", - "UKFRSHMRCRetainedLeavesResult", - "UKFRSHMRCRetainedLeavesStageTransform", - "UKFRSRawTableIdentity", - "retain_uk_frs_hmrc_leaves", -] - -FRS_SOURCE_VINTAGE = "2023-24" -FRS_SOURCE_BUILD_PERIOD = "2023" -FRS_WEEKS_IN_YEAR = 365.25 / 7 -FRS_HMRC_RETAINED_LEAVES_STAGE_NAME = "frs_hmrc_retained_leaves" - -# Full concepts use the normalized columns already consumed by the SPI/HMRC -# stage. Partial concepts are fenced under their adjudicated subset names. -FRS_HMRC_PAY_COLUMN = SPI_HMRC_PAY_COLUMN -FRS_HMRC_UBISJA_COLUMN = SPI_HMRC_UNEMPLOYMENT_BENEFIT_INCOME_COLUMN -FRS_HMRC_INCPBEN_COLUMN = SPI_HMRC_INCAPACITY_BENEFIT_INCOME_COLUMN -FRS_HMRC_OSSBEN_IDENTIFIABLE_SUBSET_COLUMN = "ossben_identifiable_subset" -FRS_HMRC_SRP_REGULAR_CODE5_COLUMN = "srp_regular_code5" -FRS_HMRC_RETAINED_LEAF_COLUMNS = ( - FRS_HMRC_PAY_COLUMN, - FRS_HMRC_UBISJA_COLUMN, - FRS_HMRC_INCPBEN_COLUMN, - FRS_HMRC_OSSBEN_IDENTIFIABLE_SUBSET_COLUMN, - FRS_HMRC_SRP_REGULAR_CODE5_COLUMN, -) - -FRS_HMRC_RETAINED_LEAF_SOURCE_EVIDENCE: dict[str, dict[str, object]] = { - FRS_HMRC_PAY_COLUMN: { - "spi_concept": "PAY", - "scope": "full", - "raw_sources": ["ADULT.INEARNS"], - "formula": "max(0, ADULT.INEARNS) * (365.25 / 7)", - }, - FRS_HMRC_UBISJA_COLUMN: { - "spi_concept": "UBISJA", - "scope": "full", - "raw_sources": [ - "BENEFITS.BENEFIT=14:BENAMT", - "BENEFITS.BENEFIT=19:BENAMT", - ], - "formula": "sum(BENAMT where BENEFIT in {14, 19}) * (365.25 / 7)", - }, - FRS_HMRC_INCPBEN_COLUMN: { - "spi_concept": "INCPBEN", - "scope": "full", - "raw_sources": ["BENEFITS.BENEFIT=17:BENAMT"], - "formula": "sum(BENAMT where BENEFIT == 17) * (365.25 / 7)", - }, - FRS_HMRC_OSSBEN_IDENTIFIABLE_SUBSET_COLUMN: { - "spi_concept": "OSSBEN", - "scope": "identifiable_subset", - "raw_sources": [ - "BENEFITS.BENEFIT=13:BENAMT", - "BENEFITS.BENEFIT=16,VAR2 in {1,3}:BENAMT", - ], - "formula": ( - "sum(BENAMT where BENEFIT == 13 or " - "(BENEFIT == 16 and VAR2 in {1, 3})) * (365.25 / 7)" - ), - }, - FRS_HMRC_SRP_REGULAR_CODE5_COLUMN: { - "spi_concept": "SRP", - "scope": "regular_code5_subset", - "raw_sources": ["BENEFITS.BENEFIT=5:BENAMT"], - "formula": "sum(BENAMT where BENEFIT == 5) * (365.25 / 7)", - }, -} - -_ADULT_REQUIRED_COLUMNS = ("sernum", "person", "inearns") -_BENEFITS_REQUIRED_COLUMNS = ( - "sernum", - "person", - "benefit", - "benamt", - "var2", -) -_CAPITAL_GAINS_FLAG = "household_is_capital_gains_clone" - - -@dataclass(frozen=True) -class UKFRSRawTableIdentity: - """Stable identity and extraction surface for one raw FRS table.""" - - path: Path - filename: str - source_vintage: str - sha256: str - size_bytes: int - rows: int - extracted_columns: tuple[str, ...] - - def evidence(self) -> dict[str, object]: - """Return JSON-safe source evidence.""" - - return { - "path": str(self.path), - "filename": self.filename, - "source_vintage": self.source_vintage, - "sha256": self.sha256, - "size_bytes": self.size_bytes, - "rows": self.rows, - "extracted_columns": list(self.extracted_columns), - } - - -@dataclass(frozen=True) -class UKFRSHMRCRetainedLeavesResult: - """National frame plus raw-source and lineage evidence. - - ``input_content_identity`` and ``output_content_identity`` are the - content identities (:func:`uk_frame_content_identity`) of the frame this - stage consumed and the frame it produced, derived inside the attesting - run. The SPI stage's descent fence compares against them, so the - guarantee survives a process boundary: a checkpoint-rehydrated frame is - content-identical to the one that was checkpointed, while a substituted - or tampered frame is not. - """ - - frame: Frame - adult_source: UKFRSRawTableIdentity - benefits_source: UKFRSRawTableIdentity - clone_id_multiplier: int - spi_person_id_offset: int - capital_gains_person_id_offset: int - raw_source_people: int - candidate_people: int - source_signal_rows: dict[str, int] - structural_zero_columns: tuple[str, ...] - input_content_identity: str - output_content_identity: str - #: Raw-survey people outside the candidate base. Zero on a full-scale - #: build (the completeness fence raises otherwise); on a #627 rung - #: sample it receipts how much of the raw surface the rung dropped. - source_people_outside_candidate: int = 0 - - def evidence(self) -> dict[str, object]: - """Return aggregate, JSON-safe evidence for a national build driver.""" - - return { - "stage": FRS_HMRC_RETAINED_LEAVES_STAGE_NAME, - "source_vintage": FRS_SOURCE_VINTAGE, - "mapped_build_period": uk_time_period(self.frame), - "sources": { - "adult": self.adult_source.evidence(), - "benefits": self.benefits_source.evidence(), - }, - "annualization": { - "days_per_year": 365.25, - "days_per_week": 7, - "weeks_per_year": FRS_WEEKS_IN_YEAR, - }, - "lineage": { - "clone_id_multiplier": self.clone_id_multiplier, - "spi_person_id_offset": self.spi_person_id_offset, - "capital_gains_person_id_offset": (self.capital_gains_person_id_offset), - "raw_source_people": self.raw_source_people, - "candidate_people": self.candidate_people, - "source_people_outside_candidate": ( - self.source_people_outside_candidate - ), - }, - "retained_leaves": { - column: { - **FRS_HMRC_RETAINED_LEAF_SOURCE_EVIDENCE[column], - "source_signal_rows": self.source_signal_rows[column], - "structural_zero": column in self.structural_zero_columns, - } - for column in FRS_HMRC_RETAINED_LEAF_COLUMNS - }, - } - - -@dataclass -class UKFRSHMRCRetainedLeavesStageTransform: - """Callable national-stage adapter retaining the last run's evidence.""" - - adult_tab_path: Path - benefits_tab_path: Path - #: Declared #627 rung build: relaxes the raw-surface completeness fence - #: into a receipted count. Never set on a release build. - sampled_rung: bool = False - last_result: UKFRSHMRCRetainedLeavesResult | None = field( - default=None, - init=False, - ) - - @classmethod - def from_raw_frs_directory( - cls, - raw_frs_directory: str | Path, - *, - sampled_rung: bool = False, - ) -> UKFRSHMRCRetainedLeavesStageTransform: - """Resolve the two permitted tables from a CLI-supplied directory.""" - - directory = Path(raw_frs_directory).expanduser() - return cls( - adult_tab_path=directory / "adult.tab", - benefits_tab_path=directory / "benefits.tab", - sampled_rung=sampled_rung, - ) - - def __call__(self, frame: Frame) -> Frame: - # The result records the content identities of the frame this stage - # consumed and produced, so the SPI stage's fence can assert descent - # from the frame the driver loaded and bound — including across a - # process boundary, where object identity cannot travel. - self.last_result = retain_uk_frs_hmrc_leaves( - frame, - adult_tab_path=self.adult_tab_path, - benefits_tab_path=self.benefits_tab_path, - sampled_rung=self.sampled_rung, - ) - return self.last_result.frame - - def checkpoint_metadata(self) -> dict[str, object]: - """JSON-safe evidence the stage checkpoint carries for a resume. - - The SPI stage consumes the retained-leaves evidence and the descent - identities; persisting them on the completed stage's run-context - record is what lets a later process resume past this stage without - re-running it. - """ - - if self.last_result is None: - raise RuntimeError( - "checkpoint metadata requires a completed retained-leaves run." - ) - return { - "evidence": self.last_result.evidence(), - "input_content_identity": self.last_result.input_content_identity, - "output_content_identity": self.last_result.output_content_identity, - } - - def resume_from_checkpoint( - self, - metadata: Mapping[str, object], - frame: Frame, - ) -> None: - """Rehydrate a completed run's evidence from its checkpoint record. - - ``frame`` is the stage's checkpointed output; the rehydrated result - exposes exactly the surface the SPI stage's descent fence reads. The - recorded output identity must match the loaded frame's content — a - mismatch means the record and the checkpoint have drifted apart, and - the resume fails closed. - """ - - evidence = metadata.get("evidence") - input_identity = metadata.get("input_content_identity") - output_identity = metadata.get("output_content_identity") - if ( - not isinstance(evidence, Mapping) - or not isinstance(input_identity, str) - or not isinstance(output_identity, str) - ): - raise RuntimeError( - "retained-leaves resume requires the checkpoint record to " - "carry the run's evidence and content identities; a record " - "without them cannot prove descent." - ) - if uk_frame_content_identity(frame) != output_identity: - raise RuntimeError( - "retained-leaves checkpoint content does not match its " - "recorded output identity; refusing to resume from a " - "drifted record." - ) - self.last_result = _ResumedRetainedLeaves( - frame=frame, - evidence_payload=dict(evidence), - input_content_identity=input_identity, - output_content_identity=output_identity, - ) - - -@dataclass(frozen=True) -class _ResumedRetainedLeaves: - """A completed retained-leaves run rehydrated from its checkpoint. - - Carries exactly the surface the SPI stage consumes: the output frame, - the JSON-safe evidence, and the descent content identities. - """ - - frame: Frame - evidence_payload: dict[str, object] - input_content_identity: str - output_content_identity: str - - def evidence(self) -> dict[str, object]: - return dict(self.evidence_payload) - - -@dataclass(frozen=True) -class _FileFingerprint: - device: int - inode: int - size_bytes: int - modified_ns: int - changed_ns: int - - -@dataclass(frozen=True) -class _CandidateLineage: - source_person_ids: np.ndarray - clone_id_multiplier: int - spi_person_id_offset: int - capital_gains_person_id_offset: int - canonical_raw_person_ids: frozenset[int] - - -def retain_uk_frs_hmrc_leaves( - frame: Frame, - *, - adult_tab_path: str | Path, - benefits_tab_path: str | Path, - sampled_rung: bool = False, -) -> UKFRSHMRCRetainedLeavesResult: - """Read two raw FRS tables and retain the adjudicated HMRC constituents. - - ``sampled_rung`` declares a #627 scale-ladder build: the candidate base - deliberately carries only a sampled subset of source families, so the - completeness fence (every raw-survey person present in the base) cannot - hold. The raw surface is restricted to surviving canonicals and the - dropped count is receipted instead — never silently. Full-scale builds - keep the strict fence. - """ - - validate_uk_national_frame(frame) - input_content_identity = uk_frame_content_identity(frame) - time_period = uk_time_period(frame) - if time_period not in {FRS_SOURCE_BUILD_PERIOD, FRS_SOURCE_VINTAGE}: - raise ValueError( - f"Raw FRS {FRS_SOURCE_VINTAGE} leaves may only map to build period " - f"{FRS_SOURCE_BUILD_PERIOD!r}; got {time_period!r}." - ) - - adult, adult_source = _read_raw_frs_table( - adult_tab_path, - expected_filename="adult.tab", - required_columns=_ADULT_REQUIRED_COLUMNS, - ) - benefits, benefits_source = _read_raw_frs_table( - benefits_tab_path, - expected_filename="benefits.tab", - required_columns=_BENEFITS_REQUIRED_COLUMNS, - ) - source_leaves = _materialize_source_leaves(adult, benefits) - lineage = _resolve_candidate_lineage(frame) - unknown_source_ids = sorted( - set(source_leaves.index) - lineage.canonical_raw_person_ids - ) - if unknown_source_ids and not sampled_rung: - raise ValueError( - "Raw FRS retained leaves contain person identity value(s) absent " - f"from the certified candidate base: {unknown_source_ids[:5]}." - ) - source_people_outside_candidate = len(unknown_source_ids) - # Signal-row evidence stays a fact about the SOURCE at every rung: - # structural_zero must never be asserted from a sampled-away surface - # (adversarial-review finding). The rung also cannot distinguish a - # compact genuinely missing raw people from sampling loss — that check - # remains the full-scale fence's, which stays strict. - full_source_leaves = source_leaves - if unknown_source_ids: - # A rung sample deliberately drops most source families; restrict the - # raw surface to the surviving canonicals and receipt the count. - source_leaves = source_leaves.loc[ - source_leaves.index.isin(list(lineage.canonical_raw_person_ids)) - ] - - person = frame.table("person").copy() - aligned = source_leaves.reindex(lineage.source_person_ids, fill_value=0.0) - if aligned.isna().any().any(): # pragma: no cover - defensive - raise RuntimeError("Raw FRS retained-leaf alignment produced missing values.") - values = aligned.to_numpy(dtype=float) - if not np.isfinite(values).all() or (values < 0.0).any(): - raise RuntimeError( - "Raw FRS retained-leaf alignment produced non-finite or negative values." - ) - for column in FRS_HMRC_RETAINED_LEAF_COLUMNS: - person[column] = aligned[column].to_numpy(dtype=float) - - # Person-only replacement: mass is untouched, so the kind and mass log - # carry through unchanged; Frame construction re-runs linkage validation. - result_frame = uk_national_frame( - person=person, - benunit=frame.table("benunit"), - household=frame.table("household"), - time_period=time_period, - weight_kind=uk_household_weight_kind(frame), - household_weights=frame.weights_for("household").values, - mass_log=frame.mass_log, - ) - validate_uk_national_frame(result_frame) - _validate_retained_leaf_propagation( - result_frame.table("person"), - source_person_ids=lineage.source_person_ids, - source_leaves=source_leaves, - ) - source_signal_rows = { - column: int((full_source_leaves[column] > 0.0).sum()) - for column in FRS_HMRC_RETAINED_LEAF_COLUMNS - } - structural_zero_columns = tuple( - column - for column in FRS_HMRC_RETAINED_LEAF_COLUMNS - if source_signal_rows[column] == 0 - ) - return UKFRSHMRCRetainedLeavesResult( - frame=result_frame, - adult_source=adult_source, - benefits_source=benefits_source, - clone_id_multiplier=lineage.clone_id_multiplier, - spi_person_id_offset=lineage.spi_person_id_offset, - capital_gains_person_id_offset=lineage.capital_gains_person_id_offset, - raw_source_people=len(source_leaves), - candidate_people=len(person), - source_people_outside_candidate=source_people_outside_candidate, - source_signal_rows=source_signal_rows, - structural_zero_columns=structural_zero_columns, - input_content_identity=input_content_identity, - output_content_identity=uk_frame_content_identity(result_frame), - ) - - -def _read_raw_frs_table( - path: str | Path, - *, - expected_filename: str, - required_columns: tuple[str, ...], -) -> tuple[pd.DataFrame, UKFRSRawTableIdentity]: - source_path = Path(path).expanduser().resolve() - if source_path.name.lower() != expected_filename: - raise ValueError( - f"Expected raw FRS table {expected_filename!r}, got {source_path.name!r}." - ) - if not source_path.is_file(): - raise FileNotFoundError(f"Raw FRS table not found: {source_path}.") - before = _file_fingerprint(source_path) - digest = _sha256(source_path) - after_hash = _file_fingerprint(source_path) - if after_hash != before: - raise RuntimeError(f"Raw FRS table changed while hashing: {source_path}.") - required = set(required_columns) - frame = pd.read_csv( - source_path, - sep="\t", - usecols=lambda column: str(column).strip().lower() in required, - ) - after_read = _file_fingerprint(source_path) - if after_read != before: - raise RuntimeError(f"Raw FRS table changed while reading: {source_path}.") - frame.columns = frame.columns.astype(str).str.strip().str.lower() - if frame.columns.duplicated().any(): - duplicates = frame.columns[frame.columns.duplicated()].tolist() - raise ValueError( - f"Raw FRS {expected_filename} has duplicate normalized columns: " - f"{duplicates}." - ) - missing = sorted(required - set(frame.columns)) - if missing: - raise ValueError( - f"Raw FRS {expected_filename} is missing required column(s): {missing}." - ) - frame = frame.loc[:, list(required_columns)] - identity = UKFRSRawTableIdentity( - path=source_path, - filename=expected_filename, - source_vintage=FRS_SOURCE_VINTAGE, - sha256=digest, - size_bytes=before.size_bytes, - rows=len(frame), - extracted_columns=required_columns, - ) - return frame, identity - - -def _materialize_source_leaves( - adult: pd.DataFrame, - benefits: pd.DataFrame, -) -> pd.DataFrame: - adult_ids = _raw_source_person_ids(adult, label="ADULT") - if pd.Index(adult_ids).duplicated().any(): - duplicates = pd.Index(adult_ids)[pd.Index(adult_ids).duplicated()].unique() - raise ValueError( - "Raw FRS ADULT person identities must be unique; duplicate " - f"value(s): {duplicates[:5].tolist()}." - ) - earnings = _finite_numeric(adult["inearns"], label="ADULT.INEARNS") - pay = np.maximum(earnings, 0.0) * FRS_WEEKS_IN_YEAR - adult_leaf = pd.DataFrame( - {FRS_HMRC_PAY_COLUMN: pay}, - index=pd.Index(adult_ids, name="source_person_id"), - ) - - benefit_ids = _raw_source_person_ids(benefits, label="BENEFITS") - benefit_codes = _strict_integer_values( - benefits["benefit"], - label="BENEFITS.BENEFIT", - minimum=0, - ) - relevant = np.isin(benefit_codes, (5, 13, 14, 16, 17, 19)) - amounts = np.zeros(len(benefits), dtype=float) - if relevant.any(): - relevant_amounts = _finite_numeric( - benefits.loc[relevant, "benamt"], - label="relevant BENEFITS.BENAMT", - ) - if (relevant_amounts < 0.0).any(): - raise ValueError("Relevant BENEFITS.BENAMT values must be non-negative.") - amounts[relevant] = relevant_amounts - - code16 = benefit_codes == 16 - contribution_based_esa = np.zeros(len(benefits), dtype=bool) - if code16.any(): - var2 = _strict_integer_values( - benefits.loc[code16, "var2"], - label="BENEFITS.VAR2 for BENEFIT=16", - ) - contribution_based_esa[code16] = np.isin(var2, (1, 3)) - - benefit_leaf = pd.DataFrame( - { - FRS_HMRC_UBISJA_COLUMN: amounts * np.isin(benefit_codes, (14, 19)), - FRS_HMRC_INCPBEN_COLUMN: amounts * (benefit_codes == 17), - FRS_HMRC_OSSBEN_IDENTIFIABLE_SUBSET_COLUMN: amounts - * ((benefit_codes == 13) | contribution_based_esa), - FRS_HMRC_SRP_REGULAR_CODE5_COLUMN: amounts * (benefit_codes == 5), - }, - index=pd.Index(benefit_ids, name="source_person_id"), - ) - benefit_leaf = benefit_leaf.groupby(level=0, sort=False).sum() - benefit_leaf *= FRS_WEEKS_IN_YEAR - - source_ids = adult_leaf.index.union(benefit_leaf.index, sort=False) - result = pd.DataFrame( - 0.0, - index=source_ids, - columns=FRS_HMRC_RETAINED_LEAF_COLUMNS, - ) - result.loc[adult_leaf.index, FRS_HMRC_PAY_COLUMN] = adult_leaf[FRS_HMRC_PAY_COLUMN] - for column in benefit_leaf.columns: - result.loc[benefit_leaf.index, column] = benefit_leaf[column] - numeric = result.to_numpy(dtype=float) - if not np.isfinite(numeric).all() or (numeric < 0.0).any(): - raise RuntimeError("Raw FRS source-leaf materialization is invalid.") - return result - - -def _resolve_candidate_lineage(frame: Frame) -> _CandidateLineage: - person = frame.table("person") - household = frame.table("household") - _require_columns( - person, - ("person_id", "person_household_id"), - label="candidate person", - ) - _require_columns( - household, - ( - "household_id", - "clone_index", - HOUSEHOLD_IS_SPI_SYNTHETIC_COLUMN, - _CAPITAL_GAINS_FLAG, - ), - label="candidate household", - ) - person_ids = _strict_integer_values( - person["person_id"], label="candidate person_id", minimum=1 - ) - person_household_ids = _strict_integer_values( - person["person_household_id"], - label="candidate person_household_id", - minimum=1, - ) - household_ids = _strict_integer_values( - household["household_id"], label="candidate household_id", minimum=1 - ) - clone_index = _strict_integer_values( - household["clone_index"], label="candidate clone_index", minimum=0 - ) - spi = _strict_bool_values( - household[HOUSEHOLD_IS_SPI_SYNTHETIC_COLUMN], - label=HOUSEHOLD_IS_SPI_SYNTHETIC_COLUMN, - ) - capital_gains = _strict_bool_values( - household[_CAPITAL_GAINS_FLAG], label=_CAPITAL_GAINS_FLAG - ) - household_metadata = pd.DataFrame( - { - "household_id": household_ids, - "clone_index": clone_index, - "spi": spi, - "capital_gains": capital_gains, - } - ).set_index("household_id") - mapped = household_metadata.reindex(person_household_ids) - if mapped.isna().any().any(): - raise ValueError( - "Candidate person_household_id cannot map every person to lineage metadata." - ) - person_clone_index = mapped["clone_index"].to_numpy(dtype=np.int64) - person_spi = mapped["spi"].to_numpy(dtype=bool) - person_capital_gains = mapped["capital_gains"].to_numpy(dtype=bool) - - canonical_households = clone_index == 0 - canonical_people = person_clone_index == 0 - if not canonical_households.any() or not canonical_people.any(): - raise ValueError("Candidate lineage requires clone_index=0 rows.") - canonical_max = max( - int(household_ids[canonical_households].max()), - int(person_ids[canonical_people].max()), - ) - clone_multiplier = 10 ** max(1, len(str(canonical_max))) - clone_reversed_household_ids = household_ids - clone_index * clone_multiplier - clone_reversed_person_ids = person_ids - person_clone_index * clone_multiplier - clone_reversed_person_households = ( - person_household_ids - person_clone_index * clone_multiplier - ) - if ( - (clone_reversed_household_ids <= 0).any() - or (clone_reversed_person_ids <= 0).any() - or (clone_reversed_person_households <= 0).any() - ): - raise ValueError("Candidate clone reversal produced non-positive IDs.") - - canonical_household_metadata = pd.DataFrame( - { - "clone_household_id": household_ids[canonical_households], - "spi": spi[canonical_households], - "capital_gains": capital_gains[canonical_households], - } - ).set_index("clone_household_id") - expected_household_metadata = canonical_household_metadata.reindex( - clone_reversed_household_ids - ) - if expected_household_metadata.isna().any().any(): - raise ValueError( - "Candidate geography-clone household IDs do not reverse to the " - "clone_index=0 surface." - ) - if not np.array_equal( - expected_household_metadata["spi"].to_numpy(dtype=bool), spi - ) or not np.array_equal( - expected_household_metadata["capital_gains"].to_numpy(dtype=bool), - capital_gains, - ): - raise ValueError( - "Candidate geography clones disagree with canonical household flags." - ) - - canonical_person = pd.DataFrame( - { - "clone_person_id": person_ids[canonical_people], - "clone_household_id": person_household_ids[canonical_people], - "spi": person_spi[canonical_people], - "capital_gains": person_capital_gains[canonical_people], - } - ).set_index("clone_person_id") - expected_people = canonical_person.reindex(clone_reversed_person_ids) - if expected_people.isna().any().any(): - raise ValueError( - "Candidate geography-clone person IDs do not reverse to the " - "clone_index=0 surface." - ) - if not np.array_equal( - expected_people["clone_household_id"].to_numpy(dtype=np.int64), - clone_reversed_person_households, - ): - raise ValueError( - "Candidate geography-clone person/household memberships are inconsistent." - ) - descriptors = pd.DataFrame( - { - "clone_person_id": clone_reversed_person_ids, - "clone_index": person_clone_index, - } - ) - if descriptors.duplicated().any(): - raise ValueError( - "Candidate geography-clone person lineage contains duplicate descendants." - ) - - canonical_person_ids = person_ids[canonical_people] - canonical_person_spi = person_spi[canonical_people] - canonical_person_capital_gains = person_capital_gains[canonical_people] - raw_person = ~canonical_person_spi & ~canonical_person_capital_gains - pre_capital_gains = ~canonical_person_capital_gains - if not raw_person.any(): - raise ValueError("Candidate lineage has no canonical raw FRS people.") - spi_offset = int(canonical_person_ids[raw_person].max()) + 1 - capital_gains_offset = int(canonical_person_ids[pre_capital_gains].max()) + 1 - source_person_ids = ( - clone_reversed_person_ids - - person_spi.astype(np.int64) * spi_offset - - person_capital_gains.astype(np.int64) * capital_gains_offset - ) - canonical_raw_ids = frozenset( - int(value) for value in canonical_person_ids[raw_person] - ) - if (source_person_ids <= 0).any() or not set(source_person_ids).issubset( - canonical_raw_ids - ): - bad = sorted(set(source_person_ids) - canonical_raw_ids) - raise ValueError( - "Candidate SPI/capital-gains person IDs do not reverse to the raw " - f"FRS surface: {bad[:5]}." - ) - - raw_household = ~spi[canonical_households] & ~capital_gains[canonical_households] - pre_capital_household = ~capital_gains[canonical_households] - canonical_household_ids = household_ids[canonical_households] - if not raw_household.any(): - raise ValueError("Candidate lineage has no canonical raw FRS households.") - spi_household_offset = int(canonical_household_ids[raw_household].max()) + 1 - capital_household_offset = ( - int(canonical_household_ids[pre_capital_household].max()) + 1 - ) - source_household_ids = ( - clone_reversed_person_households - - person_spi.astype(np.int64) * spi_household_offset - - person_capital_gains.astype(np.int64) * capital_household_offset - ) - if not np.array_equal(source_person_ids // 1000, source_household_ids): - raise ValueError( - "Candidate reversed person IDs disagree with reversed household IDs." - ) - lineage_descriptors = pd.DataFrame( - { - "source_person_id": source_person_ids, - "clone_index": person_clone_index, - "spi": person_spi, - "capital_gains": person_capital_gains, - } - ) - if lineage_descriptors.duplicated().any(): - raise ValueError( - "Candidate person lineage contains duplicate stack identities." - ) - return _CandidateLineage( - source_person_ids=source_person_ids, - clone_id_multiplier=clone_multiplier, - spi_person_id_offset=spi_offset, - capital_gains_person_id_offset=capital_gains_offset, - canonical_raw_person_ids=canonical_raw_ids, - ) - - -def _validate_retained_leaf_propagation( - person: pd.DataFrame, - *, - source_person_ids: np.ndarray, - source_leaves: pd.DataFrame, -) -> None: - expected = source_leaves.reindex(source_person_ids, fill_value=0.0) - actual = person.loc[:, list(FRS_HMRC_RETAINED_LEAF_COLUMNS)].apply( - pd.to_numeric, errors="coerce" - ) - actual_values = actual.to_numpy(dtype=float) - if not np.isfinite(actual_values).all() or (actual_values < 0.0).any(): - raise RuntimeError("Retained FRS HMRC leaves must be finite and non-negative.") - if not np.array_equal(actual_values, expected.to_numpy(dtype=float)): - raise RuntimeError("Retained FRS HMRC leaves lost source-person alignment.") - - -def _raw_source_person_ids(frame: pd.DataFrame, *, label: str) -> np.ndarray: - households = _strict_integer_values( - frame["sernum"], label=f"{label}.SERNUM", minimum=1 - ) - people = _strict_integer_values(frame["person"], label=f"{label}.PERSON", minimum=1) - if (people >= 1000).any(): - raise ValueError(f"{label}.PERSON must be less than 1000.") - maximum_household = (np.iinfo(np.int64).max - people) // 1000 - if (households > maximum_household).any(): - raise ValueError(f"{label} source person identity exceeds int64 range.") - return households * 1000 + people - - -def _strict_integer_values( - values: pd.Series, - *, - label: str, - minimum: int | None = None, -) -> np.ndarray: - numeric = pd.to_numeric(values, errors="coerce").to_numpy( - dtype=float, na_value=np.nan - ) - if not np.isfinite(numeric).all(): - raise ValueError(f"{label} must contain finite numeric values.") - if not np.equal(numeric, np.floor(numeric)).all(): - raise ValueError(f"{label} must contain integer values.") - if (np.abs(numeric) > np.iinfo(np.int64).max).any(): - raise ValueError(f"{label} exceeds int64 range.") - result = numeric.astype(np.int64) - if minimum is not None and (result < minimum).any(): - raise ValueError(f"{label} must be at least {minimum}.") - return result - - -def _strict_bool_values(values: pd.Series, *, label: str) -> np.ndarray: - if pd.api.types.is_bool_dtype(values.dtype): - if values.isna().any(): - raise ValueError(f"{label} must not contain missing values.") - return values.to_numpy(dtype=bool) - numeric = _strict_integer_values(values, label=label) - if not np.isin(numeric, (0, 1)).all(): - raise ValueError(f"{label} must contain only boolean or 0/1 values.") - return numeric.astype(bool) - - -def _finite_numeric(values: pd.Series, *, label: str) -> np.ndarray: - numeric = pd.to_numeric(values, errors="coerce").to_numpy( - dtype=float, na_value=np.nan - ) - if not np.isfinite(numeric).all(): - raise ValueError(f"{label} must contain finite numeric values.") - return numeric - - -def _require_columns( - frame: pd.DataFrame, - columns: tuple[str, ...], - *, - label: str, -) -> None: - missing = sorted(set(columns) - set(frame.columns)) - if missing: - raise ValueError(f"{label} is missing required column(s): {missing}.") - - -def _file_fingerprint(path: Path) -> _FileFingerprint: - stat = path.stat() - return _FileFingerprint( - device=stat.st_dev, - inode=stat.st_ino, - size_bytes=stat.st_size, - modified_ns=stat.st_mtime_ns, - changed_ns=stat.st_ctime_ns, - ) - - -def _sha256(path: Path) -> str: - digest = hashlib.sha256() - with path.open("rb") as source: - for chunk in iter(lambda: source.read(1024 * 1024), b""): - digest.update(chunk) - return digest.hexdigest() diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/frs_hmrc_source.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/frs_hmrc_source.py new file mode 100644 index 000000000..d0dd4c5f9 --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/frs_hmrc_source.py @@ -0,0 +1,311 @@ +"""Source-faithful raw FRS extraction for the canonical HMRC spine stages. + +This module reads raw survey tables and preserves named partial income concepts. +It does not restore or accept a pre-existing candidate population. +""" + +from __future__ import annotations + +import hashlib +from dataclasses import dataclass +from pathlib import Path + +import numpy as np +import pandas as pd + +from microcosm.build.uk_runtime.spi_support import ( + SPI_HMRC_INCAPACITY_BENEFIT_INCOME_COLUMN, + SPI_HMRC_PAY_COLUMN, + SPI_HMRC_UNEMPLOYMENT_BENEFIT_INCOME_COLUMN, +) + +FRS_WEEKS_IN_YEAR = 365.25 / 7 + + +FRS_HMRC_PAY_COLUMN = SPI_HMRC_PAY_COLUMN + + +FRS_HMRC_UBISJA_COLUMN = SPI_HMRC_UNEMPLOYMENT_BENEFIT_INCOME_COLUMN + + +FRS_HMRC_INCPBEN_COLUMN = SPI_HMRC_INCAPACITY_BENEFIT_INCOME_COLUMN + + +FRS_HMRC_OSSBEN_IDENTIFIABLE_SUBSET_COLUMN = "ossben_identifiable_subset" + + +FRS_HMRC_SRP_REGULAR_CODE5_COLUMN = "srp_regular_code5" + + +FRS_HMRC_RETAINED_LEAF_COLUMNS = ( + FRS_HMRC_PAY_COLUMN, + FRS_HMRC_UBISJA_COLUMN, + FRS_HMRC_INCPBEN_COLUMN, + FRS_HMRC_OSSBEN_IDENTIFIABLE_SUBSET_COLUMN, + FRS_HMRC_SRP_REGULAR_CODE5_COLUMN, +) + + +FRS_HMRC_RETAINED_LEAF_SOURCE_EVIDENCE: dict[str, dict[str, object]] = { + FRS_HMRC_PAY_COLUMN: { + "spi_concept": "PAY", + "scope": "full", + "raw_sources": ["ADULT.INEARNS"], + "formula": "max(0, ADULT.INEARNS) * (365.25 / 7)", + }, + FRS_HMRC_UBISJA_COLUMN: { + "spi_concept": "UBISJA", + "scope": "full", + "raw_sources": [ + "BENEFITS.BENEFIT=14:BENAMT", + "BENEFITS.BENEFIT=19:BENAMT", + ], + "formula": "sum(BENAMT where BENEFIT in {14, 19}) * (365.25 / 7)", + }, + FRS_HMRC_INCPBEN_COLUMN: { + "spi_concept": "INCPBEN", + "scope": "full", + "raw_sources": ["BENEFITS.BENEFIT=17:BENAMT"], + "formula": "sum(BENAMT where BENEFIT == 17) * (365.25 / 7)", + }, + FRS_HMRC_OSSBEN_IDENTIFIABLE_SUBSET_COLUMN: { + "spi_concept": "OSSBEN", + "scope": "identifiable_subset", + "raw_sources": [ + "BENEFITS.BENEFIT=13:BENAMT", + "BENEFITS.BENEFIT=16,VAR2 in {1,3}:BENAMT", + ], + "formula": ( + "sum(BENAMT where BENEFIT == 13 or " + "(BENEFIT == 16 and VAR2 in {1, 3})) * (365.25 / 7)" + ), + }, + FRS_HMRC_SRP_REGULAR_CODE5_COLUMN: { + "spi_concept": "SRP", + "scope": "regular_code5_subset", + "raw_sources": ["BENEFITS.BENEFIT=5:BENAMT"], + "formula": "sum(BENAMT where BENEFIT == 5) * (365.25 / 7)", + }, +} + + +@dataclass(frozen=True) +class UKFRSRawTableIdentity: + """Stable identity and extraction surface for one raw FRS table.""" + + path: Path + filename: str + source_vintage: str + sha256: str + size_bytes: int + rows: int + extracted_columns: tuple[str, ...] + + def evidence(self) -> dict[str, object]: + """Return JSON-safe source evidence.""" + + return { + "path": str(self.path), + "filename": self.filename, + "source_vintage": self.source_vintage, + "sha256": self.sha256, + "size_bytes": self.size_bytes, + "rows": self.rows, + "extracted_columns": list(self.extracted_columns), + } + + +@dataclass(frozen=True) +class _FileFingerprint: + device: int + inode: int + size_bytes: int + modified_ns: int + changed_ns: int + + +def _read_raw_frs_table( + path: str | Path, + *, + expected_filename: str, + required_columns: tuple[str, ...], + source_vintage: str = "unspecified", +) -> tuple[pd.DataFrame, UKFRSRawTableIdentity]: + source_path = Path(path).expanduser().resolve() + if source_path.name.lower() != expected_filename: + raise ValueError( + f"Expected raw FRS table {expected_filename!r}, got {source_path.name!r}." + ) + if not source_path.is_file(): + raise FileNotFoundError(f"Raw FRS table not found: {source_path}.") + before = _file_fingerprint(source_path) + digest = _sha256(source_path) + after_hash = _file_fingerprint(source_path) + if after_hash != before: + raise RuntimeError(f"Raw FRS table changed while hashing: {source_path}.") + required = set(required_columns) + frame = pd.read_csv( + source_path, + sep="\t", + usecols=lambda column: str(column).strip().lower() in required, + ) + after_read = _file_fingerprint(source_path) + if after_read != before: + raise RuntimeError(f"Raw FRS table changed while reading: {source_path}.") + frame.columns = frame.columns.astype(str).str.strip().str.lower() + if frame.columns.duplicated().any(): + duplicates = frame.columns[frame.columns.duplicated()].tolist() + raise ValueError( + f"Raw FRS {expected_filename} has duplicate normalized columns: " + f"{duplicates}." + ) + missing = sorted(required - set(frame.columns)) + if missing: + raise ValueError( + f"Raw FRS {expected_filename} is missing required column(s): {missing}." + ) + frame = frame.loc[:, list(required_columns)] + identity = UKFRSRawTableIdentity( + path=source_path, + filename=expected_filename, + source_vintage=source_vintage, + sha256=digest, + size_bytes=before.size_bytes, + rows=len(frame), + extracted_columns=required_columns, + ) + return frame, identity + + +def _materialize_source_leaves( + adult: pd.DataFrame, + benefits: pd.DataFrame, +) -> pd.DataFrame: + adult_ids = _raw_source_person_ids(adult, label="ADULT") + if pd.Index(adult_ids).duplicated().any(): + duplicates = pd.Index(adult_ids)[pd.Index(adult_ids).duplicated()].unique() + raise ValueError( + "Raw FRS ADULT person identities must be unique; duplicate " + f"value(s): {duplicates[:5].tolist()}." + ) + earnings = _finite_numeric(adult["inearns"], label="ADULT.INEARNS") + pay = np.maximum(earnings, 0.0) * FRS_WEEKS_IN_YEAR + adult_leaf = pd.DataFrame( + {FRS_HMRC_PAY_COLUMN: pay}, + index=pd.Index(adult_ids, name="source_person_id"), + ) + + benefit_ids = _raw_source_person_ids(benefits, label="BENEFITS") + benefit_codes = _strict_integer_values( + benefits["benefit"], + label="BENEFITS.BENEFIT", + minimum=0, + ) + relevant = np.isin(benefit_codes, (5, 13, 14, 16, 17, 19)) + amounts = np.zeros(len(benefits), dtype=float) + if relevant.any(): + relevant_amounts = _finite_numeric( + benefits.loc[relevant, "benamt"], + label="relevant BENEFITS.BENAMT", + ) + if (relevant_amounts < 0.0).any(): + raise ValueError("Relevant BENEFITS.BENAMT values must be non-negative.") + amounts[relevant] = relevant_amounts + + code16 = benefit_codes == 16 + contribution_based_esa = np.zeros(len(benefits), dtype=bool) + if code16.any(): + var2 = _strict_integer_values( + benefits.loc[code16, "var2"], + label="BENEFITS.VAR2 for BENEFIT=16", + ) + contribution_based_esa[code16] = np.isin(var2, (1, 3)) + + benefit_leaf = pd.DataFrame( + { + FRS_HMRC_UBISJA_COLUMN: amounts * np.isin(benefit_codes, (14, 19)), + FRS_HMRC_INCPBEN_COLUMN: amounts * (benefit_codes == 17), + FRS_HMRC_OSSBEN_IDENTIFIABLE_SUBSET_COLUMN: amounts + * ((benefit_codes == 13) | contribution_based_esa), + FRS_HMRC_SRP_REGULAR_CODE5_COLUMN: amounts * (benefit_codes == 5), + }, + index=pd.Index(benefit_ids, name="source_person_id"), + ) + benefit_leaf = benefit_leaf.groupby(level=0, sort=False).sum() + benefit_leaf *= FRS_WEEKS_IN_YEAR + + source_ids = adult_leaf.index.union(benefit_leaf.index, sort=False) + result = pd.DataFrame( + 0.0, + index=source_ids, + columns=FRS_HMRC_RETAINED_LEAF_COLUMNS, + ) + result.loc[adult_leaf.index, FRS_HMRC_PAY_COLUMN] = adult_leaf[FRS_HMRC_PAY_COLUMN] + for column in benefit_leaf.columns: + result.loc[benefit_leaf.index, column] = benefit_leaf[column] + numeric = result.to_numpy(dtype=float) + if not np.isfinite(numeric).all() or (numeric < 0.0).any(): + raise RuntimeError("Raw FRS source-leaf materialization is invalid.") + return result + + +def _raw_source_person_ids(frame: pd.DataFrame, *, label: str) -> np.ndarray: + households = _strict_integer_values( + frame["sernum"], label=f"{label}.SERNUM", minimum=1 + ) + people = _strict_integer_values(frame["person"], label=f"{label}.PERSON", minimum=1) + if (people >= 1000).any(): + raise ValueError(f"{label}.PERSON must be less than 1000.") + maximum_household = (np.iinfo(np.int64).max - people) // 1000 + if (households > maximum_household).any(): + raise ValueError(f"{label} source person identity exceeds int64 range.") + return households * 1000 + people + + +def _strict_integer_values( + values: pd.Series, + *, + label: str, + minimum: int | None = None, +) -> np.ndarray: + numeric = pd.to_numeric(values, errors="coerce").to_numpy( + dtype=float, na_value=np.nan + ) + if not np.isfinite(numeric).all(): + raise ValueError(f"{label} must contain finite numeric values.") + if not np.equal(numeric, np.floor(numeric)).all(): + raise ValueError(f"{label} must contain integer values.") + if (np.abs(numeric) > np.iinfo(np.int64).max).any(): + raise ValueError(f"{label} exceeds int64 range.") + result = numeric.astype(np.int64) + if minimum is not None and (result < minimum).any(): + raise ValueError(f"{label} must be at least {minimum}.") + return result + + +def _finite_numeric(values: pd.Series, *, label: str) -> np.ndarray: + numeric = pd.to_numeric(values, errors="coerce").to_numpy( + dtype=float, na_value=np.nan + ) + if not np.isfinite(numeric).all(): + raise ValueError(f"{label} must contain finite numeric values.") + return numeric + + +def _file_fingerprint(path: Path) -> _FileFingerprint: + stat = path.stat() + return _FileFingerprint( + device=stat.st_dev, + inode=stat.st_ino, + size_bytes=stat.st_size, + modified_ns=stat.st_mtime_ns, + changed_ns=stat.st_ctime_ns, + ) + + +def _sha256(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as source: + for chunk in iter(lambda: source.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/full_build_cli.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/full_build_cli.py new file mode 100644 index 000000000..8cca240c0 --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/full_build_cli.py @@ -0,0 +1,798 @@ +"""Execute the single UK full build; all geographies are selected by default. + +Numerical operations and verdicts belong to the composed graph. This module +resolves requests, executes graph endpoints and atomically materializes their +stored artifacts. Publication and signing remain explicit external services. +""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import sys +import tempfile +from dataclasses import asdict, dataclass, replace +from datetime import date +from pathlib import Path + +from microcosm.build.artifact_files import file_artifact, materialize_bytes +from microcosm.graph import ( + ArtifactInput, + ContentStore, + Graph, + KernelRegistry, + SourceRef, + compile_graph, + graph_to_json, + run_graph, +) +from microcosm.graph.canonical import canonical_json + +from .frs_release import load_uk_frs_release +from .full_certification import ( + append_uk_full_certification_node, + register_uk_full_certification_kernel, +) +from .graph_build import ( + SPINE_PROVENANCE_TYPE, + UKFullBuildConfig, + UKFullGraph, + bound_spine_graph, + register_uk_full_kernels, + uk_full_graph, +) +from .graph_calibration import UKGraphCalibrationConfig +from .graph_targets import TARGET_SELECTION_TYPE +from .graph_terminal import ( + FULL_DIAGNOSTICS_CSV_TYPE, + FULL_DIAGNOSTICS_TYPE, + FULL_GATE_REPORT_TYPE, + FULL_HOLDOUT_TYPE, + FULL_SUPPORT_CSV_TYPE, + add_uk_export_continuation, + add_uk_export_preparation, + append_uk_full_gate_nodes, + decode_full_gate_report, + materialize_uk_export, + materialize_uk_terminal_artifacts, + register_uk_full_gate_kernels, + register_uk_terminal_kernels, +) +from .local_doctrine import ( + UK_LOCAL_CLONE_COUNT, + UK_LOCAL_MAX_WEIGHT_RATIO, + UK_LOCAL_SOLVE_DOCTRINE, + UK_LOCAL_SOLVE_EPOCHS, + UK_LOCAL_TARGET_LOSS_CAP, +) +from .national_chronicle_feed import load_uk_national_chronicle_feed +from .national_frame import load_uk_national_frame +from .national_sampling import UK_SAMPLE_SEED_DEFAULT + + +def _target_geographies(value: str) -> tuple[str, ...] | None: + if value == "all": + return None + levels = tuple(value.split(",")) + if ( + not levels + or len(levels) != len(set(levels)) + or set(levels) - {"country", "region", "constituency", "la"} + ): + raise argparse.ArgumentTypeError( + "Use all or a comma-separated subset of country,region,constituency,la." + ) + return levels + + +def parse_args(argv: list[str] | None = None) -> argparse.Namespace: + parser = argparse.ArgumentParser(description=__doc__) + population = parser.add_mutually_exclusive_group(required=True) + population.add_argument( + "--input-h5", + type=Path, + help="Canonical spine checkpoint with bound build and gate sidecars.", + ) + population.add_argument( + "--spine-request", + type=Path, + help="JSON array of raw FRS spine arguments; these stages execute in the same graph.", + ) + parser.add_argument("--input-sidecar", type=Path) + parser.add_argument("--input-spine-gates", type=Path) + parser.add_argument("--input-sha256") + parser.add_argument("--ladder", type=Path, required=True) + parser.add_argument("--ladder-sha256") + parser.add_argument( + "--ledger-facts", + type=Path, + required=True, + help="Complete Chronicle artifact directory matching the committed feed pins.", + ) + parser.add_argument("--ledger-facts-sha256") + parser.add_argument("--ledger-manifest-sha256") + parser.add_argument("--measure-exclusions", type=Path) + parser.add_argument("--register-json", type=Path) + parser.add_argument("--input-mass-reference", type=Path) + parser.add_argument( + "--native-scorecard", + type=Path, + help="Measured incumbent-surface comparison bound to immutable candidate.json and its exact output bytes.", + ) + parser.add_argument( + "--matched-size-scorecard", + type=Path, + help="Additional measured comparison with both populations at requested k.", + ) + parser.add_argument( + "--target-geographies", + type=_target_geographies, + default=None, + metavar="all|country,...", + help="Default all: calibrate all applicable geographies together. country is an explicit filter in this same build.", + ) + parser.add_argument( + "--n-clones", + type=int, + default=UK_LOCAL_CLONE_COUNT, + help="Geographic pool copies K, independent of target scope and exported size k.", + ) + parser.add_argument( + "--dataset-households", + type=int, + help="Exact exported household count k via informed L0, draw and refit.", + ) + parser.add_argument( + "--sample-fraction", + type=float, + default=1.0, + help="Optional pool sampling before cloning. Cannot resample an already sampled source spine.", + ) + parser.add_argument("--sample-seed", type=int, default=UK_SAMPLE_SEED_DEFAULT) + parser.add_argument("--seed", type=int, default=42) + parser.add_argument("--selection-seed", type=int) + parser.add_argument("--selection-pi-hi", type=float, default=1.0) + parser.add_argument("--epochs", type=int, default=UK_LOCAL_SOLVE_EPOCHS) + parser.add_argument("--learning-rate", type=float, default=0.15) + parser.add_argument( + "--target-weight-rule", + choices=("uniform", "grain_equal"), + default=UK_LOCAL_SOLVE_DOCTRINE.target_weight_rule, + ) + parser.add_argument("--engine-blocks", type=int, default=1) + parser.add_argument("--source-year", type=int) + parser.add_argument("--calibration-year", type=int) + parser.add_argument("--source-lineage-modulus", type=int) + parser.add_argument("--expected-constituency-vintage", default="2024_pcon") + parser.add_argument("--skip-holdout", action="store_true") + parser.add_argument("--release-candidate", action="store_true") + parser.add_argument("--review-date", type=date.fromisoformat, default=date.today()) + parser.add_argument( + "--resume-size-checkpoint", + type=Path, + help="Import an identity-verified historical size search, skipping its dense solve and search.", + ) + parser.add_argument( + "--graph-store", + type=Path, + help="Persistent shared graph store; default /.graph-store.", + ) + parser.add_argument( + "--resume", choices=("auto", "require", "forbid"), default="auto" + ) + parser.add_argument("--out", type=Path, required=True) + parser.add_argument( + "--dry-run", + action="store_true", + help="Validate the request and print the compiled operation inventory without fitting or writing files.", + ) + args = parser.parse_args(argv) + if args.input_h5 is None and any( + (args.input_sidecar, args.input_spine_gates, args.input_sha256) + ): + parser.error("Input H5 sidecar/pin options require --input-h5.") + if args.dataset_households is None and ( + args.selection_seed is not None + or args.selection_pi_hi != 1.0 + or args.resume_size_checkpoint + ): + parser.error("Selection options require --dataset-households.") + if args.matched_size_scorecard is not None and args.dataset_households is None: + parser.error("A matched-size scorecard requires --dataset-households.") + if args.release_candidate and args.skip_holdout: + parser.error("A release candidate must evaluate applicable holdouts.") + if args.release_candidate: + if args.dataset_households is not None: + parser.error( + "Exact-count builds need separate matched-size evidence before release promotion." + ) + if ( + args.epochs != UK_LOCAL_SOLVE_EPOCHS + or args.n_clones != UK_LOCAL_CLONE_COUNT + or args.target_weight_rule != UK_LOCAL_SOLVE_DOCTRINE.target_weight_rule + or args.engine_blocks != 1 + or args.measure_exclusions is not None + ): + parser.error( + "A release candidate must use the maintained solve doctrine, pool count, single engine and reviewed exclusions." + ) + if args.ladder_sha256 is None or ( + args.input_h5 is not None and args.input_sha256 is None + ): + parser.error( + "A release candidate requires explicit input H5 and ladder digest pins." + ) + if args.resume_size_checkpoint and args.input_h5 is None: + parser.error( + "Historical size checkpoints bind an input H5; raw builds resume using --graph-store." + ) + return args + + +def _pin(path: Path, expected: str | None = None) -> dict: + record = file_artifact(path) + if expected is not None and expected != record["sha256"]: + raise ValueError(f"Input digest differs from the requested pin: {path}.") + return {"sha256": record["sha256"], "size_bytes": record["size_bytes"]} + + +def _checkpoint_identity(args, config, pins) -> dict | None: + if args.resume_size_checkpoint is None: + return None + # Match the existing checkpoint schema exactly. The import kernel also + # verifies ordered targets, weights, household axis and recomputed losses. + return { + "dataset_pin": pins["dataset"], + "ladder_pin": pins["ladder"], + "ledger_facts_sha256": args.ledger_facts_sha256, + "ledger_manifest_sha256": args.ledger_manifest_sha256, + "seed": args.seed, + "selection_seed": args.seed + if args.selection_seed is None + else args.selection_seed, + "n_clones": args.n_clones, + "dataset_households": args.dataset_households, + "epochs": args.epochs, + "learning_rate": args.learning_rate, + "sample_fraction": args.sample_fraction, + "sample_seed": args.sample_seed, + "source_year": config.source_year, + "source_lineage_modulus": args.source_lineage_modulus, + "calibration_year": config.calibration_year, + "target_weight_rule": args.target_weight_rule, + "engine_blocks": args.engine_blocks, + "measure_exclusions": None + if args.measure_exclusions is None + else str(args.measure_exclusions), + "doctrine": { + "target_loss_cap": float(UK_LOCAL_TARGET_LOSS_CAP), + "max_weight_ratio": float(UK_LOCAL_MAX_WEIGHT_RATIO), + "scale_rule": UK_LOCAL_SOLVE_DOCTRINE.scale_rule, + "target_weight_rule": UK_LOCAL_SOLVE_DOCTRINE.target_weight_rule, + "solve_epochs": int(UK_LOCAL_SOLVE_EPOCHS), + "clone_count": int(UK_LOCAL_CLONE_COUNT), + }, + } + + +@dataclass(frozen=True) +class PreparedUKFullBuild: + full: UKFullGraph + kernels: KernelRegistry + sources: dict[str, Path] + bindings: dict + spine_provenance: ArtifactInput | None = None + comparison_sources: dict[str, Path] | None = None + + +def prepare_full_build(args: argparse.Namespace) -> PreparedUKFullBuild: + from .calibration_run import load_bound_spine_checkpoint + from .graph import uk_spine_endpoint + from .spine_build import ( + _rules_engine, + _rules_engine_provenance, + parse_uk_spine_args, + prepare_uk_spine_execution, + ) + + release = load_uk_frs_release() + pins = {"ladder": _pin(args.ladder, args.ladder_sha256)} + sources = {"uk_ladder": args.ladder, "uk_ledger_facts": args.ledger_facts} + provenance = None + if args.input_h5 is not None: + pins["dataset"] = _pin(args.input_h5, args.input_sha256) + frame, _ = load_uk_national_frame(args.input_h5) + sidecar_path = args.input_sidecar or args.input_h5.with_suffix(".build.json") + gates_path = args.input_spine_gates or args.input_h5.with_suffix( + ".spine_gates.json" + ) + sidecar = load_bound_spine_checkpoint( + sidecar_path, frame, gate_report_path=gates_path + ) + spine = bound_spine_graph(frame) + endpoint = "uk.full.spine_checkpoint" + weight_kind = frame.weights_for("household").kind.value + time_period = str(frame.metadata["time_period"]) + source_fraction = float((sidecar.get("sampling") or {}).get("fraction", 1.0)) + stages = tuple(sidecar["stages"]) + engine = _rules_engine() + engine_identity = hashlib.sha256( + canonical_json(_rules_engine_provenance()) + ).hexdigest() + kernels = KernelRegistry() + sources.update( + uk_spine=args.input_h5, + uk_spine_evidence=sidecar_path, + uk_spine_gates=gates_path, + ) + provenance = ArtifactInput( + "spine_provenance", endpoint, "spine_provenance", SPINE_PROVENANCE_TYPE + ) + else: + raw_arguments = json.loads(args.spine_request.read_text()) + if not isinstance(raw_arguments, list) or not all( + isinstance(x, str) for x in raw_arguments + ): + raise ValueError( + "The spine request must be a JSON array of command arguments." + ) + # --spine-h5 is an output control of the checkpoint command. Preparation + # uses it only for path configuration; the full graph writes its own H5. + if "--spine-h5" not in raw_arguments and not any( + x.startswith("--spine-h5=") for x in raw_arguments + ): + raw_arguments += ["--spine-h5", str(args.out / "spine.h5")] + raw = parse_uk_spine_args(raw_arguments) + if args.release_candidate: + raw.release_candidate = True + prepared = prepare_uk_spine_execution(raw) + spine, kernels = prepared.graph, prepared.kernels + sources.update(prepared.sources) + endpoint = uk_spine_endpoint(spine).population + weight_kind = "importance" + time_period = prepared.frs_release.time_period + source_fraction = raw.sample_fraction + stages = prepared.stage_names + engine, engine_identity = prepared.engine, prepared.engine_identity + config = UKFullBuildConfig( + calibration_year=args.calibration_year or release.calibration_year, + time_period=time_period, + source_year=args.source_year + if args.source_year is not None + else int(time_period), + geography_levels=args.target_geographies, + n_clones=args.n_clones, + sample_fraction=args.sample_fraction, + source_sample_fraction=source_fraction, + sample_seed=args.sample_seed, + seed=args.seed, + engine_blocks=args.engine_blocks, + constituency_vintage=args.expected_constituency_vintage, + source_lineage_modulus=args.source_lineage_modulus, + calibration=UKGraphCalibrationConfig( + epochs=args.epochs, + learning_rate=args.learning_rate, + seed=args.seed, + dataset_households=args.dataset_households, + selection_seed=args.selection_seed, + selection_pi_hi=args.selection_pi_hi, + target_weight_rule=args.target_weight_rule, + ), + ) + if args.release_candidate and config.effective_sample_fraction != 1.0: + raise ValueError( + "Sampled builds cannot request release-candidate certification." + ) + feed = load_uk_national_chronicle_feed() + for supplied, committed in ( + (args.ledger_facts_sha256, feed.facts_sha256), + (args.ledger_manifest_sha256, feed.manifest_sha256), + ): + if supplied is not None and supplied != committed: + raise ValueError( + "Requested Chronicle pin differs from the committed full-build feed." + ) + optional = [] + for name, path in ( + ("uk_measure_exclusions", args.measure_exclusions), + ("uk_frozen_register", args.register_json), + ): + if path is not None: + sources[name] = path + optional.append(name) + if args.input_mass_reference is not None: + sources["uk_input_mass_reference"] = args.input_mass_reference + spine = replace( + spine, + sources=( + *spine.sources, + SourceRef("uk_input_mass_reference", "raw-bytes-v1"), + ), + ) + full = uk_full_graph( + config, + spine=spine, + spine_population=endpoint, + spine_weight_kind=weight_kind, + optional_target_sources=tuple(optional), + review_date=args.review_date.isoformat(), + checkpoint_identity=_checkpoint_identity(args, config, pins), + ) + if args.resume_size_checkpoint: + from .size_checkpoint import ( + SIZE_CHECKPOINT_ARRAYS_FILENAME, + SIZE_CHECKPOINT_MANIFEST_FILENAME, + ) + + sources["uk_size_checkpoint_manifest"] = ( + args.resume_size_checkpoint / SIZE_CHECKPOINT_MANIFEST_FILENAME + ) + sources["uk_size_checkpoint_arrays"] = ( + args.resume_size_checkpoint / SIZE_CHECKPOINT_ARRAYS_FILENAME + ) + graph = append_uk_full_gate_nodes( + full.graph, + calibration=full.calibration, + spine_stage_names=stages, + engine_identity=engine_identity, + review_date=args.review_date, + sample_fraction=config.effective_sample_fraction, + release_candidate=args.release_candidate, + spine_provenance=provenance, + skip_holdout=args.skip_holdout, + ) + bindings = { + "schema": "microcosm.uk.full-build-request.v1", + "configuration": asdict(config), + "target_scope": "all" + if config.geography_levels is None + else list(config.geography_levels), + "review_date": args.review_date.isoformat(), + "engine_identity": engine_identity, + "release_candidate": args.release_candidate, + "skip_holdout": args.skip_holdout, + "ledger": { + "facts_sha256": feed.facts_sha256, + "manifest_sha256": feed.manifest_sha256, + }, + } + graph = add_uk_export_preparation( + graph, + population=full.population, + bindings=bindings, + artifact_inputs=( + ArtifactInput( + "full_gates", + "uk.full.gates.calibrated", + "gate_report", + FULL_GATE_REPORT_TYPE, + ), + ), + ) + register_uk_full_kernels(kernels) + register_uk_full_gate_kernels( + kernels, coverage_engine=engine, engine_identity=engine_identity + ) + register_uk_terminal_kernels(kernels) + register_uk_full_certification_kernel(kernels) + full = replace(full, graph=graph) + compile_graph(graph) + comparisons = { + name: path + for name, path in ( + ("uk_native_scorecard", args.native_scorecard), + ("uk_matched_size_scorecard", args.matched_size_scorecard), + ) + if path is not None + } + return PreparedUKFullBuild( + full, kernels, sources, bindings, provenance, comparisons + ) + + +def _through(graph: Graph, endpoint: str) -> Graph: + """Execute an actual ancestor-closed checkpoint of the one declared graph.""" + compiled = compile_graph(graph) + needed, pending = {endpoint}, [endpoint] + while pending: + for parent in compiled.predecessors[pending.pop()]: + if parent not in needed: + needed.add(parent) + pending.append(parent) + return replace( + graph, nodes=tuple(node for node in graph.nodes if node.id in needed) + ) + + +def _payload(manifest, store, node: str, artifact: str) -> bytes: + return store.load_bytes(manifest.nodes[node].opaque_artifacts[artifact]) + + +def _materialize_evidence(manifest, store, out: Path) -> dict: + inventory = {} + for node_id, receipt in manifest.nodes.items(): + for name, key in receipt.opaque_artifacts.items(): + payload = store.load_bytes(key) + suffix = ".json" if payload.startswith((b"{", b"[")) else ".artifact" + filename = f"{node_id}.{name}{suffix}" + if Path(filename).name != filename: + raise ValueError( + "Graph artifact name cannot be materialized as a bundle filename." + ) + inventory[f"{node_id}/{name}"] = { + "key": key, + **materialize_bytes(payload, out / filename), + } + materialize_bytes(canonical_json(inventory), out / "evidence-index.json") + return inventory + + +def _output_locations(prepared: PreparedUKFullBuild, args: argparse.Namespace): + output = args.out.resolve() + graph_store = (args.graph_store or output / ".graph-store").resolve() + for source in {**prepared.sources, **(prepared.comparison_sources or {})}.values(): + source = source.resolve() + if source.is_relative_to(output): + raise ValueError( + "Full-build output directory must not contain an input source." + ) + if source.is_dir() and ( + output.is_relative_to(source) or graph_store.is_relative_to(source) + ): + raise ValueError( + "Output files and graph store cannot alter a declared source directory." + ) + return output, graph_store + + +def execute_full_build(prepared: PreparedUKFullBuild, args: argparse.Namespace) -> int: + """Stage complete files, then publish their completion marker last.""" + if args.dry_run: + return _execute_full_build(prepared, args) + from microcosm.build.artifact_files import publish_staged_bundle + + output, graph_store = _output_locations(prepared, args) + output.parent.mkdir(parents=True, exist_ok=True) + with tempfile.TemporaryDirectory( + prefix=f".{output.name}.full-build-", dir=output.parent + ) as temporary: + staged_args = argparse.Namespace(**vars(args)) + staged_args.out = Path(temporary) + staged_args.graph_store = graph_store + status = _execute_full_build(prepared, staged_args) + if not (staged_args.out / "build.json").exists(): + materialize_bytes( + canonical_json( + { + "schema_version": 1, + "kind": "uk_full_build_refused", + "request": prepared.bindings, + "release_authorized": False, + "artifact_permitted": False, + } + ), + staged_args.out / "build.json", + ) + staged = { + ("manifest" if path.name == "build.json" else path.name): path + for path in staged_args.out.iterdir() + if path.is_file() + } + destinations = {role: output / path.name for role, path in staged.items()} + publish_staged_bundle(staged, destinations, completion_role="manifest") + if status == 0: + print( + f"UK full build: {output / f'microcosm_uk_{prepared.full.config.calibration_year}.h5'}; " + f"target scope {prepared.bindings['target_scope']}." + ) + return status + + +def _execute_full_build(prepared: PreparedUKFullBuild, args: argparse.Namespace) -> int: + full, kernels, sources = prepared.full, prepared.kernels, prepared.sources + if args.dry_run: + print(json.dumps(full.operation_inventory(), indent=2)) + return 0 + args.out.mkdir(parents=True, exist_ok=True) + store = ContentStore(args.graph_store or args.out / ".graph-store") + graph = full.graph + materialize_bytes(graph_to_json(graph).encode(), args.out / "graph.json") + materialize_bytes( + canonical_json(full.operation_inventory()), args.out / "operations.json" + ) + # Persist preflight outcomes before any solver can reject them. The second + # endpoint reuses the same stored node identities, never repeats fitting. + preflight_graph = _through(graph, "uk.full.gates.preflight") + preflight = run_graph( + compile_graph(preflight_graph), + sources=sources, + store=store, + kernels=kernels, + resume=args.resume, + population_retention="lazy", + ) + preflight.save(args.out / "preflight.graph.json") + _materialize_evidence(preflight, store, args.out) + _, admission = decode_full_gate_report( + _payload(preflight, store, "uk.full.gates.preflight", "gate_report") + ) + if not admission["artifact_permitted"]: + return 1 + manifest = run_graph( + compile_graph(_through(graph, "uk.full.gates.calibrated")), + sources=sources, + store=store, + kernels=kernels, + resume="require" if args.resume == "require" else "auto", + population_retention="lazy", + ) + manifest.save(args.out / "numerical.graph.json") + _materialize_evidence(manifest, store, args.out) + terminal_files = materialize_uk_terminal_artifacts( + manifest, + store, + directory=args.out, + stem=f"microcosm_uk_{full.config.calibration_year}", + ) + _, enforcement = decode_full_gate_report( + _payload(manifest, store, "uk.full.gates.calibrated", "gate_report") + ) + if not enforcement["artifact_permitted"]: + return 1 + manifest = run_graph( + compile_graph(graph), + sources=sources, + store=store, + kernels=kernels, + resume="require" if args.resume == "require" else "auto", + population_retention="lazy", + ) + descriptor = json.loads( + _payload(manifest, store, "uk.full.export.prepare", "export_descriptor") + ) + dataset = args.out / f"microcosm_uk_{full.config.calibration_year}.h5" + materialize_uk_export(manifest.population(full.population), descriptor, dataset) + graph = add_uk_export_continuation( + graph, + population=full.population, + manifest_binding={ + "graph_sha256": hashlib.sha256(graph_to_json(graph).encode()).hexdigest() + }, + artifact_inputs=( + ArtifactInput( + "full_gates", + "uk.full.gates.calibrated", + "gate_report", + FULL_GATE_REPORT_TYPE, + ), + ArtifactInput( + "diagnostics", + "uk.full.gates.calibrated", + "calibration_diagnostics", + FULL_DIAGNOSTICS_TYPE, + ), + ArtifactInput("holdout", "uk.full.holdout", "holdout", FULL_HOLDOUT_TYPE), + ArtifactInput( + "target_diagnostics", + "uk.full.gates.calibrated", + "target_diagnostics_csv", + FULL_DIAGNOSTICS_CSV_TYPE, + ), + ArtifactInput( + "area_support", + "uk.full.gates.calibrated", + "area_support_csv", + FULL_SUPPORT_CSV_TYPE, + ), + ArtifactInput( + "target_registry", + "uk.full.target_selection", + "selection", + TARGET_SELECTION_TYPE, + ), + ), + evidence_files={ + "full_gates": "uk.full.gates.calibrated.gate_report.json", + "diagnostics": terminal_files["calibration_diagnostics"]["filename"], + "holdout": terminal_files["holdout"]["filename"], + "target_diagnostics": terminal_files["target_diagnostics"]["filename"], + "area_support": terminal_files["area_support"]["filename"], + "target_registry": terminal_files["target_registry"]["filename"], + }, + ) + evidence_sources = { + "exported_evidence_full_gates": args.out + / "uk.full.gates.calibrated.gate_report.json", + "exported_evidence_diagnostics": args.out + / terminal_files["calibration_diagnostics"]["filename"], + "exported_evidence_holdout": args.out / terminal_files["holdout"]["filename"], + "exported_evidence_target_diagnostics": args.out + / terminal_files["target_diagnostics"]["filename"], + "exported_evidence_area_support": args.out + / terminal_files["area_support"]["filename"], + "exported_evidence_target_registry": args.out + / terminal_files["target_registry"]["filename"], + } + comparisons = prepared.comparison_sources or {} + graph = append_uk_full_certification_node( + graph, + population=full.population, + spine_provenance=prepared.spine_provenance, + comparison_sources=tuple(comparisons), + ) + final = run_graph( + compile_graph(graph), + sources={ + **sources, + "exported_dataset": dataset, + **evidence_sources, + **comparisons, + }, + store=store, + kernels=kernels, + resume="auto", + population_retention="lazy", + ) + final.save(args.out / "build.graph.json") + materialize_bytes(graph_to_json(graph).encode(), args.out / "graph.json") + _materialize_evidence(final, store, args.out) + package = json.loads(_payload(final, store, "uk.full.package", "package_inventory")) + candidate_file = materialize_bytes( + canonical_json(package), args.out / "candidate.json" + ) + certification_payload = _payload( + final, store, "uk.full.certification", "certification_readiness" + ) + certification_file = materialize_bytes( + certification_payload, args.out / "certification.json" + ) + completion = { + **package, + "kind": "uk_full_build_completion", + "candidate_manifest": candidate_file, + "certification": { + **certification_file, + "graph_artifact_key": final.nodes["uk.full.certification"].opaque_artifacts[ + "certification_readiness" + ], + }, + } + materialize_bytes(canonical_json(completion), args.out / "build.json") + return ( + 0 if package["readback_passed"] and not enforcement["enforced_blocking"] else 1 + ) + + +def main(argv: list[str] | None = None) -> int: + args = parse_args(argv) + prepared = None + try: + prepared = prepare_full_build(args) + return execute_full_build(prepared, args) + except Exception as error: + safe_output = False + if prepared is not None: + try: + _output_locations(prepared, args) + safe_output = True + except ValueError: + pass + if not args.dry_run and safe_output: + materialize_bytes( + canonical_json( + { + "schema": "microcosm.uk.full-build-failure.v1", + "error_type": type(error).__name__, + "message": str(error), + "release_authorized": False, + } + ), + args.out / "failure.json", + ) + print(f"UK full build failed: {error}", file=sys.stderr) + return 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/full_certification.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/full_certification.py new file mode 100644 index 000000000..8393023ee --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/full_certification.py @@ -0,0 +1,524 @@ +"""Unsigned certification readiness from one UK graph's identified artifacts. + +This consumer does not rerun a battery, solve weights or authorize publication. +Historical split-lane signatures remain verifiable in release_certification; +current builds bind one complete gate roster and the selected target scope. +""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import math +import sys +from collections.abc import Mapping +from dataclasses import replace +from pathlib import Path + +from microcosm.graph import ( + ArtifactInput, + ArtifactOutput, + ArtifactType, + Capabilities, + ContentStore, + Determinism, + Graph, + KernelBase, + KernelContext, + KernelRegistry, + KernelResult, + Node, + Numeric, + RunManifest, + SeedSource, + SourceRef, + source_hash, +) +from microcosm.graph.canonical import canonical_json + +from ..artifact_files import file_artifact, materialize_bytes +from ..country_spec import load_country_spec +from ..gate_battery import GateStatus, gate_phase_report_from_payload +from .full_gates import uk_full_gate_manifest, uk_full_gate_scope_receipt +from .graph_evidence import SPINE_GATE_REPORT_TYPE, uk_spine_gate_manifest +from .graph_targets import TARGET_SELECTION_TYPE, TARGET_SURFACE_TYPE +from .graph_terminal import ( + EXPORT_DESCRIPTOR_TYPE, + EXPORT_READBACK_TYPE, + FULL_DIAGNOSTICS_TYPE, + FULL_GATE_REPORT_TYPE, + FULL_HOLDOUT_TYPE, + PACKAGE_INVENTORY_TYPE, + decode_full_gate_report, +) + +FULL_CERTIFICATION_TYPE = ArtifactType("microcosm.uk.full-certification-readiness", 1) +_REQUIRED = frozenset( + { + "package", + "export_descriptor", + "export_readback", + "preflight", + "full_gates", + "diagnostics", + "holdout", + "selection", + "surface", + } +) +_COMPARISON_SOURCES = { + "native_scorecard": "uk_native_scorecard", + "matched_size_scorecard": "uk_matched_size_scorecard", +} + + +def _object(payload: bytes) -> dict: + value = json.loads(payload) + if not isinstance(value, dict): + raise ValueError("UK certification input must be a JSON object.") + return value + + +def _equal(actual, expected, label: str) -> None: + if actual != expected: + raise ValueError(f"UK certification {label} differs from its graph artifact.") + + +def _scorecard_status(payload, *, expected_identity, matched_households=None): + """Use the existing release-quality assessment, preserving incomplete evidence.""" + from microcosm.data.contract import _check_uk_incumbent_surface_evaluation + + if payload is None: + return { + "status": "evidence_absent", + "failures": ["No declared scorecard source."], + } + failures = [] + _check_uk_incumbent_surface_evaluation( + payload, failures, expected_identity=expected_identity + ) + if matched_households is not None: + comparison = payload.get("comparison", {}) + if ( + not isinstance(comparison, Mapping) + or comparison.get("kind") != "matched_size" + ): + failures.append( + "Matched-size scorecard needs an explicit matched_size comparison identity." + ) + elif any( + type(comparison.get(field)) is not int + or comparison[field] != matched_households + for field in ("candidate_households", "incumbent_households") + ): + failures.append( + "Matched-size scorecard population counts must both equal the exported k." + ) + return {"status": "passed" if not failures else "failed", "failures": failures} + + +def compose_uk_full_certification_readiness( + artifacts: Mapping[str, tuple[str, bytes]], + *, + comparison_sources: Mapping[str, tuple[Mapping, bytes]] | None = None, +) -> dict: + """Verify one graph's byte joins and report readiness for external review.""" + missing = _REQUIRED - set(artifacts) + if missing: + raise ValueError( + f"UK certification is missing graph artifacts {sorted(missing)}." + ) + documents = {name: _object(payload) for name, (_, payload) in artifacts.items()} + keys = {name: key for name, (key, _) in artifacts.items()} + package, readback = documents["package"], documents["export_readback"] + descriptor = documents["export_descriptor"] + if ( + package.get("kind") != "uk_full_build_package" + or package.get("schema_version") != 1 + or package.get("readback_passed") is not True + or package.get("release_authorized") is not False + ): + raise ValueError("UK certification requires the full graph package inventory.") + if ( + descriptor.get("kind") != "uk_full_build_export" + or descriptor.get("schema_version") != 1 + ): + raise ValueError("UK certification requires a typed export descriptor.") + if ( + readback.get("kind") != "uk_full_build_export_readback" + or readback.get("passed") is not True + ): + raise ValueError("UK certification requires passing exported-byte readback.") + _equal(package["dataset"], readback["dataset"], "candidate bytes") + _equal(package["content_sha256"], readback["content_sha256"], "candidate contents") + _equal( + readback["content_sha256"], descriptor["content_sha256"], "export descriptor" + ) + _equal(readback["bindings"], descriptor["bindings"], "export bindings") + _equal(package["build_bindings"], readback["bindings"], "package bindings") + for name in ("full_gates", "diagnostics", "holdout"): + _equal(package["artifacts"].get(name), keys[name], f"packaged {name}") + _equal( + package["artifacts"].get("export_readback"), + keys["export_readback"], + "packaged readback", + ) + selection = documents["selection"]["receipt"] + scope = uk_full_gate_scope_receipt(selection) + manifest = uk_full_gate_manifest(selection) + outcomes = {} + phase_names = [] + for name in ("preflight", "full_gates"): + document = documents[name] + report, _ = decode_full_gate_report(document) + _equal(document["selection_receipt"], selection, f"{name} target selection") + _equal(document["scope"], scope, f"{name} declared scope") + for dependency in ("selection", "surface"): + _equal( + document["artifacts"].get(dependency), + keys[dependency], + f"{name} {dependency}", + ) + phase_names.append(report.phase) + for outcome in report.outcomes: + if outcome.entry.id in outcomes: + raise ValueError("UK certification has duplicate gate outcomes.") + outcomes[outcome.entry.id] = outcome.to_payload() + final = documents["full_gates"] + _equal( + final["artifacts"].get("preflight"), keys["preflight"], "calibrated preflight" + ) + _equal(final["artifacts"].get("holdout"), keys["holdout"], "calibrated holdout") + _equal( + final["sample_fraction"], + documents["preflight"]["sample_fraction"], + "sample fraction", + ) + _equal( + final["release_candidate"], + documents["preflight"]["release_candidate"], + "release posture", + ) + if "spine_provenance" in documents: + if {"spine_assembled", "spine_transferred"} & set(documents): + raise ValueError( + "Use raw spine reports or bound checkpoint provenance, not both." + ) + provenance = documents["spine_provenance"] + _equal( + final["artifacts"].get("spine_provenance"), + keys["spine_provenance"], + "spine checkpoint provenance", + ) + report = provenance["spine_gate_report"]["payload"] + if not provenance.get("uk_frame_content_identity") or not provenance[ + "spine_gate_report" + ].get("sha256"): + raise ValueError("UK certification requires strict bound spine provenance.") + spine_gates = uk_spine_gate_manifest(load_country_spec("uk")) + expected = {entry.id: entry for entry in spine_gates.gates} + _equal(set(report["gates"]), set(expected), "checkpoint spine scope") + _equal(report.get("blocked_at_phase"), None, "checkpoint spine gate completion") + for gate_id, entry in expected.items(): + outcome = report["gates"][gate_id] + _equal( + outcome.get("criticality"), entry.criticality, f"{gate_id} criticality" + ) + _equal(outcome.get("phase"), entry.phase, f"{gate_id} phase") + outcomes[gate_id] = outcome + phase_names.extend(spine_gates.phases) + else: + spine_gates = uk_spine_gate_manifest(load_country_spec("uk")) + for name, phase in ( + ("spine_assembled", "assembled"), + ("spine_transferred", "transferred"), + ): + if name not in documents: + raise ValueError( + f"UK certification needs {name} or strict bound spine provenance." + ) + report = gate_phase_report_from_payload(documents[name], gates=spine_gates) + _equal(report.phase, phase, "spine phase") + phase_names.append(report.phase) + for outcome in report.outcomes: + if outcome.entry.id in outcomes: + raise ValueError( + "UK certification has duplicate spine gate outcomes." + ) + outcomes[outcome.entry.id] = outcome.to_payload() + _equal( + set(outcomes), {entry.id for entry in manifest.gates}, "complete gate roster" + ) + _equal(set(phase_names), set(manifest.phases), "complete gate phases") + reviewed_inapplicable = { + entry.id for entry in manifest.gates if entry.not_applicable is not None + } + failed_gates = sorted( + gate_id + for gate_id, outcome in outcomes.items() + if outcome["criticality"] == "release_blocking" + and outcome["status"] != GateStatus.PASSED.value + and not ( + gate_id in reviewed_inapplicable + and outcome["status"] == GateStatus.NOT_APPLICABLE.value + ) + ) + validation = documents["surface"]["source_validation"] + ledger = validation["ledger_provenance"] + expected_identity = { + "candidate_dataset_sha256": package["dataset"]["sha256"], + "candidate_manifest_sha256": hashlib.sha256( + artifacts["package"][1] + ).hexdigest(), + "candidate_diagnostics_sha256": hashlib.sha256( + artifacts["diagnostics"][1] + ).hexdigest(), + "ledger_facts_sha256": ledger["facts_sha256"], + "ledger_manifest_sha256": ledger["manifest_sha256"], + } + config = package["build_bindings"].get("configuration", {}) + requested_k = config.get("calibration", {}).get("dataset_households") + actual_k = descriptor["tables"]["household"]["rows"] + if requested_k is not None: + _equal(actual_k, requested_k, "exported exact household count") + comparisons = {} + comparison_sources = {} if comparison_sources is None else comparison_sources + if set(comparison_sources) - set(_COMPARISON_SOURCES): + raise ValueError("Unknown UK certification scorecard source.") + for role in _COMPARISON_SOURCES: + source = comparison_sources.get(role) + if role == "matched_size_scorecard" and requested_k is None: + comparisons[role] = { + "status": "not_required", + "reason": "No exported exact-count k was requested.", + } + if source is not None: + raise ValueError( + "Matched-size scorecard supplied to a build without requested k." + ) + continue + payload = None if source is None else _object(source[1]) + comparisons[role] = _scorecard_status( + payload, + expected_identity=expected_identity, + matched_households=actual_k if role == "matched_size_scorecard" else None, + ) + if source is not None: + file = dict(source[0]) + _equal( + file["sha256"], + hashlib.sha256(source[1]).hexdigest(), + f"{role} source bytes", + ) + _equal(file["size_bytes"], len(source[1]), f"{role} source length") + comparisons[role]["source"] = file + reasons = [] + if failed_gates: + reasons.append("Non-passing release-blocking gates: " + ", ".join(failed_gates)) + if final["sample_fraction"] != 1.0: + reasons.append( + "Development sample fraction does not establish native population readiness." + ) + holdout = documents["holdout"] + if scope["local_fit_claim"] and ( + holdout.get("skipped") + or holdout.get("method") != "rotated_folds" + or holdout.get("n_folds") != 5 + or len(holdout.get("folds", ())) != 5 + or any( + isinstance(holdout.get(field), bool) + or not isinstance(holdout.get(field), int | float) + or not math.isfinite(holdout[field]) + for field in ("mean_holdout_loss", "worst_holdout_loss") + ) + ): + reasons.append("The declared five-fold local holdout is skipped or incomplete.") + reasons.extend( + f"{role}: {record['status']}" + for role, record in comparisons.items() + if record["status"] not in {"passed", "not_required"} + ) + return { + "schema_version": 1, + "kind": "uk_full_build_certification_readiness", + "candidate": package["dataset"], + "content_sha256": package["content_sha256"], + "target_scope": scope, + "gate_coverage": { + "declared": sorted(outcomes), + "phases": sorted(phase_names), + "scope_exclusions": scope["scope_exclusions"], + }, + "gate_outcomes": outcomes, + "source_validation": validation, + "comparisons": comparisons, + "artifacts": { + name: { + "graph_artifact_key": key, + "sha256": hashlib.sha256(payload).hexdigest(), + "size_bytes": len(payload), + } + for name, (key, payload) in sorted(artifacts.items()) + }, + "ready_for_external_review": not reasons, + "readiness_failures": reasons, + "subnational_fit_certified": False, + "release_authorized": False, + "signing": "external", + } + + +class UKFullCertificationKernel(KernelBase): + ref = "uk.full.certification-readiness@1" + capabilities = Capabilities( + Determinism.DETERMINISTIC, numeric=Numeric.BITWISE, seed_source=SeedSource.NONE + ) + + def implementation_hash(self): + from microcosm.data import contract + + from . import full_gates, graph_terminal + + return source_hash(sys.modules[__name__], contract, full_gates, graph_terminal) + + def run(self, context: KernelContext) -> KernelResult: + sources = {} + for role, source_name in _COMPARISON_SOURCES.items(): + if source_name in context.sources: + path = Path(context.sources[source_name]) + payload = path.read_bytes() + sources[role] = (file_artifact(path), payload) + report = compose_uk_full_certification_readiness( + { + name: (value.key, value.payload) + for name, value in context.artifacts.items() + }, + comparison_sources=sources, + ) + return KernelResult( + artifacts={"certification_readiness": canonical_json(report)} + ) + + +def append_uk_full_certification_node( + graph: Graph, + *, + population: str, + spine_provenance: ArtifactInput | None = None, + comparison_sources: tuple[str, ...] = (), +) -> Graph: + """Append readiness after packaging; comparison sources are explicit inputs.""" + if set(comparison_sources) - set(_COMPARISON_SOURCES.values()): + raise ValueError("Unknown full-build comparison source.") + inputs = [ + ArtifactInput( + "package", "uk.full.package", "package_inventory", PACKAGE_INVENTORY_TYPE + ), + ArtifactInput( + "export_descriptor", + "uk.full.export.prepare", + "export_descriptor", + EXPORT_DESCRIPTOR_TYPE, + ), + ArtifactInput( + "export_readback", + "uk.full.export.readback", + "export_readback", + EXPORT_READBACK_TYPE, + ), + ArtifactInput( + "preflight", "uk.full.gates.preflight", "gate_report", FULL_GATE_REPORT_TYPE + ), + ArtifactInput( + "full_gates", + "uk.full.gates.calibrated", + "gate_report", + FULL_GATE_REPORT_TYPE, + ), + ArtifactInput( + "diagnostics", + "uk.full.gates.calibrated", + "calibration_diagnostics", + FULL_DIAGNOSTICS_TYPE, + ), + ArtifactInput("holdout", "uk.full.holdout", "holdout", FULL_HOLDOUT_TYPE), + ArtifactInput( + "selection", "uk.full.target_selection", "selection", TARGET_SELECTION_TYPE + ), + ArtifactInput( + "surface", "uk.full.target_compilation", "surface", TARGET_SURFACE_TYPE + ), + ] + if spine_provenance is None: + inputs.extend( + ArtifactInput( + f"spine_{phase}", + f"spine.gates.{phase}", + "gate_report", + SPINE_GATE_REPORT_TYPE, + ) + for phase in ("assembled", "transferred") + ) + else: + inputs.append(replace(spine_provenance, name="spine_provenance")) + existing = {source.name for source in graph.sources} + return replace( + graph, + sources=( + *graph.sources, + *( + SourceRef(name, "raw-bytes-v1") + for name in comparison_sources + if name not in existing + ), + ), + nodes=( + *graph.nodes, + Node( + "uk.full.certification", + UKFullCertificationKernel.ref, + population=population, + sources=comparison_sources, + artifact_inputs=tuple(inputs), + artifact_outputs=( + ArtifactOutput("certification_readiness", FULL_CERTIFICATION_TYPE), + ), + description="Bind complete selected-scope gates, exact exported bytes and native/size comparison readiness; publication and signing remain external.", + ), + ), + ) + + +def register_uk_full_certification_kernel(registry: KernelRegistry) -> None: + registry.register(UKFullCertificationKernel()) + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser( + description="Materialize one full UK graph's unsigned certification readiness; historical split-lane inputs are retired." + ) + parser.add_argument("--graph-manifest", required=True, type=Path) + parser.add_argument("--graph-store", required=True, type=Path) + parser.add_argument("--candidate-h5", required=True, type=Path) + parser.add_argument("--certification-json", required=True, type=Path) + args = parser.parse_args(argv) + store = ContentStore(args.graph_store) + manifest = RunManifest.load(args.graph_manifest, store) + key = manifest.nodes["uk.full.certification"].opaque_artifacts[ + "certification_readiness" + ] + payload = store.load_bytes(key) + report = _object(payload) + if ( + report.get("kind") != "uk_full_build_certification_readiness" + or report.get("release_authorized") is not False + ): + raise ValueError( + "Expected an unsigned full-graph certification readiness artifact." + ) + _equal( + file_artifact(args.candidate_h5), report["candidate"], "current candidate bytes" + ) + materialize_bytes(payload, args.certification_json) + return 0 if report["ready_for_external_review"] else 1 diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/full_gates.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/full_gates.py new file mode 100644 index 000000000..4e1fe4005 --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/full_gates.py @@ -0,0 +1,339 @@ +"""Complete UK gate ownership for one full build and explicit target filters. + +Country-only target selection removes local fit claims, while geographic +integrity, source coverage, register completeness and population checks remain. +Gate evaluation and persistence use the shared battery and graph artifacts. +""" + +from __future__ import annotations + +import hashlib +from collections.abc import Mapping +from dataclasses import replace +from types import SimpleNamespace +from typing import Any + +import numpy as np +from scipy import sparse + +from microcosm.build.country_spec import GatesManifest, load_country_spec +from microcosm.build.gate_battery import EvidenceContext +from microcosm.build.uk_runtime.calibration_run import uk_aggregate_admin_totals +from microcosm.build.uk_runtime.parity_reference import load_efrs_parity_reference +from microcosm.calibrate import TargetRegistry +from microcosm.calibrate.artifacts import OrderedProblem, OrderedSolution +from microcosm.calibrate.solve import CalibrationResult, _build_diagnostics +from microcosm.frame import Frame +from microcosm.graph.canonical import canonical_json + +_LOCAL_FIT_GATES = frozenset( + {"uk_local_target_fit", "uk_local_per_family_fit", "uk_local_area_support"} +) +_LOCAL_LEVELS = frozenset({"constituency", "local_authority", "la"}) + + +def _scope(selection_receipt: Mapping[str, Any] | None) -> tuple[bool, dict[str, str]]: + if selection_receipt is None: + return True, {} + if selection_receipt.get("schema") != "microcosm.calibrate.target-selection.v1": + raise ValueError("UK gate scope requires a versioned target-selection receipt.") + included = selection_receipt.get("included") + if not isinstance(included, list) or not included: + raise ValueError("UK gate scope requires nonempty selected targets.") + levels = {str(row["geography_level"]) for row in included} + unknown = levels - (_LOCAL_LEVELS | {"country", "region"}) + if unknown: + raise ValueError( + f"UK gate scope has unknown geography levels {sorted(unknown)}." + ) + has_local = bool(levels & _LOCAL_LEVELS) + selector = selection_receipt.get("selector", {}) + if selector.get("geography_levels") is None: + # An unfiltered source that omits all local targets is a completeness + # failure, not permission to declare national-only scope. + if not has_local: + raise ValueError("An unfiltered UK full build must include local targets.") + return True, {} + excluded = ( + {} + if has_local + else { + gate_id: "No local targets were selected; local fit and per-area fit-support claims are inapplicable." + for gate_id in sorted(_LOCAL_FIT_GATES) + } + ) + return has_local, excluded + + +def uk_full_gate_manifest( + selection_receipt: Mapping[str, Any] | None = None, + *, + source: GatesManifest | None = None, +) -> GatesManifest: + """Return every declared UK gate, except explicitly inapplicable fit checks.""" + source = load_country_spec("uk").gates if source is None else source + _, exclusions = _scope(selection_receipt) + return replace( + source, + policy=f"{source.policy}; full_build_scope", + gates=tuple(entry for entry in source.gates if entry.id not in exclusions), + ) + + +def uk_full_gate_scope_receipt( + selection_receipt: Mapping[str, Any] | None = None, +) -> dict[str, object]: + """Describe selected-fit scope without changing invariant gate ownership.""" + has_local, exclusions = _scope(selection_receipt) + return { + "schema": "microcosm.build.uk.full-gate-scope.v1", + "posture": "full_build" if has_local else "full_build_filtered_targets", + "local_fit_claim": has_local, + "scope_exclusions": exclusions, + "target_selection": None + if selection_receipt is None + else dict(selection_receipt), + } + + +def build_full_gate_context( + frame: Frame, + *, + ordered_problem: OrderedProblem, + solution: OrderedSolution, + selection_receipt: Mapping[str, Any], + stage_evidence: Mapping[str, Mapping[str, Any]], + supporting_evidence: Mapping[str, Any], +) -> EvidenceContext: + """Reconstruct gate evidence from identified rows and the installed solution. + + Supporting source/validation artifacts are forwarded under their existing + gate-binding names. Missing artifacts remain missing so shared evaluation + records an evidence failure; measured quantities are never filled with zero. + """ + _scope(selection_receipt) + bindings = ordered_problem.bindings + embedded = bindings.get("target_selection") + digest = bindings.get("target_selection_sha256") + if embedded is None and digest is None: + raise ValueError("Gate problem has no target-selection binding.") + if embedded is not None and canonical_json(embedded) != canonical_json( + selection_receipt + ): + raise ValueError("Gate target selection differs from the problem binding.") + if ( + digest is not None + and digest != hashlib.sha256(canonical_json(selection_receipt)).hexdigest() + ): + raise ValueError( + "Gate target-selection digest differs from the problem binding." + ) + if solution.problem_sha256 != ordered_problem.sha256: + raise ValueError("Gate solution belongs to a different ordered problem.") + if solution.entity_ids != ordered_problem.entity_ids: + raise ValueError("Gate solution and problem have different household axes.") + entity = ordered_problem.problem.weight_entity + ids = tuple(frame.table(entity)[frame.schema.entity_id_column(entity)]) + if ids != ordered_problem.entity_ids: + raise ValueError( + "Gate population differs from the ordered problem household axis." + ) + if not np.array_equal(frame.weights_for(entity).values, solution.weights): + raise ValueError("Gate population does not carry the bound solution weights.") + if ordered_problem.problem.skipped: + raise ValueError( + "Full-build gate context cannot omit skipped selected targets." + ) + problem = ordered_problem.problem + if len(ordered_problem.target_metadata) != len(problem.targets): + raise ValueError("Gate target metadata does not align with the ordered matrix.") + calibration_result = supporting_evidence.get("calibration_result") + if calibration_result is None: + solved_rows = _build_diagnostics( + problem, frame, problem.initial_weights.values, solution.weights + ) + else: + if not isinstance(calibration_result, CalibrationResult): + raise TypeError( + "Gate calibration_result must be a decoded CalibrationResult." + ) + result_problem = calibration_result.problem + result_ids = tuple( + calibration_result.frame.table(entity)[ + calibration_result.frame.schema.entity_id_column(entity) + ] + ) + if ( + calibration_result.weight_entity != entity + or result_ids != ordered_problem.entity_ids + or not np.array_equal(calibration_result.weights, solution.weights) + or not np.array_equal( + calibration_result.initial_weights, problem.initial_weights.values + ) + or result_problem.names != problem.names + or result_problem.matrix.shape != problem.matrix.shape + or ( + sparse.csr_array(result_problem.matrix) + != sparse.csr_array(problem.matrix) + ).nnz + or not np.array_equal(result_problem.target_vector, problem.target_vector) + or calibration_result.skipped + ): + raise ValueError( + "Gate calibration_result differs from the bound numerical problem/solution." + ) + solved_rows = calibration_result.diagnostics + if tuple( + row.name for row in solved_rows + ) != problem.names or not np.array_equal( + [row.target for row in solved_rows], problem.target_vector + ): + raise ValueError( + "Gate calibration_result diagnostics differ from the target axis." + ) + national_errors: dict[str, float] = {} + local_rows = [] + diagnostics = [] + for index, (target, metadata) in enumerate( + zip(problem.targets, ordered_problem.target_metadata, strict=True) + ): + level = str(metadata.get("geography_level", "")) + materialization = metadata.get("materialization") + if level not in _LOCAL_LEVELS | {"country", "region"}: + raise ValueError(f"Target {target.row_name!r} lacks classified geography.") + if materialization not in {"uk_local_surface", "uk_national_measure"}: + raise ValueError( + f"Target {target.row_name!r} lacks a materialization owner." + ) + row = { + "name": target.row_name, + "target_name": target.row_name, + "target": float(target.value), + "estimate": float(solved_rows[index].final_estimate), + "relative_error": float(solved_rows[index].relative_error), + "abs_relative_error": float(abs(solved_rows[index].relative_error)), + "family": str(metadata.get("family", "")), + "geography_level": level, + "area_type": "local_authority" if level == "la" else level, + "area_code": str(metadata.get("geography_id", "")), + "metric": str( + metadata.get("metric", metadata.get("contract_target_id", target.name)) + ), + } + if not row["family"]: + raise ValueError(f"Target {target.row_name!r} lacks a diagnostic family.") + diagnostics.append(row) + if materialization == "uk_local_surface": + local_rows.append(row) + else: + national_errors[target.row_name] = float(solved_rows[index].relative_error) + reference_registry = supporting_evidence.get("reference_registry") + if not isinstance(reference_registry, TargetRegistry): + raise ValueError( + "Full-build gates require the complete approved national reference registry." + ) + selected = { + (str(row["name"]), row["period"]) for row in selection_receipt["included"] + } + national_reference = { + spec.to_target().row_name + for spec in reference_registry.specs + if (spec.name, spec.period) in selected + } + reference = load_efrs_parity_reference() + manifest = uk_full_gate_manifest(selection_receipt) + admin_totals, admin_receipt = uk_aggregate_admin_totals(frame, manifest) + artifacts = dict(supporting_evidence) + artifacts.update( + { + "stage_evidence": dict(stage_evidence), + "build_stage_names": tuple(stage_evidence), + "national_calibration": { + "activated_reference_count": len(selection_receipt["included"]), + "resolved_reference_count": len(problem.targets), + "matrix_target_count": len(problem.names), + }, + "parity_evidence": SimpleNamespace( + candidate_columns={ + f"{entity}.{column}" + for entity in frame.entities + for column in frame.table(entity).columns + }, + reference_columns={ + f"{entity}.{name}" + for name, entity in reference.input_entities.items() + }, + candidate_targets=set(national_errors), + reference_targets=national_reference, + target_relative_errors=national_errors, + ), + "local_target_diagnostics": local_rows, + "target_diagnostics": diagnostics, + "aggregate_admin": admin_totals, + "aggregate_admin_measurement": admin_receipt, + "full_gate_scope": uk_full_gate_scope_receipt(selection_receipt), + } + ) + if "rules_engine" not in artifacts and "coverage_engine" in artifacts: + artifacts["rules_engine"] = artifacts["coverage_engine"] + return EvidenceContext(frame=frame, artifacts=artifacts) + + +def classify_full_gate_outcomes( + report, + *, + sample_fraction: float, + release_candidate: bool, +) -> dict[str, object]: + """Preserve declaration enforcement and the existing local export exception. + + The combined driver exports a diagnostic candidate after failures of its + five local fit/support/weight checks, but stops on a non-passing geography + ladder. Below f100 those five checks are recorded without enforcement. + All newly included national/source checks keep the shared battery's own + BLOCKS_ARTIFACT policy, including its declared missing-evidence rules. + """ + from microcosm.build.gate_battery import GatePhaseReport, GateStatus + from microcosm.build.uk_runtime.calibration_run import UK_LOCAL_GATE_SCOPE + + if not isinstance(report, GatePhaseReport): + raise TypeError("Classify a validated GatePhaseReport, not unbound JSON.") + if not 0.0 < sample_fraction <= 1.0: + raise ValueError("sample_fraction must be in (0, 1].") + geography = "uk_local_geography_ladder_post_calibration" + local_export_exception = set(UK_LOCAL_GATE_SCOPE) - {geography} + blocking = { + outcome.entry.id + for outcome in report.blocking_outcomes(release_candidate=release_candidate) + } + stop, exported_failures, unenforced, diagnostic = [], [], [], [] + for outcome in report.outcomes: + gate_id = outcome.entry.id + if gate_id == geography and outcome.status is not GateStatus.PASSED: + stop.append(gate_id) + continue + if outcome.status not in {GateStatus.FAILED, GateStatus.EVIDENCE_ABSENT}: + continue + if outcome.entry.criticality == "diagnostic": + diagnostic.append(gate_id) + elif gate_id in local_export_exception: + if gate_id in blocking and sample_fraction == 1.0: + exported_failures.append(gate_id) + else: + unenforced.append(gate_id) + elif gate_id in blocking: + stop.append(gate_id) + else: + unenforced.append(gate_id) + return { + "schema": "microcosm.build.uk.full-gate-enforcement.v1", + "sample_fraction": sample_fraction, + "release_candidate": release_candidate, + "structural_failures": stop, + "enforced_blocking": stop + exported_failures, + "exportable_blocking": exported_failures, + "unenforced_release_failures": unenforced, + "diagnostic_failures": diagnostic, + "artifact_permitted": not stop, + "release_blocking_gates_passed": not (stop or exported_failures or unenforced), + } diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/full_measure.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/full_measure.py new file mode 100644 index 000000000..a2ec27aac --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/full_measure.py @@ -0,0 +1,283 @@ +"""Rules-engine evaluation for the selected full-build target surface. + +Temporary inputs and prepared columns remain inside this operation. Graph +consumers receive compiled numerical contributions, never injected engine state. +""" + +from __future__ import annotations + +from collections.abc import Mapping +from pathlib import Path +from typing import Any + +import numpy as np +import pandas as pd + +from microcosm.build.target_materialization import resolve_target_measures +from microcosm.build.uk_runtime import ( + CalibrationFrameAdapter, + UKRowwiseNationalRows, + compute_household_metrics, + drop_injected_measure_inputs, + inject_measure_inputs, + ladder_clone_index_column, + materialize_uk_ledger_targets, +) +from microcosm.build.uk_runtime.measure_simulation import UKMeasureResolver +from microcosm.frame import Frame, MassChangeRecord + +UK_BLOCK_SENSITIVE_MEASURE_COLUMNS = ( + "ons/corporate_land_value", + "ons/land_value", + "slc/student_loan_repayment/england", +) + + +def resolve_uk_full_measures( + frame, + national_registry, + *, + period: int, + scratch_dir: Path, + band_edge_registry=None, + resolver_factory=UKMeasureResolver, + blocks: int = 1, + local_grains: tuple[str, ...] = ("constituency", "la"), +) -> tuple[Any, Any, UKRowwiseNationalRows, dict[str, pd.DataFrame], dict[str, Any]]: + """Resolve national inputs and local metrics on the cloned frame. + + ``blocks=1`` uses one scratch-mode engine for the whole clone. The + reviewed escape hatch ``blocks=K`` resolves each clone index separately, + then rejoins every entity-level prepared column by its stable entity id so + the full-frame target materialization and single solve retain frame order. + """ + + household = frame.table("household") + if blocks < 1: + raise ValueError("engine resolution blocks must be positive.") + if blocks == 1: + block_frames = [(None, frame)] + else: + clone_column = ladder_clone_index_column("household") + if clone_column not in household.columns: + raise ValueError(f"per-clone engine resolution requires {clone_column}.") + clone_indices = tuple(sorted(household[clone_column].unique().tolist())) + if len(clone_indices) != blocks: + raise ValueError( + "engine resolution blocks must match the realized clone indices: " + f"requested {blocks}, found {clone_indices}." + ) + person = frame.table("person") + block_frames = [] + for clone_index in clone_indices: + household_ids = set( + household.loc[ + household[clone_column] == clone_index, + "household_id", + ].tolist() + ) + person_mask = person["person_household_id"].isin(household_ids) + block = frame.select(person_mask) + # The block carries a K-th of the cloned mass while its log still + # ends on the full-clone record, and the scratch export validates + # the chain. Declare the subset explicitly: old = the cloned + # total, new = the block total, reason naming the block. The block + # frame is engine scratch and is discarded after resolution. + block_weights = block.weights_for("household") + full_total = float(frame.weights_for("household").total) + block_total = float(block_weights.total) + subset_record = MassChangeRecord( + entity="household", + old_total=full_total, + new_total=block_total, + declared_factor=block_total / full_total, + reason=( + f"engine resolution block {clone_index} of {blocks}: " + "scratch subset of the cloned frame for measure " + "resolution only, discarded after resolution" + ), + ) + block = Frame( + { + **{name: block.table(name) for name in block.entities}, + **{name: block.link(name) for name in block.links}, + }, + block.schema, + { + entity: block.weights_for(entity) + for entity in block.weighted_entities + }, + block.strata, + mass_log=(*block.mass_log, subset_record), + metadata=block.metadata, + ) + block_frames.append((clone_index, block)) + + measure_parts: dict[tuple[str, str], list[pd.Series]] = {} + metric_parts: dict[str, list[pd.DataFrame]] = {grain: [] for grain in local_grains} + resolver_receipts: list[Mapping[str, Any]] = [] + national_input_keys: set[tuple[str, str]] | None = None + for clone_index, block_frame in block_frames: + block_scratch = ( + scratch_dir if clone_index is None else scratch_dir / f"clone-{clone_index}" + ) + resolver = resolver_factory( + simulation_source=None, + scratch_dir=block_scratch, + year=period, + frame=block_frame, + ) + resolution = resolve_target_measures( + lambda block_frame=block_frame: CalibrationFrameAdapter(block_frame), + national_registry, + resolver, + period=period, + ) + keys = set(resolution.measure_inputs) + if national_input_keys is None: + national_input_keys = keys + elif keys != national_input_keys: + raise RuntimeError( + "per-clone engine resolution returned inconsistent national inputs." + ) + for (entity, variable), values in resolution.measure_inputs.items(): + entity_table = block_frame.table(entity) + entity_id = f"{entity}_id" + measure_parts.setdefault((entity, variable), []).append( + pd.Series( + np.asarray(values), + index=entity_table[entity_id].tolist(), + ) + ) + block_household_ids = block_frame.table("household")["household_id"].tolist() + for area_type in metric_parts: + metric_parts[area_type].append( + compute_household_metrics( + resolver.simulation, + area_type, + period=period, + household_ids=block_household_ids, + ) + ) + resolver_receipts.append(resolver.receipt()) + del resolver + simulation_input = block_scratch / "simulation-input.h5" + simulation_input.unlink(missing_ok=True) + try: + block_scratch.rmdir() + except OSError: + pass + + measure_inputs: dict[tuple[str, str], np.ndarray] = {} + for (entity, variable), parts in measure_parts.items(): + combined = pd.concat(parts) + if combined.index.has_duplicates: + raise RuntimeError( + f"per-clone engine resolution duplicated {entity} ids for {variable}." + ) + ordered_ids = frame.table(entity)[f"{entity}_id"] + ordered = combined.reindex(ordered_ids.tolist()) + if ordered.isna().any(): + raise RuntimeError( + f"per-clone engine resolution missed {entity} rows for {variable}." + ) + measure_inputs[(entity, variable)] = ordered.to_numpy() + + full_household_ids = household["household_id"].tolist() + local_metrics = {} + for area_type, parts in metric_parts.items(): + combined = pd.concat(parts) + if combined.index.has_duplicates: + raise RuntimeError( + f"per-clone engine resolution duplicated {area_type} household ids." + ) + ordered = combined.reindex(full_household_ids) + if ordered.isna().any().any(): + raise RuntimeError( + f"per-clone engine resolution missed {area_type} household rows." + ) + local_metrics[area_type] = ordered + + adapter = CalibrationFrameAdapter(frame) + # Injected engine inputs are scratch state for materialization only: + # they must be dropped before the prepared frame is assembled, or the + # flattening rule refuses columns that now exist on two entities + # (region, esa_* on the live spine). Same lifecycle as the national stage. + original_columns = { + entity: set(table.columns) for entity, table in adapter.tables.items() + } + inject_measure_inputs(adapter, measure_inputs) + materialized = materialize_uk_ledger_targets( + adapter, + national_registry, + period=period, + band_edge_registry=( + national_registry if band_edge_registry is None else band_edge_registry + ), + ) + if materialized.skipped: + raise RuntimeError( + "candidate national target materialization skipped row(s): " + f"{[skip.__dict__ for skip in materialized.skipped]}." + ) + modes = {receipt.get("mode") for receipt in resolver_receipts} + versions = {receipt.get("policyengine_uk_version") for receipt in resolver_receipts} + if len(modes) != 1 or len(versions) != 1: + raise RuntimeError("per-clone engine resolver provenance is inconsistent.") + cgt_period_contract = resolver_receipts[0].get("cgt_period_contract") + if any( + block_receipt.get("cgt_period_contract") != cgt_period_contract + for block_receipt in resolver_receipts[1:] + ): + raise RuntimeError("per-clone CGT period contract is inconsistent.") + receipt = { + "mode": next(iter(modes)), + "engine_version": next(iter(versions)), + "households": len(frame.table("household")), + "persons": len(frame.table("person")), + "benunits": len(frame.table("benunit")), + "national_inputs": len(measure_inputs), + "local_metrics": { + area_type: len(metrics.columns) + for area_type, metrics in local_metrics.items() + }, + "blocks": blocks, + } + if cgt_period_contract is not None: + receipt["cgt_period_contract"] = cgt_period_contract + if blocks > 1: + receipt["deviation"] = "per_clone_block_engine_resolution" + present = sorted( + column + for column in UK_BLOCK_SENSITIVE_MEASURE_COLUMNS + if column in {variable for _, variable in measure_inputs} + ) + receipt["block_sensitivity"] = { + "known_population_normalised_measures": list( + UK_BLOCK_SENSITIVE_MEASURE_COLUMNS + ), + "present_in_this_run": present, + "caveat": ( + "per-block engine resolution mis-measures population-normalised " + "formulas (each block reproduces a national aggregate); rows " + "on these measures are not evidence for adjudication from this " + "run. Resolve in a single block before ruling on them." + ), + } + try: + scratch_dir.rmdir() + except OSError: + pass + drop_injected_measure_inputs(adapter, measure_inputs, original_columns) + national_rows = UKRowwiseNationalRows( + targets=national_registry.to_target_set(), + registry=national_registry, + families=tuple(sorted({spec.family for spec in national_registry.specs})), + ) + return ( + adapter.prepared_frame(), + adapter.restore, + national_rows, + local_metrics, + receipt, + ) diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/full_problem.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/full_problem.py new file mode 100644 index 000000000..9aeb3814f --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/full_problem.py @@ -0,0 +1,205 @@ +"""Compile selected geographic contribution rows in the full UK build.""" + +from __future__ import annotations + +from collections.abc import Mapping +from typing import Any + +import numpy as np +import pandas as pd + +from microcosm.build.uk_runtime import ( + UkOaLadder, + UKRowwiseLocalMatrix, + uk_local_target_surface, +) +from microcosm.build.uk_runtime.local_rowwise import ( + build_uk_rowwise_local_surface_matrix, + empty_uk_local_problem, +) +from microcosm.calibrate import TargetRegistry + + +def _national_contract_target_ids(registry: TargetRegistry) -> tuple[str, ...]: + return tuple( + sorted( + { + str(spec.metadata.get("contract_target_id", spec.name)) + for spec in registry.specs + } + ) + ) + + +def _joint_surface_registry( + local_registry: TargetRegistry, + national_registry: TargetRegistry, +) -> TargetRegistry: + """Put national controls beside local cells for cross-grain reconciliation.""" + + return TargetRegistry( + [*local_registry.specs, *national_registry.specs], + country="uk", + ) + + +def build_uk_full_local_problem( + assignment: Any, + *, + target_ladder: UkOaLadder, + local_registry: TargetRegistry, + national_registry: TargetRegistry, + local_metrics: Mapping[str, pd.DataFrame], + period: int, + sample_fraction: float, + reviewed_unbound_higher_targets: Mapping[str, Mapping[str, object]], + ladder_household_uprating: Mapping[str, Any] | None = None, + selected_surface: pd.DataFrame | None = None, + surface_receipt: Mapping[str, Any] | None = None, +) -> tuple[ + pd.DataFrame, + UKRowwiseLocalMatrix, + dict[str, Any], + tuple[str, ...], + dict[str, Any], +]: + if assignment.ladder is not target_ladder: + raise ValueError( + "assignment and targets must come from the same loaded UK OA ladder object." + ) + household = assignment.result.frame.table("household").reset_index(drop=True) + household_index = pd.Index(household["household_id"], name="household_id") + metrics = { + grain: frame.set_axis(household_index, axis="index") + for grain, frame in local_metrics.items() + } + assigned = { + "constituency": pd.Series( + household["constituency_code"].astype(str).to_numpy(), + index=household_index, + ), + "la": pd.Series( + household["local_authority_code"].astype(str).to_numpy(), + index=household_index, + ), + } + assigned = {grain: assigned[grain] for grain in metrics} + national_ids = _national_contract_target_ids(national_registry) + if selected_surface is None: + surface, cross_grain = uk_local_target_surface( + _joint_surface_registry(local_registry, national_registry), + target_ladder, + bound_national_target_ids=national_ids, + period=period, + reviewed_unbound_higher_targets=reviewed_unbound_higher_targets, + ladder_household_uprating=ladder_household_uprating, + ) + else: + surface = selected_surface.copy() + cross_grain = dict(surface_receipt or {}) + covered = { + grain: set(values.astype(str).tolist()) for grain, values in assigned.items() + } + covered_mask = pd.Series( + [ + str(row.area_code) in covered[str(row.area_type)] + for row in surface.itertuples(index=False) + ], + index=surface.index, + dtype=bool, + ) + dropped = surface.loc[~covered_mask] + if sample_fraction < 1.0: + surface = surface.loc[covered_mask].reset_index(drop=True) + # Below f100 a covered area can still carry a nonzero cell with no metric + # support in the sample (no self-employed household among three drawn + # rows). The builder refuses such a cell at every rung; at development + # rungs the cell is dropped here and receipted instead. f100 stays strict. + unreachable = surface.iloc[0:0] + if sample_fraction < 1.0 and len(surface): + nonzero_by_grain = { + grain: (metrics[grain] != 0).groupby(assigned[grain]).sum() + for grain in metrics + } + unreachable_mask = pd.Series( + [ + float(row.value) != 0.0 + and str(row.metric) in nonzero_by_grain[str(row.area_type)].columns + and str(row.area_code) in nonzero_by_grain[str(row.area_type)].index + and int( + nonzero_by_grain[str(row.area_type)].loc[ + str(row.area_code), str(row.metric) + ] + ) + == 0 + for row in surface.itertuples(index=False) + ], + index=surface.index, + dtype=bool, + ) + unreachable = surface.loc[unreachable_mask] + surface = surface.loc[~unreachable_mask].reset_index(drop=True) + rung_surface = { + "dropped_unreachable_cells": int(len(unreachable)), + "dropped_unreachable_by_grain": { + str(key): int(value) + for key, value in unreachable.groupby("area_type").size().items() + }, + "dropped_unreachable_by_family": { + str(key): int(value) + for key, value in unreachable.groupby("family").size().items() + }, + "fraction": float(sample_fraction), + "dropped_cells": int(len(dropped) if sample_fraction < 1.0 else 0), + "dropped_by_grain": ( + { + str(key): int(value) + for key, value in dropped.groupby("area_type").size().items() + } + if sample_fraction < 1.0 + else {} + ), + "dropped_by_family": ( + { + str(key): int(value) + for key, value in dropped.groupby("family").size().items() + } + if sample_fraction < 1.0 + else {} + ), + } + rosters = { + "constituency": tuple(map(str, np.unique(target_ladder.constituency_code))), + "la": tuple(map(str, np.unique(target_ladder.local_authority_code))), + } + if surface.empty: + problem = empty_uk_local_problem(household_index) + else: + problem = build_uk_rowwise_local_surface_matrix( + metrics, + assigned, + surface, + area_codes_by_grain={grain: rosters[grain] for grain in metrics}, + require_every_assigned_area_covered=(sample_fraction == 1.0), + ) + local_bound = tuple( + sorted( + { + f"{row.family}/{row.area_type}" + for row in surface[["family", "area_type"]] + .drop_duplicates() + .itertuples(index=False) + } + ) + ) + national_bound = tuple( + f"national/{family}" + for family in sorted({spec.family for spec in national_registry.specs}) + ) + return ( + household, + problem, + cross_grain, + (*local_bound, *national_bound), + rung_surface, + ) diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/full_targets.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/full_targets.py new file mode 100644 index 000000000..de01b89d1 --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/full_targets.py @@ -0,0 +1,151 @@ +"""Pinned national and local target inputs for the single UK full build. + +Compilation and approved measure exclusions precede geography selection. The +unreduced national register remains available for band edges, and independent +reference-period compilations remain available for release validation. +""" + +from __future__ import annotations + +from datetime import date +from pathlib import Path +from typing import Any + +from microcosm.build.ledger_artifact import load_ledger_consumer_artifact +from microcosm.build.uk_runtime.calibration_run import ( + _ledger_provenance, + _validate_band_edge_registry, +) +from microcosm.build.uk_runtime.frs_release import load_uk_frs_release +from microcosm.build.uk_runtime.ledger_targets import ( + compile_uk_local_target_registry, + compile_uk_target_registry, + load_uk_local_area_crosswalk, +) +from microcosm.build.uk_runtime.measure_simulation import ( + apply_uk_calibration_measure_exclusions, + load_uk_calibration_measure_exclusions, +) +from microcosm.build.uk_runtime.national_chronicle_feed import ( + load_uk_national_chronicle_feed, +) +from microcosm.build.uk_runtime.weighted_integrity import exclusion_evaluation_date +from microcosm.calibrate import TargetRegistry + + +def load_uk_full_target_inputs( + facts_path: str | Path, + *, + expected_facts_sha256: str | None = None, + expected_manifest_sha256: str | None = None, + measure_exclusions: str | Path | None = None, + register_json: str | Path | None = None, + calibration_year: int | None = None, + exclusions_evaluated_on: date | None = None, +) -> dict[str, Any]: + """Compile the full target surface with the national source/review contract. + + Default hashes are the reviewed national feed pins. Explicit hashes must + agree with those pins as well: a target-scope filter does not authorize a + different source. The optional frozen register compares the complete, + pre-exclusion national register, matching its completeness role. + """ + pin = load_uk_national_chronicle_feed() + for label, supplied, committed in ( + ("facts", expected_facts_sha256, pin.facts_sha256), + ("manifest", expected_manifest_sha256, pin.manifest_sha256), + ): + if supplied is not None and supplied != committed: + raise ValueError( + f"UK full-build {label} SHA differs from the committed national feed pin." + ) + artifact = load_ledger_consumer_artifact( + Path(facts_path), + expected_facts_sha256=pin.facts_sha256, + expected_manifest_sha256=pin.manifest_sha256, + ) + if ( + artifact.facts_sha256 != pin.facts_sha256 + or artifact.manifest_sha256 != pin.manifest_sha256 + ): + raise ValueError( + "UK full-build Ledger artifact differs from the national feed pin." + ) + year = ( + load_uk_frs_release().calibration_year + if calibration_year is None + else calibration_year + ) + if type(year) is not int or year <= 0: + raise ValueError("calibration_year must be a positive integer.") + evaluated_on = exclusion_evaluation_date(exclusions_evaluated_on) + crosswalk = load_uk_local_area_crosswalk() + national_registries = {} + local_registries = {} + for period in sorted({2023, 2025, year}): + compilation = compile_uk_target_registry(artifact.facts, target_period=period) + if compilation.unsupported: + raise ValueError( + f"UK national target references failed to compile for {period}: " + f"{compilation.unsupported!r}." + ) + national_registries[period] = compilation.registry + for period in sorted({2025, year}): + compilation = compile_uk_local_target_registry( + artifact.facts, target_period=period, crosswalk=crosswalk + ) + if compilation.unsupported: + raise ValueError( + f"UK local target references failed to compile for {period}: " + f"{compilation.unsupported!r}." + ) + local_registries[period] = compilation.registry + band_edges = national_registries[year] + frozen_version = None + if register_json is not None: + frozen = TargetRegistry.from_json(Path(register_json)) + frozen_version = frozen.version + if frozen.version != band_edges.version: + raise ValueError( + "Re-derived full national register differs from the frozen scoring register: " + f"{band_edges.version} vs {frozen.version}." + ) + exclusions = load_uk_calibration_measure_exclusions( + None if measure_exclusions is None else Path(measure_exclusions) + ) + national_registry, exclusion_receipt = apply_uk_calibration_measure_exclusions( + band_edges, exclusions, now=evaluated_on + ) + _validate_band_edge_registry( + register_registry=national_registry, + band_edge_registry=band_edges, + exclusion_receipt=exclusion_receipt, + ) + by_name = {spec.name: spec for spec in band_edges.specs} + reviewed_unbound = { + str(by_name[name].metadata.get("contract_target_id", name)): record + for name, record in exclusion_receipt.items() + } + return { + "artifact": artifact, + "calibration_year": year, + "national_registry": national_registry, + "band_edge_registry": band_edges, + "local_registry": local_registries[year], + "measure_exclusions": exclusion_receipt, + "reviewed_unbound_higher_targets": reviewed_unbound, + "national_source_pin": pin.to_dict(), + "ledger_provenance": _ledger_provenance(artifact), + "register_completeness": { + "compiled_registry_version": band_edges.version, + "approved_registry_version": national_registry.version, + "frozen_registry_version": frozen_version, + "compiled_reference_count": len(band_edges.specs), + "approved_reference_count": len(national_registry.specs), + "measure_exclusion_count": len(exclusion_receipt), + "exclusions_evaluated_on": evaluated_on.isoformat(), + "band_edge_registry_reconciled": True, + }, + "uk_ledger_compiled_registries": national_registries, + "uk_ledger_compiled_local_registries": local_registries, + } diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/geography_ladder.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/geography_ladder.py index b5015c4b1..f63c6c6aa 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/geography_ladder.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/geography_ladder.py @@ -328,28 +328,15 @@ def load_uk_oa_ladder(path: str | Path) -> UkOaLadder: ) -def assign_uk_geography_ladder( +def draw_uk_ladder_locations( household: pd.DataFrame, ladder: UkOaLadder, *, seed: int = 0, expected_constituency_vintage: str | None = None, region_column: str = "region", -) -> pd.DataFrame: - """Assign each household one OA and the derived ladder columns. - - Two seeded draws under the build's seed discipline: a 2024 constituency is - sampled within the household's calibrated region proportional to - constituency household counts, then an OA is sampled within that - constituency proportional to 2021 Census OA population. Every finer and - coarser layer then derives from the OA, so the calibrated region marginal - is preserved exactly while every grain becomes filterable. - - Requires region assignment to have run first (the FRS carries it). A - household region absent from the ladder is an error, never a silent partial - join — for an England-&-Wales ladder that is exactly how a Scottish or - Northern Irish household is refused until those rungs are pinned. - """ +) -> np.ndarray: + """Draw the atomic locations using the existing sequential two-stage RNG.""" if region_column not in household.columns: raise ValueError( @@ -371,12 +358,35 @@ def assign_uk_geography_ladder( region_column=region_column, ) - assigned_index = _sample_oa_indices( + return _sample_oa_indices( region_codes.to_numpy(), ladder=ladder, seed=seed, ) + +def derive_uk_ladder_locations( + household: pd.DataFrame, + ladder: UkOaLadder, + assigned_index: np.ndarray, + *, + region_column: str = "region", +) -> pd.DataFrame: + """Derive every geography from a validated, already drawn atomic location.""" + + assigned_index = np.asarray(assigned_index) + if ( + assigned_index.shape != (len(household),) + or assigned_index.dtype.kind not in "iu" + or (assigned_index < 0).any() + or (assigned_index >= len(ladder)).any() + ): + raise ValueError("location indices must align with households and the ladder.") + region_codes = _validated_household_ladder_region_codes( + household, + ladder, + region_column=region_column, + ) assigned_region = ladder.region_code[assigned_index] mismatched = assigned_region != region_codes.to_numpy() if mismatched.any(): @@ -412,6 +422,44 @@ def assign_uk_geography_ladder( return assigned +def assign_uk_geography_ladder( + household: pd.DataFrame, + ladder: UkOaLadder, + *, + seed: int = 0, + expected_constituency_vintage: str | None = None, + region_column: str = "region", +) -> pd.DataFrame: + """Assign each household one OA and the derived ladder columns. + + Two seeded draws under the build's seed discipline: a 2024 constituency is + sampled within the household's calibrated region proportional to + constituency household counts, then an OA is sampled within that + constituency proportional to 2021 Census OA population. Every finer and + coarser layer then derives from the OA, so the calibrated region marginal + is preserved exactly while every grain becomes filterable. + + Requires region assignment to have run first (the FRS carries it). A + household region absent from the ladder is an error, never a silent partial + join — for an England-&-Wales ladder that is exactly how a Scottish or + Northern Irish household is refused until those rungs are pinned. + """ + + assigned_index = draw_uk_ladder_locations( + household, + ladder, + seed=seed, + expected_constituency_vintage=expected_constituency_vintage, + region_column=region_column, + ) + return derive_uk_ladder_locations( + household, + ladder, + assigned_index, + region_column=region_column, + ) + + def expected_uk_ladder_area_support( household: pd.DataFrame, ladder: UkOaLadder, @@ -453,10 +501,7 @@ def expected_uk_ladder_area_support( region_households = float(constituency_weight.sum()) for constituency_code, household_count in constituency_weight.items(): expected_rows = ( - n_clones - * int(n_region) - * float(household_count) - / region_households + n_clones * int(n_region) * float(household_count) / region_households ) constituency_key = str(constituency_code) constituency_expected[constituency_key] += expected_rows @@ -504,9 +549,13 @@ def uk_region_mix( """Summarize household row and weight shares by normalized UK region.""" if region_column not in household.columns: - raise ValueError(f"household table must contain region column {region_column!r}.") + raise ValueError( + f"household table must contain region column {region_column!r}." + ) if weight_column not in household.columns: - raise ValueError(f"household table must contain weight column {weight_column!r}.") + raise ValueError( + f"household table must contain weight column {weight_column!r}." + ) region_codes = _household_region_codes( household[region_column], @@ -518,7 +567,9 @@ def uk_region_mix( if not np.isfinite(weights).all() or (weights < 0).any(): raise ValueError(f"{weight_column} must be finite and non-negative.") if len(weights) == 0: - raise ValueError("household table must contain at least one row for region mix.") + raise ValueError( + "household table must contain at least one row for region mix." + ) total_weight = float(weights.sum()) if total_weight <= 0: raise ValueError(f"{weight_column} must carry positive total weight.") diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/graph.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/graph.py index a00abb0ad..945b6c3e4 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/graph.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/graph.py @@ -18,8 +18,10 @@ import json from collections.abc import Iterable, Mapping from dataclasses import dataclass +from importlib import metadata from microcosm.graph import ( + ArtifactOutput, Graph, KernelRegistry, Node, @@ -27,28 +29,22 @@ Slice, SourceRef, StructuralDelta, + compile_graph, ) from ..country_spec import CountrySpec, load_country_spec +from ..stage_evidence import STAGE_EVIDENCE_TYPE from .national_sampling import UK_SAMPLE_SEED_DEFAULT __all__ = [ - "UK_SPINE_EXCLUSIONS", "UK_SPINE_STRUCTURAL_STAGES", "uk_registry", + "uk_spine_endpoint", "uk_spine_graph", + "uk_spine_operation_inventory", ] -UK_SPINE_EXCLUSIONS = frozenset( - { - # These are the certified-candidate/H5 alternatives to the raw-FRS - # spine stages named below, not additional steps in this pipeline. - "frs_hmrc_retained_leaves", - "hmrc_spi_income", - } -) - UK_SPINE_STRUCTURAL_STAGES = frozenset( {"spi_support_channel", "cgt_incidence_clone", "cgt_band_donors"} ) @@ -712,9 +708,7 @@ def _deduplicate(cells: Iterable[_Cell]) -> tuple[_Cell, ...]: def _manifest_stages(spec: CountrySpec) -> tuple[object, ...]: if spec.sources is None: raise ValueError("The UK graph requires a source-stage manifest.") - selected = tuple( - stage for stage in spec.sources.stages if stage.stage not in UK_SPINE_EXCLUSIONS - ) + selected = spec.sources.stages if not selected or selected[0].stage != "frs_spine": raise ValueError("The UK FRS spine manifest must begin with 'frs_spine'.") unknown = [stage.stage for stage in selected[1:] if stage.stage not in _STAGE_CELLS] @@ -794,6 +788,29 @@ def _source_refs(source_mode: str) -> tuple[SourceRef, ...]: ) +def _numerical_dependency_versions() -> tuple[tuple[str, str], ...]: + """Bind installed behavior-bearing libraries, including optional engines. + + The missing marker permits source-only declarations without engine extras; + installing the engine then produces a different identity, never a false hit. + """ + versions = [] + for name in ( + "policyengine-uk", + "policyengine-core", + "numpy", + "pandas", + "scikit-learn", + "quantile-forest", + ): + try: + version = metadata.version(name) + except metadata.PackageNotFoundError: + version = "not-installed" + versions.append((name, version)) + return tuple(versions) + + def uk_spine_graph( spec: CountrySpec | None = None, *, @@ -813,6 +830,7 @@ def uk_spine_graph( raise ValueError("UK graph sample_seed must be non-negative.") resolved = load_country_spec("uk") if spec is None else spec stages = _manifest_stages(resolved) + dependency_versions = _numerical_dependency_versions() # The root transform loads the complete national-frame seed schema even # when a reduced hermetic manifest names only the output under test. # CREATE must declare every loaded cell, never merely the StagePlan's @@ -826,12 +844,14 @@ def uk_spine_graph( kernel="uk.create@1", outputs=tuple(cell.owned() for cell in root_cells), structural=StructuralDelta.CREATE, + artifact_outputs=(ArtifactOutput("stage_evidence", STAGE_EVIDENCE_TYPE),), sources=_source_names("frs_spine", source_mode), params={ "time_period": "2024", "stage_contract_sha256": _stage_contract_sha256(stages[0], resolved), "sample_fraction": float(sample_fraction), "sample_seed": int(sample_seed), + "numerical_dependencies": dependency_versions, }, description="Load the source-bound UK FRS root population.", ) @@ -883,6 +903,7 @@ def uk_spine_graph( ), params={ "stage": stage_name, + "numerical_dependencies": dependency_versions, "time_period": "2024", "expand_cells": tuple( (cell.entity, cell.column, cell.dtype) for cell in cells @@ -894,6 +915,9 @@ def uk_spine_graph( ), }, structural=StructuralDelta.EXPAND, + artifact_outputs=( + ArtifactOutput("stage_evidence", STAGE_EVIDENCE_TYPE), + ), base=current_population, sources=_source_names(stage_name, source_mode), mass=_STRUCTURAL_MASS[stage_name], @@ -964,8 +988,12 @@ def uk_spine_graph( for cell in cells ), population=current_population, + artifact_outputs=( + ArtifactOutput("stage_evidence", STAGE_EVIDENCE_TYPE), + ), params={ "stage": stage_name, + "numerical_dependencies": dependency_versions, "time_period": "2024", "stage_contract_sha256": _stage_contract_sha256( manifest_stage, resolved @@ -1017,3 +1045,74 @@ def uk_registry( uk_spine_graph() if graph is None else graph, {} if implementations is None else implementations, ) + + +@dataclass(frozen=True) +class UKSpineEndpoint: + """The final population and complete declared cell surface for composition.""" + + population: str + inputs: tuple[Slice, ...] + stage_names: tuple[str, ...] + + +def uk_spine_endpoint(graph: Graph) -> UKSpineEndpoint: + stages = ( + "frs_spine", + *(str(node.params["stage"]) for node in graph.nodes if "stage" in node.params), + ) + live = {} + for node in graph.nodes: + for owned in node.outputs: + live[(owned.entity, owned.column)] = _Cell( + owned.entity, owned.column, owned.dtype + ) + return UKSpineEndpoint( + population=compile_graph(graph).versions[stages[-1]], + inputs=_slices(live), + stage_names=stages, + ) + + +def uk_spine_operation_inventory( + graph: Graph, spec: CountrySpec | None = None +) -> tuple[dict[str, object], ...]: + """Generate truthful operation ownership from the executable stage roster. + + Conditional fits/draw chains remain one coupled execution unit. In + particular WAS encoding observes donors and recipients jointly; its fit + is not advertised as an independently reusable donor-only artifact. + """ + resolved = load_country_spec("uk") if spec is None else spec + nodes = {node.id: node for node in graph.nodes} + rows = [] + for stage in _manifest_stages(resolved): + node_id = "create_uk_frs" if stage.stage == "frs_spine" else stage.stage + node = nodes[node_id] + rows.append( + { + "stage": stage.stage, + "node": node_id, + "kernel": node.kernel, + "operations": [ + {"kind": operation.kind, "parameters": dict(operation.parameters)} + for operation in stage.operations + ], + "execution_unit": "composite" + if len(stage.operations) > 1 + else "single", + "source_inputs": list(node.sources), + "artifact_outputs": [output.name for output in node.artifact_outputs], + "randomness": "Existing literal/child seeds and draw order are preserved inside the registered transform.", + "coupling": ( + "Donor and recipient region encoding, four segmented fit/draw chains and their child seeds remain coupled." + if stage.stage == "was_wealth" + else "Source assembly, declared household sample selection and same-kind mass normalization execute once in CREATE." + if stage.stage == "frs_spine" + else "Declared preparation, fit and application operations execute once in this stage; intermediate models are not independently cached." + if any("qrf" in operation.kind for operation in stage.operations) + else None + ), + } + ) + return tuple(rows) diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_build.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_build.py new file mode 100644 index 000000000..2e78cc9fa --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_build.py @@ -0,0 +1,333 @@ +"""Compose the canonical UK full build on the shared executable graph. + +The same graph handles all targets (the default), explicit target filters, +dense exports and exact-count exports. A bound spine checkpoint resumes this +composition; an arbitrary historical uk-data H5 is not a build source. +""" + +from __future__ import annotations + +from dataclasses import asdict, dataclass, field, replace +from pathlib import Path + +import pandas as pd + +from microcosm.frame import Frame +from microcosm.graph import ( + ArtifactOutput, + ArtifactType, + Capabilities, + Determinism, + Graph, + KernelBase, + KernelContext, + KernelRegistry, + KernelResult, + Node, + Owned, + SourceRef, + StructuralDelta, + compile_graph, + source_hash, +) +from microcosm.graph.canonical import canonical_json +from microcosm.graph.codecs import SOURCE_CODECS + +from . import calibration_run, national_frame, release_certification +from .frs_release import load_uk_frs_release +from .graph import uk_spine_endpoint, uk_spine_graph +from .graph_calibration import ( + UKCalibrationNodes, + UKGraphCalibrationConfig, + register_uk_calibration_kernels, + uk_calibration_nodes, +) +from .graph_kernels import UKClaimKernel, UKIdentityKernel, _normalize_create_frame +from .graph_population import ( + append_uk_population_nodes, + population_columns, + population_slices, + register_uk_population_kernels, +) +from .graph_targets import append_uk_target_nodes, register_uk_target_kernels +from .local_doctrine import UK_LOCAL_CLONE_COUNT +from .national_sampling import UK_SAMPLE_SEED_DEFAULT + +SPINE_PROVENANCE_TYPE = ArtifactType("microcosm.uk.bound-spine-provenance", 1) + + +@dataclass(frozen=True) +class UKFullBuildConfig: + """Three independent controls: geography target scope, pool K, export k.""" + + calibration_year: int + time_period: str = field(default_factory=lambda: load_uk_frs_release().time_period) + source_year: int = field(default_factory=lambda: load_uk_frs_release().survey_year) + geography_levels: tuple[str, ...] | None = None + n_clones: int = UK_LOCAL_CLONE_COUNT + sample_fraction: float = 1.0 + source_sample_fraction: float = 1.0 + sample_seed: int = UK_SAMPLE_SEED_DEFAULT + seed: int = 42 + engine_blocks: int = 1 + constituency_vintage: str = "2024_pcon" + source_lineage_modulus: int | None = None + calibration: UKGraphCalibrationConfig = UKGraphCalibrationConfig() + + def __post_init__(self) -> None: + from .national_sampling import validate_sample_fraction + + validate_sample_fraction(self.sample_fraction, label="UK full pool") + validate_sample_fraction(self.source_sample_fraction, label="UK source spine") + if self.sample_fraction != 1.0 and self.source_sample_fraction != 1.0: + raise ValueError( + "A sampled spine cannot be sampled a second time in the full build." + ) + if type(self.n_clones) is not int or self.n_clones < 1: + raise ValueError("Geographic pool K must be a positive integer.") + if self.engine_blocks not in {1, self.n_clones}: + raise ValueError( + "Engine blocks must be one or equal the geographic pool K." + ) + if self.geography_levels is not None: + if not self.geography_levels or set(self.geography_levels) - { + "country", + "region", + "constituency", + "la", + }: + raise ValueError( + "Use explicit supported geography levels or omit the selector for all." + ) + if len(set(self.geography_levels)) != len(self.geography_levels): + raise ValueError("Geographic target levels must not repeat.") + if self.seed != self.calibration.seed: + raise ValueError( + "Pool and dense solve share the existing build seed; selection_seed is separate." + ) + + @property + def effective_sample_fraction(self) -> float: + return self.sample_fraction * self.source_sample_fraction + + +@dataclass(frozen=True) +class UKFullGraph: + graph: Graph + calibration: UKCalibrationNodes + config: UKFullBuildConfig + + @property + def population(self) -> str: + return self.calibration.population + + def operation_inventory(self) -> dict: + compiled = compile_graph(self.graph) + return { + "schema": "microcosm.uk.full-build-operations.v1", + "default_scope": "all_geographies", + "configuration": asdict(self.config), + "nodes": [ + { + "id": node_id, + "kernel": self.graph.node(node_id).kernel, + "description": self.graph.node(node_id).description, + "population": compiled.versions[node_id], + "dependencies": list(compiled.predecessors[node_id]), + "artifacts": [ + o.name for o in self.graph.node(node_id).artifact_outputs + ], + } + for node_id in compiled.order + ], + } + + +def uk_full_graph( + config: UKFullBuildConfig, + *, + spine: Graph | None = None, + spine_population: str | None = None, + spine_weight_kind: str = "importance", + optional_target_sources: tuple[str, ...] = (), + checkpoint_identity: dict | None = None, + review_date: str | None = None, +) -> UKFullGraph: + """Append the full build to the existing source-owned UK spine graph.""" + + initial = uk_spine_graph(source_mode="split") if spine is None else spine + endpoint = ( + uk_spine_endpoint(initial).population + if spine_population is None + else spine_population + ) + graph = append_uk_population_nodes( + initial, + population=endpoint, + time_period=config.time_period, + weight_kind=spine_weight_kind, + sample_fraction=config.sample_fraction, + sample_seed=config.sample_seed, + n_clones=config.n_clones, + seed=config.seed, + source_year=config.source_year, + constituency_vintage=config.constituency_vintage, + source_lineage_modulus=config.source_lineage_modulus, + ) + graph = append_uk_target_nodes( + graph, + calibration_year=config.calibration_year, + time_period=config.time_period, + geography_levels=config.geography_levels, + engine_blocks=config.engine_blocks, + sample_fraction=config.effective_sample_fraction, + target_weight_rule=config.calibration.target_weight_rule, + optional_sources=optional_target_sources, + review_date=review_date, + ) + cells = population_columns(graph, "uk.full.expand") + # This structural checkpoint depends on every pool operation, including + # evidence-only gates and the contribution problem. It binds the complete + # incoming ledger before downstream nodes reconstruct any Frame slices. + graph = replace( + graph, + nodes=( + *graph.nodes, + Node( + "uk.full.pool", + UKIdentityKernel.ref, + structural=StructuralDelta.FILTER, + base="uk.full.expand", + inputs=population_slices(cells), + description="Checkpoint the complete geographic pool and selected-target problem.", + ), + ), + ) + calibration = uk_calibration_nodes( + base="uk.full.pool", + columns=cells, + problem_producer="uk.full.problem", + config=config.calibration, + checkpoint_identity=checkpoint_identity, + ) + checkpoint_sources = ( + () + if checkpoint_identity is None + else ( + SourceRef("uk_size_checkpoint_manifest", "raw-bytes-v1"), + SourceRef("uk_size_checkpoint_arrays", "raw-bytes-v1"), + ) + ) + graph = replace( + graph, + nodes=(*graph.nodes, *calibration.nodes), + sources=(*graph.sources, *checkpoint_sources), + ) + compile_graph(graph) + return UKFullGraph(graph, calibration, config) + + +def _load_spine_h5(path: Path) -> Frame: + return national_frame.load_uk_national_frame(path)[0] + + +SOURCE_CODECS.register("uk-spine-h5-v1", _load_spine_h5) + + +class UKBoundSpineKernel(KernelBase): + ref = "uk.full.bound_spine@1" + capabilities = Capabilities( + Determinism.DETERMINISTIC, structural=StructuralDelta.CREATE + ) + + def implementation_hash(self) -> str: + return source_hash(type(self), national_frame, calibration_run, release_certification) + + def run(self, context: KernelContext) -> KernelResult: + if dict(context.params["spine_gate_digests"]) != calibration_run.uk_spine_checkpoint_gate_digests(): + raise ValueError("Bound spine gate declarations differ from the compiled checkpoint request.") + frame = _load_spine_h5(context.sources["uk_spine"]) + sidecar_path = context.sources["uk_spine_evidence"] + sidecar = calibration_run.load_bound_spine_checkpoint( + sidecar_path, frame, gate_report_path=context.sources["uk_spine_gates"] + ) + provenance = calibration_run.strict_spine_provenance_from_sidecar( + sidecar_path, sidecar, gate_report_path=context.sources["uk_spine_gates"] + ) + return KernelResult( + frame=_normalize_create_frame(frame, context), + artifacts={"spine_provenance": canonical_json(provenance)}, + ) + + +def bound_spine_graph(frame: Frame) -> Graph: + """Declare a checkpoint schema; the CREATE kernel verifies bound evidence.""" + + structural = { + "person_id", + "person_household_id", + "person_benunit_id", + "household_id", + "benunit_id", + } + + def token(dtype): + if isinstance(dtype, pd.StringDtype) or dtype.kind in "OUS": + return "string" + return str(dtype) + + outputs = tuple( + Owned(entity, str(column), token(frame.table(entity)[column].dtype)) + for entity in frame.entities + for column in frame.table(entity).columns + if column not in structural + ) + return Graph( + "uk", + ( + SourceRef( + "uk_spine", + "uk-spine-h5-v1", + "Bound canonical Microcosm spine checkpoint.", + ), + SourceRef( + "uk_spine_evidence", + "raw-bytes-v1", + "Exact source lineage, graph and gate evidence for the checkpoint.", + ), + SourceRef( + "uk_spine_gates", + "raw-bytes-v1", + "Exact gate report bound by the canonical spine sidecar.", + ), + ), + ( + Node( + "uk.full.spine_checkpoint", + UKBoundSpineKernel.ref, + structural=StructuralDelta.CREATE, + sources=("uk_spine", "uk_spine_evidence", "uk_spine_gates"), + params={"spine_gate_digests": calibration_run.uk_spine_checkpoint_gate_digests()}, + outputs=outputs, + artifact_outputs=( + ArtifactOutput("spine_provenance", SPINE_PROVENANCE_TYPE), + ), + description="Resume a canonical spine with its bound lineage and source evidence.", + ), + ), + ) + + +def register_uk_full_kernels(registry: KernelRegistry) -> KernelRegistry: + """Extend the existing UK source/stage registry with the full build.""" + + # Source graph registries already contain these primitive kernels. + for kernel in (UKBoundSpineKernel(), UKIdentityKernel(), UKClaimKernel()): + try: + registry.get(kernel.ref) + except KeyError: + registry.register(kernel) + register_uk_population_kernels(registry) + register_uk_target_kernels(registry) + register_uk_calibration_kernels(registry) + return registry diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_calibration.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_calibration.py new file mode 100644 index 000000000..87166ecb9 --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_calibration.py @@ -0,0 +1,851 @@ +"""UK solver bindings over shared, ordered calibration artifacts. + +Every solver consumes the original pool. Dense weights are not a selection +prior; filtering and installing the completed solution are separate structural +operations. Completed search and draw artifacts resume without repeating RNG. +""" + +from __future__ import annotations + +import hashlib +import json +from collections.abc import Mapping +from dataclasses import asdict, dataclass, replace + +import numpy as np +import pandas as pd + +from microcosm.calibrate import artifacts as calibration_artifacts +from microcosm.calibrate import calibrate +from microcosm.calibrate.artifacts import ( + PROBLEM_TYPE, + RESULT_TYPE, + SOLUTION_TYPE, + decode_calibration_result, + decode_problem, + decode_solution, + encode_calibration_result, + encode_problem, + encode_solution, +) +from microcosm.calibrate.exact_k import select_exact_k +from microcosm.calibrate.gates import HardConcrete +from microcosm.calibrate.initialization import contribution_initialization +from microcosm.calibrate.kernels import CalibrateAdamKernel +from microcosm.frame import Frame, MassChangeRecord, WeightKind, Weights +from microcosm.graph import ( + ArtifactInput, + ArtifactOutput, + ArtifactType, + Capabilities, + Determinism, + KernelBase, + KernelContext, + KernelRegistry, + KernelResult, + Node, + SeedSource, + StructuralDelta, + WeightTransition, + source_hash, +) +from microcosm.graph.canonical import canonical_json + +from . import dataset_size, size_checkpoint +from .dataset_size import UKSizeDraw, UKSizeSelection +from .graph_population import context_frame, population_slices +from .local_doctrine import UK_LOCAL_SOLVE_EPOCHS + +SIZE_SEARCH_TYPE = ArtifactType("microcosm.uk.size-search", 1) +SIZE_DRAW_TYPE = ArtifactType("microcosm.uk.size-draw", 1) +SIZE_RECEIPT_TYPE = ArtifactType("microcosm.uk.size-receipt", 1) +_DEPENDENCIES = ("numpy", "pandas", "scipy", "torch") + + +@dataclass(frozen=True) +class UKGraphCalibrationConfig: + epochs: int = UK_LOCAL_SOLVE_EPOCHS + learning_rate: float = 0.15 + seed: int = 42 + dataset_households: int | None = None + selection_seed: int | None = None + selection_pi_hi: float = 1.0 + target_weight_rule: str = "uniform" + + def __post_init__(self): + if type(self.epochs) is not int or self.epochs < 1: + raise ValueError("Calibration epochs must be a positive integer.") + if not np.isfinite(self.learning_rate) or self.learning_rate <= 0: + raise ValueError("Calibration learning rate must be positive and finite.") + for seed in (self.seed, self.selection_seed): + if seed is not None and (type(seed) is not int or seed < 0): + raise ValueError("Calibration seeds must be nonnegative integers.") + if self.dataset_households is not None and ( + type(self.dataset_households) is not int or self.dataset_households < 1 + ): + raise ValueError("Dataset household count must be a positive integer.") + dataset_size._check_pi_hi(self.selection_pi_hi) + + +@dataclass(frozen=True) +class UKCalibrationNodes: + nodes: tuple[Node, ...] + population: str + result_producer: str + problem_producer: str + solution_producer: str + size_producer: str | None + dense_producer: str + + +def _axis(frame: Frame) -> list: + return frame.table("household")["household_id"].tolist() + + +def _inputs(context: KernelContext): + frame = context_frame(context) + problem = decode_problem(context.artifacts["problem"].payload) + if ( + tuple(_axis(frame)) != problem.entity_ids + or problem.problem.weight_entity != "household" + ): + raise ValueError("UK solve requires its exact original household axis.") + weights = frame.weights_for("household") + if weights.kind != problem.problem.initial_weights.kind or not np.array_equal( + weights.values, problem.problem.initial_weights.values + ): + raise ValueError("UK solve requires the original pool weights.") + if problem.problem.skipped: + raise ValueError("UK solve cannot silently omit selected targets.") + return frame, problem + + +def _dense(context, frame, problem): + return decode_calibration_result( + context.artifacts["dense"].payload, frame=frame, problem=problem + ) + + +def _solution(result, frame, problem): + if result.frame.mass_log[: len(frame.mass_log)] != frame.mass_log: + raise ValueError("Calibration replaced the original pool mass ledger.") + return encode_solution( + result.weights, + entity_ids=_axis(result.frame), + problem_sha256=problem.sha256, + diagnostics={ + "frame_mass_log_append": [ + asdict(record) + for record in result.frame.mass_log[len(frame.mass_log) :] + ] + }, + ) + + +def restore_uk_graph_result( + pool: Frame, + *, + problem_payload: bytes, + result_payload: bytes, + solution_payload: bytes, + original_problem_payload: bytes | None = None, +): + """Rebuild completed diagnostics and exact legacy mass evidence, never solve. + + A compact result has its own HT-normalized initial weights and matrix. + Its original problem is required to authenticate the supplied full pool. + ``solution_payload`` is the original-bound installation solution. + """ + problem = decode_problem(problem_payload) + original = ( + problem + if original_problem_payload is None + else decode_problem(original_problem_payload) + ) + if ( + tuple(_axis(pool)) != original.entity_ids + or not np.array_equal( + pool.weights_for("household").values, + original.problem.initial_weights.values, + ) + or pool.weights_for("household").kind != original.problem.initial_weights.kind + ): + raise ValueError("Completed result requires its authenticated original pool.") + if ( + problem.sha256 != original.sha256 + and problem.bindings.get("original_problem_sha256") != original.sha256 + ): + raise ValueError( + "Compact result is not bound to the supplied original problem." + ) + solution = decode_solution( + solution_payload, problem_sha256=original.sha256, entity_ids=problem.entity_ids + ) + ids = set(problem.entity_ids) + selected = pool.select( + pool.table("person")["person_household_id"].isin(ids).to_numpy() + ) + if tuple(_axis(selected)) != problem.entity_ids: + raise ValueError("Completed result is not an ordered subset of the pool.") + initial_frame = Frame( + {e: selected.table(e) for e in selected.entities}, + selected.schema, + {"household": problem.problem.initial_weights}, + selected.strata, + mass_log=pool.mass_log, + metadata=pool.metadata, + ) + result = decode_calibration_result( + result_payload, frame=initial_frame, problem=problem + ) + if not np.array_equal(result.weights, solution.weights): + raise ValueError("Completed result disagrees with its installed solution.") + records = tuple( + MassChangeRecord(**dict(row)) + for row in solution.diagnostics["frame_mass_log_append"] + ) + if records and ( + not np.isclose(records[0].old_total, selected.weights_for("household").total) + or not np.isclose(records[-1].new_total, float(result.weights.sum())) + ): + raise ValueError("Completed result mass evidence differs from its boundary.") + final_frame = Frame( + {e: selected.table(e) for e in selected.entities}, + selected.schema, + {"household": result.frame.weights_for("household")}, + selected.strata, + mass_log=(*pool.mass_log, *records), + metadata=pool.metadata, + ) + return replace(result, frame=final_frame) + + +class _CalibrationKernel(KernelBase): + def implementation_hash(self): + return hashlib.sha256( + canonical_json( + { + "solver": CalibrateAdamKernel().implementation_hash(), + "adapter": source_hash( + type(self), + dataset_size, + size_checkpoint, + calibration_artifacts, + context_frame, + select_exact_k, + HardConcrete, + contribution_initialization, + dependencies=self.capabilities.dependencies, + ), + } + ) + ).hexdigest() + + +class UKSizeCheckpointImportKernel(_CalibrationKernel): + """Authenticate existing external checkpoints as explicit graph sources.""" + + ref = "uk.full.size_checkpoint_import@1" + capabilities = Capabilities(Determinism.DETERMINISTIC, dependencies=_DEPENDENCIES) + + def run(self, context): + frame, problem = _inputs(context) + manifest_path = context.sources[context.params["manifest_source"]] + arrays_path = context.sources[context.params["arrays_source"]] + identity = json.loads(context.params["identity_json"]) + stored = json.loads(manifest_path.read_bytes()) + # The historical loader accepts requested subsets. Graph import must + # authenticate every stored caller pin rather than leave old source, + # selector or K settings outside the comparison. + if set(identity) != set(stored.get("identity", {})): + raise ValueError( + "Size checkpoint import requires every original identity field." + ) + restored = size_checkpoint.load_uk_size_checkpoint_files( + manifest_path, + arrays_path, + frame=frame, + target_set=problem.to_target_set(), + identity=identity, + ) + dense, selection = restored.dense, restored.selection + for key in ("epochs", "learning_rate"): + if dense.options[key] != context.params[key]: + raise ValueError(f"Imported dense solve has different {key}.") + if dense.options["seed"] != context.params["dense_seed"]: + raise ValueError("Imported dense solve has a different seed.") + for key in ("households", "epochs", "learning_rate", "seed"): + if getattr(selection, key) != context.params[key]: + raise ValueError(f"Imported size search has different {key}.") + binding = problem.bindings + if ( + dense.options["mass"] != "free" + or dense.options["mass_reason"] != binding["mass_reason"] + or dense.options["max_weight_ratio"] != binding["max_weight_ratio"] + or dense.target_loss_cap != binding["target_loss_cap"] + or not np.array_equal( + dense.target_loss_weights, binding["target_loss_weights"] + ) + ): + raise ValueError("Imported dense solve has a different solve doctrine.") + metadata = { + "method": "contribution_informed_l0", + "problem_sha256": problem.sha256, + "protected": selection.protected.tolist(), + "households": selection.households, + "epochs": selection.epochs, + "learning_rate": selection.learning_rate, + "seed": selection.seed, + "pi_hi": selection.search_pi_hi, + } + return KernelResult( + artifacts={ + "dense": encode_calibration_result( + dense, entity_ids=problem.entity_ids, problem_sha256=problem.sha256 + ), + "search": encode_calibration_result( + selection.selection, + entity_ids=problem.entity_ids, + problem_sha256=problem.sha256, + ), + "selection": canonical_json(metadata), + } + ) + + +class UKDenseSolveKernel(_CalibrationKernel): + ref = "uk.full.dense@1" + capabilities = Capabilities( + Determinism.DETERMINISTIC, + seed_source=SeedSource.PARAM, + dependencies=_DEPENDENCIES, + ) + + def run(self, context): + from .graph_terminal import decode_full_gate_report + + if "preflight" not in context.artifacts: + raise ValueError( + "Dense calibration requires its source preflight artifact." + ) + report, classification = decode_full_gate_report( + context.artifacts["preflight"].payload + ) + if report.phase != "preflight": + raise ValueError("Dense calibration requires a preflight phase report.") + if not classification["artifact_permitted"]: + raise ValueError("Dense calibration refused by the source preflight.") + frame, problem = _inputs(context) + if "imported_dense" in context.artifacts: + result = decode_calibration_result( + context.artifacts["imported_dense"].payload, + frame=frame, + problem=problem, + ) + return KernelResult( + artifacts={ + "result": context.artifacts["imported_dense"].payload, + "solution": _solution(result, frame, problem), + } + ) + binding = problem.bindings + # These are the maintained full-build doctrine values carried by the + # selected problem, regardless of the geographic scope of its rows. + required = { + "mass_reason", + "max_weight_ratio", + "target_loss_weights", + "target_loss_cap", + } + if not required <= set(binding): + raise ValueError("UK ordered problem is missing its solve doctrine.") + result = calibrate( + frame, + problem.to_target_set(), + weight_entity="household", + epochs=context.params["epochs"], + learning_rate=context.params["learning_rate"], + seed=context.params["seed"], + mass="free", + mass_reason=binding["mass_reason"], + max_weight_ratio=binding["max_weight_ratio"], + target_loss_weights=np.asarray( + binding["target_loss_weights"], dtype=np.float64 + ), + target_loss_cap=binding["target_loss_cap"], + ) + return KernelResult( + artifacts={ + "result": encode_calibration_result( + result, entity_ids=problem.entity_ids, problem_sha256=problem.sha256 + ), + "solution": _solution(result, frame, problem), + } + ) + + +def _full_pool(frame, context): + k = context.params["households"] + if k > frame.n("household"): + raise ValueError( + "Requested dataset size exceeds the original pool; never clamped." + ) + return k == frame.n("household") + + +class UKSizeSearchKernel(_CalibrationKernel): + ref = "uk.full.size_search@1" + capabilities = Capabilities( + Determinism.DETERMINISTIC, + seed_source=SeedSource.PARAM, + dependencies=_DEPENDENCIES, + ) + + def run(self, context): + frame, problem = _inputs(context) + dense = _dense(context, frame, problem) + if "search" in context.artifacts: + _selection(context, frame, problem) + return KernelResult( + artifacts={ + "result": context.artifacts["search"].payload, + "selection": context.artifacts["selection"].payload, + } + ) + if _full_pool(frame, context): + metadata = { + "method": "full_pool", + "problem_sha256": problem.sha256, + **dict(context.params), + } + result_payload = context.artifacts["dense"].payload + else: + selection = dataset_size.select_uk_dataset_size( + frame, dense, **dict(context.params) + ) + metadata = { + "method": "contribution_informed_l0", + "problem_sha256": problem.sha256, + "protected": selection.protected.tolist(), + **dict(context.params), + } + result_payload = encode_calibration_result( + selection.selection, + entity_ids=problem.entity_ids, + problem_sha256=problem.sha256, + ) + return KernelResult( + artifacts={"result": result_payload, "selection": canonical_json(metadata)} + ) + + +def _selection(context, frame, problem): + metadata = json.loads(context.artifacts["selection"].payload) + if metadata["problem_sha256"] != problem.sha256: + raise ValueError("Size search belongs to a different ordered problem.") + for key in ("households", "epochs", "learning_rate", "seed"): + if metadata[key] != context.params[key]: + raise ValueError(f"Size search has a different {key}.") + if metadata["method"] == "full_pool": + if not _full_pool(frame, context): + raise ValueError("Full-pool receipt used for a compact request.") + return None + if metadata["method"] != "contribution_informed_l0": + raise ValueError("Unknown UK size-search method.") + raw_protected = np.asarray(metadata["protected"]) + if raw_protected.dtype != np.bool_ or raw_protected.shape != ( + frame.n("household"), + ): + raise ValueError("Size search protected mask is not aligned.") + return UKSizeSelection( + decode_calibration_result( + context.artifacts["search"].payload, frame=frame, problem=problem + ), + raw_protected, + metadata["households"], + metadata["epochs"], + metadata["learning_rate"], + metadata["seed"], + metadata["pi_hi"], + ) + + +class UKSizeDrawKernel(_CalibrationKernel): + ref = "uk.full.size_draw@1" + capabilities = Capabilities( + Determinism.DETERMINISTIC, + seed_source=SeedSource.PARAM, + dependencies=_DEPENDENCIES, + ) + + def run(self, context): + frame, problem = _inputs(context) + dense = _dense(context, frame, problem) + selection = _selection(context, frame, problem) + if selection is None: + draw = {"method": "full_pool", "support": list(range(frame.n("household")))} + else: + result = dataset_size.draw_uk_dataset_size( + frame, + dense, + selection=selection, + households=context.params["households"], + seed=context.params["seed"], + pi_hi=context.params["pi_hi"], + ) + draw = {"method": "exact_count", **asdict(result)} + for key in ("support", "inclusion_probabilities"): + draw[key] = draw[key].tolist() + return KernelResult( + artifacts={ + "draw": canonical_json({"problem_sha256": problem.sha256, **draw}) + } + ) + + +class UKSizeRefitKernel(_CalibrationKernel): + ref = "uk.full.size_refit@1" + capabilities = Capabilities( + Determinism.DETERMINISTIC, + seed_source=SeedSource.PARAM, + dependencies=_DEPENDENCIES, + ) + + def run(self, context): + frame, problem = _inputs(context) + dense = _dense(context, frame, problem) + selection = _selection(context, frame, problem) + draw = json.loads(context.artifacts["draw"].payload) + if draw.pop("problem_sha256") != problem.sha256: + raise ValueError("Exact-count draw belongs to another ordered problem.") + method = draw.pop("method") + if selection is None: + if method != "full_pool" or draw["support"] != list( + range(frame.n("household")) + ): + raise ValueError("Full-pool draw has a different support.") + cached_draw = None + else: + if method != "exact_count": + raise ValueError("Compact refit requires a completed exact-count draw.") + cached_draw = UKSizeDraw( + **{ + **draw, + "support": np.asarray(draw["support"]), + "inclusion_probabilities": np.asarray( + draw["inclusion_probabilities"], dtype=np.float64 + ), + } + ) + compact = dataset_size.refit_uk_dataset_size( + frame, dense, selection=selection, draw=cached_draw, **dict(context.params) + ) + result = compact.result + ids = _axis(result.frame) + if selection is None: + compact_payload = context.artifacts["problem"].payload + result_payload = context.artifacts["dense"].payload + compact_problem = problem + else: + compact_payload = encode_problem( + result.problem, + entity_ids=ids, + target_metadata=problem.target_metadata, + bindings={ + **dict(problem.bindings), + "original_problem_sha256": problem.sha256, + }, + ) + compact_problem = decode_problem(compact_payload) + result_payload = encode_calibration_result( + result, entity_ids=ids, problem_sha256=compact_problem.sha256 + ) + return KernelResult( + artifacts={ + "problem": compact_payload, + "result": result_payload, + "solution": _solution(result, frame, problem), + "refit_solution": _solution(result, frame, compact_problem), + "size": canonical_json( + { + **compact.receipt, + "problem_sha256": problem.sha256, + "household_ids": ids, + } + ), + } + ) + + +class UKSizeFilterKernel(_CalibrationKernel): + ref = "uk.full.selected@1" + capabilities = Capabilities( + Determinism.DETERMINISTIC, + structural=StructuralDelta.FILTER, + dependencies=_DEPENDENCIES, + ) + + def run(self, context): + frame = context_frame(context) + problem = decode_problem(context.artifacts["problem"].payload) + solution = decode_solution( + context.artifacts["solution"].payload, problem_sha256=problem.sha256 + ) + selected = set(solution.entity_ids) + if [i for i in _axis(frame) if i in selected] != list(solution.entity_ids): + raise ValueError("Selected household IDs are not an ordered pool subset.") + person = frame.table("person") + keep = pd.Series( + person["person_household_id"].isin(selected).to_numpy(), + index=pd.Index(person["person_id"], name="person_id"), + dtype=bool, + ) + return KernelResult(keep=keep) + + +class UKInstallCalibrationKernel(_CalibrationKernel): + ref = "uk.full.calibrated@1" + capabilities = Capabilities( + Determinism.DETERMINISTIC, + structural=StructuralDelta.REWEIGHT, + dependencies=_DEPENDENCIES, + ) + + def run(self, context): + frame = context_frame(context) + problem = decode_problem(context.artifacts["problem"].payload) + solution = decode_solution( + context.artifacts["solution"].payload, + problem_sha256=problem.sha256, + entity_ids=_axis(frame), + ) + return KernelResult( + weights=Weights(solution.weights, kind=WeightKind.CALIBRATED), + receipt={ + "frame_mass_log_append": solution.diagnostics["frame_mass_log_append"] + }, + ) + + +def uk_calibration_nodes( + *, + base: str, + columns: Mapping[tuple[str, str], str], + problem_producer: str, + problem_artifact: str = "problem", + prefix: str = "uk.full", + config: UKGraphCalibrationConfig | None = None, + checkpoint_identity: Mapping | None = None, + checkpoint_sources: tuple[str, str] = ( + "uk_size_checkpoint_manifest", + "uk_size_checkpoint_arrays", + ), +) -> UKCalibrationNodes: + """Compose the same numerical route for every selected target scope.""" + config = UKGraphCalibrationConfig() if config is None else config + inputs = population_slices(columns) + problem_input = ArtifactInput( + "problem", problem_producer, problem_artifact, PROBLEM_TYPE + ) + dense_id = f"{prefix}.dense" + checkpoint_id = f"{prefix}.size_checkpoint_import" + imported_dense = ( + () + if checkpoint_identity is None + else (ArtifactInput("imported_dense", checkpoint_id, "dense", RESULT_TYPE),) + ) + nodes = [ + Node( + id=dense_id, + kernel=UKDenseSolveKernel.ref, + population=base, + inputs=inputs, + params={ + "epochs": config.epochs, + "learning_rate": config.learning_rate, + "seed": config.seed, + }, + artifact_inputs=(problem_input, *imported_dense), + artifact_outputs=( + ArtifactOutput("result", RESULT_TYPE), + ArtifactOutput("solution", SOLUTION_TYPE), + ), + ) + ] + if checkpoint_identity is not None: + if config.dataset_households is None: + raise ValueError( + "Importing a size checkpoint requires an explicit dataset size." + ) + nodes.insert( + 0, + Node( + id=checkpoint_id, + kernel=UKSizeCheckpointImportKernel.ref, + population=base, + inputs=inputs, + sources=checkpoint_sources, + params={ + "identity_json": canonical_json(checkpoint_identity).decode(), + "manifest_source": checkpoint_sources[0], + "arrays_source": checkpoint_sources[1], + "epochs": config.epochs, + "learning_rate": config.learning_rate, + "dense_seed": config.seed, + "seed": config.seed + if config.selection_seed is None + else config.selection_seed, + "households": config.dataset_households, + }, + artifact_inputs=(problem_input,), + artifact_outputs=( + ArtifactOutput("dense", RESULT_TYPE), + ArtifactOutput("search", RESULT_TYPE), + ArtifactOutput("selection", SIZE_SEARCH_TYPE), + ), + ), + ) + solution_producer = result_producer = dense_id + result_problem_producer = problem_producer + size_producer = None + final_base = base + if config.dataset_households is not None: + params = { + "epochs": config.epochs, + "learning_rate": config.learning_rate, + "seed": config.seed + if config.selection_seed is None + else config.selection_seed, + "households": config.dataset_households, + "pi_hi": config.selection_pi_hi, + } + dense_input = ArtifactInput("dense", dense_id, "result", RESULT_TYPE) + search_id, draw_id, refit_id = ( + f"{prefix}.{s}" for s in ("size_search", "size_draw", "size_refit") + ) + nodes.append( + Node( + id=search_id, + kernel=UKSizeSearchKernel.ref, + population=base, + inputs=inputs, + params=params, + artifact_inputs=( + problem_input, + dense_input, + *( + () + if checkpoint_identity is None + else ( + ArtifactInput( + "search", checkpoint_id, "search", RESULT_TYPE + ), + ArtifactInput( + "selection", + checkpoint_id, + "selection", + SIZE_SEARCH_TYPE, + ), + ) + ), + ), + artifact_outputs=( + ArtifactOutput("result", RESULT_TYPE), + ArtifactOutput("selection", SIZE_SEARCH_TYPE), + ), + ) + ) + search_inputs = ( + problem_input, + dense_input, + ArtifactInput("search", search_id, "result", RESULT_TYPE), + ArtifactInput("selection", search_id, "selection", SIZE_SEARCH_TYPE), + ) + nodes.append( + Node( + id=draw_id, + kernel=UKSizeDrawKernel.ref, + population=base, + inputs=inputs, + params=params, + artifact_inputs=search_inputs, + artifact_outputs=(ArtifactOutput("draw", SIZE_DRAW_TYPE),), + ) + ) + nodes.append( + Node( + id=refit_id, + kernel=UKSizeRefitKernel.ref, + population=base, + inputs=inputs, + params=params, + artifact_inputs=( + *search_inputs, + ArtifactInput("draw", draw_id, "draw", SIZE_DRAW_TYPE), + ), + artifact_outputs=( + ArtifactOutput("result", RESULT_TYPE), + ArtifactOutput("problem", PROBLEM_TYPE), + ArtifactOutput("solution", SOLUTION_TYPE), + ArtifactOutput("refit_solution", SOLUTION_TYPE), + ArtifactOutput("size", SIZE_RECEIPT_TYPE), + ), + ) + ) + solution_producer = result_producer = result_problem_producer = ( + size_producer + ) = refit_id + final_base = f"{prefix}.selected" + nodes.append( + Node( + id=final_base, + kernel=UKSizeFilterKernel.ref, + base=base, + inputs=inputs, + structural=StructuralDelta.FILTER, + mass="free", + artifact_inputs=( + problem_input, + ArtifactInput("solution", refit_id, "solution", SOLUTION_TYPE), + ), + ) + ) + calibrated = f"{prefix}.calibrated" + nodes.append( + Node( + id=calibrated, + kernel=UKInstallCalibrationKernel.ref, + base=final_base, + inputs=inputs, + structural=StructuralDelta.REWEIGHT, + mass="free", + weights=WeightTransition("household", "calibrated", mass="free"), + artifact_inputs=( + problem_input, + ArtifactInput("solution", solution_producer, "solution", SOLUTION_TYPE), + ), + ) + ) + return UKCalibrationNodes( + tuple(nodes), + calibrated, + result_producer, + result_problem_producer, + solution_producer, + size_producer, + dense_id, + ) + + +def register_uk_calibration_kernels(registry: KernelRegistry) -> KernelRegistry: + for kernel in ( + UKSizeCheckpointImportKernel, + UKDenseSolveKernel, + UKSizeSearchKernel, + UKSizeDrawKernel, + UKSizeRefitKernel, + UKSizeFilterKernel, + UKInstallCalibrationKernel, + ): + registry.register(kernel()) + return registry diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_evidence.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_evidence.py new file mode 100644 index 000000000..e2f663c06 --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_evidence.py @@ -0,0 +1,385 @@ +"""UK spine evidence and gate bindings for the shared graph/store contracts.""" + +from __future__ import annotations + +import hashlib +import json +from collections.abc import Mapping, Sequence +from dataclasses import replace + +from microcosm.graph import ( + ArtifactInput, + ArtifactOutput, + ArtifactType, + Capabilities, + Determinism, + Graph, + KernelBase, + KernelContext, + KernelRegistry, + KernelResult, + KernelRole, + Node, + Numeric, + SeedSource, + compile_graph, + source_hash, +) + +from .. import gate_battery +from ..country_spec import CountrySpec, GatesManifest +from ..gate_battery import ( + BlockingMode, + EvidenceContext, + GateBatteryRun, + evaluate_phase, + gate_phase_report_from_payload, + gate_phase_report_payload, +) +from ..stage_evidence import ( + STAGE_EVIDENCE_TYPE, + decode_stage_evidence, + encode_stage_evidence, +) +from . import battery_bindings +from .calibration_run import UK_SPINE_GATE_SCOPE, uk_scoped_gate_manifest + +SPINE_GATE_REPORT_TYPE = ArtifactType("microcosm.gate-phase-report", 1) + + +def require_uk_spine_gate_admission( + context: KernelContext, *, alias: str = "spine_gate" +) -> None: + """Enforce a declared, already persisted spine phase before doing more work.""" + artifact = context.artifacts.get(alias) + if artifact is None: + if any(edge.name == alias for edge in context.node.artifact_inputs): + raise ValueError("The declared spine gate admission artifact is absent.") + return + from ..country_spec import load_country_spec + + report = gate_phase_report_from_payload( + json.loads(artifact.payload), + gates=uk_spine_gate_manifest(load_country_spec("uk")), + ) + if report.phase != context.params["spine_gate_phase"]: + raise ValueError("Spine admission report belongs to a different phase.") + blocking = report.blocking_outcomes( + release_candidate=bool(context.params["spine_gate_release_candidate"]) + ) + if blocking: + raise ValueError( + f"Stored {report.phase} spine gates block downstream execution: " + + ", ".join(outcome.entry.id for outcome in blocking) + ) + + +def uk_spine_gate_manifest(spec: CountrySpec) -> GatesManifest | None: + if getattr(spec, "gates", None) is None: + return None + return uk_scoped_gate_manifest( + UK_SPINE_GATE_SCOPE, + phases=("assembled", "transferred"), + policy_suffix="spine_build_scope", + source=spec.gates, + ) + + +def load_spine_stage_artifacts( + manifest, store, *, stage_names: Sequence[str] +) -> dict[str, dict[str, object]]: + """Read every requested stage from verified bytes, with no live transforms.""" + result = {} + for stage in stage_names: + node = "create_uk_frs" if stage == "frs_spine" else stage + receipt = manifest.nodes[node] + try: + key = receipt.opaque_artifacts["stage_evidence"] + except KeyError as error: + raise ValueError( + f"Spine stage {stage!r} has no stored evidence artifact." + ) from error + result[stage] = decode_stage_evidence(store.load_bytes(key), stage=stage) + return result + + +def spine_sidecar_evidence( + artifacts: Mapping[str, Mapping[str, object]], +) -> dict[str, object]: + """Project portable stage contracts onto the maintained sidecar schema.""" + return { + "stage_evidence": { + stage: document["evidence"] + for stage, document in artifacts.items() + if document["evidence"] is not None + }, + "fit_weight_records": { + stage: document["fit_weight_records"] + for stage, document in artifacts.items() + if "fit_weight_records" in document + }, + "sampling": artifacts["frs_spine"]["sampling"], + } + + +def add_uk_spine_gate_nodes( + graph: Graph, + *, + spec: CountrySpec, + engine_identity: str, + release_candidate: bool = False, +) -> Graph: + """Attach gates to exact assembled/transferred versions and stored evidence. + + The assembled gate reads the version frozen *by* the BRMA checkpoint, + before subsequent wealth rewrites. Signing and output paths stay external. + """ + from .graph import uk_spine_endpoint + + gates = uk_spine_gate_manifest(spec) + if gates is None: + return graph + if not engine_identity: + raise ValueError("Spine gates require a declared rules-engine identity.") + if any(node.kernel == UKSpineGateKernel.ref for node in graph.nodes): + raise ValueError("Spine gate nodes are already registered.") + compiled = compile_graph(graph) + endpoint = uk_spine_endpoint(graph) + stages = endpoint.stage_names + if "frs_brma" not in stages: + raise ValueError("Spine gates require the assembled frs_brma boundary.") + end = stages.index("frs_brma") + 1 + checkpoints = ( + ( + "assembled", + compiled.versions["frs_brma"], + graph.node("frs_brma.checkpoint").inputs, + stages[:end], + ), + ("transferred", endpoint.population, endpoint.inputs, stages), + ) + nodes = list(graph.nodes) + for phase, population, inputs, phase_stages in checkpoints: + if phase == "transferred" and end == len(stages): + continue + nodes.append( + Node( + id=f"spine.gates.{phase}", + kernel=UKSpineGateKernel.ref, + inputs=inputs, + population=population, + artifact_inputs=tuple( + ArtifactInput( + stage, + "create_uk_frs" if stage == "frs_spine" else stage, + "stage_evidence", + STAGE_EVIDENCE_TYPE, + ) + for stage in phase_stages + ) + + ( + ( + ArtifactInput( + "previous_gate", + "spine.gates.assembled", + "gate_report", + SPINE_GATE_REPORT_TYPE, + ), + ) + if phase == "transferred" + else () + ), + artifact_outputs=( + ArtifactOutput("gate_report", SPINE_GATE_REPORT_TYPE), + ), + params={ + "phase": phase, + "time_period": "2024", + "stage_names": phase_stages, + "gate_manifest": json.dumps( + gate_battery._gates_manifest_payload(gates), sort_keys=True + ), + "engine_identity": engine_identity, + "release_candidate": release_candidate, + }, + description=f"Evaluate the {phase} spine battery against its exact population and stage evidence.", + ) + ) + if end < len(stages): + # GATE receipts preserve failures but do not themselves stop kernels. + # Make the original assembled admission boundary an explicit dependency + # of the first later model, after the report has reached the store. + next_stage = stages[end] + nodes = [ + replace( + node, + artifact_inputs=( + *node.artifact_inputs, + ArtifactInput( + "spine_gate", + "spine.gates.assembled", + "gate_report", + SPINE_GATE_REPORT_TYPE, + ), + ), + params={ + **node.params, + "spine_gate_phase": "assembled", + "spine_gate_release_candidate": release_candidate, + }, + ) + if node.id == next_stage + else node + for node in nodes + ] + return replace(graph, nodes=tuple(nodes)) + + +class UKSpineGateKernel(KernelBase): + ref = "uk.spine-gates@1" + capabilities = Capabilities( + determinism=Determinism.DETERMINISTIC, + numeric=Numeric.BITWISE, + seed_source=SeedSource.NONE, + role=KernelRole.GATE, + ) + + def __init__(self, *, gates: GatesManifest, engine: object, engine_identity: str): + self.gates = gates + self.engine = engine + self.engine_identity = engine_identity + + def implementation_hash(self) -> str: + from microcosm.graph.canonical import canonical_json + + from ..country_spec import load_country_spec + + return hashlib.sha256( + canonical_json( + { + "code": source_hash(type(self), gate_battery, battery_bindings), + "country_resources": load_country_spec("uk").fingerprint, + } + ) + ).hexdigest() + + def run(self, context: KernelContext) -> KernelResult: + from microcosm.frame import Frame, MassChangeRecord + + from .graph_kernels import _minimal_frame + + expected = json.dumps( + gate_battery._gates_manifest_payload(self.gates), sort_keys=True + ) + if ( + context.params["gate_manifest"] != expected + or context.params["engine_identity"] != self.engine_identity + ): + raise ValueError( + "Spine gate binding differs from its declared manifest or engine." + ) + previous = context.artifacts.get("previous_gate") + if previous is not None: + previous_report = gate_phase_report_from_payload( + json.loads(previous.payload), gates=self.gates + ) + if previous_report.blocking_outcomes( + release_candidate=bool(context.params["release_candidate"]) + ): + report = gate_battery.GatePhaseReport( + phase=str(context.params["phase"]), + outcomes=tuple( + gate_battery.GateOutcome( + entry=entry, + status=gate_battery.GateStatus.UNREACHED, + reason="The assembled spine gate blocked this phase.", + ) + for entry in self.gates.gates + if entry.phase == context.params["phase"] + ), + ) + return KernelResult( + artifacts={ + "gate_report": encode_stage_evidence( + gate_phase_report_payload(report, gates=self.gates) + ) + }, + receipt={"outcome": "unreached", "phase": report.phase}, + ) + artifacts = { + stage: decode_stage_evidence(context.artifacts[stage].payload, stage=stage) + for stage in context.params["stage_names"] + } + evidence = spine_sidecar_evidence(artifacts) + minimal = _minimal_frame(context) + root_context = artifacts["frs_spine"]["frame_context"] + mass_records = [ + MassChangeRecord(**row) + for document in artifacts.values() + for row in document["frame_mass_log_append"] + ] + frame = Frame( + {entity: minimal.table(entity) for entity in minimal.entities}, + minimal.schema, + { + entity: minimal.weights_for(entity) + for entity in minimal.weighted_entities + }, + minimal.strata, + mass_log=tuple(mass_records), + metadata=root_context["metadata"], + ) + report = evaluate_phase( + self.gates, + str(context.params["phase"]), + EvidenceContext( + frame=frame, + artifacts={**evidence, "rules_engine": self.engine}, + ), + registry=battery_bindings.UK_GATE_REGISTRY, + ) + blocked = report.blocking_outcomes( + release_candidate=bool(context.params["release_candidate"]) + ) + return KernelResult( + artifacts={ + "gate_report": encode_stage_evidence( + gate_phase_report_payload(report, gates=self.gates) + ) + }, + receipt={"outcome": "fail" if blocked else "pass", "phase": report.phase}, + ) + + +def register_spine_gate_kernel( + registry: KernelRegistry, + *, + spec: CountrySpec, + engine: object, + engine_identity: str, +) -> None: + gates = uk_spine_gate_manifest(spec) + if gates is not None: + registry.register( + UKSpineGateKernel( + gates=gates, engine=engine, engine_identity=engine_identity + ) + ) + + +def materialize_spine_gate_reports( + manifest, store, *, battery: GateBatteryRun, gates: GatesManifest +) -> None: + """Restore cached verdicts, then use the original write-before-block policy.""" + for phase in gates.phases: + node = f"spine.gates.{phase}" + if node not in manifest.nodes: + continue + key = manifest.nodes[node].opaque_artifacts.get("gate_report") + if key is None: + raise ValueError(f"Spine gate {phase!r} produced no persisted report.") + report = gate_phase_report_from_payload( + json.loads(store.load_bytes(key)), gates=gates + ) + battery.record_phase(report) + battery.enforce(phase, mode=BlockingMode.BLOCKS_ARTIFACT) diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_kernels.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_kernels.py index 530e61470..6975f9fde 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_kernels.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_kernels.py @@ -44,7 +44,8 @@ ) from microcosm.graph.population import dtype_for_token -from . import uc_relationships +from .. import stage_evidence +from . import frs_hmrc_source, uc_relationships from .national_frame import UK_NATIONAL_SCHEMA from .rowwise_geography import id_multiplier_for_values @@ -79,7 +80,7 @@ "lcfs_consumption": "lcfs_consumption", "etb_vat": "etb_vat", "etb_services": "etb_services", - "frs_hmrc_spine_leaves": "frs_hmrc_leaves", + "frs_hmrc_spine_leaves": "spi_spine", "spi_support_channel": "spi_spine", "hmrc_spi_income_spine": "spi_spine", "uc_reporter_redraw": "uc_reporter_redraw", @@ -96,6 +97,7 @@ # Imported modules are not traversed by ``source_hash``. Bind relationship # helpers and the adapter's input-retention checks into every consuming stage. _STAGE_HELPER_MODULES = { + "frs_hmrc_spine_leaves": (frs_hmrc_source,), "frs_spine": (uc_relationships,), "frs_legacy_proxies": (uk_engine_adapter,), "frs_education_grant_split": (uk_engine_adapter,), @@ -144,13 +146,22 @@ def _stage_module(stage: str): def _implementation_hash(kernel: object, stage: str, transform: object | None) -> str: + from . import graph_evidence + # The stage module is the behavior-bearing source in every mode. Hashing # an injected transform's dynamic test wrapper would make # hermetic registries unhashable and, more importantly, would fail to bind # production edits made elsewhere in that stage's module. - del transform + dependencies = getattr(transform, "graph_implementation_dependencies", None) return source_hash( - type(kernel), _stage_module(stage), *_STAGE_HELPER_MODULES.get(stage, ()) + type(kernel), + stage_evidence, + graph_evidence, + _stage_artifacts, + _mass_log_payload, + _stage_module(stage), + *_STAGE_HELPER_MODULES.get(stage, ()), + *(dependencies() if callable(dependencies) else ()), ) @@ -170,6 +181,29 @@ def _mass_log_payload(before: Frame, after: Frame) -> list[dict[str, object]]: ] +def _stage_artifacts( + stage: str, transform: object | None, before: Frame | None, after: Frame +) -> dict[str, bytes]: + document = stage_evidence.snapshot_stage_evidence(stage, transform) + document["frame_mass_log_append"] = ( + [ + { + "entity": record.entity, + "old_total": record.old_total, + "new_total": record.new_total, + "declared_factor": record.declared_factor, + "reason": record.reason, + } + for record in after.mass_log + ] + if before is None + else _mass_log_payload(before, after) + ) + if before is None: + document["frame_context"] = {"metadata": dict(after.metadata)} + return {"stage_evidence": stage_evidence.encode_stage_evidence(document)} + + def _invoke_transform(transform: object, frame: Frame, context: KernelContext): """Invoke a stage, giving context-bound adapters only declared sources.""" @@ -669,7 +703,10 @@ def run(self, context: KernelContext) -> KernelResult: raise TypeError( f"The UK root transform returned {type(frame).__name__}, not Frame." ) - return KernelResult(frame=_normalize_create_frame(frame, context)) + return KernelResult( + frame=_normalize_create_frame(frame, context), + artifacts=_stage_artifacts("frs_spine", self.transform, None, frame), + ) class UKIdentityKernel(KernelBase): @@ -732,6 +769,9 @@ def implementation_hash(self) -> str: return _implementation_hash(self, self.stage, self.transform) def run(self, context: KernelContext) -> KernelResult: + from .graph_evidence import require_uk_spine_gate_admission + + require_uk_spine_gate_admission(context) transform = self.transform if transform is None and self.fixture_resolver is not None: transform = self.fixture_resolver.resolve(self.stage, context) @@ -754,6 +794,7 @@ def run(self, context: KernelContext) -> KernelResult: } return KernelResult( columns=MappingProxyType(columns), + artifacts=_stage_artifacts(self.stage, transform, before, after), receipt={ "stage": self.stage, "frame_mass_log_append": _mass_log_payload(before, after), @@ -840,6 +881,9 @@ def implementation_hash(self) -> str: return _implementation_hash(self, self.stage, self.transform) def run(self, context: KernelContext) -> KernelResult: + from .graph_evidence import require_uk_spine_gate_admission + + require_uk_spine_gate_admission(context) transform = self.transform if transform is None and self.fixture_resolver is not None: transform = self.fixture_resolver.resolve(self.stage, context) @@ -901,6 +945,7 @@ def run(self, context: KernelContext) -> KernelResult: columns=MappingProxyType(columns), expand=MappingProxyType(expand), weights=after_weights, + artifacts=_stage_artifacts(self.stage, transform, before, after), receipt=receipt, ) @@ -985,7 +1030,11 @@ def build_uk_registry( else: registry.register(UKStageKernel(stage, transform, fixture_resolver)) - required = {node.kernel for node in graph.nodes} + # Gate bindings carry the live rules engine and are registered separately + # after population-stage construction by the composing build. + required = { + node.kernel for node in graph.nodes if node.kernel != "uk.spine-gates@1" + } if set(registry.refs()) != required: missing = sorted(required - set(registry.refs())) extra = sorted(set(registry.refs()) - required) diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_population.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_population.py new file mode 100644 index 000000000..d060b6303 --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_population.py @@ -0,0 +1,541 @@ +"""Population operations in the single UK full-build graph. + +Source interpretation stays in UK adapters. Selection, row ancestry, typed +weights, content storage and replay are enforced by the shared graph runtime. +The legacy numerical functions remain the only implementations of the draws. +""" + +from __future__ import annotations + +import json +from collections.abc import Mapping +from dataclasses import asdict, replace + +import numpy as np +import pandas as pd + +from microcosm.frame import Frame, Weights +from microcosm.graph import ( + ArtifactInput, + ArtifactOutput, + ArtifactType, + Capabilities, + Determinism, + Graph, + KernelBase, + KernelContext, + KernelRegistry, + KernelResult, + Node, + Owned, + SeedSource, + Slice, + SourceRef, + StructuralDelta, + WeightUpdate, + compile_graph, + source_hash, +) +from microcosm.graph.canonical import canonical_json +from microcosm.graph.population import dtype_for_token +from microcosm.graph.weight_update import weight_update_receipt + +from . import geography_ladder, national_sampling, rowwise_dataset +from .geography_ladder import ( + UK_GEOGRAPHY_LADDER_COLUMNS, + derive_uk_ladder_locations, + draw_uk_ladder_locations, + load_uk_oa_ladder, +) +from .national_frame import UK_NATIONAL_SCHEMA +from .rowwise_dataset import expand_uk_geographic_pool, ladder_clone_index_column + +POPULATION_RECEIPT_TYPE = ArtifactType("microcosm.uk.population-receipt", 1) +LOCATION_DRAW_TYPE = ArtifactType("microcosm.uk.ladder-location-draw", 1) +GEOGRAPHY_GATE_TYPE = ArtifactType("microcosm.uk.geography-gate", 1) + + +def context_frame(context: KernelContext) -> Frame: + """Rebuild only declared slices, with the shared immutable weight context.""" + + return Frame( + { + entity: context.tables[entity] + .loc[ + :, + list( + context.frame_column_order.get( + entity, tuple(context.tables[entity].columns) + ) + ), + ] + .copy(deep=True) + for entity in UK_NATIONAL_SCHEMA.entities + }, + UK_NATIONAL_SCHEMA, + {"household": context.weights["household"]}, + context.strata.copy(deep=True), + mass_log=getattr(context, "frame_mass_log", ()), + metadata=getattr(context, "frame_metadata", {}) + or {"time_period": str(context.params["time_period"])}, + ) + + +def population_columns(graph: Graph, population: str) -> dict[tuple[str, str], str]: + """Resolve the carried and newly owned columns in a compiled version.""" + + compiled = compile_graph(graph) + holder = graph.node(population) + cells = {} if holder.base is None else population_columns(graph, holder.base) + for node in graph.nodes: + if compiled.versions[node.id] == population: + cells.update({(o.entity, o.column): o.dtype for o in node.outputs}) + return cells + + +def population_slices(cells: Mapping[tuple[str, str], str]) -> tuple[Slice, ...]: + return tuple( + Slice(entity, tuple(sorted(c for e, c in cells if e == entity))) + for entity in sorted({e for e, _ in cells}) + ) + + +def _series(frame: Frame, entity: str, column: str, dtype: str) -> pd.Series: + table = frame.table(entity) + ids = pd.Index( + table[frame.schema.entity_id_column(entity)], + name=frame.schema.entity_id_column(entity), + ) + return pd.Series(table[column].array, index=ids, name=column).astype( + dtype_for_token(dtype) + ) + + +def _mass_records(before: Frame, after: Frame) -> list[dict]: + if after.mass_log[: len(before.mass_log)] != before.mass_log: + raise ValueError( + "A graph population operation replaced its incoming mass ledger." + ) + return [asdict(record) for record in after.mass_log[len(before.mass_log) :]] + + +def _declared_mass(before: Frame, after: Frame) -> dict: + old, new = before.stratum_mass(), after.stratum_mass() + return { + "policy": "declared", + "before": float(old.sum()), + "after": float(new.sum()), + "stratum_before": {k: float(v) for k, v in old.items()}, + "stratum_after": {k: float(v) for k, v in new.items()}, + } + + +class _PopulationKernel(KernelBase): + def implementation_hash(self) -> str: + from . import graph_evidence + + return source_hash( + type(self), geography_ladder, national_sampling, rowwise_dataset, graph_evidence + ) + + +class UKFamilySampleKernel(_PopulationKernel): + ref = "uk.full.sample@1" + capabilities = Capabilities( + Determinism.DETERMINISTIC, + seed_source=SeedSource.PARAM, + structural=StructuralDelta.FILTER, + ) + + def run(self, context: KernelContext) -> KernelResult: + from .graph_evidence import require_uk_spine_gate_admission + + require_uk_spine_gate_admission(context) + before = context_frame(context) + fraction = float(context.params["fraction"]) + seed = int(context.params["seed"]) + if fraction == 1.0: + after = before + receipt = { + "fraction": fraction, + "seed": seed, + "sampled": False, + "pre_household_count": before.n("household"), + "post_household_count": before.n("household"), + "rung_token": "f100", + } + else: + after, receipt = national_sampling.sample_uk_spine_frame( + before, fraction=fraction, seed=seed + ) + receipt = {"sampled": True, **receipt} + ids = before.table("person")["person_id"] + keep = pd.Series( + ids.isin(after.table("person")["person_id"]).to_numpy(), + index=pd.Index(ids, name="person_id"), + dtype=bool, + ) + # The sampling function computes the historical normalization once. + # Installing its weights is a separate declared same-kind operation. + payload = { + "receipt": receipt, + "household_ids": after.table("household")["household_id"].tolist(), + "weights": after.weights_for("household").values.tolist(), + "mass_log_append": _mass_records(before, after), + } + return KernelResult(keep=keep, artifacts={"sampling": canonical_json(payload)}) + + +class UKSampleNormalizationKernel(_PopulationKernel): + ref = "uk.full.normalize@1" + capabilities = Capabilities( + Determinism.DETERMINISTIC, structural=StructuralDelta.REWEIGHT + ) + + def run(self, context: KernelContext) -> KernelResult: + frame = context_frame(context) + payload = json.loads(context.artifacts["sampling"].payload) + ids = frame.table("household")["household_id"].tolist() + if ids != payload["household_ids"]: + raise ValueError("Sample normalization household axis changed.") + weights = Weights( + np.asarray(payload["weights"], dtype=np.float64), + kind=context.weights["household"].kind, + ) + after = Frame( + {e: frame.table(e) for e in frame.entities}, + frame.schema, + {"household": weights}, + frame.strata, + metadata=frame.metadata, + ) + return KernelResult( + weights=weights, + receipt={ + "weight_update": weight_update_receipt(ids), + "mass": _declared_mass(frame, after), + "frame_mass_log_append": payload["mass_log_append"], + }, + ) + + +class UKGeographicExpansionKernel(_PopulationKernel): + ref = "uk.full.expand@1" + capabilities = Capabilities( + Determinism.DETERMINISTIC, structural=StructuralDelta.EXPAND + ) + + def run(self, context: KernelContext) -> KernelResult: + before = context_frame(context) + household = before.table("household").copy() + household["household_weight"] = before.weights_for("household").values + pool = expand_uk_geographic_pool( + person=before.table("person"), + benunit=before.table("benunit"), + household=household, + n_clones=int(context.params["n_clones"]), + source_year=int(context.params["source_year"]), + time_period=str(context.params["time_period"]), + household_weight_kind=before.weights_for("household").kind, + mass_log=before.mass_log, + source_lineage_modulus=context.params.get("source_lineage_modulus"), + ) + after = pool.frame + ancestry = {} + for entity in before.entities: + table = after.table(entity) + id_column = before.schema.entity_id_column(entity) + added = table.iloc[before.n(entity) :] + source_ids = ( + added[id_column].to_numpy() + - added[ladder_clone_index_column(entity)].to_numpy() + * pool.id_multiplier + ) + ancestry[entity] = pd.Series( + source_ids, index=pd.Index(added[id_column], name=id_column) + ) + columns = { + (e, c): _series(after, e, c, dtype) + for e, c, dtype in context.params["expand_cells"] + } + return KernelResult( + expand=ancestry, + columns=columns, + weights=after.weights_for("household"), + artifacts={ + "expansion": canonical_json( + {"n_clones": pool.n_clones, "id_multiplier": pool.id_multiplier} + ) + }, + receipt={"frame_mass_log_append": _mass_records(before, after)}, + ) + + +class UKLocationDrawKernel(_PopulationKernel): + ref = "uk.full.locations@1" + capabilities = Capabilities(Determinism.DETERMINISTIC, seed_source=SeedSource.PARAM) + + def run(self, context: KernelContext) -> KernelResult: + ladder = load_uk_oa_ladder(context.sources["uk_ladder"]) + household = context.tables["household"] + indices = draw_uk_ladder_locations( + household, + ladder, + seed=int(context.params["seed"]), + expected_constituency_vintage=str(context.params["constituency_vintage"]), + ) + return KernelResult( + artifacts={ + "locations": canonical_json( + { + "household_ids": household["household_id"].tolist(), + "indices": indices.tolist(), + "seed": int(context.params["seed"]), + "layer_vintages": ladder.layer_vintages, + "constituency_sampling_basis": ladder.metadata[ + "constituency_sampling_basis" + ], + "oa_sampling_basis": ladder.metadata["oa_sampling_basis"], + } + ) + } + ) + + +class UKGeographyMappingKernel(_PopulationKernel): + ref = "uk.full.geography_mapping@1" + capabilities = Capabilities(Determinism.DETERMINISTIC) + + def run(self, context: KernelContext) -> KernelResult: + payload = json.loads(context.artifacts["locations"].payload) + household = context.tables["household"] + if household["household_id"].tolist() != payload["household_ids"]: + raise ValueError("Location draw is bound to a different household axis.") + assigned = derive_uk_ladder_locations( + household, + load_uk_oa_ladder(context.sources["uk_ladder"]), + np.asarray(payload["indices"], dtype=np.int64), + ) + index = pd.Index(household["household_id"], name="household_id") + return KernelResult( + columns={ + (owned.entity, owned.column): pd.Series( + assigned[owned.column].array, index=index, name=owned.column + ).astype(dtype_for_token(owned.dtype)) + for owned in context.node.outputs + } + ) + + +class UKGeographyGateKernel(_PopulationKernel): + ref = "uk.full.geography_gate@1" + capabilities = Capabilities(Determinism.DETERMINISTIC) + + def run(self, context: KernelContext) -> KernelResult: + frame = context_frame(context) + gate = geography_ladder.uk_geography_ladder_gate( + frame.table("household"), frame.weights_for("household").values + ) + # Persist the failure. Downstream target materialization explicitly + # refuses this outcome; cached failure must never turn into a pass. + payload = asdict(gate) + return KernelResult( + artifacts={"gate": canonical_json(payload)}, + receipt={"outcome": "pass" if gate.passed else "fail", "evidence": payload}, + ) + + +def append_uk_population_nodes( + graph: Graph, + *, + population: str, + time_period: str, + weight_kind: str, + sample_fraction: float = 1.0, + sample_seed: int = national_sampling.UK_SAMPLE_SEED_DEFAULT, + n_clones: int = 1, + seed: int = 42, + source_year: int = 2024, + constituency_vintage: str = "2024_pcon", + source_lineage_modulus: int | None = None, +) -> Graph: + """Compose sampling → replication → location draw → derivation → gate.""" + + from .graph_kernels import UKClaimKernel + + cells = population_columns(graph, population) + if type(n_clones) is not int or n_clones < 1: + raise ValueError("Geographic replicate count K must be a positive integer.") + national_sampling.validate_sample_fraction(sample_fraction, label="UK full build") + national_sampling.validate_sample_seed(sample_seed, label="UK full build") + common = {"time_period": str(time_period)} + spine_gate_inputs = () + spine_gate_params = {} + if any(node.id == "spine.gates.transferred" for node in graph.nodes): + from .graph_evidence import SPINE_GATE_REPORT_TYPE + + gate = graph.node("spine.gates.transferred") + spine_gate_inputs = (ArtifactInput("spine_gate", gate.id, "gate_report", SPINE_GATE_REPORT_TYPE),) + spine_gate_params = {"spine_gate_phase": "transferred", "spine_gate_release_candidate": bool(gate.params["release_candidate"])} + nodes = list(graph.nodes) + nodes.append( + Node( + id="uk.full.sample", + kernel=UKFamilySampleKernel.ref, + inputs=population_slices(cells), + base=population, + structural=StructuralDelta.FILTER, + mass="free", + params={**common, "fraction": float(sample_fraction), "seed": sample_seed, **spine_gate_params}, + artifact_inputs=spine_gate_inputs, + artifact_outputs=(ArtifactOutput("sampling", POPULATION_RECEIPT_TYPE),), + description="Select intact source families; calculate their historical mass normalization.", + ) + ) + nodes.append( + Node( + id="uk.full.normalize", + kernel=UKSampleNormalizationKernel.ref, + inputs=population_slices(cells), + base="uk.full.sample", + structural=StructuralDelta.REWEIGHT, + mass="declared", + weights=WeightUpdate( + "household", weight_kind, "Normalize sampled source-family mass." + ), + artifact_inputs=( + ArtifactInput( + "sampling", "uk.full.sample", "sampling", POPULATION_RECEIPT_TYPE + ), + ), + params=common, + description="Install normalized weights without changing their kind.", + ) + ) + expansion_cells = { + ("household", "source_household_id"): "int64", + ("household", "source_year"): "int64", + ("household", "source_household_key"): "string", + **{ + (entity, ladder_clone_index_column(entity)): "int64" + for entity in UK_NATIONAL_SCHEMA.entities + }, + } + # Existing source lineage is carried unchanged, except the explicitly + # requested historical modulus conversion, which owns its declarations. + if source_lineage_modulus is not None: + expansion_cells.update( + {("household", rowwise_dataset.POOL_SOURCE_LINEAGE_COLUMN): "int64"} + ) + expansion_cells = { + key: cells.get(key, dtype) for key, dtype in expansion_cells.items() + } + nodes.append( + Node( + id="uk.full.expand", + kernel=UKGeographicExpansionKernel.ref, + inputs=population_slices(cells), + structural=StructuralDelta.EXPAND, + base="uk.full.normalize", + mass="conserve", + params={ + **common, + "source_year": source_year, + "n_clones": n_clones, + "source_lineage_modulus": source_lineage_modulus, + "expand_weight_entity": "household", + "expand_weight_kind": weight_kind, + "expand_cells": tuple( + (e, c, dtype) for (e, c), dtype in sorted(expansion_cells.items()) + ), + }, + artifact_outputs=(ArtifactOutput("expansion", POPULATION_RECEIPT_TYPE),), + description="Prepare source lineage and expand linked entities into K geographic copies.", + ) + ) + nodes.append( + Node( + id="uk.full.expand.owned", + kernel=UKClaimKernel.ref, + population="uk.full.expand", + outputs=tuple( + Owned(e, c, dtype, rewrite=(e, c) in cells) + for (e, c), dtype in sorted(expansion_cells.items()) + ), + params={ + "materialized_expand_outputs": tuple( + f"{e}.{c}" for e, c in expansion_cells if (e, c) not in cells + ) + }, + description="Declare the lineage and replicate columns materialized by expansion.", + ) + ) + cells.update(expansion_cells) + nodes.append( + Node( + id="uk.full.locations", + kernel=UKLocationDrawKernel.ref, + population="uk.full.expand", + inputs=(Slice("household", ("region",)),), + sources=("uk_ladder",), + params={"seed": seed, "constituency_vintage": constituency_vintage}, + artifact_outputs=(ArtifactOutput("locations", LOCATION_DRAW_TYPE),), + description="Draw constituency then atomic area with the current sequential RNG.", + ) + ) + nodes.append( + Node( + id="uk.full.geography_mapping", + kernel=UKGeographyMappingKernel.ref, + population="uk.full.expand", + inputs=(Slice("household", ("region",)),), + sources=("uk_ladder",), + artifact_inputs=( + ArtifactInput( + "locations", "uk.full.locations", "locations", LOCATION_DRAW_TYPE + ), + ), + outputs=tuple( + Owned("household", col, "string", rewrite=("household", col) in cells) + for col in UK_GEOGRAPHY_LADDER_COLUMNS + ), + description="Derive every geography from the drawn atomic-area index.", + ) + ) + cells.update({("household", col): "string" for col in UK_GEOGRAPHY_LADDER_COLUMNS}) + nodes.append( + Node( + id="uk.full.geography_gate", + kernel=UKGeographyGateKernel.ref, + population="uk.full.expand", + inputs=population_slices(cells), + params=common, + artifact_outputs=(ArtifactOutput("gate", GEOGRAPHY_GATE_TYPE),), + description="Validate geography integrity before selected-target contributions.", + ) + ) + sources = ( + graph.sources + if any(s.name == "uk_ladder" for s in graph.sources) + else ( + *graph.sources, + SourceRef( + "uk_ladder", + "raw-bytes-v1", + "Full UK atomic-area ladder with pinned vintages.", + ), + ) + ) + return replace(graph, nodes=tuple(nodes), sources=tuple(sources)) + + +def register_uk_population_kernels(registry: KernelRegistry) -> None: + for kernel in ( + UKFamilySampleKernel(), + UKSampleNormalizationKernel(), + UKGeographicExpansionKernel(), + UKLocationDrawKernel(), + UKGeographyMappingKernel(), + UKGeographyGateKernel(), + ): + registry.register(kernel) diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_targets.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_targets.py new file mode 100644 index 000000000..c6e7d53db --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_targets.py @@ -0,0 +1,617 @@ +"""One full UK target surface: source compilation, selection and contributions.""" + +from __future__ import annotations + +import hashlib +import json +import tempfile +from collections.abc import Mapping +from dataclasses import asdict, dataclass, replace +from datetime import date +from pathlib import Path +from types import SimpleNamespace + +import numpy as np +import pandas as pd + +from microcosm.build.country_spec import load_country_spec +from microcosm.calibrate import TargetRegistry, TargetSpec +from microcosm.calibrate.artifacts import ( + PROBLEM_TYPE, + _pack, + _unpack, + decode_problem, + encode_problem, +) +from microcosm.calibrate.matrix import build_constraint_matrix +from microcosm.calibrate.target_selection import select_targets +from microcosm.graph import ( + ArtifactInput, + ArtifactOutput, + ArtifactType, + Capabilities, + Determinism, + Graph, + KernelBase, + KernelContext, + KernelRegistry, + KernelResult, + Node, + SourceRef, + source_hash, +) +from microcosm.graph.canonical import canonical_json + +from . import full_measure, full_problem, ledger_targets, local_doctrine +from .full_measure import resolve_uk_full_measures +from .full_problem import build_uk_full_local_problem +from .geography_ladder import load_uk_oa_ladder +from .graph_population import ( + GEOGRAPHY_GATE_TYPE, + context_frame, + population_columns, + population_slices, +) +from .ledger_targets import uk_ladder_household_uprating, uk_ledger_households_total +from .local_rowwise import UKRowwiseNationalRows, prepare_uk_full_solve + +TARGET_SURFACE_TYPE = ArtifactType("microcosm.uk.full-target-surface", 1) +TARGET_SELECTION_TYPE = ArtifactType("microcosm.uk.full-target-selection", 1) +MEASURE_TYPE = ArtifactType("microcosm.uk.full-measured-contributions", 1) + + +def registry_payload(registry: TargetRegistry) -> dict: + return {"country": "uk", "specs": [asdict(spec) for spec in registry.specs]} + + +def registry_from_payload(payload: Mapping) -> TargetRegistry: + if payload.get("country") != "uk" or not isinstance(payload.get("specs"), list): + raise ValueError("Invalid UK target registry artifact.") + return TargetRegistry( + [TargetSpec(**spec) for spec in payload["specs"]], country="uk" + ) + + +def target_geography(spec: TargetSpec) -> str: + """Resolve explicit metadata without interpreting legacy registry buckets.""" + + metadata = spec.metadata + local = metadata.get("geography_level") + ledger = metadata.get("ledger_geography_level") + if local and ledger and local != ledger: + raise ValueError(f"Target {spec.name!r} has contradictory geography levels.") + level = local or ledger + if level not in {"country", "region", "constituency", "la"}: + raise ValueError( + f"Target {spec.name!r} has unsupported geography level {level!r}." + ) + return str(level) + + +def _surface_records(frame: pd.DataFrame) -> list[dict]: + def native(value): + if isinstance(value, np.generic): + value = value.item() + if value is pd.NA or (isinstance(value, float) and np.isnan(value)): + return None + return value + + # Preserve the exact binary float values; decimal-rounding table codecs + # can silently change target values on their first graph registration. + return [ + {key: native(value) for key, value in row.items()} + for row in frame.to_dict(orient="records") + ] + + +def _local_specs(surface: pd.DataFrame) -> list[TargetSpec]: + return [ + TargetSpec( + name=str(row["target_name"]), + entity="household", + measure=str(row["metric"]), + value=float(row["value"]), + period=row.get("period", 0), + source=str(row.get("source", "uk_rowwise_local_surface")), + family=str(row["family"]), + metadata={ + **row, + "geography_level": str(row["area_type"]), + "geography_id": str(row["area_code"]), + "materialization": "uk_local_surface", + }, + ) + for row in _surface_records(surface) + ] + + +def _selected_local_surface(surface: pd.DataFrame, specs) -> pd.DataFrame: + """Keep exact selected facts, including repeated names in different years.""" + keys = {spec.key for spec in specs} + periods = surface["period"] if "period" in surface else [0] * len(surface) + keep = [ + (str(name), period) in keys + for name, period in zip(surface["target_name"], periods, strict=True) + ] + return surface.loc[keep].reset_index(drop=True) + + +class _TargetKernel(KernelBase): + capabilities = Capabilities( + Determinism.DETERMINISTIC, dependencies=("policyengine-uk",) + ) + + def implementation_hash(self) -> str: + from . import full_targets + + implementation = source_hash( + type(self), + full_measure, + full_problem, + full_targets, + ledger_targets, + local_doctrine, + ) + return hashlib.sha256( + canonical_json( + { + "implementation": implementation, + "country_spec": load_country_spec("uk").fingerprint, + } + ) + ).hexdigest() + + +class UKFullTargetCompilationKernel(_TargetKernel): + ref = "uk.full.target_compilation@1" + + def run(self, context: KernelContext) -> KernelResult: + from .full_targets import load_uk_full_target_inputs + from .ledger_targets import uk_local_target_surface + + inputs = load_uk_full_target_inputs( + context.sources["uk_ledger_facts"], + measure_exclusions=context.sources.get("uk_measure_exclusions"), + register_json=context.sources.get("uk_frozen_register"), + calibration_year=int(context.params["calibration_year"]), + exclusions_evaluated_on=date.fromisoformat( + str(context.params["review_date"]) + ), + ) + ladder = load_uk_oa_ladder(context.sources["uk_ladder"]) + period = int(inputs["calibration_year"]) + national = inputs["national_registry"] + local = inputs["local_registry"] + uprating = uk_ladder_household_uprating( + ladder, + uk_ledger_households_total(inputs["artifact"].facts, period=period), + period=period, + ) + surface, reconciliation = uk_local_target_surface( + full_problem._joint_surface_registry(local, national), + ladder, + bound_national_target_ids=full_problem._national_contract_target_ids( + national + ), + period=period, + reviewed_unbound_higher_targets=inputs["reviewed_unbound_higher_targets"], + ladder_household_uprating=uprating, + ) + full = TargetRegistry( + [ + *_local_specs(surface), + *[ + replace( + spec, + metadata={ + **spec.metadata, + "materialization": "uk_national_measure", + "geography_level": target_geography(spec), + }, + ) + for spec in national.specs + ], + ], + country="uk", + ) + payload = { + "registry": registry_payload(full), + "local_registry": registry_payload(local), + "national_registry": registry_payload(national), + "band_edge_registry": registry_payload(inputs["band_edge_registry"]), + "surface": _surface_records(surface), + "surface_columns": surface.columns.tolist(), + "cross_geography": reconciliation, + "ladder_household_uprating": uprating, + "measure_exclusions": inputs["measure_exclusions"], + "reviewed_unbound_higher_targets": inputs[ + "reviewed_unbound_higher_targets" + ], + "source_validation": { + "national_source_pin": inputs["national_source_pin"], + "register_completeness": inputs["register_completeness"], + "ledger_provenance": inputs["ledger_provenance"], + }, + "uk_ledger_compiled_registries": { + str(period): registry_payload(registry) + for period, registry in inputs["uk_ledger_compiled_registries"].items() + }, + "uk_ledger_compiled_local_registries": { + str(period): registry_payload(registry) + for period, registry in inputs[ + "uk_ledger_compiled_local_registries" + ].items() + }, + "calibration_year": period, + } + return KernelResult(artifacts={"surface": canonical_json(payload)}) + + +class UKFullTargetSelectionKernel(_TargetKernel): + ref = "uk.full.target_selection@1" + + def run(self, context: KernelContext) -> KernelResult: + full = json.loads(context.artifacts["surface"].payload) + registry = registry_from_payload(full["registry"]) + levels = context.params.get("geography_levels") + selected = select_targets( + registry, geography_levels=levels, geography_resolver=target_geography + ) + if not len(selected.registry): + raise ValueError("Full build target selection contains no constraints.") + payload = { + "registry": registry_payload(selected.registry), + "receipt": selected.receipt, + } + return KernelResult(artifacts={"selection": canonical_json(payload)}) + + +class UKFullMeasureKernel(_TargetKernel): + ref = "uk.full.measures@1" + + def run(self, context: KernelContext) -> KernelResult: + gate = json.loads(context.artifacts["geography_gate"].payload) + if not gate["passed"]: + raise ValueError( + "Geography integrity gate failed: " + "; ".join(gate["failures"]) + ) + frame = context_frame(context) + full = json.loads(context.artifacts["surface"].payload) + selected = registry_from_payload( + json.loads(context.artifacts["selection"].payload)["registry"] + ) + national = selected.select( + predicate=lambda spec: ( + spec.metadata["materialization"] == "uk_national_measure" + ) + ) + grains = tuple( + sorted( + { + target_geography(spec) + for spec in selected + if spec.metadata["materialization"] == "uk_local_surface" + } + ) + ) + with tempfile.TemporaryDirectory( + prefix="microcosm-uk-full-measures-" + ) as scratch: + prepared, restore, rows, metrics, evidence = resolve_uk_full_measures( + frame, + national, + period=int(full["calibration_year"]), + scratch_dir=Path(scratch), + band_edge_registry=registry_from_payload(full["band_edge_registry"]), + blocks=int(context.params["engine_blocks"]), + local_grains=grains, + ) + # Compile national measures while temporary columns exist. The + # immutable pool, rather than that evaluation Frame, goes forward. + national_problem = ( + build_constraint_matrix(prepared, rows.targets, "household") + if len(rows.targets) + else None + ) + if national_problem is not None and national_problem.skipped: + failures = "; ".join( + f"{item.target.key}: {item.reason}" + for item in national_problem.skipped + ) + raise ValueError( + "Selected national constraints failed to compile: " + failures + ) + clean = restore(prepared) + for entity in frame.entities: + pd.testing.assert_frame_equal(clean.table(entity), frame.table(entity)) + arrays = { + f"metrics_{grain}": metrics[grain].to_numpy(dtype=np.float64) + for grain in grains + } + if national_problem is not None: + arrays["national_problem"] = np.frombuffer( + encode_problem( + national_problem, + entity_ids=frame.table("household")["household_id"].tolist(), + ), + dtype=np.uint8, + ) + metadata = { + "schema": MEASURE_TYPE.name + ".v1", + "household_ids": frame.table("household")["household_id"].tolist(), + "grains": {grain: metrics[grain].columns.tolist() for grain in grains}, + "has_national": national_problem is not None, + "evidence": evidence, + } + return KernelResult(artifacts={"measures": _pack(metadata, arrays)}) + + +def decode_measures(payload: bytes, *, selected: TargetRegistry) -> tuple[dict, dict]: + grains = { + target_geography(spec) + for spec in selected + if spec.metadata["materialization"] == "uk_local_surface" + } + has_national = any( + spec.metadata["materialization"] == "uk_national_measure" for spec in selected + ) + members = {f"metrics_{grain}" for grain in grains} + if has_national: + members.add("national_problem") + return _unpack(payload, schema=MEASURE_TYPE.name + ".v1", members=members) + + +@dataclass(frozen=True) +class UKFullProblemInputs: + """The exact admitted problem inputs shared by solve and holdout branches.""" + + frame: object + local_problem: object + national_rows: UKRowwiseNationalRows | None + bound_families: tuple[str, ...] + metadata: dict + full: dict + selection: dict + cross: dict + rung: object + national: TargetRegistry + selected: TargetRegistry + + +def reconstruct_uk_full_problem_inputs(context: KernelContext) -> UKFullProblemInputs: + """Reconstruct stored contributions without evaluating engine measures again.""" + frame = context_frame(context) + full = json.loads(context.artifacts["surface"].payload) + selection = json.loads(context.artifacts["selection"].payload) + selected = registry_from_payload(selection["registry"]) + national = selected.select( + predicate=lambda spec: spec.metadata["materialization"] == "uk_national_measure" + ) + local_specs = [ + spec + for spec in selected + if spec.metadata["materialization"] == "uk_local_surface" + ] + metadata, arrays = decode_measures( + context.artifacts["measures"].payload, selected=selected + ) + ids = frame.table("household")["household_id"].tolist() + if ids != metadata["household_ids"]: + raise ValueError("Measured contributions belong to a different household axis.") + metrics = { + grain: pd.DataFrame(arrays[f"metrics_{grain}"], columns=columns, index=ids) + for grain, columns in metadata["grains"].items() + } + surface = pd.DataFrame(full["surface"], columns=full["surface_columns"]) + surface = _selected_local_surface(surface, local_specs) + ladder = load_uk_oa_ladder(context.sources["uk_ladder"]) + _, local_problem, cross, bound_families, rung = build_uk_full_local_problem( + SimpleNamespace(result=SimpleNamespace(frame=frame), ladder=ladder), + target_ladder=ladder, + local_registry=registry_from_payload(full["local_registry"]), + national_registry=national, + local_metrics=metrics, + period=int(full["calibration_year"]), + sample_fraction=float(context.params["sample_fraction"]), + reviewed_unbound_higher_targets=full["reviewed_unbound_higher_targets"], + selected_surface=surface, + surface_receipt=full["cross_geography"], + ) + national_rows = None + if metadata["has_national"]: + national_problem = decode_problem(arrays["national_problem"].tobytes()) + national_rows = UKRowwiseNationalRows( + national_problem.to_target_set(), + national, + tuple(sorted({spec.family for spec in national})), + ) + return UKFullProblemInputs( + frame, + local_problem, + national_rows, + tuple(bound_families), + metadata, + full, + selection, + cross, + rung, + national, + selected, + ) + + +class UKFullProblemKernel(_TargetKernel): + ref = "uk.full.problem@1" + + def run(self, context: KernelContext) -> KernelResult: + inputs = reconstruct_uk_full_problem_inputs(context) + frame, local_problem = inputs.frame, inputs.local_problem + national_rows, bound_families = inputs.national_rows, inputs.bound_families + selection, selected = inputs.selection, inputs.selected + metadata, full = inputs.metadata, inputs.full + cross, rung = inputs.cross, inputs.rung + ids = frame.table("household")["household_id"].tolist() + prepared = prepare_uk_full_solve( + frame, + local_problem, + bound_families=bound_families, + national_rows=national_rows, + target_weight_rule=str(context.params["target_weight_rule"]), + ) + problem = build_constraint_matrix(frame, prepared.target_set, "household") + if problem.skipped: + raise ValueError("Selected full-build constraints failed to compile.") + by_key = {spec.key: spec for spec in selected} + target_metadata = [] + for target in problem.targets: + spec = by_key[target.key] + target_metadata.append( + { + **spec.metadata, + "family": spec.family, + "source": spec.source, + "geography_level": target_geography(spec), + } + ) + doctrine = local_doctrine.UK_LOCAL_SOLVE_DOCTRINE + payload = encode_problem( + problem, + entity_ids=ids, + target_metadata=target_metadata, + bindings={ + "target_selection": selection["receipt"], + "target_selection_sha256": hashlib.sha256( + canonical_json(selection["receipt"]) + ).hexdigest(), + "source_surface_sha256": context.artifacts["surface"].key, + "target_loss_weights": ( + np.ones(problem.n_targets, dtype=np.float64) + if prepared.target_loss_weights is None + else prepared.target_loss_weights + ).tolist(), + "mass_reason": prepared.mass_reason, + "target_loss_cap": doctrine.target_loss_cap, + "max_weight_ratio": doctrine.max_weight_ratio, + "binding_adjudications": prepared.binding_adjudications, + "rung_surface": rung, + "bound_families": list(bound_families), + "measure_resolution": metadata["evidence"], + "cross_geography": cross, + "measure_exclusions": full["measure_exclusions"], + "calibration_year": full["calibration_year"], + }, + ) + return KernelResult(artifacts={"problem": payload}) + + +def append_uk_target_nodes( + graph: Graph, + *, + population: str = "uk.full.expand", + calibration_year: int, + time_period: str, + geography_levels: tuple[str, ...] | None = None, + engine_blocks: int = 1, + sample_fraction: float = 1.0, + target_weight_rule: str = "uniform", + optional_sources: tuple[str, ...] = (), + review_date: str | None = None, +) -> Graph: + """Default all geographies. Scope is independent of K, k and solver outcomes.""" + + if geography_levels is not None and not geography_levels: + raise ValueError("An explicit geography selector must contain levels.") + cells = population_columns(graph, population) + slices = population_slices(cells) + compile_sources = ("uk_ladder", "uk_ledger_facts", *optional_sources) + common = {"time_period": time_period} + review_date = date.today().isoformat() if review_date is None else review_date + date.fromisoformat(review_date) + contract_identity = load_country_spec("uk").fingerprint + surface = ArtifactInput( + "surface", "uk.full.target_compilation", "surface", TARGET_SURFACE_TYPE + ) + selection = ArtifactInput( + "selection", "uk.full.target_selection", "selection", TARGET_SELECTION_TYPE + ) + nodes = ( + Node( + "uk.full.target_compilation", + UKFullTargetCompilationKernel.ref, + population=population, + sources=compile_sources, + params={ + "calibration_year": calibration_year, + "review_date": review_date, + "country_contract_sha256": contract_identity, + }, + artifact_outputs=(ArtifactOutput("surface", TARGET_SURFACE_TYPE),), + description="Compile pinned national and local evidence plus ladder target rows.", + ), + Node( + "uk.full.target_selection", + UKFullTargetSelectionKernel.ref, + population=population, + params={"geography_levels": geography_levels}, + artifact_inputs=(surface,), + artifact_outputs=(ArtifactOutput("selection", TARGET_SELECTION_TYPE),), + description="Select all geographies unless a target filter is explicitly requested.", + ), + Node( + "uk.full.measures", + UKFullMeasureKernel.ref, + population=population, + inputs=slices, + params={**common, "engine_blocks": engine_blocks}, + artifact_inputs=( + surface, + selection, + ArtifactInput( + "geography_gate", + "uk.full.geography_gate", + "gate", + GEOGRAPHY_GATE_TYPE, + ), + ), + artifact_outputs=(ArtifactOutput("measures", MEASURE_TYPE),), + description="Evaluate selected measures and compile temporary national contributions.", + ), + Node( + "uk.full.problem", + UKFullProblemKernel.ref, + population=population, + inputs=slices, + params={ + **common, + "sample_fraction": float(sample_fraction), + "target_weight_rule": target_weight_rule, + }, + sources=("uk_ladder",), + artifact_inputs=( + surface, + selection, + ArtifactInput("measures", "uk.full.measures", "measures", MEASURE_TYPE), + ), + artifact_outputs=(ArtifactOutput("problem", PROBLEM_TYPE),), + description="Build the one ordered selected-target problem and admission receipts.", + ), + ) + existing = {source.name for source in graph.sources} + sources = tuple( + SourceRef(name, "raw-bytes-v1") + for name in compile_sources + if name not in existing + ) + return replace( + graph, nodes=(*graph.nodes, *nodes), sources=(*graph.sources, *sources) + ) + + +def register_uk_target_kernels(registry: KernelRegistry) -> None: + for kernel in ( + UKFullTargetCompilationKernel(), + UKFullTargetSelectionKernel(), + UKFullMeasureKernel(), + UKFullProblemKernel(), + ): + registry.register(kernel) diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_terminal.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_terminal.py new file mode 100644 index 000000000..e53333a39 --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_terminal.py @@ -0,0 +1,1113 @@ +"""Full-build terminal artifacts, with streamed H5 materialization and readback. + +Graph kernels describe and validate the exact numerical export. Atomic disk +materialization is an outer service, so a cache hit cannot silently skip a +required file write. The resulting H5 is a content-bound source to the graph +continuation; large H5 payloads are never duplicated in a byte artifact. +""" + +from __future__ import annotations + +import hashlib +import json +import sys +from collections.abc import Mapping +from dataclasses import asdict, replace +from importlib import metadata +from pathlib import Path + +import numpy as np +import pandas as pd + +from microcosm.frame import Frame, engine_tables +from microcosm.graph import ( + ArtifactInput, + ArtifactOutput, + ArtifactType, + Capabilities, + Determinism, + Graph, + KernelBase, + KernelContext, + KernelRegistry, + KernelResult, + KernelRole, + Node, + Numeric, + SeedSource, + SourceRef, + source_hash, +) +from microcosm.graph.canonical import canonical_json +from microcosm.graph.codecs import SOURCE_CODECS + +from ..artifact_files import file_artifact +from . import geography_ladder, national_frame +from .geography_ladder import uk_geography_ladder_gate +from .graph_population import context_frame, population_columns, population_slices +from .national_frame import ( + _read_uk_national_tables, + _write_uk_single_year_tables, + uk_household_weight_kind, + uk_time_period, + validate_uk_national_frame, +) +from .rowwise_dataset import ( + ARTIFACT_CLONE_INDEX_COLUMN, + ladder_clone_index_column, + load_uk_rowwise_dataset, +) + +EXPORT_DESCRIPTOR_TYPE = ArtifactType("microcosm.uk.full-export-descriptor", 1) +EXPORT_READBACK_TYPE = ArtifactType("microcosm.uk.full-export-readback", 1) +PACKAGE_INVENTORY_TYPE = ArtifactType("microcosm.full-package-inventory", 1) +EXPORT_SOURCE_CODEC = "uk-single-year-h5@1" + + +def _tables(frame: Frame) -> dict[str, pd.DataFrame]: + tables = engine_tables(frame, weighted_entities=("household",)) + renamed = {} + for entity in ("person", "benunit", "household"): + column = ladder_clone_index_column(entity) + if column not in tables[entity]: + raise ValueError( + f"Full-build export lacks {entity} geographic replicate lineage {column!r}." + ) + renamed[entity] = tables[entity].rename( + columns={column: ARTIFACT_CLONE_INDEX_COLUMN} + ) + return renamed + + +def _table_description(table: pd.DataFrame) -> dict[str, object]: + descriptor = { + "rows": len(table), + "columns": list(table.columns), + "dtypes": [str(dtype) for dtype in table.dtypes], + "index_dtype": str(table.index.dtype), + } + digest = hashlib.sha256(canonical_json(descriptor)) + digest.update( + np.ascontiguousarray( + pd.util.hash_pandas_object(table, index=True).to_numpy() + ).tobytes() + ) + return {**descriptor, "content_sha256": digest.hexdigest()} + + +def _content_descriptor( + tables, *, time_period, weight_kind, mass_log +) -> dict[str, object]: + descriptor = { + "tables": { + entity: _table_description(tables[entity]) + for entity in ("person", "benunit", "household") + }, + "time_period": str(time_period), + "weight_kind": weight_kind.value, + "mass_log": [asdict(record) for record in mass_log], + "hash_environment": { + "pandas": metadata.version("pandas"), + "numpy": metadata.version("numpy"), + }, + } + return { + **descriptor, + "content_sha256": hashlib.sha256(canonical_json(descriptor)).hexdigest(), + } + + +def describe_uk_export( + frame: Frame, *, bindings: Mapping[str, object] +) -> dict[str, object]: + """Validate and describe the maintained H5 layout without serializing it.""" + validate_uk_national_frame(frame) + tables = _tables(frame) + gate = uk_geography_ladder_gate( + tables["household"], frame.weights_for("household").values + ) + if not gate.passed: + raise ValueError( + "UK export geography integrity failed: " + "; ".join(gate.failures) + ) + return { + "schema_version": 1, + "kind": "uk_full_build_export", + **_content_descriptor( + tables, + time_period=uk_time_period(frame), + weight_kind=uk_household_weight_kind(frame), + mass_log=frame.mass_log, + ), + "bindings": dict(bindings), + "geography_integrity": {"passed": gate.passed, "failures": list(gate.failures)}, + "graph_only_metadata": [ + "strata", + "metadata_other_than_time_period", + "structural_ancestry", + ], + } + + +def _check_descriptor(descriptor: Mapping[str, object]) -> None: + if ( + descriptor.get("schema_version") != 1 + or descriptor.get("kind") != "uk_full_build_export" + ): + raise ValueError("Unsupported UK export descriptor.") + fields = ("tables", "time_period", "weight_kind", "mass_log", "hash_environment") + payload = {key: descriptor[key] for key in fields} + if hashlib.sha256(canonical_json(payload)).hexdigest() != descriptor.get( + "content_sha256" + ): + raise ValueError("UK export descriptor content identity is inconsistent.") + + +def materialize_uk_export( + frame: Frame, descriptor: Mapping[str, object], path: str | Path +) -> dict[str, object]: + """Perform only the declared serialization, recreating files on cache hits.""" + _check_descriptor(descriptor) + tables = _tables(frame) + current = _content_descriptor( + tables, + time_period=uk_time_period(frame), + weight_kind=uk_household_weight_kind(frame), + mass_log=frame.mass_log, + ) + if current["content_sha256"] != descriptor["content_sha256"]: + raise ValueError("UK export population differs from its graph descriptor.") + destination = Path(path) + if destination.suffix != ".h5": + raise ValueError("UK full-build dataset must have an .h5 filename.") + _write_uk_single_year_tables( + **tables, + time_period=uk_time_period(frame), + weight_kind=uk_household_weight_kind(frame), + mass_log=frame.mass_log, + path=destination, + ) + return file_artifact(destination) + + +def validate_uk_export( + path: str | Path, descriptor: Mapping[str, object] +) -> dict[str, object]: + """Compare actual written table values/dtypes/weights/periods to the request.""" + _check_descriptor(descriptor) + dataset = file_artifact(path) + payload, _, _ = _read_uk_national_tables(path) + actual = _content_descriptor( + payload, + time_period=payload["time_period"], + weight_kind=payload["household_weight_kind"], + mass_log=payload["mass_log"], + ) + failures = [ + f"Exported {entity} table differs from its graph descriptor." + for entity in ("person", "benunit", "household") + if actual["tables"][entity] != descriptor["tables"][entity] + ] + for key in ("time_period", "weight_kind", "mass_log", "hash_environment"): + if actual[key] != descriptor[key]: + failures.append(f"Exported {key} differs from its graph descriptor.") + if file_artifact(path) != dataset: + raise ValueError("UK exported file changed during graph readback validation.") + return { + "schema_version": 1, + "kind": "uk_full_build_export_readback", + "passed": not failures, + "failures": failures, + "dataset": dataset, + "content_sha256": actual["content_sha256"], + "expected_content_sha256": descriptor["content_sha256"], + "bindings": dict(descriptor["bindings"]), + } + + +class UKExportPrepareKernel(KernelBase): + ref = "uk.full-export.prepare@1" + capabilities = Capabilities( + Determinism.DETERMINISTIC, numeric=Numeric.BITWISE, seed_source=SeedSource.NONE + ) + + def implementation_hash(self) -> str: + return source_hash(sys.modules[__name__], national_frame, geography_ladder) + + def run(self, context: KernelContext) -> KernelResult: + for value in context.artifacts.values(): + if value.type == FULL_GATE_REPORT_TYPE: + _, enforcement = decode_full_gate_report(value.payload) + if not enforcement["artifact_permitted"]: + raise ValueError( + "UK export preparation refused by a structural full-build gate." + ) + bindings = json.loads(str(context.params["bindings"])) + bindings["artifacts"] = { + name: value.key for name, value in context.artifacts.items() + } + descriptor = describe_uk_export(context_frame(context), bindings=bindings) + return KernelResult(artifacts={"export_descriptor": canonical_json(descriptor)}) + + +class UKExportReadbackKernel(KernelBase): + ref = "uk.full-export.readback@1" + capabilities = Capabilities( + Determinism.DETERMINISTIC, + numeric=Numeric.BITWISE, + seed_source=SeedSource.NONE, + role=KernelRole.GATE, + ) + + def implementation_hash(self) -> str: + return source_hash(sys.modules[__name__], national_frame) + + def run(self, context: KernelContext) -> KernelResult: + descriptor = json.loads(context.artifacts["export_descriptor"].payload) + report = validate_uk_export(context.sources["exported_dataset"], descriptor) + return KernelResult( + artifacts={"export_readback": canonical_json(report)}, + receipt={"outcome": "pass" if report["passed"] else "fail"}, + ) + + +class UKPackageInventoryKernel(KernelBase): + ref = "uk.full-export.package@1" + capabilities = Capabilities( + Determinism.DETERMINISTIC, numeric=Numeric.BITWISE, seed_source=SeedSource.NONE + ) + + def run(self, context: KernelContext) -> KernelResult: + report = json.loads(context.artifacts["export_readback"].payload) + if ( + report.get("kind") != "uk_full_build_export_readback" + or report.get("schema_version") != 1 + ): + raise ValueError( + "Package inventory requires a typed export readback report." + ) + if not report["passed"]: + raise ValueError("UK package refused because exported H5 readback failed.") + manifest = json.loads(str(context.params["manifest_binding"])) + files = {} + for alias, filename in json.loads( + str(context.params.get("evidence_files", "{}")) + ).items(): + record = file_artifact(context.sources["exported_evidence_" + alias]) + payload = context.artifacts[alias].payload + if ( + record["filename"] != filename + or record["sha256"] != hashlib.sha256(payload).hexdigest() + or record["size_bytes"] != len(payload) + ): + raise ValueError( + f"Materialized UK evidence {alias!r} differs from its graph artifact." + ) + files[alias] = { + **record, + "graph_artifact_key": context.artifacts[alias].key, + } + return KernelResult( + artifacts={ + "package_inventory": canonical_json( + { + "schema_version": 1, + "kind": "uk_full_build_package", + "readback_passed": report["passed"], + "dataset": report["dataset"], + "content_sha256": report["content_sha256"], + "build_bindings": report["bindings"], + "numerical_graph": manifest, + "artifacts": { + name: value.key for name, value in context.artifacts.items() + }, + "evidence_files": files, + "release_authorized": False, + } + ) + } + ) + + +def add_uk_export_preparation( + graph: Graph, + *, + population: str, + bindings: Mapping[str, object], + artifact_inputs: tuple[ArtifactInput, ...] = (), +) -> Graph: + node = Node( + id="uk.full.export.prepare", + kernel=UKExportPrepareKernel.ref, + population=population, + inputs=population_slices(population_columns(graph, population)), + params={"bindings": canonical_json(dict(bindings)).decode()}, + artifact_inputs=artifact_inputs, + artifact_outputs=(ArtifactOutput("export_descriptor", EXPORT_DESCRIPTOR_TYPE),), + description="Validate and describe exact H5 values, schema, weights and geographic lineage.", + ) + return replace(graph, nodes=(*graph.nodes, node)) + + +def add_uk_export_continuation( + graph: Graph, + *, + population: str, + manifest_binding: Mapping[str, object], + artifact_inputs: tuple[ArtifactInput, ...] = (), + evidence_files: Mapping[str, str] | None = None, +) -> Graph: + """Continue the same graph after its declared H5 materialization boundary.""" + evidence_files = {} if evidence_files is None else dict(evidence_files) + aliases = {item.name for item in artifact_inputs} + for alias, filename in evidence_files.items(): + if alias not in aliases or not alias.replace("_", "").isalnum(): + raise ValueError( + "Materialized evidence must name a declared simple artifact alias." + ) + if not filename or Path(filename).name != filename: + raise ValueError( + "Materialized evidence filenames must be simple path components." + ) + evidence_sources = tuple("exported_evidence_" + alias for alias in evidence_files) + readback = Node( + id="uk.full.export.readback", + kernel=UKExportReadbackKernel.ref, + population=population, + sources=("exported_dataset",), + artifact_inputs=( + ArtifactInput( + "export_descriptor", + "uk.full.export.prepare", + "export_descriptor", + EXPORT_DESCRIPTOR_TYPE, + ), + ), + artifact_outputs=(ArtifactOutput("export_readback", EXPORT_READBACK_TYPE),), + description="Read the written H5 and compare exact exported tables to the graph descriptor.", + ) + package = Node( + id="uk.full.package", + sources=evidence_sources, + kernel=UKPackageInventoryKernel.ref, + population=population, + artifact_inputs=( + ArtifactInput( + "export_readback", readback.id, "export_readback", EXPORT_READBACK_TYPE + ), + *artifact_inputs, + ), + artifact_outputs=(ArtifactOutput("package_inventory", PACKAGE_INVENTORY_TYPE),), + params={ + "manifest_binding": canonical_json(dict(manifest_binding)).decode(), + "evidence_files": canonical_json(evidence_files).decode(), + }, + description="Bind output bytes, numerical graph identity, scope/sizing and terminal evidence.", + ) + return replace( + graph, + sources=( + *graph.sources, + *(SourceRef(name, "raw-bytes-v1") for name in evidence_sources), + SourceRef( + "exported_dataset", + EXPORT_SOURCE_CODEC, + "Materialized UK H5; streamed identity and graph-owned readback.", + ), + ), + nodes=(*graph.nodes, readback, package), + ) + + +def _load_export_frame(path: Path) -> Frame: + return load_uk_rowwise_dataset(path)[0] + + +def register_uk_terminal_kernels(registry: KernelRegistry) -> None: + SOURCE_CODECS.register(EXPORT_SOURCE_CODEC, _load_export_frame) + for kernel in ( + UKExportPrepareKernel(), + UKExportReadbackKernel(), + UKPackageInventoryKernel(), + ): + registry.register(kernel) + + +FULL_GATE_REPORT_TYPE = ArtifactType("microcosm.uk.full-gate-report", 1) +FULL_DIAGNOSTICS_TYPE = ArtifactType("microcosm.uk.full-calibration-diagnostics", 1) +FULL_DIAGNOSTICS_CSV_TYPE = ArtifactType("microcosm.uk.full-target-diagnostics-csv", 1) +FULL_SUPPORT_CSV_TYPE = ArtifactType("microcosm.uk.full-area-support-csv", 1) +FULL_HOLDOUT_TYPE = ArtifactType("microcosm.uk.full-rotated-holdout", 1) + + +def _full_gate_enforcement(document: Mapping, report): + """Carry earlier phase policy forward without reevaluating its population.""" + from ..country_spec import load_country_spec + from ..gate_battery import gate_phase_report_from_payload + from .full_gates import classify_full_gate_outcomes, uk_full_gate_manifest + from .graph_evidence import uk_spine_gate_manifest + + upstream = document.get("upstream_phase_reports", {}) + allowed = {"spine_assembled": "assembled", "spine_transferred": "transferred"} + if report.phase == "terminal": + allowed["full_preflight"] = "preflight" + if not isinstance(upstream, Mapping) or set(upstream) - set(allowed): + raise ValueError("Unexpected upstream phase in the full gate artifact.") + reports = [] + for name, payload in upstream.items(): + gates = ( + uk_full_gate_manifest(document["selection_receipt"]) + if name == "full_preflight" + else uk_spine_gate_manifest(load_country_spec("uk")) + ) + previous = gate_phase_report_from_payload(payload, gates=gates) + if previous.phase != allowed[name]: + raise ValueError("Upstream full gate report has a different phase.") + reports.append(previous) + reports.append(report) + classifications = [ + classify_full_gate_outcomes( + item, + sample_fraction=document["sample_fraction"], + release_candidate=document["release_candidate"], + ) + for item in reports + ] + enforcement = dict(classifications[-1]) + for key in ( + "structural_failures", + "enforced_blocking", + "exportable_blocking", + "unenforced_release_failures", + "diagnostic_failures", + ): + enforcement[key] = list( + dict.fromkeys(value for item in classifications for value in item[key]) + ) + enforcement["artifact_permitted"] = all( + item["artifact_permitted"] for item in classifications + ) + enforcement["release_blocking_gates_passed"] = all( + item["release_blocking_gates_passed"] for item in classifications + ) + return enforcement + + +def decode_full_gate_report(payload: bytes | Mapping): + """Restore a phase report only after checking its declared gate scope.""" + from ..gate_battery import gate_phase_report_from_payload + from .full_gates import uk_full_gate_manifest + + document = json.loads(payload) if isinstance(payload, bytes) else dict(payload) + if ( + document.get("schema_version") != 1 + or document.get("kind") != "uk_full_gate_report" + ): + raise ValueError("Unsupported UK full gate artifact.") + gates = uk_full_gate_manifest(document["selection_receipt"]) + report = gate_phase_report_from_payload(document["report"], gates=gates) + enforcement = _full_gate_enforcement(document, report) + if enforcement != document["enforcement"]: + raise ValueError("UK gate enforcement differs from its bound phase report.") + return report, enforcement + + +def _spine_gate_evidence(context: KernelContext): + from ..stage_evidence import decode_stage_evidence + + names = tuple(context.params["spine_stage_names"]) + if "spine_provenance" in context.artifacts: + provenance = json.loads(context.artifacts["spine_provenance"].payload) + if tuple(provenance["stages"]) != names: + raise ValueError( + "Bound spine stage roster differs from its declared gate input." + ) + evidence = { + name: provenance.get("stage_evidence", {}).get(name) for name in names + } + records = provenance.get("fit_weight_records") + return evidence, records + documents = { + name: decode_stage_evidence(context.artifacts[name].payload, stage=name) + for name in names + } + return ( + {name: document["evidence"] for name, document in documents.items()}, + { + name: document["fit_weight_records"] + for name, document in documents.items() + if "fit_weight_records" in document + }, + ) + + +def _source_gate_evidence(context: KernelContext, engine): + from datetime import date + + from .graph_targets import registry_from_payload + + surface = json.loads(context.artifacts["surface"].payload) + return { + "coverage_engine": engine, + "build_stage_names": tuple(context.params["spine_stage_names"]), + "reference_registry": registry_from_payload(surface["national_registry"]), + "uk_ledger_compiled_registries": { + int(period): registry_from_payload(registry) + for period, registry in surface["uk_ledger_compiled_registries"].items() + }, + "uk_ledger_compiled_local_registries": { + int(period): registry_from_payload(registry) + for period, registry in surface[ + "uk_ledger_compiled_local_registries" + ].items() + }, + "exclusions_evaluated_on": date.fromisoformat( + str(context.params["review_date"]) + ), + } + + +class UKFullGateKernel(KernelBase): + ref = "uk.full-gates@1" + capabilities = Capabilities( + Determinism.DETERMINISTIC, + numeric=Numeric.BITWISE, + seed_source=SeedSource.NONE, + role=KernelRole.GATE, + dependencies=("policyengine-uk",), + ) + + def __init__(self, *, coverage_engine, engine_identity: str): + self.engine = coverage_engine + self.engine_identity = engine_identity + + def implementation_hash(self) -> str: + from .. import gate_battery + from ..country_spec import load_country_spec + from . import ( + battery_bindings, + diagnostics, + full_gates, + local_rowwise, + weighted_integrity, + ) + + return hashlib.sha256( + canonical_json( + { + "code": source_hash( + sys.modules[__name__], + gate_battery, + battery_bindings, + full_gates, + local_rowwise, + weighted_integrity, + diagnostics, + ), + "country_resources": load_country_spec("uk").fingerprint, + } + ) + ).hexdigest() + + def run(self, context: KernelContext) -> KernelResult: + from microcosm.calibrate.artifacts import decode_problem, decode_solution + + from ..gate_battery import ( + EvidenceContext, + evaluate_phase, + gate_phase_report_payload, + ) + from ..stage_evidence import encode_stage_evidence + from .battery_bindings import UK_GATE_REGISTRY + from .full_gates import ( + build_full_gate_context, + uk_full_gate_manifest, + uk_full_gate_scope_receipt, + ) + from .geography_ladder import load_uk_oa_ladder + from .local_rowwise import uk_ladder_area_support_summary + from .release_certification import rehydrate_uk_fit_weight_records + from .weighted_integrity import load_uk_input_mass_reference + + if context.params["engine_identity"] != self.engine_identity: + raise ValueError( + "UK full gate engine identity differs from its declared binding." + ) + selection = json.loads(context.artifacts["selection"].payload)["receipt"] + gates = uk_full_gate_manifest(selection) + from ..gate_battery import _gates_manifest_payload, _json_safe + + if ( + canonical_json(_gates_manifest_payload(uk_full_gate_manifest())).decode() + != context.params["gate_manifest"] + ): + raise ValueError("UK full gate manifest differs from its declared binding.") + stage_evidence, fit_weight_records = _spine_gate_evidence(context) + supporting = _source_gate_evidence(context, self.engine) + phase = str(context.params["phase"]) + diagnostics = [] + support = [] + output_artifacts = {} + upstream_reports = { + name: json.loads(context.artifacts[name].payload) + for name in ("spine_assembled", "spine_transferred") + if name in context.artifacts + } + if phase == "preflight": + # These source/reference and roster checks have no final-weight or + # contribution-matrix dependency. They run before dense calibration. + evidence = EvidenceContext(artifacts=supporting) + elif phase == "terminal": + preflight_document = json.loads(context.artifacts["preflight"].payload) + _, preflight = decode_full_gate_report(preflight_document) + if not preflight["artifact_permitted"]: + raise ValueError( + "Full calibration reached terminal gates despite a blocking source preflight." + ) + upstream_reports = { + **preflight_document.get("upstream_phase_reports", {}), + "full_preflight": preflight_document["report"], + } + frame = context_frame(context) + problem = decode_problem(context.artifacts["problem"].payload) + solution = decode_solution(context.artifacts["solution"].payload) + from microcosm.calibrate.artifacts import decode_calibration_result + + # The ordered result binds its initial weights. Rehydrate against + # the same selected table axis, then install the actual graph Frame + # so scoring uses the graph-owned mass ledger and final population. + initial_frame = Frame( + {entity: frame.table(entity) for entity in frame.entities}, + frame.schema, + {"household": problem.problem.initial_weights}, + frame.strata, + metadata=frame.metadata, + ) + result = decode_calibration_result( + context.artifacts["result"].payload, + frame=initial_frame, + problem=problem, + ) + if not np.array_equal(result.weights, solution.weights): + raise ValueError( + "Final diagnostics result differs from the installed solution." + ) + result = replace(result, frame=frame) + supporting["calibration_result"] = result + fit_records = rehydrate_uk_fit_weight_records( + {"fit_weight_records": fit_weight_records} + ) + if fit_records is not None: + supporting["fit_weight_records"] = fit_records + if "uk_input_mass_reference" in context.sources: + supporting["input_mass_reference"] = load_uk_input_mass_reference( + context.sources["uk_input_mass_reference"] + ) + household = frame.table("household").copy() + household["household_weight"] = frame.weights_for("household").values + summaries = uk_ladder_area_support_summary( + household, load_uk_oa_ladder(context.sources["uk_ladder"]) + ) + support_frame = pd.concat( + ( + summaries["constituency"].assign(geography_level="constituency"), + summaries["la"].assign(geography_level="local_authority"), + ), + ignore_index=True, + ) + supporting["uk_area_support_summary"] = support_frame + evidence = build_full_gate_context( + frame, + ordered_problem=problem, + solution=solution, + selection_receipt=selection, + stage_evidence=stage_evidence, + supporting_evidence=supporting, + ) + diagnostics = evidence.artifacts["target_diagnostics"] + support = support_frame.to_dict(orient="records") + from .diagnostics import uk_calibration_diagnostics_payload + from .graph_targets import registry_from_payload + + selected_registry = registry_from_payload( + json.loads(context.artifacts["selection"].payload)["registry"] + ) + holdout = json.loads(context.artifacts["holdout"].payload) + complete_diagnostics = uk_calibration_diagnostics_payload( + result, + frame, + target_geography_levels={ + target.row_name: str(row["geography_level"]) + for target, row in zip( + problem.problem.targets, problem.target_metadata, strict=True + ) + }, + target_registry=selected_registry, + local_area_support=support_frame, + rotated_holdout=holdout, + build={ + "build_kind": "uk_full_build", + "target_scope": selection["selector"], + }, + ) + output_artifacts.update( + { + "calibration_diagnostics": encode_stage_evidence( + _json_safe(complete_diagnostics) + ), + "target_diagnostics_csv": pd.DataFrame(diagnostics) + .to_csv(index=False) + .encode(), + "area_support_csv": support_frame.to_csv(index=False).encode(), + } + ) + else: + raise ValueError(f"Unknown full gate phase {phase!r}.") + report = evaluate_phase( + gates, phase=phase, context=evidence, registry=UK_GATE_REGISTRY + ) + payload = { + "schema_version": 1, + "kind": "uk_full_gate_report", + "selection_receipt": selection, + "sample_fraction": float(context.params["sample_fraction"]), + "release_candidate": bool(context.params["release_candidate"]), + "scope": uk_full_gate_scope_receipt(selection), + "report": gate_phase_report_payload(report, gates=gates), + "upstream_phase_reports": upstream_reports, + "target_diagnostics": _json_safe(diagnostics), + "area_support": _json_safe(support), + "artifacts": {name: value.key for name, value in context.artifacts.items()}, + } + enforcement = _full_gate_enforcement(payload, report) + payload["enforcement"] = enforcement + return KernelResult( + artifacts={ + "gate_report": encode_stage_evidence(payload), + **output_artifacts, + }, + receipt={ + "outcome": "pass" if enforcement["artifact_permitted"] else "fail" + }, + ) + + +def append_uk_full_gate_nodes( + graph: Graph, + *, + calibration, + spine_stage_names: tuple[str, ...], + engine_identity: str, + review_date, + sample_fraction: float = 1.0, + release_candidate: bool = False, + spine_provenance: ArtifactInput | None = None, + input_population: str = "uk.full.pool", + skip_holdout: bool = False, +) -> Graph: + """Own source preflight before solve and final diagnostics after installation.""" + from microcosm.calibrate.artifacts import PROBLEM_TYPE, RESULT_TYPE, SOLUTION_TYPE + + from ..gate_battery import _gates_manifest_payload + from ..stage_evidence import STAGE_EVIDENCE_TYPE + from .full_gates import uk_full_gate_manifest + from .graph_evidence import SPINE_GATE_REPORT_TYPE + from .graph_targets import TARGET_SELECTION_TYPE, TARGET_SURFACE_TYPE + + if not engine_identity: + raise ValueError("Full-build gates require a declared engine identity.") + if any(node.id == "uk.full.gates.preflight" for node in graph.nodes): + raise ValueError("Full gate nodes are already registered.") + params = { + "spine_stage_names": tuple(spine_stage_names), + "engine_identity": engine_identity, + "review_date": str(review_date), + "sample_fraction": sample_fraction, + "release_candidate": release_candidate, + "gate_manifest": canonical_json( + _gates_manifest_payload(uk_full_gate_manifest()) + ).decode(), + } + provenance = ( + (replace(spine_provenance, name="spine_provenance"),) + if spine_provenance + else tuple( + ArtifactInput( + name, + "create_uk_frs" if name == "frs_spine" else name, + "stage_evidence", + STAGE_EVIDENCE_TYPE, + ) + for name in spine_stage_names + ) + ) + common = ( + ArtifactInput( + "surface", "uk.full.target_compilation", "surface", TARGET_SURFACE_TYPE + ), + ArtifactInput( + "selection", "uk.full.target_selection", "selection", TARGET_SELECTION_TYPE + ), + *provenance, + *( + ArtifactInput( + f"spine_{phase}", + f"spine.gates.{phase}", + "gate_report", + SPINE_GATE_REPORT_TYPE, + ) + for phase in ("assembled", "transferred") + if any(node.id == f"spine.gates.{phase}" for node in graph.nodes) + ), + ) + preflight = Node( + "uk.full.gates.preflight", + UKFullGateKernel.ref, + population=input_population, + params={**params, "phase": "preflight"}, + artifact_inputs=common, + artifact_outputs=(ArtifactOutput("gate_report", FULL_GATE_REPORT_TYPE),), + description="Validate complete source/reference registries and spine stage ownership before dense calibration.", + ) + prerequisite = ArtifactInput( + "preflight", preflight.id, "gate_report", FULL_GATE_REPORT_TYPE + ) + nodes = tuple( + replace(node, artifact_inputs=(*node.artifact_inputs, prerequisite)) + if node.id == calibration.dense_producer + else node + for node in graph.nodes + ) + sources = ("uk_ladder",) + ( + ("uk_input_mass_reference",) + if any(source.name == "uk_input_mass_reference" for source in graph.sources) + else () + ) + final = Node( + "uk.full.gates.calibrated", + UKFullGateKernel.ref, + population=calibration.population, + inputs=population_slices(population_columns(graph, calibration.population)), + sources=sources, + params={**params, "phase": "terminal"}, + artifact_inputs=( + *common, + prerequisite, + ArtifactInput( + "problem", calibration.problem_producer, "problem", PROBLEM_TYPE + ), + ArtifactInput( + "solution", + calibration.solution_producer, + "refit_solution" if calibration.size_producer else "solution", + SOLUTION_TYPE, + ), + ), + artifact_outputs=(ArtifactOutput("gate_report", FULL_GATE_REPORT_TYPE),), + description="Compute final identified-row diagnostics once and evaluate all applicable national/local release gates.", + ) + holdout = uk_full_holdout_node( + graph, + calibration=calibration, + input_population=input_population, + skip_holdout=skip_holdout, + ) + final = replace( + final, + artifact_inputs=( + *final.artifact_inputs, + ArtifactInput("result", calibration.result_producer, "result", RESULT_TYPE), + ArtifactInput("holdout", holdout.id, "holdout", FULL_HOLDOUT_TYPE), + ), + artifact_outputs=( + *final.artifact_outputs, + ArtifactOutput("calibration_diagnostics", FULL_DIAGNOSTICS_TYPE), + ArtifactOutput("target_diagnostics_csv", FULL_DIAGNOSTICS_CSV_TYPE), + ArtifactOutput("area_support_csv", FULL_SUPPORT_CSV_TYPE), + ), + ) + return replace(graph, nodes=(*nodes, preflight, holdout, final)) + + +def register_uk_full_gate_kernels( + registry: KernelRegistry, *, coverage_engine, engine_identity: str +) -> None: + registry.register( + UKFullGateKernel( + coverage_engine=coverage_engine, engine_identity=engine_identity + ) + ) + + registry.register(UKFullHoldoutKernel()) + + +class UKFullHoldoutKernel(KernelBase): + ref = "uk.full.rotated-holdout@1" + capabilities = Capabilities( + Determinism.DETERMINISTIC, + numeric=Numeric.PLATFORM_BITWISE, + seed_source=SeedSource.PARAM, + dependencies=("policyengine-uk", "torch"), + ) + + def implementation_hash(self) -> str: + from ..country_spec import load_country_spec + from . import dataset_size, graph_targets, local_rowwise + + return hashlib.sha256( + canonical_json( + { + "code": source_hash( + sys.modules[__name__], + graph_targets, + local_rowwise, + dataset_size, + dependencies=self.capabilities.dependencies, + ), + "country_resources": load_country_spec("uk").fingerprint, + } + ) + ).hexdigest() + + def run(self, context: KernelContext) -> KernelResult: + from microcosm.calibrate.artifacts import decode_problem + + from ..gate_battery import _json_safe + from ..stage_evidence import encode_stage_evidence + from .graph_targets import reconstruct_uk_full_problem_inputs + from .local_rowwise import rotated_uk_local_holdout + + _, preflight = decode_full_gate_report(context.artifacts["preflight"].payload) + if not preflight["artifact_permitted"]: + raise ValueError( + "Rotated holdout cannot run after a blocking source preflight." + ) + if context.params["skip_holdout"]: + report = { + "report_only": True, + "skipped": True, + "reason": "Explicit development request; no holdout claim.", + } + else: + inputs = reconstruct_uk_full_problem_inputs(context) + original = decode_problem(context.artifacts["problem"].payload) + if original.entity_ids != tuple( + inputs.frame.table("household")["household_id"] + ): + raise ValueError( + "Holdout original pool differs from its bound problem axis." + ) + report = rotated_uk_local_holdout( + inputs.frame, + inputs.local_problem, + bound_families=inputs.bound_families, + national_rows=inputs.national_rows, + target_weight_rule=str(context.params["target_weight_rule"]), + epochs=int(context.params["epochs"]), + learning_rate=float(context.params["learning_rate"]), + conserve_mass=False, + target_records=None, + l0_lambda=0.0, + budget_iters=10, + dataset_households=context.params.get("dataset_households"), + solve_seed=int(context.params["seed"]), + selection_seed=context.params.get("selection_seed"), + selection_pi_hi=float(context.params["selection_pi_hi"]), + ) + report = { + **report, + "graph_binding": { + "original_problem_artifact": context.artifacts["problem"].key, + "artifacts": { + name: value.key for name, value in context.artifacts.items() + }, + }, + } + return KernelResult( + artifacts={"holdout": encode_stage_evidence(_json_safe(report))} + ) + + +def uk_full_holdout_node( + graph: Graph, *, calibration, input_population: str, skip_holdout: bool +) -> Node: + from microcosm.calibrate.artifacts import PROBLEM_TYPE + + original = graph.node("uk.full.problem") + dense = graph.node(calibration.dense_producer) + size = ( + None + if calibration.size_producer is None + else graph.node(calibration.size_producer) + ) + return Node( + "uk.full.holdout", + UKFullHoldoutKernel.ref, + population=input_population, + inputs=population_slices(population_columns(graph, input_population)), + sources=original.sources, + params={ + **original.params, + **dense.params, + "skip_holdout": skip_holdout, + "dataset_households": None if size is None else size.params["households"], + "selection_seed": None if size is None else size.params["seed"], + "selection_pi_hi": 1.0 if size is None else size.params["pi_hi"], + }, + artifact_inputs=( + *original.artifact_inputs, + ArtifactInput("problem", original.id, "problem", PROBLEM_TYPE), + ArtifactInput( + "preflight", + "uk.full.gates.preflight", + "gate_report", + FULL_GATE_REPORT_TYPE, + ), + ), + artifact_outputs=(ArtifactOutput("holdout", FULL_HOLDOUT_TYPE),), + description="Preserve five rotated local-target holdouts with national constraints fixed in training and unchanged sizing doctrine.", + ) + + +def materialize_uk_terminal_artifacts( + manifest, store, *, directory: str | Path, stem: str +) -> dict[str, dict[str, object]]: + """Write graph-produced diagnostic bytes atomically, including after cache hits.""" + from ..artifact_files import materialize_bytes + + if not stem or Path(stem).name != stem: + raise ValueError( + "UK terminal artifact stem must be a simple filename component." + ) + root = Path(directory) + artifacts = { + "calibration_diagnostics": ( + "uk.full.gates.calibrated", + "calibration_diagnostics", + ".diagnostics.json", + ), + "target_diagnostics": ( + "uk.full.gates.calibrated", + "target_diagnostics_csv", + ".targets.csv", + ), + "area_support": ( + "uk.full.gates.calibrated", + "area_support_csv", + ".area_support.csv", + ), + "holdout": ("uk.full.holdout", "holdout", ".holdout.json"), + "target_registry": ( + "uk.full.target_selection", + "selection", + ".target_selection.json", + ), + } + inventory = {} + for role, (node, output, suffix) in artifacts.items(): + key = manifest.nodes[node].opaque_artifacts[output] + inventory[role] = { + **materialize_bytes(store.load_bytes(key), root / (stem + suffix)), + "graph_artifact_key": key, + } + return inventory diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/hmrc_source_contract.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/hmrc_source_contract.py index 946dc08cc..85c1803d1 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/hmrc_source_contract.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/hmrc_source_contract.py @@ -2,14 +2,13 @@ from __future__ import annotations -import copy import json from collections.abc import Mapping, Sequence from importlib.resources import files from pathlib import Path from typing import Any -from microcosm.build.uk_runtime.frs_hmrc_leaves import ( +from microcosm.build.uk_runtime.frs_hmrc_source import ( FRS_HMRC_RETAINED_LEAF_COLUMNS, FRS_HMRC_RETAINED_LEAF_SOURCE_EVIDENCE, ) @@ -31,7 +30,6 @@ CANONICAL_HMRC_FACT_FENCES, FULL_FRS_TI_BAND_FENCE_ID, ) -from microcosm.build.uk_runtime.release_identity import UK_RELEASE_TIER_FRS from microcosm.build.uk_runtime.spi_income import ( DEFAULT_SPI_DONOR_SAMPLE_SIZE, SPI_DERIVED_POLICYENGINE_SOURCE_COLUMNS, @@ -59,35 +57,19 @@ FRS_ONLY_SPI_FILL_PREDICTOR_COLUMNS, SPI_HMRC_DERIVED_AUXILIARY_COLUMNS, SPI_INCOME_QRF_OUTPUT_COLUMNS, - SPI_PRIOR_MASS_CHANGE_REASON, - SPI_REPLACEMENT_STRATA_COLUMNS, ) __all__ = [ - "CERTIFIED_UK_CANDIDATE_FILENAME", - "CERTIFIED_UK_CANDIDATE_REVISION", - "CERTIFIED_UK_CANDIDATE_SHA256", - "CERTIFIED_UK_CANDIDATE_SIZE_BYTES", - "CERTIFIED_UK_CANDIDATE_TIER", "HMRC_DISTRIBUTIONAL_INPUTS", - "UK_HMRC_INCOME_SOURCE_STAGES_RESOURCE", "assert_uk_hmrc_income_source_contract_current", "uk_hmrc_weighted_qrf_output_columns", ] -UK_HMRC_INCOME_SOURCE_STAGES_RESOURCE = "hmrc_income_source_stages.json" UK_CANONICAL_SOURCE_STAGES_RESOURCE = "source_stages.json" HMRC_DISTRIBUTIONAL_INPUTS = ( "gift_aid", "charitable_investment_gifts", ) -CERTIFIED_UK_CANDIDATE_FILENAME = "populace_uk_2023.h5" -CERTIFIED_UK_CANDIDATE_REVISION = "populace-uk-2023-dd68c73-4aa4b14-20260619T023711Z" -CERTIFIED_UK_CANDIDATE_TIER = UK_RELEASE_TIER_FRS -CERTIFIED_UK_CANDIDATE_SHA256 = ( - "f17306ccb2aad7ff0130be3589b560afb2e2a12a943570911cd0c77f07934833" -) -CERTIFIED_UK_CANDIDATE_SIZE_BYTES = 1_315_880_118 _STAGE2_SOURCE_FAITHFUL_INCOME_PREDICTORS = ( "employment_income", "self_employment_income", @@ -113,13 +95,16 @@ } _EXPECTED_OPERATION_KINDS = ( - "verify_certified_candidate", "retain_adjudicated_frs_hmrc_leaves", + "derive", + "stack_zero_weight_donors", + "gate_zero_weight_strata", + "allocate_zero_weight_prior_mass", "verify_pinned_hmrc_source_pair", - "replace_zero_weight_spi_support", "strict_read_private_table", "fit_weighted_qrf_stage1", "fit_weighted_qrf_stage2", + "redraw_columns_from_fitted_qrf", "materialize_hmrc_income_bands_fail_closed", "classify_hmrc_income_facts_with_reviewed_fences", "gate_distributional_effective_mass", @@ -141,47 +126,9 @@ def assert_uk_hmrc_income_source_contract_current( failures.append(f"stages: expected exactly one stage, got {len(stages)}") _raise_failures(failures) stage = stages[0] - _expect(failures, "stage.stage", stage.get("stage"), "hmrc_spi_income") + _expect(failures, "stage.stage", stage.get("stage"), "hmrc_spi_income_spine") _expect(failures, "stage.grain", stage.get("grain"), "person") - base = _mapping(stage.get("base_candidate"), "base_candidate", failures) - _expect( - failures, - "base_candidate.filename", - base.get("filename"), - CERTIFIED_UK_CANDIDATE_FILENAME, - ) - _expect( - failures, - "base_candidate.tier", - base.get("tier"), - CERTIFIED_UK_CANDIDATE_TIER, - ) - _expect( - failures, - "base_candidate.revision", - base.get("revision"), - CERTIFIED_UK_CANDIDATE_REVISION, - ) - _expect( - failures, - "base_candidate.sha256", - base.get("sha256"), - CERTIFIED_UK_CANDIDATE_SHA256, - ) - _expect( - failures, - "base_candidate.size_bytes", - base.get("size_bytes"), - CERTIFIED_UK_CANDIDATE_SIZE_BYTES, - ) - _expect( - failures, - "base_candidate.runtime_sha256_required", - base.get("runtime_sha256_required"), - True, - ) - artifacts = _keyed_items( stage.get("artifacts"), key="role", @@ -303,10 +250,6 @@ def assert_uk_hmrc_income_source_contract_current( _EXPECTED_OPERATION_KINDS, ) - verify = operations.get("verify_certified_candidate", {}) - _expect(failures, "verify.artifact", verify.get("artifact"), "base_candidate") - _expect(failures, "verify.fail_on_mismatch", verify.get("fail_on_mismatch"), True) - frs_leaves = operations.get("retain_adjudicated_frs_hmrc_leaves", {}) _expect( failures, @@ -416,55 +359,20 @@ def assert_uk_hmrc_income_source_contract_current( ): _expect(failures, f"source_pair.{flag}", source_pair.get(flag), True) - prior = operations.get("replace_zero_weight_spi_support", {}) - _expect(failures, "prior.existing_channel", prior.get("existing_channel"), "spi") - _expect( - failures, - "prior.require_existing_weight", - prior.get("require_existing_weight"), - 0, - ) - _expect( - failures, - "prior.replacement_strata", - tuple(prior.get("replacement_strata", ())), - SPI_REPLACEMENT_STRATA_COLUMNS, - ) + prior = operations.get("allocate_zero_weight_prior_mass", {}) _expect( - failures, - "prior.mass_share", - prior.get("spi_prior_national_household_mass_share"), - DEFAULT_SPI_PRIOR_MASS_SHARE, + failures, "prior.mass_share", prior.get("share"), DEFAULT_SPI_PRIOR_MASS_SHARE ) + _expect(failures, "prior.strata", tuple(prior.get("strata", ())), ("region",)) _expect( - failures, - "prior.output_weight_kind", - prior.get("output_weight_kind"), - "importance", + failures, "prior.output_weight_kind", prior.get("weight_kind_out"), "importance" ) + _expect(failures, "prior.conservation", prior.get("conservation"), "exact_total") _expect( failures, - "prior.preserve_total_household_mass", - prior.get("preserve_total_household_mass"), - True, - ) - _expect( - failures, - "prior.require_mass_change_record", - prior.get("require_mass_change_record"), - True, - ) - _expect( - failures, - "prior.mass_change_reason", - prior.get("mass_change_reason"), - SPI_PRIOR_MASS_CHANGE_REASON, - ) - _expect( - failures, - "prior.fail_on_live_existing_spi_mass", - prior.get("fail_on_live_existing_spi_mass"), - True, + "frs_leaves.population", + frs_leaves.get("population"), + "uk_frs_raw_spine", ) strict = operations.get("strict_read_private_table", {}) @@ -629,7 +537,7 @@ def assert_uk_hmrc_income_source_contract_current( failures, "stage2.predictors", tuple(stage2.get("predictors", ())), - _STAGE2_SOURCE_FAITHFUL_PREDICTORS, + (*_STAGE2_SOURCE_FAITHFUL_PREDICTORS, "state_pension_receipt"), ) _expect( failures, @@ -900,32 +808,35 @@ def assert_uk_hmrc_income_source_contract_current( failures, "effective.fail_below_floor", effective.get("fail_below_floor"), True ) - _expect( - failures, - "stage.official_table_components", - tuple(stage.get("official_table_components", ())), - HMRC_SPI_INCOME_COMPONENTS, - ) - _expect( - failures, - "stage.donor_relief_outputs", - tuple(stage.get("donor_relief_outputs", ())), - HMRC_DISTRIBUTIONAL_INPUTS, + # The current spine declares new columns separately from rewrites; the + # legacy candidate stage's flat output list is not an ownership contract. + from microcosm.build.source_manifest import SourceStageSpec + from microcosm.build.uk_runtime.spi_spine import ( + UK_SPI_INCOME_SPINE_OUTPUT_COLUMNS, + UK_SPI_INCOME_SPINE_REWRITE_COLUMNS, + _assert_income_stage_parameters, + _support_stage_parameters, ) + + declared = set(stage.get("outputs", ())) | set(stage.get("rewrites", ())) _expect( failures, "stage.outputs", - tuple(stage.get("outputs", ())), - ( - *( - "hmrc_spi_state_pension_income" - if component == "state_pension" - else component - for component in HMRC_SPI_INCOME_COMPONENTS - ), - *HMRC_DISTRIBUTIONAL_INPUTS, - *SPI_HMRC_DERIVED_AUXILIARY_COLUMNS, - ), + declared, + set(UK_SPI_INCOME_SPINE_OUTPUT_COLUMNS) + | set(UK_SPI_INCOME_SPINE_REWRITE_COLUMNS), + ) + _raise_failures(failures) + declared_stages = payload["source_stages"] + _support_stage_parameters( + SourceStageSpec.from_mapping(declared_stages["spi_support_channel"]), + seed=42, + ) + _assert_income_stage_parameters( + SourceStageSpec.from_mapping(declared_stages["hmrc_spi_income_spine"]), + seed=42, + qrf_estimators=100, + donor_sample_size=DEFAULT_SPI_DONOR_SAMPLE_SIZE, ) _raise_failures(failures) @@ -988,71 +899,46 @@ def uk_hmrc_weighted_qrf_output_columns( def _load_payload(resource: Any | None) -> Mapping[str, Any]: - if resource is None: - frozen_payload = json.loads( - files("microcosm.build.uk") - .joinpath(UK_HMRC_INCOME_SOURCE_STAGES_RESOURCE) - .read_text(encoding="utf-8") - ) - if not isinstance(frozen_payload, Mapping): - raise ValueError("UK HMRC source manifest root must be a JSON object.") - frozen_stages = frozen_payload.get("stages") - if not isinstance(frozen_stages, Sequence) or isinstance( - frozen_stages, (str, bytes) - ): - raise ValueError("UK HMRC source manifest stages must be a list.") - if len(frozen_stages) != 1 or not isinstance(frozen_stages[0], Mapping): - raise ValueError( - "UK HMRC source manifest must contain exactly one source stage." - ) - payload = json.loads( - files("microcosm.build.uk") - .joinpath(UK_CANONICAL_SOURCE_STAGES_RESOURCE) - .read_text(encoding="utf-8") - ) - if not isinstance(payload, Mapping): - raise ValueError("UK source manifest root must be a JSON object.") - stages = payload.get("stages") - if not isinstance(stages, Sequence) or isinstance(stages, (str, bytes)): - raise ValueError("UK source manifest stages must be a list.") - retained = [ - stage - for stage in stages - if isinstance(stage, Mapping) - and stage.get("stage") == "frs_hmrc_retained_leaves" - ] - hmrc = [ - stage - for stage in stages - if isinstance(stage, Mapping) and stage.get("stage") == "hmrc_spi_income" - ] - if len(retained) != 1 or len(hmrc) != 1: - raise ValueError( - "UK source manifest must contain exactly one " - "frs_hmrc_retained_leaves stage and one hmrc_spi_income stage." - ) - stage = copy.deepcopy(dict(hmrc[0])) - stage["base_candidate"] = copy.deepcopy( - dict(frozen_stages[0].get("base_candidate", {})) - ) - stage["operations"] = [ - *copy.deepcopy(list(retained[0].get("operations", ()))), - *copy.deepcopy(list(hmrc[0].get("operations", ()))), - ] - return { - "country": payload.get("country"), - "version": payload.get("version"), - "stages": [stage], - } - target = resource - if hasattr(target, "read_text"): - raw = target.read_text(encoding="utf-8") - else: - raw = Path(target).read_text(encoding="utf-8") + target = ( + files("microcosm.build.uk").joinpath(UK_CANONICAL_SOURCE_STAGES_RESOURCE) + if resource is None + else resource + ) + raw = ( + target.read_text(encoding="utf-8") + if hasattr(target, "read_text") + else Path(target).read_text(encoding="utf-8") + ) payload = json.loads(raw) if not isinstance(payload, Mapping): - raise ValueError("UK HMRC source manifest root must be a JSON object.") - return payload + raise ValueError("UK source manifest root must be a JSON object.") + stages = payload.get("stages") + if not isinstance(stages, list) or not all( + isinstance(stage, Mapping) for stage in stages + ): + raise ValueError("UK source manifest stages must be a list of objects.") + names = ("frs_hmrc_spine_leaves", "spi_support_channel", "hmrc_spi_income_spine") + selected = {} + for name in names: + matches = [stage for stage in stages if stage.get("stage") == name] + if len(matches) != 1: + raise ValueError( + f"UK source manifest must contain exactly one {name} stage." + ) + selected[name] = matches[0] + # Audit the connected income family without inventing an executable stage. + income = dict(selected["hmrc_spi_income_spine"]) + income["operations"] = [ + operation + for name in names + for operation in selected[name].get("operations", ()) + ] + return { + "country": payload.get("country"), + "version": payload.get("version"), + "stages": [income], + "source_stages": selected, + } def _mapping(value: object, label: str, failures: list[str]) -> Mapping[str, Any]: diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/incumbent_surface_evaluation.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/incumbent_surface_evaluation.py index 5084f6fcd..8b38c541f 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/incumbent_surface_evaluation.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/incumbent_surface_evaluation.py @@ -22,6 +22,7 @@ import re from collections.abc import Iterable, Mapping from importlib import resources as importlib_resources +from pathlib import Path from typing import Any import numpy as np @@ -556,3 +557,37 @@ def render_markdown( + (" …" if len(sub) > 12 else "") ) return "\n".join(lines) + "\n" + + +def candidate_evaluation_manifest(payload: dict) -> dict: + """Expose the same measured-byte identity from a canonical candidate package. + + Archived candidate manifests retain their existing schema for historical + comparisons. Current full builds use immutable candidate.json; the separate + completion marker may subsequently bind a certification readiness artifact. + """ + if payload.get("kind") != "uk_full_build_package": + if "outputs" not in payload or "identity" not in payload: + raise ValueError( + "Evaluation requires an immutable candidate package, not a completion marker." + ) + return payload + if payload.get("schema_version") != 1 or payload.get("readback_passed") is not True: + raise ValueError("Evaluation requires a checked UK full-build export.") + outputs = {} + for role, record in ( + ("dataset", payload["dataset"]), + ("calibration_diagnostics", payload["evidence_files"]["diagnostics"]), + ): + filename = record.get("filename") + if not isinstance(filename, str) or Path(filename).name != filename: + raise ValueError("Candidate package filenames must be simple components.") + outputs[role] = { + "path": filename, + "sha256": record["sha256"], + "bytes": record["size_bytes"], + } + return { + "outputs": outputs, + "identity": {"ledger": payload["build_bindings"]["ledger"]}, + } diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/local_rowwise.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/local_rowwise.py index 33f25cc2b..b294ee9a0 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/local_rowwise.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/local_rowwise.py @@ -359,6 +359,42 @@ def build_uk_rowwise_local_matrix( ) +def empty_uk_local_problem(household_ids: Sequence[Any]) -> UKRowwiseLocalMatrix: + """An explicitly filtered local surface on the full pool household axis. + + Composition must still supply at least one selected target. This does + not manufacture local constraints or imply that local fit was assessed. + """ + + ids = tuple(household_ids) + if len(set(ids)) != len(ids): + raise ValueError("household IDs must be unique.") + return UKRowwiseLocalMatrix( + matrix=sp.csr_matrix((0, len(ids)), dtype=np.float64), + targets=np.empty(0, dtype=np.float64), + target_frame=pd.DataFrame( + columns=[ + "target_index", + "area_type", + "area_code", + "metric", + "value", + "target_name", + "family", + ] + ), + area_codes=(), + metric_names=(), + household_ids=ids, + assigned_areas=(), + metric_values=np.empty((len(ids), 0), dtype=np.float64), + area_codes_by_grain={}, + metric_names_by_grain={}, + assigned_areas_by_grain={}, + metric_values_by_grain={}, + ) + + def build_uk_rowwise_local_surface_matrix( metrics_by_grain: Mapping[str, pd.DataFrame], assigned_by_grain: Mapping[str, pd.Series | Sequence[str]], @@ -851,11 +887,6 @@ def _normalise_uk_local_bound_families( "sequence of family/area_type strings, not one string." ) declared = tuple(str(name) for name in bound_families) - if not declared: - raise ValueError( - "UK local binding declarations: bound_families must name at " - "least one family/area_type pair." - ) blanks = [name for name in declared if not name.strip()] if blanks: raise ValueError( @@ -1035,65 +1066,34 @@ def vector(frame: Frame) -> np.ndarray: return TargetSet(targets) -def solve_uk_rowwise_weights_under_doctrine( +@dataclass(frozen=True) +class UKPreparedFullSolve: + """Declared target problem before any weight or size operation.""" + + frame: Frame + problem: UKRowwiseLocalMatrix + national_rows: UKRowwiseNationalRows | None + target_set: TargetSet + target_loss_weights: np.ndarray + binding_adjudications: Mapping[str, Any] + mass_reason: str + + +def prepare_uk_full_solve( frame: Frame, problem: UKRowwiseLocalMatrix, *, bound_families: Sequence[str], national_rows: UKRowwiseNationalRows | None = None, target_weight_rule: str = "uniform", - restore: Callable[[Frame], Frame] | None = None, - epochs: int = 512, - learning_rate: float = 0.15, - conserve_mass: bool = False, - target_records: int | None = None, - dataset_households: int | None = None, - l0_lambda: float = 0.0, - budget_iters: int = 10, - seed: int = 0, - selection_seed: int | None = None, - selection_pi_hi: float = 1.0, - size_checkpoint_dir: Path | None = None, - resume_size_checkpoint: Path | None = None, - checkpoint_identity: Mapping[str, Any] | None = None, - checkpoint_provenance: Mapping[str, Any] | None = None, - progress: Callable[[str], None] | None = None, -) -> UKRowwiseDoctrineSolve: - """Solve rowwise household weights under the reviewed doctrine. - - ``selection_seed`` (default ``seed``) seeds only the size selection — - the informed L0 search, the exact-count draw and the refit — so two - selections can be compared on one pool and one dense reference. - - ``size_checkpoint_dir`` persists the dense solve and the informed L0 - search of a ``dataset_households`` solve before the exact-count draw - (:mod:`microcosm.build.uk_runtime.size_checkpoint`), stamped with - ``checkpoint_identity``; ``resume_size_checkpoint`` restores such a - checkpoint instead of solving and searching again, refusing when the - identity, the pool or the target surface differ. The draw's threshold - (``selection_pi_hi``) may differ from the one the search stopped on; the - size receipt records both. - - ``progress`` receives one readable line per hundred epochs of the dense - solve, of every budget probe and of the refit, one line per finished - probe with its drawability verdict, and one when the search stops - (:func:`~microcosm.build.uk_runtime.solve_progress.uk_solve_progress_callback`). +) -> UKPreparedFullSolve: + """Validate the selected surface with one doctrine for every geography.""" - Structurally knob-free like before the ``calibrate()`` migration: no - per-target parameters and no doctrine parameter — the bounds always come - from :data:`UK_LOCAL_SOLVE_DOCTRINE` and ride into the public front door - as explicit arguments. Initial weights are the frame's typed household - weights directly (a rowwise household exists in exactly one area, so - nothing is split); zero weights are refused — a dead row must be dropped - or revived upstream with a recorded mass change, never resurrected by a - solver floor. The kernel enforces the ``CALIBRATED`` kind transition and - mints the mass record (reason from - :func:`rowwise_calibration_mass_reason`); the returned frame carries - both, with the persisted ``household_weight`` column refreshed. - """ - - doctrine = UK_LOCAL_SOLVE_DOCTRINE _require_uniform_target_surface(problem) + if not len(problem.targets) and ( + national_rows is None or not len(national_rows.targets) + ): + raise ValueError("full calibration requires at least one selected target.") local_bound_families = tuple( family for family in bound_families if not str(family).startswith("national/") ) @@ -1181,6 +1181,8 @@ def solve_uk_rowwise_weights_under_doctrine( *(() if national_rows is None else national_rows.targets.targets), ] ) + if not len(target_set): + raise ValueError("full calibration requires at least one selected target.") local_count = len(local_target_set) national_count = len(target_set) - local_count grain_labels = [ @@ -1191,6 +1193,117 @@ def solve_uk_rowwise_weights_under_doctrine( grain_labels, rule=target_weight_rule, ) + return UKPreparedFullSolve( + frame=frame, + problem=problem, + national_rows=national_rows, + target_set=target_set, + target_loss_weights=target_loss_weights, + binding_adjudications=binding_adjudications, + mass_reason=mass_reason, + ) + + +def solve_uk_dense_reference( + prepared: UKPreparedFullSolve, + *, + epochs: int = 512, + learning_rate: float = 0.15, + conserve_mass: bool = False, + target_records: int | None = None, + l0_lambda: float = 0.0, + budget_iters: int = 10, + seed: int = 0, + progress_callback: Callable | None = None, +) -> CalibrationResult: + """Execute the shared solver on the original pool, before size selection.""" + + doctrine = UK_LOCAL_SOLVE_DOCTRINE + return calibrate( + prepared.frame, + prepared.target_set, + weight_entity="household", + epochs=epochs, + learning_rate=learning_rate, + mass=CONSERVE_MASS if conserve_mass else FREE_MASS, + mass_reason=None if conserve_mass else prepared.mass_reason, + max_weight_ratio=doctrine.max_weight_ratio, + target_records=target_records, + l0_lambda=l0_lambda, + budget_iters=budget_iters, + seed=seed, + target_loss_weights=prepared.target_loss_weights, + target_loss_cap=doctrine.target_loss_cap, + progress_callback=progress_callback, + ) + + +def solve_uk_rowwise_weights_under_doctrine( + frame: Frame, + problem: UKRowwiseLocalMatrix, + *, + bound_families: Sequence[str], + national_rows: UKRowwiseNationalRows | None = None, + target_weight_rule: str = "uniform", + restore: Callable[[Frame], Frame] | None = None, + epochs: int = 512, + learning_rate: float = 0.15, + conserve_mass: bool = False, + target_records: int | None = None, + dataset_households: int | None = None, + l0_lambda: float = 0.0, + budget_iters: int = 10, + seed: int = 0, + selection_seed: int | None = None, + selection_pi_hi: float = 1.0, + size_checkpoint_dir: Path | None = None, + resume_size_checkpoint: Path | None = None, + checkpoint_identity: Mapping[str, Any] | None = None, + checkpoint_provenance: Mapping[str, Any] | None = None, + progress: Callable[[str], None] | None = None, +) -> UKRowwiseDoctrineSolve: + """Solve rowwise household weights under the reviewed doctrine. + + ``selection_seed`` (default ``seed``) seeds only the size selection — + the informed L0 search, the exact-count draw and the refit — so two + selections can be compared on one pool and one dense reference. + + ``size_checkpoint_dir`` persists the dense solve and the informed L0 + search of a ``dataset_households`` solve before the exact-count draw + (:mod:`microcosm.build.uk_runtime.size_checkpoint`), stamped with + ``checkpoint_identity``; ``resume_size_checkpoint`` restores such a + checkpoint instead of solving and searching again, refusing when the + identity, the pool or the target surface differ. The draw's threshold + (``selection_pi_hi``) may differ from the one the search stopped on; the + size receipt records both. + + ``progress`` receives one readable line per hundred epochs of the dense + solve, of every budget probe and of the refit, one line per finished + probe with its drawability verdict, and one when the search stops + (:func:`~microcosm.build.uk_runtime.solve_progress.uk_solve_progress_callback`). + + Structurally knob-free like before the ``calibrate()`` migration: no + per-target parameters and no doctrine parameter — the bounds always come + from :data:`UK_LOCAL_SOLVE_DOCTRINE` and ride into the public front door + as explicit arguments. Initial weights are the frame's typed household + weights directly (a rowwise household exists in exactly one area, so + nothing is split); zero weights are refused — a dead row must be dropped + or revived upstream with a recorded mass change, never resurrected by a + solver floor. The kernel enforces the ``CALIBRATED`` kind transition and + mints the mass record (reason from + :func:`rowwise_calibration_mass_reason`); the returned frame carries + both, with the persisted ``household_weight`` column refreshed. + """ + + prepared = prepare_uk_full_solve( + frame, + problem, + bound_families=bound_families, + national_rows=national_rows, + target_weight_rule=target_weight_rule, + ) + doctrine = UK_LOCAL_SOLVE_DOCTRINE + target_set = prepared.target_set if (size_checkpoint_dir is not None or resume_size_checkpoint is not None) and ( dataset_households is None ): @@ -1238,21 +1351,15 @@ def solve_uk_rowwise_weights_under_doctrine( + "." ) else: - result = calibrate( - frame, - target_set, - weight_entity="household", + result = solve_uk_dense_reference( + prepared, epochs=epochs, learning_rate=learning_rate, - mass=CONSERVE_MASS if conserve_mass else FREE_MASS, - mass_reason=None if conserve_mass else mass_reason, - max_weight_ratio=doctrine.max_weight_ratio, + conserve_mass=conserve_mass, target_records=target_records, l0_lambda=l0_lambda, budget_iters=budget_iters, seed=seed, - target_loss_weights=target_loss_weights, - target_loss_cap=doctrine.target_loss_cap, progress_callback=progress_callback, ) selected_support = None @@ -1320,6 +1427,35 @@ def solve_uk_rowwise_weights_under_doctrine( # always hands the refit the selection it just searched or restored. size_receipt["selection_reused"] = restored is not None size_receipt["checkpoint"] = checkpoint_receipt + return finish_uk_full_solve( + prepared, + result, + restore=restore, + selected_support=selected_support, + size_receipt=size_receipt, + dense_result=dense_result, + ) + + +def finish_uk_full_solve( + prepared: UKPreparedFullSolve, + result: CalibrationResult, + *, + restore: Callable[[Frame], Frame] | None = None, + selected_support: np.ndarray | None = None, + size_receipt: Mapping[str, Any] | None = None, + dense_result: CalibrationResult | None = None, +) -> UKRowwiseDoctrineSolve: + """Restore clean inputs and label evidence after the one selected solve.""" + + frame = prepared.frame + problem = prepared.problem + national_rows = prepared.national_rows + target_set = prepared.target_set + target_loss_weights = prepared.target_loss_weights + binding_adjudications = prepared.binding_adjudications + local_count = len(problem.targets) + doctrine = UK_LOCAL_SOLVE_DOCTRINE evidence = _doctrine_solve_evidence( result, target_set=target_set, @@ -1534,10 +1670,19 @@ def _doctrine_solve_evidence( ) scales = default_target_loss_scales(targets_vec) - local_targets_vec = targets_vec[:local_count] - local_scales = scales[:local_count] - local_initial = initial_estimates[:local_count] - local_final = final_estimates[:local_count] + # Identity joins avoid making local-prefix/national-suffix ordering a + # public diagnostic contract. Compilation order is still checked above. + row_positions = { + diagnostic.name: i for i, diagnostic in enumerate(result.diagnostics) + } + local_positions = np.asarray( + [row_positions[target.row_name] for target in _rowwise_target_set(problem)], + dtype=np.int64, + ) + local_targets_vec = targets_vec[local_positions] + local_scales = scales[local_positions] + local_initial = initial_estimates[local_positions] + local_final = final_estimates[local_positions] diagnostics = problem.target_frame.copy() diagnostics["target"] = local_targets_vec diagnostics["initial_estimate"] = local_initial @@ -1558,10 +1703,14 @@ def _doctrine_solve_evidence( target_frame=problem.target_frame, ) national_specs = () if national_rows is None else national_rows.registry.specs - national_initial = initial_estimates[local_count:] - national_final = final_estimates[local_count:] - national_targets_vec = targets_vec[local_count:] - national_scales = scales[local_count:] + national_positions = np.asarray( + [row_positions[spec.to_target().row_name] for spec in national_specs], + dtype=np.int64, + ) + national_initial = initial_estimates[national_positions] + national_final = final_estimates[national_positions] + national_targets_vec = targets_vec[national_positions] + national_scales = scales[national_positions] national_diagnostics = pd.DataFrame( { "name": [spec.to_target().row_name for spec in national_specs], @@ -1658,6 +1807,18 @@ def rotated_uk_local_holdout( ) -> dict[str, object]: """Run five local-row rotations with national rows fixed in training.""" + if not len(problem.targets): + return { + "report_only": True, + "method": "rotated_folds", + "outcome": "not_applicable", + "reason": "No local targets were selected for calibration.", + "n_folds": 0, + "folds": [], + "training_national_rows": ( + 0 if national_rows is None else len(national_rows.targets) + ), + } folds = rotated_folds( len(problem.targets), n_folds=UK_LOCAL_HOLDOUT_FOLDS, diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/national_calibration.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/national_calibration.py index ba7cd6d5d..9c9786618 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/national_calibration.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/national_calibration.py @@ -1,349 +1,31 @@ -"""Ledger-backed calibration stage for the UK national build.""" +"""UK temporary-measure and frame adapters shared by full builds and scoring.""" from __future__ import annotations -from collections.abc import Mapping, Sequence +from collections.abc import Mapping from typing import Any import numpy as np -from microcosm.build.plan import Stage from microcosm.build.target_materialization import ( - MeasureResolution, resolve_target_measures, ) -from microcosm.build.uk_runtime.content_identity import uk_frame_content_identity from microcosm.build.uk_runtime.ledger_targets import ( UKFrameTargetAdapter, - UKLedgerTargetCompilation, materialize_uk_ledger_targets, ) -from microcosm.build.uk_runtime.national_doctrine import ( - UK_NATIONAL_SOLVE_DOCTRINE, - UKNationalSolveDoctrine, - uk_national_target_loss_weights, -) from microcosm.calibrate import ( - CalibrationResult, TargetRegistry, - calibrate, - effective_sample_size, ) -from microcosm.frame import Frame, WeightKind, Weights +from microcosm.frame import Frame, Weights __all__ = [ "CalibrationFrameAdapter", - "UKNationalCalibrationStage", "drop_injected_measure_inputs", "inject_measure_inputs", - "national_calibration_mass_reason", - "uk_national_calibration_stage", ] -class UKNationalCalibrationStage: - """Fail-closed national calibration transform and its manifest evidence.""" - - def __init__( - self, - registry: UKLedgerTargetCompilation | TargetRegistry, - *, - period: int, - doctrine: UKNationalSolveDoctrine = UK_NATIONAL_SOLVE_DOCTRINE, - measure_resolver: object | None = None, - band_edge_registry: TargetRegistry, - ) -> None: - self.compilation = ( - registry - if isinstance(registry, UKLedgerTargetCompilation) - else UKLedgerTargetCompilation(registry=registry, unsupported=()) - ) - self.registry = self.compilation.registry - # Required, never defaulted: the stage cannot tell a pruned registry - # from a full one, so a fallback to self.registry would quietly - # restate #792 for any caller holding an exclusion-pruned roster. - # A caller whose registry is unpruned passes it explicitly. - self.band_edge_registry = band_edge_registry - # The materialization period is the declared calibration year the - # registry was compiled at — never the input frame's base-year - # time_period, which lags it (survey 2024, calibration 2025). - # A binding can separately declare its observed measurement period, - # as the three individual CGT rows do for FY2024-25. - if not isinstance(period, int) or isinstance(period, bool) or period <= 0: - raise ValueError( - f"period must be the declared calibration year, got {period!r}." - ) - self.period = period - self.doctrine = doctrine - self.measure_resolver = measure_resolver - self.manifest: dict[str, object] | None = None - self.diagnostics: tuple[dict[str, object], ...] = () - self.solve_result: CalibrationResult | None = None - self.output_content_identity: str | None = None - - def __call__(self, frame: Frame) -> Frame: - declared = len(self.registry.specs) + len(self.compilation.unsupported) - resolved = len(self.registry.specs) - if self.compilation.unsupported: - raise RuntimeError( - "UK national calibration resolved " - f"{resolved} of {declared} activated target references; " - f"unsupported={self.compilation.unsupported!r}." - ) - adapter = _CalibrationFrameAdapter(frame) - original_columns = { - entity: set(table.columns) for entity, table in adapter.tables.items() - } - measure_resolution = self._resolve_measures(frame) - if measure_resolution is not None: - _inject_measure_inputs(adapter, measure_resolution.measure_inputs) - materialized = materialize_uk_ledger_targets( - adapter, - self.registry, - period=self.period, - band_edge_registry=self.band_edge_registry, - ) - if materialized.skipped: - skipped = [skip.__dict__ for skip in materialized.skipped] - raise RuntimeError( - "UK national calibration could not materialize every activated " - f"target reference: skipped={skipped}." - ) - if measure_resolution is not None: - _drop_injected_measure_inputs( - adapter, - measure_resolution.measure_inputs, - original_columns, - ) - prepared = adapter.prepared_frame() - mass_reason = national_calibration_mass_reason( - spec.family for spec in self.registry.specs - ) - mass_log_records_before_calibration = len(frame.mass_log) - # Target-set rows follow registry spec order, so the doctrine weight - # vector aligns positionally; under the default "uniform" rule this - # is None — the kernel's own equal weighting. - target_loss_weights = uk_national_target_loss_weights( - [spec.family for spec in self.registry.specs], - rule=self.doctrine.target_weight_rule, - ) - result = calibrate( - prepared, - self.registry.to_target_set(), - weight_entity="household", - epochs=self.doctrine.epochs, - learning_rate=self.doctrine.learning_rate, - mass=self.doctrine.mass_rule, - mass_reason=mass_reason, - max_weight_ratio=self.doctrine.max_weight_ratio, - seed=self.doctrine.seed, - l0_lambda=self.doctrine.l0_lambda, - target_loss_cap=self.doctrine.target_loss_cap, - target_loss_weights=target_loss_weights, - ) - if result.skipped or len(result.problem.names) != declared: - skipped = [item.name for item in result.skipped] - raise RuntimeError( - "UK national calibration matrix did not contain every activated " - f"reference: declared={declared}, rows={len(result.problem.names)}, " - f"skipped={skipped}." - ) - clean_frame = adapter.restore(result.frame) - self.solve_result = result - calibration_record = _post_solve_calibration_record( - frame, - clean_frame, - before_count=mass_log_records_before_calibration, - ) - self.diagnostics = tuple( - { - "name": row.name, - "estimate": row.final_estimate, - "target": row.target, - "relative_error": row.relative_error, - } - for row in result.diagnostics - ) - ratios = result.weights / result.initial_weights - old_total = float(calibration_record.old_total) - new_total = float(calibration_record.new_total) - before_kind = frame.weights_for("household").kind - after_kind = clean_frame.weights_for("household").kind - manifest = { - "activated_reference_count": declared, - "resolved_reference_count": resolved, - "matrix_target_count": len(result.problem.names), - "loss": result.final_loss, - "effective_sample_size": effective_sample_size(result.weights), - "max_weight_ratio": float(ratios.max()), - "max_weight_ratio_bound": self.doctrine.max_weight_ratio, - "target_materialization": materialized.report(), - "weights": { - "household_weight_kind": after_kind.value, - "household_weight_kind_chain": [ - {"stage": "staging", "kind": before_kind.value}, - {"stage": "national_calibration", "kind": after_kind.value}, - ], - "mass_log_records_before_calibration": ( - mass_log_records_before_calibration - ), - "mass_log_records": len(clean_frame.mass_log), - "calibration_mass_change": { - "entity": str(calibration_record.entity), - "old_total": old_total, - "new_total": new_total, - "relative_shift": (new_total - old_total) / old_total, - "declared_factor": calibration_record.declared_factor, - "reason": str(calibration_record.reason), - }, - }, - "solve": { - "n_targets": len(result.problem.names), - "n_households": len(clean_frame.table("household")), - "initial_loss": float(result.initial_loss), - "final_loss": float(result.final_loss), - "n_nonzero": int(np.count_nonzero(result.weights)), - }, - "parameters": {"doctrine": _doctrine_bounds(self.doctrine)}, - } - if measure_resolution is not None: - manifest["measure_resolution"] = dict(measure_resolution.receipt) - self.manifest = manifest - self.output_content_identity = uk_frame_content_identity(clean_frame) - return clean_frame - - def _resolve_measures(self, frame: Frame) -> MeasureResolution | None: - if self.measure_resolver is None: - return None - resolve = getattr(self.measure_resolver, "resolve", None) - if callable(resolve): - return resolve( - lambda: _CalibrationFrameAdapter(frame), - self.registry, - period=self.period, - band_edge_registry=self.band_edge_registry, - ) - return resolve_target_measures( - lambda: _CalibrationFrameAdapter(frame), - self.registry, - self.measure_resolver, - period=self.period, - band_edge_registry=self.band_edge_registry, - ) - - def checkpoint_metadata(self) -> Mapping[str, object]: - if self.manifest is None: - raise RuntimeError("UK national calibration has not run.") - return { - "calibration": self.manifest, - "diagnostics": self.diagnostics, - "output_content_identity": self.output_content_identity, - } - - def resume_from_checkpoint( - self, - metadata: Mapping[str, object], - frame: Frame, - ) -> None: - """Rehydrate completed calibration evidence from its checkpoint record.""" - - calibration = metadata.get("calibration") - diagnostics = metadata.get("diagnostics") - output_identity = metadata.get("output_content_identity") - count_keys = ( - "activated_reference_count", - "resolved_reference_count", - "matrix_target_count", - ) - if ( - not isinstance(calibration, Mapping) - or not all(key in calibration for key in count_keys) - or not isinstance(diagnostics, list) - or not all(isinstance(row, Mapping) for row in diagnostics) - or not isinstance(output_identity, str) - or not output_identity - ): - raise RuntimeError( - "UK national calibration resume requires the checkpoint record " - "to carry calibration counts, diagnostics, and output content " - "identity; a record without them cannot feed the calibration " - "reference coverage gate or the drift check." - ) - if uk_frame_content_identity(frame) != output_identity: - raise RuntimeError( - "UK national calibration checkpoint content does not match its " - "recorded output identity; refusing to resume from a drifted " - "record." - ) - self.manifest = dict(calibration) - self.diagnostics = tuple(dict(row) for row in diagnostics) - self.output_content_identity = output_identity - - -def uk_national_calibration_stage( - registry: UKLedgerTargetCompilation, **kwargs: Any -) -> Stage: - """Return the named ordered-stage entry for national calibration.""" - - transform = UKNationalCalibrationStage(registry, **kwargs) - return Stage(name="national_calibration", transform=transform) - - -def national_calibration_mass_reason(bound_families: Sequence[str]) -> str: - """The mass-record reason a national doctrine calibration declares.""" - - families = sorted({str(name) for name in bound_families}) - if not families or any(not name.strip() for name in families): - raise ValueError("bound_families must name at least one target family.") - return ( - "National doctrine calibration to bound target family(ies) " - f"{', '.join(families)}; total household mass moved with the targets." - ) - - -def _post_solve_calibration_record( - before: Frame, - after: Frame, - *, - before_count: int, -): - if after.weights_for("household").kind is not WeightKind.CALIBRATED: - raise RuntimeError( - "UK national calibration returned household weights whose kind is " - f"{after.weights_for('household').kind.value!r}, not 'calibrated'." - ) - if len(after.mass_log) != before_count + 1: - raise RuntimeError( - "UK national calibration must append exactly one mass record; " - f"before={before_count}, after={len(after.mass_log)}." - ) - if before.mass_log != after.mass_log[:before_count]: - raise RuntimeError( - "UK national calibration changed pre-existing mass-log records." - ) - record = after.mass_log[-1] - if record.entity != "household" or "calibration" not in record.reason: - raise RuntimeError( - "UK national calibration latest mass record is not the calibration " - f"record: entity={record.entity!r}, reason={record.reason!r}." - ) - return record - - -def _doctrine_bounds(doctrine: UKNationalSolveDoctrine) -> dict[str, object]: - return { - "epochs": doctrine.epochs, - "learning_rate": doctrine.learning_rate, - "max_weight_ratio": doctrine.max_weight_ratio, - "seed": doctrine.seed, - "target_loss_cap": doctrine.target_loss_cap, - "scale_rule": doctrine.scale_rule, - "target_weight_rule": doctrine.target_weight_rule, - "mass_rule": doctrine.mass_rule, - "l0_lambda": doctrine.l0_lambda, - } - - def inject_measure_inputs( adapter: UKFrameTargetAdapter, measure_inputs: Mapping[tuple[str, str], np.ndarray], @@ -419,7 +101,7 @@ def prepare_uk_target_frame( ) -> tuple[Frame, Mapping[str, Any] | None]: """Materialize a registry's measures onto a frame, for scoring. - The same resolve-inject-materialize route the calibration stage takes, + The same resolve-inject-materialize route the full build takes, without the solve: scoring a UK register against a raw exported H5 cannot work, because every packaged reference binds a slash-named prepared measure that calibration deliberately strips before export. A skipped diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/release_certification.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/release_certification.py index 278ecccea..e15902fa6 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/release_certification.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/release_certification.py @@ -1,24 +1,8 @@ -"""UK release-cut certification: the national battery's runner and composer. - -The June national driver retired with microcosm#757 and took the only -executor of the 16 declared national preflight/terminal gates with it. -This module is their executable home (issue #757 item B5): a scoped -``GateBatteryRun`` over ``UK_NATIONAL_GATE_SCOPE`` evaluated against the -calibrated candidate, plus the **multi-part release certification** the -2026-08-25 audit (issue comment 5413502559) specified — the spine build's -battery report, the calibration seam's battery report, and the release-cut -battery report must union to the full declared gate-entry set with no gap -and no overlap beyond ``UK_SHARED_GATE_IDS``, each part signed by its -producer, with the phase and digest checks moving from per-report to -per-certification. A candidate's shippability verdict comes only from the -certification, never from a single scoped report. - -Evidence adaptation only, never verdict re-implementation: every gate in -the release-cut battery runs the same ``UK_GATE_REGISTRY`` binding the June -runner used; this module reconstructs the evidence the retired runner drew -from live stage objects out of the artifacts the split pipeline persists -(the spine build sidecar, the seam's diagnostics and build record, the -per-run licensed input-mass reference). +"""Historical UK split-lane certification verification and shared evidence helpers. + +Current builds use full_certification: one graph carries the full gate roster. +The old independent national battery is retired; signed historical receipts +remain verifiable without introducing a second current build path. """ from __future__ import annotations @@ -37,8 +21,6 @@ from microcosm.build.country_spec import GatesManifest, load_country_spec from microcosm.build.gate_battery import ( - BlockingMode, - EvidenceContext, GateBatteryRun, gate_signing_key_env, ) @@ -51,8 +33,6 @@ UK_NATIONAL_GATE_SCOPE, UK_SHARED_GATE_IDS, UK_SPINE_GATE_SCOPE, - finalize_uk_scoped_gate_report, - uk_aggregate_admin_totals, uk_scoped_gate_manifest, ) @@ -63,7 +43,6 @@ "UKReleaseCertificationError", "compose_uk_release_certification", "rehydrate_uk_fit_weight_records", - "run_uk_release_cut_battery", "uk_national_gate_manifest", "uk_release_cut_scope_exclusions", "uk_release_parity_evidence", @@ -249,78 +228,6 @@ def uk_release_parity_evidence( ) -def run_uk_release_cut_battery( - frame: Any, - *, - report_path: Path, - release_id: str, - diagnostics_sha256: str, - coverage_engine: Any, - build_stage_names: Sequence[str], - ledger_registries: Mapping[object, Any], - local_ledger_registries: Mapping[object, Any], - parity_evidence: Any, - fit_weight_records: tuple[FitWeightRecord, ...] | None, - input_mass_reference: Mapping[str, Any], - exclusions_evaluated_on: date, - gate_registry: Mapping[str, Any] | None = None, -) -> dict[str, Any]: - """Run the 18 national gates over the calibrated candidate, signed. - - Always release-candidate strict: this battery exists to certify a cut, - so an ``evidence_absent`` gap blocks rather than being tolerated, and a - blocked phase persists its report and raises before any composition. - The local-surface compile gates run here too: the certification is the - declared owner of the whole national scope, so the caller supplies both - the national and the local compiled registries. - """ - - battery = GateBatteryRun( - uk_national_gate_manifest(), - release_id=release_id, - report_path=report_path, - release_candidate=True, - registry=UK_GATE_REGISTRY if gate_registry is None else gate_registry, - release_evidence={"calibration_diagnostics_sha256": diagnostics_sha256}, - ) - preflight_artifacts: dict[str, Any] = { - "coverage_engine": coverage_engine, - "build_stage_names": tuple(str(name) for name in build_stage_names), - "uk_ledger_compiled_registries": dict(ledger_registries), - "uk_ledger_compiled_local_registries": dict(local_ledger_registries), - } - battery.run_phase("preflight", EvidenceContext(artifacts=preflight_artifacts)) - battery.enforce("preflight", mode=BlockingMode.BLOCKS_ARTIFACT) - - admin_totals, admin_receipt = uk_aggregate_admin_totals( - frame, uk_national_gate_manifest() - ) - terminal_artifacts: dict[str, Any] = { - "coverage_engine": coverage_engine, - "rules_engine": coverage_engine, - "build_stage_names": tuple(str(name) for name in build_stage_names), - "exclusions_evaluated_on": exclusions_evaluated_on, - "parity_evidence": parity_evidence, - "aggregate_admin": admin_totals, - "input_mass_reference": input_mass_reference, - } - if fit_weight_records is not None: - terminal_artifacts["fit_weight_records"] = fit_weight_records - battery.run_phase( - "terminal", EvidenceContext(frame=frame, artifacts=terminal_artifacts) - ) - battery.enforce("terminal", mode=BlockingMode.BLOCKS_ARTIFACT) - payload = battery.report_payload() - finalize_uk_scoped_gate_report( - payload, - posture=UK_RELEASE_CUT_POSTURE, - scope_exclusions=uk_release_cut_scope_exclusions(), - aggregate_admin_measurement=admin_receipt, - ) - _write_json(report_path, payload) - return payload - - # --------------------------------------------------------------------------- # The multi-part certification composer # --------------------------------------------------------------------------- diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/release_input_coverage.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/release_input_coverage.py index 4ebb3a528..69f777f4e 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/release_input_coverage.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/release_input_coverage.py @@ -185,28 +185,26 @@ def reviewed_exclusions(self) -> dict[str, str]: @property def required_build_stages(self) -> frozenset[str]: - """National stages that the checked-in family contract makes mandatory.""" + """Canonical producers and predecessors required by the family contract.""" return frozenset( - str(family["stage"]) - for family in self.family_coverage.values() - if family.get("status") == _REQUIRED_AT_BUILD_STATUS + stage + for stages in self.required_build_stage_options.values() + for stage in stages ) @property def required_build_stage_options(self) -> Mapping[str, tuple[str, ...]]: - """Per-family executable stage alternatives declared by the manifest.""" + """All mandatory stages per family; alternative producers are unsupported.""" - options: dict[str, tuple[str, ...]] = {} - for name, family in self.family_coverage.items(): - if family.get("status") != _REQUIRED_AT_BUILD_STATUS: - continue - stages = [str(family["stage"])] - superseded_by = family.get("superseded_by") - if isinstance(superseded_by, Mapping): - stages.append(str(superseded_by["stage"])) - options[str(name)] = tuple(dict.fromkeys(stages)) - return options + return { + str(name): ( + *tuple(family.get("required_predecessor_stages", ())), + str(family["stage"]), + ) + for name, family in self.family_coverage.items() + if family.get("status") == _REQUIRED_AT_BUILD_STATUS + } def _resource_text(resource: str) -> str: @@ -304,48 +302,24 @@ def _parse_family_coverage( f"{resource}: family {name!r} needs a lowercase SHA-256 " "for source_manifest_sha256." ) - superseded_by = raw_family.get("superseded_by") - parsed_superseded_by: dict[str, Any] | None = None - if superseded_by is not None: - if not isinstance(superseded_by, Mapping): - raise ValueError( - f"{resource}: family {name!r} superseded_by must be an object." - ) - superseding_stage = str(superseded_by.get("stage", "")).strip() - superseding_manifest = str( - superseded_by.get("source_manifest", "") - ).strip() - superseding_sha = str( - superseded_by.get("source_manifest_sha256", "") - ).strip() - supersession_reason = str(superseded_by.get("reason", "")).strip() - if not superseding_stage: - raise ValueError( - f"{resource}: family {name!r} superseded_by needs a stage." - ) - if not superseding_manifest: - raise ValueError( - f"{resource}: family {name!r} superseded_by needs a " - "source_manifest." - ) - if len(superseding_sha) != 64 or any( - character not in "0123456789abcdef" for character in superseding_sha - ): - raise ValueError( - f"{resource}: family {name!r} superseded_by needs a " - "lowercase SHA-256 for source_manifest_sha256." - ) - if not supersession_reason: - raise ValueError( - f"{resource}: family {name!r} superseded_by needs a reason." - ) - parsed_superseded_by = { - **dict(superseded_by), - "stage": superseding_stage, - "source_manifest": superseding_manifest, - "source_manifest_sha256": superseding_sha, - "reason": supersession_reason, - } + if "superseded_by" in raw_family: + raise ValueError( + f"{resource}: family {name!r} must name its canonical producer; " + "superseded_by alternatives are no longer supported." + ) + predecessors = raw_family.get("required_predecessor_stages", []) + if ( + not isinstance(predecessors, list) + or any( + not isinstance(value, str) or not value.strip() + for value in predecessors + ) + or len(set(predecessors)) != len(predecessors) + or stage in predecessors + ): + raise ValueError( + f"{resource}: family {name!r} has invalid required_predecessor_stages." + ) try: base_candidate_tier = validate_uk_release_tier( raw_family.get("base_candidate_tier") @@ -438,11 +412,6 @@ def _parse_family_coverage( "stage": stage, "source_manifest": source_manifest, "source_manifest_sha256": source_manifest_sha256, - **( - {"superseded_by": parsed_superseded_by} - if parsed_superseded_by is not None - else {} - ), "base_candidate_tier": base_candidate_tier, "output_weight_kind": output_weight_kind, "required_mass_change_reason": required_mass_change_reason, @@ -1439,7 +1408,7 @@ def assert_uk_release_input_coverage_build_stages( missing = sorted( family for family, options in manifest.required_build_stage_options.items() - if actual.isdisjoint(options) + if not set(options).issubset(actual) ) if missing: raise ValueError( diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/rowwise_dataset.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/rowwise_dataset.py index c49a78a1e..1749bca1c 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/rowwise_dataset.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/rowwise_dataset.py @@ -296,27 +296,36 @@ class UKLadderRowwiseDatasetResult: output_path: Path | None = None -def clone_uk_dataset_tables_with_ladder_geography( +@dataclass(frozen=True) +class UKGeographicPool: + """Linked geographic copies before any location draw. + + ``n_clones`` is the total copies K, independent of output household size. + Source IDs, replicate IDs and the original SPI/CGT ancestry stay distinct. + """ + + frame: Frame + n_clones: int + id_multiplier: int + + +def expand_uk_geographic_pool( *, person: pd.DataFrame, benunit: pd.DataFrame, household: pd.DataFrame, - ladder: UkOaLadder, n_clones: int = 1, - seed: int = 42, time_period: int | str | None = None, source_year: int | None = None, id_multiplier: int | None = None, - expected_constituency_vintage: str | None = None, - region_column: str = "region", household_weight_kind: WeightKind = WeightKind.DESIGN, mass_log: tuple[MassChangeRecord, ...] = (), source_lineage_modulus: int | None = None, -) -> UKLadderRowwiseDatasetResult: - """Clone UK tables and assign geography through the OA ladder. +) -> UKGeographicPool: + """Prepare lineage and expand linked entities using shared clone operations. - The result carries a validated UK national frame; clone indices land on - the canonical per-entity :func:`ladder_clone_index_column` names. + The existing clone-major row order, ID multiplier and weight split are + preserved. This operation consumes no randomness. """ _validate_weight_metadata(household_weight_kind, mass_log) @@ -410,16 +419,8 @@ def clone_uk_dataset_tables_with_ladder_geography( _assert_clone_link_alignment(cloned_person, cloned_household) - assigned = assign_uk_geography_ladder( - cloned_household, - ladder, - seed=seed, - expected_constituency_vintage=expected_constituency_vintage, - region_column=region_column, - ).reset_index(drop=True) - output_total = float( - np.asarray(assigned["household_weight"], dtype=np.float64).sum() + np.asarray(cloned_household["household_weight"], dtype=np.float64).sum() ) _assert_household_mass_conserved(input_total, output_total) clone_record = MassChangeRecord( @@ -434,9 +435,59 @@ def clone_uk_dataset_tables_with_ladder_geography( ), ) + return UKGeographicPool( + frame=uk_national_frame( + person=cloned_person, + benunit=cloned_benunit, + household=cloned_household, + time_period=_normalise_time_period(time_period, source_year=source_year), + weight_kind=household_weight_kind, + mass_log=(*mass_log, clone_record), + ), + n_clones=n_clones, + id_multiplier=id_multiplier, + ) + + +def assign_uk_geographic_pool( + pool: UKGeographicPool, + ladder: UkOaLadder, + *, + seed: int = 42, + expected_constituency_vintage: str | None = None, + region_column: str = "region", +) -> Frame: + """Draw and derive current ladder geography, retaining legacy RNG order.""" + + frame = pool.frame + assigned = assign_uk_geography_ladder( + frame.table("household"), + ladder, + seed=seed, + expected_constituency_vintage=expected_constituency_vintage, + region_column=region_column, + ).reset_index(drop=True) + return uk_national_frame( + person=frame.table("person"), + benunit=frame.table("benunit"), + household=assigned, + time_period=uk_time_period(frame), + weight_kind=frame.weights_for("household").kind, + household_weights=frame.weights_for("household").values, + mass_log=frame.mass_log, + ) + + +def validate_uk_geographic_pool( + frame: Frame, + *, + region_column: str = "region", +) -> GateResult: + """Check assigned geography and entity links before contributions compile.""" + gate = uk_geography_ladder_gate( - assigned, - np.asarray(assigned["household_weight"], dtype=np.float64), + frame.table("household"), + frame.weights_for("household").values, region_column=region_column, ) if not gate.passed: @@ -444,23 +495,60 @@ def clone_uk_dataset_tables_with_ladder_geography( "UK geography ladder gate failed on the cloned assignment: " + "; ".join(gate.failures) ) + validate_uk_ladder_rowwise_dataset_tables( + frame.table("person"), frame.table("benunit"), frame.table("household") + ) + return gate - validate_uk_ladder_rowwise_dataset_tables(cloned_person, cloned_benunit, assigned) - # The frame construction re-runs linkage validation and binds the typed - # household weights, the mass log, and the time period to the carrier. - frame = uk_national_frame( - person=cloned_person, - benunit=cloned_benunit, - household=assigned, - time_period=_normalise_time_period(time_period, source_year=source_year), - weight_kind=household_weight_kind, - mass_log=(*mass_log, clone_record), + +def clone_uk_dataset_tables_with_ladder_geography( + *, + person: pd.DataFrame, + benunit: pd.DataFrame, + household: pd.DataFrame, + ladder: UkOaLadder, + n_clones: int = 1, + seed: int = 42, + time_period: int | str | None = None, + source_year: int | None = None, + id_multiplier: int | None = None, + expected_constituency_vintage: str | None = None, + region_column: str = "region", + household_weight_kind: WeightKind = WeightKind.DESIGN, + mass_log: tuple[MassChangeRecord, ...] = (), + source_lineage_modulus: int | None = None, +) -> UKLadderRowwiseDatasetResult: + """Clone UK tables and assign geography through the OA ladder. + + The result carries a validated UK national frame; clone indices land on + the canonical per-entity :func:`ladder_clone_index_column` names. + """ + + pool = expand_uk_geographic_pool( + person=person, + benunit=benunit, + household=household, + n_clones=n_clones, + time_period=time_period, + source_year=source_year, + id_multiplier=id_multiplier, + household_weight_kind=household_weight_kind, + mass_log=mass_log, + source_lineage_modulus=source_lineage_modulus, ) + frame = assign_uk_geographic_pool( + pool, + ladder, + seed=seed, + expected_constituency_vintage=expected_constituency_vintage, + region_column=region_column, + ) + gate = validate_uk_geographic_pool(frame, region_column=region_column) return UKLadderRowwiseDatasetResult( frame=frame, gate=gate, n_clones=n_clones, - id_multiplier=id_multiplier, + id_multiplier=pool.id_multiplier, ) diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/size_checkpoint.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/size_checkpoint.py index 6f659849d..f588be885 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/size_checkpoint.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/size_checkpoint.py @@ -232,8 +232,26 @@ def load_uk_size_checkpoint( with the losses recomputed on the compiled system. """ directory = Path(directory) - arrays_path = directory / SIZE_CHECKPOINT_ARRAYS_FILENAME - manifest_path = directory / SIZE_CHECKPOINT_MANIFEST_FILENAME + return load_uk_size_checkpoint_files( + directory / SIZE_CHECKPOINT_MANIFEST_FILENAME, + directory / SIZE_CHECKPOINT_ARRAYS_FILENAME, + frame=frame, + target_set=target_set, + identity=identity, + ) + + +def load_uk_size_checkpoint_files( + manifest_path: Path, + arrays_path: Path, + *, + frame: Frame, + target_set: TargetSet, + identity: Mapping[str, Any], +) -> UKSizeCheckpointRestore: + """Read the same checkpoint from separately content-verified source paths.""" + manifest_path, arrays_path = Path(manifest_path), Path(arrays_path) + directory = manifest_path.parent for required in (arrays_path, manifest_path): if not required.is_file(): raise FileNotFoundError(f"size checkpoint file missing: {required}.") diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/source_runtime.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/source_runtime.py index c36abf6cb..342c41c92 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/source_runtime.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/source_runtime.py @@ -49,8 +49,6 @@ def _uk_nonnegative_outputs_by_stage() -> dict[str, tuple[str, ...]]: def uk_stage_implementations( *, - retained_leaves_transform: Callable[[Frame], Frame], - hmrc_income_transform: Callable[[Frame], Frame], frs_spine_transform: Callable[[Frame], Frame] | None = None, frs_employment_transform: Callable[[Frame], Frame] | None = None, frs_council_tax_transform: Callable[[Frame], Frame] | None = None, @@ -80,10 +78,6 @@ def uk_stage_implementations( """Return the whole-stage implementation map for the UK source plan.""" implementations = { - "frs_hmrc_retained_leaves": retained_leaves_transform, - "hmrc_spi_income": hmrc_income_transform, - } - optional = { "frs_spine": frs_spine_transform, "frs_employment": frs_employment_transform, "frs_council_tax": frs_council_tax_transform, @@ -110,14 +104,11 @@ def uk_stage_implementations( "salary_sacrifice": salary_sacrifice_transform, "student_loans": student_loans_transform, } - implementations.update( - { - name: transform - for name, transform in optional.items() - if transform is not None - } - ) - return implementations + return { + name: transform + for name, transform in implementations.items() + if transform is not None + } def uk_source_operation_handlers() -> Mapping[str, SourceOperationHandler]: diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/spi_income.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/spi_income.py index dc06ec036..720648e93 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/spi_income.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/spi_income.py @@ -18,7 +18,7 @@ UKDWPDisabilityCategoryRates, UKDWPDisabilityFlagRates, ) -from microcosm.build.uk_runtime.frs_hmrc_leaves import ( +from microcosm.build.uk_runtime.frs_hmrc_source import ( FRS_HMRC_INCPBEN_COLUMN, FRS_HMRC_OSSBEN_IDENTIFIABLE_SUBSET_COLUMN, FRS_HMRC_PAY_COLUMN, diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/spi_spine.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/spi_spine.py index fb3ee169e..2540a4ad9 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/spi_spine.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/spi_spine.py @@ -12,7 +12,7 @@ from microcosm.build.gates import FitWeightRecord from microcosm.build.source_manifest import SourceStageSpec -from microcosm.build.uk_runtime.frs_hmrc_leaves import ( +from microcosm.build.uk_runtime.frs_hmrc_source import ( FRS_HMRC_INCPBEN_COLUMN, FRS_HMRC_PAY_COLUMN, FRS_HMRC_RETAINED_LEAF_COLUMNS, @@ -286,11 +286,13 @@ def __call__(self, frame: Frame) -> Frame: adult, adult_identity = _read_raw_frs_table( self.frs_raw_dir / str(artifacts["adult"]["locator"]), expected_filename="adult.tab", + source_vintage=str(artifacts["adult"].get("vintage", "unspecified")), required_columns=("sernum", "person", "inearns"), ) benefits, benefits_identity = _read_raw_frs_table( self.frs_raw_dir / str(artifacts["benefits"]["locator"]), expected_filename="benefits.tab", + source_vintage=str(artifacts["benefits"].get("vintage", "unspecified")), required_columns=("sernum", "person", "benefit", "benamt", "var2"), ) _assert_identity_matches_artifact(adult_identity.evidence(), artifacts["adult"]) @@ -308,7 +310,7 @@ def __call__(self, frame: Frame) -> Frame: if missing_ids: # A rung sample deliberately drops most source people; restrict # the raw surface to the survivors (the full-scale fence above - # stays strict — mirrors frs_hmrc_leaves' sampled_rung posture). + # stays strict). source_leaves = source_leaves.loc[ source_leaves.index.isin(person["person_id"].to_numpy()) ] diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/spine_build.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/spine_build.py new file mode 100644 index 000000000..927f54cad --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/spine_build.py @@ -0,0 +1,1548 @@ +"""Build the raw UK FRS spine Frame from pinned local tabs.""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import sys +import time +from collections.abc import Mapping, Sequence +from dataclasses import dataclass +from datetime import UTC, datetime +from importlib import metadata +from pathlib import Path + +from microcosm.build.country_spec import ( + GatesManifest, + load_country_spec, +) +from microcosm.build.frame_sampling import ( + normalize_sampled_household_mass, + sample_frame_households, +) +from microcosm.build.gate_battery import BlockingMode, EvidenceContext, GateBatteryRun +from microcosm.build.logbook import canonical_json_bytes +from microcosm.build.logbook_adoption import ( + AttemptState, + append_phase, + apply_error_verdict, + atomic_write_json, + error_receipt_path, + git_code_pin, + local_artifact_reference, + preflight_digest, + record_terminal_attempt, + resolve_predecessor, + role_pins_digest, + sha256_argument, + write_error_receipt, +) +from microcosm.build.plan import StageRecord +from microcosm.build.uk_runtime.age_tail import UKAgeTailStageTransform +from microcosm.build.uk_runtime.battery_bindings import UK_GATE_REGISTRY +from microcosm.build.uk_runtime.calibration_run import ( + UK_SPINE_GATE_SCOPE, + uk_scoped_gate_manifest, +) +from microcosm.build.uk_runtime.cgt_imputation import uk_cgt_spine_stage_transform +from microcosm.build.uk_runtime.cgt_structure import ( + UKCGTBandDonorStageTransform, + UKCGTIncidenceCloneStageTransform, +) +from microcosm.build.uk_runtime.content_identity import uk_frame_content_identity +from microcosm.build.uk_runtime.etb_services import UKETBServicesStageTransform +from microcosm.build.uk_runtime.etb_vat import UKETBVATStageTransform +from microcosm.build.uk_runtime.frs_brma import UKFRSBRMAStageTransform +from microcosm.build.uk_runtime.frs_council_tax import UKFRSCouncilTaxStageTransform +from microcosm.build.uk_runtime.frs_disability import UKFRSDisabilityStageTransform +from microcosm.build.uk_runtime.frs_education import UKFRSEducationStageTransform +from microcosm.build.uk_runtime.frs_education_grants import ( + FRS_EDUCATION_GRANT_REWRITES, + UKFRSEducationGrantSplitStageTransform, +) +from microcosm.build.uk_runtime.frs_employment import UKFRSEmploymentStageTransform +from microcosm.build.uk_runtime.frs_household_draws import ( + UKFRSHouseholdDrawsStageTransform, +) +from microcosm.build.uk_runtime.frs_legacy_proxies import ( + UKFRSLegacyProxiesStageTransform, +) +from microcosm.build.uk_runtime.frs_person_draws import UKFRSPersonDrawsStageTransform +from microcosm.build.uk_runtime.frs_release import load_uk_frs_release +from microcosm.build.uk_runtime.frs_spine import ( + UKFRSSpineStageTransform, + uk_frs_spine_seed_frame, +) +from microcosm.build.uk_runtime.frs_take_up import UKFRSTakeUpStageTransform +from microcosm.build.uk_runtime.graph import ( + uk_registry, + uk_spine_graph, + uk_spine_operation_inventory, +) +from microcosm.build.uk_runtime.graph_evidence import ( + add_uk_spine_gate_nodes, + load_spine_stage_artifacts, + materialize_spine_gate_reports, + register_spine_gate_kernel, + spine_sidecar_evidence, +) +from microcosm.build.uk_runtime.lcfs_consumption import ( + UKLCFSConsumptionStageTransform, +) +from microcosm.build.uk_runtime.national_frame import ( + uk_household_weight_kind, + write_uk_national_frame, +) +from microcosm.build.uk_runtime.national_sampling import ( + UK_SAMPLE_RUNG_TOKENS, + UK_SAMPLE_SEED_DEFAULT, +) +from microcosm.build.uk_runtime.regional_uprating import ( + UKRegionalPropertyUpratingStageTransform, +) +from microcosm.build.uk_runtime.salary_sacrifice import UKSalarySacrificeStageTransform +from microcosm.build.uk_runtime.spi_spine import ( + UKFRSHMRCSpineLeavesStageTransform, + UKSPIIncomeSpineStageTransform, + UKSPISupportChannelStageTransform, +) +from microcosm.build.uk_runtime.student_loans import UKStudentLoansStageTransform +from microcosm.build.uk_runtime.take_up_contract import load_uk_take_up_contract +from microcosm.build.uk_runtime.uc_capital_coherence import ( + UKUCCapitalCoherenceStageTransform, +) +from microcosm.build.uk_runtime.uc_deduction_attributes import ( + UKUCDeductionAttributesStageTransform, +) +from microcosm.build.uk_runtime.uc_reporter_redraw import ( + UKUCReporterRedrawStageTransform, +) +from microcosm.build.uk_runtime.was_wealth import UKWASWealthStageTransform +from microcosm.frame.adapters.policyengine_uk import PolicyEngineUKEngine +from microcosm.graph import ContentStore, compile_graph, run_graph + +_PIPELINE = "uk-frs-spine" +_REPOSITORY = next( + ( + parent + for parent in Path(__file__).resolve().parents + if (parent / "pyproject.toml").is_file() and (parent / "packages").is_dir() + ), + Path.cwd(), +) +_RUNG_NAMED_EDGE_SIGNATURE = "The least populated classes in y have only 1 member" +_RUNG_ABORT_EXIT_CODE = 3 +#: The last stage of the assembled checkpoint: everything through the base +#: FRS mapping and the stochastic draws. A name, not an index — a position +#: standing in for a key is correct only while two independently-maintained +#: orderings happen to agree (the uk-data#468 class). +UK_SPINE_ASSEMBLED_FINAL_STAGE = "frs_brma" + + +def _uk_spine_stage_names(spec) -> tuple[str, ...]: + """Derive the runnable manifest stages from graph ownership edges.""" + + if spec.sources is None: + raise ValueError("UK country spec has no source stages.") + declared = {stage.stage for stage in spec.sources.stages} + compiled = compile_graph(uk_spine_graph(spec)) + ordered = tuple(node_id for node_id in compiled.order if node_id in declared) + if set(ordered) != declared: + raise ValueError( + "UK spine graph and manifest stage roster disagree: " + f"graph={list(ordered)!r}, manifest={sorted(declared)!r}." + ) + return ordered + + +def _rung_sample_fraction(value: str) -> float: + """CLI rung policy (#624) over the permissive library validator.""" + + try: + fraction = float(value) + except ValueError as error: + raise argparse.ArgumentTypeError( + f"sample fraction must be a number; got {value!r}." + ) from error + if fraction not in UK_SAMPLE_RUNG_TOKENS: + raise argparse.ArgumentTypeError( + "sample fraction must be one of 0.01, 0.10, or 1.0 (the #624 rungs)." + ) + return fraction + + +def _parse_args(argv: list[str] | None = None) -> argparse.Namespace: + parser = argparse.ArgumentParser( + description=( + "Build the UK FRS spine from pinned raw tabs, preserving each " + "stage's declared seeds and draw order." + ) + ) + parser.add_argument( + "--frs-raw-dir", + type=Path, + required=True, + help="Directory containing the 14 licensed FRS 2024-25 tab files.", + ) + parser.add_argument( + "--spine-h5", + type=Path, + required=True, + help="Output H5 path for the raw FRS spine Frame.", + ) + parser.add_argument( + "--spi-tab", + type=Path, + required=True, + help="Pinned local SPI 2022-23 put2223uk.tab path.", + ) + parser.add_argument( + "--hmrc-ods", + type=Path, + required=True, + help="Pinned local HMRC collated ODS path.", + ) + parser.add_argument( + "--cgt-ods", + type=Path, + help="Pinned local HMRC Capital Gains Tax Table 3 ODS path.", + ) + parser.add_argument( + "--checkpoint-dir", + type=Path, + help="Optional directory for a copy of the completed spine checkpoint.", + ) + parser.add_argument( + "--sample-fraction", + type=_rung_sample_fraction, + default=1.0, + help=( + "Scale-ladder rung (#624): 0.01 smoke, 0.10 dev, or 1.0 full. " + "Below 1.0 the raw FRS spine is sampled immediately after ingest, " + "renormalized to full household mass, and treated as a receipt." + ), + ) + parser.add_argument( + "--release-candidate", + action="store_true", + help=( + "Evaluate the spine battery at release-candidate strictness: " + "evidence_absent gaps block instead of being tolerated. Explicit " + "by design - a full-scale developer build is not a release " + "candidate unless the caller says so." + ), + ) + parser.add_argument( + "--sample-seed", + type=int, + default=UK_SAMPLE_SEED_DEFAULT, + help=f"Raw FRS spine sampling seed (default: {UK_SAMPLE_SEED_DEFAULT}).", + ) + parser.add_argument( + "--was-tab", + type=Path, + help="Caller-supplied private WAS round-8 household tab for was_wealth.", + ) + parser.add_argument( + "--lcfs-hh-tab", + type=Path, + help="Caller-supplied private LCFS 2023-24 household tab for lcfs_consumption.", + ) + parser.add_argument( + "--lcfs-person-tab", + type=Path, + help="Caller-supplied private LCFS 2023-24 person tab for lcfs_consumption.", + ) + parser.add_argument( + "--etb-tab", + type=Path, + help="Caller-supplied private ETB 1977-2024 household tab for ETB stages.", + ) + parser.add_argument( + "--emit-nonzero-shares", + type=Path, + help="Optional JSON path for unweighted per-produced-column nonzero shares.", + ) + parser.add_argument( + "--logbook-prev-row-digest", + type=sha256_argument, + help="Optional current Logbook chain head.", + ) + args = parser.parse_args(argv) + if args.sample_seed < 0: + parser.error("sample seed must be a non-negative integer.") + if args.sample_fraction != 1.0 and args.checkpoint_dir is not None: + parser.error( + "sampled spine rungs refuse --checkpoint-dir; rung artifacts are " + "receipts, never releases." + ) + return args + + +def _validate_args(args: argparse.Namespace) -> None: + if not args.frs_raw_dir.is_dir(): + raise ValueError( + f"--frs-raw-dir must be an existing directory: {args.frs_raw_dir}" + ) + if args.spine_h5.suffix != ".h5": + raise ValueError("--spine-h5 must end with '.h5'.") + if not args.spi_tab.is_file(): + raise ValueError(f"--spi-tab must be an existing file: {args.spi_tab}") + if args.spi_tab.name != "put2223uk.tab": + raise ValueError("--spi-tab must name put2223uk.tab.") + if not args.hmrc_ods.is_file(): + raise ValueError(f"--hmrc-ods must be an existing file: {args.hmrc_ods}") + if args.hmrc_ods.suffix.lower() != ".ods": + raise ValueError("--hmrc-ods must end with '.ods'.") + if args.cgt_ods is not None: + if not args.cgt_ods.is_file(): + raise ValueError(f"--cgt-ods must be an existing file: {args.cgt_ods}") + if args.cgt_ods.suffix.lower() != ".ods": + raise ValueError("--cgt-ods must end with '.ods'.") + paths = { + "spine_h5": args.spine_h5, + "build_sidecar": args.spine_h5.with_suffix(".build.json"), + "hmrc_replay_sidecar": args.spine_h5.with_suffix(".hmrc_replay.json"), + } + if args.emit_nonzero_shares is not None: + paths["emit_nonzero_shares"] = args.emit_nonzero_shares + resolved: dict[Path, str] = {} + for label, path in paths.items(): + target = Path(path).expanduser().resolve() + other = resolved.get(target) + if other is not None: + raise ValueError(f"{label} path collides with {other}: {target}.") + resolved[target] = label + + +def _artifact_pins(stages) -> dict[str, dict[str, object]]: + pins = {} + for stage in stages: + for artifact in stage.artifacts: + key = artifact.get("table", artifact.get("filename")) + if key is None: + continue + key = str(key) + pin = { + "locator": str(artifact["locator"]), + "sha256": str(artifact["sha256"]), + "size_bytes": int(artifact["size_bytes"]), + } + if key in pins and pins[key] != pin: + raise ValueError( + f"UK source artifact {key!r} has inconsistent pins across stages." + ) + pins[key] = pin + return dict(sorted(pins.items())) + + +def _stage_artifact_pins(stage) -> dict[str, dict[str, object]]: + return { + str(artifact.get("table", artifact.get("filename"))): { + "locator": str(artifact["locator"]), + "sha256": str(artifact["sha256"]), + "size_bytes": int(artifact["size_bytes"]), + } + for artifact in stage.artifacts + if "table" in artifact or "filename" in artifact + } + + +def _resource_pins(stages, spec) -> dict[str, str]: + """Country-package resources the selected stages declare as inputs. + + Non-tab artifacts reference committed resources by filename; their bytes + are hashed by load_country_spec, so the pin is the spec's recorded sha. + """ + + pins: dict[str, str] = {} + for stage in stages: + for artifact in stage.artifacts: + if "resource" not in artifact: + continue + resource = str(artifact["resource"]) + sha256 = spec.resource_hashes.get(resource) + if sha256 is None: + raise ValueError( + f"stage {stage.stage!r} declares resource artifact " + f"{resource!r} which is not a declared country-package " + "resource." + ) + pins[resource] = str(sha256) + return dict(sorted(pins.items())) + + +def _input_artifact_pins(stages) -> dict[str, dict[str, object]]: + """Caller-supplied private input artifacts, pinned by role. + + Non-table, non-resource artifacts (the SPI donor tab and the HMRC ODS) + carry their own sha256/size pins in the manifest. Binding them here puts + the pins in the build sidecar and the Logbook input-pins digest, so two + runs with different high-impact source inputs can never share build-side + provenance (adversarial-review finding on #717). + """ + + pins: dict[str, dict[str, object]] = {} + for stage in stages: + for artifact in stage.artifacts: + if "table" in artifact or "resource" in artifact: + continue + if "sha256" not in artifact: + continue + role = str(artifact.get("role") or artifact.get("filename") or "") + if not role: + raise ValueError( + f"stage {stage.stage!r} declares a pinned input artifact " + "without a role or filename." + ) + pin = { + "filename": str( + artifact.get("filename") or artifact.get("locator") or "" + ), + "kind": str(artifact.get("kind", "")), + "sha256": str(artifact["sha256"]), + "size_bytes": int(artifact["size_bytes"]), + } + if role in pins and pins[role] != pin: + raise ValueError( + f"input artifact role {role!r} has inconsistent pins across stages." + ) + pins[role] = pin + return dict(sorted(pins.items())) + + +def _role_pins(pins: dict[str, dict[str, object]]) -> dict[str, dict[str, object]]: + return { + table: { + "sha256": str(pin["sha256"]), + "size_bytes": int(pin["size_bytes"]), + } + for table, pin in pins.items() + } + + +def _entity_row_counts(frame) -> dict[str, int]: + return {entity: int(len(frame.table(entity))) for entity in frame.entities} + + +def _rules_engine() -> PolicyEngineUKEngine: + try: + import policyengine_uk # noqa: F401 + except ImportError as exc: + raise ImportError( + "build_uk_frs_spine requires the microcosm-build 'uk' extra " + "(policyengine-uk). Run: uv sync --all-packages --extra uk" + ) from exc + return PolicyEngineUKEngine() + + +def _rules_engine_provenance() -> dict[str, str]: + try: + version = metadata.version("policyengine-uk") + except metadata.PackageNotFoundError: + return {"package": "policyengine-uk", "version": "unavailable"} + return {"package": "policyengine-uk", "version": version} + + +def _declared_seeds(stages) -> dict[str, dict[str, int]]: + declared: dict[str, dict[str, int]] = {} + for stage in stages: + stage_seeds: dict[str, int] = {} + for operation in stage.operations: + output = operation.parameters.get("output") + seed = operation.parameters.get("seed") + if seed is None: + seed = operation.parameters.get("seed_base") + if isinstance(output, str) and isinstance(seed, int): + stage_seeds[output] = seed + elif isinstance(seed, int): + if operation.kind == "stack_zero_weight_donors": + stage_seeds["stack_zero_weight_donors"] = seed + elif operation.kind == "strict_read_private_table": + stage_seeds["donor_bootstrap"] = seed + elif operation.kind == "fit_weighted_qrf_stage1": + stage_seeds["stage1"] = seed + elif operation.kind == "fit_weighted_qrf_stage2": + stage_seeds["stage2"] = seed + elif operation.kind == "bridge_donor_column_via_qrf": + stage_seeds["bridge_donor_column_via_qrf"] = seed + elif operation.kind == "assign_binary_from_rate": + target = operation.parameters.get("target") + if isinstance(target, str): + stage_seeds[target] = seed + else: + stage_seeds["assign_binary_from_rate"] = seed + elif operation.kind == "fit_weighted_qrf_chain": + stage_seeds[stage.stage] = seed + elif operation.kind == "fit_weighted_qrf": + stage_seeds[stage.stage] = seed + elif operation.kind == "draw_capital_gains_prior_from_banded_quantiles": + stage_seeds[str(operation.parameters["salt"])] = seed + elif operation.kind == "stack_band_donor_households": + stage_seeds["stack_band_donor_households"] = seed + elif operation.kind == "within_band_draws": + stage_seeds["within_band_draws"] = seed + elif operation.kind == "convert_donors_to_target_stock": + stage_seeds[str(operation.parameters["salt"])] = seed + elif operation.kind == "top_up_to_stock": + stage_seeds[str(operation.parameters["salt"])] = seed + if stage_seeds: + declared[stage.stage] = stage_seeds + return declared + + +def _result_evidence(result: object) -> object: + if isinstance(result, dict): + return result + evidence = getattr(result, "evidence", None) + if callable(evidence): + return evidence() + return None + + +def _collect_stage_evidence( + *, + stage_names: Sequence[str], + implementations: Mapping[str, object], +) -> dict[str, object]: + evidence_by_stage: dict[str, object] = {} + for stage_name in stage_names: + implementation = implementations.get(stage_name) + if implementation is None: + continue + metadata = None + metadata_hook = getattr(implementation, "checkpoint_metadata", None) + if callable(metadata_hook): + metadata = dict(metadata_hook()) + evidence = metadata.get("evidence", metadata) + else: + evidence = _result_evidence(getattr(implementation, "last_result", None)) + if evidence is not None: + evidence_by_stage[stage_name] = evidence + return evidence_by_stage + + +def _collect_fit_weight_records( + *, + stage_names: Sequence[str], + implementations: Mapping[str, object], +) -> dict[str, list[dict[str, str]]]: + """Persist each fitting stage's resolved weight kinds into the sidecar. + + The terminal weights audit (``uk_weights_audit``) consumes + :class:`FitWeightRecord` evidence that only exists on live stage + objects; the release-cut certification producer runs in a later + process, so the sidecar carries the records across the run boundary. + Duck-typed like ``stage_evidence``: every stage whose transform exposes + ``fit_weight_records`` contributes, in stage order. A fitting stage + whose records are missing, unreadable, or empty records an empty list — + the audit binding fails an empty record set, so the gap stays visible + rather than vanishing from the sidecar. + """ + + records_by_stage: dict[str, list[dict[str, str]]] = {} + for stage_name in stage_names: + implementation = implementations.get(stage_name) + if implementation is None: + continue + # Detect the hook without evaluating it: a raising property must + # count as a fitting stage with unreadable records, not vanish. + exposes_records = getattr( + type(implementation), "fit_weight_records", None + ) is not None or "fit_weight_records" in getattr(implementation, "__dict__", {}) + if not exposes_records: + continue + try: + records = tuple(implementation.fit_weight_records or ()) + except Exception: # noqa: BLE001 - unreadable records fail the audit + records_by_stage[stage_name] = [] + continue + records_by_stage[stage_name] = [ + { + "fit_name": str(record.fit_name), + "weight_kind": str(record.weight_kind), + } + for record in records + ] + return records_by_stage + + +def _build_sidecar( + *, + frame, + stages, + records, + artifact_pins, + resource_pins: dict[str, str], + input_artifact_pins: dict[str, dict[str, object]], + hmrc_replay: dict[str, object], + stochastic_contract_sha256: str, + frs_vintage: str, + sampling: dict[str, object] | None, + spine_gate_report: dict[str, object] | None = None, +) -> dict[str, object]: + household_weight = frame.weights_for("household") + return { + "schema_version": 2, + "pipeline": _PIPELINE, + "uk_frame_content_identity": uk_frame_content_identity(frame), + "stages": [stage.stage for stage in stages], + "time_period": str(frame.metadata["time_period"]), + "household_weight_kind": uk_household_weight_kind(frame).value, + "household_weight_total": float(household_weight.values.sum()), + "entity_row_counts": _entity_row_counts(frame), + "artifact_pins": artifact_pins, + "resource_pins": resource_pins, + "input_artifact_pins": input_artifact_pins, + "hmrc_replay": hmrc_replay, + "stage_artifact_pins": { + stage.stage: _stage_artifact_pins(stage) for stage in stages + }, + "stage_records": [ + { + "stage": record.stage, + "produced": list(record.produced), + "nonzero_share": dict(record.nonzero_share), + "seconds": record.seconds, + } + for record in records + ], + "operations": { + stage.stage: [operation.kind for operation in stage.operations] + for stage in stages + }, + "declared_seeds": _declared_seeds(stages), + "source_vintages": {"frs": frs_vintage}, + "sampling": sampling, + "spine_gate_report": spine_gate_report, + "stochastic_contract_sha256": stochastic_contract_sha256, + "rules_engine": _rules_engine_provenance(), + } + + +def _nonzero_shares(frame, columns: list[str]) -> dict[str, float]: + shares: dict[str, float] = {} + for column in columns: + for entity in frame.entities: + table = frame.table(entity) + if column not in table.columns: + continue + values = table[column] + if values.dtype == object: + shares[column] = float(values.astype(str).ne("").mean()) + else: + shares[column] = float((values != 0).mean()) + break + return shares + + +def _series_nonzero_share(values) -> float: + if values.dtype == object or str(values.dtype).startswith("string"): + return float(values.fillna("").astype(str).ne("").mean()) + return float((values != 0).mean()) + + +def _graph_stage_records( + *, + manifest, + store: ContentStore, + stages, + frame, +) -> tuple[StageRecord, ...]: + """Project immediate node artifacts onto the legacy record schema. + + Entity ids and memberships are executor-carried context, not owned cells, + so the root node exposes no artifact for them although ``frs_spine`` + declares them as outputs. Their share is read from the final population + instead, which is what the legacy plan recorded (identity columns are + never zero, so the value is 1.0 on every vintage). + """ + + structural = _structural_columns(frame) + records: list[StageRecord] = [] + for stage in stages: + output_node = ( + f"{stage.stage}.owned" + if f"{stage.stage}.owned" in manifest.nodes + else stage.stage + ) + output_receipt = manifest.nodes[output_node] + shares: dict[str, float] = {} + for column in stage.outputs: + matches = [ + (coordinate, key) + for coordinate, key in output_receipt.artifacts.items() + if coordinate[1] == column + ] + if not matches and column in structural: + shares[column] = _nonzero_shares(frame, [column])[column] + continue + if len(matches) != 1: + raise RuntimeError( + f"graph stage {stage.stage!r} exposes {len(matches)} artifacts " + f"for declared output {column!r}." + ) + shares[column] = _series_nonzero_share(store.load_column(matches[0][1])) + execution_node = "create_uk_frs" if stage.stage == "frs_spine" else stage.stage + records.append( + StageRecord( + stage=stage.stage, + produced=stage.outputs, + donor_survey=stage.survey, + nonzero_share=shares, + seconds=manifest.nodes[execution_node].wall_time, + ) + ) + return tuple(records) + + +def _structural_columns(frame) -> frozenset[str]: + """Entity id and membership columns the executor carries outside owned cells.""" + + schema = frame.schema + columns = {schema.entity_id_column(entity) for entity in frame.entities} + columns.update(schema.membership_column(group) for group in schema.group_entities) + return frozenset(columns) + + +def _new_build_id(timestamp: datetime) -> str: + return f"uk-frs-spine-{timestamp.strftime('%Y%m%dT%H%M%SZ')}" + + +def _record_attempt( + *, + state: AttemptState, + started_at: float, + started_ts: datetime, + code_pin: str, + disposition: str, + predecessor: str | None, + rung: str, + spool_dir: Path, +) -> Path: + return record_terminal_attempt( + state=state, + started_at=started_at, + started_ts=started_ts, + pipeline=_PIPELINE, + rung=rung, + seed=None, + code_pin=code_pin, + disposition=disposition, + predecessor=predecessor, + spool_dir=spool_dir, + ) + + +def _sample_spine_frame( + frame, + *, + fraction: float, + seed: int, +) -> tuple[object, dict[str, object] | None]: + if fraction == 1.0: + return frame, None + household_weight = frame.weights_for("household") + pre_households = int(len(frame.table("household"))) + sampled, receipt = sample_frame_households( + frame, + fraction=fraction, + seed=seed, + source_name="UK FRS spine", + ) + normalized, factor = normalize_sampled_household_mass( + sampled, + target_mass=float(household_weight.total), + source_name="UK FRS spine", + ) + return normalized, { + "fraction": float(fraction), + "seed": int(seed), + "rung_token": UK_SAMPLE_RUNG_TOKENS[fraction], + "pre_household_count": pre_households, + "post_household_count": int(len(normalized.table("household"))), + "normalization_factor": float(factor), + "receipt": dict(receipt), + } + + +class _SampledGraphRootTransform: + """CREATE-stage adapter applying the declared sampling rung at ingest.""" + + def __init__(self, transform, *, fraction: float, seed: int) -> None: + self.transform = transform + self.fraction = fraction + self.seed = seed + self.sampling: dict[str, object] | None = None + + def _sample(self, assembled): + sampled, self.sampling = _sample_spine_frame( + assembled, + fraction=self.fraction, + seed=self.seed, + ) + return sampled + + def __call__(self, frame): + return self._sample(self.transform(frame)) + + def run_with_sources(self, frame, sources): + runner = getattr(self.transform, "run_with_sources", None) + assembled = ( + runner(frame, sources) if callable(runner) else self.transform(frame) + ) + return self._sample(assembled) + + def checkpoint_metadata(self) -> dict[str, object]: + hook = getattr(self.transform, "checkpoint_metadata", None) + if not callable(hook): + raise RuntimeError("FRS root transform exposes no checkpoint metadata.") + return dict(hook()) + + def graph_implementation_dependencies(self): + from microcosm.build import frame_sampling + + return (type(self), _sample_spine_frame, frame_sampling) + + +class _GraphSourceTransform: + """Build a file-reading stage from only the node's declared source paths.""" + + def __init__(self, factory) -> None: + self.factory = factory + self.transform = None + + def run_with_sources(self, frame, sources): + self.transform = self.factory(sources) + result = self.transform(frame) + if hasattr(self.transform, "fit_weight_records"): + self.fit_weight_records = self.transform.fit_weight_records + return result + + def __getattr__(self, name: str): + transform = self.__dict__.get("transform") + if transform is None: + raise AttributeError(name) + return getattr(transform, name) + + +def _run_plan_with_spine_sampling( + plan, + *, + sample_fraction: float, + sample_seed: int, + spine_battery: GateBatteryRun | None = None, + stage_evidence_provider=None, + gate_artifacts: Mapping[str, object] | None = None, +) -> tuple[object, tuple[object, ...], dict[str, object] | None]: + if not plan.stages or plan.stages[0].name != "frs_spine": + frame, records = plan.run(uk_frs_spine_seed_frame()) + return frame, records, None + + from microcosm.build.plan import StagePlan + + spine_frame, spine_records = StagePlan(plan.stages[:1]).run( + uk_frs_spine_seed_frame() + ) + spine_frame, sampling = _sample_spine_frame( + spine_frame, + fraction=sample_fraction, + seed=sample_seed, + ) + if len(plan.stages) == 1: + return spine_frame, spine_records, sampling + names = tuple(stage.name for stage in plan.stages) + if UK_SPINE_ASSEMBLED_FINAL_STAGE in names: + assembled_end = names.index(UK_SPINE_ASSEMBLED_FINAL_STAGE) + 1 + elif spine_battery is not None: + raise RuntimeError( + "spine battery is armed but the declared assembled-boundary stage " + f"{UK_SPINE_ASSEMBLED_FINAL_STAGE!r} is not in the plan; a stage " + "plan change must move the boundary declaration with it." + ) + else: + assembled_end = len(plan.stages) + frame, assembled_records = StagePlan(plan.stages[1:assembled_end]).run(spine_frame) + # Each boundary offers only the stages that have actually run: asking a + # later stage for checkpoint evidence would (correctly) raise, and the + # first licensed battery run did exactly that at the assembled boundary. + executed = tuple(stage.name for stage in plan.stages[:assembled_end]) + if spine_battery is not None: + _run_spine_gate_phase( + spine_battery, + "assembled", + frame=frame, + stage_evidence=( + stage_evidence_provider(executed) + if stage_evidence_provider is not None + else {} + ), + gate_artifacts=gate_artifacts, + ) + if assembled_end == len(plan.stages): + return frame, (*spine_records, *assembled_records), sampling + frame, tail_records = StagePlan(plan.stages[assembled_end:]).run(frame) + executed = tuple(stage.name for stage in plan.stages) + if spine_battery is not None: + _run_spine_gate_phase( + spine_battery, + "transferred", + frame=frame, + stage_evidence=( + stage_evidence_provider(executed) + if stage_evidence_provider is not None + else {} + ), + gate_artifacts=gate_artifacts, + ) + return frame, (*spine_records, *assembled_records, *tail_records), sampling + + +def _run_spine_gate_phase( + battery: GateBatteryRun, + phase: str, + *, + frame, + stage_evidence: Mapping[str, object], + gate_artifacts: Mapping[str, object] | None = None, +) -> None: + artifacts: dict[str, object] = {"stage_evidence": dict(stage_evidence)} + # The enum-domain gate resolves its domain from the live rules engine, + # exactly as the national terminal battery supplied it. + artifacts.update(dict(gate_artifacts or {})) + battery.run_phase( + phase, + EvidenceContext(frame=frame, artifacts=artifacts), + ) + battery.enforce(phase, mode=BlockingMode.BLOCKS_ARTIFACT) + + +def _spine_gate_report_path(spine_h5: Path) -> Path: + return spine_h5.with_suffix(".spine_gates.json") + + +def _spine_gate_manifest_from_spec(spec) -> GatesManifest | None: + """The spine build's scoped battery manifest, from the shared helper. + + A spec without a gates block leaves the battery unarmed (``None``), + exactly as before; when armed, the filtering runs through the one + scope-filtering implementation every scoped producer shares. The + driver passes the spec it already loaded, which is also the hermetic + tests' stub point. Digests are identical to the previous local copy + because entries, phases, and the policy suffix are unchanged. + """ + + source = getattr(spec, "gates", None) + if source is None: + return None + return uk_scoped_gate_manifest( + UK_SPINE_GATE_SCOPE, + phases=("assembled", "transferred"), + policy_suffix="spine_build_scope", + source=source, + ) + + +def _rung_abort_receipt( + args: argparse.Namespace, + *, + error: BaseException, +) -> dict[str, object]: + return { + "schema_version": 1, + "artifact_kind": "uk_frs_spine_rung_abort_receipt", + "build_kind": "uk_frs_spine", + "sampling": { + "sample_fraction": float(args.sample_fraction), + "sample_seed": int(args.sample_seed), + "rung_token": UK_SAMPLE_RUNG_TOKENS[args.sample_fraction], + }, + "named_edge": "spine_split_singleton_class", + "stage": "frs_spine", + "error": str(error), + "disposition": "aborted_with_receipt", + "remedy": ( + "Re-roll --sample-seed; accepted dev-scale statistical edge. " + "The computation is never altered to avoid it." + ), + } + + +def _exception_chain_contains(error: BaseException, text: str) -> bool: + """Match a named rung edge through graph execution wrappers.""" + + seen: set[int] = set() + current: BaseException | None = error + while current is not None and id(current) not in seen: + seen.add(id(current)) + if text in str(current): + return True + current = current.__cause__ or current.__context__ + return False + + +@dataclass(frozen=True) +class PreparedUKSpineExecution: + """Declared raw-source graph and bindings, prepared without numerical execution.""" + + graph: object + kernels: object + sources: Mapping[str, Path] + spec: object + stage_names: tuple[str, ...] + stages: tuple[object, ...] + engine: object + engine_identity: str + stochastic_contract: object + frs_release: object + + +def parse_uk_spine_args(argv: list[str] | None = None) -> argparse.Namespace: + """Parse the maintained raw-source request independently of execution.""" + return _parse_args(argv) + + +def prepare_uk_spine_execution(args: argparse.Namespace) -> PreparedUKSpineExecution: + """Bind source paths and lazy transforms without fitting models or writing files.""" + _validate_args(args) + spec = load_country_spec("uk") + if spec.sources is None: + raise ValueError("UK country spec has no source stages.") + stages_by_name = spec.sources.stage_map() + graph = uk_spine_graph( + spec, + source_mode="split", + sample_fraction=args.sample_fraction, + sample_seed=args.sample_seed, + ) + stage_names = _uk_spine_stage_names(spec) + if "hmrc_cgt_gains_spine" in stage_names and args.cgt_ods is None: + raise ValueError( + "--cgt-ods is required when hmrc_cgt_gains_spine is scheduled." + ) + if "was_wealth" in stage_names and args.was_tab is None: + raise ValueError( + "--was-tab is required when the was_wealth stage is scheduled." + ) + if "lcfs_consumption" in stage_names: + missing_lcfs = [ + flag + for flag, value in ( + ("--lcfs-hh-tab", args.lcfs_hh_tab), + ("--lcfs-person-tab", args.lcfs_person_tab), + ("--was-tab", args.was_tab), + ) + if value is None + ] + if missing_lcfs: + raise ValueError( + "lcfs_consumption requires caller-supplied private inputs: " + f"{', '.join(missing_lcfs)}." + ) + if ( + "etb_vat" in stage_names or "etb_services" in stage_names + ) and args.etb_tab is None: + raise ValueError( + "--etb-tab is required when etb_vat or etb_services is scheduled." + ) + engine = _rules_engine() + stochastic_contract = load_uk_take_up_contract() + frs_release = load_uk_frs_release() + hmrc_spine_transform = _GraphSourceTransform( + lambda sources: UKSPIIncomeSpineStageTransform( + sources["spi"], + sources["hmrc_income"], + stage=stages_by_name["hmrc_spi_income_spine"], + sampled_rung=args.sample_fraction != 1.0, + ) + ) + implementations = { + "frs_spine": _GraphSourceTransform( + lambda sources: UKFRSSpineStageTransform( + sources["frs"], + stage=stages_by_name["frs_spine"], + ) + ), + "frs_employment": _GraphSourceTransform( + lambda sources: UKFRSEmploymentStageTransform( + sources["frs"], + stage=stages_by_name["frs_employment"], + ) + ), + "frs_council_tax": _GraphSourceTransform( + lambda sources: UKFRSCouncilTaxStageTransform( + sources["frs"], + stage=stages_by_name["frs_council_tax"], + ) + ), + "frs_disability": UKFRSDisabilityStageTransform( + stage=stages_by_name["frs_disability"], + ), + "frs_education": _GraphSourceTransform( + lambda sources: UKFRSEducationStageTransform( + sources["frs"], + stage=stages_by_name["frs_education"], + ) + ), + "frs_legacy_proxies": _GraphSourceTransform( + lambda sources: UKFRSLegacyProxiesStageTransform( + sources["frs"], + stage=stages_by_name["frs_legacy_proxies"], + engine=engine, + ) + ), + "frs_education_grant_split": ( + UKFRSEducationGrantSplitStageTransform( + stage=stages_by_name["frs_education_grant_split"], + engine=engine, + ) + ), + "frs_take_up": UKFRSTakeUpStageTransform( + contract=stochastic_contract, + stage=stages_by_name["frs_take_up"], + ), + "frs_person_draws": UKFRSPersonDrawsStageTransform( + contract=stochastic_contract, + stage=stages_by_name["frs_person_draws"], + ), + "frs_household_draws": UKFRSHouseholdDrawsStageTransform( + contract=stochastic_contract, + stage=stages_by_name["frs_household_draws"], + ), + "frs_brma": UKFRSBRMAStageTransform( + stage=stages_by_name["frs_brma"], + engine=engine, + ), + } + if "was_wealth" in stage_names: + implementations["was_wealth"] = _GraphSourceTransform( + lambda sources: UKWASWealthStageTransform( + stage=stages_by_name["was_wealth"], + engine=engine, + was_tab_path=sources["was"], + ) + ) + if "regional_property_uprating" in stage_names: + implementations["regional_property_uprating"] = ( + UKRegionalPropertyUpratingStageTransform( + stage=stages_by_name["regional_property_uprating"], + ) + ) + if "lcfs_consumption" in stage_names: + implementations["lcfs_consumption"] = _GraphSourceTransform( + lambda sources: UKLCFSConsumptionStageTransform( + stage=stages_by_name["lcfs_consumption"], + engine=engine, + lcfs_hh_tab_path=sources["lcfs_household"], + lcfs_person_tab_path=sources["lcfs_person"], + was_tab_path=sources["was"], + ) + ) + if "etb_vat" in stage_names: + implementations["etb_vat"] = _GraphSourceTransform( + lambda sources: UKETBVATStageTransform( + stage=stages_by_name["etb_vat"], + engine=engine, + etb_tab_path=sources["etb"], + ) + ) + if "etb_services" in stage_names: + implementations["etb_services"] = _GraphSourceTransform( + lambda sources: UKETBServicesStageTransform( + stage=stages_by_name["etb_services"], + engine=engine, + etb_tab_path=sources["etb"], + ) + ) + implementations["frs_hmrc_spine_leaves"] = _GraphSourceTransform( + lambda sources: UKFRSHMRCSpineLeavesStageTransform( + sources["frs"], + stage=stages_by_name["frs_hmrc_spine_leaves"], + sampled_rung=args.sample_fraction != 1.0, + ) + ) + implementations["spi_support_channel"] = UKSPISupportChannelStageTransform( + stage=stages_by_name["spi_support_channel"], + sample_fraction=args.sample_fraction, + ) + implementations["hmrc_spi_income_spine"] = hmrc_spine_transform + if "uc_reporter_redraw" in stage_names: + implementations["uc_reporter_redraw"] = UKUCReporterRedrawStageTransform( + stage=stages_by_name["uc_reporter_redraw"], + engine=engine, + ) + if "uc_capital_coherence" in stage_names: + implementations["uc_capital_coherence"] = UKUCCapitalCoherenceStageTransform( + stage=stages_by_name["uc_capital_coherence"] + ) + if "uc_deduction_attributes" in stage_names: + implementations["uc_deduction_attributes"] = ( + UKUCDeductionAttributesStageTransform( + stage=stages_by_name["uc_deduction_attributes"] + ) + ) + if "cgt_incidence_clone" in stage_names: + implementations["cgt_incidence_clone"] = UKCGTIncidenceCloneStageTransform( + stage=stages_by_name["cgt_incidence_clone"] + ) + if "cgt_band_donors" in stage_names: + implementations["cgt_band_donors"] = UKCGTBandDonorStageTransform( + stage=stages_by_name["cgt_band_donors"] + ) + if "hmrc_cgt_gains_spine" in stage_names: + implementations["hmrc_cgt_gains_spine"] = _GraphSourceTransform( + lambda sources: uk_cgt_spine_stage_transform( + stages_by_name["hmrc_cgt_gains_spine"], + sources["hmrc_cgt"], + ) + ) + if "salary_sacrifice" in stage_names: + implementations["salary_sacrifice"] = UKSalarySacrificeStageTransform( + stage=stages_by_name["salary_sacrifice"] + ) + if "student_loans" in stage_names: + implementations["student_loans"] = UKStudentLoansStageTransform( + stage=stages_by_name["student_loans"], + calibration_year=frs_release.calibration_year, + ) + if "age_tail" in stage_names: + implementations["age_tail"] = UKAgeTailStageTransform( + stage=stages_by_name["age_tail"] + ) + sampled_root = _SampledGraphRootTransform( + implementations["frs_spine"], + fraction=args.sample_fraction, + seed=args.sample_seed, + ) + implementations["frs_spine"] = sampled_root + graph_sources = {"frs": args.frs_raw_dir} + if "was_wealth" in stage_names or "lcfs_consumption" in stage_names: + graph_sources["was"] = args.was_tab + if "lcfs_consumption" in stage_names: + graph_sources["lcfs_household"] = args.lcfs_hh_tab + graph_sources["lcfs_person"] = args.lcfs_person_tab + if "etb_vat" in stage_names or "etb_services" in stage_names: + graph_sources["etb"] = args.etb_tab + if "hmrc_spi_income_spine" in stage_names: + graph_sources["spi"] = args.spi_tab + graph_sources["hmrc_income"] = args.hmrc_ods + if "hmrc_cgt_gains_spine" in stage_names: + graph_sources["hmrc_cgt"] = args.cgt_ods + engine_identity = hashlib.sha256( + canonical_json_bytes(_rules_engine_provenance()) + ).hexdigest() + graph = add_uk_spine_gate_nodes( + graph, + spec=spec, + engine_identity=engine_identity, + release_candidate=args.release_candidate, + ) + kernels = uk_registry(implementations, graph=graph) + register_spine_gate_kernel( + kernels, spec=spec, engine=engine, engine_identity=engine_identity + ) + return PreparedUKSpineExecution( + graph=graph, + kernels=kernels, + sources=graph_sources, + spec=spec, + stage_names=stage_names, + stages=tuple(stages_by_name[name] for name in stage_names), + engine=engine, + engine_identity=engine_identity, + stochastic_contract=stochastic_contract, + frs_release=frs_release, + ) + + +def main(argv: list[str] | None = None) -> int: + args = _parse_args(argv) + rung = UK_SAMPLE_RUNG_TOKENS[args.sample_fraction] + started_at = time.perf_counter() + started_ts = datetime.now(UTC) + predecessor = resolve_predecessor(args.logbook_prev_row_digest) + digest = preflight_digest(_PIPELINE) + state = AttemptState( + build_id=_new_build_id(started_ts), + identity_digest=digest, + input_pins_digest=digest, + phases_reached=["attempt_started"], + gate_verdicts={ + "pipeline": { + "verdict": "running", + "receipt": "pending-build-scoped-spine-receipt", + } + }, + ) + code_pin = "unresolved-local-git-code-pin" + spool_dir = args.spine_h5.parent / "logbook-spool" + try: + _validate_args(args) + # A crash between the H5 write and the sidecar writes must never + # leave a stale sidecar beside a fresh H5 (adversarial-review + # finding on #717): clear every output up front, and treat the + # build sidecar - written last, binding the replay hash - as the + # marker that the bundle is complete. + stale_outputs = [ + args.spine_h5, + args.spine_h5.with_suffix(".build.json"), + args.spine_h5.with_suffix(".hmrc_replay.json"), + _spine_gate_report_path(args.spine_h5), + args.spine_h5.with_suffix(".rung_abort.json"), + ] + if args.emit_nonzero_shares is not None: + stale_outputs.append(args.emit_nonzero_shares) + for stale in stale_outputs: + stale.unlink(missing_ok=True) + code_pin = git_code_pin(_REPOSITORY) + append_phase(state, "configured") + prepared = prepare_uk_spine_execution(args) + spec = prepared.spec + graph = prepared.graph + stage_names = prepared.stage_names + stages_by_name = spec.sources.stage_map() + stages = [stages_by_name[name] for name in stage_names] + artifact_pins = _artifact_pins(stages) + resource_pins = _resource_pins(stages, spec) + input_artifact_pins = _input_artifact_pins(stages) + overlapping_pin_roles = set(artifact_pins) & set(input_artifact_pins) + if overlapping_pin_roles: + raise ValueError( + "input artifact roles collide with FRS tab names: " + f"{sorted(overlapping_pin_roles)}." + ) + state.input_pins_digest = role_pins_digest( + _role_pins({**artifact_pins, **input_artifact_pins}) + ) + run_config = { + "pipeline": _PIPELINE, + "stages": list(stage_names), + "artifact_pins_digest": state.input_pins_digest, + "spine_h5": str(args.spine_h5), + } + state.identity_digest = hashlib.sha256( + canonical_json_bytes(run_config) + ).hexdigest() + append_phase(state, "inputs_pinned") + stochastic_contract = prepared.stochastic_contract + frs_release = prepared.frs_release + spine_gate_path = _spine_gate_report_path(args.spine_h5) + spine_gate_manifest = _spine_gate_manifest_from_spec(spec) + spine_battery = ( + GateBatteryRun( + spine_gate_manifest, + release_id=state.build_id, + report_path=spine_gate_path, + release_candidate=args.release_candidate, + registry=UK_GATE_REGISTRY, + ) + if spine_gate_manifest is not None + else None + ) + checkpoint_root = ( + args.checkpoint_dir + if args.checkpoint_dir is not None + else args.spine_h5.parent / f".{args.spine_h5.stem}.checkpoints" + ) + graph_sources = prepared.sources + kernels = prepared.kernels + compiled_graph = compile_graph(graph) + graph_store = ContentStore(checkpoint_root / "node-graph") + graph_manifest = run_graph( + compiled_graph, + sources=graph_sources, + store=graph_store, + kernels=kernels, + resume="auto", + decisions=(), + ) + graph_manifest.save(checkpoint_root / "spine.graph.json") + final_version = compiled_graph.versions[stage_names[-1]] + frame = graph_manifest.population(final_version) + records = _graph_stage_records( + manifest=graph_manifest, + store=graph_store, + stages=stages, + frame=frame, + ) + stored_stages = load_spine_stage_artifacts( + graph_manifest, graph_store, stage_names=stage_names + ) + stored_evidence = spine_sidecar_evidence(stored_stages) + sampling = stored_evidence["sampling"] + if spine_battery is not None: + materialize_spine_gate_reports( + graph_manifest, + graph_store, + battery=spine_battery, + gates=spine_gate_manifest, + ) + if spine_battery is not None: + append_phase(state, "spine_gates_evaluated") + append_phase(state, "spine_built") + output = write_uk_national_frame(frame, args.spine_h5) + append_phase(state, "spine_written") + if args.checkpoint_dir is not None: + args.checkpoint_dir.mkdir(parents=True, exist_ok=True) + write_uk_national_frame(frame, args.checkpoint_dir / "frs_spine.h5") + append_phase(state, "checkpoint_written") + sidecar_path = output.with_suffix(".build.json") + replay_sidecar_path = output.with_suffix(".hmrc_replay.json") + replay_metadata = stored_stages["hmrc_spi_income_spine"]["checkpoint_metadata"] + if ( + not isinstance(replay_metadata, dict) + or "replay_payload" not in replay_metadata + ): + raise RuntimeError("HMRC SPI spine stage did not record replay evidence.") + atomic_write_json(replay_sidecar_path, replay_metadata["replay_payload"]) + append_phase(state, "hmrc_replay_sidecar_written") + replay_bytes = replay_sidecar_path.read_bytes() + replay_binding = { + "filename": replay_sidecar_path.name, + "report_kind": str(json.loads(replay_bytes).get("report_kind", "")), + "sha256": hashlib.sha256(replay_bytes).hexdigest(), + } + sidecar = _build_sidecar( + frame=frame, + stages=stages, + records=records, + artifact_pins=artifact_pins, + resource_pins=resource_pins, + input_artifact_pins=input_artifact_pins, + hmrc_replay=replay_binding, + stochastic_contract_sha256=stochastic_contract.resource_sha256, + frs_vintage=frs_release.vintage, + sampling=sampling, + spine_gate_report=( + { + "path": str(spine_gate_path), + "sha256": hashlib.sha256(spine_gate_path.read_bytes()).hexdigest(), + } + if spine_gate_path.is_file() + else None + ), + ) + sidecar["operation_inventory"] = list(uk_spine_operation_inventory(graph, spec)) + sidecar["graph_manifest"] = { + "path": str(checkpoint_root / "spine.graph.json"), + "key": graph_manifest.key, + } + stage_evidence = stored_evidence["stage_evidence"] + if stage_evidence: + sidecar["stage_evidence"] = stage_evidence + fit_weight_records = stored_evidence["fit_weight_records"] + if fit_weight_records: + sidecar["fit_weight_records"] = fit_weight_records + atomic_write_json(sidecar_path, sidecar) + append_phase(state, "build_sidecar_written") + if args.emit_nonzero_shares is not None: + final_columns = list( + dict.fromkeys( + [column for record in records for column in record.produced] + + list(FRS_EDUCATION_GRANT_REWRITES) + ) + ) + atomic_write_json( + args.emit_nonzero_shares, + { + "stages": { + record.stage: dict(record.nonzero_share) for record in records + }, + "final": _nonzero_shares(frame, final_columns), + }, + ) + append_phase(state, "nonzero_shares_written") + state.artifact_location = local_artifact_reference( + output, + repository_hint=_REPOSITORY, + ) + state.gate_verdicts = { + "pipeline": { + "verdict": "passed", + "receipt": local_artifact_reference( + sidecar_path, repository_hint=_REPOSITORY + ), + } + } + if spine_gate_path.is_file(): + gate_payload = json.loads(spine_gate_path.read_text(encoding="utf-8")) + for gate_id, payload in gate_payload.get("gates", {}).items(): + state.gate_verdicts[str(gate_id)] = { + "verdict": str(payload.get("status")), + "receipt": ( + f"{local_artifact_reference(spine_gate_path, repository_hint=_REPOSITORY)}" + f"#/gates/{gate_id}" + ), + } + spool_path = _record_attempt( + state=state, + started_at=started_at, + started_ts=started_ts, + code_pin=code_pin, + disposition="iterating", + predecessor=predecessor, + rung=rung, + spool_dir=spool_dir, + ) + print(f"Wrote FRS spine H5: {output}", file=sys.stderr) + print(f"Wrote Logbook row: {spool_path}", file=sys.stderr) + return 0 + except Exception as error: + if args.sample_fraction != 1.0 and _exception_chain_contains( + error, _RUNG_NAMED_EDGE_SIGNATURE + ): + rung_abort_path = args.spine_h5.with_suffix(".rung_abort.json") + receipt = _rung_abort_receipt(args, error=error) + atomic_write_json(rung_abort_path, receipt) + state.gate_verdicts = { + "uk_frs_spine_rung_abort": { + "verdict": "aborted", + "receipt": ( + f"{local_artifact_reference(rung_abort_path, repository_hint=_REPOSITORY)}" + "#/named_edge" + ), + } + } + append_phase(state, "rung_aborted") + _record_attempt( + state=state, + started_at=started_at, + started_ts=started_ts, + code_pin=code_pin, + disposition="discarded", + predecessor=predecessor, + rung=rung, + spool_dir=spool_dir, + ) + print(json.dumps(receipt, indent=2, sort_keys=True)) + return _RUNG_ABORT_EXIT_CODE + try: + receipt_path = write_error_receipt( + error_receipt_path(args.spine_h5.parent, build_id=state.build_id), + state=state, + pipeline=_PIPELINE, + error=error, + ) + apply_error_verdict( + state, + local_artifact_reference(receipt_path, repository_hint=_REPOSITORY), + ) + _record_attempt( + state=state, + started_at=started_at, + started_ts=started_ts, + code_pin=code_pin, + disposition="failed", + predecessor=predecessor, + rung=rung, + spool_dir=spool_dir, + ) + except Exception: + pass + print(f"UK FRS spine build failed: {error}", file=sys.stderr) + return 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/packages/microcosm-build/tests/test_country_spec.py b/packages/microcosm-build/tests/test_country_spec.py index 84eb4e1b1..55695a8f3 100644 --- a/packages/microcosm-build/tests/test_country_spec.py +++ b/packages/microcosm-build/tests/test_country_spec.py @@ -919,7 +919,6 @@ def test_spi_spine_adds_no_country_package_resources(self) -> None: "cgt_band_donor_support_bounds.json", "hmrc_income_release_gate_report.json", "hmrc_income_replay_report.json", - "hmrc_income_source_stages.json", "need_energy_targets.json", "lcfs_consumption_anchors.json", "etb_policy_anchors.json", @@ -1013,7 +1012,6 @@ def test_uk_package_loads(self) -> None: "cgt_band_donor_support_bounds.json", "hmrc_income_release_gate_report.json", "hmrc_income_replay_report.json", - "hmrc_income_source_stages.json", "need_energy_targets.json", "lcfs_consumption_anchors.json", "etb_policy_anchors.json", diff --git a/packages/microcosm-build/tests/test_gate_battery_contract_pins.py b/packages/microcosm-build/tests/test_gate_battery_contract_pins.py index cdf822c1b..508a09ac0 100644 --- a/packages/microcosm-build/tests/test_gate_battery_contract_pins.py +++ b/packages/microcosm-build/tests/test_gate_battery_contract_pins.py @@ -400,25 +400,15 @@ def test_entry_ids_mirror_the_local_battery_scope(self) -> None: ) def test_scoped_digests_mirror_the_live_local_manifest(self) -> None: - import importlib.util - from pathlib import Path - from microcosm.build.uk_runtime.calibration_run import UK_LOCAL_GATE_SCOPE from microcosm.build.uk_runtime.release_certification import _scoped_digests - spec = importlib.util.spec_from_file_location( - "build_uk_rowwise_candidate", - Path(__file__).resolve().parents[3] - / "tools" - / "build_uk_rowwise_candidate.py", - ) - builder = importlib.util.module_from_spec(spec) - assert spec.loader is not None - spec.loader.exec_module(builder) + # Archived dense certificates retain their original policy identity; + # the retired candidate command no longer owns executable gate policy. live = _scoped_digests( frozenset(UK_LOCAL_GATE_SCOPE), phases=tuple(data_contract._UK_DENSE_GATE_PHASES), - policy_suffix=str(builder._LOCAL_GATE_POLICY_SUFFIX), + policy_suffix="local_candidate", ) for field, mirrored in data_contract._UK_DENSE_GATE_DIGESTS.items(): assert mirrored == live[field], field diff --git a/packages/microcosm-build/tests/test_spec_engine_country_bundles.py b/packages/microcosm-build/tests/test_spec_engine_country_bundles.py index 1169547c3..70133dfea 100644 --- a/packages/microcosm-build/tests/test_spec_engine_country_bundles.py +++ b/packages/microcosm-build/tests/test_spec_engine_country_bundles.py @@ -25,7 +25,7 @@ "spine", "vintages", } -AM_SPEC_SHA256 = "d983a5e6bb2f91f9abd44669fe5b1c795d1a7597c4b6acbe9282eaa956bb326d" +AM_SPEC_SHA256 = "e2b275861eee20ee8184d030aa22a6d7fad2bc2fd9922588b3ca8566f81c5878" @pytest.mark.parametrize( @@ -45,7 +45,7 @@ ), ( "be", - "2a7d83483c90abe31108f8c3a77c053b4e6f2fd50d3cf3290a85e63ea3b9fbac", + "938e842e4e72f9319b06edfe4055ef2c0190ae942c8dcb801a2effe1fd6f4b98", { "household.household_id", "person.person_id", @@ -55,7 +55,7 @@ ), ( "uk", - "2269bc281ada99dce077730657e3868972e025b48be5f9db8c11b3cc63285bac", + "39dd6f0c15c4d57d4053f8ac74e4d9a6eb4a6fb154a004908ed2d9cbf9cf9a83", { "benunit.benunit_id", "household.household_id", @@ -134,7 +134,7 @@ def test_country_kernel_contract_ids_are_closed_in_the_compiler_registry() -> No "silc_load", "clone_assign_communes", "be_commune_geography_gate", - "load_uk_national_frame", + "build_uk_frs_spine", "assign_uk_geography_ladder", "uk_geography_ladder_gate", } diff --git a/packages/microcosm-build/tests/test_uk_battery_bindings.py b/packages/microcosm-build/tests/test_uk_battery_bindings.py index 465b89ef8..f2c96abf2 100644 --- a/packages/microcosm-build/tests/test_uk_battery_bindings.py +++ b/packages/microcosm-build/tests/test_uk_battery_bindings.py @@ -194,9 +194,9 @@ def _run_battery(tables, *, parity=None, fit_records=None, armed=True, clock=CLO artifacts: dict[str, object] = { "coverage_engine": object(), "exclusions_evaluated_on": clock, - # The staging pipeline's two scheduled stages declare no nonnegative - # outputs, so the nonnegative gate passes with zero required columns. - "build_stage_names": ("frs_hmrc_retained_leaves", "hmrc_spi_income"), + # This binding fixture schedules a stage with no declared nonnegative + # outputs; canonical family completeness has dedicated tests. + "build_stage_names": ("frs_household_draws",), } if fit_records is not None: artifacts["fit_weight_records"] = fit_records @@ -427,18 +427,12 @@ def test_nonnegative_binding_passes_clean_scheduled_columns(self) -> None: assert result.passed is True def test_nonnegative_binding_does_not_demand_unscheduled_stages(self) -> None: - # The national staging build schedules only the two HMRC stages, - # which declare no nonnegative outputs — the gate passes honestly - # with zero required columns rather than by silent pre-filtering. + # This isolated stage declares no nonnegative outputs. Outputs of + # unscheduled employment and income stages must not be demanded. binding = UK_GATE_REGISTRY["nonnegative_columns"] context = EvidenceContext( frame=self._nonnegative_frame(sic=None), - artifacts={ - "build_stage_names": ( - "frs_hmrc_retained_leaves", - "hmrc_spi_income", - ) - }, + artifacts={"build_stage_names": ("frs_household_draws",)}, ) result = binding.evaluate(context, {}) diff --git a/packages/microcosm-build/tests/test_uk_calibration_run.py b/packages/microcosm-build/tests/test_uk_calibration_run.py index 2ca21390d..c9248235f 100644 --- a/packages/microcosm-build/tests/test_uk_calibration_run.py +++ b/packages/microcosm-build/tests/test_uk_calibration_run.py @@ -1,10 +1,8 @@ from __future__ import annotations import hashlib -import hmac import json from pathlib import Path -from types import SimpleNamespace import numpy as np import pandas as pd @@ -16,29 +14,19 @@ PUBLISHED_CONSUMER_ARTIFACT_SCHEMA_VERSION, ) from microcosm.build.country_spec import load_country_spec -from microcosm.build.gate_battery import ( - _canonical_json_bytes as canonical_json_bytes, -) from microcosm.build.ledger_artifact import load_ledger_consumer_artifact from microcosm.build.uk_runtime import calibration_run from microcosm.build.uk_runtime.calibration_run import ( UK_CALIBRATION_GATE_SCOPE, UK_CALIBRATION_GATE_SCOPE_EXCLUSIONS, - UK_SPINE_GATE_SCOPE, - UKCalibrationRunPaths, - run_uk_calibration, ) +from microcosm.build.uk_runtime.content_identity import uk_frame_content_identity from microcosm.build.uk_runtime.etb_services import ( UK_NHS_SPENDING_COMPONENT_COLUMNS, ) -from microcosm.build.uk_runtime.national_doctrine import UKNationalSolveDoctrine from microcosm.build.uk_runtime.national_frame import ( - load_uk_national_frame, - uk_household_weight_kind, uk_national_frame, - write_uk_national_frame, ) -from microcosm.calibrate import TargetRegistry, TargetSpec from microcosm.frame import WeightKind SIGNING_KEY = "MDEyMzQ1Njc4OWFiY2RlZjAxMjM0NTY3ODlhYmNkZWY=" @@ -79,108 +67,101 @@ def _frame(): ) -def _registry(): - return TargetRegistry( - [ - TargetSpec( - name="dwp.uc.households", - entity="benunit", - measure="dwp/uc/households", - value=20.0, - source="test", - family="dwp_universal_credit", - metadata={"contract_target_id": "dwp.uc.households"}, - ) - ], - country="uk", - ) - - -def _paths(tmp_path: Path) -> UKCalibrationRunPaths: - return UKCalibrationRunPaths( - input_h5=tmp_path / "input.h5", - staging_h5=tmp_path / "staged.h5", - diagnostics_json=tmp_path / "diagnostics.json", - build_record_json=tmp_path / "build_record.json", - terminal_gate_json=tmp_path / "terminal_gates.json", - ) - - -def _sha(path: Path) -> str: - return hashlib.sha256(path.read_bytes()).hexdigest() - - -def _write_spine_sidecar( - input_h5: Path, - frame=None, - **overrides, -) -> dict[str, object]: - frame = _frame() if frame is None else frame - sidecar = { - "schema_version": 2, - "pipeline": "uk-frs-spine", - "stages": ["frs_spine", "was_wealth"], - "stage_records": [ - { - "stage": "was_wealth", - "produced": ["property_wealth"], - "nonzero_share": {"property_wealth": 1.0}, - "seconds": 0.1, - } - ], - "stage_evidence": { - "was_wealth": { - "stage": "was_wealth", - "support_clip": {"columns": {}}, - } +def _bound_checkpoint(tmp_path, frame): + report_path = tmp_path / "spine.spine_gates.json" + report = { + **calibration_run.uk_spine_checkpoint_gate_digests(), + "blocked_at_phase": None, + "gates": { + entry.id: {"status": "passed", "criticality": entry.criticality} + for entry in load_country_spec("uk").gates.gates + if entry.id in calibration_run.UK_SPINE_GATE_SCOPE }, - "artifact_pins": {"person": "a" * 64}, - "input_artifact_pins": {"was_qrf_donor": {"sha256": "b" * 64}}, - "resource_pins": {"wealth.json": "c" * 64}, - "stage_artifact_pins": {"was_wealth": {"was_qrf_donor": "d" * 64}}, - "declared_seeds": {"was_wealth": {"was_wealth": 0}}, - "rules_engine": {"package": "policyengine-uk", "version": "unavailable"}, - "source_vintages": {"frs": "2024_25"}, - "stochastic_contract_sha256": "e" * 64, + } + report_path.write_text(json.dumps(report)) + sidecar = { "entity_row_counts": { - entity: int(len(frame.table(entity))) for entity in frame.entities + entity: len(frame.table(entity)) for entity in frame.entities }, - "household_weight_kind": uk_household_weight_kind(frame).value, + "household_weight_kind": frame.weights_for("household").kind.value, "household_weight_total": float(frame.weights_for("household").values.sum()), + "uk_frame_content_identity": uk_frame_content_identity(frame), + "spine_gate_report": { + "sha256": hashlib.sha256(report_path.read_bytes()).hexdigest() + }, + "fit_weight_records": {"model": {"fit_weights_used": True}}, } - sidecar.update(overrides) - input_h5.with_suffix(".build.json").write_text( - json.dumps(sidecar, indent=2, sort_keys=True) + "\n", - encoding="utf-8", - ) - input_h5.with_suffix(".spine_gates.json").write_text( - json.dumps( - { - "blocked_at_phase": None, - "gates": { - gate_id: { - "criticality": "release_blocking", - "status": "passed", - } - for gate_id in UK_SPINE_GATE_SCOPE - }, - }, - indent=2, - sort_keys=True, - ) - + "\n", - encoding="utf-8", + sidecar_path = tmp_path / "spine.build.json" + sidecar_path.write_text(json.dumps(sidecar)) + return sidecar_path, report_path, sidecar + + +def test_strict_checkpoint_binds_contents_and_retains_gate_payload(tmp_path): + frame = _frame() + path, gate_path, _ = _bound_checkpoint(tmp_path, frame) + sidecar = calibration_run.load_bound_spine_checkpoint(path, frame) + provenance = calibration_run.strict_spine_provenance_from_sidecar(path, sidecar) + assert provenance["fit_weight_records"] == sidecar["fit_weight_records"] + assert provenance["spine_gate_report"]["payload"] == json.loads( + gate_path.read_bytes() ) - return sidecar -def _admin_anchor_values(): - values = {} - for entry in load_country_spec("uk").gates.gates: - if entry.id == "uk_aggregate_admin": - for anchor in entry.parameters["anchors"]: - values[str(anchor["name"])] = float(anchor["value"]) - return values +@pytest.mark.parametrize( + "mutation", + [ + "missing_identity", + "wrong_identity", + "bypass", + "gate_bytes", + "missing_gate_binding", + "gate_roster", + "gate_policy", + ], +) +def test_strict_checkpoint_rejects_unbound_or_changed_evidence(tmp_path, mutation): + frame = _frame() + path, gate_path, sidecar = _bound_checkpoint(tmp_path, frame) + if mutation == "missing_identity": + sidecar.pop("uk_frame_content_identity") + elif mutation == "wrong_identity": + sidecar["uk_frame_content_identity"] = "f" * 64 + elif mutation == "bypass": + sidecar["spine_gate_bypass"] = {"reviewed": True, "reason": "historical"} + elif mutation == "gate_bytes": + gate_path.write_text(gate_path.read_text() + "\n") + elif mutation == "missing_gate_binding": + sidecar.pop("spine_gate_report") + elif mutation == "gate_policy": + report = json.loads(gate_path.read_bytes()) + report["policy_sha256"] = "f" * 64 + gate_path.write_text(json.dumps(report)) + sidecar["spine_gate_report"]["sha256"] = hashlib.sha256( + gate_path.read_bytes() + ).hexdigest() + else: + report = json.loads(gate_path.read_bytes()) + report["gates"].pop(next(iter(report["gates"]))) + gate_path.write_text(json.dumps(report)) + sidecar["spine_gate_report"]["sha256"] = hashlib.sha256( + gate_path.read_bytes() + ).hexdigest() + path.write_text(json.dumps(sidecar)) + with pytest.raises(ValueError): + calibration_run.load_bound_spine_checkpoint(path, frame) + + +def test_strict_checkpoint_accepts_explicit_declared_gate_path(tmp_path): + frame = _frame() + path, gate_path, _ = _bound_checkpoint(tmp_path, frame) + moved = gate_path.rename(tmp_path / "declared-gates.json") + sidecar = calibration_run.load_bound_spine_checkpoint( + path, frame, gate_report_path=moved + ) + provenance = calibration_run.strict_spine_provenance_from_sidecar( + path, sidecar, gate_report_path=moved + ) + assert provenance["spine_gate_report"]["path"] == str(moved) def test_gate_scope_classifies_every_uk_gate(): @@ -202,469 +183,6 @@ def test_import_hygiene_does_not_load_national_build_in_fresh_subprocess(): assert " ".join(("from", legacy_module, "import")) not in source -def test_run_uk_calibration_writes_cross_pinned_outputs(monkeypatch, tmp_path: Path): - pytest.importorskip("tables") # pandas HDF backend - monkeypatch.setattr( - calibration_run, - "uk_aggregate_admin_totals", - lambda frame, manifest: (_admin_anchor_values(), []), - ) - input_h5 = tmp_path / "input.h5" - frame = _frame() - write_uk_national_frame(frame, input_h5) - spine_sidecar = _write_spine_sidecar(input_h5, frame) - paths = UKCalibrationRunPaths( - input_h5=input_h5, - staging_h5=tmp_path / "staged.h5", - diagnostics_json=tmp_path / "diagnostics.json", - build_record_json=tmp_path / "build_record.json", - terminal_gate_json=tmp_path / "terminal_gates.json", - ) - source_pins = { - "input_h5": {"sha256": _sha(input_h5), "size_bytes": input_h5.stat().st_size}, - "ledger_facts": {"sha256": "a" * 64, "size_bytes": 1}, - } - - result = run_uk_calibration( - paths=paths, - input_sha256=_sha(input_h5), - ledger_artifact=object(), - register_registry=_registry(), - band_edge_registry=_registry(), - calibration_year=2025, - exclusion_receipt={}, - doctrine=UKNationalSolveDoctrine(epochs=5), - doctrine_overrides={}, - measure_resolver=None, - source_pins=source_pins, - run_config_extra={"calibration_year": 2025}, - release_id="test-run", - ) - - assert paths.staging_h5.exists() - assert paths.diagnostics_json.exists() - assert paths.build_record_json.exists() - assert paths.terminal_gate_json.exists() - assert result.build_record["artifacts"]["staging_h5"]["sha256"] == _sha( - paths.staging_h5 - ) - assert result.build_record["artifacts"]["diagnostics_json"]["sha256"] == _sha( - paths.diagnostics_json - ) - assert result.build_record["artifacts"]["terminal_gate_json"]["sha256"] == _sha( - paths.terminal_gate_json - ) - # The record makes no shippability claim of its own — the hand-written - # literal retired with the #757 release-cut audit — and instead points - # at the certification artifact whose verdict is authoritative. - assert "shippable" not in result.build_record - assert "shippable_reason" not in result.build_record - certification = result.build_record["certification"] - assert certification["producer"] == "tools/certify_uk_release_cut.py" - assert certification["expected_artifact"] == str( - paths.staging_h5.with_suffix(".release_certification.json") - ) - spine_provenance = result.build_record["spine_provenance"] - assert spine_provenance["stages"] == spine_sidecar["stages"] - assert spine_provenance["stage_records"] == spine_sidecar["stage_records"] - assert spine_provenance["stage_evidence"] == spine_sidecar["stage_evidence"] - assert spine_provenance["artifact_pins"] == spine_sidecar["artifact_pins"] - assert ( - spine_provenance["input_artifact_pins"] == spine_sidecar["input_artifact_pins"] - ) - assert spine_provenance["resource_pins"] == spine_sidecar["resource_pins"] - assert ( - spine_provenance["stage_artifact_pins"] == spine_sidecar["stage_artifact_pins"] - ) - assert spine_provenance["declared_seeds"] == spine_sidecar["declared_seeds"] - assert spine_provenance["rules_engine"] == spine_sidecar["rules_engine"] - assert spine_provenance["source_vintages"] == spine_sidecar["source_vintages"] - assert ( - spine_provenance["stochastic_contract_sha256"] - == spine_sidecar["stochastic_contract_sha256"] - ) - diagnostics = json.loads(paths.diagnostics_json.read_text()) - assert diagnostics["build"]["spine_provenance"] == spine_provenance - staged, _ = load_uk_national_frame(paths.staging_h5) - assert staged.weights_for("household").kind is WeightKind.CALIBRATED - report = json.loads(paths.terminal_gate_json.read_text()) - assert report["posture"] == "calibration_seam" - assert set(report["scope_exclusions"]) == set(UK_CALIBRATION_GATE_SCOPE_EXCLUSIONS) - attestation = report["attestation"] - signature = attestation["signature"] - attestation["signature"] = None - key = b"0123456789abcdef0123456789abcdef" - assert ( - hmac.new(key, canonical_json_bytes(report), hashlib.sha256).hexdigest() - == signature - ) - assert result.logbook_spool.exists() - - -def test_run_uk_calibration_requires_the_band_edge_register( - tmp_path: Path, -): - # Required, never defaulted: an empty receipt is a claim that nothing was - # pruned, not permission to skip the reconciliation, so the seam takes no - # register-without-edges path at all (#803 review findings 1 and 3). - paths = _paths(tmp_path) - - with pytest.raises(TypeError, match="band_edge_registry"): - run_uk_calibration( - paths=paths, - input_sha256="a" * 64, - ledger_artifact=object(), - register_registry=_registry(), - calibration_year=2025, - exclusion_receipt={"excluded.target": {"reason": "reviewed"}}, - doctrine=UKNationalSolveDoctrine(epochs=1), - doctrine_overrides={}, - measure_resolver=None, - source_pins={}, - run_config_extra={}, - release_id="pruned-without-edge-register", - ) - - assert not paths.staging_h5.exists() - assert not paths.diagnostics_json.exists() - assert not paths.build_record_json.exists() - - -def test_run_uk_calibration_reconciles_an_empty_receipt_as_no_prunes( - tmp_path: Path, -): - # A pruned register handed in with an empty receipt must refuse: with - # nothing declared excluded, the two rosters have to be name-identical. - paths = _paths(tmp_path) - full = _registry() - pruned = TargetRegistry([], country="uk") - - with pytest.raises(ValueError, match="exclusion receipt"): - run_uk_calibration( - paths=paths, - input_sha256="a" * 64, - ledger_artifact=object(), - register_registry=pruned, - band_edge_registry=full, - calibration_year=2025, - exclusion_receipt={}, - doctrine=UKNationalSolveDoctrine(epochs=1), - doctrine_overrides={}, - measure_resolver=None, - source_pins={}, - run_config_extra={}, - release_id="empty-receipt-pruned-register", - ) - - assert not paths.staging_h5.exists() - assert not paths.diagnostics_json.exists() - assert not paths.build_record_json.exists() - - -def test_run_uk_calibration_refuses_incoherent_band_edge_register(tmp_path: Path): - paths = _paths(tmp_path) - edge_registry = TargetRegistry( - [ - *_registry().specs, - TargetSpec( - name="different.excluded", - entity="benunit", - measure="different/excluded", - value=1.0, - source="test", - metadata={"contract_target_id": "different.excluded"}, - ), - ], - country="uk", - ) - - with pytest.raises(ValueError, match="exclusion receipt"): - run_uk_calibration( - paths=paths, - input_sha256="a" * 64, - ledger_artifact=object(), - register_registry=_registry(), - band_edge_registry=edge_registry, - calibration_year=2025, - exclusion_receipt={"other.excluded": {"reason": "reviewed"}}, - doctrine=UKNationalSolveDoctrine(epochs=1), - doctrine_overrides={}, - measure_resolver=None, - source_pins={}, - run_config_extra={}, - release_id="incoherent-edge-register", - ) - - assert not paths.staging_h5.exists() - assert not paths.diagnostics_json.exists() - assert not paths.build_record_json.exists() - - -def test_run_uk_calibration_records_band_edge_register_sha256( - monkeypatch, tmp_path: Path -): - pytest.importorskip("tables") # pandas HDF backend - monkeypatch.setattr( - calibration_run, - "uk_aggregate_admin_totals", - lambda frame, manifest: (_admin_anchor_values(), []), - ) - input_h5 = tmp_path / "input.h5" - frame = _frame() - write_uk_national_frame(frame, input_h5) - _write_spine_sidecar(input_h5, frame) - paths = _paths(tmp_path) - register = _registry() - edge_registry = TargetRegistry( - [ - TargetSpec( - name="dwp.uc.households", - entity="benunit", - measure="dwp/uc/households", - value=99.0, - source="test", - family="dwp_universal_credit", - metadata={"contract_target_id": "dwp.uc.households"}, - ) - ], - country="uk", - ) - - result = run_uk_calibration( - paths=paths, - input_sha256=_sha(input_h5), - ledger_artifact=object(), - register_registry=register, - band_edge_registry=edge_registry, - calibration_year=2025, - exclusion_receipt={}, - doctrine=UKNationalSolveDoctrine(epochs=5), - doctrine_overrides={}, - measure_resolver=None, - source_pins={ - "input_h5": { - "sha256": _sha(input_h5), - "size_bytes": input_h5.stat().st_size, - } - }, - run_config_extra={}, - release_id="band-edge-provenance", - ) - - assert ( - result.build_record["run_config"]["band_edge_register_sha256"] - == edge_registry.version - ) - - -def test_run_uk_calibration_refuses_input_sha_before_outputs(tmp_path: Path): - pytest.importorskip("tables") # pandas HDF backend - input_h5 = tmp_path / "input.h5" - write_uk_national_frame(_frame(), input_h5) - paths = UKCalibrationRunPaths( - input_h5=input_h5, - staging_h5=tmp_path / "staged.h5", - diagnostics_json=tmp_path / "diagnostics.json", - build_record_json=tmp_path / "build_record.json", - terminal_gate_json=tmp_path / "terminal_gates.json", - ) - with pytest.raises(ValueError, match="sha mismatch"): - run_uk_calibration( - paths=paths, - input_sha256="0" * 64, - ledger_artifact=object(), - register_registry=_registry(), - band_edge_registry=_registry(), - calibration_year=2025, - exclusion_receipt={}, - doctrine=UKNationalSolveDoctrine(epochs=1), - doctrine_overrides={}, - measure_resolver=None, - source_pins={ - "input_h5": { - "sha256": _sha(input_h5), - "size_bytes": input_h5.stat().st_size, - } - }, - run_config_extra={"calibration_year": 2025}, - release_id="bad-sha", - ) - assert not paths.staging_h5.exists() - assert not paths.diagnostics_json.exists() - - -def test_run_uk_calibration_refuses_absent_input_sidecar(tmp_path: Path): - pytest.importorskip("tables") # pandas HDF backend - input_h5 = tmp_path / "input.h5" - write_uk_national_frame(_frame(), input_h5) - paths = UKCalibrationRunPaths( - input_h5=input_h5, - staging_h5=tmp_path / "staged.h5", - diagnostics_json=tmp_path / "diagnostics.json", - build_record_json=tmp_path / "build_record.json", - terminal_gate_json=tmp_path / "terminal_gates.json", - ) - - with pytest.raises(ValueError, match="build sidecar absent"): - run_uk_calibration( - paths=paths, - input_sha256=_sha(input_h5), - ledger_artifact=object(), - register_registry=_registry(), - band_edge_registry=_registry(), - calibration_year=2025, - exclusion_receipt={}, - doctrine=UKNationalSolveDoctrine(epochs=1), - doctrine_overrides={}, - measure_resolver=None, - source_pins={ - "input_h5": { - "sha256": _sha(input_h5), - "size_bytes": input_h5.stat().st_size, - } - }, - run_config_extra={"calibration_year": 2025}, - release_id="missing-sidecar", - ) - - assert not paths.staging_h5.exists() - assert not paths.diagnostics_json.exists() - assert not paths.terminal_gate_json.exists() - - -@pytest.mark.parametrize( - ("override", "message"), - [ - ( - {"entity_row_counts": {"person": 999, "benunit": 4, "household": 4}}, - "row-count mismatch", - ), - ({"household_weight_total": 1.0}, "household_weight_total mismatch"), - ], -) -def test_run_uk_calibration_refuses_unbound_input_sidecar( - override, message, tmp_path: Path -): - pytest.importorskip("tables") # pandas HDF backend - frame = _frame() - input_h5 = tmp_path / "input.h5" - write_uk_national_frame(frame, input_h5) - _write_spine_sidecar(input_h5, frame, **override) - paths = UKCalibrationRunPaths( - input_h5=input_h5, - staging_h5=tmp_path / "staged.h5", - diagnostics_json=tmp_path / "diagnostics.json", - build_record_json=tmp_path / "build_record.json", - terminal_gate_json=tmp_path / "terminal_gates.json", - ) - - with pytest.raises(ValueError, match=message): - run_uk_calibration( - paths=paths, - input_sha256=_sha(input_h5), - ledger_artifact=object(), - register_registry=_registry(), - band_edge_registry=_registry(), - calibration_year=2025, - exclusion_receipt={}, - doctrine=UKNationalSolveDoctrine(epochs=1), - doctrine_overrides={}, - measure_resolver=None, - source_pins={ - "input_h5": { - "sha256": _sha(input_h5), - "size_bytes": input_h5.stat().st_size, - } - }, - run_config_extra={"calibration_year": 2025}, - release_id="unbound-sidecar", - ) - - assert not paths.staging_h5.exists() - assert not paths.diagnostics_json.exists() - assert not paths.terminal_gate_json.exists() - - -def test_seam_never_modifies_data_variables(monkeypatch, tmp_path: Path): - """The seam's defining invariant: weights move, data never does. - - Every data column of every entity table in the staged H5 must be - byte-identical to the input; only the household weights, the weight - kind, and exactly one appended mass record may differ. - """ - - pytest.importorskip("tables") # pandas HDF backend - - monkeypatch.setattr( - calibration_run, - "uk_aggregate_admin_totals", - lambda frame, manifest: (_admin_anchor_values(), []), - ) - input_h5 = tmp_path / "input.h5" - frame = _frame() - write_uk_national_frame(frame, input_h5) - _write_spine_sidecar(input_h5, frame) - paths = UKCalibrationRunPaths( - input_h5=input_h5, - staging_h5=tmp_path / "staged.h5", - diagnostics_json=tmp_path / "diagnostics.json", - build_record_json=tmp_path / "build_record.json", - terminal_gate_json=tmp_path / "terminal_gates.json", - ) - - # Target 30 against an initial weighted UC count of 20, so the solve - # genuinely has to move weights while the data stays untouched. - pulling_registry = TargetRegistry( - [ - TargetSpec( - name="dwp.uc.households", - entity="benunit", - measure="dwp/uc/households", - value=30.0, - source="test", - family="dwp_universal_credit", - metadata={"contract_target_id": "dwp.uc.households"}, - ) - ], - country="uk", - ) - run_uk_calibration( - paths=paths, - input_sha256=_sha(input_h5), - ledger_artifact=object(), - register_registry=pulling_registry, - band_edge_registry=pulling_registry, - calibration_year=2025, - exclusion_receipt={}, - doctrine=UKNationalSolveDoctrine(epochs=50), - doctrine_overrides={}, - measure_resolver=None, - source_pins={ - "input_h5": { - "sha256": _sha(input_h5), - "size_bytes": input_h5.stat().st_size, - } - }, - run_config_extra={}, - release_id="invariant-run", - ) - - source, _ = load_uk_national_frame(input_h5) - staged, _ = load_uk_national_frame(paths.staging_h5) - for entity in ("person", "benunit", "household"): - left = source.table(entity) - right = staged.table(entity) - assert list(left.columns) == list(right.columns), entity - for column in left.columns: - pd.testing.assert_series_equal(left[column], right[column]) - assert staged.weights_for("household").kind is WeightKind.CALIBRATED - assert not np.allclose( - staged.weights_for("household").values, - source.weights_for("household").values, - ) - assert len(staged.mass_log) == len(source.mass_log) + 1 - - def test_aggregate_admin_measurement_convention_and_refusals(): frame = _frame() manifest = calibration_run._calibration_gate_manifest() @@ -728,189 +246,6 @@ def test_partly_carried_derived_anchor_refuses_and_names_the_missing_part(): calibration_run.uk_aggregate_admin_totals(frame, manifest) -def test_seam_pipeline_derives_a_ratified_logbook_scope(): - """The seam appends to the FRS line's chain, not a new unratified one.""" - - logbook_tool = _load_logbook_tool() - - scope = logbook_tool._chain_scope(calibration_run._PIPELINE) - - assert scope == "uk/frs" - assert scope in logbook_tool.DECLARED_SCOPES - - -def _load_logbook_tool(): - import importlib.util - - path = Path(__file__).resolve().parents[3] / "tools" / "logbook.py" - spec = importlib.util.spec_from_file_location("_logbook_tool", path) - module = importlib.util.module_from_spec(spec) - spec.loader.exec_module(module) - return module - - -def test_refusal_records_a_failed_attempt_and_stages_nothing(tmp_path: Path): - pytest.importorskip("tables") # pandas HDF backend - input_h5 = tmp_path / "input.h5" - write_uk_national_frame(_frame(), input_h5) - paths = UKCalibrationRunPaths( - input_h5=input_h5, - staging_h5=tmp_path / "staged.h5", - diagnostics_json=tmp_path / "diagnostics.json", - build_record_json=tmp_path / "build_record.json", - terminal_gate_json=tmp_path / "terminal_gates.json", - ) - - with pytest.raises(ValueError, match="sha mismatch"): - run_uk_calibration( - paths=paths, - input_sha256="0" * 64, - ledger_artifact=object(), - register_registry=_registry(), - band_edge_registry=_registry(), - calibration_year=2025, - exclusion_receipt={}, - doctrine=UKNationalSolveDoctrine(epochs=1), - doctrine_overrides={}, - measure_resolver=None, - source_pins={ - "input_h5": { - "sha256": _sha(input_h5), - "size_bytes": input_h5.stat().st_size, - } - }, - run_config_extra={"calibration_year": 2025}, - release_id="refused-run", - ) - - # Every terminal disposition is a row; a refusal that left the chain - # silent would hide the attempt entirely. - spooled = sorted((tmp_path / "logbook-spool").rglob("*.json")) - assert spooled, "refusal recorded no Logbook row" - rows = [json.loads(path.read_text()) for path in spooled] - assert [row["disposition"] for row in rows] == ["failed"] - assert rows[0]["pipeline"] == calibration_run._PIPELINE - # The refusal is explained on disk, not only in the row. - receipts = sorted((tmp_path / "logbook-receipts").rglob("error.json")) - assert len(receipts) == 1 - assert not paths.staging_h5.exists() - assert not paths.diagnostics_json.exists() - assert not paths.terminal_gate_json.exists() - - -def test_attempt_ids_are_unique_across_reruns_of_one_release( - monkeypatch, tmp_path: Path -): - pytest.importorskip("tables") # pandas HDF backend - monkeypatch.setattr( - calibration_run, - "uk_aggregate_admin_totals", - lambda frame, manifest: (_admin_anchor_values(), []), - ) - input_h5 = tmp_path / "input.h5" - frame = _frame() - write_uk_national_frame(frame, input_h5) - _write_spine_sidecar(input_h5, frame) - source_pins = { - "input_h5": {"sha256": _sha(input_h5), "size_bytes": input_h5.stat().st_size} - } - build_ids = [] - for attempt in ("a", "b"): - run_dir = tmp_path / attempt - run_dir.mkdir() - result = run_uk_calibration( - paths=UKCalibrationRunPaths( - input_h5=input_h5, - staging_h5=run_dir / "staged.h5", - diagnostics_json=run_dir / "diagnostics.json", - build_record_json=run_dir / "build_record.json", - terminal_gate_json=run_dir / "terminal_gates.json", - ), - input_sha256=_sha(input_h5), - ledger_artifact=object(), - register_registry=_registry(), - band_edge_registry=_registry(), - calibration_year=2025, - exclusion_receipt={}, - doctrine=UKNationalSolveDoctrine(epochs=5), - doctrine_overrides={}, - measure_resolver=None, - source_pins=source_pins, - run_config_extra={"calibration_year": 2025}, - release_id="one-release-id", - ) - build_ids.append(result.build_record["build_id"]) - - # Both the local chain and the store reject a duplicate build id, so one - # release re-run twice must not collide. - assert build_ids[0] != build_ids[1] - assert all(value.startswith("uk-frs-calibration-attempt-") for value in build_ids) - - -def test_verified_ledger_identity_reaches_the_run_evidence(monkeypatch, tmp_path: Path): - pytest.importorskip("tables") # pandas HDF backend - monkeypatch.setattr( - calibration_run, - "uk_aggregate_admin_totals", - lambda frame, manifest: (_admin_anchor_values(), []), - ) - input_h5 = tmp_path / "input.h5" - frame = _frame() - write_uk_national_frame(frame, input_h5) - _write_spine_sidecar(input_h5, frame) - artifact = SimpleNamespace( - facts_sha256="d" * 64, - fact_row_count=107_550, - manifest_sha256="e" * 64, - manifest={ - "artifact_id": "chronicle-uk-artifact-1cab809", - "profile": "uk-national", - "schema_version": 1, - "unrelated": "not carried", - }, - ) - - result = run_uk_calibration( - paths=UKCalibrationRunPaths( - input_h5=input_h5, - staging_h5=tmp_path / "staged.h5", - diagnostics_json=tmp_path / "diagnostics.json", - build_record_json=tmp_path / "build_record.json", - terminal_gate_json=tmp_path / "terminal_gates.json", - ), - input_sha256=_sha(input_h5), - ledger_artifact=artifact, - register_registry=_registry(), - band_edge_registry=_registry(), - calibration_year=2025, - exclusion_receipt={}, - doctrine=UKNationalSolveDoctrine(epochs=5), - doctrine_overrides={}, - measure_resolver=None, - source_pins={ - "input_h5": { - "sha256": _sha(input_h5), - "size_bytes": input_h5.stat().st_size, - } - }, - run_config_extra={"calibration_year": 2025}, - release_id="ledger-identity", - ) - - ledger = result.build_record["run_config"]["ledger"] - assert ledger["facts_sha256"] == "d" * 64 - assert ledger["manifest_sha256"] == "e" * 64 - assert ledger["fact_row_count"] == 107_550 - assert ledger["manifest"] == { - "artifact_id": "chronicle-uk-artifact-1cab809", - "profile": "uk-national", - "schema_version": 1, - } - # A bare feed carries no manifest, and that absence is recorded rather - # than invented. - assert calibration_run._ledger_provenance(object())["manifest_sha256"] is None - - def _consumer_fact_row( *, aggregate_fact_key: str, @@ -983,79 +318,6 @@ def _mixed_epoch_artifact_dir(tmp_path: Path) -> Path: return artifact_dir -def test_mixed_epoch_feed_epochs_reach_the_uk_release_evidence(monkeypatch, tmp_path): - """A UK run says which Chronicle era resolved its targets. - - The run's own provenance block used to be assembled field by field from - the artifact, which is how it came to carry the hashes but not the epoch - witnesses the loader had already computed. It now delegates to the shared - block, so a cutover-window feed is visible in the signed diagnostics and - in the build record — including the one Chronicle-namespace spelling this - build does not declare, named rather than folded into an era. - """ - pytest.importorskip("tables") # pandas HDF backend - monkeypatch.setattr( - calibration_run, - "uk_aggregate_admin_totals", - lambda frame, manifest: (_admin_anchor_values(), []), - ) - input_h5 = tmp_path / "input.h5" - frame = _frame() - write_uk_national_frame(frame, input_h5) - _write_spine_sidecar(input_h5, frame) - artifact = load_ledger_consumer_artifact(_mixed_epoch_artifact_dir(tmp_path)) - diagnostics_json = tmp_path / "diagnostics.json" - - result = run_uk_calibration( - paths=UKCalibrationRunPaths( - input_h5=input_h5, - staging_h5=tmp_path / "staged.h5", - diagnostics_json=diagnostics_json, - build_record_json=tmp_path / "build_record.json", - terminal_gate_json=tmp_path / "terminal_gates.json", - ), - input_sha256=_sha(input_h5), - ledger_artifact=artifact, - register_registry=_registry(), - band_edge_registry=_registry(), - calibration_year=2025, - exclusion_receipt={}, - doctrine=UKNationalSolveDoctrine(epochs=5), - doctrine_overrides={}, - measure_resolver=None, - source_pins={ - "input_h5": { - "sha256": _sha(input_h5), - "size_bytes": input_h5.stat().st_size, - } - }, - run_config_extra={"calibration_year": 2025}, - release_id="chronicle-mixed-epoch", - ) - - ledger = result.build_record["run_config"]["ledger"] - # The observed manifest id, verbatim, and the era it belongs to. - assert ledger["manifest"]["schema_version"] == ( - PUBLISHED_CONSUMER_ARTIFACT_SCHEMA_VERSION - ) - assert ledger["schema_epoch"] == "ledger" - # The feed straddles the cutover, and says so rather than reporting one era. - assert ledger["fact_key_epochs"] == ["ledger", "chronicle", "undeclared"] - assert ledger["undeclared_fact_key_domains"] == ["chronicle.source_release.v9"] - assert ledger["fact_schema_versions"] == [ - CHRONICLE_CONSUMER_FACT_SCHEMA_VERSION, - LEDGER_CONSUMER_FACT_SCHEMA_VERSION, - ] - # The hashes the block always carried are unchanged by the delegation. - assert ledger["facts_sha256"] == artifact.facts_sha256 - assert ledger["fact_row_count"] == 3 - - # The same block is what the signed diagnostics carry, so the evidence a - # release assembler reads witnesses the era too. - diagnostics = json.loads(diagnostics_json.read_text()) - assert diagnostics["build"]["ledger"] == ledger - - def test_the_uk_block_delegates_rather_than_reassembling_the_shared_one(tmp_path): """Every field of the shared provenance block reaches the UK block. diff --git a/packages/microcosm-build/tests/test_uk_calibration_seam_driver.py b/packages/microcosm-build/tests/test_uk_calibration_seam_driver.py index 6b431828f..d3022113e 100644 --- a/packages/microcosm-build/tests/test_uk_calibration_seam_driver.py +++ b/packages/microcosm-build/tests/test_uk_calibration_seam_driver.py @@ -1,287 +1,65 @@ +"""The former national CLI is an alias for the canonical full build.""" + from __future__ import annotations import importlib.util +import sys from pathlib import Path from types import SimpleNamespace import pytest -from microcosm.build.uk_runtime.ledger_targets import UKLedgerTargetCompilation -from microcosm.calibrate import TargetRegistry, TargetSpec - def _load_driver_module(): - root = Path(__file__).resolve().parents[3] - path = root / "tools" / "calibrate_uk_national_dataset.py" + path = ( + Path(__file__).resolve().parents[3] / "tools/calibrate_uk_national_dataset.py" + ) spec = importlib.util.spec_from_file_location("calibrate_uk_national_dataset", path) module = importlib.util.module_from_spec(spec) - assert spec.loader is not None spec.loader.exec_module(module) return module -def _registry(): - return TargetRegistry( - [ - TargetSpec( - name="dwp.uc.households", - entity="benunit", - measure="dwp/uc/households", - value=1.0, - source="test", - metadata={"contract_target_id": "dwp.uc.households"}, - ) - ], - country="uk", +@pytest.mark.parametrize("selector", [[], ["--target-geographies", "country"]]) +def test_old_command_delegates_without_inventing_a_target_filter(monkeypatch, selector): + calls = [] + monkeypatch.setitem( + sys.modules, + "microcosm.build.uk_runtime.full_build_cli", + SimpleNamespace(main=lambda args: calls.append(args) or 7), ) - - -def _args(tmp_path: Path) -> list[str]: - paths = { - "input": tmp_path / "input.h5", - "ledger": tmp_path / "ledger", - "staging": tmp_path / "staging.h5", - "diagnostics": tmp_path / "diagnostics.json", - "record": tmp_path / "record.json", - } - paths["ledger"].mkdir(exist_ok=True) - for key, path in paths.items(): - if key != "ledger": - path.write_bytes(key.encode()) - return [ + arguments = [ "--input-h5", - str(paths["input"]), - "--input-sha256", - "a" * 64, + "bound-spine.h5", + "--ladder", + "ladder.zip", "--ledger-facts", - str(paths["ledger"]), - "--ledger-facts-sha256", - "b" * 64, - "--ledger-manifest-sha256", - "c" * 64, - "--staging-h5", - str(paths["staging"]), - "--diagnostics-json", - str(paths["diagnostics"]), - "--build-record-json", - str(paths["record"]), - "--release-id", - "dev-calibration", + "facts.jsonl", + *selector, ] + assert _load_driver_module().main(arguments) == 7 + assert calls == [arguments] -def test_driver_refuses_release_candidate_outright(tmp_path: Path): - # The seam's scoped battery covers 6 of the declared entries and must - # never sign a shippability claim (the #757 release-cut audit); the - # release verdict belongs to the release-cut certification producer. - driver = _load_driver_module() - with pytest.raises(SystemExit): - driver._parse_args(_args(tmp_path) + ["--release-candidate"]) - # The refusal is unconditional — an otherwise doctrine-clean invocation - # is refused too, not just ones with override flags. - with pytest.raises(SystemExit): - driver._parse_args(_args(tmp_path) + ["--release-candidate", "--epochs", "128"]) - - -def test_driver_refuses_canonical_release_ids(tmp_path: Path): - # Canonical release ids name shippable candidates; the seam runs under - # staging or dev ids only, and redirects canonical ids to the - # release-cut producer. - driver = _load_driver_module() - base = _args(tmp_path) - release_index = base.index("--release-id") - for canonical in ( - "populace-uk-2024-frs-k100", - "populace-uk-2023-dd68c73-4aa4b14-20260619T023711Z", +@pytest.mark.parametrize( + "flag", + [ + "--staging-h5", + "--diagnostics-json", + "--build-record-json", + "--terminal-gate-json", + "--allow-unpinned-feed", + ], +) +def test_retired_national_output_controls_explain_migration(flag): + with pytest.raises( + SystemExit, match="independent UK national calibration driver is retired" ): - args = [*base] - args[release_index + 1] = canonical - with pytest.raises(SystemExit): - driver._parse_args(args) + _load_driver_module().main([flag, "old-output"]) -def test_driver_accepts_operator_exclusions_on_staging_posture(tmp_path: Path): +def test_old_command_cannot_invoke_a_second_solver(): driver = _load_driver_module() - exclusions = tmp_path / "operator.json" - exclusions.write_text("{}", encoding="utf-8") - parsed = driver._parse_args( - _args(tmp_path) + ["--measure-exclusions", str(exclusions)] - ) - assert parsed.measure_exclusions == exclusions - - -def test_driver_refuses_bad_sha_and_path_alias(tmp_path: Path): - driver = _load_driver_module() - base_args = _args(tmp_path) - with pytest.raises(SystemExit): - driver._parse_args([*base_args[:3], "not-a-sha", *base_args[4:]]) - - args = _args(tmp_path) - staging_index = args.index("--staging-h5") + 1 - args[staging_index] = args[args.index("--input-h5") + 1] - with pytest.raises(SystemExit, match="distinct paths"): - driver._parse_args(args) - - -def test_driver_refuses_feed_outside_the_committed_pin_without_override(): - driver = _load_driver_module() - - with pytest.raises(SystemExit, match="committed UK national feed pin"): - driver._check_committed_ledger_feed_pin( - "b" * 64, - manifest_sha256=driver.load_uk_national_chronicle_feed().manifest_sha256, - allow_unpinned_feed=False, - ) - - driver._check_committed_ledger_feed_pin( - "b" * 64, - manifest_sha256="c" * 64, - allow_unpinned_feed=True, - ) - - -@pytest.mark.parametrize("allow_unpinned_feed", [False, True]) -def test_driver_threads_registry_exclusions_resolver_and_overrides( - monkeypatch, tmp_path, capsys, allow_unpinned_feed -): - driver = _load_driver_module() - calls = [] - registry = _registry() - pruned_registry = TargetRegistry([], country="uk") - pin = driver.load_uk_national_chronicle_feed() - artifact = SimpleNamespace( - path=tmp_path / "ledger", - facts=({"fact": 1},), - facts_sha256="b" * 64 if allow_unpinned_feed else pin.facts_sha256, - manifest_sha256="c" * 64 if allow_unpinned_feed else pin.manifest_sha256, - ) - artifact.path.mkdir() - (artifact.path / "consumer_facts.jsonl").write_text("{}", encoding="utf-8") - monkeypatch.setattr( - driver, "load_ledger_consumer_artifact", lambda *a, **k: artifact - ) - monkeypatch.setattr( - driver, - "compile_uk_target_registry", - lambda facts, target_period: UKLedgerTargetCompilation(registry, ()), - ) - monkeypatch.setattr( - driver, - "load_uk_frs_release", - lambda: SimpleNamespace(calibration_year=2025), - ) - monkeypatch.setattr( - driver, "load_uk_calibration_measure_exclusions", lambda path: () - ) - monkeypatch.setattr( - driver, - "apply_uk_calibration_measure_exclusions", - lambda reg, exclusions: (pruned_registry, {"excluded": "reviewed"}), - ) - - class FakeResolver: - def __init__(self, **kwargs): - self.kwargs = kwargs - - monkeypatch.setattr(driver, "UKMeasureResolver", FakeResolver) - - def fake_run(**kwargs): - calls.append(kwargs) - return SimpleNamespace( - staging_sha256="1" * 64, - diagnostics_sha256="2" * 64, - terminal_gate_sha256="3" * 64, - build_record_sha256="4" * 64, - build_record={"gate_summary": {"uk_target_fit": "passed"}}, - ) - - monkeypatch.setattr(driver, "run_uk_calibration", fake_run) - - argv = _args(tmp_path) - # The driver verifies the input pin before the resolver reads the H5, so - # the threading test must pin the fixture file's real digest. - argv[argv.index("--input-sha256") + 1] = driver._sha256_file( - Path(argv[argv.index("--input-h5") + 1]) - ) - extra = ["--allow-unpinned-feed"] if allow_unpinned_feed else [] - result = driver.main(argv + ["--epochs", "128", *extra]) - - assert result == 0 - call = calls[0] - assert call["register_registry"] is pruned_registry - assert call["band_edge_registry"] is registry - assert call["calibration_year"] == 2025 - assert call["exclusion_receipt"] == {"excluded": "reviewed"} - assert call["doctrine"].epochs == 128 - assert call["doctrine_overrides"] == {"epochs": {"default": 256, "effective": 128}} - assert isinstance(call["measure_resolver"], FakeResolver) - assert ( - call["measure_resolver"].kwargs["simulation_source"] == call["paths"].input_h5 - ) - assert call["source_pins"]["ledger_facts"] == { - "sha256": artifact.facts_sha256, - "size_bytes": 2, - } - assert call["run_config_extra"] == { - "calibration_year": 2025, - "allow_unpinned_feed": allow_unpinned_feed, - "national_chronicle_feed_pin": pin.to_dict(), - } - assert "uk_target_fit" in capsys.readouterr().out - - -def test_driver_refuses_the_national_release_id(tmp_path: Path): - # The constant national id names a shippable release (ruling 2026-08-27); - # the seam runs under staging or dev ids only. - driver = _load_driver_module() - base = _args(tmp_path) - args = [*base] - args[base.index("--release-id") + 1] = "microcosm-uk-2024-25-national" - with pytest.raises(SystemExit): - driver._parse_args(args) - - -def test_driver_accepts_the_merged_national_feed_without_local_promotion(): - from microcosm.build.uk_runtime.local_target_census import _LEDGER_FACT_FEED_PIN - - driver = _load_driver_module() - driver._check_committed_ledger_feed_pin( - "4a50ee9568a01bbb57f73d927084ed6b4b9e52249b51a2338455874ae6e382b5", - manifest_sha256="a95d0ee9f87f36947eaecdb3de29cf81a91e47ccaa822fed42da677eedca877f", - allow_unpinned_feed=False, - ) - assert _LEDGER_FACT_FEED_PIN["facts_sha256"] == ( - "6ae49d7d7ab297df25a0b9bfe2d6776827c672d284fbb360957fe8337089549f" - ) - assert _LEDGER_FACT_FEED_PIN["manifest_sha256"] == ( - "dcda51d6496aea67f768a284e7955c7520e7c8b91e2bed3569f247567b7153f0" - ) - - -@pytest.mark.parametrize("manifest_sha256", ["c" * 64, None]) -def test_driver_refuses_unpinned_national_manifest(manifest_sha256): - driver = _load_driver_module() - with pytest.raises(SystemExit, match="manifest"): - driver._check_committed_ledger_feed_pin( - "4a50ee9568a01bbb57f73d927084ed6b4b9e52249b51a2338455874ae6e382b5", - manifest_sha256=manifest_sha256, - allow_unpinned_feed=False, - ) - - -def test_driver_checks_loaded_manifest_before_compiling_targets(monkeypatch, tmp_path): - driver = _load_driver_module() - pin = driver.load_uk_national_chronicle_feed() - artifact = SimpleNamespace(facts_sha256=pin.facts_sha256, manifest_sha256="c" * 64) - monkeypatch.setattr( - driver, "load_ledger_consumer_artifact", lambda *a, **k: artifact - ) - monkeypatch.setattr( - driver, - "compile_uk_target_registry", - lambda *a, **k: pytest.fail( - "Targets must not compile from an unpinned artifact" - ), - ) - with pytest.raises(SystemExit, match="manifest"): - driver.main(_args(tmp_path)) + assert not hasattr(driver, "run_uk_calibration") + assert not hasattr(driver, "UKMeasureResolver") + assert not hasattr(driver, "_parse_args") diff --git a/packages/microcosm-build/tests/test_uk_cgt_observation_period.py b/packages/microcosm-build/tests/test_uk_cgt_observation_period.py index 88d7952f2..c9f5e38e3 100644 --- a/packages/microcosm-build/tests/test_uk_cgt_observation_period.py +++ b/packages/microcosm-build/tests/test_uk_cgt_observation_period.py @@ -1,9 +1,7 @@ -"""Dated CGT measurements retain base-year rows through both calibration paths.""" +"""Dated CGT measurements retain base-year rows for every full-build target scope.""" from __future__ import annotations -import importlib.util -from pathlib import Path from types import SimpleNamespace import numpy as np @@ -12,8 +10,6 @@ from microcosm.build.uk_runtime import measure_simulation from microcosm.build.uk_runtime.measure_simulation import UKMeasureResolver -from microcosm.build.uk_runtime.national_calibration import UKNationalCalibrationStage -from microcosm.build.uk_runtime.national_doctrine import UKNationalSolveDoctrine from microcosm.build.uk_runtime.national_frame import ( load_uk_national_frame, uk_national_frame, @@ -119,9 +115,9 @@ def _assert_export(original, fitted, path): ) -@pytest.mark.parametrize("route", ["national", "local"]) +@pytest.mark.parametrize("target_scope", ["country", "all"]) def test_dated_cgt_fit_restores_base2024_values_with_fitted_weights( - monkeypatch, tmp_path, route + monkeypatch, tmp_path, target_scope ): pytest.importorskip("tables") pytest.importorskip("h5py") @@ -139,65 +135,52 @@ def resolver_factory(**kwargs): **kwargs, microsimulation_factory=lambda **_: simulation ) + from microcosm.build.uk_runtime import full_measure + from microcosm.build.uk_runtime.local_rowwise import ( + build_uk_rowwise_local_matrix, + empty_uk_local_problem, + solve_uk_rowwise_weights_under_doctrine, + ) + registry = _registry() - if route == "national": - resolver = resolver_factory( - frame=original, scratch_dir=tmp_path / "engine", year=2025 - ) - stage = UKNationalCalibrationStage( + monkeypatch.setattr( + full_measure, + "compute_household_metrics", + lambda _sim, _area, *, period, household_ids: pd.DataFrame( + {"households": np.ones(len(household_ids))}, index=household_ids + ), + ) + prepared, restore, national, metrics, engine_receipt = ( + full_measure.resolve_uk_full_measures( + original, registry, - band_edge_registry=registry, period=2025, - doctrine=UKNationalSolveDoctrine(epochs=2), - measure_resolver=resolver, + scratch_dir=tmp_path / "engine", + resolver_factory=resolver_factory, + band_edge_registry=registry, + local_grains=() if target_scope == "country" else ("constituency",), ) - fitted = stage(original) - receipt = stage.manifest["measure_resolution"]["provider"][ - "cgt_period_contract" - ] + ) + receipt = engine_receipt["cgt_period_contract"] + if target_scope == "country": + local = empty_uk_local_problem((0, 1, 2)) + bound_families = ["national/hmrc_cgt"] else: - from microcosm.build.uk_runtime.local_rowwise import ( - build_uk_rowwise_local_matrix, - solve_uk_rowwise_weights_under_doctrine, - ) - - spec = importlib.util.spec_from_file_location( - "cgt_rowwise_builder", - Path(__file__).resolve().parents[3] / "tools/build_uk_rowwise_candidate.py", - ) - builder = importlib.util.module_from_spec(spec) - spec.loader.exec_module(builder) - monkeypatch.setattr( - builder, - "compute_household_metrics", - lambda _sim, _area, *, period, household_ids: pd.DataFrame( - {"households": np.ones(len(household_ids))}, index=household_ids - ), - ) - prepared, restore, national, metrics, engine_receipt = ( - builder._resolve_candidate_engine_surface( - original, - registry, - period=2025, - scratch_dir=tmp_path / "engine", - resolver_factory=resolver_factory, - ) - ) - receipt = engine_receipt["cgt_period_contract"] local = build_uk_rowwise_local_matrix( metrics["constituency"], pd.Series(["A", "A", "A"], index=[0, 1, 2]), pd.DataFrame({"code": ["A"], "households": [20.0]}), ) - result = solve_uk_rowwise_weights_under_doctrine( - prepared, - local, - bound_families=["census_households/constituency", "national/hmrc_cgt"], - national_rows=national, - restore=restore, - epochs=2, - ) - fitted = result.frame + bound_families = ["census_households/constituency", "national/hmrc_cgt"] + result = solve_uk_rowwise_weights_under_doctrine( + prepared, + local, + bound_families=bound_families, + national_rows=national, + restore=restore, + epochs=2, + ) + fitted = result.frame assert all(spec.period == 2025 for spec in registry.specs) assert receipt["input_period"] == "2024" assert receipt["calibration_period"] == 2025 @@ -215,7 +198,7 @@ def resolver_factory(**kwargs): ("capital_gains", 2024), ("capital_gains_tax", 2024), } - _assert_export(original, fitted, tmp_path / f"{route}.h5") + _assert_export(original, fitted, tmp_path / f"{target_scope}.h5") @pytest.mark.requires_uk diff --git a/packages/microcosm-build/tests/test_uk_cgt_source_manifest.py b/packages/microcosm-build/tests/test_uk_cgt_source_manifest.py index 2348f23ea..41cf69a15 100644 --- a/packages/microcosm-build/tests/test_uk_cgt_source_manifest.py +++ b/packages/microcosm-build/tests/test_uk_cgt_source_manifest.py @@ -95,17 +95,18 @@ def test_receipt_reason_matches_the_stage_constant() -> None: assert receipt["declared_factor"] == 1.0 -def test_family_coverage_carries_the_stage_as_required_at_build() -> None: +def test_family_coverage_requires_the_canonical_spine_stage() -> None: manifest = load_uk_release_input_coverage_manifest() - family = manifest.family_coverage[UK_CGT_IMPUTATION_STAGE_NAME] - + family = manifest.family_coverage["hmrc_cgt_gains_spine"] assert family["status"] == "required_at_build" - assert family["source_manifest"] == _MANIFEST_PATH.name + assert family["source_manifest"] == "source_stages.json" assert family["calibration_permitted"] is False - assert family["outputs"] == ["capital_gains"] + assert "capital_gains" in (*family["outputs"], *family["rewrites"]) assert family["output_weight_kind"] == "importance" - assert family["required_mass_change_reason"] == UK_CGT_MASS_CONSERVATION_REASON - assert UK_CGT_IMPUTATION_STAGE_NAME in manifest.required_build_stages + assert ( + family["required_mass_change_reason"] == UK_CGT_SPINE_MASS_CONSERVATION_REASON + ) + assert "hmrc_cgt_gains_spine" in manifest.required_build_stages def test_the_shipped_family_contracts_pass_the_terminal_gate_shape() -> None: @@ -215,12 +216,14 @@ def test_the_shipped_family_contracts_pass_the_terminal_gate_shape() -> None: ) -def test_certified_and_spine_families_require_distinct_receipts() -> None: - """One record must never satisfy both CGT families (review finding).""" - manifest = load_uk_release_input_coverage_manifest() - families = manifest.family_coverage - certified = families["hmrc_cgt_gains"]["required_mass_change_reason"] - spine = families["hmrc_cgt_gains_spine"]["required_mass_change_reason"] - assert certified == UK_CGT_MASS_CONSERVATION_REASON - assert spine == UK_CGT_SPINE_MASS_CONSERVATION_REASON - assert certified != spine +def test_legacy_cgt_family_is_absent_from_the_executable_contract() -> None: + families = load_uk_release_input_coverage_manifest().family_coverage + assert "hmrc_cgt_gains" not in families + assert ( + families["hmrc_cgt_gains_spine"]["required_mass_change_reason"] + == UK_CGT_SPINE_MASS_CONSERVATION_REASON + ) + assert ( + families["hmrc_cgt_gains_spine"]["required_mass_change_reason"] + != UK_CGT_MASS_CONSERVATION_REASON + ) diff --git a/packages/microcosm-build/tests/test_uk_country_adapter.py b/packages/microcosm-build/tests/test_uk_country_adapter.py new file mode 100644 index 000000000..bf6d18433 --- /dev/null +++ b/packages/microcosm-build/tests/test_uk_country_adapter.py @@ -0,0 +1,52 @@ +"""The UK country adapter uses raw FRS sources and the canonical full graph.""" + +import shutil +from pathlib import Path + +import pytest + +from microcosm.build.country_spec import load_country_spec +from microcosm.build.uk_runtime.country_adapter import ( + build_uk_country_graph, + validate_uk_country_source_projection, +) +from microcosm.build.uk_runtime.frs_release import load_uk_frs_release + + +def test_country_raw_source_projection_is_current(): + spec = load_country_spec("uk") + validate_uk_country_source_projection(spec) + sources = spec.resolved_spec.resource("sources").domain.to_wire()["sources"] + assert all(row["role"] == "frs_raw_table" for row in sources) + assert all(row["loader"] == "kernel:build_uk_frs_spine" for row in sources) + assert {"frs_adult", "frs_benefits", "frs_child", "frs_househol"} <= { + row["id"] for row in sources + } + assert "uk_national_candidate_2023" not in str(sources) + + +def test_country_source_projection_refuses_divergent_header_pin(tmp_path): + source = Path(__file__).parents[1] / "src/microcosm/build/uk" + destination = tmp_path / "uk" + shutil.copytree(source, destination) + path = destination / "spec/sources.yaml" + text = path.read_text() + first_sha = text.split(" sha256: ", 1)[1].splitlines()[0] + path.write_text(text.replace(first_sha, "f" * 64, 1)) + spec = load_country_spec(destination) + with pytest.raises(ValueError, match="raw-source pins differ"): + validate_uk_country_source_projection(spec) + + +@pytest.mark.requires_uk +def test_country_adapter_compiles_the_same_full_graph_with_all_targets_default(): + built = build_uk_country_graph() + release = load_uk_frs_release() + assert built.config.geography_levels is None + assert built.config.calibration_year == release.calibration_year + assert built.config.source_year == release.survey_year + kernels = {node.kernel for node in built.graph.nodes} + assert {"uk.create@1", "uk.full.target_compilation@1", "uk.full.dense@1"} <= kernels + assert not any( + "national_candidate" in source.name for source in built.graph.sources + ) diff --git a/packages/microcosm-build/tests/test_uk_frs_hmrc_leaves.py b/packages/microcosm-build/tests/test_uk_frs_hmrc_leaves.py index 80d81d5ee..03c657362 100644 --- a/packages/microcosm-build/tests/test_uk_frs_hmrc_leaves.py +++ b/packages/microcosm-build/tests/test_uk_frs_hmrc_leaves.py @@ -1,149 +1,26 @@ +"""Raw FRS extraction shared by the canonical spine income stages.""" + from __future__ import annotations import hashlib +import importlib.util from pathlib import Path import numpy as np import pandas as pd import pytest -from microcosm.build.uk_runtime.frs_hmrc_leaves import ( +from microcosm.build.uk_runtime.frs_hmrc_source import ( FRS_HMRC_INCPBEN_COLUMN, FRS_HMRC_OSSBEN_IDENTIFIABLE_SUBSET_COLUMN, FRS_HMRC_PAY_COLUMN, FRS_HMRC_RETAINED_LEAF_COLUMNS, - FRS_HMRC_RETAINED_LEAF_SOURCE_EVIDENCE, - FRS_HMRC_RETAINED_LEAVES_STAGE_NAME, FRS_HMRC_SRP_REGULAR_CODE5_COLUMN, FRS_HMRC_UBISJA_COLUMN, FRS_WEEKS_IN_YEAR, - UKFRSHMRCRetainedLeavesStageTransform, - retain_uk_frs_hmrc_leaves, -) -from microcosm.build.uk_runtime.national_frame import ( - UKNationalStage, - uk_national_frame, + _materialize_source_leaves, + _read_raw_frs_table, ) -from microcosm.build.uk_runtime.spi_support import ( - SPI_HMRC_EMPLOYMENT_BENEFITS_COLUMN, - SPI_HMRC_EMPLOYMENT_EXPENSES_COLUMN, - SPI_HMRC_MISCELLANEOUS_EMPLOYMENT_INCOME_COLUMN, - SPI_HMRC_OTHER_INCOME_COLUMN, - SPI_HMRC_OTHER_SOCIAL_SECURITY_INCOME_COLUMN, - SPI_HMRC_STATE_PENSION_INCOME_COLUMN, - SPI_HMRC_TAXABLE_TERMINATION_PAY_COLUMN, -) -from microcosm.frame import Frame - - -def _candidate() -> tuple[Frame, np.ndarray]: - raw_households = { - 1: (1001, 1002), - 2: (2001,), - } - raw_person_ids = tuple( - person_id for people in raw_households.values() for person_id in people - ) - spi_person_offset = max(raw_person_ids) + 1 - spi_household_offset = max(raw_households) + 1 - - pre_capital_stacks = [ - (household_id, False, people) for household_id, people in raw_households.items() - ] - pre_capital_stacks.append( - ( - 1 + spi_household_offset, - True, - tuple(person_id + spi_person_offset for person_id in raw_households[1]), - ) - ) - capital_person_offset = ( - max( - person_id - for _household_id, _spi, people in pre_capital_stacks - for person_id in people - ) - + 1 - ) - capital_household_offset = ( - max(household_id for household_id, _spi, _people in pre_capital_stacks) + 1 - ) - clone_zero_stacks = [ - (household_id, spi, False, people) - for household_id, spi, people in pre_capital_stacks - ] + [ - ( - household_id + capital_household_offset, - spi, - True, - tuple(person_id + capital_person_offset for person_id in people), - ) - for household_id, spi, people in pre_capital_stacks - ] - clone_multiplier = 10 ** len( - str( - max( - person_id - for _household_id, _spi, _capital, people in clone_zero_stacks - for person_id in people - ) - ) - ) - - household_rows: list[dict[str, object]] = [] - person_rows: list[dict[str, object]] = [] - for clone_index in range(2): - clone_offset = clone_index * clone_multiplier - for household_id, spi, capital_gains, people in clone_zero_stacks: - descendant_household_id = household_id + clone_offset - household_rows.append( - { - "household_id": descendant_household_id, - "household_weight": 0.0 if spi else 1.0, - "clone_index": clone_index, - "household_is_spi_synthetic": spi, - "household_is_capital_gains_clone": capital_gains, - } - ) - for person_id in people: - descendant_person_id = person_id + clone_offset - source_person_id = ( - person_id - - int(spi) * spi_person_offset - - int(capital_gains) * capital_person_offset - ) - person_rows.append( - { - "person_id": descendant_person_id, - "person_household_id": descendant_household_id, - "person_benunit_id": descendant_person_id, - "expected_source_person_id": source_person_id, - } - ) - # Frame requires group ids sorted ascending (it raises, never reorders), - # so the group tables sort; the person table stays SHUFFLED, which keeps - # this fixture's teeth: person row i never corresponds positionally to - # household row i, so lineage resolution must stay id-keyed — the - # 2024-25 FRS bug class this candidate exists to catch. - household = pd.DataFrame(household_rows).sort_values( - "household_id", ignore_index=True - ) - person = pd.DataFrame(person_rows).sample( - frac=1.0, random_state=7, ignore_index=True - ) - source_person_ids = person.pop("expected_source_person_id").to_numpy(dtype=int) - benunit = pd.DataFrame( - {"benunit_id": person["person_benunit_id"].copy()} - ).sort_values("benunit_id", ignore_index=True) - return ( - uk_national_frame( - person=person, - benunit=benunit, - household=household, - time_period="2023", - ), - source_person_ids, - ) def _write_raw_tables( @@ -211,346 +88,62 @@ def _expected_leaves(source_person_ids: np.ndarray) -> pd.DataFrame: return expected -def test_retains_source_faithful_leaves_across_all_candidate_descendants( - tmp_path: Path, -) -> None: - dataset, source_person_ids = _candidate() - adult_path, benefits_path = _write_raw_tables(tmp_path) - - result = retain_uk_frs_hmrc_leaves( - dataset, - adult_tab_path=adult_path, - benefits_tab_path=benefits_path, - ) - - actual = result.frame.person.loc[:, list(FRS_HMRC_RETAINED_LEAF_COLUMNS)] - assert np.array_equal( - actual.to_numpy(), _expected_leaves(source_person_ids).to_numpy() - ) - assert result.clone_id_multiplier == 10_000 - assert result.spi_person_id_offset == 2_002 - assert result.capital_gains_person_id_offset == 3_005 - assert result.raw_source_people == 3 - assert result.candidate_people == len(dataset.person) - assert result.structural_zero_columns == () - assert result.source_signal_rows == { - FRS_HMRC_PAY_COLUMN: 2, - FRS_HMRC_UBISJA_COLUMN: 1, - FRS_HMRC_INCPBEN_COLUMN: 1, - FRS_HMRC_OSSBEN_IDENTIFIABLE_SUBSET_COLUMN: 1, - FRS_HMRC_SRP_REGULAR_CODE5_COLUMN: 1, - } - assert ( - result.adult_source.sha256 - == hashlib.sha256(adult_path.read_bytes()).hexdigest() - ) - assert ( - result.benefits_source.sha256 - == hashlib.sha256(benefits_path.read_bytes()).hexdigest() - ) - assert result.adult_source.extracted_columns == ("sernum", "person", "inearns") - evidence = result.evidence() - assert evidence["stage"] == FRS_HMRC_RETAINED_LEAVES_STAGE_NAME - assert evidence["retained_leaves"][FRS_HMRC_PAY_COLUMN]["spi_concept"] == "PAY" - assert ( - evidence["retained_leaves"][FRS_HMRC_OSSBEN_IDENTIFIABLE_SUBSET_COLUMN]["scope"] - == "identifiable_subset" - ) - - -def test_partial_leaves_never_populate_full_ossben_or_srp_columns( - tmp_path: Path, -) -> None: - dataset, _source_person_ids = _candidate() - adult_path, benefits_path = _write_raw_tables(tmp_path) - - result = retain_uk_frs_hmrc_leaves( - dataset, - adult_tab_path=adult_path, - benefits_tab_path=benefits_path, - ) - - forbidden = { - SPI_HMRC_EMPLOYMENT_BENEFITS_COLUMN, - SPI_HMRC_EMPLOYMENT_EXPENSES_COLUMN, - SPI_HMRC_OTHER_SOCIAL_SECURITY_INCOME_COLUMN, - SPI_HMRC_TAXABLE_TERMINATION_PAY_COLUMN, - SPI_HMRC_MISCELLANEOUS_EMPLOYMENT_INCOME_COLUMN, - SPI_HMRC_OTHER_INCOME_COLUMN, - SPI_HMRC_STATE_PENSION_INCOME_COLUMN, - } - assert forbidden.isdisjoint(result.frame.person.columns) - assert ( - FRS_HMRC_RETAINED_LEAF_SOURCE_EVIDENCE[ - FRS_HMRC_OSSBEN_IDENTIFIABLE_SUBSET_COLUMN - ]["scope"] - == "identifiable_subset" - ) - - -def test_incpben_is_an_honest_structural_zero_when_code17_is_unobserved( - tmp_path: Path, -) -> None: - dataset, _source_person_ids = _candidate() - adult_path, benefits_path = _write_raw_tables(tmp_path, include_incapacity=False) - - result = retain_uk_frs_hmrc_leaves( - dataset, - adult_tab_path=adult_path, - benefits_tab_path=benefits_path, - ) - - assert (result.frame.person[FRS_HMRC_INCPBEN_COLUMN] == 0.0).all() - assert FRS_HMRC_INCPBEN_COLUMN in result.structural_zero_columns - assert result.evidence()["retained_leaves"][FRS_HMRC_INCPBEN_COLUMN][ - "structural_zero" - ] - - -def test_future_code17_observation_flows_without_a_schema_change( - tmp_path: Path, -) -> None: - dataset, source_person_ids = _candidate() - adult_path, benefits_path = _write_raw_tables(tmp_path, include_incapacity=True) - - result = retain_uk_frs_hmrc_leaves( - dataset, - adult_tab_path=adult_path, - benefits_tab_path=benefits_path, - ) - - expected = np.where(source_person_ids == 2001, 3.0 * FRS_WEEKS_IN_YEAR, 0.0) - assert np.array_equal( - result.frame.person[FRS_HMRC_INCPBEN_COLUMN].to_numpy(), expected +def _read_sources(directory): + adult, identity = _read_raw_frs_table( + directory / "adult.tab", + expected_filename="adult.tab", + source_vintage="2024-25", + required_columns=("sernum", "person", "inearns"), ) - assert FRS_HMRC_INCPBEN_COLUMN not in result.structural_zero_columns - - -def test_transform_is_national_stage_compatible_and_retains_evidence( - tmp_path: Path, -) -> None: - dataset, _source_person_ids = _candidate() - _write_raw_tables(tmp_path) - transform = UKFRSHMRCRetainedLeavesStageTransform.from_raw_frs_directory(tmp_path) - stage = UKNationalStage( - name=FRS_HMRC_RETAINED_LEAVES_STAGE_NAME, - transform=transform, + benefits, _ = _read_raw_frs_table( + directory / "benefits.tab", + expected_filename="benefits.tab", + required_columns=("sernum", "person", "benefit", "benamt", "var2"), ) + return adult, benefits, identity - staged = stage.run(dataset) - - assert set(FRS_HMRC_RETAINED_LEAF_COLUMNS).issubset(staged.person.columns) - assert transform.last_result is not None - assert transform.last_result.frame is staged - - -def test_raw_source_identity_must_exist_on_candidate_base(tmp_path: Path) -> None: - dataset, _source_person_ids = _candidate() - adult_path, benefits_path = _write_raw_tables(tmp_path) - adult = pd.read_csv(adult_path, sep="\t") - adult.loc[len(adult)] = {"SERNUM": 9, "PERSON": 1, "INEARNS": 1, "UNUSED": "x"} - adult.to_csv(adult_path, sep="\t", index=False) - with pytest.raises(ValueError, match="absent from the certified candidate base"): - retain_uk_frs_hmrc_leaves( - dataset, - adult_tab_path=adult_path, - benefits_tab_path=benefits_path, - ) - - -def test_sampled_rung_receipts_the_dropped_raw_surface(tmp_path: Path) -> None: - """A #627 rung build restricts the raw surface and receipts the drop. - - The completeness fence (every raw-survey person present in the base) - cannot hold when the base deliberately carries a sampled subset of - source families; declaring ``sampled_rung`` converts the raise into a - receipted count while the surviving surface stays source-faithful. - """ - - dataset, _source_person_ids = _candidate() - (tmp_path / "clean").mkdir() - (tmp_path / "extra").mkdir() - clean_adult_path, clean_benefits_path = _write_raw_tables(tmp_path / "clean") - strict = retain_uk_frs_hmrc_leaves( - dataset, - adult_tab_path=clean_adult_path, - benefits_tab_path=clean_benefits_path, - ) - adult_path, benefits_path = _write_raw_tables(tmp_path / "extra") - adult = pd.read_csv(adult_path, sep="\t") - adult.loc[len(adult)] = {"SERNUM": 9, "PERSON": 1, "INEARNS": 1, "UNUSED": "x"} - adult.to_csv(adult_path, sep="\t", index=False) +def test_raw_source_extraction_preserves_income_concepts(tmp_path): + adult_path, _ = _write_raw_tables(tmp_path) + adult, benefits, identity = _read_sources(tmp_path) + actual = _materialize_source_leaves(adult, benefits) + expected = _expected_leaves(actual.index.to_numpy()) + pd.testing.assert_frame_equal(actual.reset_index(drop=True), expected) + assert list(adult) == ["sernum", "person", "inearns"] + assert identity.sha256 == hashlib.sha256(adult_path.read_bytes()).hexdigest() + assert identity.rows == 3 - result = retain_uk_frs_hmrc_leaves( - dataset, - adult_tab_path=adult_path, - benefits_tab_path=benefits_path, - sampled_rung=True, - ) - assert result.source_people_outside_candidate == 1 - assert result.evidence()["lineage"]["source_people_outside_candidate"] == 1 - assert strict.source_people_outside_candidate == 0 - # The surviving surface attaches exactly what the strict run attaches. - pd.testing.assert_frame_equal( - result.frame.table("person"), strict.frame.table("person") - ) - # Signal-row evidence remains a fact about the SOURCE: the extra raw - # person's pay carrier is counted even though the rung dropped the row, - # so structural_zero can never be asserted from a sampled-away surface. +def test_absent_incapacity_stays_zero(tmp_path): + _write_raw_tables(tmp_path, include_incapacity=False) + adult, benefits, _ = _read_sources(tmp_path) assert ( - result.source_signal_rows[FRS_HMRC_PAY_COLUMN] - == strict.source_signal_rows[FRS_HMRC_PAY_COLUMN] + 1 - ) - - -def test_candidate_clone_identity_mismatch_fails_closed(tmp_path: Path) -> None: - dataset, _source_person_ids = _candidate() - adult_path, benefits_path = _write_raw_tables(tmp_path) - person = dataset.person.copy() - household = dataset.table("household") - clone_households = set(household.loc[household["clone_index"] == 1, "household_id"]) - tampered_row = person["person_household_id"].isin(clone_households).idxmax() - person.loc[tampered_row, "person_id"] += 500 - tampered = uk_national_frame( - person=person, - benunit=dataset.table("benunit"), - household=household, - time_period="2023", - household_weights=dataset.weights_for("household").values, - ) - - with pytest.raises(ValueError, match="person IDs do not reverse"): - retain_uk_frs_hmrc_leaves( - tampered, - adult_tab_path=adult_path, - benefits_tab_path=benefits_path, - ) + _materialize_source_leaves(adult, benefits)[FRS_HMRC_INCPBEN_COLUMN] == 0 + ).all() @pytest.mark.parametrize( - ("column", "message"), + "mutation,match", [ - ("INEARNS", "missing required column"), - ("PERSON", "missing required column"), + ("duplicate", "unique"), + ("negative_benefit", "non-negative"), + ("nan_earnings", "finite"), ], ) -def test_missing_required_raw_column_fails_closed( - tmp_path: Path, - column: str, - message: str, -) -> None: - dataset, _source_person_ids = _candidate() - adult_path, benefits_path = _write_raw_tables(tmp_path) - adult = pd.read_csv(adult_path, sep="\t").drop(columns=[column]) - adult.to_csv(adult_path, sep="\t", index=False) - - with pytest.raises(ValueError, match=message): - retain_uk_frs_hmrc_leaves( - dataset, - adult_tab_path=adult_path, - benefits_tab_path=benefits_path, - ) - - -def test_negative_relevant_benefit_amount_fails_closed(tmp_path: Path) -> None: - dataset, _source_person_ids = _candidate() - adult_path, benefits_path = _write_raw_tables(tmp_path) - benefits = pd.read_csv(benefits_path, sep="\t") - benefits.loc[benefits["BENEFIT"] == 14, "BENAMT"] = -1.0 - benefits.to_csv(benefits_path, sep="\t", index=False) - - with pytest.raises(ValueError, match="must be non-negative"): - retain_uk_frs_hmrc_leaves( - dataset, - adult_tab_path=adult_path, - benefits_tab_path=benefits_path, - ) - - -@pytest.mark.parametrize( - ("table", "column", "message"), - [ - ("adult", "INEARNS", "ADULT.INEARNS"), - ("benefits", "BENAMT", "relevant BENEFITS.BENAMT"), - ], -) -def test_nonfinite_source_amount_fails_closed( - tmp_path: Path, - table: str, - column: str, - message: str, -) -> None: - dataset, _source_person_ids = _candidate() - adult_path, benefits_path = _write_raw_tables(tmp_path) - path = adult_path if table == "adult" else benefits_path - frame = pd.read_csv(path, sep="\t") - frame.loc[0, column] = np.inf - frame.to_csv(path, sep="\t", index=False) - - with pytest.raises(ValueError, match=message): - retain_uk_frs_hmrc_leaves( - dataset, - adult_tab_path=adult_path, - benefits_tab_path=benefits_path, - ) - - -def test_missing_code16_var2_fails_closed(tmp_path: Path) -> None: - dataset, _source_person_ids = _candidate() - adult_path, benefits_path = _write_raw_tables(tmp_path) - benefits = pd.read_csv(benefits_path, sep="\t") - benefits.loc[benefits["BENEFIT"] == 16, "VAR2"] = np.nan - benefits.to_csv(benefits_path, sep="\t", index=False) - - with pytest.raises(ValueError, match="VAR2 for BENEFIT=16"): - retain_uk_frs_hmrc_leaves( - dataset, - adult_tab_path=adult_path, - benefits_tab_path=benefits_path, - ) - - -def test_checkpoint_metadata_round_trips_the_descent_evidence(tmp_path) -> None: - """A fresh process resumes the retained stage from its record alone. - - The rehydrated result exposes exactly the surface the SPI stage's - descent fence reads — evidence and both content identities — and a - checkpoint whose content no longer matches its recorded output identity - is refused as drifted. - """ - - from microcosm.build.uk_runtime.content_identity import ( - uk_frame_content_identity, - ) - - dataset, _source_person_ids = _candidate() +def test_invalid_raw_source_values_are_refused(tmp_path, mutation, match): _write_raw_tables(tmp_path) - transform = UKFRSHMRCRetainedLeavesStageTransform.from_raw_frs_directory(tmp_path) - staged = transform(dataset) - metadata = transform.checkpoint_metadata() - - resumed = UKFRSHMRCRetainedLeavesStageTransform.from_raw_frs_directory(tmp_path) - resumed.resume_from_checkpoint(metadata, staged) - assert resumed.last_result is not None - assert resumed.last_result.frame is staged - assert resumed.last_result.evidence() == transform.last_result.evidence() - assert resumed.last_result.input_content_identity == uk_frame_content_identity( - dataset - ) - assert resumed.last_result.output_content_identity == uk_frame_content_identity( - staged + adult, benefits, _ = _read_sources(tmp_path) + if mutation == "duplicate": + adult = pd.concat([adult, adult.iloc[:1]], ignore_index=True) + elif mutation == "negative_benefit": + benefits.loc[0, "benamt"] = -1 + else: + adult.loc[0, "inearns"] = np.nan + with pytest.raises(ValueError, match=match): + _materialize_source_leaves(adult, benefits) + + +def test_candidate_restoration_module_is_removed(): + assert ( + importlib.util.find_spec("microcosm.build.uk_runtime.frs_hmrc_leaves") is None ) - - drifted = UKFRSHMRCRetainedLeavesStageTransform.from_raw_frs_directory(tmp_path) - with pytest.raises(RuntimeError, match="drifted record"): - drifted.resume_from_checkpoint(metadata, dataset) - - empty = UKFRSHMRCRetainedLeavesStageTransform.from_raw_frs_directory(tmp_path) - with pytest.raises(RuntimeError, match="cannot prove descent"): - empty.resume_from_checkpoint({}, staged) - - unrun = UKFRSHMRCRetainedLeavesStageTransform.from_raw_frs_directory(tmp_path) - with pytest.raises(RuntimeError, match="completed retained-leaves run"): - unrun.checkpoint_metadata() diff --git a/packages/microcosm-build/tests/test_uk_frs_spine.py b/packages/microcosm-build/tests/test_uk_frs_spine.py index 18e35e77c..8f9303447 100644 --- a/packages/microcosm-build/tests/test_uk_frs_spine.py +++ b/packages/microcosm-build/tests/test_uk_frs_spine.py @@ -1,7 +1,6 @@ from __future__ import annotations import hashlib -import importlib.util import json import sys from importlib import metadata @@ -40,16 +39,11 @@ ) from microcosm.frame import Frame, WeightKind, engine_tables -_TOOL_PATH = Path(__file__).resolve().parents[3] / "tools" / "build_uk_frs_spine.py" - def _load_tool(): - spec = importlib.util.spec_from_file_location("build_uk_frs_spine", _TOOL_PATH) - assert spec is not None - assert spec.loader is not None - module = importlib.util.module_from_spec(spec) - spec.loader.exec_module(module) - return module + from microcosm.build.uk_runtime import spine_build + + return spine_build def _write_tab(root: Path, table: str, rows: list[dict[str, object]]) -> None: @@ -1188,18 +1182,17 @@ def __call__(self, frame: Frame) -> Frame: self.last_result = SimpleNamespace(replay_report={"report_kind": "fake"}) return result - def _write_fake_replay(report, path): - output = Path(path) - output.write_text( - json.dumps({"report_kind": "fake_spine_replay"}) + "\n", - encoding="utf-8", - ) - return output + def checkpoint_metadata(self): + if self.last_result is None: + raise RuntimeError("Stage evidence requires completed computation.") + return { + "evidence": {"stage": self.stage.stage}, + "replay_payload": {"report_kind": "fake_spine_replay"}, + } monkeypatch.setattr(tool, "UKFRSHMRCSpineLeavesStageTransform", _FakeStageTransform) monkeypatch.setattr(tool, "UKSPISupportChannelStageTransform", _FakeStageTransform) monkeypatch.setattr(tool, "UKSPIIncomeSpineStageTransform", _FakeStageTransform) - monkeypatch.setattr(tool, "write_hmrc_replay_report", _write_fake_replay) return spi_tab, hmrc_ods diff --git a/packages/microcosm-build/tests/test_uk_full_build_cli.py b/packages/microcosm-build/tests/test_uk_full_build_cli.py new file mode 100644 index 000000000..3194cf27c --- /dev/null +++ b/packages/microcosm-build/tests/test_uk_full_build_cli.py @@ -0,0 +1,402 @@ +"""The canonical CLI restores declared files and preserves failure/scope semantics.""" + +import hashlib +import json +from dataclasses import replace + +import pytest +from test_uk_graph_terminal import _frame + +from microcosm.build.gate_battery import ( + GateOutcome, + GatePhaseReport, + GateStatus, + gate_phase_report_payload, +) +from microcosm.build.gates import GateResult +from microcosm.build.uk_runtime import full_build_cli as cli +from microcosm.build.uk_runtime.full_certification import FULL_CERTIFICATION_TYPE +from microcosm.build.uk_runtime.full_gates import ( + classify_full_gate_outcomes, + uk_full_gate_manifest, +) +from microcosm.build.uk_runtime.graph_build import UKFullBuildConfig, UKFullGraph +from microcosm.build.uk_runtime.graph_calibration import UKCalibrationNodes +from microcosm.build.uk_runtime.graph_targets import TARGET_SELECTION_TYPE +from microcosm.build.uk_runtime.graph_terminal import ( + FULL_DIAGNOSTICS_CSV_TYPE, + FULL_DIAGNOSTICS_TYPE, + FULL_GATE_REPORT_TYPE, + FULL_HOLDOUT_TYPE, + FULL_SUPPORT_CSV_TYPE, + add_uk_export_preparation, + register_uk_terminal_kernels, +) +from microcosm.graph import ( + ArtifactInput, + ArtifactOutput, + Capabilities, + Determinism, + Graph, + KernelBase, + KernelRegistry, + KernelResult, + Node, + Owned, + SourceRef, + StructuralDelta, +) +from microcosm.graph.canonical import canonical_json + + +def arguments(tmp_path, *extra): + return cli.parse_args( + [ + "--input-h5", + str(tmp_path / "spine.h5"), + "--ladder", + str(tmp_path / "ladder.npz"), + "--ledger-facts", + str(tmp_path / "ledger"), + "--out", + str(tmp_path / "out"), + *extra, + ] + ) + + +def test_every_scope_and_size_control_keeps_default_all(tmp_path): + for extra in ( + (), + ("--target-geographies", "all"), + ("--n-clones", "1"), + ("--dataset-households", "10"), + ("--n-clones", "1", "--dataset-households", "10"), + ): + assert arguments(tmp_path, *extra).target_geographies is None + assert arguments( + tmp_path, "--target-geographies", "country" + ).target_geographies == ("country",) + with pytest.raises(SystemExit): + arguments(tmp_path, "--target-geographies", "national") + + +def test_source_sampling_cannot_be_reapplied_as_pool_sampling(): + config = UKFullBuildConfig(calibration_year=2025, source_sample_fraction=0.1) + assert config.sample_fraction == 1.0 + assert config.effective_sample_fraction == 0.1 + with pytest.raises(ValueError, match="second time"): + replace(config, sample_fraction=0.1) + + +def gate_payload(phase, failed=None): + selection = { + "schema": "microcosm.calibrate.target-selection.v1", + "selector": {"geography_levels": None, "explicit": False}, + "included": [ + {"name": "count", "period": 2025, "geography_level": "country"}, + {"name": "local", "period": 2025, "geography_level": "constituency"}, + ], + "excluded": [], + } + gates = uk_full_gate_manifest(selection) + report = GatePhaseReport( + phase, + tuple( + GateOutcome( + entry, + GateStatus.FAILED if entry.id == failed else GateStatus.PASSED, + GateResult( + name=entry.id, + passed=entry.id != failed, + details={}, + failures=("synthetic failure",) if entry.id == failed else (), + ), + ) + for entry in gates.gates + if entry.phase == phase + ), + ) + return canonical_json( + { + "schema_version": 1, + "kind": "uk_full_gate_report", + "selection_receipt": selection, + "sample_fraction": 1.0, + "release_candidate": False, + "report": gate_phase_report_payload(report, gates=gates), + "enforcement": classify_full_gate_outcomes( + report, sample_fraction=1.0, release_candidate=False + ), + } + ) + + +class Fixture(KernelBase): + ref = "uk.test.cli-frame@1" + capabilities = Capabilities( + Determinism.DETERMINISTIC, structural=StructuralDelta.CREATE + ) + + def implementation_hash(self): + return hashlib.sha256(self.ref.encode()).hexdigest() + + def run(self, context): + return KernelResult(frame=_frame()) + + +class Evidence(KernelBase): + ref = "uk.test.cli-evidence@1" + capabilities = Capabilities(Determinism.DETERMINISTIC) + + def implementation_hash(self): + return hashlib.sha256(self.ref.encode()).hexdigest() + + def run(self, context): + phase = context.params["phase"] + artifacts = {"gate_report": gate_payload(phase, context.params.get("failed"))} + if phase == "terminal": + artifacts.update( + calibration_diagnostics=b'{"fixture":true}', + target_diagnostics_csv=b"name,actual\ncount,100\n", + area_support_csv=b"area,households\nfixture,2\n", + ) + return KernelResult(artifacts=artifacts) + + +class Holdout(Evidence): + ref = "uk.test.cli-holdout@1" + + def run(self, context): + return KernelResult( + artifacts={"holdout": b'{"fixture":true}', "selection": b'{"fixture":true}'} + ) + + +class Certification(Evidence): + ref = "uk.test.cli-certification@1" + + def run(self, context): + return KernelResult( + artifacts={ + "certification_readiness": b'{"fixture":true,"release_authorized":false}' + } + ) + + +@pytest.fixture(autouse=True) +def certification_service_fixture(monkeypatch): + # Scientific certification validation has its own graph-artifact tests. + # This suite tests the filesystem/execution service with synthetic evidence. + def append(graph, *, population, **kwargs): + return replace( + graph, + nodes=( + *graph.nodes, + Node( + "uk.full.certification", + Certification.ref, + population=population, + artifact_outputs=( + ArtifactOutput( + "certification_readiness", FULL_CERTIFICATION_TYPE + ), + ), + ), + ), + ) + + monkeypatch.setattr(cli, "append_uk_full_certification_node", append) + + +def prepared(tmp_path, failed=None): + frame = _frame() + fixture = tmp_path / "fixture.txt" + fixture.write_text("constant source") + identifiers = { + "person_id", + "person_household_id", + "person_benunit_id", + "household_id", + "benunit_id", + } + root = Node( + "uk.full.calibrated", + Fixture.ref, + structural=StructuralDelta.CREATE, + sources=("fixture",), + outputs=tuple( + Owned( + e, + str(c), + "string" + if frame.table(e)[c].dtype.kind in "OUS" + else str(frame.table(e)[c].dtype), + ) + for e in frame.entities + for c in frame.table(e).columns + if c not in identifiers + ), + ) + nodes = [ + root, + Node( + "uk.full.gates.preflight", + Evidence.ref, + population=root.id, + params={"phase": "preflight", "failed": failed}, + artifact_outputs=(ArtifactOutput("gate_report", FULL_GATE_REPORT_TYPE),), + ), + Node( + "uk.full.gates.calibrated", + Evidence.ref, + population=root.id, + params={"phase": "terminal", "failed": failed}, + artifact_outputs=( + ArtifactOutput("gate_report", FULL_GATE_REPORT_TYPE), + ArtifactOutput("calibration_diagnostics", FULL_DIAGNOSTICS_TYPE), + ArtifactOutput("target_diagnostics_csv", FULL_DIAGNOSTICS_CSV_TYPE), + ArtifactOutput("area_support_csv", FULL_SUPPORT_CSV_TYPE), + ), + ), + Node( + "uk.full.holdout", + Holdout.ref, + population=root.id, + artifact_outputs=( + ArtifactOutput("holdout", FULL_HOLDOUT_TYPE), + ArtifactOutput("selection", TARGET_SELECTION_TYPE), + ), + ), + ] + # CLI materialization consumes the public target-selection endpoint. + nodes.append( + Node( + "uk.full.target_selection", + Holdout.ref, + population=root.id, + artifact_outputs=( + ArtifactOutput("holdout", FULL_HOLDOUT_TYPE), + ArtifactOutput("selection", TARGET_SELECTION_TYPE), + ), + ) + ) + nodes = [ + replace( + node, + artifact_inputs=( + ArtifactInput( + "preflight", + "uk.full.gates.preflight", + "gate_report", + FULL_GATE_REPORT_TYPE, + ), + ArtifactInput( + "holdout", "uk.full.holdout", "holdout", FULL_HOLDOUT_TYPE + ), + ArtifactInput( + "selection", + "uk.full.target_selection", + "selection", + TARGET_SELECTION_TYPE, + ), + ), + ) + if node.id == "uk.full.gates.calibrated" + else node + for node in nodes + ] + graph = Graph("uk", (SourceRef("fixture", "raw-bytes-v1"),), tuple(nodes)) + graph = add_uk_export_preparation( + graph, + population=root.id, + bindings={"target_scope": "all"}, + artifact_inputs=( + ArtifactInput( + "gates", + "uk.full.gates.calibrated", + "gate_report", + FULL_GATE_REPORT_TYPE, + ), + ), + ) + calibration = UKCalibrationNodes( + (), root.id, "unused", "unused", "unused", None, "unused" + ) + full = UKFullGraph(graph, calibration, UKFullBuildConfig(calibration_year=2025)) + kernels = KernelRegistry() + for kernel in (Fixture(), Evidence(), Holdout(), Certification()): + kernels.register(kernel) + register_uk_terminal_kernels(kernels) + return cli.PreparedUKFullBuild( + full, kernels, {"fixture": fixture}, {"target_scope": "all"} + ) + + +def test_cli_cold_and_required_replay_recreate_dataset_and_sidecars( + tmp_path, monkeypatch +): + pytest.importorskip("tables") + args = arguments(tmp_path) + first = prepared(tmp_path) + assert cli.execute_full_build(first, args) == 0 + out = args.out + expected = json.loads((out / "build.json").read_text()) + assert expected["readback_passed"] is True + assert expected["release_authorized"] is False + assert (out / "microcosm_uk_2025.targets.csv").read_text().startswith("name,actual") + for path in out.iterdir(): + if path.is_file(): + path.unlink() + + def forbidden(*args): + raise AssertionError( + "Required replay repeated a completed numerical/evidence node" + ) + + for kernel in (Fixture, Evidence, Holdout): + monkeypatch.setattr(kernel, "run", forbidden) + args.resume = "require" + assert cli.execute_full_build(prepared(tmp_path), args) == 0 + actual = json.loads((out / "build.json").read_text()) + assert actual["content_sha256"] == expected["content_sha256"] + assert (out / "microcosm_uk_2025.h5").is_file() + assert (out / "microcosm_uk_2025.holdout.json").is_file() + + +@pytest.mark.parametrize( + "failure,exported", + [ + ("uk_local_geography_ladder_post_calibration", False), + ("uk_local_target_fit", True), + ], +) +def test_cli_retains_failed_evidence_and_correct_status(tmp_path, failure, exported): + if exported: + pytest.importorskip("tables") + args = arguments(tmp_path) + build = prepared(tmp_path, failure) + assert cli.execute_full_build(build, args) == 1 + assert (args.out / "uk.full.gates.calibrated.gate_report.json").is_file() + assert (args.out / "microcosm_uk_2025.h5").exists() == exported + + +def test_dry_run_has_no_files_or_kernel_execution(tmp_path, monkeypatch, capsys): + args = arguments(tmp_path, "--dry-run") + build = prepared(tmp_path) + monkeypatch.setattr( + cli, "run_graph", lambda *a, **k: pytest.fail("dry run executed graph") + ) + assert cli.execute_full_build(build, args) == 0 + assert json.loads(capsys.readouterr().out)["default_scope"] == "all_geographies" + assert not args.out.exists() + + +def test_rejected_output_inside_source_never_writes_failure_sidecar(tmp_path, monkeypatch): + args = arguments(tmp_path) + build = prepared(tmp_path) + build = replace(build, sources={"fixture": tmp_path}) + monkeypatch.setattr(cli, "parse_args", lambda argv: args) + monkeypatch.setattr(cli, "prepare_full_build", lambda args: build) + assert cli.main([]) == 1 + assert not args.out.exists() diff --git a/packages/microcosm-build/tests/test_uk_full_build_preparation.py b/packages/microcosm-build/tests/test_uk_full_build_preparation.py new file mode 100644 index 000000000..51ab2d86f --- /dev/null +++ b/packages/microcosm-build/tests/test_uk_full_build_preparation.py @@ -0,0 +1,110 @@ +"""Canonical CLI preparation authenticates real H5 checkpoint boundaries.""" + +import json + +import pytest +from test_uk_calibration_run import _bound_checkpoint +from test_uk_full_population_graph import source_frame +from test_uk_ladder_rowwise_clone import toy_ladder as toy_ladder + +from microcosm.build.uk_runtime import full_build_cli as cli +from microcosm.build.uk_runtime import spine_build +from microcosm.build.uk_runtime.national_frame import ( + load_uk_national_frame, + write_uk_national_frame, +) +from microcosm.graph import ContentStore, compile_graph, run_graph + + +@pytest.fixture +def checkpoint_request(tmp_path, toy_ladder, monkeypatch): + pytest.importorskip("tables") + _, ladder_path = toy_ladder + path = write_uk_national_frame(source_frame(), tmp_path / "spine.h5") + frame, _ = load_uk_national_frame(path) + sidecar_path, gates_path, sidecar = _bound_checkpoint(tmp_path, frame) + sidecar["stages"] = ["frs_spine"] + sidecar["sampling"] = {"fraction": 1.0, "seed": 7} + sidecar_path.write_text(json.dumps(sidecar)) + ledger = tmp_path / "ledger" + ledger.mkdir() + (ledger / "facts.csv").write_text("fixture-only; unused during preparation") + monkeypatch.setattr(spine_build, "_rules_engine", lambda: object()) + monkeypatch.setattr( + spine_build, "_rules_engine_provenance", lambda: {"version": "fixture"} + ) + args = cli.parse_args( + [ + "--input-h5", + str(path), + "--input-sidecar", + str(sidecar_path), + "--input-spine-gates", + str(gates_path), + "--ladder", + str(ladder_path), + "--ledger-facts", + str(ledger), + "--out", + str(tmp_path / "out"), + "--n-clones", + "1", + "--epochs", + "8", + ] + ) + return args, sidecar_path, gates_path + + +def test_prepare_real_checkpoint_preserves_source_year_and_wires_preflight( + checkpoint_request, tmp_path +): + args, _, _ = checkpoint_request + prepared = cli.prepare_full_build(args) + assert prepared.full.config.source_year == 2023 + assert prepared.full.config.geography_levels is None + dense = prepared.full.graph.node("uk.full.dense") + assert any( + a.name == "preflight" and a.producer == "uk.full.gates.preflight" + for a in dense.artifact_inputs + ) + endpoint = cli._through(prepared.full.graph, "uk.full.spine_checkpoint") + store = ContentStore(tmp_path / "store") + first = run_graph( + compile_graph(endpoint), + sources=prepared.sources, + store=store, + kernels=prepared.kernels, + ) + provenance_key = first.nodes["uk.full.spine_checkpoint"].opaque_artifacts[ + "spine_provenance" + ] + provenance = json.loads(store.load_bytes(provenance_key)) + assert provenance["stages"] == ["frs_spine"] + assert provenance["fit_weight_records"] == {"model": {"fit_weights_used": True}} + fresh = cli.prepare_full_build(args) + replay = run_graph( + compile_graph(cli._through(fresh.full.graph, "uk.full.spine_checkpoint")), + sources=fresh.sources, + store=store, + kernels=fresh.kernels, + resume="require", + ) + assert replay.nodes["uk.full.spine_checkpoint"].hit + assert not args.out.exists() + + +@pytest.mark.parametrize("damage", ["identity", "gate_bytes"]) +def test_prepare_refuses_checkpoint_drift_before_registering_a_full_build( + checkpoint_request, damage +): + args, sidecar_path, gates_path = checkpoint_request + if damage == "identity": + value = json.loads(sidecar_path.read_text()) + value["uk_frame_content_identity"] = "f" * 64 + sidecar_path.write_text(json.dumps(value)) + else: + gates_path.write_text(gates_path.read_text() + "\n") + with pytest.raises(ValueError, match="identity mismatch|SHA-256 mismatch"): + cli.prepare_full_build(args) + assert not args.out.exists() diff --git a/packages/microcosm-build/tests/test_uk_full_calibration_graph.py b/packages/microcosm-build/tests/test_uk_full_calibration_graph.py new file mode 100644 index 000000000..f300eeb6a --- /dev/null +++ b/packages/microcosm-build/tests/test_uk_full_calibration_graph.py @@ -0,0 +1,487 @@ +"""The full graph preserves dense and exact-count numerical paths and replay.""" + +import hashlib +import json +from dataclasses import replace + +import numpy as np +import pandas as pd +import pytest +from test_uk_local_rowwise import _clone_frame + +from microcosm.build.uk_runtime import dataset_size +from microcosm.build.uk_runtime.graph_calibration import ( + UKGraphCalibrationConfig, + register_uk_calibration_kernels, + restore_uk_graph_result, + uk_calibration_nodes, +) +from microcosm.build.uk_runtime.graph_terminal import FULL_GATE_REPORT_TYPE +from microcosm.calibrate import Target, TargetSet, build_constraint_matrix, calibrate +from microcosm.calibrate.artifacts import ( + PROBLEM_TYPE, + decode_calibration_result, + decode_problem, + encode_problem, +) +from microcosm.frame import Frame +from microcosm.graph import ( + ArtifactInput, + ArtifactOutput, + Capabilities, + ContentStore, + Determinism, + Graph, + KernelBase, + KernelRegistry, + KernelResult, + Node, + Owned, + SourceRef, + StructuralDelta, + compile_graph, + run_graph, +) +from microcosm.graph.canonical import canonical_json + + +def source_frame(): + frame = _clone_frame() + tables = {entity: frame.table(entity).copy() for entity in frame.entities} + tables["household"]["marker"] = [5, 7, 9] + tables["person"]["age"] = [30, 40, 50] + tables["benunit"]["eligible"] = [True, True, False] + return Frame( + tables, + frame.schema, + {"household": frame.weights_for("household")}, + frame.strata, + mass_log=frame.mass_log, + metadata=frame.metadata, + ) + + +def targets(): + return TargetSet( + [Target("count", "household", lambda f: np.ones(f.n("household")), 3)] + ) + + +def binding(): + return { + "mass_reason": "Fixture selected constraints", + "max_weight_ratio": 10.0, + "target_loss_weights": [1.0], + "target_loss_cap": 10.0, + "selector": "all", + } + + +def problem_payload(): + frame = source_frame() + return encode_problem( + build_constraint_matrix(frame, targets(), weight_entity="household"), + entity_ids=frame.table("household")["household_id"].tolist(), + bindings=binding(), + ) + + +def preflight_payload(passed=True, selection=None): + from microcosm.build.gate_battery import ( + GateOutcome, + GatePhaseReport, + GateStatus, + gate_phase_report_payload, + ) + from microcosm.build.gates import GateResult + from microcosm.build.uk_runtime.full_gates import ( + classify_full_gate_outcomes, + uk_full_gate_manifest, + ) + + selection = ( + selection + if selection is not None + else { + "schema": "microcosm.calibrate.target-selection.v1", + "selector": {"geography_levels": ["country"], "explicit": True}, + "included": [{"name": "count", "period": 0, "geography_level": "country"}], + "excluded": [], + } + ) + gates = uk_full_gate_manifest(selection) + report = GatePhaseReport( + "preflight", + tuple( + GateOutcome( + entry=entry, + status=GateStatus.PASSED if passed else GateStatus.FAILED, + result=GateResult( + name=entry.gate, + passed=passed, + failures=() if passed else ("fixture blocked",), + ), + ) + for entry in gates.gates + if entry.phase == "preflight" + ), + ) + return canonical_json( + { + "schema_version": 1, + "kind": "uk_full_gate_report", + "selection_receipt": selection, + "sample_fraction": 1.0, + "release_candidate": True, + "report": gate_phase_report_payload(report, gates=gates), + "enforcement": classify_full_gate_outcomes( + report, sample_fraction=1.0, release_candidate=True + ), + } + ) + + +class Source(KernelBase): + ref = "uk.test.solver-source@1" + capabilities = Capabilities( + Determinism.DETERMINISTIC, structural=StructuralDelta.CREATE + ) + + def implementation_hash(self): + return hashlib.sha256(self.ref.encode()).hexdigest() + + def run(self, context): + return KernelResult( + frame=source_frame(), + artifacts={ + "problem": problem_payload(), + "preflight": preflight_payload(context.params["preflight_passed"]), + }, + ) + + +def compiled(k, checkpoint_identity=None, preflight_passed=True): + calibration = uk_calibration_nodes( + base="pool", + columns={ + ("household", "marker"): "int64", + ("person", "age"): "int64", + ("benunit", "eligible"): "bool", + }, + problem_producer="pool", + config=UKGraphCalibrationConfig( + epochs=2, learning_rate=0.02, seed=7, dataset_households=k + ), + checkpoint_identity=checkpoint_identity, + ) + graph = Graph( + "uk", + ( + SourceRef("fixture", "raw-bytes-v1"), + *( + () + if checkpoint_identity is None + else ( + SourceRef("uk_size_checkpoint_manifest", "raw-bytes-v1"), + SourceRef("uk_size_checkpoint_arrays", "raw-bytes-v1"), + ) + ), + ), + ( + Node( + "pool", + Source.ref, + sources=("fixture",), + params={"preflight_passed": preflight_passed}, + structural=StructuralDelta.CREATE, + outputs=( + Owned("household", "marker", "int64"), + Owned("person", "age", "int64"), + Owned("benunit", "eligible", "bool"), + ), + artifact_outputs=( + ArtifactOutput("problem", PROBLEM_TYPE), + ArtifactOutput("preflight", FULL_GATE_REPORT_TYPE), + ), + ), + *( + replace( + node, + artifact_inputs=( + *node.artifact_inputs, + ArtifactInput( + "preflight", "pool", "preflight", FULL_GATE_REPORT_TYPE + ), + ), + ) + if node.id == calibration.dense_producer + else node + for node in calibration.nodes + ), + ), + ) + return compile_graph(graph), calibration + + +def registry(): + kernels = KernelRegistry() + kernels.register(Source()) + return register_uk_calibration_kernels(kernels) + + +@pytest.mark.parametrize("k", [None, 2, 3]) +def test_graph_preserves_numerical_path_and_complete_resume(k, tmp_path, monkeypatch): + frame = source_frame() + dense = calibrate( + frame, + targets(), + weight_entity="household", + epochs=2, + learning_rate=0.02, + seed=7, + mass="free", + **binding_solver(), + ) + expected = ( + dense + if k is None + else dataset_size.refit_uk_dataset_size( + frame, dense, households=k, epochs=2, learning_rate=0.02, seed=7 + ).result + ) + graph, endpoints = compiled(k) + fixture = tmp_path / "fixture" + fixture.write_bytes(b"fixture") + store = ContentStore(tmp_path / "store") + first = run_graph( + graph, sources={"fixture": fixture}, store=store, kernels=registry() + ) + actual = first.population(endpoints.population) + for entity in frame.entities: + pd.testing.assert_frame_equal( + actual.table(entity), expected.frame.table(entity) + ) + np.testing.assert_array_equal( + actual.weights_for("household").values, expected.weights + ) + assert actual.mass_log == expected.frame.mass_log + assert actual.weights_for("household").kind.value == "calibrated" + + def payload(producer, name): + return store.load_bytes(first.node(producer).opaque_artifacts[name]) + + restored = restore_uk_graph_result( + frame, + problem_payload=payload(endpoints.problem_producer, "problem"), + result_payload=payload(endpoints.result_producer, "result"), + solution_payload=payload(endpoints.solution_producer, "solution"), + original_problem_payload=problem_payload(), + ) + assert restored.frame.mass_log == expected.frame.mass_log + np.testing.assert_array_equal(restored.initial_weights, expected.initial_weights) + np.testing.assert_array_equal(restored.loss_trajectory, expected.loss_trajectory) + assert restored.options == expected.options + if k == 2: + assert graph.versions["uk.full.size_search"] == "pool" + assert graph.versions["uk.full.size_refit"] == "pool" + assert graph.graph.node("uk.full.selected").structural is StructuralDelta.FILTER + if k == 3: + receipt = json.loads( + store.load_bytes(first.node("uk.full.size_refit").opaque_artifacts["size"]) + ) + assert receipt["method"] == "full_pool" + # Identity is source-authored; prohibit execution while retaining the + # original kernel source identity in this simulated fresh registry. + original_registry = registry() + for kernel in original_registry.as_mapping().values(): + monkeypatch.setattr( + kernel, "run", lambda *a, **kw: pytest.fail("replay executed") + ) + replay = run_graph( + graph, + sources={"fixture": fixture}, + store=store, + kernels=original_registry, + resume="require", + ) + np.testing.assert_array_equal( + replay.population(endpoints.population).weights_for("household").values, + expected.weights, + ) + + +def binding_solver(): + value = binding() + value.pop("selector") + return value + + +def test_reused_draw_skips_rng_and_rejects_changed_binding(monkeypatch): + frame = source_frame() + dense = calibrate(frame, targets(), epochs=2) + options = dict(households=2, epochs=2, learning_rate=0.02, seed=7) + selection = dataset_size.select_uk_dataset_size(frame, dense, **options) + draw = dataset_size.draw_uk_dataset_size( + frame, dense, selection=selection, households=2, seed=7 + ) + expected = dataset_size.refit_uk_dataset_size( + frame, dense, selection=selection, draw=draw, **options + ) + monkeypatch.setattr( + dataset_size, "select_exact_k", lambda *a, **kw: pytest.fail("draw repeated") + ) + again = dataset_size.refit_uk_dataset_size( + frame, dense, selection=selection, draw=draw, **options + ) + np.testing.assert_array_equal(expected.result.weights, again.result.weights) + from dataclasses import replace + + with pytest.raises(ValueError, match="differs from its selection"): + dataset_size.refit_uk_dataset_size( + frame, + dense, + selection=selection, + draw=replace(draw, probabilities_sha256="0" * 64), + **options, + ) + + +def test_completed_result_can_rebuild_without_optimizer(): + # A separately serialized result is also independently inspectable; the + # graph cache is not the only way to obtain diagnostics on resume. + from microcosm.calibrate.artifacts import encode_calibration_result + + frame = source_frame() + problem = decode_problem(problem_payload()) + result = calibrate(frame, problem.to_target_set(), epochs=2) + replay = decode_calibration_result( + encode_calibration_result( + result, entity_ids=problem.entity_ids, problem_sha256=problem.sha256 + ), + frame=frame, + problem=problem, + ) + assert replay.problem.names == result.problem.names + np.testing.assert_array_equal(replay.initial_weights, result.initial_weights) + + +def test_external_search_checkpoint_is_imported_without_repeating_solves( + tmp_path, monkeypatch +): + from microcosm.build.uk_runtime import graph_calibration, size_checkpoint + from microcosm.graph.errors import NodeRejectedError + + frame = source_frame() + dense = calibrate( + frame, + targets(), + weight_entity="household", + epochs=2, + learning_rate=0.02, + seed=7, + mass="free", + **binding_solver(), + ) + selection = dataset_size.select_uk_dataset_size( + frame, dense, households=2, epochs=2, learning_rate=0.02, seed=7 + ) + expected = dataset_size.refit_uk_dataset_size( + frame, + dense, + selection=selection, + households=2, + epochs=2, + learning_rate=0.02, + seed=7, + ) + identity = {"pool": "fixture", "K": 1, "selector": "all", "k": 2} + directory = tmp_path / "legacy" + size_checkpoint.write_uk_size_checkpoint( + directory, frame=frame, dense=dense, selection=selection, identity=identity + ) + fixture = tmp_path / "fixture" + fixture.write_bytes(b"fixture") + sources = { + "fixture": fixture, + "uk_size_checkpoint_manifest": directory + / size_checkpoint.SIZE_CHECKPOINT_MANIFEST_FILENAME, + "uk_size_checkpoint_arrays": directory + / size_checkpoint.SIZE_CHECKPOINT_ARRAYS_FILENAME, + } + monkeypatch.setattr( + graph_calibration, + "calibrate", + lambda *a, **kw: pytest.fail("dense solve repeated"), + ) + monkeypatch.setattr( + dataset_size, + "select_uk_dataset_size", + lambda *a, **kw: pytest.fail("search repeated"), + ) + graph, endpoints = compiled(2, checkpoint_identity=identity) + result = run_graph( + graph, + sources=sources, + store=ContentStore(tmp_path / "store"), + kernels=registry(), + ) + np.testing.assert_array_equal( + result.population(endpoints.population).weights_for("household").values, + expected.result.weights, + ) + assert "uk.full.size_checkpoint_import" in graph.predecessors["uk.full.dense"] + assert "uk.full.size_checkpoint_import" in graph.predecessors["uk.full.size_search"] + for changed in ({**identity, "K": 2}, {"K": 1}): + drift, _ = compiled(2, checkpoint_identity=changed) + with pytest.raises(NodeRejectedError, match="identity"): + run_graph( + drift, + sources=sources, + store=ContentStore(tmp_path / "drift"), + kernels=registry(), + ) + + +def test_changing_k_reuses_dense_but_source_bytes_invalidate_it(tmp_path, monkeypatch): + fixture = tmp_path / "fixture" + fixture.write_bytes(b"first source") + store = ContentStore(tmp_path / "store") + first_graph, _ = compiled(2) + run_graph( + first_graph, sources={"fixture": fixture}, store=store, kernels=registry() + ) + second_graph, _ = compiled(3) + kernels = registry() + for ref in (Source.ref, "uk.full.dense@1"): + monkeypatch.setattr( + kernels.get(ref), + "run", + lambda *a, **kw: pytest.fail("k reran upstream work"), + ) + resized = run_graph( + second_graph, sources={"fixture": fixture}, store=store, kernels=kernels + ) + assert resized.node("pool").hit + assert resized.node("uk.full.dense").hit + assert not resized.node("uk.full.size_search").hit + fixture.write_bytes(b"changed source") + rebuilt = run_graph( + second_graph, sources={"fixture": fixture}, store=store, kernels=registry() + ) + assert not rebuilt.node("pool").hit + assert not rebuilt.node("uk.full.dense").hit + + +@pytest.mark.parametrize("imported", [False, True]) +def test_blocking_preflight_refuses_before_dense_or_checkpoint_work(imported): + from types import SimpleNamespace + + from microcosm.build.uk_runtime.graph_calibration import UKDenseSolveKernel + + artifacts = {"preflight": SimpleNamespace(payload=preflight_payload(False))} + if imported: + # Deliberately unreadable result: the preflight must refuse before + # decoding an imported solution, just as it must before optimization. + artifacts["imported_dense"] = SimpleNamespace(payload=b"must not read") + with pytest.raises(ValueError, match="refused by the source preflight"): + UKDenseSolveKernel().run(SimpleNamespace(artifacts=artifacts)) diff --git a/packages/microcosm-build/tests/test_uk_full_certification.py b/packages/microcosm-build/tests/test_uk_full_certification.py new file mode 100644 index 000000000..aef69f2ae --- /dev/null +++ b/packages/microcosm-build/tests/test_uk_full_certification.py @@ -0,0 +1,288 @@ +"""One full graph certifies its own scope and bytes without signed lane joins.""" + +import hashlib +import json +import sys + +import pytest + +from microcosm.build.country_spec import load_country_spec +from microcosm.build.gate_battery import ( + GateOutcome, + GatePhaseReport, + GateStatus, + gate_phase_report_payload, +) +from microcosm.build.gates import GateResult +from microcosm.build.uk_runtime import full_certification as runtime +from microcosm.build.uk_runtime.full_gates import ( + classify_full_gate_outcomes, + uk_full_gate_manifest, + uk_full_gate_scope_receipt, +) +from microcosm.build.uk_runtime.graph_evidence import uk_spine_gate_manifest +from microcosm.graph.canonical import canonical_json + + +@pytest.fixture(scope="module", autouse=True) +def _one_validated_country_spec(): + """These artifact tests share immutable declarations; source loading is separate.""" + from microcosm.build.uk_runtime import full_gates + + spec = load_country_spec("uk") + with pytest.MonkeyPatch.context() as patch: + for module in (runtime, full_gates, sys.modules[__name__]): + patch.setattr(module, "load_country_spec", lambda country: spec) + yield + + +def _passed(gates, phase): + return GatePhaseReport( + phase, + tuple( + GateOutcome(entry, GateStatus.PASSED, GateResult(entry.gate, True, (), {})) + for entry in gates.gates + if entry.phase == phase + ), + ) + + +def _fixture(*, country=False, k=None): + selection = { + "schema": "microcosm.calibrate.target-selection.v1", + "selector": {"geography_levels": ["country"] if country else None}, + "included": [ + {"name": "national", "period": 2025, "geography_level": "country"} + ], + "excluded": [], + } + if not country: + selection["included"].append( + {"name": "local", "period": 2025, "geography_level": "constituency"} + ) + names = set(runtime._REQUIRED) | {"spine_assembled", "spine_transferred"} + keys = {name: hashlib.sha256(name.encode()).hexdigest() for name in names} + gate_manifest = uk_full_gate_manifest(selection) + documents = { + "selection": {"receipt": selection}, + "diagnostics": {"targets": []}, + "holdout": { + "skipped": False, + "method": "rotated_folds", + "n_folds": 5, + "folds": [{"holdout_loss": 0.01}] * 5, + "mean_holdout_loss": 0.01, + "worst_holdout_loss": 0.01, + }, + } + for role, phase in (("preflight", "preflight"), ("full_gates", "terminal")): + report = _passed(gate_manifest, phase) + documents[role] = { + "schema_version": 1, + "kind": "uk_full_gate_report", + "selection_receipt": selection, + "sample_fraction": 1.0, + "release_candidate": False, + "scope": uk_full_gate_scope_receipt(selection), + "report": gate_phase_report_payload(report, gates=gate_manifest), + "enforcement": classify_full_gate_outcomes( + report, sample_fraction=1.0, release_candidate=False + ), + "artifacts": { + name: keys[name] + for name in ("surface", "selection", "preflight", "holdout") + }, + } + spine = uk_spine_gate_manifest(load_country_spec("uk")) + for phase in ("assembled", "transferred"): + documents[f"spine_{phase}"] = gate_phase_report_payload( + _passed(spine, phase), gates=spine + ) + bindings = {"configuration": {"calibration": {"dataset_households": k}}} + dataset = {"filename": "candidate.h5", "sha256": "d" * 64, "size_bytes": 123} + documents["export_descriptor"] = { + "kind": "uk_full_build_export", + "schema_version": 1, + "content_sha256": "a" * 64, + "bindings": bindings, + "tables": {"household": {"rows": k or 12}}, + } + documents["export_readback"] = { + "kind": "uk_full_build_export_readback", + "passed": True, + "dataset": dataset, + "content_sha256": "a" * 64, + "bindings": bindings, + } + documents["package"] = { + "kind": "uk_full_build_package", + "readback_passed": True, + "release_authorized": False, + "schema_version": 1, + "dataset": dataset, + "content_sha256": "a" * 64, + "build_bindings": bindings, + "artifacts": { + name: keys[name] + for name in ("full_gates", "diagnostics", "holdout", "export_readback") + }, + } + documents["surface"] = { + "source_validation": { + "ledger_provenance": {"facts_sha256": "b" * 64, "manifest_sha256": "c" * 64} + } + } + return { + name: (keys[name], canonical_json(document)) + for name, document in documents.items() + } + + +def _rewrite(artifacts, name, mutate): + key, payload = artifacts[name] + document = json.loads(payload) + mutate(document) + artifacts[name] = (key, canonical_json(document)) + + +def _scores(artifacts, monkeypatch, *, k=None): + def validate(payload, failures, *, expected_identity): + if payload.get("identity") != expected_identity: + failures.append("scorecard candidate identity mismatch") + + monkeypatch.setattr( + "microcosm.data.contract._check_uk_incumbent_surface_evaluation", validate + ) + identity = { + "candidate_dataset_sha256": "d" * 64, + "candidate_manifest_sha256": hashlib.sha256( + artifacts["package"][1] + ).hexdigest(), + "candidate_diagnostics_sha256": hashlib.sha256( + artifacts["diagnostics"][1] + ).hexdigest(), + "ledger_facts_sha256": "b" * 64, + "ledger_manifest_sha256": "c" * 64, + } + sources = {} + for role in ( + ("native_scorecard", "matched_size_scorecard") if k else ("native_scorecard",) + ): + document = {"identity": identity} + if role == "matched_size_scorecard": + document["comparison"] = { + "kind": "matched_size", + "candidate_households": k, + "incumbent_households": k, + } + payload = canonical_json(document) + sources[role] = ( + { + "filename": role + ".json", + "sha256": hashlib.sha256(payload).hexdigest(), + "size_bytes": len(payload), + }, + payload, + ) + return sources + + +def test_complete_graph_retains_explicit_missing_native_evidence(): + report = runtime.compose_uk_full_certification_readiness(_fixture()) + assert report["comparisons"]["native_scorecard"]["status"] == "evidence_absent" + assert report["comparisons"]["matched_size_scorecard"]["status"] == "not_required" + assert not report["ready_for_external_review"] + assert not report["release_authorized"] + assert set(report["gate_coverage"]["declared"]) == { + entry.id for entry in load_country_spec("uk").gates.gates + } + + +@pytest.mark.parametrize( + "country,k", [(False, None), (True, None), (False, 5), (True, 5)] +) +def test_bound_comparisons_can_complete_readiness_without_publication( + monkeypatch, country, k +): + artifacts = _fixture(country=country, k=k) + report = runtime.compose_uk_full_certification_readiness( + artifacts, comparison_sources=_scores(artifacts, monkeypatch, k=k) + ) + assert report["ready_for_external_review"] + assert not report["release_authorized"] + assert not report["subnational_fit_certified"] + assert report["target_scope"]["local_fit_claim"] is (not country) + + +@pytest.mark.parametrize( + "name,field", + [ + ("package", "content_sha256"), + ("export_readback", "content_sha256"), + ("full_gates", "selection_receipt"), + ], +) +def test_certification_rejects_foreign_graph_identity(name, field): + artifacts = _fixture() + _rewrite(artifacts, name, lambda document: document.__setitem__(field, "foreign")) + with pytest.raises((ValueError, AttributeError)): + runtime.compose_uk_full_certification_readiness(artifacts) + + +def test_actual_surface_validator_refuses_old_score_summary(): + artifacts = _fixture() + payload = canonical_json({"candidate_train_loss": 0.01}) + sources = { + "native_scorecard": ( + {"sha256": hashlib.sha256(payload).hexdigest(), "size_bytes": len(payload)}, + payload, + ) + } + report = runtime.compose_uk_full_certification_readiness( + artifacts, comparison_sources=sources + ) + assert report["comparisons"]["native_scorecard"]["status"] == "failed" + assert any( + "schema 2" in failure + for failure in report["comparisons"]["native_scorecard"]["failures"] + ) + + +def test_exact_size_requires_its_own_bound_comparison(monkeypatch): + artifacts = _fixture(k=5) + sources = _scores(artifacts, monkeypatch, k=5) + sources.pop("matched_size_scorecard") + report = runtime.compose_uk_full_certification_readiness( + artifacts, comparison_sources=sources + ) + assert ( + report["comparisons"]["matched_size_scorecard"]["status"] == "evidence_absent" + ) + assert not report["ready_for_external_review"] + + +def test_bound_spine_provenance_replaces_raw_spine_reports(): + artifacts = _fixture() + gates = uk_spine_gate_manifest(load_country_spec("uk")) + report = { + "blocked_at_phase": None, + "gates": { + outcome.entry.id: outcome.to_payload() + for phase in gates.phases + for outcome in _passed(gates, phase).outcomes + }, + } + provenance = { + "uk_frame_content_identity": "e" * 64, + "spine_gate_report": {"sha256": "f" * 64, "payload": report}, + } + artifacts["spine_provenance"] = ("1" * 64, canonical_json(provenance)) + for phase in ("assembled", "transferred"): + artifacts.pop(f"spine_{phase}") + _rewrite( + artifacts, + "full_gates", + lambda doc: doc["artifacts"].__setitem__("spine_provenance", "1" * 64), + ) + result = runtime.compose_uk_full_certification_readiness(artifacts) + assert result["artifacts"]["spine_provenance"]["graph_artifact_key"] == "1" * 64 diff --git a/packages/microcosm-build/tests/test_uk_full_gates.py b/packages/microcosm-build/tests/test_uk_full_gates.py new file mode 100644 index 000000000..c11c462f0 --- /dev/null +++ b/packages/microcosm-build/tests/test_uk_full_gates.py @@ -0,0 +1,317 @@ +"""Scope and population identity checks for the complete UK gate context.""" + +from dataclasses import replace +from types import SimpleNamespace + +import numpy as np +import pandas as pd +import pytest +from scipy import sparse + +from microcosm.build.country_spec import load_country_spec +from microcosm.build.uk_runtime import full_gates as runtime +from microcosm.calibrate import TargetRegistry, TargetSpec +from microcosm.calibrate.artifacts import OrderedProblem, OrderedSolution +from microcosm.calibrate.matrix import CalibrationProblem +from microcosm.calibrate.solve import CalibrationResult +from microcosm.frame import WeightKind, Weights + + +def _selection(levels=None): + rows = [{"name": "national", "period": 2024, "geography_level": "country"}] + if levels is None: + rows.append( + {"name": "local", "period": 2024, "geography_level": "constituency"} + ) + return { + "schema": "microcosm.calibrate.target-selection.v1", + "selector": {"geography_levels": levels, "explicit": levels is not None}, + "included": rows, + "excluded": [], + } + + +def test_default_full_gate_scope_owns_every_country_gate(): + assert {gate.id for gate in runtime.uk_full_gate_manifest().gates} == { + gate.id for gate in load_country_spec("uk").gates.gates + } + assert runtime.uk_full_gate_scope_receipt()["scope_exclusions"] == {} + + +def test_country_filter_retains_integrity_and_registry_checks_without_local_fit_claims(): + selected = _selection(["country"]) + receipt = runtime.uk_full_gate_scope_receipt(selected) + gates = {gate.id for gate in runtime.uk_full_gate_manifest(selected).gates} + assert receipt["local_fit_claim"] is False + assert set(receipt["scope_exclusions"]) == { + "uk_local_target_fit", + "uk_local_per_family_fit", + "uk_local_area_support", + } + assert { + "uk_local_geography_ladder_post_calibration", + "uk_weight_ess", + "uk_release_family_build_stages", + "uk_ledger_compile_parity_local_incumbent_2025", + "uk_target_surface_local_default_2025", + } <= gates + + +def test_unfiltered_build_cannot_silently_become_national_only(): + selection = _selection(["country"]) + selection["selector"]["geography_levels"] = None + with pytest.raises(ValueError, match="unfiltered UK full build"): + runtime.uk_full_gate_manifest(selection) + + +@pytest.fixture +def evidence(monkeypatch): + specs = [ + TargetSpec( + name="national", + entity="household", + value=0.5, + measure="income", + period=2024, + source="test", + family="income", + ), + TargetSpec( + name="local", + entity="household", + value=2.0, + measure="income", + period=2024, + source="test", + family="population", + ), + ] + targets = tuple(spec.to_target() for spec in specs) + weights = Weights(np.ones(2), WeightKind.CALIBRATED) + problem = CalibrationProblem( + sparse.csr_array([[0.1, 0.2], [1.0, 1.0]]), + np.array([0.5, 2.0]), + tuple(target.row_name for target in targets), + Weights(np.ones(2), WeightKind.IMPORTANCE), + "household", + targets, + ) + metadata = ( + { + "geography_level": "country", + "geography_id": "UK", + "family": "income", + "materialization": "uk_national_measure", + }, + { + "geography_level": "constituency", + "geography_id": "E14000001", + "family": "population", + "materialization": "uk_local_surface", + }, + ) + ordered = OrderedProblem( + problem, (1, 2), metadata, {"target_selection": _selection()}, "a" * 64 + ) + solution = OrderedSolution(np.ones(2), (1, 2), "a" * 64, {}, "b" * 64) + table = pd.DataFrame({"household_id": [1, 2], "income": [1.0, 1.0]}) + frame = SimpleNamespace( + entities=("household",), + table=lambda entity: table, + schema=SimpleNamespace(entity_id_column=lambda entity: "household_id"), + weights_for=lambda entity: weights, + ) + monkeypatch.setattr( + runtime, + "load_efrs_parity_reference", + lambda: SimpleNamespace(input_entities={"income": "household"}), + ) + monkeypatch.setattr( + runtime, + "uk_aggregate_admin_totals", + lambda frame, manifest: ({"admin": 2.0}, [{"measured": 2.0}]), + ) + return ( + frame, + ordered, + solution, + { + "reference_registry": TargetRegistry([specs[0]], country="uk"), + "coverage_engine": object(), + }, + ) + + +def _context(evidence, **overrides): + frame, ordered, solution, supporting = evidence + return runtime.build_full_gate_context( + frame, + ordered_problem=ordered, + solution=overrides.get("solution", solution), + selection_receipt=_selection(), + stage_evidence={"frs_spine": {}}, + supporting_evidence=supporting, + ) + + +def test_gate_diagnostics_use_bound_matrix_and_row_identity(evidence): + context = _context(evidence) + assert context.artifacts["parity_evidence"].target_relative_errors == pytest.approx( + {"national@2024": -0.4} + ) + + assert context.artifacts["parity_evidence"].reference_targets == {"national@2024"} + assert context.artifacts["local_target_diagnostics"][0]["area_code"] == "E14000001" + assert context.artifacts["local_target_diagnostics"][0]["relative_error"] == 0.0 + assert context.artifacts["national_calibration"]["matrix_target_count"] == 2 + assert context.artifacts["rules_engine"] is context.artifacts["coverage_engine"] + + +def _completed_result(evidence): + frame, ordered, solution, _ = evidence + problem = ordered.problem + return CalibrationResult( + frame=frame, + weight_entity="household", + weights=solution.weights, + initial_weights=problem.initial_weights.values, + diagnostics=runtime._build_diagnostics( + problem, frame, problem.initial_weights.values, solution.weights + ), + loss_trajectory=np.array([0.2]), + skipped=(), + problem=problem, + l0_lambda=0.0, + n_nonzero=2, + closing_loss=0.2, + target_loss_weights=np.ones(2), + target_loss_scales=np.ones(2), + target_loss_cap=10.0, + ) + + +def test_gate_context_reuses_authenticated_final_diagnostics(evidence, monkeypatch): + evidence[3]["calibration_result"] = _completed_result(evidence) + monkeypatch.setattr( + runtime, + "_build_diagnostics", + lambda *args: pytest.fail("diagnostics computed twice"), + ) + assert _context(evidence).artifacts["parity_evidence"].target_relative_errors[ + "national@2024" + ] == pytest.approx(-0.4) + + +@pytest.mark.parametrize("mutation", ["weights", "problem", "diagnostic_axis"]) +def test_gate_context_rejects_foreign_completed_result(evidence, mutation): + result = _completed_result(evidence) + if mutation == "weights": + result = replace(result, weights=np.array([1.5, 0.5])) + elif mutation == "problem": + result = replace( + result, problem=replace(result.problem, matrix=2 * result.problem.matrix) + ) + else: + result = replace(result, diagnostics=result.diagnostics[::-1]) + evidence[3]["calibration_result"] = result + with pytest.raises(ValueError, match="calibration_result"): + _context(evidence) + + +@pytest.mark.parametrize( + "change,match", + [ + ({"problem_sha256": "c" * 64}, "different ordered problem"), + ({"entity_ids": (2, 1)}, "different household axes"), + ({"weights": np.array([2.0, 1.0])}, "bound solution weights"), + ], +) +def test_gate_context_rejects_wrong_solution_identity(evidence, change, match): + with pytest.raises(ValueError, match=match): + _context(evidence, solution=replace(evidence[2], **change)) + + +def test_gate_context_rejects_selection_binding_drift(evidence): + frame, ordered, solution, supporting = evidence + ordered = replace(ordered, bindings={"target_selection_sha256": "c" * 64}) + with pytest.raises(ValueError, match="target-selection digest"): + _context((frame, ordered, solution, supporting)) + + +def _phase_failure(gate_id, *, absent=False): + from microcosm.build.gate_battery import GateOutcome, GatePhaseReport, GateStatus + from microcosm.build.gates import GateResult + + entry = next(g for g in load_country_spec("uk").gates.gates if g.id == gate_id) + return GatePhaseReport( + entry.phase, + ( + GateOutcome( + entry=entry, + status=GateStatus.EVIDENCE_ABSENT if absent else GateStatus.FAILED, + result=None + if absent + else GateResult( + name=entry.gate, passed=False, failures=("test failure",) + ), + reason="missing evidence" if absent else None, + ), + ), + ) + + +def test_existing_local_statistical_failure_can_export_but_blocks_full_release(): + report = _phase_failure("uk_local_target_fit") + full = runtime.classify_full_gate_outcomes( + report, sample_fraction=1.0, release_candidate=True + ) + assert full["artifact_permitted"] is True + assert full["exportable_blocking"] == ["uk_local_target_fit"] + assert full["release_blocking_gates_passed"] is False + rung = runtime.classify_full_gate_outcomes( + report, sample_fraction=0.1, release_candidate=False + ) + assert rung["enforced_blocking"] == [] + assert rung["unenforced_release_failures"] == ["uk_local_target_fit"] + + +@pytest.mark.parametrize( + "gate_id", + [ + "uk_local_geography_ladder_post_calibration", + "uk_nonnegative_columns", + "uk_input_mass_parity", + "uk_target_fit", + "uk_calibration_reference_coverage", + ], +) +def test_geography_and_migrated_national_gates_keep_artifact_enforcement(gate_id): + for fraction in (0.1, 1.0): + result = runtime.classify_full_gate_outcomes( + _phase_failure(gate_id), sample_fraction=fraction, release_candidate=False + ) + assert result["artifact_permitted"] is False + assert result["structural_failures"] == [gate_id] + + +def test_missing_evidence_uses_declared_development_policy(): + ordinary = _phase_failure("uk_input_mass_parity", absent=True) + assert ( + runtime.classify_full_gate_outcomes( + ordinary, sample_fraction=1.0, release_candidate=False + )["artifact_permitted"] + is True + ) + assert ( + runtime.classify_full_gate_outcomes( + ordinary, sample_fraction=1.0, release_candidate=True + )["artifact_permitted"] + is False + ) + strict = _phase_failure("uk_weights_audit", absent=True) + assert ( + runtime.classify_full_gate_outcomes( + strict, sample_fraction=1.0, release_candidate=False + )["artifact_permitted"] + is False + ) diff --git a/packages/microcosm-build/tests/test_uk_full_measure.py b/packages/microcosm-build/tests/test_uk_full_measure.py new file mode 100644 index 000000000..5b4f47a46 --- /dev/null +++ b/packages/microcosm-build/tests/test_uk_full_measure.py @@ -0,0 +1,272 @@ +"""Shared full-build measure evaluation retains legacy engine/RNG boundaries.""" + +from types import SimpleNamespace + +import numpy as np +import pandas as pd +import pytest +from test_uk_full_population_graph import source_frame +from test_uk_ladder_rowwise_clone import toy_ladder as toy_ladder + +from microcosm.build.uk_runtime import full_measure +from microcosm.build.uk_runtime.rowwise_dataset import ( + clone_uk_dataset_with_ladder_geography, +) +from microcosm.calibrate import TargetRegistry, TargetSpec + + +def test_full_measure_reuses_one_resolver( + monkeypatch, + tmp_path, +) -> None: + frame = source_frame() + constructions = [] + cgt_period_contract = { + "version": "uk-cgt-measurement-v2", + "input_period": "2024", + "calibration_period": 2025, + "bound_measurements": { + "cgt_2024_gains": { + "model_variable": "capital_gains", + "measurement_period": 2024, + } + }, + } + + class StubResolver: + def __init__(self, **kwargs): + constructions.append(kwargs) + self.simulation = object() + self.contract_targets = {} + + def receipt(self): + return { + "mode": "stub", + "policyengine_uk_version": "test", + "cgt_period_contract": cgt_period_contract, + } + + monkeypatch.setattr( + full_measure, + "compute_household_metrics", + lambda _simulation, area_type, *, household_ids, **_kwargs: pd.DataFrame( + {f"{area_type}_metric": np.ones(len(household_ids))}, + index=household_ids, + ), + ) + registry = TargetRegistry([], country="uk") + prepared, restore, national, local_metrics, receipt = ( + full_measure.resolve_uk_full_measures( + frame, + registry, + period=2025, + scratch_dir=tmp_path / "scratch", + resolver_factory=StubResolver, + ) + ) + + assert len(constructions) == 1 + assert receipt == { + "mode": "stub", + "engine_version": "test", + "households": frame.n("household"), + "persons": frame.n("person"), + "benunits": frame.n("benunit"), + "national_inputs": 0, + "local_metrics": {"constituency": 1, "la": 1}, + "blocks": 1, + "cgt_period_contract": cgt_period_contract, + } + assert set(local_metrics) == {"constituency", "la"} + assert len(national.targets) == 0 + assert restore(prepared).table("household").equals(frame.table("household")) + + +@pytest.mark.parametrize("second_cgt_period", [2024, 2025, None]) +def test_full_measure_resolves_real_per_clone_blocks( + monkeypatch, + tmp_path, + second_cgt_period, + toy_ladder, +) -> None: + frame = source_frame() + ladder, _ = toy_ladder + clone = clone_uk_dataset_with_ladder_geography( + frame, + ladder, + n_clones=2, + seed=7, + source_year=2023, + expected_constituency_vintage="2024_pcon", + source_lineage_modulus=None, + ) + constructions = [] + + class StubResolver: + def __init__(self, **kwargs): + constructions.append(kwargs) + self.simulation = object() + self.contract_targets = {} + self.cgt_period = 2024 if len(constructions) == 1 else second_cgt_period + + def receipt(self): + receipt = {"mode": "stub", "policyengine_uk_version": "test"} + if self.cgt_period is not None: + receipt["cgt_period_contract"] = { + "version": "uk-cgt-measurement-v2", + "input_period": "2024", + "calibration_period": 2025, + "bound_measurements": { + "cgt_2024_gains": { + "model_variable": "capital_gains", + "measurement_period": self.cgt_period, + } + }, + } + return receipt + + monkeypatch.setattr( + full_measure, + "compute_household_metrics", + lambda _simulation, area_type, *, household_ids, **_kwargs: pd.DataFrame( + {f"{area_type}_metric": np.arange(len(household_ids), dtype=float)}, + index=household_ids, + ), + ) + + def resolve(): + return full_measure.resolve_uk_full_measures( + clone.frame, + TargetRegistry([], country="uk"), + period=2025, + scratch_dir=tmp_path / "block-scratch", + resolver_factory=StubResolver, + blocks=2, + ) + + if second_cgt_period != 2024: + with pytest.raises(RuntimeError, match="CGT period contract is inconsistent"): + resolve() + return + + prepared, restore, _, metrics, receipt = resolve() + + assert len(constructions) == 2 + assert [len(call["frame"].table("household")) for call in constructions] == [ + frame.n("household"), + frame.n("household"), + ] + assert receipt["blocks"] == 2 + assert receipt["cgt_period_contract"]["bound_measurements"] == { + "cgt_2024_gains": { + "model_variable": "capital_gains", + "measurement_period": 2024, + } + } + assert receipt["deviation"] == "per_clone_block_engine_resolution" + sensitivity = receipt["block_sensitivity"] + assert ( + "ons/corporate_land_value" + in sensitivity["known_population_normalised_measures"] + ) + assert set(sensitivity["present_in_this_run"]) <= set( + sensitivity["known_population_normalised_measures"] + ) + assert "not evidence for adjudication" in sensitivity["caveat"] + assert ( + metrics["constituency"].index.tolist() + == clone.frame.table("household")["household_id"].tolist() + ) + assert restore(prepared).table("household").equals(clone.frame.table("household")) + + +@pytest.mark.parametrize("blocks", [1, 2]) +def test_prepared_measures_preserve_ids_and_remove_duplicate_scratch_inputs( + monkeypatch, tmp_path, toy_ladder, blocks +): + frame = source_frame() + if blocks == 2: + frame = clone_uk_dataset_with_ladder_geography( + frame, + toy_ladder[0], + n_clones=2, + seed=7, + source_year=2023, + expected_constituency_vintage="2024_pcon", + ).frame + registry = TargetRegistry( + [ + TargetSpec( + name="national", + entity="household", + measure="prepared_count", + value=33.0, + period=2025, + family="fixture", + source="fixture", + ) + ], + country="uk", + ) + + class Resolver: + def __init__(self, *, frame, **kwargs): + self.frame = frame + self.simulation = object() + + def receipt(self): + return {"mode": "stub", "policyengine_uk_version": "test"} + + monkeypatch.setattr( + full_measure, + "resolve_target_measures", + lambda _factory, _registry, provider, **kwargs: SimpleNamespace( + measure_inputs={ + ("person", "region"): np.zeros(provider.frame.n("person")), + ("household", "raw_engine_input"): provider.frame.table("household")[ + "household_id" + ].to_numpy(), + ("household", "ons/corporate_land_value"): np.ones( + provider.frame.n("household") + ), + } + ), + ) + + def materialize(adapter, _registry, **kwargs): + # The duplicate scratch region must be usable while materializing, + # then removed before assembling the flattened prepared Frame. + assert "region" in adapter.tables["person"] + table = adapter.tables["household"] + table["prepared_count"] = table["raw_engine_input"].to_numpy() + return SimpleNamespace(skipped=()) + + monkeypatch.setattr(full_measure, "materialize_uk_ledger_targets", materialize) + prepared, restore, national, metrics, receipt = ( + full_measure.resolve_uk_full_measures( + frame, + registry, + period=2025, + scratch_dir=tmp_path / "scratch", + resolver_factory=Resolver, + blocks=blocks, + local_grains=(), + ) + ) + assert "region" not in prepared.table("person") + assert "raw_engine_input" not in prepared.table("household") + np.testing.assert_array_equal( + prepared.table("household")["prepared_count"], + frame.table("household")["household_id"], + ) + for entity in frame.entities: + pd.testing.assert_frame_equal( + restore(prepared).table(entity), frame.table(entity) + ) + assert len(national.targets) == 1 + assert metrics == {} + assert receipt["national_inputs"] == 3 + if blocks == 2: + assert receipt["block_sensitivity"]["present_in_this_run"] == [ + "ons/corporate_land_value" + ] diff --git a/packages/microcosm-build/tests/test_uk_full_population_graph.py b/packages/microcosm-build/tests/test_uk_full_population_graph.py new file mode 100644 index 000000000..38408f709 --- /dev/null +++ b/packages/microcosm-build/tests/test_uk_full_population_graph.py @@ -0,0 +1,160 @@ +"""Full-build population graph preserves the maintained geography operation.""" + +import hashlib + +import numpy as np +import pandas as pd +import pytest +from test_uk_ladder_rowwise_clone import _seam_frame +from test_uk_ladder_rowwise_clone import toy_ladder as toy_ladder + +from microcosm.build.uk_runtime.graph_kernels import UKClaimKernel +from microcosm.build.uk_runtime.graph_population import ( + append_uk_population_nodes, + register_uk_population_kernels, +) +from microcosm.build.uk_runtime.rowwise_dataset import ( + clone_uk_dataset_with_ladder_geography, +) +from microcosm.frame import Frame +from microcosm.graph import ( + Capabilities, + ContentStore, + Determinism, + Graph, + KernelBase, + KernelRegistry, + KernelResult, + Node, + Owned, + SourceRef, + StructuralDelta, + compile_graph, + run_graph, +) + + +def source_frame(): + original = _seam_frame() + tables = {e: original.table(e).copy() for e in original.entities} + tables["person"]["age"] = 40 + tables["benunit"]["would_claim_uc"] = True + tables["household"]["region"] = tables["household"]["region"].astype("string") + return Frame( + tables, + original.schema, + {"household": original.weights_for("household")}, + original.strata, + mass_log=original.mass_log, + metadata=original.metadata, + ) + + +class Source(KernelBase): + ref = "uk.test.full-source@1" + capabilities = Capabilities( + Determinism.DETERMINISTIC, structural=StructuralDelta.CREATE + ) + + def implementation_hash(self): + return hashlib.sha256(self.ref.encode()).hexdigest() + + def run(self, context): + return KernelResult(frame=source_frame()) + + +def graph_and_registry(k): + frame = source_frame() + source = Node( + "source", + Source.ref, + structural=StructuralDelta.CREATE, + sources=("fixture",), + outputs=tuple( + Owned( + entity, + col, + "string" if table[col].dtype.kind in "OUS" else str(table[col].dtype), + ) + for entity in frame.entities + for table in [frame.table(entity)] + for col in table.columns + if col + not in { + "person_id", + "person_household_id", + "person_benunit_id", + "household_id", + "benunit_id", + } + ), + ) + graph = append_uk_population_nodes( + Graph("uk", (SourceRef("fixture", "raw-bytes-v1"),), (source,)), + population="source", + time_period="2023", + weight_kind="importance", + n_clones=k, + seed=7, + source_year=2023, + ) + registry = KernelRegistry() + registry.register(Source()) + registry.register(UKClaimKernel()) + register_uk_population_kernels(registry) + return graph, registry + + +@pytest.mark.parametrize("k", [1, 2, 5]) +def test_population_graph_preserves_rows_weights_geography_and_replays( + k, toy_ladder, tmp_path +): + ladder, path = toy_ladder + expected = clone_uk_dataset_with_ladder_geography( + source_frame(), + ladder, + n_clones=k, + seed=7, + source_year=2023, + expected_constituency_vintage="2024_pcon", + ).frame + graph, registry = graph_and_registry(k) + store = ContentStore(tmp_path / "store") + compiled = compile_graph(graph) + assert "uk.full.locations" in compiled.predecessors["uk.full.geography_mapping"] + first = run_graph( + compiled, + sources={"uk_ladder": path, "fixture": path}, + store=store, + kernels=registry, + ) + actual = first.population("uk.full.expand") + for entity in expected.entities: + pd.testing.assert_frame_equal( + actual.table(entity)[expected.table(entity).columns], + expected.table(entity), + check_dtype=False, + ) + np.testing.assert_array_equal( + actual.weights_for("household").values, expected.weights_for("household").values + ) + assert actual.mass_log == expected.mass_log + # A new registry cannot obtain results from mutable objects of the cold run. + _, fresh_registry = graph_and_registry(k) + replay = run_graph( + compiled, + sources={"uk_ladder": path, "fixture": path}, + store=store, + kernels=fresh_registry, + resume="require", + ) + assert replay.population("uk.full.expand").mass_log == actual.mass_log + + +def test_k_changes_expansion_but_never_sampling(): + first, _ = graph_and_registry(1) + second, _ = graph_and_registry(3) + assert first.node("uk.full.sample") == second.node("uk.full.sample") + assert first.node("uk.full.normalize") == second.node("uk.full.normalize") + assert first.node("uk.full.expand").params["n_clones"] == 1 + assert second.node("uk.full.expand").params["n_clones"] == 3 diff --git a/packages/microcosm-build/tests/test_uk_full_solve_scope.py b/packages/microcosm-build/tests/test_uk_full_solve_scope.py new file mode 100644 index 000000000..c3ed9f8bb --- /dev/null +++ b/packages/microcosm-build/tests/test_uk_full_solve_scope.py @@ -0,0 +1,82 @@ +"""A filtered full problem has zero local rows and uses the same solver.""" + +import numpy as np +import pytest +from test_uk_local_rowwise import _clone_frame + +from microcosm.build.uk_runtime.local_rowwise import ( + UKRowwiseNationalRows, + empty_uk_local_problem, + finish_uk_full_solve, + prepare_uk_full_solve, + rotated_uk_local_holdout, + solve_uk_dense_reference, + solve_uk_rowwise_weights_under_doctrine, +) +from microcosm.calibrate import Target, TargetRegistry, TargetSet, TargetSpec + + +def national_rows(): + registry = TargetRegistry( + [ + TargetSpec( + name="households", + entity="household", + value=6.0, + measure="household_id", + period=2026, + family="fixture", + source="fixture", + ) + ], + country="uk", + ) + return UKRowwiseNationalRows( + TargetSet( + [ + Target( + name="households", + entity="household", + value=6.0, + measure=lambda frame: np.ones(frame.n("household")), + period=2026, + ) + ] + ), + registry, + ("fixture",), + ) + + +def test_zero_local_scope_uses_same_solver_and_has_no_fake_holdout(): + frame = _clone_frame() + local = empty_uk_local_problem(frame.table("household")["household_id"]) + rows = national_rows() + prepared = prepare_uk_full_solve( + frame, local, bound_families=("national/fixture",), national_rows=rows + ) + dense = solve_uk_dense_reference(prepared, epochs=8, seed=17) + finished = finish_uk_full_solve(prepared, dense) + existing = solve_uk_rowwise_weights_under_doctrine( + frame, + local, + bound_families=("national/fixture",), + national_rows=rows, + epochs=8, + seed=17, + ) + np.testing.assert_array_equal(finished.weights, existing.weights) + assert finished.diagnostics.empty + assert len(finished.national_diagnostics) == 1 + assert dense.problem.n_targets == 1 + holdout = rotated_uk_local_holdout(frame, local, national_rows=rows) + assert holdout["outcome"] == "not_applicable" + assert holdout["n_folds"] == 0 + assert holdout["folds"] == [] + + +def test_empty_total_surface_refuses_instead_of_manufacturing_constraints(): + frame = _clone_frame() + local = empty_uk_local_problem(frame.table("household")["household_id"]) + with pytest.raises(ValueError, match="at least one selected target"): + prepare_uk_full_solve(frame, local, bound_families=()) diff --git a/packages/microcosm-build/tests/test_uk_full_target_graph.py b/packages/microcosm-build/tests/test_uk_full_target_graph.py new file mode 100644 index 000000000..8d50492f6 --- /dev/null +++ b/packages/microcosm-build/tests/test_uk_full_target_graph.py @@ -0,0 +1,447 @@ +"""Actual full graph on synthetic target and engine-source adapters.""" + +import json +from dataclasses import replace +from types import SimpleNamespace + +import numpy as np +import pandas as pd +import pytest +from test_uk_full_calibration_graph import preflight_payload +from test_uk_full_population_graph import Source, graph_and_registry +from test_uk_ladder_rowwise_clone import toy_ladder as toy_ladder + +from microcosm.build.uk_runtime import full_targets, graph_targets, ledger_targets +from microcosm.build.uk_runtime.graph_build import ( + UKFullBuildConfig, + register_uk_full_kernels, + uk_full_graph, +) +from microcosm.build.uk_runtime.graph_calibration import UKGraphCalibrationConfig +from microcosm.build.uk_runtime.graph_terminal import FULL_GATE_REPORT_TYPE +from microcosm.build.uk_runtime.local_rowwise import UKRowwiseNationalRows +from microcosm.calibrate import TargetRegistry, TargetSpec +from microcosm.calibrate.artifacts import decode_problem +from microcosm.frame import Frame +from microcosm.graph import ( + ArtifactInput, + ArtifactOutput, + Capabilities, + ContentStore, + Determinism, + Graph, + KernelBase, + KernelRegistry, + KernelResult, + Node, + compile_graph, + run_graph, +) + + +@pytest.fixture +def target_inputs(monkeypatch): + national = TargetRegistry( + [ + TargetSpec( + name="country_households", + entity="household", + measure="test_ones", + value=34.0, + period=2026, + family="fixture", + source="fixture", + metadata={"geography_level": "country", "geography_id": "UK"}, + ), + TargetSpec( + # Repeated names across periods must retain distinct metadata. + name="country_households", + entity="household", + measure="test_ones", + value=4.0, + period=2025, + family="fixture", + source="fixture", + filter="test_london", + metadata={"geography_level": "region", "geography_id": "LONDON"}, + ), + ], + country="uk", + ) + empty = TargetRegistry([], country="uk") + monkeypatch.setattr( + full_targets, + "load_uk_full_target_inputs", + lambda *args, **kwargs: { + "national_registry": national, + "band_edge_registry": national, + "local_registry": empty, + "artifact": SimpleNamespace(facts=()), + "calibration_year": 2026, + "measure_exclusions": {}, + "reviewed_unbound_higher_targets": {}, + "national_source_pin": {"fixture": True}, + "register_completeness": {"fixture": True}, + "ledger_provenance": {"fixture": True}, + "uk_ledger_compiled_registries": {2026: national}, + "uk_ledger_compiled_local_registries": {2026: empty}, + }, + ) + monkeypatch.setattr( + graph_targets, "uk_ledger_households_total", lambda *args, **kwargs: 33.0 + ) + monkeypatch.setattr( + graph_targets, + "uk_ladder_household_uprating", + lambda *args, **kwargs: {"applied": True}, + ) + + def surface(registry, ladder, **kwargs): + rows = [] + for grain, codes in ( + ("constituency", ladder.constituency_code), + ("la", ladder.local_authority_code), + ): + for i, code in enumerate(codes): + rows.append( + { + "area_type": grain, + "area_code": str(code), + "metric": "households", + "value": 3.0 if i < 2 else 10.0, + "target_name": f"{grain}/{code}/households", + "family": "census_households", + "period": 2026, + "source": "fixture", + } + ) + return pd.DataFrame(rows), {"fixture": True} + + monkeypatch.setattr(ledger_targets, "uk_local_target_surface", surface) + + def measures(frame, national_registry, *, local_grains, **kwargs): + tables = {e: frame.table(e).copy() for e in frame.entities} + tables["household"]["test_ones"] = 1.0 + tables["household"]["test_london"] = ( + tables["household"]["region"] == "LONDON" + ).astype(float) + prepared = Frame( + tables, + frame.schema, + {"household": frame.weights_for("household")}, + frame.strata, + mass_log=frame.mass_log, + metadata=frame.metadata, + ) + return ( + prepared, + lambda _: frame, + UKRowwiseNationalRows( + national_registry.to_target_set(), national_registry, ("fixture",) + ), + { + g: pd.DataFrame( + {"households": np.ones(frame.n("household"))}, + index=frame.table("household")["household_id"], + ) + for g in local_grains + }, + {"fixture": True}, + ) + + monkeypatch.setattr(graph_targets, "resolve_uk_full_measures", measures) + return {"national": national, "surface": surface, "measures": measures} + + +class Preflight(KernelBase): + """Synthetic source verdict: tests below exercise numerical graph ownership.""" + + ref = "uk.test.target-preflight@1" + capabilities = Capabilities(Determinism.DETERMINISTIC) + + def run(self, context): + selection = json.loads(context.artifacts["selection"].payload)["receipt"] + return KernelResult( + artifacts={"preflight": preflight_payload(selection=selection)} + ) + + +def build( + tmp_path, + ladder_path, + levels, + *, + n_clones=1, + seed=7, + dataset_households=None, + resume="auto", + forbid_execution=False, +): + primitive, _ = graph_and_registry(1) + base = Graph( + "uk", + tuple(s for s in primitive.sources if s.name == "fixture"), + (primitive.node("source"),), + ) + config = UKFullBuildConfig( + calibration_year=2026, + time_period="2023", + source_year=2023, + n_clones=n_clones, + geography_levels=levels, + seed=seed, + calibration=UKGraphCalibrationConfig( + epochs=8, seed=seed, dataset_households=dataset_households + ), + ) + full = uk_full_graph(config, spine=base, spine_population="source") + preflight = Node( + "fixture.preflight", + Preflight.ref, + population="uk.full.pool", + artifact_inputs=( + ArtifactInput( + "selection", + "uk.full.target_selection", + "selection", + graph_targets.TARGET_SELECTION_TYPE, + ), + ), + artifact_outputs=(ArtifactOutput("preflight", FULL_GATE_REPORT_TYPE),), + ) + full = replace( + full, + graph=replace( + full.graph, + nodes=( + *( + replace( + node, + artifact_inputs=( + *node.artifact_inputs, + ArtifactInput( + "preflight", + preflight.id, + "preflight", + FULL_GATE_REPORT_TYPE, + ), + ), + ) + if node.id == full.calibration.dense_producer + else node + for node in full.graph.nodes + ), + preflight, + ), + ), + ) + registry = KernelRegistry() + registry.register(Source()) + registry.register(Preflight()) + register_uk_full_kernels(registry) + if forbid_execution: + for kernel in registry.as_mapping().values(): + kernel.run = lambda *args, **kwargs: pytest.fail( + "cached full graph executed" + ) + store = ContentStore(tmp_path / "store") + manifest = run_graph( + compile_graph(full.graph), + sources={ + "fixture": ladder_path, + "uk_ladder": ladder_path, + "uk_ledger_facts": ladder_path, + }, + store=store, + kernels=registry, + resume=resume, + ) + problem = decode_problem( + store.load_bytes(manifest.nodes["uk.full.problem"].opaque_artifacts["problem"]) + ) + return full, manifest, problem + + +@pytest.mark.requires_uk +def test_explicit_country_filter_runs_same_full_graph_without_local_constraints( + target_inputs, toy_ladder, tmp_path +): + _, path = toy_ladder + full, manifest, problem = build(tmp_path, path, ("country",)) + assert problem.problem.names == ("country_households@2026",) + assert {m["geography_level"] for m in problem.target_metadata} == {"country"} + assert manifest.population(full.population).n("household") == 4 + assert len(problem.bindings["target_selection"]["excluded"]) > 0 + assert "uk.full.dense" in manifest.nodes + assert "uk.full.locations" in manifest.nodes + + +def test_default_scope_contains_all_levels_and_never_depends_on_k_or_k_small(): + default = UKFullBuildConfig(calibration_year=2026) + for config in ( + default, + replace(default, n_clones=1), + replace( + default, calibration=replace(default.calibration, dataset_households=20) + ), + ): + graph = uk_full_graph(config).graph + assert graph.node("uk.full.target_selection").params["geography_levels"] is None + + +@pytest.mark.requires_uk +def test_default_all_has_direct_matrix_and_solver_parity_and_replays( + target_inputs, toy_ladder, tmp_path +): + from test_uk_full_population_graph import source_frame + + from microcosm.build.uk_runtime.full_problem import build_uk_full_local_problem + from microcosm.build.uk_runtime.local_rowwise import ( + prepare_uk_full_solve, + solve_uk_dense_reference, + ) + from microcosm.build.uk_runtime.rowwise_dataset import ( + clone_uk_dataset_with_ladder_geography, + ) + + ladder, path = toy_ladder + default, default_run, default_problem = build( + tmp_path / "default", path, None, n_clones=10 + ) + explicit, explicit_run, explicit_problem = build( + tmp_path / "explicit", + path, + ("country", "region", "constituency", "la"), + n_clones=10, + ) + assert {row["geography_level"] for row in default_problem.target_metadata} == { + "country", + "region", + "constituency", + "la", + } + assert default_problem.problem.n_targets == 12 + assert default_problem.problem.names == explicit_problem.problem.names + np.testing.assert_array_equal( + default_problem.problem.matrix.toarray(), + explicit_problem.problem.matrix.toarray(), + ) + np.testing.assert_array_equal( + default_problem.problem.target_vector, explicit_problem.problem.target_vector + ) + np.testing.assert_array_equal( + default_run.population(default.population).weights_for("household").values, + explicit_run.population(explicit.population).weights_for("household").values, + ) + assert default_problem.bindings["target_selection"]["selector"]["explicit"] is False + assert explicit_problem.bindings["target_selection"]["selector"]["explicit"] is True + + # Independently execute the maintained pre-graph numerical helpers on the + # same original spine, legacy location draw, target rows and solver options. + assignment = clone_uk_dataset_with_ladder_geography( + source_frame(), + ladder, + n_clones=10, + seed=7, + source_year=2023, + expected_constituency_vintage="2024_pcon", + ) + prepared_frame, _, national_rows, metrics, _ = target_inputs["measures"]( + assignment.frame, target_inputs["national"], local_grains=("constituency", "la") + ) + surface, cross = target_inputs["surface"](None, ladder) + _, local, _, families, _ = build_uk_full_local_problem( + SimpleNamespace(result=SimpleNamespace(frame=prepared_frame), ladder=ladder), + target_ladder=ladder, + local_registry=TargetRegistry([], country="uk"), + national_registry=target_inputs["national"], + local_metrics=metrics, + period=2026, + sample_fraction=1.0, + reviewed_unbound_higher_targets={}, + selected_surface=surface, + surface_receipt=cross, + ) + prepared = prepare_uk_full_solve( + prepared_frame, + local, + bound_families=families, + national_rows=national_rows, + target_weight_rule="uniform", + ) + direct = solve_uk_dense_reference(prepared, epochs=8, seed=7) + np.testing.assert_array_equal( + default_problem.problem.matrix.toarray(), direct.problem.matrix.toarray() + ) + np.testing.assert_array_equal( + default_run.population(default.population).weights_for("household").values, + direct.weights, + ) + + _, replay, _ = build( + tmp_path / "default", + path, + None, + n_clones=10, + resume="require", + forbid_execution=True, + ) + assert all(receipt.hit for receipt in replay.nodes.values()) + np.testing.assert_array_equal( + replay.population(default.population).weights_for("household").values, + direct.weights, + ) + + +@pytest.mark.requires_uk +def test_unsupported_default_all_refuses_without_narrowing( + target_inputs, toy_ladder, tmp_path +): + from microcosm.graph.errors import NodeRejectedError + + _, path = toy_ladder + # At K=1 London's only household cannot occupy both positive area cells. + with pytest.raises(NodeRejectedError, match="[Ss]upport|unassigned|positive"): + build(tmp_path / "all", path, None, n_clones=1) + country, result, problem = build( + tmp_path / "country", path, ("country",), n_clones=1 + ) + assert problem.problem.names == ("country_households@2026",) + assert result.population(country.population).n("household") == 4 + + +def test_local_surface_selection_keeps_exact_target_periods(): + rows = pd.DataFrame( + { + "target_name": ["same", "same", "different"], + "period": [2025, 2026, 2026], + "value": [2.0, 3.0, 4.0], + } + ) + selected = [ + TargetSpec( + name="same", + entity="household", + measure="count", + value=3.0, + period=2026, + source="fixture", + family="fixture", + ) + ] + actual = graph_targets._selected_local_surface(rows, selected) + assert actual.to_dict(orient="records") == [ + {"target_name": "same", "period": 2026, "value": 3.0} + ] + + +@pytest.mark.requires_uk +def test_target_kernel_identity_binds_country_reference_resources(monkeypatch): + kernel = graph_targets.UKFullProblemKernel() + original = kernel.implementation_hash() + monkeypatch.setattr( + graph_targets, + "load_country_spec", + lambda _country: SimpleNamespace(fingerprint="changed-reference-resource"), + ) + assert kernel.implementation_hash() != original diff --git a/packages/microcosm-build/tests/test_uk_full_targets.py b/packages/microcosm-build/tests/test_uk_full_targets.py new file mode 100644 index 000000000..f54964a88 --- /dev/null +++ b/packages/microcosm-build/tests/test_uk_full_targets.py @@ -0,0 +1,147 @@ +"""Source pins and unreduced-register ownership in the full UK target path.""" + +from datetime import date +from types import SimpleNamespace + +import pytest + +from microcosm.build.uk_runtime import full_targets as runtime +from microcosm.calibrate import TargetRegistry, TargetSpec + + +def _registry(*names): + return TargetRegistry( + [ + TargetSpec( + name=name, + entity="person", + value=1.0, + measure="age", + period=2024, + source="test", + family="population", + metadata={"contract_target_id": name}, + ) + for name in names + ], + country="uk", + ) + + +@pytest.fixture +def prepared(monkeypatch): + national = _registry("retained", "excluded") + approved = _registry("retained") + local = _registry("local") + calls = [] + artifact = SimpleNamespace( + facts=({"test": True},), facts_sha256="a" * 64, manifest_sha256="b" * 64 + ) + pin = SimpleNamespace( + facts_sha256=artifact.facts_sha256, + manifest_sha256=artifact.manifest_sha256, + to_dict=lambda: { + "facts_sha256": artifact.facts_sha256, + "manifest_sha256": artifact.manifest_sha256, + }, + ) + monkeypatch.setattr(runtime, "load_uk_national_chronicle_feed", lambda: pin) + monkeypatch.setattr( + runtime, "load_ledger_consumer_artifact", lambda *a, **kw: artifact + ) + monkeypatch.setattr(runtime, "load_uk_local_area_crosswalk", lambda: {}) + + def compile_national(facts, *, target_period): + calls.append(("national", target_period)) + return SimpleNamespace(registry=national, unsupported=()) + + def compile_local(facts, *, target_period, crosswalk): + calls.append(("local", target_period)) + return SimpleNamespace(registry=local, unsupported=()) + + monkeypatch.setattr(runtime, "compile_uk_target_registry", compile_national) + monkeypatch.setattr(runtime, "compile_uk_local_target_registry", compile_local) + monkeypatch.setattr( + runtime, "load_uk_calibration_measure_exclusions", lambda path: () + ) + monkeypatch.setattr( + runtime, + "apply_uk_calibration_measure_exclusions", + lambda registry, exclusions, now: ( + approved, + {"excluded": {"reason": "reviewed"}}, + ), + ) + return national, approved, local, artifact, calls + + +def _load(**kwargs): + return runtime.load_uk_full_target_inputs( + "facts.jsonl", + calibration_year=2024, + exclusions_evaluated_on=date(2026, 9, 10), + **kwargs, + ) + + +def test_full_inputs_preserve_band_edges_and_validation_periods(prepared): + national, approved, local, artifact, calls = prepared + result = _load() + assert result["artifact"] is artifact + assert result["band_edge_registry"] is national + assert result["national_registry"] is approved + assert result["local_registry"] is local + assert calls == [ + ("national", 2023), + ("national", 2024), + ("national", 2025), + ("local", 2024), + ("local", 2025), + ] + assert result["register_completeness"]["compiled_reference_count"] == 2 + assert result["register_completeness"]["approved_reference_count"] == 1 + assert result["reviewed_unbound_higher_targets"] == { + "excluded": {"reason": "reviewed"} + } + + +def test_source_pin_mismatch_refuses_before_read(prepared, monkeypatch): + monkeypatch.setattr( + runtime, + "load_ledger_consumer_artifact", + lambda *a, **kw: pytest.fail("source read before pin agreement"), + ) + with pytest.raises(ValueError, match="committed national feed"): + _load(expected_facts_sha256="c" * 64) + + +def test_loaded_source_mismatch_refuses(prepared): + prepared[3].manifest_sha256 = "c" * 64 + with pytest.raises(ValueError, match="Ledger artifact"): + _load() + + +def test_frozen_register_compares_complete_not_measure_pruned_surface( + prepared, tmp_path +): + path = tmp_path / "register.json" + prepared[0].to_json(path) + assert ( + _load(register_json=path)["register_completeness"]["frozen_registry_version"] + == prepared[0].version + ) + prepared[1].to_json(path) + with pytest.raises(ValueError, match="full national register differs"): + _load(register_json=path) + + +def test_validation_reference_compilation_is_fail_closed(prepared, monkeypatch): + monkeypatch.setattr( + runtime, + "compile_uk_target_registry", + lambda *a, **kw: SimpleNamespace( + registry=prepared[0], unsupported=({"target": "missing"},) + ), + ) + with pytest.raises(ValueError, match="failed to compile for 2023"): + _load() diff --git a/packages/microcosm-build/tests/test_uk_graph.py b/packages/microcosm-build/tests/test_uk_graph.py index 80a39f041..0b9f2a63c 100644 --- a/packages/microcosm-build/tests/test_uk_graph.py +++ b/packages/microcosm-build/tests/test_uk_graph.py @@ -10,7 +10,6 @@ from microcosm.build.country_spec import load_country_spec from microcosm.build.uk_runtime.graph import ( - UK_SPINE_EXCLUSIONS, UK_SPINE_STRUCTURAL_STAGES, uk_registry, uk_spine_graph, @@ -152,27 +151,17 @@ def test_uk_expand_contract_rejects_unknown_source_ids() -> None: patch(_expand_population(), _expand_node(), _expand_result(bad_source=True)) -def test_uk_spine_graph_contains_manifest_stages_and_named_exclusions() -> None: +def test_uk_spine_graph_contains_all_canonical_manifest_stages() -> None: spec = load_country_spec("uk") assert spec.sources is not None - expected = tuple( - stage.stage - for stage in spec.sources.stages - if stage.stage not in UK_SPINE_EXCLUSIONS - ) + expected = tuple(stage.stage for stage in spec.sources.stages) graph = uk_spine_graph(spec) ids = {node.id for node in graph.nodes} - # 28 with the #832 uc_reporter_redraw and #685 uc_deduction_attributes - # stages; the two named exclusions are the certified-pair alternatives, - # not steps of this pipeline. assert len(expected) == 28 - assert UK_SPINE_EXCLUSIONS == { - "frs_hmrc_retained_leaves", - "hmrc_spi_income", - } assert set(expected) <= ids - assert not (UK_SPINE_EXCLUSIONS & ids) + assert "frs_hmrc_retained_leaves" not in ids + assert "hmrc_spi_income" not in ids root_dtypes = { (owned.entity, owned.column): owned.dtype for owned in graph.node("create_uk_frs").outputs @@ -186,11 +175,7 @@ def test_uk_spine_graph_contains_manifest_stages_and_named_exclusions() -> None: def test_uk_spine_compile_order_is_derived_from_declared_inputs() -> None: spec = load_country_spec("uk") assert spec.sources is not None - expected = tuple( - stage.stage - for stage in spec.sources.stages - if stage.stage not in UK_SPINE_EXCLUSIONS - ) + expected = tuple(stage.stage for stage in spec.sources.stages) compiled = compile_graph(uk_spine_graph(spec)) stage_order = tuple(node_id for node_id in compiled.order if node_id in expected) @@ -511,6 +496,7 @@ def test_spi_support_fixture_changes_person_mass_and_conserves_household_mass() assert after.mass_log[-1].new_total == before_household_mass +@pytest.mark.requires_uk @pytest.mark.requires_uk def test_driver_projects_a_stage_record_for_every_graph_stage_on_the_fixture( tmp_path, @@ -525,41 +511,83 @@ def test_driver_projects_a_stage_record_for_every_graph_stage_on_the_fixture( in CI's engine lane instead. """ - import importlib.util + import json from pathlib import Path + from microcosm.build.gate_battery import ( + EvidenceContext, + evaluate_phase, + gate_phase_report_payload, + ) + from microcosm.build.uk_runtime.battery_bindings import UK_GATE_REGISTRY + from microcosm.build.uk_runtime.graph_evidence import ( + add_uk_spine_gate_nodes, + register_spine_gate_kernel, + uk_spine_gate_manifest, + ) from microcosm.build.uk_runtime.graph_kernels import fixture_stage_plan_inputs + from microcosm.frame.adapters.policyengine_uk import PolicyEngineUKEngine from microcosm.graph import ContentStore, run_graph root = Path(__file__).resolve().parents[3] fixture = root / "packages/microcosm-graph/tests/fixtures/parity/uk_spine" if not fixture.exists(): pytest.skip("UK spine parity fixture is not present") - spec = importlib.util.spec_from_file_location( - "build_uk_frs_spine", root / "tools" / "build_uk_frs_spine.py" - ) - driver = importlib.util.module_from_spec(spec) - spec.loader.exec_module(driver) + from microcosm.build.uk_runtime import spine_build as driver country = load_country_spec("uk") - stages = [ - stage - for stage in country.sources.stages - if stage.stage not in UK_SPINE_EXCLUSIONS - ] + stages = [stage for stage in country.sources.stages] _, implementations = fixture_stage_plan_inputs(fixture / "sources") - graph = uk_spine_graph() + graph = add_uk_spine_gate_nodes( + uk_spine_graph(), spec=country, engine_identity="fixture-engine" + ) compiled = compile_graph(graph) store = ContentStore(tmp_path / "store") + engine = PolicyEngineUKEngine() + registry = uk_registry(dict(implementations), graph=graph) + register_spine_gate_kernel( + registry, spec=country, engine=engine, engine_identity="fixture-engine" + ) manifest = run_graph( compiled, sources={"frs": fixture / "sources"}, store=store, - kernels=uk_registry(dict(implementations)), + kernels=registry, resume="forbid", decisions=(), ) - final = manifest.population(compiled.versions[compiled.order[-1]]) + final = manifest.population(compiled.versions[stages[-1].stage]) + + # The new stored gate nodes must reproduce the maintained evaluators on + # the checkpoint populations and live transform evidence they used before. + gates = uk_spine_gate_manifest(country) + names = tuple(stage.stage for stage in stages) + for phase, stage_names, population in ( + ( + "assembled", + names[: names.index("frs_brma") + 1], + manifest.population(compiled.versions["frs_brma"]), + ), + ("transferred", names, final), + ): + key = manifest.nodes[f"spine.gates.{phase}"].opaque_artifacts["gate_report"] + stored = json.loads(store.load_bytes(key)) + expected = evaluate_phase( + gates, + phase, + EvidenceContext( + frame=population, + artifacts={ + "stage_evidence": driver._collect_stage_evidence( + stage_names=stage_names, + implementations=implementations, + ), + "rules_engine": engine, + }, + ), + registry=UK_GATE_REGISTRY, + ) + assert stored == gate_phase_report_payload(expected, gates=gates) records = driver._graph_stage_records( manifest=manifest, store=store, stages=stages, frame=final diff --git a/packages/microcosm-build/tests/test_uk_graph_evidence.py b/packages/microcosm-build/tests/test_uk_graph_evidence.py new file mode 100644 index 000000000..563a0e4f1 --- /dev/null +++ b/packages/microcosm-build/tests/test_uk_graph_evidence.py @@ -0,0 +1,234 @@ +"""Spine evidence survives cached execution and a completely fresh process.""" + +import json +import subprocess +import sys +from dataclasses import replace +from pathlib import Path +from types import SimpleNamespace + +import pytest + +from microcosm.build.country_spec import load_country_spec +from microcosm.build.uk_runtime.graph import ( + uk_registry, + uk_spine_endpoint, + uk_spine_graph, + uk_spine_operation_inventory, +) +from microcosm.build.uk_runtime.graph_evidence import ( + add_uk_spine_gate_nodes, + load_spine_stage_artifacts, + spine_sidecar_evidence, +) +from microcosm.graph import ContentStore, compile_graph, load_source, run_graph + + +def test_spine_evidence_replays_without_instantiating_original_transform(tmp_path): + country = load_country_spec("uk") + country = replace( + country, sources=replace(country.sources, stages=country.sources.stages[:1]) + ) + graph = uk_spine_graph(country) + compiled = compile_graph(graph) + source = ( + Path(__file__).resolve().parents[2] + / "microcosm-graph/tests/fixtures/parity/uk_spine/sources" + ) + + class Root: + sampling = {"fraction": 0.1, "seed": 7} + fit_weight_records = ( + SimpleNamespace(fit_name="fixture-fit", weight_kind="design"), + ) + + def run_with_sources(self, frame, sources): + return load_source("csv-tables", sources["frs"]) + + def checkpoint_metadata(self): + return { + "evidence": {"rows": 8}, + "replay_payload": {"classification": "fixture"}, + } + + store = ContentStore(tmp_path / "store") + cold = run_graph( + compiled, + sources={"frs": source}, + store=store, + kernels=uk_registry({"frs_spine": Root()}, graph=graph), + resume="forbid", + ) + expected = load_spine_stage_artifacts(cold, store, stage_names=("frs_spine",)) + warm = run_graph( + compiled, + sources={"frs": source}, + store=store, + kernels=uk_registry(graph=graph), + resume="require", + ) + assert all(receipt.store_hit for receipt in warm.nodes.values()) + assert ( + load_spine_stage_artifacts(warm, store, stage_names=("frs_spine",)) == expected + ) + path = tmp_path / "manifest.json" + warm.save(path) + script = """ +import json, sys +from microcosm.graph import ContentStore, RunManifest +from microcosm.build.uk_runtime.graph_evidence import load_spine_stage_artifacts +store = ContentStore(sys.argv[2]) +manifest = RunManifest.load(sys.argv[1], store) +print(json.dumps(load_spine_stage_artifacts(manifest, store, stage_names=('frs_spine',)), sort_keys=True)) +""" + result = subprocess.run( + [sys.executable, "-c", script, str(path), str(store.root)], + check=True, + text=True, + capture_output=True, + ) + assert json.loads(result.stdout) == expected + sidecar = spine_sidecar_evidence(expected) + assert sidecar["sampling"] == {"fraction": 0.1, "seed": 7} + assert sidecar["fit_weight_records"]["frs_spine"][0]["weight_kind"] == "design" + + +def test_spine_inventory_is_roster_derived_and_gates_bind_checkpoint_versions(): + country = load_country_spec("uk") + spine = uk_spine_graph(country) + endpoint = uk_spine_endpoint(spine) + inventory = uk_spine_operation_inventory(spine, country) + assert tuple(row["stage"] for row in inventory) == endpoint.stage_names + assert inventory[0]["node"] == "create_uk_frs" + assert "normalization" in inventory[0]["coupling"] + wealth = next(row for row in inventory if row["stage"] == "was_wealth") + assert "Donor and recipient" in wealth["coupling"] + graph = add_uk_spine_gate_nodes(spine, spec=country, engine_identity="test-engine") + compiled = compile_graph(graph) + assembled = graph.node("spine.gates.assembled") + transferred = graph.node("spine.gates.transferred") + assert assembled.population == compiled.versions["frs_brma"] + assert assembled.population != compiled.versions["was_wealth"] + assert transferred.population == endpoint.population + assert {edge.producer for edge in transferred.artifact_inputs} >= { + "spine.gates.assembled", + "was_wealth", + } + assert "spine.gates.assembled" in compiled.predecessors["frs_brma.checkpoint"] + assert "spine.gates.assembled" in compiled.predecessors["was_wealth"] + assert graph.node("was_wealth").params["spine_gate_phase"] == "assembled" + + +def _assembled_report(status): + from microcosm.build.gate_battery import ( + GateOutcome, + GatePhaseReport, + GateStatus, + gate_phase_report_payload, + ) + from microcosm.build.gates import GateResult + from microcosm.build.uk_runtime.graph_evidence import uk_spine_gate_manifest + + gates = uk_spine_gate_manifest(load_country_spec("uk")) + entries = tuple(entry for entry in gates.gates if entry.phase == "assembled") + selected = next( + entry + for entry in entries + if entry.criticality == "release_blocking" and not entry.evidence_absent_blocks + ) + outcomes = [] + for entry in entries: + state = status if entry == selected else GateStatus.PASSED + evaluated = state in (GateStatus.PASSED, GateStatus.FAILED) + outcomes.append( + GateOutcome( + entry, + state, + result=GateResult( + entry.gate, + state is GateStatus.PASSED, + () if state is GateStatus.PASSED else ("fixture failure",), + ) + if evaluated + else None, + reason=None if evaluated else "fixture missing reference", + ) + ) + return gate_phase_report_payload( + GatePhaseReport("assembled", tuple(outcomes)), gates=gates + ), selected.id + + +def test_assembled_admission_refuses_before_model_and_preserves_development_policy( + tmp_path, +): + from microcosm.build.gate_battery import GateStatus + from microcosm.build.uk_runtime.graph_evidence import ( + require_uk_spine_gate_admission, + ) + from microcosm.build.uk_runtime.graph_kernels import UKStageKernel + + report, failed = _assembled_report(GateStatus.FAILED) + path = tmp_path / "stored-phase.json" + path.write_text(json.dumps(report)) + context = SimpleNamespace( + artifacts={"spine_gate": SimpleNamespace(payload=path.read_bytes())}, + params={"spine_gate_phase": "assembled", "spine_gate_release_candidate": False}, + ) + + class UntouchedModel: + def __call__(self, frame): + raise AssertionError("A blocked assembled spine must not run a donor model") + + with pytest.raises(ValueError, match=failed): + UKStageKernel("was_wealth", UntouchedModel()).run(context) + assert json.loads(path.read_bytes()) == report + report, _ = _assembled_report(GateStatus.EVIDENCE_ABSENT) + context.artifacts["spine_gate"].payload = json.dumps(report).encode() + require_uk_spine_gate_admission(context) + context.params["spine_gate_release_candidate"] = True + with pytest.raises(ValueError, match="block downstream"): + require_uk_spine_gate_admission(context) + + +def test_full_gate_replay_carries_spine_failure_and_checks_bound_policy(): + from microcosm.build.gate_battery import ( + GateOutcome, + GatePhaseReport, + GateStatus, + gate_phase_report_payload, + ) + from microcosm.build.gates import GateResult + from microcosm.build.uk_runtime.full_gates import uk_full_gate_manifest + from microcosm.build.uk_runtime.graph_terminal import ( + _full_gate_enforcement, + decode_full_gate_report, + ) + + gates = uk_full_gate_manifest() + report = GatePhaseReport( + "preflight", + tuple( + GateOutcome(entry, GateStatus.PASSED, GateResult(entry.gate, True)) + for entry in gates.gates + if entry.phase == "preflight" + ), + ) + previous, failed = _assembled_report(GateStatus.FAILED) + document = { + "schema_version": 1, + "kind": "uk_full_gate_report", + "selection_receipt": None, + "sample_fraction": 0.01, + "release_candidate": False, + "report": gate_phase_report_payload(report, gates=gates), + "upstream_phase_reports": {"spine_assembled": previous}, + } + document["enforcement"] = _full_gate_enforcement(document, report) + restored, enforcement = decode_full_gate_report(document) + assert restored.phase == "preflight" + assert enforcement["artifact_permitted"] is False + assert failed in enforcement["structural_failures"] + document["upstream_phase_reports"] = {} + with pytest.raises(ValueError, match="enforcement"): + decode_full_gate_report(document) diff --git a/packages/microcosm-build/tests/test_uk_graph_terminal.py b/packages/microcosm-build/tests/test_uk_graph_terminal.py new file mode 100644 index 000000000..b6430b60b --- /dev/null +++ b/packages/microcosm-build/tests/test_uk_graph_terminal.py @@ -0,0 +1,724 @@ +"""Export artifacts bind exact H5 content and survive filesystem recreation.""" + +import numpy as np +import pandas as pd +import pytest + +from microcosm.build.uk_runtime.graph_terminal import ( + add_uk_export_continuation, + add_uk_export_preparation, + describe_uk_export, + materialize_uk_export, + register_uk_terminal_kernels, + validate_uk_export, +) +from microcosm.build.uk_runtime.national_frame import uk_national_frame +from microcosm.frame import MassChangeRecord, WeightKind + + +def _frame(): + household = pd.DataFrame( + { + "household_id": [1, 2], + "household_clone_index": [0, 0], + "region": ["LONDON", "SOUTH_EAST"], + "oa_code": ["E00000001", "E00000002"], + "lsoa_code": ["E01000001", "E01000002"], + "msoa_code": ["E02000001", "E02000002"], + "local_authority_code": ["E09000001", "E07000002"], + "ward_code": ["E05000001", "E05000002"], + "constituency_code": ["E14000001", "E14000002"], + "region_code": ["E12000007", "E12000008"], + "itl3_code": ["TLI31", "TLJ31"], + "itl2_code": ["TLI3", "TLJ3"], + "itl1_code": ["TLI", "TLJ"], + } + ) + for column in household.select_dtypes(include=["str", "object"]).columns: + household[column] = household[column].astype("string") + return uk_national_frame( + person=pd.DataFrame( + { + "person_id": [1, 2], + "person_household_id": [1, 2], + "person_benunit_id": [1, 2], + "person_clone_index": [0, 0], + "income": pd.Series([10.5, 20.5], dtype="float32"), + } + ), + benunit=pd.DataFrame({"benunit_id": [1, 2], "benunit_clone_index": [0, 0]}), + household=household, + time_period="2024", + weight_kind=WeightKind.CALIBRATED, + household_weights=np.array([13.0, 87.0]), + mass_log=(MassChangeRecord("household", 100.0, 100.0, 1.0, "calibration"),), + ) + + +def test_export_roundtrip_preserves_dtype_weights_lineage_period_and_gate(tmp_path): + pytest.importorskip("tables") + frame = _frame() + descriptor = describe_uk_export( + frame, + bindings={"pool_replicates": 1, "dataset_households": 2, "target_scope": "all"}, + ) + path = tmp_path / "full.h5" + record = materialize_uk_export(frame, descriptor, path) + report = validate_uk_export(path, descriptor) + assert report["passed"] is True + assert record == report["dataset"] + assert descriptor["tables"]["person"]["dtypes"][-1] == "float32" + assert descriptor["time_period"] == "2024" + assert descriptor["weight_kind"] == "calibrated" + path.unlink() + materialize_uk_export(frame, descriptor, path) + assert validate_uk_export(path, descriptor)["passed"] is True + + +def test_export_refuses_population_changed_after_descriptor(tmp_path): + frame = _frame() + descriptor = describe_uk_export(frame, bindings={}) + frame.table("person").loc[0, "income"] = 99.0 + with pytest.raises(ValueError, match="descriptor"): + materialize_uk_export(frame, descriptor, tmp_path / "changed.h5") + + +def test_export_readback_reports_changed_stored_values(tmp_path): + pytest.importorskip("tables") + frame = _frame() + descriptor = describe_uk_export(frame, bindings={}) + path = tmp_path / "full.h5" + materialize_uk_export(frame, descriptor, path) + with pd.HDFStore(path) as store: + person = store["person"] + person.loc[0, "income"] = np.float32(77.0) + store.put("person", person, format="table") + report = validate_uk_export(path, descriptor) + assert report["passed"] is False + assert any("person" in failure for failure in report["failures"]) + + +def test_graph_export_continuation_reuses_numerics_and_validates_recreated_file( + tmp_path, +): + import json + + from microcosm.graph import ( + Capabilities, + ContentStore, + Determinism, + Graph, + KernelBase, + KernelRegistry, + KernelResult, + Node, + Owned, + SourceRef, + StructuralDelta, + compile_graph, + run_graph, + ) + + pytest.importorskip("tables") + + class Create(KernelBase): + ref = "fixture.create@1" + capabilities = Capabilities( + Determinism.DETERMINISTIC, structural=StructuralDelta.CREATE + ) + + def run(self, context): + return KernelResult(frame=_frame()) + + frame = _frame() + identifiers = { + "person_id", + "benunit_id", + "household_id", + "person_household_id", + "person_benunit_id", + } + source = tmp_path / "fixture.txt" + source.write_text("deterministic export fixture") + graph = Graph( + "uk", + (SourceRef("fixture", "raw-bytes-v1"),), + ( + Node( + id="root", + kernel=Create.ref, + structural=StructuralDelta.CREATE, + sources=("fixture",), + outputs=tuple( + Owned(entity, column, str(table[column].dtype)) + for entity in frame.entities + for table in (frame.table(entity),) + for column in table.columns + if column not in identifiers + ), + ), + ), + ) + graph = add_uk_export_preparation( + graph, + population="root", + bindings={"pool_replicates": 1, "dataset_households": 2, "target_scope": "all"}, + ) + registry = KernelRegistry() + registry.register(Create()) + register_uk_terminal_kernels(registry) + store = ContentStore(tmp_path / "store") + numerical = run_graph( + compile_graph(graph), sources={"fixture": source}, store=store, kernels=registry + ) + key = numerical.nodes["uk.full.export.prepare"].opaque_artifacts[ + "export_descriptor" + ] + descriptor = json.loads(store.load_bytes(key)) + path = tmp_path / "full.h5" + materialize_uk_export(numerical.population("root"), descriptor, path) + continued = add_uk_export_continuation( + graph, population="root", manifest_binding={"key": numerical.key} + ) + terminal = run_graph( + compile_graph(continued), + sources={"fixture": source, "exported_dataset": path}, + store=store, + kernels=registry, + ) + assert terminal.nodes["root"].store_hit + assert terminal.nodes["uk.full.export.prepare"].store_hit + assert terminal.nodes["uk.full.export.readback"].receipt["outcome"] == "pass" + inventory = json.loads( + store.load_bytes( + terminal.nodes["uk.full.package"].opaque_artifacts["package_inventory"] + ) + ) + assert inventory["numerical_graph"]["key"] == numerical.key + assert inventory["build_bindings"]["target_scope"] == "all" + path.unlink() + materialize_uk_export(numerical.population("root"), descriptor, path) + replay = run_graph( + compile_graph(continued), + sources={"fixture": source, "exported_dataset": path}, + store=store, + kernels=registry, + ) + assert replay.nodes["root"].store_hit + assert replay.nodes["uk.full.export.readback"].receipt["outcome"] == "pass" + + +@pytest.mark.parametrize("households", [None, 2]) +def test_full_gate_nodes_precede_dense_and_bind_final_problem_axis(households): + from microcosm.build.uk_runtime.graph import uk_spine_endpoint, uk_spine_graph + from microcosm.build.uk_runtime.graph_build import UKFullBuildConfig, uk_full_graph + from microcosm.build.uk_runtime.graph_calibration import UKGraphCalibrationConfig + from microcosm.build.uk_runtime.graph_terminal import append_uk_full_gate_nodes + from microcosm.graph import compile_graph + + raw = uk_spine_graph(source_mode="split") + full = uk_full_graph( + UKFullBuildConfig( + calibration_year=2025, + calibration=UKGraphCalibrationConfig(dataset_households=households), + ), + spine=raw, + ) + graph = append_uk_full_gate_nodes( + full.graph, + calibration=full.calibration, + spine_stage_names=uk_spine_endpoint(raw).stage_names, + engine_identity="fixture-engine", + review_date="2026-09-10", + ) + compiled = compile_graph(graph) + assert ( + "uk.full.gates.preflight" + in compiled.predecessors[full.calibration.dense_producer] + ) + preflight = graph.node("uk.full.gates.preflight") + assert not preflight.inputs + assert not {"problem", "solution"} & { + item.name for item in preflight.artifact_inputs + } + final = graph.node("uk.full.gates.calibrated") + assert final.population == full.calibration.population + inputs = {item.name: item for item in final.artifact_inputs} + assert inputs["problem"].producer == full.calibration.problem_producer + assert inputs["solution"].artifact == ( + "solution" if households is None else "refit_solution" + ) + + +def test_full_preflight_persists_real_source_failures_without_matrix_diagnostics( + monkeypatch, +): + import json + from types import SimpleNamespace + + from microcosm.build.gate_battery import _gates_manifest_payload + from microcosm.build.uk_runtime import full_gates + from microcosm.build.uk_runtime.graph_terminal import ( + UKFullGateKernel, + decode_full_gate_report, + ) + from microcosm.graph.canonical import canonical_json + + selection = { + "schema": "microcosm.calibrate.target-selection.v1", + "selector": {"geography_levels": None}, + "included": [ + {"name": "n", "period": 2025, "geography_level": "country"}, + {"name": "l", "period": 2025, "geography_level": "constituency"}, + ], + "excluded": [], + } + empty_registry = {"country": "uk", "specs": []} + + def artifact(payload): + return SimpleNamespace(payload=canonical_json(payload), key="a" * 64) + + context = SimpleNamespace( + params={ + "engine_identity": "fixture", + "phase": "preflight", + "gate_manifest": canonical_json( + _gates_manifest_payload(full_gates.uk_full_gate_manifest()) + ).decode(), + "spine_stage_names": ("frs_spine",), + "review_date": "2026-09-10", + "sample_fraction": 1.0, + "release_candidate": True, + }, + artifacts={ + "selection": artifact({"receipt": selection}), + "surface": artifact( + { + "national_registry": empty_registry, + "uk_ledger_compiled_registries": { + "2023": empty_registry, + "2025": empty_registry, + }, + "uk_ledger_compiled_local_registries": {"2025": empty_registry}, + } + ), + "spine_provenance": artifact( + {"stages": ["frs_spine"], "stage_evidence": {}} + ), + }, + ) + + def forbidden(*args, **kwargs): + raise AssertionError( + "Source preflight must not compute final matrix diagnostics" + ) + + monkeypatch.setattr(full_gates, "build_full_gate_context", forbidden) + result = UKFullGateKernel(coverage_engine=object(), engine_identity="fixture").run( + context + ) + report, enforcement = decode_full_gate_report(result.artifacts["gate_report"]) + assert report.phase == "preflight" + assert len(report.outcomes) == 6 + assert enforcement["artifact_permitted"] is False + assert "uk_release_family_build_stages" in enforcement["structural_failures"] + document = json.loads(result.artifacts["gate_report"]) + assert document["target_diagnostics"] == [] + document["enforcement"]["artifact_permitted"] = True + with pytest.raises(ValueError, match="enforcement"): + decode_full_gate_report(document) + + +def test_terminal_byte_materialization_recreates_exact_files(tmp_path): + import hashlib + from types import SimpleNamespace + + from microcosm.build.uk_runtime.graph_terminal import ( + materialize_uk_terminal_artifacts, + ) + from microcosm.graph import ContentStore + + payloads = { + "calibration_diagnostics": b'{"score":1}', + "target_diagnostics_csv": b"target,estimate\na,1\n", + "area_support_csv": b"area,rows\na,2\n", + "holdout": b'{"report_only":true}', + "selection": b'{"registry":{}}', + } + store = ContentStore(tmp_path / "store") + keys = {} + for name, payload in payloads.items(): + key = hashlib.sha256(payload).hexdigest() + store.put_bytes(key, payload) + keys[name] = key + manifest = SimpleNamespace( + nodes={ + "uk.full.gates.calibrated": SimpleNamespace(opaque_artifacts=keys), + "uk.full.holdout": SimpleNamespace(opaque_artifacts=keys), + "uk.full.target_selection": SimpleNamespace(opaque_artifacts=keys), + } + ) + first = materialize_uk_terminal_artifacts( + manifest, store, directory=tmp_path, stem="full" + ) + for record in first.values(): + (tmp_path / record["filename"]).unlink() + assert ( + materialize_uk_terminal_artifacts( + manifest, store, directory=tmp_path, stem="full" + ) + == first + ) + + +def test_holdout_kernel_preserves_existing_rotations_and_seed_settings(monkeypatch): + import json + from types import SimpleNamespace + + from test_uk_full_calibration_graph import preflight_payload + from test_uk_local_rowwise import _assigned, _clone_frame + + from microcosm.build.uk_runtime import graph_targets, local_rowwise + from microcosm.build.uk_runtime.graph_terminal import UKFullHoldoutKernel + from microcosm.calibrate import Target, TargetSet, build_constraint_matrix + from microcosm.calibrate.artifacts import encode_problem + + frame = _clone_frame() + names = ("households", "tenure/social_rent", "tenure/private_rent") + problem = local_rowwise.build_uk_rowwise_local_matrix( + pd.DataFrame({name: [1.0, 2.0, 3.0] for name in names}, index=[101, 102, 103]), + _assigned(), + pd.DataFrame( + {"code": ["E001", "S001"], **{name: [3.0, 3.0] for name in names}} + ), + ) + calls = [] + + def solve(frame, training, **kwargs): + calls.append( + (kwargs["seed"], kwargs["dataset_households"], kwargs["selection_seed"]) + ) + return SimpleNamespace(weights=np.ones(2), selected_support=np.array([0, 2])) + + monkeypatch.setattr(local_rowwise, "solve_uk_rowwise_weights_under_doctrine", solve) + monkeypatch.setattr( + local_rowwise, + "_derive_uk_local_bound_families_from_target_frame", + lambda *a, **k: (), + ) + monkeypatch.setattr( + graph_targets, + "reconstruct_uk_full_problem_inputs", + lambda context: SimpleNamespace( + frame=frame, local_problem=problem, national_rows=None, bound_families=() + ), + ) + original = encode_problem( + build_constraint_matrix( + frame, + TargetSet([Target("count", "household", lambda f: np.ones(3), 3.0)]), + "household", + ), + entity_ids=[101, 102, 103], + ) + context = SimpleNamespace( + params={ + "skip_holdout": False, + "target_weight_rule": "uniform", + "epochs": 1, + "learning_rate": 0.1, + "dataset_households": 2, + "seed": 42, + "selection_seed": 17, + "selection_pi_hi": 1.0, + }, + artifacts={ + "preflight": SimpleNamespace(payload=preflight_payload(), key="a" * 64), + "problem": SimpleNamespace(payload=original, key="b" * 64), + }, + ) + result = json.loads(UKFullHoldoutKernel().run(context).artifacts["holdout"]) + expected = local_rowwise.rotated_uk_local_holdout( + frame, + problem, + bound_families=(), + epochs=1, + learning_rate=0.1, + dataset_households=2, + solve_seed=42, + selection_seed=17, + ) + assert { + key: value for key, value in result.items() if key != "graph_binding" + } == expected + assert calls == [(42, 2, 17)] * 10 + + +def test_final_gate_kernel_owns_complete_diagnostics_and_reuses_decoded_result( + monkeypatch, +): + import json + from dataclasses import replace + from types import SimpleNamespace + + from test_uk_full_calibration_graph import preflight_payload + + from microcosm.build.gate_battery import _gates_manifest_payload + from microcosm.build.uk_runtime import full_gates, geography_ladder + from microcosm.build.uk_runtime.graph_targets import registry_payload + from microcosm.build.uk_runtime.graph_terminal import ( + UKFullGateKernel, + decode_full_gate_report, + ) + from microcosm.calibrate import ( + TargetRegistry, + TargetSet, + TargetSpec, + build_constraint_matrix, + calibrate, + ) + from microcosm.calibrate.artifacts import ( + decode_problem, + encode_calibration_result, + encode_problem, + encode_solution, + ) + from microcosm.frame import Frame, Weights + from microcosm.graph.canonical import canonical_json + + original = _frame() + tables = {entity: original.table(entity).copy() for entity in original.entities} + tables["household"]["source_household_id"] = [1, 2] + tables["household"]["household_is_spi_synthetic"] = False + tables["household"]["household_is_capital_gains_clone"] = False + initial = Frame( + tables, + original.schema, + {"household": Weights(np.array([13.0, 87.0]), WeightKind.IMPORTANCE)}, + original.strata, + metadata=original.metadata, + ) + specs = [ + TargetSpec( + name="national", + entity="household", + value=100.0, + measure="household_id", + period=2024, + source="fixture", + family="households", + metadata={ + "geography_level": "country", + "geography_id": "UK", + "materialization": "uk_national_measure", + }, + ), + TargetSpec( + name="local", + entity="household", + value=13.0, + measure="household_id", + period=2024, + source="fixture", + family="census_households", + metadata={ + "geography_level": "constituency", + "geography_id": "E14000001", + "materialization": "uk_local_surface", + "area_type": "constituency", + "area_code": "E14000001", + "metric": "households", + }, + ), + ] + targets = TargetSet( + [ + replace(specs[0].to_target(), measure=lambda f: np.ones(2)), + replace(specs[1].to_target(), measure=lambda f: np.array([1.0, 0.0])), + ] + ) + selection = { + "schema": "microcosm.calibrate.target-selection.v1", + "selector": {"geography_levels": None}, + "included": [ + { + "name": spec.name, + "period": 2024, + "geography_level": spec.metadata["geography_level"], + } + for spec in specs + ], + "excluded": [], + } + problem = build_constraint_matrix(initial, targets, "household") + problem_bytes = encode_problem( + problem, + entity_ids=[1, 2], + target_metadata=[{**spec.metadata, "family": spec.family} for spec in specs], + bindings={"target_selection": selection}, + ) + ordered = decode_problem(problem_bytes) + result = calibrate(initial, targets, weight_entity="household", epochs=1, seed=42) + frame = result.frame + + def artifact(payload, raw=False): + return SimpleNamespace( + payload=payload if raw else canonical_json(payload), key="a" * 64 + ) + + empty = {"country": "uk", "specs": []} + artifacts = { + "problem": artifact(problem_bytes, True), + "result": artifact( + encode_calibration_result( + result, entity_ids=[1, 2], problem_sha256=ordered.sha256 + ), + True, + ), + "solution": artifact( + encode_solution( + result.weights, entity_ids=[1, 2], problem_sha256=ordered.sha256 + ), + True, + ), + "selection": artifact( + { + "receipt": selection, + "registry": registry_payload(TargetRegistry(specs, country="uk")), + } + ), + "preflight": artifact(preflight_payload(selection=selection), True), + "spine_provenance": artifact( + {"stages": ["frs_spine"], "stage_evidence": {}, "fit_weight_records": {}} + ), + "surface": artifact( + { + "national_registry": registry_payload( + TargetRegistry(specs[:1], country="uk") + ), + "uk_ledger_compiled_registries": {"2023": empty, "2025": empty}, + "uk_ledger_compiled_local_registries": {"2025": empty}, + } + ), + "holdout": artifact({"report_only": True, "outcome": "fixture"}), + } + monkeypatch.setattr( + geography_ladder, + "load_uk_oa_ladder", + lambda path: SimpleNamespace( + constituency_code=np.array(["E14000001", "E14000002"]), + local_authority_code=np.array(["E09000001", "E07000002"]), + ), + ) + monkeypatch.setattr( + full_gates, + "load_efrs_parity_reference", + lambda: SimpleNamespace(input_entities={}), + ) + monkeypatch.setattr( + full_gates, "uk_aggregate_admin_totals", lambda frame, gates: ({}, []) + ) + + def forbidden(*args, **kwargs): + raise AssertionError("The decoded result diagnostics must be reused") + + monkeypatch.setattr(full_gates, "_build_diagnostics", forbidden) + context = SimpleNamespace( + tables={entity: frame.table(entity) for entity in frame.entities}, + weights={"household": frame.weights_for("household")}, + strata=frame.strata, + frame_metadata=frame.metadata, + frame_mass_log=frame.mass_log, + frame_column_order={}, + params={ + "phase": "terminal", + "engine_identity": "fixture", + "review_date": "2026-09-10", + "sample_fraction": 1.0, + "release_candidate": False, + "spine_stage_names": ("frs_spine",), + "gate_manifest": canonical_json( + _gates_manifest_payload(full_gates.uk_full_gate_manifest()) + ).decode(), + }, + artifacts=artifacts, + sources={"uk_ladder": "fixture"}, + ) + stored = UKFullGateKernel(coverage_engine=object(), engine_identity="fixture").run( + context + ) + phase, _ = decode_full_gate_report(stored.artifacts["gate_report"]) + assert phase.phase == "terminal" + document = json.loads(stored.artifacts["calibration_diagnostics"]) + assert document["uk_diagnostics"]["rotated_holdout"]["outcome"] == "fixture" + assert len(document["targets"]) == 2 + assert ( + len(pd.read_csv(__import__("io").BytesIO(stored.artifacts["area_support_csv"]))) + == 4 + ) + + +def test_package_validates_materialized_evidence_against_graph_bytes(tmp_path): + import json + from types import SimpleNamespace + + from microcosm.build.uk_runtime.graph_terminal import UKPackageInventoryKernel + from microcosm.graph.canonical import canonical_json + + payload = b'{"diagnostics":"graph-owned"}' + evidence = tmp_path / "full.diagnostics.json" + evidence.write_bytes(payload) + readback = { + "schema_version": 1, + "kind": "uk_full_build_export_readback", + "passed": True, + "dataset": {"filename": "full.h5", "sha256": "b" * 64, "size_bytes": 10}, + "content_sha256": "c" * 64, + "bindings": {"K": 20, "k": 2, "scope": "all"}, + } + context = SimpleNamespace( + params={ + "manifest_binding": "{}", + "evidence_files": json.dumps({"diagnostics": evidence.name}), + }, + sources={"exported_evidence_diagnostics": evidence}, + artifacts={ + "export_readback": SimpleNamespace( + payload=canonical_json(readback), key="d" * 64 + ), + "diagnostics": SimpleNamespace(payload=payload, key="a" * 64), + }, + ) + document = json.loads( + UKPackageInventoryKernel().run(context).artifacts["package_inventory"] + ) + assert document["evidence_files"]["diagnostics"]["graph_artifact_key"] == "a" * 64 + evidence.write_bytes(b"tampered") + with pytest.raises(ValueError, match="differs from its graph artifact"): + UKPackageInventoryKernel().run(context) + readback["passed"] = False + context.artifacts["export_readback"].payload = canonical_json(readback) + with pytest.raises(ValueError, match="H5 readback failed"): + UKPackageInventoryKernel().run(context) + + +@pytest.mark.requires_uk +def test_gate_cache_identity_includes_country_reference_resource_bytes(monkeypatch): + from types import SimpleNamespace + + from microcosm.build import country_spec + from microcosm.build.uk_runtime.graph_terminal import UKFullGateKernel + + kernel = UKFullGateKernel(coverage_engine=object(), engine_identity="fixture") + monkeypatch.setattr( + country_spec, + "load_country_spec", + lambda country: SimpleNamespace(fingerprint="a" * 64), + ) + first = kernel.implementation_hash() + monkeypatch.setattr( + country_spec, + "load_country_spec", + lambda country: SimpleNamespace(fingerprint="b" * 64), + ) + assert kernel.implementation_hash() != first diff --git a/packages/microcosm-build/tests/test_uk_hmrc_income_source_manifest.py b/packages/microcosm-build/tests/test_uk_hmrc_income_source_manifest.py index 92a4009a3..857db2ac7 100644 --- a/packages/microcosm-build/tests/test_uk_hmrc_income_source_manifest.py +++ b/packages/microcosm-build/tests/test_uk_hmrc_income_source_manifest.py @@ -1,4 +1,4 @@ -"""Contract tests for the raw UK HMRC/SPI income source manifest.""" +"""Canonical raw-spine HMRC contracts bind current producers and official sources.""" from __future__ import annotations @@ -9,691 +9,192 @@ from microcosm.build.uk_runtime.hmrc_source_contract import ( assert_uk_hmrc_income_source_contract_current, + uk_hmrc_weighted_qrf_output_columns, ) -_MANIFEST_PATH = ( - Path(__file__).resolve().parents[1] - / "src" - / "microcosm" - / "build" - / "uk" - / "hmrc_income_source_stages.json" +MANIFEST = ( + Path(__file__).resolve().parents[1] / "src/microcosm/build/uk/source_stages.json" ) -_CANONICAL_SOURCE_STAGES_PATH = ( - Path(__file__).resolve().parents[1] - / "src" - / "microcosm" - / "build" - / "uk" - / "source_stages.json" -) -_COLLATED_ODS_URL = ( - "https://assets.publishing.service.gov.uk/media/" - "69f1f12d2fae53a03709682f/Collated_Tables_3_1_to_3_11_2324.ods" -) -_COLLATED_ODS_SHA256 = ( - "ad063b06b2bdeef8600dbbb09d48153337a4966f8c7eea50df7a2e0304ebd73e" -) -_COLLATED_ODS_SIZE_BYTES = 166_693 -_SPI_DONOR_SHA256 = "5ef829461060c91a2a47be59ad541d9b519fc3976d66ca80d4920f711bb96f66" -_SPI_DONOR_SIZE_BYTES = 141_323_762 -_OFFICIAL_COMPONENTS = [ - "employment_income", - "self_employment_income", - "state_pension", - "private_pension_income", - "property_income", - "savings_interest_income", - "dividend_income", - "other_investment_income", -] -_STAGE1_OUTPUTS = [ - "self_employment_income", - "savings_interest_income", - "dividend_income", - "private_pension_income", - "property_income", - "other_investment_income", - "gift_aid", - "charitable_investment_gifts", - "hmrc_spi_pay", - "hmrc_spi_employment_benefits", - "hmrc_spi_employment_expenses", - "hmrc_spi_incapacity_benefit_income", - "hmrc_spi_other_social_security_income", - "hmrc_spi_taxable_termination_pay", - "hmrc_spi_unemployment_benefit_income", - "hmrc_spi_miscellaneous_employment_income", - "hmrc_spi_other_income", - "hmrc_spi_state_pension_income", -] -_STAGE2_INCOME_PREDICTORS = [ - "employment_income", - "self_employment_income", - "savings_interest_income", - "dividend_income", - "private_pension_income", - "property_income", -] -_FRS_HMRC_FULL_CONSTITUENTS = [ - "hmrc_spi_pay", - "hmrc_spi_unemployment_benefit_income", - "hmrc_spi_incapacity_benefit_income", -] -_FRS_HMRC_NAMED_SUBSETS = [ - "ossben_identifiable_subset", - "srp_regular_code5", -] -_FRS_HMRC_SOURCE_ABSENT = [ - "EPB", - "EXPS", - "TAXTERM", - "MOTHINC", - "OTHERINC", -] -_DERIVED_AUXILIARIES = [ - "hmrc_spi_employed_income", - "hmrc_spi_total_earned_income", - "hmrc_spi_total_investment_income", - "hmrc_spi_assessable_income", -] -_COMPONENT_COLUMNS = { - "employment_income": ("Table_3_6", 4, 5), - "self_employment_income": ("Table_3_6", 1, 2), - "state_pension": ("Table_3_6", 7, 8), - "private_pension_income": ("Table_3_6", 10, 11), - "property_income": ("Table_3_7", 1, 2), - "savings_interest_income": ("Table_3_7", 4, 5), - "dividend_income": ("Table_3_7", 7, 8), - "other_investment_income": ("Table_3_7", 10, 11), -} - - -def _manifest() -> dict[str, object]: - return json.loads(_MANIFEST_PATH.read_text(encoding="utf-8")) - - -def _runtime_manifest() -> dict[str, object]: - payload = json.loads(_CANONICAL_SOURCE_STAGES_PATH.read_text(encoding="utf-8")) - retained = [ - stage - for stage in payload["stages"] - if stage["stage"] == "frs_hmrc_retained_leaves" - ] - hmrc = [stage for stage in payload["stages"] if stage["stage"] == "hmrc_spi_income"] - assert len(retained) == 1 - assert len(hmrc) == 1 - stage = dict(hmrc[0]) - stage["base_candidate"] = dict(_manifest()["stages"][0]["base_candidate"]) - stage["operations"] = [ - *retained[0]["operations"], - *hmrc[0]["operations"], - ] - return { - "country": payload["country"], - "version": payload["version"], - "stages": [stage], - } - - -def _stage(payload: dict[str, object]) -> dict[str, object]: - stages = payload["stages"] - assert isinstance(stages, list) - assert len(stages) == 1 - stage = stages[0] - assert isinstance(stage, dict) - return stage - +INCOME = "hmrc_spi_income_spine" -def _by_role(stage: dict[str, object]) -> dict[str, dict[str, object]]: - artifacts = stage["artifacts"] - assert isinstance(artifacts, list) - assert all(isinstance(artifact, dict) for artifact in artifacts) - return {artifact["role"]: artifact for artifact in artifacts} +def _payload(): + return json.loads(MANIFEST.read_text()) -def _by_kind(stage: dict[str, object]) -> dict[str, dict[str, object]]: - operations = stage["operations"] - assert isinstance(operations, list) - assert all(isinstance(operation, dict) for operation in operations) - return {operation["kind"]: operation for operation in operations} +def _stage(payload, name=INCOME): + return next(stage for stage in payload["stages"] if stage["stage"] == name) -def test__given_uk_hmrc_manifest__then_one_scoped_stage_is_declared() -> None: - payload = _manifest() - stage = _stage(payload) - assert payload["version"] == 1 - assert payload["country"] == "uk" - assert stage["stage"] == "hmrc_spi_income" - assert stage["grain"] == "person" - assert stage["official_table_components"] == _OFFICIAL_COMPONENTS - assert stage["donor_relief_outputs"] == [ - "gift_aid", - "charitable_investment_gifts", - ] - assert stage["outputs"] == [ - *_OFFICIAL_COMPONENTS, - "gift_aid", - "charitable_investment_gifts", - *_DERIVED_AUXILIARIES, - ] - assert [operation["kind"] for operation in stage["operations"]] == [ - "verify_certified_candidate", - "retain_adjudicated_frs_hmrc_leaves", - "verify_pinned_hmrc_source_pair", - "replace_zero_weight_spi_support", - "strict_read_private_table", - "fit_weighted_qrf_stage1", - "fit_weighted_qrf_stage2", - "materialize_hmrc_income_bands_fail_closed", - "classify_hmrc_income_facts_with_reviewed_fences", - "gate_distributional_effective_mass", - ] - - -def test__given_hmrc_source_artifacts__then_vintages_and_runtime_hashes_are_exact() -> ( - None -): - stage = _stage(_manifest()) - artifacts = _by_role(stage) - - assert len(stage["artifacts"]) == 2 - assert set(artifacts) == {"qrf_donor", "published_fact_surface"} - donor = artifacts["qrf_donor"] - assert donor["survey"] == "Survey of Personal Incomes Public Use Tape 2022-23" - assert donor["vintage"] == "2022-23" - assert donor["ukds_study_number"] == "SN 9422" - assert donor["doi"] == "10.5255/UKDA-SN-9422-1" - assert donor["filename"] == "put2223uk.tab" - assert donor["sha256"] == _SPI_DONOR_SHA256 - assert donor["size_bytes"] == _SPI_DONOR_SIZE_BYTES - assert donor["access"] == "private_local_input" - assert donor["locator"] == "caller-supplied local input" - assert donor["runtime_sha256_required"] is True - - surface = artifacts["published_fact_surface"] - assert surface["vintage"] == "2023-24" - assert surface["locator"] == _COLLATED_ODS_URL - assert surface["sha256"] == _COLLATED_ODS_SHA256 - assert surface["size_bytes"] == _COLLATED_ODS_SIZE_BYTES - assert surface["mime_type"] == ("application/vnd.oasis.opendocument.spreadsheet") - assert surface["sheets"] == ["Table_3_6", "Table_3_7"] - assert surface["mapped_build_period"] == 2023 - assert surface["period_mapping"] == "tax_year_start" - assert surface["runtime_sha256_required"] is True - assert "tax-year-2023-to-2024" in surface["publication"] - - base = stage["base_candidate"] - assert base["tier"] == "frs" - assert base["sha256"] == ( - "f17306ccb2aad7ff0130be3589b560afb2e2a12a943570911cd0c77f07934833" +def _operation(payload, name, kind): + return next( + operation + for operation in _stage(payload, name)["operations"] + if operation["kind"] == kind ) - assert base["size_bytes"] == 1_315_880_118 - assert base["runtime_sha256_required"] is True -def test__given_frs_channel__then_adjudicated_leaves_and_subsets_are_explicit() -> None: - operations = _by_kind(_stage(_manifest())) - leaves = operations["retain_adjudicated_frs_hmrc_leaves"] - - assert list(leaves["retained_full_constituents"]) == _FRS_HMRC_FULL_CONSTITUENTS - assert list(leaves["retained_named_subsets"]) == _FRS_HMRC_NAMED_SUBSETS - assert leaves["source_absent_full_constituents"] == _FRS_HMRC_SOURCE_ABSENT - assert leaves["status"] == "adjudicated_partial_replay" - assert leaves["source_vintage"] == "2023-24" - assert leaves["mapped_build_period"] == 2023 - assert leaves["retained_full_constituents"]["hmrc_spi_pay"] == { - "spi_concept": "PAY", - "scope": "full", - "raw_sources": ["ADULT.INEARNS"], - "formula": "max(0, ADULT.INEARNS) * (365.25 / 7)", - } - assert leaves["retained_full_constituents"]["hmrc_spi_incapacity_benefit_income"][ - "observed_support" - ].startswith("structural zero") - assert ( - leaves["retained_named_subsets"]["ossben_identifiable_subset"]["scope"] - == "identifiable_subset" - ) - assert leaves["retained_named_subsets"]["srp_regular_code5"]["scope"] == ( - "regular_code5_subset" - ) - assert leaves["forbid_proxy_substitution"] == [ - "employment_income", - "miscellaneous_income", - ] - assert leaves["fail_on_missing_retained_constituent"] is True - assert leaves["fail_on_full_concept_alias"] is True - - source_pair = operations["verify_pinned_hmrc_source_pair"] - assert source_pair == { - "kind": "verify_pinned_hmrc_source_pair", - "artifact_roles": ["qrf_donor", "published_fact_surface"], - "require_before_source_read": True, - "runtime_sha256_required": True, - "fail_on_mismatch": True, - } - - -def test__given_spi_donor__then_both_qrf_stages_are_weighted_and_strict() -> None: - operations = _by_kind(_stage(_manifest())) - - strict_read = operations["strict_read_private_table"] - assert strict_read["artifact_role"] == "qrf_donor" - assert strict_read["filename"] == "put2223uk.tab" - assert strict_read["weight"] == "FACT" - assert strict_read["runtime_sha256_required"] is True - assert strict_read["fail_on_missing_file"] is True - assert strict_read["fail_on_missing_columns"] is True - assert strict_read["fail_on_invalid_weight"] is True - assert { - "EXPS", - "INCPBEN", - "OSSBEN", - "UBISJA", - "MOTHINC", - "OTHERINC", - "CAPALL", - "LOSSBF", - "SRP", - "TI", - } <= set(strict_read["required_columns"]) - - stage1 = operations["fit_weighted_qrf_stage1"] - assert stage1["source_sampling_weight"] == "FACT" - assert stage1["sample_size"] == 100_000 - assert stage1["sample_with_replacement"] is True - assert stage1["post_sample_fit_weight"] == "uniform" - assert stage1["fit_weight_kind"] == "design" - assert stage1["double_apply_source_weight"] is False - assert stage1["outputs"] == _STAGE1_OUTPUTS - employment_derivation = stage1["derived_policyengine_outputs"]["employment_income"] - assert employment_derivation["source_columns"] == [ - "PAY", - "EPB", - "TAXTERM", - ] - assert employment_derivation["formula"] == ( - "hmrc_spi_pay + hmrc_spi_employment_benefits + hmrc_spi_taxable_termination_pay" - ) - assert employment_derivation["derive_after_draw"] is True - assert "employment_income" not in stage1["source_columns"] - assert "employment_income" not in stage1["outputs"] - assert stage1["source_columns"]["other_investment_income"] == ["OTHERINV"] - assert stage1["source_columns"]["hmrc_spi_other_income"] == ["OTHERINC"] - assert stage1["source_columns"]["hmrc_spi_state_pension_income"] == ["SRP"] - assert stage1["ti_identity_absolute_tolerance_gbp"] == 5 - assert stage1["source_ti_identity_fields"] == ["TI", "TEI", "TII"] - reconciliation = stage1["source_leaf_reconciliation"] - assert reconciliation["composite_indicator"] == "AGERANGE == -1" - assert reconciliation["formulas"]["TEI"].startswith("max(0, PAY + EPB - EXPS)") - assert reconciliation["formulas"]["TII"] == ( - "OTHERINV + DIVIDENDS + INCPROP + INCBBS" - ) - assert reconciliation["formulas"]["TI"] == "TEI + TII" - assert reconciliation["maximum_absolute_difference_gbp"] == { - "ordinary": {"TEI": 15, "TII": 10, "TI": 20}, - "composite": {"TEI": 180, "TII": 10, "TI": 180}, - } - assert stage1["stochastic_aggregates_forbidden"] == _DERIVED_AUXILIARIES - assert all( - column not in stage1["source_columns"] for column in _DERIVED_AUXILIARIES - ) - assert all(column not in stage1["outputs"] for column in _DERIVED_AUXILIARIES) - assert "TEI + TII" in stage1["assessable_income_source_semantics"] - assert "deterministic post-draw" in stage1["assessable_income_source_semantics"] - assert stage1["source_columns"]["gift_aid"] == ["GIFTAID"] - assert stage1["source_columns"]["charitable_investment_gifts"] == ["GIFTINV"] - assert "state_pension" not in stage1["outputs"] - assert stage1["joint_draw"] is True - assert stage1["require_all_predictors"] is True - assert stage1["require_all_outputs"] is True - - stage2 = operations["fit_weighted_qrf_stage2"] - assert stage2["weight"] == "household_weight" - assert stage2["weight_mapping"] == "household_to_person" - assert stage2["predictors"] == [ - "age", - "gender", - "region", - *_STAGE2_INCOME_PREDICTORS, - ] - assert "other_investment_income" not in stage2["predictors"] - assert set(stage2["reviewed_absent_predictors"]) == {"other_investment_income"} - assert ( - "stage-1 SPI draw" - in stage2["reviewed_absent_predictors"]["other_investment_income"] - ) +def test_canonical_hmrc_family_matches_runtime_and_has_no_candidate_dependency(): + assert_uk_hmrc_income_source_contract_current() + payload = _payload() + names = {stage["stage"] for stage in payload["stages"]} + assert {"frs_hmrc_spine_leaves", "spi_support_channel", INCOME} <= names + assert not ({"frs_hmrc_retained_leaves", "hmrc_spi_income"} & names) + for name in ("frs_hmrc_spine_leaves", "spi_support_channel", INCOME): + stage = _stage(payload, name) + assert "base_candidate" not in stage + assert "verify_certified_candidate" not in { + o["kind"] for o in stage["operations"] + } + assert not MANIFEST.with_name("hmrc_income_source_stages.json").exists() + + +def test_official_hmrc_sources_and_sampling_weights_remain_bound(): + stage = _stage(_payload()) + artifacts = {artifact["role"]: artifact for artifact in stage["artifacts"]} assert ( - "exactly six income predictors" - in stage2["reviewed_absent_predictors"]["other_investment_income"] + artifacts["qrf_donor"]["sha256"] + == "5ef829461060c91a2a47be59ad541d9b519fc3976d66ca80d4920f711bb96f66" ) assert ( - "no other_investment_income column" - in stage2["reviewed_absent_predictors"]["other_investment_income"] - ) - assert "state_pension_reported" in stage2["outputs"] - assert "universal_credit_reported" in stage2["outputs"] - assert "employee_pension_contributions" in stage2["outputs"] - assert "incapacity_benefit_reported" not in stage2["outputs"] - assert "maternity_allowance_reported" not in stage2["outputs"] - assert set(stage2["reviewed_absent_outputs"]) == { - "incapacity_benefit_reported", - "maternity_allowance_reported", - } - assert stage2["require_all_predictors"] is True - assert stage2["require_all_materializable_outputs"] is True - assert stage2["require_all_outputs"] is False - assert stage2["postprocess"]["gross_savings_interest_income"] == ( - "stage1 INCBBS draw + stage2 tax_free_savings_income" + artifacts["published_fact_surface"]["sha256"] + == "ad063b06b2bdeef8600dbbb09d48153337a4966f8c7eea50df7a2e0304ebd73e" ) - assert "pip_dl_category" in stage2["postprocess"]["refresh_disability_categories"] - assert ( - "is_disabled_for_benefits" in stage2["postprocess"]["refresh_disability_flags"] - ) - - -def test__given_official_tables__then_all_eight_components_fail_closed() -> None: - operations = _by_kind(_stage(_manifest())) - materializer = operations["materialize_hmrc_income_bands_fail_closed"] - - actual_columns = { - component: ( - spec["sheet"], - spec["count_column_index"], - spec["amount_column_index"], - ) - for component, spec in materializer["component_columns"].items() - } - assert actual_columns == _COMPONENT_COLUMNS - assert materializer["mapped_build_period"] == 2023 - assert materializer["period_mapping"] == "tax_year_start" - assert materializer["required_measures"] == ["count", "amount"] - assert materializer["required_band_lower_bounds_gbp"] == [ - 12_570, - 15_000, - 20_000, - 30_000, - 40_000, - 50_000, - 70_000, - 100_000, - 150_000, - 200_000, - 300_000, - 500_000, - 1_000_000, - ] - assert materializer["fail_on_missing_sheet"] is True - assert materializer["fail_on_missing_component"] is True - assert materializer["fail_on_missing_band"] is True - assert materializer["fail_on_non_numeric_value"] is True - - -def test__given_materialized_hmrc_targets__then_all_facts_are_fenced_without_calibration() -> ( - None -): - operations = _by_kind(_stage(_manifest())) - - classification = operations["classify_hmrc_income_facts_with_reviewed_fences"] - assert classification["components"] == _OFFICIAL_COMPONENTS - assert classification["breakdown_dependency"] == "hmrc_spi_assessable_income" - assert classification["frs_breakdown_status"] == "unavailable_full_measure" - assert classification["input_weight_kind"] == "importance" - assert classification["output_weight_kind"] == "importance" - assert classification["calibration_permitted"] is False - assert classification["required_fact_count"] == 208 - assert classification["outcome_counts"] == { - "exact_pass": 0, - "exact_fail": 0, - "directional_pass": 0, - "directional_fail": 0, - "excluded_with_fence": 208, - } - fences = {fence["fence_id"]: fence for fence in classification["reviewed_fences"]} - assert set(fences) == { - "frs_epb_source_absent", - "frs_exps_source_absent", - "frs_taxterm_source_absent", - "frs_mothinc_source_absent", - "frs_otherinc_source_absent", - "frs_ossben_identifiable_subset", - "frs_srp_regular_code5_subset", - "full_frs_tei_band_unavailable", - } - for fence in fences.values(): - assert fence["constituents"] - assert fence["finding"].strip() - assert fence["mass_implication"].strip() - assert fence["rationale"].strip() - assert fences["frs_epb_source_absent"]["raw_sources_searched"] == [ - "JOB.EXPBEN01-EXPBEN13", - "JOB.CARVAL", - "JOB.CARAMT", - "JOB.FUELAMT", - "JOB.VCHAMT", - "JOB.CHVAMT", - ] - assert fences["frs_ossben_identifiable_subset"]["constituents"] == [ - "OSSBEN", - "ossben_identifiable_subset", - ] - assert fences["frs_srp_regular_code5_subset"]["constituents"] == [ - "SRP", - "srp_regular_code5", - ] - full_ti = fences["full_frs_tei_band_unavailable"] - assert len(full_ti["dependent_fence_ids"]) == 7 - assert "non-overlapping" in full_ti["rationale"] - assert classification["fact_fence_id"] == "full_frs_tei_band_unavailable" - assert classification["fail_on_unfenced_exclusion"] is True - assert classification["fail_on_fact_count_mismatch"] is True - assert classification["forbid_biased_estimate_or_delta"] is True - - prior = operations["replace_zero_weight_spi_support"] - assert prior["require_existing_weight"] == 0 - assert prior["spi_prior_national_household_mass_share"] == 0.5 - assert prior["output_weight_kind"] == "importance" - assert prior["preserve_total_household_mass"] is True - assert prior["require_mass_change_record"] is True - - effective = operations["gate_distributional_effective_mass"] - assert effective["columns"] == [ - "gift_aid", - "charitable_investment_gifts", - ] - assert effective["minimum_nondefault_mass_share"] == 0.000001 - assert effective["fail_below_floor"] is True - - -def test__given_standalone_contract__then_sources_are_explicit_artifacts() -> None: - artifacts = _by_role(_stage(_manifest())) - - assert artifacts["qrf_donor"]["access"] == "private_local_input" - assert artifacts["qrf_donor"]["locator"] == "caller-supplied local input" - assert artifacts["published_fact_surface"]["locator"] == _COLLATED_ODS_URL - assert all(artifact["runtime_sha256_required"] for artifact in artifacts.values()) - - -def test_runtime_source_contract_matches_committed_manifest() -> None: - assert_uk_hmrc_income_source_contract_current() + assert artifacts["published_fact_surface"]["mapped_build_period"] == 2024 + assert artifacts["published_fact_surface"]["vintage"] == "2023-24" + payload = _payload() + first = _operation(payload, INCOME, "fit_weighted_qrf_stage1") + second = _operation(payload, INCOME, "fit_weighted_qrf_stage2") + assert first["source_sampling_weight"] == "FACT" + assert first["post_sample_fit_weight"] == "uniform" + assert first["double_apply_source_weight"] is False + assert first["seed"] == 42 and second["seed"] == 43 + assert second["weight_mapping"] == "household_to_person" @pytest.mark.parametrize( - ("path", "replacement", "match"), + "stage_name,kind,field,replacement,match", [ ( - ("stages", 0, "base_candidate", "tier"), - "public", - "base_candidate.tier", - ), - ( - ("stages", 0, "base_candidate", "sha256"), - "0" * 64, - "base_candidate.sha256", - ), - ( - ("stages", 0, "artifacts", 1, "vintage"), - "2022-23", - "published_fact_surface.vintage", - ), - ( - ("stages", 0, "artifacts", 0, "sha256"), - "0" * 64, - "qrf_donor.sha256", + "frs_hmrc_spine_leaves", + "retain_adjudicated_frs_hmrc_leaves", + "population", + "candidate_h5", + "frs_leaves.population", ), ( - ("stages", 0, "artifacts", 1, "size_bytes"), - 1, - "published_fact_surface.size_bytes", + "frs_hmrc_spine_leaves", + "retain_adjudicated_frs_hmrc_leaves", + "source_vintage", + "2023-24", + "source_vintage", ), ( - ("stages", 0, "operations", 1, "status"), - "ready", - "frs_leaves.status", - ), - ( - ( - "stages", - 0, - "operations", - 3, - "spi_prior_national_household_mass_share", - ), - 0.25, + "spi_support_channel", + "allocate_zero_weight_prior_mass", + "share", + 0.1, "prior.mass_share", ), ( - ("stages", 0, "operations", 5, "sample_size"), - 50_000, - "stage1.sample_size", - ), - ( - ("stages", 0, "operations", 5, "outputs"), - ["employment_income"], - "stage1.outputs", - ), - ( - ("stages", 0, "operations", 5, "source_ti_identity_fields"), - ["TI"], - "stage1.source_ti_identity_fields", - ), - ( - ( - "stages", - 0, - "operations", - 5, - "derived_policyengine_outputs", - "employment_income", - "formula", - ), - "hmrc_spi_pay", - "stage1.derived_policyengine_outputs.employment_income.formula", - ), - ( - ( - "stages", - 0, - "operations", - 5, - "source_leaf_reconciliation", - "maximum_absolute_difference_gbp", - "ordinary", - "TEI", - ), - 1_000, - "stage1.source_leaf_reconciliation.maximum_absolute_difference_gbp", - ), - ( - ("stages", 0, "operations", 6, "reviewed_absent_outputs"), - {"maternity_allowance_reported": "changed"}, - "stage2.reviewed_absent_outputs", + "spi_support_channel", + "allocate_zero_weight_prior_mass", + "strata", + [], + "prior.strata", ), ( - ("stages", 0, "operations", 6, "reviewed_absent_predictors"), - {"other_investment_income": "changed"}, - "stage2.reviewed_absent_predictors", + "spi_support_channel", + "stack_zero_weight_donors", + "count", + 0, + "count drifted", ), + (INCOME, "strict_read_private_table", "weight", "uniform", "strict.weight"), ( - ( - "stages", - 0, - "operations", - 7, - "component_columns", - "savings_interest_income", - "amount_column_index", - ), - 6, - "materialize.component_columns", + INCOME, + "fit_weighted_qrf_stage1", + "post_sample_fit_weight", + "FACT", + "post_sample_fit_weight", ), + (INCOME, "fit_weighted_qrf_stage1", "source_columns", {}, "source_columns"), + (INCOME, "fit_weighted_qrf_stage1", "seed", 43, "seed drifted"), + (INCOME, "fit_weighted_qrf_stage2", "predictors", ["age"], "stage2.predictors"), + (INCOME, "redraw_columns_from_fitted_qrf", "rows", "all", "base redraw"), ( - ("stages", 0, "operations", 8, "output_weight_kind"), - "calibrated", - "classification.output_weight_kind", + INCOME, + "materialize_hmrc_income_bands_fail_closed", + "component_columns", + {}, + "component_columns", ), ( - ("stages", 0, "operations", 8, "required_fact_count"), + INCOME, + "classify_hmrc_income_facts_with_reviewed_fences", + "required_fact_count", 207, - "classification.required_fact_count", + "required_fact_count", ), ( - ( - "stages", - 0, - "operations", - 8, - "outcome_counts", - "excluded_with_fence", - ), - 207, - "classification.outcome_counts", - ), - ( - ("stages", 0, "operations", 9, "required_support_channel"), + INCOME, + "gate_distributional_effective_mass", + "required_support_channel", "frs", - "effective.required_support_channel", + "required_support_channel", ), ( - ("stages", 0, "operations", 9, "minimum_nondefault_mass_share"), + INCOME, + "gate_distributional_effective_mass", + "minimum_nondefault_mass_share", 0.01, - "effective.minimum_nondefault_mass_share", + "minimum_nondefault_mass_share", ), ], ) -def test_runtime_source_contract_rejects_manifest_drift( - tmp_path, - path, - replacement, - match, -) -> None: - payload = _runtime_manifest() - cursor = payload - for segment in path[:-1]: - cursor = cursor[segment] - cursor[path[-1]] = replacement - tampered = tmp_path / "hmrc_income_source_stages.json" - tampered.write_text(json.dumps(payload), encoding="utf-8") - +def test_source_contract_rejects_drift( + tmp_path, stage_name, kind, field, replacement, match +): + payload = _payload() + _operation(payload, stage_name, kind)[field] = replacement + path = tmp_path / "sources.json" + path.write_text(json.dumps(payload)) with pytest.raises(ValueError, match=match): - assert_uk_hmrc_income_source_contract_current(tampered) + assert_uk_hmrc_income_source_contract_current(path) @pytest.mark.parametrize( - ("collection", "key"), [("artifacts", "role"), ("operations", "kind")] + "collection,key", [("artifacts", "role"), ("operations", "kind")] ) -def test_runtime_source_contract_rejects_duplicate_keys( - tmp_path, - collection, - key, -) -> None: - payload = _runtime_manifest() - values = payload["stages"][0][collection] +def test_duplicate_source_declarations_are_refused(tmp_path, collection, key): + payload = _payload() + values = _stage(payload)[collection] values.append(dict(values[0])) - tampered = tmp_path / f"duplicate_{key}.json" - tampered.write_text(json.dumps(payload), encoding="utf-8") - + path = tmp_path / "sources.json" + path.write_text(json.dumps(payload)) with pytest.raises(ValueError, match=f"duplicate {key}"): - assert_uk_hmrc_income_source_contract_current(tampered) + assert_uk_hmrc_income_source_contract_current(path) + + +def test_missing_canonical_stage_is_refused(tmp_path): + payload = _payload() + payload["stages"] = [ + s for s in payload["stages"] if s["stage"] != "spi_support_channel" + ] + path = tmp_path / "sources.json" + path.write_text(json.dumps(payload)) + with pytest.raises(ValueError, match="exactly one spi_support_channel"): + assert_uk_hmrc_income_source_contract_current(path) + + +def test_tail_concentration_surface_contains_both_canonical_model_stages(): + payload = _payload() + expected = tuple( + dict.fromkeys( + output + for kind in ("fit_weighted_qrf_stage1", "fit_weighted_qrf_stage2") + for output in _operation(payload, INCOME, kind)["outputs"] + ) + ) + assert uk_hmrc_weighted_qrf_output_columns() == expected + assert {"gift_aid", "self_employment_income"} <= set(expected) diff --git a/packages/microcosm-build/tests/test_uk_hmrc_replay_artifacts.py b/packages/microcosm-build/tests/test_uk_hmrc_replay_artifacts.py index cf2c149c9..48c9af1ad 100644 --- a/packages/microcosm-build/tests/test_uk_hmrc_replay_artifacts.py +++ b/packages/microcosm-build/tests/test_uk_hmrc_replay_artifacts.py @@ -7,6 +7,8 @@ from importlib.resources import files from typing import Any +import pytest + from microcosm.build.uk_runtime.hmrc_income import ( HMRC_SPI_COLLATED_ODS_SHA256, HMRC_SPI_INCOME_BAND_LOWER_BOUNDS, @@ -16,6 +18,7 @@ from microcosm.build.uk_runtime.hmrc_replay import FULL_FRS_TI_BAND_FENCE_ID from microcosm.build.uk_runtime.release_input_coverage import ( DEFAULT_MINIMUM_NONDEFAULT_MASS_SHARE, + assert_uk_release_input_coverage_build_stages, ) from microcosm.build.uk_runtime.spi_income import ( SPI_DONOR_SHA256, @@ -121,22 +124,8 @@ def test_real_replay_binds_sources_identity_and_positive_mass() -> None: } assert sources["hmrc_surface"]["sha256"] == HMRC_SPI_COLLATED_ODS_SHA256 assert sources["hmrc_surface"]["mapped_build_period"] == "2023" - # June-freeze partition, made self-describing (adversarial-review - # disposition, microcosm#723): this report is evidence for the - # grandfathered June release and binds to the FROZEN manifest's period - # mapping - it deliberately does NOT follow the live build period, which - # moved to "2024" with the #723 signed re-map. It retires with the frozen - # manifest after #686 (#687's disposition), never regenerates against a - # different vintage. - frozen_stage = _resource("hmrc_income_source_stages.json")["stages"][0] - frozen_surface = next( - artifact - for artifact in frozen_stage["artifacts"] - if artifact.get("role") == "published_fact_surface" - ) - assert sources["hmrc_surface"]["mapped_build_period"] == str( - frozen_surface["mapped_build_period"] - ) + # This immutable June replay remains historical evidence. Current source + # contracts use the canonical spine stages and the 2024 build period. assert qrf["fits"] == { "uk_frs_only_spi_fill": {"weight_kind": "importance"}, "uk_spi_2022_23_income": {"weight_kind": "design"}, @@ -256,3 +245,9 @@ def test_committed_replay_artifacts_contain_no_row_level_payloads_or_local_paths serialized = json.dumps(payload, allow_nan=False, sort_keys=True) assert "/Users/" not in serialized assert "put2223uk.tab" not in serialized + + +def test_historical_candidate_stages_cannot_satisfy_current_build_contract(): + record = _resource(_BUILD_RECORD_RESOURCE) + with pytest.raises(ValueError, match="hmrc_spi_income"): + assert_uk_release_input_coverage_build_stages(record["stages"]) diff --git a/packages/microcosm-build/tests/test_uk_incumbent_surface_evaluation.py b/packages/microcosm-build/tests/test_uk_incumbent_surface_evaluation.py index a38154087..c0f0cdaa4 100644 --- a/packages/microcosm-build/tests/test_uk_incumbent_surface_evaluation.py +++ b/packages/microcosm-build/tests/test_uk_incumbent_surface_evaluation.py @@ -408,3 +408,37 @@ def test_evaluator_cli_rejects_blocks_before_opening_inputs(blocks): assert result.returncode == 2 assert "--engine-blocks" in result.stderr assert "Traceback" not in result.stderr + + +def test_full_candidate_package_preserves_measured_byte_bindings(): + from microcosm.build.uk_runtime.incumbent_surface_evaluation import ( + candidate_evaluation_manifest, + ) + + package = { + "schema_version": 1, + "kind": "uk_full_build_package", + "readback_passed": True, + "dataset": {"filename": "full.h5", "sha256": "a" * 64, "size_bytes": 42}, + "evidence_files": { + "diagnostics": { + "filename": "full.diagnostics.json", + "sha256": "b" * 64, + "size_bytes": 21, + } + }, + "build_bindings": { + "ledger": {"facts_sha256": "c" * 64, "manifest_sha256": "d" * 64} + }, + } + result = candidate_evaluation_manifest(package) + assert result["outputs"]["dataset"] == { + "path": "full.h5", + "sha256": "a" * 64, + "bytes": 42, + } + assert result["outputs"]["calibration_diagnostics"]["sha256"] == "b" * 64 + assert result["identity"]["ledger"] == package["build_bindings"]["ledger"] + package["dataset"]["filename"] = "../other.h5" + with pytest.raises(ValueError, match="filenames"): + candidate_evaluation_manifest(package) diff --git a/packages/microcosm-build/tests/test_uk_ladder_rowwise_clone.py b/packages/microcosm-build/tests/test_uk_ladder_rowwise_clone.py index 6f7934c74..c076e5de7 100644 --- a/packages/microcosm-build/tests/test_uk_ladder_rowwise_clone.py +++ b/packages/microcosm-build/tests/test_uk_ladder_rowwise_clone.py @@ -359,19 +359,6 @@ def test_write_round_trip_preserves_ladder_columns(toy_ladder, tmp_path) -> None assert "itl1_code" in household.columns -def _load_builder_module(): - import importlib.util - from pathlib import Path - - root = Path(__file__).resolve().parents[3] - path = root / "tools" / "build_uk_rowwise_dataset.py" - spec = importlib.util.spec_from_file_location("build_uk_rowwise_dataset", path) - module = importlib.util.module_from_spec(spec) - assert spec.loader is not None - spec.loader.exec_module(module) - return module - - def _write_seam_h5(path, *, household: pd.DataFrame | None = None) -> None: from microcosm.build.uk_runtime import write_uk_national_frame from microcosm.build.uk_runtime.national_frame import uk_national_frame @@ -395,522 +382,6 @@ def _write_seam_h5(path, *, household: pd.DataFrame | None = None) -> None: write_uk_national_frame(dataset, path) -def test_driver_ladder_route_builds_with_gate(monkeypatch, toy_ladder, tmp_path): - pytest.importorskip("tables") - pytest.importorskip("h5py") - import json - import sys - - ladder, ladder_path = toy_ladder - builder = _load_builder_module() - input_h5 = tmp_path / "staging.h5" - _write_seam_h5(input_h5) - output_dir = tmp_path / "out" - - monkeypatch.setattr( - sys, - "argv", - [ - "build_uk_rowwise_dataset.py", - "--input-h5", - str(input_h5), - "--out", - str(output_dir), - "--ladder", - str(ladder_path), - "--n-clones", - "2", - "--seed", - "7", - ], - ) - assert builder.main() == 0 - manifest = json.loads((output_dir / builder.MANIFEST_FILENAME).read_text()) - assert manifest["parameters"]["assignment_route"] == "ladder" - assert manifest["inputs"]["ladder"]["sha256"] - assert manifest["inputs"]["dataset"]["pin_verified"] is False - assert manifest["inputs"]["ladder"]["pin_verified"] is False - assert manifest["inputs"]["ladder"]["matches_local_area_crosswalk_pin"] is False - summary = manifest["rowwise_dataset"] - assert summary["gate"]["passed"] is True - assert summary["missing_geography_rows"] == 0 - assert summary["assigned_constituencies"] >= 4 - assert summary["weights"]["household_weight_kind"] == "importance" - assert summary["weights"]["mass_conservation"]["passed"] is True - assert summary["source_lineage"]["explicit"] is None - assert summary["area_support"]["source_basis"] == "source_household_id" - assert summary["area_support"]["constituency"]["n_areas"] == len( - np.unique(ladder.constituency_code) - ) - assert summary["area_support"]["la"]["n_areas"] == len( - np.unique(ladder.local_authority_code) - ) - assert set(summary["area_support"]["constituency"]) == { - "n_areas", - "rows_basis", - "min_rows", - "median_rows", - "min_ess", - "median_ess", - "min_distinct_sources", - "median_distinct_sources", - "bottom_by_rows", - "bottom_by_ess", - } - assert sum(row["row_share"] for row in summary["region_mix"]) == pytest.approx( - 1.0 - ) - assert sum( - row["weight_share"] for row in summary["region_mix"] - ) == pytest.approx(1.0) - area_support_path = output_dir / builder.AREA_SUPPORT_FILENAME - assert area_support_path.exists() - assert manifest["outputs"]["area_support_summary"]["path"] == str( - area_support_path - ) - area_support = pd.read_csv(area_support_path) - assert area_support.columns.tolist() == [ - "area_type", - "area_code", - "assigned_households", - "nonzero_households", - "nonzero_source_households", - "weight_sum", - "max_weight", - "effective_sample_size", - ] - assert area_support["area_type"].tolist() == [ - *(["constituency"] * len(np.unique(ladder.constituency_code))), - *(["la"] * len(np.unique(ladder.local_authority_code))), - ] - assert (output_dir / "staging_rowwise.h5").exists() - - -def test_driver_ladder_dry_run_matches_real_assignment( - monkeypatch, toy_ladder, tmp_path -): - pytest.importorskip("tables") - pytest.importorskip("h5py") - import json - import sys - - ladder, ladder_path = toy_ladder - builder = _load_builder_module() - input_h5 = tmp_path / "staging.h5" - _write_seam_h5(input_h5) - plan_dir = tmp_path / "plan" - build_dir = tmp_path / "build" - input_sha256 = builder._sha256(input_h5) - ladder_sha256 = builder._sha256(ladder_path) - - base_argv = [ - "build_uk_rowwise_dataset.py", - "--input-h5", - str(input_h5), - "--input-sha256", - input_sha256, - "--ladder", - str(ladder_path), - "--ladder-sha256", - ladder_sha256, - "--n-clones", - "2", - "--seed", - "7", - ] - monkeypatch.setattr(sys, "argv", [*base_argv, "--out", str(plan_dir), "--dry-run"]) - assert builder.main() == 0 - plan = json.loads((plan_dir / builder.DRY_RUN_PLAN_FILENAME).read_text()) - assert not (plan_dir / "staging_rowwise.h5").exists() - expected_support = plan["expected_support"] - assert expected_support["basis"] == ( - "analytic expectation: constituency household-count share within region x " - "the input's region mix x n_clones; OA population shares within " - "constituency for LA support" - ) - for area_type in ("constituency", "la"): - area_support = expected_support[area_type] - # Same-named stats keys carry their semantics: an expected-mass - # figure must never be silently compared against a realized count. - assert area_support["rows_basis"] == "expected_rows" - assert plan["realized_support"][area_type]["rows_basis"] == "assigned_rows" - assert area_support["n_areas"] <= builder.EXPECTED_SUPPORT_BOTTOM_AREAS - assert len(area_support["bottom"]) == area_support["n_areas"] - assert sum(row["rows"] for row in area_support["bottom"]) == pytest.approx( - plan["plan"]["rows"]["household"] - ) - - monkeypatch.setattr(sys, "argv", [*base_argv, "--out", str(build_dir)]) - assert builder.main() == 0 - manifest = json.loads((build_dir / builder.MANIFEST_FILENAME).read_text()) - assert plan["input"]["dataset"]["pin_verified"] is True - assert plan["input"]["ladder"]["pin_verified"] is True - assert plan["input"]["ladder"]["matches_local_area_crosswalk_pin"] is False - assert manifest["inputs"]["dataset"]["pin_verified"] is True - assert manifest["inputs"]["ladder"]["pin_verified"] is True - assert manifest["inputs"]["ladder"]["matches_local_area_crosswalk_pin"] is False - assert plan["area_support"] == manifest["rowwise_dataset"]["area_support"] - assert plan["region_mix"] == manifest["rowwise_dataset"]["region_mix"] - - # The dry-run's realized support is exact: identical draws to the build. - realized = { - row["area_code"]: row["rows"] - for row in plan["realized_support"]["constituency"]["bottom"] - } - with pd.HDFStore(build_dir / "staging_rowwise.h5", mode="r") as store: - household = store["household"] - built_counts = household["constituency_code"].value_counts() - for code, rows in realized.items(): - assert built_counts.get(code, 0) == rows - assert ( - plan["realized_support"]["constituency"]["n_areas"] - >= manifest["rowwise_dataset"]["assigned_constituencies"] - ) - assert plan["source_lineage"]["explicit"] is None - assert manifest["rowwise_dataset"]["source_lineage"]["explicit"] is None - - -def test_driver_ladder_candidate_k_matches_independent_single_k_plan( - monkeypatch, toy_ladder, tmp_path -) -> None: - pytest.importorskip("tables") - pytest.importorskip("h5py") - import json - import sys - - _, ladder_path = toy_ladder - builder = _load_builder_module() - input_h5 = tmp_path / "staging.h5" - _write_seam_h5(input_h5) - candidate_dir = tmp_path / "candidates" - independent_dir = tmp_path / "independent" - base_argv = [ - "build_uk_rowwise_dataset.py", - "--input-h5", - str(input_h5), - "--ladder", - str(ladder_path), - "--seed", - "7", - ] - - monkeypatch.setattr( - sys, - "argv", - [ - *base_argv, - "--out", - str(candidate_dir), - "--n-clones", - "2", - "--candidate-clone-counts", - "3,1,3", - "--dry-run", - ], - ) - assert builder.main() == 0 - candidate_plan = json.loads( - (candidate_dir / builder.DRY_RUN_PLAN_FILENAME).read_text() - ) - - monkeypatch.setattr( - sys, - "argv", - [ - *base_argv, - "--out", - str(independent_dir), - "--n-clones", - "3", - "--dry-run", - ], - ) - assert builder.main() == 0 - independent_plan = json.loads( - (independent_dir / builder.DRY_RUN_PLAN_FILENAME).read_text() - ) - - assert candidate_plan["plan"]["n_clones"] == 2 - candidates = candidate_plan["candidates"] - assert candidates["clone_counts"] == [1, 3] - assert [candidate["n_clones"] for candidate in candidates["plans"]] == [1, 3] - input_bytes = input_h5.stat().st_size - base_rows = {"person": 5, "benunit": 4, "household": 4} - for candidate in candidates["plans"]: - n_clones = candidate["n_clones"] - assert candidate["rows"] == { - name: rows * n_clones for name, rows in base_rows.items() - } - assert candidate["output_bytes_estimate"] == input_bytes * n_clones - assert set(candidate) == { - "n_clones", - "rows", - "output_bytes_estimate", - "realized_support", - "expected_support", - "area_support", - } - - candidate_k3 = candidates["plans"][1] - assert candidate_k3["realized_support"] == { - area_type: independent_plan["realized_support"][area_type] - for area_type in ("constituency", "la") - } - assert candidate_k3["expected_support"] == { - area_type: independent_plan["expected_support"][area_type] - for area_type in ("constituency", "la") - } - assert candidate_k3["area_support"] == independent_plan["area_support"] - - -def test_driver_ladder_spine_lineage_plan_manifest_parity( - monkeypatch, toy_ladder, tmp_path -): - pytest.importorskip("tables") - pytest.importorskip("h5py") - import json - import sys - - _, ladder_path = toy_ladder - builder = _load_builder_module() - input_h5 = tmp_path / "spine.h5" - _write_seam_h5(input_h5, household=_spine_household_frame()) - plan_dir = tmp_path / "plan" - build_dir = tmp_path / "build" - base_argv = [ - "build_uk_rowwise_dataset.py", - "--input-h5", - str(input_h5), - "--ladder", - str(ladder_path), - "--n-clones", - "2", - "--seed", - "7", - ] - - monkeypatch.setattr(sys, "argv", [*base_argv, "--out", str(plan_dir), "--dry-run"]) - assert builder.main() == 0 - plan = json.loads((plan_dir / builder.DRY_RUN_PLAN_FILENAME).read_text()) - - monkeypatch.setattr(sys, "argv", [*base_argv, "--out", str(build_dir)]) - assert builder.main() == 0 - manifest = json.loads((build_dir / builder.MANIFEST_FILENAME).read_text()) - - explicit = plan["source_lineage"]["explicit"] - assert explicit == manifest["rowwise_dataset"]["source_lineage"]["explicit"] - assert plan["area_support"]["source_basis"] == "source_household_id" - assert ( - manifest["rowwise_dataset"]["area_support"]["source_basis"] - == "source_household_id" - ) - assert explicit == { - "basis": "explicit_lineage_columns", - "columns_present": [ - "source_household_id", - "household_support_channel", - "household_support_clone_index", - "household_is_spi_synthetic", - "household_is_capital_gains_clone", - "household_is_cgt_band_donor", - ], - "distinct_source_households": 3, - "distinct_by_support_channel": {"frs": 1, "spi": 2}, - "flag_counts": { - "household_is_spi_synthetic": 2, - "household_is_capital_gains_clone": 1, - "household_is_cgt_band_donor": 1, - }, - } - - -def test_driver_ladder_refuses_crosswalk_combo(monkeypatch, toy_ladder, tmp_path): - pytest.importorskip("tables") - import sys - - _, ladder_path = toy_ladder - builder = _load_builder_module() - input_h5 = tmp_path / "staging.h5" - _write_seam_h5(input_h5) - monkeypatch.setattr( - sys, - "argv", - [ - "build_uk_rowwise_dataset.py", - "--input-h5", - str(input_h5), - "--out", - str(tmp_path / "out"), - "--ladder", - str(ladder_path), - "--crosswalk", - str(tmp_path / "crosswalk.csv"), - ], - ) - with pytest.raises(ValueError, match="mutually exclusive"): - builder.main() - - -def test_driver_ladder_sha256_refuses_crosswalk(monkeypatch, tmp_path) -> None: - pytest.importorskip("tables") - import sys - - builder = _load_builder_module() - input_h5 = tmp_path / "staging.h5" - _write_seam_h5(input_h5) - monkeypatch.setattr( - sys, - "argv", - [ - "build_uk_rowwise_dataset.py", - "--input-h5", - str(input_h5), - "--crosswalk", - str(tmp_path / "crosswalk.csv"), - "--ladder-sha256", - "0" * 64, - "--out", - str(tmp_path / "out"), - "--dry-run", - ], - ) - - with pytest.raises(ValueError, match="ladder-sha256.*ladder"): - builder.main() - - -def test_driver_ladder_sha256_refuses_generated_crosswalk_route( - monkeypatch, tmp_path -) -> None: - # Neither --ladder nor --crosswalk: the driver would download and build a - # crosswalk, and there is no ladder artifact the pin could verify. A - # silently ignored pin would report verification that never happened. - pytest.importorskip("tables") - import sys - - builder = _load_builder_module() - input_h5 = tmp_path / "staging.h5" - _write_seam_h5(input_h5) - monkeypatch.setattr( - sys, - "argv", - [ - "build_uk_rowwise_dataset.py", - "--input-h5", - str(input_h5), - "--ladder-sha256", - "0" * 64, - "--out", - str(tmp_path / "out"), - "--dry-run", - ], - ) - - with pytest.raises(ValueError, match="ladder-sha256 requires --ladder"): - builder.main() - - -def test_area_support_stats_read_nonzero_rows_not_assigned() -> None: - # An area whose only rows are zero-weight synthetics shapes no estimate - # there: the stats and the thinness ranking must read mass-carrying rows, - # with the assigned count still visible per bottom area. - builder = _load_builder_module() - support = pd.DataFrame( - { - "area_code": ["A1", "A2"], - "assigned_households": [40, 3], - "nonzero_households": [0, 3], - "nonzero_source_households": [0, 3], - "weight_sum": [0.0, 30.0], - "max_weight": [0.0, 10.0], - "effective_sample_size": [0.0, 3.0], - } - ) - - stats = builder._area_support_stats(support) - - assert stats["rows_basis"] == "nonzero_households" - assert stats["min_rows"] == 0 - assert stats["bottom_by_rows"][0] == { - "area_code": "A1", - "rows": 0, - "assigned": 40, - } - - -def test_driver_ladder_pin_mismatch_refuses_before_parse( - monkeypatch, toy_ladder, tmp_path -) -> None: - pytest.importorskip("tables") - pytest.importorskip("h5py") - import sys - - _, ladder_path = toy_ladder - builder = _load_builder_module() - input_h5 = tmp_path / "staging.h5" - _write_seam_h5(input_h5) - - def unexpected_parse(_path): - raise AssertionError("ladder pin mismatch must refuse before NPZ parsing") - - monkeypatch.setattr(builder, "load_uk_oa_ladder", unexpected_parse) - monkeypatch.setattr( - sys, - "argv", - [ - "build_uk_rowwise_dataset.py", - "--input-h5", - str(input_h5), - "--ladder", - str(ladder_path), - "--ladder-sha256", - "0" * 64, - "--out", - str(tmp_path / "out"), - "--dry-run", - ], - ) - - with pytest.raises(SystemExit, match=r"--ladder sha mismatch: measured"): - builder.main() - - -def test_driver_ladder_dry_run_refuses_legacy_preassigned_geography( - monkeypatch, toy_ladder, tmp_path -): - pytest.importorskip("tables") - pytest.importorskip("h5py") - import sys - - _, ladder_path = toy_ladder - builder = _load_builder_module() - input_h5 = tmp_path / "preassigned.h5" - household = _household_frame().assign( - constituency_code_oa="stale-constituency" - ) - _write_seam_h5(input_h5, household=household) - monkeypatch.setattr( - sys, - "argv", - [ - "build_uk_rowwise_dataset.py", - "--input-h5", - str(input_h5), - "--ladder", - str(ladder_path), - "--out", - str(tmp_path / "plan"), - "--dry-run", - ], - ) - - with pytest.raises( - ValueError, - match=r"stale \*_oa columns cannot ride through", - ): - builder.main() - - def test_ladder_clone_pins_per_copy_weights_and_fk_alignment(toy_ladder) -> None: ladder, _ = toy_ladder result = clone_uk_dataset_with_ladder_geography(_seam_frame(), ladder, n_clones=2) @@ -999,43 +470,6 @@ def test_write_refuses_post_gate_geography_mutation(toy_ladder, tmp_path) -> Non assert not (tmp_path / "mutated.h5").exists() -def test_dry_run_bottom_covers_every_toy_area(monkeypatch, toy_ladder, tmp_path): - pytest.importorskip("tables") - pytest.importorskip("h5py") - import json - import sys - - _, ladder_path = toy_ladder - builder = _load_builder_module() - input_h5 = tmp_path / "staging.h5" - _write_seam_h5(input_h5) - plan_dir = tmp_path / "plan" - monkeypatch.setattr( - sys, - "argv", - [ - "build_uk_rowwise_dataset.py", - "--input-h5", - str(input_h5), - "--ladder", - str(ladder_path), - "--out", - str(plan_dir), - "--n-clones", - "2", - "--dry-run", - ], - ) - assert builder.main() == 0 - plan = json.loads((plan_dir / builder.DRY_RUN_PLAN_FILENAME).read_text()) - constituency = plan["realized_support"]["constituency"] - # The toy surface must stay within the bottom cap so the exactness test - # keeps comparing every area; growing the fixture past the cap should - # fail here loudly instead of silently weakening the comparison. - assert constituency["n_areas"] <= builder.EXPECTED_SUPPORT_BOTTOM_AREAS - assert len(constituency["bottom"]) == constituency["n_areas"] - - def test_inherited_clone_index_is_replaced_like_the_pre_frame_writer( toy_ladder, tmp_path ) -> None: @@ -1080,3 +514,24 @@ def test_reserved_in_memory_clone_names_fail_closed(toy_ladder) -> None: ) with pytest.raises(ValueError, match="reserved in-memory clone column"): clone_uk_dataset_with_ladder_geography(poisoned, ladder, n_clones=1) + + +def test_area_support_counts_only_mass_carrying_rows(toy_ladder): + from microcosm.build.uk_runtime.local_rowwise import uk_ladder_area_support_summary + + ladder, _ = toy_ladder + households = pd.DataFrame( + { + "source_household_id": [1, 2, 3], + "constituency_code": ["E14000001", "E14000001", "E14000002"], + "local_authority_code": ["E09000001", "E09000001", "E09000002"], + "household_weight": [0.0, 0.0, 10.0], + } + ) + support = uk_ladder_area_support_summary(households, ladder)[ + "constituency" + ].set_index("area_code") + assert support.loc["E14000001", "assigned_households"] == 2 + assert support.loc["E14000001", "nonzero_households"] == 0 + assert support.loc["E14000001", "nonzero_source_households"] == 0 + assert support.loc["E14000002", "nonzero_households"] == 1 diff --git a/packages/microcosm-build/tests/test_uk_national_calibration.py b/packages/microcosm-build/tests/test_uk_national_calibration.py index ad4615440..90595c84c 100644 --- a/packages/microcosm-build/tests/test_uk_national_calibration.py +++ b/packages/microcosm-build/tests/test_uk_national_calibration.py @@ -1,4 +1,4 @@ -"""The UK national Ledger-backed calibration stage.""" +"""Shared UK measure materialization and target-loss contracts.""" from __future__ import annotations @@ -21,47 +21,15 @@ UK_NATIONAL_TARGET_LOSS_CAP, UK_NATIONAL_TARGET_WEIGHT_RULE, UKNationalSolveDoctrine, - national_calibration, uk_doctrine_with_overrides, uk_national_target_loss_weights, ) -from microcosm.build.uk_runtime.ledger_targets import ( - UKLedgerTargetCompilation, - _uk_contract_targets, -) -from microcosm.build.uk_runtime.national_calibration import ( - UKNationalCalibrationStage, - _post_solve_calibration_record, - national_calibration_mass_reason, -) -from microcosm.build.uk_runtime.national_frame import ( - validate_uk_national_frame, - write_uk_national_frame, -) from microcosm.calibrate import TargetRegistry, TargetSpec from microcosm.frame import EntitySchema, Frame, WeightKind, Weights ACTIVE_REFERENCE_COUNT = 415 -def _uc_reference(**overrides) -> LedgerTargetReference: - values = { - "name": "dwp.uc.households", - "ledger_selector": { - "source_name": "dwp", - "source_concept": "dwp.uc_benefit_units", - "geography_level": "country", - }, - "entity": "benunit", - "measure": "dwp/uc/households", - "family": "dwp_uc", - "period": 2025, - "metadata": {"contract_target_id": "dwp.uc.households"}, - } - values.update(overrides) - return LedgerTargetReference(**values) - - def _fact( *, concept: str = "dwp.uc_benefit_units", @@ -85,141 +53,6 @@ def _fact( } -def _registry(*, value: float = 30.0) -> TargetRegistry: - return TargetRegistry( - [ - TargetSpec( - name="dwp.uc.households", - entity="benunit", - measure="dwp/uc/households", - value=value, - source="test", - family="dwp_universal_credit", - metadata={"contract_target_id": "dwp.uc.households"}, - ) - ], - country="uk", - ) - - -def _frame() -> Frame: - ids = np.arange(4, dtype="int64") - return Frame( - { - "person": pd.DataFrame( - { - "person_id": ids, - "person_benunit_id": ids, - "person_household_id": ids, - } - ), - "benunit": pd.DataFrame( - {"benunit_id": ids, "universal_credit": [1.0, 1.0, 0.0, 0.0]} - ), - "household": pd.DataFrame({"household_id": ids, "region": "LONDON"}), - }, - EntitySchema(group_entities=("benunit", "household")), - {"household": Weights(np.full(4, 10.0), WeightKind.DESIGN)}, - metadata={"time_period": "2023"}, - ) - - -def _frame_without_uc_column() -> Frame: - frame = _frame() - return Frame( - { - "person": frame.table("person"), - "benunit": frame.table("benunit").drop(columns=["universal_credit"]), - "household": frame.table("household"), - }, - frame.schema, - {"household": frame.weights_for("household")}, - frame.strata, - mass_log=frame.mass_log, - metadata=frame.metadata, - ) - - -class StubMeasureResolver: - contract_targets = { - "dwp.uc.households": { - "bindings": { - "policyengine": { - "from_entity": "benunit", - "value_variable": "universal_credit", - } - } - } - } - - def __init__(self): - self.calls = [] - - def knows(self, entity, variable): - return (entity, variable) == ("benunit", "universal_credit") - - def compute(self, entity, variable): - self.calls.append((entity, variable)) - return np.array([1.0, 1.0, 0.0, 0.0]), "stub_uc" - - def receipt(self): - return {"provider": "stub_uc"} - - -class StubCrosstabResolver: - """Supply separate prepared benefit-unit flags and affected-child counts. - - The national stage must inject both measurements temporarily, materialize - the target count, then remove the scratch inputs from the returned frame. - """ - - contract_targets = None - measures = { - "uc_tcl_affected_benunit_proxy": np.array([True, False, True]), - "uc_tcl_affected_child_count_proxy": np.array([2.0, 0.0, 3.0]), - } - - def knows(self, entity, variable): - return (entity, variable) == ("person", "cgt_2024_gains") or ( - entity == "benunit" and variable in self.measures - ) - - def compute(self, entity, variable): - if (entity, variable) == ("person", "cgt_2024_gains"): - return np.array([0.0, 7000.0, 12000.0, 500.0, 0.0, 0.0]), "stub_engine_year" - assert self.knows(entity, variable) - return self.measures[variable].copy(), "stub_benunit_tcl_measure" - - def receipt(self): - return {"provider": "stub_crosstab_flag"} - - -def _nested_frame() -> Frame: - return Frame( - { - "person": pd.DataFrame( - { - "person_id": np.arange(6, dtype="int64"), - "person_benunit_id": [0, 0, 1, 2, 3, 3], - "person_household_id": [0, 0, 0, 1, 2, 2], - } - ), - "benunit": pd.DataFrame( - { - "benunit_id": np.arange(4, dtype="int64"), - "universal_credit": [1.0, 0.0, 1.0, 1.0], - } - ), - "household": pd.DataFrame( - {"household_id": np.arange(3, dtype="int64"), "region": "LONDON"} - ), - }, - EntitySchema(group_entities=("benunit", "household")), - {"household": Weights(np.array([10.0, 20.0, 30.0]), WeightKind.DESIGN)}, - metadata={"time_period": "2023"}, - ) - - def _reference_by_name(name: str) -> LedgerTargetReference: from microcosm.build.country_spec import load_country_spec @@ -330,154 +163,6 @@ def _materialization_binding_frame( ) -def test_uc_calibration_compiles_and_moves_weighted_count_towards_fact() -> None: - frame = _frame() - stage = UKNationalCalibrationStage( - _registry(), - band_edge_registry=_registry(), - period=2025, - doctrine=UKNationalSolveDoctrine(epochs=200, learning_rate=0.05), - ) - - result = stage(frame) - - before = 20.0 - after = float(result.weights_for("household").values[:2].sum()) - assert abs(after - 30.0) < abs(before - 30.0) - assert stage.manifest["activated_reference_count"] == 1 - assert stage.manifest["resolved_reference_count"] == 1 - assert stage.manifest["matrix_target_count"] == 1 - assert stage.diagnostics[0]["target"] == 30.0 - assert result.weights_for("household").kind is WeightKind.CALIBRATED - assert len(result.mass_log) == 1 - mass_change = stage.manifest["weights"]["calibration_mass_change"] - assert mass_change["entity"] == "household" - assert "National doctrine calibration" in mass_change["reason"] - assert stage.manifest["weights"]["household_weight_kind_chain"] == [ - {"stage": "staging", "kind": "design"}, - {"stage": "national_calibration", "kind": "calibrated"}, - ] - assert stage.manifest["weights"]["mass_log_records_before_calibration"] == 0 - assert stage.manifest["weights"]["mass_log_records"] == 1 - assert stage.manifest["solve"]["n_targets"] == 1 - assert stage.manifest["solve"]["n_households"] == 4 - - -def test_uc_calibration_stage_accepts_benunit_grain_reference_on_nested_frame() -> None: - frame = _nested_frame() - stage = UKNationalCalibrationStage( - _registry(value=60.0), - band_edge_registry=_registry(value=60.0), - period=2025, - doctrine=UKNationalSolveDoctrine(epochs=5), - ) - - result = stage(frame) - - assert stage.manifest["activated_reference_count"] == 1 - assert stage.manifest["resolved_reference_count"] == 1 - assert stage.manifest["matrix_target_count"] == 1 - assert stage.diagnostics[0]["target"] == 60.0 - assert stage.diagnostics[0]["estimate"] == pytest.approx(60.0) - validate_uk_national_frame(result) - - -def test_stage_measure_resolver_injects_columns_then_restores_pristine_output() -> None: - frame = _frame_without_uc_column() - resolver = StubMeasureResolver() - original_columns = { - entity: set(frame.table(entity).columns) for entity in frame.entities - } - stage = UKNationalCalibrationStage( - _registry(), - band_edge_registry=_registry(), - period=2025, - doctrine=UKNationalSolveDoctrine(epochs=5), - measure_resolver=resolver, - ) - - result = stage(frame) - - assert resolver.calls == [("benunit", "universal_credit")] - assert stage.manifest["measure_resolution"]["provider"] == {"provider": "stub_uc"} - assert stage.manifest["measure_resolution"]["attached"] == { - "benunit.universal_credit": "stub_uc" - } - for entity in frame.entities: - assert set(result.table(entity).columns) == original_columns[entity] - assert "universal_credit" not in result.table("benunit") - validate_uk_national_frame(result) - - -def test_stage_manifest_omits_measure_resolution_without_resolver() -> None: - stage = UKNationalCalibrationStage( - _registry(), - band_edge_registry=_registry(), - period=2025, - doctrine=UKNationalSolveDoctrine(epochs=5), - ) - - stage(_frame()) - - assert "measure_resolution" not in stage.manifest - - -def test_stage_threads_band_edge_registry_to_materialization(monkeypatch) -> None: - captured = [] - real_materialize = national_calibration.materialize_uk_ledger_targets - - def capture_materialize(*args, **kwargs): - captured.append(kwargs) - return real_materialize(*args, **kwargs) - - monkeypatch.setattr( - national_calibration, - "materialize_uk_ledger_targets", - capture_materialize, - ) - sentinel = TargetRegistry([], country="uk") - stage = UKNationalCalibrationStage( - _registry(), - period=2025, - doctrine=UKNationalSolveDoctrine(epochs=1), - band_edge_registry=sentinel, - ) - - stage(_frame()) - - assert captured[-1]["band_edge_registry"] is sentinel - - # The parameter is required, never defaulted: a stage cannot tell a - # pruned registry from a full one (#803 review finding 1). - with pytest.raises(TypeError, match="band_edge_registry"): - UKNationalCalibrationStage( - _registry(), - period=2025, - doctrine=UKNationalSolveDoctrine(epochs=1), - ) - - -def test_activated_unresolvable_compiled_reference_aborts_loudly() -> None: - stage = UKNationalCalibrationStage( - UKLedgerTargetCompilation( - registry=TargetRegistry([], country="uk"), - unsupported=( - { - "name": "dwp.uc.households", - "period": "2025", - "reason": "did not match a Ledger fact selector", - }, - ), - ), - band_edge_registry=TargetRegistry([], country="uk"), - period=2025, - doctrine=UKNationalSolveDoctrine(epochs=1), - ) - - with pytest.raises(RuntimeError, match="did not match a Ledger fact selector"): - stage(_frame()) - - def test_chronicle_184_uc_and_obr_references_compile_fail_closed() -> None: from microcosm.build.country_spec import load_country_spec from microcosm.build.ledger_targets import compile_ledger_target_references @@ -543,244 +228,6 @@ def test_chronicle_184_uc_and_obr_references_compile_fail_closed() -> None: assert {spec.name for spec in registry.specs} == {"obr.universal_credit_in_cap"} -def test_packaged_binding_classes_materialize_through_national_stage() -> None: - selected_names = ( - "dwp.uc.households", - "obr.esa", - "hmrc.cgt.taxpayers_total", - "dwp.uc.two_child_limit.children_affected", - "hmrc.salary_sacrifice.it_relief_basic_rate", - ) - references = tuple(_reference_by_name(name) for name in selected_names) - facts = [ - fact - for reference, value in zip( - references, - (20.0, 33.0, 2.0, 5.0, 3.0), - strict=True, - ) - for fact in _facts_for_reference(reference, value) - ] - from microcosm.build.ledger_targets import compile_ledger_target_references - from microcosm.build.uk_runtime.ledger_targets import ( - UKFrameTargetAdapter, - materialize_uk_ledger_targets, - ) - - registry = compile_ledger_target_references(facts, references, country="uk") - headline = next(spec for spec in registry.specs if spec.name == "dwp.uc.households") - assert headline.value == 20.0 - assert headline.metadata["ledger_member_fact_count"] == "120" - resolver = StubCrosstabResolver() - resolver.contract_targets = _uk_contract_targets() - stage = UKNationalCalibrationStage( - registry, - band_edge_registry=registry, - period=2025, - doctrine=UKNationalSolveDoctrine(epochs=1, learning_rate=0.01), - measure_resolver=resolver, - ) - - input_frame = _materialization_binding_frame() - original_columns = { - entity: set(input_frame.table(entity).columns) - for entity in input_frame.entities - } - - result = stage(input_frame) - - assert stage.manifest["activated_reference_count"] == len(selected_names) - assert stage.manifest["resolved_reference_count"] == len(selected_names) - assert stage.manifest["matrix_target_count"] == len(selected_names) - # The staged frame stays writer-clean: no prepared scratch column survives - # onto the returned tables (adjudicated lifecycle; slash-named scratch - # crashes the HDFStore staging writer). Original input columns — including - # the fixture's precomputed counterfactual delta — are exactly preserved. - for entity in input_frame.entities: - assert set(result.table(entity).columns) == original_columns[entity] - # The binding classes produce the right prepared values on the adapter… - adapter = UKFrameTargetAdapter(_materialization_binding_frame()) - # The same table-scoped injection the resolution loop performs. - for variable, values in resolver.measures.items(): - adapter.tables["benunit"][variable] = values.copy() - adapter.tables["person"]["cgt_2024_gains"] = np.array( - [0.0, 7000.0, 12000.0, 500.0, 0.0, 0.0] - ) - materialize_uk_ledger_targets(adapter, registry, period=2025) - materialized = { - ("benunit", "dwp/uc/households"): [1.0, 0.0, 1.0], - ("household", "obr/esa"): [11.0, 22.0, 0.0], - ("person", "hmrc/cgt_taxpayers"): [0.0, 1.0, 1.0, 0.0, 0.0, 0.0], - # The prepared affected-child counts stay distinct from the claim - # indicator [1.0, 0.0, 1.0] and legacy person-native entitlement flags. - ("benunit", "dwp/uc/two_child_limit/children_affected"): [2.0, 0.0, 3.0], - ("person", "hmrc/salary_sacrifice_it_relief_basic_rate"): [ - 1.0, - 2.0, - 0.0, - 0.0, - 0.0, - 0.0, - ], - } - for (entity, measure), expected in materialized.items(): - assert adapter.tables[entity][measure].tolist() == expected - - -def test_packaged_materialization_skip_aborts_national_stage() -> None: - from microcosm.build.ledger_targets import compile_ledger_target_references - - reference = _reference_by_name("hmrc.salary_sacrifice.it_relief_basic_rate") - registry = compile_ledger_target_references( - [_fact_for_reference(reference, 3.0)], - [reference], - country="uk", - ) - stage = UKNationalCalibrationStage( - registry, - band_edge_registry=registry, - period=2025, - doctrine=UKNationalSolveDoctrine(epochs=1), - ) - - with pytest.raises(RuntimeError, match="could not materialize every"): - stage(_materialization_binding_frame(include_counterfactual_delta=False)) - - -def test_calibration_preserves_entity_ids_and_national_integrity() -> None: - frame = _frame() - stage = UKNationalCalibrationStage( - _registry(), - band_edge_registry=_registry(), - period=2025, - doctrine=UKNationalSolveDoctrine(epochs=5), - ) - - result = stage(frame) - - for entity in frame.entities: - id_column = f"{entity}_id" - assert result.table(entity)[id_column].equals(frame.table(entity)[id_column]) - validate_uk_national_frame(result) - - -def test_checkpoint_metadata_round_trips_calibration_evidence() -> None: - frame = _frame() - stage = UKNationalCalibrationStage( - _registry(), - band_edge_registry=_registry(), - period=2025, - doctrine=UKNationalSolveDoctrine(epochs=5), - ) - - staged = stage(frame) - metadata = json.loads(json.dumps(stage.checkpoint_metadata())) - - resumed = UKNationalCalibrationStage( - _registry(), - band_edge_registry=_registry(), - period=2025, - doctrine=UKNationalSolveDoctrine(epochs=5), - ) - resumed.resume_from_checkpoint(metadata, staged) - - assert resumed.manifest == stage.manifest - assert resumed.diagnostics == stage.diagnostics - assert resumed.output_content_identity == metadata["output_content_identity"] - - drifted = UKNationalCalibrationStage( - _registry(), - band_edge_registry=_registry(), - period=2025, - doctrine=UKNationalSolveDoctrine(epochs=5), - ) - with pytest.raises(RuntimeError, match="drifted record"): - drifted.resume_from_checkpoint(metadata, frame) - - empty = UKNationalCalibrationStage( - _registry(), - band_edge_registry=_registry(), - period=2025, - doctrine=UKNationalSolveDoctrine(epochs=5), - ) - with pytest.raises(RuntimeError, match="calibration counts"): - empty.resume_from_checkpoint({}, staged) - - missing_count = dict(metadata) - missing_count["calibration"] = { - key: value - for key, value in metadata["calibration"].items() - if key != "activated_reference_count" - } - with pytest.raises(RuntimeError, match="calibration counts"): - empty.resume_from_checkpoint(missing_count, staged) - - unrun = UKNationalCalibrationStage( - _registry(), - band_edge_registry=_registry(), - period=2025, - doctrine=UKNationalSolveDoctrine(epochs=5), - ) - with pytest.raises(RuntimeError, match="has not run"): - unrun.checkpoint_metadata() - - -def test_prepared_slash_columns_are_not_returned_to_the_writer(tmp_path) -> None: - pytest.importorskip("tables") - stage = UKNationalCalibrationStage( - _registry(), - band_edge_registry=_registry(), - period=2025, - doctrine=UKNationalSolveDoctrine(epochs=5), - ) - - result = stage(_frame()) - - assert "dwp/uc/households" not in result.table("benunit") - write_uk_national_frame(result, tmp_path / "staging.h5") - - -def test_national_calibration_mass_reason_is_canonical() -> None: - assert national_calibration_mass_reason( - ["dwp_universal_credit", "hmrc", "dwp_universal_credit"] - ) == ( - "National doctrine calibration to bound target family(ies) " - "dwp_universal_credit, hmrc; total household mass moved with the targets." - ) - with pytest.raises(ValueError, match="bound_families"): - national_calibration_mass_reason([]) - - -def test_post_solve_fence_requires_calibrated_kind_and_mass_record() -> None: - before = _frame() - uncalibrated = Frame( - {entity: before.table(entity) for entity in before.entities}, - before.schema, - {"household": Weights(np.full(4, 10.0), WeightKind.DESIGN)}, - before.strata, - mass_log=before.mass_log, - metadata=before.metadata, - ) - - with pytest.raises(RuntimeError, match="not 'calibrated'"): - _post_solve_calibration_record(before, uncalibrated, before_count=0) - - calibrated_without_record = Frame( - {entity: before.table(entity) for entity in before.entities}, - before.schema, - {"household": Weights(np.full(4, 10.0), WeightKind.CALIBRATED)}, - before.strata, - mass_log=before.mass_log, - metadata=before.metadata, - ) - with pytest.raises(RuntimeError, match="exactly one mass record"): - _post_solve_calibration_record( - before, - calibrated_without_record, - before_count=0, - ) - - def test_national_doctrine_constants_are_the_declared_contract() -> None: assert UK_NATIONAL_SOLVE_EPOCHS == 256 assert UK_NATIONAL_LEARNING_RATE == 0.02 @@ -881,51 +328,6 @@ def test_national_doctrine_rejects_tampered_bounds() -> None: UKNationalSolveDoctrine(l0_lambda=-0.1) -@pytest.mark.parametrize( - ("rule", "expected"), - [("uniform", None), ("family_equal", [1.0])], -) -def test_doctrine_target_weight_rule_reaches_the_solver( - monkeypatch, rule, expected -) -> None: - """The declared rule must reach calibrate(), not just the manifest echo. - - First-armed-run finding (2026-08-23): the doctrine declared a - target_weight_rule the stage never passed to the kernel, so the solve - silently ran uniform whatever the doctrine said. The doctrine vector now - travels explicitly; "uniform" maps to None — the kernel's own default — - so the shipped identity is unchanged under the default rule. - """ - - from microcosm.calibrate import calibrate as real_calibrate - - captured: dict[str, object] = {} - - def capturing_calibrate(*args, **kwargs): - captured["target_loss_weights"] = kwargs["target_loss_weights"] - return real_calibrate(*args, **kwargs) - - monkeypatch.setattr( - "microcosm.build.uk_runtime.national_calibration.calibrate", - capturing_calibrate, - ) - stage = UKNationalCalibrationStage( - _registry(), - band_edge_registry=_registry(), - period=2025, - doctrine=UKNationalSolveDoctrine(epochs=5, target_weight_rule=rule), - ) - - stage(_frame()) - - weights = captured["target_loss_weights"] - if expected is None: - assert weights is None - else: - assert weights is not None - assert weights.tolist() == expected - - def test_measure_resolution_never_touches_the_source_frame() -> None: """Injection lands on adapter table copies, not on the caller's frame. diff --git a/packages/microcosm-build/tests/test_uk_national_sampling.py b/packages/microcosm-build/tests/test_uk_national_sampling.py index 000cea2ce..da83fd649 100644 --- a/packages/microcosm-build/tests/test_uk_national_sampling.py +++ b/packages/microcosm-build/tests/test_uk_national_sampling.py @@ -420,31 +420,6 @@ def test_full_fraction_is_a_structural_no_op() -> None: assert receipt["uk_policy"]["spi_replacement_quota_checked"] is True -def test_sampled_frames_pass_the_real_stage_fence() -> None: - """The sampler's arithmetic is proven against the fence itself. - - The first credentialed rung run died because the sampler and - ``_resolve_candidate_lineage`` disagreed; this test closes the coverage - hole the adversarial review found by running the REAL fence over sampled - frames: every draw must resolve with the full frame's exact multiplier - and person-level SPI/CG offsets. - """ - - from microcosm.build.uk_runtime.frs_hmrc_leaves import ( - _resolve_candidate_lineage, - ) - - frame = _source_family_frame() - full = _resolve_candidate_lineage(frame) - for seed in (0, 3, 11, 42): - sampled, _receipt = sample_uk_national_frame(frame, fraction=0.5, seed=seed) - lineage = _resolve_candidate_lineage(sampled) - assert lineage.clone_id_multiplier == full.clone_id_multiplier - assert lineage.spi_person_id_offset == full.spi_person_id_offset - assert ( - lineage.capital_gains_person_id_offset - == full.capital_gains_person_id_offset - ) def test_spine_source_units_use_raw_family_regions() -> None: diff --git a/packages/microcosm-build/tests/test_uk_release_certification.py b/packages/microcosm-build/tests/test_uk_release_certification.py index 086987c3c..f31108bf7 100644 --- a/packages/microcosm-build/tests/test_uk_release_certification.py +++ b/packages/microcosm-build/tests/test_uk_release_certification.py @@ -6,26 +6,21 @@ import hashlib import importlib.util import json -from datetime import date from pathlib import Path import pytest from microcosm.build.country_spec import load_country_spec from microcosm.build.gate_battery import gate_signing_key_env -from microcosm.build.uk_runtime import release_certification from microcosm.build.uk_runtime.calibration_run import ( - UK_CALIBRATION_GATE_SCOPE, UK_LOCAL_GATE_SCOPE, UK_NATIONAL_GATE_SCOPE, UK_SHARED_GATE_IDS, - UK_SPINE_GATE_SCOPE, ) from microcosm.build.uk_runtime.release_certification import ( UKReleaseCertificationError, compose_uk_release_certification, rehydrate_uk_fit_weight_records, - run_uk_release_cut_battery, uk_release_cut_scope_exclusions, ) @@ -289,39 +284,3 @@ def test_compose_refuses_absent_signing_key(green_certification_inputs, monkeypa monkeypatch.delenv(gate_signing_key_env("uk")) with pytest.raises(UKReleaseCertificationError, match="must be set"): compose_uk_release_certification(**green_certification_inputs) - - -def test_release_cut_battery_runs_and_signs(tmp_path: Path, monkeypatch): - monkeypatch.setattr( - release_certification, - "uk_aggregate_admin_totals", - lambda frame, manifest: ({}, {"stub": True}), - ) - report_path = tmp_path / "release_cut_gates.json" - payload = run_uk_release_cut_battery( - object(), - report_path=report_path, - release_id="uk-757-first-certified-cut", - diagnostics_sha256="a" * 64, - coverage_engine=object(), - build_stage_names=("frs_spine",), - ledger_registries={2023: object(), 2025: object()}, - local_ledger_registries={2025: object()}, - parity_evidence=object(), - fit_weight_records=None, - input_mass_reference={}, - exclusions_evaluated_on=date(2026, 8, 27), - gate_registry=_stub_registry(), - ) - assert payload["posture"] == "release_cut" - assert payload["release_candidate"] is True - assert payload["shippable"] is True - assert set(payload["gates"]) == set(UK_NATIONAL_GATE_SCOPE) - assert payload["blocked_at_phase"] is None - assert set(payload["scope_exclusions"]) == ( - set(UK_SPINE_GATE_SCOPE) - | set(UK_CALIBRATION_GATE_SCOPE) - | set(UK_LOCAL_GATE_SCOPE) - ) - set(UK_NATIONAL_GATE_SCOPE) - on_disk = json.loads(report_path.read_text(encoding="utf-8")) - assert on_disk["attestation"]["signature"] == payload["attestation"]["signature"] diff --git a/packages/microcosm-build/tests/test_uk_release_input_coverage.py b/packages/microcosm-build/tests/test_uk_release_input_coverage.py index e9755e947..8ad8c6964 100644 --- a/packages/microcosm-build/tests/test_uk_release_input_coverage.py +++ b/packages/microcosm-build/tests/test_uk_release_input_coverage.py @@ -139,7 +139,7 @@ def _hmrc_family_coverage() -> dict[str, dict[str, object]]: return { "hmrc_spi_income": { "status": "required_at_build", - "stage": "hmrc_spi_income", + "stage": "hmrc_spi_income_spine", "effective_mass_requirements": { "gift_aid": { "status": "distributional_required", @@ -603,8 +603,9 @@ def test_shipped_manifest_is_current(self) -> None: assert load_efrs_parity_known_gaps() == () assert manifest.required_build_stages == frozenset( { - "hmrc_spi_income", - "hmrc_cgt_gains", + "frs_hmrc_spine_leaves", + "spi_support_channel", + "hmrc_spi_income_spine", "cgt_incidence_clone", "cgt_band_donors", "hmrc_cgt_gains_spine", @@ -722,47 +723,45 @@ def test_required_family_stage_cannot_be_omitted(self) -> None: assert_uk_release_input_coverage_build_stages((), manifest=manifest) result = assert_uk_release_input_coverage_build_stages( - ("hmrc_spi_income",), + ("hmrc_spi_income_spine",), manifest=manifest, ) assert result is None - def test_spine_posture_satisfies_superseded_required_families(self) -> None: + def test_canonical_producers_and_predecessors_satisfy_required_families( + self, + ) -> None: manifest = load_uk_release_input_coverage_manifest() - spine_stages = tuple( - stage - for stage in manifest.required_build_stages - if stage not in {"hmrc_spi_income", "hmrc_cgt_gains"} - ) - result = assert_uk_release_input_coverage_build_stages( - (*spine_stages, "hmrc_spi_income_spine"), - manifest=manifest, + assert_uk_release_input_coverage_build_stages( + tuple(manifest.required_build_stages), manifest=manifest ) - assert result is None - assert ( - manifest.family_coverage["hmrc_spi_income"]["superseded_by"]["stage"] - == "hmrc_spi_income_spine" + assert { + "frs_hmrc_spine_leaves", + "spi_support_channel", + "hmrc_spi_income_spine", + } <= manifest.required_build_stages + assert {"hmrc_spi_income", "hmrc_cgt_gains"}.isdisjoint( + manifest.required_build_stages ) - assert ( - manifest.family_coverage["hmrc_cgt_gains"]["superseded_by"]["stage"] - == "hmrc_cgt_gains_spine" + assert all( + "superseded_by" not in family + for family in manifest.family_coverage.values() ) - def test_supersession_does_not_hide_a_genuinely_missing_family(self) -> None: + @pytest.mark.parametrize( + "missing_stage", + [ + "frs_hmrc_spine_leaves", + "spi_support_channel", + "hmrc_spi_income_spine", + "student_loans", + ], + ) + def test_each_canonical_family_dependency_is_required(self, missing_stage) -> None: manifest = load_uk_release_input_coverage_manifest() - spine_stages = tuple( - stage - for stage in manifest.required_build_stages - if stage - not in { - "hmrc_spi_income", - "hmrc_cgt_gains", - "student_loans", - } - ) - with pytest.raises(ValueError, match="student_loans"): + with pytest.raises(ValueError, match="hmrc_spi_income|student_loans"): assert_uk_release_input_coverage_build_stages( - (*spine_stages, "hmrc_spi_income_spine"), + tuple(manifest.required_build_stages - {missing_stage}), manifest=manifest, ) diff --git a/packages/microcosm-build/tests/test_uk_release_input_coverage_manifest.py b/packages/microcosm-build/tests/test_uk_release_input_coverage_manifest.py index 504f17999..14d7f0139 100644 --- a/packages/microcosm-build/tests/test_uk_release_input_coverage_manifest.py +++ b/packages/microcosm-build/tests/test_uk_release_input_coverage_manifest.py @@ -89,6 +89,7 @@ def test_known_gap_register_records_post_candidate_restoration_separately() -> N assert gaps["exclusion_policy"]["tracking_note"].strip() for name, evidence in gaps["restored_required_columns"].items(): assert evidence["stage"] == "hmrc_spi_income" + assert evidence["current_producer_stage"] == "hmrc_spi_income_spine" assert evidence["support_channel"] == "spi" assert ( evidence["effective_signal_mass_share"] @@ -137,52 +138,31 @@ def test_frozen_candidate_retains_its_original_engine_provenance() -> None: def test_hmrc_family_period_fields_come_from_the_bytes_their_hash_names() -> None: - # Adversarial-review finding (2026-08-20): the family block renders the - # #723 re-mapped period fields from the CANONICAL manifest while the - # frozen mirror keeps its June bytes; each field set must bind to the - # sha256 of the file it actually came from. import hashlib - manifest = _resource("release_input_coverage_manifest.json") - family = manifest["family_coverage"]["hmrc_spi_income"] - - frozen_bytes = ( - files(_UK_PACKAGE).joinpath("hmrc_income_source_stages.json").read_bytes() - ) + family = _resource("release_input_coverage_manifest.json")["family_coverage"][ + "hmrc_spi_income" + ] canonical_bytes = files(_UK_PACKAGE).joinpath("source_stages.json").read_bytes() - assert family["source_manifest_sha256"] == hashlib.sha256(frozen_bytes).hexdigest() + assert family["source_manifest"] == "source_stages.json" assert ( - family["canonical_source_manifest_sha256"] - == hashlib.sha256(canonical_bytes).hexdigest() - ) - - frozen_stage = json.loads(frozen_bytes)["stages"][0] - frozen_surface = next( - artifact - for artifact in frozen_stage["artifacts"] - if artifact.get("role") == "published_fact_surface" + family["source_manifest_sha256"] == hashlib.sha256(canonical_bytes).hexdigest() ) - canonical_stage = next( + stage = next( stage for stage in json.loads(canonical_bytes)["stages"] - if stage.get("stage") == "hmrc_spi_income" + if stage["stage"] == "hmrc_spi_income_spine" ) - canonical_surface = next( + surface = next( artifact - for artifact in canonical_stage["artifacts"] - if artifact.get("role") == "published_fact_surface" + for artifact in stage["artifacts"] + if artifact["role"] == "published_fact_surface" ) - # The re-mapped fields equal the canonical declaration; the frozen mirror - # still declares the June mapping (its bytes are pinned elsewhere). assert family["source_vintages"]["mapped_build_period"] == str( - canonical_surface["mapped_build_period"] + surface["mapped_build_period"] ) - assert ( - family["source_vintages"]["period_mapping"] - == canonical_surface["period_mapping"] - ) - assert str(frozen_surface["mapped_build_period"]) == "2023" - assert frozen_surface["period_mapping"] == "tax_year_start" + assert family["source_vintages"]["period_mapping"] == surface["period_mapping"] + assert "canonical_source_manifest" not in family def test_promoted_manifest_requires_the_full_reference_surface() -> None: @@ -221,7 +201,7 @@ def test_hmrc_stage_is_required_while_the_208_fact_replay_remains_fenced() -> No assert family["status"] == "required_at_build" assert family["restoration_status"] == "adjudicated_partial_replay" - assert family["source_manifest"] == "hmrc_income_source_stages.json" + assert family["source_manifest"] == "source_stages.json" assert len(family["source_manifest_sha256"]) == 64 assert family["base_candidate_tier"] == "frs" assert family["source_vintages"] == { @@ -231,8 +211,10 @@ def test_hmrc_stage_is_required_while_the_208_fact_replay_remains_fenced() -> No "period_mapping": "latest_published_tax_year", } assert family["spi_prior_national_household_mass_share"] == 0.5 - assert family["canonical_source_manifest"] == "source_stages.json" - assert len(family["canonical_source_manifest_sha256"]) == 64 + assert family["required_predecessor_stages"] == [ + "frs_hmrc_spine_leaves", + "spi_support_channel", + ] assert family["required_mass_change_reason"] == ( "Allocate 50% of certified UK national household prior mass to the " "rebuilt 2022-23 SPI support channel; total national mass is conserved." @@ -329,18 +311,26 @@ def test_manifest_generation_rejects_candidate_tier_drift() -> None: generator.build_manifest(reference=reference, known_gaps_payload=gaps) -def test_hmrc_family_rejects_source_stage_tier_mismatch( - monkeypatch: pytest.MonkeyPatch, - tmp_path: Path, +def test_hmrc_family_rejects_canonical_source_contract_drift( + monkeypatch, tmp_path ) -> None: generator = _load_generator() - source_stages = _resource("hmrc_income_source_stages.json") - source_stages["stages"][0]["base_candidate"]["tier"] = "cps-transfer" - drifted = tmp_path / "hmrc_income_source_stages.json" + source_stages = _resource("source_stages.json") + stage = next( + stage + for stage in source_stages["stages"] + if stage["stage"] == "spi_support_channel" + ) + operation = next( + operation + for operation in stage["operations"] + if operation["kind"] == "allocate_zero_weight_prior_mass" + ) + operation["share"] = 0.1 + drifted = tmp_path / "source_stages.json" drifted.write_text(json.dumps(source_stages), encoding="utf-8") - monkeypatch.setattr(generator, "HMRC_SOURCE_STAGES_PATH", drifted) - - with pytest.raises(ValueError, match="disagrees with the certified candidate"): + monkeypatch.setattr(generator, "SOURCE_STAGES_PATH", drifted) + with pytest.raises(ValueError, match="prior.mass_share"): generator._hmrc_family_coverage_contract(candidate_source={"tier": "frs"}) diff --git a/packages/microcosm-build/tests/test_uk_rowwise_build_driver.py b/packages/microcosm-build/tests/test_uk_rowwise_build_driver.py index d65a6d487..16da11033 100644 --- a/packages/microcosm-build/tests/test_uk_rowwise_build_driver.py +++ b/packages/microcosm-build/tests/test_uk_rowwise_build_driver.py @@ -1,694 +1,64 @@ -from __future__ import annotations +"""The older geography-only builder delegates to the single full build.""" import importlib.util -import json import sys from pathlib import Path +from types import SimpleNamespace -import pandas as pd import pytest -from microcosm.build.logbook import LOGBOOK_ROW_FIELDS, load_spool_rows - -@pytest.fixture(autouse=True) -def _spool_only_by_default(monkeypatch: pytest.MonkeyPatch) -> None: - monkeypatch.delenv("POPULACE_LEDGER_URL", raising=False) - monkeypatch.delenv("POPULACE_LEDGER_KEY", raising=False) - monkeypatch.delenv("POPULACE_LEDGER_API_KEY", raising=False) - monkeypatch.delenv("POPULACE_LOGBOOK_PREV_ROW_DIGEST", raising=False) - - -def _spool_rows(output_dir: Path): - rows = load_spool_rows(output_dir / "logbook-spool") - for row in rows: - assert frozenset(row.to_mapping()) == LOGBOOK_ROW_FIELDS - return rows - - -def _local_ref(path: Path) -> str: - return f"local://{path.resolve().as_posix().lstrip('/')}" - - -def _load_builder_module(): - root = Path(__file__).resolve().parents[3] - path = root / "tools" / "build_uk_rowwise_dataset.py" +def _driver(): + path = Path(__file__).resolve().parents[3] / "tools/build_uk_rowwise_dataset.py" spec = importlib.util.spec_from_file_location("build_uk_rowwise_dataset", path) module = importlib.util.module_from_spec(spec) - assert spec.loader is not None spec.loader.exec_module(module) return module -def _write_toy_h5( - path: Path, - *, - regions: tuple[str, str] = ("LONDON", "WALES"), - time_period: str = "2023", -) -> None: - with pd.HDFStore(path) as store: - store.put( - "household", - pd.DataFrame( - { - "household_id": [1, 2], - "household_weight": [10.0, 20.0], - "region": list(regions), - } - ), - format="table", - data_columns=True, - ) - store.put( - "person", - pd.DataFrame( - { - "person_id": [1001, 2001, 2002], - "person_household_id": [1, 2, 2], - "person_benunit_id": [101, 201, 201], - } - ), - format="table", - data_columns=True, - ) - store.put( - "benunit", - pd.DataFrame({"benunit_id": [101, 201]}), - format="table", - data_columns=True, - ) - store.put( - "time_period", - pd.Series([time_period]), - format="table", - data_columns=True, - ) - - -def _crosswalk_frame() -> pd.DataFrame: - return pd.DataFrame( - [ - { - "oa_code": "E0001", - "lsoa_code": "E0101", - "msoa_code": "E0201", - "la_code": "E06000063", - "constituency_code": "E14000001", - "region_code": "E12000007", - "country": "England", - "population": 100, - }, - { - "oa_code": "W0001", - "lsoa_code": "W0101", - "msoa_code": "W0201", - "la_code": "W06000001", - "constituency_code": "W07000041", - "region_code": "W99999999", - "country": "Wales", - "population": 80, - }, - { - "oa_code": "S0001", - "lsoa_code": "S0101", - "msoa_code": "S0201", - "la_code": "S12000033", - "constituency_code": "S14000001", - "region_code": "S99999999", - "country": "Scotland", - "population": 90, - }, - { - "oa_code": "N20000001", - "lsoa_code": "N20000001", - "msoa_code": "N21000001", - "la_code": "N09000001", - "constituency_code": "N05000001", - "region_code": "N99999999", - "country": "Northern Ireland", - "population": 70, - }, - ] - ) - - -@pytest.mark.parametrize("route_option", ["--crosswalk", "--ladder"]) -def test_input_pin_mismatch_refuses_before_side_effects( - monkeypatch, - tmp_path, - route_option, -) -> None: - pytest.importorskip("tables") - builder = _load_builder_module() - input_h5 = tmp_path / "input.h5" - route_artifact = tmp_path / "route-artifact" - output_dir = tmp_path / "out" - _write_toy_h5(input_h5) - route_artifact.write_bytes(b"must not be read") - - def unexpected_h5_read(_path): - raise AssertionError("input pin mismatch must refuse before H5 parsing") - - monkeypatch.setattr(builder, "_h5_summary", unexpected_h5_read) - - with pytest.raises(SystemExit, match=r"--input-h5 sha mismatch: measured"): - builder.main( - [ - "--input-h5", - str(input_h5), - "--input-sha256", - "0" * 64, - "--out", - str(output_dir), - route_option, - str(route_artifact), - ] - ) - - assert not output_dir.exists() - - -def test_build_uk_rowwise_dataset_writes_manifest_and_outputs( - monkeypatch, tmp_path, capsys -): - pytest.importorskip("tables") - builder = _load_builder_module() - input_h5 = tmp_path / "populace_uk_2023.h5" - crosswalk_path = tmp_path / "crosswalk.csv.gz" - constituency_codes = tmp_path / "constituencies.csv" - la_codes = tmp_path / "local_authorities.csv" - output_dir = tmp_path / "out" - _write_toy_h5(input_h5) - input_sha256 = builder._sha256(input_h5) - _crosswalk_frame().to_csv(crosswalk_path, index=False) - output_dir.mkdir() - stale_area_support = output_dir / builder.AREA_SUPPORT_FILENAME - stale_area_support.write_text("stale") - pd.DataFrame({"code": ["E14000001", "W07000041", "S14000001", "N05000001"]}).to_csv( - constituency_codes, index=False - ) - pd.DataFrame({"code": ["E06000063", "W06000001", "S12000033", "N09000001"]}).to_csv( - la_codes, index=False - ) - monkeypatch.setattr( - sys, - "argv", - [ - "build_uk_rowwise_dataset.py", - "--input-h5", - str(input_h5), - "--input-sha256", - input_sha256, - "--out", - str(output_dir), - "--crosswalk", - str(crosswalk_path), - "--constituency-codes", - str(constituency_codes), - "--la-codes", - str(la_codes), - "--n-clones", - "2", - "--allow-missing-country", - "--allow-constituency-collisions", - ], - ) - - assert builder.main() == 0 - captured = capsys.readouterr() - assert "Wrote Logbook row:" in captured.err - - output_h5 = output_dir / "populace_uk_2023_rowwise.h5" - manifest_path = output_dir / builder.MANIFEST_FILENAME - coverage_path = output_dir / builder.COVERAGE_FILENAME - assert output_h5.exists() - assert manifest_path.exists() - assert coverage_path.exists() - assert not stale_area_support.exists() - manifest = json.loads(manifest_path.read_text()) - assert manifest["build_kind"] == "uk_rowwise_local_geography_dataset" - assert manifest["parameters"]["n_clones"] == 2 - assert manifest["parameters"]["source_year"] == 2023 - assert manifest["parameters"]["require_all_countries"] is False - assert manifest["inputs"]["dataset"]["pin_verified"] is True - assert manifest["base_dataset"]["household_weight_sum"] == pytest.approx(30.0) - assert manifest["rowwise_dataset"]["household_weight_sum"] == pytest.approx(30.0) - assert manifest["rowwise_dataset"]["household_weight_delta"] == pytest.approx(0.0) - assert manifest["rowwise_dataset"]["missing_geography_rows"] == 0 - assert manifest["rowwise_dataset"]["assigned_constituencies"] == 2 - assert manifest["rowwise_dataset"]["assigned_local_authorities"] == 2 - assert manifest["coverage"][0]["covered_areas"] == 4 - assert manifest["outputs"]["crosswalk"] is None - assert manifest["outputs"]["area_support_summary"] is None - with pd.HDFStore(output_h5, mode="r") as store: - assert store["household"].shape[0] == 4 - assert store["person"].shape[0] == 6 - assert store["benunit"].shape[0] == 4 - rows = _spool_rows(output_dir) - assert len(rows) == 1 - first_row = rows[0] - assert first_row.pipeline == "uk-local-rowwise" - assert first_row.rung == "f100" - assert first_row.seed == 42 - assert first_row.disposition == "iterating" - assert first_row.artifact_location == _local_ref(output_h5) - assert first_row.gate_verdicts == { - "uk_mass_conservation": { - "verdict": "passed", - "receipt": f"{_local_ref(manifest_path)}#/rowwise_dataset/weights/mass_conservation", - }, - "uk_coverage": { - "verdict": "passed", - "receipt": f"{_local_ref(manifest_path)}#/rowwise_dataset/coverage", - }, - } - - monkeypatch.setattr( - sys, - "argv", - [ - "build_uk_rowwise_dataset.py", - "--input-h5", - str(input_h5), - "--out", - str(output_dir), - "--crosswalk", - str(crosswalk_path), - "--n-clones", - "1", - "--allow-missing-country", - "--logbook-prev-row-digest", - first_row.row_digest, - ], - ) - - assert builder.main() == 0 - assert not coverage_path.exists() - manifest = json.loads(manifest_path.read_text()) - assert manifest["coverage"] == [] - assert manifest["outputs"]["coverage_summary"] is None - rows = _spool_rows(output_dir) - assert [row.prev_row_digest for row in rows] == [None, first_row.row_digest] - assert rows[1].gate_verdicts == { - "uk_mass_conservation": { - "verdict": "passed", - "receipt": f"{_local_ref(manifest_path)}#/rowwise_dataset/weights/mass_conservation", - } - } - - -def test_build_uk_rowwise_dataset_rejects_target_csv_without_code(tmp_path): - builder = _load_builder_module() - bad_codes = tmp_path / "bad.csv" - bad_codes.write_text("name\nAldershot\n") - - with pytest.raises(ValueError, match="code"): - builder._read_code_csv(bad_codes) - - -def test_build_uk_rowwise_dataset_counts_blank_geography(monkeypatch, tmp_path): - pytest.importorskip("tables") - builder = _load_builder_module() - input_h5 = tmp_path / "populace_uk_2023.h5" - crosswalk_path = tmp_path / "england_only_crosswalk.csv.gz" - output_dir = tmp_path / "out" - _write_toy_h5(input_h5, regions=("LONDON", "SCOTLAND")) - _crosswalk_frame().iloc[:1].to_csv(crosswalk_path, index=False) - monkeypatch.setattr( - sys, - "argv", - [ - "build_uk_rowwise_dataset.py", - "--input-h5", - str(input_h5), - "--out", - str(output_dir), - "--crosswalk", - str(crosswalk_path), - "--n-clones", - "1", - "--allow-missing-country", - ], - ) - - assert builder.main() == 0 - - manifest = json.loads((output_dir / builder.MANIFEST_FILENAME).read_text()) - assert manifest["rowwise_dataset"]["missing_geography_rows"] == 1 - assert manifest["rowwise_dataset"]["assigned_constituencies"] == 1 - assert manifest["rowwise_dataset"]["assigned_local_authorities"] == 1 - assert ( - manifest["rowwise_dataset"]["duplicate_source_household_constituency_pairs"] - == 0 - ) - - -def test_build_uk_rowwise_dataset_infers_source_year_from_h5(monkeypatch, tmp_path): - pytest.importorskip("tables") - builder = _load_builder_module() - input_h5 = tmp_path / "microcosm_uk_2024.h5" - crosswalk_path = tmp_path / "crosswalk.csv.gz" - output_dir = tmp_path / "out" - _write_toy_h5(input_h5, time_period="2024") - _crosswalk_frame().to_csv(crosswalk_path, index=False) - monkeypatch.setattr( - sys, - "argv", - [ - "build_uk_rowwise_dataset.py", - "--input-h5", - str(input_h5), - "--out", - str(output_dir), - "--crosswalk", - str(crosswalk_path), - "--n-clones", - "1", - "--allow-missing-country", - "--allow-constituency-collisions", - ], - ) - - assert builder.main() == 0 - - manifest = json.loads((output_dir / builder.MANIFEST_FILENAME).read_text()) - assert manifest["parameters"]["source_year"] == 2024 - assert manifest["rowwise_dataset"]["time_period"] == "2024" - assert (output_dir / "microcosm_uk_2024_rowwise.h5").exists() - with pd.HDFStore(output_dir / "microcosm_uk_2024_rowwise.h5", mode="r") as store: - household = store["household"] - assert household["source_year"].unique().tolist() == [2024] - assert household["source_household_key"].tolist() == [ - "2024:1", - "2024:2", +@pytest.mark.parametrize("selector", [[], ["--target-geographies", "country"]]) +def test_geography_command_preserves_explicit_scope_only(monkeypatch, selector): + calls = [] + monkeypatch.setitem( + sys.modules, + "microcosm.build.uk_runtime.full_build_cli", + SimpleNamespace(main=lambda arguments: calls.append(arguments) or 0), + ) + arguments = [ + "--input-h5", + "bound-spine.h5", + "--ladder", + "ladder.npz", + "--ledger-facts", + "facts.jsonl", + "--out", + "out", + "--n-clones", + "2", + *selector, ] - - -def test_build_uk_rowwise_dataset_ladder_route_records_gate_verdict( - monkeypatch, - tmp_path, -) -> None: - pytest.importorskip("tables") - builder = _load_builder_module() - input_h5 = tmp_path / "populace_uk_2023.h5" - ladder_path = tmp_path / "ladder.npz" - output_dir = tmp_path / "out" - _write_toy_h5(input_h5) - ladder_path.write_bytes(b"ladder") - - def fake_clone(*_args, output_path: Path, **_kwargs): - output_path.write_bytes(b"rowwise") - household = pd.DataFrame( - { - "household_id": [1, 2], - "household_weight": [10.0, 20.0], - "oa_code": ["E0001", "W0001"], - "lsoa_code": ["E0101", "W0101"], - "msoa_code": ["E0201", "W0201"], - "local_authority_code": ["E06000063", "W06000001"], - "ward_code": ["E05000001", "W05000001"], - "constituency_code": ["E14000001", "W07000041"], - "region_code": ["E12000007", "W99999999"], - "region": ["LONDON", "WALES"], - "itl3_code": ["TLI", "TLL"], - "itl2_code": ["TL", "TL"], - "itl1_code": ["T", "T"], - "country": ["England", "Wales"], - "rowwise_household_clone_index": [0, 0], - } - ) - return type( - "Result", - (), - { - "person": pd.DataFrame({"person_id": [1, 2]}), - "benunit": pd.DataFrame({"benunit_id": [1, 2]}), - "household": household, - "household_weight_kind": type("WeightKind", (), {"value": "design"})(), - "mass_log": (), - "time_period": "2023", - "n_clones": 1, - "id_multiplier": 10, - "gate": type( - "Gate", - (), - {"passed": True, "details": {"areas": 2}}, - )(), - }, - )() - - ladder = type( - "Ladder", - (), - { - "constituency_code": pd.Series(["E14000001", "W07000041"]).to_numpy(), - "local_authority_code": pd.Series( - ["E06000063", "W06000001"] - ).to_numpy(), - }, - )() - monkeypatch.setattr(builder, "load_uk_oa_ladder", lambda _path: ladder) - monkeypatch.setattr( - builder, - "clone_uk_dataset_with_ladder_geography", - fake_clone, - ) - monkeypatch.setattr( - sys, - "argv", - [ - "build_uk_rowwise_dataset.py", - "--input-h5", - str(input_h5), - "--out", - str(output_dir), - "--ladder", - str(ladder_path), - "--n-clones", - "1", - ], - ) - - assert builder.main() == 0 - - manifest_path = output_dir / builder.MANIFEST_FILENAME - manifest = json.loads(manifest_path.read_text()) - assert manifest["rowwise_dataset"]["area_support"]["source_basis"] == ( - "household_id" - ) - rows = _spool_rows(output_dir) - assert len(rows) == 1 - assert rows[0].gate_verdicts == { - "uk_geography_ladder": { - "verdict": "passed", - "receipt": f"{_local_ref(manifest_path)}#/rowwise_dataset/gate", - } - } - - -def test_build_uk_rowwise_dataset_failure_records_pipeline_error( - monkeypatch, - tmp_path, -) -> None: - pytest.importorskip("tables") - builder = _load_builder_module() - input_h5 = tmp_path / "populace_uk_2023.h5" - crosswalk_path = tmp_path / "crosswalk.csv.gz" - output_dir = tmp_path / "out" - _write_toy_h5(input_h5) - _crosswalk_frame().to_csv(crosswalk_path, index=False) - - def fail_clone(*_args, **_kwargs): - raise RuntimeError("rowwise clone failed") - - monkeypatch.setattr(builder, "clone_uk_dataset_with_rowwise_geography", fail_clone) - monkeypatch.setattr( - sys, - "argv", - [ - "build_uk_rowwise_dataset.py", - "--input-h5", - str(input_h5), - "--out", - str(output_dir), - "--crosswalk", - str(crosswalk_path), - "--n-clones", - "1", - "--allow-missing-country", - ], - ) - - with pytest.raises(RuntimeError, match="rowwise clone failed"): - builder.main() - - rows = _spool_rows(output_dir) - assert len(rows) == 1 - row = rows[0] - assert row.disposition == "failed" - assert row.gate_verdicts["pipeline_error"]["verdict"] == "error" - assert row.gate_verdicts["pipeline_error"]["receipt"].endswith("#/error_type") - - -def test_candidate_clone_counts_refused_on_real_build(tmp_path) -> None: - builder = _load_builder_module() - output_dir = tmp_path / "out" - - with pytest.raises(ValueError, match="candidate-K planning is a dry-run surface"): - builder.main( - [ - "--input-h5", - str(tmp_path / "not-read.h5"), - "--out", - str(output_dir), - "--candidate-clone-counts", - "1,2,4", - ] - ) - - assert not output_dir.exists() + assert _driver().main(arguments) == 0 + assert calls == [arguments] @pytest.mark.parametrize( - "dataset_filename", + "flag", [ - "../escaped.h5", - "/tmp/escaped.h5", - "rowwise_build_manifest.json", - "geography_coverage_summary.csv", - "area_support_summary.csv", - "uk_official_geography_crosswalk.csv.gz", + "--crosswalk", + "--candidate-clone-counts", + "--allow-cross-region-assignment", + "--dataset-filename", ], ) -def test_dataset_output_path_rejects_paths_and_reserved_names( - dataset_filename, tmp_path -): - builder = _load_builder_module() - - with pytest.raises(ValueError, match="dataset-filename"): - builder._dataset_output_path( - tmp_path, - dataset_filename=dataset_filename, - input_stem="microcosm_uk_2024", - source_year=2023, - ) - - -def test_validate_output_paths_rejects_crosswalk_collision(tmp_path): - builder = _load_builder_module() - crosswalk = tmp_path / "rowwise.h5" - args = type( - "Args", - (), - { - "out": tmp_path, - "crosswalk": crosswalk, - }, - ) - - with pytest.raises(ValueError, match="differ"): - builder._validate_output_paths( - input_h5=tmp_path / "source.h5", - output_h5=crosswalk, - args=args, - ) - - -@pytest.mark.parametrize( - "sidecar_name", - [ - "rowwise_build_manifest.json", - "geography_coverage_summary.csv", - "area_support_summary.csv", - ], -) -def test_validate_output_paths_rejects_supplied_crosswalk_sidecar_collision( - sidecar_name, - tmp_path, -): - builder = _load_builder_module() - sidecar_path = tmp_path / sidecar_name - args = type( - "Args", - (), - { - "out": tmp_path, - "crosswalk": sidecar_path, - }, - ) - - with pytest.raises(ValueError, match="crosswalk.*sidecars"): - builder._validate_output_paths( - input_h5=tmp_path / "source.h5", - output_h5=tmp_path / "rowwise.h5", - args=args, - ) - - -def test_load_or_build_crosswalk_unlinks_stale_generated_sidecar(tmp_path): - builder = _load_builder_module() - output_dir = tmp_path / "out" - output_dir.mkdir() - supplied_crosswalk = tmp_path / "supplied_crosswalk.csv.gz" - stale_generated_crosswalk = output_dir / builder.CROSSWALK_FILENAME - _crosswalk_frame().to_csv(supplied_crosswalk, index=False) - stale_generated_crosswalk.write_text("stale") - args = type( - "Args", - (), - { - "out": output_dir, - "crosswalk": supplied_crosswalk, - }, - ) - - source = builder._load_or_build_crosswalk(args) - - assert source.generated is False - assert source.path == supplied_crosswalk.resolve() - assert not stale_generated_crosswalk.exists() - - -def test_load_or_build_crosswalk_keeps_supplied_generated_path(tmp_path): - builder = _load_builder_module() - output_dir = tmp_path / "out" - output_dir.mkdir() - supplied_crosswalk = output_dir / builder.CROSSWALK_FILENAME - _crosswalk_frame().to_csv(supplied_crosswalk, index=False) - args = type( - "Args", - (), - { - "out": output_dir, - "crosswalk": supplied_crosswalk, - }, - ) - - source = builder._load_or_build_crosswalk(args) - - assert source.generated is False - assert source.path == supplied_crosswalk.resolve() - assert supplied_crosswalk.exists() - - -def test_build_uk_rowwise_dataset_rejects_overwriting_input(monkeypatch, tmp_path): - pytest.importorskip("tables") - builder = _load_builder_module() - input_h5 = tmp_path / "populace_uk_2023_rowwise.h5" - _write_toy_h5(input_h5) - monkeypatch.setattr( - sys, - "argv", - [ - "build_uk_rowwise_dataset.py", - "--input-h5", - str(input_h5), - "--out", - str(tmp_path), - "--dataset-filename", - input_h5.name, - ], - ) - - with pytest.raises(ValueError, match="must differ"): - builder.main() +def test_removed_geography_workflow_options_explain_migration(flag): + with pytest.raises( + SystemExit, match="independent UK geography-only build driver is retired" + ): + _driver().main([flag, "old-value"]) + + +def test_command_contains_no_independent_cloning_export_path(): + driver = _driver() + assert not hasattr(driver, "_main_impl") + assert not hasattr(driver, "clone_uk_dataset_with_ladder_geography") + assert not hasattr(driver, "_load_or_build_crosswalk") diff --git a/packages/microcosm-build/tests/test_uk_rowwise_candidate.py b/packages/microcosm-build/tests/test_uk_rowwise_candidate.py index 9c0f3aafa..156b39acf 100644 --- a/packages/microcosm-build/tests/test_uk_rowwise_candidate.py +++ b/packages/microcosm-build/tests/test_uk_rowwise_candidate.py @@ -1,2240 +1,85 @@ -"""Synthetic end-to-end contract for the first UK rowwise candidate.""" +"""Retired rowwise command delegates exactly to the canonical full build.""" from __future__ import annotations -import base64 -import hashlib import importlib.util -import json -from dataclasses import replace from pathlib import Path -from types import SimpleNamespace -import numpy as np -import pandas as pd import pytest -from microcosm.build.logbook import LOGBOOK_ROW_FIELDS, load_spool_rows -from microcosm.build.uk_runtime import ( - assemble_uk_oa_ladder, - ladder_target_provenance, - load_uk_oa_ladder, - read_uk_single_year_weight_metadata, - write_uk_national_frame, -) -from microcosm.build.uk_runtime.national_frame import ( - uk_national_frame, - validate_uk_national_frame, -) -from microcosm.calibrate import TargetRegistry, TargetSpec -from microcosm.frame import MassChangeRecord, WeightKind - - -@pytest.fixture(autouse=True) -def _empty_support_exclusions_for_synthetic_rosters( - monkeypatch: pytest.MonkeyPatch, -) -> None: - """Synthetic candidates never carry the committed micro-LA exclusions. - - The committed ``local_area_support_exclusions.json`` names real local - authorities measured on the licensed spine; a synthetic roster either - lacks them (unknown) or meets the floor (stale), and the gate rightly - fails either way. These tests exercise the machinery, so the support - register is pinned empty here; the committed entries are covered by - ``test_uk_battery_bindings.py``. - """ - - import microcosm.build.uk_runtime.battery_bindings as battery_bindings - - real_loader = battery_bindings.load_uk_reviewed_exclusion_register - - def _loader(path, *, resource, **kwargs): - if resource == "local_area_support_exclusions.json": - return {} - return real_loader(path, resource=resource, **kwargs) - - monkeypatch.setattr( - battery_bindings, "load_uk_reviewed_exclusion_register", _loader - ) - - -@pytest.fixture(autouse=True) -def _spool_only_by_default(monkeypatch: pytest.MonkeyPatch) -> None: - monkeypatch.delenv("POPULACE_LEDGER_URL", raising=False) - monkeypatch.delenv("POPULACE_LEDGER_KEY", raising=False) - monkeypatch.delenv("POPULACE_LEDGER_API_KEY", raising=False) - monkeypatch.delenv("POPULACE_LOGBOOK_PREV_ROW_DIGEST", raising=False) - monkeypatch.setenv( - "MICROCOSM_UK_TERMINAL_GATE_SIGNING_KEY", - base64.b64encode(b"\x07" * 32).decode("ascii"), - ) - - -def _spool_rows(output_dir: Path): - rows = load_spool_rows(output_dir / "logbook-spool") - for row in rows: - assert frozenset(row.to_mapping()) == LOGBOOK_ROW_FIELDS - return rows - - -def _local_ref(path: Path) -> str: - return f"local://{path.resolve().as_posix().lstrip('/')}" +from microcosm.build.uk_runtime import full_build_cli def _load_builder_module(): - root = Path(__file__).resolve().parents[3] - path = root / "tools" / "build_uk_rowwise_candidate.py" - spec = importlib.util.spec_from_file_location( - "build_uk_rowwise_candidate", - path, + path = ( + Path(__file__).resolve().parents[3] / "tools" / "build_uk_rowwise_candidate.py" ) + spec = importlib.util.spec_from_file_location("build_uk_rowwise_candidate", path) module = importlib.util.module_from_spec(spec) assert spec.loader is not None spec.loader.exec_module(module) return module -def _ladder_metadata() -> dict[str, object]: - def layer(vintage: str) -> dict[str, object]: - return {"vintage": vintage, "source": "synthetic test source"} - - return { - "schema_version": 1, - "kind": "uk_oa_ladder", - "coverage": "uk", - "oa_vintage": "synthetic", - "constituency_sampling_basis": "synthetic household counts", - "oa_sampling_basis": "synthetic population", - "layers": { - "constituency": layer("2024_pcon"), - "lsoa": layer("synthetic"), - "msoa": layer("synthetic"), - "local_authority": layer("synthetic"), - "ward": layer("synthetic"), - "itl": layer("2021_itl"), - "region": layer("synthetic"), - }, - } - - -def _ladder_frame( - household_counts: tuple[float, float, float, float] = ( - 3.0, - 10.0, - 10.0, - 10.0, - ), -) -> pd.DataFrame: - rows = [ - ( - "E00000001", - "E12000007", - "E14000001", - "E05014284", - "E09000001", - "TLI31", - ), - ( - "W00000001", - "W99999999", - "W07000041", - "W05001517", - "W06000001", - "TLL11", - ), - ( - "S00000001", - "S99999999", - "S14000001", - "S13002835", - "S12000033", - "TLM50", - ), - ( - "N20000001", - "N99999999", - "N05000001", - "N10000104", - "N09000001", - "TLN0A", - ), - ] - return pd.DataFrame( - [ - { - "oa_code": oa, - "population": 100.0, - "households": households, - "constituency_code": constituency, - "region_code": region, - "lsoa_code": oa, - "msoa_code": oa, - "local_authority_code": local_authority, - "ward_code": ward, - "itl3_code": itl3, - } - for ( - oa, - region, - constituency, - ward, - local_authority, - itl3, - ), households in zip(rows, household_counts, strict=True) - ] - ) - - -def _write_ladder( - path: Path, - *, - household_counts: tuple[float, float, float, float] = ( - 3.0, - 10.0, - 10.0, - 10.0, - ), -): - payload = assemble_uk_oa_ladder( - _ladder_frame(household_counts), - _ladder_metadata(), - ) - np.savez_compressed(path, **payload) - return load_uk_oa_ladder(path) - - -def _write_staging_h5( - path: Path, - *, - households_per_region: int = 3, - region_masses: tuple[float, float, float, float] = (3.0, 10.0, 10.0, 10.0), -) -> None: - if households_per_region < 3: - raise ValueError("spine fixture needs one raw row and two derivatives") - region_names = ( - "LONDON", - "WALES", - "SCOTLAND", - "NORTHERN_IRELAND", - ) - household_ids = list(range(1, 4 * households_per_region + 1)) - source_household_ids: list[int] = [] - support_clone_indices: list[int] = [] - spi_flags: list[bool] = [] - for region_index in range(4): - first = region_index * households_per_region + 1 - raw_count = households_per_region - 2 - source_household_ids.extend(range(first, first + raw_count)) - source_household_ids.extend([first, first]) - support_clone_indices.extend([0] * raw_count + [0, 1]) - spi_flags.extend([False] * raw_count + [True, False]) - household = pd.DataFrame( - { - "household_id": household_ids, - "household_weight": [ - mass / households_per_region - for mass in region_masses - for _ in range(households_per_region) - ], - "region": [ - region for region in region_names for _ in range(households_per_region) - ], - "source_household_id": source_household_ids, - "source_household_key": [ - f"2023:{source_id}" for source_id in source_household_ids - ], - "household_source_id": source_household_ids, - "household_support_clone_index": support_clone_indices, - "household_is_spi_synthetic": spi_flags, - "household_is_capital_gains_clone": [False] * len(household_ids), - "household_is_cgt_band_donor": [False] * len(household_ids), - } - ) - person_ids = [10_000 + household_id for household_id in household_ids] - benunit_ids = [20_000 + household_id for household_id in household_ids] - person = pd.DataFrame( - { - "person_id": person_ids, - "person_household_id": household_ids, - "person_benunit_id": benunit_ids, - } - ) - benunit = pd.DataFrame({"benunit_id": benunit_ids}) - dataset = uk_national_frame( - person=person, - benunit=benunit, - household=household, - time_period="2023", - weight_kind=WeightKind.IMPORTANCE, - mass_log=( - MassChangeRecord( - entity="household", - old_total=33.0, - new_total=33.0, - declared_factor=1.0, - reason="Synthetic staging mass record.", - ), - ), - ) - write_uk_national_frame(dataset, path) - - -def test_candidate_build_writes_calibrated_h5_and_evidence( - monkeypatch, tmp_path -) -> None: - pytest.importorskip("tables") - pytest.importorskip("h5py") - builder = _load_builder_module() - input_h5 = tmp_path / "staging.h5" - ladder_path = tmp_path / "ladder.npz" - output_dir = tmp_path / "candidate" - _write_staging_h5(input_h5, households_per_region=52) - ladder = _write_ladder(ladder_path) - import microcosm.build.uk_runtime.battery_bindings as battery_bindings - - monkeypatch.setattr( - battery_bindings, - "_local_area_roster", - lambda _resource, levels: { - "constituency": tuple(sorted(set(ladder.constituency_code))), - "local_authority": tuple(sorted(set(ladder.local_authority_code))), - }, - ) - holdout = { - "report_only": True, - "method": "rotated_folds", - "n_folds": 5, - "seed": 20260529, - "solve_seed": 7, - "mean_holdout_loss": 0.1, - "worst_holdout_loss": 0.2, - "fold_losses": [0.1, 0.1, 0.2, 0.05, 0.05], - "folds": [], - } - monkeypatch.setattr( - builder, - "rotated_uk_local_holdout", - lambda *_args, **_kwargs: holdout, - ) - - assert ( - builder.main( - [ - "--input-h5", - str(input_h5), - "--ladder", - str(ladder_path), - "--out", - str(output_dir), - "--n-clones", - "2", - "--seed", - "7", - "--epochs", - "2", - ] - ) - == 0 - ) - - candidate_h5 = output_dir / builder.CANDIDATE_FILENAME_TEMPLATE.format( - calibration_year=2025 - ) - expected_sidecars = { - builder.MANIFEST_FILENAME, - builder.SOLVE_DIAGNOSTICS_FILENAME, - builder.AREA_SUPPORT_FILENAME, - builder.PAST_CAP_FILENAME, - builder.CALIBRATION_DIAGNOSTICS_FILENAME, - builder.LOCAL_REGISTRY_FILENAME, - builder.LOCAL_GATE_REPORT_FILENAME_TEMPLATE.format(calibration_year=2025), - } - assert candidate_h5.exists() - assert expected_sidecars <= {path.name for path in output_dir.iterdir()} - - candidate_kind, candidate_mass_log = read_uk_single_year_weight_metadata( - candidate_h5 - ) - with pd.HDFStore(candidate_h5, mode="r") as store: - candidate_household = store["household"] - assert candidate_kind is WeightKind.CALIBRATED - assert candidate_household["source_year"].unique().tolist() == [2023] - assert len(set(candidate_household["source_household_key"])) == 200 - assert {"2023:1", "2023:206"} <= set(candidate_household["source_household_key"]) - assert len(candidate_mass_log) == 3 - calibration_records = [ - record - for record in candidate_mass_log - if "census_households/constituency" in record.reason - ] - assert calibration_records == [candidate_mass_log[-1]] - # The kernel-minted record declares the realized factor (the hand-minted - # predecessor left it None) — declared-vs-realized is validated by the - # kernel at with_weights time. - record = candidate_mass_log[-1] - assert record.declared_factor == pytest.approx(record.new_total / record.old_total) - - manifest = json.loads((output_dir / builder.MANIFEST_FILENAME).read_text()) - assert manifest["candidate_scope"] == "adjudicated_partial" - assert manifest["bound_target_families"] == ["census_households/constituency"] - adjudications = manifest["binding_adjudications"] - assert adjudications["register_resource"] == "local_binding_adjudications.json" - assert adjudications["bound_families"] == ["census_households/constituency"] - assert adjudications["evaluated_on"] - seed = adjudications["stood_on"]["census_households/constituency"][ - "census_disclosure_control_noise" - ] - assert seed["approved_by"] == "juaristi22" - assert seed["adjudication"] == "microcosm#802" - assert seed["approved_on"] == "2026-08-31" - assert seed["expires_on"] == "2026-11-30" - assert adjudications["dormant"] == [ - "full_frs_tei_band_unavailable", - "hmrc_spi_frame_model_proxy", - "population_universe_private_households", - "uc_unit_vs_household_grain", - "voa_dwellings_vs_household_frame", - ] - cross_grain = manifest["cross_grain"] - assert cross_grain["bound_national_targets"] == [] - assert cross_grain["bound_higher_targets"] == [] - assert cross_grain["inconsistencies_in_force"] == [] - assert cross_grain["groups"] == [] - assert cross_grain["empty_legs_licensed"] == [] - assert cross_grain["controls_without_lower_rows"] == [] - assert cross_grain["absence"] - assert manifest["ladder_target_provenance"] == ladder_target_provenance(ladder) - assert manifest["gate"]["passed"] is True - assert manifest["gate"]["phase"] == "post_calibration" - assert manifest["gate"]["details"] - assert ( - manifest["inputs"]["dataset"]["sha256"] - == hashlib.sha256(input_h5.read_bytes()).hexdigest() - ) - assert manifest["inputs"]["dataset"]["bytes"] == input_h5.stat().st_size - assert ( - manifest["inputs"]["ladder"]["sha256"] - == hashlib.sha256(ladder_path.read_bytes()).hexdigest() - ) - assert manifest["inputs"]["ladder"]["bytes"] == ladder_path.stat().st_size - assert manifest["parameters"]["n_clones"] == 2 - assert manifest["parameters"]["seed"] == 7 - assert manifest["parameters"]["source_year"] == 2023 - assert manifest["parameters"]["source_lineage_modulus"] is None - assert manifest["parameters"]["epochs"] == 2 - assert manifest["parameters"]["learning_rate"] == pytest.approx(0.15) - assert manifest["parameters"]["expected_constituency_vintage"] == "2024_pcon" - assert [ - row["kind"] for row in manifest["weights"]["household_weight_kind_chain"] - ] == ["importance", "importance", "calibrated"] - assert manifest["weights"]["mass_log_records_before_calibration"] == 2 - assert manifest["weights"]["mass_log_records"] == 3 - mass_change = manifest["weights"]["calibration_mass_change"] - assert mass_change["old_total"] == pytest.approx(33.0) - assert mass_change["new_total"] == pytest.approx( - candidate_household["household_weight"].sum() - ) - assert mass_change["relative_shift"] == pytest.approx( - (mass_change["new_total"] - 33.0) / 33.0 - ) - assert manifest["parameters"]["doctrine"] == { - "target_loss_cap": 10.0, - "max_weight_ratio": 10.0, - "scale_rule": "default_target_loss_scales", - "target_weight_rule": "grain_equal", - "solve_epochs": 1500, - "clone_count": 15, - } - assert manifest["ladder_household_uprating"]["applied"] is False - assert manifest["solve"]["n_targets"] == 4 - assert manifest["solve"]["n_households"] == 416 - assert np.isfinite(manifest["solve"]["initial_loss"]) - assert np.isfinite(manifest["solve"]["final_loss"]) - assert np.isfinite(manifest["solve"]["max_abs_relative_error"]) - assert np.isfinite(manifest["solve"]["median_abs_relative_error"]) - assert manifest["solve"]["past_cap"]["n_targets"] == 4 - assert manifest["support"]["min_assigned_households"] == 104 - assert manifest["support"]["min_nonzero_households"] == 104 - assert manifest["support"]["min_effective_sample_size"] == pytest.approx(104.0) - - diagnostics = pd.read_csv(output_dir / builder.SOLVE_DIAGNOSTICS_FILENAME) - support = pd.read_csv(output_dir / builder.AREA_SUPPORT_FILENAME) - past_cap = json.loads((output_dir / builder.PAST_CAP_FILENAME).read_text()) - calibration_diagnostics = json.loads( - (output_dir / builder.CALIBRATION_DIAGNOSTICS_FILENAME).read_text() - ) - assert len(diagnostics) == 4 - assert diagnostics["metric"].unique().tolist() == ["households"] - assert len(support) == 8 - assert past_cap["n_targets"] == 4 - assert calibration_diagnostics["schema_version"] == 6 - uk_diagnostics = calibration_diagnostics["uk_diagnostics"] - assert len(uk_diagnostics["weakest_families"]) == 1 - assert len(uk_diagnostics["weakest_areas_by_fit"]["bottom_by_fit"]) == 4 - assert uk_diagnostics["weakest_areas_by_fit"]["n_areas_scored"] == 4 - assert { - row["country"] for row in uk_diagnostics["weakest_areas_by_fit"]["countries"] - } == { - "England", - "Northern Ireland", - "Scotland", - "Wales", - } - assert ( - manifest["diagnostics"]["weakest_families"] - == uk_diagnostics["weakest_families"] - ) - assert uk_diagnostics["rotated_holdout"] == holdout - assert manifest["diagnostics"]["rotated_holdout"] == holdout - assert "calibration_diagnostics" in manifest["outputs"] - rows = _spool_rows(output_dir) - assert len(rows) == 1 - row = rows[0] - assert row.pipeline == "uk-local-candidate" - assert row.rung == "f100" - assert row.seed == 7 - assert row.disposition == "iterating" - assert row.artifact_location == _local_ref(candidate_h5) - assert set(row.gate_verdicts) == set(builder.UK_LOCAL_GATE_SCOPE) - assert {item["verdict"] for item in row.gate_verdicts.values()} == {"passed"} - assert all( - ".local_gates.json#/gates/" in item["receipt"] - for item in row.gate_verdicts.values() - ) - - -def test_candidate_dry_run_plans_without_solve_or_write( - monkeypatch, - capsys, - tmp_path, -) -> None: - pytest.importorskip("tables") - pytest.importorskip("h5py") - builder = _load_builder_module() - input_h5 = tmp_path / "staging.h5" - ladder_path = tmp_path / "ladder.npz" - output_dir = tmp_path / "dry-run-output" - _write_staging_h5(input_h5) - ladder = _write_ladder(ladder_path) - - def forbidden(*_args, **_kwargs): - pytest.fail("dry run called a solve or dataset writer") - - monkeypatch.setattr( - builder, - "solve_uk_rowwise_weights_under_doctrine", - forbidden, - ) - monkeypatch.setattr(builder, "write_uk_rowwise_dataset", forbidden) - - assert ( - builder.main( - [ - "--input-h5", - str(input_h5), - "--ladder", - str(ladder_path), - "--out", - str(output_dir), - "--n-clones", - "2", - "--seed", - "7", - "--dry-run", - ] - ) - == 0 - ) - captured = capsys.readouterr() - plan = json.loads(captured.out) - assert plan["dry_run"] is True - assert plan["sampling"] == { - "fraction": 1.0, - "seed": 578, - "rung_token": "f100", - "sampled": False, - "pre_household_count": 12, - "post_household_count": 12, - } - assert plan["bound_target_families"] == ["census_households/constituency"] - adjudications = plan["binding_adjudications"] - assert adjudications["register_resource"] == "local_binding_adjudications.json" - assert adjudications["bound_families"] == ["census_households/constituency"] - assert adjudications["evaluated_on"] - assert ( - "census_disclosure_control_noise" - in adjudications["stood_on"]["census_households/constituency"] - ) - assert adjudications["dormant"] == [ - "full_frs_tei_band_unavailable", - "hmrc_spi_frame_model_proxy", - "population_universe_private_households", - "uc_unit_vs_household_grain", - "voa_dwellings_vs_household_frame", - ] - cross_grain = plan["cross_grain"] - assert cross_grain["bound_national_targets"] == [] - assert cross_grain["bound_higher_targets"] == [] - assert cross_grain["inconsistencies_in_force"] == [] - assert cross_grain["groups"] == [] - assert cross_grain["empty_legs_licensed"] == [] - assert cross_grain["controls_without_lower_rows"] == [] - assert cross_grain["absence"] - assert plan["ladder_target_provenance"] == ladder_target_provenance(ladder) - assert plan["shapes"]["person"][0] == 24 - assert plan["shapes"]["benunit"][0] == 24 - assert plan["shapes"]["household"][0] == 24 - assert plan["shapes"]["local_matrix"] == [4, 24] - assert plan["target_count"] == 4 - assert not output_dir.exists() - assert not (output_dir / "logbook-spool").exists() - - -def test_candidate_sampling_rung_receipt_and_engine_block_validation( - monkeypatch, - capsys, - tmp_path, -) -> None: - pytest.importorskip("tables") - pytest.importorskip("h5py") - builder = _load_builder_module() - input_h5 = tmp_path / "staging.h5" - ladder_path = tmp_path / "ladder.npz" - output_dir = tmp_path / "dry-run-output" - _write_staging_h5(input_h5) - _write_ladder(ladder_path) - - def compact_sampler_forbidden(*_args, **_kwargs): - pytest.fail("rowwise spine path called the certified-compact sampler") - - monkeypatch.setattr( - builder, - "sample_uk_national_frame", - compact_sampler_forbidden, - raising=False, - ) - monkeypatch.setattr( - builder, - "sample_uk_spine_frame", - lambda frame, **_kwargs: ( - frame, - { - "fraction": 0.01, - "seed": 578, - "rung_token": "f001", - "pre_household_count": 12, - "post_household_count": 12, - "pre_family_count": 4, - "post_family_count": 4, - "normalization_factor": 1.0, - "strata_count": 4, - "receipt": {"synthetic_fixture": True}, - }, - ), - raising=False, - ) - - assert ( - builder._parse_args( - [ - "--input-h5", - str(input_h5), - "--ladder", - str(ladder_path), - "--out", - str(output_dir), - ] - ).n_clones - == builder.UK_LOCAL_CLONE_COUNT - ) - with pytest.raises(ValueError, match="must equal --n-clones"): - builder.main( - [ - "--input-h5", - str(input_h5), - "--ladder", - str(ladder_path), - "--out", - str(output_dir), - "--n-clones", - "4", - "--engine-blocks", - "2", - "--dry-run", - ] - ) - assert ( - builder.main( - [ - "--input-h5", - str(input_h5), - "--ladder", - str(ladder_path), - "--out", - str(output_dir), - "--n-clones", - "2", - "--sample-fraction", - "0.01", - "--sample-seed", - "578", - "--dry-run", - ] - ) - == 0 - ) - plan = json.loads(capsys.readouterr().out) - assert plan["sampling"]["fraction"] == 0.01 - assert plan["sampling"]["rung_token"] == "f001" - assert plan["sampling"]["pre_household_count"] == 12 - assert plan["sampling"]["post_household_count"] >= 1 - assert plan["sampling"]["normalization_factor"] > 0 - - -def test_candidate_f100_does_not_call_any_sampler(monkeypatch, tmp_path) -> None: - pytest.importorskip("tables") - builder = _load_builder_module() - input_h5 = tmp_path / "staging.h5" - _write_staging_h5(input_h5) - frame, _ = builder.load_uk_national_frame(input_h5) - - def forbidden(*_args, **_kwargs): - pytest.fail("f100 called a sampler") - - monkeypatch.setattr(builder, "sample_uk_national_frame", forbidden, raising=False) - monkeypatch.setattr(builder, "sample_uk_spine_frame", forbidden, raising=False) - - sampled, receipt = builder._sample_candidate_frame( - frame, - fraction=1.0, - seed=578, - ) - - assert sampled is frame - assert receipt == { - "fraction": 1.0, - "seed": 578, - "rung_token": "f100", - "sampled": False, - "pre_household_count": 12, - "post_household_count": 12, - } - - -def test_candidate_clone_count_planning_is_dry_run_only(tmp_path) -> None: - builder = _load_builder_module() - with pytest.raises(ValueError, match="only with --dry-run"): - builder.main( - [ - "--input-h5", - str(tmp_path / "missing.h5"), - "--ladder", - str(tmp_path / "missing.npz"), - "--out", - str(tmp_path / "out"), - "--candidate-clone-counts", - "1,2,4", - ] - ) - - -def test_candidate_engine_surface_reuses_one_resolver( - monkeypatch, - tmp_path, -) -> None: - pytest.importorskip("tables") - builder = _load_builder_module() - input_h5 = tmp_path / "staging.h5" - _write_staging_h5(input_h5) - frame, _ = builder.load_uk_national_frame(input_h5) - constructions = [] - cgt_period_contract = { - "version": "uk-cgt-measurement-v2", - "input_period": "2024", - "calibration_period": 2025, - "bound_measurements": { - "cgt_2024_gains": { - "model_variable": "capital_gains", - "measurement_period": 2024, - } - }, - } - - class StubResolver: - def __init__(self, **kwargs): - constructions.append(kwargs) - self.simulation = object() - self.contract_targets = {} - - def receipt(self): - return { - "mode": "stub", - "policyengine_uk_version": "test", - "cgt_period_contract": cgt_period_contract, - } - - monkeypatch.setattr( - builder, - "compute_household_metrics", - lambda _simulation, area_type, *, household_ids, **_kwargs: pd.DataFrame( - {f"{area_type}_metric": np.ones(len(household_ids))}, - index=household_ids, - ), - ) - registry = TargetRegistry([], country="uk") - prepared, restore, national, local_metrics, receipt = ( - builder._resolve_candidate_engine_surface( - frame, - registry, - period=2025, - scratch_dir=tmp_path / "scratch", - resolver_factory=StubResolver, - ) - ) - - assert len(constructions) == 1 - assert receipt == { - "mode": "stub", - "engine_version": "test", - "households": 12, - "persons": 12, - "benunits": 12, - "national_inputs": 0, - "local_metrics": {"constituency": 1, "la": 1}, - "blocks": 1, - "cgt_period_contract": cgt_period_contract, - } - assert set(local_metrics) == {"constituency", "la"} - assert len(national.targets) == 0 - assert restore(prepared).table("household").equals(frame.table("household")) - - -@pytest.mark.parametrize("second_cgt_period", [2024, 2025, None]) -def test_candidate_engine_surface_resolves_real_per_clone_blocks( - monkeypatch, - tmp_path, - second_cgt_period, -) -> None: - pytest.importorskip("tables") - builder = _load_builder_module() - input_h5 = tmp_path / "staging.h5" - ladder_path = tmp_path / "ladder.npz" - _write_staging_h5(input_h5) - frame, _ = builder.load_uk_national_frame(input_h5) - ladder = _write_ladder(ladder_path) - clone = builder._clone_with_ladder_binding( - frame, - ladder, - n_clones=2, - seed=7, - source_year=2023, - expected_constituency_vintage="2024_pcon", - source_lineage_modulus=None, - ).result - constructions = [] - - class StubResolver: - def __init__(self, **kwargs): - constructions.append(kwargs) - self.simulation = object() - self.contract_targets = {} - self.cgt_period = 2024 if len(constructions) == 1 else second_cgt_period - - def receipt(self): - receipt = {"mode": "stub", "policyengine_uk_version": "test"} - if self.cgt_period is not None: - receipt["cgt_period_contract"] = { - "version": "uk-cgt-measurement-v2", - "input_period": "2024", - "calibration_period": 2025, - "bound_measurements": { - "cgt_2024_gains": { - "model_variable": "capital_gains", - "measurement_period": self.cgt_period, - } - }, - } - return receipt - - monkeypatch.setattr( - builder, - "compute_household_metrics", - lambda _simulation, area_type, *, household_ids, **_kwargs: pd.DataFrame( - {f"{area_type}_metric": np.arange(len(household_ids), dtype=float)}, - index=household_ids, - ), - ) - - def resolve(): - return builder._resolve_candidate_engine_surface( - clone.frame, - TargetRegistry([], country="uk"), - period=2025, - scratch_dir=tmp_path / "block-scratch", - resolver_factory=StubResolver, - blocks=2, - ) - - if second_cgt_period != 2024: - with pytest.raises(RuntimeError, match="CGT period contract is inconsistent"): - resolve() - return - - prepared, restore, _, metrics, receipt = resolve() - - assert len(constructions) == 2 - assert [len(call["frame"].table("household")) for call in constructions] == [ - 12, - 12, - ] - assert receipt["blocks"] == 2 - assert receipt["cgt_period_contract"]["bound_measurements"] == { - "cgt_2024_gains": { - "model_variable": "capital_gains", - "measurement_period": 2024, - } - } - assert receipt["deviation"] == "per_clone_block_engine_resolution" - sensitivity = receipt["block_sensitivity"] - assert ( - "ons/corporate_land_value" - in sensitivity["known_population_normalised_measures"] - ) - assert set(sensitivity["present_in_this_run"]) <= set( - sensitivity["known_population_normalised_measures"] - ) - assert "not evidence for adjudication" in sensitivity["caveat"] - assert ( - metrics["constituency"].index.tolist() - == clone.frame.table("household")["household_id"].tolist() - ) - assert restore(prepared).table("household").equals(clone.frame.table("household")) - - -def test_joint_candidate_f100_and_f001_end_to_end( - monkeypatch, - tmp_path, - capsys, -) -> None: - """The driver solves one local/ladder/national matrix at both rung postures.""" - - pytest.importorskip("tables") - pytest.importorskip("h5py") - builder = _load_builder_module() - input_h5 = tmp_path / "staging.h5" - ladder_path = tmp_path / "ladder.npz" - _write_staging_h5( - input_h5, - households_per_region=200, - region_masses=(4.0, 10.0, 10.0, 9.0), - ) - ladder_frame = _ladder_frame() - ladder_frame.loc[0, "households"] = 1.0 - english = ladder_frame.iloc[0].copy() - extra_english = [] - for suffix in (2, 3): - row = english.copy() - row["oa_code"] = f"E0000000{suffix}" - row["lsoa_code"] = row["oa_code"] - row["msoa_code"] = row["oa_code"] - row["constituency_code"] = f"E1400000{suffix}" - row["local_authority_code"] = f"E0900000{suffix}" - row["households"] = 1.0 - extra_english.append(row) - ladder_frame = pd.concat( - [ladder_frame, pd.DataFrame(extra_english)], ignore_index=True - ) - payload = assemble_uk_oa_ladder(ladder_frame, _ladder_metadata()) - np.savez_compressed(ladder_path, **payload) - ladder = load_uk_oa_ladder(ladder_path) - - from microcosm.build.uk_runtime.ledger_targets import UK_CROSS_GRAIN_BRIDGES - - household_bridge = UK_CROSS_GRAIN_BRIDGES[0] - reviewed_missing = { - "ons.household_composition.unrelated_adult_households", - "ons.household_composition.lone_parent_non_dependent_children_households", - "ons.household_composition.multi_family_households", - } - selected_composition = tuple( - target_id - for target_id in household_bridge.higher_target_ids - if target_id not in reviewed_missing - ) - fanout_target_id = "dwp.uc.payment_distribution_single" - fanout_names = tuple(f"payment-band-{index}" for index in range(3)) - national_registry = TargetRegistry( - [ - *[ - TargetSpec( - name=target_id, - entity="household", - measure=f"national/composition_{index}", - value=33.0, - period=2025, - source="synthetic national fixture", - family="ons", - metadata={ - "contract_target_id": target_id, - "ledger_geography_level": "country", - "ledger_geography_id": "K02000001", - }, - ) - for index, target_id in enumerate(selected_composition) - ], - *[ - TargetSpec( - name=name, - entity="household", - measure=f"national/payment_band_{index}", - value=33.0, - period=2025, - source="synthetic fan-out fixture", - family="dwp_uc", - metadata={ - "contract_target_id": fanout_target_id, - "ledger_geography_level": "country", - "ledger_geography_id": "K03000001", - }, - ) - for index, name in enumerate(fanout_names) - ], - ], - country="uk", - ) - local_registry = TargetRegistry( - [ - TargetSpec( - name="ons.tenure.owned_outright@E09000001", - entity="household", - measure="tenure/owned_outright", - value=1.0, - period=2025, - source="synthetic local fact fixture", - family="ons", - metadata={ - "contract_target_id": "ons.tenure.owned_outright", - "geography_level": "local_authority", - "geography_id": "E09000001", - "ledger_fact_period": "2023", - }, - ) - ], - country="uk", - ) - artifact = SimpleNamespace( - provenance=lambda: { - "facts_sha256": "1" * 64, - "manifest_sha256": "2" * 64, - "artifact_id": "synthetic-joint-fixture", - } - ) - joint_inputs = { - "artifact": artifact, - "calibration_year": 2025, - "national_registry": national_registry, - "band_edge_registry": national_registry, - "local_registry": local_registry, - "measure_exclusions": { - f"compiled::{target_id}": { - "tracking": "microcosm#791", - "reason": "relationship-to-head is unavailable", - } - for target_id in reviewed_missing - }, - "reviewed_unbound_higher_targets": { - target_id: { - "tracking": "microcosm#791", - "reason": "relationship-to-head is unavailable", - } - for target_id in reviewed_missing - }, - } - monkeypatch.setattr( - builder, "_load_joint_target_inputs", lambda _args: joint_inputs - ) - - constructions = [] - - class StubResolver: - def __init__(self, **kwargs): - constructions.append(kwargs) - self.frame = kwargs["frame"] - # A live resolver writes the frame to a scratch H5 through the - # national-frame writer, which validates the mass chain; a block - # must therefore carry a record whose total equals its weights. - validate_uk_national_frame(self.frame) - self.simulation = object() - self.contract_targets = {} - - def receipt(self): - return {"mode": "stub", "policyengine_uk_version": "test"} - - monkeypatch.setattr( - builder, - "resolve_target_measures", - # A live resolver injects ENGINE INPUTS (scratch columns the - # materialization reads), some of which also exist on another entity - # (region, esa_* on the spine); the driver must drop them before the - # prepared frame or the flattening rule refuses the duplicate column. - lambda _factory, _registry, provider, **_kwargs: SimpleNamespace( - measure_inputs={ - ("household", "stub_engine_input"): np.ones( - len(provider.frame.table("household")), dtype=float - ), - ("person", "region"): np.zeros( - len(provider.frame.table("person")), dtype=float - ), - } - ), - ) - - def _stub_materialize(adapter, registry, *, period, band_edge_registry=None): - # Materialization is what mints the prepared measure columns the - # national rows compile against; the stub writes them from the - # injected input so the lifecycle matches the real stage. - for spec in registry.specs: - table = adapter.tables[spec.entity] - table[spec.measure] = np.ones(len(table), dtype=float) - return SimpleNamespace(skipped=()) - - monkeypatch.setattr(builder, "materialize_uk_ledger_targets", _stub_materialize) - monkeypatch.setattr( - builder, - "compute_household_metrics", - lambda _simulation, area_type, *, household_ids, **_kwargs: pd.DataFrame( - { - "households": np.ones(len(household_ids), dtype=float), - **( - {"tenure/owned_outright": np.ones(len(household_ids), dtype=float)} - if area_type == "la" - else {} - ), - }, - index=household_ids, - ), - ) - real_resolve = builder._resolve_candidate_engine_surface - monkeypatch.setattr( - builder, - "_resolve_candidate_engine_surface", - lambda *args, **kwargs: real_resolve( - *args, resolver_factory=StubResolver, **kwargs - ), - ) - monkeypatch.setattr( - builder, - "rotated_uk_local_holdout", - lambda *_args, **_kwargs: {"report_only": True, "folds": []}, - ) - import microcosm.build.uk_runtime.battery_bindings as battery_bindings - - monkeypatch.setattr( - battery_bindings, - "_local_area_roster", - lambda _resource, levels: { - "constituency": tuple(sorted(set(ladder.constituency_code))), - "local_authority": tuple(sorted(set(ladder.local_authority_code))), - }, - ) - real_support_summary = builder.uk_ladder_area_support_summary - - def support_summary(household, ladder_arg): - if "household_weight" in household: - return real_support_summary(household, ladder_arg) - support = pd.DataFrame( - { - "nonzero_households": [len(household)], - "effective_sample_size": [float(len(household))], - "nonzero_source_households": [ - household["source_household_id"].nunique() - ], - } - ) - return {"constituency": support, "la": support} - - monkeypatch.setattr(builder, "uk_ladder_area_support_summary", support_summary) - - dry_out = tmp_path / "joint-dry" - assert ( - builder.main( - [ - "--input-h5", - str(input_h5), - "--ladder", - str(ladder_path), - "--out", - str(dry_out), - "--n-clones", - "2", - "--dry-run", - ] - ) - == 0 - ) - dry_plan = json.loads(capsys.readouterr().out) - dry_unbound = dry_plan["cross_grain"]["unbound_bridges"] - assert dry_plan["cross_grain"]["empty_legs_licensed"] == [] - assert dry_plan["cross_grain"]["controls_without_lower_rows"] == [] - assert [entry["bridge_id"] for entry in dry_unbound] == [household_bridge.bridge_id] - assert dry_unbound[0]["missing"] == sorted(reviewed_missing) - dry_fanout = dry_plan["cross_grain"]["fanout_targets_not_controls"] - assert dry_fanout == [ - { - "target_id": fanout_target_id, - "geography_id": "K03000001", - "cells": 3, - "cell_names": list(fanout_names), - "activated_sum": 99.0, - "reason": ( - "The activated cells are a band subset, so this distribution " - "is not a cross-grain control." - ), - } - ] - assert "fanout_controls_summed" not in dry_plan["cross_grain"] - assert not dry_out.exists() - - f100_out = tmp_path / "joint-f100" - assert ( - builder.main( - [ - "--input-h5", - str(input_h5), - "--ladder", - str(ladder_path), - "--out", - str(f100_out), - "--n-clones", - "2", - "--epochs", - "2", - "--skip-holdout", - ] - ) - == 0 - ) - f100 = json.loads((f100_out / builder.MANIFEST_FILENAME).read_text()) - assert f100["schema_version"] == 2 - # The written rowwise artifact carries the shared ``clone_index`` name on - # every table: the compact national loader must refuse it (flattening - # rule) and the rowwise reader must undo the export rename. - from microcosm.build.uk_runtime.rowwise_dataset import ( - ladder_clone_index_column, - load_uk_rowwise_dataset, - ) - - dataset_path = Path(f100["outputs"]["dataset"]["path"]) - with pytest.raises(ValueError, match="globally unique"): - builder.load_uk_national_frame(dataset_path) - reloaded, provenance = load_uk_rowwise_dataset(dataset_path) - assert provenance.source_h5 == dataset_path.resolve() or str( - provenance.source_h5 - ).endswith(dataset_path.name) - for entity in ("person", "benunit", "household"): - assert ladder_clone_index_column(entity) in reloaded.table(entity).columns - assert "clone_index" not in reloaded.table(entity).columns - assert len(reloaded.table("household")) == f100["solve"]["n_households"] - assert reloaded.weights_for("household").total == pytest.approx( - f100["weights"]["calibration_mass_change"]["new_total"] - ) - # Exact: the reader undoes the export rename and nothing else, so every - # table equals the written one with clone_index renamed back. - for entity in ("person", "benunit", "household"): - written = pd.read_hdf(dataset_path, entity) - expected = written.rename( - columns={"clone_index": ladder_clone_index_column(entity)} - ) - if entity == "household": - # The frame carries the weight as its typed vector, not a column. - np.testing.assert_array_equal( - reloaded.weights_for("household").values, - expected["household_weight"].to_numpy(dtype="float64"), - ) - expected = expected.drop(columns=["household_weight"]) - got = reloaded.table(entity) - assert sorted(got.columns) == sorted(expected.columns) - pd.testing.assert_frame_equal( - got[sorted(got.columns)].reset_index(drop=True), - expected[sorted(expected.columns)].reset_index(drop=True), - check_dtype=True, - ) - assert f100["solve"]["n_targets_by_kind"] == { - "local": 1, - "ladder": 12, - "national": len(national_registry.specs), - } - assert f100["solve"]["n_targets"] == 13 + len(national_registry.specs) - assert f100["solve"]["measure_resolution"]["mode"] == "stub" - assert f100["cross_grain"]["unbound_bridges"] == dry_unbound - assert f100["solve"]["cross_grain"]["unbound_bridges"] == dry_unbound - assert f100["cross_grain"]["empty_legs_licensed"] == [] - assert f100["solve"]["cross_grain"]["empty_legs_licensed"] == [] - assert f100["cross_grain"]["controls_without_lower_rows"] == [] - assert f100["solve"]["cross_grain"]["controls_without_lower_rows"] == [] - assert f100["cross_grain"]["fanout_targets_not_controls"] == dry_fanout - assert f100["solve"]["cross_grain"]["fanout_targets_not_controls"] == dry_fanout - assert "fanout_controls_summed" not in f100["cross_grain"] - assert "fanout_controls_summed" not in f100["solve"]["cross_grain"] - assert f100["releasable"] is True - assert f100["measure_exclusions"] == joint_inputs["measure_exclusions"] - assert f100["ladder_household_uprating"]["applied"] is False - assert _spool_rows(f100_out)[0].rung == "f100" - - f001_out = tmp_path / "joint-f001" - assert ( - builder.main( - [ - "--input-h5", - str(input_h5), - "--ladder", - str(ladder_path), - "--out", - str(f001_out), - "--n-clones", - "1", - "--sample-fraction", - "0.01", - "--epochs", - "2", - "--skip-holdout", - ] - ) - == 0 - ) - f001 = json.loads((f001_out / builder.MANIFEST_FILENAME).read_text()) - assert f001["rung_surface"]["dropped_cells"] > 0 - assert f001["rung_surface"]["dropped_unreachable_cells"] >= 0 - assert isinstance(f001["rung_surface"]["dropped_unreachable_by_grain"], dict) - assert f001["rung_surface"]["dropped_by_grain"]["constituency"] >= 1 - assert f001["rung_surface"]["dropped_by_grain"]["la"] >= 1 - assert "uk_local_area_support" in f001["failing_gate_ids"] - assert f001["releasable"] is False - assert _spool_rows(f001_out)[0].rung == "f001" - assert len(constructions) == 2 - - -def test_candidate_refusal_records_receipt_and_reraises( - monkeypatch, - tmp_path, -) -> None: - pytest.importorskip("tables") - pytest.importorskip("h5py") - builder = _load_builder_module() - input_h5 = tmp_path / "staging.h5" - ladder_path = tmp_path / "ladder.npz" - output_dir = tmp_path / "candidate" - _write_staging_h5(input_h5) - _write_ladder(ladder_path) - - def failing_gate(*_args, **_kwargs): - return builder.GateResult( - name="spine_agreement", - passed=False, - failures=("post-calibration coverage failed",), - details={"minimum": 0}, - ) - - original = builder.UK_GATE_REGISTRY["spine_agreement"] - monkeypatch.setattr( - builder, - "UK_GATE_REGISTRY", - { - **builder.UK_GATE_REGISTRY, - "spine_agreement": replace(original, evaluator=failing_gate), - }, - ) - - with pytest.raises( - builder.GateBatteryBlockedError, match="post-calibration coverage failed" - ): - builder.main( - [ - "--input-h5", - str(input_h5), - "--ladder", - str(ladder_path), - "--out", - str(output_dir), - "--n-clones", - "2", - "--seed", - "7", - "--epochs", - "2", - ] - ) - - rows = _spool_rows(output_dir) - assert len(rows) == 1 - row = rows[0] - assert row.disposition == "failed" - gate_report_path = output_dir / builder.LOCAL_GATE_REPORT_FILENAME_TEMPLATE.format( - calibration_year=2025 - ) - assert gate_report_path.exists() - assert row.gate_verdicts["uk_local_geography_ladder_post_calibration"] == { - "verdict": "failed", - "receipt": ( - f"{_local_ref(gate_report_path)}" - "#/gates/uk_local_geography_ladder_post_calibration" - ), - } - assert row.gate_verdicts["pipeline_error"]["verdict"] == "error" - assert row.gate_verdicts["pipeline_error"]["receipt"].endswith("#/error_type") - - -def test_candidate_binding_adjudication_failure_records_failed_row( - monkeypatch, - tmp_path, -) -> None: - pytest.importorskip("tables") - pytest.importorskip("h5py") - builder = _load_builder_module() - input_h5 = tmp_path / "staging.h5" - ladder_path = tmp_path / "ladder.npz" - output_dir = tmp_path / "candidate" - _write_staging_h5(input_h5) - _write_ladder(ladder_path) - - import microcosm.build.uk_runtime.local_rowwise as local_rowwise - - monkeypatch.setattr( - local_rowwise, - "load_uk_reviewed_exclusion_register", - lambda *_args, **_kwargs: {}, - ) - - with pytest.raises(ValueError, match="census_disclosure_control_noise"): - builder.main( - [ - "--input-h5", - str(input_h5), - "--ladder", - str(ladder_path), - "--out", - str(output_dir), - "--n-clones", - "2", - "--seed", - "7", - "--epochs", - "2", - ] - ) - - rows = _spool_rows(output_dir) - assert len(rows) == 1 - row = rows[0] - assert row.disposition == "failed" - assert "targets_bound" in row.phases_reached - assert "solved" not in row.phases_reached - assert row.gate_verdicts["pipeline_error"]["verdict"] == "error" - assert row.gate_verdicts["pipeline_error"]["receipt"].endswith("#/error_type") - - -def test_candidate_setup_failure_records_failed_row(monkeypatch, tmp_path) -> None: - """A pre-solve setup failure (ladder load) still spools a failed row. - - Adversarial-review finding on #666: input verification, frame/ladder - loading, cloning, and target binding used to run before the recording - envelope opened, so their failures escaped with no Logbook row. - """ - - pytest.importorskip("tables") - pytest.importorskip("h5py") - builder = _load_builder_module() - input_h5 = tmp_path / "staging.h5" - ladder_path = tmp_path / "ladder.npz" - output_dir = tmp_path / "candidate" - _write_staging_h5(input_h5) - _write_ladder(ladder_path) - - def failing_ladder_load(_path): - raise RuntimeError("ladder artifact refused to parse") - - monkeypatch.setattr(builder, "load_uk_oa_ladder", failing_ladder_load) - - with pytest.raises(RuntimeError, match="ladder artifact refused to parse"): - builder.main( - [ - "--input-h5", - str(input_h5), - "--ladder", - str(ladder_path), - "--out", - str(output_dir), - "--n-clones", - "2", - "--seed", - "7", - "--epochs", - "2", - ] - ) - - rows = _spool_rows(output_dir) - assert len(rows) == 1 - row = rows[0] - assert row.disposition == "failed" - assert row.gate_verdicts["pipeline_error"]["verdict"] == "error" - assert row.gate_verdicts["pipeline_error"]["receipt"].endswith("#/error_type") - assert "inputs_pinned" in row.phases_reached - assert "cloned" not in row.phases_reached - # Real input pins were promoted before the failure; the preflight - # placeholder digest must not survive into the row. - assert row.input_pins_digest != builder.preflight_digest( - builder._UK_CANDIDATE_PIPELINE - ) - - -def test_candidate_refuses_separate_assignment_and_target_ladders( - tmp_path, -) -> None: - pytest.importorskip("tables") - pytest.importorskip("h5py") - builder = _load_builder_module() - input_h5 = tmp_path / "staging.h5" - first_path = tmp_path / "assignment_ladder.npz" - second_path = tmp_path / "target_ladder.npz" - _write_staging_h5(input_h5) - assignment_ladder = _write_ladder(first_path) - target_ladder = _write_ladder( - second_path, - household_counts=(4.0, 9.0, 10.0, 10.0), - ) - assignment = builder._clone_with_ladder_binding( - input_h5, - assignment_ladder, - n_clones=2, - seed=7, - source_year=2023, - expected_constituency_vintage="2024_pcon", - source_lineage_modulus=None, - ) - - with pytest.raises(ValueError, match="same loaded"): - builder._build_bound_problem( - assignment, - target_ladder=target_ladder, - ) - - -def test_candidate_dry_run_refuses_ladder_sidecar_collision( - tmp_path, -) -> None: - pytest.importorskip("tables") - pytest.importorskip("h5py") - builder = _load_builder_module() - input_h5 = tmp_path / "staging.h5" - output_dir = tmp_path / "candidate" - temporary_ladder = tmp_path / "ladder.npz" - ladder_path = output_dir / builder.MANIFEST_FILENAME - _write_staging_h5(input_h5) - _write_ladder(temporary_ladder) - output_dir.mkdir() - temporary_ladder.replace(ladder_path) - ladder_bytes = ladder_path.read_bytes() - - with pytest.raises(ValueError, match="differ"): - builder.main( - [ - "--input-h5", - str(input_h5), - "--ladder", - str(ladder_path), - "--out", - str(output_dir), - "--dry-run", - ] - ) - - assert ladder_path.read_bytes() == ladder_bytes - assert list(output_dir.iterdir()) == [ladder_path] - - -def test_candidate_publication_rolls_back_on_interrupt( - monkeypatch, - tmp_path, -) -> None: - builder = _load_builder_module() - staging_dir = tmp_path / "staging" - output_dir = tmp_path / "candidate" - staging_dir.mkdir() - output_paths = builder._output_paths( - output_dir, - source_year=2023, - calibration_year=2025, - ) - staged = {key: staging_dir / path.name for key, path in output_paths.items()} - for path in staged.values(): - path.write_text("complete staged artifact\n") - - original_replace = Path.replace - - def interrupt_support(self, target): - if Path(target) == output_paths["support"]: - raise KeyboardInterrupt - return original_replace(self, target) - - monkeypatch.setattr(Path, "replace", interrupt_support) - with pytest.raises(KeyboardInterrupt): - builder._publish_staged_files(staged, output_paths) - - assert not output_dir.exists() - - -def _failing_gate_evaluator(builder, name: str, message: str): - def evaluator(*_args, **_kwargs): - return builder.GateResult( - name=name, passed=False, failures=(message,), details={"minimum": 0} - ) - - return evaluator - - -def _joint_f100_args(input_h5: Path, ladder_path: Path, output_dir: Path) -> list[str]: +def _arguments(tmp_path): return [ - "--input-h5", - str(input_h5), - "--ladder", - str(ladder_path), - "--out", - str(output_dir), - "--n-clones", - "2", - "--seed", - "7", - "--epochs", - "2", - "--skip-holdout", - ] - - -def test_candidate_weight_ratio_failure_is_reported_and_blocks( - monkeypatch, tmp_path, capsys -) -> None: - pytest.importorskip("tables") - pytest.importorskip("h5py") - builder = _load_builder_module() - input_h5 = tmp_path / "staging.h5" - ladder_path = tmp_path / "ladder.npz" - output_dir = tmp_path / "candidate" - _write_staging_h5( - input_h5, households_per_region=200, region_masses=(4.0, 10.0, 10.0, 9.0) - ) - _write_ladder(ladder_path) - ladder = load_uk_oa_ladder(ladder_path) - import microcosm.build.uk_runtime.battery_bindings as battery_bindings - - monkeypatch.setattr( - battery_bindings, - "_local_area_roster", - lambda _resource, levels: { - "constituency": tuple(sorted(set(ladder.constituency_code))), - "local_authority": tuple(sorted(set(ladder.local_authority_code))), - }, - ) - ratio = builder.UK_GATE_REGISTRY["weight_ratio"] - monkeypatch.setattr( - builder, - "UK_GATE_REGISTRY", - { - **builder.UK_GATE_REGISTRY, - "weight_ratio": replace( - ratio, - evaluator=_failing_gate_evaluator( - builder, "weight_ratio", "ratio 104.6 > 100" - ), - ), - }, - ) - - assert builder.main(_joint_f100_args(input_h5, ladder_path, output_dir)) == 1 - - capsys.readouterr() - manifest = json.loads((output_dir / builder.MANIFEST_FILENAME).read_text()) - assert manifest["failing_gate_ids"] == ["uk_local_weight_ratio"] - assert manifest["blocked_at_f100"] is True - assert manifest["diagnostic_failures"] == [] - assert manifest["blocking_failures"] == [ - "[uk_local_weight_ratio] ratio 104.6 > 100" - ] - assert manifest["releasable"] is False - report = json.loads( - Path(manifest["outputs"]["local_gate_report"]["path"]).read_text() - ) - assert report["gates"]["uk_local_weight_ratio"]["criticality"] == "release_blocking" - assert report["gates"]["uk_local_weight_ratio"]["status"] == "failed" - assert _spool_rows(output_dir)[0].disposition == "failed" - - -def test_candidate_block_partitions_failures_by_criticality( - monkeypatch, tmp_path, capsys -) -> None: - pytest.importorskip("tables") - pytest.importorskip("h5py") - builder = _load_builder_module() - input_h5 = tmp_path / "staging.h5" - ladder_path = tmp_path / "ladder.npz" - output_dir = tmp_path / "candidate" - _write_staging_h5( - input_h5, households_per_region=200, region_masses=(4.0, 10.0, 10.0, 9.0) - ) - _write_ladder(ladder_path) - registry = builder.UK_GATE_REGISTRY - monkeypatch.setattr( - builder, - "UK_GATE_REGISTRY", - { - **registry, - "area_support": replace( - registry["area_support"], - evaluator=_failing_gate_evaluator( - builder, "area_support", "ESS 42.3 < 50" - ), - ), - "weight_ratio": replace( - registry["weight_ratio"], - evaluator=_failing_gate_evaluator( - builder, "weight_ratio", "ratio 578 > 100" - ), - ), - }, - ) - - assert builder.main(_joint_f100_args(input_h5, ladder_path, output_dir)) == 1 - - captured = capsys.readouterr() - assert "artifact unreleasable" in captured.err - manifest = json.loads((output_dir / builder.MANIFEST_FILENAME).read_text()) - assert manifest["failing_gate_ids"] == [ - "uk_local_area_support", - "uk_local_weight_ratio", - ] - assert manifest["blocked_at_f100"] is True - assert manifest["blocking_failures"] == [ - "[uk_local_area_support] ESS 42.3 < 50", - "[uk_local_weight_ratio] ratio 578 > 100", - ] - assert manifest["diagnostic_failures"] == [] - assert manifest["releasable"] is False - assert _spool_rows(output_dir)[0].disposition == "failed" - - -def test_release_verdict_requires_single_block_engine() -> None: - builder = _load_builder_module() - releasable, posture = builder._release_verdict( - sample_fraction=1.0, engine_blocks=1, release_blocking_gates_passed=True - ) - assert releasable is True and all(posture.values()) - # A per-block engine resolution never writes a releasable artifact, even - # with every release-blocking gate passed on the full rung (#736 erratum). - releasable, posture = builder._release_verdict( - sample_fraction=1.0, engine_blocks=15, release_blocking_gates_passed=True - ) - assert releasable is False - assert posture == { - "full_rung": True, - "single_block_engine": False, - "release_blocking_gates_passed": True, - } - assert ( - builder._release_verdict( - sample_fraction=0.1, engine_blocks=1, release_blocking_gates_passed=True - )[0] - is False - ) - - -def test_gate_criticality_reads_fail_closed() -> None: - builder = _load_builder_module() - assert builder._is_release_blocking({"criticality": "release_blocking"}) is True - assert builder._is_release_blocking({"criticality": "diagnostic"}) is False - # Missing or unknown criticality vetoes: schema drift on one entry cannot - # drop a failed gate out of both the blocking list and all_gates_passed. - assert builder._is_release_blocking({}) is True - assert builder._is_release_blocking({"criticality": "advisory"}) is True - blocking, diagnostic = builder._gate_failures_by_criticality( - { - "gates": { - "uk_local_area_support": { - "status": "failed", - "failures": ["ESS 42.3 < 50"], - }, - "uk_local_weight_ratio": { - "status": "failed", - "criticality": "diagnostic", - "failures": ["ratio 578 > 100"], - }, - "uk_local_target_fit": { - "status": "passed", - "criticality": "diagnostic", - }, - } - } - ) - assert blocking == ["[uk_local_area_support] ESS 42.3 < 50"] - assert diagnostic == ["[uk_local_weight_ratio] ratio 578 > 100"] - - -def test_release_candidate_refuses_non_doctrine_solve_settings(tmp_path) -> None: - builder = _load_builder_module() - pin = "0" * 64 - base = [ "--input-h5", str(tmp_path / "spine.h5"), - "--input-sha256", - pin, "--ladder", - str(tmp_path / "ladder.npz"), - "--ladder-sha256", - pin, + str(tmp_path / "ladder.parquet"), "--ledger-facts", - str(tmp_path / "ledger"), - "--ledger-facts-sha256", - pin, - "--ledger-manifest-sha256", - pin, + str(tmp_path / "chronicle"), "--out", str(tmp_path / "out"), - "--release-candidate", ] - # The doctrine defaults are the release posture: nothing to refuse. - args = builder._parse_args(base) - builder._validate_cli_args(args) - assert args.n_clones == builder.UK_LOCAL_CLONE_COUNT == 15 - assert args.epochs == builder.UK_LOCAL_SOLVE_EPOCHS == 1500 - assert args.target_weight_rule == "grain_equal" - with pytest.raises(ValueError, match=r"--epochs != doctrine 1500"): - builder._validate_cli_args(builder._parse_args([*base, "--epochs", "512"])) - with pytest.raises(ValueError, match=r"--n-clones != doctrine 15"): - builder._validate_cli_args(builder._parse_args([*base, "--n-clones", "10"])) - with pytest.raises(ValueError, match=r"--target-weight-rule"): - builder._validate_cli_args( - builder._parse_args([*base, "--target-weight-rule", "uniform"]) - ) +def test_compatibility_command_is_exact_full_build_redirect(tmp_path, monkeypatch): + seen = [] -def test_candidate_multi_block_engine_run_is_never_releasable( - monkeypatch, tmp_path, capsys -) -> None: - """End to end: ``--engine-blocks K`` on f100 writes ``releasable: false``. + def execute(prepared, args): + seen.append((prepared, args)) + return 17 - Every release-blocking gate passes here; the posture alone withholds the - verdict, and the manifest names the leg (``single_block_engine``). This is - the assertion that catches a future caller bypassing ``_release_verdict``. - """ + monkeypatch.setattr(full_build_cli, "prepare_full_build", lambda args: "prepared") + monkeypatch.setattr(full_build_cli, "execute_full_build", execute) + module = _load_builder_module() + assert module.main is full_build_cli.main + assert module.main(_arguments(tmp_path)) == 17 + assert seen[0][1].target_geographies is None - pytest.importorskip("tables") - pytest.importorskip("h5py") - builder = _load_builder_module() - input_h5 = tmp_path / "staging.h5" - ladder_path = tmp_path / "ladder.npz" - output_dir = tmp_path / "candidate" - _write_staging_h5( - input_h5, households_per_region=200, region_masses=(4.0, 10.0, 10.0, 9.0) - ) - _write_ladder(ladder_path) - ladder = load_uk_oa_ladder(ladder_path) - import microcosm.build.uk_runtime.battery_bindings as battery_bindings +def test_country_only_is_an_explicit_filter_in_the_same_command(tmp_path, monkeypatch): + seen = [] + monkeypatch.setattr(full_build_cli, "prepare_full_build", lambda args: args) monkeypatch.setattr( - battery_bindings, - "_local_area_roster", - lambda _resource, levels: { - "constituency": tuple(sorted(set(ladder.constituency_code))), - "local_authority": tuple(sorted(set(ladder.local_authority_code))), - }, + full_build_cli, + "execute_full_build", + lambda prepared, args: seen.append(args) or 0, ) - - args = [ - *_joint_f100_args(input_h5, ladder_path, output_dir), - "--engine-blocks", - "2", - ] - assert builder.main(args) == 0 - - capsys.readouterr() - manifest = json.loads((output_dir / builder.MANIFEST_FILENAME).read_text()) - assert manifest["parameters"]["engine_blocks"] == 2 - assert manifest["blocking_failures"] == [] - assert manifest["releasable"] is False - assert manifest["release_posture"] == { - "full_rung": True, - "single_block_engine": False, - "release_blocking_gates_passed": True, - } - - -def test_size_candidate_exports_compact_links_and_cannot_claim_dense_release( - monkeypatch, tmp_path -): - pytest.importorskip("tables") - pytest.importorskip("h5py") - builder = _load_builder_module() - input_h5 = tmp_path / "spine.h5" - ladder_path = tmp_path / "ladder.npz" - out = tmp_path / "k300" - _write_staging_h5(input_h5, households_per_region=52) - ladder = _write_ladder(ladder_path) - import microcosm.build.uk_runtime.battery_bindings as bindings - - monkeypatch.setattr( - bindings, - "_local_area_roster", - lambda _resource, levels: { - "constituency": tuple(sorted(set(ladder.constituency_code))), - "local_authority": tuple(sorted(set(ladder.local_authority_code))), - }, - ) - status = builder.main( - [ - "--input-h5", - str(input_h5), - "--ladder", - str(ladder_path), - "--out", - str(out), - "--n-clones", - "2", - "--dataset-households", - "300", - "--selection-seed", - "11", - "--epochs", - "2", - "--skip-holdout", - "--seed", - "7", - ] - ) - assert status in (0, 1) # Gate failures remain reportable candidates. - manifest = json.loads((out / builder.MANIFEST_FILENAME).read_text()) - assert manifest["releasable"] is False - assert manifest["release_posture"]["size_certification_present"] is False - size = manifest["solve"]["dataset_size"] - assert size["requested_households"] == size["realized_households"] == 300 - assert size["pool_households"] == 416 - assert manifest["parameters"]["n_clones"] == 2 assert ( - manifest["weights"]["stretch_reference"] - == "normalized_horvitz_thompson_w_over_q" - ) - path = out / builder.CANDIDATE_FILENAME_TEMPLATE.format(calibration_year=2025) - with pd.HDFStore(path, "r") as store: - households = store["household"] - persons = store["person"] - benunits = store["benunit"] - assert len(households) == 300 - assert set(persons.person_household_id) == set(households.household_id) - assert set(persons.person_benunit_id) == set(benunits.benunit_id) - assert len(_spool_rows(out)) == 1 - - # The selection seed moves the draw only; the manifest records both seeds. - assert manifest["parameters"]["seed"] == 7 - assert manifest["parameters"]["selection_seed"] == 11 - assert size["seed"] == 11 - assert manifest["parameters"]["selection_pi_hi"] == 1.0 - assert size["selection_pi_hi"] == 1.0 - assert size["selection_receipt"]["pi_hi"] == 1.0 - assert size["selection_feasibility"]["requested_pi_hi"] == 1.0 - assert size["selection_feasibility"]["feasible_at_requested_pi_hi"] is True - - # The dense solve the selection was cut from ships as evidence. - dense = size["dense_reference"] - assert dense["final_loss"] == size["dense_loss"] - assert dense["n_households"] == 416 - assert dense["weights"]["n_records"] == 416 - assert {"effective_sample_size", "max_to_median_positive_weight"} <= set( - dense["weights"] - ) - assert dense["diagnostics_file"] == builder.DENSE_REFERENCE_DIAGNOSTICS_FILENAME - outputs = manifest["outputs"] - dense_csv = out / builder.DENSE_REFERENCE_DIAGNOSTICS_FILENAME - selection_csv = out / builder.DATASET_SIZE_SELECTION_FILENAME - assert Path(outputs["dense_reference_diagnostics"]["path"]) == dense_csv.resolve() - assert Path(outputs["dataset_size_selection"]["path"]) == selection_csv.resolve() - assert ( - outputs["dataset_size_selection"]["sha256"] - == hashlib.sha256(selection_csv.read_bytes()).hexdigest() - ) - dense_rows = pd.read_csv(dense_csv) - assert len(dense_rows) == manifest["solve"]["n_targets"] - assert dense_rows.columns[0] == "grain" - assert {"target", "final_estimate", "abs_relative_error"} <= set(dense_rows.columns) - selection = pd.read_csv(selection_csv) - assert list(selection.columns) == [ - "pool_row_index", - "household_id", - "clone_index", - "design_weight", - "inclusion_probability", - "certainty", - "ht_baseline_weight", - "refit_weight", - ] - assert len(selection) == 300 - assert selection["pool_row_index"].is_unique - assert selection["pool_row_index"].max() < 416 - assert set(selection["household_id"]) == set(households.household_id) - assert (selection["refit_weight"] > 0).all() - assert (selection["design_weight"] > 0).all() - assert ( - int(selection["certainty"].sum()) - == size["selection_receipt"]["certainty_count"] - ) - assert int(selection["certainty"].sum()) == size["protected_carriers"] - - -def test_selection_seed_requires_a_dataset_size(tmp_path): - builder = _load_builder_module() - args = builder._parse_args( - [ - "--input-h5", - str(tmp_path / "spine.h5"), - "--ladder", - str(tmp_path / "ladder.npz"), - "--out", - str(tmp_path / "out"), - "--selection-seed", - "11", - ] + _load_builder_module().main( + _arguments(tmp_path) + ["--target-geographies", "country"] + ) + == 0 ) - with pytest.raises(ValueError, match="requires --dataset-households"): - builder._validate_cli_args(args) + assert seen[0].target_geographies == ("country",) @pytest.mark.parametrize( - ("argv_tail", "message"), + "retired", [ - (["--selection-pi-hi", "0.95"], "requires --dataset-households"), - (["--dataset-households", "10", "--selection-pi-hi", "0"], r"in \(0, 1\]"), - (["--dataset-households", "10", "--selection-pi-hi", "1.5"], r"in \(0, 1\]"), + "--constituency-household-targets", + "--staging-h5", + "--allow-unpinned-feed", + "--logbook-dir", ], ) -def test_selection_pi_hi_is_candidate_only_and_bounded(tmp_path, argv_tail, message): - builder = _load_builder_module() - args = builder._parse_args( - [ - "--input-h5", - str(tmp_path / "spine.h5"), - "--ladder", - str(tmp_path / "ladder.npz"), - "--out", - str(tmp_path / "out"), - *argv_tail, - ] - ) - with pytest.raises(ValueError, match=message): - builder._validate_cli_args(args) - - -def test_dense_candidate_manifest_has_no_size_sidecars(tmp_path): - builder = _load_builder_module() - paths = builder._output_paths(tmp_path, source_year=2024, calibration_year=2025) - assert paths["dense_reference"].name == builder.DENSE_REFERENCE_DIAGNOSTICS_FILENAME - assert paths["selection"].name == builder.DATASET_SIZE_SELECTION_FILENAME - assert builder._SIZE_RUN_ONLY_OUTPUTS == {"dense_reference", "selection"} - - -def test_size_cli_refuses_promotion_without_separate_certification(tmp_path): - builder = _load_builder_module() - args = builder._parse_args( - [ - "--input-h5", - str(tmp_path / "spine.h5"), - "--ladder", - str(tmp_path / "ladder.npz"), - "--out", - str(tmp_path / "out"), - "--dataset-households", - "50000", - "--release-candidate", - ] - ) - with pytest.raises(ValueError, match="candidate-only"): - builder._validate_cli_args(args) - - -def test_size_candidate_checkpoints_before_the_draw_and_resumes_from_it( - monkeypatch, tmp_path, capsys +def test_retired_options_refuse_instead_of_selecting_a_legacy_route( + tmp_path, capsys, retired ): - pytest.importorskip("tables") - pytest.importorskip("h5py") - builder = _load_builder_module() - from microcosm.build.uk_runtime.size_checkpoint import ( - SIZE_CHECKPOINT_ARRAYS_FILENAME, - SIZE_CHECKPOINT_MANIFEST_FILENAME, - ) - - input_h5 = tmp_path / "spine.h5" - ladder_path = tmp_path / "ladder.npz" - _write_staging_h5(input_h5, households_per_region=52) - ladder = _write_ladder(ladder_path) - import microcosm.build.uk_runtime.battery_bindings as bindings - - monkeypatch.setattr( - bindings, - "_local_area_roster", - lambda _resource, levels: { - "constituency": tuple(sorted(set(ladder.constituency_code))), - "local_authority": tuple(sorted(set(ladder.local_authority_code))), - }, - ) - common = [ - "--input-h5", - str(input_h5), - "--ladder", - str(ladder_path), - "--n-clones", - "2", - "--dataset-households", - "300", - "--epochs", - "2", - "--skip-holdout", - "--seed", - "7", - ] - first = tmp_path / "first" - status = builder.main([*common, "--out", str(first), "--selection-pi-hi", "0.5"]) - assert status in (0, 1) - # The solve is no longer silent: probe verdicts and the search stop reach - # stderr as they happen, beside the phase lines. - err = capsys.readouterr().err - assert "probe 1/10 done:" in err and "search stopped:" in err - assert "dense solve: epoch 2/2" in err and "refit: epoch 2/2" in err - assert "size selection checkpoint written to" in err - assert (first / SIZE_CHECKPOINT_ARRAYS_FILENAME).is_file() - checkpoint = json.loads((first / SIZE_CHECKPOINT_MANIFEST_FILENAME).read_text()) - assert checkpoint["selection"]["households"] == 300 - assert checkpoint["selection"]["search_pi_hi"] == 0.5 - assert checkpoint["identity"]["dataset_households"] == 300 - assert checkpoint["identity"]["epochs"] == 2 - # The identity carries the solve doctrine; the provenance names the - # writing run (reported on resume, not compared). - assert checkpoint["identity"]["doctrine"] == builder._doctrine_bounds() - assert set(checkpoint["provenance"]) == {"code_pin", "build_id"} - manifest = json.loads((first / builder.MANIFEST_FILENAME).read_text()) - written = manifest["solve"]["dataset_size"]["checkpoint"]["written"] - assert "written_at" not in written and "directory" not in written - size_first = manifest["solve"]["dataset_size"] - assert 0 < size_first["certainty_share"] <= 1 - assert ( - size_first["boundary_draws"] - == 300 - (size_first["selection_receipt"]["certainty_count"]) - ) - assert size_first["zero_target_rows"] >= 0 - weights_block = manifest["weights"] - assert weights_block["stretch_reference"] == "normalized_horvitz_thompson_w_over_q" - assert weights_block["realized_max_weight_ratio_vs_stretch_reference"] > 0 - assert weights_block["realized_max_weight_ratio_vs_design"] > 0 - assert manifest["parameters"]["size_checkpoint"] is True - assert manifest["parameters"]["resume_size_checkpoint"] is None - written = manifest["solve"]["dataset_size"]["checkpoint"]["written"] - assert written["arrays_sha256"] == checkpoint["arrays_sha256"] - assert manifest["solve"]["dataset_size"]["selection_reused"] is False - rows = _spool_rows(first) - assert len(rows) == 1 - assert "size_selection_checkpointed" in rows[0].phases_reached - - # Resume on the same inputs: no dense solve, no search, same draw and refit. - second = tmp_path / "second" - status = builder.main( - [ - *common, - "--out", - str(second), - "--selection-pi-hi", - "0.5", - "--resume-size-checkpoint", - str(first), - ] - ) - assert status in (0, 1) - assert not (second / SIZE_CHECKPOINT_ARRAYS_FILENAME).exists() - resumed = json.loads((second / builder.MANIFEST_FILENAME).read_text()) - assert resumed["parameters"]["size_checkpoint"] is False - assert resumed["parameters"]["resume_size_checkpoint"] == str(first.resolve()) - size = resumed["solve"]["dataset_size"] - assert size["selection_reused"] is True - assert size["selection_search_pi_hi"] == 0.5 - assert ( - size["checkpoint"]["resumed_from"]["arrays_sha256"] - == (checkpoint["arrays_sha256"]) - ) - assert size["dense_loss"] == manifest["solve"]["dataset_size"]["dense_loss"] - assert ( - size["selection_l0_lambda"] - == (manifest["solve"]["dataset_size"]["selection_l0_lambda"]) - ) - first_selection = pd.read_csv(first / builder.DATASET_SIZE_SELECTION_FILENAME) - second_selection = pd.read_csv(second / builder.DATASET_SIZE_SELECTION_FILENAME) - pd.testing.assert_frame_equal(first_selection, second_selection) - assert "size_selection_resumed" in _spool_rows(second)[0].phases_reached - - # Another threshold re-draws from the same checkpoint and records both. - third = tmp_path / "third" - status = builder.main( - [ - *common, - "--out", - str(third), - "--selection-pi-hi", - "1.0", - "--resume-size-checkpoint", - str(first), - ] - ) - assert status in (0, 1) - redrawn = json.loads((third / builder.MANIFEST_FILENAME).read_text()) - assert redrawn["solve"]["dataset_size"]["selection_pi_hi"] == 1.0 - assert redrawn["solve"]["dataset_size"]["selection_search_pi_hi"] == 0.5 - - # A resume whose inputs differ refuses by name, before any solve. - different_epochs = list(common) - different_epochs[different_epochs.index("--epochs") + 1] = "3" - with pytest.raises(ValueError, match="epochs: checkpoint 2 != run 3"): - builder.main( - [ - *different_epochs, - "--out", - str(tmp_path / "fourth"), - "--resume-size-checkpoint", - str(first), - ] - ) - with pytest.raises(ValueError, match="requires --dataset-households"): - builder.main( - [ - "--input-h5", - str(input_h5), - "--ladder", - str(ladder_path), - "--out", - str(tmp_path / "fifth"), - "--resume-size-checkpoint", - str(first), - ] - ) - # An --out that already holds a checkpoint refuses before any solve - # (Vahid's should-fix 2): the checkpoint writer's own refusal came hours - # too late. - stale = tmp_path / "stale" - stale.mkdir() - (stale / SIZE_CHECKPOINT_ARRAYS_FILENAME).write_bytes(b"stale") - (stale / SIZE_CHECKPOINT_MANIFEST_FILENAME).write_text("{}") - with pytest.raises(FileExistsError, match="already holds a size checkpoint"): - builder.main([*common, "--out", str(stale)]) - assert not (stale / builder.MANIFEST_FILENAME).exists() - assert manifest["solve"]["dataset_size"]["checkpoint"]["written"]["stage"] == ( - "before_exact_count_draw" - ) - resumed_receipt = resumed["solve"]["dataset_size"]["checkpoint"]["resumed_from"] - assert ( - resumed_receipt["provenance"]["build_id"] - == checkpoint["provenance"]["build_id"] - ) - assert "written_at" not in resumed_receipt + with pytest.raises(SystemExit) as error: + _load_builder_module().main(_arguments(tmp_path) + [retired, "legacy"]) + assert error.value.code == 2 + assert "unrecognized arguments" in capsys.readouterr().err + assert not (tmp_path / "out").exists() diff --git a/packages/microcosm-build/tests/test_uk_rowwise_dry_run.py b/packages/microcosm-build/tests/test_uk_rowwise_dry_run.py index b1cb6c77b..b668caac8 100644 --- a/packages/microcosm-build/tests/test_uk_rowwise_dry_run.py +++ b/packages/microcosm-build/tests/test_uk_rowwise_dry_run.py @@ -1,39 +1,11 @@ -"""Dry-run plan and manifest weight-chain tests for the rowwise driver. - -microcosm#495 increment 2. The dry-run computes the clone plan — rows and -byte estimates at ``n_clones``, the input's weight-kind chain, both lineage -layers, and per-area support — without writing a dataset, so clone-count -adjudication happens before a multi-gigabyte build. Support ships twice, -honestly labelled: the *realized* assignment (the real sampler at the build -seed — identical draws to the real build, collision avoidance included) and -the analytic *collision-free* expectation, which can differ substantially -when ``n_clones`` is comparable to a group's sampleable constituency count. -""" +"""Analytical geography-support expectations retained independently of the CLI.""" from __future__ import annotations -import argparse -import importlib.util -import json -import sys -from pathlib import Path - import pandas as pd import pytest from microcosm.build.uk_runtime import expected_uk_rowwise_area_support -from microcosm.build.uk_runtime.national_frame import uk_national_frame -from microcosm.frame import MassChangeRecord, WeightKind - - -def _load_builder_module(): - root = Path(__file__).resolve().parents[3] - path = root / "tools" / "build_uk_rowwise_dataset.py" - spec = importlib.util.spec_from_file_location("build_uk_rowwise_dataset", path) - module = importlib.util.module_from_spec(spec) - assert spec.loader is not None - spec.loader.exec_module(module) - return module def _household_frame() -> pd.DataFrame: @@ -46,20 +18,6 @@ def _household_frame() -> pd.DataFrame: ) -def _person_frame() -> pd.DataFrame: - return pd.DataFrame( - { - "person_id": [1001, 2001, 2002], - "person_household_id": [1, 2, 2], - "person_benunit_id": [101, 201, 201], - } - ) - - -def _benunit_frame() -> pd.DataFrame: - return pd.DataFrame({"benunit_id": [101, 201]}) - - def _crosswalk_frame() -> pd.DataFrame: return pd.DataFrame( [ @@ -97,73 +55,6 @@ def _crosswalk_frame() -> pd.DataFrame: ) -def _write_toy_h5( - path: Path, - *, - household: pd.DataFrame | None = None, - fmt: str = "table", -) -> None: - with pd.HDFStore(path) as store: - store.put( - "household", - _household_frame() if household is None else household, - format=fmt, - data_columns=fmt == "table", - ) - store.put("person", _person_frame(), format=fmt, data_columns=fmt == "table") - store.put("benunit", _benunit_frame(), format=fmt, data_columns=fmt == "table") - store.put( - "time_period", - pd.Series(["2023"]), - format=fmt, - data_columns=fmt == "table", - ) - - -def _dry_run_argv( - input_h5: Path, - output_dir: Path, - crosswalk_path: Path, - *extra: str, -) -> list[str]: - return [ - "build_uk_rowwise_dataset.py", - "--input-h5", - str(input_h5), - "--out", - str(output_dir), - "--crosswalk", - str(crosswalk_path), - "--n-clones", - "2", - "--allow-missing-country", - "--dry-run", - *extra, - ] - - -def test_candidate_clone_counts_parser_deduplicates_and_sorts() -> None: - builder = _load_builder_module() - - assert builder._candidate_clone_counts_argument("10, 2,4,2,1") == ( - 1, - 2, - 4, - 10, - ) - - -@pytest.mark.parametrize( - "value", - ["", " ", ",", "0", "-1", "1,0", "1,-2", "1,junk", "1,,2"], -) -def test_candidate_clone_counts_parser_refuses_invalid_values(value) -> None: - builder = _load_builder_module() - - with pytest.raises(argparse.ArgumentTypeError, match="positive integers"): - builder._candidate_clone_counts_argument(value) - - def test_collision_free_expectation_matches_distribution_math() -> None: support = expected_uk_rowwise_area_support( _household_frame(), @@ -210,347 +101,3 @@ def test_collision_free_expectation_requires_covered_countries() -> None: "E06000063", "E06000064", } - - -def test_driver_dry_run_writes_plan_only(monkeypatch, tmp_path) -> None: - pytest.importorskip("tables") - pytest.importorskip("h5py") - builder = _load_builder_module() - input_h5 = tmp_path / "populace_uk_2023.h5" - crosswalk_path = tmp_path / "crosswalk.csv.gz" - output_dir = tmp_path / "out" - _write_toy_h5(input_h5) - _crosswalk_frame().to_csv(crosswalk_path, index=False) - - monkeypatch.setattr( - sys, "argv", _dry_run_argv(input_h5, output_dir, crosswalk_path) - ) - assert builder.main() == 0 - - plan_path = output_dir / builder.DRY_RUN_PLAN_FILENAME - assert plan_path.exists() - assert not (output_dir / "populace_uk_2023_rowwise.h5").exists() - assert not (output_dir / builder.MANIFEST_FILENAME).exists() - assert not (output_dir / "logbook-spool").exists() - - plan = json.loads(plan_path.read_text()) - assert plan["build_kind"] == "uk_rowwise_local_geography_dry_run" - assert plan["parameters"]["n_clones"] == 2 - assert plan["input"]["dataset"]["sha256"] - assert plan["input"]["dataset"]["pin_verified"] is False - assert plan["input"]["household_weight_kind"] == WeightKind.DESIGN.value - assert plan["input"]["mass_log_records"] == 0 - assert plan["plan"]["rows"] == { - "person": 6, - "benunit": 4, - "household": 4, - } - assert plan["plan"]["output_bytes_estimate"] >= input_h5.stat().st_size - assert "lower-bound" in plan["plan"]["output_bytes_estimate_basis"] - - # Realized support: collision avoidance forces the London household's two - # clones into distinct England constituencies, and the single Welsh - # constituency absorbs both Welsh clones. - realized = plan["realized_support"] - assert "realized assignment at seed 42" in realized["basis"] - constituency = realized["constituency"] - assert constituency["n_areas"] == 3 - assert constituency["min_rows"] == pytest.approx(1.0) - by_code = {row["area_code"]: row["rows"] for row in constituency["bottom"]} - assert by_code["E14000001"] == pytest.approx(1.0) - assert by_code["E14000002"] == pytest.approx(1.0) - assert by_code["W07000041"] == pytest.approx(2.0) - assert realized["la"]["n_areas"] == 3 - assert realized["la"]["min_rows"] == pytest.approx(1.0) - - collision_free = plan["collision_free_expected_support"] - assert collision_free["constituency"]["min_rows"] == pytest.approx(0.5) - - assert plan["source_lineage"]["pool_modulus"] is None - assert plan["source_lineage"]["pool"] is None - assert plan["source_lineage"]["explicit"] is None - - -def test_crosswalk_dry_run_reports_candidate_k_rows_bytes_and_support( - monkeypatch, tmp_path -) -> None: - pytest.importorskip("tables") - pytest.importorskip("h5py") - builder = _load_builder_module() - input_h5 = tmp_path / "populace_uk_2023.h5" - crosswalk_path = tmp_path / "crosswalk.csv.gz" - output_dir = tmp_path / "out" - _write_toy_h5(input_h5) - _crosswalk_frame().to_csv(crosswalk_path, index=False) - - monkeypatch.setattr( - sys, - "argv", - _dry_run_argv( - input_h5, - output_dir, - crosswalk_path, - "--candidate-clone-counts", - "3,1,3", - ), - ) - assert builder.main() == 0 - plan = json.loads((output_dir / builder.DRY_RUN_PLAN_FILENAME).read_text()) - - assert plan["parameters"]["n_clones"] == 2 - assert plan["parameters"]["candidate_clone_counts"] == [1, 3] - candidates = plan["candidates"] - assert candidates["clone_counts"] == [1, 3] - assert [candidate["n_clones"] for candidate in candidates["plans"]] == [1, 3] - input_bytes = input_h5.stat().st_size - base_rows = {"person": 3, "benunit": 2, "household": 2} - for candidate in candidates["plans"]: - n_clones = candidate["n_clones"] - assert candidate["rows"] == { - name: rows * n_clones for name, rows in base_rows.items() - } - assert candidate["output_bytes_estimate"] == input_bytes * n_clones - assert set(candidate["realized_support"]) == {"constituency", "la"} - assert set(candidate["collision_free_expected_support"]) == { - "constituency", - "la", - } - assert "area_support" not in candidate - - -def test_driver_dry_run_reports_weight_chain_and_pool_lineage( - monkeypatch, tmp_path -) -> None: - pytest.importorskip("tables") - pytest.importorskip("h5py") - from microcosm.build.uk_runtime import write_uk_national_frame - - builder = _load_builder_module() - staging = tmp_path / "staging.h5" - household = pd.DataFrame( - { - "household_id": [101, 102, 100000101, 100000102], - "household_weight": [10.0, 20.0, 10.0, 20.0], - "region": ["LONDON", "WALES", "LONDON", "WALES"], - "source_household_id": [901, 902, 903, 904], - } - ) - person = pd.DataFrame( - { - "person_id": [1, 2, 3, 4], - "person_household_id": [101, 102, 100000101, 100000102], - "person_benunit_id": [11, 12, 13, 14], - } - ) - benunit = pd.DataFrame({"benunit_id": [11, 12, 13, 14]}) - dataset = uk_national_frame( - person=person, - benunit=benunit, - household=household, - time_period="2023", - weight_kind=WeightKind.IMPORTANCE, - mass_log=( - MassChangeRecord( - entity="household", - old_total=60.0, - new_total=60.0, - declared_factor=1.0, - reason="Toy reviewed SPI-channel allocation record.", - ), - ), - ) - write_uk_national_frame(dataset, staging) - crosswalk_path = tmp_path / "crosswalk.csv.gz" - _crosswalk_frame().to_csv(crosswalk_path, index=False) - output_dir = tmp_path / "out" - - monkeypatch.setattr( - sys, - "argv", - _dry_run_argv( - staging, - output_dir, - crosswalk_path, - "--allow-constituency-collisions", - "--source-lineage-modulus", - "100000000", - ), - ) - assert builder.main() == 0 - plan = json.loads((output_dir / builder.DRY_RUN_PLAN_FILENAME).read_text()) - assert plan["input"]["household_weight_kind"] == WeightKind.IMPORTANCE.value - assert plan["input"]["mass_log_records"] == 1 - - lineage = plan["source_lineage"] - assert lineage["pool_modulus"] == 100000000 - assert lineage["pool"]["distinct_pool_source_households"] == 2 - assert lineage["pool"]["pool_copies_per_source"]["min"] == 2 - assert lineage["pool"]["pool_copies_per_source"]["max"] == 2 - # The staging input's immediate layer is reported untouched alongside. - assert lineage["immediate"]["distinct_source_households"] == 4 - assert lineage["explicit"] is None - - -def test_driver_dry_run_supports_fixed_format_stores(monkeypatch, tmp_path) -> None: - pytest.importorskip("tables") - pytest.importorskip("h5py") - builder = _load_builder_module() - input_h5 = tmp_path / "fixed.h5" - _write_toy_h5(input_h5, fmt="fixed") - crosswalk_path = tmp_path / "crosswalk.csv.gz" - _crosswalk_frame().to_csv(crosswalk_path, index=False) - output_dir = tmp_path / "out" - - monkeypatch.setattr( - sys, "argv", _dry_run_argv(input_h5, output_dir, crosswalk_path) - ) - assert builder.main() == 0 - plan = json.loads((output_dir / builder.DRY_RUN_PLAN_FILENAME).read_text()) - assert plan["plan"]["rows"]["person"] == 6 - - -def test_driver_dry_run_preserves_generated_crosswalk_cache( - monkeypatch, tmp_path -) -> None: - pytest.importorskip("tables") - pytest.importorskip("h5py") - builder = _load_builder_module() - input_h5 = tmp_path / "populace_uk_2023.h5" - _write_toy_h5(input_h5) - crosswalk_path = tmp_path / "crosswalk.csv.gz" - _crosswalk_frame().to_csv(crosswalk_path, index=False) - output_dir = tmp_path / "out" - output_dir.mkdir() - cache = output_dir / builder.CROSSWALK_FILENAME - cache.write_text("previously generated cache") - - monkeypatch.setattr( - sys, "argv", _dry_run_argv(input_h5, output_dir, crosswalk_path) - ) - assert builder.main() == 0 - assert cache.read_text() == "previously generated cache" - - -def test_driver_dry_run_rejects_broken_links(monkeypatch, tmp_path) -> None: - pytest.importorskip("tables") - pytest.importorskip("h5py") - builder = _load_builder_module() - input_h5 = tmp_path / "broken.h5" - person = _person_frame() - person.loc[0, "person_household_id"] = 999 - with pd.HDFStore(input_h5) as store: - store.put("household", _household_frame(), format="table", data_columns=True) - store.put("person", person, format="table", data_columns=True) - store.put("benunit", _benunit_frame(), format="table", data_columns=True) - store.put("time_period", pd.Series(["2023"]), format="table", data_columns=True) - crosswalk_path = tmp_path / "crosswalk.csv.gz" - _crosswalk_frame().to_csv(crosswalk_path, index=False) - output_dir = tmp_path / "out" - - monkeypatch.setattr( - sys, "argv", _dry_run_argv(input_h5, output_dir, crosswalk_path) - ) - with pytest.raises(ValueError, match="person_household_id"): - builder.main() - - -def test_driver_full_build_records_weight_chain_and_lineage( - monkeypatch, tmp_path -) -> None: - pytest.importorskip("tables") - pytest.importorskip("h5py") - builder = _load_builder_module() - input_h5 = tmp_path / "pool.h5" - with pd.HDFStore(input_h5) as store: - store.put( - "household", - pd.DataFrame( - { - "household_id": [101, 102, 100000101, 100000102], - "household_weight": [10.0, 20.0, 10.0, 20.0], - "region": ["LONDON", "WALES", "LONDON", "WALES"], - } - ), - format="table", - data_columns=True, - ) - store.put( - "person", - pd.DataFrame( - { - "person_id": [1, 2, 3, 4], - "person_household_id": [101, 102, 100000101, 100000102], - "person_benunit_id": [11, 12, 13, 14], - } - ), - format="table", - data_columns=True, - ) - store.put( - "benunit", - pd.DataFrame({"benunit_id": [11, 12, 13, 14]}), - format="table", - data_columns=True, - ) - store.put( - "time_period", - pd.Series(["2023"]), - format="table", - data_columns=True, - ) - crosswalk_path = tmp_path / "crosswalk.csv.gz" - _crosswalk_frame().to_csv(crosswalk_path, index=False) - output_dir = tmp_path / "out" - - monkeypatch.setattr( - sys, - "argv", - [ - "build_uk_rowwise_dataset.py", - "--input-h5", - str(input_h5), - "--out", - str(output_dir), - "--crosswalk", - str(crosswalk_path), - "--n-clones", - "2", - "--allow-missing-country", - "--allow-constituency-collisions", - "--source-lineage-modulus", - "100000000", - ], - ) - assert builder.main() == 0 - manifest = json.loads((output_dir / builder.MANIFEST_FILENAME).read_text()) - - weights = manifest["rowwise_dataset"]["weights"] - assert weights["household_weight_kind"] == WeightKind.DESIGN.value - assert weights["mass_log_records"] == 1 - conservation = weights["mass_conservation"] - assert conservation["passed"] is True - assert conservation["input_total"] == pytest.approx(60.0) - assert conservation["output_total"] == pytest.approx(60.0) - assert conservation["relative_tolerance"] > 0 - - lineage = manifest["rowwise_dataset"]["source_lineage"] - assert lineage["pool_modulus"] == 100000000 - assert lineage["pool"]["distinct_pool_source_households"] == 2 - assert lineage["pool"]["pool_copies_per_source"]["min"] == 2 - assert lineage["pool"]["pool_copies_per_source"]["max"] == 2 - assert lineage["immediate"] is None - assert lineage["explicit"] is None - assert manifest["base_dataset"]["distinct_source_households"] is None - - output_h5 = output_dir / "pool_rowwise.h5" - import h5py - - from microcosm.build.uk_runtime.national_frame import ( - UK_HOUSEHOLD_WEIGHT_KIND_ATTR, - ) - - with h5py.File(output_h5, mode="r") as file: - stored = file.attrs[UK_HOUSEHOLD_WEIGHT_KIND_ATTR] - if isinstance(stored, bytes): - stored = stored.decode("utf-8") - assert stored == WeightKind.DESIGN.value diff --git a/packages/microcosm-build/tests/test_uk_source_runtime.py b/packages/microcosm-build/tests/test_uk_source_runtime.py index 9aa93d945..2f0df6f4e 100644 --- a/packages/microcosm-build/tests/test_uk_source_runtime.py +++ b/packages/microcosm-build/tests/test_uk_source_runtime.py @@ -112,8 +112,6 @@ def hmrc(frame: Frame) -> Frame: return frame assert uk_stage_implementations( - retained_leaves_transform=retained, - hmrc_income_transform=hmrc, was_wealth_transform=retained, uc_deduction_attributes_transform=hmrc, regional_property_uprating_transform=hmrc, @@ -126,8 +124,6 @@ def hmrc(frame: Frame) -> Frame: salary_sacrifice_transform=hmrc, student_loans_transform=retained, ) == { - "frs_hmrc_retained_leaves": retained, - "hmrc_spi_income": hmrc, "was_wealth": retained, "uc_deduction_attributes": hmrc, "regional_property_uprating": hmrc, @@ -231,3 +227,16 @@ def test_materialize_rules_engine_predictors_refuses_an_unknown_year_rule() -> N with pytest.raises(SourceRuntimeError, match="Unknown UK year_rule"): handler(None, operation, _context(engine=_PeriodRecordingEngine())) + + +def test_canonical_source_roster_has_no_candidate_migration_stages() -> None: + from microcosm.build.country_spec import load_country_spec + + stages = load_country_spec("uk").sources.stage_map() + assert "frs_hmrc_retained_leaves" not in stages + assert "hmrc_spi_income" not in stages + assert { + "frs_hmrc_spine_leaves", + "spi_support_channel", + "hmrc_spi_income_spine", + } <= stages.keys() diff --git a/packages/microcosm-build/tests/test_uk_source_stages.py b/packages/microcosm-build/tests/test_uk_source_stages.py index 56786b7fe..bcd4a7b94 100644 --- a/packages/microcosm-build/tests/test_uk_source_stages.py +++ b/packages/microcosm-build/tests/test_uk_source_stages.py @@ -1,7 +1,5 @@ from __future__ import annotations -import copy -import hashlib import json from pathlib import Path @@ -12,13 +10,12 @@ FORBIDDEN_SOURCE_DEPENDENCIES, SourceManifest, ) -from microcosm.build.uk_runtime.graph import UK_SPINE_EXCLUSIONS, uk_spine_graph +from microcosm.build.uk_runtime.graph import uk_spine_graph from microcosm.frame import Frame from microcosm.graph import compile_graph ROOT = Path(__file__).resolve().parents[3] UK_PACKAGE = ROOT / "packages/microcosm-build/src/microcosm/build/uk" -FROZEN_SOURCE_STAGES = UK_PACKAGE / "hmrc_income_source_stages.json" CANONICAL_SOURCE_STAGES = UK_PACKAGE / "source_stages.json" E3_STAGE_NAMES = [ "frs_employment", @@ -79,12 +76,7 @@ *UC_COHERENCE_STAGE_NAMES, *E9_STAGE_NAMES, *E8_STAGE_NAMES, - "frs_hmrc_retained_leaves", - "hmrc_spi_income", ] -FROZEN_SOURCE_STAGES_SHA256 = ( - "c0341af7166ae3a85a3c1164e7d9e880c4b4aec122f1a8fa90c73b46c596e1ea" -) def _load_json(path: Path) -> dict: @@ -96,11 +88,7 @@ def _identity(frame: Frame) -> Frame: def _uk_graph_stage_names(spec) -> list[str]: - manifest_stages = { - stage.stage - for stage in spec.sources.stages - if stage.stage not in UK_SPINE_EXCLUSIONS - } + manifest_stages = {stage.stage for stage in spec.sources.stages} return [ node_id for node_id in compile_graph(uk_spine_graph(spec)).order @@ -114,20 +102,6 @@ def _assert_no_forbidden_dependency(value: object) -> None: assert dependency not in text -def _expected_reviewed_source() -> str: - return ( - "PolicyEngine licensed UKDS mirror (private Hugging Face repository), " - "spi_2022_23.zip" - ) - - -def _rephrase_stage2_predictor_note(value: str) -> str: - return value.replace( - "policyengine-" + "uk-data frs_only.py", - "the incumbent UK data build's frs_only.py", - ) - - class TestUKSourceStagesManifest: def test_source_stages_json_loads_as_shared_manifest(self) -> None: manifest = SourceManifest.from_mapping(_load_json(CANONICAL_SOURCE_STAGES)) @@ -170,7 +144,7 @@ def test_e7_block_sits_between_e6_and_e8(self) -> None: def test_age_tail_runs_immediately_after_frs_spine(self) -> None: canonical = _load_json(CANONICAL_SOURCE_STAGES) - spine = [stage["stage"] for stage in canonical["stages"][:-2]] + spine = [stage["stage"] for stage in canonical["stages"]] assert spine[1] == "age_tail" @@ -185,97 +159,13 @@ def test_age_tail_position_owns_the_only_later_age_rewrite_guard(self) -> None: assert "age" not in stage.outputs, stage.stage assert "age" not in stage.rewrites, stage.stage - def test_e8_block_is_final_and_the_certified_pair_stays_last(self) -> None: - # The E8 stages stay contiguous at the end of the spine, while the - # certified pair stays at [-2:] (the frozen-copy lockstep test reads - # them from there). age_tail is now the post-frs_spine block, before - # every stage that conditions on age. + def test_e8_block_is_final_and_all_stages_are_canonical(self) -> None: canonical = _load_json(CANONICAL_SOURCE_STAGES) names = [stage["stage"] for stage in canonical["stages"]] - - assert names[-2:] == ["frs_hmrc_retained_leaves", "hmrc_spi_income"] - spine = names[:-2] - start = spine.index(E8_STAGE_NAMES[0]) - assert spine[start : start + len(E8_STAGE_NAMES)] == E8_STAGE_NAMES - assert spine[start + len(E8_STAGE_NAMES) :] == [] - - def test_copy_is_lockstep_with_frozen_original_except_citation_rewrites( - self, - ) -> None: - frozen = _load_json(FROZEN_SOURCE_STAGES) - canonical = _load_json(CANONICAL_SOURCE_STAGES) - frozen_stage = frozen["stages"][0] - stage1, stage2 = canonical["stages"][-2:] - - expected_operations = copy.deepcopy(frozen_stage["operations"]) - predictor_note = expected_operations[6]["reviewed_absent_predictors"][ - "other_investment_income" - ] - expected_operations[6]["reviewed_absent_predictors"][ - "other_investment_income" - ] = _rephrase_stage2_predictor_note(predictor_note) - # FRS retained leaves now come from the FRS 2024-25 spine while the - # frozen HMRC fact surface stays byte-pinned. - expected_operations[1]["source_vintage"] = "2024-25" - expected_operations[1]["mapped_build_period"] = 2024 - # Signed period re-map (#723) for materialized HMRC SPI facts. - expected_operations[7]["mapped_build_period"] = 2024 - expected_operations[7]["period_mapping"] = "latest_published_tax_year" - - assert stage1["operations"] + stage2["operations"] == expected_operations - _assert_no_forbidden_dependency( - stage2["operations"][4]["reviewed_absent_predictors"][ - "other_investment_income" - ] - ) - - expected_artifacts = copy.deepcopy(frozen_stage["artifacts"]) - expected_artifacts[0]["reviewed_source"] = _expected_reviewed_source() - # Signed period re-map (#723): the ODS source surface remains the - # frozen 2023-24 file, but the canonical manifest declares that it is - # replayed against build period 2024. - expected_artifacts[1]["mapped_build_period"] = 2024 - expected_artifacts[1]["period_mapping"] = "latest_published_tax_year" - # Declared output-name correction (licensed-data acceptance finding): - # the frozen original listed the SPI concept "state_pension", but the - # stage writes the auxiliary column SPI_HMRC_STATE_PENSION_INCOME_COLUMN - # ("hmrc_spi_state_pension_income") — the model input state_pension is - # formula-owned and never a frame column here. Outputs became - # load-bearing when country_stage_plan compiled them into - # StagePlan.produces, so the copy declares the persisted truth. The - # operation payloads keep the concept name unchanged. - expected_outputs = [ - "hmrc_spi_state_pension_income" if name == "state_pension" else name - for name in frozen_stage["outputs"] - ] - assert stage2["outputs"] == expected_outputs - assert stage2["grain"] == frozen_stage["grain"] - assert stage2["artifacts"] == expected_artifacts - _assert_no_forbidden_dependency(stage2["artifacts"]) - _assert_no_forbidden_dependency(stage2["notes"]) - - def test_frozen_original_bytes_are_pinned(self) -> None: - digest = hashlib.sha256(FROZEN_SOURCE_STAGES.read_bytes()).hexdigest() - - assert digest == FROZEN_SOURCE_STAGES_SHA256 - - def test_country_stage_plan_assembles_two_certified_uk_national_stages( - self, - ) -> None: - spec = load_country_spec("uk") - plan = country_stage_plan( - spec, - { - "frs_hmrc_retained_leaves": _identity, - "hmrc_spi_income": _identity, - }, - stage_names=("frs_hmrc_retained_leaves", "hmrc_spi_income"), - ) - - assert [stage.name for stage in plan.stages] == [ - "frs_hmrc_retained_leaves", - "hmrc_spi_income", - ] + start = names.index(E8_STAGE_NAMES[0]) + assert names[start:] == E8_STAGE_NAMES + assert "frs_hmrc_retained_leaves" not in names + assert "hmrc_spi_income" not in names def test_country_stage_plan_assembles_spine_plan(self) -> None: spec = load_country_spec("uk") @@ -292,7 +182,7 @@ def test_country_stage_plan_assembles_spine_plan(self) -> None: @pytest.mark.parametrize( "implementations, match", [ - ({"frs_hmrc_retained_leaves": _identity}, "missing"), + ({"frs_spine": _identity}, "missing"), ( { "frs_spine": _identity, @@ -323,8 +213,6 @@ def test_country_stage_plan_assembles_spine_plan(self) -> None: "salary_sacrifice": _identity, "student_loans": _identity, "age_tail": _identity, - "frs_hmrc_retained_leaves": _identity, - "hmrc_spi_income": _identity, "hmrc_spi_income_fallback": _identity, }, "Unknown stage implementation", @@ -355,14 +243,17 @@ class TestDeclaredOutputsAreWrittenColumns: """ def test_stage1_outputs_are_exactly_the_retained_leaf_columns(self) -> None: - from microcosm.build.uk_runtime.frs_hmrc_leaves import ( + from microcosm.build.uk_runtime.frs_hmrc_source import ( FRS_HMRC_RETAINED_LEAF_COLUMNS, ) spec = load_country_spec("uk") stages = {stage.stage: stage for stage in spec.sources.stages} - stage1 = stages["frs_hmrc_retained_leaves"] - assert stage1.outputs == tuple(FRS_HMRC_RETAINED_LEAF_COLUMNS) + stage1 = stages["frs_hmrc_spine_leaves"] + assert stage1.outputs == ( + *FRS_HMRC_RETAINED_LEAF_COLUMNS, + "employer_pension_contributions", + ) def test_e3_outputs_are_backed_by_runtime_written_columns(self) -> None: from microcosm.build.uk_runtime.etb_services import ( @@ -1007,7 +898,7 @@ def test_stage2_outputs_are_backed_by_runtime_written_columns(self) -> None: spec = load_country_spec("uk") stages = {stage.stage: stage for stage in spec.sources.stages} - stage2 = stages["hmrc_spi_income"] + stage2 = stages["hmrc_spi_income_spine"] written = ( set(SPI_INCOME_IMPUTATION_COLUMNS) | set(SPI_HMRC_QRF_AUXILIARY_COLUMNS) diff --git a/packages/microcosm-build/tests/test_uk_spi_income.py b/packages/microcosm-build/tests/test_uk_spi_income.py index 14d2b7314..6e6fee1a0 100644 --- a/packages/microcosm-build/tests/test_uk_spi_income.py +++ b/packages/microcosm-build/tests/test_uk_spi_income.py @@ -10,7 +10,7 @@ import pytest from microcosm.build.uk_runtime import frs_disability, spi_income -from microcosm.build.uk_runtime.frs_hmrc_leaves import ( +from microcosm.build.uk_runtime.frs_hmrc_source import ( FRS_HMRC_OSSBEN_IDENTIFIABLE_SUBSET_COLUMN, FRS_HMRC_RETAINED_LEAF_COLUMNS, FRS_HMRC_SRP_REGULAR_CODE5_COLUMN, diff --git a/packages/microcosm-build/tests/test_uk_spine_acceptance_receipt.py b/packages/microcosm-build/tests/test_uk_spine_acceptance_receipt.py index 2c57fbb70..c1f336989 100644 --- a/packages/microcosm-build/tests/test_uk_spine_acceptance_receipt.py +++ b/packages/microcosm-build/tests/test_uk_spine_acceptance_receipt.py @@ -14,10 +14,7 @@ from importlib.resources import files from microcosm.build.country_spec import load_country_spec -from microcosm.build.uk_runtime.graph import ( - UK_SPINE_EXCLUSIONS, - uk_spine_graph, -) +from microcosm.build.uk_runtime.graph import uk_spine_graph from microcosm.graph import compile_graph @@ -32,11 +29,8 @@ def _receipt() -> dict: def _production_graph_stage_names() -> tuple[str, ...]: spec = load_country_spec("uk") assert spec.sources is not None - declared = { - stage.stage - for stage in spec.sources.stages - if stage.stage not in UK_SPINE_EXCLUSIONS - } + declared = {stage.stage for stage in spec.sources.stages} + assert declared.isdisjoint({"frs_hmrc_retained_leaves", "hmrc_spi_income"}) compiled = compile_graph(uk_spine_graph(spec)) return tuple(node_id for node_id in compiled.order if node_id in declared) diff --git a/packages/microcosm-build/tests/test_us_plan.py b/packages/microcosm-build/tests/test_us_plan.py index e9688c654..5cecd6e3c 100644 --- a/packages/microcosm-build/tests/test_us_plan.py +++ b/packages/microcosm-build/tests/test_us_plan.py @@ -984,7 +984,6 @@ def test_no_incumbent_data_package_references_in_live_tree(self) -> None: # never import or execute the retired data package. "packages/microcosm-build/src/microcosm/build/uk/efrs_parity_reference.json", "packages/microcosm-build/src/microcosm/build/uk/frs_release.json", - "packages/microcosm-build/src/microcosm/build/uk/hmrc_income_source_stages.json", # The UK population contract's registry-parity accounting names the # retired data package by necessity: 651 rows at pinned ref ebf733c # = 609 mapped + 42 signed exclusions + 3 unmapped declarations. diff --git a/packages/microcosm-graph/tests/fixtures/parity/uk_spine/uk_spine.json b/packages/microcosm-graph/tests/fixtures/parity/uk_spine/uk_spine.json index 1a3698802..bd6d14ecd 100644 --- a/packages/microcosm-graph/tests/fixtures/parity/uk_spine/uk_spine.json +++ b/packages/microcosm-graph/tests/fixtures/parity/uk_spine/uk_spine.json @@ -1 +1 @@ -{"country":"uk","nodes":[{"base":null,"citation":"","description":"Load the source-bound UK FRS root population.","id":"create_uk_frs","inputs":[],"kernel":"uk.create@1","mass":"conserve","outputs":[{"column":"age","dtype":"int64","entity":"person","ownership":"produced","rows":"all"},{"column":"gender","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"marital_status","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"hours_worked","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"is_household_head","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"is_benunit_head","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"is_parent","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"is_uc_claimant","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"employment_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"self_employment_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"private_pension_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"tax_free_savings_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"savings_interest_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"dividend_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"property_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"maintenance_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"miscellaneous_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"private_transfer_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"lump_sum_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"student_loan_repayments","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"statutory_sick_pay","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"statutory_maternity_pay","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"student_loans","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"access_fund","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"education_grants","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"healthy_start_vouchers","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"free_school_breakfasts","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"free_school_fruit_veg","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"free_school_meals","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"council_tax_benefit_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"maintenance_expenses","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"childcare_expenses","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"personal_pension_contributions","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"employee_pension_contributions","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"pension_contributions_via_salary_sacrifice","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"salary_sacrifice_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"salary_sacrifice_asked","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"child_benefit_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"income_support_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"housing_benefit_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"attendance_allowance_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"dla_sc_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"dla_m_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"iidb_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"carers_allowance_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"sda_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"afcs_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"ssmg_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"pension_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"child_tax_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"working_tax_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"state_pension_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"winter_fuel_allowance_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"incapacity_benefit_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"universal_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"pip_m_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"pip_dl_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"jsa_contrib_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"jsa_income_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"esa_contrib_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"esa_income_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"bsp_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"frs_benunit_capital","dtype":"float64","entity":"benunit","ownership":"produced","rows":"all"},{"column":"is_married","dtype":"bool","entity":"benunit","ownership":"produced","rows":"all"},{"column":"dependent_children","dtype":"int64","entity":"benunit","ownership":"produced","rows":"all"},{"column":"region","dtype":"string","entity":"household","ownership":"produced","rows":"all"},{"column":"tenure_type","dtype":"string","entity":"household","ownership":"produced","rows":"all"},{"column":"accommodation_type","dtype":"string","entity":"household","ownership":"produced","rows":"all"},{"column":"num_bedrooms","dtype":"int64","entity":"household","ownership":"produced","rows":"all"},{"column":"council_tax_reported","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"council_tax_band","dtype":"string","entity":"household","ownership":"produced","rows":"all"},{"column":"council_tax_rebate","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"council_tax_single_adult_raw","dtype":"int64","entity":"household","ownership":"produced","rows":"all"},{"column":"water_and_sewerage_charges","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"domestic_rates","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"rent","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"subrent","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"mortgage_interest_repayment","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"mortgage_capital_repayment","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"structural_insurance_payments","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"housing_service_charges","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"external_child_payments","dtype":"float64","entity":"household","ownership":"produced","rows":"all"}],"params":{"sample_fraction":1.0,"sample_seed":578,"stage_contract_sha256":"7ed918cd9dfa27874110bececf0c2513e7348df10b79ed55ea432e0e9ed70f3f","time_period":"2024"},"population":null,"sources":["frs"],"structural":"create","weights":null},{"base":"create_uk_frs","citation":"","description":"Ownership boundary for the source-assembling root stage.","id":"frs_spine.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments"],"entity":"household","rows":"all"},{"columns":["age","gender","marital_status","hours_worked","is_household_head","is_benunit_head","is_parent","is_uc_claimant","employment_income","self_employment_income","private_pension_income","tax_free_savings_income","savings_interest_income","dividend_income","property_income","maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","bsp_reported"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"base":null,"citation":"","description":"Claim the cells assembled by the UK FRS root transform.","id":"frs_spine","inputs":[],"kernel":"uk.claim@1","mass":"conserve","outputs":[{"column":"age","dtype":"int64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"gender","dtype":"string","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"marital_status","dtype":"string","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"hours_worked","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_household_head","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_benunit_head","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_parent","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_uc_claimant","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"employment_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"self_employment_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"private_pension_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"tax_free_savings_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"savings_interest_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"dividend_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"property_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"maintenance_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"miscellaneous_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"private_transfer_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"lump_sum_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"student_loan_repayments","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"statutory_sick_pay","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"statutory_maternity_pay","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"student_loans","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"access_fund","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"education_grants","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"healthy_start_vouchers","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"free_school_breakfasts","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"free_school_fruit_veg","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"free_school_meals","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"council_tax_benefit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"maintenance_expenses","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"childcare_expenses","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"personal_pension_contributions","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"employee_pension_contributions","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"pension_contributions_via_salary_sacrifice","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"salary_sacrifice_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"salary_sacrifice_asked","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"child_benefit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"income_support_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"housing_benefit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"attendance_allowance_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"dla_sc_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"dla_m_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"iidb_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"carers_allowance_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"sda_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"afcs_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"ssmg_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"pension_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"child_tax_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"working_tax_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"state_pension_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"winter_fuel_allowance_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"incapacity_benefit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"universal_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"pip_m_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"pip_dl_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"jsa_contrib_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"jsa_income_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"esa_contrib_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"esa_income_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"bsp_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"frs_benunit_capital","dtype":"float64","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_married","dtype":"bool","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"},{"column":"dependent_children","dtype":"int64","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"},{"column":"region","dtype":"string","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"tenure_type","dtype":"string","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"accommodation_type","dtype":"string","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"num_bedrooms","dtype":"int64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"council_tax_reported","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"council_tax_band","dtype":"string","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"council_tax_rebate","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"council_tax_single_adult_raw","dtype":"int64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"water_and_sewerage_charges","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"domestic_rates","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"rent","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"subrent","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"mortgage_interest_repayment","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"mortgage_capital_repayment","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"structural_insurance_payments","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"housing_service_charges","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"external_child_payments","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"}],"params":{},"population":"frs_spine.boundary","sources":[],"structural":"none","weights":null},{"base":"frs_spine.boundary","citation":"","description":"Ownership boundary before age_tail rewrites.","id":"age_tail.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments"],"entity":"household","rows":"all"},{"columns":["age","gender","marital_status","hours_worked","is_household_head","is_benunit_head","is_parent","is_uc_claimant","employment_income","self_employment_income","private_pension_income","tax_free_savings_income","savings_interest_income","dividend_income","property_income","maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","bsp_reported"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"base":null,"citation":"","description":"Run UK spine stage age_tail.","id":"age_tail","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["external_child_payments","region"],"entity":"household","rows":"all"},{"columns":["gender"],"entity":"person","rows":"all"}],"kernel":"uk.stage.age_tail@1","mass":"conserve","outputs":[{"column":"age","dtype":"int64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"stage":"age_tail","stage_contract_sha256":"963a34f38caede2525e3b8735f904bb70a1bbef5a272f0eefd4346343444484f","time_period":"2024"},"population":"age_tail.boundary","sources":["frs"],"structural":"none","weights":null},{"base":null,"citation":"","description":"Run UK spine stage frs_employment.","id":"frs_employment","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["region"],"entity":"household","rows":"all"},{"columns":["age"],"entity":"person","rows":"all"}],"kernel":"uk.stage.frs_employment@1","mass":"conserve","outputs":[{"column":"employment_status","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"employment_sector","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"sic_industry_division","dtype":"int64","entity":"person","ownership":"produced","rows":"all"}],"params":{"stage":"frs_employment","stage_contract_sha256":"ddbefaf05b44788d794a6e4b0e8926c14318d66e50d2f11f50ca549538bbf60c","time_period":"2024"},"population":"age_tail.boundary","sources":["frs"],"structural":"none","weights":null},{"base":null,"citation":"","description":"Run UK spine stage frs_council_tax.","id":"frs_council_tax","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["region"],"entity":"household","rows":"all"},{"columns":["age","sic_industry_division"],"entity":"person","rows":"all"}],"kernel":"uk.stage.frs_council_tax@1","mass":"conserve","outputs":[{"column":"council_tax","dtype":"float64","entity":"household","ownership":"produced","rows":"all"}],"params":{"stage":"frs_council_tax","stage_contract_sha256":"3381cd9f7a736514d5073c57480e07f5098890f3ca9ed3865392043594771eb9","time_period":"2024"},"population":"age_tail.boundary","sources":["frs"],"structural":"none","weights":null},{"base":null,"citation":"","description":"Run UK spine stage frs_disability.","id":"frs_disability","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["council_tax","region"],"entity":"household","rows":"all"},{"columns":["afcs_reported","age","attendance_allowance_reported","dla_m_reported","dla_sc_reported","esa_contrib_reported","esa_income_reported","iidb_reported","incapacity_benefit_reported","pip_dl_reported","pip_m_reported","sda_reported"],"entity":"person","rows":"all"}],"kernel":"uk.stage.frs_disability@1","mass":"conserve","outputs":[{"column":"aa_category","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"dla_sc_category","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"dla_m_category","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"pip_m_category","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"pip_dl_category","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"is_disabled_for_benefits","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"is_enhanced_disabled_for_benefits","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"is_severely_disabled_for_benefits","dtype":"bool","entity":"person","ownership":"produced","rows":"all"}],"params":{"stage":"frs_disability","stage_contract_sha256":"ac500b4cb1bc04d4fb72d198592dd376d61da8c150921a6ac53914ac18be0508","time_period":"2024"},"population":"age_tail.boundary","sources":["frs"],"structural":"none","weights":null},{"base":null,"citation":"","description":"Run UK spine stage frs_education.","id":"frs_education","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["region"],"entity":"household","rows":"all"},{"columns":["age","esa_contrib_reported","esa_income_reported","is_severely_disabled_for_benefits","jsa_contrib_reported","jsa_income_reported","universal_credit_reported"],"entity":"person","rows":"all"}],"kernel":"uk.stage.frs_education@1","mass":"conserve","outputs":[{"column":"current_education","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"highest_education","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"is_in_non_advanced_education","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"is_in_approved_training","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"age_started_or_accepted_current_education_or_training","dtype":"int64","entity":"person","ownership":"produced","rows":"all"},{"column":"is_before_universal_credit_qualifying_young_person_terminal_date","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"adult_ema","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"child_ema","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"receives_benefits_in_own_right","dtype":"bool","entity":"person","ownership":"produced","rows":"all"}],"params":{"stage":"frs_education","stage_contract_sha256":"836cb0f5a582fee1425190e96c9cb81bdef859bd236c8b6bc27660b6e3d0c2f8","time_period":"2024"},"population":"age_tail.boundary","sources":["frs"],"structural":"none","weights":null},{"base":null,"citation":"","description":"Run UK spine stage frs_legacy_proxies.","id":"frs_legacy_proxies","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","council_tax"],"entity":"household","rows":"all"},{"columns":["age","gender","marital_status","hours_worked","is_household_head","is_benunit_head","is_parent","is_uc_claimant","employment_income","self_employment_income","private_pension_income","tax_free_savings_income","savings_interest_income","dividend_income","property_income","maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","bsp_reported","employment_status","employment_sector","sic_industry_division","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right"],"entity":"person","rows":"all"}],"kernel":"uk.stage.frs_legacy_proxies@1","mass":"conserve","outputs":[{"column":"legacy_jobseeker_proxy","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"esa_health_condition_proxy","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"esa_support_group_proxy","dtype":"bool","entity":"person","ownership":"produced","rows":"all"}],"params":{"stage":"frs_legacy_proxies","stage_contract_sha256":"1ecf761cf3fa7180da15659e138e67a8654e58272b4fde28344aa27a874ad0aa","time_period":"2024"},"population":"age_tail.boundary","sources":["frs"],"structural":"none","weights":null},{"base":"age_tail.boundary","citation":"","description":"Ownership boundary before frs_education_grant_split rewrites.","id":"frs_education_grant_split.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","council_tax"],"entity":"household","rows":"all"},{"columns":["age","gender","marital_status","hours_worked","is_household_head","is_benunit_head","is_parent","is_uc_claimant","employment_income","self_employment_income","private_pension_income","tax_free_savings_income","savings_interest_income","dividend_income","property_income","maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","bsp_reported","employment_status","employment_sector","sic_industry_division","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"base":null,"citation":"","description":"Run UK spine stage frs_education_grant_split.","id":"frs_education_grant_split","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","council_tax"],"entity":"household","rows":"all"},{"columns":["age","gender","marital_status","hours_worked","is_household_head","is_benunit_head","is_parent","is_uc_claimant","employment_income","self_employment_income","private_pension_income","tax_free_savings_income","savings_interest_income","dividend_income","property_income","maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","student_loans","access_fund","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","bsp_reported","employment_status","employment_sector","sic_industry_division","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy"],"entity":"person","rows":"all"}],"kernel":"uk.stage.frs_education_grant_split@1","mass":"conserve","outputs":[{"column":"disabled_students_allowance_eligible_expenses","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"education_grants","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"stage":"frs_education_grant_split","stage_contract_sha256":"8718f014b90498c2cc7c754289775ca4a41452d9b978d8cbc5f34b64cbdc7df2","time_period":"2024"},"population":"frs_education_grant_split.boundary","sources":["frs"],"structural":"none","weights":null},{"base":null,"citation":"","description":"Run UK spine stage frs_take_up.","id":"frs_take_up","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["region"],"entity":"household","rows":"all"},{"columns":["age","child_benefit_reported","education_grants","pension_credit_reported","universal_credit_reported"],"entity":"person","rows":"all"}],"kernel":"uk.stage.frs_take_up@1","mass":"conserve","outputs":[{"column":"would_claim_child_benefit","dtype":"bool","entity":"benunit","ownership":"produced","rows":"all"},{"column":"child_benefit_opts_out","dtype":"bool","entity":"benunit","ownership":"produced","rows":"all"},{"column":"would_claim_pc","dtype":"bool","entity":"benunit","ownership":"produced","rows":"all"},{"column":"would_claim_uc","dtype":"bool","entity":"benunit","ownership":"produced","rows":"all"},{"column":"would_claim_tfc","dtype":"bool","entity":"benunit","ownership":"produced","rows":"all"},{"column":"would_claim_extended_childcare","dtype":"bool","entity":"benunit","ownership":"produced","rows":"all"},{"column":"would_claim_universal_childcare","dtype":"bool","entity":"benunit","ownership":"produced","rows":"all"},{"column":"would_claim_targeted_childcare","dtype":"bool","entity":"benunit","ownership":"produced","rows":"all"},{"column":"maximum_extended_childcare_hours_usage","dtype":"float64","entity":"benunit","ownership":"produced","rows":"all"}],"params":{"stage":"frs_take_up","stage_contract_sha256":"d10ca01bfa5d719640ded8e62196b0692236d3be0a2417899deeefd241088abf","time_period":"2024"},"population":"frs_education_grant_split.boundary","sources":["frs"],"structural":"none","weights":null},{"base":null,"citation":"","description":"Run UK spine stage frs_person_draws.","id":"frs_person_draws","inputs":[{"columns":["frs_benunit_capital","maximum_extended_childcare_hours_usage"],"entity":"benunit","rows":"all"},{"columns":["region"],"entity":"household","rows":"all"},{"columns":["age"],"entity":"person","rows":"all"}],"kernel":"uk.stage.frs_person_draws@1","mass":"conserve","outputs":[{"column":"would_claim_marriage_allowance","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"would_claim_scp","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"attends_private_school_random_draw","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"tax_free_childcare_spend_routed_share","dtype":"float64","entity":"person","ownership":"produced","rows":"all"}],"params":{"stage":"frs_person_draws","stage_contract_sha256":"e87b779c3977aa520c298f9bca54b675a8cacb8baed0b58412a1f783b42968ff","time_period":"2024"},"population":"frs_education_grant_split.boundary","sources":["frs"],"structural":"none","weights":null},{"base":null,"citation":"","description":"Run UK spine stage frs_household_draws.","id":"frs_household_draws","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["region"],"entity":"household","rows":"all"},{"columns":["age","tax_free_childcare_spend_routed_share"],"entity":"person","rows":"all"}],"kernel":"uk.stage.frs_household_draws@1","mass":"conserve","outputs":[{"column":"household_owns_tv","dtype":"bool","entity":"household","ownership":"produced","rows":"all"},{"column":"would_evade_tv_licence_fee","dtype":"bool","entity":"household","ownership":"produced","rows":"all"},{"column":"main_residential_property_purchased_is_first_home","dtype":"bool","entity":"household","ownership":"produced","rows":"all"},{"column":"property_purchased","dtype":"bool","entity":"household","ownership":"produced","rows":"all"}],"params":{"stage":"frs_household_draws","stage_contract_sha256":"c74c63b09264319c4bf0049dabba00ecd0ce660beb7b53d6dae71518434bc949","time_period":"2024"},"population":"frs_education_grant_split.boundary","sources":["frs"],"structural":"none","weights":null},{"base":null,"citation":"","description":"Run UK spine stage frs_brma.","id":"frs_brma","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","maximum_extended_childcare_hours_usage"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased"],"entity":"household","rows":"all"},{"columns":["age","gender","marital_status","hours_worked","is_household_head","is_benunit_head","is_parent","is_uc_claimant","employment_income","self_employment_income","private_pension_income","tax_free_savings_income","savings_interest_income","dividend_income","property_income","maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","bsp_reported","employment_status","employment_sector","sic_industry_division","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share"],"entity":"person","rows":"all"}],"kernel":"uk.stage.frs_brma@1","mass":"conserve","outputs":[{"column":"brma","dtype":"string","entity":"household","ownership":"produced","rows":"all"}],"params":{"stage":"frs_brma","stage_contract_sha256":"5b8f0b361310c7cd3efbaae3762b347649b7d4afb9e9943131d9bef9266fc3c8","time_period":"2024"},"population":"frs_education_grant_split.boundary","sources":["frs"],"structural":"none","weights":null},{"base":"frs_education_grant_split.boundary","citation":"","description":"Freeze the assembled-spine gate population.","id":"frs_brma.checkpoint","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","maximum_extended_childcare_hours_usage"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma"],"entity":"household","rows":"all"},{"columns":["age","gender","marital_status","hours_worked","is_household_head","is_benunit_head","is_parent","is_uc_claimant","employment_income","self_employment_income","private_pension_income","tax_free_savings_income","savings_interest_income","dividend_income","property_income","maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","bsp_reported","employment_status","employment_sector","sic_industry_division","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"base":null,"citation":"","description":"Run UK spine stage was_wealth.","id":"was_wealth","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","maximum_extended_childcare_hours_usage"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma"],"entity":"household","rows":"all"},{"columns":["age","gender","marital_status","hours_worked","is_household_head","is_benunit_head","is_parent","is_uc_claimant","employment_income","self_employment_income","private_pension_income","tax_free_savings_income","savings_interest_income","dividend_income","property_income","maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","bsp_reported","employment_status","employment_sector","sic_industry_division","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share"],"entity":"person","rows":"all"}],"kernel":"uk.stage.was_wealth@1","mass":"conserve","outputs":[{"column":"owned_land","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"property_wealth","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"corporate_wealth","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"private_pension_wealth","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"gross_financial_wealth","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"net_financial_wealth","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"main_residence_value","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"other_residential_property_value","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"non_residential_property_value","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"savings","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"num_vehicles","dtype":"int64","entity":"household","ownership":"produced","rows":"all"},{"column":"cash_isa","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"stocks_and_shares_isa","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"mortgage_debt","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"consumer_debt","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"student_loan_balance","dtype":"float64","entity":"person","ownership":"produced","rows":"all"}],"params":{"stage":"was_wealth","stage_contract_sha256":"c499dcb7c49a0176229dbc363b400415cb5c1da44af266ba2155f9a62a0bd2ed","time_period":"2024"},"population":"frs_brma.checkpoint","sources":["frs"],"structural":"none","weights":null},{"base":"frs_brma.checkpoint","citation":"","description":"Ownership boundary before regional_property_uprating rewrites.","id":"regional_property_uprating.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","maximum_extended_childcare_hours_usage"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt"],"entity":"household","rows":"all"},{"columns":["age","gender","marital_status","hours_worked","is_household_head","is_benunit_head","is_parent","is_uc_claimant","employment_income","self_employment_income","private_pension_income","tax_free_savings_income","savings_interest_income","dividend_income","property_income","maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","bsp_reported","employment_status","employment_sector","sic_industry_division","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","student_loan_balance"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"base":null,"citation":"","description":"Run UK spine stage regional_property_uprating.","id":"regional_property_uprating","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["region"],"entity":"household","rows":"all"},{"columns":["age","student_loan_balance"],"entity":"person","rows":"all"}],"kernel":"uk.stage.regional_property_uprating@1","mass":"conserve","outputs":[{"column":"main_residence_value","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"property_wealth","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"stage":"regional_property_uprating","stage_contract_sha256":"4304356c6c5ba91a04883148cbf9078761ee926c8387aadb6c15b6b70ebb31f6","time_period":"2024"},"population":"regional_property_uprating.boundary","sources":["frs"],"structural":"none","weights":null},{"base":null,"citation":"","description":"Run UK spine stage lcfs_consumption.","id":"lcfs_consumption","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","maximum_extended_childcare_hours_usage"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt"],"entity":"household","rows":"all"},{"columns":["age","gender","marital_status","hours_worked","is_household_head","is_benunit_head","is_parent","is_uc_claimant","employment_income","self_employment_income","private_pension_income","tax_free_savings_income","savings_interest_income","dividend_income","property_income","maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","bsp_reported","employment_status","employment_sector","sic_industry_division","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","student_loan_balance"],"entity":"person","rows":"all"}],"kernel":"uk.stage.lcfs_consumption@1","mass":"conserve","outputs":[{"column":"food_and_non_alcoholic_beverages_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"alcohol_and_tobacco_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"clothing_and_footwear_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"housing_water_and_electricity_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"household_furnishings_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"health_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"transport_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"communication_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"recreation_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"education_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"restaurants_and_hotels_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"miscellaneous_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"petrol_spending","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"diesel_spending","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"bus_fare_spending","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"domestic_energy_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"electricity_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"gas_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"has_fuel_consumption","dtype":"bool","entity":"household","ownership":"produced","rows":"all"}],"params":{"stage":"lcfs_consumption","stage_contract_sha256":"eb0a529c357c84290a001209a18c5a43b0a0310553c77963d6842fd0111b73ad","time_period":"2024"},"population":"regional_property_uprating.boundary","sources":["frs"],"structural":"none","weights":null},{"base":null,"citation":"","description":"Run UK spine stage etb_vat.","id":"etb_vat","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","maximum_extended_childcare_hours_usage"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","has_fuel_consumption"],"entity":"household","rows":"all"},{"columns":["age","gender","marital_status","hours_worked","is_household_head","is_benunit_head","is_parent","is_uc_claimant","employment_income","self_employment_income","private_pension_income","tax_free_savings_income","savings_interest_income","dividend_income","property_income","maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","bsp_reported","employment_status","employment_sector","sic_industry_division","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","student_loan_balance"],"entity":"person","rows":"all"}],"kernel":"uk.stage.etb_vat@1","mass":"conserve","outputs":[{"column":"full_rate_vat_expenditure_rate","dtype":"float64","entity":"household","ownership":"produced","rows":"all"}],"params":{"stage":"etb_vat","stage_contract_sha256":"99a6756256adfe8255672fafda89710c08ae56bd2b8f011b20c3b8381eabfa1e","time_period":"2024"},"population":"regional_property_uprating.boundary","sources":["frs"],"structural":"none","weights":null},{"base":null,"citation":"","description":"Run UK spine stage etb_services.","id":"etb_services","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","maximum_extended_childcare_hours_usage"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","has_fuel_consumption","full_rate_vat_expenditure_rate"],"entity":"household","rows":"all"},{"columns":["age","gender","marital_status","hours_worked","is_household_head","is_benunit_head","is_parent","is_uc_claimant","employment_income","self_employment_income","private_pension_income","tax_free_savings_income","savings_interest_income","dividend_income","property_income","maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","bsp_reported","employment_status","employment_sector","sic_industry_division","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","student_loan_balance"],"entity":"person","rows":"all"}],"kernel":"uk.stage.etb_services@1","mass":"conserve","outputs":[{"column":"dfe_education_spending","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"rail_subsidy_spending","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"bus_subsidy_spending","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"rail_usage","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"a_and_e_visits","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"admitted_patient_visits","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"outpatient_visits","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"nhs_a_and_e_spending","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"nhs_admitted_patient_spending","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"nhs_outpatient_spending","dtype":"float64","entity":"person","ownership":"produced","rows":"all"}],"params":{"stage":"etb_services","stage_contract_sha256":"48294c9d86dd422238a723aca5845e9ceb9eda2903a3bb87bb32a86092447e19","time_period":"2024"},"population":"regional_property_uprating.boundary","sources":["frs"],"structural":"none","weights":null},{"base":null,"citation":"","description":"Run UK spine stage frs_hmrc_spine_leaves.","id":"frs_hmrc_spine_leaves","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["region"],"entity":"household","rows":"all"},{"columns":["age","employee_pension_contributions","nhs_outpatient_spending"],"entity":"person","rows":"all"}],"kernel":"uk.stage.frs_hmrc_spine_leaves@1","mass":"conserve","outputs":[{"column":"hmrc_spi_pay","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"hmrc_spi_unemployment_benefit_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"hmrc_spi_incapacity_benefit_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"ossben_identifiable_subset","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"srp_regular_code5","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"employer_pension_contributions","dtype":"float64","entity":"person","ownership":"produced","rows":"all"}],"params":{"stage":"frs_hmrc_spine_leaves","stage_contract_sha256":"2bb3d068003489b47cc8676ac26c203ce3c39a9b6e92f3ecd0a3c06acd6addc4","time_period":"2024"},"population":"regional_property_uprating.boundary","sources":["frs"],"structural":"none","weights":null},{"base":"regional_property_uprating.boundary","citation":"","description":"Run structural UK stage spi_support_channel.","id":"spi_support_channel","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","maximum_extended_childcare_hours_usage"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","has_fuel_consumption","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage"],"entity":"household","rows":"all"},{"columns":["age","gender","marital_status","hours_worked","is_household_head","is_benunit_head","is_parent","is_uc_claimant","employment_income","self_employment_income","private_pension_income","tax_free_savings_income","savings_interest_income","dividend_income","property_income","maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","bsp_reported","employment_status","employment_sector","sic_industry_division","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","student_loan_balance","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions"],"entity":"person","rows":"all"}],"kernel":"uk.stage.expand.spi_support_channel@1","mass":"declared","outputs":[],"params":{"expand_cells":[["person","person_source_id","int64"],["person","person_support_channel","string"],["person","person_support_clone_index","int64"],["benunit","benunit_source_id","int64"],["benunit","benunit_support_channel","string"],["benunit","benunit_support_clone_index","int64"],["household","source_household_id","int64"],["household","source_year","int64"],["household","source_household_key","string"],["household","household_source_id","int64"],["household","household_support_channel","string"],["household","household_support_clone_index","int64"],["household","household_is_spi_synthetic","bool"]],"expand_weight_entity":"household","expand_weight_kind":"importance","stage":"spi_support_channel","stage_contract_sha256":"4b10f1a4a215cdf2c406c9974de2d1e580eb3cd3a25ed34973c1b63abf4a54bb","time_period":"2024"},"population":null,"sources":["frs"],"structural":"expand","weights":null},{"base":null,"citation":"","description":"Own the cells materialized by spi_support_channel.","id":"spi_support_channel.owned","inputs":[],"kernel":"uk.claim@1","mass":"conserve","outputs":[{"column":"person_source_id","dtype":"int64","entity":"person","ownership":"produced","rows":"all"},{"column":"person_support_channel","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"person_support_clone_index","dtype":"int64","entity":"person","ownership":"produced","rows":"all"},{"column":"benunit_source_id","dtype":"int64","entity":"benunit","ownership":"produced","rows":"all"},{"column":"benunit_support_channel","dtype":"string","entity":"benunit","ownership":"produced","rows":"all"},{"column":"benunit_support_clone_index","dtype":"int64","entity":"benunit","ownership":"produced","rows":"all"},{"column":"source_household_id","dtype":"int64","entity":"household","ownership":"produced","rows":"all"},{"column":"source_year","dtype":"int64","entity":"household","ownership":"produced","rows":"all"},{"column":"source_household_key","dtype":"string","entity":"household","ownership":"produced","rows":"all"},{"column":"household_source_id","dtype":"int64","entity":"household","ownership":"produced","rows":"all"},{"column":"household_support_channel","dtype":"string","entity":"household","ownership":"produced","rows":"all"},{"column":"household_support_clone_index","dtype":"int64","entity":"household","ownership":"produced","rows":"all"},{"column":"household_is_spi_synthetic","dtype":"bool","entity":"household","ownership":"produced","rows":"all"}],"params":{"materialized_expand_outputs":["person.person_source_id","person.person_support_channel","person.person_support_clone_index","benunit.benunit_source_id","benunit.benunit_support_channel","benunit.benunit_support_clone_index","household.source_household_id","household.source_year","household.source_household_key","household.household_source_id","household.household_support_channel","household.household_support_clone_index","household.household_is_spi_synthetic"]},"population":"spi_support_channel","sources":[],"structural":"none","weights":null},{"base":null,"citation":"","description":"Run UK spine stage hmrc_spi_income_spine.","id":"hmrc_spi_income_spine","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","has_fuel_consumption","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic"],"entity":"household","rows":"all"},{"columns":["age","gender","marital_status","hours_worked","is_household_head","is_benunit_head","is_parent","is_uc_claimant","maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","maintenance_expenses","childcare_expenses","salary_sacrifice_reported","salary_sacrifice_asked","ssmg_reported","incapacity_benefit_reported","employment_status","employment_sector","sic_industry_division","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","student_loan_balance","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending","ossben_identifiable_subset","srp_regular_code5","person_source_id","person_support_channel","person_support_clone_index"],"entity":"person","rows":"all"}],"kernel":"uk.stage.hmrc_spi_income_spine@1","mass":"conserve","outputs":[{"column":"charitable_investment_gifts","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"gift_aid","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"other_investment_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"hmrc_spi_employment_benefits","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"hmrc_spi_employment_expenses","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"hmrc_spi_other_social_security_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"hmrc_spi_taxable_termination_pay","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"hmrc_spi_miscellaneous_employment_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"hmrc_spi_other_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"hmrc_spi_state_pension_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"hmrc_spi_employed_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"hmrc_spi_total_earned_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"hmrc_spi_total_investment_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"hmrc_spi_assessable_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"employment_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"self_employment_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"savings_interest_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"dividend_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"private_pension_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"property_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"employee_pension_contributions","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"employer_pension_contributions","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"personal_pension_contributions","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"pension_contributions_via_salary_sacrifice","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"tax_free_savings_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"universal_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"pension_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"child_benefit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"housing_benefit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"income_support_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"working_tax_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"child_tax_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"attendance_allowance_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"state_pension_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"dla_sc_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"dla_m_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"pip_m_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"pip_dl_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"sda_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"carers_allowance_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"iidb_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"afcs_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"bsp_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"winter_fuel_allowance_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"council_tax_benefit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"jsa_contrib_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"jsa_income_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"esa_contrib_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"esa_income_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"hmrc_spi_pay","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"hmrc_spi_unemployment_benefit_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"hmrc_spi_incapacity_benefit_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"aa_category","dtype":"string","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"dla_sc_category","dtype":"string","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"dla_m_category","dtype":"string","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"pip_m_category","dtype":"string","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"pip_dl_category","dtype":"string","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_disabled_for_benefits","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_enhanced_disabled_for_benefits","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_severely_disabled_for_benefits","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"stage":"hmrc_spi_income_spine","stage_contract_sha256":"4c2edca1f77416a00cff10c7e9d0a9da50e332a2bb6a995ef9ac1bdd00ff97a9","time_period":"2024"},"population":"spi_support_channel","sources":["frs"],"structural":"none","weights":null},{"base":"spi_support_channel","citation":"","description":"Ownership boundary before uc_reporter_redraw rewrites.","id":"uc_reporter_redraw.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","has_fuel_consumption","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic"],"entity":"household","rows":"all"},{"columns":["age","gender","marital_status","hours_worked","is_household_head","is_benunit_head","is_parent","is_uc_claimant","employment_income","self_employment_income","private_pension_income","tax_free_savings_income","savings_interest_income","dividend_income","property_income","maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","bsp_reported","employment_status","employment_sector","sic_industry_division","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","student_loan_balance","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","person_source_id","person_support_channel","person_support_clone_index","charitable_investment_gifts","gift_aid","other_investment_income","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"base":null,"citation":"","description":"Run UK spine stage uc_reporter_redraw.","id":"uc_reporter_redraw","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","has_fuel_consumption","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic"],"entity":"household","rows":"all"},{"columns":["age","gender","marital_status","hours_worked","is_household_head","is_benunit_head","is_parent","is_uc_claimant","employment_income","self_employment_income","private_pension_income","tax_free_savings_income","savings_interest_income","dividend_income","property_income","maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","bsp_reported","employment_status","employment_sector","sic_industry_division","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","student_loan_balance","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","person_source_id","person_support_channel","person_support_clone_index","charitable_investment_gifts","gift_aid","other_investment_income","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income"],"entity":"person","rows":"all"}],"kernel":"uk.stage.uc_reporter_redraw@1","mass":"conserve","outputs":[{"column":"universal_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"stage":"uc_reporter_redraw","stage_contract_sha256":"7e94e034cad29c5bae566ffb875c42cddf3a1025c15496907e783ceff1144f8b","time_period":"2024"},"population":"uc_reporter_redraw.boundary","sources":["frs"],"structural":"none","weights":null},{"base":"uc_reporter_redraw.boundary","citation":"","description":"Ownership boundary before uc_capital_coherence rewrites.","id":"uc_capital_coherence.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","has_fuel_consumption","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic"],"entity":"household","rows":"all"},{"columns":["age","gender","marital_status","hours_worked","is_household_head","is_benunit_head","is_parent","is_uc_claimant","employment_income","self_employment_income","private_pension_income","tax_free_savings_income","savings_interest_income","dividend_income","property_income","maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","bsp_reported","employment_status","employment_sector","sic_industry_division","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","student_loan_balance","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","person_source_id","person_support_channel","person_support_clone_index","charitable_investment_gifts","gift_aid","other_investment_income","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"base":null,"citation":"","description":"Run UK spine stage uc_capital_coherence.","id":"uc_capital_coherence","inputs":[{"columns":["benunit_support_channel","dependent_children","is_married"],"entity":"benunit","rows":"all"},{"columns":["region"],"entity":"household","rows":"all"},{"columns":["age","is_benunit_head","is_parent","person_support_channel","universal_credit_reported"],"entity":"person","rows":"all"}],"kernel":"uk.stage.uc_capital_coherence@1","mass":"conserve","outputs":[{"column":"uc_reported_capital","dtype":"float64","entity":"benunit","ownership":"produced","rows":"all"},{"column":"frs_benunit_capital","dtype":"float64","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"},{"column":"would_claim_uc","dtype":"bool","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"stage":"uc_capital_coherence","stage_contract_sha256":"059f27ee687ee15385b246d43068b17aeecbe3645ccc41fd52fcad06a006a081","time_period":"2024"},"population":"uc_capital_coherence.boundary","sources":["frs"],"structural":"none","weights":null},{"base":null,"citation":"","description":"Run UK spine stage uc_deduction_attributes.","id":"uc_deduction_attributes","inputs":[{"columns":["frs_benunit_capital","would_claim_uc"],"entity":"benunit","rows":"all"},{"columns":["region"],"entity":"household","rows":"all"},{"columns":["age"],"entity":"person","rows":"all"}],"kernel":"uk.stage.uc_deduction_attributes@1","mass":"conserve","outputs":[{"column":"uc_deduction_random_draw","dtype":"float64","entity":"benunit","ownership":"produced","rows":"all"},{"column":"uc_deduction_type_random_draw","dtype":"float64","entity":"benunit","ownership":"produced","rows":"all"},{"column":"uc_latent_deduction_rate","dtype":"float64","entity":"benunit","ownership":"produced","rows":"all"},{"column":"uc_deduction_combination","dtype":"string","entity":"benunit","ownership":"produced","rows":"all"}],"params":{"stage":"uc_deduction_attributes","stage_contract_sha256":"6bc7e3ec1a5712e23ca1b7d2956e8b17f2526b94431ba3e5a9294b04658e4fe0","time_period":"2024"},"population":"uc_capital_coherence.boundary","sources":["frs"],"structural":"none","weights":null},{"base":"uc_capital_coherence.boundary","citation":"","description":"Run structural UK stage cgt_incidence_clone.","id":"cgt_incidence_clone","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","uc_deduction_random_draw","uc_deduction_type_random_draw","uc_latent_deduction_rate","uc_deduction_combination"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","has_fuel_consumption","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic"],"entity":"household","rows":"all"},{"columns":["age","gender","marital_status","hours_worked","is_household_head","is_benunit_head","is_parent","is_uc_claimant","employment_income","self_employment_income","private_pension_income","tax_free_savings_income","savings_interest_income","dividend_income","property_income","maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","bsp_reported","employment_status","employment_sector","sic_industry_division","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","student_loan_balance","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","person_source_id","person_support_channel","person_support_clone_index","charitable_investment_gifts","gift_aid","other_investment_income","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income"],"entity":"person","rows":"all"}],"kernel":"uk.stage.expand.cgt_incidence_clone@1","mass":"conserve","outputs":[],"params":{"expand_cells":[["household","household_is_capital_gains_clone","bool"],["person","capital_gains","float64"]],"expand_weight_entity":"household","expand_weight_kind":"importance","stage":"cgt_incidence_clone","stage_contract_sha256":"ee30278543cc0297a5366d855b7753aa04af5518971594c193c52be36bf8a7b4","time_period":"2024"},"population":null,"sources":["frs"],"structural":"expand","weights":null},{"base":null,"citation":"","description":"Own the cells materialized by cgt_incidence_clone.","id":"cgt_incidence_clone.owned","inputs":[],"kernel":"uk.claim@1","mass":"conserve","outputs":[{"column":"household_is_capital_gains_clone","dtype":"bool","entity":"household","ownership":"produced","rows":"all"},{"column":"capital_gains","dtype":"float64","entity":"person","ownership":"produced","rows":"all"}],"params":{"materialized_expand_outputs":["household.household_is_capital_gains_clone","person.capital_gains"]},"population":"cgt_incidence_clone","sources":[],"structural":"none","weights":null},{"base":"cgt_incidence_clone","citation":"","description":"Run structural UK stage cgt_band_donors.","id":"cgt_band_donors","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","uc_deduction_random_draw","uc_deduction_type_random_draw","uc_latent_deduction_rate","uc_deduction_combination"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","has_fuel_consumption","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_capital_gains_clone"],"entity":"household","rows":"all"},{"columns":["age","gender","marital_status","hours_worked","is_household_head","is_benunit_head","is_parent","is_uc_claimant","employment_income","self_employment_income","private_pension_income","tax_free_savings_income","savings_interest_income","dividend_income","property_income","maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","bsp_reported","employment_status","employment_sector","sic_industry_division","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","student_loan_balance","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","person_source_id","person_support_channel","person_support_clone_index","charitable_investment_gifts","gift_aid","other_investment_income","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","capital_gains"],"entity":"person","rows":"all"}],"kernel":"uk.stage.expand.cgt_band_donors@1","mass":"free","outputs":[],"params":{"expand_cells":[["household","household_is_cgt_band_donor","bool"],["person","capital_gains","float64"]],"expand_weight_entity":"household","expand_weight_kind":"importance","stage":"cgt_band_donors","stage_contract_sha256":"e10e10c49c0ca2a6a65048c91b683f818e8b6e207a6c6f57a20d76ea299160b0","time_period":"2024"},"population":null,"sources":["frs"],"structural":"expand","weights":null},{"base":null,"citation":"","description":"Own the cells materialized by cgt_band_donors.","id":"cgt_band_donors.owned","inputs":[],"kernel":"uk.claim@1","mass":"conserve","outputs":[{"column":"household_is_cgt_band_donor","dtype":"bool","entity":"household","ownership":"produced","rows":"all"},{"column":"capital_gains","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"materialized_expand_outputs":["household.household_is_cgt_band_donor"]},"population":"cgt_band_donors","sources":[],"structural":"none","weights":null},{"base":"cgt_band_donors","citation":"","description":"Ownership boundary before hmrc_cgt_gains_spine rewrites.","id":"hmrc_cgt_gains_spine.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","uc_deduction_random_draw","uc_deduction_type_random_draw","uc_latent_deduction_rate","uc_deduction_combination"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","has_fuel_consumption","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_capital_gains_clone","household_is_cgt_band_donor"],"entity":"household","rows":"all"},{"columns":["age","gender","marital_status","hours_worked","is_household_head","is_benunit_head","is_parent","is_uc_claimant","employment_income","self_employment_income","private_pension_income","tax_free_savings_income","savings_interest_income","dividend_income","property_income","maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","bsp_reported","employment_status","employment_sector","sic_industry_division","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","student_loan_balance","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","person_source_id","person_support_channel","person_support_clone_index","charitable_investment_gifts","gift_aid","other_investment_income","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","capital_gains"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"base":null,"citation":"","description":"Run UK spine stage hmrc_cgt_gains_spine.","id":"hmrc_cgt_gains_spine","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["region"],"entity":"household","rows":"all"},{"columns":["age","dividend_income","employment_income","miscellaneous_income","private_pension_income","property_income","savings_interest_income","self_employment_income","state_pension_reported","tax_free_savings_income"],"entity":"person","rows":"all"}],"kernel":"uk.stage.hmrc_cgt_gains_spine@1","mass":"conserve","outputs":[{"column":"capital_gains","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"stage":"hmrc_cgt_gains_spine","stage_contract_sha256":"20ada8d8ee94400bd160223e865a910db7aff9c40a7f0c8ee8ca8c40ed901b72","time_period":"2024"},"population":"hmrc_cgt_gains_spine.boundary","sources":["frs"],"structural":"none","weights":null},{"base":null,"citation":"","description":"Run UK spine stage salary_sacrifice.","id":"salary_sacrifice","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","uc_deduction_random_draw","uc_deduction_type_random_draw","uc_latent_deduction_rate","uc_deduction_combination"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","has_fuel_consumption","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_capital_gains_clone","household_is_cgt_band_donor"],"entity":"household","rows":"all"},{"columns":["age","gender","marital_status","hours_worked","is_household_head","is_benunit_head","is_parent","is_uc_claimant","employment_income","self_employment_income","private_pension_income","tax_free_savings_income","savings_interest_income","dividend_income","property_income","maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","salary_sacrifice_reported","salary_sacrifice_asked","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","bsp_reported","employment_status","employment_sector","sic_industry_division","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","student_loan_balance","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","person_source_id","person_support_channel","person_support_clone_index","charitable_investment_gifts","gift_aid","other_investment_income","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","capital_gains"],"entity":"person","rows":"all"}],"kernel":"uk.stage.salary_sacrifice@1","mass":"conserve","outputs":[{"column":"pension_contributions_via_salary_sacrifice","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"employee_pension_contributions","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"stage":"salary_sacrifice","stage_contract_sha256":"a4af925af21cc6766eb7c8855b6743a3aa4eeeb38df12cfc83063d7ad6890539","time_period":"2024"},"population":"hmrc_cgt_gains_spine.boundary","sources":["frs"],"structural":"none","weights":null},{"base":null,"citation":"","description":"Run UK spine stage student_loans.","id":"student_loans","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["region"],"entity":"household","rows":"all"},{"columns":["age","current_education","employee_pension_contributions","highest_education","student_loan_repayments","student_loans"],"entity":"person","rows":"all"}],"kernel":"uk.stage.student_loans@1","mass":"conserve","outputs":[{"column":"student_loan_plan","dtype":"string","entity":"person","ownership":"produced","rows":"all"}],"params":{"stage":"student_loans","stage_contract_sha256":"7c4c76398c2e80b41c7f941a5f06246b57d89b5eb0c36d42e3116ac35e82e203","time_period":"2024"},"population":"hmrc_cgt_gains_spine.boundary","sources":["frs"],"structural":"none","weights":null}],"sources":[{"codec":"csv-tables","description":"Content-bound UK FRS and donor fixture/source bundle.","name":"frs"}]} +{"country":"uk","nodes":[{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Load the source-bound UK FRS root population.","id":"create_uk_frs","inputs":[],"kernel":"uk.create@1","mass":"conserve","outputs":[{"column":"age","dtype":"int64","entity":"person","ownership":"produced","rows":"all"},{"column":"gender","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"marital_status","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"hours_worked","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"is_household_head","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"is_benunit_head","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"is_parent","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"is_uc_claimant","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"employment_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"self_employment_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"private_pension_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"tax_free_savings_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"savings_interest_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"dividend_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"property_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"maintenance_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"miscellaneous_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"private_transfer_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"lump_sum_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"student_loan_repayments","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"statutory_sick_pay","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"statutory_maternity_pay","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"student_loans","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"access_fund","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"education_grants","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"healthy_start_vouchers","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"free_school_breakfasts","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"free_school_fruit_veg","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"free_school_meals","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"council_tax_benefit_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"maintenance_expenses","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"childcare_expenses","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"personal_pension_contributions","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"employee_pension_contributions","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"pension_contributions_via_salary_sacrifice","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"salary_sacrifice_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"salary_sacrifice_asked","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"child_benefit_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"income_support_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"housing_benefit_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"attendance_allowance_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"dla_sc_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"dla_m_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"iidb_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"carers_allowance_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"sda_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"afcs_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"ssmg_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"pension_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"child_tax_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"working_tax_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"state_pension_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"winter_fuel_allowance_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"incapacity_benefit_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"universal_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"pip_m_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"pip_dl_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"jsa_contrib_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"jsa_income_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"esa_contrib_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"esa_income_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"bsp_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"frs_benunit_capital","dtype":"float64","entity":"benunit","ownership":"produced","rows":"all"},{"column":"is_married","dtype":"bool","entity":"benunit","ownership":"produced","rows":"all"},{"column":"dependent_children","dtype":"int64","entity":"benunit","ownership":"produced","rows":"all"},{"column":"region","dtype":"string","entity":"household","ownership":"produced","rows":"all"},{"column":"tenure_type","dtype":"string","entity":"household","ownership":"produced","rows":"all"},{"column":"accommodation_type","dtype":"string","entity":"household","ownership":"produced","rows":"all"},{"column":"num_bedrooms","dtype":"int64","entity":"household","ownership":"produced","rows":"all"},{"column":"council_tax_reported","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"council_tax_band","dtype":"string","entity":"household","ownership":"produced","rows":"all"},{"column":"council_tax_rebate","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"council_tax_single_adult_raw","dtype":"int64","entity":"household","ownership":"produced","rows":"all"},{"column":"water_and_sewerage_charges","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"domestic_rates","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"rent","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"subrent","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"mortgage_interest_repayment","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"mortgage_capital_repayment","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"structural_insurance_payments","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"housing_service_charges","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"external_child_payments","dtype":"float64","entity":"household","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.97.0"],["policyengine-core","3.31.0"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"sample_fraction":1.0,"sample_seed":578,"stage_contract_sha256":"7ed918cd9dfa27874110bececf0c2513e7348df10b79ed55ea432e0e9ed70f3f","time_period":"2024"},"population":null,"sources":["frs"],"structural":"create","weights":null},{"base":"create_uk_frs","citation":"","description":"Ownership boundary for the source-assembling root stage.","id":"frs_spine.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments"],"entity":"household","rows":"all"},{"columns":["age","gender","marital_status","hours_worked","is_household_head","is_benunit_head","is_parent","is_uc_claimant","employment_income","self_employment_income","private_pension_income","tax_free_savings_income","savings_interest_income","dividend_income","property_income","maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","bsp_reported"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"base":null,"citation":"","description":"Claim the cells assembled by the UK FRS root transform.","id":"frs_spine","inputs":[],"kernel":"uk.claim@1","mass":"conserve","outputs":[{"column":"age","dtype":"int64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"gender","dtype":"string","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"marital_status","dtype":"string","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"hours_worked","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_household_head","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_benunit_head","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_parent","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_uc_claimant","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"employment_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"self_employment_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"private_pension_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"tax_free_savings_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"savings_interest_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"dividend_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"property_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"maintenance_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"miscellaneous_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"private_transfer_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"lump_sum_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"student_loan_repayments","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"statutory_sick_pay","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"statutory_maternity_pay","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"student_loans","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"access_fund","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"education_grants","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"healthy_start_vouchers","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"free_school_breakfasts","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"free_school_fruit_veg","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"free_school_meals","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"council_tax_benefit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"maintenance_expenses","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"childcare_expenses","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"personal_pension_contributions","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"employee_pension_contributions","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"pension_contributions_via_salary_sacrifice","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"salary_sacrifice_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"salary_sacrifice_asked","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"child_benefit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"income_support_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"housing_benefit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"attendance_allowance_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"dla_sc_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"dla_m_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"iidb_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"carers_allowance_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"sda_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"afcs_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"ssmg_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"pension_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"child_tax_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"working_tax_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"state_pension_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"winter_fuel_allowance_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"incapacity_benefit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"universal_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"pip_m_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"pip_dl_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"jsa_contrib_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"jsa_income_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"esa_contrib_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"esa_income_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"bsp_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"frs_benunit_capital","dtype":"float64","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_married","dtype":"bool","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"},{"column":"dependent_children","dtype":"int64","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"},{"column":"region","dtype":"string","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"tenure_type","dtype":"string","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"accommodation_type","dtype":"string","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"num_bedrooms","dtype":"int64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"council_tax_reported","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"council_tax_band","dtype":"string","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"council_tax_rebate","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"council_tax_single_adult_raw","dtype":"int64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"water_and_sewerage_charges","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"domestic_rates","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"rent","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"subrent","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"mortgage_interest_repayment","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"mortgage_capital_repayment","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"structural_insurance_payments","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"housing_service_charges","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"external_child_payments","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"}],"params":{},"population":"frs_spine.boundary","sources":[],"structural":"none","weights":null},{"base":"frs_spine.boundary","citation":"","description":"Ownership boundary before age_tail rewrites.","id":"age_tail.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments"],"entity":"household","rows":"all"},{"columns":["age","gender","marital_status","hours_worked","is_household_head","is_benunit_head","is_parent","is_uc_claimant","employment_income","self_employment_income","private_pension_income","tax_free_savings_income","savings_interest_income","dividend_income","property_income","maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","bsp_reported"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage age_tail.","id":"age_tail","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["external_child_payments","region"],"entity":"household","rows":"all"},{"columns":["gender"],"entity":"person","rows":"all"}],"kernel":"uk.stage.age_tail@1","mass":"conserve","outputs":[{"column":"age","dtype":"int64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.97.0"],["policyengine-core","3.31.0"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"age_tail","stage_contract_sha256":"963a34f38caede2525e3b8735f904bb70a1bbef5a272f0eefd4346343444484f","time_period":"2024"},"population":"age_tail.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage frs_employment.","id":"frs_employment","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["region"],"entity":"household","rows":"all"},{"columns":["age"],"entity":"person","rows":"all"}],"kernel":"uk.stage.frs_employment@1","mass":"conserve","outputs":[{"column":"employment_status","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"employment_sector","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"sic_industry_division","dtype":"int64","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.97.0"],["policyengine-core","3.31.0"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"frs_employment","stage_contract_sha256":"ddbefaf05b44788d794a6e4b0e8926c14318d66e50d2f11f50ca549538bbf60c","time_period":"2024"},"population":"age_tail.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage frs_council_tax.","id":"frs_council_tax","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["region"],"entity":"household","rows":"all"},{"columns":["age","sic_industry_division"],"entity":"person","rows":"all"}],"kernel":"uk.stage.frs_council_tax@1","mass":"conserve","outputs":[{"column":"council_tax","dtype":"float64","entity":"household","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.97.0"],["policyengine-core","3.31.0"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"frs_council_tax","stage_contract_sha256":"3381cd9f7a736514d5073c57480e07f5098890f3ca9ed3865392043594771eb9","time_period":"2024"},"population":"age_tail.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage frs_disability.","id":"frs_disability","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["council_tax","region"],"entity":"household","rows":"all"},{"columns":["afcs_reported","age","attendance_allowance_reported","dla_m_reported","dla_sc_reported","esa_contrib_reported","esa_income_reported","iidb_reported","incapacity_benefit_reported","pip_dl_reported","pip_m_reported","sda_reported"],"entity":"person","rows":"all"}],"kernel":"uk.stage.frs_disability@1","mass":"conserve","outputs":[{"column":"aa_category","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"dla_sc_category","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"dla_m_category","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"pip_m_category","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"pip_dl_category","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"is_disabled_for_benefits","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"is_enhanced_disabled_for_benefits","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"is_severely_disabled_for_benefits","dtype":"bool","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.97.0"],["policyengine-core","3.31.0"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"frs_disability","stage_contract_sha256":"ac500b4cb1bc04d4fb72d198592dd376d61da8c150921a6ac53914ac18be0508","time_period":"2024"},"population":"age_tail.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage frs_education.","id":"frs_education","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["region"],"entity":"household","rows":"all"},{"columns":["age","esa_contrib_reported","esa_income_reported","is_severely_disabled_for_benefits","jsa_contrib_reported","jsa_income_reported","universal_credit_reported"],"entity":"person","rows":"all"}],"kernel":"uk.stage.frs_education@1","mass":"conserve","outputs":[{"column":"current_education","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"highest_education","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"is_in_non_advanced_education","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"is_in_approved_training","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"age_started_or_accepted_current_education_or_training","dtype":"int64","entity":"person","ownership":"produced","rows":"all"},{"column":"is_before_universal_credit_qualifying_young_person_terminal_date","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"adult_ema","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"child_ema","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"receives_benefits_in_own_right","dtype":"bool","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.97.0"],["policyengine-core","3.31.0"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"frs_education","stage_contract_sha256":"836cb0f5a582fee1425190e96c9cb81bdef859bd236c8b6bc27660b6e3d0c2f8","time_period":"2024"},"population":"age_tail.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage frs_legacy_proxies.","id":"frs_legacy_proxies","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","council_tax"],"entity":"household","rows":"all"},{"columns":["age","gender","marital_status","hours_worked","is_household_head","is_benunit_head","is_parent","is_uc_claimant","employment_income","self_employment_income","private_pension_income","tax_free_savings_income","savings_interest_income","dividend_income","property_income","maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","bsp_reported","employment_status","employment_sector","sic_industry_division","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right"],"entity":"person","rows":"all"}],"kernel":"uk.stage.frs_legacy_proxies@1","mass":"conserve","outputs":[{"column":"legacy_jobseeker_proxy","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"esa_health_condition_proxy","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"esa_support_group_proxy","dtype":"bool","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.97.0"],["policyengine-core","3.31.0"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"frs_legacy_proxies","stage_contract_sha256":"1ecf761cf3fa7180da15659e138e67a8654e58272b4fde28344aa27a874ad0aa","time_period":"2024"},"population":"age_tail.boundary","sources":["frs"],"structural":"none","weights":null},{"base":"age_tail.boundary","citation":"","description":"Ownership boundary before frs_education_grant_split rewrites.","id":"frs_education_grant_split.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","council_tax"],"entity":"household","rows":"all"},{"columns":["age","gender","marital_status","hours_worked","is_household_head","is_benunit_head","is_parent","is_uc_claimant","employment_income","self_employment_income","private_pension_income","tax_free_savings_income","savings_interest_income","dividend_income","property_income","maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","bsp_reported","employment_status","employment_sector","sic_industry_division","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage frs_education_grant_split.","id":"frs_education_grant_split","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","council_tax"],"entity":"household","rows":"all"},{"columns":["age","gender","marital_status","hours_worked","is_household_head","is_benunit_head","is_parent","is_uc_claimant","employment_income","self_employment_income","private_pension_income","tax_free_savings_income","savings_interest_income","dividend_income","property_income","maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","student_loans","access_fund","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","bsp_reported","employment_status","employment_sector","sic_industry_division","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy"],"entity":"person","rows":"all"}],"kernel":"uk.stage.frs_education_grant_split@1","mass":"conserve","outputs":[{"column":"disabled_students_allowance_eligible_expenses","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"education_grants","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.97.0"],["policyengine-core","3.31.0"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"frs_education_grant_split","stage_contract_sha256":"8718f014b90498c2cc7c754289775ca4a41452d9b978d8cbc5f34b64cbdc7df2","time_period":"2024"},"population":"frs_education_grant_split.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage frs_take_up.","id":"frs_take_up","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["region"],"entity":"household","rows":"all"},{"columns":["age","child_benefit_reported","education_grants","pension_credit_reported","universal_credit_reported"],"entity":"person","rows":"all"}],"kernel":"uk.stage.frs_take_up@1","mass":"conserve","outputs":[{"column":"would_claim_child_benefit","dtype":"bool","entity":"benunit","ownership":"produced","rows":"all"},{"column":"child_benefit_opts_out","dtype":"bool","entity":"benunit","ownership":"produced","rows":"all"},{"column":"would_claim_pc","dtype":"bool","entity":"benunit","ownership":"produced","rows":"all"},{"column":"would_claim_uc","dtype":"bool","entity":"benunit","ownership":"produced","rows":"all"},{"column":"would_claim_tfc","dtype":"bool","entity":"benunit","ownership":"produced","rows":"all"},{"column":"would_claim_extended_childcare","dtype":"bool","entity":"benunit","ownership":"produced","rows":"all"},{"column":"would_claim_universal_childcare","dtype":"bool","entity":"benunit","ownership":"produced","rows":"all"},{"column":"would_claim_targeted_childcare","dtype":"bool","entity":"benunit","ownership":"produced","rows":"all"},{"column":"maximum_extended_childcare_hours_usage","dtype":"float64","entity":"benunit","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.97.0"],["policyengine-core","3.31.0"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"frs_take_up","stage_contract_sha256":"d10ca01bfa5d719640ded8e62196b0692236d3be0a2417899deeefd241088abf","time_period":"2024"},"population":"frs_education_grant_split.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage frs_person_draws.","id":"frs_person_draws","inputs":[{"columns":["frs_benunit_capital","maximum_extended_childcare_hours_usage"],"entity":"benunit","rows":"all"},{"columns":["region"],"entity":"household","rows":"all"},{"columns":["age"],"entity":"person","rows":"all"}],"kernel":"uk.stage.frs_person_draws@1","mass":"conserve","outputs":[{"column":"would_claim_marriage_allowance","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"would_claim_scp","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"attends_private_school_random_draw","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"tax_free_childcare_spend_routed_share","dtype":"float64","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.97.0"],["policyengine-core","3.31.0"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"frs_person_draws","stage_contract_sha256":"e87b779c3977aa520c298f9bca54b675a8cacb8baed0b58412a1f783b42968ff","time_period":"2024"},"population":"frs_education_grant_split.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage frs_household_draws.","id":"frs_household_draws","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["region"],"entity":"household","rows":"all"},{"columns":["age","tax_free_childcare_spend_routed_share"],"entity":"person","rows":"all"}],"kernel":"uk.stage.frs_household_draws@1","mass":"conserve","outputs":[{"column":"household_owns_tv","dtype":"bool","entity":"household","ownership":"produced","rows":"all"},{"column":"would_evade_tv_licence_fee","dtype":"bool","entity":"household","ownership":"produced","rows":"all"},{"column":"main_residential_property_purchased_is_first_home","dtype":"bool","entity":"household","ownership":"produced","rows":"all"},{"column":"property_purchased","dtype":"bool","entity":"household","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.97.0"],["policyengine-core","3.31.0"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"frs_household_draws","stage_contract_sha256":"c74c63b09264319c4bf0049dabba00ecd0ce660beb7b53d6dae71518434bc949","time_period":"2024"},"population":"frs_education_grant_split.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage frs_brma.","id":"frs_brma","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","maximum_extended_childcare_hours_usage"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased"],"entity":"household","rows":"all"},{"columns":["age","gender","marital_status","hours_worked","is_household_head","is_benunit_head","is_parent","is_uc_claimant","employment_income","self_employment_income","private_pension_income","tax_free_savings_income","savings_interest_income","dividend_income","property_income","maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","bsp_reported","employment_status","employment_sector","sic_industry_division","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share"],"entity":"person","rows":"all"}],"kernel":"uk.stage.frs_brma@1","mass":"conserve","outputs":[{"column":"brma","dtype":"string","entity":"household","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.97.0"],["policyengine-core","3.31.0"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"frs_brma","stage_contract_sha256":"5b8f0b361310c7cd3efbaae3762b347649b7d4afb9e9943131d9bef9266fc3c8","time_period":"2024"},"population":"frs_education_grant_split.boundary","sources":["frs"],"structural":"none","weights":null},{"base":"frs_education_grant_split.boundary","citation":"","description":"Freeze the assembled-spine gate population.","id":"frs_brma.checkpoint","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","maximum_extended_childcare_hours_usage"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma"],"entity":"household","rows":"all"},{"columns":["age","gender","marital_status","hours_worked","is_household_head","is_benunit_head","is_parent","is_uc_claimant","employment_income","self_employment_income","private_pension_income","tax_free_savings_income","savings_interest_income","dividend_income","property_income","maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","bsp_reported","employment_status","employment_sector","sic_industry_division","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage was_wealth.","id":"was_wealth","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","maximum_extended_childcare_hours_usage"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma"],"entity":"household","rows":"all"},{"columns":["age","gender","marital_status","hours_worked","is_household_head","is_benunit_head","is_parent","is_uc_claimant","employment_income","self_employment_income","private_pension_income","tax_free_savings_income","savings_interest_income","dividend_income","property_income","maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","bsp_reported","employment_status","employment_sector","sic_industry_division","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share"],"entity":"person","rows":"all"}],"kernel":"uk.stage.was_wealth@1","mass":"conserve","outputs":[{"column":"owned_land","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"property_wealth","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"corporate_wealth","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"private_pension_wealth","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"gross_financial_wealth","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"net_financial_wealth","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"main_residence_value","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"other_residential_property_value","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"non_residential_property_value","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"savings","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"num_vehicles","dtype":"int64","entity":"household","ownership":"produced","rows":"all"},{"column":"cash_isa","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"stocks_and_shares_isa","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"mortgage_debt","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"consumer_debt","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"student_loan_balance","dtype":"float64","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.97.0"],["policyengine-core","3.31.0"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"was_wealth","stage_contract_sha256":"c499dcb7c49a0176229dbc363b400415cb5c1da44af266ba2155f9a62a0bd2ed","time_period":"2024"},"population":"frs_brma.checkpoint","sources":["frs"],"structural":"none","weights":null},{"base":"frs_brma.checkpoint","citation":"","description":"Ownership boundary before regional_property_uprating rewrites.","id":"regional_property_uprating.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","maximum_extended_childcare_hours_usage"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt"],"entity":"household","rows":"all"},{"columns":["age","gender","marital_status","hours_worked","is_household_head","is_benunit_head","is_parent","is_uc_claimant","employment_income","self_employment_income","private_pension_income","tax_free_savings_income","savings_interest_income","dividend_income","property_income","maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","bsp_reported","employment_status","employment_sector","sic_industry_division","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","student_loan_balance"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage regional_property_uprating.","id":"regional_property_uprating","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["region"],"entity":"household","rows":"all"},{"columns":["age","student_loan_balance"],"entity":"person","rows":"all"}],"kernel":"uk.stage.regional_property_uprating@1","mass":"conserve","outputs":[{"column":"main_residence_value","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"property_wealth","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.97.0"],["policyengine-core","3.31.0"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"regional_property_uprating","stage_contract_sha256":"4304356c6c5ba91a04883148cbf9078761ee926c8387aadb6c15b6b70ebb31f6","time_period":"2024"},"population":"regional_property_uprating.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage lcfs_consumption.","id":"lcfs_consumption","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","maximum_extended_childcare_hours_usage"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt"],"entity":"household","rows":"all"},{"columns":["age","gender","marital_status","hours_worked","is_household_head","is_benunit_head","is_parent","is_uc_claimant","employment_income","self_employment_income","private_pension_income","tax_free_savings_income","savings_interest_income","dividend_income","property_income","maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","bsp_reported","employment_status","employment_sector","sic_industry_division","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","student_loan_balance"],"entity":"person","rows":"all"}],"kernel":"uk.stage.lcfs_consumption@1","mass":"conserve","outputs":[{"column":"food_and_non_alcoholic_beverages_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"alcohol_and_tobacco_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"clothing_and_footwear_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"housing_water_and_electricity_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"household_furnishings_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"health_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"transport_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"communication_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"recreation_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"education_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"restaurants_and_hotels_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"miscellaneous_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"petrol_spending","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"diesel_spending","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"bus_fare_spending","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"domestic_energy_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"electricity_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"gas_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"has_fuel_consumption","dtype":"bool","entity":"household","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.97.0"],["policyengine-core","3.31.0"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"lcfs_consumption","stage_contract_sha256":"eb0a529c357c84290a001209a18c5a43b0a0310553c77963d6842fd0111b73ad","time_period":"2024"},"population":"regional_property_uprating.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage etb_vat.","id":"etb_vat","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","maximum_extended_childcare_hours_usage"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","has_fuel_consumption"],"entity":"household","rows":"all"},{"columns":["age","gender","marital_status","hours_worked","is_household_head","is_benunit_head","is_parent","is_uc_claimant","employment_income","self_employment_income","private_pension_income","tax_free_savings_income","savings_interest_income","dividend_income","property_income","maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","bsp_reported","employment_status","employment_sector","sic_industry_division","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","student_loan_balance"],"entity":"person","rows":"all"}],"kernel":"uk.stage.etb_vat@1","mass":"conserve","outputs":[{"column":"full_rate_vat_expenditure_rate","dtype":"float64","entity":"household","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.97.0"],["policyengine-core","3.31.0"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"etb_vat","stage_contract_sha256":"99a6756256adfe8255672fafda89710c08ae56bd2b8f011b20c3b8381eabfa1e","time_period":"2024"},"population":"regional_property_uprating.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage etb_services.","id":"etb_services","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","maximum_extended_childcare_hours_usage"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","has_fuel_consumption","full_rate_vat_expenditure_rate"],"entity":"household","rows":"all"},{"columns":["age","gender","marital_status","hours_worked","is_household_head","is_benunit_head","is_parent","is_uc_claimant","employment_income","self_employment_income","private_pension_income","tax_free_savings_income","savings_interest_income","dividend_income","property_income","maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","bsp_reported","employment_status","employment_sector","sic_industry_division","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","student_loan_balance"],"entity":"person","rows":"all"}],"kernel":"uk.stage.etb_services@1","mass":"conserve","outputs":[{"column":"dfe_education_spending","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"rail_subsidy_spending","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"bus_subsidy_spending","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"rail_usage","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"a_and_e_visits","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"admitted_patient_visits","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"outpatient_visits","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"nhs_a_and_e_spending","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"nhs_admitted_patient_spending","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"nhs_outpatient_spending","dtype":"float64","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.97.0"],["policyengine-core","3.31.0"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"etb_services","stage_contract_sha256":"48294c9d86dd422238a723aca5845e9ceb9eda2903a3bb87bb32a86092447e19","time_period":"2024"},"population":"regional_property_uprating.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage frs_hmrc_spine_leaves.","id":"frs_hmrc_spine_leaves","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["region"],"entity":"household","rows":"all"},{"columns":["age","employee_pension_contributions","nhs_outpatient_spending"],"entity":"person","rows":"all"}],"kernel":"uk.stage.frs_hmrc_spine_leaves@1","mass":"conserve","outputs":[{"column":"hmrc_spi_pay","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"hmrc_spi_unemployment_benefit_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"hmrc_spi_incapacity_benefit_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"ossben_identifiable_subset","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"srp_regular_code5","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"employer_pension_contributions","dtype":"float64","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.97.0"],["policyengine-core","3.31.0"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"frs_hmrc_spine_leaves","stage_contract_sha256":"2bb3d068003489b47cc8676ac26c203ce3c39a9b6e92f3ecd0a3c06acd6addc4","time_period":"2024"},"population":"regional_property_uprating.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":"regional_property_uprating.boundary","citation":"","description":"Run structural UK stage spi_support_channel.","id":"spi_support_channel","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","maximum_extended_childcare_hours_usage"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","has_fuel_consumption","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage"],"entity":"household","rows":"all"},{"columns":["age","gender","marital_status","hours_worked","is_household_head","is_benunit_head","is_parent","is_uc_claimant","employment_income","self_employment_income","private_pension_income","tax_free_savings_income","savings_interest_income","dividend_income","property_income","maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","bsp_reported","employment_status","employment_sector","sic_industry_division","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","student_loan_balance","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions"],"entity":"person","rows":"all"}],"kernel":"uk.stage.expand.spi_support_channel@1","mass":"declared","outputs":[],"params":{"expand_cells":[["person","person_source_id","int64"],["person","person_support_channel","string"],["person","person_support_clone_index","int64"],["benunit","benunit_source_id","int64"],["benunit","benunit_support_channel","string"],["benunit","benunit_support_clone_index","int64"],["household","source_household_id","int64"],["household","source_year","int64"],["household","source_household_key","string"],["household","household_source_id","int64"],["household","household_support_channel","string"],["household","household_support_clone_index","int64"],["household","household_is_spi_synthetic","bool"]],"expand_weight_entity":"household","expand_weight_kind":"importance","numerical_dependencies":[["policyengine-uk","2.97.0"],["policyengine-core","3.31.0"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"spi_support_channel","stage_contract_sha256":"4b10f1a4a215cdf2c406c9974de2d1e580eb3cd3a25ed34973c1b63abf4a54bb","time_period":"2024"},"population":null,"sources":["frs"],"structural":"expand","weights":null},{"base":null,"citation":"","description":"Own the cells materialized by spi_support_channel.","id":"spi_support_channel.owned","inputs":[],"kernel":"uk.claim@1","mass":"conserve","outputs":[{"column":"person_source_id","dtype":"int64","entity":"person","ownership":"produced","rows":"all"},{"column":"person_support_channel","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"person_support_clone_index","dtype":"int64","entity":"person","ownership":"produced","rows":"all"},{"column":"benunit_source_id","dtype":"int64","entity":"benunit","ownership":"produced","rows":"all"},{"column":"benunit_support_channel","dtype":"string","entity":"benunit","ownership":"produced","rows":"all"},{"column":"benunit_support_clone_index","dtype":"int64","entity":"benunit","ownership":"produced","rows":"all"},{"column":"source_household_id","dtype":"int64","entity":"household","ownership":"produced","rows":"all"},{"column":"source_year","dtype":"int64","entity":"household","ownership":"produced","rows":"all"},{"column":"source_household_key","dtype":"string","entity":"household","ownership":"produced","rows":"all"},{"column":"household_source_id","dtype":"int64","entity":"household","ownership":"produced","rows":"all"},{"column":"household_support_channel","dtype":"string","entity":"household","ownership":"produced","rows":"all"},{"column":"household_support_clone_index","dtype":"int64","entity":"household","ownership":"produced","rows":"all"},{"column":"household_is_spi_synthetic","dtype":"bool","entity":"household","ownership":"produced","rows":"all"}],"params":{"materialized_expand_outputs":["person.person_source_id","person.person_support_channel","person.person_support_clone_index","benunit.benunit_source_id","benunit.benunit_support_channel","benunit.benunit_support_clone_index","household.source_household_id","household.source_year","household.source_household_key","household.household_source_id","household.household_support_channel","household.household_support_clone_index","household.household_is_spi_synthetic"]},"population":"spi_support_channel","sources":[],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage hmrc_spi_income_spine.","id":"hmrc_spi_income_spine","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","has_fuel_consumption","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic"],"entity":"household","rows":"all"},{"columns":["age","gender","marital_status","hours_worked","is_household_head","is_benunit_head","is_parent","is_uc_claimant","maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","maintenance_expenses","childcare_expenses","salary_sacrifice_reported","salary_sacrifice_asked","ssmg_reported","incapacity_benefit_reported","employment_status","employment_sector","sic_industry_division","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","student_loan_balance","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending","ossben_identifiable_subset","srp_regular_code5","person_source_id","person_support_channel","person_support_clone_index"],"entity":"person","rows":"all"}],"kernel":"uk.stage.hmrc_spi_income_spine@1","mass":"conserve","outputs":[{"column":"charitable_investment_gifts","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"gift_aid","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"other_investment_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"hmrc_spi_employment_benefits","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"hmrc_spi_employment_expenses","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"hmrc_spi_other_social_security_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"hmrc_spi_taxable_termination_pay","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"hmrc_spi_miscellaneous_employment_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"hmrc_spi_other_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"hmrc_spi_state_pension_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"hmrc_spi_employed_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"hmrc_spi_total_earned_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"hmrc_spi_total_investment_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"hmrc_spi_assessable_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"employment_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"self_employment_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"savings_interest_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"dividend_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"private_pension_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"property_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"employee_pension_contributions","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"employer_pension_contributions","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"personal_pension_contributions","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"pension_contributions_via_salary_sacrifice","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"tax_free_savings_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"universal_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"pension_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"child_benefit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"housing_benefit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"income_support_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"working_tax_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"child_tax_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"attendance_allowance_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"state_pension_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"dla_sc_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"dla_m_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"pip_m_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"pip_dl_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"sda_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"carers_allowance_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"iidb_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"afcs_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"bsp_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"winter_fuel_allowance_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"council_tax_benefit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"jsa_contrib_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"jsa_income_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"esa_contrib_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"esa_income_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"hmrc_spi_pay","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"hmrc_spi_unemployment_benefit_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"hmrc_spi_incapacity_benefit_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"aa_category","dtype":"string","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"dla_sc_category","dtype":"string","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"dla_m_category","dtype":"string","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"pip_m_category","dtype":"string","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"pip_dl_category","dtype":"string","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_disabled_for_benefits","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_enhanced_disabled_for_benefits","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_severely_disabled_for_benefits","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.97.0"],["policyengine-core","3.31.0"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"hmrc_spi_income_spine","stage_contract_sha256":"4c2edca1f77416a00cff10c7e9d0a9da50e332a2bb6a995ef9ac1bdd00ff97a9","time_period":"2024"},"population":"spi_support_channel","sources":["frs"],"structural":"none","weights":null},{"base":"spi_support_channel","citation":"","description":"Ownership boundary before uc_reporter_redraw rewrites.","id":"uc_reporter_redraw.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","has_fuel_consumption","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic"],"entity":"household","rows":"all"},{"columns":["age","gender","marital_status","hours_worked","is_household_head","is_benunit_head","is_parent","is_uc_claimant","employment_income","self_employment_income","private_pension_income","tax_free_savings_income","savings_interest_income","dividend_income","property_income","maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","bsp_reported","employment_status","employment_sector","sic_industry_division","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","student_loan_balance","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","person_source_id","person_support_channel","person_support_clone_index","charitable_investment_gifts","gift_aid","other_investment_income","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage uc_reporter_redraw.","id":"uc_reporter_redraw","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","has_fuel_consumption","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic"],"entity":"household","rows":"all"},{"columns":["age","gender","marital_status","hours_worked","is_household_head","is_benunit_head","is_parent","is_uc_claimant","employment_income","self_employment_income","private_pension_income","tax_free_savings_income","savings_interest_income","dividend_income","property_income","maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","bsp_reported","employment_status","employment_sector","sic_industry_division","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","student_loan_balance","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","person_source_id","person_support_channel","person_support_clone_index","charitable_investment_gifts","gift_aid","other_investment_income","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income"],"entity":"person","rows":"all"}],"kernel":"uk.stage.uc_reporter_redraw@1","mass":"conserve","outputs":[{"column":"universal_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.97.0"],["policyengine-core","3.31.0"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"uc_reporter_redraw","stage_contract_sha256":"7e94e034cad29c5bae566ffb875c42cddf3a1025c15496907e783ceff1144f8b","time_period":"2024"},"population":"uc_reporter_redraw.boundary","sources":["frs"],"structural":"none","weights":null},{"base":"uc_reporter_redraw.boundary","citation":"","description":"Ownership boundary before uc_capital_coherence rewrites.","id":"uc_capital_coherence.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","has_fuel_consumption","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic"],"entity":"household","rows":"all"},{"columns":["age","gender","marital_status","hours_worked","is_household_head","is_benunit_head","is_parent","is_uc_claimant","employment_income","self_employment_income","private_pension_income","tax_free_savings_income","savings_interest_income","dividend_income","property_income","maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","bsp_reported","employment_status","employment_sector","sic_industry_division","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","student_loan_balance","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","person_source_id","person_support_channel","person_support_clone_index","charitable_investment_gifts","gift_aid","other_investment_income","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage uc_capital_coherence.","id":"uc_capital_coherence","inputs":[{"columns":["benunit_support_channel","dependent_children","is_married"],"entity":"benunit","rows":"all"},{"columns":["region"],"entity":"household","rows":"all"},{"columns":["age","is_benunit_head","is_parent","person_support_channel","universal_credit_reported"],"entity":"person","rows":"all"}],"kernel":"uk.stage.uc_capital_coherence@1","mass":"conserve","outputs":[{"column":"uc_reported_capital","dtype":"float64","entity":"benunit","ownership":"produced","rows":"all"},{"column":"frs_benunit_capital","dtype":"float64","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"},{"column":"would_claim_uc","dtype":"bool","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.97.0"],["policyengine-core","3.31.0"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"uc_capital_coherence","stage_contract_sha256":"059f27ee687ee15385b246d43068b17aeecbe3645ccc41fd52fcad06a006a081","time_period":"2024"},"population":"uc_capital_coherence.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage uc_deduction_attributes.","id":"uc_deduction_attributes","inputs":[{"columns":["frs_benunit_capital","would_claim_uc"],"entity":"benunit","rows":"all"},{"columns":["region"],"entity":"household","rows":"all"},{"columns":["age"],"entity":"person","rows":"all"}],"kernel":"uk.stage.uc_deduction_attributes@1","mass":"conserve","outputs":[{"column":"uc_deduction_random_draw","dtype":"float64","entity":"benunit","ownership":"produced","rows":"all"},{"column":"uc_deduction_type_random_draw","dtype":"float64","entity":"benunit","ownership":"produced","rows":"all"},{"column":"uc_latent_deduction_rate","dtype":"float64","entity":"benunit","ownership":"produced","rows":"all"},{"column":"uc_deduction_combination","dtype":"string","entity":"benunit","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.97.0"],["policyengine-core","3.31.0"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"uc_deduction_attributes","stage_contract_sha256":"6bc7e3ec1a5712e23ca1b7d2956e8b17f2526b94431ba3e5a9294b04658e4fe0","time_period":"2024"},"population":"uc_capital_coherence.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":"uc_capital_coherence.boundary","citation":"","description":"Run structural UK stage cgt_incidence_clone.","id":"cgt_incidence_clone","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","uc_deduction_random_draw","uc_deduction_type_random_draw","uc_latent_deduction_rate","uc_deduction_combination"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","has_fuel_consumption","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic"],"entity":"household","rows":"all"},{"columns":["age","gender","marital_status","hours_worked","is_household_head","is_benunit_head","is_parent","is_uc_claimant","employment_income","self_employment_income","private_pension_income","tax_free_savings_income","savings_interest_income","dividend_income","property_income","maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","bsp_reported","employment_status","employment_sector","sic_industry_division","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","student_loan_balance","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","person_source_id","person_support_channel","person_support_clone_index","charitable_investment_gifts","gift_aid","other_investment_income","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income"],"entity":"person","rows":"all"}],"kernel":"uk.stage.expand.cgt_incidence_clone@1","mass":"conserve","outputs":[],"params":{"expand_cells":[["household","household_is_capital_gains_clone","bool"],["person","capital_gains","float64"]],"expand_weight_entity":"household","expand_weight_kind":"importance","numerical_dependencies":[["policyengine-uk","2.97.0"],["policyengine-core","3.31.0"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"cgt_incidence_clone","stage_contract_sha256":"ee30278543cc0297a5366d855b7753aa04af5518971594c193c52be36bf8a7b4","time_period":"2024"},"population":null,"sources":["frs"],"structural":"expand","weights":null},{"base":null,"citation":"","description":"Own the cells materialized by cgt_incidence_clone.","id":"cgt_incidence_clone.owned","inputs":[],"kernel":"uk.claim@1","mass":"conserve","outputs":[{"column":"household_is_capital_gains_clone","dtype":"bool","entity":"household","ownership":"produced","rows":"all"},{"column":"capital_gains","dtype":"float64","entity":"person","ownership":"produced","rows":"all"}],"params":{"materialized_expand_outputs":["household.household_is_capital_gains_clone","person.capital_gains"]},"population":"cgt_incidence_clone","sources":[],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":"cgt_incidence_clone","citation":"","description":"Run structural UK stage cgt_band_donors.","id":"cgt_band_donors","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","uc_deduction_random_draw","uc_deduction_type_random_draw","uc_latent_deduction_rate","uc_deduction_combination"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","has_fuel_consumption","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_capital_gains_clone"],"entity":"household","rows":"all"},{"columns":["age","gender","marital_status","hours_worked","is_household_head","is_benunit_head","is_parent","is_uc_claimant","employment_income","self_employment_income","private_pension_income","tax_free_savings_income","savings_interest_income","dividend_income","property_income","maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","bsp_reported","employment_status","employment_sector","sic_industry_division","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","student_loan_balance","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","person_source_id","person_support_channel","person_support_clone_index","charitable_investment_gifts","gift_aid","other_investment_income","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","capital_gains"],"entity":"person","rows":"all"}],"kernel":"uk.stage.expand.cgt_band_donors@1","mass":"free","outputs":[],"params":{"expand_cells":[["household","household_is_cgt_band_donor","bool"],["person","capital_gains","float64"]],"expand_weight_entity":"household","expand_weight_kind":"importance","numerical_dependencies":[["policyengine-uk","2.97.0"],["policyengine-core","3.31.0"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"cgt_band_donors","stage_contract_sha256":"e10e10c49c0ca2a6a65048c91b683f818e8b6e207a6c6f57a20d76ea299160b0","time_period":"2024"},"population":null,"sources":["frs"],"structural":"expand","weights":null},{"base":null,"citation":"","description":"Own the cells materialized by cgt_band_donors.","id":"cgt_band_donors.owned","inputs":[],"kernel":"uk.claim@1","mass":"conserve","outputs":[{"column":"household_is_cgt_band_donor","dtype":"bool","entity":"household","ownership":"produced","rows":"all"},{"column":"capital_gains","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"materialized_expand_outputs":["household.household_is_cgt_band_donor"]},"population":"cgt_band_donors","sources":[],"structural":"none","weights":null},{"base":"cgt_band_donors","citation":"","description":"Ownership boundary before hmrc_cgt_gains_spine rewrites.","id":"hmrc_cgt_gains_spine.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","uc_deduction_random_draw","uc_deduction_type_random_draw","uc_latent_deduction_rate","uc_deduction_combination"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","has_fuel_consumption","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_capital_gains_clone","household_is_cgt_band_donor"],"entity":"household","rows":"all"},{"columns":["age","gender","marital_status","hours_worked","is_household_head","is_benunit_head","is_parent","is_uc_claimant","employment_income","self_employment_income","private_pension_income","tax_free_savings_income","savings_interest_income","dividend_income","property_income","maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","bsp_reported","employment_status","employment_sector","sic_industry_division","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","student_loan_balance","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","person_source_id","person_support_channel","person_support_clone_index","charitable_investment_gifts","gift_aid","other_investment_income","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","capital_gains"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage hmrc_cgt_gains_spine.","id":"hmrc_cgt_gains_spine","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["region"],"entity":"household","rows":"all"},{"columns":["age","dividend_income","employment_income","miscellaneous_income","private_pension_income","property_income","savings_interest_income","self_employment_income","state_pension_reported","tax_free_savings_income"],"entity":"person","rows":"all"}],"kernel":"uk.stage.hmrc_cgt_gains_spine@1","mass":"conserve","outputs":[{"column":"capital_gains","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.97.0"],["policyengine-core","3.31.0"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"hmrc_cgt_gains_spine","stage_contract_sha256":"20ada8d8ee94400bd160223e865a910db7aff9c40a7f0c8ee8ca8c40ed901b72","time_period":"2024"},"population":"hmrc_cgt_gains_spine.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage salary_sacrifice.","id":"salary_sacrifice","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","uc_deduction_random_draw","uc_deduction_type_random_draw","uc_latent_deduction_rate","uc_deduction_combination"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","has_fuel_consumption","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_capital_gains_clone","household_is_cgt_band_donor"],"entity":"household","rows":"all"},{"columns":["age","gender","marital_status","hours_worked","is_household_head","is_benunit_head","is_parent","is_uc_claimant","employment_income","self_employment_income","private_pension_income","tax_free_savings_income","savings_interest_income","dividend_income","property_income","maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","salary_sacrifice_reported","salary_sacrifice_asked","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","bsp_reported","employment_status","employment_sector","sic_industry_division","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","student_loan_balance","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","person_source_id","person_support_channel","person_support_clone_index","charitable_investment_gifts","gift_aid","other_investment_income","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","capital_gains"],"entity":"person","rows":"all"}],"kernel":"uk.stage.salary_sacrifice@1","mass":"conserve","outputs":[{"column":"pension_contributions_via_salary_sacrifice","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"employee_pension_contributions","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.97.0"],["policyengine-core","3.31.0"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"salary_sacrifice","stage_contract_sha256":"a4af925af21cc6766eb7c8855b6743a3aa4eeeb38df12cfc83063d7ad6890539","time_period":"2024"},"population":"hmrc_cgt_gains_spine.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage student_loans.","id":"student_loans","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["region"],"entity":"household","rows":"all"},{"columns":["age","current_education","employee_pension_contributions","highest_education","student_loan_repayments","student_loans"],"entity":"person","rows":"all"}],"kernel":"uk.stage.student_loans@1","mass":"conserve","outputs":[{"column":"student_loan_plan","dtype":"string","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.97.0"],["policyengine-core","3.31.0"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"student_loans","stage_contract_sha256":"7c4c76398c2e80b41c7f941a5f06246b57d89b5eb0c36d42e3116ac35e82e203","time_period":"2024"},"population":"hmrc_cgt_gains_spine.boundary","sources":["frs"],"structural":"none","weights":null}],"sources":[{"codec":"csv-tables","description":"Content-bound UK FRS and donor fixture/source bundle.","name":"frs"}]} diff --git a/tools/build_uk_frs_spine.py b/tools/build_uk_frs_spine.py index 3512a79e2..493b09350 100644 --- a/tools/build_uk_frs_spine.py +++ b/tools/build_uk_frs_spine.py @@ -1,1493 +1,6 @@ -"""Build the raw UK FRS spine Frame from pinned local tabs.""" - -from __future__ import annotations - -import argparse -import hashlib -import json -import sys -import time -from collections.abc import Mapping, Sequence -from datetime import UTC, datetime -from importlib import metadata -from pathlib import Path - -from microcosm.build.country_spec import ( - GatesManifest, - load_country_spec, -) -from microcosm.build.frame_sampling import ( - normalize_sampled_household_mass, - sample_frame_households, -) -from microcosm.build.gate_battery import BlockingMode, EvidenceContext, GateBatteryRun -from microcosm.build.logbook import canonical_json_bytes -from microcosm.build.logbook_adoption import ( - AttemptState, - append_phase, - apply_error_verdict, - atomic_write_json, - error_receipt_path, - git_code_pin, - local_artifact_reference, - preflight_digest, - record_terminal_attempt, - resolve_predecessor, - role_pins_digest, - sha256_argument, - write_error_receipt, -) -from microcosm.build.plan import StageRecord -from microcosm.build.uk_runtime.age_tail import UKAgeTailStageTransform -from microcosm.build.uk_runtime.battery_bindings import UK_GATE_REGISTRY -from microcosm.build.uk_runtime.calibration_run import ( - UK_SPINE_GATE_SCOPE, - uk_scoped_gate_manifest, -) -from microcosm.build.uk_runtime.cgt_imputation import uk_cgt_spine_stage_transform -from microcosm.build.uk_runtime.cgt_structure import ( - UKCGTBandDonorStageTransform, - UKCGTIncidenceCloneStageTransform, -) -from microcosm.build.uk_runtime.content_identity import uk_frame_content_identity -from microcosm.build.uk_runtime.etb_services import UKETBServicesStageTransform -from microcosm.build.uk_runtime.etb_vat import UKETBVATStageTransform -from microcosm.build.uk_runtime.frs_brma import UKFRSBRMAStageTransform -from microcosm.build.uk_runtime.frs_council_tax import UKFRSCouncilTaxStageTransform -from microcosm.build.uk_runtime.frs_disability import UKFRSDisabilityStageTransform -from microcosm.build.uk_runtime.frs_education import UKFRSEducationStageTransform -from microcosm.build.uk_runtime.frs_education_grants import ( - FRS_EDUCATION_GRANT_REWRITES, - UKFRSEducationGrantSplitStageTransform, -) -from microcosm.build.uk_runtime.frs_employment import UKFRSEmploymentStageTransform -from microcosm.build.uk_runtime.frs_household_draws import ( - UKFRSHouseholdDrawsStageTransform, -) -from microcosm.build.uk_runtime.frs_legacy_proxies import ( - UKFRSLegacyProxiesStageTransform, -) -from microcosm.build.uk_runtime.frs_person_draws import UKFRSPersonDrawsStageTransform -from microcosm.build.uk_runtime.frs_release import load_uk_frs_release -from microcosm.build.uk_runtime.frs_spine import ( - UKFRSSpineStageTransform, - uk_frs_spine_seed_frame, -) -from microcosm.build.uk_runtime.frs_take_up import UKFRSTakeUpStageTransform -from microcosm.build.uk_runtime.graph import ( - UK_SPINE_EXCLUSIONS, - uk_registry, - uk_spine_graph, -) -from microcosm.build.uk_runtime.hmrc_replay import write_hmrc_replay_report -from microcosm.build.uk_runtime.lcfs_consumption import ( - UKLCFSConsumptionStageTransform, -) -from microcosm.build.uk_runtime.national_frame import ( - uk_household_weight_kind, - write_uk_national_frame, -) -from microcosm.build.uk_runtime.national_sampling import ( - UK_SAMPLE_RUNG_TOKENS, - UK_SAMPLE_SEED_DEFAULT, -) -from microcosm.build.uk_runtime.regional_uprating import ( - UKRegionalPropertyUpratingStageTransform, -) -from microcosm.build.uk_runtime.salary_sacrifice import UKSalarySacrificeStageTransform -from microcosm.build.uk_runtime.spi_spine import ( - UKFRSHMRCSpineLeavesStageTransform, - UKSPIIncomeSpineStageTransform, - UKSPISupportChannelStageTransform, -) -from microcosm.build.uk_runtime.student_loans import UKStudentLoansStageTransform -from microcosm.build.uk_runtime.take_up_contract import load_uk_take_up_contract -from microcosm.build.uk_runtime.uc_capital_coherence import ( - UKUCCapitalCoherenceStageTransform, -) -from microcosm.build.uk_runtime.uc_deduction_attributes import ( - UKUCDeductionAttributesStageTransform, -) -from microcosm.build.uk_runtime.uc_reporter_redraw import ( - UKUCReporterRedrawStageTransform, -) -from microcosm.build.uk_runtime.was_wealth import UKWASWealthStageTransform -from microcosm.frame.adapters.policyengine_uk import PolicyEngineUKEngine -from microcosm.graph import ContentStore, compile_graph, run_graph - -_PIPELINE = "uk-frs-spine" -_REPOSITORY = Path(__file__).resolve().parents[1] -_RUNG_NAMED_EDGE_SIGNATURE = "The least populated classes in y have only 1 member" -_RUNG_ABORT_EXIT_CODE = 3 -#: The last stage of the assembled checkpoint: everything through the base -#: FRS mapping and the stochastic draws. A name, not an index — a position -#: standing in for a key is correct only while two independently-maintained -#: orderings happen to agree (the uk-data#468 class). -UK_SPINE_ASSEMBLED_FINAL_STAGE = "frs_brma" - - -def _uk_spine_stage_names(spec) -> tuple[str, ...]: - """Derive the runnable manifest stages from graph ownership edges.""" - - if spec.sources is None: - raise ValueError("UK country spec has no source stages.") - declared = { - stage.stage - for stage in spec.sources.stages - if stage.stage not in UK_SPINE_EXCLUSIONS - } - compiled = compile_graph(uk_spine_graph(spec)) - ordered = tuple(node_id for node_id in compiled.order if node_id in declared) - if set(ordered) != declared: - raise ValueError( - "UK spine graph and manifest stage roster disagree: " - f"graph={list(ordered)!r}, manifest={sorted(declared)!r}." - ) - return ordered - - -def _rung_sample_fraction(value: str) -> float: - """CLI rung policy (#624) over the permissive library validator.""" - - try: - fraction = float(value) - except ValueError as error: - raise argparse.ArgumentTypeError( - f"sample fraction must be a number; got {value!r}." - ) from error - if fraction not in UK_SAMPLE_RUNG_TOKENS: - raise argparse.ArgumentTypeError( - "sample fraction must be one of 0.01, 0.10, or 1.0 (the #624 rungs)." - ) - return fraction - - -def _parse_args(argv: list[str] | None = None) -> argparse.Namespace: - parser = argparse.ArgumentParser( - description=( - "Build the deterministic UK FRS spine from pinned raw tabs. Every " - "stochastic stage draws identity-keyed from seeds declared in the " - "manifest, so two runs from the same inputs are payload-identical." - ) - ) - parser.add_argument( - "--frs-raw-dir", - type=Path, - required=True, - help="Directory containing the 14 licensed FRS 2024-25 tab files.", - ) - parser.add_argument( - "--spine-h5", - type=Path, - required=True, - help="Output H5 path for the raw FRS spine Frame.", - ) - parser.add_argument( - "--spi-tab", - type=Path, - required=True, - help="Pinned local SPI 2022-23 put2223uk.tab path.", - ) - parser.add_argument( - "--hmrc-ods", - type=Path, - required=True, - help="Pinned local HMRC collated ODS path.", - ) - parser.add_argument( - "--cgt-ods", - type=Path, - help="Pinned local HMRC Capital Gains Tax Table 3 ODS path.", - ) - parser.add_argument( - "--checkpoint-dir", - type=Path, - help="Optional directory for a copy of the completed spine checkpoint.", - ) - parser.add_argument( - "--sample-fraction", - type=_rung_sample_fraction, - default=1.0, - help=( - "Scale-ladder rung (#624): 0.01 smoke, 0.10 dev, or 1.0 full. " - "Below 1.0 the raw FRS spine is sampled immediately after ingest, " - "renormalized to full household mass, and treated as a receipt." - ), - ) - parser.add_argument( - "--release-candidate", - action="store_true", - help=( - "Evaluate the spine battery at release-candidate strictness: " - "evidence_absent gaps block instead of being tolerated. Explicit " - "by design - a full-scale developer build is not a release " - "candidate unless the caller says so." - ), - ) - parser.add_argument( - "--sample-seed", - type=int, - default=UK_SAMPLE_SEED_DEFAULT, - help=f"Raw FRS spine sampling seed (default: {UK_SAMPLE_SEED_DEFAULT}).", - ) - parser.add_argument( - "--was-tab", - type=Path, - help="Caller-supplied private WAS round-8 household tab for was_wealth.", - ) - parser.add_argument( - "--lcfs-hh-tab", - type=Path, - help="Caller-supplied private LCFS 2023-24 household tab for lcfs_consumption.", - ) - parser.add_argument( - "--lcfs-person-tab", - type=Path, - help="Caller-supplied private LCFS 2023-24 person tab for lcfs_consumption.", - ) - parser.add_argument( - "--etb-tab", - type=Path, - help="Caller-supplied private ETB 1977-2024 household tab for ETB stages.", - ) - parser.add_argument( - "--emit-nonzero-shares", - type=Path, - help="Optional JSON path for unweighted per-produced-column nonzero shares.", - ) - parser.add_argument( - "--logbook-prev-row-digest", - type=sha256_argument, - help="Optional current Logbook chain head.", - ) - args = parser.parse_args(argv) - if args.sample_seed < 0: - parser.error("sample seed must be a non-negative integer.") - if args.sample_fraction != 1.0 and args.checkpoint_dir is not None: - parser.error( - "sampled spine rungs refuse --checkpoint-dir; rung artifacts are " - "receipts, never releases." - ) - return args - - -def _validate_args(args: argparse.Namespace) -> None: - if not args.frs_raw_dir.is_dir(): - raise ValueError( - f"--frs-raw-dir must be an existing directory: {args.frs_raw_dir}" - ) - if args.spine_h5.suffix != ".h5": - raise ValueError("--spine-h5 must end with '.h5'.") - if not args.spi_tab.is_file(): - raise ValueError(f"--spi-tab must be an existing file: {args.spi_tab}") - if args.spi_tab.name != "put2223uk.tab": - raise ValueError("--spi-tab must name put2223uk.tab.") - if not args.hmrc_ods.is_file(): - raise ValueError(f"--hmrc-ods must be an existing file: {args.hmrc_ods}") - if args.hmrc_ods.suffix.lower() != ".ods": - raise ValueError("--hmrc-ods must end with '.ods'.") - if args.cgt_ods is not None: - if not args.cgt_ods.is_file(): - raise ValueError(f"--cgt-ods must be an existing file: {args.cgt_ods}") - if args.cgt_ods.suffix.lower() != ".ods": - raise ValueError("--cgt-ods must end with '.ods'.") - paths = { - "spine_h5": args.spine_h5, - "build_sidecar": args.spine_h5.with_suffix(".build.json"), - "hmrc_replay_sidecar": args.spine_h5.with_suffix(".hmrc_replay.json"), - } - if args.emit_nonzero_shares is not None: - paths["emit_nonzero_shares"] = args.emit_nonzero_shares - resolved: dict[Path, str] = {} - for label, path in paths.items(): - target = Path(path).expanduser().resolve() - other = resolved.get(target) - if other is not None: - raise ValueError(f"{label} path collides with {other}: {target}.") - resolved[target] = label - - -def _artifact_pins(stages) -> dict[str, dict[str, object]]: - pins = {} - for stage in stages: - for artifact in stage.artifacts: - key = artifact.get("table", artifact.get("filename")) - if key is None: - continue - key = str(key) - pin = { - "locator": str(artifact["locator"]), - "sha256": str(artifact["sha256"]), - "size_bytes": int(artifact["size_bytes"]), - } - if key in pins and pins[key] != pin: - raise ValueError( - f"UK source artifact {key!r} has inconsistent pins across stages." - ) - pins[key] = pin - return dict(sorted(pins.items())) - - -def _stage_artifact_pins(stage) -> dict[str, dict[str, object]]: - return { - str(artifact.get("table", artifact.get("filename"))): { - "locator": str(artifact["locator"]), - "sha256": str(artifact["sha256"]), - "size_bytes": int(artifact["size_bytes"]), - } - for artifact in stage.artifacts - if "table" in artifact or "filename" in artifact - } - - -def _resource_pins(stages, spec) -> dict[str, str]: - """Country-package resources the selected stages declare as inputs. - - Non-tab artifacts reference committed resources by filename; their bytes - are hashed by load_country_spec, so the pin is the spec's recorded sha. - """ - - pins: dict[str, str] = {} - for stage in stages: - for artifact in stage.artifacts: - if "resource" not in artifact: - continue - resource = str(artifact["resource"]) - sha256 = spec.resource_hashes.get(resource) - if sha256 is None: - raise ValueError( - f"stage {stage.stage!r} declares resource artifact " - f"{resource!r} which is not a declared country-package " - "resource." - ) - pins[resource] = str(sha256) - return dict(sorted(pins.items())) - - -def _input_artifact_pins(stages) -> dict[str, dict[str, object]]: - """Caller-supplied private input artifacts, pinned by role. - - Non-table, non-resource artifacts (the SPI donor tab and the HMRC ODS) - carry their own sha256/size pins in the manifest. Binding them here puts - the pins in the build sidecar and the Logbook input-pins digest, so two - runs with different high-impact source inputs can never share build-side - provenance (adversarial-review finding on #717). - """ - - pins: dict[str, dict[str, object]] = {} - for stage in stages: - for artifact in stage.artifacts: - if "table" in artifact or "resource" in artifact: - continue - if "sha256" not in artifact: - continue - role = str(artifact.get("role") or artifact.get("filename") or "") - if not role: - raise ValueError( - f"stage {stage.stage!r} declares a pinned input artifact " - "without a role or filename." - ) - pin = { - "filename": str( - artifact.get("filename") or artifact.get("locator") or "" - ), - "kind": str(artifact.get("kind", "")), - "sha256": str(artifact["sha256"]), - "size_bytes": int(artifact["size_bytes"]), - } - if role in pins and pins[role] != pin: - raise ValueError( - f"input artifact role {role!r} has inconsistent pins across stages." - ) - pins[role] = pin - return dict(sorted(pins.items())) - - -def _role_pins(pins: dict[str, dict[str, object]]) -> dict[str, dict[str, object]]: - return { - table: { - "sha256": str(pin["sha256"]), - "size_bytes": int(pin["size_bytes"]), - } - for table, pin in pins.items() - } - - -def _entity_row_counts(frame) -> dict[str, int]: - return {entity: int(len(frame.table(entity))) for entity in frame.entities} - - -def _rules_engine() -> PolicyEngineUKEngine: - try: - import policyengine_uk # noqa: F401 - except ImportError as exc: - raise ImportError( - "build_uk_frs_spine requires the microcosm-build 'uk' extra " - "(policyengine-uk). Run: uv sync --all-packages --extra uk" - ) from exc - return PolicyEngineUKEngine() - - -def _rules_engine_provenance() -> dict[str, str]: - try: - version = metadata.version("policyengine-uk") - except metadata.PackageNotFoundError: - return {"package": "policyengine-uk", "version": "unavailable"} - return {"package": "policyengine-uk", "version": version} - - -def _declared_seeds(stages) -> dict[str, dict[str, int]]: - declared: dict[str, dict[str, int]] = {} - for stage in stages: - stage_seeds: dict[str, int] = {} - for operation in stage.operations: - output = operation.parameters.get("output") - seed = operation.parameters.get("seed") - if seed is None: - seed = operation.parameters.get("seed_base") - if isinstance(output, str) and isinstance(seed, int): - stage_seeds[output] = seed - elif isinstance(seed, int): - if operation.kind == "stack_zero_weight_donors": - stage_seeds["stack_zero_weight_donors"] = seed - elif operation.kind == "strict_read_private_table": - stage_seeds["donor_bootstrap"] = seed - elif operation.kind == "fit_weighted_qrf_stage1": - stage_seeds["stage1"] = seed - elif operation.kind == "fit_weighted_qrf_stage2": - stage_seeds["stage2"] = seed - elif operation.kind == "bridge_donor_column_via_qrf": - stage_seeds["bridge_donor_column_via_qrf"] = seed - elif operation.kind == "assign_binary_from_rate": - target = operation.parameters.get("target") - if isinstance(target, str): - stage_seeds[target] = seed - else: - stage_seeds["assign_binary_from_rate"] = seed - elif operation.kind == "fit_weighted_qrf_chain": - stage_seeds[stage.stage] = seed - elif operation.kind == "fit_weighted_qrf": - stage_seeds[stage.stage] = seed - elif operation.kind == "draw_capital_gains_prior_from_banded_quantiles": - stage_seeds[str(operation.parameters["salt"])] = seed - elif operation.kind == "stack_band_donor_households": - stage_seeds["stack_band_donor_households"] = seed - elif operation.kind == "within_band_draws": - stage_seeds["within_band_draws"] = seed - elif operation.kind == "convert_donors_to_target_stock": - stage_seeds[str(operation.parameters["salt"])] = seed - elif operation.kind == "top_up_to_stock": - stage_seeds[str(operation.parameters["salt"])] = seed - if stage_seeds: - declared[stage.stage] = stage_seeds - return declared - - -def _result_evidence(result: object) -> object: - if isinstance(result, dict): - return result - evidence = getattr(result, "evidence", None) - if callable(evidence): - return evidence() - return None - - -def _collect_stage_evidence( - *, - stage_names: Sequence[str], - implementations: Mapping[str, object], -) -> dict[str, object]: - evidence_by_stage: dict[str, object] = {} - for stage_name in stage_names: - implementation = implementations.get(stage_name) - if implementation is None: - continue - metadata = None - metadata_hook = getattr(implementation, "checkpoint_metadata", None) - if callable(metadata_hook): - metadata = dict(metadata_hook()) - evidence = metadata.get("evidence", metadata) - else: - evidence = _result_evidence(getattr(implementation, "last_result", None)) - if evidence is not None: - evidence_by_stage[stage_name] = evidence - return evidence_by_stage - - -def _collect_fit_weight_records( - *, - stage_names: Sequence[str], - implementations: Mapping[str, object], -) -> dict[str, list[dict[str, str]]]: - """Persist each fitting stage's resolved weight kinds into the sidecar. - - The terminal weights audit (``uk_weights_audit``) consumes - :class:`FitWeightRecord` evidence that only exists on live stage - objects; the release-cut certification producer runs in a later - process, so the sidecar carries the records across the run boundary. - Duck-typed like ``stage_evidence``: every stage whose transform exposes - ``fit_weight_records`` contributes, in stage order. A fitting stage - whose records are missing, unreadable, or empty records an empty list — - the audit binding fails an empty record set, so the gap stays visible - rather than vanishing from the sidecar. - """ - - records_by_stage: dict[str, list[dict[str, str]]] = {} - for stage_name in stage_names: - implementation = implementations.get(stage_name) - if implementation is None: - continue - # Detect the hook without evaluating it: a raising property must - # count as a fitting stage with unreadable records, not vanish. - exposes_records = getattr( - type(implementation), "fit_weight_records", None - ) is not None or "fit_weight_records" in getattr(implementation, "__dict__", {}) - if not exposes_records: - continue - try: - records = tuple(implementation.fit_weight_records or ()) - except Exception: # noqa: BLE001 - unreadable records fail the audit - records_by_stage[stage_name] = [] - continue - records_by_stage[stage_name] = [ - { - "fit_name": str(record.fit_name), - "weight_kind": str(record.weight_kind), - } - for record in records - ] - return records_by_stage - - -def _build_sidecar( - *, - frame, - stages, - records, - artifact_pins, - resource_pins: dict[str, str], - input_artifact_pins: dict[str, dict[str, object]], - hmrc_replay: dict[str, object], - stochastic_contract_sha256: str, - frs_vintage: str, - sampling: dict[str, object] | None, - spine_gate_report: dict[str, object] | None = None, -) -> dict[str, object]: - household_weight = frame.weights_for("household") - return { - "schema_version": 2, - "pipeline": _PIPELINE, - "uk_frame_content_identity": uk_frame_content_identity(frame), - "stages": [stage.stage for stage in stages], - "time_period": str(frame.metadata["time_period"]), - "household_weight_kind": uk_household_weight_kind(frame).value, - "household_weight_total": float(household_weight.values.sum()), - "entity_row_counts": _entity_row_counts(frame), - "artifact_pins": artifact_pins, - "resource_pins": resource_pins, - "input_artifact_pins": input_artifact_pins, - "hmrc_replay": hmrc_replay, - "stage_artifact_pins": { - stage.stage: _stage_artifact_pins(stage) for stage in stages - }, - "stage_records": [ - { - "stage": record.stage, - "produced": list(record.produced), - "nonzero_share": dict(record.nonzero_share), - "seconds": record.seconds, - } - for record in records - ], - "operations": { - stage.stage: [operation.kind for operation in stage.operations] - for stage in stages - }, - "declared_seeds": _declared_seeds(stages), - "source_vintages": {"frs": frs_vintage}, - "sampling": sampling, - "spine_gate_report": spine_gate_report, - "stochastic_contract_sha256": stochastic_contract_sha256, - "rules_engine": _rules_engine_provenance(), - } - - -def _nonzero_shares(frame, columns: list[str]) -> dict[str, float]: - shares: dict[str, float] = {} - for column in columns: - for entity in frame.entities: - table = frame.table(entity) - if column not in table.columns: - continue - values = table[column] - if values.dtype == object: - shares[column] = float(values.astype(str).ne("").mean()) - else: - shares[column] = float((values != 0).mean()) - break - return shares - - -def _series_nonzero_share(values) -> float: - if values.dtype == object or str(values.dtype).startswith("string"): - return float(values.fillna("").astype(str).ne("").mean()) - return float((values != 0).mean()) - - -def _graph_stage_records( - *, - manifest, - store: ContentStore, - stages, - frame, -) -> tuple[StageRecord, ...]: - """Project immediate node artifacts onto the legacy record schema. - - Entity ids and memberships are executor-carried context, not owned cells, - so the root node exposes no artifact for them although ``frs_spine`` - declares them as outputs. Their share is read from the final population - instead, which is what the legacy plan recorded (identity columns are - never zero, so the value is 1.0 on every vintage). - """ - - structural = _structural_columns(frame) - records: list[StageRecord] = [] - for stage in stages: - output_node = ( - f"{stage.stage}.owned" - if f"{stage.stage}.owned" in manifest.nodes - else stage.stage - ) - output_receipt = manifest.nodes[output_node] - shares: dict[str, float] = {} - for column in stage.outputs: - matches = [ - (coordinate, key) - for coordinate, key in output_receipt.artifacts.items() - if coordinate[1] == column - ] - if not matches and column in structural: - shares[column] = _nonzero_shares(frame, [column])[column] - continue - if len(matches) != 1: - raise RuntimeError( - f"graph stage {stage.stage!r} exposes {len(matches)} artifacts " - f"for declared output {column!r}." - ) - shares[column] = _series_nonzero_share(store.load_column(matches[0][1])) - execution_node = "create_uk_frs" if stage.stage == "frs_spine" else stage.stage - records.append( - StageRecord( - stage=stage.stage, - produced=stage.outputs, - donor_survey=stage.survey, - nonzero_share=shares, - seconds=manifest.nodes[execution_node].wall_time, - ) - ) - return tuple(records) - - -def _structural_columns(frame) -> frozenset[str]: - """Entity id and membership columns the executor carries outside owned cells.""" - - schema = frame.schema - columns = {schema.entity_id_column(entity) for entity in frame.entities} - columns.update(schema.membership_column(group) for group in schema.group_entities) - return frozenset(columns) - - -def _new_build_id(timestamp: datetime) -> str: - return f"uk-frs-spine-{timestamp.strftime('%Y%m%dT%H%M%SZ')}" - - -def _record_attempt( - *, - state: AttemptState, - started_at: float, - started_ts: datetime, - code_pin: str, - disposition: str, - predecessor: str | None, - rung: str, - spool_dir: Path, -) -> Path: - return record_terminal_attempt( - state=state, - started_at=started_at, - started_ts=started_ts, - pipeline=_PIPELINE, - rung=rung, - seed=None, - code_pin=code_pin, - disposition=disposition, - predecessor=predecessor, - spool_dir=spool_dir, - ) - - -def _sample_spine_frame( - frame, - *, - fraction: float, - seed: int, -) -> tuple[object, dict[str, object] | None]: - if fraction == 1.0: - return frame, None - household_weight = frame.weights_for("household") - pre_households = int(len(frame.table("household"))) - sampled, receipt = sample_frame_households( - frame, - fraction=fraction, - seed=seed, - source_name="UK FRS spine", - ) - normalized, factor = normalize_sampled_household_mass( - sampled, - target_mass=float(household_weight.total), - source_name="UK FRS spine", - ) - return normalized, { - "fraction": float(fraction), - "seed": int(seed), - "rung_token": UK_SAMPLE_RUNG_TOKENS[fraction], - "pre_household_count": pre_households, - "post_household_count": int(len(normalized.table("household"))), - "normalization_factor": float(factor), - "receipt": dict(receipt), - } - - -class _SampledGraphRootTransform: - """CREATE-stage adapter applying the declared sampling rung at ingest.""" - - def __init__(self, transform, *, fraction: float, seed: int) -> None: - self.transform = transform - self.fraction = fraction - self.seed = seed - self.sampling: dict[str, object] | None = None - - def _sample(self, assembled): - sampled, self.sampling = _sample_spine_frame( - assembled, - fraction=self.fraction, - seed=self.seed, - ) - return sampled - - def __call__(self, frame): - return self._sample(self.transform(frame)) - - def run_with_sources(self, frame, sources): - runner = getattr(self.transform, "run_with_sources", None) - assembled = ( - runner(frame, sources) if callable(runner) else self.transform(frame) - ) - return self._sample(assembled) - - def checkpoint_metadata(self) -> dict[str, object]: - hook = getattr(self.transform, "checkpoint_metadata", None) - if not callable(hook): - raise RuntimeError("FRS root transform exposes no checkpoint metadata.") - return dict(hook()) - - -class _GraphSourceTransform: - """Build a file-reading stage from only the node's declared source paths.""" - - def __init__(self, factory) -> None: - self.factory = factory - self.transform = None - - def run_with_sources(self, frame, sources): - self.transform = self.factory(sources) - result = self.transform(frame) - if hasattr(self.transform, "fit_weight_records"): - self.fit_weight_records = self.transform.fit_weight_records - return result - - def __getattr__(self, name: str): - transform = self.__dict__.get("transform") - if transform is None: - raise AttributeError(name) - return getattr(transform, name) - - -def _run_plan_with_spine_sampling( - plan, - *, - sample_fraction: float, - sample_seed: int, - spine_battery: GateBatteryRun | None = None, - stage_evidence_provider=None, - gate_artifacts: Mapping[str, object] | None = None, -) -> tuple[object, tuple[object, ...], dict[str, object] | None]: - if not plan.stages or plan.stages[0].name != "frs_spine": - frame, records = plan.run(uk_frs_spine_seed_frame()) - return frame, records, None - - from microcosm.build.plan import StagePlan - - spine_frame, spine_records = StagePlan(plan.stages[:1]).run( - uk_frs_spine_seed_frame() - ) - spine_frame, sampling = _sample_spine_frame( - spine_frame, - fraction=sample_fraction, - seed=sample_seed, - ) - if len(plan.stages) == 1: - return spine_frame, spine_records, sampling - names = tuple(stage.name for stage in plan.stages) - if UK_SPINE_ASSEMBLED_FINAL_STAGE in names: - assembled_end = names.index(UK_SPINE_ASSEMBLED_FINAL_STAGE) + 1 - elif spine_battery is not None: - raise RuntimeError( - "spine battery is armed but the declared assembled-boundary stage " - f"{UK_SPINE_ASSEMBLED_FINAL_STAGE!r} is not in the plan; a stage " - "plan change must move the boundary declaration with it." - ) - else: - assembled_end = len(plan.stages) - frame, assembled_records = StagePlan(plan.stages[1:assembled_end]).run(spine_frame) - # Each boundary offers only the stages that have actually run: asking a - # later stage for checkpoint evidence would (correctly) raise, and the - # first licensed battery run did exactly that at the assembled boundary. - executed = tuple(stage.name for stage in plan.stages[:assembled_end]) - if spine_battery is not None: - _run_spine_gate_phase( - spine_battery, - "assembled", - frame=frame, - stage_evidence=( - stage_evidence_provider(executed) - if stage_evidence_provider is not None - else {} - ), - gate_artifacts=gate_artifacts, - ) - if assembled_end == len(plan.stages): - return frame, (*spine_records, *assembled_records), sampling - frame, tail_records = StagePlan(plan.stages[assembled_end:]).run(frame) - executed = tuple(stage.name for stage in plan.stages) - if spine_battery is not None: - _run_spine_gate_phase( - spine_battery, - "transferred", - frame=frame, - stage_evidence=( - stage_evidence_provider(executed) - if stage_evidence_provider is not None - else {} - ), - gate_artifacts=gate_artifacts, - ) - return frame, (*spine_records, *assembled_records, *tail_records), sampling - - -def _run_spine_gate_phase( - battery: GateBatteryRun, - phase: str, - *, - frame, - stage_evidence: Mapping[str, object], - gate_artifacts: Mapping[str, object] | None = None, -) -> None: - artifacts: dict[str, object] = {"stage_evidence": dict(stage_evidence)} - # The enum-domain gate resolves its domain from the live rules engine, - # exactly as the national terminal battery supplied it. - artifacts.update(dict(gate_artifacts or {})) - battery.run_phase( - phase, - EvidenceContext(frame=frame, artifacts=artifacts), - ) - battery.enforce(phase, mode=BlockingMode.BLOCKS_ARTIFACT) - - -def _spine_gate_report_path(spine_h5: Path) -> Path: - return spine_h5.with_suffix(".spine_gates.json") - - -def _spine_gate_manifest_from_spec(spec) -> GatesManifest | None: - """The spine build's scoped battery manifest, from the shared helper. - - A spec without a gates block leaves the battery unarmed (``None``), - exactly as before; when armed, the filtering runs through the one - scope-filtering implementation every scoped producer shares. The - driver passes the spec it already loaded, which is also the hermetic - tests' stub point. Digests are identical to the previous local copy - because entries, phases, and the policy suffix are unchanged. - """ - - source = getattr(spec, "gates", None) - if source is None: - return None - return uk_scoped_gate_manifest( - UK_SPINE_GATE_SCOPE, - phases=("assembled", "transferred"), - policy_suffix="spine_build_scope", - source=source, - ) - - -def _rung_abort_receipt( - args: argparse.Namespace, - *, - error: BaseException, -) -> dict[str, object]: - return { - "schema_version": 1, - "artifact_kind": "uk_frs_spine_rung_abort_receipt", - "build_kind": "uk_frs_spine", - "sampling": { - "sample_fraction": float(args.sample_fraction), - "sample_seed": int(args.sample_seed), - "rung_token": UK_SAMPLE_RUNG_TOKENS[args.sample_fraction], - }, - "named_edge": "spine_split_singleton_class", - "stage": "frs_spine", - "error": str(error), - "disposition": "aborted_with_receipt", - "remedy": ( - "Re-roll --sample-seed; accepted dev-scale statistical edge. " - "The computation is never altered to avoid it." - ), - } - - -def _exception_chain_contains(error: BaseException, text: str) -> bool: - """Match a named rung edge through graph execution wrappers.""" - - seen: set[int] = set() - current: BaseException | None = error - while current is not None and id(current) not in seen: - seen.add(id(current)) - if text in str(current): - return True - current = current.__cause__ or current.__context__ - return False - - -def main(argv: list[str] | None = None) -> int: - args = _parse_args(argv) - rung = UK_SAMPLE_RUNG_TOKENS[args.sample_fraction] - started_at = time.perf_counter() - started_ts = datetime.now(UTC) - predecessor = resolve_predecessor(args.logbook_prev_row_digest) - digest = preflight_digest(_PIPELINE) - state = AttemptState( - build_id=_new_build_id(started_ts), - identity_digest=digest, - input_pins_digest=digest, - phases_reached=["attempt_started"], - gate_verdicts={ - "pipeline": { - "verdict": "running", - "receipt": "pending-build-scoped-spine-receipt", - } - }, - ) - code_pin = "unresolved-local-git-code-pin" - spool_dir = args.spine_h5.parent / "logbook-spool" - try: - _validate_args(args) - # A crash between the H5 write and the sidecar writes must never - # leave a stale sidecar beside a fresh H5 (adversarial-review - # finding on #717): clear every output up front, and treat the - # build sidecar - written last, binding the replay hash - as the - # marker that the bundle is complete. - stale_outputs = [ - args.spine_h5, - args.spine_h5.with_suffix(".build.json"), - args.spine_h5.with_suffix(".hmrc_replay.json"), - _spine_gate_report_path(args.spine_h5), - args.spine_h5.with_suffix(".rung_abort.json"), - ] - if args.emit_nonzero_shares is not None: - stale_outputs.append(args.emit_nonzero_shares) - for stale in stale_outputs: - stale.unlink(missing_ok=True) - code_pin = git_code_pin(_REPOSITORY) - append_phase(state, "configured") - spec = load_country_spec("uk") - if spec.sources is None: - raise ValueError("UK country spec has no source stages.") - stages_by_name = spec.sources.stage_map() - graph = uk_spine_graph( - spec, - source_mode="split", - sample_fraction=args.sample_fraction, - sample_seed=args.sample_seed, - ) - compiled_graph = compile_graph(graph) - stage_names = _uk_spine_stage_names(spec) - if "hmrc_cgt_gains_spine" in stage_names and args.cgt_ods is None: - raise ValueError( - "--cgt-ods is required when hmrc_cgt_gains_spine is scheduled." - ) - if "was_wealth" in stage_names and args.was_tab is None: - raise ValueError( - "--was-tab is required when the was_wealth stage is scheduled." - ) - if "lcfs_consumption" in stage_names: - missing_lcfs = [ - flag - for flag, value in ( - ("--lcfs-hh-tab", args.lcfs_hh_tab), - ("--lcfs-person-tab", args.lcfs_person_tab), - ("--was-tab", args.was_tab), - ) - if value is None - ] - if missing_lcfs: - raise ValueError( - "lcfs_consumption requires caller-supplied private inputs: " - f"{', '.join(missing_lcfs)}." - ) - if ( - "etb_vat" in stage_names or "etb_services" in stage_names - ) and args.etb_tab is None: - raise ValueError( - "--etb-tab is required when etb_vat or etb_services is scheduled." - ) - stages = [stages_by_name[name] for name in stage_names] - artifact_pins = _artifact_pins(stages) - resource_pins = _resource_pins(stages, spec) - input_artifact_pins = _input_artifact_pins(stages) - overlapping_pin_roles = set(artifact_pins) & set(input_artifact_pins) - if overlapping_pin_roles: - raise ValueError( - "input artifact roles collide with FRS tab names: " - f"{sorted(overlapping_pin_roles)}." - ) - state.input_pins_digest = role_pins_digest( - _role_pins({**artifact_pins, **input_artifact_pins}) - ) - run_config = { - "pipeline": _PIPELINE, - "stages": list(stage_names), - "artifact_pins_digest": state.input_pins_digest, - "spine_h5": str(args.spine_h5), - } - state.identity_digest = hashlib.sha256( - canonical_json_bytes(run_config) - ).hexdigest() - append_phase(state, "inputs_pinned") - engine = _rules_engine() - stochastic_contract = load_uk_take_up_contract() - frs_release = load_uk_frs_release() - hmrc_spine_transform = _GraphSourceTransform( - lambda sources: UKSPIIncomeSpineStageTransform( - sources["spi"], - sources["hmrc_income"], - stage=stages_by_name["hmrc_spi_income_spine"], - sampled_rung=args.sample_fraction != 1.0, - ) - ) - implementations = { - "frs_spine": _GraphSourceTransform( - lambda sources: UKFRSSpineStageTransform( - sources["frs"], - stage=stages_by_name["frs_spine"], - ) - ), - "frs_employment": _GraphSourceTransform( - lambda sources: UKFRSEmploymentStageTransform( - sources["frs"], - stage=stages_by_name["frs_employment"], - ) - ), - "frs_council_tax": _GraphSourceTransform( - lambda sources: UKFRSCouncilTaxStageTransform( - sources["frs"], - stage=stages_by_name["frs_council_tax"], - ) - ), - "frs_disability": UKFRSDisabilityStageTransform( - stage=stages_by_name["frs_disability"], - ), - "frs_education": _GraphSourceTransform( - lambda sources: UKFRSEducationStageTransform( - sources["frs"], - stage=stages_by_name["frs_education"], - ) - ), - "frs_legacy_proxies": _GraphSourceTransform( - lambda sources: UKFRSLegacyProxiesStageTransform( - sources["frs"], - stage=stages_by_name["frs_legacy_proxies"], - engine=engine, - ) - ), - "frs_education_grant_split": ( - UKFRSEducationGrantSplitStageTransform( - stage=stages_by_name["frs_education_grant_split"], - engine=engine, - ) - ), - "frs_take_up": UKFRSTakeUpStageTransform( - contract=stochastic_contract, - stage=stages_by_name["frs_take_up"], - ), - "frs_person_draws": UKFRSPersonDrawsStageTransform( - contract=stochastic_contract, - stage=stages_by_name["frs_person_draws"], - ), - "frs_household_draws": UKFRSHouseholdDrawsStageTransform( - contract=stochastic_contract, - stage=stages_by_name["frs_household_draws"], - ), - "frs_brma": UKFRSBRMAStageTransform( - stage=stages_by_name["frs_brma"], - engine=engine, - ), - } - if "was_wealth" in stage_names: - implementations["was_wealth"] = _GraphSourceTransform( - lambda sources: UKWASWealthStageTransform( - stage=stages_by_name["was_wealth"], - engine=engine, - was_tab_path=sources["was"], - ) - ) - if "regional_property_uprating" in stage_names: - implementations["regional_property_uprating"] = ( - UKRegionalPropertyUpratingStageTransform( - stage=stages_by_name["regional_property_uprating"], - ) - ) - if "lcfs_consumption" in stage_names: - implementations["lcfs_consumption"] = _GraphSourceTransform( - lambda sources: UKLCFSConsumptionStageTransform( - stage=stages_by_name["lcfs_consumption"], - engine=engine, - lcfs_hh_tab_path=sources["lcfs_household"], - lcfs_person_tab_path=sources["lcfs_person"], - was_tab_path=sources["was"], - ) - ) - if "etb_vat" in stage_names: - implementations["etb_vat"] = _GraphSourceTransform( - lambda sources: UKETBVATStageTransform( - stage=stages_by_name["etb_vat"], - engine=engine, - etb_tab_path=sources["etb"], - ) - ) - if "etb_services" in stage_names: - implementations["etb_services"] = _GraphSourceTransform( - lambda sources: UKETBServicesStageTransform( - stage=stages_by_name["etb_services"], - engine=engine, - etb_tab_path=sources["etb"], - ) - ) - implementations["frs_hmrc_spine_leaves"] = _GraphSourceTransform( - lambda sources: UKFRSHMRCSpineLeavesStageTransform( - sources["frs"], - stage=stages_by_name["frs_hmrc_spine_leaves"], - sampled_rung=args.sample_fraction != 1.0, - ) - ) - implementations["spi_support_channel"] = UKSPISupportChannelStageTransform( - stage=stages_by_name["spi_support_channel"], - sample_fraction=args.sample_fraction, - ) - implementations["hmrc_spi_income_spine"] = hmrc_spine_transform - if "uc_reporter_redraw" in stage_names: - implementations["uc_reporter_redraw"] = UKUCReporterRedrawStageTransform( - stage=stages_by_name["uc_reporter_redraw"], - engine=engine, - ) - if "uc_capital_coherence" in stage_names: - implementations["uc_capital_coherence"] = ( - UKUCCapitalCoherenceStageTransform( - stage=stages_by_name["uc_capital_coherence"] - ) - ) - if "uc_deduction_attributes" in stage_names: - implementations["uc_deduction_attributes"] = ( - UKUCDeductionAttributesStageTransform( - stage=stages_by_name["uc_deduction_attributes"] - ) - ) - if "cgt_incidence_clone" in stage_names: - implementations["cgt_incidence_clone"] = UKCGTIncidenceCloneStageTransform( - stage=stages_by_name["cgt_incidence_clone"] - ) - if "cgt_band_donors" in stage_names: - implementations["cgt_band_donors"] = UKCGTBandDonorStageTransform( - stage=stages_by_name["cgt_band_donors"] - ) - if "hmrc_cgt_gains_spine" in stage_names: - implementations["hmrc_cgt_gains_spine"] = _GraphSourceTransform( - lambda sources: uk_cgt_spine_stage_transform( - stages_by_name["hmrc_cgt_gains_spine"], - sources["hmrc_cgt"], - ) - ) - if "salary_sacrifice" in stage_names: - implementations["salary_sacrifice"] = UKSalarySacrificeStageTransform( - stage=stages_by_name["salary_sacrifice"] - ) - if "student_loans" in stage_names: - implementations["student_loans"] = UKStudentLoansStageTransform( - stage=stages_by_name["student_loans"], - calibration_year=frs_release.calibration_year, - ) - if "age_tail" in stage_names: - implementations["age_tail"] = UKAgeTailStageTransform( - stage=stages_by_name["age_tail"] - ) - sampled_root = _SampledGraphRootTransform( - implementations["frs_spine"], - fraction=args.sample_fraction, - seed=args.sample_seed, - ) - implementations["frs_spine"] = sampled_root - spine_gate_path = _spine_gate_report_path(args.spine_h5) - spine_gate_manifest = _spine_gate_manifest_from_spec(spec) - spine_battery = ( - GateBatteryRun( - spine_gate_manifest, - release_id=state.build_id, - report_path=spine_gate_path, - release_candidate=args.release_candidate, - registry=UK_GATE_REGISTRY, - ) - if spine_gate_manifest is not None - else None - ) - checkpoint_root = ( - args.checkpoint_dir - if args.checkpoint_dir is not None - else args.spine_h5.parent / f".{args.spine_h5.stem}.checkpoints" - ) - graph_sources = {"frs": args.frs_raw_dir} - if "was_wealth" in stage_names or "lcfs_consumption" in stage_names: - graph_sources["was"] = args.was_tab - if "lcfs_consumption" in stage_names: - graph_sources["lcfs_household"] = args.lcfs_hh_tab - graph_sources["lcfs_person"] = args.lcfs_person_tab - if "etb_vat" in stage_names or "etb_services" in stage_names: - graph_sources["etb"] = args.etb_tab - if "hmrc_spi_income_spine" in stage_names: - graph_sources["spi"] = args.spi_tab - graph_sources["hmrc_income"] = args.hmrc_ods - if "hmrc_cgt_gains_spine" in stage_names: - graph_sources["hmrc_cgt"] = args.cgt_ods - graph_store = ContentStore(checkpoint_root / "node-graph") - graph_manifest = run_graph( - compiled_graph, - sources=graph_sources, - store=graph_store, - kernels=uk_registry(implementations, graph=graph), - resume="forbid", - decisions=(), - ) - final_version = compiled_graph.versions[compiled_graph.order[-1]] - frame = graph_manifest.population(final_version) - records = _graph_stage_records( - manifest=graph_manifest, - store=graph_store, - stages=stages, - frame=frame, - ) - sampling = sampled_root.sampling - if spine_battery is not None: - if UK_SPINE_ASSEMBLED_FINAL_STAGE not in stage_names: - raise RuntimeError( - "spine battery is armed but the graph has no assembled " - f"boundary stage {UK_SPINE_ASSEMBLED_FINAL_STAGE!r}." - ) - assembled_index = stage_names.index(UK_SPINE_ASSEMBLED_FINAL_STAGE) + 1 - assembled_version = compiled_graph.versions[UK_SPINE_ASSEMBLED_FINAL_STAGE] - _run_spine_gate_phase( - spine_battery, - "assembled", - frame=graph_manifest.population(assembled_version), - stage_evidence=_collect_stage_evidence( - stage_names=stage_names[:assembled_index], - implementations=implementations, - ), - gate_artifacts={"rules_engine": engine}, - ) - if assembled_index < len(stage_names): - _run_spine_gate_phase( - spine_battery, - "transferred", - frame=frame, - stage_evidence=_collect_stage_evidence( - stage_names=stage_names, - implementations=implementations, - ), - gate_artifacts={"rules_engine": engine}, - ) - if spine_battery is not None: - append_phase(state, "spine_gates_evaluated") - append_phase(state, "spine_built") - output = write_uk_national_frame(frame, args.spine_h5) - append_phase(state, "spine_written") - if args.checkpoint_dir is not None: - args.checkpoint_dir.mkdir(parents=True, exist_ok=True) - write_uk_national_frame(frame, args.checkpoint_dir / "frs_spine.h5") - append_phase(state, "checkpoint_written") - sidecar_path = output.with_suffix(".build.json") - replay_sidecar_path = output.with_suffix(".hmrc_replay.json") - if hmrc_spine_transform.last_result is None: - raise RuntimeError("HMRC SPI spine stage did not record replay evidence.") - write_hmrc_replay_report( - hmrc_spine_transform.last_result.replay_report, - replay_sidecar_path, - ) - append_phase(state, "hmrc_replay_sidecar_written") - replay_bytes = replay_sidecar_path.read_bytes() - replay_binding = { - "filename": replay_sidecar_path.name, - "report_kind": str(json.loads(replay_bytes).get("report_kind", "")), - "sha256": hashlib.sha256(replay_bytes).hexdigest(), - } - sidecar = _build_sidecar( - frame=frame, - stages=stages, - records=records, - artifact_pins=artifact_pins, - resource_pins=resource_pins, - input_artifact_pins=input_artifact_pins, - hmrc_replay=replay_binding, - stochastic_contract_sha256=stochastic_contract.resource_sha256, - frs_vintage=frs_release.vintage, - sampling=sampling, - spine_gate_report=( - { - "path": str(spine_gate_path), - "sha256": hashlib.sha256(spine_gate_path.read_bytes()).hexdigest(), - } - if spine_gate_path.is_file() - else None - ), - ) - stage_evidence = _collect_stage_evidence( - stage_names=stage_names, - implementations=implementations, - ) - if stage_evidence: - sidecar["stage_evidence"] = stage_evidence - fit_weight_records = _collect_fit_weight_records( - stage_names=stage_names, - implementations=implementations, - ) - if fit_weight_records: - sidecar["fit_weight_records"] = fit_weight_records - atomic_write_json(sidecar_path, sidecar) - append_phase(state, "build_sidecar_written") - if args.emit_nonzero_shares is not None: - final_columns = list( - dict.fromkeys( - [column for record in records for column in record.produced] - + list(FRS_EDUCATION_GRANT_REWRITES) - ) - ) - atomic_write_json( - args.emit_nonzero_shares, - { - "stages": { - record.stage: dict(record.nonzero_share) for record in records - }, - "final": _nonzero_shares(frame, final_columns), - }, - ) - append_phase(state, "nonzero_shares_written") - state.artifact_location = local_artifact_reference( - output, - repository_hint=_REPOSITORY, - ) - state.gate_verdicts = { - "pipeline": { - "verdict": "passed", - "receipt": local_artifact_reference( - sidecar_path, repository_hint=_REPOSITORY - ), - } - } - if spine_gate_path.is_file(): - gate_payload = json.loads(spine_gate_path.read_text(encoding="utf-8")) - for gate_id, payload in gate_payload.get("gates", {}).items(): - state.gate_verdicts[str(gate_id)] = { - "verdict": str(payload.get("status")), - "receipt": ( - f"{local_artifact_reference(spine_gate_path, repository_hint=_REPOSITORY)}" - f"#/gates/{gate_id}" - ), - } - spool_path = _record_attempt( - state=state, - started_at=started_at, - started_ts=started_ts, - code_pin=code_pin, - disposition="iterating", - predecessor=predecessor, - rung=rung, - spool_dir=spool_dir, - ) - print(f"Wrote FRS spine H5: {output}", file=sys.stderr) - print(f"Wrote Logbook row: {spool_path}", file=sys.stderr) - return 0 - except Exception as error: - if args.sample_fraction != 1.0 and _exception_chain_contains( - error, _RUNG_NAMED_EDGE_SIGNATURE - ): - rung_abort_path = args.spine_h5.with_suffix(".rung_abort.json") - receipt = _rung_abort_receipt(args, error=error) - atomic_write_json(rung_abort_path, receipt) - state.gate_verdicts = { - "uk_frs_spine_rung_abort": { - "verdict": "aborted", - "receipt": ( - f"{local_artifact_reference(rung_abort_path, repository_hint=_REPOSITORY)}" - "#/named_edge" - ), - } - } - append_phase(state, "rung_aborted") - _record_attempt( - state=state, - started_at=started_at, - started_ts=started_ts, - code_pin=code_pin, - disposition="discarded", - predecessor=predecessor, - rung=rung, - spool_dir=spool_dir, - ) - print(json.dumps(receipt, indent=2, sort_keys=True)) - return _RUNG_ABORT_EXIT_CODE - try: - receipt_path = write_error_receipt( - error_receipt_path(args.spine_h5.parent, build_id=state.build_id), - state=state, - pipeline=_PIPELINE, - error=error, - ) - apply_error_verdict( - state, - local_artifact_reference(receipt_path, repository_hint=_REPOSITORY), - ) - _record_attempt( - state=state, - started_at=started_at, - started_ts=started_ts, - code_pin=code_pin, - disposition="failed", - predecessor=predecessor, - rung=rung, - spool_dir=spool_dir, - ) - except Exception: - pass - print(f"UK FRS spine build failed: {error}", file=sys.stderr) - return 1 +"""Build the UK raw-source spine using the installed package driver.""" +from microcosm.build.uk_runtime.spine_build import main if __name__ == "__main__": raise SystemExit(main()) diff --git a/tools/build_uk_full.py b/tools/build_uk_full.py new file mode 100644 index 000000000..8d60eee8a --- /dev/null +++ b/tools/build_uk_full.py @@ -0,0 +1,6 @@ +"""Canonical UK full build: all applicable geographies calibrated together.""" + +from microcosm.build.uk_runtime.full_build_cli import main + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tools/build_uk_release_input_coverage_manifest.py b/tools/build_uk_release_input_coverage_manifest.py index 61fda0eea..0dd4890b8 100644 --- a/tools/build_uk_release_input_coverage_manifest.py +++ b/tools/build_uk_release_input_coverage_manifest.py @@ -43,7 +43,6 @@ REFERENCE_PATH = UK_PACKAGE_DIR / "efrs_parity_reference.json" KNOWN_GAPS_PATH = UK_PACKAGE_DIR / "efrs_parity_known_gaps.json" MANIFEST_PATH = UK_PACKAGE_DIR / "release_input_coverage_manifest.json" -HMRC_SOURCE_STAGES_PATH = UK_PACKAGE_DIR / "hmrc_income_source_stages.json" CGT_SOURCE_STAGES_PATH = UK_PACKAGE_DIR / "cgt_source_stages.json" SOURCE_STAGES_PATH = UK_PACKAGE_DIR / "source_stages.json" @@ -96,6 +95,7 @@ "promotion_basis": "weighted release gate stale-exclusion remediation", "reviewed_on": "2026-07-13", "stage": "hmrc_spi_income", + "current_producer_stage": "hmrc_spi_income_spine", "support_channel": "spi", }, "gift_aid": { @@ -105,6 +105,7 @@ "promotion_basis": "weighted release gate stale-exclusion remediation", "reviewed_on": "2026-07-13", "stage": "hmrc_spi_income", + "current_producer_stage": "hmrc_spi_income_spine", "support_channel": "spi", }, } @@ -809,9 +810,6 @@ def build_manifest( stage_name="student_loans", candidate_source=candidate_source, ), - "hmrc_cgt_gains": _cgt_family_coverage_contract( - candidate_source=candidate_source, - ), "hmrc_spi_income": _hmrc_family_coverage_contract( candidate_source=candidate_source ), @@ -857,118 +855,6 @@ def build_manifest( } -def _cgt_family_coverage_contract( - *, - candidate_source: dict[str, Any], -) -> dict[str, Any]: - """Validate the CGT source manifest and emit its family contract. - - The stage redraws capital gains amounts only, so unlike the SPI family it - moves no mass and declares no distributional effective-mass requirement: - ``capital_gains`` already carries hard release status from the candidate, - and the stage replaces its values in place. - """ - - payload = _load(CGT_SOURCE_STAGES_PATH) - stages = payload.get("stages") - if not isinstance(stages, list) or len(stages) != 1: - raise ValueError( - f"{CGT_SOURCE_STAGES_PATH}: expected exactly one source stage." - ) - stage = stages[0] - if not isinstance(stage, dict) or stage.get("stage") != "hmrc_cgt_gains": - raise ValueError(f"{CGT_SOURCE_STAGES_PATH}: expected hmrc_cgt_gains stage.") - base_candidate = stage.get("base_candidate") - if not isinstance(base_candidate, dict): - raise ValueError(f"{CGT_SOURCE_STAGES_PATH}: base_candidate must be an object.") - source_tier = validate_uk_release_tier(candidate_source.get("tier")) - base_candidate_tier = validate_uk_release_tier(base_candidate.get("tier")) - if base_candidate_tier != source_tier: - raise ValueError( - "CGT source-stage base candidate tier disagrees with the certified " - f"candidate evidence: {base_candidate_tier!r} != {source_tier!r}." - ) - artifacts = { - artifact["role"]: artifact - for artifact in stage.get("artifacts", []) - if isinstance(artifact, dict) and isinstance(artifact.get("role"), str) - } - operations = { - operation["kind"]: operation - for operation in stage.get("operations", []) - if isinstance(operation, dict) and isinstance(operation.get("kind"), str) - } - required_artifacts = {"published_fact_surface", "policy_parameters"} - missing_artifacts = sorted(required_artifacts - set(artifacts)) - required_operations = { - "verify_certified_candidate", - "verify_pinned_cgt_ods", - "taxable_income_proxy", - "rank_preserving_allocation", - "within_band_draws", - "sub_aea_remainder", - "record_mass_conservation_receipt", - "classify_cgt_band_facts_with_reviewed_fence", - } - missing_operations = sorted(required_operations - set(operations)) - if missing_artifacts or missing_operations: - raise ValueError( - f"{CGT_SOURCE_STAGES_PATH}: incomplete CGT family contract; " - f"missing_artifacts={missing_artifacts}, " - f"missing_operations={missing_operations}." - ) - surface = artifacts["published_fact_surface"] - verify = operations["verify_pinned_cgt_ods"] - fence = operations["classify_cgt_band_facts_with_reviewed_fence"] - if not bool(verify.get("require_before_source_read")): - raise ValueError( - f"{CGT_SOURCE_STAGES_PATH}: the pinned ODS must be verified before " - "it is read." - ) - if bool(fence.get("calibration_permitted", True)): - raise ValueError( - f"{CGT_SOURCE_STAGES_PATH}: the band-fact fence must keep " - "calibration_permitted false; promotion goes through a separately " - "reviewed target profile." - ) - if str(surface.get("sha256", "")) == "" or int(surface.get("size_bytes", 0)) <= 0: - raise ValueError( - f"{CGT_SOURCE_STAGES_PATH}: published_fact_surface must pin sha256 " - "and size_bytes." - ) - return { - "status": "required_at_build", - "stage": "hmrc_cgt_gains", - "source_manifest": CGT_SOURCE_STAGES_PATH.name, - "source_manifest_sha256": _sha256(CGT_SOURCE_STAGES_PATH), - "superseded_by": { - "stage": "hmrc_cgt_gains_spine", - "source_manifest": SOURCE_STAGES_PATH.name, - "source_manifest_sha256": _sha256(SOURCE_STAGES_PATH), - "reason": ( - "The FRS spine build executes hmrc_cgt_gains_spine, which " - "applies the same HMRC Table 3 amounts redraw directly in " - "source_stages.json before calibration." - ), - }, - "base_candidate_sha256": str(base_candidate["sha256"]), - "base_candidate_tier": base_candidate_tier, - "source_vintages": { - "hmrc_surface": str(surface["vintage"]), - "mapped_build_period": str(surface["mapped_build_period"]), - }, - "output_weight_kind": str(stage["output_weight_kind"]), - "required_mass_change_reason": str( - operations["record_mass_conservation_receipt"]["reason"] - ), - "calibration_permitted": bool(fence["calibration_permitted"]), - "fact_fence_id": str(fence["fact_fence_id"]), - "fenced_fact_count": int(fence["fenced_fact_count"]), - "outputs": list(stage.get("outputs", [])), - "effective_mass_requirements": {}, - } - - def _source_stage_family_coverage_contract( *, stage_name: str, @@ -1121,77 +1007,28 @@ def _hmrc_family_coverage_contract( *, candidate_source: dict[str, Any], ) -> dict[str, Any]: - payload = _load(HMRC_SOURCE_STAGES_PATH) - stages = payload.get("stages") - if not isinstance(stages, list) or len(stages) != 1: - raise ValueError( - f"{HMRC_SOURCE_STAGES_PATH}: expected exactly one source stage." - ) - stage = stages[0] - if not isinstance(stage, dict) or stage.get("stage") != "hmrc_spi_income": - raise ValueError(f"{HMRC_SOURCE_STAGES_PATH}: expected hmrc_spi_income stage.") - canonical_payload = _load(SOURCE_STAGES_PATH) - canonical_stages = canonical_payload.get("stages") - if not isinstance(canonical_stages, list): - raise ValueError(f"{SOURCE_STAGES_PATH}: expected source stages list.") - canonical_matches = [ - candidate - for candidate in canonical_stages - if isinstance(candidate, dict) and candidate.get("stage") == "hmrc_spi_income" - ] - if len(canonical_matches) != 1: - raise ValueError( - f"{SOURCE_STAGES_PATH}: expected exactly one hmrc_spi_income stage." - ) - canonical_stage = canonical_matches[0] - base_candidate = stage.get("base_candidate") - if not isinstance(base_candidate, dict): - raise ValueError( - f"{HMRC_SOURCE_STAGES_PATH}: base_candidate must be an object." - ) - source_tier = validate_uk_release_tier(candidate_source.get("tier")) - base_candidate_tier = validate_uk_release_tier(base_candidate.get("tier")) - if base_candidate_tier != source_tier: - raise ValueError( - "HMRC source-stage base candidate tier disagrees with the certified " - f"candidate evidence: {base_candidate_tier!r} != {source_tier!r}." - ) - artifacts = { - artifact["role"]: artifact - for artifact in stage.get("artifacts", []) - if isinstance(artifact, dict) and isinstance(artifact.get("role"), str) - } - canonical_artifacts = { - artifact["role"]: artifact - for artifact in canonical_stage.get("artifacts", []) - if isinstance(artifact, dict) and isinstance(artifact.get("role"), str) - } - operations = { - operation["kind"]: operation - for operation in stage.get("operations", []) - if isinstance(operation, dict) and isinstance(operation.get("kind"), str) - } - required_artifacts = {"qrf_donor", "published_fact_surface"} - missing_artifacts = sorted(required_artifacts - set(artifacts)) - missing_canonical_artifacts = sorted(required_artifacts - set(canonical_artifacts)) - required_operations = { - "retain_adjudicated_frs_hmrc_leaves", - "verify_pinned_hmrc_source_pair", - "replace_zero_weight_spi_support", - "classify_hmrc_income_facts_with_reviewed_fences", - "gate_distributional_effective_mass", - } - missing_operations = sorted(required_operations - set(operations)) - if missing_artifacts or missing_canonical_artifacts or missing_operations: - raise ValueError( - f"{HMRC_SOURCE_STAGES_PATH}: incomplete HMRC family contract; " - f"missing_artifacts={missing_artifacts}, " - f"missing_canonical_artifacts={missing_canonical_artifacts}, " - f"missing_operations={missing_operations}." - ) + from microcosm.build.uk_runtime.hmrc_source_contract import ( + assert_uk_hmrc_income_source_contract_current, + ) + from microcosm.build.uk_runtime.spi_support import SPI_PRIOR_MASS_CHANGE_REASON + + assert_uk_hmrc_income_source_contract_current(SOURCE_STAGES_PATH) + payload = _load(SOURCE_STAGES_PATH) + stages = {stage["stage"]: stage for stage in payload["stages"]} + stage = stages["hmrc_spi_income_spine"] + artifacts = {artifact["role"]: artifact for artifact in stage["artifacts"]} + operations = {operation["kind"]: operation for operation in stage["operations"]} + frs_leaves = next( + operation + for operation in stages["frs_hmrc_spine_leaves"]["operations"] + if operation["kind"] == "retain_adjudicated_frs_hmrc_leaves" + ) + prior = next( + operation + for operation in stages["spi_support_channel"]["operations"] + if operation["kind"] == "allocate_zero_weight_prior_mass" + ) classification = operations["classify_hmrc_income_facts_with_reviewed_fences"] - frs_leaves = operations["retain_adjudicated_frs_hmrc_leaves"] - prior = operations["replace_zero_weight_spi_support"] effective = operations["gate_distributional_effective_mass"] floor = float(effective["minimum_nondefault_mass_share"]) if floor != EFFECTIVE_MASS_COVERAGE["minimum_nondefault_mass_share"]: @@ -1230,42 +1067,23 @@ def _hmrc_family_coverage_contract( # truthfully retains the 208-fact adjudicated-partial-replay verdict. "status": "required_at_build", "restoration_status": str(frs_leaves["status"]), - "stage": "hmrc_spi_income", - "source_manifest": HMRC_SOURCE_STAGES_PATH.name, - "source_manifest_sha256": _sha256(HMRC_SOURCE_STAGES_PATH), - # The two re-mapped period fields below come from the CANONICAL - # manifest (the #723 signed re-map lives there; the frozen mirror - # keeps its June bytes), so the bytes they derive from are pinned - # separately - evidence fields and their hash must name the same - # source (adversarial-review finding, 2026-08-20). - "canonical_source_manifest": SOURCE_STAGES_PATH.name, - "canonical_source_manifest_sha256": _sha256(SOURCE_STAGES_PATH), - "superseded_by": { - "stage": "hmrc_spi_income_spine", - "source_manifest": SOURCE_STAGES_PATH.name, - "source_manifest_sha256": _sha256(SOURCE_STAGES_PATH), - "reason": ( - "The FRS spine build executes hmrc_spi_income_spine, which " - "supersedes the June retained-leaves/hmrc_spi_income pair " - "inside source_stages.json." - ), - }, - "base_candidate_sha256": str(base_candidate["sha256"]), - "base_candidate_tier": base_candidate_tier, + "stage": "hmrc_spi_income_spine", + "source_manifest": SOURCE_STAGES_PATH.name, + "source_manifest_sha256": _sha256(SOURCE_STAGES_PATH), + "base_candidate_tier": validate_uk_release_tier(candidate_source["tier"]), + "required_predecessor_stages": ["frs_hmrc_spine_leaves", "spi_support_channel"], "source_vintages": { "spi_donor": str(artifacts["qrf_donor"]["vintage"]), "hmrc_surface": str(artifacts["published_fact_surface"]["vintage"]), "mapped_build_period": str( - canonical_artifacts["published_fact_surface"]["mapped_build_period"] + artifacts["published_fact_surface"]["mapped_build_period"] ), "period_mapping": str( - canonical_artifacts["published_fact_surface"]["period_mapping"] + artifacts["published_fact_surface"]["period_mapping"] ), }, - "spi_prior_national_household_mass_share": float( - prior["spi_prior_national_household_mass_share"] - ), - "required_mass_change_reason": str(prior["mass_change_reason"]), + "spi_prior_national_household_mass_share": float(prior["share"]), + "required_mass_change_reason": SPI_PRIOR_MASS_CHANGE_REASON, "input_weight_kind": str(classification["input_weight_kind"]), "output_weight_kind": str(classification["output_weight_kind"]), "calibration_permitted": bool(classification["calibration_permitted"]), diff --git a/tools/build_uk_rowwise_candidate.py b/tools/build_uk_rowwise_candidate.py index e683d4664..0faeb0dd0 100644 --- a/tools/build_uk_rowwise_candidate.py +++ b/tools/build_uk_rowwise_candidate.py @@ -1,3189 +1,13 @@ -"""Build a joint local, ladder, and national UK rowwise candidate. +"""Compatibility entry point for the canonical UK full build. -Pinned Ledger facts supply the local and national registries. The command -samples before cloning, resolves both local grains and national measures on the -cloned frame, and calibrates every row in one doctrine solve. A dry run compiles -the registries and reports analytical matrix/support evidence without running -the policy engine, solving, or writing output files. - -For pre-#762 synthetic fixtures, omitting the Ledger arguments retains the -adjudicated constituency-household compatibility path. +The independent rowwise candidate driver is retired. Both command names now +use the same graph and default to all applicable target geographies. Request +--target-geographies country explicitly for a country-only target filter. +Legacy census-only, independent solver, and logbook options are no longer +accepted; use --help for the maintained full-build request and output options. """ -from __future__ import annotations - -import argparse -import dataclasses -import hashlib -import json -import shutil -import subprocess -import sys -import tempfile -import time -import uuid -from collections.abc import Mapping -from datetime import UTC, date, datetime -from pathlib import Path -from typing import Any - -import numpy as np -import pandas as pd - -from microcosm.build.gate_battery import ( - BlockingMode, - EvidenceContext, - GateBatteryBlockedError, - GateBatteryRun, -) -from microcosm.build.gates import GateResult -from microcosm.build.ledger_artifact import load_ledger_consumer_artifact -from microcosm.build.logbook import canonical_json_bytes -from microcosm.build.logbook_adoption import ( - AttemptState, - append_phase, - apply_error_verdict, - atomic_write_json, - error_receipt_path, - git_code_pin, - local_artifact_reference, - preflight_digest, - record_terminal_attempt, - resolve_predecessor, - role_pins_digest, - sha256_argument, - write_error_receipt, -) -from microcosm.build.target_materialization import resolve_target_measures -from microcosm.build.uk_runtime import ( - UK_GATE_REGISTRY, - UK_LOCAL_CLONE_COUNT, - UK_LOCAL_MAX_WEIGHT_RATIO, - UK_LOCAL_SOLVE_DOCTRINE, - UK_LOCAL_SOLVE_EPOCHS, - UK_LOCAL_TARGET_LOSS_CAP, - CalibrationFrameAdapter, - UKLadderRowwiseDatasetResult, - UkOaLadder, - UKRowwiseDoctrineSolve, - UKRowwiseLocalMatrix, - UKRowwiseNationalRows, - apply_uk_cross_grain_reconciliation, - build_uk_rowwise_local_matrix, - build_uk_rowwise_local_surface_matrix, - clone_uk_dataset_with_ladder_geography, - compile_uk_local_target_registry, - compile_uk_target_registry, - compute_household_metrics, - constituency_household_targets, - drop_injected_measure_inputs, - inject_measure_inputs, - ladder_clone_index_column, - ladder_target_provenance, - load_bound_spine_sidecar, - load_uk_local_area_crosswalk, - load_uk_national_frame, - load_uk_oa_ladder, - local_target_census, - materialize_uk_ledger_targets, - require_adjudicated_uk_local_binding, - rotated_uk_local_holdout, - runtime_provenance, - solve_uk_rowwise_weights_under_doctrine, - spine_provenance_from_sidecar, - uk_fit_by_family, - uk_household_weight_kind, - uk_ladder_area_support_summary, - uk_ladder_household_uprating, - uk_ledger_households_total, - uk_local_doctrine_with_overrides, - uk_local_target_surface, - uk_support_limited_misses, - uk_time_period, - uk_weight_summary, - write_uk_calibration_diagnostics, - write_uk_rowwise_dataset, -) -from microcosm.build.uk_runtime.calibration_run import ( - UK_LOCAL_GATE_SCOPE, - finalize_uk_scoped_gate_report, - uk_local_gate_scope_exclusions, - uk_scoped_gate_manifest, -) -from microcosm.build.uk_runtime.frs_release import load_uk_frs_release -from microcosm.build.uk_runtime.ledger_targets import _spec_geography -from microcosm.build.uk_runtime.measure_simulation import ( - UKMeasureResolver, - apply_uk_calibration_measure_exclusions, - load_uk_calibration_measure_exclusions, -) -from microcosm.build.uk_runtime.national_sampling import ( - UK_SAMPLE_RUNG_TOKENS, - UK_SAMPLE_SEED_DEFAULT, - sample_uk_spine_frame, -) -from microcosm.calibrate import TargetRegistry, TargetSpec -from microcosm.frame import Frame, MassChangeRecord - -BOUND_TARGET_FAMILIES = ("census_households/constituency",) -BOUND_NATIONAL_TARGETS: tuple[str, ...] = () -CANDIDATE_FILENAME_TEMPLATE = "microcosm_uk_{calibration_year}_local.h5" -LOCAL_GATE_REPORT_FILENAME_TEMPLATE = ( - "microcosm_uk_{calibration_year}_local.local_gates.json" -) -MANIFEST_FILENAME = "rowwise_candidate_manifest.json" -SOLVE_DIAGNOSTICS_FILENAME = "solve_diagnostics.csv" -CALIBRATION_DIAGNOSTICS_FILENAME = "calibration_diagnostics.json" -AREA_SUPPORT_FILENAME = "area_support_summary.csv" -PAST_CAP_FILENAME = "past_cap_census.json" -LOCAL_REGISTRY_FILENAME = "local_target_registry.json" -DENSE_REFERENCE_DIAGNOSTICS_FILENAME = "dense_reference_diagnostics.csv" -DATASET_SIZE_SELECTION_FILENAME = "dataset_size_selection.csv" - -#: Outputs a run writes only when ``--dataset-households`` is set. -_SIZE_RUN_ONLY_OUTPUTS = frozenset({"dense_reference", "selection"}) - -_CONSERVE_MASS = False -_TARGET_RECORDS: int | None = None -_L0_LAMBDA = 0.0 -_BUDGET_ITERS = 10 -_UK_CANDIDATE_PIPELINE = "uk-local-candidate" -_LOCAL_GATE_POLICY_SUFFIX = "local_candidate" -_REPOSITORY = Path(__file__).resolve().parents[1] -_PAST_CAP_COUNT_KEYS = ( - "n_targets", - "past_at_init", - "past_at_final", - "escaped", - "frozen", - "pushed_out", -) - - -class _LadderAssignment: - """A clone paired in memory with the exact ladder object that produced it.""" - - def __init__( - self, - result: UKLadderRowwiseDatasetResult, - ladder: UkOaLadder, - ) -> None: - self.result = result - self.ladder = ladder - - -def _new_candidate_build_id( - *, seed: int, timestamp: datetime, rung: str = "f100" -) -> str: - instant = timestamp.astimezone(UTC) - return ( - f"uk-local-candidate-{rung}-s{seed}-" - f"{instant.strftime('%Y%m%dT%H%M%SZ')}-{uuid.uuid4().hex[:8]}" - ) - - -def _candidate_clone_counts_argument(value: str) -> tuple[int, ...]: - parts = value.split(",") - if not value.strip() or any(not part.strip() for part in parts): - raise argparse.ArgumentTypeError( - "candidate clone counts must be a non-empty comma list of positive integers" - ) - try: - counts = [int(part.strip()) for part in parts] - except ValueError as error: - raise argparse.ArgumentTypeError( - "candidate clone counts must be a comma list of positive integers" - ) from error - if any(count <= 0 for count in counts): - raise argparse.ArgumentTypeError( - "candidate clone counts must all be positive integers" - ) - return tuple(sorted(set(counts))) - - -def _sample_candidate_frame( - frame, - *, - fraction: float, - seed: int, -) -> tuple[Any, dict[str, Any]]: - """Sample spine families below f100; keep the full rung untouched.""" - - pre_count = len(frame.table("household")) - if fraction == 1.0: - return frame, { - "fraction": 1.0, - "seed": int(seed), - "rung_token": UK_SAMPLE_RUNG_TOKENS[fraction], - "sampled": False, - "pre_household_count": int(pre_count), - "post_household_count": int(pre_count), - } - - sampled, receipt = sample_uk_spine_frame( - frame, - fraction=fraction, - seed=seed, - ) - return sampled, {"sampled": True, **receipt} - - -def _resolve_candidate_engine_surface( - frame, - national_registry, - *, - period: int, - scratch_dir: Path, - band_edge_registry=None, - resolver_factory=UKMeasureResolver, - blocks: int = 1, -) -> tuple[Any, Any, UKRowwiseNationalRows, dict[str, pd.DataFrame], dict[str, Any]]: - """Resolve national inputs and local metrics on the cloned frame. - - ``blocks=1`` uses one scratch-mode engine for the whole clone. The - reviewed escape hatch ``blocks=K`` resolves each clone index separately, - then rejoins every entity-level prepared column by its stable entity id so - the full-frame target materialization and single solve retain frame order. - """ - - household = frame.table("household") - if blocks < 1: - raise ValueError("engine resolution blocks must be positive.") - if blocks == 1: - block_frames = [(None, frame)] - else: - clone_column = ladder_clone_index_column("household") - if clone_column not in household.columns: - raise ValueError(f"per-clone engine resolution requires {clone_column}.") - clone_indices = tuple(sorted(household[clone_column].unique().tolist())) - if len(clone_indices) != blocks: - raise ValueError( - "engine resolution blocks must match the realized clone indices: " - f"requested {blocks}, found {clone_indices}." - ) - person = frame.table("person") - block_frames = [] - for clone_index in clone_indices: - household_ids = set( - household.loc[ - household[clone_column] == clone_index, - "household_id", - ].tolist() - ) - person_mask = person["person_household_id"].isin(household_ids) - block = frame.select(person_mask) - # The block carries a K-th of the cloned mass while its log still - # ends on the full-clone record, and the scratch export validates - # the chain. Declare the subset explicitly: old = the cloned - # total, new = the block total, reason naming the block. The block - # frame is engine scratch and is discarded after resolution. - block_weights = block.weights_for("household") - full_total = float(frame.weights_for("household").total) - block_total = float(block_weights.total) - subset_record = MassChangeRecord( - entity="household", - old_total=full_total, - new_total=block_total, - declared_factor=block_total / full_total, - reason=( - f"engine resolution block {clone_index} of {blocks}: " - "scratch subset of the cloned frame for measure " - "resolution only, discarded after resolution" - ), - ) - block = Frame( - { - **{name: block.table(name) for name in block.entities}, - **{name: block.link(name) for name in block.links}, - }, - block.schema, - { - entity: block.weights_for(entity) - for entity in block.weighted_entities - }, - block.strata, - mass_log=(*block.mass_log, subset_record), - metadata=block.metadata, - ) - block_frames.append((clone_index, block)) - - measure_parts: dict[tuple[str, str], list[pd.Series]] = {} - metric_parts: dict[str, list[pd.DataFrame]] = { - "constituency": [], - "la": [], - } - resolver_receipts: list[Mapping[str, Any]] = [] - national_input_keys: set[tuple[str, str]] | None = None - for clone_index, block_frame in block_frames: - block_scratch = ( - scratch_dir if clone_index is None else scratch_dir / f"clone-{clone_index}" - ) - resolver = resolver_factory( - simulation_source=None, - scratch_dir=block_scratch, - year=period, - frame=block_frame, - ) - resolution = resolve_target_measures( - lambda block_frame=block_frame: CalibrationFrameAdapter(block_frame), - national_registry, - resolver, - period=period, - ) - keys = set(resolution.measure_inputs) - if national_input_keys is None: - national_input_keys = keys - elif keys != national_input_keys: - raise RuntimeError( - "per-clone engine resolution returned inconsistent national inputs." - ) - for (entity, variable), values in resolution.measure_inputs.items(): - entity_table = block_frame.table(entity) - entity_id = f"{entity}_id" - measure_parts.setdefault((entity, variable), []).append( - pd.Series( - np.asarray(values), - index=entity_table[entity_id].tolist(), - ) - ) - block_household_ids = block_frame.table("household")["household_id"].tolist() - for area_type in metric_parts: - metric_parts[area_type].append( - compute_household_metrics( - resolver.simulation, - area_type, - period=period, - household_ids=block_household_ids, - ) - ) - resolver_receipts.append(resolver.receipt()) - del resolver - simulation_input = block_scratch / "simulation-input.h5" - simulation_input.unlink(missing_ok=True) - try: - block_scratch.rmdir() - except OSError: - pass - - measure_inputs: dict[tuple[str, str], np.ndarray] = {} - for (entity, variable), parts in measure_parts.items(): - combined = pd.concat(parts) - if combined.index.has_duplicates: - raise RuntimeError( - f"per-clone engine resolution duplicated {entity} ids for {variable}." - ) - ordered_ids = frame.table(entity)[f"{entity}_id"] - ordered = combined.reindex(ordered_ids.tolist()) - if ordered.isna().any(): - raise RuntimeError( - f"per-clone engine resolution missed {entity} rows for {variable}." - ) - measure_inputs[(entity, variable)] = ordered.to_numpy() - - full_household_ids = household["household_id"].tolist() - local_metrics = {} - for area_type, parts in metric_parts.items(): - combined = pd.concat(parts) - if combined.index.has_duplicates: - raise RuntimeError( - f"per-clone engine resolution duplicated {area_type} household ids." - ) - ordered = combined.reindex(full_household_ids) - if ordered.isna().any().any(): - raise RuntimeError( - f"per-clone engine resolution missed {area_type} household rows." - ) - local_metrics[area_type] = ordered - - adapter = CalibrationFrameAdapter(frame) - # Injected engine inputs are scratch state for materialization only: - # they must be dropped before the prepared frame is assembled, or the - # flattening rule refuses columns that now exist on two entities - # (region, esa_* on the live spine). Same lifecycle as the national stage. - original_columns = { - entity: set(table.columns) for entity, table in adapter.tables.items() - } - inject_measure_inputs(adapter, measure_inputs) - materialized = materialize_uk_ledger_targets( - adapter, - national_registry, - period=period, - band_edge_registry=( - national_registry if band_edge_registry is None else band_edge_registry - ), - ) - if materialized.skipped: - raise RuntimeError( - "candidate national target materialization skipped row(s): " - f"{[skip.__dict__ for skip in materialized.skipped]}." - ) - modes = {receipt.get("mode") for receipt in resolver_receipts} - versions = {receipt.get("policyengine_uk_version") for receipt in resolver_receipts} - if len(modes) != 1 or len(versions) != 1: - raise RuntimeError("per-clone engine resolver provenance is inconsistent.") - cgt_period_contract = resolver_receipts[0].get("cgt_period_contract") - if any( - block_receipt.get("cgt_period_contract") != cgt_period_contract - for block_receipt in resolver_receipts[1:] - ): - raise RuntimeError("per-clone CGT period contract is inconsistent.") - receipt = { - "mode": next(iter(modes)), - "engine_version": next(iter(versions)), - "households": len(frame.table("household")), - "persons": len(frame.table("person")), - "benunits": len(frame.table("benunit")), - "national_inputs": len(measure_inputs), - "local_metrics": { - area_type: len(metrics.columns) - for area_type, metrics in local_metrics.items() - }, - "blocks": blocks, - } - if cgt_period_contract is not None: - receipt["cgt_period_contract"] = cgt_period_contract - if blocks > 1: - receipt["deviation"] = "per_clone_block_engine_resolution" - present = sorted( - column - for column in UK_BLOCK_SENSITIVE_MEASURE_COLUMNS - if column in measure_inputs - ) - receipt["block_sensitivity"] = { - "known_population_normalised_measures": list( - UK_BLOCK_SENSITIVE_MEASURE_COLUMNS - ), - "present_in_this_run": present, - "caveat": ( - "per-block engine resolution mis-measures population-normalised " - "formulas (each block reproduces a national aggregate); rows " - "on these measures are not evidence for adjudication from this " - "run. Resolve in a single block before ruling on them." - ), - } - try: - scratch_dir.rmdir() - except OSError: - pass - drop_injected_measure_inputs(adapter, measure_inputs, original_columns) - national_rows = UKRowwiseNationalRows( - targets=national_registry.to_target_set(), - registry=national_registry, - families=tuple(sorted({spec.family for spec in national_registry.specs})), - ) - return ( - adapter.prepared_frame(), - adapter.restore, - national_rows, - local_metrics, - receipt, - ) - - -def _pin_from_artifact(info: Mapping[str, Any]) -> dict[str, object]: - return { - "sha256": str(info["sha256"]), - "size_bytes": int(info["bytes"]), - } - - -def _stderr_progress(line: str) -> None: - """Solver progress (epoch losses, budget probes, the search verdict).""" - print(line, file=sys.stderr, flush=True) - - -def _refuse_stale_size_checkpoint(args: argparse.Namespace, out_dir: Path) -> None: - """Refuse an --out holding a checkpoint before the solve, not after it. - - The checkpoint writer refuses to overwrite, but it runs after the dense - solve and the search; a stale checkpoint in --out must fail here, before - the hours are spent. - """ - if args.dataset_households is None or args.no_size_checkpoint: - return - if args.resume_size_checkpoint is not None: - return - from microcosm.build.uk_runtime.size_checkpoint import ( - SIZE_CHECKPOINT_ARRAYS_FILENAME, - SIZE_CHECKPOINT_MANIFEST_FILENAME, - ) - - existing = sorted( - str(out_dir / name) - for name in (SIZE_CHECKPOINT_ARRAYS_FILENAME, SIZE_CHECKPOINT_MANIFEST_FILENAME) - if (out_dir / name).exists() - ) - if existing: - raise FileExistsError( - "refusing to run into an --out that already holds a size checkpoint: " - f"{existing}. Resume from it with --resume-size-checkpoint, or choose " - "another --out." - ) - - -def _size_checkpoint_identity( - args: argparse.Namespace, - *, - pins: Mapping[str, Mapping[str, object]], - source_year: int, -) -> dict[str, object]: - """Everything a size checkpoint must share with the run that resumes it. - - The pool (spine, ladder, clones, seed, sampling), the target surface - (ledger digests, year, rule, engine blocks) and the solve settings the - checkpointed dense solve and search were made with. The draw threshold is - deliberately absent: re-drawing at another threshold is the point. - """ - return { - "dataset_pin": dict(pins["dataset"]), - "ladder_pin": dict(pins["ladder"]), - "ledger_facts_sha256": args.ledger_facts_sha256, - "ledger_manifest_sha256": args.ledger_manifest_sha256, - "seed": int(args.seed), - "selection_seed": int( - args.seed if args.selection_seed is None else args.selection_seed - ), - "n_clones": int(args.n_clones), - "dataset_households": args.dataset_households, - "epochs": int(args.epochs), - "learning_rate": float(args.learning_rate), - "sample_fraction": float(args.sample_fraction), - "sample_seed": int(args.sample_seed), - "source_year": int(source_year), - "source_lineage_modulus": args.source_lineage_modulus, - "calibration_year": getattr(args, "_calibration_year", None), - "target_weight_rule": args.target_weight_rule, - "engine_blocks": int(args.engine_blocks), - "measure_exclusions": ( - None if args.measure_exclusions is None else str(args.measure_exclusions) - ), - # The solve doctrine the dense solve and the search run under: a - # resume after a doctrine change must refuse, not run under the old - # bound while the manifest declares the new one. - "doctrine": _doctrine_bounds(), - } - - -def _candidate_identity_digest( - *, - pins: dict[str, dict[str, object]], - args: argparse.Namespace, - source_year: int, -) -> str: - payload = { - "build_kind": "uk_rowwise_calibrated_candidate", - "inputs": pins, - "parameters": _parameters(args, source_year=source_year), - "source_year": source_year, - } - return hashlib.sha256(canonical_json_bytes(payload)).hexdigest() - - -def _record_candidate_attempt( - *, - state: AttemptState, - started_at: float, - started_ts: datetime, - seed: int, - code_pin: str, - disposition: str, - predecessor: str | None, - spool_dir: Path, - rung: str = "f100", -) -> Path: - return record_terminal_attempt( - state=state, - started_at=started_at, - started_ts=started_ts, - pipeline=_UK_CANDIDATE_PIPELINE, - rung=rung, - seed=seed, - code_pin=code_pin, - disposition=disposition, - predecessor=predecessor, - spool_dir=spool_dir, - ) - - -def _record_candidate_error( - *, - error: BaseException, - state: AttemptState, - started_at: float, - started_ts: datetime, - seed: int, - code_pin: str, - predecessor: str | None, - base_dir: Path, - spool_dir: Path, - rung: str = "f100", -) -> None: - error_path = write_error_receipt( - error_receipt_path(base_dir, build_id=state.build_id), - state=state, - pipeline=_UK_CANDIDATE_PIPELINE, - error=error, - ) - apply_error_verdict( - state, - f"{local_artifact_reference(error_path, repository_hint=_REPOSITORY)}#/error_type", - ) - _record_candidate_attempt( - state=state, - started_at=started_at, - started_ts=started_ts, - seed=seed, - code_pin=code_pin, - disposition="failed", - predecessor=predecessor, - spool_dir=spool_dir, - rung=rung, - ) - - -def _parse_args(argv: list[str] | None = None) -> argparse.Namespace: - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument( - "--input-h5", - type=Path, - required=True, - help="National Microcosm UK staging H5.", - ) - parser.add_argument( - "--input-sha256", - type=sha256_argument, - help="Pinned SHA-256 of --input-h5 (required for the joint registry path).", - ) - parser.add_argument( - "--ladder", - type=Path, - required=True, - help="Full-UK OA geography ladder NPZ.", - ) - parser.add_argument( - "--ladder-sha256", - type=sha256_argument, - help="Pinned SHA-256 of --ladder (required for the joint registry path).", - ) - parser.add_argument("--ledger-facts", type=Path) - parser.add_argument("--ledger-facts-sha256", type=sha256_argument) - parser.add_argument("--ledger-manifest-sha256", type=sha256_argument) - parser.add_argument("--measure-exclusions", type=Path) - parser.add_argument("--register-json", type=Path) - parser.add_argument( - "--target-weight-rule", - choices=("uniform", "grain_equal"), - default=UK_LOCAL_SOLVE_DOCTRINE.target_weight_rule, - ) - parser.add_argument("--release-candidate", action="store_true") - parser.add_argument("--skip-holdout", action="store_true") - parser.add_argument( - "--out", - type=Path, - required=True, - help="Output directory for the candidate H5 and evidence sidecars.", - ) - parser.add_argument( - "--dataset-households", - type=int, - help="Exact output household count after informed L0 and refit; pool clone K is unchanged. Candidate-only until size certification.", - ) - parser.add_argument("--n-clones", type=int, default=UK_LOCAL_CLONE_COUNT) - parser.add_argument( - "--candidate-clone-counts", - type=_candidate_clone_counts_argument, - help="Dry-run only comma-separated candidate clone counts.", - ) - parser.add_argument("--seed", type=int, default=42) - parser.add_argument( - "--selection-seed", - type=int, - help=( - "Seed for the size selection only (informed L0 search, exact-count " - "draw, refit); defaults to --seed. The pool, ladder assignment and " - "dense reference stay on --seed, so two selections compare on one " - "pool. Requires --dataset-households." - ), - ) - parser.add_argument( - "--selection-pi-hi", - type=float, - default=1.0, - help=( - "Certainty threshold of the exact-count draw: gates whose learned open " - "probability reaches it are taken with certainty. 1.0 (default) keeps " - "only the protected carriers certain; a lower value promotes learned " - "near-certain gates (the US exact-k ladder runs 0.95). Candidate-only; " - "recorded in the size receipt. Requires --dataset-households." - ), - ) - parser.add_argument( - "--no-size-checkpoint", - action="store_true", - help=( - "Do not persist the dense solve and the informed L0 search before the " - "exact-count draw. By default a --dataset-households run writes " - "size_selection_checkpoint.{npz,json} into --out so a draw refusal " - "costs a re-draw, not the pool solve (microcosm#355)." - ), - ) - parser.add_argument( - "--resume-size-checkpoint", - type=Path, - help=( - "Directory holding a size_selection_checkpoint written by an earlier " - "--dataset-households run on the same inputs: the pool and the target " - "surface are re-derived and verified, the dense solve and the search " - "are restored, and the run continues at the exact-count draw " - "(--selection-pi-hi may differ; both thresholds are recorded). " - "Requires --dataset-households and the same seeds, epochs and pins." - ), - ) - parser.add_argument( - "--sample-fraction", - type=float, - default=1.0, - help="Spine sampling rung: 0.01, 0.10, or 1.0.", - ) - parser.add_argument( - "--sample-seed", - type=int, - default=UK_SAMPLE_SEED_DEFAULT, - ) - parser.add_argument( - "--engine-blocks", - type=int, - default=1, - help="Resolve one engine or one block per clone (must equal --n-clones).", - ) - parser.add_argument( - "--source-year", - type=int, - help="Survey year recorded for lineage (calibration uses the FRS release year).", - ) - parser.add_argument("--source-lineage-modulus", type=int) - parser.add_argument("--epochs", type=int, default=UK_LOCAL_SOLVE_EPOCHS) - parser.add_argument("--learning-rate", type=float, default=0.15) - parser.add_argument( - "--expected-constituency-vintage", - default="2024_pcon", - help="Constituency vintage required from the ladder.", - ) - parser.add_argument( - "--dry-run", - action="store_true", - help=( - "Print the fenced clone/matrix plan without solving or writing any file." - ), - ) - parser.add_argument( - "--logbook-prev-row-digest", - type=sha256_argument, - help=( - "Optional current Logbook chain head. If omitted, " - "POPULACE_LOGBOOK_PREV_ROW_DIGEST is used, then genesis null." - ), - ) - return parser.parse_args(argv) - - -def main(argv: list[str] | None = None) -> int: - """Run the rowwise candidate build.""" - - args = _parse_args(argv) - _validate_cli_args(args) - if args.candidate_clone_counts is not None and not args.dry_run: - raise ValueError("--candidate-clone-counts is valid only with --dry-run.") - if _CONSERVE_MASS: - raise NotImplementedError( - "the candidate manifest's calibration_mass_change block reads " - "the kernel's free-mass record; a conserve-mass doctrine run " - "appends no record and needs its own reviewed manifest shape " - "before this constant may flip." - ) - if args.dry_run: - # Dry runs plan without solving or writing and record no Logbook - # row on any path, so they need no chain configuration. - return _run_candidate(args, attempt=None) - started_at = time.perf_counter() - started_ts = datetime.now(UTC) - digest = preflight_digest(_UK_CANDIDATE_PIPELINE) - state = AttemptState( - build_id=_new_candidate_build_id( - seed=args.seed, - timestamp=started_ts, - rung=UK_SAMPLE_RUNG_TOKENS[args.sample_fraction], - ), - identity_digest=digest, - input_pins_digest=digest, - phases_reached=["attempt_started"], - gate_verdicts={ - "pipeline": { - "verdict": "running", - "receipt": "pending-build-scoped-terminal-receipt", - } - }, - ) - return _run_candidate( - args, - attempt={ - "state": state, - "started_at": started_at, - "started_ts": started_ts, - "code_pin": "unresolved-local-git-code-pin", - # Logbook chain configuration is validated before any terminal - # work: a malformed or conflicting head refuses the run with no - # row and no side effects (#666 adversarial-review finding). - "predecessor": resolve_predecessor(args.logbook_prev_row_digest), - }, - ) - - -def _run_candidate( - args: argparse.Namespace, - *, - attempt: dict[str, object] | None, -) -> int: - """Build the candidate, recording every non-dry terminal outcome. - - The recording envelope opens before input verification so that setup - failures — unreadable inputs, frame or ladder load errors, clone and - target-binding refusals — still spool a failed row (#666 - adversarial-review finding). Dry runs pass ``attempt=None`` and record - nothing. - """ - - out_dir = args.out.expanduser().resolve() - try: - input_h5 = _require_file(args.input_h5, label="--input-h5") - ladder_path = _require_file(args.ladder, label="--ladder") - if out_dir.exists() and not out_dir.is_dir(): - raise ValueError(f"--out must be a directory path, got {out_dir}.") - - input_artifact = _artifact_info(input_h5) - ladder_artifact = _artifact_info(ladder_path) - _verify_requested_pin("--input-h5", input_artifact, requested=args.input_sha256) - _verify_requested_pin("--ladder", ladder_artifact, requested=args.ladder_sha256) - pins = { - "dataset": _pin_from_artifact(input_artifact), - "ladder": _pin_from_artifact(ladder_artifact), - } - state: AttemptState | None = None - if attempt is not None: - unpacked_state = attempt["state"] - assert isinstance(unpacked_state, AttemptState) - state = unpacked_state - attempt["code_pin"] = git_code_pin(_REPOSITORY) - state.input_pins_digest = role_pins_digest(pins) - append_phase(state, "configured") - append_phase(state, "inputs_pinned") - national_frame, _national_provenance = load_uk_national_frame(input_h5) - calibration_year = int(load_uk_frs_release().calibration_year) - args._calibration_year = calibration_year - if args.ledger_facts is not None: - spine_sidecar_path = input_h5.with_suffix(".build.json") - spine_sidecar = load_bound_spine_sidecar( - spine_sidecar_path, - national_frame, - ) - args._spine_provenance = spine_provenance_from_sidecar( - spine_sidecar_path, - spine_sidecar, - ) - else: - args._spine_provenance = {} - national_frame, sampling = _sample_candidate_frame( - national_frame, - fraction=args.sample_fraction, - seed=args.sample_seed, - ) - args._sampling_receipt = sampling - source_year = _source_year( - args.source_year, - time_period=uk_time_period(national_frame), - ) - if state is not None: - # The identity digest waits on the frame-derived source year; - # earlier failures record with the preflight placeholder. - state.identity_digest = _candidate_identity_digest( - pins=pins, - args=args, - source_year=source_year, - ) - output_paths = _output_paths( - out_dir, - source_year=source_year, - calibration_year=calibration_year, - ) - _validate_output_paths( - output_paths, - input_h5=input_h5, - ladder_path=ladder_path, - ) - _refuse_stale_size_checkpoint(args, out_dir) - ladder = load_uk_oa_ladder(ladder_path) - target_provenance = ladder_target_provenance(ladder) - joint_inputs = _load_joint_target_inputs(args) - if joint_inputs is not None: - # microcosm#762 A15: the ladder's census household counts bind at - # the calibration year through one national factor from the - # Ledger's published UK household total (fail-closed by name on a - # real Ledger artifact; a synthetic artifact without facts binds - # the rows as published and says so, which the release posture - # refuses). - facts = getattr(joint_inputs.get("artifact"), "facts", None) - if facts is None: - joint_inputs["ladder_household_uprating"] = { - "applied": False, - "reason": ( - "the joint target inputs carry no Ledger facts; ladder " - "household rows bind at their census vintage." - ), - } - else: - joint_inputs["ladder_household_uprating"] = ( - uk_ladder_household_uprating( - ladder, - uk_ledger_households_total( - facts, period=joint_inputs["calibration_year"] - ), - period=joint_inputs["calibration_year"], - ) - ) - if args.release_candidate and not joint_inputs[ - "ladder_household_uprating" - ].get("applied"): - raise SystemExit( - "error: --release-candidate requires the A15 ladder household " - "uprating: " - + str(joint_inputs["ladder_household_uprating"].get("reason")) - ) - doctrine, doctrine_override = uk_local_doctrine_with_overrides( - UK_LOCAL_SOLVE_DOCTRINE, - ( - {} - if args.target_weight_rule == UK_LOCAL_SOLVE_DOCTRINE.target_weight_rule - else {"target_weight_rule": args.target_weight_rule} - ), - ) - args._doctrine_override_receipt = doctrine_override - if doctrine.target_weight_rule != args.target_weight_rule: - raise RuntimeError( - "local doctrine override did not bind the requested rule." - ) - - print("cloning through the ladder route...", file=sys.stderr, flush=True) - assignment = _clone_with_ladder_binding( - national_frame, - ladder, - n_clones=args.n_clones, - seed=args.seed, - source_year=source_year, - expected_constituency_vintage=args.expected_constituency_vintage, - source_lineage_modulus=args.source_lineage_modulus, - ) - clone = assignment.result - if ( - args.dataset_households is not None - and args.dataset_households > clone.frame.n("household") - ): - raise ValueError( - "--dataset-households exceeds the cloned pool; selection never clamps the request." - ) - if state is not None: - append_phase(state, "cloned") - - if joint_inputs is not None and args.dry_run: - plan = _joint_dry_run_plan( - args, - clone=clone, - sampled_spine=national_frame, - ladder=ladder, - joint_inputs=joint_inputs, - source_year=source_year, - input_artifact=input_artifact, - ladder_artifact=ladder_artifact, - target_provenance=target_provenance, - ) - _assert_artifacts_unchanged( - input_h5=input_h5, - input_artifact=input_artifact, - ladder_path=ladder_path, - ladder_artifact=ladder_artifact, - ) - print(_json_text(plan), end="") - return 0 - - if joint_inputs is None: - print("binding census household targets...", file=sys.stderr, flush=True) - household, problem, cross_grain = _build_bound_problem( - assignment, - target_ladder=ladder, - ) - solve_frame = clone.frame - restore = None - national_rows = None - bound_families = BOUND_TARGET_FAMILIES - measure_resolution: Mapping[str, Any] = {} - args._rung_surface = { - "fraction": float(args.sample_fraction), - "dropped_cells": 0, - "dropped_by_grain": {}, - "dropped_by_family": {}, - } - else: - print("resolving joint local and national surface...", file=sys.stderr) - ( - solve_frame, - restore, - national_rows, - local_metrics, - measure_resolution, - ) = _resolve_candidate_engine_surface( - clone.frame, - joint_inputs["national_registry"], - period=joint_inputs["calibration_year"], - scratch_dir=out_dir.parent - / f".{out_dir.name}.candidate-engine-scratch", - band_edge_registry=joint_inputs["band_edge_registry"], - blocks=args.engine_blocks, - ) - ( - household, - problem, - cross_grain, - bound_families, - rung_surface, - ) = _build_joint_problem( - assignment, - target_ladder=ladder, - local_registry=joint_inputs["local_registry"], - national_registry=joint_inputs["national_registry"], - local_metrics=local_metrics, - period=joint_inputs["calibration_year"], - sample_fraction=args.sample_fraction, - reviewed_unbound_higher_targets=joint_inputs[ - "reviewed_unbound_higher_targets" - ], - ladder_household_uprating=joint_inputs.get("ladder_household_uprating"), - ) - args._rung_surface = rung_surface - args._bound_families = tuple(bound_families) - args._joint_inputs_receipt = joint_inputs - args._measure_resolution = dict(measure_resolution) - if state is not None: - append_phase(state, "targets_bound") - - if args.dry_run: - binding_adjudications = require_adjudicated_uk_local_binding( - bound_families, - problem.target_frame, - ) - _assert_artifacts_unchanged( - input_h5=input_h5, - input_artifact=input_artifact, - ladder_path=ladder_path, - ladder_artifact=ladder_artifact, - ) - plan = _dry_run_plan( - args, - clone=clone, - problem=problem, - source_year=source_year, - input_artifact=input_artifact, - ladder_artifact=ladder_artifact, - target_provenance=target_provenance, - binding_adjudications=binding_adjudications, - cross_grain=cross_grain, - ) - print(_json_text(plan), end="") - return 0 - - assert attempt is not None - assert state is not None - started_at = attempt["started_at"] - started_ts = attempt["started_ts"] - assert isinstance(started_at, float) - assert isinstance(started_ts, datetime) - predecessor = attempt["predecessor"] - assert predecessor is None or isinstance(predecessor, str) - code_pin = str(attempt["code_pin"]) - - print( - f"solving {problem.matrix.shape[0]} targets x " - f"{problem.matrix.shape[1]} households under the doctrine...", - file=sys.stderr, - flush=True, - ) - checkpoint_identity = _size_checkpoint_identity( - args, pins=pins, source_year=source_year - ) - resume_checkpoint = ( - None - if args.resume_size_checkpoint is None - else args.resume_size_checkpoint.expanduser().resolve() - ) - write_checkpoint = ( - args.dataset_households is not None - and not args.no_size_checkpoint - and resume_checkpoint is None - ) - if resume_checkpoint is not None: - print( - f"resuming the size selection from {resume_checkpoint}...", - file=sys.stderr, - flush=True, - ) - solve = solve_uk_rowwise_weights_under_doctrine( - solve_frame, - problem, - bound_families=bound_families, - national_rows=national_rows, - target_weight_rule=args.target_weight_rule, - restore=restore, - epochs=args.epochs, - learning_rate=args.learning_rate, - conserve_mass=_CONSERVE_MASS, - target_records=_TARGET_RECORDS, - dataset_households=args.dataset_households, - l0_lambda=_L0_LAMBDA, - budget_iters=_BUDGET_ITERS, - seed=args.seed, - selection_seed=args.selection_seed, - selection_pi_hi=args.selection_pi_hi, - size_checkpoint_dir=out_dir if write_checkpoint else None, - resume_size_checkpoint=resume_checkpoint, - checkpoint_identity=checkpoint_identity, - checkpoint_provenance={"code_pin": code_pin, "build_id": state.build_id}, - progress=_stderr_progress, - ) - _validate_solve_result(solve, problem=problem) - if solve.size_receipt is not None and solve.size_receipt.get("checkpoint"): - checkpoint = solve.size_receipt["checkpoint"] - if "written" in checkpoint: - append_phase(state, "size_selection_checkpointed") - print( - f"size selection checkpoint written to {out_dir}", - file=sys.stderr, - flush=True, - ) - elif "resumed_from" in checkpoint: - append_phase(state, "size_selection_resumed") - append_phase(state, "solved") - - # The kernel minted the calibration mass record inside calibrate() (the - # CALIBRATED kind transition is enforced there too); the record names - # the bound families via the doctrine's mass reason. - calibration_record = solve.frame.mass_log[-1] - if "calibration" not in calibration_record.reason: - raise ValueError( - "calibrated frame's latest mass record is not the calibration " - f"record: {calibration_record.reason!r}." - ) - support = _candidate_area_support( - solve.frame.table("household"), - ladder, - weights=solve.weights, - ) - _validate_support_summary(support) - local_diagnostics = _local_gate_diagnostics(solve.diagnostics) - target_registry, target_geography_levels = _local_diagnostics_registry( - solve, - problem, - national_registry=( - None if joint_inputs is None else joint_inputs["national_registry"] - ), - ) - try: - gate_report, candidate_gate = _run_local_gate_battery( - frame=solve.frame, - support=support, - diagnostics=local_diagnostics, - report_path=output_paths["local_gates"], - release_id=state.build_id, - evaluated_on=started_ts.date(), - enforce_only=( - None - if args.sample_fraction == 1.0 - else ("uk_local_geography_ladder_post_calibration",) - ), - # The battery attests the posture it ran under: a release - # candidate blocks on absent evidence and can be shippable. - release_candidate=bool(args.release_candidate), - ) - except GateBatteryBlockedError: - # Write-then-block, extended to the whole evidence bundle: a - # release-blocking failure at f100 still writes the diagnostics, - # manifest and artifact (marked unreleasable) so the block can be - # reviewed; only a non-passing ladder verdict is structural and - # re-raises. The Logbook row records the attempt as failed. - gate_report = json.loads( - output_paths["local_gates"].read_text(encoding="utf-8") - ) - _apply_gate_verdicts(state, gate_report, output_paths["local_gates"]) - ladder_entry = gate_report["gates"].get( - "uk_local_geography_ladder_post_calibration", {} - ) - if ladder_entry.get("status") != "passed": - raise - candidate_gate = clone.gate - blocked_failures, diagnostic_failures = _gate_failures_by_criticality( - gate_report - ) - if not blocked_failures: - # The battery blocked, yet the persisted report names no - # failed release-blocking entry: the report and the error - # disagree, which is structural. - raise - unenforced_failures = [] - else: - _apply_gate_verdicts(state, gate_report, output_paths["local_gates"]) - # Nothing blocked. Release-blocking entries can still hold a - # failure here: below f100 only the ladder gate is enforced, and - # a dev build tolerates absent evidence. Those lines are reported - # as not enforced, never as a block. - unenforced_failures, diagnostic_failures = _gate_failures_by_criticality( - gate_report - ) - blocked_failures = [] - args._gate_report = gate_report - args._blocked_failures = blocked_failures - args._diagnostic_failures = diagnostic_failures - args._unenforced_release_failures = ( - [] if blocked_failures else unenforced_failures - ) - append_phase( - state, "candidate_gated" if not blocked_failures else "candidate_blocked" - ) - - if args.skip_holdout: - rotated_holdout = {"skipped": True} - else: - rotated_holdout = rotated_uk_local_holdout( - solve_frame, - problem, - bound_families=bound_families, - national_rows=national_rows, - target_weight_rule=args.target_weight_rule, - restore=restore, - epochs=args.epochs, - learning_rate=args.learning_rate, - conserve_mass=_CONSERVE_MASS, - target_records=_TARGET_RECORDS, - dataset_households=args.dataset_households, - l0_lambda=_L0_LAMBDA, - budget_iters=_BUDGET_ITERS, - solve_seed=args.seed, - selection_seed=args.selection_seed, - selection_pi_hi=args.selection_pi_hi, - ) - args._rotated_holdout = rotated_holdout - - candidate = dataclasses.replace( - clone, - frame=solve.frame, - gate=candidate_gate, - output_path=None, - ) - support_by_grain = { - ("la" if grain == "local_authority" else str(grain)): rows.reset_index( - drop=True - ) - for grain, rows in support.groupby("geography_level", sort=True) - } - args._support_limited_misses = uk_support_limited_misses( - solve.diagnostics, - support_by_grain, - max_abs_relative_error=0.25, - ) - _assert_artifacts_unchanged( - input_h5=input_h5, - input_artifact=input_artifact, - ladder_path=ladder_path, - ladder_artifact=ladder_artifact, - ) - - manifest = _write_output_bundle( - args, - candidate=candidate, - clone=clone, - problem=problem, - solve=solve, - local_diagnostics=local_diagnostics, - target_registry=target_registry, - target_geography_levels=target_geography_levels, - rotated_holdout=rotated_holdout, - support=support, - calibration_record=calibration_record, - source_year=source_year, - output_paths=output_paths, - input_artifact=input_artifact, - ladder_artifact=ladder_artifact, - target_provenance=target_provenance, - cross_grain=cross_grain, - ) - append_phase(state, "published") - state.artifact_location = local_artifact_reference( - output_paths["dataset"], - repository_hint=_REPOSITORY, - ) - spool_path = _record_candidate_attempt( - state=state, - started_at=started_at, - started_ts=started_ts, - seed=args.seed, - code_pin=code_pin, - disposition="failed" if blocked_failures else "iterating", - predecessor=predecessor, - spool_dir=out_dir / "logbook-spool", - rung=UK_SAMPLE_RUNG_TOKENS[args.sample_fraction], - ) - print(f"Wrote Logbook row: {spool_path}", file=sys.stderr) - print(_json_text(manifest), end="") - if blocked_failures: - print( - "Gate battery blocked the artifact at f100; evidence bundle " - f"written, artifact unreleasable: {blocked_failures[:5]}", - file=sys.stderr, - ) - return 1 - return 0 - except Exception as error: - if attempt is None: - # Dry runs record no row on any path, including failures. - raise - failed_state = attempt["state"] - assert isinstance(failed_state, AttemptState) - failed_started_at = attempt["started_at"] - failed_started_ts = attempt["started_ts"] - assert isinstance(failed_started_at, float) - assert isinstance(failed_started_ts, datetime) - failed_predecessor = attempt["predecessor"] - assert failed_predecessor is None or isinstance(failed_predecessor, str) - _record_candidate_error( - error=error, - state=failed_state, - started_at=failed_started_at, - started_ts=failed_started_ts, - seed=args.seed, - code_pin=str(attempt["code_pin"]), - predecessor=failed_predecessor, - base_dir=out_dir, - spool_dir=out_dir / "logbook-spool", - rung=UK_SAMPLE_RUNG_TOKENS[args.sample_fraction], - ) - raise - - -def _clone_with_ladder_binding( - dataset: Any, - ladder: UkOaLadder, - *, - n_clones: int, - seed: int, - source_year: int, - expected_constituency_vintage: str | None, - source_lineage_modulus: int | None, -) -> _LadderAssignment: - clone = clone_uk_dataset_with_ladder_geography( - dataset, - ladder, - n_clones=n_clones, - seed=seed, - source_year=source_year, - expected_constituency_vintage=expected_constituency_vintage, - source_lineage_modulus=source_lineage_modulus, - ) - return _LadderAssignment(clone, ladder) - - -def _verify_requested_pin( - label: str, - artifact: Mapping[str, Any], - *, - requested: str | None, -) -> None: - if requested is None: - artifact["pin_verified"] = False - return - measured = str(artifact["sha256"]) - if measured != requested: - raise SystemExit( - f"error: {label} sha mismatch: measured {measured}, pinned {requested}" - ) - artifact["pin_verified"] = True - - -def _load_joint_target_inputs(args: argparse.Namespace) -> dict[str, Any] | None: - if args.ledger_facts is None: - return None - artifact = load_ledger_consumer_artifact( - args.ledger_facts, - expected_facts_sha256=args.ledger_facts_sha256, - expected_manifest_sha256=args.ledger_manifest_sha256, - ) - calibration_year = int(load_uk_frs_release().calibration_year) - national_compilation = compile_uk_target_registry( - artifact.facts, target_period=calibration_year - ) - if national_compilation.unsupported: - raise SystemExit( - f"{len(national_compilation.unsupported)} national target references " - "failed to compile" - ) - local_compilation = compile_uk_local_target_registry( - artifact.facts, - target_period=calibration_year, - crosswalk=load_uk_local_area_crosswalk(), - ) - if local_compilation.unsupported: - raise SystemExit( - f"{len(local_compilation.unsupported)} local target references " - "failed to compile" - ) - exclusions = load_uk_calibration_measure_exclusions(args.measure_exclusions) - national_registry, exclusion_receipt = apply_uk_calibration_measure_exclusions( - national_compilation.registry, exclusions - ) - national_specs_by_name = { - spec.name: spec for spec in national_compilation.registry.specs - } - reviewed_unbound_higher_targets = { - str( - national_specs_by_name[name].metadata.get( - "contract_target_id", national_specs_by_name[name].name - ) - ): record - for name, record in exclusion_receipt.items() - } - if args.register_json is not None: - try: - frozen = TargetRegistry.from_json(args.register_json) - except ValueError as error: - raise SystemExit( - f"error: frozen scoring register is unusable: {error}" - ) from error - if frozen.version != national_registry.version: - raise SystemExit( - "re-derived register differs from the frozen scoring register: " - f"{national_registry.version} vs {frozen.version}" - ) - return { - "artifact": artifact, - "calibration_year": calibration_year, - "national_registry": national_registry, - "band_edge_registry": national_compilation.registry, - "local_registry": local_compilation.registry, - "measure_exclusions": exclusion_receipt, - "reviewed_unbound_higher_targets": reviewed_unbound_higher_targets, - } - - -def _national_contract_target_ids(registry: TargetRegistry) -> tuple[str, ...]: - return tuple( - sorted( - { - str(spec.metadata.get("contract_target_id", spec.name)) - for spec in registry.specs - } - ) - ) - - -def _joint_surface_registry( - local_registry: TargetRegistry, - national_registry: TargetRegistry, -) -> TargetRegistry: - """Put national controls beside local cells for cross-grain reconciliation.""" - - return TargetRegistry( - [*local_registry.specs, *national_registry.specs], - country="uk", - ) - - -def _build_joint_problem( - assignment: _LadderAssignment, - *, - target_ladder: UkOaLadder, - local_registry: TargetRegistry, - national_registry: TargetRegistry, - local_metrics: Mapping[str, pd.DataFrame], - period: int, - sample_fraction: float, - reviewed_unbound_higher_targets: Mapping[str, Mapping[str, object]], - ladder_household_uprating: Mapping[str, Any] | None = None, -) -> tuple[ - pd.DataFrame, - UKRowwiseLocalMatrix, - dict[str, Any], - tuple[str, ...], - dict[str, Any], -]: - if assignment.ladder is not target_ladder: - raise ValueError( - "assignment and targets must come from the same loaded UK OA ladder object." - ) - household = assignment.result.frame.table("household").reset_index(drop=True) - household_index = pd.Index(household["household_id"], name="household_id") - metrics = { - grain: frame.set_axis(household_index, axis="index") - for grain, frame in local_metrics.items() - } - assigned = { - "constituency": pd.Series( - household["constituency_code"].astype(str).to_numpy(), - index=household_index, - ), - "la": pd.Series( - household["local_authority_code"].astype(str).to_numpy(), - index=household_index, - ), - } - national_ids = _national_contract_target_ids(national_registry) - surface, cross_grain = uk_local_target_surface( - _joint_surface_registry(local_registry, national_registry), - target_ladder, - bound_national_target_ids=national_ids, - period=period, - reviewed_unbound_higher_targets=reviewed_unbound_higher_targets, - ladder_household_uprating=ladder_household_uprating, - ) - covered = { - grain: set(values.astype(str).tolist()) for grain, values in assigned.items() - } - covered_mask = pd.Series( - [ - str(row.area_code) in covered[str(row.area_type)] - for row in surface.itertuples(index=False) - ], - index=surface.index, - dtype=bool, - ) - dropped = surface.loc[~covered_mask] - if sample_fraction < 1.0: - surface = surface.loc[covered_mask].reset_index(drop=True) - # Below f100 a covered area can still carry a nonzero cell with no metric - # support in the sample (no self-employed household among three drawn - # rows). The builder refuses such a cell at every rung; at development - # rungs the cell is dropped here and receipted instead. f100 stays strict. - unreachable = surface.iloc[0:0] - if sample_fraction < 1.0 and len(surface): - nonzero_by_grain = { - grain: (metrics[grain] != 0).groupby(assigned[grain]).sum() - for grain in metrics - } - unreachable_mask = pd.Series( - [ - float(row.value) != 0.0 - and str(row.metric) in nonzero_by_grain[str(row.area_type)].columns - and str(row.area_code) in nonzero_by_grain[str(row.area_type)].index - and int( - nonzero_by_grain[str(row.area_type)].loc[ - str(row.area_code), str(row.metric) - ] - ) - == 0 - for row in surface.itertuples(index=False) - ], - index=surface.index, - dtype=bool, - ) - unreachable = surface.loc[unreachable_mask] - surface = surface.loc[~unreachable_mask].reset_index(drop=True) - rung_surface = { - "dropped_unreachable_cells": int(len(unreachable)), - "dropped_unreachable_by_grain": { - str(key): int(value) - for key, value in unreachable.groupby("area_type").size().items() - }, - "dropped_unreachable_by_family": { - str(key): int(value) - for key, value in unreachable.groupby("family").size().items() - }, - "fraction": float(sample_fraction), - "dropped_cells": int(len(dropped) if sample_fraction < 1.0 else 0), - "dropped_by_grain": ( - { - str(key): int(value) - for key, value in dropped.groupby("area_type").size().items() - } - if sample_fraction < 1.0 - else {} - ), - "dropped_by_family": ( - { - str(key): int(value) - for key, value in dropped.groupby("family").size().items() - } - if sample_fraction < 1.0 - else {} - ), - } - rosters = { - "constituency": tuple(map(str, np.unique(target_ladder.constituency_code))), - "la": tuple(map(str, np.unique(target_ladder.local_authority_code))), - } - problem = build_uk_rowwise_local_surface_matrix( - metrics, - assigned, - surface, - area_codes_by_grain=rosters, - require_every_assigned_area_covered=(sample_fraction == 1.0), - ) - local_bound = tuple( - sorted( - { - f"{row.family}/{row.area_type}" - for row in surface[["family", "area_type"]] - .drop_duplicates() - .itertuples(index=False) - } - ) - ) - national_bound = tuple( - f"national/{family}" - for family in sorted({spec.family for spec in national_registry.specs}) - ) - return ( - household, - problem, - cross_grain, - (*local_bound, *national_bound), - rung_surface, - ) - - -def _joint_dry_run_plan( - args: argparse.Namespace, - *, - clone: UKLadderRowwiseDatasetResult, - sampled_spine: Any, - ladder: UkOaLadder, - joint_inputs: Mapping[str, Any], - source_year: int, - input_artifact: Mapping[str, Any], - ladder_artifact: Mapping[str, Any], - target_provenance: Mapping[str, Any], -) -> dict[str, Any]: - national_registry = joint_inputs["national_registry"] - surface, cross_grain = uk_local_target_surface( - _joint_surface_registry( - joint_inputs["local_registry"], - national_registry, - ), - ladder, - bound_national_target_ids=_national_contract_target_ids(national_registry), - period=joint_inputs["calibration_year"], - reviewed_unbound_higher_targets=joint_inputs["reviewed_unbound_higher_targets"], - ladder_household_uprating=joint_inputs.get("ladder_household_uprating"), - ) - household = clone.frame.table("household") - covered = { - "constituency": set(household["constituency_code"].astype(str)), - "la": set(household["local_authority_code"].astype(str)), - } - covered_mask = pd.Series( - [ - str(row.area_code) in covered[str(row.area_type)] - for row in surface.itertuples(index=False) - ], - index=surface.index, - dtype=bool, - ) - dropped = surface.loc[~covered_mask] - active_surface = ( - surface.loc[covered_mask].reset_index(drop=True) - if args.sample_fraction < 1.0 - else surface - ) - household_count = len(clone.frame.table("household")) - clone_support: dict[str, object] = {} - for clone_count in args.candidate_clone_counts or (args.n_clones,): - candidate = ( - clone - if clone_count == args.n_clones - else _clone_with_ladder_binding( - sampled_spine, - ladder, - n_clones=clone_count, - seed=args.seed, - source_year=source_year, - expected_constituency_vintage=args.expected_constituency_vintage, - source_lineage_modulus=args.source_lineage_modulus, - ).result - ) - # The typed frame weights are the authority; the persisted - # household_weight column is an export artefact the loaded spine - # does not carry, so attach them the way the real support path does. - candidate_household = candidate.frame.table("household").copy() - candidate_household["household_weight"] = np.asarray( - candidate.frame.weights_for("household").values, dtype=np.float64 - ) - summaries = uk_ladder_area_support_summary(candidate_household, ladder) - clone_support[str(clone_count)] = { - grain: { - "minimum_rows": int(rows["nonzero_households"].min()), - "minimum_effective_sample_size": float( - rows["effective_sample_size"].min() - ), - "minimum_distinct_sources": int( - rows["nonzero_source_households"].min() - ), - } - for grain, rows in summaries.items() - } - return { - "schema_version": 2, - "build_kind": "uk_rowwise_calibrated_candidate_plan", - "dry_run": True, - "survey_year": source_year, - "calibration_year": joint_inputs["calibration_year"], - "identity": { - "spine": dict(input_artifact), - "ladder": dict(ladder_artifact), - "ledger": joint_inputs["artifact"].provenance(), - }, - "sampling": dict(args._sampling_receipt), - "rung_surface": { - "rung": UK_SAMPLE_RUNG_TOKENS[args.sample_fraction], - "fraction": args.sample_fraction, - "dropped_cells": int(len(dropped) if args.sample_fraction < 1.0 else 0), - "dropped_by_grain": ( - { - str(key): int(value) - for key, value in dropped.groupby("area_type").size().items() - } - if args.sample_fraction < 1.0 - else {} - ), - "dropped_by_family": ( - { - str(key): int(value) - for key, value in dropped.groupby("family").size().items() - } - if args.sample_fraction < 1.0 - else {} - ), - "unreachable_check": "deferred_to_build", - }, - "vintages": _local_vintage_census(joint_inputs["local_registry"]), - "cross_grain": cross_grain, - "matrix": { - "rows": int(len(active_surface) + len(national_registry.specs)), - "columns": household_count, - "local_rows": len(active_surface), - "national_rows": len(national_registry.specs), - }, - "candidate_clone_counts": list(args.candidate_clone_counts or (args.n_clones,)), - "candidate_clone_support": clone_support, - "parameters": _parameters(args, source_year=source_year), - "releasable": False, - "engine": "not_run", - "ladder_target_provenance": dict(target_provenance), - } - - -def _local_vintage_census(registry: TargetRegistry) -> list[dict[str, object]]: - counts: dict[tuple[str, str, str, str], int] = {} - for spec in registry.specs: - resolved = str(spec.metadata.get("ledger_fact_period", "")) - target = str(spec.period) - if not resolved or resolved == target: - continue - level, _ = _spec_geography(spec) - key = (spec.family, level, resolved, target) - counts[key] = counts.get(key, 0) + 1 - return [ - { - "family": family, - "geography_level": level, - "resolved_period": resolved, - "target_period": target, - "cells": cells, - } - for (family, level, resolved, target), cells in sorted(counts.items()) - ] - - -def _build_bound_problem( - assignment: _LadderAssignment, - *, - target_ladder: UkOaLadder, -) -> tuple[pd.DataFrame, UKRowwiseLocalMatrix, dict[str, Any]]: - """Bind the one target family, refusing separately loaded ladders.""" - - if assignment.ladder is not target_ladder: - raise ValueError( - "assignment and targets must come from the same loaded UK OA ladder object." - ) - clone = assignment.result - household = clone.frame.table("household").reset_index(drop=True) - household_index = pd.Index( - household["household_id"], - name="household_id", - ) - metrics = pd.DataFrame( - {"households": np.ones(len(household), dtype=np.float64)}, - index=household_index, - ) - assigned = pd.Series( - household["constituency_code"].astype(str).to_numpy(), - index=household_index, - name="constituency_code", - ) - targets = constituency_household_targets(target_ladder) - local_surface = pd.DataFrame( - { - "grain": "constituency", - "geography_id": targets["code"].astype(str), - "target_id": "external:census_households/households", - "value": targets["households"].to_numpy(dtype=np.float64), - } - ) - reconciled_surface, cross_grain = apply_uk_cross_grain_reconciliation( - local_surface, - BOUND_NATIONAL_TARGETS, - ) - targets = targets.copy() - targets["households"] = reconciled_surface["value"].to_numpy(dtype=np.float64) - problem = build_uk_rowwise_local_matrix( - metrics, - assigned, - targets, - area_type="constituency", - code_column="code", - ) - return ( - household, - problem, - { - "bound_national_targets": list(BOUND_NATIONAL_TARGETS), - **cross_grain, - }, - ) - - -def _candidate_area_support( - household: pd.DataFrame, - ladder: UkOaLadder, - *, - weights: np.ndarray, -) -> pd.DataFrame: - weighted_household = household.copy() - weighted_household["household_weight"] = np.asarray(weights, dtype=np.float64) - summaries = uk_ladder_area_support_summary(weighted_household, ladder) - return pd.concat( - ( - summaries["constituency"].assign(geography_level="constituency"), - summaries["la"].assign(geography_level="local_authority"), - ), - ignore_index=True, - )[ - [ - "geography_level", - "area_code", - "assigned_households", - "nonzero_households", - "nonzero_source_households", - "weight_sum", - "max_weight", - "effective_sample_size", - ] - ] - - -def _local_gate_diagnostics(diagnostics: pd.DataFrame) -> pd.DataFrame: - result = diagnostics.copy() - required = {"family", "area_type", "area_code", "metric"} - missing = sorted(required - set(result.columns)) - if missing: - raise ValueError(f"local diagnostics are missing binding columns {missing}.") - if result[list(required)].isna().any().any(): - raise ValueError("local diagnostics contain unclassified binding rows.") - return result - - -def _local_diagnostics_registry( - solve: UKRowwiseDoctrineSolve, - problem: UKRowwiseLocalMatrix, - *, - national_registry: TargetRegistry | None = None, -) -> tuple[TargetRegistry, dict[str, str]]: - targets = tuple(solve.calibration_result.problem.targets) - expected = len(problem.target_frame) + ( - 0 if national_registry is None else len(national_registry.specs) - ) - if len(targets) != expected: - raise RuntimeError( - "candidate diagnostics registry is not aligned to the solve." - ) - specs: list[TargetSpec] = [] - geography: dict[str, str] = {} - for target, row in zip( - targets[: len(problem.target_frame)], - problem.target_frame.itertuples(index=False), - strict=True, - ): - metric = str(row.metric) - family = ( - str(row.family) - if "family" in problem.target_frame.columns - else local_target_census.family_for_metric(metric) - ) - spec = TargetSpec( - name=str(target.name), - entity=str(target.entity), - value=float(target.value), - measure=f"rowwise_metric:{metric}", - filter=f"rowwise_area:{row.area_code}", - period=target.period, - source=str(target.source), - family=family, - metadata={key: str(value) for key, value in target.metadata.items()}, - ) - specs.append(spec) - geography[spec.to_target().row_name] = str(row.area_type) - if national_registry is not None: - specs.extend(national_registry.specs) - for spec in national_registry.specs: - level, _ = _spec_geography(spec) - geography[spec.to_target().row_name] = level - return TargetRegistry(specs, country="uk"), geography - - -#: Measure columns whose policyengine-uk formulas normalise by a population -#: total (a fixed national aggregate allocated by each household's share of -#: total weighted corporate wealth, or a term scaled by a weight sum). Under -#: ``--engine-blocks K`` the engine sees one clone block at a time, so each -#: block reproduces the whole aggregate and the column comes out K× (receipt -#: R15 in ``experiments/762-uk-rowwise-candidate-receipts.md``: corporate -#: land value ×15.000 at K=15). Evidence from a per-block run must not -#: adjudicate these rows; the release posture is single-block. -UK_BLOCK_SENSITIVE_MEASURE_COLUMNS = ( - "ons/corporate_land_value", - "ons/land_value", - "slc/student_loan_repayment/england", -) - - -def _run_local_gate_battery( - *, - frame: Any, - support: pd.DataFrame, - diagnostics: pd.DataFrame, - report_path: Path, - release_id: str, - evaluated_on: date, - enforce_only: tuple[str, ...] | None = None, - release_candidate: bool = False, -) -> tuple[dict[str, object], GateResult]: - manifest = uk_scoped_gate_manifest( - UK_LOCAL_GATE_SCOPE, - phases=("terminal",), - policy_suffix=_LOCAL_GATE_POLICY_SUFFIX, - ) - battery = GateBatteryRun( - manifest, - release_id=release_id, - report_path=report_path, - release_candidate=release_candidate, - registry=UK_GATE_REGISTRY, - ) - phase = battery.run_phase( - "terminal", - EvidenceContext( - frame=frame, - artifacts={ - "uk_area_support_summary": support, - "local_target_diagnostics": diagnostics, - # The run's own clock, not the wall clock: the same artifact - # must reproduce the same exclusion verdicts. - "exclusions_evaluated_on": evaluated_on, - }, - ), - ) - if enforce_only is None: - try: - battery.enforce("terminal", mode=BlockingMode.BLOCKS_ARTIFACT) - except GateBatteryBlockedError: - payload = battery.report_payload() - finalize_uk_scoped_gate_report( - payload, - posture="local_candidate", - scope_exclusions=uk_local_gate_scope_exclusions(), - aggregate_admin_measurement=None, - ) - atomic_write_json(report_path, payload) - raise - else: - unknown = sorted(set(enforce_only) - set(UK_LOCAL_GATE_SCOPE)) - if unknown: - raise ValueError(f"enforce_only names unknown local gates: {unknown}.") - selected_blocking = [ - outcome - for outcome in phase.blocking_outcomes(release_candidate=release_candidate) - if outcome.entry.id in enforce_only - ] - if selected_blocking: - payload = battery.report_payload() - finalize_uk_scoped_gate_report( - payload, - posture="local_candidate", - scope_exclusions=uk_local_gate_scope_exclusions(), - aggregate_admin_measurement=None, - ) - atomic_write_json(report_path, payload) - failures = [ - failure - for outcome in selected_blocking - if outcome.result is not None - for failure in outcome.result.failures - ] - raise GateBatteryBlockedError("terminal", failures, report_path) - payload = battery.report_payload() - finalize_uk_scoped_gate_report( - payload, - posture="local_candidate", - scope_exclusions=uk_local_gate_scope_exclusions(), - aggregate_admin_measurement=None, - ) - atomic_write_json(report_path, payload) - ladder = next( - outcome - for outcome in phase.outcomes - if outcome.entry.id == "uk_local_geography_ladder_post_calibration" - ) - if ladder.result is None or not ladder.result.passed: - raise RuntimeError( - "a non-passing local geography-ladder result escaped battery enforcement." - ) - return payload, ladder.result - - -def _gate_failures_by_criticality( - gate_report: Mapping[str, Any], -) -> tuple[list[str], list[str]]: - """Split a persisted battery report's failure lines by criticality. - - Returns ``(release_blocking, diagnostic)``, each entry-prefixed like - :class:`GateBatteryBlockedError`'s lines. Only ``failed`` and - ``evidence_absent`` entries are failures; ``not_applicable`` and - ``unreached`` entries are not. - """ - - blocking: list[str] = [] - diagnostic: list[str] = [] - gates = gate_report.get("gates", {}) - if not isinstance(gates, Mapping): - return blocking, diagnostic - for gate_id, payload in gates.items(): - if not isinstance(payload, Mapping): - continue - status = payload.get("status") - if status not in {"failed", "evidence_absent"}: - continue - lines = [f"[{gate_id}] {line}" for line in payload.get("failures") or ()] - if not lines: - lines = [f"[{gate_id}] {payload.get('reason') or status}"] - bucket = blocking if _is_release_blocking(payload) else diagnostic - bucket.extend(lines) - return blocking, diagnostic - - -def _is_release_blocking(payload: Mapping[str, Any]) -> bool: - """Fail-closed criticality read. - - Only an entry that explicitly declares ``criticality: diagnostic`` is - exempt from vetoing the release; a missing or unknown criticality is - treated as release-blocking, so partial schema drift on one persisted - entry cannot drop a failed gate out of both the blocking list and - ``all_gates_passed``. - """ - - return payload.get("criticality") != "diagnostic" - - -def _release_verdict( - *, - sample_fraction: float, - engine_blocks: int, - release_blocking_gates_passed: bool, -) -> tuple[bool, dict[str, bool]]: - """``releasable`` needs the full rung, a single-block engine resolution and - every release-blocking gate passed. - - Per-block engine resolution mis-measures population-normalised formulas - (each block reproduces a national aggregate: the ×K land-value artefact - behind the #736 erratum), so a run resolved in more than one block is - diagnostic-only whatever its gates say. The posture is written beside the - verdict so a reader sees which leg failed. - """ - - posture = { - "full_rung": float(sample_fraction) == 1.0, - "single_block_engine": int(engine_blocks) == 1, - "release_blocking_gates_passed": bool(release_blocking_gates_passed), - } - return all(posture.values()), posture - - -def _apply_gate_verdicts( - state: AttemptState, - report: Mapping[str, object], - report_path: Path, -) -> None: - gates = report.get("gates") - if not isinstance(gates, Mapping) or set(gates) != set(UK_LOCAL_GATE_SCOPE): - raise RuntimeError("local gate report does not cover the declared scope.") - receipt = local_artifact_reference(report_path, repository_hint=_REPOSITORY) - state.gate_verdicts = { - gate_id: { - "verdict": str(payload["status"]), - "receipt": f"{receipt}#/gates/{gate_id}", - } - for gate_id, payload in gates.items() - if isinstance(payload, Mapping) - } - if set(state.gate_verdicts) != set(UK_LOCAL_GATE_SCOPE): - raise RuntimeError("local gate verdicts are malformed.") - - -def _dry_run_plan( - args: argparse.Namespace, - *, - clone: UKLadderRowwiseDatasetResult, - problem: UKRowwiseLocalMatrix, - source_year: int, - input_artifact: Mapping[str, Any], - ladder_artifact: Mapping[str, Any], - target_provenance: Mapping[str, Any], - binding_adjudications: Mapping[str, Any], - cross_grain: Mapping[str, Any], -) -> dict[str, Any]: - return { - "schema_version": 2, - "build_kind": "uk_rowwise_calibrated_candidate_plan", - "dry_run": True, - "candidate_scope": "adjudicated_partial", - "bound_target_families": list(args._bound_families), - "binding_adjudications": dict(binding_adjudications), - "cross_grain": dict(cross_grain), - "ladder_target_provenance": dict(target_provenance), - "inputs": { - "dataset": dict(input_artifact), - "ladder": dict(ladder_artifact), - }, - "sampling": dict(args._sampling_receipt), - "survey_year": source_year, - "calibration_year": ( - args._joint_inputs_receipt["calibration_year"] - if args._joint_inputs_receipt is not None - else source_year - ), - "rung_surface": { - "rung": UK_SAMPLE_RUNG_TOKENS[args.sample_fraction], - "fraction": args.sample_fraction, - "unreachable_check": "completed", - }, - "releasable": args.sample_fraction == 1.0 - and args.engine_blocks == 1 - and args.dataset_households is None, - "parameters": _parameters(args, source_year=source_year), - "shapes": { - "person": list(clone.frame.table("person").shape), - "benunit": list(clone.frame.table("benunit").shape), - "household": list(clone.frame.table("household").shape), - "local_matrix": list(problem.matrix.shape), - }, - "target_count": int(len(problem.targets)), - "gate": _gate_payload(clone.gate, phase="post_clone"), - } - - -def _write_output_bundle( - args: argparse.Namespace, - *, - candidate: UKLadderRowwiseDatasetResult, - clone: UKLadderRowwiseDatasetResult, - problem: UKRowwiseLocalMatrix, - solve: UKRowwiseDoctrineSolve, - local_diagnostics: pd.DataFrame, - target_registry: TargetRegistry, - target_geography_levels: Mapping[str, str], - rotated_holdout: Mapping[str, object], - support: pd.DataFrame, - calibration_record: MassChangeRecord, - source_year: int, - output_paths: Mapping[str, Path], - input_artifact: Mapping[str, Any], - ladder_artifact: Mapping[str, Any], - target_provenance: Mapping[str, Any], - cross_grain: Mapping[str, Any], -) -> dict[str, Any]: - """Stage the complete bundle, then publish atomically per file.""" - - out_dir = output_paths["manifest"].parent - out_dir.parent.mkdir(parents=True, exist_ok=True) - staging_dir = Path( - tempfile.mkdtemp( - prefix=f".{out_dir.name}.rowwise-candidate.", - dir=out_dir.parent, - ) - ) - try: - staged = {key: staging_dir / path.name for key, path in output_paths.items()} - print( - f"staging candidate for {output_paths['dataset']}...", - file=sys.stderr, - flush=True, - ) - write_uk_rowwise_dataset(candidate, staged["dataset"]) - solve.diagnostics.to_csv(staged["diagnostics"], index=False) - if solve.dense_reference is not None: - _dense_reference_diagnostics_frame(solve).to_csv( - staged["dense_reference"], index=False - ) - _dataset_size_selection_frame(solve, problem=problem, clone=clone).to_csv( - staged["selection"], index=False - ) - support = support.copy() - support["support_below_floor"] = ( - (support["assigned_households"] < 50) - | (support["effective_sample_size"] < 50.0) - | (support["nonzero_source_households"] < 50) - ) - support.to_csv(staged["support"], index=False) - staged["past_cap"].write_text(_json_text(dict(solve.past_cap_census or {}))) - local_registry = _local_output_registry( - problem, - period=( - args._joint_inputs_receipt["calibration_year"] - if args._joint_inputs_receipt is not None - else source_year - ), - ) - local_registry.to_json(staged["local_registry"]) - write_uk_calibration_diagnostics( - solve.calibration_result, - staged["calibration_diagnostics"], - solve.frame, - target_geography_levels=target_geography_levels, - target_registry=target_registry, - local_area_support=support, - rotated_holdout=rotated_holdout, - build={ - "build_kind": "uk_rowwise_calibrated_candidate", - "candidate_scope": "adjudicated_partial", - }, - ) - calibration_diagnostics = json.loads( - staged["calibration_diagnostics"].read_text(encoding="utf-8") - ) - args._weakest_areas_by_fit = calibration_diagnostics["uk_diagnostics"][ - "weakest_areas_by_fit" - ] - - outputs = { - "dataset": _artifact_info( - staged["dataset"], - reported_path=output_paths["dataset"], - ), - "solve_diagnostics": _artifact_info( - staged["diagnostics"], - reported_path=output_paths["diagnostics"], - ), - "area_support_summary": _artifact_info( - staged["support"], - reported_path=output_paths["support"], - ), - "past_cap_census": _artifact_info( - staged["past_cap"], - reported_path=output_paths["past_cap"], - ), - "calibration_diagnostics": _artifact_info( - staged["calibration_diagnostics"], - reported_path=output_paths["calibration_diagnostics"], - ), - "local_gate_report": _artifact_info(output_paths["local_gates"]), - "local_target_registry": _artifact_info( - staged["local_registry"], - reported_path=output_paths["local_registry"], - ), - } - if solve.dense_reference is not None: - outputs["dense_reference_diagnostics"] = _artifact_info( - staged["dense_reference"], - reported_path=output_paths["dense_reference"], - ) - outputs["dataset_size_selection"] = _artifact_info( - staged["selection"], - reported_path=output_paths["selection"], - ) - manifest = _manifest( - args, - candidate=candidate, - clone=clone, - problem=problem, - solve=solve, - support=support, - calibration_record=calibration_record, - source_year=source_year, - input_artifact=input_artifact, - ladder_artifact=ladder_artifact, - target_provenance=target_provenance, - cross_grain=cross_grain, - calibration_diagnostics=calibration_diagnostics, - outputs=outputs, - ) - staged["manifest"].write_text(_json_text(manifest)) - _publish_staged_files(staged, output_paths) - return manifest - finally: - shutil.rmtree(staging_dir) - - -def _manifest( - args: argparse.Namespace, - *, - candidate: UKLadderRowwiseDatasetResult, - clone: UKLadderRowwiseDatasetResult, - problem: UKRowwiseLocalMatrix, - solve: UKRowwiseDoctrineSolve, - support: pd.DataFrame, - calibration_record: MassChangeRecord, - source_year: int, - input_artifact: Mapping[str, Any], - ladder_artifact: Mapping[str, Any], - target_provenance: Mapping[str, Any], - cross_grain: Mapping[str, Any], - calibration_diagnostics: Mapping[str, Any], - outputs: Mapping[str, Any], -) -> dict[str, Any]: - abs_errors = solve.diagnostics["abs_relative_error"].to_numpy(dtype=np.float64) - old_total = float(calibration_record.old_total) - new_total = float(calibration_record.new_total) - past_cap = dict(solve.past_cap_census or {}) - gate_rows = args._gate_report.get("gates", {}) - # ``releasable`` follows the battery's own doctrine: release-blocking - # entries decide, diagnostic entries (target_fit, weight_ratio, ...) are - # reported but never veto. ``failing_gate_ids`` still lists every - # non-passing entry of either criticality. - release_gate_rows = { - gate_id: payload - for gate_id, payload in gate_rows.items() - if isinstance(payload, Mapping) and _is_release_blocking(payload) - } - all_gates_passed = bool(release_gate_rows) and all( - payload.get("status") == "passed" for payload in release_gate_rows.values() - ) - releasable, release_posture = _release_verdict( - sample_fraction=args.sample_fraction, - engine_blocks=args.engine_blocks, - release_blocking_gates_passed=all_gates_passed, - ) - area_gate = gate_rows.get("uk_local_area_support", {}) - area_exclusion_details = ( - area_gate.get("details", {}) if isinstance(area_gate, Mapping) else {} - ) - ladder_rows = int( - problem.target_frame["target_name"] - .astype(str) - .str.startswith("external:census_households/households@") - .sum() - ) - local_rows = int(len(problem.target_frame) - ladder_rows) - sample_stage = ( - [] - if args.sample_fraction == 1.0 - else [ - { - "stage": "sample", - "kind": uk_household_weight_kind(clone.frame).value, - } - ] - ) - return { - "schema_version": 2, - "build_kind": "uk_rowwise_calibrated_candidate", - "candidate_scope": "adjudicated_partial", - "created_at": datetime.now(UTC).isoformat(), - "git_commit": _git_commit(), - "git_dirty": _git_dirty(), - "bound_target_families": list(args._bound_families), - "binding_adjudications": dict(solve.binding_adjudications), - "cross_grain": dict(cross_grain), - "ladder_target_provenance": dict(target_provenance), - "parameters": _parameters(args, source_year=source_year), - "inputs": { - "dataset": dict(input_artifact), - "ladder": dict(ladder_artifact), - }, - "identity": { - "spine": { - **dict(input_artifact), - "spine_provenance": dict(args._spine_provenance), - }, - "ladder": { - **dict(ladder_artifact), - "layer_vintages": dict(target_provenance), - "matches_local_area_crosswalk_pin": True, - }, - **( - {"ledger": args._joint_inputs_receipt["artifact"].provenance()} - if args._joint_inputs_receipt is not None - else {} - ), - "code": {"git_commit": _git_commit(), "git_dirty": _git_dirty()}, - "runtime": runtime_provenance(), - "sampling": dict(args._sampling_receipt), - "survey_year": source_year, - "calibration_year": ( - args._joint_inputs_receipt["calibration_year"] - if args._joint_inputs_receipt is not None - else source_year - ), - }, - "sampling": dict(args._sampling_receipt), - "rung_surface": { - **dict(args._rung_surface), - "rung": UK_SAMPLE_RUNG_TOKENS[args.sample_fraction], - "fraction": args.sample_fraction, - "unreachable_check": "completed", - }, - "outputs": dict(outputs), - "geography": { - "constituencies_assigned": int( - support.loc[ - support["geography_level"] == "constituency", - "area_code", - ].nunique() - ), - "local_authorities_assigned": int( - support.loc[ - support["geography_level"] == "local_authority", - "area_code", - ].nunique() - ), - "missing_geography_rows": 0, - "ladder_gate": _gate_payload(candidate.gate, phase="post_calibration"), - }, - "gate": _gate_payload(candidate.gate, phase="post_calibration"), - "weights": { - "household_weight_kind": uk_household_weight_kind(candidate.frame).value, - "household_weight_kind_chain": [ - { - "stage": "staging", - "kind": uk_household_weight_kind(clone.frame).value, - }, - *sample_stage, - { - "stage": "ladder_clone", - "kind": uk_household_weight_kind(clone.frame).value, - }, - { - "stage": "rowwise_calibration", - "kind": uk_household_weight_kind(candidate.frame).value, - }, - ], - "mass_log_records_before_calibration": len(clone.frame.mass_log), - "mass_log_records": len(candidate.frame.mass_log), - "calibration_mass_change": { - "entity": str(calibration_record.entity), - "old_total": old_total, - "new_total": new_total, - "relative_shift": (new_total - old_total) / old_total, - "declared_factor": calibration_record.declared_factor, - "reason": str(calibration_record.reason), - }, - "abs_delta": abs(new_total - old_total), - "declared_stretch_bound": float(UK_LOCAL_MAX_WEIGHT_RATIO), - "stretch_reference": "pool_design" - if solve.size_receipt is None - else "normalized_horvitz_thompson_w_over_q", - # Against the frame the refit started from (the pool design on a - # dense run, the Horvitz-Thompson baseline on a size run)... - "realized_max_weight_ratio_vs_stretch_reference": float( - np.max( - np.divide( - np.asarray(solve.weights, dtype=np.float64), - np.asarray(solve.initial_weights), - ) - ) - ), - # ...and always against the pool design weights themselves. - "realized_max_weight_ratio_vs_design": float( - np.max( - np.divide( - np.asarray(solve.weights, dtype=np.float64), - np.asarray(_design_weights_for(solve), dtype=np.float64), - ) - ) - ), - }, - "solve": { - "n_targets": int(len(problem.targets) + len(solve.national_diagnostics)), - "n_targets_by_kind": { - "local": local_rows, - "ladder": ladder_rows, - "national": int(len(solve.national_diagnostics)), - }, - "n_households": int(solve.frame.n("household")), - "pool_households": int(problem.n_households), - "dataset_size": None - if solve.size_receipt is None - else { - **dict(solve.size_receipt), - "dense_reference": _dense_reference_summary(solve), - }, - "initial_loss": float(solve.initial_loss), - "final_loss": float(solve.final_loss), - "max_abs_relative_error": float(abs_errors.max()), - "median_abs_relative_error": float(np.median(abs_errors)), - "n_nonzero": int(solve.n_nonzero), - "past_cap": {key: int(past_cap[key]) for key in _PAST_CAP_COUNT_KEYS}, - "loss_shape": "capped_relative_error", - "target_weight_rule": args.target_weight_rule, - "target_weight_rule_override": dict(args._doctrine_override_receipt), - "measure_resolution": dict(args._measure_resolution), - "cross_grain": dict(cross_grain), - "binding_adjudications": dict(solve.binding_adjudications), - "area_support_exclusions": { - "resource": "local_area_support_exclusions.json", - "entries_stood_on": sorted( - area_exclusion_details.get("reviewed_exclusions", {}) - ), - "stale": list(area_exclusion_details.get("stale_exclusions", [])), - "unknown": list(area_exclusion_details.get("unknown_exclusions", [])), - }, - "past_cap_by_kind": { - "local": dict(solve.past_cap_census or {}), - "national": dict(solve.national_past_cap_census or {}), - "all": dict(solve.all_past_cap_census or {}), - }, - }, - "diagnostics": { - "schema_version": calibration_diagnostics["schema_version"], - "target_registry": calibration_diagnostics["target_registry"], - "weakest_families": calibration_diagnostics["uk_diagnostics"][ - "weakest_families" - ], - "weakest_areas_by_fit": calibration_diagnostics["uk_diagnostics"][ - "weakest_areas_by_fit" - ], - "rotated_holdout": calibration_diagnostics["uk_diagnostics"][ - "rotated_holdout" - ], - }, - "support": { - "min_assigned_households": int(support["assigned_households"].min()), - "min_nonzero_households": int(support["nonzero_households"].min()), - "min_effective_sample_size": float(support["effective_sample_size"].min()), - "by_geography_level": { - str(level): { - "min_assigned_households": int(rows["assigned_households"].min()), - "min_nonzero_households": int(rows["nonzero_households"].min()), - "min_effective_sample_size": float( - rows["effective_sample_size"].min() - ), - "min_nonzero_source_households": int( - rows["nonzero_source_households"].min() - ), - } - for level, rows in support.groupby("geography_level", sort=True) - }, - }, - "fit": { - "local_by_family": uk_fit_by_family(solve.diagnostics), - "national_by_family": uk_fit_by_family(solve.national_diagnostics), - "weakest_families": sorted( - [ - *uk_fit_by_family(solve.diagnostics), - *uk_fit_by_family(solve.national_diagnostics), - ], - key=lambda row: ( - -float(row["worst_abs_relative_error"]), - row["family"], - ), - )[:10], - "weakest_areas_by_fit": dict(args._weakest_areas_by_fit), - "support_limited_misses": dict(args._support_limited_misses), - "rotated_holdout": dict(args._rotated_holdout), - }, - "vintages": ( - _local_vintage_census(args._joint_inputs_receipt["local_registry"]) - if args._joint_inputs_receipt is not None - else [] - ), - "failing_gate_ids": sorted( - gate_id - for gate_id, payload in gate_rows.items() - if not isinstance(payload, Mapping) or payload.get("status") != "passed" - ), - "releasable": releasable and args.dataset_households is None, - "release_posture": { - **release_posture, - **( - {} - if args.dataset_households is None - else {"size_certification_present": False} - ), - }, - "ladder_household_uprating": dict( - cross_grain.get("ladder_household_uprating") - or {"applied": False, "reason": "no cross-grain receipt"} - ), - # The reviewed measure exclusions the national compile stood on - # (name -> register record), so the narrowing is in the evidence. - "measure_exclusions": { - str(name): dict(record) - for name, record in sorted( - ( - (getattr(args, "_joint_inputs_receipt", None) or {}).get( - "measure_exclusions" - ) - or {} - ).items() - ) - }, - "blocked_at_f100": bool(getattr(args, "_blocked_failures", [])), - "blocking_failures": list(getattr(args, "_blocked_failures", [])), - "diagnostic_failures": list(getattr(args, "_diagnostic_failures", [])), - "release_gate_failures_not_enforced": list( - getattr(args, "_unenforced_release_failures", []) - ), - } - - -def _parameters(args: argparse.Namespace, *, source_year: int) -> dict[str, Any]: - return { - "n_clones": int(args.n_clones), - "dataset_households": args.dataset_households, - "seed": int(args.seed), - "selection_seed": None - if args.dataset_households is None - else int(args.seed if args.selection_seed is None else args.selection_seed), - "selection_pi_hi": None - if args.dataset_households is None - else float(args.selection_pi_hi), - "size_checkpoint": bool( - args.dataset_households is not None - and not args.no_size_checkpoint - and args.resume_size_checkpoint is None - ), - "resume_size_checkpoint": None - if args.resume_size_checkpoint is None - else str(args.resume_size_checkpoint.expanduser().resolve()), - "source_year": source_year, - "source_lineage_modulus": args.source_lineage_modulus, - "sample_fraction": float(args.sample_fraction), - "sample_seed": int(args.sample_seed), - "engine_blocks": int(args.engine_blocks), - "target_weight_rule": args.target_weight_rule, - "release_candidate": bool(args.release_candidate), - "skip_holdout": bool(args.skip_holdout), - "epochs": int(args.epochs), - "learning_rate": float(args.learning_rate), - "expected_constituency_vintage": str(args.expected_constituency_vintage), - "doctrine": _doctrine_bounds(), - "solve_options": { - "conserve_mass": _CONSERVE_MASS, - "target_records": _TARGET_RECORDS, - "l0_lambda": _L0_LAMBDA, - "budget_iters": _BUDGET_ITERS, - }, - } - - -def _design_weights_for(solve: UKRowwiseDoctrineSolve) -> np.ndarray: - """The pool design weights aligned to the solve's exported rows.""" - if solve.selected_support is None or solve.dense_reference is None: - return np.asarray(solve.initial_weights, dtype=np.float64) - return np.asarray(solve.dense_reference.initial_weights, dtype=np.float64)[ - np.asarray(solve.selected_support, dtype=np.int64) - ] - - -def _dense_reference_summary(solve: UKRowwiseDoctrineSolve) -> dict[str, Any] | None: - """Manifest-sized evidence of the dense solve a size run was cut from.""" - - dense = solve.dense_reference - if dense is None: - return None - local_errors = dense.diagnostics["abs_relative_error"].to_numpy(dtype=np.float64) - national_errors = dense.national_diagnostics["abs_relative_error"].to_numpy( - dtype=np.float64 - ) - past_cap = dict(dense.past_cap_census or {}) - return { - "initial_loss": float(dense.initial_loss), - "final_loss": float(dense.final_loss), - "n_nonzero": int(dense.n_nonzero), - "n_households": int(dense.weights.size), - "max_abs_relative_error": float(local_errors.max()) - if local_errors.size - else None, - "median_abs_relative_error": float(np.median(local_errors)) - if local_errors.size - else None, - "national_max_abs_relative_error": float(national_errors.max()) - if national_errors.size - else None, - "past_cap": {key: int(past_cap[key]) for key in _PAST_CAP_COUNT_KEYS}, - "weights": uk_weight_summary(dense.weights), - "local_by_family": uk_fit_by_family(dense.diagnostics), - "national_by_family": uk_fit_by_family(dense.national_diagnostics), - "diagnostics_file": DENSE_REFERENCE_DIAGNOSTICS_FILENAME, - } - - -def _dense_reference_diagnostics_frame(solve: UKRowwiseDoctrineSolve) -> pd.DataFrame: - """Every target's dense-reference estimate, local rows then national rows.""" - - dense = solve.dense_reference - assert dense is not None - local = dense.diagnostics.copy() - local.insert(0, "grain", local["area_type"].astype(str)) - national = dense.national_diagnostics.copy() - national.insert(0, "grain", "national") - return pd.concat([local, national], ignore_index=True, sort=False) - - -def _dataset_size_selection_frame( - solve: UKRowwiseDoctrineSolve, - *, - problem: UKRowwiseLocalMatrix, - clone: UKLadderRowwiseDatasetResult, -) -> pd.DataFrame: - """One row per selected pool household: identity, design, draw and refit.""" - - dense = solve.dense_reference - receipt = solve.size_receipt - assert dense is not None and receipt is not None - support = np.asarray(solve.selected_support, dtype=np.int64) - household = clone.frame.table("household") - clone_column = ladder_clone_index_column("household") - ids = household["household_id"].to_numpy()[support] - expected = np.asarray([problem.household_ids[i] for i in support]) - if not np.array_equal(ids, expected): - raise RuntimeError( - "the cloned pool's household order does not match the solve's " - "matrix columns; the selection sidecar would misattribute rows." - ) - inclusion = np.asarray(receipt["inclusion_probabilities"], dtype=np.float64) - if inclusion.shape != support.shape: - raise RuntimeError("selection receipt inclusion probabilities are misaligned.") - return pd.DataFrame( - { - "pool_row_index": support, - "household_id": ids, - "clone_index": household[clone_column].to_numpy()[support] - if clone_column in household.columns - else np.zeros(support.size, dtype=np.int64), - "design_weight": dense.initial_weights[support], - "inclusion_probability": inclusion, - "certainty": inclusion >= 1.0, - "ht_baseline_weight": np.asarray(solve.initial_weights, dtype=np.float64), - "refit_weight": np.asarray(solve.weights, dtype=np.float64), - } - ) - - -def _local_output_registry( - problem: UKRowwiseLocalMatrix, - *, - period: int, -) -> TargetRegistry: - specs = [] - for row in problem.target_frame.itertuples(index=False): - payload = row._asdict() - specs.append( - TargetSpec( - name=str(payload["target_name"]), - entity="household", - value=float(payload["value"]), - measure=str(payload["metric"]), - period=int(payload.get("period", period)), - source=str(payload.get("source", "uk_rowwise_local_surface")), - family=str(payload["family"]), - metadata={ - "area_type": str(payload["area_type"]), - "area_code": str(payload["area_code"]), - "metric": str(payload["metric"]), - }, - ) - ) - return TargetRegistry(specs, country="uk") - - -def _doctrine_bounds() -> dict[str, Any]: - return { - "target_loss_cap": float(UK_LOCAL_TARGET_LOSS_CAP), - "max_weight_ratio": float(UK_LOCAL_MAX_WEIGHT_RATIO), - "scale_rule": UK_LOCAL_SOLVE_DOCTRINE.scale_rule, - "target_weight_rule": UK_LOCAL_SOLVE_DOCTRINE.target_weight_rule, - "solve_epochs": int(UK_LOCAL_SOLVE_EPOCHS), - "clone_count": int(UK_LOCAL_CLONE_COUNT), - } - - -def _gate_payload(gate: GateResult, *, phase: str) -> dict[str, Any]: - return { - "name": str(gate.name), - "passed": bool(gate.passed), - "failures": list(gate.failures), - "details": dict(gate.details), - "phase": phase, - } - - -def _validate_solve_result( - solve: UKRowwiseDoctrineSolve, - *, - problem: UKRowwiseLocalMatrix, -) -> None: - if solve.past_cap_census is None: - raise RuntimeError( - "doctrine solve returned no past-cap census; refusing candidate." - ) - expected_count = ( - problem.n_households - if solve.selected_support is None - else len(solve.selected_support) - ) - if len(solve.weights) != expected_count: - raise RuntimeError( - "doctrine solve returned a weight vector with the wrong length." - ) - weights = np.asarray(solve.weights, dtype=np.float64) - if not np.isfinite(weights).all() or (weights < 0).any(): - raise RuntimeError( - "doctrine solve returned non-finite or negative household weights." - ) - if not np.isfinite([solve.initial_loss, solve.final_loss]).all(): - raise RuntimeError("doctrine solve returned a non-finite loss.") - errors = solve.diagnostics["abs_relative_error"].to_numpy(dtype=np.float64) - if len(errors) != len(problem.targets) or not np.isfinite(errors).all(): - raise RuntimeError( - "doctrine solve returned incomplete or non-finite diagnostics." - ) - missing_counts = sorted(set(_PAST_CAP_COUNT_KEYS) - set(solve.past_cap_census)) - if missing_counts: - raise RuntimeError( - f"past-cap census is missing count field(s): {missing_counts}." - ) - - -def _validate_support_summary(support: pd.DataFrame) -> None: - required = { - "geography_level", - "area_code", - "assigned_households", - "nonzero_households", - "nonzero_source_households", - "effective_sample_size", - } - missing = sorted(required - set(support.columns)) - if missing or support.empty: - raise RuntimeError( - f"area support summary is empty or missing required columns: {missing}." - ) - numeric = sorted(required - {"geography_level", "area_code"}) - values = support[numeric].to_numpy(dtype=np.float64) - if not np.isfinite(values).all() or (values < 0).any(): - raise RuntimeError("area support summary contains invalid values.") - - -def _validate_cli_args(args: argparse.Namespace) -> None: - ledger_values = ( - args.ledger_facts, - args.ledger_facts_sha256, - args.ledger_manifest_sha256, - ) - if any(value is not None for value in ledger_values) and not all( - value is not None for value in ledger_values - ): - raise ValueError( - "--ledger-facts, --ledger-facts-sha256, and " - "--ledger-manifest-sha256 must be supplied together." - ) - if args.ledger_facts is not None and ( - args.input_sha256 is None or args.ladder_sha256 is None - ): - raise ValueError( - "the joint registry path requires --input-sha256 and --ladder-sha256." - ) - if args.selection_seed is not None and args.dataset_households is None: - raise ValueError("--selection-seed requires --dataset-households.") - if not (0.0 < args.selection_pi_hi <= 1.0): - raise ValueError("--selection-pi-hi must be in (0, 1].") - if args.selection_pi_hi != 1.0 and args.dataset_households is None: - raise ValueError("--selection-pi-hi requires --dataset-households.") - if args.no_size_checkpoint and args.dataset_households is None: - raise ValueError("--no-size-checkpoint requires --dataset-households.") - if args.resume_size_checkpoint is not None: - if args.dataset_households is None: - raise ValueError("--resume-size-checkpoint requires --dataset-households.") - if args.no_size_checkpoint: - raise ValueError( - "--resume-size-checkpoint already implies no new checkpoint; " - "drop --no-size-checkpoint." - ) - if args.dataset_households is not None: - if args.dataset_households <= 0: - raise ValueError("--dataset-households must be positive.") - if args.release_candidate: - raise ValueError( - "--dataset-households is candidate-only: size-specific matched comparison and promotion scorecard are required before release." - ) - if args.release_candidate: - required_release = { - "--input-sha256": args.input_sha256, - "--ladder-sha256": args.ladder_sha256, - "--ledger-facts": args.ledger_facts, - "--ledger-facts-sha256": args.ledger_facts_sha256, - "--ledger-manifest-sha256": args.ledger_manifest_sha256, - } - missing_release = [ - name for name, value in required_release.items() if value is None - ] - if missing_release: - raise ValueError( - "--release-candidate requires pinned joint inputs: " - + ", ".join(missing_release) - ) - refused = [] - if args.target_weight_rule != UK_LOCAL_SOLVE_DOCTRINE.target_weight_rule: - refused.append("--target-weight-rule") - if args.epochs != UK_LOCAL_SOLVE_EPOCHS: - refused.append(f"--epochs != doctrine {UK_LOCAL_SOLVE_EPOCHS}") - if args.n_clones != UK_LOCAL_CLONE_COUNT: - refused.append(f"--n-clones != doctrine {UK_LOCAL_CLONE_COUNT}") - if args.measure_exclusions is not None: - refused.append("--measure-exclusions") - if args.skip_holdout: - refused.append("--skip-holdout") - if args.engine_blocks > 1: - refused.append("--engine-blocks > 1") - if args.sample_fraction != 1.0: - refused.append("--sample-fraction != 1.0") - if refused: - raise ValueError( - "--release-candidate refuses non-release settings: " - + ", ".join(refused) - ) - if args.n_clones <= 0: - raise ValueError("--n-clones must be positive.") - if args.seed < 0: - raise ValueError("--seed must be non-negative.") - if args.sample_fraction not in UK_SAMPLE_RUNG_TOKENS: - raise ValueError( - "--sample-fraction must be one of " - f"{sorted(UK_SAMPLE_RUNG_TOKENS)}, got {args.sample_fraction!r}." - ) - if args.sample_seed < 0: - raise ValueError("--sample-seed must be non-negative.") - if args.engine_blocks <= 0: - raise ValueError("--engine-blocks must be positive.") - if args.engine_blocks > 1 and args.engine_blocks != args.n_clones: - raise ValueError("--engine-blocks greater than one must equal --n-clones.") - if args.source_year is not None and args.source_year <= 0: - raise ValueError("--source-year must be positive.") - if args.epochs <= 0: - raise ValueError("--epochs must be positive.") - if not np.isfinite(args.learning_rate) or args.learning_rate <= 0: - raise ValueError("--learning-rate must be positive and finite.") - if not str(args.expected_constituency_vintage).strip(): - raise ValueError("--expected-constituency-vintage must be non-empty.") - - -def _output_paths( - out_dir: Path, - *, - source_year: int, - calibration_year: int, -) -> dict[str, Path]: - dataset = out_dir / CANDIDATE_FILENAME_TEMPLATE.format( - calibration_year=calibration_year - ) - return { - "dataset": dataset, - "manifest": out_dir / MANIFEST_FILENAME, - "diagnostics": out_dir / SOLVE_DIAGNOSTICS_FILENAME, - "support": out_dir / AREA_SUPPORT_FILENAME, - "past_cap": out_dir / PAST_CAP_FILENAME, - "calibration_diagnostics": out_dir / CALIBRATION_DIAGNOSTICS_FILENAME, - "local_gates": out_dir - / LOCAL_GATE_REPORT_FILENAME_TEMPLATE.format(calibration_year=calibration_year), - "local_registry": out_dir / LOCAL_REGISTRY_FILENAME, - "dense_reference": out_dir / DENSE_REFERENCE_DIAGNOSTICS_FILENAME, - "selection": out_dir / DATASET_SIZE_SELECTION_FILENAME, - } - - -def _validate_output_paths( - output_paths: Mapping[str, Path], - *, - input_h5: Path, - ladder_path: Path, -) -> None: - resolved = {name: path.resolve() for name, path in output_paths.items()} - if len(set(resolved.values())) != len(resolved): - raise ValueError("candidate output paths must be distinct.") - protected = {input_h5.resolve(), ladder_path.resolve()} - collisions = sorted(str(path) for path in resolved.values() if path in protected) - if collisions: - raise ValueError( - "candidate outputs must differ from --input-h5 and --ladder; " - f"collision(s): {collisions}." - ) - existing = sorted(str(path) for path in resolved.values() if path.exists()) - if existing: - raise FileExistsError( - f"refusing to overwrite existing candidate artifact(s): {existing}." - ) - - -def _publish_staged_files( - staged: Mapping[str, Path], - output_paths: Mapping[str, Path], -) -> None: - out_dir = output_paths["manifest"].parent - created_out_dir = not out_dir.exists() - out_dir.mkdir(parents=True, exist_ok=True) - publish_order = ( - "dataset", - "diagnostics", - "support", - "past_cap", - "calibration_diagnostics", - "local_registry", - "dense_reference", - "selection", - "manifest", - ) - published: list[Path] = [] - succeeded = False - try: - for key in publish_order: - if key in _SIZE_RUN_ONLY_OUTPUTS and not staged[key].exists(): - continue - destination = output_paths[key] - if destination.exists(): - raise FileExistsError( - "candidate output appeared during publication; refusing " - f"to overwrite {destination}." - ) - staged[key].replace(destination) - published.append(destination) - succeeded = True - finally: - if not succeeded: - for path in reversed(published): - path.unlink(missing_ok=True) - if created_out_dir: - try: - out_dir.rmdir() - except OSError: - pass - - -def _assert_artifacts_unchanged( - *, - input_h5: Path, - input_artifact: Mapping[str, Any], - ladder_path: Path, - ladder_artifact: Mapping[str, Any], -) -> None: - for label, path, before in ( - ("input H5", input_h5, input_artifact), - ("ladder", ladder_path, ladder_artifact), - ): - after = _artifact_info(path) - if after["sha256"] != before["sha256"] or after["bytes"] != before["bytes"]: - raise RuntimeError( - f"{label} changed during the candidate build; refusing to " - "bind mixed source bytes." - ) - - -def _require_file(path: Path, *, label: str) -> Path: - resolved = path.expanduser().resolve() - if not resolved.is_file(): - raise FileNotFoundError(f"{label} artifact not found: {resolved}.") - return resolved - - -def _source_year(requested: int | None, *, time_period: str) -> int: - if requested is not None: - if requested <= 0: - raise ValueError("--source-year must be positive.") - return requested - prefix = str(time_period).strip()[:4] - if len(prefix) != 4 or not prefix.isdigit(): - raise ValueError( - "Could not infer source year from input H5 time_period; pass --source-year." - ) - return int(prefix) - - -def _artifact_info( - path: Path, - *, - reported_path: Path | None = None, -) -> dict[str, Any]: - digest = hashlib.sha256() - with path.open("rb") as stream: - for chunk in iter(lambda: stream.read(1024 * 1024), b""): - digest.update(chunk) - return { - "path": str((reported_path or path).resolve()), - "sha256": digest.hexdigest(), - "bytes": int(path.stat().st_size), - } - - -def _git_commit() -> str | None: - result = subprocess.run( - ["git", "rev-parse", "HEAD"], - check=False, - capture_output=True, - text=True, - ) - if result.returncode != 0: - return None - return result.stdout.strip() - - -def _git_dirty() -> bool | None: - """Measured, not asserted: tracked modifications in the working tree. - - ``None`` when git cannot answer (no repository), so a downstream - assembler records the pin as unmeasured rather than clean. - """ - - result = subprocess.run( - ["git", "status", "--porcelain", "--untracked-files=no"], - check=False, - capture_output=True, - text=True, - ) - if result.returncode != 0: - return None - return bool(result.stdout.strip()) - - -def _json_text(payload: Any) -> str: - return ( - json.dumps( - payload, - allow_nan=False, - indent=2, - sort_keys=True, - ) - + "\n" - ) - +from microcosm.build.uk_runtime.full_build_cli import main if __name__ == "__main__": raise SystemExit(main()) diff --git a/tools/build_uk_rowwise_dataset.py b/tools/build_uk_rowwise_dataset.py index 6f3bb2482..a56107efa 100644 --- a/tools/build_uk_rowwise_dataset.py +++ b/tools/build_uk_rowwise_dataset.py @@ -1,1913 +1,44 @@ -"""Build a Microcosm UK row-wise local-geography dataset. +"""Compatibility command for the canonical UK full build, all targets by default. -This is the narrow build driver for the UK local replacement path. Its -sanctioned input is the microcosm-built national spine — the assembly seam's -accepted, sha-pinned single-year H5 with its declared weight kind and mass -log — never the retired certified compact and never anything upstream of the -seam. The driver clones the entity tables, assigns each household a finest -available geography row (the ratified OA ladder, or the crosswalk sampler as -the coverage-check path), and writes diagnostics that prove coverage, weight -preservation, and per-area support. +Geography-only cloning/export orchestration is retired. Runtime cloning and +geography helpers remain available for analysis and graph-owned population work. """ from __future__ import annotations -import argparse -import hashlib -import json -import subprocess import sys -import time -import uuid -from datetime import UTC, datetime -from pathlib import Path -from typing import Any -import numpy as np -import pandas as pd - -from microcosm.build.logbook import canonical_json_bytes -from microcosm.build.logbook_adoption import ( - AttemptState, - append_phase, - apply_error_verdict, - error_receipt_path, - git_code_pin, - local_artifact_reference, - preflight_digest, - record_terminal_attempt, - resolve_predecessor, - role_pins_digest, - sha256_argument, - write_error_receipt, -) -from microcosm.build.uk_runtime import ( - MASS_CONSERVATION_RELATIVE_TOLERANCE, - PERSON_ID_COLUMNS, - POOL_SOURCE_LINEAGE_COLUMN, - UK_GEOGRAPHY_LADDER_COLUMNS, - UKLadderRowwiseDatasetResult, - apply_uk_source_lineage_modulus, - assign_household_geography, - assign_uk_geography_ladder, - build_official_uk_geography_crosswalk, - clone_entity_frame, - clone_uk_dataset_with_ladder_geography, - clone_uk_dataset_with_rowwise_geography, - expected_uk_ladder_area_support, - expected_uk_rowwise_area_support, - geography_coverage_summary, - id_multiplier_for_values, - ladder_clone_index_column, - load_uk_local_area_crosswalk, - load_uk_oa_ladder, - read_uk_single_year_weight_metadata, - uk_geography_ladder_gate, - uk_household_weight_kind, - uk_ladder_area_support_summary, - uk_region_mix, - uk_time_period, - validate_geography_coverage, - write_geography_crosswalk, -) -from microcosm.build.uk_runtime.rowwise_dataset import ( - UK_SPINE_LINEAGE_COLUMNS, - _refuse_preassigned_geography, -) -from microcosm.frame import engine_tables - -CROSSWALK_FILENAME = "uk_official_geography_crosswalk.csv.gz" -DATASET_FILENAME_TEMPLATE = "{input_stem}_rowwise.h5" -MANIFEST_FILENAME = "rowwise_build_manifest.json" -COVERAGE_FILENAME = "geography_coverage_summary.csv" -DRY_RUN_PLAN_FILENAME = "rowwise_dry_run_plan.json" -AREA_SUPPORT_FILENAME = "area_support_summary.csv" -EXPECTED_SUPPORT_BOTTOM_AREAS = 15 -_UK_ROWWISE_PIPELINE = "uk-local-rowwise" -_REPOSITORY = Path(__file__).resolve().parents[1] - - -def _candidate_clone_counts_argument(value: str) -> tuple[int, ...]: - parts = value.split(",") - if not value.strip() or any(not part.strip() for part in parts): - raise argparse.ArgumentTypeError( - "candidate clone counts must be a non-empty comma list of positive integers" - ) - try: - clone_counts = [int(part.strip()) for part in parts] - except ValueError as error: - raise argparse.ArgumentTypeError( - "candidate clone counts must be a comma list of positive integers" - ) from error - if any(clone_count <= 0 for clone_count in clone_counts): - raise argparse.ArgumentTypeError( - "candidate clone counts must all be positive integers" - ) - return tuple(sorted(set(clone_counts))) - - -def _parse_args(argv: list[str] | None = None) -> argparse.Namespace: - parser = argparse.ArgumentParser() - parser.add_argument( - "--input-h5", - type=Path, - required=True, - help="Compact Microcosm UK single-year H5 to clone.", - ) - parser.add_argument( - "--input-sha256", - type=sha256_argument, - help="Optional SHA-256 pin for --input-h5; mismatches fail before H5 parsing.", - ) - parser.add_argument( - "--out", - type=Path, - required=True, - help="Output directory for the row-wise H5 and diagnostics.", - ) - parser.add_argument( +_RETIRED_OPTIONS = frozenset( + { "--crosswalk", - type=Path, - help=( - "Optional existing official geography crosswalk CSV/CSV.GZ. If omitted, " - "the driver downloads public source tables and builds one." - ), - ) - parser.add_argument( - "--ladder", - type=Path, - help=( - "UK OA-ladder NPZ artifact (tools/build_uk_oa_ladder_artifact.py). " - "When set, geography is assigned through the ratified ladder route " - "with its release-blocking gate, instead of the crosswalk sampler. " - "Mutually exclusive with --crosswalk and the coverage-code checks." - ), - ) - parser.add_argument( - "--ladder-sha256", - type=sha256_argument, - help="Optional SHA-256 pin for --ladder; mismatches fail before NPZ parsing.", - ) - parser.add_argument( - "--expected-constituency-vintage", - default="2024_pcon", - help=( - "Constituency vintage the ladder artifact must declare " - "(vintage_policy: error). Applies to the --ladder route." - ), - ) - parser.add_argument( "--constituency-codes", - type=Path, - help="Optional CSV containing a `code` column for constituency coverage checks.", - ) - parser.add_argument( "--la-codes", - type=Path, - help="Optional CSV containing a `code` column for local-authority coverage checks.", - ) - parser.add_argument("--n-clones", type=int, default=2) - parser.add_argument( "--candidate-clone-counts", - type=_candidate_clone_counts_argument, - help=( - "Dry-run only: comma-separated candidate clone counts to plan " - "independently at the build seed (deduplicated and sorted)." - ), - ) - parser.add_argument("--seed", type=int, default=42) - parser.add_argument( - "--source-year", - type=int, - help="Source year for cloned household lineage. Defaults to the input H5 time_period.", - ) - parser.add_argument( "--dataset-filename", - help=( - "Output H5 filename within --out. Defaults to the input H5 stem " - "plus '_rowwise.h5'." - ), - ) - parser.add_argument( "--allow-missing-country", - action="store_true", - help="Do not require all UK countries to appear in the input H5.", - ) - parser.add_argument( "--allow-blank-constituency", - action="store_true", - help="Allow blank constituency codes in the crosswalk.", - ) - parser.add_argument( "--allow-cross-region-assignment", - action="store_true", - help="Allow households to draw geography from any UK region in their country.", - ) - parser.add_argument( "--allow-constituency-collisions", - action="store_true", - help="Allow the same source household to be assigned to the same constituency across clones.", - ) - parser.add_argument( - "--source-lineage-modulus", - type=int, - help=( - "Pool inputs only: derive pool_source_household_id before cloning " - "from household_id = tier * 10**8 + base, leaving the immediate " - "source_household_id untouched. Spine inputs instead use sernum + " - "1{spi} * 10**d + 1{cgt_clone} * 10**(d+1) + 1{band_donor} * " - "10**(d+2) and carry authoritative explicit lineage columns, so " - "the modulus is refused for them. Also refused when the pool " - "column exists or the mapping would be an identity." - ), - ) - parser.add_argument( - "--dry-run", - action="store_true", - help=( - "Compute and write the clone plan (row/byte math, weight-kind " - "chain, the realized per-area support of the real sampler at " - "this seed, and the analytic collision-free expectation) as " - f"{DRY_RUN_PLAN_FILENAME} without cloning or writing a dataset. " - "When no --crosswalk is supplied, the freshly built crosswalk " - "cache is still written to --out." - ), - ) - parser.add_argument( - "--logbook-prev-row-digest", - type=sha256_argument, - help=( - "Optional current Logbook chain head. If omitted, " - "POPULACE_LOGBOOK_PREV_ROW_DIGEST is used, then genesis null." - ), - ) - return parser.parse_args(argv) - - -def _new_rowwise_build_id( - *, - route: str, - seed: int, - timestamp: datetime, -) -> str: - instant = timestamp.astimezone(UTC) - return ( - f"uk-local-rowwise-{route}-f100-s{seed}-" - f"{instant.strftime('%Y%m%dT%H%M%SZ')}-{uuid.uuid4().hex[:8]}" - ) - - -def _pin_from_artifact(info: dict[str, Any]) -> dict[str, object]: - return { - "sha256": str(info["sha256"]), - "size_bytes": int(info["bytes"]), } - - -def _rowwise_identity_digest( - *, - route: str, - pins: dict[str, dict[str, object]], - args: argparse.Namespace, - source_year: int, -) -> str: - payload = { - "build_kind": "uk_rowwise_local_geography_dataset", - "route": route, - "inputs": pins, - "parameters": _parameters(args, source_year=source_year), - "source_year": source_year, - } - return hashlib.sha256(canonical_json_bytes(payload)).hexdigest() - - -def _record_rowwise_attempt( - *, - state: AttemptState, - started_at: float, - started_ts: datetime, - seed: int, - code_pin: str, - disposition: str, - predecessor: str | None, - spool_dir: Path, -) -> Path: - return record_terminal_attempt( - state=state, - started_at=started_at, - started_ts=started_ts, - pipeline=_UK_ROWWISE_PIPELINE, - rung="f100", - seed=seed, - code_pin=code_pin, - disposition=disposition, - predecessor=predecessor, - spool_dir=spool_dir, - ) - - -def _record_rowwise_error( - *, - error: BaseException, - state: AttemptState, - started_at: float, - started_ts: datetime, - seed: int, - code_pin: str, - predecessor: str | None, - base_dir: Path, - spool_dir: Path, -) -> None: - error_path = write_error_receipt( - error_receipt_path(base_dir, build_id=state.build_id), - state=state, - pipeline=_UK_ROWWISE_PIPELINE, - error=error, - ) - apply_error_verdict( - state, - f"{local_artifact_reference(error_path, repository_hint=_REPOSITORY)}#/error_type", - ) - _record_rowwise_attempt( - state=state, - started_at=started_at, - started_ts=started_ts, - seed=seed, - code_pin=code_pin, - disposition="failed", - predecessor=predecessor, - spool_dir=spool_dir, - ) +) def main(argv: list[str] | None = None) -> int: - args = _parse_args(argv) - if args.candidate_clone_counts is not None and not args.dry_run: - raise ValueError( - "candidate-K planning is a dry-run surface; a real build has " - "exactly one K from --n-clones." - ) - if args.dry_run: - return _main_impl(args, attempt=None) - - started_at = time.perf_counter() - started_ts = datetime.now(UTC) - digest = preflight_digest(_UK_ROWWISE_PIPELINE) - state = AttemptState( - build_id=_new_rowwise_build_id( - route="attempt", - seed=args.seed, - timestamp=started_ts, - ), - identity_digest=digest, - input_pins_digest=digest, - phases_reached=["attempt_started"], - gate_verdicts={ - "pipeline": { - "verdict": "running", - "receipt": "pending-build-scoped-terminal-receipt", - } - }, - ) - attempt: dict[str, object] = { - "state": state, - "started_at": started_at, - "started_ts": started_ts, - "code_pin": "unresolved-local-git-code-pin", - # Logbook chain configuration is validated before any side effect: a - # malformed or conflicting predecessor refuses the run here, before - # the build can unlink stale coverage/crosswalk sidecars. Config - # refusals record no row, like argparse refusals. - "predecessor": resolve_predecessor(args.logbook_prev_row_digest), - } - try: - return _main_impl(args, attempt=attempt) - except Exception as error: - _record_rowwise_error( - error=error, - state=state, - started_at=started_at, - started_ts=started_ts, - seed=args.seed, - code_pin=str(attempt["code_pin"]), - predecessor=attempt["predecessor"], - base_dir=args.out, - spool_dir=args.out / "logbook-spool", - ) - raise - - -def _main_impl( - args: argparse.Namespace, - *, - attempt: dict[str, object] | None, -) -> int: - input_h5 = args.input_h5.resolve() - input_artifact = _artifact_info(input_h5) - _verify_artifact_pin( - input_artifact, - pinned_sha256=args.input_sha256, - option="--input-h5", - ) - args.out.mkdir(parents=True, exist_ok=True) - base_summary = _h5_summary(input_h5) - source_year = _source_year(args.source_year, base_summary=base_summary) - output_h5 = _dataset_output_path( - args.out, - dataset_filename=args.dataset_filename, - input_stem=input_h5.stem, - source_year=source_year, + arguments = list(sys.argv[1:] if argv is None else argv) + retired = sorted( + {argument.split("=", 1)[0] for argument in arguments} & _RETIRED_OPTIONS ) - _validate_output_paths(input_h5=input_h5, output_h5=output_h5, args=args) - if args.ladder_sha256 is not None and args.ladder is None: - raise ValueError( - "--ladder-sha256 requires --ladder; the crosswalk routes have no " - "ladder artifact to pin, and silently ignoring a supplied pin " - "would report verification that never happened." - ) - if args.ladder is not None: - if args.crosswalk is not None: - raise ValueError("--ladder and --crosswalk are mutually exclusive.") - if args.constituency_codes is not None or args.la_codes is not None: - raise ValueError( - "--ladder does not take coverage-code checks; the ladder gate " - "validates coverage." - ) - sidecars = { - (args.out / MANIFEST_FILENAME).resolve(), - (args.out / COVERAGE_FILENAME).resolve(), - (args.out / DRY_RUN_PLAN_FILENAME).resolve(), - (args.out / CROSSWALK_FILENAME).resolve(), - (args.out / AREA_SUPPORT_FILENAME).resolve(), - } - if args.ladder.resolve() in sidecars: - raise ValueError( - "--ladder must not point at a build sidecar path inside " - "--out; it would be overwritten." - ) - # A reused crosswalk output directory must not leave stale sidecars - # beside a ladder manifest that reports no coverage output. - (args.out / COVERAGE_FILENAME).unlink(missing_ok=True) - (args.out / CROSSWALK_FILENAME).unlink(missing_ok=True) - return _run_ladder_route( - args, - input_h5=input_h5, - input_artifact=input_artifact, - output_h5=output_h5, - base_summary=base_summary, - source_year=source_year, - attempt=attempt, - ) - if not args.dry_run: - (args.out / AREA_SUPPORT_FILENAME).unlink(missing_ok=True) - crosswalk_source = _load_or_build_crosswalk(args) - crosswalk = crosswalk_source.frame - crosswalk_path = crosswalk_source.path - area_codes_by_type = _area_codes_by_type(args) - coverage = _validate_optional_coverage(crosswalk, area_codes_by_type) - - if args.dry_run: - plan = _dry_run_plan( - args, - input_h5=input_h5, - input_artifact=input_artifact, - output_h5=output_h5, - crosswalk=crosswalk, - crosswalk_source=crosswalk_source, - base_summary=base_summary, - source_year=source_year, - coverage=coverage, - ) - plan_path = args.out / DRY_RUN_PLAN_FILENAME - plan_path.write_text(json.dumps(plan, indent=2, sort_keys=True) + "\n") - print(json.dumps(plan, indent=2, sort_keys=True)) - return 0 - if attempt is not None: - state = attempt["state"] - assert isinstance(state, AttemptState) - pins = { - "dataset": _pin_from_artifact(input_artifact), - "crosswalk": _pin_from_artifact(_artifact_info(crosswalk_path)), - } - attempt["code_pin"] = git_code_pin(_REPOSITORY) - state.build_id = _new_rowwise_build_id( - route="crosswalk", - seed=args.seed, - timestamp=attempt["started_ts"], - ) - state.input_pins_digest = role_pins_digest(pins) - state.identity_digest = _rowwise_identity_digest( - route="crosswalk", - pins=pins, - args=args, - source_year=source_year, - ) - append_phase(state, "configured") - append_phase(state, "inputs_pinned") - - result = clone_uk_dataset_with_rowwise_geography( - input_h5, - crosswalk, - output_path=output_h5, - n_clones=args.n_clones, - seed=args.seed, - source_year=source_year, - require_all_countries=not args.allow_missing_country, - require_constituency=not args.allow_blank_constituency, - constrain_to_region=not args.allow_cross_region_assignment, - avoid_constituency_collisions=not args.allow_constituency_collisions, - source_lineage_modulus=args.source_lineage_modulus, - ) - if attempt is not None: - state = attempt["state"] - assert isinstance(state, AttemptState) - append_phase(state, "cloned") - rowwise_summary = _rowwise_summary( - result, - base_summary=base_summary, - source_lineage_modulus=args.source_lineage_modulus, - ) - coverage_path = args.out / COVERAGE_FILENAME - coverage_artifact = None - if not coverage.empty: - coverage.to_csv(coverage_path, index=False) - coverage_artifact = _artifact_info(coverage_path) - else: - coverage_path.unlink(missing_ok=True) - - manifest = { - "schema_version": 1, - "build_kind": "uk_rowwise_local_geography_dataset", - "created_at": datetime.now(UTC).isoformat(), - "git_commit": _git_commit(), - "parameters": _parameters(args, source_year=source_year), - "inputs": { - "dataset": input_artifact, - "crosswalk": _artifact_info(crosswalk_path), - }, - "outputs": { - "dataset": _artifact_info(output_h5), - "crosswalk": ( - _artifact_info(crosswalk_path) if crosswalk_source.generated else None - ), - "coverage_summary": coverage_artifact, - "area_support_summary": None, - }, - "base_dataset": base_summary, - "rowwise_dataset": rowwise_summary, - "coverage": coverage.to_dict("records") if not coverage.empty else [], - } - manifest_path = args.out / MANIFEST_FILENAME - manifest_path.write_text(json.dumps(manifest, indent=2, sort_keys=True) + "\n") - if attempt is not None: - state = attempt["state"] - assert isinstance(state, AttemptState) - append_phase(state, "manifest_written") - state.gate_verdicts = { - "uk_mass_conservation": { - "verdict": ( - "passed" - if rowwise_summary["weights"]["mass_conservation"]["passed"] - else "failed" - ), - "receipt": ( - f"{local_artifact_reference(manifest_path, repository_hint=_REPOSITORY)}" - "#/rowwise_dataset/weights/mass_conservation" - ), - } - } - if coverage_artifact is not None: - state.gate_verdicts["uk_coverage"] = { - "verdict": "passed", - "receipt": ( - f"{local_artifact_reference(manifest_path, repository_hint=_REPOSITORY)}" - "#/rowwise_dataset/coverage" - ), - } - state.artifact_location = local_artifact_reference( - output_h5, - repository_hint=_REPOSITORY, - ) - spool_path = _record_rowwise_attempt( - state=state, - started_at=attempt["started_at"], - started_ts=attempt["started_ts"], - seed=args.seed, - code_pin=str(attempt["code_pin"]), - disposition="iterating", - predecessor=attempt["predecessor"], - spool_dir=args.out / "logbook-spool", - ) - print(f"Wrote Logbook row: {spool_path}", file=sys.stderr) - print(json.dumps(manifest, indent=2, sort_keys=True)) - return 0 - - -class CrosswalkSource: - def __init__(self, frame: pd.DataFrame, path: Path, *, generated: bool) -> None: - self.frame = frame - self.path = path - self.generated = generated - - -def _dataset_output_path( - out_dir: Path, - *, - dataset_filename: str | None, - input_stem: str, - source_year: int, -) -> Path: - filename = dataset_filename or DATASET_FILENAME_TEMPLATE.format( - input_stem=input_stem, - source_year=source_year, - ) - path = Path(filename) - if path.is_absolute() or path.name != filename or path.name in {"", ".", ".."}: - raise ValueError("--dataset-filename must be a filename, not a path.") - reserved = { - CROSSWALK_FILENAME, - MANIFEST_FILENAME, - COVERAGE_FILENAME, - DRY_RUN_PLAN_FILENAME, - AREA_SUPPORT_FILENAME, - } - if path.name in reserved: - raise ValueError( - f"--dataset-filename must not use reserved name {path.name!r}." - ) - return out_dir / path.name - - -def _validate_output_paths( - *, - input_h5: Path, - output_h5: Path, - args: argparse.Namespace, -) -> None: - output_sidecars = { - (args.out / MANIFEST_FILENAME).resolve(), - (args.out / COVERAGE_FILENAME).resolve(), - (args.out / DRY_RUN_PLAN_FILENAME).resolve(), - (args.out / AREA_SUPPORT_FILENAME).resolve(), - } - generated_crosswalk_path = (args.out / CROSSWALK_FILENAME).resolve() - reserved_paths = { - input_h5, - *output_sidecars, - } - if getattr(args, "ladder", None) is not None: - reserved_paths.add(args.ladder.resolve()) - if args.crosswalk is None: - reserved_paths.add(generated_crosswalk_path) - else: - crosswalk_path = args.crosswalk.resolve() - if crosswalk_path in output_sidecars: - raise ValueError("--crosswalk path must differ from output sidecars.") - reserved_paths.add(crosswalk_path) - if output_h5.resolve() in reserved_paths: - raise ValueError("Output H5 path must differ from inputs and sidecars.") - - -def _source_year(cli_source_year: int | None, *, base_summary: dict[str, Any]) -> int: - if cli_source_year is not None: - return cli_source_year - time_period = base_summary.get("time_period") - if time_period is None: - raise ValueError( - "Could not infer source year from input H5 time_period; pass --source-year." - ) - try: - return int(str(time_period)[:4]) - except ValueError as exc: - raise ValueError( - "Could not infer source year from input H5 time_period; pass --source-year." - ) from exc - - -def _load_or_build_crosswalk(args: argparse.Namespace) -> CrosswalkSource: - if args.crosswalk is not None: - path = args.crosswalk.resolve() - generated_crosswalk_path = args.out / CROSSWALK_FILENAME - if path != generated_crosswalk_path.resolve() and not getattr( - args, "dry_run", False - ): - # A dry run must not delete build artifacts, including a - # previously generated crosswalk cache. - generated_crosswalk_path.unlink(missing_ok=True) - return CrosswalkSource(_read_crosswalk(path), path, generated=False) - crosswalk = build_official_uk_geography_crosswalk() - path = args.out / CROSSWALK_FILENAME - write_geography_crosswalk(crosswalk, path) - return CrosswalkSource(crosswalk, path, generated=True) - - -def _read_crosswalk(path: Path) -> pd.DataFrame: - return pd.read_csv( - path, - dtype={ - "oa_code": str, - "lsoa_code": str, - "msoa_code": str, - "la_code": str, - "constituency_code": str, - "region_code": str, - "country": str, - }, - ) - - -def _area_codes_by_type(args: argparse.Namespace) -> dict[str, list[str]]: - area_codes: dict[str, list[str]] = {} - if args.constituency_codes is not None: - area_codes["constituency"] = _read_code_csv(args.constituency_codes) - if args.la_codes is not None: - area_codes["la"] = _read_code_csv(args.la_codes) - return area_codes - - -def _read_code_csv(path: Path) -> list[str]: - frame = pd.read_csv(path, dtype=str) - if "code" not in frame.columns: - raise ValueError(f"{path} must include a `code` column.") - return frame["code"].dropna().astype(str).str.strip().tolist() - - -def _validate_optional_coverage( - crosswalk: pd.DataFrame, - area_codes_by_type: dict[str, list[str]], -) -> pd.DataFrame: - if not area_codes_by_type: - return pd.DataFrame() - validate_geography_coverage( - crosswalk, - required_countries=["England", "Wales", "Scotland", "Northern Ireland"], - area_codes_by_type=area_codes_by_type, - ) - return geography_coverage_summary(crosswalk, area_codes_by_type) - - -def _parameters(args: argparse.Namespace, *, source_year: int) -> dict[str, Any]: - ladder_route = getattr(args, "ladder", None) is not None - return { - "n_clones": args.n_clones, - "candidate_clone_counts": ( - list(args.candidate_clone_counts) - if args.candidate_clone_counts is not None - else None - ), - "seed": args.seed, - "source_year": source_year, - # Crosswalk-sampler knobs are meaningless on the ladder route and are - # recorded as null rather than falsely claimed effective. - "require_all_countries": ( - None if ladder_route else not args.allow_missing_country - ), - "require_constituency": ( - None if ladder_route else not args.allow_blank_constituency - ), - "constrain_to_region": ( - None if ladder_route else not args.allow_cross_region_assignment - ), - "avoid_constituency_collisions": ( - None if ladder_route else not args.allow_constituency_collisions - ), - "source_lineage_modulus": args.source_lineage_modulus, - "assignment_route": "ladder" if ladder_route else "crosswalk", - "expected_constituency_vintage": ( - args.expected_constituency_vintage if ladder_route else None - ), - } - - -def _h5_summary(path: Path) -> dict[str, Any]: - weight_kind, mass_log = read_uk_single_year_weight_metadata(path) - with pd.HDFStore(path, mode="r") as store: - household = store["household"] - return { - "path": str(path), - "tables": {key.strip("/"): list(store[key].shape) for key in store.keys()}, - "household_weight_sum": float(household["household_weight"].sum()), - "time_period": str(store["time_period"].iloc[0]), - "household_weight_kind": weight_kind.value, - "mass_log_records": len(mass_log), - "distinct_source_households": ( - int(household["source_household_id"].nunique()) - if "source_household_id" in household.columns - else None - ), - } - - -def _dry_run_plan( - args: argparse.Namespace, - *, - input_h5: Path, - input_artifact: dict[str, Any], - output_h5: Path, - crosswalk: pd.DataFrame, - crosswalk_source: CrosswalkSource, - base_summary: dict[str, Any], - source_year: int, - coverage: pd.DataFrame, -) -> dict[str, Any]: - """Compute the clone plan without cloning or writing a dataset.""" - - with pd.HDFStore(input_h5, mode="r") as store: - household = store["household"] - person_ids = _select_h5_columns(store, "person", list(PERSON_ID_COLUMNS)) - benunit_ids = _select_h5_columns(store, "benunit", ["benunit_id"]) - _validate_dry_run_input( - input_h5, - household=household, - person_ids=person_ids, - benunit_ids=benunit_ids, - ) - id_multiplier = id_multiplier_for_values( - household["household_id"], - person_ids["person_id"], - person_ids["person_household_id"], - person_ids["person_benunit_id"], - benunit_ids["benunit_id"], - ) - if args.source_lineage_modulus is not None: - household = apply_uk_source_lineage_modulus( - household, - modulus=args.source_lineage_modulus, - ) - # The realized assignment IS the real build's: the sampler consumes only - # the household table, and identical inputs, flags, id multiplier, and - # seed produce identical draws — so these per-area counts are exact for - # the build this plan describes, collision avoidance included. - assignment = assign_household_geography( - household, - crosswalk, - n_clones=args.n_clones, - seed=args.seed, - id_multiplier=id_multiplier, - source_year=source_year, - require_all_countries=not args.allow_missing_country, - require_constituency=not args.allow_blank_constituency, - constrain_to_region=not args.allow_cross_region_assignment, - avoid_constituency_collisions=not args.allow_constituency_collisions, - ) - realized = _realized_area_support(assignment.household, crosswalk) - collision_free = expected_uk_rowwise_area_support( - household, - crosswalk, - n_clones=args.n_clones, - source_year=source_year, - require_all_countries=not args.allow_missing_country, - require_constituency=not args.allow_blank_constituency, - constrain_to_region=not args.allow_cross_region_assignment, - ) - table_rows = { - name: base_summary["tables"][name][0] - for name in ("person", "benunit", "household") - } - input_bytes = input_h5.stat().st_size - plan = { - "schema_version": 1, - "build_kind": "uk_rowwise_local_geography_dry_run", - "created_at": datetime.now(UTC).isoformat(), - "git_commit": _git_commit(), - "parameters": _parameters(args, source_year=source_year), - "input": { - "dataset": input_artifact, - "crosswalk": _artifact_info(crosswalk_source.path), - "tables": base_summary["tables"], - "household_weight_sum": base_summary["household_weight_sum"], - "time_period": base_summary["time_period"], - "household_weight_kind": base_summary["household_weight_kind"], - "mass_log_records": base_summary["mass_log_records"], - }, - "plan": { - "n_clones": args.n_clones, - "id_multiplier": id_multiplier, - "output_h5": str(output_h5), - "rows": {name: rows * args.n_clones for name, rows in table_rows.items()}, - "output_bytes_estimate": input_bytes * args.n_clones, - "output_bytes_estimate_basis": ( - "lower-bound estimate: linear scaling of the input H5 byte " - "size by n_clones; added geography/lineage columns and HDF " - "table overhead increase the actual size" - ), - }, - "realized_support": { - "basis": ( - f"realized assignment at seed {args.seed} — identical draws " - "to the real build under these parameters; zero-row " - "sampleable areas included" - ), - **{ - area_type: _support_summary( - realized, area_type, rows_basis="assigned_rows" - ) - for area_type in ("constituency", "la") - }, - }, - "collision_free_expected_support": { - "basis": ( - "analytic collision-free expectation; diverges from realized " - "support when n_clones is comparable to a group's sampleable " - "constituency count" - ), - **{ - area_type: _support_summary( - collision_free, area_type, rows_basis="expected_rows" - ) - for area_type in ("constituency", "la") - }, - }, - "source_lineage": _source_lineage_report( - household, - modulus=args.source_lineage_modulus, - ), - "coverage": coverage.to_dict("records") if not coverage.empty else [], - } - if args.candidate_clone_counts is not None: - plan["candidates"] = _crosswalk_candidate_plans( - args, - household=household, - crosswalk=crosswalk, - source_year=source_year, - id_multiplier=id_multiplier, - table_rows=table_rows, - input_bytes=input_bytes, - ) - return plan - - -def _crosswalk_candidate_plans( - args: argparse.Namespace, - *, - household: pd.DataFrame, - crosswalk: pd.DataFrame, - source_year: int, - id_multiplier: int, - table_rows: dict[str, int], - input_bytes: int, -) -> dict[str, Any]: - clone_counts = args.candidate_clone_counts - assert clone_counts is not None - plans = [] - for n_clones in clone_counts: - assignment = assign_household_geography( - household, - crosswalk, - n_clones=n_clones, - seed=args.seed, - id_multiplier=id_multiplier, - source_year=source_year, - require_all_countries=not args.allow_missing_country, - require_constituency=not args.allow_blank_constituency, - constrain_to_region=not args.allow_cross_region_assignment, - avoid_constituency_collisions=not args.allow_constituency_collisions, - ) - realized = _realized_area_support(assignment.household, crosswalk) - collision_free = expected_uk_rowwise_area_support( - household, - crosswalk, - n_clones=n_clones, - source_year=source_year, - require_all_countries=not args.allow_missing_country, - require_constituency=not args.allow_blank_constituency, - constrain_to_region=not args.allow_cross_region_assignment, - ) - plans.append( - { - "n_clones": n_clones, - "rows": {name: rows * n_clones for name, rows in table_rows.items()}, - "output_bytes_estimate": input_bytes * n_clones, - "realized_support": { - area_type: _support_summary( - realized, area_type, rows_basis="assigned_rows" - ) - for area_type in ("constituency", "la") - }, - "collision_free_expected_support": { - area_type: _support_summary( - collision_free, area_type, rows_basis="expected_rows" - ) - for area_type in ("constituency", "la") - }, - } - ) - return { - "basis": ( - "independent crosswalk assignments at the build seed for each " - "candidate K; realized and collision-free expected support only " - "because this route has no ladder roster for area-support stats" - ), - "clone_counts": list(clone_counts), - "plans": plans, - } - - -def _run_ladder_route( - args: argparse.Namespace, - *, - input_h5: Path, - input_artifact: dict[str, Any], - output_h5: Path, - base_summary: dict[str, Any], - source_year: int, - attempt: dict[str, object] | None, -) -> int: - """Build (or dry-run plan) the rowwise dataset through the OA ladder.""" - - ladder_path = args.ladder.resolve() - ladder_artifact = _artifact_info(ladder_path) - _verify_artifact_pin( - ladder_artifact, - pinned_sha256=args.ladder_sha256, - option="--ladder", - ) - _annotate_ladder_crosswalk_pin(ladder_artifact) - ladder = load_uk_oa_ladder(ladder_path) - - if args.dry_run: - plan = _ladder_dry_run_plan( - args, - input_h5=input_h5, - input_artifact=input_artifact, - output_h5=output_h5, - base_summary=base_summary, - source_year=source_year, - ladder=ladder, - ladder_artifact=ladder_artifact, - ) - plan_path = args.out / DRY_RUN_PLAN_FILENAME - plan_path.write_text(json.dumps(plan, indent=2, sort_keys=True) + "\n") - print(json.dumps(plan, indent=2, sort_keys=True)) - return 0 - if attempt is not None: - state = attempt["state"] - assert isinstance(state, AttemptState) - pins = { - "dataset": _pin_from_artifact(input_artifact), - "ladder": _pin_from_artifact(ladder_artifact), - } - attempt["code_pin"] = git_code_pin(_REPOSITORY) - state.build_id = _new_rowwise_build_id( - route="ladder", - seed=args.seed, - timestamp=attempt["started_ts"], - ) - state.input_pins_digest = role_pins_digest(pins) - state.identity_digest = _rowwise_identity_digest( - route="ladder", - pins=pins, - args=args, - source_year=source_year, - ) - append_phase(state, "configured") - append_phase(state, "inputs_pinned") - - result = clone_uk_dataset_with_ladder_geography( - input_h5, - ladder, - output_path=output_h5, - n_clones=args.n_clones, - seed=args.seed, - source_year=source_year, - expected_constituency_vintage=args.expected_constituency_vintage, - source_lineage_modulus=args.source_lineage_modulus, - ) - if attempt is not None: - state = attempt["state"] - assert isinstance(state, AttemptState) - append_phase(state, "cloned") - rowwise_summary = _rowwise_summary( - result, - base_summary=base_summary, - source_lineage_modulus=args.source_lineage_modulus, - geo_columns=UK_GEOGRAPHY_LADDER_COLUMNS, - constituency_column="constituency_code", - la_column="local_authority_code", - ) - rowwise_summary["gate"] = { - "name": "uk_geography_ladder", - "passed": bool(result.gate.passed), - "details": dict(result.gate.details), - } - household = _result_household(result) - area_support, area_support_summaries = _ladder_area_support_diagnostics( - household, - ladder, - ) - rowwise_summary["area_support"] = area_support - rowwise_summary["region_mix"] = uk_region_mix(household).to_dict("records") - area_support_path = args.out / AREA_SUPPORT_FILENAME - _area_support_long_frame(area_support_summaries).to_csv( - area_support_path, index=False - ) - manifest = { - "schema_version": 1, - "build_kind": "uk_rowwise_local_geography_dataset", - "created_at": datetime.now(UTC).isoformat(), - "git_commit": _git_commit(), - "parameters": _parameters(args, source_year=source_year), - "inputs": { - "dataset": input_artifact, - "ladder": ladder_artifact, - }, - "outputs": { - "dataset": _artifact_info(output_h5), - "crosswalk": None, - "coverage_summary": None, - "area_support_summary": _artifact_info(area_support_path), - }, - "base_dataset": base_summary, - "rowwise_dataset": rowwise_summary, - "coverage": [], - } - manifest_path = args.out / MANIFEST_FILENAME - manifest_path.write_text(json.dumps(manifest, indent=2, sort_keys=True) + "\n") - if attempt is not None: - state = attempt["state"] - assert isinstance(state, AttemptState) - append_phase(state, "manifest_written") - state.gate_verdicts = { - "uk_geography_ladder": { - "verdict": "passed" if result.gate.passed else "failed", - "receipt": ( - f"{local_artifact_reference(manifest_path, repository_hint=_REPOSITORY)}" - "#/rowwise_dataset/gate" - ), - } - } - state.artifact_location = local_artifact_reference( - output_h5, - repository_hint=_REPOSITORY, - ) - spool_path = _record_rowwise_attempt( - state=state, - started_at=attempt["started_at"], - started_ts=attempt["started_ts"], - seed=args.seed, - code_pin=str(attempt["code_pin"]), - disposition="iterating", - predecessor=attempt["predecessor"], - spool_dir=args.out / "logbook-spool", - ) - print(f"Wrote Logbook row: {spool_path}", file=sys.stderr) - print(json.dumps(manifest, indent=2, sort_keys=True)) - return 0 - - -def _ladder_dry_run_plan( - args: argparse.Namespace, - *, - input_h5: Path, - input_artifact: dict[str, Any], - output_h5: Path, - base_summary: dict[str, Any], - source_year: int, - ladder: Any, - ladder_artifact: dict[str, Any], -) -> dict[str, Any]: - """Exact ladder-route plan: the real cloned assignment at the build seed.""" - - with pd.HDFStore(input_h5, mode="r") as store: - household = store["household"] - person_ids = _select_h5_columns(store, "person", list(PERSON_ID_COLUMNS)) - benunit_ids = _select_h5_columns(store, "benunit", ["benunit_id"]) - _validate_dry_run_input( - input_h5, - household=household, - person_ids=person_ids, - benunit_ids=benunit_ids, - ) - _refuse_preassigned_geography(household, label="household") - id_multiplier = id_multiplier_for_values( - household["household_id"], - person_ids["person_id"], - person_ids["person_household_id"], - person_ids["person_benunit_id"], - benunit_ids["benunit_id"], - ) - if args.source_lineage_modulus is not None: - household = apply_uk_source_lineage_modulus( - household, - modulus=args.source_lineage_modulus, - ) - # Fence parity with the real build (a plan must never bless a build that - # would raise): weight validity, mass-chain currency, then the release - # gate on the divided-weight assignment. - from microcosm.build.uk_runtime.rowwise_dataset import _assert_mass_log_current - - weight_values = pd.to_numeric( - household["household_weight"], errors="raise" - ).to_numpy(dtype=float) - if not (weight_values >= 0).all() or not np.isfinite(weight_values).all(): - raise ValueError("household weights must be finite and non-negative.") - if float(weight_values.sum()) <= 0.0: - raise ValueError("household weights must carry positive total mass.") - _kind, mass_log = read_uk_single_year_weight_metadata(input_h5) - _assert_mass_log_current(mass_log, float(weight_values.sum())) - assigned = _ladder_planned_assignment( - household, - ladder, - n_clones=args.n_clones, - seed=args.seed, - id_multiplier=id_multiplier, - expected_constituency_vintage=args.expected_constituency_vintage, - ) - realized = _ladder_realized_support(assigned, ladder) - expected = expected_uk_ladder_area_support( - household, - ladder, - n_clones=args.n_clones, - ) - area_support, _ = _ladder_area_support_diagnostics( - assigned, - ladder, - ) - region_mix = uk_region_mix(assigned).to_dict("records") - table_rows = { - name: base_summary["tables"][name][0] - for name in ("person", "benunit", "household") - } - input_bytes = input_h5.stat().st_size - plan = { - "schema_version": 1, - "build_kind": "uk_rowwise_local_geography_dry_run", - "created_at": datetime.now(UTC).isoformat(), - "git_commit": _git_commit(), - "parameters": _parameters(args, source_year=source_year), - "input": { - "dataset": input_artifact, - "ladder": ladder_artifact, - "tables": base_summary["tables"], - "household_weight_sum": base_summary["household_weight_sum"], - "time_period": base_summary["time_period"], - "household_weight_kind": base_summary["household_weight_kind"], - "mass_log_records": base_summary["mass_log_records"], - }, - "plan": { - "n_clones": args.n_clones, - "id_multiplier": id_multiplier, - "output_h5": str(output_h5), - "rows": {name: rows * args.n_clones for name, rows in table_rows.items()}, - "output_bytes_estimate": input_bytes * args.n_clones, - "output_bytes_estimate_basis": ( - "lower-bound estimate: linear scaling of the input H5 byte " - "size by n_clones; added geography/lineage columns and HDF " - "table overhead increase the actual size" - ), - }, - "realized_support": { - "basis": ( - f"realized ladder assignment at seed {args.seed} — identical " - "draws to the real build under these parameters; zero-row " - "ladder areas included" - ), - **{ - area_type: _support_summary( - realized, area_type, rows_basis="assigned_rows" - ) - for area_type in ("constituency", "la") - }, - }, - "expected_support": { - "basis": ( - "analytic expectation: constituency household-count share " - "within region x the input's region mix x n_clones; OA " - "population shares within constituency for LA support" - ), - **{ - area_type: _support_summary( - expected, area_type, rows_basis="expected_rows" - ) - for area_type in ("constituency", "la") - }, - }, - "area_support": area_support, - "region_mix": region_mix, - "source_lineage": _source_lineage_report( - household, - modulus=args.source_lineage_modulus, - ), - "coverage": [], - } - if args.candidate_clone_counts is not None: - plan["candidates"] = _ladder_candidate_plans( - args, - household=household, - ladder=ladder, - id_multiplier=id_multiplier, - table_rows=table_rows, - input_bytes=input_bytes, - ) - return plan - - -def _ladder_planned_assignment( - household: pd.DataFrame, - ladder: Any, - *, - n_clones: int, - seed: int, - id_multiplier: int, - expected_constituency_vintage: str, -) -> pd.DataFrame: - """Execute the fenced ladder assignment used by every dry-run K.""" - - household_for_clone = household.copy() - if "source_household_id" not in household_for_clone.columns: - household_for_clone["source_household_id"] = household_for_clone["household_id"] - cloned = clone_entity_frame( - household_for_clone, - id_columns=("household_id",), - n_clones=n_clones, - id_multiplier=id_multiplier, - clone_index_column="clone_index", - ).reset_index(drop=True) - cloned["household_weight"] = ( - pd.to_numeric(cloned["household_weight"], errors="raise").to_numpy(dtype=float) - / n_clones - ) - assigned = assign_uk_geography_ladder( - cloned, - ladder, - seed=seed, - expected_constituency_vintage=expected_constituency_vintage, - ) - gate = uk_geography_ladder_gate( - assigned, - assigned["household_weight"].to_numpy(dtype=float), - ) - if not gate.passed: - raise ValueError( - "UK geography ladder gate would fail this build: " - + "; ".join(gate.failures) - ) - return assigned - - -def _ladder_candidate_plans( - args: argparse.Namespace, - *, - household: pd.DataFrame, - ladder: Any, - id_multiplier: int, - table_rows: dict[str, int], - input_bytes: int, -) -> dict[str, Any]: - clone_counts = args.candidate_clone_counts - assert clone_counts is not None - plans = [] - for n_clones in clone_counts: - assigned = _ladder_planned_assignment( - household, - ladder, - n_clones=n_clones, - seed=args.seed, - id_multiplier=id_multiplier, - expected_constituency_vintage=args.expected_constituency_vintage, - ) - realized = _ladder_realized_support(assigned, ladder) - expected = expected_uk_ladder_area_support( - household, - ladder, - n_clones=n_clones, - ) - area_support, _ = _ladder_area_support_diagnostics( - assigned, - ladder, - ) - plans.append( - { - "n_clones": n_clones, - "rows": {name: rows * n_clones for name, rows in table_rows.items()}, - "output_bytes_estimate": input_bytes * n_clones, - "realized_support": { - area_type: _support_summary( - realized, area_type, rows_basis="assigned_rows" - ) - for area_type in ("constituency", "la") - }, - "expected_support": { - area_type: _support_summary( - expected, area_type, rows_basis="expected_rows" - ) - for area_type in ("constituency", "la") - }, - "area_support": area_support, - } - ) - return { - "basis": ( - "independent fenced ladder assignments at the build seed for each " - "candidate K; summary-only plans" - ), - "clone_counts": list(clone_counts), - "plans": plans, - } - - -def _ladder_realized_support( - assigned_household: pd.DataFrame, - ladder: Any, -) -> pd.DataFrame: - """Realized rows per ladder area, zeros included for unassigned areas.""" - - import numpy as _np - - rows: list[dict[str, Any]] = [] - for assigned_column, ladder_codes, area_type in ( - ("constituency_code", ladder.constituency_code, "constituency"), - ("local_authority_code", ladder.local_authority_code, "la"), - ): - assigned = assigned_household[assigned_column].astype(str).str.strip() - counts = assigned[assigned != ""].value_counts() - codes = sorted(set(_np.unique(ladder_codes).tolist()) | set(counts.index)) - rows.extend( - { - "area_type": area_type, - "area_code": code, - "expected_rows": float(counts.get(code, 0)), - } - for code in codes - ) - return pd.DataFrame(rows) - - -def _validate_dry_run_input( - input_h5: Path, - *, - household: pd.DataFrame, - person_ids: pd.DataFrame, - benunit_ids: pd.DataFrame, -) -> None: - """Mirror the real build's input refusals so a plan cannot bless an - input the build would reject.""" - - if input_h5.suffix != ".h5": - raise ValueError("UK single-year dataset path must end with '.h5'.") - for frame, column, label in ( - (household, "household_id", "household"), - (person_ids, "person_id", "person"), - (benunit_ids, "benunit_id", "benunit"), - ): - if frame[column].isna().any(): - raise ValueError(f"{label}.{column} contains missing values.") - if frame[column].duplicated().any(): - duplicates = frame.loc[frame[column].duplicated(), column].unique() - raise ValueError( - f"{label}.{column} must be unique; duplicate value(s): " - f"{list(map(str, duplicates[:5]))}." - ) - household_ids = set(household["household_id"]) - missing_households = sorted(set(person_ids["person_household_id"]) - household_ids) - if missing_households: - raise ValueError( - "person.person_household_id contains value(s) absent from " - f"household: {missing_households[:5]}." - ) - missing_benunits = sorted( - set(person_ids["person_benunit_id"]) - set(benunit_ids["benunit_id"]) - ) - if missing_benunits: - raise ValueError( - "person.person_benunit_id contains value(s) absent from benunit: " - f"{missing_benunits[:5]}." - ) - - -def _realized_area_support( - assigned_household: pd.DataFrame, - crosswalk: pd.DataFrame, -) -> pd.DataFrame: - """Realized rows per sampleable area, zeros included.""" - - sampleable = crosswalk[pd.to_numeric(crosswalk["population"]) > 0] - rows: list[dict[str, Any]] = [] - for assigned_column, crosswalk_column, area_type in ( - ("constituency_code_oa", "constituency_code", "constituency"), - ("la_code_oa", "la_code", "la"), - ): - assigned = assigned_household[assigned_column].astype(str).str.strip() - counts = assigned[assigned != ""].value_counts() - codes = sorted( - { - str(code).strip() - for code in sampleable[crosswalk_column] - if str(code).strip() - } - | set(counts.index) - ) - rows.extend( - { - "area_type": area_type, - "area_code": code, - "expected_rows": float(counts.get(code, 0)), - } - for code in codes - ) - return pd.DataFrame(rows) - - -def _select_h5_columns( - store: pd.HDFStore, - key: str, - columns: list[str], -) -> pd.DataFrame: - try: - return store.select(key, columns=columns) - except (TypeError, ValueError, KeyError): - return store[key][columns] - - -def _support_summary( - support: pd.DataFrame, - area_type: str, - *, - rows_basis: str, - bottom: int = EXPECTED_SUPPORT_BOTTOM_AREAS, -) -> dict[str, Any]: - subset = support[support["area_type"] == area_type] - if subset.empty: - return { - "n_areas": 0, - "rows_basis": rows_basis, - "min_rows": 0.0, - "median_rows": 0.0, - "mean_rows": 0.0, - "max_rows": 0.0, - "bottom": [], - } - values = subset["expected_rows"] - ordered = subset.sort_values( - ["expected_rows", "area_code"], - kind="mergesort", - ).head(bottom) - return { - "n_areas": int(len(subset)), - "rows_basis": rows_basis, - "min_rows": float(values.min()), - "median_rows": float(values.median()), - "mean_rows": float(values.mean()), - "max_rows": float(values.max()), - "bottom": [ - { - "area_code": str(row.area_code), - "rows": float(row.expected_rows), - } - for row in ordered.itertuples(index=False) - ], - } - - -def _result_household(result: Any) -> pd.DataFrame: - if isinstance(result, UKLadderRowwiseDatasetResult): - return engine_tables(result.frame, weighted_entities=("household",))[ - "household" - ] - return result.household - - -def _ladder_area_support_diagnostics( - household: pd.DataFrame, - ladder: Any, -) -> tuple[dict[str, Any], dict[str, pd.DataFrame]]: - source_basis = ( - "source_household_id" - if "source_household_id" in household.columns - else "household_id" - ) - summaries = uk_ladder_area_support_summary( - household, - ladder, - source_column=source_basis, - ) - block = { - "source_basis": source_basis, - **{ - area_type: _area_support_stats(summaries[area_type]) - for area_type in ("constituency", "la") - }, - } - return block, summaries - - -def _area_support_long_frame(summaries: dict[str, pd.DataFrame]) -> pd.DataFrame: - """The full per-area table, built only where a consumer writes it.""" - - long_frames = [] - for area_type in ("constituency", "la"): - frame = summaries[area_type].copy() - frame.insert(0, "area_type", area_type) - long_frames.append(frame) - return pd.concat(long_frames, ignore_index=True) - - -def _area_support_stats(support: pd.DataFrame) -> dict[str, Any]: - # Support means rows that carry mass: a row assigned to an area at zero - # weight shapes no estimate there, so the headline stats and the thinness - # ranking read nonzero_households. The assigned count stays visible per - # bottom area and in the full per-area CSV. - rows = support["nonzero_households"] - ess = support["effective_sample_size"] - sources = support["nonzero_source_households"] - bottom_by_rows = support.sort_values( - ["nonzero_households", "area_code"], - kind="mergesort", - ).head(EXPECTED_SUPPORT_BOTTOM_AREAS) - bottom_by_ess = support.sort_values( - ["effective_sample_size", "area_code"], - kind="mergesort", - ).head(EXPECTED_SUPPORT_BOTTOM_AREAS) - return { - "n_areas": int(len(support)), - "rows_basis": "nonzero_households", - "min_rows": int(rows.min()) if len(rows) else 0, - "median_rows": float(rows.median()) if len(rows) else 0.0, - "min_ess": float(ess.min()) if len(ess) else 0.0, - "median_ess": float(ess.median()) if len(ess) else 0.0, - "min_distinct_sources": int(sources.min()) if len(sources) else 0, - "median_distinct_sources": (float(sources.median()) if len(sources) else 0.0), - "bottom_by_rows": [ - { - "area_code": str(row.area_code), - "rows": int(row.nonzero_households), - "assigned": int(row.assigned_households), - } - for row in bottom_by_rows.itertuples(index=False) - ], - "bottom_by_ess": [ - { - "area_code": str(row.area_code), - "ess": float(row.effective_sample_size), - } - for row in bottom_by_ess.itertuples(index=False) - ], - } - - -def _pool_lineage_block(household: pd.DataFrame) -> dict[str, Any] | None: - if POOL_SOURCE_LINEAGE_COLUMN not in household.columns: - return None - counts = household.groupby(POOL_SOURCE_LINEAGE_COLUMN).size() - block = { - "distinct_pool_source_households": int(counts.size), - "pool_copies_per_source": { - "min": int(counts.min()), - "median": float(counts.median()), - "max": int(counts.max()), - }, - } - if "household_support_channel" in household.columns: - block["distinct_by_support_channel"] = { - str(channel): int(group[POOL_SOURCE_LINEAGE_COLUMN].nunique()) - for channel, group in household.groupby("household_support_channel") - } - return block - - -def _explicit_lineage_block(household: pd.DataFrame) -> dict[str, Any] | None: - spine_columns = [ - column for column in UK_SPINE_LINEAGE_COLUMNS if column in household.columns - ] - if not spine_columns: - return None - - columns_present = [ - column - for column in ("source_household_id", *UK_SPINE_LINEAGE_COLUMNS) - if column in household.columns - ] - block: dict[str, Any] = { - "basis": "explicit_lineage_columns", - "columns_present": columns_present, - "flag_counts": { - column: int(household[column].fillna(False).astype(bool).sum()) - for column in UK_SPINE_LINEAGE_COLUMNS - if column.startswith("household_is_") and column in household.columns - }, - } - if "source_household_id" in household.columns: - block["distinct_source_households"] = int( - household["source_household_id"].nunique() - ) - if "household_support_channel" in household.columns: - block["distinct_by_support_channel"] = { - str(channel): int(group["source_household_id"].nunique()) - for channel, group in household.groupby("household_support_channel") - } - return block - - -def _source_lineage_report( - household: pd.DataFrame, - *, - modulus: int | None, -) -> dict[str, Any]: - """Report pool, immediate, and explicit spine lineage when supported.""" - - pool = _pool_lineage_block(household) - immediate = None - if "source_household_id" in household.columns: - immediate = { - "distinct_source_households": int( - household["source_household_id"].nunique() - ), - } - return { - "pool_modulus": modulus, - "pool": pool, - "immediate": immediate, - "explicit": _explicit_lineage_block(household), - } - - -def _rowwise_summary( - result, - *, - base_summary: dict[str, Any], - source_lineage_modulus: int | None = None, - geo_columns: tuple[str, ...] = ( - "oa_code", - "lsoa_code", - "msoa_code", - "la_code_oa", - "constituency_code_oa", - "region_code_oa", - ), - constituency_column: str = "constituency_code_oa", - la_column: str = "la_code_oa", -) -> dict[str, Any]: - if isinstance(result, UKLadderRowwiseDatasetResult): - person = result.frame.table("person") - benunit = result.frame.table("benunit") - household = engine_tables(result.frame, weighted_entities=("household",))[ - "household" - ] - weight_kind = uk_household_weight_kind(result.frame) - mass_log = result.frame.mass_log - time_period = uk_time_period(result.frame) - clone_column = ladder_clone_index_column("household") - else: - person = result.person - benunit = result.benunit - household = result.household - weight_kind = result.household_weight_kind - mass_log = result.mass_log - time_period = result.time_period - clone_column = "clone_index" - missing_geography = household[list(geo_columns)].isna().any(axis=1) - for column in geo_columns: - missing_geography |= household[column].fillna("").astype(str).str.strip().eq("") - assigned_constituencies = household.loc[ - _nonblank_string_mask(household[constituency_column]), - constituency_column, - ] - assigned_las = household.loc[ - _nonblank_string_mask(household[la_column]), - la_column, - ] - by_constituency = assigned_constituencies.groupby(assigned_constituencies).size() - by_la = assigned_las.groupby(assigned_las).size() - weight_sum = float(household["household_weight"].sum()) - constituency_rows = _area_row_summary(by_constituency) - la_rows = _area_row_summary(by_la) - input_total = float(base_summary["household_weight_sum"]) - abs_delta = abs(weight_sum - input_total) - clone0 = household - if clone_column in household.columns: - clone0 = household[household[clone_column] == 0] - lineage = { - "pool_modulus": source_lineage_modulus, - "pool": _pool_lineage_block(clone0), - "immediate": ( - { - "distinct_source_households": base_summary[ - "distinct_source_households" - ], - } - if base_summary.get("distinct_source_households") is not None - else None - ), - "explicit": _explicit_lineage_block(clone0), - } - return { - "weights": { - "household_weight_kind": weight_kind.value, - "mass_log_records": len(mass_log), - "mass_conservation": { - "input_total": input_total, - "output_total": weight_sum, - "abs_delta": abs_delta, - "relative_tolerance": MASS_CONSERVATION_RELATIVE_TOLERANCE, - "passed": bool( - abs_delta <= MASS_CONSERVATION_RELATIVE_TOLERANCE * abs(input_total) - ), - }, - }, - "source_lineage": lineage, - "tables": { - "person": list(person.shape), - "benunit": list(benunit.shape), - "household": list(household.shape), - }, - "time_period": time_period, - "n_clones": result.n_clones, - "id_multiplier": result.id_multiplier, - "household_weight_sum": weight_sum, - "household_weight_delta": weight_sum - base_summary["household_weight_sum"], - "missing_geography_rows": int(missing_geography.sum()), - "assigned_constituencies": int(by_constituency.size), - "assigned_local_authorities": int(by_la.size), - "min_household_rows_by_constituency": constituency_rows["min"], - "min_household_rows_by_local_authority": la_rows["min"], - "median_household_rows_by_constituency": constituency_rows["median"], - "median_household_rows_by_local_authority": la_rows["median"], - "duplicate_source_household_constituency_pairs": ( - _duplicate_source_household_constituency_pairs( - household, - constituency_column=constituency_column, - ) - ), - } - - -def _nonblank_string_mask(values: pd.Series) -> pd.Series: - return values.notna() & values.astype(str).str.strip().ne("") - - -def _area_row_summary(counts: pd.Series) -> dict[str, int | float]: - if counts.empty: - return {"min": 0, "median": 0.0} - return {"min": int(counts.min()), "median": float(counts.median())} - - -def _duplicate_source_household_constituency_pairs( - household: pd.DataFrame, - *, - constituency_column: str = "constituency_code_oa", -) -> int: - if "source_household_id" not in household.columns: - return 0 - assigned = household[_nonblank_string_mask(household[constituency_column])] - return int(assigned.duplicated(["source_household_id", constituency_column]).sum()) - - -def _artifact_info(path: Path) -> dict[str, Any]: - return { - "path": str(path), - "sha256": _sha256(path), - "bytes": path.stat().st_size, - } - - -def _verify_artifact_pin( - artifact: dict[str, Any], - *, - pinned_sha256: str | None, - option: str, -) -> None: - measured_sha256 = str(artifact["sha256"]) - if pinned_sha256 is not None and measured_sha256 != pinned_sha256: + if retired: raise SystemExit( - f"error: {option} sha mismatch: " - f"measured {measured_sha256}, pinned {pinned_sha256}" - ) - artifact["pin_verified"] = pinned_sha256 is not None - - -def _annotate_ladder_crosswalk_pin(ladder_artifact: dict[str, Any]) -> None: - try: - crosswalk = load_uk_local_area_crosswalk() - expected_sha256 = str(crosswalk["ladder_artifact_sha256"]) - except Exception as error: - ladder_artifact["matches_local_area_crosswalk_pin"] = None - ladder_artifact["local_area_crosswalk_pin_error"] = ( - f"{type(error).__name__}: {error}" + "The independent UK geography-only build driver is retired; " + f"unsupported old controls: {', '.join(retired)}. " + "Use tools/build_uk_full.py with a bound --input-h5 or --spine-request, " + "--ladder, --ledger-facts and --out. --n-clones controls geography K; " + "--dry-run describes the full graph. All target geographies remain the default." ) - return - ladder_artifact["matches_local_area_crosswalk_pin"] = ( - str(ladder_artifact["sha256"]) == expected_sha256 - ) - + from microcosm.build.uk_runtime.full_build_cli import main as full_main -def _sha256(path: Path) -> str: - digest = hashlib.sha256() - with path.open("rb") as file: - for chunk in iter(lambda: file.read(1024 * 1024), b""): - digest.update(chunk) - return digest.hexdigest() - - -def _git_commit() -> str | None: - result = subprocess.run( - ["git", "rev-parse", "HEAD"], - check=False, - capture_output=True, - text=True, - ) - if result.returncode != 0: - return None - return result.stdout.strip() + return full_main(arguments) if __name__ == "__main__": diff --git a/tools/calibrate_uk_national_dataset.py b/tools/calibrate_uk_national_dataset.py index 6f8e758ec..e0e7693ed 100644 --- a/tools/calibrate_uk_national_dataset.py +++ b/tools/calibrate_uk_national_dataset.py @@ -1,314 +1,41 @@ -"""Calibrate an existing UK national spine H5 without replaying source stages. +"""Compatibility command for the canonical UK full build. -This driver is intentionally thin: it verifies pinned inputs, compiles the -Ledger target registry, applies the reviewed measure-exclusion register, records -any explicit doctrine overrides, and delegates the calibration/gate/logbook -work to :func:`microcosm.build.uk_runtime.calibration_run.run_uk_calibration`. - -Signed deviation for v1: no sampling rungs and no checkpointing. This seam runs -full-scale only; scale ladders and resumable source-stage checkpoints belong to -the spine build lane. +This command has the same all-geography default as build_uk_full. +Use --target-geographies country only when explicitly requesting that filter. +The old national-only solver and its separate output controls are retired. """ from __future__ import annotations -import argparse -import hashlib -import json -import re -from itertools import combinations -from pathlib import Path -from typing import Any - -from microcosm.build.ledger_artifact import load_ledger_consumer_artifact -from microcosm.build.uk_runtime.calibration_run import ( - UKCalibrationRunPaths, - run_uk_calibration, -) -from microcosm.build.uk_runtime.frs_release import load_uk_frs_release -from microcosm.build.uk_runtime.ledger_targets import compile_uk_target_registry -from microcosm.build.uk_runtime.measure_simulation import ( - UKMeasureResolver, - apply_uk_calibration_measure_exclusions, - load_uk_calibration_measure_exclusions, -) -from microcosm.build.uk_runtime.national_chronicle_feed import ( - UKNationalChronicleFeed, - load_uk_national_chronicle_feed, -) -from microcosm.build.uk_runtime.national_doctrine import uk_doctrine_with_overrides -from microcosm.build.uk_runtime.release_identity import UK_NATIONAL_RELEASE_ID -from microcosm.calibrate import TargetRegistry +import sys -_SHA256 = re.compile(r"[0-9a-f]{64}") -_CANONICAL_UK_RELEASE_ID = re.compile( - r"populace-uk-[1-9][0-9]*-[a-z0-9_]+-k[1-9][0-9]*" +_RETIRED_OPTIONS = frozenset( + { + "--staging-h5", + "--diagnostics-json", + "--build-record-json", + "--terminal-gate-json", + "--allow-unpinned-feed", + } ) -_UK_JUNE_RELEASE_ID = "populace-uk-2023-dd68c73-4aa4b14-20260619T023711Z" def main(argv: list[str] | None = None) -> int: - args = _parse_args(argv) - national_feed = load_uk_national_chronicle_feed() - artifact = load_ledger_consumer_artifact( - args.ledger_facts, - expected_facts_sha256=args.ledger_facts_sha256, - expected_manifest_sha256=args.ledger_manifest_sha256, - ) - _check_committed_ledger_feed_pin( - artifact.facts_sha256, - manifest_sha256=artifact.manifest_sha256, - pin=national_feed, - allow_unpinned_feed=args.allow_unpinned_feed, - ) - calibration_year = load_uk_frs_release().calibration_year - compilation = compile_uk_target_registry( - artifact.facts, target_period=calibration_year - ) - if compilation.unsupported: - raise SystemExit( - f"{len(compilation.unsupported)} target references failed to compile" - ) - _compare_frozen_register(args.register_json, compilation.registry) - exclusions = load_uk_calibration_measure_exclusions(args.measure_exclusions) - registry, exclusion_receipt = apply_uk_calibration_measure_exclusions( - compilation.registry, exclusions + arguments = list(sys.argv[1:] if argv is None else argv) + retired = sorted( + {argument.split("=", 1)[0] for argument in arguments} & _RETIRED_OPTIONS ) - overrides = { - key: value - for key, value in { - "epochs": args.epochs, - "target_weight_rule": args.target_weight_rule, - "learning_rate": args.learning_rate, - "target_loss_cap": args.target_loss_cap, - }.items() - if value is not None - } - doctrine, doctrine_overrides = uk_doctrine_with_overrides(**overrides) - paths = UKCalibrationRunPaths( - input_h5=args.input_h5, - staging_h5=args.staging_h5, - diagnostics_json=args.diagnostics_json, - build_record_json=args.build_record_json, - terminal_gate_json=args.terminal_gate_json, - ) - # The resolver reads the input H5 to build its simulation, so the pin is - # verified here first — no bytes are consumed before they match the CLI sha. - measured_input_sha = _sha256_file(args.input_h5) - if measured_input_sha != args.input_sha256: + if retired: raise SystemExit( - "error: --input-h5 sha mismatch: " - f"measured {measured_input_sha}, pinned {args.input_sha256}" + "The independent UK national calibration driver is retired; " + f"unsupported old option(s): {', '.join(retired)}. " + "Use tools/build_uk_full.py --help for the canonical " + "full-build options. Both command names default to all geographies; " + "--target-geographies country is an explicit target filter." ) - resolver = UKMeasureResolver( - simulation_source=args.input_h5, - scratch_dir=args.staging_h5.parent, - year=calibration_year, - frame=None, - ) - result = run_uk_calibration( - paths=paths, - input_sha256=args.input_sha256, - ledger_artifact=artifact, - register_registry=registry, - band_edge_registry=compilation.registry, - calibration_year=calibration_year, - exclusion_receipt=exclusion_receipt, - doctrine=doctrine, - doctrine_overrides=doctrine_overrides, - measure_resolver=resolver, - source_pins={ - "input_h5": { - "sha256": args.input_sha256, - "size_bytes": args.input_h5.stat().st_size, - }, - "ledger_facts": _ledger_facts_pin(artifact), - }, - run_config_extra={ - "calibration_year": calibration_year, - "allow_unpinned_feed": args.allow_unpinned_feed, - "national_chronicle_feed_pin": national_feed.to_dict(), - }, - release_id=args.release_id, - logbook_prev_row_digest=args.logbook_prev_row_digest, - ) - summary = { - "staging_h5_sha256": result.staging_sha256, - "diagnostics_sha256": result.diagnostics_sha256, - "terminal_gate_sha256": result.terminal_gate_sha256, - "build_record_sha256": result.build_record_sha256, - "gate_verdicts": result.build_record["gate_summary"], - } - print(json.dumps(summary, indent=2, sort_keys=True)) - return 0 - + from microcosm.build.uk_runtime.full_build_cli import main as full_build_main -def _parse_args(argv: list[str] | None) -> argparse.Namespace: - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("--input-h5", required=True, type=Path) - parser.add_argument("--input-sha256", required=True, type=_sha256) - parser.add_argument("--ledger-facts", required=True, type=Path) - parser.add_argument("--ledger-facts-sha256", required=True, type=_sha256) - parser.add_argument("--ledger-manifest-sha256", required=True, type=_sha256) - parser.add_argument( - "--allow-unpinned-feed", - action="store_true", - help=( - "Allow Chronicle facts or manifest hashes outside the committed UK " - "national feed pin; the override is recorded in the run manifest." - ), - ) - parser.add_argument("--staging-h5", required=True, type=Path) - parser.add_argument("--diagnostics-json", required=True, type=Path) - parser.add_argument("--build-record-json", required=True, type=Path) - parser.add_argument("--release-id", required=True) - parser.add_argument("--terminal-gate-json", type=Path) - parser.add_argument("--register-json", type=Path) - parser.add_argument("--measure-exclusions", type=Path) - parser.add_argument("--release-candidate", action="store_true") - parser.add_argument("--logbook-prev-row-digest", type=_sha256) - parser.add_argument("--epochs", type=int) - parser.add_argument("--target-weight-rule") - parser.add_argument("--learning-rate", type=float) - parser.add_argument("--target-loss-cap", type=float) - args = parser.parse_args(argv) - args.terminal_gate_json = args.terminal_gate_json or args.staging_h5.with_suffix( - ".terminal_gates.json" - ) - if args.release_candidate: - # The seam refuses release-candidate posture outright (the #757 - # release-cut audit, issue comment 5413502559): its scoped battery - # covers 6 of the declared entries and must never sign a - # shippability claim, and its evidence_absent gaps already refuse - # upstream. A candidate's verdict comes only from the release-cut - # certification producer (tools/certify_uk_release_cut.py). - parser.error( - "--release-candidate is refused on the calibration seam: the " - "seam's scoped battery cannot sign shippability; run the " - "release-cut certification producer instead" - ) - if ( - _CANONICAL_UK_RELEASE_ID.fullmatch(args.release_id) - or args.release_id == _UK_JUNE_RELEASE_ID - or args.release_id == UK_NATIONAL_RELEASE_ID - ): - parser.error( - "canonical UK release ids belong to the release-cut " - "certification producer; the seam runs under a staging or dev " - "release id" - ) - _validate_distinct_paths( - { - "--input-h5": args.input_h5, - "--ledger-facts": args.ledger_facts, - "--staging-h5": args.staging_h5, - "--diagnostics-json": args.diagnostics_json, - "--build-record-json": args.build_record_json, - "--terminal-gate-json": args.terminal_gate_json, - **({"--register-json": args.register_json} if args.register_json else {}), - **( - {"--measure-exclusions": args.measure_exclusions} - if args.measure_exclusions - else {} - ), - } - ) - return args - - -def _sha256(value: str) -> str: - if not _SHA256.fullmatch(value): - raise argparse.ArgumentTypeError("expected a lowercase SHA-256 digest") - return value - - -def _sha256_file(path: Path) -> str: - digest = hashlib.sha256() - with path.open("rb") as stream: - for chunk in iter(lambda: stream.read(1 << 20), b""): - digest.update(chunk) - return digest.hexdigest() - - -def _check_committed_ledger_feed_pin( - facts_sha256: str, - *, - manifest_sha256: str | None, - allow_unpinned_feed: bool, - pin: UKNationalChronicleFeed | None = None, -) -> None: - pin = pin or load_uk_national_chronicle_feed() - mismatches = [] - for label, loaded, committed in ( - ("facts", facts_sha256, pin.facts_sha256), - ("manifest", manifest_sha256, pin.manifest_sha256), - ): - if loaded != committed: - mismatches.append(f"{label}: loaded {loaded}, committed {committed}") - if mismatches and not allow_unpinned_feed: - raise SystemExit( - "error: Chronicle artifact differs from the committed UK national feed pin: " - + "; ".join(mismatches) - + "; pass " - "--allow-unpinned-feed only for an explicitly reviewed diagnostic run" - ) - - -def _validate_distinct_paths(paths: dict[str, Path]) -> None: - resolved = {label: path.expanduser().resolve() for label, path in paths.items()} - for (left_label, left), (right_label, right) in combinations(resolved.items(), 2): - if _paths_alias(left, right): - raise SystemExit( - f"error: {left_label} and {right_label} must be distinct paths " - f"({left} aliases {right})" - ) - - -def _paths_alias(left: Path, right: Path) -> bool: - if str(left).casefold() == str(right).casefold(): - return True - try: - left_stat = left.stat() - right_stat = right.stat() - except FileNotFoundError: - return False - return (left_stat.st_dev, left_stat.st_ino) == ( - right_stat.st_dev, - right_stat.st_ino, - ) - - -def _compare_frozen_register(path: Path | None, registry: Any) -> None: - """Refuse when the compiled register is not the frozen scoring surface. - - Both sides are compared on ``TargetRegistry.version`` — the registry's own - content hash — through the validating loader, so the same artifact serves - this check and ``tools/score_uk_national_candidate.py``, and incidental - formatting cannot make two identical registers look different. - """ - - if path is None: - return - try: - frozen = TargetRegistry.from_json(path) - except ValueError as error: - raise SystemExit( - f"error: frozen scoring register is unusable: {error}" - ) from error - if frozen.version != registry.version: - raise SystemExit( - "re-derived register differs from the frozen scoring register: " - f"{registry.version} vs {frozen.version}" - ) - - -def _ledger_facts_pin(artifact: Any) -> dict[str, object]: - facts_path = ( - artifact.path / "consumer_facts.jsonl" - if artifact.path.is_dir() - else artifact.path - ) - return {"sha256": artifact.facts_sha256, "size_bytes": facts_path.stat().st_size} + return full_build_main(arguments) if __name__ == "__main__": diff --git a/tools/certify_uk_release_cut.py b/tools/certify_uk_release_cut.py index 5d18e892f..ddb881f6d 100644 --- a/tools/certify_uk_release_cut.py +++ b/tools/certify_uk_release_cut.py @@ -1,323 +1,11 @@ -"""Certify a calibrated UK national candidate for release. +"""Materialize unsigned certification readiness from the canonical UK full graph. -The release-cut certification producer (microcosm#757 item B5): runs the 18 -declared national preflight/terminal gates over the calibrated candidate — -the executable home the June driver's retirement left empty — then composes -the multi-part certification over the spine build's battery report, the -calibration seam's battery report, and the fresh release-cut report. The -parts must union to the full declared gate-entry set with no gap and no -overlap beyond the declared shared ids, each signed by its producer, over -one closed identity join (spine report -> sidecar -> build record -> -diagnostics -> candidate bytes). A candidate's shippability verdict comes -only from the certification this driver writes. - -The battery always runs at release-candidate strictness: evidence_absent -gaps block. The rule-1 score receipt is cross-pinned into the certification -(the audit's third carried defect), so the score is signed run evidence -rather than a null slot. +Use --graph-manifest, --graph-store, --candidate-h5 and --certification-json. +The historical independent national/seam battery CLI is retired. Native and +matched-size scorecards enter as declared sources of the full graph build. """ -from __future__ import annotations - -import argparse -import hashlib -import json -import re -import time -from datetime import UTC, datetime -from pathlib import Path - -from microcosm.build.ledger_artifact import load_ledger_consumer_artifact -from microcosm.build.logbook_adoption import ( - AttemptState, - append_phase, - apply_error_verdict, - error_receipt_path, - git_code_pin, - local_artifact_reference, - record_terminal_attempt, - resolve_predecessor, - role_pins_digest, - write_error_receipt, -) -from microcosm.build.uk_runtime.frs_release import load_uk_frs_release -from microcosm.build.uk_runtime.ledger_targets import ( - compile_uk_local_target_registry, - compile_uk_target_registry, - load_uk_local_area_crosswalk, -) -from microcosm.build.uk_runtime.measure_simulation import ( - apply_uk_calibration_measure_exclusions, - load_uk_calibration_measure_exclusions, -) -from microcosm.build.uk_runtime.national_frame import load_uk_national_frame -from microcosm.build.uk_runtime.parity_reference import load_efrs_parity_reference -from microcosm.build.uk_runtime.release_certification import ( - compose_uk_release_certification, - rehydrate_uk_fit_weight_records, - run_uk_release_cut_battery, - uk_release_parity_evidence, -) -from microcosm.build.uk_runtime.release_input_coverage import ( - PolicyEngineUKCoverageEngine, -) -from microcosm.build.uk_runtime.weighted_integrity import ( - exclusion_evaluation_date, - load_uk_input_mass_reference, -) - -_SHA256 = re.compile(r"[0-9a-f]{64}") -_REPOSITORY = Path(__file__).resolve().parents[1] -_PIPELINE = "uk-frs-release-certification" -_LEDGER_COMPILE_PARITY_PERIODS = (2023, 2025) -_LOCAL_COMPILE_PARITY_PERIOD = 2025 - - -def main(argv: list[str] | None = None) -> int: - args = _parse_args(argv) - started_at = time.perf_counter() - started_ts = datetime.now(UTC) - code_pin = git_code_pin(_REPOSITORY) - predecessor = resolve_predecessor(args.logbook_prev_row_digest) - source_pins = { - "candidate_h5": {"sha256": args.candidate_sha256}, - "ledger_facts": {"sha256": args.ledger_facts_sha256}, - } - state = AttemptState( - build_id=f"{_PIPELINE}-attempt-{started_ts.strftime('%Y%m%dT%H%M%SZ')}", - identity_digest=hashlib.sha256( - json.dumps( - { - "pipeline": _PIPELINE, - "release_id": args.release_id, - "candidate_sha256": args.candidate_sha256, - }, - sort_keys=True, - ).encode("utf-8") - ).hexdigest(), - input_pins_digest=role_pins_digest(source_pins), - phases_reached=["attempt_started"], - gate_verdicts={}, - ) - spool_dir = args.certification_json.parent / "logbook-spool" - try: - summary = _run(args, state) - except BaseException as error: - error_path = write_error_receipt( - error_receipt_path( - args.certification_json.parent / "logbook-receipts", - build_id=state.build_id, - ), - state=state, - pipeline=_PIPELINE, - error=error, - ) - apply_error_verdict( - state, - f"{local_artifact_reference(error_path, repository_hint=_REPOSITORY)}" - "#/error_type", - ) - record_terminal_attempt( - state=state, - started_at=started_at, - started_ts=started_ts, - pipeline=_PIPELINE, - rung="f100", - seed=None, - code_pin=code_pin, - disposition=( - "discarded" if isinstance(error, KeyboardInterrupt) else "failed" - ), - predecessor=predecessor, - spool_dir=spool_dir, - ) - raise - state.artifact_location = local_artifact_reference( - args.certification_json, repository_hint=_REPOSITORY - ) - record_terminal_attempt( - state=state, - started_at=started_at, - started_ts=started_ts, - pipeline=_PIPELINE, - rung="f100", - seed=None, - code_pin=code_pin, - disposition="certified", - predecessor=predecessor, - spool_dir=spool_dir, - ) - print(json.dumps(summary, indent=2, sort_keys=True)) - return 0 - - -def _run(args: argparse.Namespace, state: AttemptState) -> dict[str, object]: - measured = hashlib.sha256(args.candidate_h5.read_bytes()).hexdigest() - if measured != args.candidate_sha256: - raise SystemExit( - "error: --candidate-h5 sha mismatch: " - f"measured {measured}, pinned {args.candidate_sha256}" - ) - sidecar_path = args.spine_h5.with_suffix(".build.json") - if not sidecar_path.is_file(): - raise SystemExit(f"error: spine build sidecar absent: {sidecar_path}") - sidecar = json.loads(sidecar_path.read_text(encoding="utf-8")) - spine_report_path = args.spine_h5.with_suffix(".spine_gates.json") - - diagnostics_bytes = args.diagnostics_json.read_bytes() - diagnostics = json.loads(diagnostics_bytes) - diagnostics_sha = hashlib.sha256(diagnostics_bytes).hexdigest() - build_record = json.loads(args.build_record_json.read_text(encoding="utf-8")) - recorded_diagnostics = ( - build_record.get("artifacts", {}).get("diagnostics_json", {}).get("sha256") - ) - if recorded_diagnostics != diagnostics_sha: - raise SystemExit( - "error: --diagnostics-json bytes do not match the build record's " - f"binding ({diagnostics_sha} != {recorded_diagnostics})" - ) - append_phase(state, "inputs_bound") - - artifact = load_ledger_consumer_artifact( - args.ledger_facts, - expected_facts_sha256=args.ledger_facts_sha256, - expected_manifest_sha256=args.ledger_manifest_sha256, - ) - ledger_registries = {} - for period in _LEDGER_COMPILE_PARITY_PERIODS: - compilation = compile_uk_target_registry(artifact.facts, target_period=period) - ledger_registries[period] = compilation.registry - local_compilation = compile_uk_local_target_registry( - artifact.facts, - target_period=_LOCAL_COMPILE_PARITY_PERIOD, - crosswalk=load_uk_local_area_crosswalk(), - ) - local_ledger_registries = {_LOCAL_COMPILE_PARITY_PERIOD: local_compilation.registry} - calibration_year = load_uk_frs_release().calibration_year - if calibration_year in ledger_registries: - reference_compiled = ledger_registries[calibration_year] - else: - reference_compiled = compile_uk_target_registry( - artifact.facts, target_period=calibration_year - ).registry - evaluated_on = exclusion_evaluation_date(None) - exclusions = load_uk_calibration_measure_exclusions() - reference_registry, _receipt = apply_uk_calibration_measure_exclusions( - reference_compiled, exclusions, now=evaluated_on - ) - append_phase(state, "registries_compiled") - - frame, _provenance = load_uk_national_frame(args.candidate_h5) - engine = PolicyEngineUKCoverageEngine() - parity_evidence = uk_release_parity_evidence( - frame, - diagnostics_targets=diagnostics["targets"], - reference_registry=reference_registry, - parity_reference=load_efrs_parity_reference(), - ) - report = run_uk_release_cut_battery( - frame, - report_path=args.release_cut_gate_json, - release_id=args.release_id, - diagnostics_sha256=diagnostics_sha, - coverage_engine=engine, - build_stage_names=sidecar["stages"], - ledger_registries=ledger_registries, - local_ledger_registries=local_ledger_registries, - parity_evidence=parity_evidence, - fit_weight_records=rehydrate_uk_fit_weight_records(sidecar), - input_mass_reference=load_uk_input_mass_reference(args.input_mass_reference), - exclusions_evaluated_on=evaluated_on, - ) - append_phase(state, "release_cut_gates_evaluated") - for gate_id, payload in report["gates"].items(): - state.gate_verdicts[gate_id] = { - "verdict": payload["status"], - "receipt": (f"local://{args.release_cut_gate_json.name}#/gates/{gate_id}"), - } - - certification = compose_uk_release_certification( - release_id=args.release_id, - candidate_name=args.candidate_name, - candidate_path=args.candidate_h5, - candidate_sha256=args.candidate_sha256, - spine_report_path=spine_report_path, - seam_report_path=args.seam_gate_report, - release_cut_report_path=args.release_cut_gate_json, - spine_sidecar=sidecar, - build_record=build_record, - score_receipt_path=args.score_receipt, - exclusions_evaluated_on=evaluated_on, - certification_path=args.certification_json, - ) - append_phase(state, "certification_written") - return { - "certification_json": str(args.certification_json), - "certification_sha256": hashlib.sha256( - args.certification_json.read_bytes() - ).hexdigest(), - "release_cut_gate_json": str(args.release_cut_gate_json), - "shippable": certification["shippable"], - "parts": { - name: part["statuses"] for name, part in certification["parts"].items() - }, - } - - -def _parse_args(argv: list[str] | None) -> argparse.Namespace: - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("--candidate-h5", required=True, type=Path) - parser.add_argument("--candidate-sha256", required=True, type=_sha256) - parser.add_argument( - "--candidate-name", - required=True, - help="The dataset name the certification certifies, e.g. microcosm_uk_2024.", - ) - parser.add_argument("--spine-h5", required=True, type=Path) - parser.add_argument("--diagnostics-json", required=True, type=Path) - parser.add_argument("--build-record-json", required=True, type=Path) - parser.add_argument("--seam-gate-report", required=True, type=Path) - parser.add_argument("--ledger-facts", required=True, type=Path) - parser.add_argument("--ledger-facts-sha256", required=True, type=_sha256) - parser.add_argument("--ledger-manifest-sha256", required=True, type=_sha256) - parser.add_argument("--input-mass-reference", required=True, type=Path) - parser.add_argument("--score-receipt", required=True, type=Path) - parser.add_argument("--release-id", required=True) - parser.add_argument("--release-cut-gate-json", type=Path) - parser.add_argument("--certification-json", type=Path) - parser.add_argument("--logbook-prev-row-digest", type=_sha256) - args = parser.parse_args(argv) - args.release_cut_gate_json = ( - args.release_cut_gate_json - or args.candidate_h5.with_suffix(".release_cut_gates.json") - ) - args.certification_json = args.certification_json or args.candidate_h5.with_suffix( - ".release_certification.json" - ) - distinct = { - "--candidate-h5": args.candidate_h5, - "--spine-h5": args.spine_h5, - "--diagnostics-json": args.diagnostics_json, - "--build-record-json": args.build_record_json, - "--seam-gate-report": args.seam_gate_report, - "--release-cut-gate-json": args.release_cut_gate_json, - "--certification-json": args.certification_json, - "--score-receipt": args.score_receipt, - } - resolved: dict[Path, str] = {} - for flag, path in distinct.items(): - canonical = path.resolve() - if canonical in resolved: - parser.error(f"{flag} aliases {resolved[canonical]}: {path}") - resolved[canonical] = flag - return args - - -def _sha256(value: str) -> str: - if not _SHA256.fullmatch(value): - raise argparse.ArgumentTypeError("expected a 64-character lowercase sha256") - return value - +from microcosm.build.uk_runtime.full_certification import main if __name__ == "__main__": raise SystemExit(main()) diff --git a/tools/evaluate_uk_incumbent_surface.py b/tools/evaluate_uk_incumbent_surface.py index 295074192..f63649fa3 100644 --- a/tools/evaluate_uk_incumbent_surface.py +++ b/tools/evaluate_uk_incumbent_surface.py @@ -17,7 +17,6 @@ import argparse import collections import hashlib -import importlib.util import json import sys import tempfile @@ -29,8 +28,10 @@ from microcosm.build.ledger_artifact import load_ledger_consumer_artifact from microcosm.build.uk_runtime.frs_release import load_uk_frs_release +from microcosm.build.uk_runtime.full_measure import resolve_uk_full_measures from microcosm.build.uk_runtime.incumbent_surface_evaluation import ( GSS_REGION_CODES, + candidate_evaluation_manifest, classify_local_rows, classify_national_rows, evaluation_summary, @@ -56,17 +57,6 @@ from microcosm.data.contract import uk_incumbent_surface_assessment -def _driver(): - spec = importlib.util.spec_from_file_location( - "build_uk_rowwise_candidate", - Path(__file__).resolve().with_name("build_uk_rowwise_candidate.py"), - ) - module = importlib.util.module_from_spec(spec) - assert spec.loader is not None - spec.loader.exec_module(module) - return module - - def _parse_args(argv): p = argparse.ArgumentParser(description=__doc__) p.add_argument("--candidate-h5", required=True, type=Path) @@ -85,7 +75,6 @@ def _parse_args(argv): def main(argv=None) -> int: args = _parse_args(argv) - driver = _driver() artifact = load_ledger_consumer_artifact( args.ledger_facts, expected_facts_sha256=args.ledger_facts_sha256, @@ -132,7 +121,9 @@ def _collect(node) -> None: resolver_registry = TargetRegistry( [s for s in registry.specs if s.name not in unresolvable], country="uk" ) - manifest = json.loads(args.candidate_manifest.read_text()) + manifest = candidate_evaluation_manifest( + json.loads(args.candidate_manifest.read_text()) + ) diagnostics_path = Path(str(manifest["outputs"]["calibration_diagnostics"]["path"])) if not diagnostics_path.is_absolute(): diagnostics_path = args.candidate_manifest.parent / diagnostics_path @@ -192,7 +183,7 @@ def digest(path): print("resolving the engine over the frame ...", file=sys.stderr, flush=True) with tempfile.TemporaryDirectory(prefix="uk-incumbent-eval-") as scratch: prepared_frame, _restore, national_rows, local_metrics, resolution = ( - driver._resolve_candidate_engine_surface( + resolve_uk_full_measures( frame, resolver_registry, period=period, diff --git a/tools/graph_uk_spine_fixture.py b/tools/graph_uk_spine_fixture.py index 16dcc9266..1f9b97231 100644 --- a/tools/graph_uk_spine_fixture.py +++ b/tools/graph_uk_spine_fixture.py @@ -60,7 +60,7 @@ uk_frs_spine_seed_frame, ) from microcosm.build.uk_runtime.frs_take_up import UKFRSTakeUpStageTransform -from microcosm.build.uk_runtime.graph import UK_SPINE_EXCLUSIONS, uk_spine_graph +from microcosm.build.uk_runtime.graph import uk_spine_graph from microcosm.build.uk_runtime.hmrc_capital_gains import ( HMRC_CGT_GAIN_BAND_LOWER_BOUNDS, HMRC_CGT_INCOME_BAND_LOWER_BOUNDS, @@ -820,8 +820,6 @@ def _fixture_stages( assert spec.sources is not None stages: list[SourceStageSpec] = [] for committed in spec.sources.stages: - if committed.stage in UK_SPINE_EXCLUSIONS: - continue artifacts = [ dict(frs_artifacts[str(artifact["table"])]) if artifact.get("table") in frs_artifacts diff --git a/uv.lock b/uv.lock index 715727d69..803db362d 100644 --- a/uv.lock +++ b/uv.lock @@ -750,6 +750,7 @@ dependencies = [ [package.optional-dependencies] uk = [ { name = "h5py" }, + { name = "microcosm-data" }, { name = "policyengine-uk" }, { name = "tables" }, ] @@ -773,6 +774,7 @@ requires-dist = [ { name = "huggingface-hub", specifier = ">=0.20" }, { name = "jsonschema", specifier = ">=4.23,<5" }, { name = "microcosm-calibrate", editable = "packages/microcosm-calibrate" }, + { name = "microcosm-data", marker = "extra == 'uk'", editable = "packages/microcosm-data" }, { name = "microcosm-data", marker = "extra == 'us'", editable = "packages/microcosm-data" }, { name = "microcosm-fit", editable = "packages/microcosm-fit" }, { name = "microcosm-frame", editable = "packages/microcosm-frame" }, From 1a28a4a71a278fafb6ccb98ce6388dfe50f5bd8c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mar=C3=ADa=20Juaristi?= <127882282+juaristi22@users.noreply.github.com> Date: Thu, 10 Sep 2026 18:40:08 +0200 Subject: [PATCH 3/8] Fix issues from review: align CI lock and target fixtures --- .../tests/test_uk_full_graph_admission.py | 7 ++++--- .../tests/test_uk_full_target_graph.py | 7 ++++++- .../microcosm-build/tests/test_uk_full_targets.py | 15 +++++++++++++++ tools/spec_seed_identity_diagnostics.py | 2 +- 4 files changed, 26 insertions(+), 5 deletions(-) diff --git a/packages/microcosm-build/tests/test_uk_full_graph_admission.py b/packages/microcosm-build/tests/test_uk_full_graph_admission.py index ee866ef5f..3cc951e45 100644 --- a/packages/microcosm-build/tests/test_uk_full_graph_admission.py +++ b/packages/microcosm-build/tests/test_uk_full_graph_admission.py @@ -89,13 +89,14 @@ def test_real_full_graph_preflight_replays_and_blocks_dense_export_and_certifica calibration_year=2026, time_period="2023", source_year=2023, - n_clones=1, seed=7, calibration=UKGraphCalibrationConfig(epochs=2, seed=7), ), spine=base, spine_population="source", ) + # Keep maintained K so the tiny fixture can reach every selected area; + # zero-support targets still refuse before gates at deliberately smaller K. provenance = ArtifactInput( "spine_provenance", "source", "provenance", SPINE_PROVENANCE_TYPE ) @@ -113,7 +114,7 @@ def test_real_full_graph_preflight_replays_and_blocks_dense_export_and_certifica graph = add_uk_export_preparation( graph, population=full.calibration.population, - bindings={"target_scope": "all", "n_clones": 1}, + bindings={"target_scope": "all", "n_clones": full.config.n_clones}, artifact_inputs=(final_gate,), ) graph = add_uk_export_continuation( @@ -160,7 +161,7 @@ def registry(): assert {row["geography_level"] for row in selection["included"]} >= { "country", "constituency", - "local_authority", + "la", } warm = run_graph( checkpoint, diff --git a/packages/microcosm-build/tests/test_uk_full_target_graph.py b/packages/microcosm-build/tests/test_uk_full_target_graph.py index 6f1aca04e..d7a75f3d5 100644 --- a/packages/microcosm-build/tests/test_uk_full_target_graph.py +++ b/packages/microcosm-build/tests/test_uk_full_target_graph.py @@ -371,7 +371,12 @@ def test_default_all_has_direct_matrix_and_solver_parity_and_replays( assert sum(by_grain["constituency"]) == pytest.approx(33.0) assert sum(by_grain["la"]) == pytest.approx(33.0) assert len(set(by_grain["constituency"])) == 1 - assert len(set(by_grain["la"])) == 2 + assert len(set(by_grain["la"])) > 1 + census_receipt = default_problem.bindings["cross_geography"][ + "census_household_uprating" + ] + assert census_receipt["grains"]["constituency"]["factor"] == 33.0 / 200.0 + assert census_receipt["grains"]["local_authority"]["factor"] == 33.0 / 195.0 assert default_problem.problem.names == explicit_problem.problem.names np.testing.assert_array_equal( default_problem.problem.matrix.toarray(), diff --git a/packages/microcosm-build/tests/test_uk_full_targets.py b/packages/microcosm-build/tests/test_uk_full_targets.py index b406c2940..b394883f7 100644 --- a/packages/microcosm-build/tests/test_uk_full_targets.py +++ b/packages/microcosm-build/tests/test_uk_full_targets.py @@ -138,6 +138,21 @@ def test_loaded_source_mismatch_refuses(prepared): assert prepared[4] == [] +def test_omitted_explicit_pins_still_bind_both_reviewed_source_hashes( + prepared, monkeypatch +): + calls = [] + monkeypatch.setattr( + runtime, + "load_ledger_consumer_artifact", + lambda path, **kwargs: calls.append(kwargs) or prepared[3], + ) + _load() + assert calls == [ + {"expected_facts_sha256": "a" * 64, "expected_manifest_sha256": "b" * 64} + ] + + def test_local_review_is_required_before_read_or_compilation(prepared, monkeypatch): monkeypatch.setattr( runtime, diff --git a/tools/spec_seed_identity_diagnostics.py b/tools/spec_seed_identity_diagnostics.py index 291cca3ca..1dc8ab273 100644 --- a/tools/spec_seed_identity_diagnostics.py +++ b/tools/spec_seed_identity_diagnostics.py @@ -20,7 +20,7 @@ from functools import cached_property from pathlib import Path -LOCK_SHA256 = "4ef1ef6eb39b65ebc47c2c00bafa44e7c872544b1b0146dd8493bcfe45b2da1b" +LOCK_SHA256 = "14bf10bf3c6c584886460d06f627ce59410e151f5af1d8b77c7df4ec05899026" UPLOAD_ACTION_SHA = "ea165f8d65b6e75b540449e92b4886f43607fa02" CAPS = { "candidate-digests.json": 64 * 1024, From e16acf209d2f38b4da3a0832daa16df8de9b8f78 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mar=C3=ADa=20Juaristi?= <127882282+juaristi22@users.noreply.github.com> Date: Thu, 10 Sep 2026 19:01:48 +0200 Subject: [PATCH 4/8] Fix issues from review: refresh graph identities and context fixtures --- docs/evidence/spec-engine/us-f0-coverage.json | 16 ++++----- .../build/spec_engine/inventory_coverage.py | 4 +-- .../graph_implementation_inventory.json | 35 +++++++++++++++---- .../build/us_runtime/worker_identity.py | 2 +- .../tests/test_spec_engine_loader.py | 2 +- .../tests/test_us_current_survey_puf_host.py | 11 +++++- .../tests/test_us_puf_detail_transfer.py | 10 +++++- 7 files changed, 60 insertions(+), 20 deletions(-) diff --git a/docs/evidence/spec-engine/us-f0-coverage.json b/docs/evidence/spec-engine/us-f0-coverage.json index 45e73dcbb..5c8e67132 100644 --- a/docs/evidence/spec-engine/us-f0-coverage.json +++ b/docs/evidence/spec-engine/us-f0-coverage.json @@ -1656,13 +1656,13 @@ "compiler_ir.node_slices" ], "expected": { - "map_sha256": "87ba50531d9fa6683096ecb39a31655b331ab8acff3a5876c2f62b33562a0885", - "protocol_sha256": "fd3e4b06f11be4e8c13ea19fef9469ab95cbbe3e2dcfce351e860dd3e00709e4" + "map_sha256": "c15bff65f27e51a7b4554b1e3edb43609c6d4fa34ccca613d7a6776947d2fd79", + "protocol_sha256": "a775cccdc4ffe64c79bf78be1a0a3c48ad274e3cc5a5aa49f907c320a48827b7" }, "failures": [], "observed": { - "map_sha256": "87ba50531d9fa6683096ecb39a31655b331ab8acff3a5876c2f62b33562a0885", - "protocol_sha256": "fd3e4b06f11be4e8c13ea19fef9469ab95cbbe3e2dcfce351e860dd3e00709e4" + "map_sha256": "c15bff65f27e51a7b4554b1e3edb43609c6d4fa34ccca613d7a6776947d2fd79", + "protocol_sha256": "a775cccdc4ffe64c79bf78be1a0a3c48ad274e3cc5a5aa49f907c320a48827b7" }, "status": "covered" }, @@ -1677,7 +1677,7 @@ "compiler_ir.seed_stream_map" ], "expected": { - "implementation_sha256": "fd3e4b06f11be4e8c13ea19fef9469ab95cbbe3e2dcfce351e860dd3e00709e4", + "implementation_sha256": "a775cccdc4ffe64c79bf78be1a0a3c48ad274e3cc5a5aa49f907c320a48827b7", "protocol": "legacy-v1", "streams": [ "build_model", @@ -1698,7 +1698,7 @@ }, "failures": [], "observed": { - "implementation_sha256": "fd3e4b06f11be4e8c13ea19fef9469ab95cbbe3e2dcfce351e860dd3e00709e4", + "implementation_sha256": "a775cccdc4ffe64c79bf78be1a0a3c48ad274e3cc5a5aa49f907c320a48827b7", "protocol": "legacy-v1", "streams": [ "build_model", @@ -2599,7 +2599,7 @@ "country": "us", "schema_id": "country_spec", "schema_version": 1, - "spec_sha256": "35a02b6b19c921faba1407d441e0b9d9623c496e2cd5b711be014def281a95c6" + "spec_sha256": "813fcb2bceef0673fe396ae67fe147947c897e44161e5f4e0d8d7c669132c0ed" } }, "report_schema_version": 3, @@ -2609,7 +2609,7 @@ "country": "us", "schema_id": "country_spec", "schema_version": 1, - "spec_sha256": "35a02b6b19c921faba1407d441e0b9d9623c496e2cd5b711be014def281a95c6" + "spec_sha256": "813fcb2bceef0673fe396ae67fe147947c897e44161e5f4e0d8d7c669132c0ed" }, "status": "pass" } diff --git a/packages/microcosm-build/src/microcosm/build/spec_engine/inventory_coverage.py b/packages/microcosm-build/src/microcosm/build/spec_engine/inventory_coverage.py index 89171055c..309b2f4de 100644 --- a/packages/microcosm-build/src/microcosm/build/spec_engine/inventory_coverage.py +++ b/packages/microcosm-build/src/microcosm/build/spec_engine/inventory_coverage.py @@ -359,8 +359,8 @@ "late_schedule": "e59c019d3d454eac99ac0ac209b6c5b6faaf9bdfcaeee18c36a25be19bf7da2f", "ownership": "5f64f0aac49e2313177564f71876bffc8c81b3ded4df701e70930e60e9c98356", "primary_tuples": "987b501c695e31f45521c4a178528f75ab3df22c09bc407b182213b2de99ee57", - "seed_map": "87ba50531d9fa6683096ecb39a31655b331ab8acff3a5876c2f62b33562a0885", - "seed_protocol": "fd3e4b06f11be4e8c13ea19fef9469ab95cbbe3e2dcfce351e860dd3e00709e4", + "seed_map": "c15bff65f27e51a7b4554b1e3edb43609c6d4fa34ccca613d7a6776947d2fd79", + "seed_protocol": "a775cccdc4ffe64c79bf78be1a0a3c48ad274e3cc5a5aa49f907c320a48827b7", "source_manifest": "cd5ba8924d64da5425ee14cca82a774e3f4b2bb5aabe06df291cc3cc457287a9", "take_up": "fa186daea0f8dd641cc470e41d1a2953f887d45282ec990201298f47bedf8d4d", "tail": "ac92829c88a1a4fb6460d61190918d5d99c6c377fc8dd8f62f02b332d09bf59c", diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/graph_implementation_inventory.json b/packages/microcosm-build/src/microcosm/build/us_runtime/graph_implementation_inventory.json index ebd5b5066..d5d237791 100644 --- a/packages/microcosm-build/src/microcosm/build/us_runtime/graph_implementation_inventory.json +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/graph_implementation_inventory.json @@ -1100,7 +1100,8 @@ "microcosm.graph.randomness", "microcosm.graph.serialize", "microcosm.graph.store", - "microcosm.graph.view" + "microcosm.graph.view", + "microcosm.graph.weight_update" ], "resource_accesses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", "unbound_uses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945" @@ -1197,6 +1198,7 @@ "microcosm.graph/kernel.py": { "imports": [ "microcosm.frame", + "microcosm.frame.bundle", "microcosm.graph.decl", "numpy", "pandas" @@ -1236,6 +1238,7 @@ "microcosm.graph.canonical", "microcosm.graph.decl", "microcosm.graph.kernel", + "microcosm.graph.weight_update", "numpy", "pandas" ], @@ -1277,6 +1280,13 @@ "resource_accesses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", "unbound_uses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945" }, + "microcosm.graph/weight_update.py": { + "imports": [ + "microcosm.graph.canonical" + ], + "resource_accesses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", + "unbound_uses_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945" + }, "microunit/__init__.py": { "imports": [ "microunit.core", @@ -1533,6 +1543,7 @@ "microcosm.graph.serialize": "whole-module in explicit stages", "microcosm.graph.store": "whole-module in explicit stages", "microcosm.graph.view": "whole-module in explicit stages", + "microcosm.graph.weight_update": "whole-module pure ordered-axis receipt in every shared-graph scope; no source data, external resource, policy engine or calibration route", "microunit": "whole-module in explicit stages", "microunit.core": "whole-module in explicit stages", "microunit.diagnostics": "whole-module in explicit stages", @@ -1597,7 +1608,8 @@ "schema.py", "serialize.py", "store.py", - "view.py" + "view.py", + "weight_update.py" ], "microunit": [ "__init__.py", @@ -1629,7 +1641,8 @@ "occupied_housing_review_base": "HU v1 source definition and attachment are exact approved 4b26e59b additions on d4bddcb6ac2aba52903cd8dace90330cee4a21e1; only the v2 prepared graph/transport wiring is new. Original source-verifier modules and resources are unchanged.", "operator_boundary": "Resolved output/formula registries projected; provider bodies are not called by source construction.", "other_imports": "Canonical non-stdlib import modules are explicitly classified. Ordinary symbols from a module/dependency bound in every using scope need no inventory edit. Only unbound-helper uses retain a lexical-scope fingerprint; resource-access expressions retain their own fingerprint. Expanded AST details are audit output, not packaged data. Neither fingerprint is a proof of dynamic closure.", - "resources": "Microunit rule YAML is bound. Default packaged geography/source-stage resources belong to inactive legacy entrypoints; graph lookups are explicit SourceRefs." + "resources": "Microunit rule YAML is bound. Default packaged geography/source-stage resources belong to inactive legacy entrypoints; graph lookups are explicit SourceRefs.", + "uk_full_graph_shared_contracts": "PR #901 shared-graph review against f4e5ec4eb761e26f65667f693ef37d3ea68adfc8: bind the added pure weight_update module in the existing conservative graph closure. The explicit same-kind weight contract does not activate on US source nodes. Kernel context now preserves immutable Frame metadata, mass records and projected column order; its frame.bundle freeze helper was already whole-module bound. Existing modules add only these covered imports; resource-access and unbound-use fingerprints are unchanged. Positional column patching preserves sampled pandas row labels. No source definition, target, numerical dependency, external resource or US stage route was added." }, "stages": { "acs_codec_2024": { @@ -1681,6 +1694,7 @@ "microcosm.graph/serialize.py", "microcosm.graph/store.py", "microcosm.graph/view.py", + "microcosm.graph/weight_update.py", "microunit/__init__.py", "microunit/core.py", "microunit/diagnostics.py", @@ -1755,6 +1769,7 @@ "microcosm.graph/serialize.py", "microcosm.graph/store.py", "microcosm.graph/view.py", + "microcosm.graph/weight_update.py", "microunit/__init__.py", "microunit/core.py", "microunit/diagnostics.py", @@ -1826,7 +1841,8 @@ "microcosm.graph/randomness.py", "microcosm.graph/serialize.py", "microcosm.graph/store.py", - "microcosm.graph/view.py" + "microcosm.graph/view.py", + "microcosm.graph/weight_update.py" ], "resources": [], "source_boundary_projection": true @@ -1912,6 +1928,7 @@ "microcosm.graph/serialize.py", "microcosm.graph/store.py", "microcosm.graph/view.py", + "microcosm.graph/weight_update.py", "microunit/__init__.py", "microunit/core.py", "microunit/diagnostics.py", @@ -1987,7 +2004,8 @@ "microcosm.graph/randomness.py", "microcosm.graph/serialize.py", "microcosm.graph/store.py", - "microcosm.graph/view.py" + "microcosm.graph/view.py", + "microcosm.graph/weight_update.py" ], "resources": [], "source_boundary_projection": false @@ -2055,6 +2073,7 @@ "microcosm.graph/serialize.py", "microcosm.graph/store.py", "microcosm.graph/view.py", + "microcosm.graph/weight_update.py", "microunit/__init__.py", "microunit/core.py", "microunit/diagnostics.py", @@ -2165,6 +2184,7 @@ "microcosm.graph/serialize.py", "microcosm.graph/store.py", "microcosm.graph/view.py", + "microcosm.graph/weight_update.py", "microunit/__init__.py", "microunit/core.py", "microunit/diagnostics.py", @@ -2286,6 +2306,7 @@ "microcosm.graph/serialize.py", "microcosm.graph/store.py", "microcosm.graph/view.py", + "microcosm.graph/weight_update.py", "microunit/__init__.py", "microunit/core.py", "microunit/diagnostics.py", @@ -2406,6 +2427,7 @@ "microcosm.graph/serialize.py", "microcosm.graph/store.py", "microcosm.graph/view.py", + "microcosm.graph/weight_update.py", "microunit/__init__.py", "microunit/core.py", "microunit/diagnostics.py", @@ -2479,7 +2501,8 @@ "microcosm.graph/randomness.py", "microcosm.graph/serialize.py", "microcosm.graph/store.py", - "microcosm.graph/view.py" + "microcosm.graph/view.py", + "microcosm.graph/weight_update.py" ], "resources": [], "source_boundary_projection": false diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/worker_identity.py b/packages/microcosm-build/src/microcosm/build/us_runtime/worker_identity.py index d9332b8f2..50d8f52aa 100644 --- a/packages/microcosm-build/src/microcosm/build/us_runtime/worker_identity.py +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/worker_identity.py @@ -28,7 +28,7 @@ PRIMARY_QRF_WORKER_MODULE = "microcosm.build.us_runtime.puf_qrf_worker" PRIMARY_QRF_INTERPRETER_PLACEHOLDER = "{python_interpreter}" APPROVED_UV_LOCK_SHA256 = ( - "751d5ef5d25406bbae1798667f0e29890d4aad933d323c12912c7d45d8809bb9" + "14bf10bf3c6c584886460d06f627ce59410e151f5af1d8b77c7df4ec05899026" ) LEGACY_CAMPAIGN_UV_LOCK_SHA256 = ( "27f47e385cfa35e2644a37410d1804b361ad9aee123577551c8421547bda65ee" diff --git a/packages/microcosm-build/tests/test_spec_engine_loader.py b/packages/microcosm-build/tests/test_spec_engine_loader.py index 0b583de0a..f4c01d8a6 100644 --- a/packages/microcosm-build/tests/test_spec_engine_loader.py +++ b/packages/microcosm-build/tests/test_spec_engine_loader.py @@ -236,7 +236,7 @@ def test_semantic_hash_has_golden_vector_and_surface_separation(tmp_path) -> Non # Pin the domain separator, normalization rules, schema-set receipt, and # exact normative projection as one reviewable golden vector. assert first.spec_sha256 == ( - "57026e5896dd52a382746fe7641f615dcc4c55965a1f50bea5971a8f92710d71" + "a5048106b31c856308aaf4dda1792a443e20254124785cdb4eb784adc456a1d5" ) second_root = _rich_minimal(tmp_path / "xy", note="second", store="local:b") diff --git a/packages/microcosm-build/tests/test_us_current_survey_puf_host.py b/packages/microcosm-build/tests/test_us_current_survey_puf_host.py index 50454a91d..1b6a3b721 100644 --- a/packages/microcosm-build/tests/test_us_current_survey_puf_host.py +++ b/packages/microcosm-build/tests/test_us_current_survey_puf_host.py @@ -289,9 +289,18 @@ def test_actual_current_sources_cold_and_replayed_host(tmp_path, monkeypatch): del missing[next(iter(missing))] extra = {**values, "__extra_entity__": next(iter(values.values()))} for changed_values in (missing, extra): + changes = {field: changed_values} + if field == "tables": + # Keep this synthetic projection internally consistent so the + # host, rather than KernelContext construction, rejects its + # missing or extra entity. + changes["frame_column_order"] = { + entity: context.frame_column_order.get(entity, tuple(table.columns)) + for entity, table in changed_values.items() + } with pytest.raises(ValueError, match="^" + reason + "$"): bridge._current_context_frame( - replace(context, **{field: changed_values}), expanded.frame + replace(context, **changes), expanded.frame ) person = context.tables[expanded.frame.schema.person_entity] diff --git a/packages/microcosm-build/tests/test_us_puf_detail_transfer.py b/packages/microcosm-build/tests/test_us_puf_detail_transfer.py index f2433c649..794d55c7b 100644 --- a/packages/microcosm-build/tests/test_us_puf_detail_transfer.py +++ b/packages/microcosm-build/tests/test_us_puf_detail_transfer.py @@ -1020,7 +1020,15 @@ def test_context_strips_only_tax_unit_owned_names(cold): tables = {e: table.copy(deep=True) for e, table in context.tables.items()} tables["tax_unit"].drop(columns=detail.MASK, inplace=True) tables["person"][detail.MASK] = False - restored = graph_detail.context_frame(replace(context, tables=tables)) + # Move the synthetic column's order metadata with its entity ownership. + column_order = dict(context.frame_column_order) + column_order["tax_unit"] = tuple( + column for column in column_order["tax_unit"] if column != detail.MASK + ) + column_order["person"] = (*column_order["person"], detail.MASK) + restored = graph_detail.context_frame( + replace(context, tables=tables, frame_column_order=column_order) + ) assert detail.MASK in restored.person and detail.MASK not in restored.table( "tax_unit" ) From 6fd47f7eea2180a54fdaa189cdf88fb5ee568e57 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mar=C3=ADa=20Juaristi?= <127882282+juaristi22@users.noreply.github.com> Date: Thu, 10 Sep 2026 19:25:05 +0200 Subject: [PATCH 5/8] Fix issues from review: admit verified diagnostic import inputs --- .github/workflows/test.yml | 1 + ...est_spec_seed_identity_parameter_assets.py | 340 ++++++++++++++++++ ...est_spec_seed_identity_per_code_context.py | 4 +- ...test_spec_seed_identity_system_metadata.py | 11 +- tools/spec_seed_identity_diagnostics.py | 245 ++++++++++++- 5 files changed, 595 insertions(+), 6 deletions(-) create mode 100644 packages/microcosm-build/tests/test_spec_seed_identity_parameter_assets.py diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index a9883d361..ab7883a46 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -243,6 +243,7 @@ jobs: packages/microcosm-build/tests/test_spec_seed_identity_owned_temp.py packages/microcosm-build/tests/test_spec_seed_identity_system_metadata.py packages/microcosm-build/tests/test_spec_seed_identity_per_code_context.py + packages/microcosm-build/tests/test_spec_seed_identity_parameter_assets.py - name: Derive candidate spec and seed identities (diagnostic only) id: spec_seed_diagnostics timeout-minutes: 15 diff --git a/packages/microcosm-build/tests/test_spec_seed_identity_parameter_assets.py b/packages/microcosm-build/tests/test_spec_seed_identity_parameter_assets.py new file mode 100644 index 000000000..d6ca64cba --- /dev/null +++ b/packages/microcosm-build/tests/test_spec_seed_identity_parameter_assets.py @@ -0,0 +1,340 @@ +"""Invented package inputs and real refusal probes; no engine or data import.""" + +import base64 +import builtins +import hashlib +import importlib.metadata +import importlib.util +import json +import os +import socket +import subprocess +import sys +from pathlib import Path +from types import SimpleNamespace + +import pytest + + +@pytest.fixture(scope="module") +def diagnostic(): + source = ( + Path(__file__).resolve().parents[3] / "tools/spec_seed_identity_diagnostics.py" + ) + spec = importlib.util.spec_from_file_location("parameter_asset_controls", source) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +@pytest.fixture +def invented_package(diagnostic, monkeypatch, tmp_path): + prefix = tmp_path / "venv" + site = prefix / "site-packages" + package = site / "policyengine_us" + path = package / "parameters/gov/invented.csv" + path.parent.mkdir(parents=True) + content = b"invented,value\nA,1\n" + path.write_bytes(content) + sha = hashlib.sha256(content).hexdigest() + record = importlib.metadata.PackagePath( + "policyengine_us/parameters/gov/invented.csv" + ) + record.size = len(content) + record.hash = SimpleNamespace( + mode="sha256", + value=base64.urlsafe_b64encode(bytes.fromhex(sha)).decode().rstrip("="), + ) + distribution = SimpleNamespace( + version=diagnostic.PARAMETER_ASSET_VERSION, + files=[record], + locate_file=lambda name: site / str(name), + ) + spec = SimpleNamespace( + origin=str(package / "__init__.py"), submodule_search_locations=[str(package)] + ) + monkeypatch.setattr( + diagnostic, "PARAMETER_ASSETS", (("gov/invented.csv", len(content), sha),) + ) + monkeypatch.setattr(diagnostic.sys, "prefix", str(prefix)) + monkeypatch.setattr( + diagnostic.importlib.metadata, "distribution", lambda _: distribution + ) + monkeypatch.setattr(diagnostic.importlib.util, "find_spec", lambda _: spec) + hooks = [] + monkeypatch.setattr(diagnostic.sys, "addaudithook", hooks.append) + policy = diagnostic.VerifiedParameterAssets() + refusals = diagnostic.install_boundary( + tmp_path / "repo", + tmp_path / "owned", + tmp_path / "output", + parameter_assets=policy, + ) + return SimpleNamespace( + path=path, + package=package, + record=record, + distribution=distribution, + spec=spec, + policy=policy, + hook=hooks[0], + refusals=refusals, + content=content, + ) + + +def test_exact_record_and_bytes_enable_only_read_access(diagnostic, invented_package): + case = invented_package + with pytest.raises(diagnostic.RefusalError, match="^DATA_FILE$"): + case.hook("open", (str(case.path), "r", os.O_RDONLY)) + evidence = {} + case.policy.verify(evidence) + case.hook("open", (str(case.path), "r", os.O_RDONLY)) + assert case.policy.permits(case.path, writing=False) + assert not case.policy.permits(case.path, writing=True) + assert evidence["engine_parameter_assets"]["assets"] == [ + { + "path": str(case.record), + "size_bytes": len(case.content), + "sha256": hashlib.sha256(case.content).hexdigest(), + } + ] + assert ( + evidence["engine_parameter_assets"]["record_origin_and_content_verified"] + is True + ) + # A recheck retains exactly the same declaration; it creates no new grant. + case.policy.verify(evidence) + assert case.policy._verified == frozenset({case.path}) + + +@pytest.mark.parametrize( + "kind", + ( + "version", + "record_hash", + "record_size", + "duplicate", + "origin", + "bytes", + "size", + "symlink", + ), +) +def test_invalid_package_input_never_grants_reads(diagnostic, invented_package, kind): + case = invented_package + code = "PARAMETER_CONTENT" + if kind == "version": + case.distribution.version = "unreviewed" + code = "PARAMETER_VERSION" + elif kind == "record_hash": + case.record.hash.value = "wrong" + code = "PARAMETER_RECORD" + elif kind == "record_size": + case.record.size += 1 + code = "PARAMETER_RECORD" + elif kind == "duplicate": + case.distribution.files.append(case.record) + code = "PARAMETER_RECORD" + elif kind == "origin": + case.spec.origin = str(case.package / "different.py") + code = "PARAMETER_ORIGIN" + elif kind == "bytes": + case.path.write_bytes(case.content.replace(b"1", b"2")) + elif kind == "size": + case.path.write_bytes(case.content + b"changed") + else: + other = case.path.with_name("other.csv") + other.write_bytes(case.content) + case.path.unlink() + case.path.symlink_to(other) + code = "PARAMETER_ORIGIN" + with pytest.raises(diagnostic.RefusalError, match="^" + code + "$"): + case.policy.verify({}) + assert case.policy._verified == frozenset() + assert case.policy._checking is None + + +def test_changed_bytes_revoke_a_previously_verified_roster( + diagnostic, invented_package +): + case = invented_package + case.policy.verify({}) + case.path.write_bytes(case.content.replace(b"1", b"2")) + with pytest.raises(diagnostic.RefusalError, match="^PARAMETER_CONTENT$"): + case.policy.verify({}) + with pytest.raises(diagnostic.RefusalError, match="^DATA_FILE$"): + case.hook("open", (str(case.path), "r", os.O_RDONLY)) + + +@pytest.mark.parametrize( + "kind", ("neighbor", "outside", "h5", "parquet", "gzip", "write", "read_write") +) +def test_roster_does_not_admit_other_data_or_asset_mutations( + diagnostic, invented_package, kind +): + case = invented_package + case.policy.verify({}) + path = { + "neighbor": case.path.with_name("neighbor.csv"), + "outside": case.package.parent / "invented.csv", + "h5": case.path.with_suffix(".h5"), + "parquet": case.path.with_suffix(".parquet"), + "gzip": case.path.with_suffix(".gz"), + "write": case.path, + "read_write": case.path, + }[kind] + flags = {"write": os.O_WRONLY, "read_write": os.O_RDWR}.get(kind, os.O_RDONLY) + with pytest.raises(diagnostic.RefusalError, match="^DATA_FILE$"): + case.hook("open", (str(path), "r", flags)) + + +@pytest.fixture +def invented_urllib3(diagnostic, monkeypatch, tmp_path): + package = tmp_path / "urllib3" + spec = SimpleNamespace( + origin=str(package / "__init__.py"), submodule_search_locations=[str(package)] + ) + distribution = SimpleNamespace( + version=diagnostic.URLLIB3_VERSION, locate_file=lambda _: package + ) + monkeypatch.setattr(diagnostic.sys, "prefix", str(tmp_path)) + monkeypatch.setattr( + diagnostic.importlib.metadata, "distribution", lambda _: distribution + ) + monkeypatch.setattr(diagnostic.importlib.util, "find_spec", lambda _: spec) + for name in tuple(sys.modules): + if name == "urllib3" or name.startswith("urllib3."): + monkeypatch.delitem(sys.modules, name) + connection = SimpleNamespace( + __file__=str(package / "util/connection.py"), + HAS_IPV6=False, + allowed_gai_family=lambda: socket.AF_INET, + ) + case = SimpleNamespace( + package=package, + connection=connection, + fail=False, + distribution=distribution, + spec=spec, + ) + original_import = builtins.__import__ + + def import_dependency(name, *args, **kwargs): + if name != "urllib3.util": + return original_import(name, *args, **kwargs) + assert socket.has_ipv6 is False + if case.fail: + raise RuntimeError("invented import failure") + monkeypatch.setitem( + sys.modules, + "urllib3", + SimpleNamespace(__file__=str(package / "__init__.py")), + ) + return SimpleNamespace(connection=connection) + + monkeypatch.setattr(builtins, "__import__", import_dependency) + return case + + +@pytest.mark.parametrize("fails", (False, True)) +def test_ipv4_bootstrap_restores_scalar_and_preserves_callables( + diagnostic, invented_urllib3, fails +): + case = invented_urllib3 + case.fail = fails + original = socket.has_ipv6 + callables = (socket.socket, socket.getaddrinfo, socket.create_connection) + evidence = {} + if fails: + with pytest.raises(RuntimeError, match="invented import failure"): + diagnostic.bootstrap_no_network_urllib3(evidence) + else: + diagnostic.bootstrap_no_network_urllib3(evidence) + assert socket.has_ipv6 is original + assert callables == (socket.socket, socket.getaddrinfo, socket.create_connection) + assert evidence["urllib3"]["socket_flag_restored"] is True + assert evidence["urllib3"]["socket_callables_unchanged"] is True + assert evidence["urllib3"]["ipv4_fallback_verified"] is not fails + + +@pytest.mark.parametrize("kind", ("preloaded", "version", "origin", "fallback")) +def test_unreviewed_urllib3_path_refuses( + diagnostic, invented_urllib3, monkeypatch, kind +): + case = invented_urllib3 + code = "URLLIB3_" + kind.upper() + if kind == "preloaded": + monkeypatch.setitem(sys.modules, "urllib3.util", SimpleNamespace()) + elif kind == "version": + case.distribution.version = "unreviewed" + elif kind == "origin": + case.spec.origin = "/outside/urllib3/__init__.py" + else: + case.connection.HAS_IPV6 = True + original = socket.has_ipv6 + with pytest.raises(diagnostic.RefusalError, match="^" + code + "$"): + diagnostic.bootstrap_no_network_urllib3({}) + assert socket.has_ipv6 is original + + +def test_non_stdlib_socket_origin_refuses_before_scalar_change( + diagnostic, invented_urllib3, monkeypatch +): + original = socket.has_ipv6 + monkeypatch.setattr(socket, "__file__", "/invented/socket.py") + with pytest.raises(diagnostic.RefusalError, match="^SOCKET_ORIGIN$"): + diagnostic.bootstrap_no_network_urllib3({}) + assert socket.has_ipv6 is original + + +def test_real_audit_still_refuses_socket_and_child_before_either_is_created( + diagnostic, tmp_path +): + # Only the harness starts a child. The child installs the real audit hook; + # its own attempted network and child operations must both stop at that hook. + script = """ +import base64, hashlib, importlib.util, json, pathlib, socket, subprocess, sys +from types import SimpleNamespace +spec = importlib.util.spec_from_file_location('diagnostic', sys.argv[1]) +diagnostic = importlib.util.module_from_spec(spec) +spec.loader.exec_module(diagnostic) +root = pathlib.Path(sys.argv[2]) +prefix = root / 'environment' +package = prefix / 'policyengine_us' +asset = package / 'parameters/gov/invented.csv' +asset.parent.mkdir(parents=True) +content = b'invented,value\\nA,1\\n' +asset.write_bytes(content) +sha = hashlib.sha256(content).hexdigest() +record = diagnostic.importlib.metadata.PackagePath('policyengine_us/parameters/gov/invented.csv') +record.size = len(content) +record.hash = SimpleNamespace(mode='sha256', value=base64.urlsafe_b64encode(bytes.fromhex(sha)).decode().rstrip('=')) +diagnostic.PARAMETER_ASSETS = (('gov/invented.csv', len(content), sha),) +distribution = SimpleNamespace(version=diagnostic.PARAMETER_ASSET_VERSION, files=[record], locate_file=lambda name: prefix / str(name)) +diagnostic.importlib.metadata.distribution = lambda _: distribution +diagnostic.importlib.util.find_spec = lambda _: SimpleNamespace(origin=str(package / '__init__.py'), submodule_search_locations=[str(package)]) +sys.prefix = str(prefix) +policy = diagnostic.VerifiedParameterAssets() +refusals = diagnostic.install_boundary(root, root / 'owned', root / 'output', parameter_assets=policy) +policy.verify({}) +assert asset.read_bytes() == content +policy.verify({}) +assert not refusals +observed = [] +for action in (lambda: socket.socket(socket.AF_INET), lambda: subprocess.run([sys.executable, '-c', 'raise SystemExit(99)'])): + try: + action() + except diagnostic.RefusalError as error: + observed.append(str(error)) +assert observed == ['NETWORK_OR_CHILD', 'NETWORK_OR_CHILD'] +assert refusals == ['NETWORK_OR_CHILD'] +print(json.dumps(observed)) +""" + result = subprocess.run( + [sys.executable, "-c", script, diagnostic.__file__, str(tmp_path)], + check=True, + capture_output=True, + text=True, + ) + assert json.loads(result.stdout) == ["NETWORK_OR_CHILD", "NETWORK_OR_CHILD"] diff --git a/packages/microcosm-build/tests/test_spec_seed_identity_per_code_context.py b/packages/microcosm-build/tests/test_spec_seed_identity_per_code_context.py index 4d889c763..79f80b731 100644 --- a/packages/microcosm-build/tests/test_spec_seed_identity_per_code_context.py +++ b/packages/microcosm-build/tests/test_spec_seed_identity_per_code_context.py @@ -42,7 +42,7 @@ def test_caught_middle_child_denial_keeps_own_context(diagnostic, audited): terminal = None secret = "invented-secret-never-retain" for event, args, code in ( - ("open", ("/proc/stat", "r", os.O_RDONLY), "READ_SCOPE"), + ("open", ("/proc/unlisted-stat", "r", os.O_RDONLY), "READ_SCOPE"), ( "subprocess.Popen", (secret, [secret], secret, {secret: secret}), @@ -112,7 +112,7 @@ def test_eight_record_capacity_never_discards_or_grows(diagnostic, audited): contexts.update({code: diagnostic.encoded({"code": code}) for code in codes}) retained = tuple(contexts.items()) with pytest.raises(diagnostic.RefusalError, match="^READ_SCOPE$") as error: - hook("open", ("/proc/stat", "r", os.O_RDONLY)) + hook("open", ("/proc/unlisted-stat", "r", os.O_RDONLY)) assert tuple(contexts.items()) == retained and len(contexts) == 8 assert refusals == distinct == ["READ_SCOPE"] assert first == [error.value.boundary_context] diff --git a/packages/microcosm-build/tests/test_spec_seed_identity_system_metadata.py b/packages/microcosm-build/tests/test_spec_seed_identity_system_metadata.py index 8b4c89b98..6ebef9480 100644 --- a/packages/microcosm-build/tests/test_spec_seed_identity_system_metadata.py +++ b/packages/microcosm-build/tests/test_spec_seed_identity_system_metadata.py @@ -129,13 +129,20 @@ def resolve(path): return hooks[0], refusals -@pytest.mark.parametrize("path", ("/proc/self/maps", "/proc/321/maps")) -def test_only_logical_and_resolved_own_maps_are_admitted(audited, path): +@pytest.mark.parametrize("path", ("/proc/self/maps", "/proc/321/maps", "/proc/stat")) +def test_reviewed_maps_and_cpu_tuple_metadata_are_admitted(audited, path): hook, refusals = audited hook("open", (path, "r", os.O_RDONLY)) assert refusals == [] +def test_cpu_tuple_metadata_is_read_only(diagnostic, audited): + hook, refusals = audited + with pytest.raises(diagnostic.RefusalError, match="^WRITE_SCOPE$"): + hook("open", ("/proc/stat", "w", os.O_WRONLY)) + assert refusals == ["WRITE_SCOPE"] + + @pytest.mark.parametrize( "kind", ("other_process", "other_file", "relative", "descriptor", "child") ) diff --git a/tools/spec_seed_identity_diagnostics.py b/tools/spec_seed_identity_diagnostics.py index 1dc8ab273..6057ee9d8 100644 --- a/tools/spec_seed_identity_diagnostics.py +++ b/tools/spec_seed_identity_diagnostics.py @@ -4,9 +4,11 @@ from __future__ import annotations import argparse +import base64 import contextlib import hashlib import importlib.metadata +import importlib.util import json import os import platform @@ -54,6 +56,53 @@ "referencing", "scikit-learn", "torch", + "urllib3", +) +URLLIB3_VERSION = "2.7.0" +# Public model parameters eagerly loaded by the locked live engine ABI path. +# These are reviewed wheel RECORD identities, not permission for other CSVs. +PARAMETER_ASSET_VERSION = "1.819.0" +PARAMETER_ASSETS = ( + ( + "gov/hud/income_limits/section8_income_limits.csv", + 1570746, + "2a8de86c81a8806e75eb15b278a9708e855cd241ba4cc564871c94a565f46442", + ), + ( + "gov/hud/payment_standards/zip_code_payment_standards.csv", + 84054, + "66179687d3e0d9d6c99a58a528aad672ee6a3af8820189209eac6be1ba02d097", + ), + ( + "gov/hud/fmr/fair_market_rents.csv", + 1314027, + "aaaa3fe7935e553c5333da51722640704351040092dbf8102c2b19b86a373183", + ), + ( + "gov/hud/fmr/small_area_fair_market_rents.csv", + 246698, + "3ae1edbe6491bafd0b4c4d66401d914e9edf37304f8d93eb43a786a3e4d98ec1", + ), + ( + "gov/hud/utility_allowance/county_utility_allowances.csv", + 3913, + "0642a75980473ffa33be12d51a3c58ed735472598109b0998b3634d918563809", + ), + ( + "gov/hhs/medicaid/geography/medicaid_rating_areas.csv", + 59225, + "98bd49798e56026262002fb82699b7940d6bfb26589934909682f355ee4e086e", + ), + ( + "gov/hhs/medicaid/geography/second_lowest_silver_plan_cost.csv", + 293454, + "b2e415151d5ecc662b62e8b05ea3e1878dcad9e63384c255a314ea3254e47c55", + ), + ( + "gov/hhs/medicaid/geography/aca_rating_areas.csv", + 68571, + "a953284013a051956e1951f26e31252fcdf2275010953d09fd16546cb3df508e", + ), ) # Filled from the reviewed source-only roster; no import is used to construct it. SOURCE_PATHS = ( @@ -323,6 +372,186 @@ def read_refusal_context(code, event, path, requested, roots) -> bytes: return payload +def bootstrap_no_network_urllib3(bootstrap: dict[str, object]) -> None: + """Use urllib3's real optional IPv4 fallback without opening a probe socket.""" + import socket + + stdlib_socket = ( + Path(sys.base_prefix) + / "lib" + / f"python{sys.version_info.major}.{sys.version_info.minor}" + / "socket.py" + ).resolve() + require( + Path(socket.__file__).resolve() == stdlib_socket + and socket.__spec__ is not None + and Path(socket.__spec__.origin).resolve() == stdlib_socket, + "SOCKET_ORIGIN", + ) + require( + not any( + name == "urllib3" or name.startswith("urllib3.") for name in sys.modules + ), + "URLLIB3_PRELOADED", + ) + distribution = importlib.metadata.distribution("urllib3") + package = Path(distribution.locate_file("urllib3")).absolute() + spec = importlib.util.find_spec("urllib3") + require(distribution.version == URLLIB3_VERSION, "URLLIB3_VERSION") + require( + package == package.resolve() + and package.is_relative_to(Path(sys.prefix).resolve()) + and spec is not None + and spec.origin == str(package / "__init__.py") + and tuple(spec.submodule_search_locations or ()) == (str(package),), + "URLLIB3_ORIGIN", + ) + original = socket.has_ipv6 + require(type(original) is bool, "URLLIB3_FEATURE_FLAG") + callables = (socket.socket, socket.getaddrinfo, socket.create_connection) + evidence = { + "policy": "real_urllib3_optional_ipv4_fallback_v1", + "version": URLLIB3_VERSION, + "scope": "fresh_urllib3_import_only", + "original_has_ipv6": original, + "stdlib_socket_origin_verified": True, + "socket_flag_restored": False, + "socket_callables_unchanged": False, + "ipv4_fallback_verified": False, + "network_and_child_events_remain_denied": True, + } + bootstrap["urllib3"] = evidence + socket.has_ipv6 = False + try: + from urllib3.util import connection + finally: + socket.has_ipv6 = original + evidence["socket_flag_restored"] = socket.has_ipv6 is original + evidence["socket_callables_unchanged"] = callables == ( + socket.socket, + socket.getaddrinfo, + socket.create_connection, + ) + require( + evidence["socket_flag_restored"] + and evidence["socket_callables_unchanged"] + and sys.modules["urllib3"].__file__ == str(package / "__init__.py") + and connection.__file__ == str(package / "util/connection.py") + and connection.HAS_IPV6 is False + and connection.allowed_gai_family() == socket.AF_INET, + "URLLIB3_FALLBACK", + ) + evidence["ipv4_fallback_verified"] = True + + +class VerifiedParameterAssets: + """Exact package inputs; no country imports or directory-wide permission. + + The audit hook admits one read-only verification read at a time. No asset + becomes an engine input until the complete roster passes, and any failed + verification revokes the entire roster. The main routine rechecks it after + derivation and before publishing success. + """ + + def __init__(self): + self._checking: Path | None = None + self._verified: frozenset[Path] = frozenset() + self._declarations: tuple | None = None + + def permits(self, path: Path, *, writing: bool) -> bool: + return not writing and (path == self._checking or path in self._verified) + + def _resolve(self) -> tuple: + distribution = importlib.metadata.distribution("policyengine-us") + require(distribution.version == PARAMETER_ASSET_VERSION, "PARAMETER_VERSION") + package = Path(distribution.locate_file("policyengine_us")).absolute() + require( + package == package.resolve() + and package.is_relative_to(Path(sys.prefix).resolve()), + "PARAMETER_ORIGIN", + ) + spec = importlib.util.find_spec("policyengine_us") + require( + spec is not None + and spec.origin == str(package / "__init__.py") + and tuple(spec.submodule_search_locations or ()) == (str(package),), + "PARAMETER_ORIGIN", + ) + files = tuple(distribution.files or ()) + result = [] + for relative, size, expected in PARAMETER_ASSETS: + name = "policyengine_us/parameters/" + relative + records = [file for file in files if str(file) == name] + require(len(records) == 1, "PARAMETER_RECORD") + record = records[0] + require( + record.size == size + and record.hash is not None + and record.hash.mode == "sha256" + and record.hash.value + == base64.urlsafe_b64encode(bytes.fromhex(expected)) + .decode() + .rstrip("="), + "PARAMETER_RECORD", + ) + path = Path(distribution.locate_file(record)).absolute() + require( + path == package / "parameters" / relative and path == path.resolve(), + "PARAMETER_ORIGIN", + ) + result.append((path, name, size, expected)) + return tuple(result) + + def verify(self, bootstrap: dict[str, object]) -> None: + self._verified = frozenset() + try: + declarations = self._resolve() + require( + self._declarations is None or declarations == self._declarations, + "PARAMETER_CHANGED", + ) + for path, _, size, expected in declarations: + info = path.lstat() + require( + stat.S_ISREG(info.st_mode) and info.st_size == size, + "PARAMETER_CONTENT", + ) + self._checking = path + try: + # O_NOFOLLOW also rejects a last-component substitution + # between origin checking and the verification open. + with open( + path, + "rb", + opener=lambda name, flags: os.open(name, flags | os.O_NOFOLLOW), + ) as stream: + content = stream.read(size + 1) + finally: + self._checking = None + require( + len(content) == size and digest(content) == expected, + "PARAMETER_CONTENT", + ) + self._declarations = declarations + self._verified = frozenset(item[0] for item in declarations) + bootstrap["engine_parameter_assets"] = { + "policy": "exact_locked_public_parameters_v1", + "distribution": "policyengine-us", + "version": PARAMETER_ASSET_VERSION, + "record_origin_and_content_verified": True, + "recheck_before_publication_required": True, + "read_only": True, + "assets": [ + {"path": name, "size_bytes": size, "sha256": expected} + for _, name, size, expected in declarations + ], + } + except Exception: + self._checking = None + self._verified = frozenset() + raise + + def install_boundary( root: Path, owned: Path, @@ -331,6 +560,7 @@ def install_boundary( first_refusal: list[bytes] | None = None, distinct_refusals: list[str] | None = None, code_contexts: dict[str, bytes] | None = None, + parameter_assets: VerifiedParameterAssets | None = None, ) -> list[str]: refusals: list[str] = [] blocked_files = ( @@ -365,6 +595,8 @@ def install_boundary( Path("/dev/null"), Path("/proc/cpuinfo"), Path("/proc/meminfo"), + # psutil defines its CPU-time tuple from this public OS metadata. + Path("/proc/stat"), Path("/proc/self/maps"), # operation_path resolves /proc/self to this process's numeric PID. Path(f"/proc/{os.getpid()}/maps"), @@ -485,13 +717,16 @@ def audit(event: str, args: tuple[object, ...]) -> None: if event == "open" and args: path = operation_path(args[0]) name = str(path) - if name.lower().endswith(blocked_files): - refuse("DATA_FILE", event="open", path=path, requested=args[0]) flags = args[2] if len(args) > 2 else 0 writing = isinstance(flags, int) and bool( flags & (os.O_WRONLY | os.O_RDWR | os.O_CREAT | os.O_TRUNC | os.O_APPEND) ) + if name.lower().endswith(blocked_files) and not ( + parameter_assets is not None + and parameter_assets.permits(path, writing=writing) + ): + refuse("DATA_FILE", event="open", path=path, requested=args[0]) if writing and name != os.devnull: if not any(path.is_relative_to(p) for p in (owned, output)): refuse("WRITE_SCOPE") @@ -959,6 +1194,7 @@ def main() -> int: first_refusal: list[bytes] = [] distinct_refusals: list[str] = [] code_contexts: dict[str, bytes] = {} + parameter_assets = VerifiedParameterAssets() refusals = install_boundary( root, owned, @@ -966,6 +1202,7 @@ def main() -> int: first_refusal=first_refusal, distinct_refusals=distinct_refusals, code_contexts=code_contexts, + parameter_assets=parameter_assets, ) # Dependency-created temporaries must stay inside this invocation's scope. tempfile.tempdir = str(owned) @@ -1001,6 +1238,9 @@ def main() -> int: before = source_stamps(root) lock = tomllib.loads((root / "uv.lock").read_text()) installed = versions(lock) + require("policyengine_us" not in sys.modules, "PARAMETER_PRELOADED") + parameter_assets.verify(bootstrap) + bootstrap_no_network_urllib3(bootstrap) phase = "derive" processor_state = prime_processor_metadata(bootstrap) with ( @@ -1020,6 +1260,7 @@ def main() -> int: payloads = derive( root, owned, run, before, installed, torch, controls, bootstrap ) + parameter_assets.verify(bootstrap) verify_processor_metadata(processor_state, bootstrap) require(not refusals, "BOUNDARY_REFUSAL") exit_code = 0 From 4057df7d4ea7373b46c6e82e638ed59d562d538d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mar=C3=ADa=20Juaristi?= <127882282+juaristi22@users.noreply.github.com> Date: Thu, 10 Sep 2026 19:47:24 +0200 Subject: [PATCH 6/8] Fix issues from review: complete diagnostic seed module roster --- .../test_spec_seed_identity_cpu_bootstrap.py | 40 +++++++++++++++++++ tools/spec_seed_identity_diagnostics.py | 4 ++ 2 files changed, 44 insertions(+) diff --git a/packages/microcosm-build/tests/test_spec_seed_identity_cpu_bootstrap.py b/packages/microcosm-build/tests/test_spec_seed_identity_cpu_bootstrap.py index 9c31d579c..31ef0af68 100644 --- a/packages/microcosm-build/tests/test_spec_seed_identity_cpu_bootstrap.py +++ b/packages/microcosm-build/tests/test_spec_seed_identity_cpu_bootstrap.py @@ -2,6 +2,7 @@ from __future__ import annotations +import ast import importlib import importlib.machinery import importlib.util @@ -24,6 +25,45 @@ def diagnostic(): return module +def _maintained_seed_modules(diagnostic): + """Read literal attestations without importing their scientific modules.""" + source = ( + Path(diagnostic.__file__).resolve().parents[1] + / "packages/microcosm-build/src/microcosm/build/spec_engine/seeds.py" + ) + names = {"_DIRECT_KERNEL_MODULES", "_QRF_KERNEL_MODULES"} + declarations = { + node.targets[0].id: ast.literal_eval(node.value) + for node in ast.parse(source.read_text()).body + if isinstance(node, ast.Assign) + and len(node.targets) == 1 + and isinstance(node.targets[0], ast.Name) + and node.targets[0].id in names + } + assert set(declarations) == names + return tuple( + sorted({name for modules in declarations.values() for name in modules}) + ) + + +def test_diagnostic_roster_exactly_covers_maintained_seed_attestations(diagnostic): + assert diagnostic.SEED_MODULES == _maintained_seed_modules(diagnostic) + assert len(set(diagnostic.SEED_MODULES)) == len(diagnostic.SEED_MODULES) + + +def test_each_attested_seed_module_has_an_explicit_source_stamp(diagnostic): + paths = { + "packages/microcosm-" + + module.split(".")[1] + + "/src/" + + module.replace(".", "/") + + ".py" + for module in _maintained_seed_modules(diagnostic) + } + assert not paths.difference(diagnostic.SOURCE_PATHS) + assert len(set(diagnostic.SOURCE_PATHS)) == len(diagnostic.SOURCE_PATHS) + + @pytest.fixture def fresh_import_state(monkeypatch): # Keep the process import roster isolated without importing any dependency. diff --git a/tools/spec_seed_identity_diagnostics.py b/tools/spec_seed_identity_diagnostics.py index 6057ee9d8..d831a4e46 100644 --- a/tools/spec_seed_identity_diagnostics.py +++ b/tools/spec_seed_identity_diagnostics.py @@ -162,6 +162,8 @@ "packages/microcosm-build/src/microcosm/build/us_runtime/workers_compensation.py", "packages/microcosm-build/tests/test_spec_engine_loader.py", "packages/microcosm-calibrate/src/microcosm/calibrate/exact_k.py", + "packages/microcosm-calibrate/src/microcosm/calibrate/gates.py", + "packages/microcosm-calibrate/src/microcosm/calibrate/initialization.py", "packages/microcosm-calibrate/src/microcosm/calibrate/solve.py", "packages/microcosm-fit/src/microcosm/fit/qrf.py", "packages/microcosm-frame/src/microcosm/frame/adapters/_policyengine_us_source_index.py", @@ -208,6 +210,8 @@ "microcosm.build.us_runtime.wic_claim", "microcosm.build.us_runtime.workers_compensation", "microcosm.calibrate.exact_k", + "microcosm.calibrate.gates", + "microcosm.calibrate.initialization", "microcosm.calibrate.solve", "microcosm.fit.qrf", ) From 798fefb5e70a0cb5abb15fb61a4a9030671aa955 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mar=C3=ADa=20Juaristi?= <127882282+juaristi22@users.noreply.github.com> Date: Thu, 10 Sep 2026 22:04:45 +0200 Subject: [PATCH 7/8] Fix issues from review: restore US fixture and stage guard coverage --- .../survey_calibration_diagnostics.py | 6 + .../fixtures/us_spine_stage_contracts.json | 11981 ++++++++++++++++ .../tests/test_us_acs_housing_source.py | 6 + ...t_us_acs_person_coverage_authentication.py | 3 + .../tests/test_us_acs_population_catalogue.py | 3 + .../tests/test_us_acs_source_compile_cache.py | 3 + .../tests/test_us_acs_transfer.py | 23 +- .../tests/test_us_asec_checkpoint.py | 117 +- .../test_us_asec_coverage_authentication.py | 55 +- .../tests/test_us_multispine_pool_tool.py | 5 +- .../tests/test_us_spine_blindness.py | 866 +- .../test_us_survey_age_calibration_run.py | 29 +- .../tests/test_us_survey_calibration.py | 64 + 13 files changed, 13079 insertions(+), 82 deletions(-) create mode 100644 packages/microcosm-build/tests/fixtures/us_spine_stage_contracts.json diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/survey_calibration_diagnostics.py b/packages/microcosm-build/src/microcosm/build/us_runtime/survey_calibration_diagnostics.py index 25d0bb9b7..ce459f366 100644 --- a/packages/microcosm-build/src/microcosm/build/us_runtime/survey_calibration_diagnostics.py +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/survey_calibration_diagnostics.py @@ -135,6 +135,12 @@ def validate_survey_calibration_diagnostics( final = score_targets(work, targets, weights=weights, target_loss_cap=10.0) _require(not initial.skipped and not final.skipped, "SKIPPED_TARGETS") options = { + # This fixed grouped profile supplies neither informed gates nor a + # target-record budget. Reconstruct the solver defaults independently. + "gate_initialization_supplied": False, + "budget_basis": "nonzero_count", + "feasible_draw_pi_hi": None, + "budget_search": None, "grouped_preserve_zeros": { "enabled": True, "fixed_zero_count": int(np.count_nonzero(bounds.incoming == 0)), diff --git a/packages/microcosm-build/tests/fixtures/us_spine_stage_contracts.json b/packages/microcosm-build/tests/fixtures/us_spine_stage_contracts.json new file mode 100644 index 000000000..59f86a86e --- /dev/null +++ b/packages/microcosm-build/tests/fixtures/us_spine_stage_contracts.json @@ -0,0 +1,11981 @@ +{ + "bindings": { + "microcosm.build.us_runtime.acs_native_coverage_binding::AuthenticatedACSNativeCoverage.frame": { + "basis": "reviewed entry/caller-chain body", + "body_sha256": "890317c8161a8fb9a72d81f35296ed11dfdc8b72b47c0484d82c092aa1b887a6", + "references": [ + { + "caller": "microcosm.build.us_runtime.asec_current_money_source::AuthenticatedCurrentMoneySource.validate", + "expression": "self.frame", + "expression_sha256": "bc9202c1b911354870c877faf15851546d34fe5d0a4cf140ef8fd985c75cf832", + "resolution": "unresolved-us-candidate", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.asec_current_money_source::AuthenticatedCurrentMoneySource.validate", + "expression": "self.frame", + "expression_sha256": "bc9202c1b911354870c877faf15851546d34fe5d0a4cf140ef8fd985c75cf832", + "resolution": "unresolved-us-candidate", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.asec_current_money_source::AuthenticatedCurrentMoneySource.views", + "expression": "self.frame", + "expression_sha256": "bc9202c1b911354870c877faf15851546d34fe5d0a4cf140ef8fd985c75cf832", + "resolution": "unresolved-us-candidate", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.asec_current_money_units::CurrentMoneyTaxUnitResult.validate", + "expression": "self.frame", + "expression_sha256": "bc9202c1b911354870c877faf15851546d34fe5d0a4cf140ef8fd985c75cf832", + "resolution": "unresolved-us-candidate", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.asec_demographic_source::DemographicSourceAttachedAsec.validate", + "expression": "self.frame", + "expression_sha256": "bc9202c1b911354870c877faf15851546d34fe5d0a4cf140ef8fd985c75cf832", + "resolution": "unresolved-us-candidate", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.asec_demographic_source::DemographicSourceAttachedAsec.validate", + "expression": "self.frame", + "expression_sha256": "bc9202c1b911354870c877faf15851546d34fe5d0a4cf140ef8fd985c75cf832", + "resolution": "unresolved-us-candidate", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.asec_demographic_source::DemographicSourceAttachedAsec.validate", + "expression": "self.frame", + "expression_sha256": "bc9202c1b911354870c877faf15851546d34fe5d0a4cf140ef8fd985c75cf832", + "resolution": "unresolved-us-candidate", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.asec_demographic_source::DemographicSourceAttachedAsec.validate", + "expression": "self.frame", + "expression_sha256": "bc9202c1b911354870c877faf15851546d34fe5d0a4cf140ef8fd985c75cf832", + "resolution": "unresolved-us-candidate", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.asec_housing_status_source::HousingStatusAttachedAsec.validate", + "expression": "self.frame", + "expression_sha256": "bc9202c1b911354870c877faf15851546d34fe5d0a4cf140ef8fd985c75cf832", + "resolution": "unresolved-us-candidate", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.asec_housing_status_source::HousingStatusAttachedAsec.validate", + "expression": "self.frame", + "expression_sha256": "bc9202c1b911354870c877faf15851546d34fe5d0a4cf140ef8fd985c75cf832", + "resolution": "unresolved-us-candidate", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.asec_housing_universe_source::HousingUniverseAttachedAsec.validate", + "expression": "self.frame", + "expression_sha256": "bc9202c1b911354870c877faf15851546d34fe5d0a4cf140ef8fd985c75cf832", + "resolution": "unresolved-us-candidate", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.asec_housing_universe_source::HousingUniverseAttachedAsec.validate", + "expression": "self.frame", + "expression_sha256": "bc9202c1b911354870c877faf15851546d34fe5d0a4cf140ef8fd985c75cf832", + "resolution": "unresolved-us-candidate", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.asec_student_controls::StudentControlsAttachedAsec.validate", + "expression": "self.frame", + "expression_sha256": "bc9202c1b911354870c877faf15851546d34fe5d0a4cf140ef8fd985c75cf832", + "resolution": "unresolved-us-candidate", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.asec_student_controls::StudentControlsAttachedAsec.validate", + "expression": "self.frame", + "expression_sha256": "bc9202c1b911354870c877faf15851546d34fe5d0a4cf140ef8fd985c75cf832", + "resolution": "unresolved-us-candidate", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.asec_student_controls::StudentControlsAttachedAsec.validate", + "expression": "self.frame", + "expression_sha256": "bc9202c1b911354870c877faf15851546d34fe5d0a4cf140ef8fd985c75cf832", + "resolution": "unresolved-us-candidate", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.asec_student_controls::StudentControlsAttachedAsec.validate", + "expression": "self.frame", + "expression_sha256": "bc9202c1b911354870c877faf15851546d34fe5d0a4cf140ef8fd985c75cf832", + "resolution": "unresolved-us-candidate", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.graph_current_survey_predictors::CurrentSurveyPredictorAttachKernel.run", + "expression": "self.clone_population.frame", + "expression_sha256": "15f45b915a6b89ff7fb8924635f396d00ff022fa0b1ddd806d2c64dff205966a", + "resolution": "unresolved-us-candidate", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.graph_current_survey_predictors::CurrentSurveyPredictorAttachKernel.run", + "expression": "self.clone_population.frame", + "expression_sha256": "15f45b915a6b89ff7fb8924635f396d00ff022fa0b1ddd806d2c64dff205966a", + "resolution": "unresolved-us-candidate", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.graph_current_survey_predictors::_Kernel._qualified", + "expression": "self.clone_population.frame", + "expression_sha256": "15f45b915a6b89ff7fb8924635f396d00ff022fa0b1ddd806d2c64dff205966a", + "resolution": "unresolved-us-candidate", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.graph_current_survey_puf_transfer::CurrentSurveyAttachKernel.run", + "expression": "self.clone_population.frame", + "expression_sha256": "15f45b915a6b89ff7fb8924635f396d00ff022fa0b1ddd806d2c64dff205966a", + "resolution": "unresolved-us-candidate", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.graph_current_survey_puf_transfer::CurrentSurveyAttachKernel.run", + "expression": "self.clone_population.frame", + "expression_sha256": "15f45b915a6b89ff7fb8924635f396d00ff022fa0b1ddd806d2c64dff205966a", + "resolution": "unresolved-us-candidate", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.graph_current_survey_puf_transfer::CurrentSurveyAttachKernel.run", + "expression": "self.clone_population.frame", + "expression_sha256": "15f45b915a6b89ff7fb8924635f396d00ff022fa0b1ddd806d2c64dff205966a", + "resolution": "unresolved-us-candidate", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.graph_current_survey_puf_transfer::CurrentSurveyAttachKernel.run", + "expression": "self.clone_population.frame", + "expression_sha256": "15f45b915a6b89ff7fb8924635f396d00ff022fa0b1ddd806d2c64dff205966a", + "resolution": "unresolved-us-candidate", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.graph_current_survey_puf_transfer::CurrentSurveyPlacementKernel.run", + "expression": "self.clone_population.frame", + "expression_sha256": "15f45b915a6b89ff7fb8924635f396d00ff022fa0b1ddd806d2c64dff205966a", + "resolution": "unresolved-us-candidate", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.graph_current_survey_puf_transfer::CurrentSurveyPlacementKernel.run", + "expression": "self.clone_population.frame", + "expression_sha256": "15f45b915a6b89ff7fb8924635f396d00ff022fa0b1ddd806d2c64dff205966a", + "resolution": "unresolved-us-candidate", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.graph_current_survey_puf_transfer::CurrentSurveyPlacementKernel.run", + "expression": "self.clone_population.frame", + "expression_sha256": "15f45b915a6b89ff7fb8924635f396d00ff022fa0b1ddd806d2c64dff205966a", + "resolution": "unresolved-us-candidate", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.graph_current_survey_puf_transfer::CurrentSurveyPlacementKernel.run", + "expression": "self.clone_population.frame", + "expression_sha256": "15f45b915a6b89ff7fb8924635f396d00ff022fa0b1ddd806d2c64dff205966a", + "resolution": "unresolved-us-candidate", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.graph_full_puf_enrichment::FullPufMaskKernel.run", + "expression": "binding.expected_population.frame", + "expression_sha256": "76de9d83af4ba326b6212754b1034d89debb1d7d1a5ca4478bcbc2a34a696926", + "resolution": "unresolved-us-candidate", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.graph_full_puf_enrichment::_context_projection", + "expression": "frame", + "expression_sha256": "2eb691dff6a08513007b0d79ee5680d4ed46afeddfdca5913b12e520a9a61614", + "resolution": "unresolved-us-candidate", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.graph_full_puf_enrichment::_context_projection", + "expression": "frame", + "expression_sha256": "2eb691dff6a08513007b0d79ee5680d4ed46afeddfdca5913b12e520a9a61614", + "resolution": "unresolved-us-candidate", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.graph_full_puf_enrichment::_context_projection", + "expression": "frame", + "expression_sha256": "2eb691dff6a08513007b0d79ee5680d4ed46afeddfdca5913b12e520a9a61614", + "resolution": "unresolved-us-candidate", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.graph_full_puf_enrichment::_context_projection", + "expression": "frame", + "expression_sha256": "2eb691dff6a08513007b0d79ee5680d4ed46afeddfdca5913b12e520a9a61614", + "resolution": "unresolved-us-candidate", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.graph_full_puf_enrichment::_context_projection", + "expression": "frame", + "expression_sha256": "2eb691dff6a08513007b0d79ee5680d4ed46afeddfdca5913b12e520a9a61614", + "resolution": "unresolved-us-candidate", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.graph_full_puf_enrichment::_context_projection", + "expression": "frame", + "expression_sha256": "2eb691dff6a08513007b0d79ee5680d4ed46afeddfdca5913b12e520a9a61614", + "resolution": "unresolved-us-candidate", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.graph_full_puf_enrichment::_context_projection", + "expression": "frame", + "expression_sha256": "2eb691dff6a08513007b0d79ee5680d4ed46afeddfdca5913b12e520a9a61614", + "resolution": "unresolved-us-candidate", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.graph_full_puf_enrichment::_finalized_columns", + "expression": "frame", + "expression_sha256": "2eb691dff6a08513007b0d79ee5680d4ed46afeddfdca5913b12e520a9a61614", + "resolution": "unresolved-us-candidate", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.graph_full_puf_enrichment::_finalized_columns", + "expression": "frame", + "expression_sha256": "2eb691dff6a08513007b0d79ee5680d4ed46afeddfdca5913b12e520a9a61614", + "resolution": "unresolved-us-candidate", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.graph_full_puf_enrichment::_finalized_columns", + "expression": "frame", + "expression_sha256": "2eb691dff6a08513007b0d79ee5680d4ed46afeddfdca5913b12e520a9a61614", + "resolution": "unresolved-us-candidate", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.graph_full_puf_enrichment::_finalized_columns", + "expression": "frame", + "expression_sha256": "2eb691dff6a08513007b0d79ee5680d4ed46afeddfdca5913b12e520a9a61614", + "resolution": "unresolved-us-candidate", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.graph_full_puf_enrichment::_finalized_columns", + "expression": "binding.expected_population.frame", + "expression_sha256": "76de9d83af4ba326b6212754b1034d89debb1d7d1a5ca4478bcbc2a34a696926", + "resolution": "unresolved-us-candidate", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.graph_full_puf_enrichment::_inputs", + "expression": "frame", + "expression_sha256": "2eb691dff6a08513007b0d79ee5680d4ed46afeddfdca5913b12e520a9a61614", + "resolution": "unresolved-us-candidate", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.graph_full_puf_enrichment::_inputs", + "expression": "frame", + "expression_sha256": "2eb691dff6a08513007b0d79ee5680d4ed46afeddfdca5913b12e520a9a61614", + "resolution": "unresolved-us-candidate", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.graph_full_puf_enrichment::_inputs", + "expression": "frame", + "expression_sha256": "2eb691dff6a08513007b0d79ee5680d4ed46afeddfdca5913b12e520a9a61614", + "resolution": "unresolved-us-candidate", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.graph_full_puf_enrichment::_mask_result", + "expression": "frame", + "expression_sha256": "2eb691dff6a08513007b0d79ee5680d4ed46afeddfdca5913b12e520a9a61614", + "resolution": "unresolved-us-candidate", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.graph_full_puf_enrichment::_mask_result", + "expression": "frame", + "expression_sha256": "2eb691dff6a08513007b0d79ee5680d4ed46afeddfdca5913b12e520a9a61614", + "resolution": "unresolved-us-candidate", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.graph_full_puf_enrichment::_mask_result", + "expression": "frame", + "expression_sha256": "2eb691dff6a08513007b0d79ee5680d4ed46afeddfdca5913b12e520a9a61614", + "resolution": "unresolved-us-candidate", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.graph_full_puf_enrichment::_mask_result", + "expression": "binding.expected_population.frame", + "expression_sha256": "76de9d83af4ba326b6212754b1034d89debb1d7d1a5ca4478bcbc2a34a696926", + "resolution": "unresolved-us-candidate", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.graph_full_puf_enrichment::_masks", + "expression": "frame", + "expression_sha256": "2eb691dff6a08513007b0d79ee5680d4ed46afeddfdca5913b12e520a9a61614", + "resolution": "unresolved-us-candidate", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.graph_full_puf_enrichment::_masks", + "expression": "frame", + "expression_sha256": "2eb691dff6a08513007b0d79ee5680d4ed46afeddfdca5913b12e520a9a61614", + "resolution": "unresolved-us-candidate", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.graph_full_puf_enrichment::_masks", + "expression": "frame", + "expression_sha256": "2eb691dff6a08513007b0d79ee5680d4ed46afeddfdca5913b12e520a9a61614", + "resolution": "unresolved-us-candidate", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.graph_full_puf_enrichment::_outputs", + "expression": "frame", + "expression_sha256": "2eb691dff6a08513007b0d79ee5680d4ed46afeddfdca5913b12e520a9a61614", + "resolution": "unresolved-us-candidate", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.graph_full_puf_enrichment::_placement", + "expression": "frame", + "expression_sha256": "2eb691dff6a08513007b0d79ee5680d4ed46afeddfdca5913b12e520a9a61614", + "resolution": "unresolved-us-candidate", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.graph_full_puf_enrichment::_placement", + "expression": "frame", + "expression_sha256": "2eb691dff6a08513007b0d79ee5680d4ed46afeddfdca5913b12e520a9a61614", + "resolution": "unresolved-us-candidate", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.graph_full_puf_enrichment::_placement", + "expression": "frame", + "expression_sha256": "2eb691dff6a08513007b0d79ee5680d4ed46afeddfdca5913b12e520a9a61614", + "resolution": "unresolved-us-candidate", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.graph_full_puf_enrichment::_placement", + "expression": "binding.expected_population.frame", + "expression_sha256": "76de9d83af4ba326b6212754b1034d89debb1d7d1a5ca4478bcbc2a34a696926", + "resolution": "unresolved-us-candidate", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.graph_full_puf_enrichment::_population_stamp", + "expression": "frame", + "expression_sha256": "2eb691dff6a08513007b0d79ee5680d4ed46afeddfdca5913b12e520a9a61614", + "resolution": "unresolved-us-candidate", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.graph_full_puf_enrichment::_population_stamp", + "expression": "frame", + "expression_sha256": "2eb691dff6a08513007b0d79ee5680d4ed46afeddfdca5913b12e520a9a61614", + "resolution": "unresolved-us-candidate", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.graph_full_puf_enrichment::_population_stamp", + "expression": "frame", + "expression_sha256": "2eb691dff6a08513007b0d79ee5680d4ed46afeddfdca5913b12e520a9a61614", + "resolution": "unresolved-us-candidate", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.graph_full_puf_enrichment::_population_stamp", + "expression": "frame", + "expression_sha256": "2eb691dff6a08513007b0d79ee5680d4ed46afeddfdca5913b12e520a9a61614", + "resolution": "unresolved-us-candidate", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.graph_full_puf_enrichment::_population_stamp", + "expression": "frame", + "expression_sha256": "2eb691dff6a08513007b0d79ee5680d4ed46afeddfdca5913b12e520a9a61614", + "resolution": "unresolved-us-candidate", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.graph_full_puf_enrichment::_population_stamp", + "expression": "frame", + "expression_sha256": "2eb691dff6a08513007b0d79ee5680d4ed46afeddfdca5913b12e520a9a61614", + "resolution": "unresolved-us-candidate", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.graph_full_puf_enrichment::_population_stamp", + "expression": "frame", + "expression_sha256": "2eb691dff6a08513007b0d79ee5680d4ed46afeddfdca5913b12e520a9a61614", + "resolution": "unresolved-us-candidate", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.graph_full_puf_enrichment::_population_stamp", + "expression": "frame", + "expression_sha256": "2eb691dff6a08513007b0d79ee5680d4ed46afeddfdca5913b12e520a9a61614", + "resolution": "unresolved-us-candidate", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.graph_full_puf_enrichment::_population_stamp", + "expression": "frame", + "expression_sha256": "2eb691dff6a08513007b0d79ee5680d4ed46afeddfdca5913b12e520a9a61614", + "resolution": "unresolved-us-candidate", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.graph_full_puf_enrichment::_population_stamp", + "expression": "frame", + "expression_sha256": "2eb691dff6a08513007b0d79ee5680d4ed46afeddfdca5913b12e520a9a61614", + "resolution": "unresolved-us-candidate", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.graph_full_puf_enrichment::_population_stamp", + "expression": "frame", + "expression_sha256": "2eb691dff6a08513007b0d79ee5680d4ed46afeddfdca5913b12e520a9a61614", + "resolution": "unresolved-us-candidate", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.graph_full_puf_enrichment::_population_stamp", + "expression": "frame", + "expression_sha256": "2eb691dff6a08513007b0d79ee5680d4ed46afeddfdca5913b12e520a9a61614", + "resolution": "unresolved-us-candidate", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.graph_full_puf_enrichment::_population_stamp", + "expression": "frame", + "expression_sha256": "2eb691dff6a08513007b0d79ee5680d4ed46afeddfdca5913b12e520a9a61614", + "resolution": "unresolved-us-candidate", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.graph_full_puf_enrichment::_population_stamp", + "expression": "frame", + "expression_sha256": "2eb691dff6a08513007b0d79ee5680d4ed46afeddfdca5913b12e520a9a61614", + "resolution": "unresolved-us-candidate", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.graph_full_puf_enrichment::_population_stamp", + "expression": "frame", + "expression_sha256": "2eb691dff6a08513007b0d79ee5680d4ed46afeddfdca5913b12e520a9a61614", + "resolution": "unresolved-us-candidate", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.graph_full_puf_enrichment::_structural", + "expression": "frame", + "expression_sha256": "2eb691dff6a08513007b0d79ee5680d4ed46afeddfdca5913b12e520a9a61614", + "resolution": "unresolved-us-candidate", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.graph_full_puf_enrichment::full_puf_attachment_nodes", + "expression": "frame", + "expression_sha256": "2eb691dff6a08513007b0d79ee5680d4ed46afeddfdca5913b12e520a9a61614", + "resolution": "unresolved-us-candidate", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.graph_full_puf_enrichment::full_puf_attachment_nodes", + "expression": "frame", + "expression_sha256": "2eb691dff6a08513007b0d79ee5680d4ed46afeddfdca5913b12e520a9a61614", + "resolution": "unresolved-us-candidate", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.graph_full_puf_enrichment::full_puf_attachment_nodes", + "expression": "frame", + "expression_sha256": "2eb691dff6a08513007b0d79ee5680d4ed46afeddfdca5913b12e520a9a61614", + "resolution": "unresolved-us-candidate", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.graph_full_puf_enrichment::full_puf_attachment_nodes", + "expression": "binding.expected_population.frame", + "expression_sha256": "76de9d83af4ba326b6212754b1034d89debb1d7d1a5ca4478bcbc2a34a696926", + "resolution": "unresolved-us-candidate", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.graph_puf_diagnostic_consumer::CurrentSurveyHostProjectionKernel.run", + "expression": "self.clone_population.frame", + "expression_sha256": "15f45b915a6b89ff7fb8924635f396d00ff022fa0b1ddd806d2c64dff205966a", + "resolution": "unresolved-us-candidate", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.multispine_pool::MultispinePoolCheckpoint.__post_init__", + "expression": "self.frame", + "expression_sha256": "bc9202c1b911354870c877faf15851546d34fe5d0a4cf140ef8fd985c75cf832", + "resolution": "unresolved-us-candidate", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.multispine_pool::MultispinePoolCheckpoint.__post_init__", + "expression": "self.frame", + "expression_sha256": "bc9202c1b911354870c877faf15851546d34fe5d0a4cf140ef8fd985c75cf832", + "resolution": "unresolved-us-candidate", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.multispine_pool::PoolStageOutput.__post_init__", + "expression": "self.frame", + "expression_sha256": "bc9202c1b911354870c877faf15851546d34fe5d0a4cf140ef8fd985c75cf832", + "resolution": "unresolved-us-candidate", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.multispine_pool::PoolStageOutput.__post_init__", + "expression": "self.frame", + "expression_sha256": "bc9202c1b911354870c877faf15851546d34fe5d0a4cf140ef8fd985c75cf832", + "resolution": "unresolved-us-candidate", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.spine_assembly::SpinePreparation.__post_init__", + "expression": "self.frame", + "expression_sha256": "bc9202c1b911354870c877faf15851546d34fe5d0a4cf140ef8fd985c75cf832", + "resolution": "unresolved-us-candidate", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.spine_assembly::SpinePreparation.__post_init__", + "expression": "self.frame", + "expression_sha256": "bc9202c1b911354870c877faf15851546d34fe5d0a4cf140ef8fd985c75cf832", + "resolution": "unresolved-us-candidate", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.stacked_spine::StackedSpinePreparation.__post_init__", + "expression": "self.frame", + "expression_sha256": "bc9202c1b911354870c877faf15851546d34fe5d0a4cf140ef8fd985c75cf832", + "resolution": "unresolved-us-candidate", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.stacked_spine::StackedSpinePreparation.__post_init__", + "expression": "self.frame", + "expression_sha256": "bc9202c1b911354870c877faf15851546d34fe5d0a4cf140ef8fd985c75cf832", + "resolution": "unresolved-us-candidate", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.stacked_spine::StackedSpineResult.__post_init__", + "expression": "self.frame", + "expression_sha256": "bc9202c1b911354870c877faf15851546d34fe5d0a4cf140ef8fd985c75cf832", + "resolution": "unresolved-us-candidate", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.stacked_spine::StackedSpineResult.__post_init__", + "expression": "self.frame", + "expression_sha256": "bc9202c1b911354870c877faf15851546d34fe5d0a4cf140ef8fd985c75cf832", + "resolution": "unresolved-us-candidate", + "usage": "reference" + } + ] + }, + "microcosm.build.us_runtime.acs_native_coverage_binding::_producer": { + "basis": "reviewed entry/caller-chain body", + "body_sha256": "a5c814c37321f94f832b800e652b02103287b19344c9780c44137bfe596d74f5", + "references": [ + { + "caller": "microcosm.build.us_runtime.acs_native_coverage_binding::issue_acs_native_coverage", + "expression": "_producer()", + "expression_sha256": "e10192e7e675a53252f4e8d4d5d6453b471e56fd0ab142e50a492ebee999caf2", + "resolution": "resolved", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.acs_native_coverage_binding::issue_acs_native_coverage", + "expression": "_producer()", + "expression_sha256": "e10192e7e675a53252f4e8d4d5d6453b471e56fd0ab142e50a492ebee999caf2", + "resolution": "resolved", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.acs_native_coverage_binding::verify_acs_native_coverage", + "expression": "_producer()", + "expression_sha256": "e10192e7e675a53252f4e8d4d5d6453b471e56fd0ab142e50a492ebee999caf2", + "resolution": "resolved", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.acs_native_coverage_binding::verify_acs_native_coverage", + "expression": "_producer()", + "expression_sha256": "e10192e7e675a53252f4e8d4d5d6453b471e56fd0ab142e50a492ebee999caf2", + "resolution": "resolved", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.acs_population_catalogue::_producer", + "expression": "native._producer()", + "expression_sha256": "9f0f851615f256dca36fdd6fcda98c7914483217541e8b06ef472016840774df", + "resolution": "resolved", + "usage": "call" + } + ] + }, + "microcosm.build.us_runtime.acs_native_coverage_binding::issue_acs_native_coverage": { + "basis": "reviewed entry/caller-chain body", + "body_sha256": "73c0e7ee83ca99079e8f1ed2e834522487b1de7a681c2a49235905d8570b4b85", + "references": [ + { + "caller": "microcosm.build.us_runtime.survey_population_preparation::prepare_authenticated_survey_population", + "expression": "acs_native.issue_acs_native_coverage(root / 'acs', snapshot_root=snapshots, serialnos=acs_keys)", + "expression_sha256": "36c1c1d0013e1130f07d5179dbeb3ca23b60c99df77421467ad3db859b875228", + "resolution": "resolved", + "usage": "call" + } + ] + }, + "microcosm.build.us_runtime.acs_native_coverage_binding::verify_acs_native_coverage": { + "basis": "reviewed entry/caller-chain body", + "body_sha256": "dfde000006ad3ecc44bfd66796e0345cd28b597247696e94dafb946284b1d926", + "references": [ + { + "caller": "microcosm.build.us_runtime.acs_native_coverage_binding::AuthenticatedACSNativeCoverage.frame", + "expression": "verify_acs_native_coverage(self)", + "expression_sha256": "a657db06e3a0585e446b0bc2cef9e6e3f633acdb2bc2d40f63e04ef2d1cf1369", + "resolution": "resolved", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.current_survey_geography::qualify_current_survey_geography", + "expression": "source.acs_native.verify_acs_native_coverage(acs_native, acs_frame)", + "expression_sha256": "48146ffa41cea315dcf052a185b5fcaec84018b053674905e167a152933a3b5e", + "resolution": "unresolved-us-candidate", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.current_survey_geography::qualify_current_survey_geography", + "expression": "source.acs_native.verify_acs_native_coverage(acs_native, acs_frame)", + "expression_sha256": "48146ffa41cea315dcf052a185b5fcaec84018b053674905e167a152933a3b5e", + "resolution": "unresolved-us-candidate", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.current_survey_predictors::_demographic_features", + "expression": "source.acs_native.verify_acs_native_coverage(state.native[0], acs_frame)", + "expression_sha256": "29a556bbf738b3c17df419c2e61606f3a10441bda2f9c9f6cd49b0b3448585d5", + "resolution": "unresolved-us-candidate", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.survey_population_preparation::_validate", + "expression": "acs_native.verify_acs_native_coverage(state.native[0], state.source_frames[0])", + "expression_sha256": "6fb8a3af001f81c0bd7ccd70929b58e1517aa3486c9b1c6052d9fd579eb36dbe", + "resolution": "resolved", + "usage": "call" + } + ] + }, + "microcosm.build.us_runtime.acs_person_coverage_authentication::load_authenticated_acs_person_coverage": { + "basis": "reviewed entry/caller-chain body", + "body_sha256": "60c241f52b251dba1acc65a911a5841a93ff8ae39da375e40e517e62031dcbfa", + "references": [ + { + "caller": "microcosm.build.us_runtime.acs_native_coverage_binding::issue_acs_native_coverage", + "expression": "coverage.load_authenticated_acs_person_coverage(private, snapshot_root=roots[1], frame=prepared.frame)", + "expression_sha256": "f6b87ce423633d183baba93cbee22db42e95a0544aa31c03fa9305180361bca6", + "resolution": "resolved", + "usage": "call" + } + ] + }, + "microcosm.build.us_runtime.acs_person_coverage_columns::read_acs_person_coverage_columns": { + "basis": "reviewed entry/caller-chain body", + "body_sha256": "999ae849a966f5c9852f9eb83ab5f8a5b23ad3e79adeb075e3463c1b27afdf7f", + "references": [ + { + "caller": "microcosm.build.us_runtime.acs_person_coverage_authentication::load_authenticated_acs_person_coverage", + "expression": "literal.read_acs_person_coverage_columns(AcsPumsSource(paths['household'], paths['person'], vintage=2024), person_keys=keys, chunksize=min(1000, len(keys)))", + "expression_sha256": "1b87c1e3a1e470b4a8d5c023daaadf1f29ac353e2d90cbd1687dbddda71b118d", + "resolution": "resolved", + "usage": "call" + } + ] + }, + "microcosm.build.us_runtime.acs_population_catalogue::AuthenticatedACSSourceCatalogue.exclusion_ledger": { + "basis": "reviewed entry/caller-chain body", + "body_sha256": "d15045625662ec399a9423c6761d2fd921190267ae27988abad14255f4b9e4ef", + "references": [] + }, + "microcosm.build.us_runtime.acs_population_catalogue::AuthenticatedACSSourceCatalogue.households": { + "basis": "reviewed entry/caller-chain body", + "body_sha256": "4eddf8cc3408c08848b2bba751cd63b90d80d5b7ef82f5d287b82db11d27d363", + "references": [] + }, + "microcosm.build.us_runtime.acs_population_catalogue::AuthenticatedACSSourceCatalogue.lineage": { + "basis": "reviewed entry/caller-chain body", + "body_sha256": "ebc574c7aca7ea38ae54ec6c416e0dd774c55c15448081c530389ca71123c6ba", + "references": [] + }, + "microcosm.build.us_runtime.acs_population_catalogue::AuthenticatedACSSourceCatalogue.receipt": { + "basis": "reviewed entry/caller-chain body", + "body_sha256": "99f7c8414e6ac4c0b23b603c01721e5750b264fb1a62b53146223b91ebc671ef", + "references": [ + { + "caller": "microcosm.build.us_runtime.asec_current_money_units::CurrentMoneyTaxUnitResult.validate", + "expression": "self._student_source.receipt", + "expression_sha256": "b07ef0768f9115dfd6c2ccf2e0558c015159bc578c6b1c1dad955a3332bba05a", + "resolution": "unresolved-us-candidate", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.asec_demographic_source::DemographicSourceAttachedAsec.validate", + "expression": "self.observations.receipt", + "expression_sha256": "1197f225000198b5a1cb0313c842ca1a66604dc55e5d85ef3cbf846c336804a3", + "resolution": "unresolved-us-candidate", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.asec_housing_status_source::HousingStatusAttachedAsec.validate", + "expression": "self.tax_result.receipt", + "expression_sha256": "fb31b06dffb7494bb98766aeff9f28da9035003318144f4a796917c7e4f17b01", + "resolution": "unresolved-us-candidate", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.asec_prepared_source::PreparedAsecPopulation.receipt_payload", + "expression": "self.receipt", + "expression_sha256": "e092d770e10b3e310aa85b322b534aa524e35d368e3ddd2eb1665852865903b2", + "resolution": "unresolved-us-candidate", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.multispine_pool::PoolStageOutput.__post_init__", + "expression": "self.receipt", + "expression_sha256": "e092d770e10b3e310aa85b322b534aa524e35d368e3ddd2eb1665852865903b2", + "resolution": "unresolved-us-candidate", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.stacked_spine::StackedSpineHarmonization.__post_init__", + "expression": "self.receipt", + "expression_sha256": "e092d770e10b3e310aa85b322b534aa524e35d368e3ddd2eb1665852865903b2", + "resolution": "unresolved-us-candidate", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.stacked_spine::StackedSpinePreparation.__post_init__", + "expression": "self.receipt", + "expression_sha256": "e092d770e10b3e310aa85b322b534aa524e35d368e3ddd2eb1665852865903b2", + "resolution": "unresolved-us-candidate", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.stacked_spine::StackedSpinePreparation.__post_init__", + "expression": "self.receipt", + "expression_sha256": "e092d770e10b3e310aa85b322b534aa524e35d368e3ddd2eb1665852865903b2", + "resolution": "unresolved-us-candidate", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.stacked_spine::StackedSpinePreparation.__post_init__", + "expression": "self.receipt", + "expression_sha256": "e092d770e10b3e310aa85b322b534aa524e35d368e3ddd2eb1665852865903b2", + "resolution": "unresolved-us-candidate", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.stacked_spine::StackedSpineResult.__post_init__", + "expression": "self.receipt", + "expression_sha256": "e092d770e10b3e310aa85b322b534aa524e35d368e3ddd2eb1665852865903b2", + "resolution": "unresolved-us-candidate", + "usage": "reference" + } + ] + }, + "microcosm.build.us_runtime.acs_population_catalogue::AuthenticatedACSSourceCatalogue.to_bytes": { + "basis": "reviewed entry/caller-chain body", + "body_sha256": "80584af16861a8fe0859322ebe25407c611aba76516bbaa5f3ac74b0b6ec6ca1", + "references": [ + { + "caller": "microcosm.build.us_runtime.acs_population_catalogue::AuthenticatedACSSourceCatalogue.receipt", + "expression": "self.to_bytes()", + "expression_sha256": "2f1266f5888414195b694c70e99148ad094a1c83d880f7e73fd0f3774747e8d3", + "resolution": "resolved", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.survey_age_calibration::_run_survey_age_calibration", + "expression": "atomic_source.reconstruction.AtomicSurveyReconstruction.to_bytes(geography_config)", + "expression_sha256": "335e8b0f924d999b5c7198f3d9d6f20380d5cfc4c0903bcb8a5e60fab14abefc", + "resolution": "unresolved-us-candidate", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.survey_age_calibration::_run_survey_age_calibration", + "expression": "atomic_source.reconstruction.AtomicSurveyReconstruction.to_bytes(geography_config)", + "expression_sha256": "335e8b0f924d999b5c7198f3d9d6f20380d5cfc4c0903bcb8a5e60fab14abefc", + "resolution": "unresolved-us-candidate", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.survey_age_calibration::_run_survey_age_calibration", + "expression": "atomic_source.reconstruction.AtomicSurveyReconstruction.to_bytes(geography_config)", + "expression_sha256": "335e8b0f924d999b5c7198f3d9d6f20380d5cfc4c0903bcb8a5e60fab14abefc", + "resolution": "unresolved-us-candidate", + "usage": "call" + } + ] + }, + "microcosm.build.us_runtime.acs_population_catalogue::AuthenticatedACSSourceCatalogue.validate": { + "basis": "reviewed entry/caller-chain body", + "body_sha256": "678bd6178f4b2b6653a9917812c57f66154ea6dfe23096adb83825b126f1bf37", + "references": [ + { + "caller": "microcosm.build.us_runtime.asec_current_money_units::CurrentMoneyTaxUnitResult.validate", + "expression": "self._student_source.validate()", + "expression_sha256": "e277a2ec4101855cb31af89e16bea5d22eef25a4a9c50910937d2db242e1bf4a", + "resolution": "unresolved-us-candidate", + "usage": "call" + } + ] + }, + "microcosm.build.us_runtime.acs_population_catalogue::_checked": { + "basis": "reviewed entry/caller-chain body", + "body_sha256": "668c8c51a62defcd7067ccdc0eb382dd008748d9fe4dc2ea4e925f13ea0480d4", + "references": [ + { + "caller": "microcosm.build.us_runtime.acs_population_catalogue::AuthenticatedACSSourceCatalogue.exclusion_ledger", + "expression": "_checked(self)", + "expression_sha256": "5697d27f0d79b27273d191339e493291625c57d78ce16dfc5148cc68faa22cd0", + "resolution": "resolved", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.acs_population_catalogue::AuthenticatedACSSourceCatalogue.households", + "expression": "_checked(self)", + "expression_sha256": "5697d27f0d79b27273d191339e493291625c57d78ce16dfc5148cc68faa22cd0", + "resolution": "resolved", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.acs_population_catalogue::AuthenticatedACSSourceCatalogue.lineage", + "expression": "_checked(self)", + "expression_sha256": "5697d27f0d79b27273d191339e493291625c57d78ce16dfc5148cc68faa22cd0", + "resolution": "resolved", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.acs_population_catalogue::AuthenticatedACSSourceCatalogue.to_bytes", + "expression": "_checked(self)", + "expression_sha256": "5697d27f0d79b27273d191339e493291625c57d78ce16dfc5148cc68faa22cd0", + "resolution": "resolved", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.acs_population_catalogue::verify_acs_source_catalogue", + "expression": "_checked(value)", + "expression_sha256": "900eb31bef2c328a78d690f808844475801f1e95613571680dd6292c327c4b2b", + "resolution": "resolved", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.graph_current_survey_geography::CurrentSurveyGeographyKernel.run", + "expression": "preparation._checked()", + "expression_sha256": "92f12c0f1d35686015dedfb5af346caa3d48ae257295fa54a1b8dab163a111b3", + "resolution": "unresolved-us-candidate", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.graph_current_survey_geography::CurrentSurveyGeographyKernel.run", + "expression": "preparation._checked()", + "expression_sha256": "92f12c0f1d35686015dedfb5af346caa3d48ae257295fa54a1b8dab163a111b3", + "resolution": "unresolved-us-candidate", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.graph_current_survey_predictors::CurrentSurveyPredictorDonorFilterKernel.run", + "expression": "self.preparation._checked()", + "expression_sha256": "d127ce8187f0b94a74063723d91dc5c2866380eacc8ee91750e5b9694c6e9829", + "resolution": "unresolved-us-candidate", + "usage": "call" + } + ] + }, + "microcosm.build.us_runtime.acs_population_catalogue::_collect": { + "basis": "reviewed entry/caller-chain body", + "body_sha256": "d6cac5cc545d56fcc6c0718af1b06fd2f5e51d9c4b04f375f5eec06166ef7da8", + "references": [ + { + "caller": "microcosm.build.us_runtime.acs_population_catalogue::issue_acs_source_catalogue", + "expression": "_collect(projection, paths)", + "expression_sha256": "eafbbc875883d9f517345351a9d9021939b518f72d400d944ab6969aea97d4ad", + "resolution": "resolved", + "usage": "call" + } + ] + }, + "microcosm.build.us_runtime.acs_population_catalogue::_collect.consume": { + "basis": "reviewed entry/caller-chain body", + "body_sha256": "34fb7a964206acd76ea5980178fe9ee8949b6f8625f507106517389273b4bbfc", + "references": [ + { + "caller": "microcosm.build.us_runtime.acs_population_catalogue::_collect", + "expression": "consume()", + "expression_sha256": "2c8ae22edfc0b6802a5065e35c1994d7884362c2989998753ba4c7afc04b5cf9", + "resolution": "resolved", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.acs_population_catalogue::_collect", + "expression": "consume()", + "expression_sha256": "2c8ae22edfc0b6802a5065e35c1994d7884362c2989998753ba4c7afc04b5cf9", + "resolution": "resolved", + "usage": "call" + } + ] + }, + "microcosm.build.us_runtime.acs_population_catalogue::_producer": { + "basis": "reviewed entry/caller-chain body", + "body_sha256": "b504ebf9a44eaefc5d61562c74db857051b70638ac4ab2639a42dd10c89be57e", + "references": [ + { + "caller": "microcosm.build.us_runtime.acs_population_catalogue::_checked", + "expression": "_producer()", + "expression_sha256": "e10192e7e675a53252f4e8d4d5d6453b471e56fd0ab142e50a492ebee999caf2", + "resolution": "resolved", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.acs_population_catalogue::_checked", + "expression": "_producer()", + "expression_sha256": "e10192e7e675a53252f4e8d4d5d6453b471e56fd0ab142e50a492ebee999caf2", + "resolution": "resolved", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.acs_population_catalogue::issue_acs_source_catalogue", + "expression": "_producer()", + "expression_sha256": "e10192e7e675a53252f4e8d4d5d6453b471e56fd0ab142e50a492ebee999caf2", + "resolution": "resolved", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.acs_population_catalogue::issue_acs_source_catalogue", + "expression": "_producer()", + "expression_sha256": "e10192e7e675a53252f4e8d4d5d6453b471e56fd0ab142e50a492ebee999caf2", + "resolution": "resolved", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.survey_population_preparation::_producer", + "expression": "acs_catalogue._producer()", + "expression_sha256": "efea7a4060da8e4547388420496c008876bd045c036205ef7c4b18ae860db4e3", + "resolution": "resolved", + "usage": "call" + } + ] + }, + "microcosm.build.us_runtime.acs_population_catalogue::issue_acs_source_catalogue": { + "basis": "reviewed entry/caller-chain body", + "body_sha256": "e68468da74e325352e9f07f83fc98d25c7107ef99626cf3593ae7704909b11a5", + "references": [ + { + "caller": "microcosm.build.us_runtime.survey_population_preparation::prepare_authenticated_survey_population", + "expression": "acs_catalogue.issue_acs_source_catalogue(root / 'acs', snapshot_root=snapshots)", + "expression_sha256": "e0ac4ca6b8997e69ce36375d2773004d0f6db086d104a7dff16239ff18f9cb9e", + "resolution": "resolved", + "usage": "call" + } + ] + }, + "microcosm.build.us_runtime.acs_population_catalogue::verify_acs_source_catalogue": { + "basis": "reviewed entry/caller-chain body", + "body_sha256": "5876478c5677338d836052026cddc1ed8ead59ac1a6c47792d9aa6eeeeb3668c", + "references": [ + { + "caller": "microcosm.build.us_runtime.acs_population_catalogue::AuthenticatedACSSourceCatalogue.validate", + "expression": "verify_acs_source_catalogue(self)", + "expression_sha256": "9148826e0b8be0efb2b4fc8491940a0b3250a41d55bd8716f34e0c0fd2d1359e", + "resolution": "resolved", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.survey_population_preparation::_validate", + "expression": "acs_catalogue.verify_acs_source_catalogue(state.catalogues[0])", + "expression_sha256": "17ce23282aa2925461ef4e371319b3c03179cdb21f08eac57ff9ebd890460141", + "resolution": "resolved", + "usage": "call" + } + ] + }, + "microcosm.build.us_runtime.asec_2024_native_population::load_authenticated_asec_2024_native_population": { + "basis": "reviewed entry/caller-chain body", + "body_sha256": "2b629cab1cfa371728a483cf22b7028433c2f0872346bd5cb082dae9a8c1491a", + "references": [ + { + "caller": "microcosm.build.us_runtime.survey_population_preparation::prepare_authenticated_survey_population", + "expression": "asec_native.load_authenticated_asec_2024_native_population(**kwargs, selected_households=asec_keys)", + "expression_sha256": "23de8ac37d52a4df4baa7195ebbfe460aeee7ad8edf03589568cf567d234e650", + "resolution": "resolved", + "usage": "call" + } + ] + }, + "microcosm.build.us_runtime.asec_current_money_source::AuthenticatedCurrentMoneySource.ready": { + "basis": "reviewed entry/caller-chain body", + "body_sha256": "66fac69faa1a784020e807233068c031c8672352b689cc8b7ebe4cfd215abfad", + "references": [] + }, + "microcosm.build.us_runtime.asec_current_money_source::AuthenticatedCurrentMoneySource.validate": { + "basis": "reviewed entry/caller-chain body", + "body_sha256": "a718e8e39d0fa16cf2f5b37df668b8baf5609c8e16405e95ae5f844c3773ad8c", + "references": [ + { + "caller": "microcosm.build.us_runtime.asec_current_money_source::AuthenticatedCurrentMoneySource.views", + "expression": "self.validate()", + "expression_sha256": "0dbfb2abba3f214bd79b1fb5aee6394b786348c0e08bb13960aff7aa550cb432", + "resolution": "resolved", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.asec_current_money_units::CurrentMoneyTaxUnitResult.validate", + "expression": "self._student_source.validate()", + "expression_sha256": "e277a2ec4101855cb31af89e16bea5d22eef25a4a9c50910937d2db242e1bf4a", + "resolution": "unresolved-us-candidate", + "usage": "call" + } + ] + }, + "microcosm.build.us_runtime.asec_current_money_source::AuthenticatedCurrentMoneySource.views": { + "basis": "reviewed entry/caller-chain body", + "body_sha256": "a87721ccfdb113f54b560794ff3775d89cd08ee7032fa8109376d909527a94f1", + "references": [ + { + "caller": "microcosm.build.us_runtime.asec_current_money_source::AuthenticatedCurrentMoneySource.ready", + "expression": "self.views()", + "expression_sha256": "19013eef5d89483f1bc260bdccd8f521cf5338212b9dc73884b0ad2fe41999c4", + "resolution": "resolved", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.asec_current_money_source::AuthenticatedCurrentMoneySource.ready", + "expression": "views", + "expression_sha256": "8de1c42dd1e0a0fabdaea4ad5ddbb32cd7be2c4f0fafe380f8f070bd41e34784", + "resolution": "resolved", + "usage": "reference" + } + ] + }, + "microcosm.build.us_runtime.asec_current_money_units::CurrentMoneyTaxUnitResult.receipt": { + "basis": "reviewed entry/caller-chain body", + "body_sha256": "b9a94ab34e20461cea34d6f9cfff836d9d9c7fb332303cef10bc6b9a538ef4a5", + "references": [ + { + "caller": "microcosm.build.us_runtime.asec_current_money_units::CurrentMoneyTaxUnitResult.validate", + "expression": "receipt", + "expression_sha256": "12f5f9e6bbebccae7df6bbb5561c34ceacecb18b6865df7116ed480dd3dce838", + "resolution": "resolved", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.asec_current_money_units::CurrentMoneyTaxUnitResult.validate", + "expression": "receipt", + "expression_sha256": "12f5f9e6bbebccae7df6bbb5561c34ceacecb18b6865df7116ed480dd3dce838", + "resolution": "resolved", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.asec_current_money_units::CurrentMoneyTaxUnitResult.validate", + "expression": "receipt", + "expression_sha256": "12f5f9e6bbebccae7df6bbb5561c34ceacecb18b6865df7116ed480dd3dce838", + "resolution": "resolved", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.asec_current_money_units::CurrentMoneyTaxUnitResult.validate", + "expression": "receipt", + "expression_sha256": "12f5f9e6bbebccae7df6bbb5561c34ceacecb18b6865df7116ed480dd3dce838", + "resolution": "resolved", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.asec_current_money_units::CurrentMoneyTaxUnitResult.validate", + "expression": "receipt", + "expression_sha256": "12f5f9e6bbebccae7df6bbb5561c34ceacecb18b6865df7116ed480dd3dce838", + "resolution": "resolved", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.asec_current_money_units::CurrentMoneyTaxUnitResult.validate", + "expression": "receipt", + "expression_sha256": "12f5f9e6bbebccae7df6bbb5561c34ceacecb18b6865df7116ed480dd3dce838", + "resolution": "resolved", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.asec_current_money_units::CurrentMoneyTaxUnitResult.validate", + "expression": "self._student_source.receipt", + "expression_sha256": "b07ef0768f9115dfd6c2ccf2e0558c015159bc578c6b1c1dad955a3332bba05a", + "resolution": "unresolved-us-candidate", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.asec_current_money_units::CurrentMoneyTaxUnitResult.validate", + "expression": "self.receipt", + "expression_sha256": "e092d770e10b3e310aa85b322b534aa524e35d368e3ddd2eb1665852865903b2", + "resolution": "resolved", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.asec_current_money_units::_reconstruct", + "expression": "receipt", + "expression_sha256": "12f5f9e6bbebccae7df6bbb5561c34ceacecb18b6865df7116ed480dd3dce838", + "resolution": "resolved", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.asec_demographic_source::DemographicSourceAttachedAsec.validate", + "expression": "self.observations.receipt", + "expression_sha256": "1197f225000198b5a1cb0313c842ca1a66604dc55e5d85ef3cbf846c336804a3", + "resolution": "unresolved-us-candidate", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.asec_housing_status_source::HousingStatusAttachedAsec.validate", + "expression": "self.tax_result.receipt", + "expression_sha256": "fb31b06dffb7494bb98766aeff9f28da9035003318144f4a796917c7e4f17b01", + "resolution": "unresolved-us-candidate", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.asec_prepared_source::PreparedAsecPopulation.receipt_payload", + "expression": "self.receipt", + "expression_sha256": "e092d770e10b3e310aa85b322b534aa524e35d368e3ddd2eb1665852865903b2", + "resolution": "unresolved-us-candidate", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.multispine_pool::PoolStageOutput.__post_init__", + "expression": "self.receipt", + "expression_sha256": "e092d770e10b3e310aa85b322b534aa524e35d368e3ddd2eb1665852865903b2", + "resolution": "unresolved-us-candidate", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.stacked_spine::StackedSpineHarmonization.__post_init__", + "expression": "self.receipt", + "expression_sha256": "e092d770e10b3e310aa85b322b534aa524e35d368e3ddd2eb1665852865903b2", + "resolution": "unresolved-us-candidate", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.stacked_spine::StackedSpinePreparation.__post_init__", + "expression": "self.receipt", + "expression_sha256": "e092d770e10b3e310aa85b322b534aa524e35d368e3ddd2eb1665852865903b2", + "resolution": "unresolved-us-candidate", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.stacked_spine::StackedSpinePreparation.__post_init__", + "expression": "self.receipt", + "expression_sha256": "e092d770e10b3e310aa85b322b534aa524e35d368e3ddd2eb1665852865903b2", + "resolution": "unresolved-us-candidate", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.stacked_spine::StackedSpinePreparation.__post_init__", + "expression": "self.receipt", + "expression_sha256": "e092d770e10b3e310aa85b322b534aa524e35d368e3ddd2eb1665852865903b2", + "resolution": "unresolved-us-candidate", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.stacked_spine::StackedSpineResult.__post_init__", + "expression": "self.receipt", + "expression_sha256": "e092d770e10b3e310aa85b322b534aa524e35d368e3ddd2eb1665852865903b2", + "resolution": "unresolved-us-candidate", + "usage": "reference" + } + ] + }, + "microcosm.build.us_runtime.asec_current_money_units::CurrentMoneyTaxUnitResult.validate": { + "basis": "reviewed entry/caller-chain body", + "body_sha256": "e1e12966b7f1805c63c8edd6023a0a2772cdc2e1990ae37316169374f2163d9c", + "references": [ + { + "caller": "microcosm.build.us_runtime.asec_current_money_units::CurrentMoneyTaxUnitResult.validate", + "expression": "self._student_source.validate()", + "expression_sha256": "e277a2ec4101855cb31af89e16bea5d22eef25a4a9c50910937d2db242e1bf4a", + "resolution": "unresolved-us-candidate", + "usage": "call" + } + ] + }, + "microcosm.build.us_runtime.asec_current_money_units::_reconstruct": { + "basis": "reviewed entry/caller-chain body", + "body_sha256": "f781b2c35bda466a79f9caff279028b07ecf908b22ceb23519e9efac1ffb3bbd", + "references": [ + { + "caller": "microcosm.build.us_runtime.asec_current_money_units::reconstruct_current_money_tax_units", + "expression": "_reconstruct(source, ready)", + "expression_sha256": "b4f159ce808a025e94e897080fe58b79397cda7e3a99650b75d7a86cd825e157", + "resolution": "resolved", + "usage": "call" + } + ] + }, + "microcosm.build.us_runtime.asec_current_money_units::reconstruct_current_money_tax_units": { + "basis": "reviewed entry/caller-chain body", + "body_sha256": "dd9fa9851473de74ae8f835c1153e227dc54a2f0bcdc8a67881d9b8dff8f847a", + "references": [ + { + "caller": "microcosm.build.us_runtime.asec_prepared_source::prepare_asec_current_money_population", + "expression": "reconstruct_current_money_tax_units(with_students, ready)", + "expression_sha256": "f836005074eb99da3f9c3af58e8734ade5afe080a7ff313bcbba918536a57807", + "resolution": "resolved", + "usage": "call" + } + ] + }, + "microcosm.build.us_runtime.asec_demographic_source::DemographicSourceAttachedAsec.receipt": { + "basis": "reviewed entry/caller-chain body", + "body_sha256": "4be1c44f3795a5ae4e9643c4f3d2e378992b3ac736e02164c53441b1e5e95f9e", + "references": [ + { + "caller": "microcosm.build.us_runtime.asec_current_money_units::CurrentMoneyTaxUnitResult.validate", + "expression": "self._student_source.receipt", + "expression_sha256": "b07ef0768f9115dfd6c2ccf2e0558c015159bc578c6b1c1dad955a3332bba05a", + "resolution": "unresolved-us-candidate", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.asec_demographic_source::DemographicSourceAttachedAsec.validate", + "expression": "self.observations.receipt", + "expression_sha256": "1197f225000198b5a1cb0313c842ca1a66604dc55e5d85ef3cbf846c336804a3", + "resolution": "unresolved-us-candidate", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.asec_demographic_source::DemographicSourceAttachedAsec.validate", + "expression": "self.receipt", + "expression_sha256": "e092d770e10b3e310aa85b322b534aa524e35d368e3ddd2eb1665852865903b2", + "resolution": "resolved", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.asec_housing_status_source::HousingStatusAttachedAsec.validate", + "expression": "self.tax_result.receipt", + "expression_sha256": "fb31b06dffb7494bb98766aeff9f28da9035003318144f4a796917c7e4f17b01", + "resolution": "unresolved-us-candidate", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.asec_prepared_source::PreparedAsecPopulation.receipt_payload", + "expression": "self.receipt", + "expression_sha256": "e092d770e10b3e310aa85b322b534aa524e35d368e3ddd2eb1665852865903b2", + "resolution": "unresolved-us-candidate", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.multispine_pool::PoolStageOutput.__post_init__", + "expression": "self.receipt", + "expression_sha256": "e092d770e10b3e310aa85b322b534aa524e35d368e3ddd2eb1665852865903b2", + "resolution": "unresolved-us-candidate", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.stacked_spine::StackedSpineHarmonization.__post_init__", + "expression": "self.receipt", + "expression_sha256": "e092d770e10b3e310aa85b322b534aa524e35d368e3ddd2eb1665852865903b2", + "resolution": "unresolved-us-candidate", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.stacked_spine::StackedSpinePreparation.__post_init__", + "expression": "self.receipt", + "expression_sha256": "e092d770e10b3e310aa85b322b534aa524e35d368e3ddd2eb1665852865903b2", + "resolution": "unresolved-us-candidate", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.stacked_spine::StackedSpinePreparation.__post_init__", + "expression": "self.receipt", + "expression_sha256": "e092d770e10b3e310aa85b322b534aa524e35d368e3ddd2eb1665852865903b2", + "resolution": "unresolved-us-candidate", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.stacked_spine::StackedSpinePreparation.__post_init__", + "expression": "self.receipt", + "expression_sha256": "e092d770e10b3e310aa85b322b534aa524e35d368e3ddd2eb1665852865903b2", + "resolution": "unresolved-us-candidate", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.stacked_spine::StackedSpineResult.__post_init__", + "expression": "self.receipt", + "expression_sha256": "e092d770e10b3e310aa85b322b534aa524e35d368e3ddd2eb1665852865903b2", + "resolution": "unresolved-us-candidate", + "usage": "reference" + } + ] + }, + "microcosm.build.us_runtime.asec_demographic_source::DemographicSourceAttachedAsec.validate": { + "basis": "reviewed entry/caller-chain body", + "body_sha256": "a2137f27cc75e081719bf11938f13691e146ec69e70603ee86cd75de1d27e34e", + "references": [ + { + "caller": "microcosm.build.us_runtime.asec_current_money_units::CurrentMoneyTaxUnitResult.validate", + "expression": "self._student_source.validate()", + "expression_sha256": "e277a2ec4101855cb31af89e16bea5d22eef25a4a9c50910937d2db242e1bf4a", + "resolution": "unresolved-us-candidate", + "usage": "call" + } + ] + }, + "microcosm.build.us_runtime.asec_housing_status_source::HousingStatusAttachedAsec.receipt": { + "basis": "reviewed entry/caller-chain body", + "body_sha256": "b9a94ab34e20461cea34d6f9cfff836d9d9c7fb332303cef10bc6b9a538ef4a5", + "references": [ + { + "caller": "microcosm.build.us_runtime.asec_current_money_units::CurrentMoneyTaxUnitResult.validate", + "expression": "self._student_source.receipt", + "expression_sha256": "b07ef0768f9115dfd6c2ccf2e0558c015159bc578c6b1c1dad955a3332bba05a", + "resolution": "unresolved-us-candidate", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.asec_demographic_source::DemographicSourceAttachedAsec.validate", + "expression": "self.observations.receipt", + "expression_sha256": "1197f225000198b5a1cb0313c842ca1a66604dc55e5d85ef3cbf846c336804a3", + "resolution": "unresolved-us-candidate", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.asec_housing_status_source::HousingStatusAttachedAsec.validate", + "expression": "self.receipt", + "expression_sha256": "e092d770e10b3e310aa85b322b534aa524e35d368e3ddd2eb1665852865903b2", + "resolution": "resolved", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.asec_housing_status_source::HousingStatusAttachedAsec.validate", + "expression": "self.receipt", + "expression_sha256": "e092d770e10b3e310aa85b322b534aa524e35d368e3ddd2eb1665852865903b2", + "resolution": "resolved", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.asec_housing_status_source::HousingStatusAttachedAsec.validate", + "expression": "self.receipt", + "expression_sha256": "e092d770e10b3e310aa85b322b534aa524e35d368e3ddd2eb1665852865903b2", + "resolution": "resolved", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.asec_housing_status_source::HousingStatusAttachedAsec.validate", + "expression": "self.tax_result.receipt", + "expression_sha256": "fb31b06dffb7494bb98766aeff9f28da9035003318144f4a796917c7e4f17b01", + "resolution": "unresolved-us-candidate", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.asec_prepared_source::PreparedAsecPopulation.receipt_payload", + "expression": "self.receipt", + "expression_sha256": "e092d770e10b3e310aa85b322b534aa524e35d368e3ddd2eb1665852865903b2", + "resolution": "unresolved-us-candidate", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.multispine_pool::PoolStageOutput.__post_init__", + "expression": "self.receipt", + "expression_sha256": "e092d770e10b3e310aa85b322b534aa524e35d368e3ddd2eb1665852865903b2", + "resolution": "unresolved-us-candidate", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.stacked_spine::StackedSpineHarmonization.__post_init__", + "expression": "self.receipt", + "expression_sha256": "e092d770e10b3e310aa85b322b534aa524e35d368e3ddd2eb1665852865903b2", + "resolution": "unresolved-us-candidate", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.stacked_spine::StackedSpinePreparation.__post_init__", + "expression": "self.receipt", + "expression_sha256": "e092d770e10b3e310aa85b322b534aa524e35d368e3ddd2eb1665852865903b2", + "resolution": "unresolved-us-candidate", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.stacked_spine::StackedSpinePreparation.__post_init__", + "expression": "self.receipt", + "expression_sha256": "e092d770e10b3e310aa85b322b534aa524e35d368e3ddd2eb1665852865903b2", + "resolution": "unresolved-us-candidate", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.stacked_spine::StackedSpinePreparation.__post_init__", + "expression": "self.receipt", + "expression_sha256": "e092d770e10b3e310aa85b322b534aa524e35d368e3ddd2eb1665852865903b2", + "resolution": "unresolved-us-candidate", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.stacked_spine::StackedSpineResult.__post_init__", + "expression": "self.receipt", + "expression_sha256": "e092d770e10b3e310aa85b322b534aa524e35d368e3ddd2eb1665852865903b2", + "resolution": "unresolved-us-candidate", + "usage": "reference" + } + ] + }, + "microcosm.build.us_runtime.asec_housing_status_source::HousingStatusAttachedAsec.validate": { + "basis": "reviewed entry/caller-chain body", + "body_sha256": "5c32c05a7224736956c8589b7bd2f124e5e4e4ef3a7666ca0ccca0af5cfd3cf1", + "references": [ + { + "caller": "microcosm.build.us_runtime.asec_current_money_units::CurrentMoneyTaxUnitResult.validate", + "expression": "self._student_source.validate()", + "expression_sha256": "e277a2ec4101855cb31af89e16bea5d22eef25a4a9c50910937d2db242e1bf4a", + "resolution": "unresolved-us-candidate", + "usage": "call" + } + ] + }, + "microcosm.build.us_runtime.asec_housing_universe_source::HousingUniverseAttachedAsec.receipt": { + "basis": "reviewed entry/caller-chain body", + "body_sha256": "b9a94ab34e20461cea34d6f9cfff836d9d9c7fb332303cef10bc6b9a538ef4a5", + "references": [ + { + "caller": "microcosm.build.us_runtime.asec_current_money_units::CurrentMoneyTaxUnitResult.validate", + "expression": "self._student_source.receipt", + "expression_sha256": "b07ef0768f9115dfd6c2ccf2e0558c015159bc578c6b1c1dad955a3332bba05a", + "resolution": "unresolved-us-candidate", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.asec_demographic_source::DemographicSourceAttachedAsec.validate", + "expression": "self.observations.receipt", + "expression_sha256": "1197f225000198b5a1cb0313c842ca1a66604dc55e5d85ef3cbf846c336804a3", + "resolution": "unresolved-us-candidate", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.asec_housing_status_source::HousingStatusAttachedAsec.validate", + "expression": "self.tax_result.receipt", + "expression_sha256": "fb31b06dffb7494bb98766aeff9f28da9035003318144f4a796917c7e4f17b01", + "resolution": "unresolved-us-candidate", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.asec_housing_universe_source::HousingUniverseAttachedAsec.validate", + "expression": "self.receipt", + "expression_sha256": "e092d770e10b3e310aa85b322b534aa524e35d368e3ddd2eb1665852865903b2", + "resolution": "resolved", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.asec_prepared_source::PreparedAsecPopulation.receipt_payload", + "expression": "self.receipt", + "expression_sha256": "e092d770e10b3e310aa85b322b534aa524e35d368e3ddd2eb1665852865903b2", + "resolution": "unresolved-us-candidate", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.multispine_pool::PoolStageOutput.__post_init__", + "expression": "self.receipt", + "expression_sha256": "e092d770e10b3e310aa85b322b534aa524e35d368e3ddd2eb1665852865903b2", + "resolution": "unresolved-us-candidate", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.stacked_spine::StackedSpineHarmonization.__post_init__", + "expression": "self.receipt", + "expression_sha256": "e092d770e10b3e310aa85b322b534aa524e35d368e3ddd2eb1665852865903b2", + "resolution": "unresolved-us-candidate", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.stacked_spine::StackedSpinePreparation.__post_init__", + "expression": "self.receipt", + "expression_sha256": "e092d770e10b3e310aa85b322b534aa524e35d368e3ddd2eb1665852865903b2", + "resolution": "unresolved-us-candidate", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.stacked_spine::StackedSpinePreparation.__post_init__", + "expression": "self.receipt", + "expression_sha256": "e092d770e10b3e310aa85b322b534aa524e35d368e3ddd2eb1665852865903b2", + "resolution": "unresolved-us-candidate", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.stacked_spine::StackedSpinePreparation.__post_init__", + "expression": "self.receipt", + "expression_sha256": "e092d770e10b3e310aa85b322b534aa524e35d368e3ddd2eb1665852865903b2", + "resolution": "unresolved-us-candidate", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.stacked_spine::StackedSpineResult.__post_init__", + "expression": "self.receipt", + "expression_sha256": "e092d770e10b3e310aa85b322b534aa524e35d368e3ddd2eb1665852865903b2", + "resolution": "unresolved-us-candidate", + "usage": "reference" + } + ] + }, + "microcosm.build.us_runtime.asec_housing_universe_source::HousingUniverseAttachedAsec.validate": { + "basis": "reviewed entry/caller-chain body", + "body_sha256": "346890aa010b0c6c865db5f04be0a4ec63eefedecf8c5653f4cc1d5c00112ce8", + "references": [ + { + "caller": "microcosm.build.us_runtime.asec_current_money_units::CurrentMoneyTaxUnitResult.validate", + "expression": "self._student_source.validate()", + "expression_sha256": "e277a2ec4101855cb31af89e16bea5d22eef25a4a9c50910937d2db242e1bf4a", + "resolution": "unresolved-us-candidate", + "usage": "call" + } + ] + }, + "microcosm.build.us_runtime.asec_population_catalogue::issue_asec_source_catalogue": { + "basis": "reviewed entry/caller-chain body", + "body_sha256": "455b49efdf4056e56c897d8c1f3cfa471a01e161199afc0d3c93fbc7dcf54374", + "references": [ + { + "caller": "microcosm.build.us_runtime.survey_population_preparation::prepare_authenticated_survey_population", + "expression": "asec_catalogue.issue_asec_source_catalogue(**kwargs)", + "expression_sha256": "249b18e0b83d503f2b4d9184de1f8459722fa949176d5457e44bc4d105bf5eb5", + "resolution": "resolved", + "usage": "call" + } + ] + }, + "microcosm.build.us_runtime.asec_prepared_source::PreparedAsecPopulation.receipt_payload": { + "basis": "reviewed entry/caller-chain body", + "body_sha256": "f96930c435394bff3b24b825259ab9270d43d4465c9f3db2cfa39ccf1cb6ed96", + "references": [ + { + "caller": "microcosm.build.us_runtime.asec_prepared_source::PreparedAsecPopulation.receipt_sha256", + "expression": "self.receipt_payload", + "expression_sha256": "6b7521abafb0f323d97243dcb19d49ac55fed9a80e59715e42be8cd13707a6f2", + "resolution": "resolved", + "usage": "reference" + } + ] + }, + "microcosm.build.us_runtime.asec_prepared_source::PreparedAsecPopulation.receipt_sha256": { + "basis": "reviewed entry/caller-chain body", + "body_sha256": "9d0195f4c3e1a2b8e3ac532520344b5f8e4984a7cebe3f24fef9ab09d417c9aa", + "references": [] + }, + "microcosm.build.us_runtime.asec_prepared_source::load_graph_asec_prepared": { + "basis": "reviewed entry/caller-chain body", + "body_sha256": "6975d9db4d1e27267900ad97fa4929520f72be9d0c1852dea51f284c06443105", + "references": [ + { + "caller": "microcosm.build.us_runtime.graph_sources::us_source_codecs", + "expression": "load_graph_asec_prepared", + "expression_sha256": "4db79d081f11d0baef1549a891d7820306aaf3c9036885cf2730b6367a6a5fe0", + "resolution": "resolved", + "usage": "reference" + } + ] + }, + "microcosm.build.us_runtime.asec_prepared_source::prepare_asec_current_money_population": { + "basis": "reviewed entry/caller-chain body", + "body_sha256": "e81f201894d742054130a1c256b20fbd6248d7afb2a398f753eb97790dbb397e", + "references": [ + { + "caller": "microcosm.build.us_runtime.asec_prepared_source::load_graph_asec_prepared", + "expression": "prepare_asec_current_money_population(path)", + "expression_sha256": "e472b1c9c8eb6db831805b9c7902b1bd51ee8e5e10be52af428c44d762dcfd1d", + "resolution": "resolved", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.graph_asec_prepared::USAsecPreparedCreateKernel.run", + "expression": "prepare_asec_current_money_population(context.sources[ASEC_PREPARED_SOURCE_NAME])", + "expression_sha256": "f1bf041d5083f2ad29ace3a9d8666357eaadadf75a4907cb43fca7082625f199", + "resolution": "resolved", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.graph_composed_population::compose_from_sources", + "expression": "prepare_asec_current_money_population(asec_prepared_path)", + "expression_sha256": "232e5714820b7dac359460eb55c611613e17d6a1ee40e85882a8ef960feb0871", + "resolution": "resolved", + "usage": "call" + } + ] + }, + "microcosm.build.us_runtime.asec_student_controls::StudentControlsAttachedAsec.ready": { + "basis": "reviewed entry/caller-chain body", + "body_sha256": "cff930826beaf59596bfcd10c854e3c3a5959fd7d25526cb1239432c39bb5d55", + "references": [] + }, + "microcosm.build.us_runtime.asec_student_controls::StudentControlsAttachedAsec.receipt": { + "basis": "reviewed entry/caller-chain body", + "body_sha256": "b9a94ab34e20461cea34d6f9cfff836d9d9c7fb332303cef10bc6b9a538ef4a5", + "references": [ + { + "caller": "microcosm.build.us_runtime.asec_current_money_units::CurrentMoneyTaxUnitResult.validate", + "expression": "self._student_source.receipt", + "expression_sha256": "b07ef0768f9115dfd6c2ccf2e0558c015159bc578c6b1c1dad955a3332bba05a", + "resolution": "unresolved-us-candidate", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.asec_demographic_source::DemographicSourceAttachedAsec.validate", + "expression": "self.observations.receipt", + "expression_sha256": "1197f225000198b5a1cb0313c842ca1a66604dc55e5d85ef3cbf846c336804a3", + "resolution": "unresolved-us-candidate", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.asec_housing_status_source::HousingStatusAttachedAsec.validate", + "expression": "self.tax_result.receipt", + "expression_sha256": "fb31b06dffb7494bb98766aeff9f28da9035003318144f4a796917c7e4f17b01", + "resolution": "unresolved-us-candidate", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.asec_prepared_source::PreparedAsecPopulation.receipt_payload", + "expression": "self.receipt", + "expression_sha256": "e092d770e10b3e310aa85b322b534aa524e35d368e3ddd2eb1665852865903b2", + "resolution": "unresolved-us-candidate", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.asec_student_controls::StudentControlsAttachedAsec.validate", + "expression": "self.receipt", + "expression_sha256": "e092d770e10b3e310aa85b322b534aa524e35d368e3ddd2eb1665852865903b2", + "resolution": "resolved", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.multispine_pool::PoolStageOutput.__post_init__", + "expression": "self.receipt", + "expression_sha256": "e092d770e10b3e310aa85b322b534aa524e35d368e3ddd2eb1665852865903b2", + "resolution": "unresolved-us-candidate", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.stacked_spine::StackedSpineHarmonization.__post_init__", + "expression": "self.receipt", + "expression_sha256": "e092d770e10b3e310aa85b322b534aa524e35d368e3ddd2eb1665852865903b2", + "resolution": "unresolved-us-candidate", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.stacked_spine::StackedSpinePreparation.__post_init__", + "expression": "self.receipt", + "expression_sha256": "e092d770e10b3e310aa85b322b534aa524e35d368e3ddd2eb1665852865903b2", + "resolution": "unresolved-us-candidate", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.stacked_spine::StackedSpinePreparation.__post_init__", + "expression": "self.receipt", + "expression_sha256": "e092d770e10b3e310aa85b322b534aa524e35d368e3ddd2eb1665852865903b2", + "resolution": "unresolved-us-candidate", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.stacked_spine::StackedSpinePreparation.__post_init__", + "expression": "self.receipt", + "expression_sha256": "e092d770e10b3e310aa85b322b534aa524e35d368e3ddd2eb1665852865903b2", + "resolution": "unresolved-us-candidate", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.stacked_spine::StackedSpineResult.__post_init__", + "expression": "self.receipt", + "expression_sha256": "e092d770e10b3e310aa85b322b534aa524e35d368e3ddd2eb1665852865903b2", + "resolution": "unresolved-us-candidate", + "usage": "reference" + } + ] + }, + "microcosm.build.us_runtime.asec_student_controls::StudentControlsAttachedAsec.validate": { + "basis": "reviewed entry/caller-chain body", + "body_sha256": "eb2e12a43ee696f9c62ff98d539c5ff42d2fa8da46fa3b69d1f1303556b2088d", + "references": [ + { + "caller": "microcosm.build.us_runtime.asec_current_money_units::CurrentMoneyTaxUnitResult.validate", + "expression": "self._student_source.validate()", + "expression_sha256": "e277a2ec4101855cb31af89e16bea5d22eef25a4a9c50910937d2db242e1bf4a", + "resolution": "unresolved-us-candidate", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.asec_student_controls::StudentControlsAttachedAsec.ready", + "expression": "self.validate()", + "expression_sha256": "0dbfb2abba3f214bd79b1fb5aee6394b786348c0e08bb13960aff7aa550cb432", + "resolution": "resolved", + "usage": "call" + } + ] + }, + "microcosm.build.us_runtime.atomic_block_api_sources::assemble_atomic_block_api_sources": { + "basis": "reviewed entry/caller-chain body", + "body_sha256": "4912dc6b4c429ec71ac1ee1772d231f20e2d6a5abd0bfd75f2867a124de15345", + "references": [] + }, + "microcosm.build.us_runtime.atomic_block_sources::assemble_atomic_block_sources": { + "basis": "reviewed entry/caller-chain body", + "body_sha256": "b3d1024811bbcf3b94ebf87da8356211e874345c8cd6685f9af1f0b227e9df01", + "references": [] + }, + "microcosm.build.us_runtime.cps_carried_current::@CPS_CURRENT_PREDICTOR_MONEY_FIELDS": { + "basis": "finite selector/validator definition", + "body_sha256": "ff55fdbd599ab1b6fad500903fc9bc3365d1ebef24a994b4bf5b7768410f129a", + "references": [] + }, + "microcosm.build.us_runtime.cps_carried_current::@CPS_CURRENT_PREDICTOR_PERSON_LEAVES": { + "basis": "finite selector/validator definition", + "body_sha256": "3140fa368a880b52289dd29567aecb73bcd2daf31bcd4938cc9ff9efa3862457", + "references": [] + }, + "microcosm.build.us_runtime.current_survey_geography::qualify_current_survey_geography": { + "basis": "reviewed entry/caller-chain body", + "body_sha256": "b174d70c71a443bb4483aaa3dc17966dbfe372ab90e8c80466053e9ccd20a7a4", + "references": [ + { + "caller": "microcosm.build.us_runtime.survey_atomic_geography::reconstruct_atomic_survey_geography", + "expression": "observed.qualify_current_survey_geography(preparation)", + "expression_sha256": "99900b6a7c6fbe7f3f97d821ba702a625ef70d429611a9aa03222bcb29c286f8", + "resolution": "resolved", + "usage": "call" + } + ] + }, + "microcosm.build.us_runtime.current_survey_predictors::@DEMOGRAPHIC_FEATURES": { + "basis": "finite selector/validator definition", + "body_sha256": "df15b9e1fc6138d055786f5becfc831177aed133cbcd229329669c0cde220102", + "references": [] + }, + "microcosm.build.us_runtime.current_survey_predictors::@FEATURES": { + "basis": "finite selector/validator definition", + "body_sha256": "a75db7eeafa98f0a5536d0d754948f981fc799c4fab8a3e955432831a3715662", + "references": [] + }, + "microcosm.build.us_runtime.current_survey_predictors::@MONEY_FIELDS": { + "basis": "finite selector/validator definition", + "body_sha256": "a3e3c66ee27f36292cd5be01f90a18b5b72ffa70de8ab3b57ba84504bb6b8412", + "references": [] + }, + "microcosm.build.us_runtime.current_survey_predictors::@OUTPUTS": { + "basis": "finite selector/validator definition", + "body_sha256": "6dff264a99d39eb7e8cbff4d36227814bb50929e0215d28bb3142c2dd7c39d93", + "references": [] + }, + "microcosm.build.us_runtime.current_survey_predictors::@PHASE": { + "basis": "finite selector/validator definition", + "body_sha256": "5d6f3da30bec360ab6c08bd2fd45c205388dae1f2e14047024d7a0dabcd33b92", + "references": [] + }, + "microcosm.build.us_runtime.current_survey_predictors::@PROTOCOL": { + "basis": "finite selector/validator definition", + "body_sha256": "b08027e6319d7a208ea060dbde75fb9781ae7b5571fe8c45abc86bb0e8323000", + "references": [] + }, + "microcosm.build.us_runtime.current_survey_predictors::@SEED": { + "basis": "finite selector/validator definition", + "body_sha256": "ca58e5f6eb8157c7b1ce710e1d44b0439c4355bd7e4da7a3ab715ea537155a04", + "references": [] + }, + "microcosm.build.us_runtime.current_survey_predictors::@TARGETS": { + "basis": "finite selector/validator definition", + "body_sha256": "cce13707c924c812b330fcaf2ac14b7572c495799672fae6053ea6b391d16361", + "references": [] + }, + "microcosm.build.us_runtime.current_survey_predictors::_qualified_seal": { + "basis": "reviewed entry/caller-chain body", + "body_sha256": "427478741f21e522e28321b4613945423706a010be453e25157b12d2bff2f984", + "references": [ + { + "caller": "microcosm.build.us_runtime.current_survey_predictors::qualify_current_survey_predictors", + "expression": "_qualified_seal(result)", + "expression_sha256": "2aff7abea7489621f5e6f6e4714237d77b894408b8db0f7a3970049b5ff7979a", + "resolution": "resolved", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.current_survey_predictors::qualify_current_survey_predictors", + "expression": "_qualified_seal(result)", + "expression_sha256": "2aff7abea7489621f5e6f6e4714237d77b894408b8db0f7a3970049b5ff7979a", + "resolution": "resolved", + "usage": "call" + } + ] + }, + "microcosm.build.us_runtime.full_puf_enrichment::": { + "basis": "reviewed entry/caller-chain body", + "body_sha256": "7fd48e843a2524dd40393fe11e31e173b1e7095ffc79868d5aeee5518f83585e", + "references": [] + }, + "microcosm.build.us_runtime.full_puf_enrichment::@FULL65": { + "basis": "finite selector/validator definition", + "body_sha256": "e7bceee9799176db4f13300214e4175650ce32f9adf82672995ba3389d2779de", + "references": [] + }, + "microcosm.build.us_runtime.full_puf_enrichment::@PERSON_OUTPUTS": { + "basis": "finite selector/validator definition", + "body_sha256": "f903fb51217d0ff58b6f3275bd9a2702175b620c7f3d7c62c13b612cd0b8989d", + "references": [] + }, + "microcosm.build.us_runtime.full_puf_enrichment::@PHASE": { + "basis": "finite selector/validator definition", + "body_sha256": "45f43af716ecba4c9f167c49999017776f7172d09a517e5b2ee7c0cfd4a02bc7", + "references": [] + }, + "microcosm.build.us_runtime.full_puf_enrichment::@PREDICTORS": { + "basis": "finite selector/validator definition", + "body_sha256": "992a3881f87380a1362675a23742a1b88929757ee0fb4f55df742001c238d0e5", + "references": [] + }, + "microcosm.build.us_runtime.full_puf_enrichment::@PUF55_SURVEY_SS": { + "basis": "finite selector/validator definition", + "body_sha256": "abbf9c926de786ba96a0907ac55e9e147fce79c3439ab4cfe6893a48c2069ccd", + "references": [] + }, + "microcosm.build.us_runtime.full_puf_enrichment::@PUF55_SURVEY_SS_NO_TOTAL": { + "basis": "finite selector/validator definition", + "body_sha256": "0e203e45d15f6410ec0246fed18fb6d6f91bcd2e13a621399a24a1cf7aacc41d", + "references": [] + }, + "microcosm.build.us_runtime.full_puf_enrichment::@PUF55_SURVEY_SS_PERSON_OUTPUTS": { + "basis": "finite selector/validator definition", + "body_sha256": "0f49cbcf6e15d8563e1a3e9b024163097eb1e884e1eece3ecc96c94916b24162", + "references": [] + }, + "microcosm.build.us_runtime.full_puf_enrichment::@PUF55_SURVEY_SS_PREDICTORS": { + "basis": "finite selector/validator definition", + "body_sha256": "58e93f55c45d59faa01fe4409f36b9c790a7f171c6269023d80a85826c74ae70", + "references": [] + }, + "microcosm.build.us_runtime.full_puf_enrichment::@PUF59": { + "basis": "finite selector/validator definition", + "body_sha256": "56e5de0be2f095a0feaea464440b92a19d74c2ed71ee4b463cf122f4b1fd7e12", + "references": [] + }, + "microcosm.build.us_runtime.full_puf_enrichment::@PUF59_PREDICTORS": { + "basis": "finite selector/validator definition", + "body_sha256": "b3204ed11a977cc9afb88d43be7a280fd4a06b0ade9a43a496bd9a1f7ea78839", + "references": [] + }, + "microcosm.build.us_runtime.full_puf_enrichment::@PUF59_TAX_UNIT_OUTPUTS": { + "basis": "finite selector/validator definition", + "body_sha256": "fb76811a1c6b14305da63c68906187d57d6cfcd924b76ff056ed4e29eb916c5c", + "references": [] + }, + "microcosm.build.us_runtime.full_puf_enrichment::@SCF_MORTGAGE_OUTPUTS": { + "basis": "finite selector/validator definition", + "body_sha256": "02f0a296758efc46200b973a187f6b86a8e12910201dca84b8553a97d1defbb0", + "references": [] + }, + "microcosm.build.us_runtime.full_puf_enrichment::@SURVEY_SS_COMPONENTS": { + "basis": "finite selector/validator definition", + "body_sha256": "bac655a41e25f624ed9c587cfb526d6a8ebb911a97a521dfd32b7a24f6b9c5f9", + "references": [] + }, + "microcosm.build.us_runtime.full_puf_enrichment::@SURVEY_SS_TOTAL_PREDICTOR": { + "basis": "finite selector/validator definition", + "body_sha256": "1836566130cb8aed83b9ce595152b880abc0e09b81b4573c8fb53b8862ef8282", + "references": [] + }, + "microcosm.build.us_runtime.full_puf_enrichment::@TARGETS": { + "basis": "finite selector/validator definition", + "body_sha256": "8275d37dade042d574ce7b77fd5a81e507fa5c25dcc74593f1a8a3b9daeb4168", + "references": [] + }, + "microcosm.build.us_runtime.full_puf_enrichment::@TAX_UNIT_OUTPUTS": { + "basis": "finite selector/validator definition", + "body_sha256": "bc7d145bdf590a46760570164ca2f435d53f10aa42d4edf5a2d9f46e5008109f", + "references": [] + }, + "microcosm.build.us_runtime.full_puf_enrichment::FullPufInputs": { + "basis": "reviewed entry/caller-chain body", + "body_sha256": "9720f9731649c635405f2ee10eada84fb6df0e7ee93064f52de153b87265b00c", + "references": [ + { + "caller": "microcosm.build.us_runtime.full_puf_enrichment::prepare_full_puf_inputs", + "expression": "FullPufInputs(inputs.donor, inputs.donor_frame, matrix, universe, profile)", + "expression_sha256": "7bceb59f563909f29ed8b6a8d1d9bec390257d94fce1968c483c463e7c4bfc1b", + "resolution": "resolved", + "usage": "call" + } + ] + }, + "microcosm.build.us_runtime.full_puf_enrichment::PufOutputProfile": { + "basis": "finite selector/validator definition", + "body_sha256": "db88e3652e7c42964845735a76673aa7da0e2d19bdff27e298ac95295b8b69de", + "references": [ + { + "caller": "microcosm.build.us_runtime.full_puf_enrichment::", + "expression": "PufOutputProfile", + "expression_sha256": "b9b418dd4d6edf7c5868142b2404af0d4824e4be6177c1cfa8a00d8160e17d42", + "resolution": "resolved", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.full_puf_enrichment::", + "expression": "PufOutputProfile", + "expression_sha256": "b9b418dd4d6edf7c5868142b2404af0d4824e4be6177c1cfa8a00d8160e17d42", + "resolution": "resolved", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.full_puf_enrichment::", + "expression": "PufOutputProfile", + "expression_sha256": "b9b418dd4d6edf7c5868142b2404af0d4824e4be6177c1cfa8a00d8160e17d42", + "resolution": "resolved", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.full_puf_enrichment::", + "expression": "PufOutputProfile", + "expression_sha256": "b9b418dd4d6edf7c5868142b2404af0d4824e4be6177c1cfa8a00d8160e17d42", + "resolution": "resolved", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.full_puf_enrichment::FullPufInputs", + "expression": "PufOutputProfile", + "expression_sha256": "b9b418dd4d6edf7c5868142b2404af0d4824e4be6177c1cfa8a00d8160e17d42", + "resolution": "resolved", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.full_puf_enrichment::PufOutputProfile.donor_auxiliary_columns", + "expression": "self", + "expression_sha256": "7ed444d38b2676027d24aa06f77919b33308ad3d2d830e88198c8b769cbdf8dd", + "resolution": "resolved", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.full_puf_enrichment::PufOutputProfile.person_outputs", + "expression": "self", + "expression_sha256": "7ed444d38b2676027d24aa06f77919b33308ad3d2d830e88198c8b769cbdf8dd", + "resolution": "resolved", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.full_puf_enrichment::PufOutputProfile.phase", + "expression": "self", + "expression_sha256": "7ed444d38b2676027d24aa06f77919b33308ad3d2d830e88198c8b769cbdf8dd", + "resolution": "resolved", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.full_puf_enrichment::PufOutputProfile.phase", + "expression": "self", + "expression_sha256": "7ed444d38b2676027d24aa06f77919b33308ad3d2d830e88198c8b769cbdf8dd", + "resolution": "resolved", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.full_puf_enrichment::PufOutputProfile.predictors", + "expression": "self", + "expression_sha256": "7ed444d38b2676027d24aa06f77919b33308ad3d2d830e88198c8b769cbdf8dd", + "resolution": "resolved", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.full_puf_enrichment::PufOutputProfile.predictors", + "expression": "self", + "expression_sha256": "7ed444d38b2676027d24aa06f77919b33308ad3d2d830e88198c8b769cbdf8dd", + "resolution": "resolved", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.full_puf_enrichment::PufOutputProfile.source_predictors", + "expression": "self", + "expression_sha256": "7ed444d38b2676027d24aa06f77919b33308ad3d2d830e88198c8b769cbdf8dd", + "resolution": "resolved", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.full_puf_enrichment::PufOutputProfile.source_predictors", + "expression": "self", + "expression_sha256": "7ed444d38b2676027d24aa06f77919b33308ad3d2d830e88198c8b769cbdf8dd", + "resolution": "resolved", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.full_puf_enrichment::PufOutputProfile.source_predictors", + "expression": "self", + "expression_sha256": "7ed444d38b2676027d24aa06f77919b33308ad3d2d830e88198c8b769cbdf8dd", + "resolution": "resolved", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.full_puf_enrichment::PufOutputProfile.source_predictors", + "expression": "self", + "expression_sha256": "7ed444d38b2676027d24aa06f77919b33308ad3d2d830e88198c8b769cbdf8dd", + "resolution": "resolved", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.full_puf_enrichment::PufOutputProfile.targets", + "expression": "self", + "expression_sha256": "7ed444d38b2676027d24aa06f77919b33308ad3d2d830e88198c8b769cbdf8dd", + "resolution": "resolved", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.full_puf_enrichment::PufOutputProfile.targets", + "expression": "self", + "expression_sha256": "7ed444d38b2676027d24aa06f77919b33308ad3d2d830e88198c8b769cbdf8dd", + "resolution": "resolved", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.full_puf_enrichment::PufOutputProfile.tax_unit_outputs", + "expression": "self", + "expression_sha256": "7ed444d38b2676027d24aa06f77919b33308ad3d2d830e88198c8b769cbdf8dd", + "resolution": "resolved", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.full_puf_enrichment::require_puf_output_profile", + "expression": "PufOutputProfile", + "expression_sha256": "b9b418dd4d6edf7c5868142b2404af0d4824e4be6177c1cfa8a00d8160e17d42", + "resolution": "resolved", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.graph_full_puf_enrichment::FullPufAttachmentBinding", + "expression": "full.PufOutputProfile", + "expression_sha256": "2bf73ef819e554d1d5a42e270117e1a8543fa14deeafe2a4286a8eb278e81eac", + "resolution": "resolved", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.puf55_canonical_donor::canonical_puf55_donor_from_artifact", + "expression": "enrichment.PufOutputProfile", + "expression_sha256": "4fa0443d54f50e7a7cde3e504c34af6785f31143ccb608aa409c7e23961df0ff", + "resolution": "resolved", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.puf55_route_finalization::Puf55RouteDraws", + "expression": "full.PufOutputProfile", + "expression_sha256": "2bf73ef819e554d1d5a42e270117e1a8543fa14deeafe2a4286a8eb278e81eac", + "resolution": "resolved", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.puf55_route_finalization::_route_snapshot", + "expression": "full.PufOutputProfile", + "expression_sha256": "2bf73ef819e554d1d5a42e270117e1a8543fa14deeafe2a4286a8eb278e81eac", + "resolution": "resolved", + "usage": "reference" + } + ] + }, + "microcosm.build.us_runtime.full_puf_enrichment::PufOutputProfile.donor_auxiliary_columns": { + "basis": "reviewed entry/caller-chain body", + "body_sha256": "7595af6ac206110a348c97b8a68061b0e832e6863a20bae41a057d8a29f00bd4", + "references": [ + { + "caller": "microcosm.build.us_runtime.full_puf_enrichment::_validated_model_donor", + "expression": "profile.donor_auxiliary_columns", + "expression_sha256": "d95b9268e7cbf531bfb5773fc49f7f045775272996345115352f2530d24e3b87", + "resolution": "unresolved-us-candidate", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.full_puf_enrichment::_validated_model_donor", + "expression": "profile.donor_auxiliary_columns", + "expression_sha256": "d95b9268e7cbf531bfb5773fc49f7f045775272996345115352f2530d24e3b87", + "resolution": "unresolved-us-candidate", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.full_puf_enrichment::_validated_model_donor", + "expression": "profile.donor_auxiliary_columns", + "expression_sha256": "d95b9268e7cbf531bfb5773fc49f7f045775272996345115352f2530d24e3b87", + "resolution": "unresolved-us-candidate", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.full_puf_enrichment::canonical_full_puf_donor", + "expression": "profile.donor_auxiliary_columns", + "expression_sha256": "d95b9268e7cbf531bfb5773fc49f7f045775272996345115352f2530d24e3b87", + "resolution": "unresolved-us-candidate", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.full_puf_enrichment::canonical_full_puf_donor", + "expression": "profile.donor_auxiliary_columns", + "expression_sha256": "d95b9268e7cbf531bfb5773fc49f7f045775272996345115352f2530d24e3b87", + "resolution": "unresolved-us-candidate", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.full_puf_enrichment::canonical_full_puf_donor", + "expression": "profile.donor_auxiliary_columns", + "expression_sha256": "d95b9268e7cbf531bfb5773fc49f7f045775272996345115352f2530d24e3b87", + "resolution": "unresolved-us-candidate", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.full_puf_enrichment::canonical_full_puf_donor", + "expression": "profile.donor_auxiliary_columns", + "expression_sha256": "d95b9268e7cbf531bfb5773fc49f7f045775272996345115352f2530d24e3b87", + "resolution": "unresolved-us-candidate", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.full_puf_enrichment::canonical_full_puf_donor", + "expression": "profile.donor_auxiliary_columns", + "expression_sha256": "d95b9268e7cbf531bfb5773fc49f7f045775272996345115352f2530d24e3b87", + "resolution": "unresolved-us-candidate", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.full_puf_enrichment::finalize_full_puf", + "expression": "profile.donor_auxiliary_columns", + "expression_sha256": "d95b9268e7cbf531bfb5773fc49f7f045775272996345115352f2530d24e3b87", + "resolution": "unresolved-us-candidate", + "usage": "reference" + } + ] + }, + "microcosm.build.us_runtime.full_puf_enrichment::PufOutputProfile.person_outputs": { + "basis": "reviewed entry/caller-chain body", + "body_sha256": "a6c6bd75e058ed005980d3a6ca2e6b8cd72992146be97d0741d20427c2a172f0", + "references": [ + { + "caller": "microcosm.build.us_runtime.full_puf_enrichment::PufOutputProfile.targets", + "expression": "self.person_outputs", + "expression_sha256": "ecb141ff34a8527c7a259e22332f4303b168a3312b510f0b3f00f1801ca25e70", + "resolution": "resolved", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.full_puf_enrichment::_recipient_matrix", + "expression": "profile.person_outputs", + "expression_sha256": "607d545092f472a000501420b9958b45d511dbc648597271cf1b94e3cbbaa3d4", + "resolution": "unresolved-us-candidate", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.full_puf_enrichment::_validated_model_donor", + "expression": "profile.person_outputs", + "expression_sha256": "607d545092f472a000501420b9958b45d511dbc648597271cf1b94e3cbbaa3d4", + "resolution": "unresolved-us-candidate", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.full_puf_enrichment::canonical_full_puf_donor", + "expression": "profile.person_outputs", + "expression_sha256": "607d545092f472a000501420b9958b45d511dbc648597271cf1b94e3cbbaa3d4", + "resolution": "unresolved-us-candidate", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.full_puf_enrichment::canonical_full_puf_donor", + "expression": "profile.person_outputs", + "expression_sha256": "607d545092f472a000501420b9958b45d511dbc648597271cf1b94e3cbbaa3d4", + "resolution": "unresolved-us-candidate", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.full_puf_enrichment::canonical_full_puf_donor", + "expression": "profile.person_outputs", + "expression_sha256": "607d545092f472a000501420b9958b45d511dbc648597271cf1b94e3cbbaa3d4", + "resolution": "unresolved-us-candidate", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.full_puf_enrichment::canonical_full_puf_donor", + "expression": "profile.person_outputs", + "expression_sha256": "607d545092f472a000501420b9958b45d511dbc648597271cf1b94e3cbbaa3d4", + "resolution": "unresolved-us-candidate", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.full_puf_enrichment::finalize_full_puf", + "expression": "profile.person_outputs", + "expression_sha256": "607d545092f472a000501420b9958b45d511dbc648597271cf1b94e3cbbaa3d4", + "resolution": "unresolved-us-candidate", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.full_puf_enrichment::finalize_full_puf", + "expression": "profile.person_outputs", + "expression_sha256": "607d545092f472a000501420b9958b45d511dbc648597271cf1b94e3cbbaa3d4", + "resolution": "unresolved-us-candidate", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.full_puf_enrichment::prepare_full_puf_inputs", + "expression": "profile.person_outputs", + "expression_sha256": "607d545092f472a000501420b9958b45d511dbc648597271cf1b94e3cbbaa3d4", + "resolution": "unresolved-us-candidate", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.graph_full_puf_enrichment::_rosters", + "expression": "profile.person_outputs", + "expression_sha256": "607d545092f472a000501420b9958b45d511dbc648597271cf1b94e3cbbaa3d4", + "resolution": "unresolved-us-candidate", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.puf_support::PufTaxDetailChainInputs.target_order", + "expression": "self.person_outputs", + "expression_sha256": "ecb141ff34a8527c7a259e22332f4303b168a3312b510f0b3f00f1801ca25e70", + "resolution": "unresolved-us-candidate", + "usage": "reference" + } + ] + }, + "microcosm.build.us_runtime.full_puf_enrichment::PufOutputProfile.phase": { + "basis": "reviewed entry/caller-chain body", + "body_sha256": "a1920ce45b9b1cd03724e252344e902f9475b5d30df8885990e13e009500dcfc", + "references": [ + { + "caller": "microcosm.build.us_runtime.full_puf_enrichment::finalize_full_puf", + "expression": "profile.phase", + "expression_sha256": "fa3c944c99f2d6f9caae6607e9b691d0a8bd65112d579fcea80a26f0a81d795c", + "resolution": "unresolved-us-candidate", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.full_puf_enrichment::full_puf_train_apply_nodes", + "expression": "profile.phase", + "expression_sha256": "fa3c944c99f2d6f9caae6607e9b691d0a8bd65112d579fcea80a26f0a81d795c", + "resolution": "unresolved-us-candidate", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.full_puf_enrichment::full_puf_train_apply_nodes", + "expression": "profile.phase", + "expression_sha256": "fa3c944c99f2d6f9caae6607e9b691d0a8bd65112d579fcea80a26f0a81d795c", + "resolution": "unresolved-us-candidate", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.graph_full_puf_enrichment::_profile_chain", + "expression": "profile.phase", + "expression_sha256": "fa3c944c99f2d6f9caae6607e9b691d0a8bd65112d579fcea80a26f0a81d795c", + "resolution": "unresolved-us-candidate", + "usage": "reference" + } + ] + }, + "microcosm.build.us_runtime.full_puf_enrichment::PufOutputProfile.predictors": { + "basis": "reviewed entry/caller-chain body", + "body_sha256": "5b45cb572eeaf16836291dd4a84721a2e6e132ebf8d16c08d9b57912230f87c5", + "references": [ + { + "caller": "microcosm.build.us_runtime.full_puf_enrichment::PufOutputProfile.source_predictors", + "expression": "self.predictors", + "expression_sha256": "5f644d18fcf74baf351257af28250e316d6778d844c63e41dbbf1f608a65c7ce", + "resolution": "resolved", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.full_puf_enrichment::PufOutputProfile.source_predictors", + "expression": "self.predictors", + "expression_sha256": "5f644d18fcf74baf351257af28250e316d6778d844c63e41dbbf1f608a65c7ce", + "resolution": "resolved", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.full_puf_enrichment::_check_fitted_model_donor", + "expression": "profile.predictors", + "expression_sha256": "d5a67e51588f2c3977fedfedbac40b41758c63576de0e165a79457c9836c0786", + "resolution": "unresolved-us-candidate", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.full_puf_enrichment::_check_fitted_model_donor", + "expression": "profile.predictors", + "expression_sha256": "d5a67e51588f2c3977fedfedbac40b41758c63576de0e165a79457c9836c0786", + "resolution": "unresolved-us-candidate", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.full_puf_enrichment::_recipient_matrix", + "expression": "profile.predictors", + "expression_sha256": "d5a67e51588f2c3977fedfedbac40b41758c63576de0e165a79457c9836c0786", + "resolution": "unresolved-us-candidate", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.full_puf_enrichment::_recipient_matrix", + "expression": "profile.predictors", + "expression_sha256": "d5a67e51588f2c3977fedfedbac40b41758c63576de0e165a79457c9836c0786", + "resolution": "unresolved-us-candidate", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.full_puf_enrichment::_recipient_matrix", + "expression": "profile.predictors", + "expression_sha256": "d5a67e51588f2c3977fedfedbac40b41758c63576de0e165a79457c9836c0786", + "resolution": "unresolved-us-candidate", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.full_puf_enrichment::_recipient_matrix", + "expression": "profile.predictors", + "expression_sha256": "d5a67e51588f2c3977fedfedbac40b41758c63576de0e165a79457c9836c0786", + "resolution": "unresolved-us-candidate", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.full_puf_enrichment::_validated_model_donor", + "expression": "profile.predictors", + "expression_sha256": "d5a67e51588f2c3977fedfedbac40b41758c63576de0e165a79457c9836c0786", + "resolution": "unresolved-us-candidate", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.full_puf_enrichment::_validated_model_donor", + "expression": "profile.predictors", + "expression_sha256": "d5a67e51588f2c3977fedfedbac40b41758c63576de0e165a79457c9836c0786", + "resolution": "unresolved-us-candidate", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.full_puf_enrichment::_validated_model_donor", + "expression": "profile.predictors", + "expression_sha256": "d5a67e51588f2c3977fedfedbac40b41758c63576de0e165a79457c9836c0786", + "resolution": "unresolved-us-candidate", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.full_puf_enrichment::_validated_model_donor", + "expression": "profile.predictors", + "expression_sha256": "d5a67e51588f2c3977fedfedbac40b41758c63576de0e165a79457c9836c0786", + "resolution": "unresolved-us-candidate", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.full_puf_enrichment::canonical_full_puf_donor", + "expression": "profile.predictors", + "expression_sha256": "d5a67e51588f2c3977fedfedbac40b41758c63576de0e165a79457c9836c0786", + "resolution": "unresolved-us-candidate", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.full_puf_enrichment::canonical_full_puf_donor", + "expression": "profile.predictors", + "expression_sha256": "d5a67e51588f2c3977fedfedbac40b41758c63576de0e165a79457c9836c0786", + "resolution": "unresolved-us-candidate", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.full_puf_enrichment::decode_full_puf_draws", + "expression": "profile.predictors", + "expression_sha256": "d5a67e51588f2c3977fedfedbac40b41758c63576de0e165a79457c9836c0786", + "resolution": "unresolved-us-candidate", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.full_puf_enrichment::finalize_full_puf", + "expression": "profile.predictors", + "expression_sha256": "d5a67e51588f2c3977fedfedbac40b41758c63576de0e165a79457c9836c0786", + "resolution": "unresolved-us-candidate", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.full_puf_enrichment::full_puf_train_apply_nodes", + "expression": "profile.predictors", + "expression_sha256": "d5a67e51588f2c3977fedfedbac40b41758c63576de0e165a79457c9836c0786", + "resolution": "unresolved-us-candidate", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.full_puf_enrichment::prepare_full_puf_inputs", + "expression": "profile.predictors", + "expression_sha256": "d5a67e51588f2c3977fedfedbac40b41758c63576de0e165a79457c9836c0786", + "resolution": "unresolved-us-candidate", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.graph_full_puf_enrichment::_profile_chain", + "expression": "profile.predictors", + "expression_sha256": "d5a67e51588f2c3977fedfedbac40b41758c63576de0e165a79457c9836c0786", + "resolution": "unresolved-us-candidate", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.housing_inputs::AcsRentDonorPreparation.__post_init__", + "expression": "self.predictors", + "expression_sha256": "5f644d18fcf74baf351257af28250e316d6778d844c63e41dbbf1f608a65c7ce", + "resolution": "unresolved-us-candidate", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.housing_inputs::AcsRentDonorPreparation.__post_init__", + "expression": "self.predictors", + "expression_sha256": "5f644d18fcf74baf351257af28250e316d6778d844c63e41dbbf1f608a65c7ce", + "resolution": "unresolved-us-candidate", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.housing_inputs::AcsRentRecipientPreparation.__post_init__", + "expression": "self.predictors", + "expression_sha256": "5f644d18fcf74baf351257af28250e316d6778d844c63e41dbbf1f608a65c7ce", + "resolution": "unresolved-us-candidate", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.housing_inputs::AcsRentRecipientPreparation.__post_init__", + "expression": "self.predictors", + "expression_sha256": "5f644d18fcf74baf351257af28250e316d6778d844c63e41dbbf1f608a65c7ce", + "resolution": "unresolved-us-candidate", + "usage": "reference" + } + ] + }, + "microcosm.build.us_runtime.full_puf_enrichment::PufOutputProfile.source_predictors": { + "basis": "reviewed entry/caller-chain body", + "body_sha256": "7e47c178187c9dae864815108a3167956b90b7e66e76f71b8a182ee49b301f9c", + "references": [ + { + "caller": "microcosm.build.us_runtime.full_puf_enrichment::_profile_source_values", + "expression": "profile.source_predictors", + "expression_sha256": "fa600fbcf5256f5be683777fd4776ce6881f98ef01f856d8309ce63b81413f28", + "resolution": "unresolved-us-candidate", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.full_puf_enrichment::_profile_source_values", + "expression": "profile.source_predictors", + "expression_sha256": "fa600fbcf5256f5be683777fd4776ce6881f98ef01f856d8309ce63b81413f28", + "resolution": "unresolved-us-candidate", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.full_puf_enrichment::_recipient_matrix", + "expression": "profile.source_predictors", + "expression_sha256": "fa600fbcf5256f5be683777fd4776ce6881f98ef01f856d8309ce63b81413f28", + "resolution": "unresolved-us-candidate", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.full_puf_enrichment::canonical_full_puf_donor", + "expression": "profile.source_predictors", + "expression_sha256": "fa600fbcf5256f5be683777fd4776ce6881f98ef01f856d8309ce63b81413f28", + "resolution": "unresolved-us-candidate", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.full_puf_enrichment::canonical_full_puf_donor", + "expression": "profile.source_predictors", + "expression_sha256": "fa600fbcf5256f5be683777fd4776ce6881f98ef01f856d8309ce63b81413f28", + "resolution": "unresolved-us-candidate", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.puf55_canonical_donor::canonical_puf55_donor_from_artifact", + "expression": "enrichment.PUF59.source_predictors", + "expression_sha256": "abaa8c02ef68ec70213b33f8a3d5968ef0983b6383b9ed785d1dedc070eca302", + "resolution": "unresolved-us-candidate", + "usage": "reference" + } + ] + }, + "microcosm.build.us_runtime.full_puf_enrichment::PufOutputProfile.targets": { + "basis": "reviewed entry/caller-chain body", + "body_sha256": "9a43b87e1602db61257a4fbee7a9ebb5af33c1562a9c4bcdc59e1630f494f66e", + "references": [ + { + "caller": "microcosm.build.us_runtime.full_puf_enrichment::_check_fitted_model_donor", + "expression": "profile.targets", + "expression_sha256": "709b42afefce06be1e687bee5a35285a8e0a40324e0d6510b9211e9eacdd2804", + "resolution": "unresolved-us-candidate", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.full_puf_enrichment::_check_fitted_model_donor", + "expression": "profile.targets", + "expression_sha256": "709b42afefce06be1e687bee5a35285a8e0a40324e0d6510b9211e9eacdd2804", + "resolution": "unresolved-us-candidate", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.full_puf_enrichment::_check_fitted_model_donor", + "expression": "profile.targets", + "expression_sha256": "709b42afefce06be1e687bee5a35285a8e0a40324e0d6510b9211e9eacdd2804", + "resolution": "unresolved-us-candidate", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.full_puf_enrichment::_validated_model_donor", + "expression": "profile.targets", + "expression_sha256": "709b42afefce06be1e687bee5a35285a8e0a40324e0d6510b9211e9eacdd2804", + "resolution": "unresolved-us-candidate", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.full_puf_enrichment::_validated_model_donor", + "expression": "profile.targets", + "expression_sha256": "709b42afefce06be1e687bee5a35285a8e0a40324e0d6510b9211e9eacdd2804", + "resolution": "unresolved-us-candidate", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.full_puf_enrichment::canonical_full_puf_donor", + "expression": "profile.targets", + "expression_sha256": "709b42afefce06be1e687bee5a35285a8e0a40324e0d6510b9211e9eacdd2804", + "resolution": "unresolved-us-candidate", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.full_puf_enrichment::decode_full_puf_draws", + "expression": "profile.targets", + "expression_sha256": "709b42afefce06be1e687bee5a35285a8e0a40324e0d6510b9211e9eacdd2804", + "resolution": "unresolved-us-candidate", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.full_puf_enrichment::decode_full_puf_draws", + "expression": "profile.targets", + "expression_sha256": "709b42afefce06be1e687bee5a35285a8e0a40324e0d6510b9211e9eacdd2804", + "resolution": "unresolved-us-candidate", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.full_puf_enrichment::decode_full_puf_draws", + "expression": "profile.targets", + "expression_sha256": "709b42afefce06be1e687bee5a35285a8e0a40324e0d6510b9211e9eacdd2804", + "resolution": "unresolved-us-candidate", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.full_puf_enrichment::decode_full_puf_draws", + "expression": "profile.targets", + "expression_sha256": "709b42afefce06be1e687bee5a35285a8e0a40324e0d6510b9211e9eacdd2804", + "resolution": "unresolved-us-candidate", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.full_puf_enrichment::decode_full_puf_draws", + "expression": "profile.targets", + "expression_sha256": "709b42afefce06be1e687bee5a35285a8e0a40324e0d6510b9211e9eacdd2804", + "resolution": "unresolved-us-candidate", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.full_puf_enrichment::finalize_full_puf", + "expression": "profile.targets", + "expression_sha256": "709b42afefce06be1e687bee5a35285a8e0a40324e0d6510b9211e9eacdd2804", + "resolution": "unresolved-us-candidate", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.full_puf_enrichment::finalize_full_puf", + "expression": "profile.targets", + "expression_sha256": "709b42afefce06be1e687bee5a35285a8e0a40324e0d6510b9211e9eacdd2804", + "resolution": "unresolved-us-candidate", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.full_puf_enrichment::finalize_full_puf", + "expression": "profile.targets", + "expression_sha256": "709b42afefce06be1e687bee5a35285a8e0a40324e0d6510b9211e9eacdd2804", + "resolution": "unresolved-us-candidate", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.full_puf_enrichment::full_puf_train_apply_nodes", + "expression": "profile.targets", + "expression_sha256": "709b42afefce06be1e687bee5a35285a8e0a40324e0d6510b9211e9eacdd2804", + "resolution": "unresolved-us-candidate", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.graph_full_puf_enrichment::_checked_artifacts", + "expression": "binding.profile.targets", + "expression_sha256": "926bd12f40a8db38aee800bf99973e496f507ef16a7dad04b722b9fd0ba21ddc", + "resolution": "unresolved-us-candidate", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.graph_full_puf_enrichment::_finalized_columns", + "expression": "binding.profile.targets", + "expression_sha256": "926bd12f40a8db38aee800bf99973e496f507ef16a7dad04b722b9fd0ba21ddc", + "resolution": "unresolved-us-candidate", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.graph_full_puf_enrichment::_placement", + "expression": "binding.profile.targets", + "expression_sha256": "926bd12f40a8db38aee800bf99973e496f507ef16a7dad04b722b9fd0ba21ddc", + "resolution": "unresolved-us-candidate", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.graph_full_puf_enrichment::_profile_chain", + "expression": "profile.targets", + "expression_sha256": "709b42afefce06be1e687bee5a35285a8e0a40324e0d6510b9211e9eacdd2804", + "resolution": "unresolved-us-candidate", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.graph_full_puf_enrichment::_profile_chain", + "expression": "profile.targets", + "expression_sha256": "709b42afefce06be1e687bee5a35285a8e0a40324e0d6510b9211e9eacdd2804", + "resolution": "unresolved-us-candidate", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.graph_full_puf_enrichment::_profile_chain", + "expression": "profile.targets", + "expression_sha256": "709b42afefce06be1e687bee5a35285a8e0a40324e0d6510b9211e9eacdd2804", + "resolution": "unresolved-us-candidate", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.graph_full_puf_enrichment::_profile_chain", + "expression": "profile.targets", + "expression_sha256": "709b42afefce06be1e687bee5a35285a8e0a40324e0d6510b9211e9eacdd2804", + "resolution": "unresolved-us-candidate", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.us_late_producer_registry::TransferProducerGroup.__post_init__", + "expression": "self.targets", + "expression_sha256": "b2a179282323fd11f4e8be7f48ef2b42307f0fc4b26526e15a68cb9d13bdd2aa", + "resolution": "unresolved-us-candidate", + "usage": "reference" + } + ] + }, + "microcosm.build.us_runtime.full_puf_enrichment::PufOutputProfile.tax_unit_outputs": { + "basis": "reviewed entry/caller-chain body", + "body_sha256": "c05df11a9127aae39e1cc1a9fec0d6f33475d8b7842fc81a106125af5ef3b82c", + "references": [ + { + "caller": "microcosm.build.us_runtime.full_puf_enrichment::PufOutputProfile.targets", + "expression": "self.tax_unit_outputs", + "expression_sha256": "0ebdca1a7500fdbe29429b1b6f8337153117bb1ce44ef21902d7c82e1bde861c", + "resolution": "resolved", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.full_puf_enrichment::_validated_model_donor", + "expression": "profile.tax_unit_outputs", + "expression_sha256": "3beeec6d04a6b172e854e4789d0a8bc0c632e68d5d787063db99cbc9d3cc9143", + "resolution": "unresolved-us-candidate", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.full_puf_enrichment::canonical_full_puf_donor", + "expression": "profile.tax_unit_outputs", + "expression_sha256": "3beeec6d04a6b172e854e4789d0a8bc0c632e68d5d787063db99cbc9d3cc9143", + "resolution": "unresolved-us-candidate", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.full_puf_enrichment::canonical_full_puf_donor", + "expression": "profile.tax_unit_outputs", + "expression_sha256": "3beeec6d04a6b172e854e4789d0a8bc0c632e68d5d787063db99cbc9d3cc9143", + "resolution": "unresolved-us-candidate", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.full_puf_enrichment::canonical_full_puf_donor", + "expression": "profile.tax_unit_outputs", + "expression_sha256": "3beeec6d04a6b172e854e4789d0a8bc0c632e68d5d787063db99cbc9d3cc9143", + "resolution": "unresolved-us-candidate", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.full_puf_enrichment::finalize_full_puf", + "expression": "profile.tax_unit_outputs", + "expression_sha256": "3beeec6d04a6b172e854e4789d0a8bc0c632e68d5d787063db99cbc9d3cc9143", + "resolution": "unresolved-us-candidate", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.full_puf_enrichment::finalize_full_puf", + "expression": "profile.tax_unit_outputs", + "expression_sha256": "3beeec6d04a6b172e854e4789d0a8bc0c632e68d5d787063db99cbc9d3cc9143", + "resolution": "unresolved-us-candidate", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.full_puf_enrichment::prepare_full_puf_inputs", + "expression": "profile.tax_unit_outputs", + "expression_sha256": "3beeec6d04a6b172e854e4789d0a8bc0c632e68d5d787063db99cbc9d3cc9143", + "resolution": "unresolved-us-candidate", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.graph_full_puf_enrichment::_rosters", + "expression": "profile.tax_unit_outputs", + "expression_sha256": "3beeec6d04a6b172e854e4789d0a8bc0c632e68d5d787063db99cbc9d3cc9143", + "resolution": "unresolved-us-candidate", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.puf_support::PufTaxDetailChainInputs.target_order", + "expression": "self.tax_unit_outputs", + "expression_sha256": "0ebdca1a7500fdbe29429b1b6f8337153117bb1ce44ef21902d7c82e1bde861c", + "resolution": "unresolved-us-candidate", + "usage": "reference" + } + ] + }, + "microcosm.build.us_runtime.full_puf_enrichment::_check_fitted_model_donor": { + "basis": "reviewed entry/caller-chain body", + "body_sha256": "0ba9e02181667314a139b40347fe6a2efa717a0e2ce2ccd1a85b36813c47f523", + "references": [ + { + "caller": "microcosm.build.us_runtime.full_puf_enrichment::finalize_full_puf", + "expression": "_check_fitted_model_donor(inputs.donor_frame, training_state=training_state, last_model=last_model, profile=profile)", + "expression_sha256": "766d08b83c70338fd7cd9b8611620255be2e552bc56f7228dfccc8f192486e52", + "resolution": "resolved", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.puf55_route_finalization::_finalize_puf55_routes", + "expression": "full._check_fitted_model_donor(donor_frame, training_state=values[-1], last_model=last_model, profile=values[0])", + "expression_sha256": "05f61642bd9cc2b2257d7c57e148875231f534639e50c51ea3a693c525c63cbc", + "resolution": "resolved", + "usage": "call" + } + ] + }, + "microcosm.build.us_runtime.full_puf_enrichment::_recipient_matrix": { + "basis": "finite selector/validator definition", + "body_sha256": "5f3eb714819d0cca31280bea4e8e4bfbfea6d2f0fc6dc86b8bead28eb1d08725", + "references": [ + { + "caller": "microcosm.build.us_runtime.full_puf_enrichment::prepare_full_puf_inputs", + "expression": "_recipient_matrix(frame, predictor_known=predictor_known, profile=profile)", + "expression_sha256": "2f97cff8a6253ab3f512e6a3cffbbb505c92eeda3ea5f468ab5e61f654e83870", + "resolution": "resolved", + "usage": "call" + } + ] + }, + "microcosm.build.us_runtime.full_puf_enrichment::full_puf_train_apply_nodes": { + "basis": "reviewed entry/caller-chain body", + "body_sha256": "4f57a775b615f6f3967ba0b6e201884ddd27f7f89a764de51677385afdb40d25", + "references": [ + { + "caller": "microcosm.build.us_runtime.graph_full_puf_enrichment::retain_full_puf_attachment", + "expression": "full.full_puf_train_apply_nodes(donor_population=first.population, recipient_population=population.version, matrix_producer=edge.producer, seed=first.params['seed'], n_estimators=first.params['n_estimators'], zero_atol=first.params['zero_atol'], prefix=first.id.removesuffix('.fit.000'), profile=profile)", + "expression_sha256": "cbaf62623e20ef448a77244448fe133c507e69539a8aafa67273c7eb03628045", + "resolution": "resolved", + "usage": "call" + } + ] + }, + "microcosm.build.us_runtime.full_puf_enrichment::prepare_full_puf_inputs": { + "basis": "finite selector/validator definition", + "body_sha256": "78d0ad5fa4d7fbecb70b9647e9daf199cfd7ecdeaa775a57b8502547066f28a1", + "references": [ + { + "caller": "microcosm.build.us_runtime.full_puf_enrichment::finalize_full_puf", + "expression": "prepare_full_puf_inputs(frame, donor, predictor_known=predictor_known, profile=profile)", + "expression_sha256": "289f3a8a2d774d3e413ac0add0e04e7a62908ad84e03368fca1ebeb29409c0e2", + "resolution": "resolved", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.graph_full_puf_enrichment::retain_full_puf_attachment", + "expression": "full.prepare_full_puf_inputs(population.frame, donor, predictor_known=predictor_known, profile=profile)", + "expression_sha256": "dfc75708fd8bf01f25a2a726cf59e8cbd705b44411b8b0f6c6cf67c734b87ce1", + "resolution": "resolved", + "usage": "call" + } + ] + }, + "microcosm.build.us_runtime.full_puf_enrichment::require_puf_output_profile": { + "basis": "finite selector/validator definition", + "body_sha256": "8cdc366f9bdb3adc0123f51681ec02614de8a0a4b8d27c14a6564c0648ca8acc", + "references": [ + { + "caller": "microcosm.build.us_runtime.full_puf_enrichment::_recipient_matrix", + "expression": "require_puf_output_profile(profile)", + "expression_sha256": "37aa6adc0ef10375ee2dea0467221c5286f817928d6c898714c01618221161d5", + "resolution": "resolved", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.full_puf_enrichment::canonical_full_puf_donor", + "expression": "require_puf_output_profile(profile)", + "expression_sha256": "37aa6adc0ef10375ee2dea0467221c5286f817928d6c898714c01618221161d5", + "resolution": "resolved", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.full_puf_enrichment::decode_full_puf_draws", + "expression": "require_puf_output_profile(profile)", + "expression_sha256": "37aa6adc0ef10375ee2dea0467221c5286f817928d6c898714c01618221161d5", + "resolution": "resolved", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.full_puf_enrichment::finalize_full_puf", + "expression": "require_puf_output_profile(profile)", + "expression_sha256": "37aa6adc0ef10375ee2dea0467221c5286f817928d6c898714c01618221161d5", + "resolution": "resolved", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.full_puf_enrichment::full_puf_train_apply_nodes", + "expression": "require_puf_output_profile(profile)", + "expression_sha256": "37aa6adc0ef10375ee2dea0467221c5286f817928d6c898714c01618221161d5", + "resolution": "resolved", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.full_puf_enrichment::prepare_full_puf_inputs", + "expression": "require_puf_output_profile(profile)", + "expression_sha256": "37aa6adc0ef10375ee2dea0467221c5286f817928d6c898714c01618221161d5", + "resolution": "resolved", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.graph_full_puf_enrichment::_profile_chain", + "expression": "full.require_puf_output_profile(profile)", + "expression_sha256": "9e28bf56358bbadab9cdb0a7b3006f09aba6507ad55f2b7698e43dd2a85b25dc", + "resolution": "resolved", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.graph_full_puf_enrichment::_rosters", + "expression": "full.require_puf_output_profile(profile)", + "expression_sha256": "9e28bf56358bbadab9cdb0a7b3006f09aba6507ad55f2b7698e43dd2a85b25dc", + "resolution": "resolved", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.graph_full_puf_enrichment::_scope", + "expression": "full.require_puf_output_profile(profile)", + "expression_sha256": "9e28bf56358bbadab9cdb0a7b3006f09aba6507ad55f2b7698e43dd2a85b25dc", + "resolution": "resolved", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.graph_full_puf_enrichment::retain_full_puf_attachment", + "expression": "full.require_puf_output_profile(profile)", + "expression_sha256": "9e28bf56358bbadab9cdb0a7b3006f09aba6507ad55f2b7698e43dd2a85b25dc", + "resolution": "resolved", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.graph_puf55_canonical_donor::CanonicalPuf55DonorKernel.run", + "expression": "full.require_puf_output_profile(full.PUF55_SURVEY_SS)", + "expression_sha256": "99ed77bbd808b8552095cf59904c8cede061089ffc22b15e0fc12bcb01075f4c", + "resolution": "resolved", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.graph_puf55_canonical_donor::_node", + "expression": "full.require_puf_output_profile(full.PUF55_SURVEY_SS)", + "expression_sha256": "99ed77bbd808b8552095cf59904c8cede061089ffc22b15e0fc12bcb01075f4c", + "resolution": "resolved", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.puf55_route_finalization::_finalize_puf55_routes", + "expression": "full.require_puf_output_profile(values[0])", + "expression_sha256": "445171db4ca0e33ee781957fe9bebabc64f5424b4a946c88580e8bcae30686eb", + "resolution": "resolved", + "usage": "call" + } + ] + }, + "microcosm.build.us_runtime.graph_asec_prepared::USAsecPreparedCreateKernel.run": { + "basis": "reviewed entry/caller-chain body", + "body_sha256": "22b28f35e760389ea9b397c4daa4427b6bbd0d3b0a846a4dce522feff42e9c5f", + "references": [ + { + "caller": "microcosm.build.us_runtime.graph_asec_prepared::us_asec_prepared_registry", + "expression": "USAsecPreparedCreateKernel()", + "expression_sha256": "3613fa695fe0292dc73c8b295918d77a6ba3c8fb245267df041b6d5c509f1c50", + "resolution": "kernel-construction", + "usage": "call" + } + ] + }, + "microcosm.build.us_runtime.graph_asec_prepared::us_asec_prepared_registry": { + "basis": "reviewed entry/caller-chain body", + "body_sha256": "f5130559f879ede2943677b4272b76b03684125819b2cf4297dd36c1dfa5196f", + "references": [] + }, + "microcosm.build.us_runtime.graph_atomic_survey_clone::atomic_survey_clone_nodes": { + "basis": "reviewed entry/caller-chain body", + "body_sha256": "1b54fa49b424147b298dc964e3add6e06c9aff5d3660f256a748c8638f133c4e", + "references": [] + }, + "microcosm.build.us_runtime.graph_atomic_survey_financial::AtomicSurveyFinancialRunValues.checked_view": { + "basis": "reviewed entry/caller-chain body", + "body_sha256": "868ff4515bd665adb80a98b4533fd5b8c25247e2169f9c73da0e6d623b9b0eb9", + "references": [ + { + "caller": "microcosm.build.us_runtime.survey_financial_successor::admit_survey_financial_population", + "expression": "budget.checked_view()", + "expression_sha256": "303121f839037c123a3907a3ba1100bc5c224834bb11b565bbec767a9078485e", + "resolution": "unresolved-us-candidate", + "usage": "call" + } + ] + }, + "microcosm.build.us_runtime.graph_atomic_survey_financial::_issue_run": { + "basis": "reviewed entry/caller-chain body", + "body_sha256": "4184c61c6a8fa383c25a976365d9d6f660d401d8a3ef899de42859d9a83794ab", + "references": [ + { + "caller": "microcosm.build.us_runtime.graph_atomic_survey_financial::run_atomic_survey_financial", + "expression": "_issue_run(result, preparation_entry=entry, pins=pins, n_estimators=n_estimators, demographic_conditioning=demographic_conditioning, source_keys=source_keys, keys=keys, implementations=implementations, loaded=loaded, live=live)", + "expression_sha256": "ec0b5713ea472b925c57328d7e40ab1b8b51c4a8769bfdec14549d1781f6a5db", + "resolution": "resolved", + "usage": "call" + } + ] + }, + "microcosm.build.us_runtime.graph_atomic_survey_financial::_live": { + "basis": "reviewed entry/caller-chain body", + "body_sha256": "5fe544d9d35b6bffd690a9823e7b6acbe43e0ab9fb74a5fb5de4399bb84768b7", + "references": [ + { + "caller": "microcosm.build.us_runtime.graph_atomic_survey_financial::_live", + "expression": "values.host.survey_budget._live()", + "expression_sha256": "49fa8a1ce1d97cf45132f56d284e2e1d339c3ca1440a5a36698438565d5e7900", + "resolution": "unresolved-us-candidate", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.graph_atomic_survey_financial::_pure_run", + "expression": "_live()", + "expression_sha256": "50c2974f3d31d5d26c794c3a5386f7492be19afba0fbf0f83f264e667d66f712", + "resolution": "resolved", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.graph_atomic_survey_financial::run_atomic_survey_financial", + "expression": "_live()", + "expression_sha256": "50c2974f3d31d5d26c794c3a5386f7492be19afba0fbf0f83f264e667d66f712", + "resolution": "resolved", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.graph_atomic_survey_financial::run_atomic_survey_financial", + "expression": "_live()", + "expression_sha256": "50c2974f3d31d5d26c794c3a5386f7492be19afba0fbf0f83f264e667d66f712", + "resolution": "resolved", + "usage": "call" + } + ] + }, + "microcosm.build.us_runtime.graph_atomic_survey_financial::_manifest_population_seals": { + "basis": "reviewed entry/caller-chain body", + "body_sha256": "773d0b545a5bb2db374965afb9358728549e36cfa6f8de17f875dcc82c4b4e95", + "references": [ + { + "caller": "microcosm.build.us_runtime.graph_atomic_survey_financial::_issue_run", + "expression": "_manifest_population_seals(prefix.manifest, prefix.compiled)", + "expression_sha256": "8d32a52d913c31e39106352eb3245658785cd19aae65198b2468b392eb5f8d3f", + "resolution": "resolved", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.graph_atomic_survey_financial::_issue_run", + "expression": "_manifest_population_seals(result.manifest, result.compiled)", + "expression_sha256": "cd125817e8536d7ee913dca57fcce5bad0484a8365ba06c04399219e04fa700d", + "resolution": "resolved", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.graph_atomic_survey_financial::_pure_run", + "expression": "_manifest_population_seals(run.manifest, run.compiled)", + "expression_sha256": "43bbe84df3d2db7a78b7f1d0e52fac5703a5fb31c6abf4536d3858d8d27aabd0", + "resolution": "resolved", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.graph_atomic_survey_financial::_pure_run", + "expression": "_manifest_population_seals(prefix.manifest, prefix.compiled)", + "expression_sha256": "8d32a52d913c31e39106352eb3245658785cd19aae65198b2468b392eb5f8d3f", + "resolution": "resolved", + "usage": "call" + } + ] + }, + "microcosm.build.us_runtime.graph_atomic_survey_financial::check_atomic_survey_financial_run": { + "basis": "reviewed entry/caller-chain body", + "body_sha256": "af0ba35fa13a278220ddd686be9b4ad0c11d86b5238d81a14472b9bb4c99acc1", + "references": [ + { + "caller": "microcosm.build.us_runtime.graph_atomic_survey_financial::AtomicSurveyFinancialRunValues.checked_view", + "expression": "check_atomic_survey_financial_run(self)", + "expression_sha256": "78dd800e581bbceec53b217bd8d80e32492c1f02a2987376940d2ff78b09aaf3", + "resolution": "resolved", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.graph_puf55_survey_recipients::_Kernel.run", + "expression": "financial.check_atomic_survey_financial_run(run)", + "expression_sha256": "4bf03aea6db078ce7fb3d81b66eb3acfba6ac0696eb7b1690480422f7b166519", + "resolution": "unresolved-us-candidate", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.puf55_survey_recipients::qualify_puf55_survey_recipients", + "expression": "financial.check_atomic_survey_financial_run(financial_run)", + "expression_sha256": "fcb1d411ab7b88a4565c25142b368e7841335e18d7ff6d366c20a869dca06f16", + "resolution": "resolved", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.puf55_survey_recipients::qualify_puf55_survey_recipients", + "expression": "financial.check_atomic_survey_financial_run(financial_run)", + "expression_sha256": "fcb1d411ab7b88a4565c25142b368e7841335e18d7ff6d366c20a869dca06f16", + "resolution": "resolved", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.survey_financial_successor::SamplingOriginFinancialSuccessor.checked_view", + "expression": "runner.check_atomic_survey_financial_run(state.financial_run)", + "expression_sha256": "309c45107f35d1ef168125ba94dc42d5c4a9a6686105e8960d538c8bb8c1ed29", + "resolution": "resolved", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.survey_financial_successor::admit_survey_financial_population", + "expression": "runner.check_atomic_survey_financial_run(financial_run)", + "expression_sha256": "3ce7150006eeb59674812fc47090ea157fdd2f1d6f1edcc9799f8d6f9a098275", + "resolution": "resolved", + "usage": "call" + } + ] + }, + "microcosm.build.us_runtime.graph_atomic_survey_financial::run_atomic_survey_financial": { + "basis": "reviewed entry/caller-chain body", + "body_sha256": "5c5714195b91dfcfd7ff2e8f470a00c1ab5f169a1b56187d101889ad6dbe63f6", + "references": [] + }, + "microcosm.build.us_runtime.graph_atomic_survey_financial::run_atomic_survey_financial.observe": { + "basis": "reviewed entry/caller-chain body", + "body_sha256": "935d57b08febd8fe94143f4507843c75ad6ad318d6619742de478706615fe55e", + "references": [ + { + "caller": "microcosm.build.us_runtime.graph_atomic_survey_financial::run_atomic_survey_financial", + "expression": "observe", + "expression_sha256": "274f69be16100e8e78059234777f281731e85cc7e8d8ea6f15a066520f4f89c8", + "resolution": "resolved", + "usage": "reference" + } + ] + }, + "microcosm.build.us_runtime.graph_atomic_survey_population::_clone_expectations": { + "basis": "reviewed entry/caller-chain body", + "body_sha256": "7b89a2a2034ef9b758b3d4b84026334839ee6e91de3109db7788271211323372", + "references": [ + { + "caller": "microcosm.build.us_runtime.graph_atomic_survey_financial::run_atomic_survey_financial", + "expression": "atomic._clone_expectations(geography, prefix.compiled.graph.nodes, compiled)", + "expression_sha256": "ea2e9241b6f323b1e0b69f9bf6e4f3ceb38460b5709d1c9292d458b90337f740", + "resolution": "resolved", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.graph_atomic_survey_population::run_atomic_survey_population", + "expression": "_clone_expectations(geography, additions, compiled)", + "expression_sha256": "73367c39180a8d57a514e9bf93bacb025e89512fb3be283c6fa7bf868cbc995e", + "resolution": "resolved", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.survey_age_calibration::_run_survey_age_calibration", + "expression": "atomic_source._clone_expectations(geography, prefix.compiled.graph.nodes, prefix.compiled)", + "expression_sha256": "41a49946eb5b1565540e95cffb9df97e2a1201bce3c63e762b03085ca3b0edfe", + "resolution": "resolved", + "usage": "call" + } + ] + }, + "microcosm.build.us_runtime.graph_atomic_survey_population::run_atomic_survey_population": { + "basis": "reviewed entry/caller-chain body", + "body_sha256": "73335893d7c4b41a109355868ff2a1cf75ed5b5db571e9ccdc31960db5d7021e", + "references": [ + { + "caller": "microcosm.build.us_runtime.graph_atomic_survey_financial::run_atomic_survey_financial", + "expression": "atomic.run_atomic_survey_population(source_dir, snapshot_root=snapshot_root, store_root=store_root, fraction=fraction, seed=seed, geography_config=geography_config, resume=resume, return_values=True)", + "expression_sha256": "e8dd91636ad84548883159b48148cf22ced49a519b1bc0517c29acb6baecb1bd", + "resolution": "resolved", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.survey_age_calibration::_run_survey_age_calibration", + "expression": "atomic_source.run_atomic_survey_population(source_dir, snapshot_root=snapshot_root, store_root=store_root, fraction=fraction, seed=seed_value, resume=resume, geography_config=geography_config, return_values=True)", + "expression_sha256": "e00bf666e872beff96da191d597cb8b8b7e3cc0694eb1b0c52b2a93320449a62", + "resolution": "resolved", + "usage": "call" + } + ] + }, + "microcosm.build.us_runtime.graph_combined_clone::USCombinedSurveyCloneExpandKernel._receipt": { + "basis": "reviewed entry/caller-chain body", + "body_sha256": "cfbfab004a3a24ee9e1a7c2873658f7eebf99ff8f52a5bafc43a2a9425927dc4", + "references": [ + { + "caller": "microcosm.build.us_runtime.asec_current_money_units::CurrentMoneyTaxUnitResult.receipt", + "expression": "self._receipt", + "expression_sha256": "a5200a0f59f87b2ad3546c02313eb32aeab5d9209bdb2da1a51715aa694653a7", + "resolution": "unresolved-us-candidate", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.asec_demographic_source::DemographicSourceAttachedAsec.receipt", + "expression": "self._receipt", + "expression_sha256": "a5200a0f59f87b2ad3546c02313eb32aeab5d9209bdb2da1a51715aa694653a7", + "resolution": "unresolved-us-candidate", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.asec_housing_status_source::HousingStatusAttachedAsec.receipt", + "expression": "self._receipt", + "expression_sha256": "a5200a0f59f87b2ad3546c02313eb32aeab5d9209bdb2da1a51715aa694653a7", + "resolution": "unresolved-us-candidate", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.asec_housing_universe_source::HousingUniverseAttachedAsec.receipt", + "expression": "self._receipt", + "expression_sha256": "a5200a0f59f87b2ad3546c02313eb32aeab5d9209bdb2da1a51715aa694653a7", + "resolution": "unresolved-us-candidate", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.asec_student_controls::StudentControlsAttachedAsec.receipt", + "expression": "self._receipt", + "expression_sha256": "a5200a0f59f87b2ad3546c02313eb32aeab5d9209bdb2da1a51715aa694653a7", + "resolution": "unresolved-us-candidate", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.graph_atomic_survey_population::_clone_expectations", + "expression": "clone.USCombinedSurveyCloneExpandKernel._receipt(before.frame, expanded, ('acs', 'asec'), authority, facts)", + "expression_sha256": "f36da898c4e1606b3f506f707adb5322ba2db6d5f3fb868fa2af26395093de4c", + "resolution": "resolved", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.graph_combined_clone::USCombinedSurveyCloneExpandKernel.run", + "expression": "self._receipt(before, after, channels, authority, entity_facts)", + "expression_sha256": "4039fb5d88a00d42cb042e483e6e3cbf8a271d42d072cf60041fbb9ceb3758c2", + "resolution": "resolved", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.graph_survey_population::run_authenticated_survey_population.observe", + "expression": "clone.USCombinedSurveyCloneExpandKernel._receipt(allocated, population.frame, ('acs', 'asec'), authority, entity_facts)", + "expression_sha256": "e12f7f43dd7be48526ce4245a187bd4188121a178fc641efe3f1844602252cb8", + "resolution": "resolved", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.puf_growth::GrownPufTable.receipt", + "expression": "self._receipt", + "expression_sha256": "a5200a0f59f87b2ad3546c02313eb32aeab5d9209bdb2da1a51715aa694653a7", + "resolution": "unresolved-us-candidate", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.puf_growth::GrownPufTable.receipt_bytes", + "expression": "self._receipt", + "expression_sha256": "a5200a0f59f87b2ad3546c02313eb32aeab5d9209bdb2da1a51715aa694653a7", + "resolution": "unresolved-us-candidate", + "usage": "reference" + } + ] + }, + "microcosm.build.us_runtime.graph_combined_clone::USCombinedSurveyCloneExpandKernel._validated_declaration": { + "basis": "reviewed entry/caller-chain body", + "body_sha256": "6f2dfcdbc03e3494692f489e161ef8078de63d4ad24d58fbf329673a1ca4ce5d", + "references": [ + { + "caller": "microcosm.build.us_runtime.graph_combined_clone::USCombinedSurveyCloneExpandKernel.run", + "expression": "self._validated_declaration(context)", + "expression_sha256": "9ede2962084aec42b87ad3877b05f3fdcabaf19497540284454e582be35efc16", + "resolution": "resolved", + "usage": "call" + } + ] + }, + "microcosm.build.us_runtime.graph_combined_clone::USCombinedSurveyCloneExpandKernel.run": { + "basis": "reviewed entry/caller-chain body", + "body_sha256": "ceb80c33ab2f625bd9b31d999de630c4d02c5f000b25519aff10a26a5550cc12", + "references": [ + { + "caller": "microcosm.build.us_runtime.graph_combined_clone::register_us_combined_survey_clone_kernels", + "expression": "USCombinedSurveyCloneExpandKernel()", + "expression_sha256": "15a28ee160bf86e888abc1e06a6647a9e17b5c36cd519466ea01481718953f68", + "resolution": "kernel-construction", + "usage": "call" + } + ] + }, + "microcosm.build.us_runtime.graph_combined_clone::register_us_combined_survey_clone_kernels": { + "basis": "reviewed entry/caller-chain body", + "body_sha256": "02ae4b2f383b4cd78d7cdade2781708a82969ee0ee2b44a27aa070c9fbe79565", + "references": [ + { + "caller": "microcosm.build.us_runtime.graph_atomic_survey_population::run_atomic_survey_population", + "expression": "clone.register_us_combined_survey_clone_kernels(kernels)", + "expression_sha256": "34e95a52ead34ae46e223fbcc968d628f261e279ac20457d29004d4a1193284e", + "resolution": "resolved", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.graph_survey_population::run_authenticated_survey_population", + "expression": "register_us_combined_survey_clone_kernels(kernels)", + "expression_sha256": "27e91ca6cba74c62b3d82b12397ebdb4cbbc18a77640d4f1048a278cf205e651", + "resolution": "resolved", + "usage": "call" + } + ] + }, + "microcosm.build.us_runtime.graph_combined_clone::us_combined_survey_clone_nodes": { + "basis": "reviewed entry/caller-chain body", + "body_sha256": "2e30825f79415ea39f6b9abb5f1fed182f56471cb6b21089de913a5d96f5464b", + "references": [ + { + "caller": "microcosm.build.us_runtime.graph_atomic_survey_clone::atomic_survey_clone_nodes", + "expression": "us_combined_survey_clone_nodes((*columns, *(output for node in geography for output in node.outputs)), base=base, source_channels=('acs', 'asec'), prefix=clone_prefix)", + "expression_sha256": "3a132ecbc1614ec5900fc35270be6c9fe077e8ba351afb418b12e2d1b8b47997", + "resolution": "resolved", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.graph_atomic_survey_population::run_atomic_survey_population", + "expression": "clone.us_combined_survey_clone_nodes(frame_column_declarations(geography.population.frame), base=survey.ALLOCATION_NODE, source_channels=('acs', 'asec'))", + "expression_sha256": "979b2300b0e7750ad162c4abda539b1cc38685626ccd5227e39aafd58f0d7abb", + "resolution": "resolved", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.graph_survey_population::run_authenticated_survey_population", + "expression": "us_combined_survey_clone_nodes(columns, base=ALLOCATION_NODE, source_channels=('acs', 'asec'))", + "expression_sha256": "7d3319ddc540e253b8090cb892d9338f1956a42417fb4593fea8dfcc44d712b7", + "resolution": "resolved", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.survey_origin_budget::_initial", + "expression": "clone.us_combined_survey_clone_nodes(columns, base=graph.ALLOCATION_NODE, source_channels=('acs', 'asec'))", + "expression_sha256": "42b17f59e8b78b82d0047af4cfa3f7d1998c1bad522689fb267cf303c500e9e3", + "resolution": "resolved", + "usage": "call" + } + ] + }, + "microcosm.build.us_runtime.graph_composed_asec_binding::USComposedAsecBindKernel.run": { + "basis": "reviewed entry/caller-chain body", + "body_sha256": "ff82b233bec87157744a722779ae1db8d2d57abb0156f007088a01d31018113b", + "references": [ + { + "caller": "microcosm.build.us_runtime.graph_composed_asec_measures::composed_asec_population_registry", + "expression": "USComposedAsecBindKernel()", + "expression_sha256": "35521769cc710664bce28d222040ee536fa710fb98d3d63bbd7e2b84fc6b10cf", + "resolution": "kernel-construction", + "usage": "call" + } + ] + }, + "microcosm.build.us_runtime.graph_composed_asec_binding::composed_asec_bind_node": { + "basis": "reviewed entry/caller-chain body", + "body_sha256": "e9e4863840890b9a75eda25d8c96288b6eb993cb0255eafc0bdfa5a9709c8aa6", + "references": [ + { + "caller": "microcosm.build.us_runtime.graph_composed_asec_measures::composed_asec_measure_nodes", + "expression": "composed_asec_bind_node(population=population, population_context=population_context)", + "expression_sha256": "b4bc74c32c1b059842fdc882c7db0b18796635f669d3494719255fa8e0bef96b", + "resolution": "resolved", + "usage": "call" + } + ] + }, + "microcosm.build.us_runtime.graph_composed_asec_binding::resolve_composed_asec_binding": { + "basis": "reviewed entry/caller-chain body", + "body_sha256": "080dc843e752f7b119cd57fba923603cc73f709a7c51f0902bef06a5ac8b623d", + "references": [ + { + "caller": "microcosm.build.us_runtime.graph_composed_asec_binding::USComposedAsecBindKernel.run", + "expression": "resolve_composed_asec_binding({entity: context.tables[entity] for entity in US_SCHEMA.entities}, context_payload=values['frame_context'].payload, origin_payload=values['source_origin'].payload, prepared_context_payload=values['asec_frame_context'].payload, money_payload=values['current_money'].payload, receipt_payload=values['prepared_receipt'].payload, housing_payload=values['housing_universe'].payload, income_payload=values['income_observations'].payload, producers={'population': values['frame_context'].producer_key, 'prepared_source': create_producer, 'source_origin': values['source_origin'].producer_key})", + "expression_sha256": "311086079b5332ed51f5d8905c39c5730c5eec13315f9b02ad8d416a1a773181", + "resolution": "resolved", + "usage": "call" + } + ] + }, + "microcosm.build.us_runtime.graph_composed_asec_measures::composed_asec_measure_nodes": { + "basis": "reviewed entry/caller-chain body", + "body_sha256": "2b10bba21ec46537ea1dd33e68d6643db34a4af3df5d1f47f2576b11ce26ee4a", + "references": [ + { + "caller": "microcosm.build.us_runtime.graph_composed_asec_measures::composed_asec_population_graph", + "expression": "composed_asec_measure_nodes(population=f'{GEOGRAPHY_PHASE}.boundary', population_context=GEOGRAPHY_PHASE)", + "expression_sha256": "743768940d332a0d2b24971069395f7e2895da42a004b306143ec02d10cf841d", + "resolution": "resolved", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.graph_composed_asec_measures::composed_asec_population_graph", + "expression": "composed_asec_measure_nodes(population=HARMONIZE_NODE, population_context=HARMONIZE_NODE)", + "expression_sha256": "84fc8d4275b223fd937426fb2a1650d410f9504044a9f43ca03b571c9a4f342f", + "resolution": "resolved", + "usage": "call" + } + ] + }, + "microcosm.build.us_runtime.graph_composed_asec_measures::composed_asec_population_graph": { + "basis": "reviewed entry/caller-chain body", + "body_sha256": "8718150f41be52336ab429871a7f1e9c1e37b21fc191828fc08070ef6175a22f", + "references": [] + }, + "microcosm.build.us_runtime.graph_composed_asec_measures::composed_asec_population_registry": { + "basis": "reviewed entry/caller-chain body", + "body_sha256": "5f6fac89b488eb2e69b2b59b3d338662d5b2a0895e038c4f11ec8d0dcb6c7014", + "references": [] + }, + "microcosm.build.us_runtime.graph_composed_population::USComposedPopulationCreateKernel.implementation_hash": { + "basis": "reviewed entry/caller-chain body", + "body_sha256": "35f6136c5ccc90883489a51d7adc072d66308c980e75bf22018fcabf66c727f4", + "references": [ + { + "caller": "microcosm.build.us_runtime.graph_composed_population::USComposedPopulationCreateKernel.implementation_hash", + "expression": "implementation_hash(COMPOSED_STAGE)", + "expression_sha256": "551d9cdb42abbb55e67203f1017152666c2e1ef3f43ec8f91530dd575b765c45", + "resolution": "resolved", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.graph_puf55_survey_recipients::_Kernel.implementation_hash", + "expression": "source_graph._Kernel.implementation_hash(self)", + "expression_sha256": "66602d44bf4a2b5f9a0f59a08aab9a96f85070ccc4a0cb5a2528724318877495", + "resolution": "unresolved-us-candidate", + "usage": "call" + } + ] + }, + "microcosm.build.us_runtime.graph_composed_population::USComposedPopulationCreateKernel.run": { + "basis": "reviewed entry/caller-chain body", + "body_sha256": "6392e23abbc623fb19af10c5c8fe5ff513646d712d685412f3396b685ea3dfee", + "references": [ + { + "caller": "microcosm.build.us_runtime.graph_composed_population::composed_population_registry", + "expression": "USComposedPopulationCreateKernel()", + "expression_sha256": "e4e8163b38a16ace7a6347ad46595aa3007b8fd2604883d3c2922e613d78194c", + "resolution": "kernel-construction", + "usage": "call" + } + ] + }, + "microcosm.build.us_runtime.graph_composed_population::compose_from_sources": { + "basis": "reviewed entry/caller-chain body", + "body_sha256": "4dfd9d1fdb672dc597dbd9bd5445d0fb39027e68219fb2d96c820250de825368", + "references": [ + { + "caller": "microcosm.build.us_runtime.graph_composed_population::USComposedPopulationCreateKernel.run", + "expression": "compose_from_sources(context.sources[ASEC_PREPARED_SOURCE_NAME], context.sources[ACS_NATIVE_SOURCE_NAME], sample_fraction=context.params['sample_fraction'], sample_seed=context.params['sample_seed'])", + "expression_sha256": "a7067d0f6fce834eac67e66f28bccd6e596e89cf3de160c434f537e8c22a3905", + "resolution": "resolved", + "usage": "call" + } + ] + }, + "microcosm.build.us_runtime.graph_composed_population::composed_population_graph": { + "basis": "reviewed entry/caller-chain body", + "body_sha256": "d9eef4c0621a7bfa26d56bd0a0103a636a9e4b2f4cd3fde02506f52f9d5b7c00", + "references": [] + }, + "microcosm.build.us_runtime.graph_composed_population::composed_population_registry": { + "basis": "reviewed entry/caller-chain body", + "body_sha256": "770efe842df206909651279eb42fb1c2bdff7d986248359c8a8431311fff9f0e", + "references": [ + { + "caller": "microcosm.build.us_runtime.graph_composed_asec_measures::composed_asec_population_registry", + "expression": "composed_population_registry(geography=geography)", + "expression_sha256": "2be714e0c950e65d85598d8376f8213fbb73833b035af186f39dae53f0c6ecfb", + "resolution": "resolved", + "usage": "call" + } + ] + }, + "microcosm.build.us_runtime.graph_current_survey_geography::CurrentSurveyGeographyKernel.run": { + "basis": "reviewed entry/caller-chain body", + "body_sha256": "aa4a82a275c10efd745ae43f49c9ba7cdfcc22b8d98903c9f629f5baa92fa35d", + "references": [ + { + "caller": "microcosm.build.us_runtime.graph_atomic_survey_population::run_atomic_survey_population", + "expression": "projection.CurrentSurveyGeographyKernel(prefix.preparation)", + "expression_sha256": "b6b17be5b4e11fe99ae6b92cca4de51a3b4475e2d276185f3141704c1c08bd04", + "resolution": "kernel-construction", + "usage": "call" + } + ] + }, + "microcosm.build.us_runtime.graph_current_survey_geography::_check_context": { + "basis": "reviewed entry/caller-chain body", + "body_sha256": "5c7a34c0a405b11a0c5fcca299fd51f9cc4552c299f9a46bec07ab814aca7155", + "references": [ + { + "caller": "microcosm.build.us_runtime.graph_current_survey_geography::CurrentSurveyGeographyKernel.run", + "expression": "_check_context(context, payload=payload, source_frame=state.frame, receipt_sha256=receipt_sha256)", + "expression_sha256": "e395054d57ba5d10584fe8b14f414426809f02e0545c67dddd6b1ab194014056", + "resolution": "resolved", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.graph_current_survey_geography::CurrentSurveyGeographyKernel.run", + "expression": "_check_context(context, payload=payload, source_frame=state.frame, receipt_sha256=receipt_sha256)", + "expression_sha256": "e395054d57ba5d10584fe8b14f414426809f02e0545c67dddd6b1ab194014056", + "resolution": "resolved", + "usage": "call" + } + ] + }, + "microcosm.build.us_runtime.graph_current_survey_geography::current_survey_geography_node": { + "basis": "reviewed entry/caller-chain body", + "body_sha256": "5f2a0be82101b172941a96406444146a2c971b1f2e629fdca417e7c9470adcdc", + "references": [ + { + "caller": "microcosm.build.us_runtime.graph_current_survey_geography::_check_context", + "expression": "current_survey_geography_node(preparation_sha256=source_graph._sha(payload), projection_receipt_sha256=receipt_sha256, population=context.node.population, node_id=context.node.id)", + "expression_sha256": "906960aba992bb96c6200a88a3d4309174a6ef15611fd7401d2c74a44efc53a8", + "resolution": "resolved", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.survey_atomic_geography::reconstruct_atomic_survey_geography", + "expression": "observed_graph.current_survey_geography_node(preparation_sha256=_sha(payload), projection_receipt_sha256=_sha(projection_receipt), population=graph.ALLOCATION_NODE)", + "expression_sha256": "6a21f534c6c8f3c9eb17387bee7fac50c905621dc1acad959ffa6b51b707a30c", + "resolution": "resolved", + "usage": "call" + } + ] + }, + "microcosm.build.us_runtime.graph_current_survey_predictors::CurrentSurveyPredictorAttachKernel.run": { + "basis": "reviewed entry/caller-chain body", + "body_sha256": "e220624a64261fda02bf44b942db98d4a7adac0075c1246243ceca4a75451efa", + "references": [] + }, + "microcosm.build.us_runtime.graph_current_survey_predictors::CurrentSurveyPredictorDonorColumnsKernel.run": { + "basis": "reviewed entry/caller-chain body", + "body_sha256": "b97d6f733b74afba512e7f5ad310bd90e2622b3b7045efb827a9a3fce29e3099", + "references": [] + }, + "microcosm.build.us_runtime.graph_current_survey_predictors::CurrentSurveyPredictorProjectionKernel.run": { + "basis": "reviewed entry/caller-chain body", + "body_sha256": "e2a8b2ec222a57a8c8c376804b24587e77911fad3e5560fb76c01d0598d2aa16", + "references": [] + }, + "microcosm.build.us_runtime.graph_current_survey_predictors::_Kernel._qualified": { + "basis": "reviewed entry/caller-chain body", + "body_sha256": "83b50c8ffc2d1ac0a4f320804668a17e3bfd07cf0f2d46a6a2fae8544e94d149", + "references": [ + { + "caller": "microcosm.build.us_runtime.graph_current_survey_predictors::CurrentSurveyPredictorAttachKernel.run", + "expression": "self._qualified(context)", + "expression_sha256": "64795f8787d490afab7edc8bf2087e76212f5fa8aceddfc6349e3ece44cb1be1", + "resolution": "unresolved-us-candidate", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.graph_current_survey_predictors::CurrentSurveyPredictorDonorColumnsKernel.run", + "expression": "self._qualified(context)", + "expression_sha256": "64795f8787d490afab7edc8bf2087e76212f5fa8aceddfc6349e3ece44cb1be1", + "resolution": "unresolved-us-candidate", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.graph_current_survey_predictors::CurrentSurveyPredictorDonorFilterKernel.run", + "expression": "self._qualified(context)", + "expression_sha256": "64795f8787d490afab7edc8bf2087e76212f5fa8aceddfc6349e3ece44cb1be1", + "resolution": "unresolved-us-candidate", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.graph_current_survey_predictors::CurrentSurveyPredictorProjectionKernel.run", + "expression": "self._qualified(context)", + "expression_sha256": "64795f8787d490afab7edc8bf2087e76212f5fa8aceddfc6349e3ece44cb1be1", + "resolution": "unresolved-us-candidate", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.graph_current_survey_puf_transfer::CurrentSurveyAttachKernel.run", + "expression": "self._qualified(context)", + "expression_sha256": "64795f8787d490afab7edc8bf2087e76212f5fa8aceddfc6349e3ece44cb1be1", + "resolution": "unresolved-us-candidate", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.graph_current_survey_puf_transfer::CurrentSurveyPlacementKernel.run", + "expression": "self._qualified(context)", + "expression_sha256": "64795f8787d490afab7edc8bf2087e76212f5fa8aceddfc6349e3ece44cb1be1", + "resolution": "unresolved-us-candidate", + "usage": "call" + } + ] + }, + "microcosm.build.us_runtime.graph_current_survey_predictors::_Kernel.implementation_hash": { + "basis": "reviewed entry/caller-chain body", + "body_sha256": "1ec99529d0c8efb97a8ed069f09e1cd9cbe83032ee396a2cbca68ece9f61e3fc", + "references": [ + { + "caller": "microcosm.build.us_runtime.graph_puf55_survey_recipients::_Kernel.implementation_hash", + "expression": "source_graph._Kernel.implementation_hash(self)", + "expression_sha256": "66602d44bf4a2b5f9a0f59a08aab9a96f85070ccc4a0cb5a2528724318877495", + "resolution": "unresolved-us-candidate", + "usage": "call" + } + ] + }, + "microcosm.build.us_runtime.graph_current_survey_predictors::verify_materialized_current_survey_predictors": { + "basis": "reviewed entry/caller-chain body", + "body_sha256": "bb658f0357f6cb5789fe570cb4524edaa5051fafd0b737c12405f3ff35eb44f5", + "references": [ + { + "caller": "microcosm.build.us_runtime.graph_atomic_survey_financial::check_atomic_survey_financial_run", + "expression": "financial.verify_materialized_current_survey_predictors(prefix.preparation, prefix.allocated_population, prefix.clone_population, population=run.financial_population, projection=state.projection, matrix=state.matrix, matrix_producer_key=keys[financial.PROJECTION_NODE], raw_draws=tuple((loaded[f'{financial.APPLY_PREFIX}.{i:03d}', 'raw_draw'] for i in range(3))), apply_states=tuple((loaded[f'{financial.APPLY_PREFIX}.{i:03d}', 'apply_state'] for i in range(3))), host_pins=codec.decode_json(state.pins), n_estimators=state.n_estimators, demographic_conditioning=state.demographic_conditioning, geography_config=prefix.geography_config)", + "expression_sha256": "67b4410e0e510a350bf94b3b706ba4804b26bd4f8f93c6fe22971fbe7af44203", + "resolution": "resolved", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.graph_atomic_survey_financial::run_atomic_survey_financial", + "expression": "financial.verify_materialized_current_survey_predictors(prefix.preparation, prefix.allocated_population, prefix.clone_population, population=result.financial_population, projection=projection_bytes, matrix=matrix_bytes, matrix_producer_key=matrix_key, raw_draws=raw, apply_states=applications, host_pins=pins, n_estimators=n_estimators, demographic_conditioning=demographic_conditioning, geography_config=geography_config)", + "expression_sha256": "829d7c1cfdbce89a7f27d1b18afce23476e1f3072e1981ac30d0a5531c54fa76", + "resolution": "resolved", + "usage": "call" + } + ] + }, + "microcosm.build.us_runtime.graph_current_survey_puf_transfer::CurrentSurveyAttachKernel.run": { + "basis": "reviewed entry/caller-chain body", + "body_sha256": "6e2974dd037214f4dc27b5b6aef5c3595c3cb04cbb769086158a1cfdc677566e", + "references": [] + }, + "microcosm.build.us_runtime.graph_current_survey_puf_transfer::CurrentSurveyPlacementKernel.run": { + "basis": "reviewed entry/caller-chain body", + "body_sha256": "bc4cbae57f7be1b340bf68e15451f4c5f58870083f87b4c963c7ab2b27d3532d", + "references": [] + }, + "microcosm.build.us_runtime.graph_current_survey_puf_transfer::_CurrentTransferKernel._qualified": { + "basis": "reviewed entry/caller-chain body", + "body_sha256": "6ec89e1867384b56af9f2dcc5fad51f38e4b651c76380fdd135936151ae39e4b", + "references": [ + { + "caller": "microcosm.build.us_runtime.graph_current_survey_predictors::CurrentSurveyPredictorAttachKernel.run", + "expression": "self._qualified(context)", + "expression_sha256": "64795f8787d490afab7edc8bf2087e76212f5fa8aceddfc6349e3ece44cb1be1", + "resolution": "unresolved-us-candidate", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.graph_current_survey_predictors::CurrentSurveyPredictorDonorColumnsKernel.run", + "expression": "self._qualified(context)", + "expression_sha256": "64795f8787d490afab7edc8bf2087e76212f5fa8aceddfc6349e3ece44cb1be1", + "resolution": "unresolved-us-candidate", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.graph_current_survey_predictors::CurrentSurveyPredictorDonorFilterKernel.run", + "expression": "self._qualified(context)", + "expression_sha256": "64795f8787d490afab7edc8bf2087e76212f5fa8aceddfc6349e3ece44cb1be1", + "resolution": "unresolved-us-candidate", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.graph_current_survey_predictors::CurrentSurveyPredictorProjectionKernel.run", + "expression": "self._qualified(context)", + "expression_sha256": "64795f8787d490afab7edc8bf2087e76212f5fa8aceddfc6349e3ece44cb1be1", + "resolution": "unresolved-us-candidate", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.graph_current_survey_puf_transfer::CurrentSurveyAttachKernel.run", + "expression": "self._qualified(context)", + "expression_sha256": "64795f8787d490afab7edc8bf2087e76212f5fa8aceddfc6349e3ece44cb1be1", + "resolution": "unresolved-us-candidate", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.graph_current_survey_puf_transfer::CurrentSurveyPlacementKernel.run", + "expression": "self._qualified(context)", + "expression_sha256": "64795f8787d490afab7edc8bf2087e76212f5fa8aceddfc6349e3ece44cb1be1", + "resolution": "unresolved-us-candidate", + "usage": "call" + } + ] + }, + "microcosm.build.us_runtime.graph_current_survey_puf_transfer::verify_materialized_current_survey_transfer": { + "basis": "reviewed entry/caller-chain body", + "body_sha256": "418916664aebf712c83a510012d09497d08c140f6238d63a94b52d9ab2c86b81", + "references": [] + }, + "microcosm.build.us_runtime.graph_full_puf_enrichment::FullPufAttachKernel.run": { + "basis": "reviewed entry/caller-chain body", + "body_sha256": "71e8a1efb3d817af246d84b9eb899fa859e30f2484a3b8eb89c10c8de97f5668", + "references": [ + { + "caller": "microcosm.build.us_runtime.graph_full_puf_enrichment::_manifest_contract", + "expression": "FullPufAttachKernel(binding)", + "expression_sha256": "c946c106ea47b28d8d3b59f89a4751e4621300da04e8f5f4958e7ea07c2c538b", + "resolution": "kernel-construction", + "usage": "call" + } + ] + }, + "microcosm.build.us_runtime.graph_full_puf_enrichment::FullPufAttachmentBinding": { + "basis": "reviewed entry/caller-chain body", + "body_sha256": "6293a0f4e444693bf1d1f27ccddcc7a1d27f8823da4ed4221b6c2e4f67544e80", + "references": [ + { + "caller": "microcosm.build.us_runtime.graph_full_puf_enrichment::FullPufAttachmentBinding.check", + "expression": "self", + "expression_sha256": "7ed444d38b2676027d24aa06f77919b33308ad3d2d830e88198c8b769cbdf8dd", + "resolution": "resolved", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.graph_full_puf_enrichment::FullPufAttachmentBinding.check", + "expression": "self", + "expression_sha256": "7ed444d38b2676027d24aa06f77919b33308ad3d2d830e88198c8b769cbdf8dd", + "resolution": "resolved", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.graph_full_puf_enrichment::FullPufAttachmentBinding.check", + "expression": "self", + "expression_sha256": "7ed444d38b2676027d24aa06f77919b33308ad3d2d830e88198c8b769cbdf8dd", + "resolution": "resolved", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.graph_full_puf_enrichment::FullPufAttachmentBinding.check", + "expression": "self", + "expression_sha256": "7ed444d38b2676027d24aa06f77919b33308ad3d2d830e88198c8b769cbdf8dd", + "resolution": "resolved", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.graph_full_puf_enrichment::FullPufAttachmentBinding.check", + "expression": "self", + "expression_sha256": "7ed444d38b2676027d24aa06f77919b33308ad3d2d830e88198c8b769cbdf8dd", + "resolution": "resolved", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.graph_full_puf_enrichment::FullPufAttachmentBinding.check", + "expression": "self", + "expression_sha256": "7ed444d38b2676027d24aa06f77919b33308ad3d2d830e88198c8b769cbdf8dd", + "resolution": "resolved", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.graph_full_puf_enrichment::FullPufAttachmentBinding.check", + "expression": "self", + "expression_sha256": "7ed444d38b2676027d24aa06f77919b33308ad3d2d830e88198c8b769cbdf8dd", + "resolution": "resolved", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.graph_full_puf_enrichment::FullPufAttachmentBinding.check", + "expression": "self", + "expression_sha256": "7ed444d38b2676027d24aa06f77919b33308ad3d2d830e88198c8b769cbdf8dd", + "resolution": "resolved", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.graph_full_puf_enrichment::FullPufAttachmentBinding.check", + "expression": "self", + "expression_sha256": "7ed444d38b2676027d24aa06f77919b33308ad3d2d830e88198c8b769cbdf8dd", + "resolution": "resolved", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.graph_full_puf_enrichment::FullPufAttachmentBinding.check", + "expression": "self", + "expression_sha256": "7ed444d38b2676027d24aa06f77919b33308ad3d2d830e88198c8b769cbdf8dd", + "resolution": "resolved", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.graph_full_puf_enrichment::FullPufAttachmentBinding.check", + "expression": "self", + "expression_sha256": "7ed444d38b2676027d24aa06f77919b33308ad3d2d830e88198c8b769cbdf8dd", + "resolution": "resolved", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.graph_full_puf_enrichment::FullPufAttachmentBinding.check", + "expression": "self", + "expression_sha256": "7ed444d38b2676027d24aa06f77919b33308ad3d2d830e88198c8b769cbdf8dd", + "resolution": "resolved", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.graph_full_puf_enrichment::FullPufAttachmentBinding.check", + "expression": "self", + "expression_sha256": "7ed444d38b2676027d24aa06f77919b33308ad3d2d830e88198c8b769cbdf8dd", + "resolution": "resolved", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.graph_full_puf_enrichment::retain_full_puf_attachment", + "expression": "FullPufAttachmentBinding(population, expected_population, population_node, MappingProxyType(dict(input_owners)), donor, predictor_known, matrix, edge, fit_nodes, apply_nodes, prefix, _population_stamp(population), _population_stamp(expected_population), _table_stamp(donor), _table_stamp(predictor_known), profile)", + "expression_sha256": "e02298b8ebf8b6ed3076b4e05dac4691904e0454c259b733b50f31d460ae6987", + "resolution": "resolved", + "usage": "call" + } + ] + }, + "microcosm.build.us_runtime.graph_full_puf_enrichment::FullPufAttachmentBinding.check": { + "basis": "reviewed entry/caller-chain body", + "body_sha256": "974dc305c2f561a5644a0ea7a613050efac3884bd0a02fb94a3b38b1385016a0", + "references": [ + { + "caller": "microcosm.build.us_runtime.graph_full_puf_enrichment::_finalized_columns", + "expression": "binding.check()", + "expression_sha256": "8f0b002cc775c62b23467fc43b4dcdd594286e0cee46df8786bbc7dc05b758a9", + "resolution": "unresolved-us-candidate", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.graph_full_puf_enrichment::full_puf_attachment_nodes", + "expression": "binding.check()", + "expression_sha256": "8f0b002cc775c62b23467fc43b4dcdd594286e0cee46df8786bbc7dc05b758a9", + "resolution": "unresolved-us-candidate", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.graph_full_puf_enrichment::retain_full_puf_attachment", + "expression": "binding.check()", + "expression_sha256": "8f0b002cc775c62b23467fc43b4dcdd594286e0cee46df8786bbc7dc05b758a9", + "resolution": "unresolved-us-candidate", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.graph_full_puf_enrichment::verify_materialized_full_puf_attachment", + "expression": "binding.check()", + "expression_sha256": "8f0b002cc775c62b23467fc43b4dcdd594286e0cee46df8786bbc7dc05b758a9", + "resolution": "unresolved-us-candidate", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.graph_full_puf_enrichment::verify_materialized_full_puf_attachment", + "expression": "binding.check()", + "expression_sha256": "8f0b002cc775c62b23467fc43b4dcdd594286e0cee46df8786bbc7dc05b758a9", + "resolution": "unresolved-us-candidate", + "usage": "call" + } + ] + }, + "microcosm.build.us_runtime.graph_full_puf_enrichment::FullPufMaskKernel.run": { + "basis": "reviewed entry/caller-chain body", + "body_sha256": "a6140634387cea059044e94827c55cb00d07b4529e431908b9eab15e9bbe22d7", + "references": [ + { + "caller": "microcosm.build.us_runtime.graph_full_puf_enrichment::_manifest_contract", + "expression": "FullPufMaskKernel(binding)", + "expression_sha256": "72ad7bee76003f99e43ba7326a33877991647bcc66e08445d14f6a2c0f4f5ed4", + "resolution": "kernel-construction", + "usage": "call" + } + ] + }, + "microcosm.build.us_runtime.graph_full_puf_enrichment::_context_projection": { + "basis": "reviewed entry/caller-chain body", + "body_sha256": "afc3a379f5b6ad858a66155b6a735f86bd5c45e0cbc908ec0887dc9d4bb7d90b", + "references": [ + { + "caller": "microcosm.build.us_runtime.graph_full_puf_enrichment::FullPufAttachKernel.run", + "expression": "_context_projection(context, attach, expected_masked.frame)", + "expression_sha256": "589392d6fece2d4fca9d231f3454b15e991b218bf969f55ed8d3c72ca0d3d3fc", + "resolution": "resolved", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.graph_full_puf_enrichment::FullPufMaskKernel.run", + "expression": "_context_projection(context, node, binding.expected_population.frame)", + "expression_sha256": "a5dbe52c77448c16f51b17c08939951623ab7ddb2c7d824f4c8cf73bc84ab95e", + "resolution": "resolved", + "usage": "call" + } + ] + }, + "microcosm.build.us_runtime.graph_full_puf_enrichment::_inputs": { + "basis": "reviewed entry/caller-chain body", + "body_sha256": "291865866d85854aed92e76d50c888f21bf4f062a0128d5710a1223d5f820eaa", + "references": [ + { + "caller": "microcosm.build.us_runtime.graph_full_puf_enrichment::full_puf_attachment_nodes", + "expression": "_inputs(frame, masks=True, profile=binding.profile)", + "expression_sha256": "57efc3edd4c42dcf7d56f8f053fb65a2c54f5529413da7bad7370c2fda86b6c8", + "resolution": "resolved", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.graph_full_puf_enrichment::full_puf_attachment_nodes", + "expression": "_inputs(frame, profile=binding.profile)", + "expression_sha256": "632989185ad301c87ba30a4e99f6b0626161e6b3224ce8ce3f9ad34e5a0358a7", + "resolution": "resolved", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.graph_full_puf_enrichment::retain_full_puf_attachment", + "expression": "_inputs(population.frame, profile=profile)", + "expression_sha256": "fc0a0a7d900b533f6d5c3fb1eda421f200a15b06889df44439da856861387666", + "resolution": "resolved", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.graph_puf55_survey_recipients::puf55_survey_recipient_nodes", + "expression": "financial.financial._inputs(entry[2].financial_population.frame)", + "expression_sha256": "e13d917860a0b7fe5c3141fe10d1817d5695f9174086de5f694aa9f2f1350956", + "resolution": "unresolved-us-candidate", + "usage": "call" + } + ] + }, + "microcosm.build.us_runtime.graph_full_puf_enrichment::_manifest_contract": { + "basis": "reviewed entry/caller-chain body", + "body_sha256": "be2cacbc645c02590e94314ab4413d691165a21e907e49a05c15c497ecdb8e8d", + "references": [ + { + "caller": "microcosm.build.us_runtime.graph_full_puf_enrichment::load_full_puf_attachment_artifacts", + "expression": "_manifest_contract(binding, compiled, manifest)", + "expression_sha256": "a092cca4ab7f5c7623d659ce90886aa543fe98799286e4e19ac8bcbef8863756", + "resolution": "resolved", + "usage": "call" + } + ] + }, + "microcosm.build.us_runtime.graph_full_puf_enrichment::_mask_result": { + "basis": "reviewed entry/caller-chain body", + "body_sha256": "f11b1bff28ec5a6796a6f7872ab43850da3f592c2a25148b22597689a0bf5032", + "references": [ + { + "caller": "microcosm.build.us_runtime.graph_full_puf_enrichment::FullPufAttachKernel.run", + "expression": "_mask_result(binding)", + "expression_sha256": "b65c38bdb9c94b98225f8304c4d2b4fa8094bb139ba839824df8928d83c05b2e", + "resolution": "resolved", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.graph_full_puf_enrichment::FullPufMaskKernel.run", + "expression": "_mask_result(binding)", + "expression_sha256": "b65c38bdb9c94b98225f8304c4d2b4fa8094bb139ba839824df8928d83c05b2e", + "resolution": "resolved", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.graph_full_puf_enrichment::verify_materialized_full_puf_attachment", + "expression": "_mask_result(binding)", + "expression_sha256": "b65c38bdb9c94b98225f8304c4d2b4fa8094bb139ba839824df8928d83c05b2e", + "resolution": "resolved", + "usage": "call" + } + ] + }, + "microcosm.build.us_runtime.graph_full_puf_enrichment::_masks": { + "basis": "reviewed entry/caller-chain body", + "body_sha256": "6f5c189e92c4402b643be0a3631465339539c623ebc88b7236eebb1a91976e6c", + "references": [ + { + "caller": "microcosm.build.us_runtime.graph_full_puf_enrichment::_finalized_columns", + "expression": "_masks(frame)", + "expression_sha256": "35a3db044517522425fe73bc1b56cb5d001c390d02d2c2f999a63f78e8a94aae", + "resolution": "resolved", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.graph_full_puf_enrichment::_mask_result", + "expression": "_masks(frame)", + "expression_sha256": "35a3db044517522425fe73bc1b56cb5d001c390d02d2c2f999a63f78e8a94aae", + "resolution": "resolved", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.graph_full_puf_enrichment::_placement", + "expression": "_masks(frame)", + "expression_sha256": "35a3db044517522425fe73bc1b56cb5d001c390d02d2c2f999a63f78e8a94aae", + "resolution": "resolved", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.graph_full_puf_enrichment::retain_full_puf_attachment", + "expression": "_masks(population.frame)", + "expression_sha256": "467cb0971f6b11ea1a55b5435eef529da416a66881fa7644cbaace1aa83693a1", + "resolution": "resolved", + "usage": "call" + } + ] + }, + "microcosm.build.us_runtime.graph_full_puf_enrichment::_outputs": { + "basis": "reviewed entry/caller-chain body", + "body_sha256": "b179248dbf1b75fdc4d4c4ba2b12f155324f17383b3ee5059e3774bdd60bb0f1", + "references": [ + { + "caller": "microcosm.build.us_runtime.graph_full_puf_enrichment::_finalized_columns", + "expression": "_outputs(frame, profile=binding.profile)", + "expression_sha256": "57d2903f4c3a5a56555fdcaafe72645126a0645bf57a7f557c5ecf91138c1257", + "resolution": "resolved", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.graph_full_puf_enrichment::full_puf_attachment_nodes", + "expression": "_outputs(frame, profile=binding.profile)", + "expression_sha256": "57d2903f4c3a5a56555fdcaafe72645126a0645bf57a7f557c5ecf91138c1257", + "resolution": "resolved", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.graph_full_puf_enrichment::retain_full_puf_attachment", + "expression": "_outputs(population.frame, profile=profile)", + "expression_sha256": "2ec8f882ea120344f9ac80c7889e88048cad56c53b128948325719655fbf42b3", + "resolution": "resolved", + "usage": "call" + } + ] + }, + "microcosm.build.us_runtime.graph_full_puf_enrichment::_params": { + "basis": "reviewed entry/caller-chain body", + "body_sha256": "7b7053279bb636dd5412a1cf968899692de060a01bff027b0f50c4e72305cb2a", + "references": [ + { + "caller": "microcosm.build.us_runtime.graph_full_puf_enrichment::_placement", + "expression": "_params(binding)", + "expression_sha256": "6d2f42b475ebcfc24a5f69e8200585fed8f45cbd49cae87286be6425c3e9517a", + "resolution": "resolved", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.graph_full_puf_enrichment::full_puf_attachment_nodes", + "expression": "_params(binding)", + "expression_sha256": "6d2f42b475ebcfc24a5f69e8200585fed8f45cbd49cae87286be6425c3e9517a", + "resolution": "resolved", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.graph_full_puf_enrichment::full_puf_attachment_nodes", + "expression": "_params(binding)", + "expression_sha256": "6d2f42b475ebcfc24a5f69e8200585fed8f45cbd49cae87286be6425c3e9517a", + "resolution": "resolved", + "usage": "call" + } + ] + }, + "microcosm.build.us_runtime.graph_full_puf_enrichment::_placement": { + "basis": "reviewed entry/caller-chain body", + "body_sha256": "000e895a23927e69c0f26df47cd46be9f054ea78aa5c92a776c4fab27ee43d72", + "references": [ + { + "caller": "microcosm.build.us_runtime.graph_full_puf_enrichment::FullPufMaskKernel.run", + "expression": "_placement(binding)", + "expression_sha256": "34eb9e8f4fbbcd7e3417436c1706ba2e39c18941ed0ebf24310fe52c6da37e76", + "resolution": "resolved", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.graph_full_puf_enrichment::_checked_artifacts", + "expression": "_placement(binding)", + "expression_sha256": "34eb9e8f4fbbcd7e3417436c1706ba2e39c18941ed0ebf24310fe52c6da37e76", + "resolution": "resolved", + "usage": "call" + } + ] + }, + "microcosm.build.us_runtime.graph_full_puf_enrichment::_population_stamp": { + "basis": "reviewed entry/caller-chain body", + "body_sha256": "8cc9fc5cc7b493cdd7646f7d259548d1a2a6bdf3bac235b6dad7705ff8b9eabe", + "references": [ + { + "caller": "microcosm.build.us_runtime.current_survey_predictors::_qualified_seal", + "expression": "geography._population_stamp(host.survey_budget.Population.from_frame(frame, 'survey_predictors.detached_values'))", + "expression_sha256": "757a944639984ae5bab9fb445350114e5cf67bc0d4ba10e47c11cbc0ba4d9b9e", + "resolution": "unresolved-us-candidate", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.graph_atomic_survey_financial::_issue_run", + "expression": "reconstruction._population_stamp(population)", + "expression_sha256": "7806b7e36886f0090c6d84210d14645a08a0408859c9cd99b6e54054fa612518", + "resolution": "unresolved-us-candidate", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.graph_atomic_survey_financial::_manifest_population_seals", + "expression": "reconstruction._population_stamp(Population.from_frame(manifest.population(version), version, mass_ledger=manifest.mass_ledger(version)))", + "expression_sha256": "ad454b52b5ff1e0dfe94fd2d7ca87918b95075c43a1fe23fe8448e42b1182158", + "resolution": "unresolved-us-candidate", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.graph_atomic_survey_financial::_pure_run", + "expression": "reconstruction._population_stamp(actual)", + "expression_sha256": "7d5cb54ee3f56af40be19c3a07db557aa5fed9c456785a58737e82d94402a373", + "resolution": "unresolved-us-candidate", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.graph_atomic_survey_financial::run_atomic_survey_financial", + "expression": "reconstruction._population_stamp(getattr(prefix, name))", + "expression_sha256": "1dd563abd5f39f940664dcb2fe82aa82681e336cc48daa4cdcb21a45c41cec8b", + "resolution": "unresolved-us-candidate", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.graph_atomic_survey_financial::run_atomic_survey_financial", + "expression": "reconstruction._population_stamp(p)", + "expression_sha256": "444d6a47eee95054eebd7fdde9c9cb12ecacf16761e92568b30a4d019a8d16ab", + "resolution": "unresolved-us-candidate", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.graph_atomic_survey_financial::run_atomic_survey_financial", + "expression": "reconstruction._population_stamp(population)", + "expression_sha256": "7806b7e36886f0090c6d84210d14645a08a0408859c9cd99b6e54054fa612518", + "resolution": "unresolved-us-candidate", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.graph_atomic_survey_financial::run_atomic_survey_financial", + "expression": "reconstruction._population_stamp(population)", + "expression_sha256": "7806b7e36886f0090c6d84210d14645a08a0408859c9cd99b6e54054fa612518", + "resolution": "unresolved-us-candidate", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.graph_atomic_survey_financial::run_atomic_survey_financial", + "expression": "reconstruction._population_stamp(observed[node_id])", + "expression_sha256": "f61340b24375f1ace39506cda70515f3d343c7387dcf6fd9464d649061b0b9de", + "resolution": "unresolved-us-candidate", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.graph_atomic_survey_financial::run_atomic_survey_financial.observe", + "expression": "reconstruction._population_stamp(population)", + "expression_sha256": "7806b7e36886f0090c6d84210d14645a08a0408859c9cd99b6e54054fa612518", + "resolution": "unresolved-us-candidate", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.graph_full_puf_enrichment::FullPufAttachmentBinding.check", + "expression": "_population_stamp(self.population)", + "expression_sha256": "24b0aa295e7a9103699c303943db988d2732f530d423d47951633fa40745c454", + "resolution": "resolved", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.graph_full_puf_enrichment::FullPufAttachmentBinding.check", + "expression": "_population_stamp(self.expected_population)", + "expression_sha256": "b615c2f7749b2f6aae8023f074dda3623fc6a3d97b5c15286fb024ed10bad114", + "resolution": "resolved", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.graph_full_puf_enrichment::retain_full_puf_attachment", + "expression": "_population_stamp(population)", + "expression_sha256": "e8f7795a33dc8b16c80453837e4b1bd775e9fd1dde6a69119fcb55ea02430f7b", + "resolution": "resolved", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.graph_full_puf_enrichment::retain_full_puf_attachment", + "expression": "_population_stamp(expected_population)", + "expression_sha256": "eddfd1e7d4dca1a07e8bf9818ea5117f397e2845958d4175bddc3e50b9d35746", + "resolution": "resolved", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.graph_puf55_canonical_donor::_frame_seal", + "expression": "physical._population_stamp(Population.from_frame(frame, CANONICAL_DONOR_NODE))", + "expression_sha256": "f4df55a94a4fc360b01c46a6e863e4086ae402f551f6bc5556565396940c1c7e", + "resolution": "resolved", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.puf55_route_finalization::_candidate_frame_sha256", + "expression": "physical._population_stamp(physical.Population.from_frame(frame, 'puf55_route_numerical_candidate'))", + "expression_sha256": "d4223d911f87ecba1002655c8edd231da1abacc6538efe2bd864eedd761dbd56", + "resolution": "resolved", + "usage": "call" + } + ] + }, + "microcosm.build.us_runtime.graph_full_puf_enrichment::_profile_chain": { + "basis": "reviewed entry/caller-chain body", + "body_sha256": "ac4987e2f16a2b0e023938e3230bd9cef92357a5099463cdcdfbf19a636d8f44", + "references": [ + { + "caller": "microcosm.build.us_runtime.graph_full_puf_enrichment::FullPufAttachmentBinding.check", + "expression": "_profile_chain(self.profile, self.fit_nodes, self.apply_nodes)", + "expression_sha256": "57c9c8136ce19e3cfaff5dbe0b2a867ad430ce0e22c056a3d75994ce10b6d860", + "resolution": "resolved", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.graph_full_puf_enrichment::retain_full_puf_attachment", + "expression": "_profile_chain(profile, fit_nodes, apply_nodes)", + "expression_sha256": "b4c5ef4cf9d233afbc2b80537688fe7f4bd9a3670462443d4cdd9d1b8a95d986", + "resolution": "resolved", + "usage": "call" + } + ] + }, + "microcosm.build.us_runtime.graph_full_puf_enrichment::_rosters": { + "basis": "reviewed entry/caller-chain body", + "body_sha256": "3718f6a2ca2e3846bb418085d3572d5a96b1bc8abbec12d22470541026b90958", + "references": [ + { + "caller": "microcosm.build.us_runtime.graph_full_puf_enrichment::_inputs", + "expression": "_rosters(profile)", + "expression_sha256": "3c96d23a1afbfce0e34432c7fc9b273a0b2fa42d207890f4087f4faca56a0af5", + "resolution": "resolved", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.graph_full_puf_enrichment::_outputs", + "expression": "_rosters(profile)", + "expression_sha256": "3c96d23a1afbfce0e34432c7fc9b273a0b2fa42d207890f4087f4faca56a0af5", + "resolution": "resolved", + "usage": "call" + } + ] + }, + "microcosm.build.us_runtime.graph_full_puf_enrichment::_scope": { + "basis": "reviewed entry/caller-chain body", + "body_sha256": "bd29b8ab9b56e20b5c2a56b0b8e8d6d98e6c98d1e9a875c98e4fbec39643c82b", + "references": [ + { + "caller": "microcosm.build.us_runtime.graph_full_puf_enrichment::FullPufMaskKernel.run", + "expression": "_scope(binding.profile)", + "expression_sha256": "1f8538e2051175f0eee3546446164b8b5a6b05bd0bd35e67bcad626da465f9d8", + "resolution": "resolved", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.graph_full_puf_enrichment::_params", + "expression": "_scope(binding.profile)", + "expression_sha256": "1f8538e2051175f0eee3546446164b8b5a6b05bd0bd35e67bcad626da465f9d8", + "resolution": "resolved", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.graph_full_puf_enrichment::verify_materialized_full_puf_attachment", + "expression": "_scope(binding.profile)", + "expression_sha256": "1f8538e2051175f0eee3546446164b8b5a6b05bd0bd35e67bcad626da465f9d8", + "resolution": "resolved", + "usage": "call" + } + ] + }, + "microcosm.build.us_runtime.graph_full_puf_enrichment::_structural": { + "basis": "reviewed entry/caller-chain body", + "body_sha256": "a3972d085c472fbf4648e01d3ce67c3b061c6e6865013fb10305d83db835b0b1", + "references": [ + { + "caller": "microcosm.build.us_runtime.graph_full_puf_enrichment::_context_projection", + "expression": "_structural(frame, entity)", + "expression_sha256": "d4b8c658cbeeef79ca06a3fb8faa7f15b3fafd2cdbe8ca4fdf9300f98ea232b1", + "resolution": "resolved", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.graph_full_puf_enrichment::_inputs", + "expression": "_structural(frame, entity)", + "expression_sha256": "d4b8c658cbeeef79ca06a3fb8faa7f15b3fafd2cdbe8ca4fdf9300f98ea232b1", + "resolution": "resolved", + "usage": "call" + } + ] + }, + "microcosm.build.us_runtime.graph_full_puf_enrichment::full_puf_attachment_nodes": { + "basis": "reviewed entry/caller-chain body", + "body_sha256": "4e6cf7205cb8fdf0131ca5fb876cfbf7fc4ff86fb09f6f5c5082da8afeec989c", + "references": [ + { + "caller": "microcosm.build.us_runtime.graph_full_puf_enrichment::FullPufAttachKernel.run", + "expression": "full_puf_attachment_nodes(binding)", + "expression_sha256": "227a5b4b67bb18f241d6efed3ce155e87362913cdb352a1f63ad916e040481da", + "resolution": "resolved", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.graph_full_puf_enrichment::FullPufMaskKernel.run", + "expression": "full_puf_attachment_nodes(binding)", + "expression_sha256": "227a5b4b67bb18f241d6efed3ce155e87362913cdb352a1f63ad916e040481da", + "resolution": "resolved", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.graph_full_puf_enrichment::_manifest_contract", + "expression": "full_puf_attachment_nodes(binding)", + "expression_sha256": "227a5b4b67bb18f241d6efed3ce155e87362913cdb352a1f63ad916e040481da", + "resolution": "resolved", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.graph_full_puf_enrichment::load_full_puf_attachment_artifacts", + "expression": "full_puf_attachment_nodes(binding)", + "expression_sha256": "227a5b4b67bb18f241d6efed3ce155e87362913cdb352a1f63ad916e040481da", + "resolution": "resolved", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.graph_full_puf_enrichment::verify_materialized_full_puf_attachment", + "expression": "full_puf_attachment_nodes(binding)", + "expression_sha256": "227a5b4b67bb18f241d6efed3ce155e87362913cdb352a1f63ad916e040481da", + "resolution": "resolved", + "usage": "call" + } + ] + }, + "microcosm.build.us_runtime.graph_full_puf_enrichment::load_full_puf_attachment_artifacts": { + "basis": "reviewed entry/caller-chain body", + "body_sha256": "53357141f95578cdc3174e170623d91657c5d02116fa1728197ded08f6b7812e", + "references": [] + }, + "microcosm.build.us_runtime.graph_full_puf_enrichment::verify_materialized_full_puf_attachment": { + "basis": "reviewed entry/caller-chain body", + "body_sha256": "dee6cc1e375f4ad3f51634d70fa687b716de8130bc90c98ac34070d108dffc53", + "references": [] + }, + "microcosm.build.us_runtime.graph_native_household_origin::PopulationOriginBindingKernel.run": { + "basis": "reviewed entry/caller-chain body", + "body_sha256": "c318bc7a45b4dd110e30ddbd6cb54171b3c6ede817000fb7d7411c746f7e75ca", + "references": [ + { + "caller": "microcosm.build.us_runtime.graph_native_household_origin::register_native_origin_kernels", + "expression": "PopulationOriginBindingKernel()", + "expression_sha256": "528c40a820e7c222e7a0ceb2b5ccab956f59056d805cfd15d22eb4bd110de5f2", + "resolution": "kernel-construction", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.graph_puf_diagnostic_consumer::_Kernel.implementation_hash", + "expression": "origins.PopulationOriginBindingKernel()", + "expression_sha256": "5505f63852bb13223fce4942e253fc20daf5bbdf409a6d07d7824bec6a9cc2a7", + "resolution": "kernel-construction", + "usage": "call" + } + ] + }, + "microcosm.build.us_runtime.graph_native_household_origin::register_native_origin_kernels": { + "basis": "reviewed entry/caller-chain body", + "body_sha256": "ff558eac312cf97fe8d2ab3eb6c5fefea5d99daaad49bcf07d86553b7a79bda5", + "references": [] + }, + "microcosm.build.us_runtime.graph_native_household_origin::verify_materialized_population_origins": { + "basis": "reviewed entry/caller-chain body", + "body_sha256": "a7bef1650e3584226cb9704b9b666ec23b1b30d64e03dcb39b1da49198f10a2d", + "references": [ + { + "caller": "microcosm.build.us_runtime.graph_native_household_origin::verify_materialized_population_origins", + "expression": "verify_materialized_population_origins(manifest, store, population_node=parent.document['population_node'], binding_node=parent_binding_node)", + "expression_sha256": "c0db21dfb58b604588c86b14c0421c89fac03e0136a4aa4e23d80b063d6a4d25", + "resolution": "resolved", + "usage": "call" + } + ] + }, + "microcosm.build.us_runtime.graph_puf55_canonical_donor::CanonicalPuf55DonorKernel.__init__": { + "basis": "reviewed entry/caller-chain body", + "body_sha256": "f64b50a45205eff6aae3b5cd8e93677d6b50d4f8d584afef6261776b4f6664d8", + "references": [] + }, + "microcosm.build.us_runtime.graph_puf55_canonical_donor::CanonicalPuf55DonorKernel._check_state": { + "basis": "reviewed entry/caller-chain body", + "body_sha256": "58afc2365e46c9d24cf48a73cafeb53a36dad7cddc9c9a3d9031310e212075d7", + "references": [ + { + "caller": "microcosm.build.us_runtime.graph_puf55_canonical_donor::CanonicalPuf55DonorKernel.__init__", + "expression": "self._check_state(self._state)", + "expression_sha256": "0dfa9532d78ebd2c4dd935967e9a6a233095ae7f6688dab7068e0d641316da22", + "resolution": "resolved", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.graph_puf55_canonical_donor::CanonicalPuf55DonorKernel._read_resources", + "expression": "self._check_state(state)", + "expression_sha256": "6d11b042a15d921f8f89fb1fa429fb5dcadb97fe1aef7bd45144782fc83aa8c0", + "resolution": "resolved", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.graph_puf55_canonical_donor::CanonicalPuf55DonorKernel._read_sources", + "expression": "self._check_state(state)", + "expression_sha256": "6d11b042a15d921f8f89fb1fa429fb5dcadb97fe1aef7bd45144782fc83aa8c0", + "resolution": "resolved", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.graph_puf55_canonical_donor::CanonicalPuf55DonorKernel.implementation_hash", + "expression": "self._check_state(state)", + "expression_sha256": "6d11b042a15d921f8f89fb1fa429fb5dcadb97fe1aef7bd45144782fc83aa8c0", + "resolution": "resolved", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.graph_puf55_canonical_donor::CanonicalPuf55DonorKernel.implementation_hash", + "expression": "self._check_state(state)", + "expression_sha256": "6d11b042a15d921f8f89fb1fa429fb5dcadb97fe1aef7bd45144782fc83aa8c0", + "resolution": "resolved", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.graph_puf55_canonical_donor::CanonicalPuf55DonorKernel.run", + "expression": "self._check_state(state)", + "expression_sha256": "6d11b042a15d921f8f89fb1fa429fb5dcadb97fe1aef7bd45144782fc83aa8c0", + "resolution": "resolved", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.graph_puf55_canonical_donor::CanonicalPuf55DonorKernel.run", + "expression": "self._check_state(state)", + "expression_sha256": "6d11b042a15d921f8f89fb1fa429fb5dcadb97fe1aef7bd45144782fc83aa8c0", + "resolution": "resolved", + "usage": "call" + } + ] + }, + "microcosm.build.us_runtime.graph_puf55_canonical_donor::CanonicalPuf55DonorKernel._context": { + "basis": "reviewed entry/caller-chain body", + "body_sha256": "08ce4cbdf54792fda565a1c88c61b2895a9f9e8360d049d2a8248c82e588e3b5", + "references": [ + { + "caller": "microcosm.build.us_runtime.graph_puf55_canonical_donor::CanonicalPuf55DonorKernel.run", + "expression": "self._context(context, state)", + "expression_sha256": "cac33e2e7039118a438bf399e0044e12758c9242e37ab8a86ef33facc3690231", + "resolution": "resolved", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.graph_puf55_canonical_donor::CanonicalPuf55DonorKernel.run", + "expression": "self._context(context, state, paths)", + "expression_sha256": "ea9a71b2bf4efe3f3cb88d4610b19952f9f103ed392b7f3399a28112315b551c", + "resolution": "resolved", + "usage": "call" + } + ] + }, + "microcosm.build.us_runtime.graph_puf55_canonical_donor::CanonicalPuf55DonorKernel.implementation_hash": { + "basis": "reviewed entry/caller-chain body", + "body_sha256": "bb9fdc52e471db77eedb4f664b695434b6abd11930f7384d1656979436687948", + "references": [ + { + "caller": "microcosm.build.us_runtime.graph_puf55_canonical_donor::CanonicalPuf55DonorKernel.__init__", + "expression": "self.implementation_hash()", + "expression_sha256": "b53e088e47957de2cc283c6ea3b2289d9fcbdb9475ee3a4b7dee46f86d6f49ca", + "resolution": "resolved", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.graph_puf55_canonical_donor::CanonicalPuf55DonorKernel._read_resources", + "expression": "self.implementation_hash()", + "expression_sha256": "b53e088e47957de2cc283c6ea3b2289d9fcbdb9475ee3a4b7dee46f86d6f49ca", + "resolution": "resolved", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.graph_puf55_survey_recipients::_Kernel.implementation_hash", + "expression": "source_graph._Kernel.implementation_hash(self)", + "expression_sha256": "66602d44bf4a2b5f9a0f59a08aab9a96f85070ccc4a0cb5a2528724318877495", + "resolution": "unresolved-us-candidate", + "usage": "call" + } + ] + }, + "microcosm.build.us_runtime.graph_puf55_canonical_donor::CanonicalPuf55DonorKernel.run": { + "basis": "reviewed entry/caller-chain body", + "body_sha256": "5debd00f8428273b6ca8879d6f294c4f1e4e1b995027c633a70775db86541bb4", + "references": [] + }, + "microcosm.build.us_runtime.graph_puf55_canonical_donor::_definition": { + "basis": "reviewed entry/caller-chain body", + "body_sha256": "a464b4c2728ab902717f159a3eab8c5c82b08ffbc6ea33080b487fffbc30c4d0", + "references": [ + { + "caller": "microcosm.build.us_runtime.graph_puf55_canonical_donor::CanonicalPuf55DonorKernel.__init__", + "expression": "_definition(definition)", + "expression_sha256": "6f50524a71f07138c0cf1c5d1d32aaedf004204b89ac109c9b95515b900b472e", + "resolution": "resolved", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.graph_puf55_canonical_donor::CanonicalPuf55DonorKernel._check_state", + "expression": "_definition(definition)", + "expression_sha256": "6f50524a71f07138c0cf1c5d1d32aaedf004204b89ac109c9b95515b900b472e", + "resolution": "resolved", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.graph_puf55_canonical_donor::CanonicalPuf55DonorKernel._check_state", + "expression": "self._definition", + "expression_sha256": "391a3d48c252ccf052bfa639b19a8d08242dd117c1de9259b3eaea7e09a7b7a5", + "resolution": "unresolved-us-candidate", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.graph_puf55_canonical_donor::_fresh_definition", + "expression": "_definition(packaged)", + "expression_sha256": "a4afa508255ed875891bf752cb5ff00cd88ac99e2b3f0d3f047bdb23ab285324", + "resolution": "resolved", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.graph_puf55_canonical_donor::_fresh_definition", + "expression": "_definition(fixture_definition)", + "expression_sha256": "bc3a112580911e24847d0edd8f3879f22d13bcb84230654e334b0b61cbb694b3", + "resolution": "resolved", + "usage": "call" + } + ] + }, + "microcosm.build.us_runtime.graph_puf55_canonical_donor::_frame_seal": { + "basis": "reviewed entry/caller-chain body", + "body_sha256": "25e4e6450da1355ff1eb9898d86e9410e4066ff5bcbd401ce2db16520f656c01", + "references": [ + { + "caller": "microcosm.build.us_runtime.graph_puf55_canonical_donor::CanonicalPuf55DonorKernel.run", + "expression": "_frame_seal(frame)", + "expression_sha256": "92ee54d7635fdd432ee69c15963126f7cbe0fc7c79cc3f81ad8e823c248a754e", + "resolution": "resolved", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.graph_puf55_canonical_donor::CanonicalPuf55DonorKernel.run", + "expression": "_frame_seal(result.frame)", + "expression_sha256": "9ea58074a85f7657c62869e81f272a534542fe6a1bae85b70bd9e43eac3d0c1c", + "resolution": "resolved", + "usage": "call" + } + ] + }, + "microcosm.build.us_runtime.graph_puf55_canonical_donor::_fresh_definition": { + "basis": "reviewed entry/caller-chain body", + "body_sha256": "1109c15c910a48aa6188615d1754208841a18d0ba16f8ba26bceaed04c925910", + "references": [ + { + "caller": "microcosm.build.us_runtime.graph_puf55_canonical_donor::CanonicalPuf55DonorKernel.__init__", + "expression": "_fresh_definition(fixture_definition)", + "expression_sha256": "cd916300f7b62192e3aea8cce1a2bb82aeb5a8f1245d654f01afc1a4bcc71fa0", + "resolution": "resolved", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.graph_puf55_canonical_donor::canonical_puf55_donor_node", + "expression": "_fresh_definition(fixture_definition)", + "expression_sha256": "cd916300f7b62192e3aea8cce1a2bb82aeb5a8f1245d654f01afc1a4bcc71fa0", + "resolution": "resolved", + "usage": "call" + } + ] + }, + "microcosm.build.us_runtime.graph_puf55_canonical_donor::_live": { + "basis": "reviewed entry/caller-chain body", + "body_sha256": "48789b717167aeb06b3570009748aebb71a206516b53e1b86509319304ec4fb6", + "references": [ + { + "caller": "microcosm.build.us_runtime.graph_atomic_survey_financial::_live", + "expression": "values.host.survey_budget._live()", + "expression_sha256": "49fa8a1ce1d97cf45132f56d284e2e1d339c3ca1440a5a36698438565d5e7900", + "resolution": "unresolved-us-candidate", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.graph_puf55_canonical_donor::CanonicalPuf55DonorKernel.__init__", + "expression": "_live()", + "expression_sha256": "50c2974f3d31d5d26c794c3a5386f7492be19afba0fbf0f83f264e667d66f712", + "resolution": "resolved", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.graph_puf55_canonical_donor::CanonicalPuf55DonorKernel._check_state", + "expression": "_live()", + "expression_sha256": "50c2974f3d31d5d26c794c3a5386f7492be19afba0fbf0f83f264e667d66f712", + "resolution": "resolved", + "usage": "call" + } + ] + }, + "microcosm.build.us_runtime.graph_puf55_canonical_donor::_marker": { + "basis": "reviewed entry/caller-chain body", + "body_sha256": "5809e52b4e569e652da174093f73e5d4771648b551b0a3032513faa8700836b5", + "references": [ + { + "caller": "microcosm.build.us_runtime.graph_puf55_canonical_donor::_live", + "expression": "_marker(value)", + "expression_sha256": "09494a34d70b9bd93df974536e7afcb756f1ea31aa25719dde6bde473266a916", + "resolution": "resolved", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.graph_puf55_canonical_donor::_live", + "expression": "_marker(value)", + "expression_sha256": "09494a34d70b9bd93df974536e7afcb756f1ea31aa25719dde6bde473266a916", + "resolution": "resolved", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.graph_puf55_canonical_donor::_live", + "expression": "_marker(function)", + "expression_sha256": "25a15ab04c273a4f2a3aa5149ad2138b15f617af241c5b13a47672a4d27a085d", + "resolution": "resolved", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.graph_puf55_canonical_donor::_live", + "expression": "_marker(function)", + "expression_sha256": "25a15ab04c273a4f2a3aa5149ad2138b15f617af241c5b13a47672a4d27a085d", + "resolution": "resolved", + "usage": "call" + } + ] + }, + "microcosm.build.us_runtime.graph_puf55_canonical_donor::_node": { + "basis": "reviewed entry/caller-chain body", + "body_sha256": "d4bb3a355cbce2031c5b077b4e9f74f5cc9a0d64346f1a7577d77298016495f7", + "references": [ + { + "caller": "microcosm.build.us_runtime.graph_puf55_canonical_donor::CanonicalPuf55DonorKernel._context", + "expression": "_node(definition, json.loads(params))", + "expression_sha256": "2f3b9ccd2a017a2fc9fadafc5fe2273c4e7f1f892ad3cc96e11bb316c1889b0c", + "resolution": "resolved", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.graph_puf55_canonical_donor::canonical_puf55_donor_node", + "expression": "_node(definition, _params(definition, seed, growth_scheme, packaged_sha))", + "expression_sha256": "e671fdeb2791f6ad3dab353b1062cae11896bfd27fe2a2b24f3ef2761cd2634e", + "resolution": "resolved", + "usage": "call" + } + ] + }, + "microcosm.build.us_runtime.graph_puf55_canonical_donor::_params": { + "basis": "reviewed entry/caller-chain body", + "body_sha256": "7d22e1f18950ac1c910482bb85da63e7ef32de43dc3e44b072eaf26af38107c4", + "references": [ + { + "caller": "microcosm.build.us_runtime.graph_puf55_canonical_donor::CanonicalPuf55DonorKernel.__init__", + "expression": "_params(definition, seed, growth_scheme, packaged_sha)", + "expression_sha256": "60874f475437478449fef34327905166b61d9adac420b230736955b2cff7df90", + "resolution": "resolved", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.graph_puf55_canonical_donor::CanonicalPuf55DonorKernel._check_state", + "expression": "_params(definition, seed, scheme, packaged_sha)", + "expression_sha256": "544e39fa8e513a4764c8d28dfa0516330851ff453081028996198561345b1b85", + "resolution": "resolved", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.graph_puf55_canonical_donor::canonical_puf55_donor_node", + "expression": "_params(definition, seed, growth_scheme, packaged_sha)", + "expression_sha256": "60874f475437478449fef34327905166b61d9adac420b230736955b2cff7df90", + "resolution": "resolved", + "usage": "call" + } + ] + }, + "microcosm.build.us_runtime.graph_puf55_canonical_donor::canonical_puf55_donor_node": { + "basis": "reviewed entry/caller-chain body", + "body_sha256": "e21dc70b3c9000b900669390b8dc194b4fe5e9bd200112ad19249d65bdd799e4", + "references": [] + }, + "microcosm.build.us_runtime.graph_puf55_survey_recipients::_Kernel._context": { + "basis": "reviewed entry/caller-chain body", + "body_sha256": "ff7f3884c6538a003dbf0851fde6982359b228663a1e3ecd3a9233f09caa5ac7", + "references": [ + { + "caller": "microcosm.build.us_runtime.graph_puf55_survey_recipients::_Kernel.run", + "expression": "self._context(context, qualified)", + "expression_sha256": "16fd5f65f38c2cbc14f6ae49602264471192ef82250761911b21d57823e085c1", + "resolution": "resolved", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.graph_puf55_survey_recipients::_Kernel.run", + "expression": "self._context(context, qualified)", + "expression_sha256": "16fd5f65f38c2cbc14f6ae49602264471192ef82250761911b21d57823e085c1", + "resolution": "resolved", + "usage": "call" + } + ] + }, + "microcosm.build.us_runtime.graph_puf55_survey_recipients::_Kernel.implementation_hash": { + "basis": "reviewed entry/caller-chain body", + "body_sha256": "5f62a167610e8df52eecbb127926fae532fcc7254bb488038e01d41349c59856", + "references": [ + { + "caller": "microcosm.build.us_runtime.graph_puf55_survey_recipients::_Kernel.implementation_hash", + "expression": "source_graph._Kernel.implementation_hash(self)", + "expression_sha256": "66602d44bf4a2b5f9a0f59a08aab9a96f85070ccc4a0cb5a2528724318877495", + "resolution": "unresolved-us-candidate", + "usage": "call" + } + ] + }, + "microcosm.build.us_runtime.graph_puf55_survey_recipients::_Kernel.run": { + "basis": "reviewed entry/caller-chain body", + "body_sha256": "193cd27ab732e014a6121424cb2fa3ece511e44e214db25e0930bdb3ad2c4792", + "references": [ + { + "caller": "microcosm.build.us_runtime.graph_puf55_survey_recipients::_Kernel.run", + "expression": "run", + "expression_sha256": "324ca68f884c8ccf2070b1fc21958a2a3004f84b25789bd6b67084b3d1c67f75", + "resolution": "resolved", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.graph_puf55_survey_recipients::_Kernel.run", + "expression": "run", + "expression_sha256": "324ca68f884c8ccf2070b1fc21958a2a3004f84b25789bd6b67084b3d1c67f75", + "resolution": "resolved", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.graph_puf55_survey_recipients::_Kernel.run", + "expression": "run", + "expression_sha256": "324ca68f884c8ccf2070b1fc21958a2a3004f84b25789bd6b67084b3d1c67f75", + "resolution": "resolved", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.graph_puf55_survey_recipients::_Kernel.run", + "expression": "run", + "expression_sha256": "324ca68f884c8ccf2070b1fc21958a2a3004f84b25789bd6b67084b3d1c67f75", + "resolution": "resolved", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.graph_puf55_survey_recipients::_Kernel.run", + "expression": "run", + "expression_sha256": "324ca68f884c8ccf2070b1fc21958a2a3004f84b25789bd6b67084b3d1c67f75", + "resolution": "resolved", + "usage": "reference" + } + ] + }, + "microcosm.build.us_runtime.graph_puf55_survey_recipients::_check_values": { + "basis": "reviewed entry/caller-chain body", + "body_sha256": "4e79d04cc81ac3a9a6ac29e18fc1bed03aae8f27e7357fac4c883bc16357ba93", + "references": [ + { + "caller": "microcosm.build.us_runtime.graph_puf55_survey_recipients::_Kernel._context", + "expression": "_check_values(qualified)", + "expression_sha256": "240b01d06d0fb745efba8da63e09be253441a92f2e99976882e1206beba97d72", + "resolution": "resolved", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.graph_puf55_survey_recipients::puf55_survey_recipient_nodes", + "expression": "_check_values(qualified)", + "expression_sha256": "240b01d06d0fb745efba8da63e09be253441a92f2e99976882e1206beba97d72", + "resolution": "resolved", + "usage": "call" + } + ] + }, + "microcosm.build.us_runtime.graph_puf55_survey_recipients::puf55_survey_recipient_nodes": { + "basis": "reviewed entry/caller-chain body", + "body_sha256": "66ecdb7b690d391b4ba713774a8158c5a0a9b19b809f09ba0ec2ca97a63ff886", + "references": [ + { + "caller": "microcosm.build.us_runtime.graph_puf55_survey_recipients::_Kernel._context", + "expression": "puf55_survey_recipient_nodes(qualified)", + "expression_sha256": "ac47875675971adc58090f41a94ccda00f910978d804b2014463fbf5debfa91c", + "resolution": "resolved", + "usage": "call" + } + ] + }, + "microcosm.build.us_runtime.graph_puf55_survey_recipients::verify_materialized_puf55_survey_recipients": { + "basis": "reviewed entry/caller-chain body", + "body_sha256": "9e4be8e1454c52668b7de42542051cf93e79d96c18e4c3b58df97258678e2913", + "references": [] + }, + "microcosm.build.us_runtime.graph_puf_detail_transfer::DetailAttachKernel": { + "basis": "reviewed entry/caller-chain body", + "body_sha256": "730caf81eab9c252a86ddfad3cf4b061d65e81389f284b3e13ba93ec88349193", + "references": [ + { + "caller": "microcosm.build.us_runtime.graph_puf_detail_transfer::DetailAttachKernel.run", + "expression": "self", + "expression_sha256": "7ed444d38b2676027d24aa06f77919b33308ad3d2d830e88198c8b769cbdf8dd", + "resolution": "resolved", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.graph_puf_detail_transfer::DetailAttachKernel.run", + "expression": "self", + "expression_sha256": "7ed444d38b2676027d24aa06f77919b33308ad3d2d830e88198c8b769cbdf8dd", + "resolution": "resolved", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.graph_puf_detail_transfer::DetailAttachKernel.run", + "expression": "self", + "expression_sha256": "7ed444d38b2676027d24aa06f77919b33308ad3d2d830e88198c8b769cbdf8dd", + "resolution": "resolved", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.graph_puf_detail_transfer::DetailAttachKernel.run", + "expression": "self", + "expression_sha256": "7ed444d38b2676027d24aa06f77919b33308ad3d2d830e88198c8b769cbdf8dd", + "resolution": "resolved", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.graph_puf_detail_transfer::DetailAttachKernel.run", + "expression": "self", + "expression_sha256": "7ed444d38b2676027d24aa06f77919b33308ad3d2d830e88198c8b769cbdf8dd", + "resolution": "resolved", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.graph_puf_detail_transfer::DetailAttachKernel.run", + "expression": "self", + "expression_sha256": "7ed444d38b2676027d24aa06f77919b33308ad3d2d830e88198c8b769cbdf8dd", + "resolution": "resolved", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.graph_puf_detail_transfer::DetailAttachKernel.run", + "expression": "self", + "expression_sha256": "7ed444d38b2676027d24aa06f77919b33308ad3d2d830e88198c8b769cbdf8dd", + "resolution": "resolved", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.graph_puf_detail_transfer::host_nodes", + "expression": "DetailAttachKernel", + "expression_sha256": "962c65a682f50a72489097c7e1cd42d853a80a47fc23188da31594e1938f004f", + "resolution": "resolved", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.graph_puf_diagnostic_consumer::DiagnosticAttachKernel", + "expression": "shared.DetailAttachKernel", + "expression_sha256": "8d1e44a50f9cae0b4aaac8ff9509b9200951a001e133d2f1ac29770b50307bfb", + "resolution": "resolved", + "usage": "reference" + } + ] + }, + "microcosm.build.us_runtime.graph_puf_detail_transfer::DetailAttachKernel.run": { + "basis": "reviewed entry/caller-chain body", + "body_sha256": "206a8f39437ee251070917e5676786622f45f9c3605a4527afc97d06cd1fb44d", + "references": [] + }, + "microcosm.build.us_runtime.graph_puf_detail_transfer::DetailMatrixKernel": { + "basis": "reviewed entry/caller-chain body", + "body_sha256": "a3974ed18aa504351f3ce68f0461225639e050ec67103fa21690a7ebf144d4a2", + "references": [ + { + "caller": "microcosm.build.us_runtime.graph_puf_detail_transfer::DetailMatrixKernel.run", + "expression": "self", + "expression_sha256": "7ed444d38b2676027d24aa06f77919b33308ad3d2d830e88198c8b769cbdf8dd", + "resolution": "resolved", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.graph_puf_detail_transfer::DetailMatrixKernel.run", + "expression": "self", + "expression_sha256": "7ed444d38b2676027d24aa06f77919b33308ad3d2d830e88198c8b769cbdf8dd", + "resolution": "resolved", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.graph_puf_detail_transfer::DetailMatrixKernel.run", + "expression": "self", + "expression_sha256": "7ed444d38b2676027d24aa06f77919b33308ad3d2d830e88198c8b769cbdf8dd", + "resolution": "resolved", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.graph_puf_detail_transfer::DetailMatrixKernel.run", + "expression": "self", + "expression_sha256": "7ed444d38b2676027d24aa06f77919b33308ad3d2d830e88198c8b769cbdf8dd", + "resolution": "resolved", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.graph_puf_detail_transfer::DetailMatrixKernel.run", + "expression": "self", + "expression_sha256": "7ed444d38b2676027d24aa06f77919b33308ad3d2d830e88198c8b769cbdf8dd", + "resolution": "resolved", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.graph_puf_detail_transfer::DetailMatrixKernel.run", + "expression": "self", + "expression_sha256": "7ed444d38b2676027d24aa06f77919b33308ad3d2d830e88198c8b769cbdf8dd", + "resolution": "resolved", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.graph_puf_detail_transfer::host_nodes", + "expression": "DetailMatrixKernel", + "expression_sha256": "dd9cd1577a3515568d0a7f86ef28495afd7c95056432a6f73f52aab27d276c39", + "resolution": "resolved", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.graph_puf_diagnostic_consumer::DiagnosticMatrixKernel", + "expression": "shared.DetailMatrixKernel", + "expression_sha256": "e3e4bb5ad9fad27845eefb6e0769826ed7b207bafeb1e3f53c3248999fe11205", + "resolution": "resolved", + "usage": "reference" + } + ] + }, + "microcosm.build.us_runtime.graph_puf_detail_transfer::DetailMatrixKernel.run": { + "basis": "reviewed entry/caller-chain body", + "body_sha256": "754a86234ca6dc345ab7564a9bfef5ee264ad395ad23a36c0f83566907f9dcf5", + "references": [] + }, + "microcosm.build.us_runtime.graph_puf_detail_transfer::FixtureDonorKernel.run": { + "basis": "reviewed entry/caller-chain body", + "body_sha256": "8e3258cde4130075714042dcd538ed93787befa89d182442527bc98d12461b90", + "references": [] + }, + "microcosm.build.us_runtime.graph_puf_detail_transfer::FixturePriceExportKernel.run": { + "basis": "reviewed entry/caller-chain body", + "body_sha256": "d23ec89eaa113c07d9ba9d563e69e1903ce62261baa61025f350121af2f52eb2", + "references": [] + }, + "microcosm.build.us_runtime.graph_puf_detail_transfer::host_nodes": { + "basis": "reviewed entry/caller-chain body", + "body_sha256": "733886c3b39ec35a131ea13434ba84a75fd2ce8890e0620673321b7768cf5ca2", + "references": [] + }, + "microcosm.build.us_runtime.graph_puf_diagnostic_consumer::CurrentSurveyHostProjectionKernel.run": { + "basis": "reviewed entry/caller-chain body", + "body_sha256": "cc514f283819886dcdc1d74be1efad6da099a73bebd1b9c89d19e1b3879695a5", + "references": [] + }, + "microcosm.build.us_runtime.graph_puf_diagnostic_consumer::CurrentSurveyMatrixKernel.run": { + "basis": "reviewed entry/caller-chain body", + "body_sha256": "efaf6651ee41cae2f41c859dd1d1f1e0928e4947b0128200f004e574ef68e8e2", + "references": [] + }, + "microcosm.build.us_runtime.graph_puf_diagnostic_consumer::DiagnosticAttachKernel": { + "basis": "reviewed entry/caller-chain body", + "body_sha256": "a0fb1e7ea314aa60124c7f2a0379c623c2e0b63afbd1716b2d9e31ad3f4c0c57", + "references": [ + { + "caller": "microcosm.build.us_runtime.graph_puf_diagnostic_consumer::diagnostic_nodes", + "expression": "DiagnosticAttachKernel", + "expression_sha256": "953d920317b9d90ca346f7845f729f2e2f348639c011db39900c2f820c71f3a3", + "resolution": "resolved", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.graph_puf_diagnostic_consumer::register_diagnostic_kernels", + "expression": "DiagnosticAttachKernel()", + "expression_sha256": "41439597a870390dd22aaac5ffbeca0239402e7a0d76729e211d79c3fda25bd0", + "resolution": "resolved", + "usage": "call" + } + ] + }, + "microcosm.build.us_runtime.graph_puf_diagnostic_consumer::DiagnosticDonorKernel.run": { + "basis": "reviewed entry/caller-chain body", + "body_sha256": "46bee8b28d839d5469b884dddd33e19d5fd2789ab23621d517bf087244c3adcc", + "references": [ + { + "caller": "microcosm.build.us_runtime.graph_puf_diagnostic_consumer::register_diagnostic_kernels", + "expression": "DiagnosticDonorKernel()", + "expression_sha256": "dba858734d6699cc0f97c026a51562fb5686bccce9a48bb6b2ef3afbcdd1b1f7", + "resolution": "kernel-construction", + "usage": "call" + } + ] + }, + "microcosm.build.us_runtime.graph_puf_diagnostic_consumer::DiagnosticMatrixKernel": { + "basis": "reviewed entry/caller-chain body", + "body_sha256": "b9aae83cf76f67f787e439125b17200b26bf0df8ffa08c4c3035339c3c5f204a", + "references": [ + { + "caller": "microcosm.build.us_runtime.graph_puf_diagnostic_consumer::diagnostic_nodes", + "expression": "DiagnosticMatrixKernel", + "expression_sha256": "8ccd491347d9a0be3cfe3c05df0e0f8b466b7db22b67e7270674c280d8086f29", + "resolution": "resolved", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.graph_puf_diagnostic_consumer::register_diagnostic_kernels", + "expression": "DiagnosticMatrixKernel()", + "expression_sha256": "79bdc9afe49c5c946d8ae3fdf7191a24fc71eac67d2687fde93e60cbeb8e0b86", + "resolution": "resolved", + "usage": "call" + } + ] + }, + "microcosm.build.us_runtime.graph_puf_diagnostic_consumer::_Kernel.implementation_hash": { + "basis": "reviewed entry/caller-chain body", + "body_sha256": "ed9f0dadf1fc88ac269e517910cc41031168afa8b33b8243650897922804e4fc", + "references": [ + { + "caller": "microcosm.build.us_runtime.graph_puf55_survey_recipients::_Kernel.implementation_hash", + "expression": "source_graph._Kernel.implementation_hash(self)", + "expression_sha256": "66602d44bf4a2b5f9a0f59a08aab9a96f85070ccc4a0cb5a2528724318877495", + "resolution": "unresolved-us-candidate", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.graph_puf_diagnostic_consumer::DiagnosticAttachKernel", + "expression": "_Kernel.implementation_hash", + "expression_sha256": "61653c1ca77e3e1c44eec99f0fd37458b5c7ffd66a0f153f3eb7551f06dd1e5e", + "resolution": "resolved", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.graph_puf_diagnostic_consumer::DiagnosticMatrixKernel", + "expression": "_Kernel.implementation_hash", + "expression_sha256": "61653c1ca77e3e1c44eec99f0fd37458b5c7ffd66a0f153f3eb7551f06dd1e5e", + "resolution": "resolved", + "usage": "reference" + } + ] + }, + "microcosm.build.us_runtime.graph_puf_diagnostic_consumer::diagnostic_nodes": { + "basis": "reviewed entry/caller-chain body", + "body_sha256": "6fef1468164da108f21fc53ceec453cac1c6d6d5f882727c6517439835d78321", + "references": [] + }, + "microcosm.build.us_runtime.graph_puf_diagnostic_consumer::qualify_current_survey_host": { + "basis": "reviewed entry/caller-chain body", + "body_sha256": "7bb8a4815d8dcc161e1e0c192d19efe18869b8bf1058bb61ba57546d57fec261", + "references": [ + { + "caller": "microcosm.build.us_runtime.graph_current_survey_puf_transfer::_CurrentTransferKernel._qualified", + "expression": "host.qualify_current_survey_host(self.preparation, self.allocated_population, self.clone_population)", + "expression_sha256": "25319bb0116eae85f603a3f78b4cfa83fc2396e8ed341678a99003e394d14e35", + "resolution": "resolved", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.graph_current_survey_puf_transfer::verify_materialized_current_survey_transfer", + "expression": "host.qualify_current_survey_host(preparation, allocated_population, clone_population)", + "expression_sha256": "2280b1d6dbff39c782c4e2f450ecde7e54bb0af0d22a73dc152cfe5cb778ec85", + "resolution": "resolved", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.graph_puf_diagnostic_consumer::CurrentSurveyHostProjectionKernel.run", + "expression": "qualify_current_survey_host(self.preparation, self.allocated_population, self.clone_population)", + "expression_sha256": "48d6ef04be6db1d8166679f1807bba7f6514b46c4001418f39b2dd0c63b80c7c", + "resolution": "resolved", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.graph_puf_diagnostic_consumer::verify_materialized_current_survey_host", + "expression": "qualify_current_survey_host(preparation, allocated_population, clone_population)", + "expression_sha256": "c88ea0f98a3ee5cbde02f73fb5bce576ce7abf62f050a1311a40f84eb81026fb", + "resolution": "resolved", + "usage": "call" + } + ] + }, + "microcosm.build.us_runtime.graph_puf_diagnostic_consumer::register_diagnostic_kernels": { + "basis": "reviewed entry/caller-chain body", + "body_sha256": "0ff8da56e267196d139e404a9b264df7c10acc276f4b97f015e601bdec233a96", + "references": [] + }, + "microcosm.build.us_runtime.graph_puf_diagnostic_consumer::verify_host": { + "basis": "reviewed entry/caller-chain body", + "body_sha256": "74696ec4ee4b3e95ec8dc2daed4800a82c7b63c28057e5fe3383377c6ddababa", + "references": [ + { + "caller": "microcosm.build.us_runtime.graph_puf_detail_transfer::DetailAttachKernel.run", + "expression": "self.verify_host(context, frame)", + "expression_sha256": "5b0cf41aecd6f81914d0c5da0fef54da3d93ccdd10a5115fa2bea3e46b831d42", + "resolution": "unresolved-us-candidate", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.graph_puf_detail_transfer::DetailMatrixKernel.run", + "expression": "self.verify_host(context, frame)", + "expression_sha256": "5b0cf41aecd6f81914d0c5da0fef54da3d93ccdd10a5115fa2bea3e46b831d42", + "resolution": "unresolved-us-candidate", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.graph_puf_diagnostic_consumer::DiagnosticAttachKernel", + "expression": "verify_host", + "expression_sha256": "ce3fbfdd34c9a45f0b0504d29e528044f0ff2c00a2aa8ed381cf820cc6e56cf7", + "resolution": "resolved", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.graph_puf_diagnostic_consumer::DiagnosticMatrixKernel", + "expression": "verify_host", + "expression_sha256": "ce3fbfdd34c9a45f0b0504d29e528044f0ff2c00a2aa8ed381cf820cc6e56cf7", + "resolution": "resolved", + "usage": "reference" + } + ] + }, + "microcosm.build.us_runtime.graph_puf_diagnostic_consumer::verify_materialized_current_survey_host": { + "basis": "reviewed entry/caller-chain body", + "body_sha256": "483cb6c2274693da6bda5871e2e74cc269d7a578eaed2d254fe137f4e70a45c5", + "references": [] + }, + "microcosm.build.us_runtime.graph_sources::USAssemblyCreateKernel.implementation_hash": { + "basis": "reviewed entry/caller-chain body", + "body_sha256": "9b82d2a40a161c1190b71f9d1a2f023f790a880d5ac2ab95786e8ea8d4b72f16", + "references": [ + { + "caller": "microcosm.build.us_runtime.graph_puf55_survey_recipients::_Kernel.implementation_hash", + "expression": "source_graph._Kernel.implementation_hash(self)", + "expression_sha256": "66602d44bf4a2b5f9a0f59a08aab9a96f85070ccc4a0cb5a2528724318877495", + "resolution": "unresolved-us-candidate", + "usage": "call" + } + ] + }, + "microcosm.build.us_runtime.graph_sources::_source_implementation_hash": { + "basis": "reviewed entry/caller-chain body", + "body_sha256": "02def6e3d7659262a478603a54b4e61bd2efa4a32b970c61a4757cdb8544ff6b", + "references": [ + { + "caller": "microcosm.build.us_runtime.graph_sources::USAssemblyCreateKernel.implementation_hash", + "expression": "_source_implementation_hash()", + "expression_sha256": "1a4919cc4b5d120162f161437b7533c52e7f861522119fdc2976e4f949fc495f", + "resolution": "resolved", + "usage": "call" + } + ] + }, + "microcosm.build.us_runtime.graph_sources::us_source_codecs": { + "basis": "reviewed entry/caller-chain body", + "body_sha256": "2a58ac10f517503411b2df528f998c9120370b0534755215c93cbc9907dee276", + "references": [ + { + "caller": "microcosm.build.us_runtime.graph_composed_population::USComposedPopulationCreateKernel.implementation_hash", + "expression": "us_source_codecs()", + "expression_sha256": "17a42c1e702081b65ed2c774aa8cc71f8c2d8fd9bc189dd35bf6d108761fa25d", + "resolution": "resolved", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.graph_sources::_source_implementation_hash", + "expression": "us_source_codecs()", + "expression_sha256": "17a42c1e702081b65ed2c774aa8cc71f8c2d8fd9bc189dd35bf6d108761fa25d", + "resolution": "resolved", + "usage": "call" + } + ] + }, + "microcosm.build.us_runtime.graph_survey_population::SurveyPopulationAllocationKernel.run": { + "basis": "reviewed entry/caller-chain body", + "body_sha256": "c89e15138f7acc408b3cb9db0f9117ab0afcd0dbcc1aee1433cdecb04dadc3cd", + "references": [ + { + "caller": "microcosm.build.us_runtime.graph_survey_population::run_authenticated_survey_population", + "expression": "SurveyPopulationAllocationKernel(instructions, preparation_bytes=payload, context_bytes=context_bytes)", + "expression_sha256": "6a7ce7dd521157d90f058808a53d867d40ee23658eeec4e275f403be8a4fffaf", + "resolution": "kernel-construction", + "usage": "call" + } + ] + }, + "microcosm.build.us_runtime.graph_survey_population::SurveyPopulationCreateKernel.run": { + "basis": "reviewed entry/caller-chain body", + "body_sha256": "847b99bc376eeff884c11b6da3aa2b5db8504cb58b7454d8e4c62582d7b6ab7f", + "references": [ + { + "caller": "microcosm.build.us_runtime.graph_survey_population::run_authenticated_survey_population", + "expression": "SurveyPopulationCreateKernel(preparation, source_dir=source_dir)", + "expression_sha256": "6f44636795fe2a4aa4da5cc811492cbeffd160494085e9c0120baf312b2ad24a", + "resolution": "kernel-construction", + "usage": "call" + } + ] + }, + "microcosm.build.us_runtime.graph_survey_population::_allocation_output": { + "basis": "reviewed entry/caller-chain body", + "body_sha256": "96317ffb608f1e8bdcca07408108a140ddb49b980580da812f5ed3394639505e", + "references": [ + { + "caller": "microcosm.build.us_runtime.graph_atomic_survey_population::run_atomic_survey_population", + "expression": "survey._allocation_output(view.frame, view.context, instructions, survey._sha(view.payload))", + "expression_sha256": "96c886c261f9a3f48082a0957e03dc5a6e1c0f5a5bfb0883f19e8f993799252a", + "resolution": "resolved", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.graph_survey_population::SurveyPopulationAllocationKernel.run", + "expression": "_allocation_output(original, bound.payload, self._instructions, digest)", + "expression_sha256": "cf6494c500cfebdcd8ff0651a80267d25b7a90feec815728559bd136ca27135c", + "resolution": "resolved", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.graph_survey_population::run_authenticated_survey_population", + "expression": "_allocation_output(prepared_frame, context_bytes, instructions, _sha(payload))", + "expression_sha256": "061004fbb7818789fdb75f510a4deb71ed24b8ae06efd88fc4bb2dabe280a1ca", + "resolution": "resolved", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.survey_age_calibration::_run_survey_age_calibration", + "expression": "source_graph._allocation_output(source_view.frame, source_view.context, instructions, _sha(source_view.payload))", + "expression_sha256": "4456f844f85fbefad22837de7b9084a61f97b4f79b3b9bf08eea41c70dca3cb2", + "resolution": "resolved", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.survey_atomic_geography::_raw_allocation", + "expression": "graph._allocation_output(view.frame, view.context, instructions, _sha(view.payload))", + "expression_sha256": "f04fd005b0ecedaa1d3f8c1d9c5e12ddda8ac8ba4fccd183b47a5debbc6a5017", + "resolution": "resolved", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.survey_origin_budget::_financial_budget_values", + "expression": "graph._allocation_output(view.frame, view.context, instructions, _sha(view.payload))", + "expression_sha256": "f04fd005b0ecedaa1d3f8c1d9c5e12ddda8ac8ba4fccd183b47a5debbc6a5017", + "resolution": "resolved", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.survey_origin_budget::_initial", + "expression": "graph._allocation_output(view.frame, view.context, instructions, _sha(view.payload))", + "expression_sha256": "f04fd005b0ecedaa1d3f8c1d9c5e12ddda8ac8ba4fccd183b47a5debbc6a5017", + "resolution": "resolved", + "usage": "call" + } + ] + }, + "microcosm.build.us_runtime.graph_survey_population::allocation_instructions": { + "basis": "reviewed entry/caller-chain body", + "body_sha256": "e53ebacebf275380711097a52f942776575a4bae9d63e49c7c889f940d484ad2", + "references": [ + { + "caller": "microcosm.build.us_runtime.graph_atomic_survey_population::run_atomic_survey_population", + "expression": "survey.allocation_instructions(view.selection_plan, view.receipt['origins']['households'])", + "expression_sha256": "1a52835fa0ad17bb019e73b020142a83badc351222b900e9bcd7e66d1f6768e3", + "resolution": "resolved", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.graph_survey_population::run_authenticated_survey_population", + "expression": "allocation_instructions(view.selection_plan, view.receipt['origins']['households'])", + "expression_sha256": "25a1def250f189cd135a5d3230b1999cb6d744825d4f8b0d0a724296d0c03a30", + "resolution": "resolved", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.survey_age_calibration::_run_survey_age_calibration", + "expression": "source_graph.allocation_instructions(source_view.selection_plan, source_view.receipt['origins']['households'])", + "expression_sha256": "bdc0e2ea7c5d1475d2e22acbc48e7556c8bbc1b80bbc37c7b330d2120da8e08a", + "resolution": "resolved", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.survey_atomic_geography::_raw_allocation", + "expression": "graph.allocation_instructions(view.selection_plan, view.receipt['origins']['households'])", + "expression_sha256": "5baf6cb0c9d17e363f797ed4f3e66b6d407bedeb765aeb758b061116d60e2108", + "resolution": "resolved", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.survey_origin_budget::_financial_budget_values", + "expression": "graph.allocation_instructions(view.selection_plan, view.receipt['origins']['households'])", + "expression_sha256": "5baf6cb0c9d17e363f797ed4f3e66b6d407bedeb765aeb758b061116d60e2108", + "resolution": "resolved", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.survey_origin_budget::_initial", + "expression": "graph.allocation_instructions(view.selection_plan, view.receipt['origins']['households'])", + "expression_sha256": "5baf6cb0c9d17e363f797ed4f3e66b6d407bedeb765aeb758b061116d60e2108", + "resolution": "resolved", + "usage": "call" + } + ] + }, + "microcosm.build.us_runtime.graph_survey_population::run_authenticated_survey_population": { + "basis": "reviewed entry/caller-chain body", + "body_sha256": "ce3b3d9fa4c0c460328637303b532705e8931e6dec3259f7e32248adfb098d19", + "references": [ + { + "caller": "microcosm.build.us_runtime.graph_atomic_survey_population::run_atomic_survey_population", + "expression": "survey.run_authenticated_survey_population(source_dir, snapshot_root=snapshot_root, store_root=store_root, fraction=fraction, seed=seed, resume=resume, clones=False, return_values=True)", + "expression_sha256": "d3a265f6953d730ebd39752668ba116fc7a22984b7d08ba9d356c31374f4b4a8", + "resolution": "resolved", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.survey_age_calibration::_run_survey_age_calibration", + "expression": "source_graph.run_authenticated_survey_population(source_dir, snapshot_root=snapshot_root, store_root=store_root, fraction=fraction, seed=seed_value, resume=resume, clones=True, return_values=True)", + "expression_sha256": "15f1d13b3d6be093895b7acc14791139f3efb900dc155049205805ca21b1740f", + "resolution": "resolved", + "usage": "call" + } + ] + }, + "microcosm.build.us_runtime.graph_survey_population::run_authenticated_survey_population.observe": { + "basis": "reviewed entry/caller-chain body", + "body_sha256": "b434253e24467cbe23e838f09dfcded5bdd926aba6559ac79cb8954c3ce54fe9", + "references": [ + { + "caller": "microcosm.build.us_runtime.graph_survey_population::run_authenticated_survey_population", + "expression": "observe", + "expression_sha256": "274f69be16100e8e78059234777f281731e85cc7e8d8ea6f15a066520f4f89c8", + "resolution": "resolved", + "usage": "reference" + } + ] + }, + "microcosm.build.us_runtime.housing_inputs::AcsRentDonorPreparation.__post_init__": { + "basis": "reviewed entry/caller-chain body", + "body_sha256": "c39c2db99d0739d28234a0dad1a450aea0809e60528e79774e4efb789fc44cbe", + "references": [] + }, + "microcosm.build.us_runtime.housing_inputs::AcsRentRecipientPreparation.__post_init__": { + "basis": "reviewed entry/caller-chain body", + "body_sha256": "8a8ff9474f586b3662d02135f8ba73e9563b24148150d9b0d810c84349f6535b", + "references": [] + }, + "microcosm.build.us_runtime.multispine_pool::MultispinePoolCheckpoint.__post_init__": { + "basis": "reviewed entry/caller-chain body", + "body_sha256": "504199439f13beaee861b0e7a324a08d75ae7bef04606bcab5fd00d65197ab45", + "references": [] + }, + "microcosm.build.us_runtime.multispine_pool::PoolStageOutput.__post_init__": { + "basis": "reviewed entry/caller-chain body", + "body_sha256": "5b9936cfc91cce5b5669b5b978d6a0d72bfe276037ac9168a75339ae398b76e2", + "references": [] + }, + "microcosm.build.us_runtime.native_household_origin::bind_population_origins": { + "basis": "reviewed entry/caller-chain body", + "body_sha256": "19ffa056948c7aa82c2e05e05ccd825a2bd7f51d356fe402bf7e27e41f6c0e60", + "references": [ + { + "caller": "microcosm.build.us_runtime.graph_native_household_origin::PopulationOriginBindingKernel.run", + "expression": "native.bind_population_origins(frame, sources=sources, parent=parent)", + "expression_sha256": "ab7987b3a25ab63a1d811934f5b0c00cb2397b24a072b8d194fac604afbd6b3c", + "resolution": "resolved", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.graph_native_household_origin::verify_materialized_population_origins", + "expression": "native.bind_population_origins(frame, sources=sources, parent=parent)", + "expression_sha256": "ab7987b3a25ab63a1d811934f5b0c00cb2397b24a072b8d194fac604afbd6b3c", + "resolution": "resolved", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.puf_diagnostic_consumer::qualify_host_population", + "expression": "origin.bind_population_origins(frame, sources=sources, parent=parent)", + "expression_sha256": "90a7a170d34796dce74cf489a0f720173d85aef4397247c0ed672ffe735e9288", + "resolution": "resolved", + "usage": "call" + } + ] + }, + "microcosm.build.us_runtime.operator_column_contracts::@ACS_DERIVED_TRANSFER_INPUTS": { + "basis": "finite selector/validator definition", + "body_sha256": "038a3478d559bc980f22bbc2896548dbaeb934352b3d7cbe0a4b96ba0830731b", + "references": [] + }, + "microcosm.build.us_runtime.operator_column_contracts::@PUF_CAPITAL_GAINS_TAIL_APPLIED_COLUMN": { + "basis": "finite selector/validator definition", + "body_sha256": "0b0bfcec06b798e16538e3b4b049cfadefb1184bc6f1a545505af4db3f907970", + "references": [] + }, + "microcosm.build.us_runtime.operator_column_contracts::@PUF_CAPITAL_GAINS_TAIL_DONOR_AGI_BAND_COLUMN": { + "basis": "finite selector/validator definition", + "body_sha256": "8769b8205c505ffd67342cc9839ae81d356cd497c143795ee2dfc2981292506a", + "references": [] + }, + "microcosm.build.us_runtime.operator_column_contracts::@PUF_CAPITAL_GAINS_TAIL_DONOR_FILING_STATUS_COLUMN": { + "basis": "finite selector/validator definition", + "body_sha256": "17f9f7ab2956bb340a692bfc79feb10cd6d10335db89e51b696a3fefc209f437", + "references": [] + }, + "microcosm.build.us_runtime.operator_column_contracts::@PUF_CAPITAL_GAINS_TAIL_DONOR_SOURCE_ID_COLUMN": { + "basis": "finite selector/validator definition", + "body_sha256": "7ae5a6a33f3c69288166569abf4ed34ca24c83f8efa92babbc9f8aa316df1c99", + "references": [] + }, + "microcosm.build.us_runtime.operator_column_contracts::@PUF_CAPITAL_GAINS_TAIL_DONOR_SYNTHETIC_COLUMN": { + "basis": "finite selector/validator definition", + "body_sha256": "3023baaee99897424bc59a25c92612b02baf684dc7d15577dac3b94b49a8debd", + "references": [] + }, + "microcosm.build.us_runtime.operator_column_contracts::@PUF_CAPITAL_GAINS_TAIL_PERSON_COLUMNS": { + "basis": "finite selector/validator definition", + "body_sha256": "4decc304a284c21723bd4d494b6f2aafcedfaca0893c5d88165c5a885b811310", + "references": [] + }, + "microcosm.build.us_runtime.operator_column_contracts::@PUF_CAPITAL_GAINS_TAIL_TAX_UNIT_COLUMNS": { + "basis": "finite selector/validator definition", + "body_sha256": "35b20398d4d513ea4541e491d8f5eb77e79eb7f95e30a59f95555b5987cfeba2", + "references": [] + }, + "microcosm.build.us_runtime.operator_column_contracts::@PUF_CAPITAL_GAINS_TAIL_TRANSFER_WEIGHT_COLUMN": { + "basis": "finite selector/validator definition", + "body_sha256": "9782241de03f85902b67da2bcbed5b3e1b3cb68fe0adcd4a3f70f9901c889d86", + "references": [] + }, + "microcosm.build.us_runtime.operator_column_contracts::@PUF_SUPPORT_MAX_CLONE_SAFE_SOURCE_ID": { + "basis": "finite selector/validator definition", + "body_sha256": "c831326ddce7cc491368111a3d77a72736164e66f5ce8ae56854cba2917e3e4d", + "references": [] + }, + "microcosm.build.us_runtime.operator_column_contracts::@PUF_TAX_DETAIL_DEFAULT_PERSON_OUTPUTS": { + "basis": "finite selector/validator definition", + "body_sha256": "b31e473f6c8339178b47f38303e259b15cacc8e8b4300a8801e6eeb463048ccf", + "references": [] + }, + "microcosm.build.us_runtime.operator_column_contracts::@PUF_TAX_DETAIL_DEFAULT_TAX_UNIT_OUTPUTS": { + "basis": "finite selector/validator definition", + "body_sha256": "1b89f1966cb264419132e91363713b45a2a73511a4f13fd2774dd68f6fdce261", + "references": [] + }, + "microcosm.build.us_runtime.operator_column_contracts::@US_PUF_SUPPORT_STAGE_NAME": { + "basis": "finite selector/validator definition", + "body_sha256": "ed07a5b1c61f7983751e90bed04c155b8bf50327725222eb2bf0315f5044a54d", + "references": [] + }, + "microcosm.build.us_runtime.operator_column_contracts::@US_QBI_BOOLEAN_OUTPUT_COLUMNS": { + "basis": "finite selector/validator definition", + "body_sha256": "ecc5b808c4dfd240b1388c51ef3d040a717adb8749e4d3804e0acd7ce417a490", + "references": [] + }, + "microcosm.build.us_runtime.operator_column_contracts::@US_QBI_NONNEGATIVE_OUTPUT_COLUMNS": { + "basis": "finite selector/validator definition", + "body_sha256": "a76dd42da666af3da88f500241008f19c0f750c28c94a62ddead02a49c322f59", + "references": [] + }, + "microcosm.build.us_runtime.operator_column_contracts::@US_QBI_OUTPUT_COLUMNS": { + "basis": "finite selector/validator definition", + "body_sha256": "1fde30806be14e7d2d0558aa0df1a3ce8a9828b640b77056494ab20ae8d067bc", + "references": [] + }, + "microcosm.build.us_runtime.operator_column_contracts::@_GENERAL_QUALIFICATION_FLAGS": { + "basis": "finite selector/validator definition", + "body_sha256": "5fea9ecb34b10eb48fbdd26b4059758e3abb25f81baa8998cd0285d1f92d5824", + "references": [] + }, + "microcosm.build.us_runtime.operator_column_contracts::@_SSTB_QUALIFICATION_FLAG": { + "basis": "finite selector/validator definition", + "body_sha256": "8b914d27c8b379e91a720b2fd148dc2955ad0058f77b87d15d2355e61e9d675c", + "references": [] + }, + "microcosm.build.us_runtime.puf55_canonical_donor::canonical_puf55_donor_from_artifact": { + "basis": "reviewed entry/caller-chain body", + "body_sha256": "f0221d309c50fc809cd0206c814da02031d58d285355e01aff3adb6afbe63ff8", + "references": [ + { + "caller": "microcosm.build.us_runtime.graph_puf55_canonical_donor::CanonicalPuf55DonorKernel.run", + "expression": "projection.canonical_puf55_donor_from_artifact(canonical_payload, expected_artifact_sha256=_sha(canonical_payload), expected_growth_scheme=scheme, profile=full.PUF55_SURVEY_SS)", + "expression_sha256": "f0b9e37b2cf0ecb2f4e6ae9e39f70dc22ec4f3f7384571075523c7a415847c02", + "resolution": "resolved", + "usage": "call" + } + ] + }, + "microcosm.build.us_runtime.puf55_route_finalization::@FINALIZATION_PROTOCOL": { + "basis": "finite selector/validator definition", + "body_sha256": "7f66e9f5fd569a3dfa950649a33e283ba234dac3447329846f566847a53c207c", + "references": [] + }, + "microcosm.build.us_runtime.puf55_route_finalization::@PROFILES": { + "basis": "finite selector/validator definition", + "body_sha256": "f3a6a503d8909f5ee74e22154e7b8603487fd5bd2f898481f7d249a81b0ed28e", + "references": [] + }, + "microcosm.build.us_runtime.puf55_route_finalization::@PROTOCOL": { + "basis": "finite selector/validator definition", + "body_sha256": "2d345e83359e07f006bd01f48b36e538138ecbbb7ab8cc7babb044d3d7506994", + "references": [] + }, + "microcosm.build.us_runtime.puf55_route_finalization::Puf55RouteDraws": { + "basis": "reviewed entry/caller-chain body", + "body_sha256": "c3aca3b7c98138b15555b9f4330dbc64f0bd5d14871fd1c58305a6020bc2fc17", + "references": [ + { + "caller": "microcosm.build.us_runtime.puf55_route_finalization::Puf55RouteFinalizationInput", + "expression": "Puf55RouteDraws", + "expression_sha256": "227c8efe1ca420bd83d18947859c0a2680ef46615d65ac4adfdd61f247b31ea6", + "resolution": "resolved", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.puf55_route_finalization::_finalize_puf55_routes", + "expression": "Puf55RouteDraws(*values)", + "expression_sha256": "78b546d55062232e54a4ad0cabc482394dba48348dab770ecbece2c84261645b", + "resolution": "resolved", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.puf55_route_finalization::_route_snapshot", + "expression": "Puf55RouteDraws", + "expression_sha256": "227c8efe1ca420bd83d18947859c0a2680ef46615d65ac4adfdd61f247b31ea6", + "resolution": "resolved", + "usage": "reference" + } + ] + }, + "microcosm.build.us_runtime.puf55_route_finalization::Puf55RouteFinalizationInput": { + "basis": "reviewed entry/caller-chain body", + "body_sha256": "26f2f4482d448bca506c6f2af6191f0c9a3d23f62355580b8ab2d5c1fd3bc297", + "references": [ + { + "caller": "microcosm.build.us_runtime.puf55_route_finalization::_finalize_puf55_routes", + "expression": "Puf55RouteFinalizationInput", + "expression_sha256": "295868034bd17b6bbd9635c015b9ee7905d5127df5f3e9787bbd15884cdcae9f", + "resolution": "resolved", + "usage": "reference" + } + ] + }, + "microcosm.build.us_runtime.puf55_route_finalization::_candidate_frame_sha256": { + "basis": "reviewed entry/caller-chain body", + "body_sha256": "c26b07b73e2995e48662969d944fa1a60b7cc5c9c2810c5ba2f851afa30136a0", + "references": [ + { + "caller": "microcosm.build.us_runtime.puf55_route_finalization::_finalize_puf55_routes", + "expression": "_candidate_frame_sha256(candidate)", + "expression_sha256": "8f52310c3ca63510b1bcc4f99262f3c3b896ac01aaf60d7c00d95f49f980b6d2", + "resolution": "resolved", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.puf55_route_finalization::_finalize_puf55_routes", + "expression": "_candidate_frame_sha256(candidate)", + "expression_sha256": "8f52310c3ca63510b1bcc4f99262f3c3b896ac01aaf60d7c00d95f49f980b6d2", + "resolution": "resolved", + "usage": "call" + } + ] + }, + "microcosm.build.us_runtime.puf55_route_finalization::_finalize_puf55_routes": { + "basis": "reviewed entry/caller-chain body", + "body_sha256": "91ca9169b26511b35a8a9feac575f8d24981bc0d4f3ccc94efdebb274476df0b", + "references": [] + }, + "microcosm.build.us_runtime.puf55_route_finalization::_finalize_puf55_routes.check_inputs": { + "basis": "reviewed entry/caller-chain body", + "body_sha256": "286d7796365f7cb5ca44fcacc387d7e9f5b166693827928061160eda384b1d79", + "references": [ + { + "caller": "microcosm.build.us_runtime.puf55_route_finalization::_finalize_puf55_routes", + "expression": "check_inputs()", + "expression_sha256": "15ac4d12cb418d8663532ed8d5d28d27961194b856adf56bb2060bf4883ab73a", + "resolution": "resolved", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.puf55_route_finalization::_finalize_puf55_routes", + "expression": "check_inputs()", + "expression_sha256": "15ac4d12cb418d8663532ed8d5d28d27961194b856adf56bb2060bf4883ab73a", + "resolution": "resolved", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.puf55_route_finalization::_finalize_puf55_routes", + "expression": "check_inputs()", + "expression_sha256": "15ac4d12cb418d8663532ed8d5d28d27961194b856adf56bb2060bf4883ab73a", + "resolution": "resolved", + "usage": "call" + } + ] + }, + "microcosm.build.us_runtime.puf55_route_finalization::_model_donor_frame": { + "basis": "reviewed entry/caller-chain body", + "body_sha256": "c2e465fef1b8fb3284df9d86cac34569867ef2927636a3bf32dd90a3de288a52", + "references": [ + { + "caller": "microcosm.build.us_runtime.graph_puf55_canonical_donor::CanonicalPuf55DonorKernel.run", + "expression": "numerical._model_donor_frame(selected)", + "expression_sha256": "afdc650d5920972efabd98db75daa78c6214e277560de2d6d9b0d980e8e6896f", + "resolution": "resolved", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.puf55_route_finalization::_finalize_puf55_routes", + "expression": "_model_donor_frame(model_donor)", + "expression_sha256": "7a2c33f3e51769ed81b5b4ecb5c6c6ac8d36c6da3ab9aaa12ef5aceed919d0f9", + "resolution": "resolved", + "usage": "call" + } + ] + }, + "microcosm.build.us_runtime.puf55_route_finalization::_route_snapshot": { + "basis": "reviewed entry/caller-chain body", + "body_sha256": "7fad08aa9168b1c015ccab8c4ae2e50af1680a6c7c2c0fa7a57a4718cc9767a8", + "references": [ + { + "caller": "microcosm.build.us_runtime.puf55_route_finalization::_finalize_puf55_routes", + "expression": "_route_snapshot(draws)", + "expression_sha256": "f6524588dfbc4f54ec74d0458e92f1a5450582cb8000161d4b342a17586ab2b7", + "resolution": "resolved", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.puf55_route_finalization::_finalize_puf55_routes.check_inputs", + "expression": "_route_snapshot(draws)", + "expression_sha256": "f6524588dfbc4f54ec74d0458e92f1a5450582cb8000161d4b342a17586ab2b7", + "resolution": "resolved", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.puf55_route_finalization::merge_puf55_route_draws", + "expression": "_route_snapshot(route)", + "expression_sha256": "9c57361e692e427aa69c08db2c4b209cbae9d363501d3cfc1f7c22e285558c94", + "resolution": "resolved", + "usage": "call" + } + ] + }, + "microcosm.build.us_runtime.puf55_route_finalization::merge_puf55_route_draws": { + "basis": "reviewed entry/caller-chain body", + "body_sha256": "54376a11b441613848771fe1add82b998e12309f24fe265a37405ac8fe732795", + "references": [ + { + "caller": "microcosm.build.us_runtime.puf55_route_finalization::_finalize_puf55_routes", + "expression": "merge_puf55_route_draws(frame, recipient_matrices=recipient_matrices, route_draws=tuple(copies), seed=seed)", + "expression_sha256": "25cae0456c6ec15c758a080f1ae787bb74673335599d5e05aadd0b95dd463371", + "resolution": "resolved", + "usage": "call" + } + ] + }, + "microcosm.build.us_runtime.puf55_survey_recipients::_project": { + "basis": "reviewed entry/caller-chain body", + "body_sha256": "9ef922cbd80c37e6d349275911b23347b49261ea4e1deeaa62493d672cc5d097", + "references": [ + { + "caller": "microcosm.build.us_runtime.puf55_survey_recipients::qualify_puf55_survey_recipients", + "expression": "_project(source_frame, frame, fresh, fresh_measurement)", + "expression_sha256": "5f57084a346b22a89287fe6c43259def60f6c6533ec716b12b43fa414e199d1e", + "resolution": "resolved", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.puf55_survey_recipients::qualify_puf55_survey_recipients", + "expression": "_project(source_frame, frame, authoritative_report, expected_measurement)", + "expression_sha256": "c947408128449643e82bbbcf204e16f58f7e0106a471044d67b367a0f809f90d", + "resolution": "resolved", + "usage": "call" + } + ] + }, + "microcosm.build.us_runtime.puf55_survey_recipients::_result_stamp": { + "basis": "reviewed entry/caller-chain body", + "body_sha256": "91d9f535290cdd2c951777f0c35aa3dae473e1f071c854e4559a3c09ada72268", + "references": [ + { + "caller": "microcosm.build.us_runtime.graph_puf55_survey_recipients::_check_values", + "expression": "values._result_stamp(qualified)", + "expression_sha256": "e0c72fe06d97d00901362d7e934b77aa539b0ebae40c95c01634d6c4f79773f9", + "resolution": "resolved", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.puf55_survey_recipients::qualify_puf55_survey_recipients", + "expression": "_result_stamp(result)", + "expression_sha256": "b1deabd84fa6f6bde9a275d7d879b5b3cdc89b2594ed0f04b2df7ec6fcea79cb", + "resolution": "resolved", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.puf55_survey_recipients::qualify_puf55_survey_recipients", + "expression": "_result_stamp(result)", + "expression_sha256": "b1deabd84fa6f6bde9a275d7d879b5b3cdc89b2594ed0f04b2df7ec6fcea79cb", + "resolution": "resolved", + "usage": "call" + } + ] + }, + "microcosm.build.us_runtime.puf55_survey_recipients::qualify_puf55_survey_recipients": { + "basis": "reviewed entry/caller-chain body", + "body_sha256": "887d1ff21852557f43abf1d7ade7c765b0bd7c288d5f4a7cb70fe31b91bffd6c", + "references": [ + { + "caller": "microcosm.build.us_runtime.graph_puf55_survey_recipients::_Kernel.run", + "expression": "values.qualify_puf55_survey_recipients(run)", + "expression_sha256": "808328bff07759d1cedcf7b487d5aa962f3152f6f9e134d73b2d6548da34b65b", + "resolution": "resolved", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.graph_puf55_survey_recipients::verify_materialized_puf55_survey_recipients", + "expression": "values.qualify_puf55_survey_recipients(financial_run)", + "expression_sha256": "eabaede2e49a350456d4786efba1ebad415a80ba0786f6155410235374795b4c", + "resolution": "resolved", + "usage": "call" + } + ] + }, + "microcosm.build.us_runtime.puf55_survey_ss_measurement::@COLUMNS": { + "basis": "finite selector/validator definition", + "body_sha256": "2ae2a1aa719d0c81889209d4a97e9ab2f734f5b72da700cb27da170c82f58cfc", + "references": [] + }, + "microcosm.build.us_runtime.puf55_survey_ss_measurement::@KNOWN": { + "basis": "finite selector/validator definition", + "body_sha256": "4b417c8ec706363a11aba1515644a86a9a5a45fce8911dc59c56a5bc70248e3f", + "references": [] + }, + "microcosm.build.us_runtime.puf55_survey_ss_measurement::@MEASUREMENT": { + "basis": "finite selector/validator definition", + "body_sha256": "6a4aca021e5ff060feb377363304517597bc24652f1e9fb2b731a9d9980af62d", + "references": [] + }, + "microcosm.build.us_runtime.puf55_survey_ss_measurement::@PROTOCOL": { + "basis": "finite selector/validator definition", + "body_sha256": "bc48b3f7344a22979f81fcabd30328123a6f4a45e2a1818f9437d08bd3696b22", + "references": [] + }, + "microcosm.build.us_runtime.puf55_survey_ss_measurement::@ROUTE": { + "basis": "finite selector/validator definition", + "body_sha256": "9eb9467db5b48a5d8a51979fa3d7278beb90420a996d2bcdb10ab61fa9cb5ce6", + "references": [] + }, + "microcosm.build.us_runtime.puf55_survey_ss_measurement::@TOTAL": { + "basis": "finite selector/validator definition", + "body_sha256": "9658c8566d4a24d1790e624bf14f9bf2b5abaf28524981ab8ff206608bd92228", + "references": [] + }, + "microcosm.build.us_runtime.puf55_survey_ss_measurement::@_PERSON_COLUMNS": { + "basis": "finite selector/validator definition", + "body_sha256": "0866140552e36551d4f97208c4914e24024039cf0356a1f39b71c9d223a15dde", + "references": [] + }, + "microcosm.build.us_runtime.puf55_survey_ss_measurement::@_REPORT_COLUMNS": { + "basis": "finite selector/validator definition", + "body_sha256": "e8f54af94005b393ba4f700bd4a34f60518decbc4eec9b0d2013e009351e5c19", + "references": [] + }, + "microcosm.build.us_runtime.puf55_survey_ss_measurement::@_STATUSES": { + "basis": "finite selector/validator definition", + "body_sha256": "dc57a7f5a57ec543272496f5243eca0158c2132f8b3553136d2a01cf00a753c5", + "references": [] + }, + "microcosm.build.us_runtime.puf55_survey_ss_measurement::@_UNIT_COLUMNS": { + "basis": "finite selector/validator definition", + "body_sha256": "26e40fe8a76e978a5956f56fb41847f104068b204927b5784ec9589eb7b5d0b4", + "references": [] + }, + "microcosm.build.us_runtime.puf55_survey_ss_measurement::qualify_puf55_survey_ss_measurement": { + "basis": "reviewed entry/caller-chain body", + "body_sha256": "d5f5ecc948a72f70fd51dd68207bc396b47af4fb728d5fb903da72414ff9b628", + "references": [] + }, + "microcosm.build.us_runtime.puf59_canonical::@PREFIX_NAMES": { + "basis": "finite selector/validator definition", + "body_sha256": "84a10ea51a597376c918a384344cd0c8f8b228df70acc855320df1ff9eade106", + "references": [] + }, + "microcosm.build.us_runtime.puf59_canonical::construct_canonical_puf59": { + "basis": "reviewed entry/caller-chain body", + "body_sha256": "351ab8682ecc7a40a4d8449f1596f870b524c378b34f8a4a5f4ea7341c2349fe", + "references": [ + { + "caller": "microcosm.build.us_runtime.graph_puf55_canonical_donor::CanonicalPuf55DonorKernel.run", + "expression": "canonical.construct_canonical_puf59(decoded, interest_bands=interest.US_PUF_E19200_AGI_BANDS, interest_asset_sha256=canonical.INTEREST_ASSET_SHA256, seed=seed, growth_scheme=scheme)", + "expression_sha256": "67e3433ec283a7dcb660583207689d20a2bf0f8cbdcb2911cdd5177c5bba96e2", + "resolution": "resolved", + "usage": "call" + } + ] + }, + "microcosm.build.us_runtime.puf59_canonical_artifact::@INTEGERS": { + "basis": "finite selector/validator definition", + "body_sha256": "2e9fdc19617e3185a6c3bfabad1107a866b06c9b852bdd46a7215595a215a0d2", + "references": [] + }, + "microcosm.build.us_runtime.puf59_canonical_artifact::@MAGIC": { + "basis": "finite selector/validator definition", + "body_sha256": "35c57571afb51b05d108cb8397b44ee31786fce4bdc3891e6e76185b058d11aa", + "references": [] + }, + "microcosm.build.us_runtime.puf59_canonical_artifact::@MAX_BODY": { + "basis": "finite selector/validator definition", + "body_sha256": "66e28562615358e1936d54517a152463eb0d8e65f7fde143b262b73f51043f4b", + "references": [] + }, + "microcosm.build.us_runtime.puf59_canonical_artifact::@MAX_HEADER": { + "basis": "finite selector/validator definition", + "body_sha256": "7bd518225ff59f0f6c9015261721adcba6dbdca3c5c2a172250aa55806416daf", + "references": [] + }, + "microcosm.build.us_runtime.puf59_canonical_artifact::@NAMES": { + "basis": "finite selector/validator definition", + "body_sha256": "3514391bfd2283b54d2825b61bb0e336a1c6e5d44d2ca5de50248d40e42ec3dc", + "references": [] + }, + "microcosm.build.us_runtime.puf59_canonical_artifact::@PREFIX": { + "basis": "finite selector/validator definition", + "body_sha256": "5321abd1c1f85ab8e276973a7f9b9ab2fe87fb8455f2e3cee43ade8d52980c78", + "references": [] + }, + "microcosm.build.us_runtime.puf59_canonical_artifact::_bindings": { + "basis": "reviewed entry/caller-chain body", + "body_sha256": "9ad9504aef905d8bbabbc5c01a375748903137281b8441d5cde891479b5a930c", + "references": [ + { + "caller": "microcosm.build.us_runtime.puf59_canonical_artifact::_encode", + "expression": "_bindings(arrays, r)", + "expression_sha256": "67fc7236cd007f46d34c256c93e30d3cd8fb5dd3e2160216db25f6cd6eecb371", + "resolution": "resolved", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.puf59_canonical_artifact::decode_canonical_puf59", + "expression": "_bindings(arrays, receipt)", + "expression_sha256": "0e73c56162a9c69e9217373fa58e53d6db6072a6c06ca6d379603d8d0001de98", + "resolution": "resolved", + "usage": "call" + } + ] + }, + "microcosm.build.us_runtime.puf59_canonical_artifact::encode_canonical_puf59": { + "basis": "reviewed entry/caller-chain body", + "body_sha256": "0bde9e19a04628c55cb317562193ee5f8e75f0709d1bf268753623abbfc0ac59", + "references": [ + { + "caller": "microcosm.build.us_runtime.graph_puf55_canonical_donor::CanonicalPuf55DonorKernel.run", + "expression": "envelope.encode_canonical_puf59(constructed, expected_growth_scheme=scheme)", + "expression_sha256": "17fa46532b9d1db3a2a25ff9ffaa5ca25890a466fdf16951f4dd3bb2d3cc4a74", + "resolution": "resolved", + "usage": "call" + } + ] + }, + "microcosm.build.us_runtime.puf59_canonical_artifact::reencode_canonical_puf59": { + "basis": "reviewed entry/caller-chain body", + "body_sha256": "0b834ecdd77d48097ec3a3dea01fabcbdaf94c54cdc8dfb1b128c77663492e7a", + "references": [] + }, + "microcosm.build.us_runtime.puf_detail_transfer::@ARRAY_DTYPES": { + "basis": "finite selector/validator definition", + "body_sha256": "8f282b978c3a2adbcd3d3defa508e6022d0d79a3aeb945fc6b79b9d778d60349", + "references": [] + }, + "microcosm.build.us_runtime.puf_detail_transfer::@FEATURES": { + "basis": "finite selector/validator definition", + "body_sha256": "4b88fec1ae5d7b15bb9abee1076dcd4cd3f714315739120048b3d773cf1538cf", + "references": [] + }, + "microcosm.build.us_runtime.puf_detail_transfer::@HEADER_FIELDS": { + "basis": "finite selector/validator definition", + "body_sha256": "0a4970fd247da54010e199ed63b30d1bdbd62185259d9a0101f505302c66a070", + "references": [] + }, + "microcosm.build.us_runtime.puf_detail_transfer::@MAGIC": { + "basis": "finite selector/validator definition", + "body_sha256": "8cb97fbc54b865ad90d1af10d3295b831343ad80386504de95e21401298027c9", + "references": [] + }, + "microcosm.build.us_runtime.puf_detail_transfer::@MARS": { + "basis": "finite selector/validator definition", + "body_sha256": "01b04136e83610b5e8ddb86ea12c0ab53be37e107847fcca364224e5314fa84e", + "references": [] + }, + "microcosm.build.us_runtime.puf_detail_transfer::@MASK": { + "basis": "finite selector/validator definition", + "body_sha256": "8b6b8730f79dcca7981898a47e5b2de9c0e6c28c5f6db411e2b51b2b9ee57d97", + "references": [] + }, + "microcosm.build.us_runtime.puf_detail_transfer::@MAX_BYTES": { + "basis": "finite selector/validator definition", + "body_sha256": "096eb23870dc9cf1069b2c9db5d10ba71e5e3d6aae9d4db8803d6962fe4f6ef9", + "references": [] + }, + "microcosm.build.us_runtime.puf_detail_transfer::@MAX_ROWS": { + "basis": "finite selector/validator definition", + "body_sha256": "a9f1c4cae067eff5221fe56194472db91da487356cc4d33effc0a500fa51384a", + "references": [] + }, + "microcosm.build.us_runtime.puf_detail_transfer::@MONEY_FIELDS": { + "basis": "finite selector/validator definition", + "body_sha256": "f31517e37f88a94383fa8e536ca263eec05e762937d9788df0e81ed9c1a72caa", + "references": [] + }, + "microcosm.build.us_runtime.puf_detail_transfer::@OUTPUT": { + "basis": "finite selector/validator definition", + "body_sha256": "798aad7da605d0a2ebeab64615ad019e363733f516d17caf98f20eee5566380c", + "references": [] + }, + "microcosm.build.us_runtime.puf_detail_transfer::@SCOPE": { + "basis": "finite selector/validator definition", + "body_sha256": "1ba2082fc41ccf2b3a7d5a1878484e4893010cb99121df45d0a096ab5c77ddf7", + "references": [] + }, + "microcosm.build.us_runtime.puf_detail_transfer::@SOURCE_ADMISSION": { + "basis": "finite selector/validator definition", + "body_sha256": "f59bfc099e91a06fb68706caed85a2934b6ea7941ad36ec0edd2ece1592aee9f", + "references": [] + }, + "microcosm.build.us_runtime.puf_detail_transfer::@TARGET": { + "basis": "finite selector/validator definition", + "body_sha256": "12362930b1ece0911e0f8b6bc35d9264b8f9ccdcee432e7754cbbe735f071294", + "references": [] + }, + "microcosm.build.us_runtime.puf_detail_transfer::donor_frame": { + "basis": "reviewed entry/caller-chain body", + "body_sha256": "3a6abbb514377c4ca46ca3af28d3ff824dbdc06d888959df40cb638864e5c1e9", + "references": [ + { + "caller": "microcosm.build.us_runtime.graph_puf_detail_transfer::FixtureDonorKernel.run", + "expression": "detail.donor_frame(arrays, status, decoded, projection)", + "expression_sha256": "ab9978963a32b59821dbd3bc62d598261b516049432dd8324fe5eceef7e0cc00", + "resolution": "resolved", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.graph_puf_detail_transfer::FixturePriceExportKernel.run", + "expression": "detail.donor_frame(checked, status, decoded, projection)", + "expression_sha256": "93062a368515dc3ae0a331a98c1bbe694e13db02d6b7368ff9ba23582ebd8a3f", + "resolution": "resolved", + "usage": "call" + } + ] + }, + "microcosm.build.us_runtime.puf_diagnostic_consumer::recipient_matrix": { + "basis": "reviewed entry/caller-chain body", + "body_sha256": "86a0212138ae1442f3f39475a38d4165aeab4306ef86df86b94b83b280658ead", + "references": [ + { + "caller": "microcosm.build.us_runtime.graph_puf_diagnostic_consumer::DiagnosticAttachKernel", + "expression": "consumer.recipient_matrix", + "expression_sha256": "2b02d66b5d929a01ec63dd1dc33c8b4dd3065e977a12ef26421cf1a758f50de7", + "resolution": "resolved", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.graph_puf_diagnostic_consumer::DiagnosticMatrixKernel", + "expression": "consumer.recipient_matrix", + "expression_sha256": "2b02d66b5d929a01ec63dd1dc33c8b4dd3065e977a12ef26421cf1a758f50de7", + "resolution": "resolved", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.puf_diagnostic_consumer::qualify_host_population", + "expression": "recipient_matrix(frame)", + "expression_sha256": "ad231c686284707e98f7c503fb1bb58c5f750ae4f439ea755bef2680f39c9a16", + "resolution": "resolved", + "usage": "call" + } + ] + }, + "microcosm.build.us_runtime.puf_full_source::@AGGREGATE_LEXEME_MAX_CHARACTERS": { + "basis": "finite selector/validator definition", + "body_sha256": "1acce73015af549f88fbe8b6f0cdefd8aca1655551be5575ebd32e8f9307dde3", + "references": [] + }, + "microcosm.build.us_runtime.puf_full_source::@COUNT_COLUMNS": { + "basis": "finite selector/validator definition", + "body_sha256": "8f5b9f1a92e79e91b46b7630011e352ef932545b389c9e31ac68ffa16c27ec1f", + "references": [] + }, + "microcosm.build.us_runtime.puf_full_source::@DEPENDENT_COLUMNS": { + "basis": "finite selector/validator definition", + "body_sha256": "1960c2178ea2956231cc3e7b8229089a1d34a98a1e090e6a016c5914c8f94ea6", + "references": [] + }, + "microcosm.build.us_runtime.puf_full_source::@DIRECT_MAPPINGS": { + "basis": "finite selector/validator definition", + "body_sha256": "3e1687d12b31c2add2c2108e598557770e581652b44b088494461fb0962273cb", + "references": [] + }, + "microcosm.build.us_runtime.puf_full_source::@FULL_SOURCE_MAX_BYTES": { + "basis": "finite selector/validator definition", + "body_sha256": "ef54391a51199baf5e2dd1c9aea072acb8ad0c2e628ac111489d3206e93f9661", + "references": [] + }, + "microcosm.build.us_runtime.puf_full_source::@FULL_SOURCE_VERSION": { + "basis": "finite selector/validator definition", + "body_sha256": "2ede99401659739e5306c646f888cbcb296ae567cc8cda19ebcda9479f4b4c8c", + "references": [] + }, + "microcosm.build.us_runtime.puf_full_source::@MONEY_COLUMNS": { + "basis": "finite selector/validator definition", + "body_sha256": "c1ea6648fee13f0d86498d81224236618e06435402cb2dfef9173e89ba860a58", + "references": [] + }, + "microcosm.build.us_runtime.puf_full_source::@OUTSIDE_AMOUNT_UNIVERSE": { + "basis": "finite selector/validator definition", + "body_sha256": "0251cdbd78e3e2bf8bed86f14afb0c9e71341d518001f55f00d4d7ea33b8d65a", + "references": [] + }, + "microcosm.build.us_runtime.puf_full_source::@PROJECTED_COLUMNS": { + "basis": "finite selector/validator definition", + "body_sha256": "432c1bf3c4d36ab4dc33cd74ac7977eebf52febac6e61cbb7bd4020f85e3591f", + "references": [] + }, + "microcosm.build.us_runtime.puf_full_source::@_AGGREGATE": { + "basis": "finite selector/validator definition", + "body_sha256": "0b2ab177fd5d6458801b1cb08cac1396c1ebdb1740732e746beb013c8f5ce961", + "references": [] + }, + "microcosm.build.us_runtime.puf_full_source::@_COUNT": { + "basis": "finite selector/validator definition", + "body_sha256": "8fc466f89f241365e2dcd014f888e49b995e4d404cebdc5c3feb81d5c8b69079", + "references": [] + }, + "microcosm.build.us_runtime.puf_full_source::@_HEADER_LIMIT": { + "basis": "finite selector/validator definition", + "body_sha256": "65fd3fd4c3ce3a85bcb46ca41bbd3dff5423bdfdd190faa41e4fc66938558f79", + "references": [] + }, + "microcosm.build.us_runtime.puf_full_source::@_MAGIC": { + "basis": "finite selector/validator definition", + "body_sha256": "4331e5057070b8d3290e36c642e7a956b2616641d8ca8e377e94ecf4bcb41696", + "references": [] + }, + "microcosm.build.us_runtime.puf_full_source::@_MONEY": { + "basis": "finite selector/validator definition", + "body_sha256": "82e63f8c67125151763b28d623d5aeb165602dce1f3c78bd16d79348546833cc", + "references": [] + }, + "microcosm.build.us_runtime.puf_growth::@PROVENANCE_RECID_COLUMN": { + "basis": "finite selector/validator definition", + "body_sha256": "50ca64af09110c0f4faf354179f70b00549e5b1be6e9892f562f188963889fc0", + "references": [] + }, + "microcosm.build.us_runtime.puf_growth::@PROVENANCE_SOURCE_AGI_COLUMN": { + "basis": "finite selector/validator definition", + "body_sha256": "b953b317334a2a02bbbd0f5c96944cfe085c8f888abb2326695a9f8b6bf54a50", + "references": [] + }, + "microcosm.build.us_runtime.puf_growth::GrownPufTable.receipt": { + "basis": "reviewed entry/caller-chain body", + "body_sha256": "a3095838ad758393da34429867d39d57fc9bd1066646ef301f8191ae1fd3d452", + "references": [ + { + "caller": "microcosm.build.us_runtime.asec_current_money_units::CurrentMoneyTaxUnitResult.validate", + "expression": "self._student_source.receipt", + "expression_sha256": "b07ef0768f9115dfd6c2ccf2e0558c015159bc578c6b1c1dad955a3332bba05a", + "resolution": "unresolved-us-candidate", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.asec_demographic_source::DemographicSourceAttachedAsec.validate", + "expression": "self.observations.receipt", + "expression_sha256": "1197f225000198b5a1cb0313c842ca1a66604dc55e5d85ef3cbf846c336804a3", + "resolution": "unresolved-us-candidate", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.asec_housing_status_source::HousingStatusAttachedAsec.validate", + "expression": "self.tax_result.receipt", + "expression_sha256": "fb31b06dffb7494bb98766aeff9f28da9035003318144f4a796917c7e4f17b01", + "resolution": "unresolved-us-candidate", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.asec_prepared_source::PreparedAsecPopulation.receipt_payload", + "expression": "self.receipt", + "expression_sha256": "e092d770e10b3e310aa85b322b534aa524e35d368e3ddd2eb1665852865903b2", + "resolution": "unresolved-us-candidate", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.multispine_pool::PoolStageOutput.__post_init__", + "expression": "self.receipt", + "expression_sha256": "e092d770e10b3e310aa85b322b534aa524e35d368e3ddd2eb1665852865903b2", + "resolution": "unresolved-us-candidate", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.stacked_spine::StackedSpineHarmonization.__post_init__", + "expression": "self.receipt", + "expression_sha256": "e092d770e10b3e310aa85b322b534aa524e35d368e3ddd2eb1665852865903b2", + "resolution": "unresolved-us-candidate", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.stacked_spine::StackedSpinePreparation.__post_init__", + "expression": "self.receipt", + "expression_sha256": "e092d770e10b3e310aa85b322b534aa524e35d368e3ddd2eb1665852865903b2", + "resolution": "unresolved-us-candidate", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.stacked_spine::StackedSpinePreparation.__post_init__", + "expression": "self.receipt", + "expression_sha256": "e092d770e10b3e310aa85b322b534aa524e35d368e3ddd2eb1665852865903b2", + "resolution": "unresolved-us-candidate", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.stacked_spine::StackedSpinePreparation.__post_init__", + "expression": "self.receipt", + "expression_sha256": "e092d770e10b3e310aa85b322b534aa524e35d368e3ddd2eb1665852865903b2", + "resolution": "unresolved-us-candidate", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.stacked_spine::StackedSpineResult.__post_init__", + "expression": "self.receipt", + "expression_sha256": "e092d770e10b3e310aa85b322b534aa524e35d368e3ddd2eb1665852865903b2", + "resolution": "unresolved-us-candidate", + "usage": "reference" + } + ] + }, + "microcosm.build.us_runtime.puf_growth::GrownPufTable.receipt_bytes": { + "basis": "reviewed entry/caller-chain body", + "body_sha256": "0ed62f7a7d88dff8e188117a5fdd2c593e2e319e3e54115a9b69d7302fc1f506", + "references": [] + }, + "microcosm.build.us_runtime.puf_price_baseline::price_baseline_invariants": { + "basis": "reviewed entry/caller-chain body", + "body_sha256": "ea7e1d39ef9af1450e506352a8b0fe2edd96856d9e08af65b07ac36a6c24bd24", + "references": [] + }, + "microcosm.build.us_runtime.puf_qbi_model::model_full_puf_qbi": { + "basis": "reviewed entry/caller-chain body", + "body_sha256": "90028b39bacca476e75182f47a5ffa6f57c1d7a0741fe544a5ccff4bce0967de", + "references": [ + { + "caller": "microcosm.build.us_runtime.puf59_canonical::construct_canonical_puf59", + "expression": "qbi.model_full_puf_qbi(canonical, status['RECID'], known=known, input_money_year=2015, seed=seed, employment_calibration=qbi_employment_calibration)", + "expression_sha256": "80b5680346e211311deeb6e9a703628c8c57ec8985e1262cb6f8f48d4f47a21c", + "resolution": "resolved", + "usage": "call" + } + ] + }, + "microcosm.build.us_runtime.puf_support::@PUF_TAX_DETAIL_DEFAULT_PREDICTORS": { + "basis": "finite selector/validator definition", + "body_sha256": "e9c2db5d080f568f60154957be6d39dce656680fc41ebc2b26bfae21e791b375", + "references": [] + }, + "microcosm.build.us_runtime.puf_support::@PUF_TAX_DETAIL_SOCIAL_SECURITY_COMPONENT_OUTPUTS": { + "basis": "finite selector/validator definition", + "body_sha256": "5acd70c29dd6dd4cbf7c2c4ce51a7365c252add3b61d5e88468ca866f1f4b2da", + "references": [] + }, + "microcosm.build.us_runtime.puf_support::@_PUF_TAX_DETAIL_BOOLEAN_PERSON_OUTPUTS": { + "basis": "finite selector/validator definition", + "body_sha256": "3d9d8442cca6fb9232a1fd3b80ac3021505cc9391a035fe30368d807ab7b1ba0", + "references": [] + }, + "microcosm.build.us_runtime.puf_support::@_PUF_TAX_DETAIL_DISCRETE_TAX_UNIT_OUTPUTS": { + "basis": "finite selector/validator definition", + "body_sha256": "533ab5993e5ba77a153cd9415df73c177625b24856161fb634aa7120b707b43a", + "references": [] + }, + "microcosm.build.us_runtime.puf_support::PufTaxDetailChainInputs.target_order": { + "basis": "reviewed entry/caller-chain body", + "body_sha256": "5e728e43ade91fd351b1109e7d93195abcb77bafb615beadd29ab4c140f84111", + "references": [] + }, + "microcosm.build.us_runtime.puf_target2024_growth::@INCIDENCE_FIELDS": { + "basis": "finite selector/validator definition", + "body_sha256": "cb0adb0b46262e55963abc096f7ae688cab342032d76c88f93087b90ffe8f493", + "references": [] + }, + "microcosm.build.us_runtime.puf_target2024_growth::@OUTPUTS": { + "basis": "finite selector/validator definition", + "body_sha256": "9f4be494c085a52705461112c5787af374ea5c416d67609352cd5588a0d74961", + "references": [] + }, + "microcosm.build.us_runtime.puf_target2024_growth::@RECIPE_SHA256": { + "basis": "finite selector/validator definition", + "body_sha256": "c59fbb49935826bbaf5f291037361b57d70d29e442ba02e25835d64cb23497c1", + "references": [] + }, + "microcosm.build.us_runtime.puf_target2024_growth::@VERSION": { + "basis": "finite selector/validator definition", + "body_sha256": "ca65ce01c029bc2a39296880aaa9cd0cc5295ce6b542b5c3eb0a644bbcbccc16", + "references": [] + }, + "microcosm.build.us_runtime.puf_target2024_growth::@_RECIPE": { + "basis": "finite selector/validator definition", + "body_sha256": "be9ac8dcc3ce2f13bbc032696251d8040a19521fcf92ec2d4a61f60aa6ae17d3", + "references": [] + }, + "microcosm.build.us_runtime.puf_target2024_growth::@_RECIPE_JSON": { + "basis": "finite selector/validator definition", + "body_sha256": "8cf7a33e52e9ff90ded5b57c512bdf2ea06eaa83524a07d42a2a1e3425cc6631", + "references": [] + }, + "microcosm.build.us_runtime.spine_assembly::SpinePreparation.__post_init__": { + "basis": "reviewed entry/caller-chain body", + "body_sha256": "1c5e3695e9dd6334629eaebee45ce4ef095b5b49a3d6a28d9ac27a8ce7c09533", + "references": [] + }, + "microcosm.build.us_runtime.stacked_spine::StackedSpineHarmonization.__post_init__": { + "basis": "reviewed entry/caller-chain body", + "body_sha256": "da16dc40eeff680886d0759d2ad50b70872f46066aea517c8796654d4c2ad4dc", + "references": [] + }, + "microcosm.build.us_runtime.stacked_spine::StackedSpinePreparation.__post_init__": { + "basis": "reviewed entry/caller-chain body", + "body_sha256": "db610c4f913144dab2bb1d540c86101aa98ebe35080421093b61dc1b760278de", + "references": [] + }, + "microcosm.build.us_runtime.stacked_spine::StackedSpineResult.__post_init__": { + "basis": "reviewed entry/caller-chain body", + "body_sha256": "88d353a057fdc957a8a321da5a0d86778a84a514bdc126ac464d3502e0cb8b7a", + "references": [] + }, + "microcosm.build.us_runtime.survey_age_calibration::_run_survey_age_calibration": { + "basis": "reviewed entry/caller-chain body", + "body_sha256": "a8bf142b1acdccd9c6aa718517c2f2e831b8441aac335a3d27d1c6a7c7cd959b", + "references": [ + { + "caller": "microcosm.build.us_runtime.survey_age_calibration::run_survey_age_calibration", + "expression": "_run_survey_age_calibration(source_dir, snapshot_root=snapshot_root, store_root=store_root, fraction=fraction, seed_value=seed_value, epochs=epochs, learning_rate=learning_rate, resume=resume, calibration=calibration, geography_config=geography_config)", + "expression_sha256": "da835f5f4451f8fb3aa3096437950055e8ae86b23f5584ab702a14111aec4a42", + "resolution": "resolved", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.survey_age_calibration::run_survey_age_development", + "expression": "_run_survey_age_calibration(source_dir, snapshot_root=snapshot_root, store_root=store_root, fraction=fraction, seed_value=seed_value, epochs=epochs, learning_rate=learning_rate, resume=resume, calibration=calibration, age_source_dir=age_source_dir, activation=activation, geography_config=geography_config)", + "expression_sha256": "8d0fe815488fe8dbaf39bd367255e1f0540ddadf640dfb3a0d31e578bb9fb074", + "resolution": "resolved", + "usage": "call" + } + ] + }, + "microcosm.build.us_runtime.survey_age_calibration::_run_survey_age_calibration.observe": { + "basis": "reviewed entry/caller-chain body", + "body_sha256": "d6d7b817e6845871ed7eb38a5b89d85a193f0b101319329299eae0a2a9de3f97", + "references": [ + { + "caller": "microcosm.build.us_runtime.survey_age_calibration::_run_survey_age_calibration", + "expression": "observe", + "expression_sha256": "274f69be16100e8e78059234777f281731e85cc7e8d8ea6f15a066520f4f89c8", + "resolution": "resolved", + "usage": "reference" + } + ] + }, + "microcosm.build.us_runtime.survey_age_calibration::run_survey_age_calibration": { + "basis": "reviewed entry/caller-chain body", + "body_sha256": "32d10340a8e4037b21ed1cb8852507cbacaa3120f431c431a49451646bc0bbbb", + "references": [] + }, + "microcosm.build.us_runtime.survey_age_calibration::run_survey_age_development": { + "basis": "reviewed entry/caller-chain body", + "body_sha256": "c90a910ca78df2157c5143a328f167a4e48bd8e9d1195f8c6575c562684bf8c0", + "references": [] + }, + "microcosm.build.us_runtime.survey_atomic_geography::_raw_allocation": { + "basis": "reviewed entry/caller-chain body", + "body_sha256": "9221af93a1ce811e2d18c9e4efbfbb88930304328bf723122a484f239f5cdf1c", + "references": [ + { + "caller": "microcosm.build.us_runtime.survey_atomic_geography::reconstruct_atomic_survey_geography", + "expression": "_raw_allocation(view, allocated_population)", + "expression_sha256": "0aaf985425c1e4891910bad45c38145adb1bb15d0aa757956c01caca577ab5e2", + "resolution": "resolved", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.survey_atomic_geography::reconstruct_atomic_survey_geography", + "expression": "_raw_allocation(view, allocated_population)", + "expression_sha256": "0aaf985425c1e4891910bad45c38145adb1bb15d0aa757956c01caca577ab5e2", + "resolution": "resolved", + "usage": "call" + } + ] + }, + "microcosm.build.us_runtime.survey_atomic_geography::_read_support": { + "basis": "reviewed entry/caller-chain body", + "body_sha256": "123067004008b895435e7713b127b3ed366c5010e01eeea828ef4961796ba4a4", + "references": [ + { + "caller": "microcosm.build.us_runtime.current_survey_predictors::qualify_current_survey_predictors", + "expression": "host.survey_budget.geography._read_support(geography_config)", + "expression_sha256": "ee87632a20abcf146af5a0759e4e065b8dbd679177ece2e66c026ead171025bb", + "resolution": "unresolved-us-candidate", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.survey_age_calibration::_run_survey_age_calibration", + "expression": "atomic_source.reconstruction._read_support(geography_config)", + "expression_sha256": "3ffce902ac9bd83a3c8fca4892ddadd4143fd615d649f34e2fd62711a7793ffb", + "resolution": "unresolved-us-candidate", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.survey_atomic_geography::reconstruct_atomic_survey_geography", + "expression": "_read_support(config)", + "expression_sha256": "65b05d9ada5ff6aecdf472f00bc81db79d1313f78c31f57dbc0d6e138a750d11", + "resolution": "resolved", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.survey_atomic_geography::reconstruct_atomic_survey_geography", + "expression": "_read_support(config)", + "expression_sha256": "65b05d9ada5ff6aecdf472f00bc81db79d1313f78c31f57dbc0d6e138a750d11", + "resolution": "resolved", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.survey_origin_budget::_final_budget_state", + "expression": "geography._read_support(state.geography_config)", + "expression_sha256": "0862a9e0c8ba3445268c5fb039c17e270447f814fd8c4759776e7285cb4828b4", + "resolution": "resolved", + "usage": "call" + } + ] + }, + "microcosm.build.us_runtime.survey_atomic_geography::reconstruct_atomic_survey_geography": { + "basis": "reviewed entry/caller-chain body", + "body_sha256": "9761183ef0e04aec07d08f42dcb3f0eb1f422fece48dcd174574c0b682b67dc7", + "references": [ + { + "caller": "microcosm.build.us_runtime.graph_atomic_survey_financial::run_atomic_survey_financial", + "expression": "reconstruction.reconstruct_atomic_survey_geography(prefix.preparation, prefix.allocated_population, geography_config)", + "expression_sha256": "ee5f403ba4123200b042b60bab83aa6e97bc554958dbb5413280ee1832e55df3", + "resolution": "unresolved-us-candidate", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.graph_atomic_survey_population::run_atomic_survey_population", + "expression": "reconstruction.reconstruct_atomic_survey_geography(prefix.preparation, prefix.allocated_population, geography_config)", + "expression_sha256": "ee5f403ba4123200b042b60bab83aa6e97bc554958dbb5413280ee1832e55df3", + "resolution": "resolved", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.graph_atomic_survey_population::run_atomic_survey_population", + "expression": "reconstruction.reconstruct_atomic_survey_geography(prefix.preparation, prefix.allocated_population, geography_config)", + "expression_sha256": "ee5f403ba4123200b042b60bab83aa6e97bc554958dbb5413280ee1832e55df3", + "resolution": "resolved", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.survey_age_calibration::_run_survey_age_calibration", + "expression": "atomic_source.reconstruction.reconstruct_atomic_survey_geography(prefix.preparation, prefix.allocated_population, geography_config)", + "expression_sha256": "57d708f48be7b422ede7462770362e91d985287f07d918d9b200a3a2738f4a7a", + "resolution": "unresolved-us-candidate", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.survey_origin_budget::_initial", + "expression": "geography.reconstruct_atomic_survey_geography(preparation, allocated, geography_config)", + "expression_sha256": "0405c87630493b001f63c06cc897d277f34659db86b46e68d1f8ba55ccf83659", + "resolution": "resolved", + "usage": "call" + } + ] + }, + "microcosm.build.us_runtime.survey_financial_successor::SamplingOriginFinancialSuccessor.checked_view": { + "basis": "reviewed entry/caller-chain body", + "body_sha256": "04a3f6ce3bf0d883bd64e490ef3df22ddd91a06366dc67fb0d187d443fe3dffe", + "references": [ + { + "caller": "microcosm.build.us_runtime.survey_financial_successor::SamplingOriginFinancialSuccessor.to_bytes", + "expression": "self.checked_view()", + "expression_sha256": "20deed1e57f9a3029d2ca06d9aa3bc02c017456630bda13a63a7346e368821c7", + "resolution": "resolved", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.survey_financial_successor::admit_survey_financial_population", + "expression": "budget.checked_view()", + "expression_sha256": "303121f839037c123a3907a3ba1100bc5c224834bb11b565bbec767a9078485e", + "resolution": "unresolved-us-candidate", + "usage": "call" + } + ] + }, + "microcosm.build.us_runtime.survey_financial_successor::SamplingOriginFinancialSuccessor.to_bytes": { + "basis": "reviewed entry/caller-chain body", + "body_sha256": "cae2f1f3bdd3da635c49a6dc45daad8b3de4e0f3cc90d68e7b492d1150e21780", + "references": [ + { + "caller": "microcosm.build.us_runtime.survey_age_calibration::_run_survey_age_calibration", + "expression": "atomic_source.reconstruction.AtomicSurveyReconstruction.to_bytes(geography_config)", + "expression_sha256": "335e8b0f924d999b5c7198f3d9d6f20380d5cfc4c0903bcb8a5e60fab14abefc", + "resolution": "unresolved-us-candidate", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.survey_age_calibration::_run_survey_age_calibration", + "expression": "atomic_source.reconstruction.AtomicSurveyReconstruction.to_bytes(geography_config)", + "expression_sha256": "335e8b0f924d999b5c7198f3d9d6f20380d5cfc4c0903bcb8a5e60fab14abefc", + "resolution": "unresolved-us-candidate", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.survey_age_calibration::_run_survey_age_calibration", + "expression": "atomic_source.reconstruction.AtomicSurveyReconstruction.to_bytes(geography_config)", + "expression_sha256": "335e8b0f924d999b5c7198f3d9d6f20380d5cfc4c0903bcb8a5e60fab14abefc", + "resolution": "unresolved-us-candidate", + "usage": "call" + } + ] + }, + "microcosm.build.us_runtime.survey_financial_successor::_final_state": { + "basis": "reviewed entry/caller-chain body", + "body_sha256": "92f5a231c2e3da2a7fc08f9e212b778042a618b4812abe4b8bd17ebe5dc21d38", + "references": [ + { + "caller": "microcosm.build.us_runtime.survey_financial_successor::SamplingOriginFinancialSuccessor.checked_view", + "expression": "_final_state(state)", + "expression_sha256": "4341f4a186d07663194f474b27829d4d36273463e3366dafb92a7c6c5c93f9c1", + "resolution": "resolved", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.survey_financial_successor::admit_survey_financial_population", + "expression": "_final_state(state)", + "expression_sha256": "4341f4a186d07663194f474b27829d4d36273463e3366dafb92a7c6c5c93f9c1", + "resolution": "resolved", + "usage": "call" + } + ] + }, + "microcosm.build.us_runtime.survey_financial_successor::_pure_state": { + "basis": "reviewed entry/caller-chain body", + "body_sha256": "35f4642dd87bf0a2af4dc990de10dbf8ccd14086447b1811ccee0ffde5130b2d", + "references": [ + { + "caller": "microcosm.build.us_runtime.survey_financial_successor::SamplingOriginFinancialSuccessor.checked_view", + "expression": "_pure_state(state)", + "expression_sha256": "664ad95e9335555dc25bc3e0fc9f03b0ea2d480bcebaf47fb4babbcd129b1e9d", + "resolution": "resolved", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.survey_financial_successor::_final_state", + "expression": "_pure_state(state)", + "expression_sha256": "664ad95e9335555dc25bc3e0fc9f03b0ea2d480bcebaf47fb4babbcd129b1e9d", + "resolution": "resolved", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.survey_origin_budget::_final_successor_state", + "expression": "financial_successor._pure_state(state.previous_binding_entry[2])", + "expression_sha256": "bef5d2160035697534536f0ee220973e7e908004f29b4abe80f418af39efe5ed", + "resolution": "resolved", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.survey_origin_budget::_pure_successor_state", + "expression": "financial_successor._pure_state(state.previous_binding_entry[2])", + "expression_sha256": "bef5d2160035697534536f0ee220973e7e908004f29b4abe80f418af39efe5ed", + "resolution": "resolved", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.survey_origin_budget::_successor_document", + "expression": "financial_successor._pure_state(retained)", + "expression_sha256": "0ec610931e87383e20ff148dbb228a25071c2e9dc14ea7b9d354063499509643", + "resolution": "resolved", + "usage": "call" + } + ] + }, + "microcosm.build.us_runtime.survey_financial_successor::admit_survey_financial_population": { + "basis": "reviewed entry/caller-chain body", + "body_sha256": "747eaf9b40557ad83c35aff1fade9c0280c3fb08310541d1ea5df22000bf5318", + "references": [] + }, + "microcosm.build.us_runtime.survey_origin_budget::SamplingOriginBudget.checked_view": { + "basis": "reviewed entry/caller-chain body", + "body_sha256": "502cd7e02bd1339a60680f944fbd5ac705db9da3c27f86975b7b5538c3366b95", + "references": [ + { + "caller": "microcosm.build.us_runtime.survey_financial_successor::admit_survey_financial_population", + "expression": "budget.checked_view()", + "expression_sha256": "303121f839037c123a3907a3ba1100bc5c224834bb11b565bbec767a9078485e", + "resolution": "unresolved-us-candidate", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.survey_origin_budget::SamplingOriginBudget.to_bytes", + "expression": "self.checked_view()", + "expression_sha256": "20deed1e57f9a3029d2ca06d9aa3bc02c017456630bda13a63a7346e368821c7", + "resolution": "resolved", + "usage": "call" + } + ] + }, + "microcosm.build.us_runtime.survey_origin_budget::SamplingOriginBudget.to_bytes": { + "basis": "reviewed entry/caller-chain body", + "body_sha256": "cae2f1f3bdd3da635c49a6dc45daad8b3de4e0f3cc90d68e7b492d1150e21780", + "references": [ + { + "caller": "microcosm.build.us_runtime.survey_age_calibration::_run_survey_age_calibration", + "expression": "atomic_source.reconstruction.AtomicSurveyReconstruction.to_bytes(geography_config)", + "expression_sha256": "335e8b0f924d999b5c7198f3d9d6f20380d5cfc4c0903bcb8a5e60fab14abefc", + "resolution": "unresolved-us-candidate", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.survey_age_calibration::_run_survey_age_calibration", + "expression": "atomic_source.reconstruction.AtomicSurveyReconstruction.to_bytes(geography_config)", + "expression_sha256": "335e8b0f924d999b5c7198f3d9d6f20380d5cfc4c0903bcb8a5e60fab14abefc", + "resolution": "unresolved-us-candidate", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.survey_age_calibration::_run_survey_age_calibration", + "expression": "atomic_source.reconstruction.AtomicSurveyReconstruction.to_bytes(geography_config)", + "expression_sha256": "335e8b0f924d999b5c7198f3d9d6f20380d5cfc4c0903bcb8a5e60fab14abefc", + "resolution": "unresolved-us-candidate", + "usage": "call" + } + ] + }, + "microcosm.build.us_runtime.survey_origin_budget::SamplingOriginSuccessor.checked_view": { + "basis": "reviewed entry/caller-chain body", + "body_sha256": "205316ed77aeabd68762e9ba0910880f75a92959bbd54887aee5e9dfbb1c7e11", + "references": [ + { + "caller": "microcosm.build.us_runtime.survey_financial_successor::admit_survey_financial_population", + "expression": "budget.checked_view()", + "expression_sha256": "303121f839037c123a3907a3ba1100bc5c224834bb11b565bbec767a9078485e", + "resolution": "unresolved-us-candidate", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.survey_origin_budget::SamplingOriginSuccessor.to_bytes", + "expression": "self.checked_view()", + "expression_sha256": "20deed1e57f9a3029d2ca06d9aa3bc02c017456630bda13a63a7346e368821c7", + "resolution": "resolved", + "usage": "call" + } + ] + }, + "microcosm.build.us_runtime.survey_origin_budget::SamplingOriginSuccessor.to_bytes": { + "basis": "reviewed entry/caller-chain body", + "body_sha256": "cae2f1f3bdd3da635c49a6dc45daad8b3de4e0f3cc90d68e7b492d1150e21780", + "references": [ + { + "caller": "microcosm.build.us_runtime.survey_age_calibration::_run_survey_age_calibration", + "expression": "atomic_source.reconstruction.AtomicSurveyReconstruction.to_bytes(geography_config)", + "expression_sha256": "335e8b0f924d999b5c7198f3d9d6f20380d5cfc4c0903bcb8a5e60fab14abefc", + "resolution": "unresolved-us-candidate", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.survey_age_calibration::_run_survey_age_calibration", + "expression": "atomic_source.reconstruction.AtomicSurveyReconstruction.to_bytes(geography_config)", + "expression_sha256": "335e8b0f924d999b5c7198f3d9d6f20380d5cfc4c0903bcb8a5e60fab14abefc", + "resolution": "unresolved-us-candidate", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.survey_age_calibration::_run_survey_age_calibration", + "expression": "atomic_source.reconstruction.AtomicSurveyReconstruction.to_bytes(geography_config)", + "expression_sha256": "335e8b0f924d999b5c7198f3d9d6f20380d5cfc4c0903bcb8a5e60fab14abefc", + "resolution": "unresolved-us-candidate", + "usage": "call" + } + ] + }, + "microcosm.build.us_runtime.survey_origin_budget::_checked_successor_document": { + "basis": "reviewed entry/caller-chain body", + "body_sha256": "179f7ecd8fc70c2a3e20f284bbabd4ea67d9f6b106e37d70047661c5dc55ee99", + "references": [ + { + "caller": "microcosm.build.us_runtime.survey_origin_budget::SamplingOriginSuccessor.checked_view", + "expression": "_checked_successor_document(state, final)", + "expression_sha256": "6564ac278403ef3815d1bbf7164764800c2e9c31f054cc7a246b4ca667f91b37", + "resolution": "resolved", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.survey_origin_budget::SamplingOriginSuccessor.checked_view", + "expression": "_checked_successor_document(state, view)", + "expression_sha256": "6b42d54767466d3fab1cfda1b4b68393926bbc97f14f599a33d4e5d776d88348", + "resolution": "resolved", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.survey_origin_budget::admit_survey_weight_only_population", + "expression": "_checked_successor_document(state, final)", + "expression_sha256": "6564ac278403ef3815d1bbf7164764800c2e9c31f054cc7a246b4ca667f91b37", + "resolution": "resolved", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.survey_origin_budget::admit_survey_weight_only_population", + "expression": "_checked_successor_document(state, view)", + "expression_sha256": "6b42d54767466d3fab1cfda1b4b68393926bbc97f14f599a33d4e5d776d88348", + "resolution": "resolved", + "usage": "call" + } + ] + }, + "microcosm.build.us_runtime.survey_origin_budget::_final_budget_state": { + "basis": "reviewed entry/caller-chain body", + "body_sha256": "fc0032cdf62d9a9e30786493f843bf0e35bcc756646e28a7ac95c3dcb928c0eb", + "references": [ + { + "caller": "microcosm.build.us_runtime.survey_financial_successor::_final_state", + "expression": "budget._final_budget_state(state.budget_entry[2])", + "expression_sha256": "beba88ceef2cf47c3d700247c9f9b3f8827a90a66e448f059a2aa1c6589c37a3", + "resolution": "resolved", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.survey_origin_budget::SamplingOriginBudget.checked_view", + "expression": "_final_budget_state(entry[2])", + "expression_sha256": "4bff849d398525251e3350c5fb15406673ad5d030d6f1751af8770547f9d7701", + "resolution": "resolved", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.survey_origin_budget::_final_successor_state", + "expression": "_final_budget_state(budget_state)", + "expression_sha256": "4dda7c5c32e1a1dbfccda64f5b2c7c0c48bbc62d8a8e60c1e84668d5144d5991", + "resolution": "resolved", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.survey_origin_budget::freeze_survey_origin_budget", + "expression": "_final_budget_state(state)", + "expression_sha256": "0d19fe85c7f6677ad4325b0dcb85d9a25b03708b47fb833c8a82f507074d121f", + "resolution": "resolved", + "usage": "call" + } + ] + }, + "microcosm.build.us_runtime.survey_origin_budget::_final_successor_state": { + "basis": "reviewed entry/caller-chain body", + "body_sha256": "a8b17cbcbd7af4198a01b51f88bb8ec6fc044479e07169eafe3034728ea56286", + "references": [ + { + "caller": "microcosm.build.us_runtime.survey_origin_budget::SamplingOriginSuccessor.checked_view", + "expression": "_final_successor_state(state)", + "expression_sha256": "1e09edcecbc4c5b2031fed1e241b7e7a962dbaff208f0832100bcbbae0a6e243", + "resolution": "resolved", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.survey_origin_budget::admit_survey_weight_only_population", + "expression": "_final_successor_state(state)", + "expression_sha256": "1e09edcecbc4c5b2031fed1e241b7e7a962dbaff208f0832100bcbbae0a6e243", + "resolution": "resolved", + "usage": "call" + } + ] + }, + "microcosm.build.us_runtime.survey_origin_budget::_financial_budget_values": { + "basis": "reviewed entry/caller-chain body", + "body_sha256": "e764c8dc031c912d72bde80d4b57b4d658e1d5139349900f2648a525e5e3ba83", + "references": [ + { + "caller": "microcosm.build.us_runtime.survey_origin_budget::_successor_document", + "expression": "_financial_budget_values(state)", + "expression_sha256": "60715723b3e7f6c9d298846a83cd3fe79a05589030c4ed393cbc17d4f7348bbe", + "resolution": "resolved", + "usage": "call" + } + ] + }, + "microcosm.build.us_runtime.survey_origin_budget::_initial": { + "basis": "reviewed entry/caller-chain body", + "body_sha256": "f555a5e9dab95cfe9018309a69dd6d63f746bedfef5a4768cc79eb5bccc21468", + "references": [ + { + "caller": "microcosm.build.us_runtime.current_survey_predictors::qualify_current_survey_predictors", + "expression": "host.survey_budget._initial(view, allocated_population, clone_population, preparation=preparation, geography_config=geography_config, _with_geography_binding=True)", + "expression_sha256": "479a6e51f1171cf8943b9eb752bb0a9148ebeab01025cd92e90147e432671e2d", + "resolution": "unresolved-us-candidate", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.graph_puf_diagnostic_consumer::qualify_current_survey_host", + "expression": "survey_budget._initial(view, allocated_population, clone_population)", + "expression_sha256": "82ecbb1b797a5f20df04802a82717b0e77207fa64c03c2fe53dfef698c7a3164", + "resolution": "resolved", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.survey_origin_budget::_validate_budget", + "expression": "_initial(final, state.allocated, state.expanded, preparation=state.preparation, geography_config=state.geography_config, _with_geography_binding=True)", + "expression_sha256": "29b6788450b5068836ef5a9e07c12a15d426fcfb177cb4787a7d62f301b14470", + "resolution": "resolved", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.survey_origin_budget::_validate_budget", + "expression": "_initial(view, state.allocated, state.expanded, preparation=state.preparation, geography_config=state.geography_config, _with_geography_binding=True)", + "expression_sha256": "2a057aa8bf035d29a562a6c3c10f84f65a9ef72fda776b6e18d2eb2a14047040", + "resolution": "resolved", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.survey_origin_budget::freeze_survey_origin_budget", + "expression": "_initial(final, allocated_population, clone_population, preparation=preparation, geography_config=geography_config, _with_geography_binding=True)", + "expression_sha256": "3707e15a5e31b8f0f6c62492e86df9569e46117d80575afc2e2afb46c9760bc2", + "resolution": "resolved", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.survey_origin_budget::freeze_survey_origin_budget", + "expression": "_initial(view, allocated_population, clone_population, preparation=preparation, geography_config=geography_config, _with_geography_binding=True)", + "expression_sha256": "3b446c14b679a71320660f6def724291979bc1a6c6f32d64ed8b1285c4239f49", + "resolution": "resolved", + "usage": "call" + } + ] + }, + "microcosm.build.us_runtime.survey_origin_budget::_pure_budget_state": { + "basis": "reviewed entry/caller-chain body", + "body_sha256": "6311c503bfd596f8ab10bb2e6e6c48443d07e3857b2dbc439a7167adbceb3382", + "references": [ + { + "caller": "microcosm.build.us_runtime.survey_origin_budget::SamplingOriginBudget.checked_view", + "expression": "_pure_budget_state(entry[2])", + "expression_sha256": "c8ec4a6dc0796b00ee92d2eb507f79fe43d030a9d43b5e78c4fe83a77b0d4b5b", + "resolution": "resolved", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.survey_origin_budget::_final_budget_state", + "expression": "_pure_budget_state(state)", + "expression_sha256": "128d050cd17782ea75603cd1b1a0f679be8d4cf2ba32e60bdc5f1dfb67dd6866", + "resolution": "resolved", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.survey_origin_budget::_pure_successor_state", + "expression": "_pure_budget_state(budget_state)", + "expression_sha256": "7ccdbcc9194248d6100666d7a9efd40f88bb1416335c6434a1c26ca31c2fe195", + "resolution": "resolved", + "usage": "call" + } + ] + }, + "microcosm.build.us_runtime.survey_origin_budget::_pure_successor_state": { + "basis": "reviewed entry/caller-chain body", + "body_sha256": "97cbb4466beb12e1a70f6053b180b43acff233d676b6847ca2a307e618e7cf30", + "references": [ + { + "caller": "microcosm.build.us_runtime.survey_origin_budget::SamplingOriginSuccessor.checked_view", + "expression": "_pure_successor_state(state)", + "expression_sha256": "f636d1e5c8a42216a712f26ebb3e445bfe662cbd7a00b8455f05d47e9176956e", + "resolution": "resolved", + "usage": "call" + } + ] + }, + "microcosm.build.us_runtime.survey_origin_budget::_successor_document": { + "basis": "reviewed entry/caller-chain body", + "body_sha256": "b4f52e4704e902e3fe8194a7eb10ac545191e0682e1d21dfe3d226cc865a3bfb", + "references": [ + { + "caller": "microcosm.build.us_runtime.survey_origin_budget::_checked_successor_document", + "expression": "_successor_document(state, budget_view)", + "expression_sha256": "fae2329b4068c0164316bc83d119f484dfa2a87a5f4ddf59ad486ee89008a199", + "resolution": "resolved", + "usage": "call" + } + ] + }, + "microcosm.build.us_runtime.survey_origin_budget::_validate_budget": { + "basis": "reviewed entry/caller-chain body", + "body_sha256": "a06ea9bfe1653e9a236702437d3b757a16a657a0a6c8680d031a0bd3e3984143", + "references": [ + { + "caller": "microcosm.build.us_runtime.survey_origin_budget::SamplingOriginBudget.checked_view", + "expression": "_validate_budget(entry[2], entry[1])", + "expression_sha256": "461a1fc2ed42a88e80c1ac68b1cb161bcc3bc6261af6fec8094c981485be9ec2", + "resolution": "resolved", + "usage": "call" + } + ] + }, + "microcosm.build.us_runtime.survey_origin_budget::admit_survey_weight_only_population": { + "basis": "reviewed entry/caller-chain body", + "body_sha256": "2ea4f943c2d74fc220eb0eb1fb33342748e5f9bb6c1b4a200d4c22290734f474", + "references": [ + { + "caller": "microcosm.build.us_runtime.survey_age_calibration::_run_survey_age_calibration.observe", + "expression": "budgets.admit_survey_weight_only_population(budget, previous=receiving_initial, current=population)", + "expression_sha256": "80a2184a19fe48eb800888827717347bd80df52f1124749d2374416a12e875c3", + "resolution": "resolved", + "usage": "call" + } + ] + }, + "microcosm.build.us_runtime.survey_origin_budget::freeze_survey_origin_budget": { + "basis": "reviewed entry/caller-chain body", + "body_sha256": "576adba2e781880832cf70dc376fead41378650ec4f6add0fcecf1f7ccb8adb7", + "references": [ + { + "caller": "microcosm.build.us_runtime.survey_age_calibration::_run_survey_age_calibration", + "expression": "budgets.freeze_survey_origin_budget(prefix.preparation, allocated_population=prefix.allocated_population, clone_population=initial, geography_config=geography_config)", + "expression_sha256": "7050ee263f9b4779b7038178caccc7c0bc5d5d5d2e050000d43d52dc9ba40c77", + "resolution": "resolved", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.survey_age_calibration::_run_survey_age_calibration.observe", + "expression": "budgets.freeze_survey_origin_budget(prefix.preparation, allocated_population=observed[source_graph.ALLOCATION_NODE], clone_population=population, candidate=budget_payload, geography_config=geography_config)", + "expression_sha256": "51cc817817aa146552522e1e2829441583f9360cd65401e2e13455c68f4cefe0", + "resolution": "resolved", + "usage": "call" + } + ] + }, + "microcosm.build.us_runtime.survey_population_preparation::AuthenticatedSurveyPopulationPreparation._checked": { + "basis": "reviewed entry/caller-chain body", + "body_sha256": "9f5d1ef3c5b29e6eb38a1059d93813d3062fdc9330e83fdd2b6c21b3861c72b3", + "references": [ + { + "caller": "microcosm.build.us_runtime.graph_current_survey_geography::CurrentSurveyGeographyKernel.run", + "expression": "preparation._checked()", + "expression_sha256": "92f12c0f1d35686015dedfb5af346caa3d48ae257295fa54a1b8dab163a111b3", + "resolution": "unresolved-us-candidate", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.graph_current_survey_geography::CurrentSurveyGeographyKernel.run", + "expression": "preparation._checked()", + "expression_sha256": "92f12c0f1d35686015dedfb5af346caa3d48ae257295fa54a1b8dab163a111b3", + "resolution": "unresolved-us-candidate", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.graph_current_survey_predictors::CurrentSurveyPredictorDonorFilterKernel.run", + "expression": "self.preparation._checked()", + "expression_sha256": "d127ce8187f0b94a74063723d91dc5c2866380eacc8ee91750e5b9694c6e9829", + "resolution": "unresolved-us-candidate", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.survey_population_preparation::AuthenticatedSurveyPopulationPreparation.checked_view", + "expression": "self._checked()", + "expression_sha256": "da5250783e70955488eb880745cca24c177d6c66b5c0730d828328e4badd9e7a", + "resolution": "resolved", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.survey_population_preparation::AuthenticatedSurveyPopulationPreparation.context", + "expression": "self._checked()", + "expression_sha256": "da5250783e70955488eb880745cca24c177d6c66b5c0730d828328e4badd9e7a", + "resolution": "resolved", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.survey_population_preparation::AuthenticatedSurveyPopulationPreparation.frame", + "expression": "self._checked()", + "expression_sha256": "da5250783e70955488eb880745cca24c177d6c66b5c0730d828328e4badd9e7a", + "resolution": "resolved", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.survey_population_preparation::AuthenticatedSurveyPopulationPreparation.receipt", + "expression": "self._checked()", + "expression_sha256": "da5250783e70955488eb880745cca24c177d6c66b5c0730d828328e4badd9e7a", + "resolution": "resolved", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.survey_population_preparation::AuthenticatedSurveyPopulationPreparation.selection_plan", + "expression": "self._checked()", + "expression_sha256": "da5250783e70955488eb880745cca24c177d6c66b5c0730d828328e4badd9e7a", + "resolution": "resolved", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.survey_population_preparation::AuthenticatedSurveyPopulationPreparation.to_bytes", + "expression": "self._checked()", + "expression_sha256": "da5250783e70955488eb880745cca24c177d6c66b5c0730d828328e4badd9e7a", + "resolution": "resolved", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.survey_population_preparation::AuthenticatedSurveyPopulationPreparation.validate", + "expression": "self._checked()", + "expression_sha256": "da5250783e70955488eb880745cca24c177d6c66b5c0730d828328e4badd9e7a", + "resolution": "resolved", + "usage": "call" + } + ] + }, + "microcosm.build.us_runtime.survey_population_preparation::AuthenticatedSurveyPopulationPreparation.checked_view": { + "basis": "reviewed entry/caller-chain body", + "body_sha256": "4361622e11349d7c382b6647431323f9ff48e2e4fd76d756c5c6783f53fabb3b", + "references": [ + { + "caller": "microcosm.build.us_runtime.survey_financial_successor::admit_survey_financial_population", + "expression": "budget.checked_view()", + "expression_sha256": "303121f839037c123a3907a3ba1100bc5c224834bb11b565bbec767a9078485e", + "resolution": "unresolved-us-candidate", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.survey_origin_budget::_validate_budget", + "expression": "source.AuthenticatedSurveyPopulationPreparation.checked_view(state.preparation)", + "expression_sha256": "2bca9b3fe2e002695e059e4dd6735663914a82c10f6b69a697a489078327d96f", + "resolution": "resolved", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.survey_origin_budget::freeze_survey_origin_budget", + "expression": "source.AuthenticatedSurveyPopulationPreparation.checked_view(preparation)", + "expression_sha256": "9f11ba018d4c72dd05ce7842aed389b4975d1b2f3c45f545e550f4a1915f532e", + "resolution": "resolved", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.survey_origin_budget::freeze_survey_origin_budget", + "expression": "source.AuthenticatedSurveyPopulationPreparation.checked_view(preparation)", + "expression_sha256": "9f11ba018d4c72dd05ce7842aed389b4975d1b2f3c45f545e550f4a1915f532e", + "resolution": "resolved", + "usage": "call" + } + ] + }, + "microcosm.build.us_runtime.survey_population_preparation::AuthenticatedSurveyPopulationPreparation.context": { + "basis": "reviewed entry/caller-chain body", + "body_sha256": "5c235c3bb783eaa2a8be401eb834eb9e878cef5d76f39e971f01085e009b700e", + "references": [ + { + "caller": "microcosm.build.us_runtime.housing_inputs::AcsRentDonorPreparation.__post_init__", + "expression": "self.context", + "expression_sha256": "f0e122dc0f582c2e252cbdd48a46a240b73fa8d3c0f81ef301de5ae926077b76", + "resolution": "unresolved-us-candidate", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.spine_assembly::SpinePreparation.__post_init__", + "expression": "self.context", + "expression_sha256": "f0e122dc0f582c2e252cbdd48a46a240b73fa8d3c0f81ef301de5ae926077b76", + "resolution": "unresolved-us-candidate", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.spine_assembly::SpinePreparation.__post_init__", + "expression": "self.context", + "expression_sha256": "f0e122dc0f582c2e252cbdd48a46a240b73fa8d3c0f81ef301de5ae926077b76", + "resolution": "unresolved-us-candidate", + "usage": "reference" + } + ] + }, + "microcosm.build.us_runtime.survey_population_preparation::AuthenticatedSurveyPopulationPreparation.frame": { + "basis": "reviewed entry/caller-chain body", + "body_sha256": "8f6662f17b88aa0c7c2033af78c45beef984e46f1a71907089ae509325c69dbd", + "references": [ + { + "caller": "microcosm.build.us_runtime.asec_current_money_source::AuthenticatedCurrentMoneySource.validate", + "expression": "self.frame", + "expression_sha256": "bc9202c1b911354870c877faf15851546d34fe5d0a4cf140ef8fd985c75cf832", + "resolution": "unresolved-us-candidate", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.asec_current_money_source::AuthenticatedCurrentMoneySource.validate", + "expression": "self.frame", + "expression_sha256": "bc9202c1b911354870c877faf15851546d34fe5d0a4cf140ef8fd985c75cf832", + "resolution": "unresolved-us-candidate", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.asec_current_money_source::AuthenticatedCurrentMoneySource.views", + "expression": "self.frame", + "expression_sha256": "bc9202c1b911354870c877faf15851546d34fe5d0a4cf140ef8fd985c75cf832", + "resolution": "unresolved-us-candidate", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.asec_current_money_units::CurrentMoneyTaxUnitResult.validate", + "expression": "self.frame", + "expression_sha256": "bc9202c1b911354870c877faf15851546d34fe5d0a4cf140ef8fd985c75cf832", + "resolution": "unresolved-us-candidate", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.asec_demographic_source::DemographicSourceAttachedAsec.validate", + "expression": "self.frame", + "expression_sha256": "bc9202c1b911354870c877faf15851546d34fe5d0a4cf140ef8fd985c75cf832", + "resolution": "unresolved-us-candidate", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.asec_demographic_source::DemographicSourceAttachedAsec.validate", + "expression": "self.frame", + "expression_sha256": "bc9202c1b911354870c877faf15851546d34fe5d0a4cf140ef8fd985c75cf832", + "resolution": "unresolved-us-candidate", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.asec_demographic_source::DemographicSourceAttachedAsec.validate", + "expression": "self.frame", + "expression_sha256": "bc9202c1b911354870c877faf15851546d34fe5d0a4cf140ef8fd985c75cf832", + "resolution": "unresolved-us-candidate", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.asec_demographic_source::DemographicSourceAttachedAsec.validate", + "expression": "self.frame", + "expression_sha256": "bc9202c1b911354870c877faf15851546d34fe5d0a4cf140ef8fd985c75cf832", + "resolution": "unresolved-us-candidate", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.asec_housing_status_source::HousingStatusAttachedAsec.validate", + "expression": "self.frame", + "expression_sha256": "bc9202c1b911354870c877faf15851546d34fe5d0a4cf140ef8fd985c75cf832", + "resolution": "unresolved-us-candidate", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.asec_housing_status_source::HousingStatusAttachedAsec.validate", + "expression": "self.frame", + "expression_sha256": "bc9202c1b911354870c877faf15851546d34fe5d0a4cf140ef8fd985c75cf832", + "resolution": "unresolved-us-candidate", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.asec_housing_universe_source::HousingUniverseAttachedAsec.validate", + "expression": "self.frame", + "expression_sha256": "bc9202c1b911354870c877faf15851546d34fe5d0a4cf140ef8fd985c75cf832", + "resolution": "unresolved-us-candidate", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.asec_housing_universe_source::HousingUniverseAttachedAsec.validate", + "expression": "self.frame", + "expression_sha256": "bc9202c1b911354870c877faf15851546d34fe5d0a4cf140ef8fd985c75cf832", + "resolution": "unresolved-us-candidate", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.asec_student_controls::StudentControlsAttachedAsec.validate", + "expression": "self.frame", + "expression_sha256": "bc9202c1b911354870c877faf15851546d34fe5d0a4cf140ef8fd985c75cf832", + "resolution": "unresolved-us-candidate", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.asec_student_controls::StudentControlsAttachedAsec.validate", + "expression": "self.frame", + "expression_sha256": "bc9202c1b911354870c877faf15851546d34fe5d0a4cf140ef8fd985c75cf832", + "resolution": "unresolved-us-candidate", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.asec_student_controls::StudentControlsAttachedAsec.validate", + "expression": "self.frame", + "expression_sha256": "bc9202c1b911354870c877faf15851546d34fe5d0a4cf140ef8fd985c75cf832", + "resolution": "unresolved-us-candidate", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.asec_student_controls::StudentControlsAttachedAsec.validate", + "expression": "self.frame", + "expression_sha256": "bc9202c1b911354870c877faf15851546d34fe5d0a4cf140ef8fd985c75cf832", + "resolution": "unresolved-us-candidate", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.graph_current_survey_predictors::CurrentSurveyPredictorAttachKernel.run", + "expression": "self.clone_population.frame", + "expression_sha256": "15f45b915a6b89ff7fb8924635f396d00ff022fa0b1ddd806d2c64dff205966a", + "resolution": "unresolved-us-candidate", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.graph_current_survey_predictors::CurrentSurveyPredictorAttachKernel.run", + "expression": "self.clone_population.frame", + "expression_sha256": "15f45b915a6b89ff7fb8924635f396d00ff022fa0b1ddd806d2c64dff205966a", + "resolution": "unresolved-us-candidate", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.graph_current_survey_predictors::_Kernel._qualified", + "expression": "self.clone_population.frame", + "expression_sha256": "15f45b915a6b89ff7fb8924635f396d00ff022fa0b1ddd806d2c64dff205966a", + "resolution": "unresolved-us-candidate", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.graph_current_survey_puf_transfer::CurrentSurveyAttachKernel.run", + "expression": "self.clone_population.frame", + "expression_sha256": "15f45b915a6b89ff7fb8924635f396d00ff022fa0b1ddd806d2c64dff205966a", + "resolution": "unresolved-us-candidate", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.graph_current_survey_puf_transfer::CurrentSurveyAttachKernel.run", + "expression": "self.clone_population.frame", + "expression_sha256": "15f45b915a6b89ff7fb8924635f396d00ff022fa0b1ddd806d2c64dff205966a", + "resolution": "unresolved-us-candidate", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.graph_current_survey_puf_transfer::CurrentSurveyAttachKernel.run", + "expression": "self.clone_population.frame", + "expression_sha256": "15f45b915a6b89ff7fb8924635f396d00ff022fa0b1ddd806d2c64dff205966a", + "resolution": "unresolved-us-candidate", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.graph_current_survey_puf_transfer::CurrentSurveyAttachKernel.run", + "expression": "self.clone_population.frame", + "expression_sha256": "15f45b915a6b89ff7fb8924635f396d00ff022fa0b1ddd806d2c64dff205966a", + "resolution": "unresolved-us-candidate", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.graph_current_survey_puf_transfer::CurrentSurveyPlacementKernel.run", + "expression": "self.clone_population.frame", + "expression_sha256": "15f45b915a6b89ff7fb8924635f396d00ff022fa0b1ddd806d2c64dff205966a", + "resolution": "unresolved-us-candidate", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.graph_current_survey_puf_transfer::CurrentSurveyPlacementKernel.run", + "expression": "self.clone_population.frame", + "expression_sha256": "15f45b915a6b89ff7fb8924635f396d00ff022fa0b1ddd806d2c64dff205966a", + "resolution": "unresolved-us-candidate", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.graph_current_survey_puf_transfer::CurrentSurveyPlacementKernel.run", + "expression": "self.clone_population.frame", + "expression_sha256": "15f45b915a6b89ff7fb8924635f396d00ff022fa0b1ddd806d2c64dff205966a", + "resolution": "unresolved-us-candidate", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.graph_current_survey_puf_transfer::CurrentSurveyPlacementKernel.run", + "expression": "self.clone_population.frame", + "expression_sha256": "15f45b915a6b89ff7fb8924635f396d00ff022fa0b1ddd806d2c64dff205966a", + "resolution": "unresolved-us-candidate", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.graph_full_puf_enrichment::FullPufMaskKernel.run", + "expression": "binding.expected_population.frame", + "expression_sha256": "76de9d83af4ba326b6212754b1034d89debb1d7d1a5ca4478bcbc2a34a696926", + "resolution": "unresolved-us-candidate", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.graph_full_puf_enrichment::_context_projection", + "expression": "frame", + "expression_sha256": "2eb691dff6a08513007b0d79ee5680d4ed46afeddfdca5913b12e520a9a61614", + "resolution": "unresolved-us-candidate", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.graph_full_puf_enrichment::_context_projection", + "expression": "frame", + "expression_sha256": "2eb691dff6a08513007b0d79ee5680d4ed46afeddfdca5913b12e520a9a61614", + "resolution": "unresolved-us-candidate", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.graph_full_puf_enrichment::_context_projection", + "expression": "frame", + "expression_sha256": "2eb691dff6a08513007b0d79ee5680d4ed46afeddfdca5913b12e520a9a61614", + "resolution": "unresolved-us-candidate", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.graph_full_puf_enrichment::_context_projection", + "expression": "frame", + "expression_sha256": "2eb691dff6a08513007b0d79ee5680d4ed46afeddfdca5913b12e520a9a61614", + "resolution": "unresolved-us-candidate", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.graph_full_puf_enrichment::_context_projection", + "expression": "frame", + "expression_sha256": "2eb691dff6a08513007b0d79ee5680d4ed46afeddfdca5913b12e520a9a61614", + "resolution": "unresolved-us-candidate", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.graph_full_puf_enrichment::_context_projection", + "expression": "frame", + "expression_sha256": "2eb691dff6a08513007b0d79ee5680d4ed46afeddfdca5913b12e520a9a61614", + "resolution": "unresolved-us-candidate", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.graph_full_puf_enrichment::_context_projection", + "expression": "frame", + "expression_sha256": "2eb691dff6a08513007b0d79ee5680d4ed46afeddfdca5913b12e520a9a61614", + "resolution": "unresolved-us-candidate", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.graph_full_puf_enrichment::_finalized_columns", + "expression": "frame", + "expression_sha256": "2eb691dff6a08513007b0d79ee5680d4ed46afeddfdca5913b12e520a9a61614", + "resolution": "unresolved-us-candidate", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.graph_full_puf_enrichment::_finalized_columns", + "expression": "frame", + "expression_sha256": "2eb691dff6a08513007b0d79ee5680d4ed46afeddfdca5913b12e520a9a61614", + "resolution": "unresolved-us-candidate", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.graph_full_puf_enrichment::_finalized_columns", + "expression": "frame", + "expression_sha256": "2eb691dff6a08513007b0d79ee5680d4ed46afeddfdca5913b12e520a9a61614", + "resolution": "unresolved-us-candidate", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.graph_full_puf_enrichment::_finalized_columns", + "expression": "frame", + "expression_sha256": "2eb691dff6a08513007b0d79ee5680d4ed46afeddfdca5913b12e520a9a61614", + "resolution": "unresolved-us-candidate", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.graph_full_puf_enrichment::_finalized_columns", + "expression": "binding.expected_population.frame", + "expression_sha256": "76de9d83af4ba326b6212754b1034d89debb1d7d1a5ca4478bcbc2a34a696926", + "resolution": "unresolved-us-candidate", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.graph_full_puf_enrichment::_inputs", + "expression": "frame", + "expression_sha256": "2eb691dff6a08513007b0d79ee5680d4ed46afeddfdca5913b12e520a9a61614", + "resolution": "unresolved-us-candidate", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.graph_full_puf_enrichment::_inputs", + "expression": "frame", + "expression_sha256": "2eb691dff6a08513007b0d79ee5680d4ed46afeddfdca5913b12e520a9a61614", + "resolution": "unresolved-us-candidate", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.graph_full_puf_enrichment::_inputs", + "expression": "frame", + "expression_sha256": "2eb691dff6a08513007b0d79ee5680d4ed46afeddfdca5913b12e520a9a61614", + "resolution": "unresolved-us-candidate", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.graph_full_puf_enrichment::_mask_result", + "expression": "frame", + "expression_sha256": "2eb691dff6a08513007b0d79ee5680d4ed46afeddfdca5913b12e520a9a61614", + "resolution": "unresolved-us-candidate", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.graph_full_puf_enrichment::_mask_result", + "expression": "frame", + "expression_sha256": "2eb691dff6a08513007b0d79ee5680d4ed46afeddfdca5913b12e520a9a61614", + "resolution": "unresolved-us-candidate", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.graph_full_puf_enrichment::_mask_result", + "expression": "frame", + "expression_sha256": "2eb691dff6a08513007b0d79ee5680d4ed46afeddfdca5913b12e520a9a61614", + "resolution": "unresolved-us-candidate", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.graph_full_puf_enrichment::_mask_result", + "expression": "binding.expected_population.frame", + "expression_sha256": "76de9d83af4ba326b6212754b1034d89debb1d7d1a5ca4478bcbc2a34a696926", + "resolution": "unresolved-us-candidate", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.graph_full_puf_enrichment::_masks", + "expression": "frame", + "expression_sha256": "2eb691dff6a08513007b0d79ee5680d4ed46afeddfdca5913b12e520a9a61614", + "resolution": "unresolved-us-candidate", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.graph_full_puf_enrichment::_masks", + "expression": "frame", + "expression_sha256": "2eb691dff6a08513007b0d79ee5680d4ed46afeddfdca5913b12e520a9a61614", + "resolution": "unresolved-us-candidate", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.graph_full_puf_enrichment::_masks", + "expression": "frame", + "expression_sha256": "2eb691dff6a08513007b0d79ee5680d4ed46afeddfdca5913b12e520a9a61614", + "resolution": "unresolved-us-candidate", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.graph_full_puf_enrichment::_outputs", + "expression": "frame", + "expression_sha256": "2eb691dff6a08513007b0d79ee5680d4ed46afeddfdca5913b12e520a9a61614", + "resolution": "unresolved-us-candidate", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.graph_full_puf_enrichment::_placement", + "expression": "frame", + "expression_sha256": "2eb691dff6a08513007b0d79ee5680d4ed46afeddfdca5913b12e520a9a61614", + "resolution": "unresolved-us-candidate", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.graph_full_puf_enrichment::_placement", + "expression": "frame", + "expression_sha256": "2eb691dff6a08513007b0d79ee5680d4ed46afeddfdca5913b12e520a9a61614", + "resolution": "unresolved-us-candidate", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.graph_full_puf_enrichment::_placement", + "expression": "frame", + "expression_sha256": "2eb691dff6a08513007b0d79ee5680d4ed46afeddfdca5913b12e520a9a61614", + "resolution": "unresolved-us-candidate", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.graph_full_puf_enrichment::_placement", + "expression": "binding.expected_population.frame", + "expression_sha256": "76de9d83af4ba326b6212754b1034d89debb1d7d1a5ca4478bcbc2a34a696926", + "resolution": "unresolved-us-candidate", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.graph_full_puf_enrichment::_population_stamp", + "expression": "frame", + "expression_sha256": "2eb691dff6a08513007b0d79ee5680d4ed46afeddfdca5913b12e520a9a61614", + "resolution": "unresolved-us-candidate", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.graph_full_puf_enrichment::_population_stamp", + "expression": "frame", + "expression_sha256": "2eb691dff6a08513007b0d79ee5680d4ed46afeddfdca5913b12e520a9a61614", + "resolution": "unresolved-us-candidate", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.graph_full_puf_enrichment::_population_stamp", + "expression": "frame", + "expression_sha256": "2eb691dff6a08513007b0d79ee5680d4ed46afeddfdca5913b12e520a9a61614", + "resolution": "unresolved-us-candidate", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.graph_full_puf_enrichment::_population_stamp", + "expression": "frame", + "expression_sha256": "2eb691dff6a08513007b0d79ee5680d4ed46afeddfdca5913b12e520a9a61614", + "resolution": "unresolved-us-candidate", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.graph_full_puf_enrichment::_population_stamp", + "expression": "frame", + "expression_sha256": "2eb691dff6a08513007b0d79ee5680d4ed46afeddfdca5913b12e520a9a61614", + "resolution": "unresolved-us-candidate", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.graph_full_puf_enrichment::_population_stamp", + "expression": "frame", + "expression_sha256": "2eb691dff6a08513007b0d79ee5680d4ed46afeddfdca5913b12e520a9a61614", + "resolution": "unresolved-us-candidate", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.graph_full_puf_enrichment::_population_stamp", + "expression": "frame", + "expression_sha256": "2eb691dff6a08513007b0d79ee5680d4ed46afeddfdca5913b12e520a9a61614", + "resolution": "unresolved-us-candidate", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.graph_full_puf_enrichment::_population_stamp", + "expression": "frame", + "expression_sha256": "2eb691dff6a08513007b0d79ee5680d4ed46afeddfdca5913b12e520a9a61614", + "resolution": "unresolved-us-candidate", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.graph_full_puf_enrichment::_population_stamp", + "expression": "frame", + "expression_sha256": "2eb691dff6a08513007b0d79ee5680d4ed46afeddfdca5913b12e520a9a61614", + "resolution": "unresolved-us-candidate", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.graph_full_puf_enrichment::_population_stamp", + "expression": "frame", + "expression_sha256": "2eb691dff6a08513007b0d79ee5680d4ed46afeddfdca5913b12e520a9a61614", + "resolution": "unresolved-us-candidate", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.graph_full_puf_enrichment::_population_stamp", + "expression": "frame", + "expression_sha256": "2eb691dff6a08513007b0d79ee5680d4ed46afeddfdca5913b12e520a9a61614", + "resolution": "unresolved-us-candidate", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.graph_full_puf_enrichment::_population_stamp", + "expression": "frame", + "expression_sha256": "2eb691dff6a08513007b0d79ee5680d4ed46afeddfdca5913b12e520a9a61614", + "resolution": "unresolved-us-candidate", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.graph_full_puf_enrichment::_population_stamp", + "expression": "frame", + "expression_sha256": "2eb691dff6a08513007b0d79ee5680d4ed46afeddfdca5913b12e520a9a61614", + "resolution": "unresolved-us-candidate", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.graph_full_puf_enrichment::_population_stamp", + "expression": "frame", + "expression_sha256": "2eb691dff6a08513007b0d79ee5680d4ed46afeddfdca5913b12e520a9a61614", + "resolution": "unresolved-us-candidate", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.graph_full_puf_enrichment::_population_stamp", + "expression": "frame", + "expression_sha256": "2eb691dff6a08513007b0d79ee5680d4ed46afeddfdca5913b12e520a9a61614", + "resolution": "unresolved-us-candidate", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.graph_full_puf_enrichment::_structural", + "expression": "frame", + "expression_sha256": "2eb691dff6a08513007b0d79ee5680d4ed46afeddfdca5913b12e520a9a61614", + "resolution": "unresolved-us-candidate", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.graph_full_puf_enrichment::full_puf_attachment_nodes", + "expression": "frame", + "expression_sha256": "2eb691dff6a08513007b0d79ee5680d4ed46afeddfdca5913b12e520a9a61614", + "resolution": "unresolved-us-candidate", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.graph_full_puf_enrichment::full_puf_attachment_nodes", + "expression": "frame", + "expression_sha256": "2eb691dff6a08513007b0d79ee5680d4ed46afeddfdca5913b12e520a9a61614", + "resolution": "unresolved-us-candidate", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.graph_full_puf_enrichment::full_puf_attachment_nodes", + "expression": "frame", + "expression_sha256": "2eb691dff6a08513007b0d79ee5680d4ed46afeddfdca5913b12e520a9a61614", + "resolution": "unresolved-us-candidate", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.graph_full_puf_enrichment::full_puf_attachment_nodes", + "expression": "binding.expected_population.frame", + "expression_sha256": "76de9d83af4ba326b6212754b1034d89debb1d7d1a5ca4478bcbc2a34a696926", + "resolution": "unresolved-us-candidate", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.graph_puf_diagnostic_consumer::CurrentSurveyHostProjectionKernel.run", + "expression": "self.clone_population.frame", + "expression_sha256": "15f45b915a6b89ff7fb8924635f396d00ff022fa0b1ddd806d2c64dff205966a", + "resolution": "unresolved-us-candidate", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.multispine_pool::MultispinePoolCheckpoint.__post_init__", + "expression": "self.frame", + "expression_sha256": "bc9202c1b911354870c877faf15851546d34fe5d0a4cf140ef8fd985c75cf832", + "resolution": "unresolved-us-candidate", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.multispine_pool::MultispinePoolCheckpoint.__post_init__", + "expression": "self.frame", + "expression_sha256": "bc9202c1b911354870c877faf15851546d34fe5d0a4cf140ef8fd985c75cf832", + "resolution": "unresolved-us-candidate", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.multispine_pool::PoolStageOutput.__post_init__", + "expression": "self.frame", + "expression_sha256": "bc9202c1b911354870c877faf15851546d34fe5d0a4cf140ef8fd985c75cf832", + "resolution": "unresolved-us-candidate", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.multispine_pool::PoolStageOutput.__post_init__", + "expression": "self.frame", + "expression_sha256": "bc9202c1b911354870c877faf15851546d34fe5d0a4cf140ef8fd985c75cf832", + "resolution": "unresolved-us-candidate", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.spine_assembly::SpinePreparation.__post_init__", + "expression": "self.frame", + "expression_sha256": "bc9202c1b911354870c877faf15851546d34fe5d0a4cf140ef8fd985c75cf832", + "resolution": "unresolved-us-candidate", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.spine_assembly::SpinePreparation.__post_init__", + "expression": "self.frame", + "expression_sha256": "bc9202c1b911354870c877faf15851546d34fe5d0a4cf140ef8fd985c75cf832", + "resolution": "unresolved-us-candidate", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.stacked_spine::StackedSpinePreparation.__post_init__", + "expression": "self.frame", + "expression_sha256": "bc9202c1b911354870c877faf15851546d34fe5d0a4cf140ef8fd985c75cf832", + "resolution": "unresolved-us-candidate", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.stacked_spine::StackedSpinePreparation.__post_init__", + "expression": "self.frame", + "expression_sha256": "bc9202c1b911354870c877faf15851546d34fe5d0a4cf140ef8fd985c75cf832", + "resolution": "unresolved-us-candidate", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.stacked_spine::StackedSpineResult.__post_init__", + "expression": "self.frame", + "expression_sha256": "bc9202c1b911354870c877faf15851546d34fe5d0a4cf140ef8fd985c75cf832", + "resolution": "unresolved-us-candidate", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.stacked_spine::StackedSpineResult.__post_init__", + "expression": "self.frame", + "expression_sha256": "bc9202c1b911354870c877faf15851546d34fe5d0a4cf140ef8fd985c75cf832", + "resolution": "unresolved-us-candidate", + "usage": "reference" + } + ] + }, + "microcosm.build.us_runtime.survey_population_preparation::AuthenticatedSurveyPopulationPreparation.receipt": { + "basis": "reviewed entry/caller-chain body", + "body_sha256": "9bfa508267c86a961a117668dca63bce9efe332dfed135e31652e9e978d13e11", + "references": [ + { + "caller": "microcosm.build.us_runtime.asec_current_money_units::CurrentMoneyTaxUnitResult.validate", + "expression": "self._student_source.receipt", + "expression_sha256": "b07ef0768f9115dfd6c2ccf2e0558c015159bc578c6b1c1dad955a3332bba05a", + "resolution": "unresolved-us-candidate", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.asec_demographic_source::DemographicSourceAttachedAsec.validate", + "expression": "self.observations.receipt", + "expression_sha256": "1197f225000198b5a1cb0313c842ca1a66604dc55e5d85ef3cbf846c336804a3", + "resolution": "unresolved-us-candidate", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.asec_housing_status_source::HousingStatusAttachedAsec.validate", + "expression": "self.tax_result.receipt", + "expression_sha256": "fb31b06dffb7494bb98766aeff9f28da9035003318144f4a796917c7e4f17b01", + "resolution": "unresolved-us-candidate", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.asec_prepared_source::PreparedAsecPopulation.receipt_payload", + "expression": "self.receipt", + "expression_sha256": "e092d770e10b3e310aa85b322b534aa524e35d368e3ddd2eb1665852865903b2", + "resolution": "unresolved-us-candidate", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.multispine_pool::PoolStageOutput.__post_init__", + "expression": "self.receipt", + "expression_sha256": "e092d770e10b3e310aa85b322b534aa524e35d368e3ddd2eb1665852865903b2", + "resolution": "unresolved-us-candidate", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.stacked_spine::StackedSpineHarmonization.__post_init__", + "expression": "self.receipt", + "expression_sha256": "e092d770e10b3e310aa85b322b534aa524e35d368e3ddd2eb1665852865903b2", + "resolution": "unresolved-us-candidate", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.stacked_spine::StackedSpinePreparation.__post_init__", + "expression": "self.receipt", + "expression_sha256": "e092d770e10b3e310aa85b322b534aa524e35d368e3ddd2eb1665852865903b2", + "resolution": "unresolved-us-candidate", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.stacked_spine::StackedSpinePreparation.__post_init__", + "expression": "self.receipt", + "expression_sha256": "e092d770e10b3e310aa85b322b534aa524e35d368e3ddd2eb1665852865903b2", + "resolution": "unresolved-us-candidate", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.stacked_spine::StackedSpinePreparation.__post_init__", + "expression": "self.receipt", + "expression_sha256": "e092d770e10b3e310aa85b322b534aa524e35d368e3ddd2eb1665852865903b2", + "resolution": "unresolved-us-candidate", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.stacked_spine::StackedSpineResult.__post_init__", + "expression": "self.receipt", + "expression_sha256": "e092d770e10b3e310aa85b322b534aa524e35d368e3ddd2eb1665852865903b2", + "resolution": "unresolved-us-candidate", + "usage": "reference" + } + ] + }, + "microcosm.build.us_runtime.survey_population_preparation::AuthenticatedSurveyPopulationPreparation.selection_plan": { + "basis": "reviewed entry/caller-chain body", + "body_sha256": "45b26b9256d83a2df1eb25f730107bd29daa8b4339263b0c5c353857fc45579c", + "references": [] + }, + "microcosm.build.us_runtime.survey_population_preparation::AuthenticatedSurveyPopulationPreparation.to_bytes": { + "basis": "reviewed entry/caller-chain body", + "body_sha256": "788c6bb86f43da428c4f83c36d825f1d32b88319a6e48980efb285ed7a8ed81b", + "references": [ + { + "caller": "microcosm.build.us_runtime.survey_age_calibration::_run_survey_age_calibration", + "expression": "atomic_source.reconstruction.AtomicSurveyReconstruction.to_bytes(geography_config)", + "expression_sha256": "335e8b0f924d999b5c7198f3d9d6f20380d5cfc4c0903bcb8a5e60fab14abefc", + "resolution": "unresolved-us-candidate", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.survey_age_calibration::_run_survey_age_calibration", + "expression": "atomic_source.reconstruction.AtomicSurveyReconstruction.to_bytes(geography_config)", + "expression_sha256": "335e8b0f924d999b5c7198f3d9d6f20380d5cfc4c0903bcb8a5e60fab14abefc", + "resolution": "unresolved-us-candidate", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.survey_age_calibration::_run_survey_age_calibration", + "expression": "atomic_source.reconstruction.AtomicSurveyReconstruction.to_bytes(geography_config)", + "expression_sha256": "335e8b0f924d999b5c7198f3d9d6f20380d5cfc4c0903bcb8a5e60fab14abefc", + "resolution": "unresolved-us-candidate", + "usage": "call" + } + ] + }, + "microcosm.build.us_runtime.survey_population_preparation::AuthenticatedSurveyPopulationPreparation.validate": { + "basis": "reviewed entry/caller-chain body", + "body_sha256": "62c0d902d74d3a3abffc203ddf29b405e4ccecdaa6d07f98ef35426568740d00", + "references": [ + { + "caller": "microcosm.build.us_runtime.asec_current_money_units::CurrentMoneyTaxUnitResult.validate", + "expression": "self._student_source.validate()", + "expression_sha256": "e277a2ec4101855cb31af89e16bea5d22eef25a4a9c50910937d2db242e1bf4a", + "resolution": "unresolved-us-candidate", + "usage": "call" + } + ] + }, + "microcosm.build.us_runtime.survey_population_preparation::_producer": { + "basis": "reviewed entry/caller-chain body", + "body_sha256": "85e86139eda663afab38e62badaac906187f83c33529a0c6b539631a42d267c9", + "references": [ + { + "caller": "microcosm.build.us_runtime.survey_population_preparation::_validate", + "expression": "_producer()", + "expression_sha256": "e10192e7e675a53252f4e8d4d5d6453b471e56fd0ab142e50a492ebee999caf2", + "resolution": "resolved", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.survey_population_preparation::prepare_authenticated_survey_population", + "expression": "_producer()", + "expression_sha256": "e10192e7e675a53252f4e8d4d5d6453b471e56fd0ab142e50a492ebee999caf2", + "resolution": "resolved", + "usage": "call" + } + ] + }, + "microcosm.build.us_runtime.survey_population_preparation::_pure_final": { + "basis": "reviewed entry/caller-chain body", + "body_sha256": "d3b23b8c0d9a67c41c1636b8715a3d7cf710599a6daf3616832462c90755722b", + "references": [ + { + "caller": "microcosm.build.us_runtime.current_asec_demographics::qualify_current_asec_demographics", + "expression": "preparation_owner._pure_final(state)", + "expression_sha256": "007b28780d719c5373b993f3705ffe7dd6d3f4e8da6d82d084813c0987a9010d", + "resolution": "resolved", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.current_survey_geography::qualify_current_survey_geography", + "expression": "source._pure_final(state)", + "expression_sha256": "9cacc93c525e2f4d852f469517ced055697b252c8757308b396eedf8d7c6edc9", + "resolution": "resolved", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.current_survey_predictors::qualify_current_survey_predictors", + "expression": "source._pure_final(state)", + "expression_sha256": "9cacc93c525e2f4d852f469517ced055697b252c8757308b396eedf8d7c6edc9", + "resolution": "unresolved-us-candidate", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.graph_atomic_survey_financial::_pure_run", + "expression": "values.source._pure_final(state.preparation_entry[2])", + "expression_sha256": "1d0535b32ee86071073f2d48c105145455a137ddeec287934483c612afd4ea7a", + "resolution": "unresolved-us-candidate", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.graph_atomic_survey_financial::run_atomic_survey_financial", + "expression": "values.source._pure_final(entry[2])", + "expression_sha256": "646ed127e4dd921326440b2837f8c0b34030b1e49515df993ff829a3101b5809", + "resolution": "unresolved-us-candidate", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.puf55_survey_ss_measurement::qualify_puf55_survey_ss_measurement", + "expression": "source._pure_final(state)", + "expression_sha256": "9cacc93c525e2f4d852f469517ced055697b252c8757308b396eedf8d7c6edc9", + "resolution": "resolved", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.survey_atomic_geography::reconstruct_atomic_survey_geography", + "expression": "source._pure_final(state)", + "expression_sha256": "9cacc93c525e2f4d852f469517ced055697b252c8757308b396eedf8d7c6edc9", + "resolution": "resolved", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.survey_origin_budget::_financial_budget_values", + "expression": "source._pure_final(retained)", + "expression_sha256": "9e290bb89222e505a5d3e1e7883dcf55d4f635a7f7a9aa2e4e5084a182ef9d48", + "resolution": "resolved", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.survey_origin_budget::_pure_budget_state", + "expression": "source._pure_final(entry[2])", + "expression_sha256": "fa7127886e7172cb4638624ba78819c1d4846d460ec94157a83bd2d1f1ffbdab", + "resolution": "resolved", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.survey_population_preparation::_current_survey_wage_projection", + "expression": "_pure_final(state)", + "expression_sha256": "1c783fd4e1e8828e99b5941a9e959544e7a8661f99c42fc0a09e5e464d3abade", + "resolution": "resolved", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.survey_population_preparation::_validate", + "expression": "_pure_final(state)", + "expression_sha256": "1c783fd4e1e8828e99b5941a9e959544e7a8661f99c42fc0a09e5e464d3abade", + "resolution": "resolved", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.survey_population_preparation::_validate", + "expression": "_pure_final(state)", + "expression_sha256": "1c783fd4e1e8828e99b5941a9e959544e7a8661f99c42fc0a09e5e464d3abade", + "resolution": "resolved", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.survey_population_preparation::prepare_authenticated_survey_population", + "expression": "_pure_final(state)", + "expression_sha256": "1c783fd4e1e8828e99b5941a9e959544e7a8661f99c42fc0a09e5e464d3abade", + "resolution": "resolved", + "usage": "call" + } + ] + }, + "microcosm.build.us_runtime.survey_population_preparation::_validate": { + "basis": "reviewed entry/caller-chain body", + "body_sha256": "82cfc173da46e81b613df170f01ac1e4803a5a7753aa4a30bef9b58a31fca0cd", + "references": [ + { + "caller": "microcosm.build.us_runtime.asec_current_money_source::AuthenticatedCurrentMoneySource.validate", + "expression": "self.source._validate()", + "expression_sha256": "3d02eb09d76453e54943dde82eb2ef0a47e4c3866ff310c06d1ca9241422b6cc", + "resolution": "unresolved-us-candidate", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.survey_population_preparation::AuthenticatedSurveyPopulationPreparation._checked", + "expression": "_validate(entry[2])", + "expression_sha256": "ea5908ca928fbfe1b5e3654cee63b3a2b60a05b4ed313d83143919f37af79687", + "resolution": "resolved", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.survey_population_preparation::prepare_authenticated_survey_population", + "expression": "_validate(state)", + "expression_sha256": "4332e0c8c5a967034c6b35f1efcd731d8853291c15b0cdbf4402ac3de0492ba2", + "resolution": "resolved", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.survey_population_preparation::verify_materialized_survey_population", + "expression": "_validate(entry[2])", + "expression_sha256": "ea5908ca928fbfe1b5e3654cee63b3a2b60a05b4ed313d83143919f37af79687", + "resolution": "resolved", + "usage": "call" + } + ] + }, + "microcosm.build.us_runtime.survey_population_preparation::prepare_authenticated_survey_population": { + "basis": "reviewed entry/caller-chain body", + "body_sha256": "537f09b8b2b6740d757f41d9d8c6448e94e82a83fb7545c01a96bed7e97791cd", + "references": [] + }, + "microcosm.build.us_runtime.survey_population_preparation::verify_materialized_survey_population": { + "basis": "reviewed entry/caller-chain body", + "body_sha256": "66d296c955d913308e0594d752687e5c8bf2ec4c27702d3c3c8cd86e22f4f89a", + "references": [] + }, + "microcosm.build.us_runtime.survey_social_security::@COMPONENTS": { + "basis": "finite selector/validator definition", + "body_sha256": "e249bbec67e1364a5aff129b2fe2d5df96c96f58af4b2b95172a101d67c2d342", + "references": [] + }, + "microcosm.build.us_runtime.survey_social_security::@PROTOCOL": { + "basis": "finite selector/validator definition", + "body_sha256": "a94d7c120aa2fcc3bdcef9f80b21d882bd6b19afa2fc448edf884054480a414f", + "references": [] + }, + "microcosm.build.us_runtime.survey_social_security::@REASON_COMPONENTS": { + "basis": "finite selector/validator definition", + "body_sha256": "0db7434c3be926a270300515897b706063c9c56d63c90f38ec7a3b566e5c1395", + "references": [] + }, + "microcosm.build.us_runtime.survey_social_security::asec_reason_basis": { + "basis": "reviewed entry/caller-chain body", + "body_sha256": "cbac1a7e108edaecb019a61c08d8696746bb80904612726dc1e7b40e6ffa191d", + "references": [ + { + "caller": "microcosm.build.us_runtime.survey_social_security::asec_reporting_basis", + "expression": "asec_reason_basis(amount, reason_1, reason_2)", + "expression_sha256": "577848df4e6e7d97e4bc3d17dab3c7992779520cb0346dd850102454d6b9aba1", + "resolution": "resolved", + "usage": "call" + } + ] + }, + "microcosm.build.us_runtime.survey_social_security::asec_reporting_basis": { + "basis": "reviewed entry/caller-chain body", + "body_sha256": "e193f407dd576de79724c1394ca26a07beae6044e50a05bd18dc0a2f24c5d22b", + "references": [ + { + "caller": "microcosm.build.us_runtime.current_social_security_source::qualify_current_social_security", + "expression": "basis_owner.asec_reporting_basis(amount, ordered.A_AGE.to_numpy(dtype=np.float64), recipiency, reason_1, reason_2)", + "expression_sha256": "31fe127432067c1a059ffe9fcfa774b1aa3445d27ac0e5fd3bc6c89c08021f45", + "resolution": "resolved", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.puf55_survey_recipients::_source_report", + "expression": "ss.reports.basis_owner.asec_reporting_basis(ss.reports._numeric(asec.SS_VAL), ss.reports._numeric(asec.A_AGE), ss.reports._codes(literals.SS_YN, {0, 1, 2}), ss.reports._codes(literals.RESNSS1, range(9)), ss.reports._codes(literals.RESNSS2, range(9)))", + "expression_sha256": "0832bd20e51af10469920a4427df96ca83ecdc932aa30db7469aa3c2206e5936", + "resolution": "unresolved-us-candidate", + "usage": "call" + } + ] + }, + "microcosm.build.us_runtime.us_late_producer_registry::TransferProducerGroup.__post_init__": { + "basis": "reviewed entry/caller-chain body", + "body_sha256": "154e751023f6884b4ee21d81aa017e81d540c4e8609064c51bf3d7d0b75394da", + "references": [] + } + }, + "module_roles": { + "_asec_current_money_codec.py": "artifact_codec_identity", + "acs_housing_universe.py": "source_value_contract", + "acs_housing_universe_source.py": "source_authentication_projection", + "acs_native_coverage_binding.py": "source_authentication_projection", + "acs_person_coverage_authentication.py": "source_authentication_projection", + "acs_person_coverage_columns.py": "source_authentication_projection", + "acs_population_catalogue.py": "source_authentication_projection", + "asec_2024_native_population.py": "source_authentication_projection", + "asec_coverage_authentication.py": "source_authentication_projection", + "asec_current_money.py": "source_value_contract", + "asec_current_money_graph_resources.py": "source_value_contract", + "asec_current_money_resources.py": "source_value_contract", + "asec_current_money_selection.py": "source_value_contract", + "asec_current_money_source.py": "source_authentication_projection", + "asec_current_money_units.py": "source_value_contract", + "asec_demographic_source.py": "source_authentication_projection", + "asec_engine_evaluation.py": "source_value_contract", + "asec_household_coverage_fields.py": "source_authentication_projection", + "asec_household_observations.py": "source_authentication_projection", + "asec_housing_status.py": "source_value_contract", + "asec_housing_status_source.py": "source_authentication_projection", + "asec_housing_universe.py": "source_value_contract", + "asec_housing_universe_source.py": "source_authentication_projection", + "asec_income_observations.py": "source_authentication_projection", + "asec_original_household_weights.py": "source_authentication_projection", + "asec_person_coverage_source.py": "source_authentication_projection", + "asec_person_income_source.py": "source_authentication_projection", + "asec_population_catalogue.py": "source_authentication_projection", + "asec_prepared_source.py": "source_authentication_projection", + "asec_student_controls.py": "source_authentication_projection", + "atomic_block_api_sources.py": "geography", + "atomic_block_sources.py": "geography", + "atomic_block_support.py": "geography", + "cd_reference.py": "calibration_reference_diagnostics", + "cd_reference_sources.py": "calibration_reference_diagnostics", + "cps_carried_current.py": "source_value_contract", + "current_asec_demographics.py": "source_authentication_projection", + "current_social_security_source.py": "source_authentication_projection", + "current_survey_geography.py": "geography", + "current_survey_predictors.py": "model_qualification_application", + "demographic_calibration_graph.py": "graph_stage_adapter", + "full_puf_enrichment.py": "model_qualification_application", + "graph_acs_housing_universe.py": "graph_stage_adapter", + "graph_asec_income.py": "graph_stage_adapter", + "graph_asec_prepared.py": "graph_stage_adapter", + "graph_atomic_survey_clone.py": "graph_stage_adapter", + "graph_atomic_survey_financial.py": "graph_stage_adapter", + "graph_atomic_survey_population.py": "graph_stage_adapter", + "graph_combined_clone.py": "graph_stage_adapter", + "graph_composed_asec_binding.py": "graph_stage_adapter", + "graph_composed_asec_measures.py": "graph_stage_adapter", + "graph_composed_contracts.py": "graph_stage_adapter", + "graph_composed_population.py": "graph_stage_adapter", + "graph_context.py": "graph_stage_adapter", + "graph_current_survey_geography.py": "graph_stage_adapter", + "graph_current_survey_predictors.py": "graph_stage_adapter", + "graph_current_survey_puf_transfer.py": "graph_stage_adapter", + "graph_full_puf_enrichment.py": "graph_stage_adapter", + "graph_geography.py": "graph_stage_adapter", + "graph_housing_universe.py": "graph_stage_adapter", + "graph_implementation.py": "graph_stage_adapter", + "graph_national_age_counts.py": "graph_stage_adapter", + "graph_native_household_origin.py": "graph_stage_adapter", + "graph_native_origin_implementation.py": "graph_stage_adapter", + "graph_puf55_canonical_donor.py": "graph_stage_adapter", + "graph_puf55_survey_recipients.py": "graph_stage_adapter", + "graph_puf_detail_transfer.py": "graph_stage_adapter", + "graph_puf_diagnostic_consumer.py": "graph_stage_adapter", + "graph_sources.py": "graph_stage_adapter", + "graph_survey_age_artifact.py": "graph_stage_adapter", + "graph_survey_budget.py": "graph_stage_adapter", + "graph_survey_calibration.py": "graph_stage_adapter", + "graph_survey_population.py": "graph_stage_adapter", + "national_age_activation.py": "calibration_reference_diagnostics", + "native_household_origin.py": "source_selection_composition", + "puf55_canonical_donor.py": "model_qualification_application", + "puf55_route_finalization.py": "model_qualification_application", + "puf55_survey_recipients.py": "model_qualification_application", + "puf55_survey_ss_measurement.py": "model_qualification_application", + "puf59_canonical.py": "model_qualification_application", + "puf59_canonical_artifact.py": "artifact_codec_identity", + "puf_detail_transfer.py": "model_qualification_application", + "puf_diagnostic_consumer.py": "model_qualification_application", + "puf_full_source.py": "source_authentication_projection", + "puf_full_source_graph.py": "graph_stage_adapter", + "puf_growth.py": "model_qualification_application", + "puf_growth_graph.py": "graph_stage_adapter", + "puf_monetary_agi_projection.py": "source_authentication_projection", + "puf_monetary_source.py": "source_authentication_projection", + "puf_price_baseline.py": "model_qualification_application", + "puf_qbi_model.py": "model_qualification_application", + "puf_raw_source.py": "source_authentication_projection", + "puf_target2024_growth.py": "model_qualification_application", + "source_csv_builtin.py": "source_authentication_projection", + "survey_age_activation.py": "calibration_reference_diagnostics", + "survey_age_calibration.py": "calibration_reference_diagnostics", + "survey_age_sources.py": "calibration_reference_diagnostics", + "survey_atomic_geography.py": "geography", + "survey_calibration_diagnostics.py": "calibration_reference_diagnostics", + "survey_catalogue_selection.py": "source_selection_composition", + "survey_financial_successor.py": "source_selection_composition", + "survey_observed_age.py": "source_value_contract", + "survey_origin_budget.py": "source_selection_composition", + "survey_population_domains.py": "source_value_contract", + "survey_population_preparation.py": "source_selection_composition", + "survey_population_replay.py": "artifact_codec_identity", + "survey_social_security.py": "model_qualification_application" + }, + "schema_version": 1, + "scopes": { + "microcosm.build.us_runtime.acs_native_coverage_binding::_live_code": { + "access": "selector", + "basis": "Inspect installed Python definitions/compiled code identities; selectors address modules and code maps, not population source columns.", + "body_sha256": "d32202aaa1bb8d5d64cd76ed46d931d57e687da3e472a329385b516891fbfff3", + "findings": [ + { + "expression": "getattr(module, node.name, None)", + "expression_sha256": "8b7242b94a1a0a35e47aea52db0299b3d258595c7f0193f205442ceb983c966f", + "kind": "getattr with an unresolvable dynamic attribute (fail-closed)" + }, + { + "expression": "getattr(module, alias.asname or alias.name, None)", + "expression_sha256": "dddbfdcbf7a4976c37bf74513761681d8d0dec20741cbf43b7f3070482b8bfee", + "kind": "getattr with an unresolvable dynamic attribute (fail-closed)" + } + ], + "references": [ + { + "caller": "microcosm.build.us_runtime.acs_native_coverage_binding::_producer", + "expression": "_live_code(module, compiled)", + "expression_sha256": "8823bc7ecf2953ab9eac27bed8967272bbc86ec551e69536bdea6085e0ef80ac", + "resolution": "resolved", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.acs_population_catalogue::_producer", + "expression": "native._live_code(module, compiled)", + "expression_sha256": "118874ad327943e342781a0e80f32626a0de20644d8a3af64adbb46a48437304", + "resolution": "resolved", + "usage": "call" + } + ], + "role": "implementation_verification" + }, + "microcosm.build.us_runtime.acs_native_coverage_binding::_live_code.check": { + "access": "selector", + "basis": "Inspect installed Python definitions/compiled code identities; selectors address modules and code maps, not population source columns.", + "body_sha256": "f17818fad716d5e076ac5bb948213038edfff4a954cc4a4d2cd2620484c23a0a", + "findings": [ + { + "expression": "compiled[path]", + "expression_sha256": "679c08f67d7eeaa8ffa77829d51b4be54a9d8bc30d2836521c23b057332fc259", + "kind": "subscript with an unresolvable dynamic selector (fail-closed)" + } + ], + "references": [ + { + "caller": "microcosm.build.us_runtime.acs_native_coverage_binding::_live_code", + "expression": "check(method)", + "expression_sha256": "0f4905add6a7fad7ab79e5e44711219c7f3ea864c9355ae3dce56026f1451619", + "resolution": "resolved", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.acs_native_coverage_binding::_live_code", + "expression": "check(method.fset)", + "expression_sha256": "114cc8136c568762be68e312cbcbdcb350ad27c4c402fffb100d21e61a459de1", + "resolution": "resolved", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.acs_native_coverage_binding::_live_code", + "expression": "check(value)", + "expression_sha256": "368cc1b17e3b39f4b1c86224dd79eacacc64f8e5326354331dcea4f4e8a888f0", + "resolution": "resolved", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.acs_native_coverage_binding::_live_code", + "expression": "check(method.fget)", + "expression_sha256": "df198d0c5dddcdf43557fdd4ade79b4d54bc453b984dac28790bac36eb4cd9ac", + "resolution": "resolved", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.acs_native_coverage_binding::_live_code.check", + "expression": "check(cell.cell_contents)", + "expression_sha256": "560a3574bcb1366b3269099cddabe0eccf895e55be9e23423a83426ddfaef059", + "resolution": "resolved", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.graph_full_puf_enrichment::_finalized_columns", + "expression": "binding.check()", + "expression_sha256": "8f0b002cc775c62b23467fc43b4dcdd594286e0cee46df8786bbc7dc05b758a9", + "resolution": "unresolved-us-candidate", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.graph_full_puf_enrichment::full_puf_attachment_nodes", + "expression": "binding.check()", + "expression_sha256": "8f0b002cc775c62b23467fc43b4dcdd594286e0cee46df8786bbc7dc05b758a9", + "resolution": "unresolved-us-candidate", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.graph_full_puf_enrichment::retain_full_puf_attachment", + "expression": "binding.check()", + "expression_sha256": "8f0b002cc775c62b23467fc43b4dcdd594286e0cee46df8786bbc7dc05b758a9", + "resolution": "unresolved-us-candidate", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.graph_full_puf_enrichment::verify_materialized_full_puf_attachment", + "expression": "binding.check()", + "expression_sha256": "8f0b002cc775c62b23467fc43b4dcdd594286e0cee46df8786bbc7dc05b758a9", + "resolution": "unresolved-us-candidate", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.graph_full_puf_enrichment::verify_materialized_full_puf_attachment", + "expression": "binding.check()", + "expression_sha256": "8f0b002cc775c62b23467fc43b4dcdd594286e0cee46df8786bbc7dc05b758a9", + "resolution": "unresolved-us-candidate", + "usage": "call" + } + ], + "role": "implementation_verification" + }, + "microcosm.build.us_runtime.acs_person_coverage_columns::read_acs_person_coverage_columns.consume_row": { + "access": "selector", + "basis": "Decode fixed source headers and literal row cells after the maintained file/member/roster checks; no downstream model treatment.", + "body_sha256": "523dd21fcb93fe0652e6a11dd3c6b4dd939a93971f369aea77d545c8a5fa6fa5", + "findings": [ + { + "expression": "cells[0]", + "expression_sha256": "1d38133b3f738d0a1397cf51545896818fa3f9ec8c917b5e1c284ed71d15df70", + "kind": "subscript with an unresolvable dynamic selector (fail-closed)" + } + ], + "references": [ + { + "caller": "microcosm.build.us_runtime.acs_person_coverage_columns::read_acs_person_coverage_columns", + "expression": "consume_row", + "expression_sha256": "66d5dba74329be46cd3fdde985b85eddd5d94b1f17c43bd4a239b28f6e7e00a9", + "resolution": "resolved", + "usage": "reference" + } + ], + "role": "literal_source_projection" + }, + "microcosm.build.us_runtime.acs_population_catalogue::_collect.household_record": { + "access": "selector", + "basis": "Decode fixed source headers and literal row cells after the maintained file/member/roster checks; no downstream model treatment.", + "body_sha256": "2e5482ab43169555ae034fb0ad0403174d9aba502bdd1562429cbe6ae6d38536", + "findings": [ + { + "expression": "row[hcols['SERIALNO']]", + "expression_sha256": "04d019690b4e8abafbdc4697c5071501173bc9d160a0a8a1e6d994bed2854785", + "kind": "subscript with an unresolvable dynamic selector (fail-closed)" + }, + { + "expression": "row[hcols['TYPEHUGQ']]", + "expression_sha256": "8ae5ff68252173d76f871e9be3b311ab168efb82303cf9c130cbaca678611ac3", + "kind": "subscript with an unresolvable dynamic selector (fail-closed)" + }, + { + "expression": "row[hcols['NP']]", + "expression_sha256": "40af9567429887066236ead76b93e188bbfbe858895a5d4e752f0cfe21588d39", + "kind": "subscript with an unresolvable dynamic selector (fail-closed)" + }, + { + "expression": "row[hcols['WGTP']]", + "expression_sha256": "0cf2bcbad514c0684ab8b65eaffd26f9ced4abe0b9cac1ceff98df2eede60467", + "kind": "subscript with an unresolvable dynamic selector (fail-closed)" + }, + { + "expression": "row[hcols['source_member']]", + "expression_sha256": "88bae2a597fa6147363ccc8e05c85c942df5acfdf8ffbec94b5f29ba7c5a4879", + "kind": "subscript with an unresolvable dynamic selector (fail-closed)" + }, + { + "expression": "row[hcols['source_row_ordinal']]", + "expression_sha256": "9f20beeae9eeb31fefe8b892a0e7d52576a73c4229a712b9962bb62c5f26178a", + "kind": "subscript with an unresolvable dynamic selector (fail-closed)" + } + ], + "references": [ + { + "caller": "microcosm.build.us_runtime.acs_population_catalogue::_collect", + "expression": "household_record(row, ())", + "expression_sha256": "c697983b7085765be779b19449664b5ab0e83502cc86224b0693f92f0f0526e5", + "resolution": "resolved", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.acs_population_catalogue::_collect.consume", + "expression": "household_record(row, members)", + "expression_sha256": "9a6177140ccefc4c374cfa5e3a9bde7f552370e8f4837c8ec9ce9f9d0765206e", + "resolution": "resolved", + "usage": "call" + } + ], + "role": "literal_source_projection" + }, + "microcosm.build.us_runtime.acs_population_catalogue::_source_checks": { + "access": "selector", + "basis": "Decode fixed source headers and literal row cells after the maintained file/member/roster checks; no downstream model treatment.", + "body_sha256": "2f648bf395c68078cd47f2c380c71a9ac2652e6dc5fad303a5fd4248d2ca60ba", + "findings": [ + { + "expression": "paths[role]", + "expression_sha256": "ee44740a981483d4552a047bf772b92d879483e239759fa850de4202137d4086", + "kind": "subscript with an unresolvable dynamic selector (fail-closed)" + } + ], + "references": [ + { + "caller": "microcosm.build.us_runtime.acs_population_catalogue::_checked", + "expression": "_source_checks(owned.source_dir, dict(owned.paths), owned.pins)", + "expression_sha256": "d0388a94cc54990dc1a85b7fb1ff3c6e0006cedb7416a2b5b848751b54bb9e8e", + "resolution": "resolved", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.acs_population_catalogue::issue_acs_source_catalogue", + "expression": "_source_checks(source_dir, paths, pins)", + "expression_sha256": "e3f55a4a45cab938c6319af0742bfd1ce872138a5cb02ce5bc0d1fd8ac87d621", + "resolution": "resolved", + "usage": "call" + } + ], + "role": "literal_source_projection" + }, + "microcosm.build.us_runtime.asec_2024_native_population::_inputs": { + "access": "selector", + "basis": "Validate an authenticated source artifact, declared source tuple, member selection or issuer state before population/model admission.", + "body_sha256": "d47bb577eedd5ce4c0628328596313e57b38022d3a2d68fce4108472cbd6defa", + "findings": [ + { + "expression": "persons[year]", + "expression_sha256": "40a7e8f4edfb7f4e322841086c2afb7013cacaa4ae93b33eaad0d1a58cdefbc8", + "kind": "subscript with an unresolvable dynamic selector (fail-closed)" + } + ], + "references": [ + { + "caller": "microcosm.build.us_runtime.asec_2024_native_population::load_authenticated_asec_2024_native_population", + "expression": "_inputs(parent_path, household_attachment_path, person_income_attachment_path, person_member_paths, household_member_path, selected_households, candidate)", + "expression_sha256": "064e39022c67571ab4f0217339942c279139ca53e8ae014c1aa56c26c526fdec", + "resolution": "resolved", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.asec_population_catalogue::issue_asec_source_catalogue", + "expression": "native._inputs(parent_path, household_attachment_path, person_income_attachment_path, person_member_paths, household_member_path, None, candidate)", + "expression_sha256": "699ee3b4f1f70fa5508704dc9848004ae83f493d3ca66ea6478e4d6ffc85c450", + "resolution": "resolved", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.graph_puf55_survey_recipients::puf55_survey_recipient_nodes", + "expression": "financial.financial._inputs(entry[2].financial_population.frame)", + "expression_sha256": "e13d917860a0b7fe5c3141fe10d1817d5695f9174086de5f694aa9f2f1350956", + "resolution": "unresolved-us-candidate", + "usage": "call" + } + ], + "role": "source_authentication" + }, + "microcosm.build.us_runtime.atomic_block_api_sources::_raw_source": { + "access": "selector", + "basis": "Validate an authenticated source artifact, declared source tuple, member selection or issuer state before population/model admission.", + "body_sha256": "8b67416dbae3a3b5efd89046946f30fa30d43c20a5213618e1fe9af5bc146734", + "findings": [ + { + "expression": "expanded[0]", + "expression_sha256": "7b99f83f4517027e237bd868552dead87644d82c9023a97d2b6bf45246cf77f8", + "kind": "subscript with an unresolvable dynamic selector (fail-closed)" + }, + { + "expression": "expanded[0]", + "expression_sha256": "1055a5f7982807b7a92f0f83c6fb7af596f512c67397f285dcb2a8519ce13e0a", + "kind": "subscript with an unresolvable dynamic selector (fail-closed)" + } + ], + "references": [ + { + "caller": "microcosm.build.us_runtime.atomic_block_api_sources::assemble_atomic_block_api_sources", + "expression": "_raw_source(item[1], bounds=bounds, expanded=expanded)", + "expression_sha256": "07080bb6035c5036099d39f8e2b2d710d803471adb5e6ede0a8748d2f813bec3", + "resolution": "resolved", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.atomic_block_api_sources::assemble_atomic_block_api_sources", + "expression": "_raw_source(item[2], bounds=bounds, expanded=expanded)", + "expression_sha256": "b9b2bea4a74d7294254431323750a1504c52f50bfbd87e16c9ea5b887a335599", + "resolution": "resolved", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.atomic_block_api_sources::assemble_atomic_block_api_sources", + "expression": "_raw_source(tract_to_puma, bounds=bounds, expanded=expanded)", + "expression_sha256": "c9e3c0a8931f43fa90793ce9a59880375c01d8404aeb3303fd3dd7acc809b051", + "resolution": "resolved", + "usage": "call" + } + ], + "role": "source_authentication" + }, + "microcosm.build.us_runtime.atomic_block_sources::_selected_zip_member": { + "access": "selector", + "basis": "Validate an authenticated source artifact, declared source tuple, member selection or issuer state before population/model admission.", + "body_sha256": "fe9760af9e6666148c440b44b36364e742e0fc5da000c78d58cd68a00efb286f", + "findings": [ + { + "expression": "expanded[0]", + "expression_sha256": "7b99f83f4517027e237bd868552dead87644d82c9023a97d2b6bf45246cf77f8", + "kind": "subscript with an unresolvable dynamic selector (fail-closed)" + }, + { + "expression": "expanded[0]", + "expression_sha256": "1055a5f7982807b7a92f0f83c6fb7af596f512c67397f285dcb2a8519ce13e0a", + "kind": "subscript with an unresolvable dynamic selector (fail-closed)" + } + ], + "references": [ + { + "caller": "microcosm.build.us_runtime.atomic_block_api_sources::assemble_atomic_block_api_sources", + "expression": "sources._selected_zip_member(cd_archive, sources.CD_MEMBER, bounds=bounds, expanded=expanded)", + "expression_sha256": "afc0321044a4b9f0e35a24197a425f89041ded45b100e7b937c7d1d8bb75bde7", + "resolution": "resolved", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.atomic_block_sources::assemble_atomic_block_sources", + "expression": "_selected_zip_member(cd_archive, CD_MEMBER, bounds=bounds, expanded=expanded)", + "expression_sha256": "b8c2f8b487afafe7d30e5a149e356e51faab833cd02703dfc515226654caeabf", + "resolution": "resolved", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.atomic_block_sources::assemble_atomic_block_sources", + "expression": "_selected_zip_member(record, _GEO_MEMBERS[state], bounds=bounds, expanded=expanded)", + "expression_sha256": "bd65898f1c9844e4614b934b95a9b550d8ed245efc93aea45be179b99ad3bbd2", + "resolution": "resolved", + "usage": "call" + } + ], + "role": "source_authentication" + }, + "microcosm.build.us_runtime.current_asec_demographics::qualify_current_asec_demographics": { + "access": "provenance", + "basis": "Interpret and join authenticated native ASEC/ACS observations using the bound native source identity and period; preserve the declared source interpretation.", + "body_sha256": "fa379b1e210dbeb126ab7e970a5704662c19cff672da36ceadff40acbf7d5140", + "findings": [ + { + "expression": "households[support_channel_column('household')]", + "expression_sha256": "ff19233d468cd38c86b483dcdd9e38cb33fd3873f0df0bd667af4e1063d385bb", + "kind": "subscript using call to support_channel_column" + }, + { + "expression": "support_channel_column('household')", + "expression_sha256": "bb43a8f3b7827b0f6f755ff2679d6de24fd68081e9650d8261b0634d5e74f447", + "kind": "call to support_channel_column" + }, + { + "expression": "selected_hh[['household_id', spine_source_id_column('household')]]", + "expression_sha256": "68e04d26c0d6ccb08d27d633707e275ebc56b37678e916650689e364e8e712e0", + "kind": "subscript using call to spine_source_id_column" + }, + { + "expression": "spine_source_id_column('household')", + "expression_sha256": "dbaa5f748b8b686aadf9f965a360f5ed79a78ef9993133c1084c8b350f1815e9", + "kind": "call to spine_source_id_column" + }, + { + "expression": "person[support_channel_column('person')]", + "expression_sha256": "10d91707d79271c150935ed7268a132cc2f0b2189bc86107f773dbc261ed441b", + "kind": "subscript using call to support_channel_column" + }, + { + "expression": "support_channel_column('person')", + "expression_sha256": "7cf3594f675603d58586f7d491519fb21a01f0471e5905b181248d3d17223cdb", + "kind": "call to support_channel_column" + }, + { + "expression": "selected[spine_source_id_column('person')]", + "expression_sha256": "21919c11bbea71f7372242a56f733b95201f65ebedd26aff413030e134132551", + "kind": "subscript using call to spine_source_id_column" + }, + { + "expression": "spine_source_id_column('person')", + "expression_sha256": "5bff0f374c0361bf89039f0907ff51ae6e76189950720fae047b4f2df3584bf5", + "kind": "call to spine_source_id_column" + } + ], + "references": [ + { + "caller": "microcosm.build.us_runtime.current_survey_geography::qualify_current_survey_geography", + "expression": "demographics.qualify_current_asec_demographics(preparation)", + "expression_sha256": "0a005bb62988d12543700e2f8dafbb01ce94acc5d8f58e5e4aaa71238c48f178", + "resolution": "resolved", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.current_survey_predictors::_demographic_features", + "expression": "observed_geography.demographics.qualify_current_asec_demographics(preparation)", + "expression_sha256": "bd081f5f43db6bcb7078bc9db120f69e0f0f3033bb10271fe58946d0c990ccbd", + "resolution": "unresolved-us-candidate", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.graph_current_survey_predictors::_Kernel.implementation_hash", + "expression": "demographics.qualify_current_asec_demographics", + "expression_sha256": "ac2b1db3605a778198f92912ab4eb3af093bbb83344870221341bb48801033a6", + "resolution": "unresolved-us-candidate", + "usage": "reference" + } + ], + "role": "native_projection" + }, + "microcosm.build.us_runtime.current_social_security_source::qualify_current_social_security": { + "access": "provenance", + "basis": "Interpret and join authenticated native ASEC/ACS observations using the bound native source identity and period; preserve the declared source interpretation.", + "body_sha256": "2a6a5b483b094c9dc4f74b811544037f95dfaa19b18f84f4eea0c26cead47d7b", + "findings": [ + { + "expression": "people[support_channel_column('person')]", + "expression_sha256": "230a04e693074722d092c0f7cad4b0f1f7266f7db14e9db6e96d8d3f50a69775", + "kind": "subscript using call to support_channel_column" + }, + { + "expression": "support_channel_column('person')", + "expression_sha256": "7cf3594f675603d58586f7d491519fb21a01f0471e5905b181248d3d17223cdb", + "kind": "call to support_channel_column" + }, + { + "expression": "people[spine_source_id_column('person')]", + "expression_sha256": "4b7b6a2b87df2d4a901c1d33163a43df841dcf4eed197a62981d30b013ce91ad", + "kind": "subscript using call to spine_source_id_column" + }, + { + "expression": "spine_source_id_column('person')", + "expression_sha256": "5bff0f374c0361bf89039f0907ff51ae6e76189950720fae047b4f2df3584bf5", + "kind": "call to spine_source_id_column" + } + ], + "references": [ + { + "caller": "microcosm.build.us_runtime.puf55_survey_recipients::qualify_puf55_survey_recipients", + "expression": "ss.reports.qualify_current_social_security(preparation)", + "expression_sha256": "4d6e06c1cf44e39a642acfbc819b21bb68ff889b2af84bc2bd79c4899c97e7f9", + "resolution": "unresolved-us-candidate", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.puf55_survey_ss_measurement::qualify_puf55_survey_ss_measurement", + "expression": "reports.qualify_current_social_security(preparation)", + "expression_sha256": "25015a62e963e3db2a429b4cfd2e771ec6e7faca03b9614683fb8b6079a65378", + "resolution": "resolved", + "usage": "call" + } + ], + "role": "native_projection" + }, + "microcosm.build.us_runtime.current_survey_geography::_project": { + "access": "provenance", + "basis": "Verify or join exact source/clone lineage; provenance identifies a carried observation/attachment, not a generic treatment branch.", + "body_sha256": "7318336b90e197069b63e861c2ee8411eeb209b41d51867eb17ae3877b37b736", + "findings": [ + { + "expression": "support_channel_column('household')", + "expression_sha256": "bb43a8f3b7827b0f6f755ff2679d6de24fd68081e9650d8261b0634d5e74f447", + "kind": "call to support_channel_column" + }, + { + "expression": "spine_source_id_column('household')", + "expression_sha256": "dbaa5f748b8b686aadf9f965a360f5ed79a78ef9993133c1084c8b350f1815e9", + "kind": "call to spine_source_id_column" + }, + { + "expression": "households[channel_column]", + "expression_sha256": "3ce03ce5ca217ce312cb0c8d56f3bea8365db21f57d28dd656fc7c8fc1751d37", + "kind": "subscript using call to support_channel_column" + }, + { + "expression": "households[['household_id', channel_column, native_column]]", + "expression_sha256": "24134a6fb357480652c311d72e77b653999f33986b03c82f885eb38eab2e17a7", + "kind": "subscript using call to support_channel_column" + } + ], + "references": [ + { + "caller": "microcosm.build.us_runtime.current_survey_geography::qualify_current_survey_geography", + "expression": "_project(origins, state.frame.table('household'), acs_frame.table('household'), asec.household)", + "expression_sha256": "304b01b9947307c2df3d2ee8e0fb8400d066bb2d92668dcf00775ca0d49974a6", + "resolution": "resolved", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.current_survey_geography::qualify_current_survey_geography", + "expression": "_project(origins, state.frame.table('household'), acs_frame.table('household'), asec.household)", + "expression_sha256": "304b01b9947307c2df3d2ee8e0fb8400d066bb2d92668dcf00775ca0d49974a6", + "resolution": "resolved", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.current_survey_predictors::_demographic_features", + "expression": "observed_geography._project(json.loads(entry[1])['origins']['households'], state.frame.table('household'), acs_frame.table('household'), values.household)", + "expression_sha256": "4843b18930081699f6a5f4307c85a24e6a32a845ee7c5e4cccd4cd4c4beab5e4", + "resolution": "resolved", + "usage": "call" + } + ], + "role": "origin_attachment" + }, + "microcosm.build.us_runtime.current_survey_predictors::_acs_earnings": { + "access": "provenance", + "basis": "Qualify source-specific measured inputs and complete the declared pre-PUF donor/recipient surface before the source-blind model application.", + "body_sha256": "171716dfd0ee91338dc2d1fd752e39e15eaaf5af355a392c812840dc5399b07f", + "findings": [ + { + "expression": "person[provenance.support_channel_column('person')]", + "expression_sha256": "148bd90973a8acaf59f39bf2890a847d3e4774f95fd7d5b956a5f18ce33557a1", + "kind": "subscript using call to support_channel_column" + }, + { + "expression": "provenance.support_channel_column('person')", + "expression_sha256": "1ca82bc581cfd1d0a520cccb66737a50f42281b6ac1405e50a4a3083d63f2ea5", + "kind": "call to support_channel_column" + } + ], + "references": [ + { + "caller": "microcosm.build.us_runtime.current_survey_predictors::qualify_current_survey_predictors", + "expression": "_acs_earnings(state.frame)", + "expression_sha256": "a4420cb5909d4e6fd1608fbd32eb74ce7c3ebc900d6af1958ddbd9cfb666a186", + "resolution": "resolved", + "usage": "call" + } + ], + "role": "model_qualification" + }, + "microcosm.build.us_runtime.current_survey_predictors::_check_demographic_values": { + "access": "selector", + "basis": "Read declared artifacts, validation metadata, typed state tuples or columns solely for byte/value/axis identity checks; caller and body contracts bind those checks.", + "body_sha256": "e68627e10c41d3dfb82d9fcd90fbb0fea5788128ef679470fbdf9224b1539fe8", + "findings": [ + { + "expression": "getattr(values, entity)", + "expression_sha256": "62eedfd74c15e2e45b6af2f7af6ccb5eadab66180fc5d056657db72578fa05c8", + "kind": "getattr with an unresolvable dynamic attribute (fail-closed)" + } + ], + "references": [ + { + "caller": "microcosm.build.us_runtime.current_survey_predictors::_demographic_features", + "expression": "_check_demographic_values(values, receipt)", + "expression_sha256": "89c9ec06a912ee8b67724d15c789c399788367f70b3c9c877c1a7d9a84c1d20d", + "resolution": "resolved", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.current_survey_predictors::_demographic_features", + "expression": "_check_demographic_values(values, receipt)", + "expression_sha256": "89c9ec06a912ee8b67724d15c789c399788367f70b3c9c877c1a7d9a84c1d20d", + "resolution": "resolved", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.current_survey_predictors::qualify_current_survey_predictors", + "expression": "_check_demographic_values(demographic_values, demographic_receipt)", + "expression_sha256": "b5b7e6e023f528cc6f42fc0011f4929e56ae34c909c74b3e46d2b94056e57314", + "resolution": "resolved", + "usage": "call" + } + ], + "role": "artifact_verification" + }, + "microcosm.build.us_runtime.current_survey_predictors::_demographic_features": { + "access": "provenance", + "basis": "Qualify source-specific measured inputs and complete the declared pre-PUF donor/recipient surface before the source-blind model application.", + "body_sha256": "c164243cd857ae1a4813ed50356b480da1fb93d195af56cf31e99f6dcefc5864", + "findings": [ + { + "expression": "entry[2]", + "expression_sha256": "255e733c946eab6358f6001cb56e783019ff9f5d1daf0b342a6ef2baa3cb55cc", + "kind": "subscript with an unresolvable dynamic selector (fail-closed)" + }, + { + "expression": "entry[1]", + "expression_sha256": "6eb20e05f22592d515f480a018bda640991f8d6f9d208eaf86997842f151ae90", + "kind": "subscript with an unresolvable dynamic selector (fail-closed)" + }, + { + "expression": "entry[1]", + "expression_sha256": "6eb20e05f22592d515f480a018bda640991f8d6f9d208eaf86997842f151ae90", + "kind": "subscript with an unresolvable dynamic selector (fail-closed)" + }, + { + "expression": "people[provenance.support_channel_column('person')]", + "expression_sha256": "4be741c1ce94301fd1e79026254b25f342c3dc147894583489ae0c387a42b99d", + "kind": "subscript using call to support_channel_column" + }, + { + "expression": "provenance.support_channel_column('person')", + "expression_sha256": "1ca82bc581cfd1d0a520cccb66737a50f42281b6ac1405e50a4a3083d63f2ea5", + "kind": "call to support_channel_column" + }, + { + "expression": "people[provenance.spine_source_id_column('person')]", + "expression_sha256": "02d22f188dadde849f2d771bb30843a0d7c1102235e0a343d2ea3b8ed1240816", + "kind": "subscript using call to spine_source_id_column" + }, + { + "expression": "provenance.spine_source_id_column('person')", + "expression_sha256": "15b03f981ea284db003b5470bf2124ba8c096ff9331cdb987e95897a6e735f63", + "kind": "call to spine_source_id_column" + } + ], + "references": [ + { + "caller": "microcosm.build.us_runtime.current_survey_predictors::qualify_current_survey_predictors", + "expression": "_demographic_features(preparation, entry)", + "expression_sha256": "72721903f4843ceb63f55f10934f12c2693a53e4887316e7e058d14e7a16acf6", + "resolution": "resolved", + "usage": "call" + } + ], + "role": "model_qualification" + }, + "microcosm.build.us_runtime.current_survey_predictors::complete_predictor_columns": { + "access": "provenance", + "basis": "Verify or join exact source/clone lineage; provenance identifies a carried observation/attachment, not a generic treatment branch.", + "body_sha256": "0e82a853c760169791489d45cb68770e1112fdef6b5e7c92ead7f11a6caf6eaf", + "findings": [ + { + "expression": "drawn_money[target]", + "expression_sha256": "3308e90a8632b1fb4fd68700e8115bd1cbe494a5db1dccb515cd26930fad4e2c", + "kind": "subscript with an unresolvable dynamic selector (fail-closed)" + }, + { + "expression": "person[provenance.spine_source_id_column('person')]", + "expression_sha256": "a707963ec81a2ce4ab39ad89e5686bd176b0ce33c9eea6ddd933b095e41d18a3", + "kind": "subscript using call to spine_source_id_column" + }, + { + "expression": "provenance.spine_source_id_column('person')", + "expression_sha256": "15b03f981ea284db003b5470bf2124ba8c096ff9331cdb987e95897a6e735f63", + "kind": "call to spine_source_id_column" + }, + { + "expression": "person[provenance.support_channel_column('person')]", + "expression_sha256": "148bd90973a8acaf59f39bf2890a847d3e4774f95fd7d5b956a5f18ce33557a1", + "kind": "subscript using call to support_channel_column" + }, + { + "expression": "provenance.support_channel_column('person')", + "expression_sha256": "1ca82bc581cfd1d0a520cccb66737a50f42281b6ac1405e50a4a3083d63f2ea5", + "kind": "call to support_channel_column" + } + ], + "references": [ + { + "caller": "microcosm.build.us_runtime.graph_atomic_survey_financial::run_atomic_survey_financial", + "expression": "values.complete_predictor_columns(qualified, prefix.clone_population.frame, drawn)", + "expression_sha256": "210d0f57193961bb1abcef15fa688611198314407a677c5164d0a5639b0a2957", + "resolution": "unresolved-us-candidate", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.graph_current_survey_predictors::CurrentSurveyPredictorAttachKernel.run", + "expression": "values.complete_predictor_columns(qualified, self.clone_population.frame, drawn)", + "expression_sha256": "b8134f19812d9a05735d76db8ed3e3c47d0e8ee2c988356b7ddecb36e64a8004", + "resolution": "resolved", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.graph_current_survey_predictors::verify_materialized_current_survey_predictors", + "expression": "values.complete_predictor_columns(qualified, clone_population.frame, drawn)", + "expression_sha256": "e55a16e07510678da2263ab7e910567f93cb0488fb81a05f8c629a1670950f2d", + "resolution": "resolved", + "usage": "call" + } + ], + "role": "origin_attachment" + }, + "microcosm.build.us_runtime.current_survey_predictors::qualify_current_survey_predictors": { + "access": "provenance", + "basis": "Qualify source-specific measured inputs and complete the declared pre-PUF donor/recipient surface before the source-blind model application.", + "body_sha256": "c4eeee32feac6d185d56f74fcbfcbdcc375a5b58dbbce50a8ae2e4efaa494de2", + "findings": [ + { + "expression": "people[provenance.support_channel_column('person')]", + "expression_sha256": "4be741c1ce94301fd1e79026254b25f342c3dc147894583489ae0c387a42b99d", + "kind": "subscript using call to support_channel_column" + }, + { + "expression": "provenance.support_channel_column('person')", + "expression_sha256": "1ca82bc581cfd1d0a520cccb66737a50f42281b6ac1405e50a4a3083d63f2ea5", + "kind": "call to support_channel_column" + }, + { + "expression": "people[provenance.spine_source_id_column('person')]", + "expression_sha256": "02d22f188dadde849f2d771bb30843a0d7c1102235e0a343d2ea3b8ed1240816", + "kind": "subscript using call to spine_source_id_column" + }, + { + "expression": "provenance.spine_source_id_column('person')", + "expression_sha256": "15b03f981ea284db003b5470bf2124ba8c096ff9331cdb987e95897a6e735f63", + "kind": "call to spine_source_id_column" + } + ], + "references": [ + { + "caller": "microcosm.build.us_runtime.graph_atomic_survey_financial::run_atomic_survey_financial", + "expression": "values.qualify_current_survey_predictors(prefix.preparation, prefix.allocated_population, prefix.clone_population, demographic_conditioning=demographic_conditioning, geography_config=geography_config)", + "expression_sha256": "1c723c84594f66dc8160c3c2848e589f4722b18013343900252e6ed38612e13f", + "resolution": "unresolved-us-candidate", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.graph_current_survey_predictors::_Kernel._qualified", + "expression": "values.qualify_current_survey_predictors(self.preparation, self.allocated_population, self.clone_population, demographic_conditioning=self.demographic_conditioning, geography_config=self.geography_config)", + "expression_sha256": "b82d22066e1251cd1ae882465d88eb721638e131319fb16eb018c0a26b29b78c", + "resolution": "resolved", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.graph_current_survey_predictors::verify_materialized_current_survey_predictors", + "expression": "values.qualify_current_survey_predictors(preparation, allocated_population, clone_population, demographic_conditioning=demographic_conditioning, geography_config=geography_config)", + "expression_sha256": "d5ee5357dcedc7f0207cb53c942725b40d22451f3f35d8631fc5512f00fa9194", + "resolution": "resolved", + "usage": "call" + } + ], + "role": "model_qualification" + }, + "microcosm.build.us_runtime.full_puf_enrichment::_known": { + "access": "selector", + "basis": "Select a declared finite profile, canonical monetary field, knownness cell or model artifact target; no source-origin column access is granted.", + "body_sha256": "bd585c93be464d7a60c26052ae078ae1c835dfd6237eb646fec1761d4ce4d397", + "findings": [ + { + "expression": "known[column]", + "expression_sha256": "0aeb857ab0fe8083a0dfa3d052dc2c8b7fa5aed805065db637f8b6619567fcc1", + "kind": "subscript with an unresolvable dynamic selector (fail-closed)" + } + ], + "references": [ + { + "caller": "microcosm.build.us_runtime.full_puf_enrichment::_recipient_matrix", + "expression": "_known(predictor_known, predictor_known, profile.predictors, 'recipient_predictor')", + "expression_sha256": "7b0b86dd3abcf378964ce1faeb9cb0daf70f74146888aa2bb015fcb2868a6fde", + "resolution": "resolved", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.full_puf_enrichment::canonical_full_puf_donor", + "expression": "_known(person, person_known, pcols, 'person')", + "expression_sha256": "a1f6b746c73ea2cc5a0e1fd1dfc1a6eff2f19033dbb3090bb72d1889c5490fa7", + "resolution": "resolved", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.full_puf_enrichment::canonical_full_puf_donor", + "expression": "_known(tax_unit, tax_unit_known, tcols, 'tax_unit')", + "expression_sha256": "fa3c9e7fe7e4129628b7e74d01b11f2c655eeff4047bfc96c05f0fe4f9d48764", + "resolution": "resolved", + "usage": "call" + } + ], + "role": "closed_model_selector" + }, + "microcosm.build.us_runtime.full_puf_enrichment::_profile_source_values": { + "access": "selector", + "basis": "Select a declared finite profile, canonical monetary field, knownness cell or model artifact target; no source-origin column access is granted.", + "body_sha256": "a6ce699a81659568a14334794bd36fe3534178fb1b93da216d31bac83d63129c", + "findings": [ + { + "expression": "table[column]", + "expression_sha256": "6422895606da8898c2e65a4eda29c5c335cbb89f48d88b50f65e5c373568dffe", + "kind": "subscript with an unresolvable dynamic selector (fail-closed)" + } + ], + "references": [ + { + "caller": "microcosm.build.us_runtime.full_puf_enrichment::_recipient_matrix", + "expression": "_profile_source_values(selected, profile=profile)", + "expression_sha256": "76540388e79d5523b9a7447d1a7e8d9062c5293b2bf93017d5c9f36d6cfe2804", + "resolution": "resolved", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.full_puf_enrichment::_validated_model_donor", + "expression": "_profile_source_values(donor, profile=profile)", + "expression_sha256": "39574182cf75cceb7cf365ec21b7c4e3c85ff5817bf67cdc601c091832a99532", + "resolution": "resolved", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.full_puf_enrichment::canonical_full_puf_donor", + "expression": "_profile_source_values(tax_unit, profile=profile)", + "expression_sha256": "bf273f49a26a036efbfd9c39fa59e6ca191057baeec3e172bb3da0c435386da1", + "resolution": "resolved", + "usage": "call" + } + ], + "role": "closed_model_selector" + }, + "microcosm.build.us_runtime.full_puf_enrichment::_validated_model_donor": { + "access": "selector", + "basis": "Select a declared finite profile, canonical monetary field, knownness cell or model artifact target; no source-origin column access is granted.", + "body_sha256": "fc057a9fec21c1472eba444725aab5bb0b004a2cc365e4a52995f06eaf408a5a", + "findings": [ + { + "expression": "donor[column]", + "expression_sha256": "570d19cfd29a6f3cbc133decfc7f86c900b8603e88da044d708c3be262151e1a", + "kind": "subscript with an unresolvable dynamic selector (fail-closed)" + }, + { + "expression": "donor[profile.predictors[1]]", + "expression_sha256": "0a50b86700bb1389a747132e7c57a1e294001c517c8fde964d223e3c765cd79e", + "kind": "subscript with an unresolvable dynamic selector (fail-closed)" + }, + { + "expression": "donor[profile.donor_auxiliary_columns[0]]", + "expression_sha256": "ad1d1e079877ae81374b3a20c4d4f799e39886f72cd5e252f8ab7b7760862a91", + "kind": "subscript with an unresolvable dynamic selector (fail-closed)" + }, + { + "expression": "donor[profile.predictors[0]]", + "expression_sha256": "83983052822fa29dc502609a7e0f14ff949209093231d1ab81c34628a5110eeb", + "kind": "subscript with an unresolvable dynamic selector (fail-closed)" + }, + { + "expression": "donor[column]", + "expression_sha256": "570d19cfd29a6f3cbc133decfc7f86c900b8603e88da044d708c3be262151e1a", + "kind": "subscript with an unresolvable dynamic selector (fail-closed)" + }, + { + "expression": "donor[column]", + "expression_sha256": "570d19cfd29a6f3cbc133decfc7f86c900b8603e88da044d708c3be262151e1a", + "kind": "subscript with an unresolvable dynamic selector (fail-closed)" + }, + { + "expression": "donor.loc[:, [*profile.predictors, *profile.targets, 'weight']]", + "expression_sha256": "06f651eef3df2d55a51fa6e1ca1fc4d25a22d570e2c82f0976480bf3e850b393", + "kind": "subscript with an unresolvable dynamic selector (fail-closed)" + } + ], + "references": [ + { + "caller": "microcosm.build.us_runtime.full_puf_enrichment::prepare_full_puf_inputs", + "expression": "_validated_model_donor(donor, profile=profile)", + "expression_sha256": "9e17f8eb4038fb2e7b18cf35686ef7fcf14c6bbfb522fa6bd23925d84e640829", + "resolution": "resolved", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.graph_puf55_canonical_donor::CanonicalPuf55DonorKernel.run", + "expression": "full._validated_model_donor(donor, profile=profile)", + "expression_sha256": "1a82d28d58481167ccdffb665ff7f923bab703f56882f8eb59474b49ee9166e3", + "resolution": "resolved", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.puf55_route_finalization::_finalize_puf55_routes", + "expression": "full._validated_model_donor(donor, profile=profile)", + "expression_sha256": "1a82d28d58481167ccdffb665ff7f923bab703f56882f8eb59474b49ee9166e3", + "resolution": "resolved", + "usage": "call" + } + ], + "role": "closed_model_selector" + }, + "microcosm.build.us_runtime.full_puf_enrichment::canonical_full_puf_donor": { + "access": "selector", + "basis": "Select a declared finite profile, canonical monetary field, knownness cell or model artifact target; no source-origin column access is granted.", + "body_sha256": "d5a3a6942dd6de0d44e2dfb4f1968df1a59ef398b4bfdda6ce7e30d67bdb8636", + "findings": [ + { + "expression": "person[column]", + "expression_sha256": "152b79a46f5562de5718cb10ced542b876132f8e0f7c048c85f7af96bcf2481b", + "kind": "subscript with an unresolvable dynamic selector (fail-closed)" + }, + { + "expression": "tax_unit[return_capacity_column]", + "expression_sha256": "ee87378f29dffc5c361628a6b6cd34f98b60437f65e9117226f1c6214e8339db", + "kind": "subscript with an unresolvable dynamic selector (fail-closed)" + }, + { + "expression": "tax_unit[column]", + "expression_sha256": "ab1b5283a8c6873092280edb87b9c4cfb06cbde6737217c4e26f7bd862f05e32", + "kind": "subscript with an unresolvable dynamic selector (fail-closed)" + } + ], + "references": [ + { + "caller": "microcosm.build.us_runtime.puf55_canonical_donor::canonical_puf55_donor_from_artifact", + "expression": "enrichment.canonical_full_puf_donor(None, tax_unit, person_known=None, tax_unit_known=known, person_targets_at_tax_unit=profile.person_outputs, profile=profile)", + "expression_sha256": "7a6f76f68949332a02103ad838dd54ac6c21bfe78df2d5d02aa353cc05c719e4", + "resolution": "resolved", + "usage": "call" + } + ], + "role": "closed_model_selector" + }, + "microcosm.build.us_runtime.full_puf_enrichment::decode_full_puf_draws": { + "access": "selector", + "basis": "Select a declared finite profile, canonical monetary field, knownness cell or model artifact target; no source-origin column access is granted.", + "body_sha256": "f812d50cc244107a9e605cb1302e365f42e215ff14a9c47bc64667643beb3727", + "findings": [ + { + "expression": "raw_draws[target]", + "expression_sha256": "13b95ca4ade5fae4a9f6afd64f007c010587689505b370fe4e4c695ab667e2ba", + "kind": "subscript with an unresolvable dynamic selector (fail-closed)" + }, + { + "expression": "raw_draws[target]", + "expression_sha256": "13b95ca4ade5fae4a9f6afd64f007c010587689505b370fe4e4c695ab667e2ba", + "kind": "subscript with an unresolvable dynamic selector (fail-closed)" + } + ], + "references": [ + { + "caller": "microcosm.build.us_runtime.full_puf_enrichment::finalize_full_puf", + "expression": "decode_full_puf_draws(matrix=matrix, matrix_producer_key=matrix_producer_key, raw_draws=raw_draws, apply_state=apply_state, training_state=training_state, seed=seed, profile=profile)", + "expression_sha256": "deca7ce71a057c6dabcf0cf8ddfa45b0a91e570acaf59ac0b7bbb5727a5d5aff", + "resolution": "resolved", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.puf55_route_finalization::merge_puf55_route_draws", + "expression": "full.decode_full_puf_draws(matrix=matrix, matrix_producer_key=key, raw_draws=dict(raw), apply_state=apply_state, training_state=training_state, seed=seed, profile=profile)", + "expression_sha256": "afa62f25d41b7a81a6fd5804cd3443d0e01fc950fc07a7057ec1d1cb8502d982", + "resolution": "resolved", + "usage": "call" + } + ], + "role": "closed_model_selector" + }, + "microcosm.build.us_runtime.full_puf_enrichment::finalize_full_puf": { + "access": "selector", + "basis": "Select a declared finite profile, canonical monetary field, knownness cell or model artifact target; no source-origin column access is granted.", + "body_sha256": "61a222c72065837c08896fb6c5fa91302adf3ac14733b58648dcb424e0cbeed8", + "findings": [ + { + "expression": "raw_draws[target]", + "expression_sha256": "13b95ca4ade5fae4a9f6afd64f007c010587689505b370fe4e4c695ab667e2ba", + "kind": "subscript with an unresolvable dynamic selector (fail-closed)" + } + ], + "references": [ + { + "caller": "microcosm.build.us_runtime.graph_full_puf_enrichment::_finalized_columns", + "expression": "full.finalize_full_puf(frame, binding.donor, predictor_known=binding.predictor_known, matrix=values['matrix'].payload, matrix_producer_key=values['matrix'].producer_key, raw_draws={target: values[f'raw_{i:03d}'].payload for i, target in enumerate(binding.profile.targets)}, apply_state=values['apply_state'].payload, training_state=values['training_state'].payload, last_model=values['last_model'].payload, seed=binding.fit_nodes[0].params['seed'], profile=binding.profile)", + "expression_sha256": "8f7557a639cfc973b3818c3bf64cbde6237dd9f405deb173b5f2496038dab3e9", + "resolution": "resolved", + "usage": "call" + } + ], + "role": "closed_model_selector" + }, + "microcosm.build.us_runtime.graph_atomic_survey_financial::_artifacts": { + "access": "selector", + "basis": "Read declared artifacts, validation metadata, typed state tuples or columns solely for byte/value/axis identity checks; caller and body contracts bind those checks.", + "body_sha256": "df52aacec07a677ca3ed4dcb3ca9b523af33aed162af6cd7b7a14451b8d41b86", + "findings": [ + { + "expression": "kernels.get(node.kernel)", + "expression_sha256": "341f708e2f45f3dafe3895733aa637654365b5ad566abb6bb6e83e322f28575c", + "kind": ".get() with an unresolvable dynamic key (fail-closed)" + }, + { + "expression": "keys[node_id]", + "expression_sha256": "f4fd3dc86255dc07388b75243b69c60c1b0293fa890ece1e781e9e561443572c", + "kind": "subscript with an unresolvable dynamic selector (fail-closed)" + }, + { + "expression": "implementations[node_id]", + "expression_sha256": "956253306e2aebed2d3c96d3983528fb3c758b045d8d4da37b306a4dc8b1fff0", + "kind": "subscript with an unresolvable dynamic selector (fail-closed)" + } + ], + "references": [ + { + "caller": "microcosm.build.us_runtime.graph_atomic_survey_financial::check_atomic_survey_financial_run", + "expression": "_artifacts(run.manifest, run.compiled, run.store, run.kernels, keys, implementations)", + "expression_sha256": "788e1ec2b64bcf338d63824a605d2c20522157ac2d9c28b15ad03f3cc7b878f7", + "resolution": "resolved", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.graph_atomic_survey_financial::run_atomic_survey_financial", + "expression": "_artifacts(manifest, compiled, store, kernels, keys, implementations)", + "expression_sha256": "ee6300d154362f8b06ded16ed2284edbe47832d16b3d1a5dad40ac44faeb0c2f", + "resolution": "resolved", + "usage": "call" + } + ], + "role": "artifact_verification" + }, + "microcosm.build.us_runtime.graph_atomic_survey_financial::_model_receipts": { + "access": "selector", + "basis": "Read declared artifacts, validation metadata, typed state tuples or columns solely for byte/value/axis identity checks; caller and body contracts bind those checks.", + "body_sha256": "46d7190fdf12e38520baddf8e84c2968895d7061726528672eafd323d50c3e47", + "findings": [ + { + "expression": "loaded[fit.id, 'model']", + "expression_sha256": "5b51eee7e1cfe7b081c443ea2796a1d4fa73dbe570f86c04a7a70d405ffcb326", + "kind": "subscript with an unresolvable dynamic selector (fail-closed)" + }, + { + "expression": "loaded[fit.id, 'training_state']", + "expression_sha256": "dfe99e8bc9eb9eec7a75194af5584a45183f7e01af842769410ec044d2598e80", + "kind": "subscript with an unresolvable dynamic selector (fail-closed)" + }, + { + "expression": "loaded[apply_id, 'apply_state']", + "expression_sha256": "ab33433dc64adb39c4efe0c8ac51eb61acc03a4bc6f0ddb733b5f9f47a31084a", + "kind": "subscript with an unresolvable dynamic selector (fail-closed)" + }, + { + "expression": "loaded[apply_id, 'raw_draw']", + "expression_sha256": "7cdf998144ad6fcb310e03c915563db014d4f29f78f6cd7bba32cd81ad2b68f1", + "kind": "subscript with an unresolvable dynamic selector (fail-closed)" + } + ], + "references": [ + { + "caller": "microcosm.build.us_runtime.graph_atomic_survey_financial::run_atomic_survey_financial", + "expression": "_model_receipts(nodes, donor_columns, qualified, loaded, matrix_key)", + "expression_sha256": "fdc09c4a9a542875eceac009f1ad922dc2151b231607b760905f9d84f00fc3cc", + "resolution": "resolved", + "usage": "call" + } + ], + "role": "artifact_verification" + }, + "microcosm.build.us_runtime.graph_atomic_survey_financial::_pure_run": { + "access": "selector", + "basis": "Read declared artifacts, validation metadata, typed state tuples or columns solely for byte/value/axis identity checks; caller and body contracts bind those checks.", + "body_sha256": "d7020b04fb1ac589d5b729c4aca259eb896b3b1627688a2dba312eda38ee729f", + "findings": [ + { + "expression": "entry[2]", + "expression_sha256": "255e733c946eab6358f6001cb56e783019ff9f5d1daf0b342a6ef2baa3cb55cc", + "kind": "subscript with an unresolvable dynamic selector (fail-closed)" + }, + { + "expression": "entry[1]", + "expression_sha256": "6eb20e05f22592d515f480a018bda640991f8d6f9d208eaf86997842f151ae90", + "kind": "subscript with an unresolvable dynamic selector (fail-closed)" + } + ], + "references": [ + { + "caller": "microcosm.build.us_runtime.graph_atomic_survey_financial::_issue_run", + "expression": "_pure_run(result, _run_entry(result))", + "expression_sha256": "19c58b85d64cf61aa71d6ea6ce9595997a7c1295df5c789cccc93fdb83c3f369", + "resolution": "resolved", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.graph_atomic_survey_financial::check_atomic_survey_financial_run", + "expression": "_pure_run(run, entry)", + "expression_sha256": "be9f340e4a2d3ae12c9d7e9abaeae63d8f0b695c884700913fdbcde3a50e7387", + "resolution": "resolved", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.graph_atomic_survey_financial::check_atomic_survey_financial_run", + "expression": "_pure_run(run, entry)", + "expression_sha256": "be9f340e4a2d3ae12c9d7e9abaeae63d8f0b695c884700913fdbcde3a50e7387", + "resolution": "resolved", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.graph_puf55_survey_recipients::_Kernel.run", + "expression": "financial._pure_run(run, entry)", + "expression_sha256": "5b6fb4b5b100bf5574e5df7b01ff85a14a66c1d9025bc60c31acc720e0245ba8", + "resolution": "unresolved-us-candidate", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.graph_puf55_survey_recipients::_check_values", + "expression": "financial._pure_run(run, entry)", + "expression_sha256": "5b6fb4b5b100bf5574e5df7b01ff85a14a66c1d9025bc60c31acc720e0245ba8", + "resolution": "unresolved-us-candidate", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.puf55_survey_recipients::qualify_puf55_survey_recipients", + "expression": "financial._pure_run(financial_run, entry)", + "expression_sha256": "cacd7e6879ad5b697bae957048d9be4dc155186dc15cba2b557019c0c023be5b", + "resolution": "resolved", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.survey_financial_successor::_pure_state", + "expression": "runner._pure_run(state.financial_run, state.run_entry)", + "expression_sha256": "d70c2264c242054b5c5e03cd707f7a7333efabd8497470b4c1e9bdb9abceadb9", + "resolution": "resolved", + "usage": "call" + } + ], + "role": "artifact_verification" + }, + "microcosm.build.us_runtime.graph_atomic_survey_population::_states": { + "access": "selector", + "basis": "Read declared artifacts, validation metadata, typed state tuples or columns solely for byte/value/axis identity checks; caller and body contracts bind those checks.", + "body_sha256": "6f7d9dcddfd6f1212e48581ec262630a65f5af14b1e76cdb048684d264b83653", + "findings": [ + { + "expression": "expected_populations[node_id]", + "expression_sha256": "0ea1a0f2834e80128bc1b1171762fdab6ca276dad6714d2001a779ed4d00ee45", + "kind": "subscript with an unresolvable dynamic selector (fail-closed)" + }, + { + "expression": "kernels.get(node.kernel)", + "expression_sha256": "341f708e2f45f3dafe3895733aa637654365b5ad566abb6bb6e83e322f28575c", + "kind": ".get() with an unresolvable dynamic key (fail-closed)" + }, + { + "expression": "raw_receipts[node_id]", + "expression_sha256": "d03a6d8061ee241e1c8bb5326a149b8ffeee437af8623bb826a63a91ad4c7117", + "kind": "subscript with an unresolvable dynamic selector (fail-closed)" + } + ], + "references": [ + { + "caller": "microcosm.build.us_runtime.graph_atomic_survey_financial::run_atomic_survey_financial", + "expression": "atomic._states(compiled, kernels, source_keys, expected, receipts)", + "expression_sha256": "0db2875285674d21b3f29d776a4cc332ddf583af471f2430b1462779739fc0af", + "resolution": "resolved", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.graph_atomic_survey_population::run_atomic_survey_population", + "expression": "_states(compiled, kernels, source_keys, expected, receipts)", + "expression_sha256": "4df55d6d712646473444a5e881ed8305b7039584b7fbde7a91ad398572ee6ac2", + "resolution": "resolved", + "usage": "call" + } + ], + "role": "artifact_verification" + }, + "microcosm.build.us_runtime.graph_combined_clone::USCombinedSurveyCloneExpandKernel._minimal_frame": { + "access": "provenance", + "basis": "Construct/validate the source composition or its first structural clone with original lineage and mass, before generic population operators.", + "body_sha256": "58ff7305b2a2d31f69a791b8b6e9573ca44296e2eb5708995aab7b53dab9a291", + "findings": [ + { + "expression": "frame.table(entity)[support_channel_column(entity)]", + "expression_sha256": "7e2010b8b14868c784671af4b73ffe21d332be2d575fae9792cad9b5ba45b6ef", + "kind": "subscript using call to support_channel_column" + }, + { + "expression": "support_channel_column(entity)", + "expression_sha256": "c5f5055ec6fcad2e076a23218592dfd7584c9c74920d3d93a84a56493490d8f3", + "kind": "call to support_channel_column" + } + ], + "references": [ + { + "caller": "microcosm.build.us_runtime.graph_combined_clone::USCombinedSurveyCloneExpandKernel.run", + "expression": "self._minimal_frame(context, channels)", + "expression_sha256": "ba415258472287b3b87d516ff6e4308d20f85e3db0bb759ea2d8cf2c402995a3", + "resolution": "resolved", + "usage": "call" + } + ], + "role": "population_composition" + }, + "microcosm.build.us_runtime.graph_combined_clone::_channel_mass": { + "access": "provenance", + "basis": "Aggregate existing household mass by its physical source for the structural clone receipt; never decide a population treatment.", + "body_sha256": "9ce8e7d56bfbab78dd6b7e14e0491ee5b6ea89116af6efa1c79e9e8f03a790d1", + "findings": [ + { + "expression": "table[support_channel_column(COMBINED_CLONE_WEIGHT_ENTITY)]", + "expression_sha256": "40246c8b91a695b6d4cc4a3955087e7602e794af4d540e04399e3907a0aff71e", + "kind": "subscript using call to support_channel_column" + }, + { + "expression": "support_channel_column(COMBINED_CLONE_WEIGHT_ENTITY)", + "expression_sha256": "5d42cc22967dbff19659e35721b2ebbf42b5f1405ae10bed4470f8f3c0fb8524", + "kind": "call to support_channel_column" + } + ], + "references": [ + { + "caller": "microcosm.build.us_runtime.graph_combined_clone::USCombinedSurveyCloneExpandKernel._receipt", + "expression": "_channel_mass(before, channels)", + "expression_sha256": "7591a35ae19da254fd8574e0549e8ad8726b6d8afb48d3bfb20f37bfa4118032", + "resolution": "resolved", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.graph_combined_clone::USCombinedSurveyCloneExpandKernel._receipt", + "expression": "_channel_mass(after, channels)", + "expression_sha256": "ec50f01dd5511b2eb2d701e5749cf763e4adfdfb05f94965e1e1d8aea9e1a5b8", + "resolution": "resolved", + "usage": "call" + } + ], + "role": "origin_diagnostic" + }, + "microcosm.build.us_runtime.graph_combined_clone::combined_clone_provenance_columns": { + "access": "provenance", + "basis": "Declare exact graph columns/inputs/nodes and their lineage dependencies; this scope is not an imputation or treatment kernel.", + "body_sha256": "4b3e61f28684b5c8beef66b566743a27d822964bc77609b6cc28f29c7bf91f54", + "findings": [ + { + "expression": "spine_source_id_column(entity)", + "expression_sha256": "2a61eaf1e93ee9e265db5fb590763785dd34e6833b9ee69d3eaf86b69042c4a7", + "kind": "call to spine_source_id_column" + }, + { + "expression": "support_channel_column(entity)", + "expression_sha256": "c5f5055ec6fcad2e076a23218592dfd7584c9c74920d3d93a84a56493490d8f3", + "kind": "call to support_channel_column" + } + ], + "references": [ + { + "caller": "microcosm.build.us_runtime.graph_combined_clone::USCombinedSurveyCloneExpandKernel._validated_declaration", + "expression": "combined_clone_provenance_columns(entity)", + "expression_sha256": "79ba08afe42974ca51308d60b28b8ffb823c69c2dd64faaddd0c5f5a5c29b254", + "resolution": "resolved", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.graph_combined_clone::us_combined_survey_clone_nodes", + "expression": "combined_clone_provenance_columns(entity)", + "expression_sha256": "79ba08afe42974ca51308d60b28b8ffb823c69c2dd64faaddd0c5f5a5c29b254", + "resolution": "resolved", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.graph_combined_clone::us_combined_survey_clone_nodes", + "expression": "combined_clone_provenance_columns(entity)", + "expression_sha256": "79ba08afe42974ca51308d60b28b8ffb823c69c2dd64faaddd0c5f5a5c29b254", + "resolution": "resolved", + "usage": "call" + } + ], + "role": "graph_declaration" + }, + "microcosm.build.us_runtime.graph_composed_asec_binding::_origin_entity_digests": { + "access": "provenance", + "basis": "Verify or join exact source/clone lineage; provenance identifies a carried observation/attachment, not a generic treatment branch.", + "body_sha256": "419453cf07f5b57b72037970749aa53e209100d1b5c45ab0850903cea745869a", + "findings": [ + { + "expression": "support_channel_column(entity)", + "expression_sha256": "c5f5055ec6fcad2e076a23218592dfd7584c9c74920d3d93a84a56493490d8f3", + "kind": "call to support_channel_column" + }, + { + "expression": "spine_source_id_column(entity)", + "expression_sha256": "2a61eaf1e93ee9e265db5fb590763785dd34e6833b9ee69d3eaf86b69042c4a7", + "kind": "call to spine_source_id_column" + }, + { + "expression": "table[channel_column]", + "expression_sha256": "76f0a14cb634d299d0b089ef978c79d8ef921308f73ea2330f9dca1e8efa0fb5", + "kind": "subscript using call to support_channel_column" + }, + { + "expression": "table[spine_column]", + "expression_sha256": "b79b7e68a883fa09b53ebf8fd081656e35e1ac2378a24a6b96778101b55c693e", + "kind": "subscript using call to spine_source_id_column" + } + ], + "references": [ + { + "caller": "microcosm.build.us_runtime.graph_composed_asec_binding::resolve_composed_asec_binding", + "expression": "_origin_entity_digests(tables)", + "expression_sha256": "66110e3824bee943b44e26083cda206e338cbc46e827c630ec5599c36bda4114", + "resolution": "resolved", + "usage": "call" + } + ], + "role": "origin_attachment" + }, + "microcosm.build.us_runtime.graph_composed_asec_binding::_original_ids": { + "access": "provenance", + "basis": "Interpret and join authenticated native ASEC/ACS observations using the bound native source identity and period; preserve the declared source interpretation.", + "body_sha256": "c6acad35b3771162ac61243acbb62b3c005a0cc82e5aa6d19453d9246386eada", + "findings": [ + { + "expression": "spine_source_id_column(entity)", + "expression_sha256": "2a61eaf1e93ee9e265db5fb590763785dd34e6833b9ee69d3eaf86b69042c4a7", + "kind": "call to spine_source_id_column" + }, + { + "expression": "table.loc[mask, column]", + "expression_sha256": "15f3ac65bf79d31abe749a23a1236a8718d52fe83d156072525822ded63f9526", + "kind": "subscript using call to spine_source_id_column" + } + ], + "references": [ + { + "caller": "microcosm.build.us_runtime.graph_composed_asec_binding::resolve_composed_asec_binding", + "expression": "_original_ids(tables[entity], entity, masks[entity])", + "expression_sha256": "05056ff9aa1fac1622ba917abd56e6a6cffd36cffe45dc2188c3c577fbd6cdee", + "resolution": "resolved", + "usage": "call" + } + ], + "role": "native_projection" + }, + "microcosm.build.us_runtime.graph_composed_asec_binding::arm_mask": { + "access": "provenance", + "basis": "Interpret and join authenticated native ASEC/ACS observations using the bound native source identity and period; preserve the declared source interpretation.", + "body_sha256": "6b05849c1df3e7a847d6b1981a7dde9b838ff83824d1f07578c6084d00d55f8d", + "findings": [ + { + "expression": "support_channel_column(entity)", + "expression_sha256": "c5f5055ec6fcad2e076a23218592dfd7584c9c74920d3d93a84a56493490d8f3", + "kind": "call to support_channel_column" + }, + { + "expression": "table[column]", + "expression_sha256": "6422895606da8898c2e65a4eda29c5c335cbb89f48d88b50f65e5c373568dffe", + "kind": "subscript using call to support_channel_column" + } + ], + "references": [ + { + "caller": "microcosm.build.us_runtime.graph_composed_asec_binding::resolve_composed_asec_binding", + "expression": "arm_mask(tables[entity], entity)", + "expression_sha256": "b26145d05730d78f40fc329dd40d4d9cd65f106c8652642c112f709abd41ee67", + "resolution": "resolved", + "usage": "call" + } + ], + "role": "native_projection" + }, + "microcosm.build.us_runtime.graph_composed_asec_binding::bind_node_inputs": { + "access": "provenance", + "basis": "Declare exact graph columns/inputs/nodes and their lineage dependencies; this scope is not an imputation or treatment kernel.", + "body_sha256": "c7ba8807e51c9355a619b10bb499bc654ff3becee70c6cf15a53da67afcfbda1", + "findings": [ + { + "expression": "support_channel_column(entity)", + "expression_sha256": "c5f5055ec6fcad2e076a23218592dfd7584c9c74920d3d93a84a56493490d8f3", + "kind": "call to support_channel_column" + }, + { + "expression": "spine_source_id_column(entity)", + "expression_sha256": "2a61eaf1e93ee9e265db5fb590763785dd34e6833b9ee69d3eaf86b69042c4a7", + "kind": "call to spine_source_id_column" + } + ], + "references": [ + { + "caller": "microcosm.build.us_runtime.graph_composed_asec_binding::USComposedAsecBindKernel.run", + "expression": "bind_node_inputs()", + "expression_sha256": "a9bbe92a04348d3abe13d63fbfe8ac45dab9942c816152b28489abe88d71caab", + "resolution": "resolved", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.graph_composed_asec_binding::composed_asec_bind_node", + "expression": "bind_node_inputs()", + "expression_sha256": "a9bbe92a04348d3abe13d63fbfe8ac45dab9942c816152b28489abe88d71caab", + "resolution": "resolved", + "usage": "call" + } + ], + "role": "graph_declaration" + }, + "microcosm.build.us_runtime.graph_composed_population::_source_origin": { + "access": "provenance", + "basis": "Construct/validate the source composition or its first structural clone with original lineage and mass, before generic population operators.", + "body_sha256": "9440daa6edfd4181c78443b58665bf58e958f58c0189ca853565218229d67151", + "findings": [ + { + "expression": "support_channel_column(entity)", + "expression_sha256": "c5f5055ec6fcad2e076a23218592dfd7584c9c74920d3d93a84a56493490d8f3", + "kind": "call to support_channel_column" + }, + { + "expression": "spine_source_id_column(entity)", + "expression_sha256": "2a61eaf1e93ee9e265db5fb590763785dd34e6833b9ee69d3eaf86b69042c4a7", + "kind": "call to spine_source_id_column" + }, + { + "expression": "table[channel_column]", + "expression_sha256": "76f0a14cb634d299d0b089ef978c79d8ef921308f73ea2330f9dca1e8efa0fb5", + "kind": "subscript using call to support_channel_column" + }, + { + "expression": "table[spine_column]", + "expression_sha256": "b79b7e68a883fa09b53ebf8fd081656e35e1ac2378a24a6b96778101b55c693e", + "kind": "subscript using call to spine_source_id_column" + }, + { + "expression": "frame.table(entity)[support_channel_column(entity)]", + "expression_sha256": "7e2010b8b14868c784671af4b73ffe21d332be2d575fae9792cad9b5ba45b6ef", + "kind": "subscript using call to support_channel_column" + }, + { + "expression": "support_channel_column(entity)", + "expression_sha256": "c5f5055ec6fcad2e076a23218592dfd7584c9c74920d3d93a84a56493490d8f3", + "kind": "call to support_channel_column" + } + ], + "references": [ + { + "caller": "microcosm.build.us_runtime.graph_composed_population::compose_from_sources", + "expression": "_source_origin(stacked.frame, arms, sampling=stacked.receipt['sampling'], preparation_sha256=_sha(preparation_payload), prepared_receipt=prepared.receipt)", + "expression_sha256": "c93433768b9ed8bdab1556f50b64ded8ac3f025235ce4f58d055448ce91181b8", + "resolution": "resolved", + "usage": "call" + } + ], + "role": "population_composition" + }, + "microcosm.build.us_runtime.graph_composed_population::composed_population_nodes": { + "access": "provenance", + "basis": "Declare exact graph columns/inputs/nodes and their lineage dependencies; this scope is not an imputation or treatment kernel.", + "body_sha256": "c219ab82c34cc97a1d17d0d0685fed8bd9f5997b6d1fbcc1ce5d5348cccc7aca", + "findings": [ + { + "expression": "support_channel_column(entity)", + "expression_sha256": "c5f5055ec6fcad2e076a23218592dfd7584c9c74920d3d93a84a56493490d8f3", + "kind": "call to support_channel_column" + }, + { + "expression": "support_channel_column('household')", + "expression_sha256": "bb43a8f3b7827b0f6f755ff2679d6de24fd68081e9650d8261b0634d5e74f447", + "kind": "call to support_channel_column" + }, + { + "expression": "support_channel_column('person')", + "expression_sha256": "7cf3594f675603d58586f7d491519fb21a01f0471e5905b181248d3d17223cdb", + "kind": "call to support_channel_column" + } + ], + "references": [ + { + "caller": "microcosm.build.us_runtime.graph_composed_asec_measures::composed_asec_population_graph", + "expression": "composed_population_nodes(columns, sample_fraction=sample_fraction, sample_seed=sample_seed)", + "expression_sha256": "6fc6de16de30bcc5f5fd39446c774bccf999591ce9967fdc3afd9bce7624fd2c", + "resolution": "resolved", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.graph_composed_population::composed_population_graph", + "expression": "composed_population_nodes(columns, sample_fraction=sample_fraction, sample_seed=sample_seed)", + "expression_sha256": "6fc6de16de30bcc5f5fd39446c774bccf999591ce9967fdc3afd9bce7624fd2c", + "resolution": "resolved", + "usage": "call" + } + ], + "role": "graph_declaration" + }, + "microcosm.build.us_runtime.graph_current_survey_predictors::CurrentSurveyPredictorDonorFilterKernel.run": { + "access": "provenance", + "basis": "Select authenticated native whole-household ASEC DESIGN donors on the pre-allocation training branch; generic fitting/drawing gets only the declared feature matrix.", + "body_sha256": "eacc47ab05ec59d896b931afb9f520e9fd3dd344dbbc9a97e8c0d411c6d12a0e", + "findings": [ + { + "expression": "person[values.provenance.support_channel_column('person')]", + "expression_sha256": "eb7350571232cddcd05e9fc646ce9f7a42384c5f4da8836edf5af4d626cffc7b", + "kind": "subscript using call to support_channel_column" + }, + { + "expression": "values.provenance.support_channel_column('person')", + "expression_sha256": "01567522687a2223d096c3b93e1ac9db78b54bb384f0a8ef475b3d6365ea7040", + "kind": "call to support_channel_column" + } + ], + "references": [], + "role": "donor_selection" + }, + "microcosm.build.us_runtime.graph_current_survey_predictors::read_current_survey_draws": { + "access": "selector", + "basis": "Select a declared finite profile, canonical monetary field, knownness cell or model artifact target; no source-origin column access is granted.", + "body_sha256": "1ba983198b36c711d8ea3c4dfc00f6660fae8aebf2a79b53ee8e4b952c0bc852", + "findings": [ + { + "expression": "apply_states[i]", + "expression_sha256": "4361fca4938ba2e5400eb3e63b39958ad4c7af2cce80cb7483a5902538b616f0", + "kind": "subscript with an unresolvable dynamic selector (fail-closed)" + }, + { + "expression": "raw_draws[:i + 1]", + "expression_sha256": "22b3cfb826c766fe399afa7a23d7e505ed2b0a7aa94c29b55720c986c12e744f", + "kind": "subscript with an unresolvable dynamic selector (fail-closed)" + }, + { + "expression": "raw_draws[i]", + "expression_sha256": "972a53c3bfe5ae694df15f3d48fb57528b28e37d5991f6e8c0e0c5b285a0760a", + "kind": "subscript with an unresolvable dynamic selector (fail-closed)" + } + ], + "references": [ + { + "caller": "microcosm.build.us_runtime.graph_atomic_survey_financial::run_atomic_survey_financial", + "expression": "financial.read_current_survey_draws(matrix_bytes, matrix_key, raw, applications, demographic_conditioning=demographic_conditioning)", + "expression_sha256": "20e9daa0db2489476ffff419f0a00b605b229d8f99aca00ccd5229cca8c6d45b", + "resolution": "resolved", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.graph_current_survey_predictors::CurrentSurveyPredictorAttachKernel.run", + "expression": "read_current_survey_draws(matrix.payload, matrix.producer_key, tuple(raw), tuple(states), demographic_conditioning=self.demographic_conditioning)", + "expression_sha256": "cecad42ba012bb90efbfa8794f0a6b3a3d5c8624f668218a1a51909b6f689887", + "resolution": "resolved", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.graph_current_survey_predictors::verify_materialized_current_survey_predictors", + "expression": "read_current_survey_draws(matrix, matrix_producer_key, raw_draws, apply_states, demographic_conditioning=demographic_conditioning)", + "expression_sha256": "4890291d77b3c61b7098167f6189e137eea5ffecefa0d4e7c582a2d50f2a46a9", + "resolution": "resolved", + "usage": "call" + } + ], + "role": "closed_model_selector" + }, + "microcosm.build.us_runtime.graph_current_survey_puf_transfer::current_survey_transfer_nodes": { + "access": "selector", + "basis": "Declare exact graph columns/inputs/nodes and their lineage dependencies; this scope is not an imputation or treatment kernel.", + "body_sha256": "ecc0c4cb39130f72c875da73a5c4302be9bacdf88c8b6e970f269cb003cc9c98", + "findings": [ + { + "expression": "fit_nodes[0]", + "expression_sha256": "12610533f551fb2b479872a78e9bce8473bf9ec6ed5b3ef19bbe7a1fa67b220d", + "kind": "subscript with an unresolvable dynamic selector (fail-closed)" + } + ], + "references": [], + "role": "graph_declaration" + }, + "microcosm.build.us_runtime.graph_full_puf_enrichment::_checked_artifacts": { + "access": "selector", + "basis": "Read declared artifacts, validation metadata, typed state tuples or columns solely for byte/value/axis identity checks; caller and body contracts bind those checks.", + "body_sha256": "c2c3c90e11c253414cf7667db4a9739c15963ec1caa496dfc8cb8c3b71e6b066", + "findings": [ + { + "expression": "artifacts[edge.name]", + "expression_sha256": "58584cc7bbfdebd1b0d99c7919b1f6276925fdb8536d784f2c20ff12cabfd0b2", + "kind": "subscript with an unresolvable dynamic selector (fail-closed)" + }, + { + "expression": "producer_keys[edge.producer]", + "expression_sha256": "4d7cd8fdef36dbfedbed4ebd75a9e77bcbd005fccf47419e8f2e097be768a1b1", + "kind": "subscript with an unresolvable dynamic selector (fail-closed)" + } + ], + "references": [ + { + "caller": "microcosm.build.us_runtime.graph_full_puf_enrichment::FullPufAttachKernel.run", + "expression": "_checked_artifacts(binding, attach.artifact_inputs, context.artifacts)", + "expression_sha256": "a5e833b4b4f31e46cb0c0d05dd01419984c1ad8be4d7bdeb72d830e908e1a361", + "resolution": "resolved", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.graph_full_puf_enrichment::FullPufMaskKernel.run", + "expression": "_checked_artifacts(binding, node.artifact_inputs, context.artifacts)", + "expression_sha256": "0a3c2bb19f4a4308f611ce2e1b1ca1b56ed716f6fd69abaaa7b795ed655793a5", + "resolution": "resolved", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.graph_full_puf_enrichment::load_full_puf_attachment_artifacts", + "expression": "_checked_artifacts(binding, attach.artifact_inputs, artifacts, keys)", + "expression_sha256": "f9d65571a32e8ce6235f30995b0a16efaf74e70bd8f419fd5347dc18f45432cb", + "resolution": "resolved", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.graph_full_puf_enrichment::verify_materialized_full_puf_attachment", + "expression": "_checked_artifacts(binding, attach.artifact_inputs, artifacts, producer_keys)", + "expression_sha256": "f8d0ccd85280ffb5555df51c1b20d4f1d49d54ffbdb8b6e7cbfe9db11e1c5a1b", + "resolution": "resolved", + "usage": "call" + } + ], + "role": "artifact_verification" + }, + "microcosm.build.us_runtime.graph_full_puf_enrichment::_finalized_columns": { + "access": "selector", + "basis": "Select a declared finite profile, canonical monetary field, knownness cell or model artifact target; no source-origin column access is granted.", + "body_sha256": "c7cde0de4d732d2b31833216c74de6b6995d6853f58fa037fea0f836ac55d840", + "findings": [ + { + "expression": "values[f'raw_{i:03d}']", + "expression_sha256": "96ec67525e1bd9753d1c7bac95d1a5c6a8cc69cd1393ad464b40b03c8a19aeed", + "kind": "subscript with an unresolvable dynamic selector (fail-closed)" + } + ], + "references": [ + { + "caller": "microcosm.build.us_runtime.graph_full_puf_enrichment::FullPufAttachKernel.run", + "expression": "_finalized_columns(binding, values)", + "expression_sha256": "5a7e26bd09c087f094cdf50602e546146203b5ef69829be2ceae47cabbba50f1", + "resolution": "resolved", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.graph_full_puf_enrichment::verify_materialized_full_puf_attachment", + "expression": "_finalized_columns(binding, values)", + "expression_sha256": "5a7e26bd09c087f094cdf50602e546146203b5ef69829be2ceae47cabbba50f1", + "resolution": "resolved", + "usage": "call" + } + ], + "role": "closed_model_selector" + }, + "microcosm.build.us_runtime.graph_full_puf_enrichment::_table_stamp": { + "access": "selector", + "basis": "Read declared artifacts, validation metadata, typed state tuples or columns solely for byte/value/axis identity checks; caller and body contracts bind those checks.", + "body_sha256": "4b263ed54ff1c116c7ee807522a1b80c49659c86c1afd82c224db837f38f87fc", + "findings": [ + { + "expression": "table[column]", + "expression_sha256": "6422895606da8898c2e65a4eda29c5c335cbb89f48d88b50f65e5c373568dffe", + "kind": "subscript with an unresolvable dynamic selector (fail-closed)" + } + ], + "references": [ + { + "caller": "microcosm.build.us_runtime.graph_full_puf_enrichment::FullPufAttachmentBinding.check", + "expression": "_table_stamp(self.predictor_known)", + "expression_sha256": "1a2ca39b5d80027919d0d242c7bc8be8004d6e4a4aa2ee5f442dfafa6f7e2178", + "resolution": "resolved", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.graph_full_puf_enrichment::FullPufAttachmentBinding.check", + "expression": "_table_stamp(self.donor)", + "expression_sha256": "aaec30b82c40aa59a5b41ae14015682878cdc8e777ca4ae9c6cec236a0e61a75", + "resolution": "resolved", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.graph_full_puf_enrichment::_population_stamp", + "expression": "_table_stamp(frame.table(e))", + "expression_sha256": "6f7158588bec6639d808451ab64465e2e02120a103c1df76ff1a03239e91eba3", + "resolution": "resolved", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.graph_full_puf_enrichment::retain_full_puf_attachment", + "expression": "_table_stamp(predictor_known)", + "expression_sha256": "09de56f7406bf6aaee3811a61abdedc7574e6de596734f11b426fe6b7d8fddc3", + "resolution": "resolved", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.graph_full_puf_enrichment::retain_full_puf_attachment", + "expression": "_table_stamp(donor)", + "expression_sha256": "62b9c844f5ebe3d53efec0fa6ca16741b08fcd692af62adf869fe6c4d83a4c7b", + "resolution": "resolved", + "usage": "call" + } + ], + "role": "artifact_verification" + }, + "microcosm.build.us_runtime.graph_full_puf_enrichment::retain_full_puf_attachment": { + "access": "selector", + "basis": "Read declared artifacts, validation metadata, typed state tuples or columns solely for byte/value/axis identity checks; caller and body contracts bind those checks.", + "body_sha256": "af4b3a7703a00f2665b29770e4e2de52a70bc7da59ea15a06ca663ff622b5853", + "findings": [ + { + "expression": "fit_nodes[0]", + "expression_sha256": "12610533f551fb2b479872a78e9bce8473bf9ec6ed5b3ef19bbe7a1fa67b220d", + "kind": "subscript with an unresolvable dynamic selector (fail-closed)" + }, + { + "expression": "apply_nodes[0]", + "expression_sha256": "34e3b9fe997e3bea1842ccd1f33a406dcdf03d2d0357e0fe2c2470857a43c4ab", + "kind": "subscript with an unresolvable dynamic selector (fail-closed)" + } + ], + "references": [], + "role": "artifact_verification" + }, + "microcosm.build.us_runtime.graph_puf55_canonical_donor::CanonicalPuf55DonorKernel._read_resources": { + "access": "selector", + "basis": "Validate an authenticated source artifact, declared source tuple, member selection or issuer state before population/model admission.", + "body_sha256": "fcd8f343f415598850392798afeb94c7b5ed9cccca7c15192571b43750c25f08", + "findings": [ + { + "expression": "state[3]", + "expression_sha256": "32e8bde11c0fae61141a00a5bda05811e49b3c2e6cc3f9050d989897892959ce", + "kind": "subscript with an unresolvable dynamic selector (fail-closed)" + }, + { + "expression": "state[6]", + "expression_sha256": "b2d154f2b7be6ccffd2b3d6a0aee4e8d8137fc894ffe929d37f60d94b03520c4", + "kind": "subscript with an unresolvable dynamic selector (fail-closed)" + } + ], + "references": [ + { + "caller": "microcosm.build.us_runtime.graph_puf55_canonical_donor::CanonicalPuf55DonorKernel.run", + "expression": "self._read_resources(state)", + "expression_sha256": "d41c5dd39bbbc71fd51d12146608add5f1cf7b5df1d86bac2a2fc06dd503aa36", + "resolution": "resolved", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.graph_puf55_canonical_donor::CanonicalPuf55DonorKernel.run", + "expression": "self._read_resources(state)", + "expression_sha256": "d41c5dd39bbbc71fd51d12146608add5f1cf7b5df1d86bac2a2fc06dd503aa36", + "resolution": "resolved", + "usage": "call" + } + ], + "role": "source_authentication" + }, + "microcosm.build.us_runtime.graph_puf55_canonical_donor::CanonicalPuf55DonorKernel._read_sources": { + "access": "selector", + "basis": "Validate an authenticated source artifact, declared source tuple, member selection or issuer state before population/model admission.", + "body_sha256": "457f78b50b6666831c1057d4e0c3cc17b26768c738fc48f57b42b14eb717679a", + "findings": [ + { + "expression": "state[0]", + "expression_sha256": "f66ec07e8d5a5937fada61b9bbf21855813702a04a2226ca8a77556b9ead37d9", + "kind": "subscript with an unresolvable dynamic selector (fail-closed)" + } + ], + "references": [ + { + "caller": "microcosm.build.us_runtime.graph_puf55_canonical_donor::CanonicalPuf55DonorKernel.run", + "expression": "self._read_sources(paths, state)", + "expression_sha256": "3aee8b805a13b5ee9ef0474a98efad8dbfbc8cc5a7f0134e5cf811bb90a70e5b", + "resolution": "resolved", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.graph_puf55_canonical_donor::CanonicalPuf55DonorKernel.run", + "expression": "self._read_sources(paths, state)", + "expression_sha256": "3aee8b805a13b5ee9ef0474a98efad8dbfbc8cc5a7f0134e5cf811bb90a70e5b", + "resolution": "resolved", + "usage": "call" + } + ], + "role": "source_authentication" + }, + "microcosm.build.us_runtime.graph_puf55_canonical_donor::_marker.visit": { + "access": "selector", + "basis": "Validate an authenticated source artifact, declared source tuple, member selection or issuer state before population/model admission.", + "body_sha256": "3b4c2ac39d65637478019e17cd05eef70ff1dfe108a47fc4238025a305d15bf4", + "findings": [ + { + "expression": "getattr(item, field.name)", + "expression_sha256": "048b0726101ac916793599826372f3d79dd73900f133a7f315eee7cfae4dfbc9", + "kind": "getattr with an unresolvable dynamic attribute (fail-closed)" + } + ], + "references": [ + { + "caller": "microcosm.build.us_runtime.graph_puf55_canonical_donor::_marker", + "expression": "visit(value, depth)", + "expression_sha256": "0f2afc0c997a4ca5c5625a57e4bc1602a8609a42290d852e551117b898094323", + "resolution": "resolved", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.graph_puf55_canonical_donor::_marker.visit", + "expression": "visit(getattr(item, field.name), level + 1)", + "expression_sha256": "0657aec3d305965ec99a40a3dcfbcbbd38d17f2553b7f9b011d1242484fee969", + "resolution": "resolved", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.graph_puf55_canonical_donor::_marker.visit", + "expression": "visit(item.__kwdefaults__, level + 1)", + "expression_sha256": "45dc506f2b7d16e3fd19531907807dc54627e5d8ba09d808b8909d73ab34d550", + "resolution": "resolved", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.graph_puf55_canonical_donor::_marker.visit", + "expression": "visit(cell_value, level + 1)", + "expression_sha256": "77195ec6d8956e74fc945f39af392b5dfd16bcefedf8a073fb6dee283f6ff8e3", + "resolution": "resolved", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.graph_puf55_canonical_donor::_marker.visit", + "expression": "visit(k, level + 1)", + "expression_sha256": "a8016e4fdfe62a205de6a9884eb30a37561163375d6b79fb2eac4eae046d9cfa", + "resolution": "resolved", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.graph_puf55_canonical_donor::_marker.visit", + "expression": "visit(item.__defaults__, level + 1)", + "expression_sha256": "b7edc3bf5f5e92c077058448a0f1414957068e3bcb1e44566b8c76ac5be3b623", + "resolution": "resolved", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.graph_puf55_canonical_donor::_marker.visit", + "expression": "visit(child, level + 1)", + "expression_sha256": "bc3a631087c23ef19eeb45324e10c5071418378889a7b8d4a32dfa8b6f0a5dcd", + "resolution": "resolved", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.graph_puf55_canonical_donor::_marker.visit", + "expression": "visit(child, level + 1)", + "expression_sha256": "bc3a631087c23ef19eeb45324e10c5071418378889a7b8d4a32dfa8b6f0a5dcd", + "resolution": "resolved", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.graph_puf55_canonical_donor::_marker.visit", + "expression": "visit(v, level + 1)", + "expression_sha256": "de2db280d3987db39dc2cb046d3c9109e51b31d8d45e1f9b4b878a5dd50b56af", + "resolution": "resolved", + "usage": "call" + } + ], + "role": "source_authentication" + }, + "microcosm.build.us_runtime.graph_puf55_canonical_donor::_pin": { + "access": "selector", + "basis": "Validate an authenticated source artifact, declared source tuple, member selection or issuer state before population/model admission.", + "body_sha256": "d1f0216571911ce2873837109e1e9248dece8454871190a0ebcac1824800e8d3", + "findings": [ + { + "expression": "getattr(pin, field.name)", + "expression_sha256": "a99ea29e4009a437d3bcf75de26ec72a57c76a348e78191a338250eec033f8cf", + "kind": "getattr with an unresolvable dynamic attribute (fail-closed)" + } + ], + "references": [ + { + "caller": "microcosm.build.us_runtime.graph_puf55_canonical_donor::CanonicalPuf55DonorKernel._check_state", + "expression": "_pin(loader.keywords['pin'])", + "expression_sha256": "55b8a0ab9478c3ad8ed607c97b5a21dfe442180de0e71d041b91af8192bad41e", + "resolution": "resolved", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.graph_puf55_canonical_donor::CanonicalPuf55DonorKernel._check_state", + "expression": "_pin(pin)", + "expression_sha256": "e8ff13b5203d6b6e1f43e368f66b0005971972bd932d2fa255d8af42e7f60c7d", + "resolution": "resolved", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.graph_puf55_canonical_donor::_definition", + "expression": "_pin(definition.main)", + "expression_sha256": "1ff797a0b882372dfc15068555c5284ecf872ec7c326b55588438f1a0f4413bf", + "resolution": "resolved", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.graph_puf55_canonical_donor::_definition", + "expression": "_pin(definition.demographic)", + "expression_sha256": "38241c848ff5234112514aecee5490c501b0865f43656b408c17d2b4c6cfa8e6", + "resolution": "resolved", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.graph_puf55_canonical_donor::_definition", + "expression": "_pin(rebuilt.main)", + "expression_sha256": "8e990eba4cbb9166242d6496b5886c4543c2900207b0c5177357ca140db6559d", + "resolution": "resolved", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.graph_puf55_canonical_donor::_definition", + "expression": "_pin(rebuilt.demographic)", + "expression_sha256": "a10277e5c24f8c0f04fe878fa3f1df21ab4e102a19a5e674ddb247068f113210", + "resolution": "resolved", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.graph_puf55_canonical_donor::_params", + "expression": "_pin(definition.main)", + "expression_sha256": "1ff797a0b882372dfc15068555c5284ecf872ec7c326b55588438f1a0f4413bf", + "resolution": "resolved", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.graph_puf55_canonical_donor::_params", + "expression": "_pin(definition.demographic)", + "expression_sha256": "38241c848ff5234112514aecee5490c501b0865f43656b408c17d2b4c6cfa8e6", + "resolution": "resolved", + "usage": "call" + } + ], + "role": "source_authentication" + }, + "microcosm.build.us_runtime.graph_puf_detail_transfer::verify_materialized_transfer": { + "access": "selector", + "basis": "Read declared artifacts, validation metadata, typed state tuples or columns solely for byte/value/axis identity checks; caller and body contracts bind those checks.", + "body_sha256": "ae4cc24baf9e1ee4d8113a05fd008f89699c640ad6f1b19b7424a0521735487e", + "findings": [ + { + "expression": "before_design[entity]", + "expression_sha256": "99230aad23cc0b54e8869f4c44d937b97604502ded3c049baac542cddad49c70", + "kind": "subscript with an unresolvable dynamic selector (fail-closed)" + }, + { + "expression": "after_design[entity]", + "expression_sha256": "0dab52da1f8c44853ef6767ab6c5f9dd9f1683ca51625e322e629f4c0963126d", + "kind": "subscript with an unresolvable dynamic selector (fail-closed)" + }, + { + "expression": "after_ledger[-1]", + "expression_sha256": "51d0661bd5d5bb58daac942b9d668af02db0e13c0a22c87e3aa37299653e4bfc", + "kind": "subscript with an unresolvable dynamic selector (fail-closed)" + } + ], + "references": [], + "role": "artifact_verification" + }, + "microcosm.build.us_runtime.graph_sources::us_assembly_graph": { + "access": "provenance", + "basis": "Declare exact graph columns/inputs/nodes and their lineage dependencies; this scope is not an imputation or treatment kernel.", + "body_sha256": "c13e9a6ada84906581050ad47a769724abb1fb2329f2126908613fc9c6ab4faf", + "findings": [ + { + "expression": "support_channel_column('household')", + "expression_sha256": "bb43a8f3b7827b0f6f755ff2679d6de24fd68081e9650d8261b0634d5e74f447", + "kind": "call to support_channel_column" + }, + { + "expression": "support_channel_column('person')", + "expression_sha256": "7cf3594f675603d58586f7d491519fb21a01f0471e5905b181248d3d17223cdb", + "kind": "call to support_channel_column" + } + ], + "references": [], + "role": "graph_declaration" + }, + "microcosm.build.us_runtime.graph_survey_population::_fraction_pair": { + "access": "selector", + "basis": "Index bounded payload bytes, declared arrays or structured budget/tuple fields in the maintained codec, not population source columns.", + "body_sha256": "a729600ad45fa969d6ac90f4e4d8d789d001a8142c4087b7357adab0959ac751", + "findings": [ + { + "expression": "value[1]", + "expression_sha256": "82f1aa797e696a99d20b034fee07c8f49dcf39d2080d47919daaa985d97e3106", + "kind": "subscript with an unresolvable dynamic selector (fail-closed)" + } + ], + "references": [ + { + "caller": "microcosm.build.us_runtime.graph_survey_population::allocation_instructions", + "expression": "_fraction_pair(origin['original_anchor'])", + "expression_sha256": "04c620785a300b5cd146c128d3f0ba90d089b9ed878aa5e4d2853c4df057f658", + "resolution": "resolved", + "usage": "call" + } + ], + "role": "artifact_codec" + }, + "microcosm.build.us_runtime.graph_survey_population::_provenance_columns": { + "access": "provenance", + "basis": "Declare exact graph columns/inputs/nodes and their lineage dependencies; this scope is not an imputation or treatment kernel.", + "body_sha256": "b38532c2a08cf9905308e41ad0109b211d2fdf09d0c501345272eaf4e664b42f", + "findings": [ + { + "expression": "support_channel_column(entity)", + "expression_sha256": "c5f5055ec6fcad2e076a23218592dfd7584c9c74920d3d93a84a56493490d8f3", + "kind": "call to support_channel_column" + }, + { + "expression": "spine_source_id_column(entity)", + "expression_sha256": "2a61eaf1e93ee9e265db5fb590763785dd34e6833b9ee69d3eaf86b69042c4a7", + "kind": "call to spine_source_id_column" + } + ], + "references": [ + { + "caller": "microcosm.build.us_runtime.graph_current_survey_geography::_check_context", + "expression": "source_graph._provenance_columns('household')", + "expression_sha256": "b848f97020639d8837f86bc7d5d7fa1295f7ea89780b40f3ae7178d58d521610", + "resolution": "resolved", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.graph_current_survey_geography::current_survey_geography_node", + "expression": "source_graph._provenance_columns('household')", + "expression_sha256": "b848f97020639d8837f86bc7d5d7fa1295f7ea89780b40f3ae7178d58d521610", + "resolution": "resolved", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.graph_survey_population::SurveyPopulationAllocationKernel.run", + "expression": "_provenance_columns(e)", + "expression_sha256": "55b4d4386f2650309a1cc04a3bd1587e7b3302404ca1dc1381692a634032c2c8", + "resolution": "resolved", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.graph_survey_population::survey_population_nodes", + "expression": "_provenance_columns(entity)", + "expression_sha256": "3ed7c52cb81bb91b8b2381c1c21cec4a65353f3eb1c3678dfd0fdc805bf3fbeb", + "resolution": "resolved", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.graph_survey_population::survey_population_nodes", + "expression": "_provenance_columns(entity)", + "expression_sha256": "3ed7c52cb81bb91b8b2381c1c21cec4a65353f3eb1c3678dfd0fdc805bf3fbeb", + "resolution": "resolved", + "usage": "call" + } + ], + "role": "graph_declaration" + }, + "microcosm.build.us_runtime.graph_survey_population::_verify_allocation_view": { + "access": "provenance", + "basis": "Verify or join exact source/clone lineage; provenance identifies a carried observation/attachment, not a generic treatment branch.", + "body_sha256": "f8c83743aa8b9b4686809742704d17ff4e787a4be13e04cfdd7b416edffec9e1", + "findings": [ + { + "expression": "household[support_channel_column('household')]", + "expression_sha256": "4b8c5e332635d99e0246ac30b55fb055135d87f49aefe38637913b449dc43bfa", + "kind": "subscript using call to support_channel_column" + }, + { + "expression": "support_channel_column('household')", + "expression_sha256": "bb43a8f3b7827b0f6f755ff2679d6de24fd68081e9650d8261b0634d5e74f447", + "kind": "call to support_channel_column" + }, + { + "expression": "household[spine_source_id_column('household')]", + "expression_sha256": "e4f8a4d04215be665fc76885ff021b295d3131406e89dabfb00ca491e832c558", + "kind": "subscript using call to spine_source_id_column" + }, + { + "expression": "spine_source_id_column('household')", + "expression_sha256": "dbaa5f748b8b686aadf9f965a360f5ed79a78ef9993133c1084c8b350f1815e9", + "kind": "call to spine_source_id_column" + } + ], + "references": [ + { + "caller": "microcosm.build.us_runtime.graph_survey_population::_allocation_output", + "expression": "_verify_allocation_view(frame, instructions)", + "expression_sha256": "d2221978240a8288b5f5d817bb085ab2d62f5121d5bea1bf5664f4743fa703fa", + "resolution": "resolved", + "usage": "call" + } + ], + "role": "origin_attachment" + }, + "microcosm.build.us_runtime.graph_survey_population::survey_population_nodes": { + "access": "provenance", + "basis": "Declare exact graph columns/inputs/nodes and their lineage dependencies; this scope is not an imputation or treatment kernel.", + "body_sha256": "c4515dd456b41901a67c7e859deb04f7bf82ff623b895138feb41cf40bcf34ac", + "findings": [ + { + "expression": "support_channel_column(entity)", + "expression_sha256": "c5f5055ec6fcad2e076a23218592dfd7584c9c74920d3d93a84a56493490d8f3", + "kind": "call to support_channel_column" + } + ], + "references": [ + { + "caller": "microcosm.build.us_runtime.graph_survey_population::SurveyPopulationCreateKernel.run", + "expression": "survey_population_nodes(frame_column_declarations(view.frame), preparation_sha256=_sha(payload), fraction=plan.fraction, seed=plan.seed)", + "expression_sha256": "ea37d0fd7a1e42cee15b98b5686032fc9434438acd9593cebdd4685afff576de", + "resolution": "resolved", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.graph_survey_population::run_authenticated_survey_population", + "expression": "survey_population_nodes(columns, preparation_sha256=_sha(payload), fraction=fraction, seed=seed)", + "expression_sha256": "581af721acb40c6211dc832fb33b9c28e7000561b9e0551147f0b6cbd97af751", + "resolution": "resolved", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.survey_atomic_geography::_raw_allocation", + "expression": "graph.survey_population_nodes(columns, preparation_sha256=_sha(view.payload), fraction=view.selection_plan.fraction, seed=view.selection_plan.seed)", + "expression_sha256": "34aa56244328dee48347c29cba926b7911ce0b09b4598bbcc3e2d77d9b41e69c", + "resolution": "resolved", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.survey_origin_budget::_initial", + "expression": "graph.survey_population_nodes(columns, preparation_sha256=_sha(view.payload), fraction=view.selection_plan.fraction, seed=view.selection_plan.seed)", + "expression_sha256": "34aa56244328dee48347c29cba926b7911ce0b09b4598bbcc3e2d77d9b41e69c", + "resolution": "resolved", + "usage": "call" + } + ], + "role": "graph_declaration" + }, + "microcosm.build.us_runtime.native_household_origin::_bind_population_origins": { + "access": "provenance", + "basis": "Verify or join exact source/clone lineage; provenance identifies a carried observation/attachment, not a generic treatment branch.", + "body_sha256": "bf7e30327f62e44ab63cebd3c5dc9b492f36d2606b7df542358d413f1422d54e", + "findings": [ + { + "expression": "hh[support_channel_column('household')]", + "expression_sha256": "055c0ddd1d4ba7548fd84ba3a358be1a96ee9c3ef547a4b8b62d409dd8022f73", + "kind": "subscript using call to support_channel_column" + }, + { + "expression": "support_channel_column('household')", + "expression_sha256": "bb43a8f3b7827b0f6f755ff2679d6de24fd68081e9650d8261b0634d5e74f447", + "kind": "call to support_channel_column" + } + ], + "references": [ + { + "caller": "microcosm.build.us_runtime.native_household_origin::bind_population_origins", + "expression": "_bind_population_origins(frame, sources=sources, parent=parent)", + "expression_sha256": "74e5a13453e7e8dc8363843838c2bb2f38850900819583c47f4aa1387448e7a7", + "resolution": "resolved", + "usage": "call" + } + ], + "role": "origin_attachment" + }, + "microcosm.build.us_runtime.native_household_origin::_entity_rows": { + "access": "provenance", + "basis": "Verify or join exact source/clone lineage; provenance identifies a carried observation/attachment, not a generic treatment branch.", + "body_sha256": "02a5e647571845ae04cb1fb75918e392a2f75a4a992c70ed0baea53b846fca11", + "findings": [ + { + "expression": "household_origins[_integer(membership[identity])]", + "expression_sha256": "d9b1885651f504fe620fc6cba3257675ca4f42cd2a2419ba9fcd02a715a5a113", + "kind": "subscript with an unresolvable dynamic selector (fail-closed)" + }, + { + "expression": "table[support_channel_column(entity)].iloc[i]", + "expression_sha256": "affa6fa4ae65c522ee18a464714e56a2b997a58d0247571c9d122276aeb56c37", + "kind": "subscript using call to support_channel_column" + }, + { + "expression": "support_channel_column(entity)", + "expression_sha256": "c5f5055ec6fcad2e076a23218592dfd7584c9c74920d3d93a84a56493490d8f3", + "kind": "call to support_channel_column" + } + ], + "references": [ + { + "caller": "microcosm.build.us_runtime.native_household_origin::_bind_population_origins", + "expression": "_entity_rows(frame, {r['household_id']: r for r in rows})", + "expression_sha256": "1a5b06602a67eddfe9d97f76cfe890401ae69369b2cf5fc18b61aa1fc484b0b9", + "resolution": "resolved", + "usage": "call" + } + ], + "role": "origin_attachment" + }, + "microcosm.build.us_runtime.puf55_route_finalization::_donor_identity": { + "access": "selector", + "basis": "Read declared artifacts, validation metadata, typed state tuples or columns solely for byte/value/axis identity checks; caller and body contracts bind those checks.", + "body_sha256": "e68f2c1860037edee09dce3e4e595a6de9e88f02618650566d14932f36f7945d", + "findings": [ + { + "expression": "donor[name]", + "expression_sha256": "f86d570c2109821f8415180f398dfb2a8a9ad57b0c959c33e91719b53bc6d231", + "kind": "subscript with an unresolvable dynamic selector (fail-closed)" + }, + { + "expression": "donor[name]", + "expression_sha256": "f86d570c2109821f8415180f398dfb2a8a9ad57b0c959c33e91719b53bc6d231", + "kind": "subscript with an unresolvable dynamic selector (fail-closed)" + } + ], + "references": [ + { + "caller": "microcosm.build.us_runtime.puf55_route_finalization::_finalize_puf55_routes", + "expression": "_donor_identity(donor, common_columns)", + "expression_sha256": "8fe4a8defc8d47500d2677d015fe82d55ebf44ceef87e8c5bfd8acdf9a14b138", + "resolution": "resolved", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.puf55_route_finalization::_finalize_puf55_routes", + "expression": "_donor_identity(model_donor, tuple(model_donor.columns))", + "expression_sha256": "b974c83b3b23decbdf4a28b91adf5a1414fbdd80e2745453e82fa7a287aa8593", + "resolution": "resolved", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.puf55_route_finalization::_finalize_puf55_routes", + "expression": "_donor_identity(donor, tuple(donor.columns))", + "expression_sha256": "c27ec3c65dfac7a3957e33423682d919257c2e629f2cf5e3d10b1d96e9bee52a", + "resolution": "resolved", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.puf55_route_finalization::_finalize_puf55_routes.check_inputs", + "expression": "_donor_identity(model_donor, tuple(model_donor.columns))", + "expression_sha256": "b974c83b3b23decbdf4a28b91adf5a1414fbdd80e2745453e82fa7a287aa8593", + "resolution": "resolved", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.puf55_route_finalization::_finalize_puf55_routes.check_inputs", + "expression": "_donor_identity(donor, tuple(donor.columns))", + "expression_sha256": "c27ec3c65dfac7a3957e33423682d919257c2e629f2cf5e3d10b1d96e9bee52a", + "resolution": "resolved", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.puf55_route_finalization::_model_donor_frame", + "expression": "_donor_identity(model_donor, tuple(model_donor.columns))", + "expression_sha256": "b974c83b3b23decbdf4a28b91adf5a1414fbdd80e2745453e82fa7a287aa8593", + "resolution": "resolved", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.puf55_route_finalization::_model_donor_frame", + "expression": "_donor_identity(model_donor, tuple(model_donor.columns))", + "expression_sha256": "b974c83b3b23decbdf4a28b91adf5a1414fbdd80e2745453e82fa7a287aa8593", + "resolution": "resolved", + "usage": "call" + } + ], + "role": "artifact_verification" + }, + "microcosm.build.us_runtime.puf55_survey_recipients::_aligned": { + "access": "provenance", + "basis": "Verify or join exact source/clone lineage; provenance identifies a carried observation/attachment, not a generic treatment branch.", + "body_sha256": "e6fa50c85df75175adb263cb0c134f7a7af4f575272ab39392e2ff0bbe8c3a30", + "findings": [ + { + "expression": "provenance.spine_source_id_column(entity)", + "expression_sha256": "13b2755960d499d22d9a3418cc4b06df299d7cee2235b0da3f9ac602e70ed6fb", + "kind": "call to spine_source_id_column" + }, + { + "expression": "provenance.support_channel_column(entity)", + "expression_sha256": "12fccfbcc85de1ebe560d28094493019b5c41c47a4e713eb65daef8def696746", + "kind": "call to support_channel_column" + } + ], + "references": [ + { + "caller": "microcosm.build.us_runtime.puf55_survey_recipients::_project", + "expression": "_aligned(source_frame, frame, 'tax_unit')", + "expression_sha256": "7f4270d7af209ad679195dae9a81c64759a67aca6e827cd24212b1304072b2b5", + "resolution": "resolved", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.puf55_survey_recipients::_project", + "expression": "_aligned(source_frame, frame, 'person')", + "expression_sha256": "c3d46778d7668d601aae89332a45c5add1e5347b454169860c3274f62d01ed60", + "resolution": "resolved", + "usage": "call" + } + ], + "role": "origin_attachment" + }, + "microcosm.build.us_runtime.puf55_survey_recipients::_literal_snapshot": { + "access": "provenance", + "basis": "Interpret and join authenticated native ASEC/ACS observations using the bound native source identity and period; preserve the declared source interpretation.", + "body_sha256": "8bb2f7b0b89dc8145dca6744f6d21d1bcfddcafadb040d57849e7feeaf312b93", + "findings": [ + { + "expression": "people[provenance.support_channel_column('person')]", + "expression_sha256": "4be741c1ce94301fd1e79026254b25f342c3dc147894583489ae0c387a42b99d", + "kind": "subscript using call to support_channel_column" + }, + { + "expression": "provenance.support_channel_column('person')", + "expression_sha256": "1ca82bc581cfd1d0a520cccb66737a50f42281b6ac1405e50a4a3083d63f2ea5", + "kind": "call to support_channel_column" + } + ], + "references": [ + { + "caller": "microcosm.build.us_runtime.puf55_survey_recipients::qualify_puf55_survey_recipients", + "expression": "_literal_snapshot(state.preparation_entry[2])", + "expression_sha256": "65491e2eb9f08efe3f4b7ba3643321be1ed63953b194c1bf9331c5c511338c23", + "resolution": "resolved", + "usage": "call" + } + ], + "role": "native_projection" + }, + "microcosm.build.us_runtime.puf55_survey_recipients::_source_report": { + "access": "provenance", + "basis": "Interpret and join authenticated native ASEC/ACS observations using the bound native source identity and period; preserve the declared source interpretation.", + "body_sha256": "b0ff4e006eeff28179a8e935a9d28e63ed58d5d15d14c37126949b1f1d647aac", + "findings": [ + { + "expression": "people[provenance.support_channel_column('person')]", + "expression_sha256": "4be741c1ce94301fd1e79026254b25f342c3dc147894583489ae0c387a42b99d", + "kind": "subscript using call to support_channel_column" + }, + { + "expression": "provenance.support_channel_column('person')", + "expression_sha256": "1ca82bc581cfd1d0a520cccb66737a50f42281b6ac1405e50a4a3083d63f2ea5", + "kind": "call to support_channel_column" + }, + { + "expression": "people[provenance.spine_source_id_column('person')]", + "expression_sha256": "02d22f188dadde849f2d771bb30843a0d7c1102235e0a343d2ea3b8ed1240816", + "kind": "subscript using call to spine_source_id_column" + }, + { + "expression": "provenance.spine_source_id_column('person')", + "expression_sha256": "15b03f981ea284db003b5470bf2124ba8c096ff9331cdb987e95897a6e735f63", + "kind": "call to spine_source_id_column" + } + ], + "references": [ + { + "caller": "microcosm.build.us_runtime.puf55_survey_recipients::qualify_puf55_survey_recipients", + "expression": "_source_report(state.preparation_entry[2], snapshot)", + "expression_sha256": "fac6fef243eb641885942c0c57536768a7287fcff1c5c336f3b8c2b40d221e9a", + "resolution": "resolved", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.puf55_survey_recipients::qualify_puf55_survey_recipients", + "expression": "_source_report(state.preparation_entry[2], snapshot)", + "expression_sha256": "fac6fef243eb641885942c0c57536768a7287fcff1c5c336f3b8c2b40d221e9a", + "resolution": "resolved", + "usage": "call" + } + ], + "role": "native_projection" + }, + "microcosm.build.us_runtime.puf55_survey_recipients::_table_digest": { + "access": "selector", + "basis": "Read declared artifacts, validation metadata, typed state tuples or columns solely for byte/value/axis identity checks; caller and body contracts bind those checks.", + "body_sha256": "7480100b5a9c4e99dcc2e007495c89cfcf351fa58209a6d20a02093f13cd8a69", + "findings": [ + { + "expression": "table[c]", + "expression_sha256": "be8f381285b9e987d00a9cfc80e8f32733b31a6b30594e5bdac8f20ffe1dfebd", + "kind": "subscript with an unresolvable dynamic selector (fail-closed)" + } + ], + "references": [ + { + "caller": "microcosm.build.us_runtime.graph_puf55_survey_recipients::_check_values", + "expression": "values._table_digest(qualified.tax_unit)", + "expression_sha256": "26e164844ec395654932109e044f8aeb9e66ff4242acf8089033076c1156f8f5", + "resolution": "resolved", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.graph_puf55_survey_recipients::_check_values", + "expression": "values._table_digest(qualified.person)", + "expression_sha256": "f6cbf31df84dd05911812adaff27aed3630fc88b83e9281bd0216493be242982", + "resolution": "resolved", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.puf55_survey_recipients::_result_stamp", + "expression": "_table_digest(result.person)", + "expression_sha256": "78ce1031357e898052f6dab68a26cfd787a1c41a4267d94423074b93e72cf426", + "resolution": "resolved", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.puf55_survey_recipients::_result_stamp", + "expression": "_table_digest(result.tax_unit)", + "expression_sha256": "d4249854c97db5445d368025bcabe6bc4efc111ff4eb26ca7b4326e9d3569911", + "resolution": "resolved", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.puf55_survey_recipients::qualify_puf55_survey_recipients", + "expression": "_table_digest(fresh)", + "expression_sha256": "03e9deb0307089f04b1c4af5181a5d64ccc83aa6a34a3abdcb04e7a60f3354a3", + "resolution": "resolved", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.puf55_survey_recipients::qualify_puf55_survey_recipients", + "expression": "_table_digest(authoritative_report)", + "expression_sha256": "1719bdc7de47a36c8e8dc56f74fcef346e76b3873ac845217dbc4d80eaa808ce", + "resolution": "resolved", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.puf55_survey_recipients::qualify_puf55_survey_recipients", + "expression": "_table_digest(authoritative_report)", + "expression_sha256": "1719bdc7de47a36c8e8dc56f74fcef346e76b3873ac845217dbc4d80eaa808ce", + "resolution": "resolved", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.puf55_survey_recipients::qualify_puf55_survey_recipients", + "expression": "_table_digest(authoritative_report)", + "expression_sha256": "1719bdc7de47a36c8e8dc56f74fcef346e76b3873ac845217dbc4d80eaa808ce", + "resolution": "resolved", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.puf55_survey_recipients::qualify_puf55_survey_recipients", + "expression": "_table_digest(units)", + "expression_sha256": "5f196de1260821ce6b96e16350572fa5d4fccec2c8fe93bc575a361453ab4617", + "resolution": "resolved", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.puf55_survey_recipients::qualify_puf55_survey_recipients", + "expression": "_table_digest(final_units)", + "expression_sha256": "741867abc6181c230d209cd6c638a5e3b5c54c33fe9197f4599dde8b6342bc2e", + "resolution": "resolved", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.puf55_survey_recipients::qualify_puf55_survey_recipients", + "expression": "_table_digest(person)", + "expression_sha256": "936ce385ab5ab06995b1ef733eabb329ff0986984a64ec0b842425fd14fddeea", + "resolution": "resolved", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.puf55_survey_recipients::qualify_puf55_survey_recipients", + "expression": "_table_digest(report.person)", + "expression_sha256": "ce6d5b031eee3f801300527c17922f7b74c7a20296fd67d27a2e1746c2057068", + "resolution": "resolved", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.puf55_survey_recipients::qualify_puf55_survey_recipients", + "expression": "_table_digest(final_person)", + "expression_sha256": "d3427ac32f86eb0b20284a0f8000562a4851590334a49f5001bd8c7b05206b96", + "resolution": "resolved", + "usage": "call" + } + ], + "role": "artifact_verification" + }, + "microcosm.build.us_runtime.puf55_survey_ss_measurement::_measure": { + "access": "selector", + "basis": "Qualify source-specific measured inputs and complete the declared pre-PUF donor/recipient surface before the source-blind model application.", + "body_sha256": "7fb847eab34580df97faa66010586891ce50080a95a4eb17cccba5f135e822b7", + "findings": [ + { + "expression": "tax_unit[list(_UNIT_COLUMNS)]", + "expression_sha256": "4aa6bc45440bf4fadfa48ff75248a9514eed9b14fb4bcbaa24305747dfea6bda", + "kind": "subscript with an unresolvable dynamic selector (fail-closed)" + }, + { + "expression": "report[_REPORT_COLUMNS[0]]", + "expression_sha256": "2a60396d6bd2842047da6cb554930fe398895eae37b2eaa6fe593f9692ecf6fd", + "kind": "subscript with an unresolvable dynamic selector (fail-closed)" + }, + { + "expression": "report[_REPORT_COLUMNS[1]]", + "expression_sha256": "fbb43f0ff8c94a7d07ff3979d2196a597466dffbb24b1cb566d6428c8d8be321", + "kind": "subscript with an unresolvable dynamic selector (fail-closed)" + } + ], + "references": [ + { + "caller": "microcosm.build.us_runtime.puf55_survey_recipients::_project", + "expression": "ss._measure(people, units, person_report)", + "expression_sha256": "068ba76bb0b7cf1f2c56b41c8d1c7cd0fe42d40c5d59fabe771fa6c3039766bf", + "resolution": "resolved", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.puf55_survey_recipients::qualify_puf55_survey_recipients", + "expression": "ss._measure(source_frame.person, source_frame.table('tax_unit'), authoritative_report)", + "expression_sha256": "5406af007da6967505521973602134125aee9c8597b61264b68ea0f3be3bf5d2", + "resolution": "resolved", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.puf55_survey_recipients::qualify_puf55_survey_recipients", + "expression": "ss._measure(source_frame.person, source_frame.table('tax_unit'), fresh)", + "expression_sha256": "70703905f2da0c8ab0eac7909efe35f97422d0628cadc4caa557bb2544b35ca3", + "resolution": "resolved", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.puf55_survey_ss_measurement::qualify_puf55_survey_ss_measurement", + "expression": "_measure(state.frame.person, state.frame.table('tax_unit'), projection.person)", + "expression_sha256": "b817c0d8f8182d035204f4aa8e2d8d453166aefde452e88d14a64880639968f7", + "resolution": "resolved", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.puf55_survey_ss_measurement::qualify_puf55_survey_ss_measurement", + "expression": "_measure(state.frame.person, state.frame.table('tax_unit'), projection.person)", + "expression_sha256": "b817c0d8f8182d035204f4aa8e2d8d453166aefde452e88d14a64880639968f7", + "resolution": "resolved", + "usage": "call" + } + ], + "role": "model_qualification" + }, + "microcosm.build.us_runtime.puf55_survey_ss_measurement::_projection_digest": { + "access": "selector", + "basis": "Read declared artifacts, validation metadata, typed state tuples or columns solely for byte/value/axis identity checks; caller and body contracts bind those checks.", + "body_sha256": "77a799f215555fea51533b2fc9465debf68e424e73bb44e154f4846bd15b9dff", + "findings": [ + { + "expression": "table[TOTAL]", + "expression_sha256": "95527704c53c50619b69e817d42377f8e1f31add07f5bbae9f853e4811a24583", + "kind": "subscript with an unresolvable dynamic selector (fail-closed)" + } + ], + "references": [ + { + "caller": "microcosm.build.us_runtime.puf55_survey_recipients::_project", + "expression": "ss._projection_digest(expected)", + "expression_sha256": "2744f79e2366b43d43466fbd78f4becfdd8f4dca95d37ec41fdc46f3dadc3634", + "resolution": "resolved", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.puf55_survey_recipients::_project", + "expression": "ss._projection_digest(received_measurement)", + "expression_sha256": "8c082278ecf83b187582f21f62bddb38b370bd6a784eb2be731f86f19d4ff520", + "resolution": "resolved", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.puf55_survey_ss_measurement::qualify_puf55_survey_ss_measurement", + "expression": "_projection_digest(result.tax_unit)", + "expression_sha256": "68cc3198771694f4ae073c0fd2a94532af7a6f448ad503f178ab0671dcdab5db", + "resolution": "resolved", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.puf55_survey_ss_measurement::qualify_puf55_survey_ss_measurement", + "expression": "_projection_digest(values)", + "expression_sha256": "981c1b92c68da919250b6454283b36266242aa257e6e55cfed7fef6d089c9706", + "resolution": "resolved", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.puf55_survey_ss_measurement::qualify_puf55_survey_ss_measurement", + "expression": "_projection_digest(fresh)", + "expression_sha256": "a7548b968f0babce69da5d204b58430b4415552b1704a3ba7295acb2486e8d5e", + "resolution": "resolved", + "usage": "call" + } + ], + "role": "artifact_verification" + }, + "microcosm.build.us_runtime.puf55_survey_ss_measurement::_source_identity": { + "access": "provenance", + "basis": "Verify or join exact source/clone lineage; provenance identifies a carried observation/attachment, not a generic treatment branch.", + "body_sha256": "250c67dfe58383c28209c415681caba255e8d6ef292eb4e56148cdf255c4e864", + "findings": [ + { + "expression": "people[support_channel_column('person')]", + "expression_sha256": "230a04e693074722d092c0f7cad4b0f1f7266f7db14e9db6e96d8d3f50a69775", + "kind": "subscript using call to support_channel_column" + }, + { + "expression": "support_channel_column('person')", + "expression_sha256": "7cf3594f675603d58586f7d491519fb21a01f0471e5905b181248d3d17223cdb", + "kind": "call to support_channel_column" + }, + { + "expression": "people[spine_source_id_column('person')]", + "expression_sha256": "4b7b6a2b87df2d4a901c1d33163a43df841dcf4eed197a62981d30b013ce91ad", + "kind": "subscript using call to spine_source_id_column" + }, + { + "expression": "spine_source_id_column('person')", + "expression_sha256": "5bff0f374c0361bf89039f0907ff51ae6e76189950720fae047b4f2df3584bf5", + "kind": "call to spine_source_id_column" + } + ], + "references": [ + { + "caller": "microcosm.build.us_runtime.puf55_survey_recipients::qualify_puf55_survey_recipients", + "expression": "ss._source_identity(source_frame.person, report)", + "expression_sha256": "7078e46c13f9b8085be6ec587104185ac0425f5222a94e8abcd03c6a7b44616b", + "resolution": "resolved", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.puf55_survey_ss_measurement::qualify_puf55_survey_ss_measurement", + "expression": "_source_identity(state.frame.person, projection)", + "expression_sha256": "523aa6e54fa5d48a4660bd36f958f5f99e68538ef848e8c6f42c7516695b4f10", + "resolution": "resolved", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.puf55_survey_ss_measurement::qualify_puf55_survey_ss_measurement", + "expression": "_source_identity(state.frame.person, projection)", + "expression_sha256": "523aa6e54fa5d48a4660bd36f958f5f99e68538ef848e8c6f42c7516695b4f10", + "resolution": "resolved", + "usage": "call" + } + ], + "role": "origin_attachment" + }, + "microcosm.build.us_runtime.puf59_canonical_artifact::_encode": { + "access": "selector", + "basis": "Index bounded payload bytes, declared arrays or structured budget/tuple fields in the maintained codec, not population source columns.", + "body_sha256": "b3fc15be8508db719af5558b6c0db0367a9acc60c03f2fa540613a43d730a08d", + "findings": [ + { + "expression": "arrays[name]", + "expression_sha256": "35f8103eebcd189430941bcab8a6f5dfc113fb50fac232fba2b0db32d0e4c392", + "kind": "subscript with an unresolvable dynamic selector (fail-closed)" + } + ], + "references": [ + { + "caller": "microcosm.build.us_runtime.puf59_canonical_artifact::encode_canonical_puf59", + "expression": "_encode(arrays, result.receipt, expected_growth_scheme)", + "expression_sha256": "3304509568eb2e4725830f5dd09c8059a75bbcbbdc61ea5070470cc236036b9d", + "resolution": "resolved", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.puf59_canonical_artifact::reencode_canonical_puf59", + "expression": "_encode(arrays, receipt, expected_growth_scheme)", + "expression_sha256": "bccd2d5e39b05e024be926058d9313e622eb61c1796b0128853c83fe60c72d13", + "resolution": "resolved", + "usage": "call" + } + ], + "role": "artifact_codec" + }, + "microcosm.build.us_runtime.puf59_canonical_artifact::_validate": { + "access": "selector", + "basis": "Index bounded payload bytes, declared arrays or structured budget/tuple fields in the maintained codec, not population source columns.", + "body_sha256": "a88faae6ee362102c37a5178d71eeb9a84907001832b7448ab92a187efbbca8e", + "findings": [ + { + "expression": "arrays[name]", + "expression_sha256": "35f8103eebcd189430941bcab8a6f5dfc113fb50fac232fba2b0db32d0e4c392", + "kind": "subscript with an unresolvable dynamic selector (fail-closed)" + }, + { + "expression": "arrays[name]", + "expression_sha256": "35f8103eebcd189430941bcab8a6f5dfc113fb50fac232fba2b0db32d0e4c392", + "kind": "subscript with an unresolvable dynamic selector (fail-closed)" + } + ], + "references": [ + { + "caller": "microcosm.build.us_runtime.asec_current_money_source::AuthenticatedCurrentMoneySource.validate", + "expression": "self.source._validate()", + "expression_sha256": "3d02eb09d76453e54943dde82eb2ef0a47e4c3866ff310c06d1ca9241422b6cc", + "resolution": "unresolved-us-candidate", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.puf59_canonical_artifact::_encode", + "expression": "_validate(arrays, n)", + "expression_sha256": "672e74310bd2ebf0eab6c0a2adaa4439e5084977e70305b69855559047002e49", + "resolution": "resolved", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.puf59_canonical_artifact::decode_canonical_puf59", + "expression": "_validate(arrays, n)", + "expression_sha256": "672e74310bd2ebf0eab6c0a2adaa4439e5084977e70305b69855559047002e49", + "resolution": "resolved", + "usage": "call" + } + ], + "role": "artifact_codec" + }, + "microcosm.build.us_runtime.puf59_canonical_artifact::decode_canonical_puf59": { + "access": "selector", + "basis": "Index bounded payload bytes, declared arrays or structured budget/tuple fields in the maintained codec, not population source columns.", + "body_sha256": "1185720ba4a8957d40d6f80467a59465624b18b166d9792a8e56e8b0050c0901", + "findings": [ + { + "expression": "payload[:len(MAGIC)]", + "expression_sha256": "257c6a840319ffc5470310ed357cd3e7e37ce4863b87f38048783d9e03acea69", + "kind": "subscript with an unresolvable dynamic selector (fail-closed)" + }, + { + "expression": "payload[len(MAGIC):len(MAGIC) + 4]", + "expression_sha256": "ebeedf44eceabe03be3f31e9c56036d2738926272df354675d8de9fa0b01169c", + "kind": "subscript with an unresolvable dynamic selector (fail-closed)" + }, + { + "expression": "payload[start:start + length]", + "expression_sha256": "954ec8a4b44fabb5e319ec0aa9025fd40131869bdd1dae8ecc2b4873b8365c8f", + "kind": "subscript with an unresolvable dynamic selector (fail-closed)" + }, + { + "expression": "payload[start:start + length]", + "expression_sha256": "954ec8a4b44fabb5e319ec0aa9025fd40131869bdd1dae8ecc2b4873b8365c8f", + "kind": "subscript with an unresolvable dynamic selector (fail-closed)" + } + ], + "references": [ + { + "caller": "microcosm.build.us_runtime.puf55_canonical_donor::canonical_puf55_donor_from_artifact", + "expression": "decode_canonical_puf59(payload, expected_growth_scheme=expected_growth_scheme)", + "expression_sha256": "5df57e5784db3c4e1776812d32681da2b3b7201f222fe0e0ed21f74a8fd8af69", + "resolution": "resolved", + "usage": "call" + } + ], + "role": "artifact_codec" + }, + "microcosm.build.us_runtime.puf_detail_transfer::_donor_frame": { + "access": "selector", + "basis": "Qualify source-specific measured inputs and complete the declared pre-PUF donor/recipient surface before the source-blind model application.", + "body_sha256": "1f59adfdfc96af9f13ac574e24b9fa11b9ac25e4289bfa6e51756cb626587cf0", + "findings": [ + { + "expression": "arrays[growth.PROVENANCE_RECID_COLUMN]", + "expression_sha256": "9d7706f1b29f250516473a8edc47a90ec5716674bff219f92e76d6d25cd441fb", + "kind": "subscript with an unresolvable dynamic selector (fail-closed)" + }, + { + "expression": "arrays[growth.PROVENANCE_SOURCE_AGI_COLUMN]", + "expression_sha256": "a610e9a25f8ab4c189d45f6a5a0c0de0bbe6977d8b2b4660dfe9d05207c43df8", + "kind": "subscript with an unresolvable dynamic selector (fail-closed)" + } + ], + "references": [ + { + "caller": "microcosm.build.us_runtime.puf_detail_transfer::donor_frame", + "expression": "_donor_frame(arrays, status, decoded, projection, scope=SCOPE)", + "expression_sha256": "96f4868d75dd0a3fbadc7df36dd9e9ee268e077007f5983c74f4ae4979a61c8f", + "resolution": "resolved", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.puf_diagnostic_consumer::qualify_price_transport", + "expression": "detail._donor_frame(arrays, status, decoded, projection, scope=SCOPE)", + "expression_sha256": "c76847cc2db2f01ee45e8634e843f45d54b14b247c5a125a2aaa2883eb8f8618", + "resolution": "resolved", + "usage": "call" + } + ], + "role": "model_qualification" + }, + "microcosm.build.us_runtime.puf_detail_transfer::recipient_matrix": { + "access": "provenance", + "basis": "Qualify source-specific measured inputs and complete the declared pre-PUF donor/recipient surface before the source-blind model application.", + "body_sha256": "f66f614387b52320aad1d12645f02bf5474c63c939c7d04558c9cae795564bb7", + "findings": [ + { + "expression": "units[provenance.support_channel_column('tax_unit')]", + "expression_sha256": "deccd3d66d6f0eae478de24455c22be179507b57c9747515747723a4afc48549", + "kind": "subscript using call to support_channel_column" + }, + { + "expression": "provenance.support_channel_column('tax_unit')", + "expression_sha256": "367b0c3c0cce893fad9d3b1096db06ff7e93c162ca4acb3fc588ccf54fe8dbba", + "kind": "call to support_channel_column" + }, + { + "expression": "provenance.support_channel_column('tax_unit')", + "expression_sha256": "367b0c3c0cce893fad9d3b1096db06ff7e93c162ca4acb3fc588ccf54fe8dbba", + "kind": "call to support_channel_column" + } + ], + "references": [ + { + "caller": "microcosm.build.us_runtime.graph_puf_detail_transfer::DetailAttachKernel", + "expression": "detail.recipient_matrix", + "expression_sha256": "f184ddbdfaf155a2134254ba691ad8e0c33b2dac3ef9ab987894b264e278dde5", + "resolution": "resolved", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.graph_puf_detail_transfer::DetailMatrixKernel", + "expression": "detail.recipient_matrix", + "expression_sha256": "f184ddbdfaf155a2134254ba691ad8e0c33b2dac3ef9ab987894b264e278dde5", + "resolution": "resolved", + "usage": "reference" + }, + { + "caller": "microcosm.build.us_runtime.puf_diagnostic_consumer::current_survey_recipient_matrix", + "expression": "detail.recipient_matrix(frame, role_column='tax_unit_role_input', person_wages=projected)", + "expression_sha256": "b4bc5b302de1b015732d79999278d4fcfe4aea28b9d65c88c93286060607e405", + "resolution": "resolved", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.puf_diagnostic_consumer::recipient_matrix", + "expression": "detail.recipient_matrix(frame, role_column='tax_unit_role_input')", + "expression_sha256": "35021e63da651d6bc53ba057970bb543707ca130388b35439093b585bfd5fa25", + "resolution": "resolved", + "usage": "call" + } + ], + "role": "model_qualification" + }, + "microcosm.build.us_runtime.puf_diagnostic_consumer::asec_wage_evidence": { + "access": "provenance", + "basis": "Qualify source-specific measured inputs and complete the declared pre-PUF donor/recipient surface before the source-blind model application.", + "body_sha256": "4e9f78a015135ab034b44dcb61bf4e996aa6ef1a65e22e8a5e441614bef39cbf", + "findings": [ + { + "expression": "provenance.spine_source_id_column('person')", + "expression_sha256": "15b03f981ea284db003b5470bf2124ba8c096ff9331cdb987e95897a6e735f63", + "kind": "call to spine_source_id_column" + } + ], + "references": [ + { + "caller": "microcosm.build.us_runtime.puf_diagnostic_consumer::qualify_host_population", + "expression": "asec_wage_evidence(asec_person, values)", + "expression_sha256": "9f51bc1cafb3a3f55499c87a2cf51cedb9e15376c76fbb884a12edaacf895264", + "resolution": "resolved", + "usage": "call" + } + ], + "role": "model_qualification" + }, + "microcosm.build.us_runtime.puf_diagnostic_consumer::asec_wage_evidence.integer": { + "access": "selector", + "basis": "Qualify source-specific measured inputs and complete the declared pre-PUF donor/recipient surface before the source-blind model application.", + "body_sha256": "bb61d7b31e9b3401eb1e828e8a3c860fbc56b27f5a2f66e8a082f555359f16f2", + "findings": [ + { + "expression": "person[name]", + "expression_sha256": "d981adc2484f5feb61da87d94b5e5577c8e277cf8f4449e3f21a2d24e0821b85", + "kind": "subscript with an unresolvable dynamic selector (fail-closed)" + } + ], + "references": [ + { + "caller": "microcosm.build.us_runtime.puf_diagnostic_consumer::asec_wage_evidence", + "expression": "integer(provenance.support_source_id_column('person'))", + "expression_sha256": "1fd693b6ce6d31732eb576574860eecfdcbee1c4ad858a51211416c355643391", + "resolution": "resolved", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.puf_diagnostic_consumer::asec_wage_evidence", + "expression": "integer('A_AGE')", + "expression_sha256": "23f43945d470c0e0c7eab9576443773075426e2c72cb251bb0d239945e30db6a", + "resolution": "resolved", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.puf_diagnostic_consumer::asec_wage_evidence", + "expression": "integer(provenance.spine_source_id_column('person'))", + "expression_sha256": "9ae9912a9d6cee6a7c0b221a908212e7eb5ec9a9c2271118ef046abb17b8a747", + "resolution": "resolved", + "usage": "call" + } + ], + "role": "model_qualification" + }, + "microcosm.build.us_runtime.puf_diagnostic_consumer::current_survey_recipient_matrix": { + "access": "provenance", + "basis": "Qualify source-specific measured inputs and complete the declared pre-PUF donor/recipient surface before the source-blind model application.", + "body_sha256": "8a105cd628978b1e4d6cf50f548678d1e04f0fa656a341998d666194cf6369ed", + "findings": [ + { + "expression": "people[provenance.support_channel_column('person')]", + "expression_sha256": "4be741c1ce94301fd1e79026254b25f342c3dc147894583489ae0c387a42b99d", + "kind": "subscript using call to support_channel_column" + }, + { + "expression": "provenance.support_channel_column('person')", + "expression_sha256": "1ca82bc581cfd1d0a520cccb66737a50f42281b6ac1405e50a4a3083d63f2ea5", + "kind": "call to support_channel_column" + }, + { + "expression": "people[provenance.support_channel_column('person')]", + "expression_sha256": "4be741c1ce94301fd1e79026254b25f342c3dc147894583489ae0c387a42b99d", + "kind": "subscript using call to support_channel_column" + }, + { + "expression": "provenance.support_channel_column('person')", + "expression_sha256": "1ca82bc581cfd1d0a520cccb66737a50f42281b6ac1405e50a4a3083d63f2ea5", + "kind": "call to support_channel_column" + }, + { + "expression": "provenance.spine_source_id_column('person')", + "expression_sha256": "15b03f981ea284db003b5470bf2124ba8c096ff9331cdb987e95897a6e735f63", + "kind": "call to spine_source_id_column" + }, + { + "expression": "native_asec[original_id]", + "expression_sha256": "c6dcbf6fe367c59102cae8850afb3bd27d44f2e19fa3296af2820eca3011b50b", + "kind": "subscript using call to spine_source_id_column" + }, + { + "expression": "provenance.support_channel_column('person')", + "expression_sha256": "1ca82bc581cfd1d0a520cccb66737a50f42281b6ac1405e50a4a3083d63f2ea5", + "kind": "call to support_channel_column" + } + ], + "references": [ + { + "caller": "microcosm.build.us_runtime.graph_puf_diagnostic_consumer::CurrentSurveyMatrixKernel.run", + "expression": "consumer.current_survey_recipient_matrix(frame, projection)", + "expression_sha256": "b4041273e2da10e5b0689892bac6b86b8658b3190f42c4dad5546693b7c86278", + "resolution": "resolved", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.graph_puf_diagnostic_consumer::qualify_current_survey_host", + "expression": "consumer.current_survey_recipient_matrix(clone_population.frame, projection)", + "expression_sha256": "7f6a40418628b9a23cb9cc667a911df7e41b37bf5d387048e69b6679e600a321", + "resolution": "resolved", + "usage": "call" + } + ], + "role": "model_qualification" + }, + "microcosm.build.us_runtime.puf_diagnostic_consumer::qualify_host_population": { + "access": "provenance", + "basis": "Qualify source-specific measured inputs and complete the declared pre-PUF donor/recipient surface before the source-blind model application.", + "body_sha256": "f3974b1a458933fe96f95f533e2e37e83735dd624f34ec977e1f80040d4841e8", + "findings": [ + { + "expression": "native[provenance.support_channel_column('person')]", + "expression_sha256": "c891c0256c28e7ba63567242c291ab71a0b6b1a985e782ea4dcfc3363b42776a", + "kind": "subscript using call to support_channel_column" + }, + { + "expression": "provenance.support_channel_column('person')", + "expression_sha256": "1ca82bc581cfd1d0a520cccb66737a50f42281b6ac1405e50a4a3083d63f2ea5", + "kind": "call to support_channel_column" + } + ], + "references": [ + { + "caller": "microcosm.build.us_runtime.graph_puf_diagnostic_consumer::verify_host", + "expression": "consumer.qualify_host_population(frame, values)", + "expression_sha256": "eea9c690699072cdcc004f8221826661bda559c213ad0ef30d5faa8a2dc64d7e", + "resolution": "resolved", + "usage": "call" + } + ], + "role": "model_qualification" + }, + "microcosm.build.us_runtime.puf_diagnostic_consumer::qualify_price_transport": { + "access": "selector", + "basis": "Qualify source-specific measured inputs and complete the declared pre-PUF donor/recipient surface before the source-blind model application.", + "body_sha256": "2e1ceb4bd623007f12965e5a9c97e11b6a58941ccbbf471eaf69e50105c8d37f", + "findings": [ + { + "expression": "payloads[n]", + "expression_sha256": "f04a1784e8c60af5275f809b77caf175b00593f4c6bfec4a4c8c90654b3128c5", + "kind": "subscript with an unresolvable dynamic selector (fail-closed)" + }, + { + "expression": "payloads[name]", + "expression_sha256": "2d0f4168d15cee56e26a346fdf0873c8e248001521ad0b13d56bc5192a274f51", + "kind": "subscript with an unresolvable dynamic selector (fail-closed)" + } + ], + "references": [ + { + "caller": "microcosm.build.us_runtime.graph_puf_diagnostic_consumer::DiagnosticDonorKernel.run", + "expression": "consumer.qualify_price_transport(payloads, pins, route=context.params['route'], recipe=codec.decode_json(context.params['recipe'].encode()))", + "expression_sha256": "30b9962f646ee998603242b2537266cd65307ceb1543934c4122b1583cbd2806", + "resolution": "resolved", + "usage": "call" + } + ], + "role": "model_qualification" + }, + "microcosm.build.us_runtime.puf_full_source::decode_full_puf_artifact": { + "access": "selector", + "basis": "Index bounded payload bytes, declared arrays or structured budget/tuple fields in the maintained codec, not population source columns.", + "body_sha256": "3cc163a38e2d201d6548a4db9156839c52967e7b6187bd5fb087717e19769a5e", + "findings": [ + { + "expression": "payload[offset:offset + 8]", + "expression_sha256": "f3e2c0e8cc122182d410f3d0deceb60ad4b4d06da180425dd599f3acf38c5e66", + "kind": "subscript with an unresolvable dynamic selector (fail-closed)" + }, + { + "expression": "payload[offset:offset + size]", + "expression_sha256": "35ddd42dee29050033153f97cc11713776641342cac3a233fc7194d5cf01199e", + "kind": "subscript with an unresolvable dynamic selector (fail-closed)" + }, + { + "expression": "payload[offset:offset + h['status_bytes']]", + "expression_sha256": "e1c1e732c94801fcbd7a7ca47f24acdae283a999409a185719567831d216f531", + "kind": "subscript with an unresolvable dynamic selector (fail-closed)" + }, + { + "expression": "payload[offset:offset + entry['bytes']]", + "expression_sha256": "f90c2a3bd3f098678f4a67ce511b2f9864f1aed5f2b7a413d99bdacdd28347df", + "kind": "subscript with an unresolvable dynamic selector (fail-closed)" + } + ], + "references": [], + "role": "artifact_codec" + }, + "microcosm.build.us_runtime.puf_price_baseline::_violations": { + "access": "selector", + "basis": "Compare declared numeric reconciliation fields for diagnostic violations; selectors cannot confer source-origin authority.", + "body_sha256": "2c27fe877c79b76c02806a6c8ea3279cf6ddbd06ce5542acfbb9d24640bdf57f", + "findings": [ + { + "expression": "frame[lesser]", + "expression_sha256": "db5dede96ee035caeb613885553f2629d6deca5c8b22221ecdee7f67fa38204d", + "kind": "subscript with an unresolvable dynamic selector (fail-closed)" + }, + { + "expression": "frame[greater]", + "expression_sha256": "bb14bd898b28ccad79d038a5da9fa620bad3f6072cf48505c46cf7071d4aa1aa", + "kind": "subscript with an unresolvable dynamic selector (fail-closed)" + } + ], + "references": [ + { + "caller": "microcosm.build.us_runtime.puf_price_baseline::price_baseline_invariants", + "expression": "_violations(table, greater, lesser)", + "expression_sha256": "0c1f304b86988e538d51e9fc554472391631c6ac6b276146e06516268ecb68dd", + "resolution": "resolved", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.puf_price_baseline::price_baseline_invariants", + "expression": "_violations(source, greater, lesser)", + "expression_sha256": "dc529c52ac265e25350db5c6e44b694f10c956764f04e9c4618de93827fda708", + "resolution": "resolved", + "usage": "call" + } + ], + "role": "diagnostic_selector" + }, + "microcosm.build.us_runtime.puf_qbi_model::_digest_columns": { + "access": "selector", + "basis": "Read declared artifacts, validation metadata, typed state tuples or columns solely for byte/value/axis identity checks; caller and body contracts bind those checks.", + "body_sha256": "a3a9a1b5d576cdc3d3b4fc5cbf8e5dc2fadc06bc491b46264c37b24cbe16ea40", + "findings": [ + { + "expression": "ids[order]", + "expression_sha256": "6a972f94e8a8e93aeacc926bb9304bf8e258dda743a43d60ce092e1c18e993c0", + "kind": "subscript with an unresolvable dynamic selector (fail-closed)" + } + ], + "references": [ + { + "caller": "microcosm.build.us_runtime.puf_qbi_model::model_full_puf_qbi", + "expression": "_digest_columns(ids, inputs)", + "expression_sha256": "5526a8ba55d106fbe992085f01d0d5c9cba0e154acc5c7ef2fce97e411bedc21", + "resolution": "resolved", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.puf_qbi_model::model_full_puf_qbi", + "expression": "_digest_columns(ids, frozen)", + "expression_sha256": "ba7d8aa1ef0ca2799761f3a551caa998f799628cd7e1d3d4ff438851879aeb81", + "resolution": "resolved", + "usage": "call" + } + ], + "role": "artifact_verification" + }, + "microcosm.build.us_runtime.puf_qbi_model::_employee_intercept": { + "access": "selector", + "basis": "Index validated numeric vectors or Boolean masks for model arithmetic; these selectors do not name source columns.", + "body_sha256": "b5880ea0f02a22c1eeaddcbadc2c552a3e09cda051dd59d8e6880b2eca112119", + "findings": [ + { + "expression": "revenues[positive]", + "expression_sha256": "3b52181d81247061b8dbc847211b8352cbf9f7d248e20fcf36f2057f5ea502d0", + "kind": "subscript with an unresolvable dynamic selector (fail-closed)" + } + ], + "references": [ + { + "caller": "microcosm.build.us_runtime.puf_qbi_model::model_full_puf_qbi", + "expression": "_employee_intercept(revenues[np.argsort(ids)], logit['slope_per_dollar'], logit['target_share'])", + "expression_sha256": "5f226b09f7decfce5b668dbe1f4baf4ef96f5483a527ff950ec6750c7ceb7dde", + "resolution": "resolved", + "usage": "call" + } + ], + "role": "numeric_array_selector" + }, + "microcosm.build.us_runtime.puf_target2024_growth::_digest": { + "access": "selector", + "basis": "Read declared artifacts, validation metadata, typed state tuples or columns solely for byte/value/axis identity checks; caller and body contracts bind those checks.", + "body_sha256": "35193890ef2e0d6a094a7c1ebae0b30b998a5b4b1640d9cc8b675eec4e404287", + "findings": [ + { + "expression": "columns[name]", + "expression_sha256": "12336835d3860a2f874603471384dc6fa08e89b5a8db156d58ba2c8ec2a41b0c", + "kind": "subscript with an unresolvable dynamic selector (fail-closed)" + } + ], + "references": [ + { + "caller": "microcosm.build.us_runtime.puf59_canonical_artifact::_bindings", + "expression": "_digest(arrays)", + "expression_sha256": "d8f06b3ad11cd6e6c495d7050bcfe61d9e301bf756d872a1cb54956298de9c01", + "resolution": "resolved", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.puf_target2024_growth::grow_puf_2015_to_2024", + "expression": "_digest(clean)", + "expression_sha256": "226fd38c4fa997e897f54fb4a89e53d17638583aafcb5606b802cff730ef94fc", + "resolution": "resolved", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.puf_target2024_growth::grow_puf_2015_to_2024", + "expression": "_digest(outputs)", + "expression_sha256": "a85a1e52d79ed4fd7e5cbc37754d4fccaab3d6828eab6c6169dab878b2914459", + "resolution": "resolved", + "usage": "call" + } + ], + "role": "artifact_verification" + }, + "microcosm.build.us_runtime.puf_target2024_growth::grow_puf_2015_to_2024": { + "access": "selector", + "basis": "Select a declared finite profile, canonical monetary field, knownness cell or model artifact target; no source-origin column access is granted.", + "body_sha256": "8136894d2de48c03befc15a04d0b78e7b488b65173650d25186fffcc18726d42", + "findings": [ + { + "expression": "canonical[OUTPUTS[0]]", + "expression_sha256": "e3c19dff4d3968ae033b14127f00f576f14f813ce5a4724af713af3e4abdc7b7", + "kind": "subscript with an unresolvable dynamic selector (fail-closed)" + }, + { + "expression": "known[name]", + "expression_sha256": "5f33d1032e7b4864fffa30a2123a4ef7dc16554c5001acbc998fef538b388975", + "kind": "subscript with an unresolvable dynamic selector (fail-closed)" + }, + { + "expression": "canonical[name]", + "expression_sha256": "2aead51f09fa1e4c8e759c3f17c71af6d5de721a73a81ae63b1b65deb0026ba9", + "kind": "subscript with an unresolvable dynamic selector (fail-closed)" + } + ], + "references": [ + { + "caller": "microcosm.build.us_runtime.puf59_canonical::construct_canonical_puf59", + "expression": "growth.grow_puf_2015_to_2024(canonical, known=known, input_money_year=2015, scheme=growth_scheme)", + "expression_sha256": "69673a67fd9ff1ab5ec7a0921b846d00aefd5f0be6fafc01c1116b50aa6d58a5", + "resolution": "resolved", + "usage": "call" + } + ], + "role": "closed_model_selector" + }, + "microcosm.build.us_runtime.survey_atomic_geography::_file_identity": { + "access": "selector", + "basis": "Read declared artifacts, validation metadata, typed state tuples or columns solely for byte/value/axis identity checks; caller and body contracts bind those checks.", + "body_sha256": "14ef16d5ba9090685251566bb0ecba5a0f9bdcf5aa62a2cea09f71c0358e49f5", + "findings": [ + { + "expression": "getattr(value, name)", + "expression_sha256": "8431dd4061838444001aafd6bb8453f749a3e6b146acaddda4645b73b415300e", + "kind": "getattr with an unresolvable dynamic attribute (fail-closed)" + } + ], + "references": [ + { + "caller": "microcosm.build.us_runtime.survey_atomic_geography::_read_support", + "expression": "_file_identity(after)", + "expression_sha256": "11b0928621cf7d3d9d606b9e32e983087277b01111f7f85c9182c1ce48f66648", + "resolution": "resolved", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.survey_atomic_geography::_read_support", + "expression": "_file_identity(current)", + "expression_sha256": "7cdb0dccb93478eea8bf43ef48a2ede3720475573a63ba53d1f1abc4a2758a2a", + "resolution": "resolved", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.survey_atomic_geography::_read_support", + "expression": "_file_identity(current)", + "expression_sha256": "7cdb0dccb93478eea8bf43ef48a2ede3720475573a63ba53d1f1abc4a2758a2a", + "resolution": "resolved", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.survey_atomic_geography::_read_support", + "expression": "_file_identity(before)", + "expression_sha256": "81619c14d3a1452158e5cf3da96ce0ff4344535268c757f6048393cc67268688", + "resolution": "resolved", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.survey_atomic_geography::reconstruct_atomic_survey_geography", + "expression": "_file_identity(support_after)", + "expression_sha256": "519e8cb9a56a9f5670e7cb09a739a46208be2bc5f4f9ec23f37f2684ba0d4b16", + "resolution": "resolved", + "usage": "call" + } + ], + "role": "artifact_verification" + }, + "microcosm.build.us_runtime.survey_population_preparation::_bounded_append": { + "access": "selector", + "basis": "Index bounded payload bytes, declared arrays or structured budget/tuple fields in the maintained codec, not population source columns.", + "body_sha256": "d574d299f5d4605c0dfe91f238d7e8078f71c972f14a181a8970fb9fa9a9e23b", + "findings": [ + { + "expression": "budget[0]", + "expression_sha256": "d2313919d217e0de4d035fbabf6558f844e8f1d3c1885a807c96af903c971317", + "kind": "subscript with an unresolvable dynamic selector (fail-closed)" + }, + { + "expression": "budget[0]", + "expression_sha256": "2476d1143df2467c0ee277f5905523c37ef2231ded368dfb7afcf9ed8a773118", + "kind": "subscript with an unresolvable dynamic selector (fail-closed)" + } + ], + "references": [ + { + "caller": "microcosm.build.us_runtime.survey_population_preparation::_current_survey_wage_projection", + "expression": "_bounded_append(rows, [int(stacked), int(native), 2024, amount.hex(), field.status_bytes[i], field.validity_bytes[i], field.zero_origin_bytes[i]], budget)", + "expression_sha256": "1c3202bd6b09e8ebab3bb77eba6b4d45bd8d10b58750eb242f5033eef4b74c9a", + "resolution": "resolved", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.survey_population_preparation::_origins", + "expression": "_bounded_append(records, [int(new_id), channel, int(source_id)], budget)", + "expression_sha256": "1f2b34d21105900f29b18c003c23a2c23701c49a2641924045ad0aebad20f981", + "resolution": "resolved", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.survey_population_preparation::_origins", + "expression": "_bounded_append(people, [new_id, channel, 2024, 2024 if channel == 'acs' else 2025, native_hh[channel, household_id], person_key, line, source_id, household_id, maps['household'][channel, household_id]], budget)", + "expression_sha256": "2ece82e43641f7fafe3f7bf09e705aea222f27a759c8b999d7a89cfe9781e8bc", + "resolution": "resolved", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.survey_population_preparation::_origins", + "expression": "_bounded_append(households, {'household_id': new_id, 'source': channel, 'source_year': 2024, 'survey_year': 2024 if channel == 'acs' else 2025, 'raw_native_id': raw, 'selected_receiving_household_id': source_id, 'original_anchor': _value(original)}, budget)", + "expression_sha256": "aebe559714904cc6ce47a7d1af585dcfdc47bc30edbc8b625d8fb40ab02a662f", + "resolution": "resolved", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.survey_population_preparation::_plan_document", + "expression": "_bounded_append(result[name], _value(row), budget)", + "expression_sha256": "16baf4cb3e0a059f02bd6043be7bf6cecace0726c42a1277fa613122353da6ea", + "resolution": "resolved", + "usage": "call" + } + ], + "role": "artifact_codec" + }, + "microcosm.build.us_runtime.survey_population_preparation::_current_survey_wage_projection": { + "access": "provenance", + "basis": "Qualify source-specific measured inputs and complete the declared pre-PUF donor/recipient surface before the source-blind model application.", + "body_sha256": "22f6817f57dd980c519ae1cda31f93ee3c1948a79c3417b30badd46351ff8d9f", + "findings": [ + { + "expression": "entry[0]", + "expression_sha256": "84bf0646e2d29e167f0fcb16e6583b27c1db80363ac6a81ca2da8ec3ac12d9b9", + "kind": "subscript with an unresolvable dynamic selector (fail-closed)" + }, + { + "expression": "entry[1]", + "expression_sha256": "6eb20e05f22592d515f480a018bda640991f8d6f9d208eaf86997842f151ae90", + "kind": "subscript with an unresolvable dynamic selector (fail-closed)" + }, + { + "expression": "entry[2]", + "expression_sha256": "255e733c946eab6358f6001cb56e783019ff9f5d1daf0b342a6ef2baa3cb55cc", + "kind": "subscript with an unresolvable dynamic selector (fail-closed)" + }, + { + "expression": "people[support_channel_column('person')]", + "expression_sha256": "230a04e693074722d092c0f7cad4b0f1f7266f7db14e9db6e96d8d3f50a69775", + "kind": "subscript using call to support_channel_column" + }, + { + "expression": "support_channel_column('person')", + "expression_sha256": "7cf3594f675603d58586f7d491519fb21a01f0471e5905b181248d3d17223cdb", + "kind": "call to support_channel_column" + }, + { + "expression": "selected[spine_source_id_column('person')]", + "expression_sha256": "21919c11bbea71f7372242a56f733b95201f65ebedd26aff413030e134132551", + "kind": "subscript using call to spine_source_id_column" + }, + { + "expression": "spine_source_id_column('person')", + "expression_sha256": "5bff0f374c0361bf89039f0907ff51ae6e76189950720fae047b4f2df3584bf5", + "kind": "call to spine_source_id_column" + }, + { + "expression": "selected[['person_id', spine_source_id_column('person')]]", + "expression_sha256": "a37202203c880044f64bc3da62d006602b587b3fb11f439fee622e10d560a53b", + "kind": "subscript using call to spine_source_id_column" + }, + { + "expression": "spine_source_id_column('person')", + "expression_sha256": "5bff0f374c0361bf89039f0907ff51ae6e76189950720fae047b4f2df3584bf5", + "kind": "call to spine_source_id_column" + }, + { + "expression": "entry[1]", + "expression_sha256": "6eb20e05f22592d515f480a018bda640991f8d6f9d208eaf86997842f151ae90", + "kind": "subscript with an unresolvable dynamic selector (fail-closed)" + }, + { + "expression": "entry[1]", + "expression_sha256": "6eb20e05f22592d515f480a018bda640991f8d6f9d208eaf86997842f151ae90", + "kind": "subscript with an unresolvable dynamic selector (fail-closed)" + } + ], + "references": [ + { + "caller": "microcosm.build.us_runtime.graph_puf_diagnostic_consumer::qualify_current_survey_host", + "expression": "survey_source._current_survey_wage_projection(preparation, entry)", + "expression_sha256": "2c1c17503b30242030caac6c6b20d560318f3a3880354ccde4d1a9a26c6c48dd", + "resolution": "resolved", + "usage": "call" + } + ], + "role": "model_qualification" + }, + "microcosm.build.us_runtime.survey_population_preparation::_nested_seals": { + "access": "selector", + "basis": "Read declared artifacts, validation metadata, typed state tuples or columns solely for byte/value/axis identity checks; caller and body contracts bind those checks.", + "body_sha256": "50500ab0c9d61e3725427b57b17a845dd9cce0dd339e084357d11e888ab5bebc", + "findings": [ + { + "expression": "native[0]", + "expression_sha256": "76c691eae80516426677a5729bf93f5affcafee8a525507a5c682f50d616d67e", + "kind": "subscript with an unresolvable dynamic selector (fail-closed)" + }, + { + "expression": "catalogues[0]", + "expression_sha256": "fbbaffad2ffa692dc30ca00ebb9446fac7bf9352ef507a16c13509f58e05e62f", + "kind": "subscript with an unresolvable dynamic selector (fail-closed)" + }, + { + "expression": "catalogues[1]", + "expression_sha256": "925a10daed888e94fee11c3fa4d48df75229521efaa8a0eed2f9e826c56b8ad1", + "kind": "subscript with an unresolvable dynamic selector (fail-closed)" + }, + { + "expression": "native[1]", + "expression_sha256": "d36151a6d0607b577f4cbe4198b5643a7388496e7a5843f7e68f1bd61fd3663f", + "kind": "subscript with an unresolvable dynamic selector (fail-closed)" + } + ], + "references": [ + { + "caller": "microcosm.build.us_runtime.survey_population_preparation::_pure_final", + "expression": "_nested_seals(state.catalogues, state.native)", + "expression_sha256": "2c4ae9a977a838033d5e0cf91b03d922db0dee25ff8ae75c99a8a9cd35cd0506", + "resolution": "resolved", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.survey_population_preparation::prepare_authenticated_survey_population", + "expression": "_nested_seals((acs, asec), (actual_acs, actual_asec))", + "expression_sha256": "8f9ea4b6ac20d1e20c651202d8a61d02683a5a821e02ab76deb516ae0953cc78", + "resolution": "resolved", + "usage": "call" + } + ], + "role": "artifact_verification" + }, + "microcosm.build.us_runtime.survey_population_preparation::_origins": { + "access": "provenance", + "basis": "Construct/validate the source composition or its first structural clone with original lineage and mass, before generic population operators.", + "body_sha256": "7287ac475d7241f9b04bf0fb04e235a624cc0bad22851fb8c226f8ac57c4116a", + "findings": [ + { + "expression": "support_channel_column(entity)", + "expression_sha256": "c5f5055ec6fcad2e076a23218592dfd7584c9c74920d3d93a84a56493490d8f3", + "kind": "call to support_channel_column" + }, + { + "expression": "spine_source_id_column(entity)", + "expression_sha256": "2a61eaf1e93ee9e265db5fb590763785dd34e6833b9ee69d3eaf86b69042c4a7", + "kind": "call to spine_source_id_column" + }, + { + "expression": "table[[ids, channels, previous]]", + "expression_sha256": "b52e44ca8d8c578ae75b703c56fb360965c4105680ecad5156b62cd087036393", + "kind": "subscript using call to support_channel_column" + }, + { + "expression": "table[support_channel_column(entity)]", + "expression_sha256": "014b6a045590d7033107a960ab1230d69df9b39ce83277cd4a22b90d02b7a7f6", + "kind": "subscript using call to support_channel_column" + }, + { + "expression": "support_channel_column(entity)", + "expression_sha256": "c5f5055ec6fcad2e076a23218592dfd7584c9c74920d3d93a84a56493490d8f3", + "kind": "call to support_channel_column" + }, + { + "expression": "arm[spine_source_id_column(entity)]", + "expression_sha256": "35a6384de034f7f37b1b2a1e16f7b383d38869b9cea170c463ecbcb763ac1bac", + "kind": "subscript using call to spine_source_id_column" + }, + { + "expression": "spine_source_id_column(entity)", + "expression_sha256": "2a61eaf1e93ee9e265db5fb590763785dd34e6833b9ee69d3eaf86b69042c4a7", + "kind": "call to spine_source_id_column" + }, + { + "expression": "table[support_channel_column(entity)]", + "expression_sha256": "014b6a045590d7033107a960ab1230d69df9b39ce83277cd4a22b90d02b7a7f6", + "kind": "subscript using call to support_channel_column" + }, + { + "expression": "support_channel_column(entity)", + "expression_sha256": "c5f5055ec6fcad2e076a23218592dfd7584c9c74920d3d93a84a56493490d8f3", + "kind": "call to support_channel_column" + }, + { + "expression": "sources[channel]", + "expression_sha256": "88153b7803c66fa6403b46d0c113c3a382e450f3a183f8f0dcb0e4fb97732150", + "kind": "subscript with an unresolvable dynamic selector (fail-closed)" + } + ], + "references": [ + { + "caller": "microcosm.build.us_runtime.survey_population_preparation::prepare_authenticated_survey_population", + "expression": "_origins(frame, source_copies, plan.selected, native_receipts)", + "expression_sha256": "e440caff914e4143fbd94178adb49b5f7e62a34ebd8608b3c52b2f716f89dab7", + "resolution": "resolved", + "usage": "call" + } + ], + "role": "population_composition" + }, + "microcosm.build.us_runtime.survey_population_preparation::_plan_document": { + "access": "selector", + "basis": "Read declared artifacts, validation metadata, typed state tuples or columns solely for byte/value/axis identity checks; caller and body contracts bind those checks.", + "body_sha256": "cb5aacaeb76cc3188b8d8ce7fba6c18411d409f6584796e09e7549edc60f72d1", + "findings": [ + { + "expression": "getattr(plan, name)", + "expression_sha256": "c2a8d0d3291d6dda6b3b7109a5c1f362a911f723069de64f9eb01d210fec24d2", + "kind": "getattr with an unresolvable dynamic attribute (fail-closed)" + } + ], + "references": [ + { + "caller": "microcosm.build.us_runtime.survey_population_preparation::_pure_final", + "expression": "_plan_document(state.plan)", + "expression_sha256": "238a29f0933991a9e9e037057f463085133a48b5a5d3cd4d2ed620653d02bae4", + "resolution": "resolved", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.survey_population_preparation::prepare_authenticated_survey_population", + "expression": "_plan_document(plan)", + "expression_sha256": "112405e367211c95ba1ec6296b417e60fc5ce0d5c0e6447ce983e8d4d64020b1", + "resolution": "resolved", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.survey_population_preparation::prepare_authenticated_survey_population", + "expression": "_plan_document(plan)", + "expression_sha256": "112405e367211c95ba1ec6296b417e60fc5ce0d5c0e6447ce983e8d4d64020b1", + "resolution": "resolved", + "usage": "call" + } + ], + "role": "artifact_verification" + }, + "microcosm.build.us_runtime.survey_population_preparation::_value": { + "access": "selector", + "basis": "Read declared artifacts, validation metadata, typed state tuples or columns solely for byte/value/axis identity checks; caller and body contracts bind those checks.", + "body_sha256": "409c94b970156203110adcbb89d77de66864a8ad1ad8d561c30dcdec78529ee9", + "findings": [ + { + "expression": "getattr(value, field.name)", + "expression_sha256": "6589b4ab9b8ca45aafd6a8220848404e6b874abe46d71cd37a359c8e699fbbb2", + "kind": "getattr with an unresolvable dynamic attribute (fail-closed)" + } + ], + "references": [ + { + "caller": "microcosm.build.us_runtime.survey_population_preparation::_origins", + "expression": "_value(original)", + "expression_sha256": "a6b08eac754a83ca45c510db5546f1767e2a0b7222e1dfc38ec745f7cbdd6095", + "resolution": "resolved", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.survey_population_preparation::_plan_document", + "expression": "_value(row)", + "expression_sha256": "46c962db3d75f7da2022867bdc60419aa82e2f31cd383de2da305f911af22486", + "resolution": "resolved", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.survey_population_preparation::_plan_document", + "expression": "_value(plan.fraction)", + "expression_sha256": "b7100d34ef0438c33b244b9925ddd3fdabf1607e44ee8ff9685f922da516391b", + "resolution": "resolved", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.survey_population_preparation::_value", + "expression": "_value(item)", + "expression_sha256": "4c2d2a840de98ff5184a6bf87bfdb81cca123ae11e6869c216956cf933f3e702", + "resolution": "resolved", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.survey_population_preparation::_value", + "expression": "_value(item)", + "expression_sha256": "4c2d2a840de98ff5184a6bf87bfdb81cca123ae11e6869c216956cf933f3e702", + "resolution": "resolved", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.survey_population_preparation::_value", + "expression": "_value(getattr(value, field.name))", + "expression_sha256": "e181dd22c751a8b8368b990ed3193136b58ea9c56f071b782593f292b86b5a72", + "resolution": "resolved", + "usage": "call" + } + ], + "role": "artifact_verification" + }, + "microcosm.build.us_runtime.survey_social_security::_float64": { + "access": "selector", + "basis": "Index validated numeric vectors or Boolean masks for model arithmetic; these selectors do not name source columns.", + "body_sha256": "b519d0ef26ec06acfd4da23ad7940b9af9af778e0105610827417eab8c25372b", + "findings": [ + { + "expression": "values[~np.isnan(values)]", + "expression_sha256": "d45d7def3a509cece8187a96bc2e9255d37cf5a7c8b1c558720640bb180bbee7", + "kind": "subscript with an unresolvable dynamic selector (fail-closed)" + } + ], + "references": [ + { + "caller": "microcosm.build.us_runtime.survey_social_security::asec_reason_basis", + "expression": "_float64(values, shape=total.shape, nullable=True)", + "expression_sha256": "5be219146ad73204cfad2efe4b42cc54270cd0a02ff9132a75420e2bc05ced28", + "resolution": "resolved", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.survey_social_security::asec_reason_basis", + "expression": "_float64(total, shape=total.shape)", + "expression_sha256": "d064fd343c1b6c6c05d72d87ee77f46ac897688a3991ff3796e4c809be4385dd", + "resolution": "resolved", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.survey_social_security::asec_reporting_basis", + "expression": "_float64(age, shape=total.shape)", + "expression_sha256": "50963e5474748c6a22e9ac1a951bb8bb5cd78e79873e0b49a646c307cb1abac9", + "resolution": "resolved", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.survey_social_security::asec_reporting_basis", + "expression": "_float64(recipiency, shape=total.shape, nullable=True)", + "expression_sha256": "9dd0a66f9acc1b548d184a0186065d388ebb67478c3e2d5ca8b600fafa6f06dc", + "resolution": "resolved", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.survey_social_security::asec_reporting_basis", + "expression": "_float64(total, shape=total.shape)", + "expression_sha256": "d064fd343c1b6c6c05d72d87ee77f46ac897688a3991ff3796e4c809be4385dd", + "resolution": "resolved", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.survey_social_security::complete_positive_basis", + "expression": "_float64(source_basis, shape=shape, nullable=True)", + "expression_sha256": "1ce8387f2b323c8a838f2586ce2002a1e723eef9323e7afd3de97b62485bf2c5", + "resolution": "resolved", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.survey_social_security::complete_positive_basis", + "expression": "_float64(modeled_scores, shape=shape, nullable=True)", + "expression_sha256": "3889f323e697c059979611cb24094d2698bd4215ff669326298e5a48e91b50cc", + "resolution": "resolved", + "usage": "call" + }, + { + "caller": "microcosm.build.us_runtime.survey_social_security::complete_positive_basis", + "expression": "_float64(total, shape=total.shape)", + "expression_sha256": "d064fd343c1b6c6c05d72d87ee77f46ac897688a3991ff3796e4c809be4385dd", + "resolution": "resolved", + "usage": "call" + } + ], + "role": "numeric_array_selector" + }, + "microcosm.build.us_runtime.survey_social_security::complete_positive_basis": { + "access": "selector", + "basis": "Index validated numeric vectors or Boolean masks for model arithmetic; these selectors do not name source columns.", + "body_sha256": "ee65a2902b7d3a1c562a2352cd50534c404b3a0b78778dd2748721e556e07123", + "findings": [ + { + "expression": "allowed[complete]", + "expression_sha256": "cdcad01d1515b4be6b6a8b709f80cf56a6050317067ac4a125424ac91f0a5c73", + "kind": "subscript with an unresolvable dynamic selector (fail-closed)" + }, + { + "expression": "allowed[missing]", + "expression_sha256": "11b0a45f2ce6ae0aae28867982dde6a9b672ea99cac1f4553bb96bcf2a7f41ca", + "kind": "subscript with an unresolvable dynamic selector (fail-closed)" + } + ], + "references": [], + "role": "numeric_array_selector" + } + }, + "selector_domain_bindings": [ + "microcosm.build.us_runtime.cps_carried_current::@CPS_CURRENT_PREDICTOR_MONEY_FIELDS", + "microcosm.build.us_runtime.cps_carried_current::@CPS_CURRENT_PREDICTOR_PERSON_LEAVES", + "microcosm.build.us_runtime.current_survey_predictors::@DEMOGRAPHIC_FEATURES", + "microcosm.build.us_runtime.current_survey_predictors::@FEATURES", + "microcosm.build.us_runtime.current_survey_predictors::@MONEY_FIELDS", + "microcosm.build.us_runtime.current_survey_predictors::@OUTPUTS", + "microcosm.build.us_runtime.current_survey_predictors::@PHASE", + "microcosm.build.us_runtime.current_survey_predictors::@PROTOCOL", + "microcosm.build.us_runtime.current_survey_predictors::@SEED", + "microcosm.build.us_runtime.current_survey_predictors::@TARGETS", + "microcosm.build.us_runtime.full_puf_enrichment::@FULL65", + "microcosm.build.us_runtime.full_puf_enrichment::@PERSON_OUTPUTS", + "microcosm.build.us_runtime.full_puf_enrichment::@PHASE", + "microcosm.build.us_runtime.full_puf_enrichment::@PREDICTORS", + "microcosm.build.us_runtime.full_puf_enrichment::@PUF55_SURVEY_SS", + "microcosm.build.us_runtime.full_puf_enrichment::@PUF55_SURVEY_SS_NO_TOTAL", + "microcosm.build.us_runtime.full_puf_enrichment::@PUF55_SURVEY_SS_PERSON_OUTPUTS", + "microcosm.build.us_runtime.full_puf_enrichment::@PUF55_SURVEY_SS_PREDICTORS", + "microcosm.build.us_runtime.full_puf_enrichment::@PUF59", + "microcosm.build.us_runtime.full_puf_enrichment::@PUF59_PREDICTORS", + "microcosm.build.us_runtime.full_puf_enrichment::@PUF59_TAX_UNIT_OUTPUTS", + "microcosm.build.us_runtime.full_puf_enrichment::@SCF_MORTGAGE_OUTPUTS", + "microcosm.build.us_runtime.full_puf_enrichment::@SURVEY_SS_COMPONENTS", + "microcosm.build.us_runtime.full_puf_enrichment::@SURVEY_SS_TOTAL_PREDICTOR", + "microcosm.build.us_runtime.full_puf_enrichment::@TARGETS", + "microcosm.build.us_runtime.full_puf_enrichment::@TAX_UNIT_OUTPUTS", + "microcosm.build.us_runtime.full_puf_enrichment::PufOutputProfile", + "microcosm.build.us_runtime.full_puf_enrichment::_recipient_matrix", + "microcosm.build.us_runtime.full_puf_enrichment::prepare_full_puf_inputs", + "microcosm.build.us_runtime.full_puf_enrichment::require_puf_output_profile", + "microcosm.build.us_runtime.operator_column_contracts::@ACS_DERIVED_TRANSFER_INPUTS", + "microcosm.build.us_runtime.operator_column_contracts::@PUF_CAPITAL_GAINS_TAIL_APPLIED_COLUMN", + "microcosm.build.us_runtime.operator_column_contracts::@PUF_CAPITAL_GAINS_TAIL_DONOR_AGI_BAND_COLUMN", + "microcosm.build.us_runtime.operator_column_contracts::@PUF_CAPITAL_GAINS_TAIL_DONOR_FILING_STATUS_COLUMN", + "microcosm.build.us_runtime.operator_column_contracts::@PUF_CAPITAL_GAINS_TAIL_DONOR_SOURCE_ID_COLUMN", + "microcosm.build.us_runtime.operator_column_contracts::@PUF_CAPITAL_GAINS_TAIL_DONOR_SYNTHETIC_COLUMN", + "microcosm.build.us_runtime.operator_column_contracts::@PUF_CAPITAL_GAINS_TAIL_PERSON_COLUMNS", + "microcosm.build.us_runtime.operator_column_contracts::@PUF_CAPITAL_GAINS_TAIL_TAX_UNIT_COLUMNS", + "microcosm.build.us_runtime.operator_column_contracts::@PUF_CAPITAL_GAINS_TAIL_TRANSFER_WEIGHT_COLUMN", + "microcosm.build.us_runtime.operator_column_contracts::@PUF_SUPPORT_MAX_CLONE_SAFE_SOURCE_ID", + "microcosm.build.us_runtime.operator_column_contracts::@PUF_TAX_DETAIL_DEFAULT_PERSON_OUTPUTS", + "microcosm.build.us_runtime.operator_column_contracts::@PUF_TAX_DETAIL_DEFAULT_TAX_UNIT_OUTPUTS", + "microcosm.build.us_runtime.operator_column_contracts::@US_PUF_SUPPORT_STAGE_NAME", + "microcosm.build.us_runtime.operator_column_contracts::@US_QBI_BOOLEAN_OUTPUT_COLUMNS", + "microcosm.build.us_runtime.operator_column_contracts::@US_QBI_NONNEGATIVE_OUTPUT_COLUMNS", + "microcosm.build.us_runtime.operator_column_contracts::@US_QBI_OUTPUT_COLUMNS", + "microcosm.build.us_runtime.operator_column_contracts::@_GENERAL_QUALIFICATION_FLAGS", + "microcosm.build.us_runtime.operator_column_contracts::@_SSTB_QUALIFICATION_FLAG", + "microcosm.build.us_runtime.puf55_route_finalization::@FINALIZATION_PROTOCOL", + "microcosm.build.us_runtime.puf55_route_finalization::@PROFILES", + "microcosm.build.us_runtime.puf55_route_finalization::@PROTOCOL", + "microcosm.build.us_runtime.puf55_survey_ss_measurement::@COLUMNS", + "microcosm.build.us_runtime.puf55_survey_ss_measurement::@KNOWN", + "microcosm.build.us_runtime.puf55_survey_ss_measurement::@MEASUREMENT", + "microcosm.build.us_runtime.puf55_survey_ss_measurement::@PROTOCOL", + "microcosm.build.us_runtime.puf55_survey_ss_measurement::@ROUTE", + "microcosm.build.us_runtime.puf55_survey_ss_measurement::@TOTAL", + "microcosm.build.us_runtime.puf55_survey_ss_measurement::@_PERSON_COLUMNS", + "microcosm.build.us_runtime.puf55_survey_ss_measurement::@_REPORT_COLUMNS", + "microcosm.build.us_runtime.puf55_survey_ss_measurement::@_STATUSES", + "microcosm.build.us_runtime.puf55_survey_ss_measurement::@_UNIT_COLUMNS", + "microcosm.build.us_runtime.puf59_canonical::@PREFIX_NAMES", + "microcosm.build.us_runtime.puf59_canonical_artifact::@INTEGERS", + "microcosm.build.us_runtime.puf59_canonical_artifact::@MAGIC", + "microcosm.build.us_runtime.puf59_canonical_artifact::@MAX_BODY", + "microcosm.build.us_runtime.puf59_canonical_artifact::@MAX_HEADER", + "microcosm.build.us_runtime.puf59_canonical_artifact::@NAMES", + "microcosm.build.us_runtime.puf59_canonical_artifact::@PREFIX", + "microcosm.build.us_runtime.puf_detail_transfer::@ARRAY_DTYPES", + "microcosm.build.us_runtime.puf_detail_transfer::@FEATURES", + "microcosm.build.us_runtime.puf_detail_transfer::@HEADER_FIELDS", + "microcosm.build.us_runtime.puf_detail_transfer::@MAGIC", + "microcosm.build.us_runtime.puf_detail_transfer::@MARS", + "microcosm.build.us_runtime.puf_detail_transfer::@MASK", + "microcosm.build.us_runtime.puf_detail_transfer::@MAX_BYTES", + "microcosm.build.us_runtime.puf_detail_transfer::@MAX_ROWS", + "microcosm.build.us_runtime.puf_detail_transfer::@MONEY_FIELDS", + "microcosm.build.us_runtime.puf_detail_transfer::@OUTPUT", + "microcosm.build.us_runtime.puf_detail_transfer::@SCOPE", + "microcosm.build.us_runtime.puf_detail_transfer::@SOURCE_ADMISSION", + "microcosm.build.us_runtime.puf_detail_transfer::@TARGET", + "microcosm.build.us_runtime.puf_full_source::@AGGREGATE_LEXEME_MAX_CHARACTERS", + "microcosm.build.us_runtime.puf_full_source::@COUNT_COLUMNS", + "microcosm.build.us_runtime.puf_full_source::@DEPENDENT_COLUMNS", + "microcosm.build.us_runtime.puf_full_source::@DIRECT_MAPPINGS", + "microcosm.build.us_runtime.puf_full_source::@FULL_SOURCE_MAX_BYTES", + "microcosm.build.us_runtime.puf_full_source::@FULL_SOURCE_VERSION", + "microcosm.build.us_runtime.puf_full_source::@MONEY_COLUMNS", + "microcosm.build.us_runtime.puf_full_source::@OUTSIDE_AMOUNT_UNIVERSE", + "microcosm.build.us_runtime.puf_full_source::@PROJECTED_COLUMNS", + "microcosm.build.us_runtime.puf_full_source::@_AGGREGATE", + "microcosm.build.us_runtime.puf_full_source::@_COUNT", + "microcosm.build.us_runtime.puf_full_source::@_HEADER_LIMIT", + "microcosm.build.us_runtime.puf_full_source::@_MAGIC", + "microcosm.build.us_runtime.puf_full_source::@_MONEY", + "microcosm.build.us_runtime.puf_growth::@PROVENANCE_RECID_COLUMN", + "microcosm.build.us_runtime.puf_growth::@PROVENANCE_SOURCE_AGI_COLUMN", + "microcosm.build.us_runtime.puf_support::@PUF_TAX_DETAIL_DEFAULT_PREDICTORS", + "microcosm.build.us_runtime.puf_support::@PUF_TAX_DETAIL_SOCIAL_SECURITY_COMPONENT_OUTPUTS", + "microcosm.build.us_runtime.puf_support::@_PUF_TAX_DETAIL_BOOLEAN_PERSON_OUTPUTS", + "microcosm.build.us_runtime.puf_support::@_PUF_TAX_DETAIL_DISCRETE_TAX_UNIT_OUTPUTS", + "microcosm.build.us_runtime.puf_target2024_growth::@INCIDENCE_FIELDS", + "microcosm.build.us_runtime.puf_target2024_growth::@OUTPUTS", + "microcosm.build.us_runtime.puf_target2024_growth::@RECIPE_SHA256", + "microcosm.build.us_runtime.puf_target2024_growth::@VERSION", + "microcosm.build.us_runtime.puf_target2024_growth::@_RECIPE", + "microcosm.build.us_runtime.puf_target2024_growth::@_RECIPE_JSON", + "microcosm.build.us_runtime.survey_social_security::@COMPONENTS", + "microcosm.build.us_runtime.survey_social_security::@PROTOCOL", + "microcosm.build.us_runtime.survey_social_security::@REASON_COMPONENTS" + ], + "selector_domain_dependencies": { + "microcosm.build.us_runtime.cps_carried_current::@CPS_CURRENT_PREDICTOR_MONEY_FIELDS": [], + "microcosm.build.us_runtime.cps_carried_current::@CPS_CURRENT_PREDICTOR_PERSON_LEAVES": [], + "microcosm.build.us_runtime.current_survey_predictors::@DEMOGRAPHIC_FEATURES": [ + "microcosm.build.us_runtime.current_survey_predictors::@FEATURES" + ], + "microcosm.build.us_runtime.current_survey_predictors::@FEATURES": [], + "microcosm.build.us_runtime.current_survey_predictors::@MONEY_FIELDS": [ + "microcosm.build.us_runtime.cps_carried_current::@CPS_CURRENT_PREDICTOR_MONEY_FIELDS" + ], + "microcosm.build.us_runtime.current_survey_predictors::@OUTPUTS": [ + "microcosm.build.us_runtime.cps_carried_current::@CPS_CURRENT_PREDICTOR_PERSON_LEAVES" + ], + "microcosm.build.us_runtime.current_survey_predictors::@PHASE": [], + "microcosm.build.us_runtime.current_survey_predictors::@PROTOCOL": [], + "microcosm.build.us_runtime.current_survey_predictors::@SEED": [], + "microcosm.build.us_runtime.current_survey_predictors::@TARGETS": [], + "microcosm.build.us_runtime.full_puf_enrichment::@FULL65": [], + "microcosm.build.us_runtime.full_puf_enrichment::@PERSON_OUTPUTS": [ + "microcosm.build.us_runtime.operator_column_contracts::@PUF_TAX_DETAIL_DEFAULT_PERSON_OUTPUTS" + ], + "microcosm.build.us_runtime.full_puf_enrichment::@PHASE": [], + "microcosm.build.us_runtime.full_puf_enrichment::@PREDICTORS": [ + "microcosm.build.us_runtime.puf_support::@PUF_TAX_DETAIL_DEFAULT_PREDICTORS" + ], + "microcosm.build.us_runtime.full_puf_enrichment::@PUF55_SURVEY_SS": [], + "microcosm.build.us_runtime.full_puf_enrichment::@PUF55_SURVEY_SS_NO_TOTAL": [], + "microcosm.build.us_runtime.full_puf_enrichment::@PUF55_SURVEY_SS_PERSON_OUTPUTS": [ + "microcosm.build.us_runtime.full_puf_enrichment::@PERSON_OUTPUTS", + "microcosm.build.us_runtime.full_puf_enrichment::@SURVEY_SS_COMPONENTS" + ], + "microcosm.build.us_runtime.full_puf_enrichment::@PUF55_SURVEY_SS_PREDICTORS": [ + "microcosm.build.us_runtime.full_puf_enrichment::@PUF59_PREDICTORS", + "microcosm.build.us_runtime.full_puf_enrichment::@SURVEY_SS_TOTAL_PREDICTOR" + ], + "microcosm.build.us_runtime.full_puf_enrichment::@PUF59": [], + "microcosm.build.us_runtime.full_puf_enrichment::@PUF59_PREDICTORS": [ + "microcosm.build.us_runtime.full_puf_enrichment::@PREDICTORS" + ], + "microcosm.build.us_runtime.full_puf_enrichment::@PUF59_TAX_UNIT_OUTPUTS": [ + "microcosm.build.us_runtime.full_puf_enrichment::@SCF_MORTGAGE_OUTPUTS", + "microcosm.build.us_runtime.full_puf_enrichment::@TAX_UNIT_OUTPUTS" + ], + "microcosm.build.us_runtime.full_puf_enrichment::@SCF_MORTGAGE_OUTPUTS": [], + "microcosm.build.us_runtime.full_puf_enrichment::@SURVEY_SS_COMPONENTS": [ + "microcosm.build.us_runtime.puf_support::@PUF_TAX_DETAIL_SOCIAL_SECURITY_COMPONENT_OUTPUTS" + ], + "microcosm.build.us_runtime.full_puf_enrichment::@SURVEY_SS_TOTAL_PREDICTOR": [], + "microcosm.build.us_runtime.full_puf_enrichment::@TARGETS": [ + "microcosm.build.us_runtime.full_puf_enrichment::@PERSON_OUTPUTS", + "microcosm.build.us_runtime.full_puf_enrichment::@TAX_UNIT_OUTPUTS" + ], + "microcosm.build.us_runtime.full_puf_enrichment::@TAX_UNIT_OUTPUTS": [ + "microcosm.build.us_runtime.operator_column_contracts::@PUF_TAX_DETAIL_DEFAULT_TAX_UNIT_OUTPUTS" + ], + "microcosm.build.us_runtime.full_puf_enrichment::PufOutputProfile": [ + "microcosm.build.us_runtime.full_puf_enrichment::@FULL65", + "microcosm.build.us_runtime.full_puf_enrichment::@PERSON_OUTPUTS", + "microcosm.build.us_runtime.full_puf_enrichment::@PHASE", + "microcosm.build.us_runtime.full_puf_enrichment::@PREDICTORS", + "microcosm.build.us_runtime.full_puf_enrichment::@PUF55_SURVEY_SS", + "microcosm.build.us_runtime.full_puf_enrichment::@PUF55_SURVEY_SS_NO_TOTAL", + "microcosm.build.us_runtime.full_puf_enrichment::@PUF55_SURVEY_SS_PERSON_OUTPUTS", + "microcosm.build.us_runtime.full_puf_enrichment::@PUF55_SURVEY_SS_PREDICTORS", + "microcosm.build.us_runtime.full_puf_enrichment::@PUF59_PREDICTORS", + "microcosm.build.us_runtime.full_puf_enrichment::@PUF59_TAX_UNIT_OUTPUTS", + "microcosm.build.us_runtime.full_puf_enrichment::@SURVEY_SS_TOTAL_PREDICTOR", + "microcosm.build.us_runtime.full_puf_enrichment::@TAX_UNIT_OUTPUTS" + ], + "microcosm.build.us_runtime.full_puf_enrichment::_recipient_matrix": [ + "microcosm.build.us_runtime.full_puf_enrichment::@FULL65" + ], + "microcosm.build.us_runtime.full_puf_enrichment::prepare_full_puf_inputs": [ + "microcosm.build.us_runtime.full_puf_enrichment::@FULL65" + ], + "microcosm.build.us_runtime.full_puf_enrichment::require_puf_output_profile": [], + "microcosm.build.us_runtime.operator_column_contracts::@ACS_DERIVED_TRANSFER_INPUTS": [], + "microcosm.build.us_runtime.operator_column_contracts::@PUF_CAPITAL_GAINS_TAIL_APPLIED_COLUMN": [], + "microcosm.build.us_runtime.operator_column_contracts::@PUF_CAPITAL_GAINS_TAIL_DONOR_AGI_BAND_COLUMN": [], + "microcosm.build.us_runtime.operator_column_contracts::@PUF_CAPITAL_GAINS_TAIL_DONOR_FILING_STATUS_COLUMN": [], + "microcosm.build.us_runtime.operator_column_contracts::@PUF_CAPITAL_GAINS_TAIL_DONOR_SOURCE_ID_COLUMN": [], + "microcosm.build.us_runtime.operator_column_contracts::@PUF_CAPITAL_GAINS_TAIL_DONOR_SYNTHETIC_COLUMN": [], + "microcosm.build.us_runtime.operator_column_contracts::@PUF_CAPITAL_GAINS_TAIL_PERSON_COLUMNS": [], + "microcosm.build.us_runtime.operator_column_contracts::@PUF_CAPITAL_GAINS_TAIL_TAX_UNIT_COLUMNS": [], + "microcosm.build.us_runtime.operator_column_contracts::@PUF_CAPITAL_GAINS_TAIL_TRANSFER_WEIGHT_COLUMN": [], + "microcosm.build.us_runtime.operator_column_contracts::@PUF_SUPPORT_MAX_CLONE_SAFE_SOURCE_ID": [], + "microcosm.build.us_runtime.operator_column_contracts::@PUF_TAX_DETAIL_DEFAULT_PERSON_OUTPUTS": [ + "microcosm.build.us_runtime.operator_column_contracts::@US_QBI_OUTPUT_COLUMNS" + ], + "microcosm.build.us_runtime.operator_column_contracts::@PUF_TAX_DETAIL_DEFAULT_TAX_UNIT_OUTPUTS": [], + "microcosm.build.us_runtime.operator_column_contracts::@US_PUF_SUPPORT_STAGE_NAME": [], + "microcosm.build.us_runtime.operator_column_contracts::@US_QBI_BOOLEAN_OUTPUT_COLUMNS": [ + "microcosm.build.us_runtime.operator_column_contracts::@_GENERAL_QUALIFICATION_FLAGS", + "microcosm.build.us_runtime.operator_column_contracts::@_SSTB_QUALIFICATION_FLAG" + ], + "microcosm.build.us_runtime.operator_column_contracts::@US_QBI_NONNEGATIVE_OUTPUT_COLUMNS": [], + "microcosm.build.us_runtime.operator_column_contracts::@US_QBI_OUTPUT_COLUMNS": [ + "microcosm.build.us_runtime.operator_column_contracts::@US_QBI_BOOLEAN_OUTPUT_COLUMNS" + ], + "microcosm.build.us_runtime.operator_column_contracts::@_GENERAL_QUALIFICATION_FLAGS": [], + "microcosm.build.us_runtime.operator_column_contracts::@_SSTB_QUALIFICATION_FLAG": [], + "microcosm.build.us_runtime.puf55_route_finalization::@FINALIZATION_PROTOCOL": [], + "microcosm.build.us_runtime.puf55_route_finalization::@PROFILES": [ + "microcosm.build.us_runtime.full_puf_enrichment::@PUF55_SURVEY_SS", + "microcosm.build.us_runtime.full_puf_enrichment::@PUF55_SURVEY_SS_NO_TOTAL" + ], + "microcosm.build.us_runtime.puf55_route_finalization::@PROTOCOL": [], + "microcosm.build.us_runtime.puf55_survey_ss_measurement::@COLUMNS": [ + "microcosm.build.us_runtime.puf55_survey_ss_measurement::@KNOWN", + "microcosm.build.us_runtime.puf55_survey_ss_measurement::@ROUTE", + "microcosm.build.us_runtime.puf55_survey_ss_measurement::@TOTAL" + ], + "microcosm.build.us_runtime.puf55_survey_ss_measurement::@KNOWN": [], + "microcosm.build.us_runtime.puf55_survey_ss_measurement::@MEASUREMENT": [], + "microcosm.build.us_runtime.puf55_survey_ss_measurement::@PROTOCOL": [], + "microcosm.build.us_runtime.puf55_survey_ss_measurement::@ROUTE": [], + "microcosm.build.us_runtime.puf55_survey_ss_measurement::@TOTAL": [ + "microcosm.build.us_runtime.full_puf_enrichment::@SURVEY_SS_TOTAL_PREDICTOR" + ], + "microcosm.build.us_runtime.puf55_survey_ss_measurement::@_PERSON_COLUMNS": [], + "microcosm.build.us_runtime.puf55_survey_ss_measurement::@_REPORT_COLUMNS": [], + "microcosm.build.us_runtime.puf55_survey_ss_measurement::@_STATUSES": [], + "microcosm.build.us_runtime.puf55_survey_ss_measurement::@_UNIT_COLUMNS": [], + "microcosm.build.us_runtime.puf59_canonical::@PREFIX_NAMES": [], + "microcosm.build.us_runtime.puf59_canonical_artifact::@INTEGERS": [ + "microcosm.build.us_runtime.puf59_canonical_artifact::@PREFIX", + "microcosm.build.us_runtime.puf_target2024_growth::@INCIDENCE_FIELDS" + ], + "microcosm.build.us_runtime.puf59_canonical_artifact::@MAGIC": [], + "microcosm.build.us_runtime.puf59_canonical_artifact::@MAX_BODY": [], + "microcosm.build.us_runtime.puf59_canonical_artifact::@MAX_HEADER": [], + "microcosm.build.us_runtime.puf59_canonical_artifact::@NAMES": [ + "microcosm.build.us_runtime.puf59_canonical_artifact::@PREFIX", + "microcosm.build.us_runtime.puf_target2024_growth::@OUTPUTS" + ], + "microcosm.build.us_runtime.puf59_canonical_artifact::@PREFIX": [ + "microcosm.build.us_runtime.puf59_canonical::@PREFIX_NAMES" + ], + "microcosm.build.us_runtime.puf_detail_transfer::@ARRAY_DTYPES": [ + "microcosm.build.us_runtime.puf_detail_transfer::@MONEY_FIELDS", + "microcosm.build.us_runtime.puf_growth::@PROVENANCE_RECID_COLUMN", + "microcosm.build.us_runtime.puf_growth::@PROVENANCE_SOURCE_AGI_COLUMN" + ], + "microcosm.build.us_runtime.puf_detail_transfer::@FEATURES": [], + "microcosm.build.us_runtime.puf_detail_transfer::@HEADER_FIELDS": [], + "microcosm.build.us_runtime.puf_detail_transfer::@MAGIC": [], + "microcosm.build.us_runtime.puf_detail_transfer::@MARS": [], + "microcosm.build.us_runtime.puf_detail_transfer::@MASK": [], + "microcosm.build.us_runtime.puf_detail_transfer::@MAX_BYTES": [], + "microcosm.build.us_runtime.puf_detail_transfer::@MAX_ROWS": [], + "microcosm.build.us_runtime.puf_detail_transfer::@MONEY_FIELDS": [], + "microcosm.build.us_runtime.puf_detail_transfer::@OUTPUT": [], + "microcosm.build.us_runtime.puf_detail_transfer::@SCOPE": [], + "microcosm.build.us_runtime.puf_detail_transfer::@SOURCE_ADMISSION": [], + "microcosm.build.us_runtime.puf_detail_transfer::@TARGET": [], + "microcosm.build.us_runtime.puf_full_source::@AGGREGATE_LEXEME_MAX_CHARACTERS": [], + "microcosm.build.us_runtime.puf_full_source::@COUNT_COLUMNS": [], + "microcosm.build.us_runtime.puf_full_source::@DEPENDENT_COLUMNS": [], + "microcosm.build.us_runtime.puf_full_source::@DIRECT_MAPPINGS": [], + "microcosm.build.us_runtime.puf_full_source::@FULL_SOURCE_MAX_BYTES": [], + "microcosm.build.us_runtime.puf_full_source::@FULL_SOURCE_VERSION": [], + "microcosm.build.us_runtime.puf_full_source::@MONEY_COLUMNS": [], + "microcosm.build.us_runtime.puf_full_source::@OUTSIDE_AMOUNT_UNIVERSE": [], + "microcosm.build.us_runtime.puf_full_source::@PROJECTED_COLUMNS": [ + "microcosm.build.us_runtime.puf_full_source::@COUNT_COLUMNS", + "microcosm.build.us_runtime.puf_full_source::@MONEY_COLUMNS" + ], + "microcosm.build.us_runtime.puf_full_source::@_AGGREGATE": [], + "microcosm.build.us_runtime.puf_full_source::@_COUNT": [], + "microcosm.build.us_runtime.puf_full_source::@_HEADER_LIMIT": [], + "microcosm.build.us_runtime.puf_full_source::@_MAGIC": [ + "microcosm.build.us_runtime.puf_full_source::@FULL_SOURCE_VERSION" + ], + "microcosm.build.us_runtime.puf_full_source::@_MONEY": [], + "microcosm.build.us_runtime.puf_growth::@PROVENANCE_RECID_COLUMN": [], + "microcosm.build.us_runtime.puf_growth::@PROVENANCE_SOURCE_AGI_COLUMN": [], + "microcosm.build.us_runtime.puf_support::@PUF_TAX_DETAIL_DEFAULT_PREDICTORS": [], + "microcosm.build.us_runtime.puf_support::@PUF_TAX_DETAIL_SOCIAL_SECURITY_COMPONENT_OUTPUTS": [], + "microcosm.build.us_runtime.puf_support::@_PUF_TAX_DETAIL_BOOLEAN_PERSON_OUTPUTS": [ + "microcosm.build.us_runtime.operator_column_contracts::@US_QBI_BOOLEAN_OUTPUT_COLUMNS" + ], + "microcosm.build.us_runtime.puf_support::@_PUF_TAX_DETAIL_DISCRETE_TAX_UNIT_OUTPUTS": [], + "microcosm.build.us_runtime.puf_target2024_growth::@INCIDENCE_FIELDS": [ + "microcosm.build.us_runtime.puf_target2024_growth::@_RECIPE" + ], + "microcosm.build.us_runtime.puf_target2024_growth::@OUTPUTS": [ + "microcosm.build.us_runtime.puf_target2024_growth::@_RECIPE" + ], + "microcosm.build.us_runtime.puf_target2024_growth::@RECIPE_SHA256": [], + "microcosm.build.us_runtime.puf_target2024_growth::@VERSION": [], + "microcosm.build.us_runtime.puf_target2024_growth::@_RECIPE": [], + "microcosm.build.us_runtime.puf_target2024_growth::@_RECIPE_JSON": [], + "microcosm.build.us_runtime.survey_social_security::@COMPONENTS": [], + "microcosm.build.us_runtime.survey_social_security::@PROTOCOL": [], + "microcosm.build.us_runtime.survey_social_security::@REASON_COMPONENTS": [] + }, + "selector_domain_roots": [ + "microcosm.build.us_runtime.current_survey_predictors::@DEMOGRAPHIC_FEATURES", + "microcosm.build.us_runtime.current_survey_predictors::@FEATURES", + "microcosm.build.us_runtime.current_survey_predictors::@MONEY_FIELDS", + "microcosm.build.us_runtime.current_survey_predictors::@OUTPUTS", + "microcosm.build.us_runtime.current_survey_predictors::@PHASE", + "microcosm.build.us_runtime.current_survey_predictors::@PROTOCOL", + "microcosm.build.us_runtime.current_survey_predictors::@SEED", + "microcosm.build.us_runtime.current_survey_predictors::@TARGETS", + "microcosm.build.us_runtime.full_puf_enrichment::@FULL65", + "microcosm.build.us_runtime.full_puf_enrichment::@PERSON_OUTPUTS", + "microcosm.build.us_runtime.full_puf_enrichment::@PHASE", + "microcosm.build.us_runtime.full_puf_enrichment::@PREDICTORS", + "microcosm.build.us_runtime.full_puf_enrichment::@PUF55_SURVEY_SS", + "microcosm.build.us_runtime.full_puf_enrichment::@PUF55_SURVEY_SS_NO_TOTAL", + "microcosm.build.us_runtime.full_puf_enrichment::@PUF55_SURVEY_SS_PERSON_OUTPUTS", + "microcosm.build.us_runtime.full_puf_enrichment::@PUF55_SURVEY_SS_PREDICTORS", + "microcosm.build.us_runtime.full_puf_enrichment::@PUF59", + "microcosm.build.us_runtime.full_puf_enrichment::@PUF59_PREDICTORS", + "microcosm.build.us_runtime.full_puf_enrichment::@PUF59_TAX_UNIT_OUTPUTS", + "microcosm.build.us_runtime.full_puf_enrichment::@SCF_MORTGAGE_OUTPUTS", + "microcosm.build.us_runtime.full_puf_enrichment::@SURVEY_SS_COMPONENTS", + "microcosm.build.us_runtime.full_puf_enrichment::@SURVEY_SS_TOTAL_PREDICTOR", + "microcosm.build.us_runtime.full_puf_enrichment::@TARGETS", + "microcosm.build.us_runtime.full_puf_enrichment::@TAX_UNIT_OUTPUTS", + "microcosm.build.us_runtime.full_puf_enrichment::PufOutputProfile", + "microcosm.build.us_runtime.full_puf_enrichment::_recipient_matrix", + "microcosm.build.us_runtime.full_puf_enrichment::prepare_full_puf_inputs", + "microcosm.build.us_runtime.full_puf_enrichment::require_puf_output_profile", + "microcosm.build.us_runtime.operator_column_contracts::@ACS_DERIVED_TRANSFER_INPUTS", + "microcosm.build.us_runtime.operator_column_contracts::@PUF_CAPITAL_GAINS_TAIL_APPLIED_COLUMN", + "microcosm.build.us_runtime.operator_column_contracts::@PUF_CAPITAL_GAINS_TAIL_DONOR_AGI_BAND_COLUMN", + "microcosm.build.us_runtime.operator_column_contracts::@PUF_CAPITAL_GAINS_TAIL_DONOR_FILING_STATUS_COLUMN", + "microcosm.build.us_runtime.operator_column_contracts::@PUF_CAPITAL_GAINS_TAIL_DONOR_SOURCE_ID_COLUMN", + "microcosm.build.us_runtime.operator_column_contracts::@PUF_CAPITAL_GAINS_TAIL_DONOR_SYNTHETIC_COLUMN", + "microcosm.build.us_runtime.operator_column_contracts::@PUF_CAPITAL_GAINS_TAIL_PERSON_COLUMNS", + "microcosm.build.us_runtime.operator_column_contracts::@PUF_CAPITAL_GAINS_TAIL_TAX_UNIT_COLUMNS", + "microcosm.build.us_runtime.operator_column_contracts::@PUF_CAPITAL_GAINS_TAIL_TRANSFER_WEIGHT_COLUMN", + "microcosm.build.us_runtime.operator_column_contracts::@PUF_SUPPORT_MAX_CLONE_SAFE_SOURCE_ID", + "microcosm.build.us_runtime.operator_column_contracts::@PUF_TAX_DETAIL_DEFAULT_PERSON_OUTPUTS", + "microcosm.build.us_runtime.operator_column_contracts::@PUF_TAX_DETAIL_DEFAULT_TAX_UNIT_OUTPUTS", + "microcosm.build.us_runtime.operator_column_contracts::@US_PUF_SUPPORT_STAGE_NAME", + "microcosm.build.us_runtime.operator_column_contracts::@US_QBI_BOOLEAN_OUTPUT_COLUMNS", + "microcosm.build.us_runtime.operator_column_contracts::@US_QBI_NONNEGATIVE_OUTPUT_COLUMNS", + "microcosm.build.us_runtime.operator_column_contracts::@US_QBI_OUTPUT_COLUMNS", + "microcosm.build.us_runtime.operator_column_contracts::@_GENERAL_QUALIFICATION_FLAGS", + "microcosm.build.us_runtime.operator_column_contracts::@_SSTB_QUALIFICATION_FLAG", + "microcosm.build.us_runtime.puf55_route_finalization::@FINALIZATION_PROTOCOL", + "microcosm.build.us_runtime.puf55_route_finalization::@PROFILES", + "microcosm.build.us_runtime.puf55_route_finalization::@PROTOCOL", + "microcosm.build.us_runtime.puf55_survey_ss_measurement::@COLUMNS", + "microcosm.build.us_runtime.puf55_survey_ss_measurement::@KNOWN", + "microcosm.build.us_runtime.puf55_survey_ss_measurement::@MEASUREMENT", + "microcosm.build.us_runtime.puf55_survey_ss_measurement::@PROTOCOL", + "microcosm.build.us_runtime.puf55_survey_ss_measurement::@ROUTE", + "microcosm.build.us_runtime.puf55_survey_ss_measurement::@TOTAL", + "microcosm.build.us_runtime.puf55_survey_ss_measurement::@_PERSON_COLUMNS", + "microcosm.build.us_runtime.puf55_survey_ss_measurement::@_REPORT_COLUMNS", + "microcosm.build.us_runtime.puf55_survey_ss_measurement::@_STATUSES", + "microcosm.build.us_runtime.puf55_survey_ss_measurement::@_UNIT_COLUMNS", + "microcosm.build.us_runtime.puf59_canonical_artifact::@INTEGERS", + "microcosm.build.us_runtime.puf59_canonical_artifact::@MAGIC", + "microcosm.build.us_runtime.puf59_canonical_artifact::@MAX_BODY", + "microcosm.build.us_runtime.puf59_canonical_artifact::@MAX_HEADER", + "microcosm.build.us_runtime.puf59_canonical_artifact::@NAMES", + "microcosm.build.us_runtime.puf59_canonical_artifact::@PREFIX", + "microcosm.build.us_runtime.puf_detail_transfer::@ARRAY_DTYPES", + "microcosm.build.us_runtime.puf_detail_transfer::@FEATURES", + "microcosm.build.us_runtime.puf_detail_transfer::@HEADER_FIELDS", + "microcosm.build.us_runtime.puf_detail_transfer::@MAGIC", + "microcosm.build.us_runtime.puf_detail_transfer::@MARS", + "microcosm.build.us_runtime.puf_detail_transfer::@MASK", + "microcosm.build.us_runtime.puf_detail_transfer::@MAX_BYTES", + "microcosm.build.us_runtime.puf_detail_transfer::@MAX_ROWS", + "microcosm.build.us_runtime.puf_detail_transfer::@MONEY_FIELDS", + "microcosm.build.us_runtime.puf_detail_transfer::@OUTPUT", + "microcosm.build.us_runtime.puf_detail_transfer::@SCOPE", + "microcosm.build.us_runtime.puf_detail_transfer::@SOURCE_ADMISSION", + "microcosm.build.us_runtime.puf_detail_transfer::@TARGET", + "microcosm.build.us_runtime.puf_full_source::@AGGREGATE_LEXEME_MAX_CHARACTERS", + "microcosm.build.us_runtime.puf_full_source::@COUNT_COLUMNS", + "microcosm.build.us_runtime.puf_full_source::@DEPENDENT_COLUMNS", + "microcosm.build.us_runtime.puf_full_source::@DIRECT_MAPPINGS", + "microcosm.build.us_runtime.puf_full_source::@FULL_SOURCE_MAX_BYTES", + "microcosm.build.us_runtime.puf_full_source::@FULL_SOURCE_VERSION", + "microcosm.build.us_runtime.puf_full_source::@MONEY_COLUMNS", + "microcosm.build.us_runtime.puf_full_source::@OUTSIDE_AMOUNT_UNIVERSE", + "microcosm.build.us_runtime.puf_full_source::@PROJECTED_COLUMNS", + "microcosm.build.us_runtime.puf_full_source::@_AGGREGATE", + "microcosm.build.us_runtime.puf_full_source::@_COUNT", + "microcosm.build.us_runtime.puf_full_source::@_HEADER_LIMIT", + "microcosm.build.us_runtime.puf_full_source::@_MAGIC", + "microcosm.build.us_runtime.puf_full_source::@_MONEY", + "microcosm.build.us_runtime.puf_support::@PUF_TAX_DETAIL_DEFAULT_PREDICTORS", + "microcosm.build.us_runtime.puf_support::@PUF_TAX_DETAIL_SOCIAL_SECURITY_COMPONENT_OUTPUTS", + "microcosm.build.us_runtime.puf_support::@_PUF_TAX_DETAIL_BOOLEAN_PERSON_OUTPUTS", + "microcosm.build.us_runtime.puf_support::@_PUF_TAX_DETAIL_DISCRETE_TAX_UNIT_OUTPUTS", + "microcosm.build.us_runtime.puf_target2024_growth::@INCIDENCE_FIELDS", + "microcosm.build.us_runtime.puf_target2024_growth::@OUTPUTS", + "microcosm.build.us_runtime.puf_target2024_growth::@RECIPE_SHA256", + "microcosm.build.us_runtime.puf_target2024_growth::@VERSION", + "microcosm.build.us_runtime.puf_target2024_growth::@_RECIPE", + "microcosm.build.us_runtime.puf_target2024_growth::@_RECIPE_JSON", + "microcosm.build.us_runtime.survey_social_security::@COMPONENTS", + "microcosm.build.us_runtime.survey_social_security::@PROTOCOL", + "microcosm.build.us_runtime.survey_social_security::@REASON_COMPONENTS" + ] +} diff --git a/packages/microcosm-build/tests/test_us_acs_housing_source.py b/packages/microcosm-build/tests/test_us_acs_housing_source.py index eb5a461c9..4f26743ff 100644 --- a/packages/microcosm-build/tests/test_us_acs_housing_source.py +++ b/packages/microcosm-build/tests/test_us_acs_housing_source.py @@ -431,6 +431,9 @@ def output_dir(self, label="artifact"): return self.tmp_path / f"out-{self._output_index}-{label}" def produce(self, *, output_dir=None, serialnos=None, source_dir=None): + # The authenticated producer binds the optional unit-constructor source + # inventory even though this source-only test constructs no tax units. + pytest.importorskip("microunit", exc_type=ModuleNotFoundError) return source.produce_acs_housing_source( self.source_dir if source_dir is None else source_dir, snapshot_root=self.snapshot_root, @@ -439,6 +442,9 @@ def produce(self, *, output_dir=None, serialnos=None, source_dir=None): ) def load(self, output_dir, *, serialnos=None, source_dir=None): + pytest.importorskip( + "microunit", exc_type=ModuleNotFoundError + ) # Same attested producer closure as produce. return source.load_acs_housing_source( self.source_dir if source_dir is None else source_dir, output_dir, diff --git a/packages/microcosm-build/tests/test_us_acs_person_coverage_authentication.py b/packages/microcosm-build/tests/test_us_acs_person_coverage_authentication.py index ea76a97ab..23de8f93b 100644 --- a/packages/microcosm-build/tests/test_us_acs_person_coverage_authentication.py +++ b/packages/microcosm-build/tests/test_us_acs_person_coverage_authentication.py @@ -52,6 +52,9 @@ def _csv(rows, *, bom=False, ending="\n"): @pytest.fixture def invented(tmp_path, monkeypatch): + pytest.importorskip( + "microunit", exc_type=ModuleNotFoundError + ) # This fixture builds actual US tax units. monkeypatch.setattr( custody.shutil, "disk_usage", lambda _p: SimpleNamespace(free=64 * 1024**3) ) diff --git a/packages/microcosm-build/tests/test_us_acs_population_catalogue.py b/packages/microcosm-build/tests/test_us_acs_population_catalogue.py index ab98b77dc..2ee18caa1 100644 --- a/packages/microcosm-build/tests/test_us_acs_population_catalogue.py +++ b/packages/microcosm-build/tests/test_us_acs_population_catalogue.py @@ -32,6 +32,9 @@ def csv_bytes(rows): @pytest.fixture def invented(tmp_path, monkeypatch): + # Source issuance authenticates the optional unit-constructor source tree. + # Resolve that dependency before tests install their no-construction traces. + pytest.importorskip("microunit", exc_type=ModuleNotFoundError) source, snapshots = tmp_path / "source", tmp_path / "snapshots" source.mkdir() snapshots.mkdir() diff --git a/packages/microcosm-build/tests/test_us_acs_source_compile_cache.py b/packages/microcosm-build/tests/test_us_acs_source_compile_cache.py index 6d108fe7a..db6c5a75b 100644 --- a/packages/microcosm-build/tests/test_us_acs_source_compile_cache.py +++ b/packages/microcosm-build/tests/test_us_acs_source_compile_cache.py @@ -270,6 +270,9 @@ def test_live_check_reads_source_twice_even_after_warming(tmp_path, monkeypatch) def test_real_producer_keeps_read_counts_and_full_evidence_on_warm_cache(): + pytest.importorskip( + "microunit", exc_type=ModuleNotFoundError + ) # Included in the real ACS producer closure. # Settle unrelated import/manifest memoization, then compare only compile # cache cold versus warm with the real producer and unchanged source files. native._producer() diff --git a/packages/microcosm-build/tests/test_us_acs_transfer.py b/packages/microcosm-build/tests/test_us_acs_transfer.py index e9e615bec..5c124dcfc 100644 --- a/packages/microcosm-build/tests/test_us_acs_transfer.py +++ b/packages/microcosm-build/tests/test_us_acs_transfer.py @@ -1467,9 +1467,8 @@ def test_pregnancy_draws_once_per_eligible_source_person_and_fans_to_clones( ) person = result.frame.person - eligible = ( - person["is_female"].astype(bool) - & person["age"].between(15, 44, inclusive="both") + eligible = person["is_female"].astype(bool) & person["age"].between( + 15, 44, inclusive="both" ) assert person.loc[eligible, "is_pregnant"].all() assert not person.loc[~eligible, "is_pregnant"].any() @@ -1490,7 +1489,9 @@ def test_pregnancy_draws_once_per_eligible_source_person_and_fans_to_clones( .first() .sum() ) - record = next(item for item in result.imputed_inputs if item.column == "is_pregnant") + record = next( + item for item in result.imputed_inputs if item.column == "is_pregnant" + ) receipt = record.structural_receipt assert receipt is not None assert sum(pattern.recipient_rows for pattern in record.patterns) == ( @@ -1576,12 +1577,8 @@ def test_pregnancy_partial_clone_fanout_receipt_categories_are_disjoint( record = result.imputed_inputs[0] receipt = record.structural_receipt assert receipt is not None - assert receipt["preexisting_value_fanout_rows"] == int( - (missing & eligible).sum() - ) - assert receipt["ineligible_rows_assigned_false"] == int( - (missing & ~eligible).sum() - ) + assert receipt["preexisting_value_fanout_rows"] == int((missing & eligible).sum()) + assert receipt["ineligible_rows_assigned_false"] == int((missing & ~eligible).sum()) assert ( receipt["preexisting_value_fanout_rows"] + receipt["ineligible_rows_assigned_false"] @@ -2467,8 +2464,11 @@ def test_strict_leaf_audit_reports_missing_us_extra( ) -> None: from microcosm.frame.adapters import policyengine_us as adapter_module + consumer_modes: list[bool] = [] + class _MissingMetadataIndex: - def __init__(self) -> None: + def __init__(self, *, include_consumers: bool = True) -> None: + consumer_modes.append(include_consumers) raise ImportError("policyengine-us is absent") monkeypatch.setattr( @@ -2485,6 +2485,7 @@ def __init__(self) -> None: {"employment_income"}, require_known=True, ) + assert consumer_modes == [False, True] def test_all_missing_donor_target_is_refused_without_zero_fill() -> None: diff --git a/packages/microcosm-build/tests/test_us_asec_checkpoint.py b/packages/microcosm-build/tests/test_us_asec_checkpoint.py index 8ea3569cc..ba6adcf21 100644 --- a/packages/microcosm-build/tests/test_us_asec_checkpoint.py +++ b/packages/microcosm-build/tests/test_us_asec_checkpoint.py @@ -118,7 +118,7 @@ def test_restored_raw_observations_carry_to_input_leaves_without_engine() -> Non ) -def test_v4_restoration_producer_authenticates_source_and_writes_new_bundle( +def test_pinned_coverage_loader_and_exact_join_roundtrip_through_v4_codec( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, ) -> None: @@ -126,9 +126,11 @@ def test_v4_restoration_producer_authenticates_source_and_writes_new_bundle( from dataclasses import replace from microcosm.build.us_runtime import reported_coverage_source - from microcosm.build.us_runtime.asec_raw_stage_v4 import restore_asec_raw_stage_v4 legacy = _raw_us_frame() + legacy.person["source_household_id"] = [1, 1] + legacy.person["P_SEQ"] = [1, 1] + legacy.person["A_LINENO"] = [1, 1] binding = _raw_binding(legacy) binding["source_receipt"]["sources"].append( dict(binding["source_receipt"]["sources"][0], year=2023) @@ -169,53 +171,86 @@ def test_v4_restoration_producer_authenticates_source_and_writes_new_bundle( monkeypatch.setattr( reported_coverage_source, "ASEC_EDUCATION_ASSISTANCE_ARCHIVES", pins ) - output_dir = tmp_path / "restored" - receipt = restore_asec_raw_stage_v4( - input_path, - expected_sha256=input_sha, - coverage_paths=paths, - output_dir=output_dir, + source = reported_coverage_source.load_asec_reported_coverage_sources( + paths, income_years=(2022, 2023) ) - assert receipt["input_sha256"] == input_sha - output = output_dir / "asec_raw_stage.checkpoint.h5" - assert receipt["output_sha256"] == hashlib.sha256(output.read_bytes()).hexdigest() - assert hashlib.sha256(input_path.read_bytes()).hexdigest() == input_sha - restored, metadata = checkpoint_module.load_asec_raw_stage_checkpoint_v4(output) - assert restored.table("person")["NOW_MCAID"].tolist() == [1, 2] + loaded_legacy, _ = load_asec_raw_stage_checkpoint(input_path) + person = reported_coverage_source.fill_asec_reported_coverage_source( + loaded_legacy.person, source.iloc[::-1] + ) + restored = Frame( + { + entity: person if entity == "person" else loaded_legacy.table(entity) + for entity in loaded_legacy.entities + }, + loaded_legacy.schema, + { + entity: loaded_legacy.weights_for(entity) + for entity in loaded_legacy.weighted_entities + }, + loaded_legacy.strata, + ) + # The maintained codec consumes an explicitly attested v4 artifact. This + # test supplies the attestation from the real pinned reader's audit; it + # does not claim an additional production restoration/bundle writer exists. + v4_binding = _v4_binding(restored) + v4_binding["source_receipt"] = binding["source_receipt"] + for column in checkpoint_module.ASEC_REPORTED_COVERAGE_RAW_COLUMNS: + v4_binding["raw_source_mappings"][column]["audit"] = { + str(year): { + "rows": audit["rows"], + **audit["columns"][column], + } + for year, audit in source.attrs["source_audit"].items() + } + output = tmp_path / "coverage-v4.h5" + write_frame_checkpoint(output, restored, metadata=v4_binding) + reloaded, metadata = checkpoint_module.load_asec_raw_stage_checkpoint_v4(output) + for column in checkpoint_module.ASEC_REPORTED_COVERAGE_RAW_COLUMNS: + assert reloaded.person[column].tolist() == [1, 2] + assert reloaded.person[column].dtype == np.dtype("int64") assert metadata["source_receipt"] == binding["source_receipt"] - assert json.loads((output_dir / "restoration.receipt.json").read_text()) == receipt - with pytest.raises(FileExistsError): - restore_asec_raw_stage_v4( - input_path, - expected_sha256=input_sha, - coverage_paths=paths, - output_dir=output_dir, + pd.testing.assert_frame_equal( + reloaded.person.drop( + columns=list(checkpoint_module.ASEC_REPORTED_COVERAGE_RAW_COLUMNS) + ), + loaded_legacy.person, + ) + for entity in loaded_legacy.schema.group_entities: + pd.testing.assert_frame_equal( + reloaded.table(entity), loaded_legacy.table(entity) + ) + assert frame_identity(loaded_legacy) == frame_identity(legacy) + assert hashlib.sha256(input_path.read_bytes()).hexdigest() == input_sha + + with pytest.raises(ValueError, match="does not cover pooled income year"): + reported_coverage_source.fill_asec_reported_coverage_source( + loaded_legacy.person, source.loc[source.source_year.eq(2022)] + ) + inconsistent = source.copy() + inconsistent.loc[inconsistent.source_year.eq(2022), "PH_SEQ"] = 99 + with pytest.raises(ValueError, match="redundant identity mismatch"): + reported_coverage_source.fill_asec_reported_coverage_source( + loaded_legacy.person, inconsistent ) - with pytest.raises(ValueError, match="SHA-256"): - restore_asec_raw_stage_v4( - input_path, - expected_sha256="f" * 64, - coverage_paths=paths, - output_dir=tmp_path / "bad", + with pytest.raises(FileNotFoundError): + reported_coverage_source.load_asec_reported_coverage_sources( + {2022: paths[2022], 2023: tmp_path / "missing.csv"}, + income_years=(2022, 2023), ) - with pytest.raises(ValueError, match="local coverage paths"): - restore_asec_raw_stage_v4( - input_path, - expected_sha256=input_sha, - coverage_paths={2022: paths[2022]}, - output_dir=tmp_path / "missing", + original = paths[2022].read_bytes() + changed = bytearray(original) + changed[-4] ^= 1 + paths[2022].write_bytes(changed) + with pytest.raises(ValueError, match="SHA-256 mismatch"): + reported_coverage_source.load_asec_reported_coverage_sources( + paths, income_years=(2022, 2023) ) paths[2022].write_text("tampered") with pytest.raises(ValueError, match="byte length mismatch"): - restore_asec_raw_stage_v4( - input_path, - expected_sha256=input_sha, - coverage_paths=paths, - output_dir=tmp_path / "tampered", + reported_coverage_source.load_asec_reported_coverage_sources( + paths, income_years=(2022, 2023) ) - assert not (tmp_path / "bad").exists() - assert not (tmp_path / "missing").exists() - assert not (tmp_path / "tampered").exists() @pytest.mark.parametrize( diff --git a/packages/microcosm-build/tests/test_us_asec_coverage_authentication.py b/packages/microcosm-build/tests/test_us_asec_coverage_authentication.py index 22ee44b3c..ac53095e9 100644 --- a/packages/microcosm-build/tests/test_us_asec_coverage_authentication.py +++ b/packages/microcosm-build/tests/test_us_asec_coverage_authentication.py @@ -4,6 +4,7 @@ import csv import hashlib import importlib.util +import inspect import json import os import struct @@ -11,6 +12,7 @@ import sys import textwrap import traceback +from contextlib import contextmanager from dataclasses import FrozenInstanceError, replace from pathlib import Path from types import SimpleNamespace @@ -103,8 +105,8 @@ def _fixtures( "test_us_asec_person_income_source" ).invented_sources(tmp_path, monkeypatch, missing=False) if native_ids is not None: - _helper("test_us_asec_demographic_source")._repoint_published_households( - parent, attachment, monkeypatch, native_ids + _changed_parent( + parent, attachment, monkeypatch, {"source_household_id": native_ids} ) if person_changes: _changed_parent(parent, attachment, monkeypatch, person_changes) @@ -312,8 +314,11 @@ def test_merged_native_households_refuse_even_with_matching_per_row_coordinates( ): # The current-money parent already refuses this shape, before coverage # issuance. Keep that layering and also test the wrapper's own fence. - with pytest.raises(ValueError, match="SOURCE_CONTRACT_REFUSAL"): + with pytest.raises(ValueError, match="SOURCE_CONTRACT_REFUSAL") as error: _fixtures(tmp_path, monkeypatch, native_ids=[7, 8, 7, 7, 7, 7]) + assert str(error.value.__context__) == ( + "ASEC persons disagree on the source key within a household." + ) person = pd.DataFrame( { "source_year": [2022, 2022], @@ -901,14 +906,54 @@ def child(name): issued.validate() +@contextmanager +def transient_csv_rebinding(target, aliases, moment): + """Attack real CSV aliases at the maintained reader's entry or call line.""" + source, first = inspect.getsourcelines(target) + call_line = next( + first + i for i, line in enumerate(source) if "csv_reader(handle," in line + ) + original, native, fired, calls = csv.reader, _csv.reader, [], [] + + def restore(): + csv.reader, _csv.reader = original, native + + def replacement(*args, **kwargs): + calls.append(True) + restore() + return original(*args, **kwargs) + + def timing(frame, event, arg): + if frame.f_code is target.__code__: + trigger = (moment == "entry" and event == "call") or ( + moment == "before_call" + and event == "line" + and frame.f_lineno == call_line + ) + if trigger and not fired: + fired.append(True) + csv.reader = replacement + if aliases == "both": + _csv.reader = replacement + if event == "return": + restore() + return timing + + previous = sys.gettrace() + sys.settrace(timing) + try: + yield fired, calls + finally: + sys.settrace(previous) + restore() + + @pytest.mark.parametrize("aliases", ["csv", "both"]) @pytest.mark.parametrize("moment", ["entry", "before_call"]) @pytest.mark.parametrize("stage", ["literal", "header"]) def test_transient_csv_rebinding_never_supplies_person_cells( tmp_path, monkeypatch, aliases, moment, stage ): - from test_us_asec_household_coverage_fields import transient_csv_rebinding - source, paths = _fixtures(tmp_path, monkeypatch) control = _read(source, paths) coverage.verify_asec_coverage_parent(control, source) diff --git a/packages/microcosm-build/tests/test_us_multispine_pool_tool.py b/packages/microcosm-build/tests/test_us_multispine_pool_tool.py index fc076a070..a540d56ab 100644 --- a/packages/microcosm-build/tests/test_us_multispine_pool_tool.py +++ b/packages/microcosm-build/tests/test_us_multispine_pool_tool.py @@ -28,7 +28,7 @@ from microcosm.build.gates import GateReport, GateResult from microcosm.build.logbook import LOGBOOK_ROW_FIELDS, load_logbook_row from microcosm.build.serialization_dtypes import CANONICAL_STRING_DTYPE -from microcosm.build.spec_engine import LegacyPayloadMismatchError +from microcosm.build.spec_engine import LegacyPayloadMismatchError, load_bundle from microcosm.build.us_runtime.acs_transfer import transfer_acs_inputs from microcosm.build.us_runtime.acs_transfer_bank import ( ACS_TRANSFER_TARGET_BANK_MATERIALIZER_VERSION, @@ -2449,6 +2449,7 @@ def capture_equality(expected: object, actual: object) -> None: capture_equality, ) + current_spec_sha256 = load_bundle("us").spec_sha256 # This call performs the real bundle load, compilation, and field-complete # equality assertion against the live generation-0 constructors. run_config = pool_tool._stacked_run_config(args) @@ -2514,7 +2515,7 @@ def capture_equality(expected: object, actual: object) -> None: "country": "us", "schema_id": "country_spec", "schema_version": 1, - "spec_sha256": "35a02b6b19c921faba1407d441e0b9d9623c496e2cd5b711be014def281a95c6", + "spec_sha256": current_spec_sha256, }, } diff --git a/packages/microcosm-build/tests/test_us_spine_blindness.py b/packages/microcosm-build/tests/test_us_spine_blindness.py index de6f713fc..2a0ee427d 100644 --- a/packages/microcosm-build/tests/test_us_spine_blindness.py +++ b/packages/microcosm-build/tests/test_us_spine_blindness.py @@ -27,6 +27,22 @@ modules' executable dataflow; true docstrings and annotation forms are deliberately exempt. +Registered population treatments retain that raw origin-blind contract. Source +authentication/composition, source-qualified pre-PUF model inputs and donor +selection, and exact origin attachments are different phases: the reviewed +fixture accounts for particular findings in particular function bodies there. +It grants no new whole-module provenance exemption. Opaque numeric/profile +selectors likewise have exact findings and finite-domain/validator bindings; +their contracts do not permit source-origin access. + +The separate stage-reference tripwire binds supported local, qualified, +import-alias and simple-assignment references, their enclosing caller bodies, +and recursive entry chains. An unresolved US re-export is conservatively a +same-name candidate, recorded as such for review. This is a syntax boundary, +not whole-program reachability: arbitrary runtime dispatch, reflection and +mutation of imported namespaces still require review. Binding caller bodies +does protect changed selection/control flow around an unchanged supported call. + Analysis is MODULE-LOCAL with single-hop name resolution. Three classes are out of scope by design, and naming them is the honest boundary. First, cross-module static dataflow -- constant tables imported from @@ -56,7 +72,11 @@ import ast import fnmatch +import hashlib +import json import re +from collections import Counter +from functools import cache from itertools import product from pathlib import Path from string import Formatter @@ -80,6 +100,85 @@ "us_late_producer_registry.py", } ) +# Reviewed pool-tool import closure. Exact membership catches a replacement +# module even when the total count stays unchanged. +_EXPECTED_POOL_RUNTIME_MODULES = frozenset( + { + "_person_signal_summary.py", + "acs_income_universe.py", + "acs_inputs.py", + "acs_pums.py", + "acs_sources.py", + "acs_transfer.py", + "acs_transfer_bank.py", + "adult_care.py", + "alimony.py", + "asec_checkpoint.py", + "capital_gain_details.py", + "capital_gain_distributions.py", + "casualty_losses.py", + "child_support.py", + "childcare.py", + "congressional_district_geography.py", + "congressional_district_vintage.py", + "cps_carried.py", + "disability_benefits.py", + "domestic_production.py", + "education_assistance_source.py", + "education_inputs.py", + "educator_expenses.py", + "eligibility_inputs.py", + "energy_subsidy.py", + "farm_business_income.py", + "form_4952.py", + "geography_ladder.py", + "h5_io.py", + "hours_worked.py", + "housing_inputs.py", + "immigration.py", + "late_producer_dag.py", + "medicare_take_up.py", + "misc_itemized.py", + "multispine_pool.py", + "operator_boundary.py", + "operator_column_contracts.py", + "post_transfer_calibration.py", + "pregnancy.py", + "prior_year_income.py", + "public_assistance_type_source.py", + "puf_aggregate_records.py", + "puf_capital_gains_tail.py", + "puf_donor_io.py", + "puf_e01000_reconciliation.py", + "puf_interest_components.py", + "puf_qrf_chain.py", + "puf_source_agi.py", + "puf_support.py", + "puma_ladder.py", + "puma_ladder_sources.py", + "qbi_inputs.py", + "relationship_inputs.py", + "reported_coverage_source.py", + "retirement_contributions.py", + "retirement_distributions.py", + "salt_refund_income.py", + "scf_wealth.py", + "sipp_financial_assets.py", + "spine_agreement.py", + "spine_assembly.py", + "stacked_battery_contract.py", + "stacked_spine.py", + "support_provenance.py", + "take_up.py", + "take_up_contract.py", + "us_late_overlap_ownership.py", + "us_late_producer_registry.py", + "weeks_unemployed.py", + "wic_claim.py", + "worker_identity.py", + "workers_compensation.py", + } +) _RETIRED_LATE_ASSEMBLY_MODULES = frozenset( { "acs_multispine.py", @@ -217,6 +316,7 @@ _OTHER_US_RUNTIME_MODULES = frozenset( { "__init__.py", + "_person_signal_summary.py", # Gate-evidence summaries, no row treatment. # Exact source-universe validator/receipt owner; no population treatment. "acs_income_universe.py", "acs_inputs.py", @@ -252,6 +352,7 @@ "misc_itemized.py", "nonzero_shares.py", "operator_boundary.py", # Raw-stage validator; no population treatment. + "operator_column_contracts.py", # Literal stage input/output declarations. "org_wages.py", "parity_reference.py", "pregnancy.py", @@ -272,6 +373,7 @@ "reform_validation.py", "register_consistency.py", "relationship_inputs.py", + "reported_coverage_source.py", # Pinned exact native-coverage restoration. "release_gate_preflight.py", "release_input_coverage.py", "release_target_parity.py", @@ -310,8 +412,12 @@ "worker_identity.py", # Portable primary-QRF worker identity; no population treatment. } ) +_STAGE_CONTRACTS_PATH = ( + Path(__file__).with_name("fixtures") / "us_spine_stage_contracts.json" +) +_STAGE_CONTRACTS = json.loads(_STAGE_CONTRACTS_PATH.read_text()) _CLASSIFIED_US_RUNTIME_MODULES = frozenset(_SPINE_BLIND_OPERATOR_MODULES).union( - _OTHER_US_RUNTIME_MODULES + _OTHER_US_RUNTIME_MODULES, _STAGE_CONTRACTS["module_roles"] ) @@ -3093,15 +3199,483 @@ def _source_spine_accesses(source: str) -> tuple[str, ...]: ) +def _stage_ast_value(value): + """Portable syntax identity, excluding locations and absent grammar fields.""" + if isinstance(value, ast.AST): + return [ + type(value).__name__, + { + name: _stage_ast_value(item) + for name, item in ast.iter_fields(value) + if item is not None and item != [] + }, + ] + if isinstance(value, list): + return [_stage_ast_value(item) for item in value] + if isinstance(value, (str, int, bool)) or value is None: + return value + return [type(value).__name__, repr(value)] + + +def _stage_ast_sha(node): + return hashlib.sha256( + json.dumps( + _stage_ast_value(node), sort_keys=True, separators=(",", ":") + ).encode() + ).hexdigest() + + +@cache +def _stage_nodes(source): + tree = ast.parse(source) + result = {"": tree} + + def visit(node, parents): + if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef)): + parents = (*parents, node.name) + result[".".join(parents)] = node + elif not parents and isinstance(node, (ast.Assign, ast.AnnAssign)): + targets = node.targets if isinstance(node, ast.Assign) else [node.target] + for target in targets: + if isinstance(target, ast.Name): + result["@" + target.id] = node + for child in ast.iter_child_nodes(node): + visit(child, parents) + + visit(tree, ()) + return result + + +def _stage_scope(nodes, node): + candidates = [ + (name, candidate) + for name, candidate in nodes.items() + if isinstance(candidate, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef)) + and candidate.lineno <= node.lineno <= candidate.end_lineno + ] + return max( + candidates, key=lambda row: row[0].count("."), default=("", None) + )[0] + + +@cache +def _stage_findings(source): + nodes = _stage_nodes(source) + tree = nodes[""] + located = {} + for node in ast.walk(tree): + if hasattr(node, "lineno"): + located.setdefault((node.lineno, node.col_offset + 1), []).append(node) + result = [] + for finding in _source_spine_accesses(source): + match = re.fullmatch(r"line (\d+):(\d+): (.*)", finding) + assert match is not None, finding + line, column, kind = int(match[1]), int(match[2]), match[3] + expected = ast.Subscript if kind.startswith("subscript") else ast.Call + positioned = located.get((line, column), ()) + candidates = [node for node in positioned if isinstance(node, expected)] + candidates = candidates or list(positioned) + assert candidates, finding + # Chained subscripts can share a source location. Bind the enclosing + # expression as well as its descendants instead of guessing which + # same-position node the conservative raw visitor reported. + node = max(candidates, key=lambda item: len(tuple(ast.walk(item)))) + result.append( + { + "scope": _stage_scope(nodes, node), + "kind": kind, + "expression_sha256": _stage_ast_sha(node), + "expression": ast.unparse(node), + "raw": finding, + } + ) + return result + + +def _stage_finding_key(finding): + return finding["kind"], finding["expression_sha256"] + + +def _stage_module_name(path): + parts = path.parts + if "src" in parts: + parts = parts[parts.index("src") + 1 :] + else: + parts = path.relative_to(_REPOSITORY_ROOT).parts + result = ".".join((*parts[:-1], Path(parts[-1]).stem)) + return result.removesuffix(".__init__") + + +def _stage_repository_sources(): + paths = [ + *(_REPOSITORY_ROOT / "packages").glob("*/src/**/*.py"), + *(_REPOSITORY_ROOT / "tools").rglob("*.py"), + ] + return {_stage_module_name(path): path.read_text() for path in sorted(paths)} + + +def _stage_import_aliases(module, source): + aliases = {} + for node in ast.walk(_stage_nodes(source)[""]): + if isinstance(node, ast.Import): + for item in node.names: + aliases.setdefault(item.asname or item.name.split(".")[0], set()).add( + item.name if item.asname else item.name.split(".")[0] + ) + elif isinstance(node, ast.ImportFrom): + prefix = node.module or "" + if node.level: + prefix = ".".join( + module.split(".")[: -node.level] + ([prefix] if prefix else []) + ) + for item in node.names: + if item.name != "*": + aliases.setdefault(item.asname or item.name, set()).add( + prefix + "." + item.name + ) + return aliases + + +def _stage_domain_dependencies(sources, roots): + """Bind finite constants through supported local/imported uppercase aliases. + + Constant declarations are syntax-only roots, not callable scopes. Their + actual defining owners and alias dependency graph are checked separately + from the entry/caller graph. This does not evaluate arbitrary Python values. + """ + + def definition(label, seen=frozenset()): + if label in seen: + return set() + module, _, name = label.rpartition(".") + if module not in sources or not name.isupper(): + return set() + if "@" + name in _stage_nodes(sources[module]): + return {module + "::@" + name} + result = set() + for alias in _stage_import_aliases(module, sources[module]).get(name, ()): + result.update(definition(alias, seen | {label})) + return result + + result = {} + pending = list(roots) + while pending: + key = pending.pop() + if key in result: + continue + module, scope = key.split("::", 1) + node = _stage_nodes(sources[module]).get(scope) if module in sources else None + if node is None: + result[key] = () + continue + aliases = _stage_import_aliases(module, sources[module]) + + def labels(value, *, module=module, aliases=aliases): + if isinstance(value, ast.Name): + return {module + "." + value.id, *aliases.get(value.id, ())} + if isinstance(value, ast.Attribute): + return {label + "." + value.attr for label in labels(value.value)} + return set() + + dependencies = set() + for value in ast.walk(node): + if isinstance(value, (ast.Name, ast.Attribute)) and isinstance( + value.ctx, ast.Load + ): + for label in labels(value): + dependencies.update(definition(label)) + dependencies.discard(key) + result[key] = tuple(sorted(dependencies)) + pending.extend(dependencies - result.keys()) + return dict(sorted(result.items())) + + +@cache +def _stage_reference_syntax(source): + """Share immutable syntax across the reviewed closure and guard checks.""" + nodes = _stage_nodes(source) + walked = tuple(ast.walk(nodes[""])) + return ( + nodes, + walked, + {id(child): node for node in walked for child in ast.iter_child_nodes(node)}, + { + id(node): _stage_scope(nodes, node) + for node in walked + if hasattr(node, "lineno") + }, + ) + + +def _stage_reference_inventory(sources, targets): + """Conservative local/import/qualified references, including simple aliases. + + Unresolved US namespace hops are candidate references by their final name. + Arbitrary runtime dispatch/obfuscated getattr and whole-program dataflow remain + outside this tripwire. Class construction binds reviewed kernel run entries; + generic executor ``kernel.run`` dispatch does not identify a particular class. + """ + labels = { + key.replace("::", "."): key + for key in targets + if not key.split("::", 1)[1].startswith("@") + } + by_leaf = {} + constructors = {} + for label, key in labels.items(): + by_leaf.setdefault(label.rsplit(".", 1)[-1], set()).add(key) + if key.split("::", 1)[1].endswith(".run"): + constructors[label.rsplit(".", 1)[0]] = key + known_definitions = { + module + "." + name + for module, source in sources.items() + for name, node in _stage_nodes(source).items() + if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef)) + } + result = {key: [] for key in targets} + for module, source in sorted(sources.items()): + nodes, walked, parents, scopes = _stage_reference_syntax(source) + local_labels = {module + "." + name for name in nodes} + aliases = {} + for node in walked: + if isinstance(node, ast.Import): + for item in node.names: + aliases.setdefault( + item.asname or item.name.split(".")[0], set() + ).add(item.name if item.asname else item.name.split(".")[0]) + elif isinstance(node, ast.ImportFrom): + prefix = node.module or "" + if node.level: + prefix = ".".join( + module.split(".")[: -node.level] + ([prefix] if prefix else []) + ) + for item in node.names: + if item.name != "*": + aliases.setdefault(item.asname or item.name, set()).add( + prefix + "." + item.name + ) + + def resolve(node, scope, *, aliases=aliases, nodes=nodes, module=module): + if isinstance(node, ast.Name): + values = set(aliases.get(node.id, ())) + if node.id in {"self", "cls"}: + pieces = scope.split(".") + for stop in range(len(pieces), 0, -1): + name = ".".join(pieces[:stop]) + if isinstance(nodes.get(name), ast.ClassDef): + values.add(module + "." + name) + break + pieces = scope.split(".") if scope != "" else [] + for stop in range(len(pieces), -1, -1): + name = ".".join((*pieces[:stop], node.id)) + if name in nodes: + values.add(module + "." + name) + break + return values + if isinstance(node, ast.Attribute): + return {value + "." + node.attr for value in resolve(node.value, scope)} + if isinstance(node, ast.NamedExpr): + return resolve(node.value, scope) + return set() + + # Monotone alias propagation over-approximates rebinding across scopes. + # This intentionally prefers an extra review over losing a possible call. + for _ in range(len(nodes) + 1): + changed = False + for node in walked: + if not isinstance(node, (ast.Assign, ast.AnnAssign, ast.NamedExpr)): + continue + value = getattr(node, "value", None) + if value is None: + continue + names = node.targets if isinstance(node, ast.Assign) else [node.target] + resolved = resolve(value, scopes[id(node)]) + resolved = { + item + for item in resolved + if item.count(".") < 16 + and ( + item.startswith(_US_RUNTIME_IMPORT_PREFIX + ".") + or any( + label == item or label.startswith(item + ".") + for label in labels + ) + ) + } + for name in names: + if isinstance(name, ast.Name): + prior = aliases.setdefault(name.id, set()) + if not resolved <= prior: + prior.update(resolved) + changed = True + if not changed: + break + for node in walked: + if not isinstance(node, (ast.Name, ast.Attribute)) or not isinstance( + node.ctx, ast.Load + ): + continue + scope = scopes[id(node)] + resolved = resolve(node, scope) + matches = {labels[value] for value in resolved if value in labels} + resolutions = {key: "resolved" for key in matches} + parent = parents.get(id(node)) + is_call = isinstance(parent, ast.Call) and parent.func is node + if is_call: + constructed = { + constructors[value] for value in resolved if value in constructors + } + matches.update(constructed) + resolutions.update({key: "kernel-construction" for key in constructed}) + unknown_us = { + value + for value in resolved + if value.startswith(_US_RUNTIME_IMPORT_PREFIX + ".") + and value not in local_labels + and value not in known_definitions + } + if not matches and unknown_us: + # A statically unresolved US re-export is a possible reference, + # never evidence that the helper is unreachable. A known foreign + # definition does not borrow an unrelated same-name contract; + # nor can it mask another unresolved candidate in an alias set. + for value in unknown_us: + leaf = value.rsplit(".", 1)[-1] + if leaf != "run": + matches.update(by_leaf.get(leaf, ())) + resolutions.update({key: "unresolved-us-candidate" for key in matches}) + if not matches: + continue + expression = parent if is_call else node + record = { + "caller": module + "::" + scope, + "usage": "call" if is_call else "reference", + "expression_sha256": _stage_ast_sha(expression), + "expression": ast.unparse(expression), + } + for target in matches: + result[target].append({**record, "resolution": resolutions[target]}) + return { + target: sorted( + rows, + key=lambda row: (row["caller"], row["usage"], row["expression_sha256"]), + ) + for target, rows in result.items() + } + + +def _stage_reference_key(row): + return row["caller"], row["usage"], row["expression_sha256"], row["resolution"] + + +def _stage_contract_failures(sources, document): + """Validate exact entry/caller chains and finite-domain validator bindings.""" + errors = [] + if document.get("schema_version") != 1: + errors.append("unsupported stage-contract schema") + targets = set(document["scopes"]) | set(document["bindings"]) + if set(document["scopes"]) & set(document["bindings"]): + errors.append("overlapping access and supporting contracts") + domains = document.get("selector_domain_bindings", ()) + if not domains or len(domains) != len(set(domains)) or not set(domains) <= targets: + errors.append("missing, repeated or unbound selector domains") + roots = document.get("selector_domain_roots", ()) + dependencies = _stage_domain_dependencies(sources, roots) + if not roots or len(roots) != len(set(roots)) or not set(roots) <= set(domains): + errors.append("missing, repeated or unbound selector roots") + if set(dependencies) != set(domains) or dependencies != { + key: tuple(value) + for key, value in document.get("selector_domain_dependencies", {}).items() + }: + errors.append("changed finite-domain owner/dependencies") + references = _stage_reference_inventory(sources, targets) + reachable = set(document["scopes"]) | set(roots) + pending = list(reachable) + while pending: + key = pending.pop() + linked = set(dependencies.get(key, ())) | { + row["caller"] for row in references.get(key, ()) + } + additions = linked - reachable + reachable.update(additions) + pending.extend(additions) + if targets - reachable: + errors.append( + "orphan supporting contracts: " + ", ".join(sorted(targets - reachable)) + ) + for key, contract in {**document["bindings"], **document["scopes"]}.items(): + module, scope = key.split("::", 1) + node = _stage_nodes(sources[module]).get(scope) if module in sources else None + if node is None: + errors.append("missing scope: " + key) + continue + if _stage_ast_sha(node) != contract["body_sha256"]: + errors.append("changed body/domain: " + key) + if Counter(map(_stage_reference_key, references[key])) != Counter( + map(_stage_reference_key, contract["references"]) + ): + errors.append("changed callers: " + key) + if any(row["caller"] not in targets for row in references[key]): + errors.append("unbound caller body: " + key) + if not contract.get("basis"): + errors.append("missing review basis: " + key) + if key in document["scopes"]: + if contract.get("access") not in { + "selector", + "provenance", + } or not contract.get("role"): + errors.append("missing reviewed phase/access role: " + key) + actual = [ + row for row in _stage_findings(sources[module]) if row["scope"] == scope + ] + if Counter(map(_stage_finding_key, actual)) != Counter( + map(_stage_finding_key, contract["findings"]) + ): + errors.append("changed or stale findings: " + key) + if contract["access"] == "selector" and any( + "unresolvable" not in row["kind"] for row in actual + ): + errors.append("selector contract cannot grant provenance: " + key) + return tuple(errors) + + def _non_owner_source_spine_accesses( module_name: str, source: str, ) -> tuple[str, ...]: - """Apply the guard unless the module is a reviewed provenance owner.""" + """Apply raw scanning, accounting only for exact reviewed stage findings.""" if module_name in _SOURCE_SPINE_PROVENANCE_OWNERS: return () - return _source_spine_accesses(source) + relevant = { + key.split("::", 1)[1]: contract + for key, contract in _STAGE_CONTRACTS["scopes"].items() + if key.split("::", 1)[0] + == _US_RUNTIME_IMPORT_PREFIX + "." + module_name.removesuffix(".py") + } + if not relevant: + return _source_spine_accesses(source) + nodes = _stage_nodes(source) + findings = _stage_findings(source) + remaining = [] + for scope, contract in relevant.items(): + actual = [row for row in findings if row["scope"] == scope] + if ( + scope not in nodes + or _stage_ast_sha(nodes[scope]) != contract["body_sha256"] + ): + remaining.append("changed or missing reviewed stage: " + scope) + elif Counter(map(_stage_finding_key, actual)) != Counter( + map(_stage_finding_key, contract["findings"]) + ): + remaining.append("changed or stale reviewed findings: " + scope) + elif contract["access"] == "selector" and any( + "unresolvable" not in row["kind"] for row in actual + ): + remaining.append("selector contract cannot grant provenance: " + scope) + else: + findings = [row for row in findings if row["scope"] != scope] + return tuple(remaining + [row["raw"] for row in findings]) def _called_function_names(source: str) -> set[str]: @@ -3297,7 +3871,7 @@ def test_us_runtime_frame_rebuilds_preserve_immutable_metadata() -> None: def test_runtime_population_operators_are_source_spine_blind() -> None: - """Every operator obeys the strict-surface and contraband-name contract. + """Population operators stay blind; preparation uses exact phase contracts. The guard parses executable syntax rather than searching raw text, so comments, annotations, and docstrings may explain the invariant. Executable @@ -3326,11 +3900,284 @@ def test_runtime_population_operators_are_source_spine_blind() -> None: assert not offenders, ( "US runtime population operators must be source-spine blind. Route " "PUF-detail behavior with support clone indices; source-spine " - "provenance may be inspected only by the reviewed owner modules. " + "provenance may be inspected only by reviewed owners or exact reviewed " + "source/model preparation and attachment scopes. " f"Found: {offenders}" ) +def _current_handler_modules(source): + """Resolve the maintained literal handler table; refuse opaque registration.""" + nodes = _stage_nodes(source) + registry = nodes["us_source_operation_handlers"] + returns = [node for node in ast.walk(registry) if isinstance(node, ast.Return)] + assert len(returns) == 1 and isinstance(returns[0].value, ast.Dict) + table = returns[0].value + assert all( + isinstance(key, ast.Constant) and isinstance(key.value, str) + for key in table.keys + ) + assert len({key.value for key in table.keys}) == len(table.keys) + imports = { + alias.asname or alias.name: node.module + for node in ast.walk(nodes[""]) + if isinstance(node, ast.ImportFrom) and node.module + for alias in node.names + } + modules = {"source_runtime.py"} + for callback in table.values: + assert isinstance(callback, ast.Name), ( + "handler registration needs explicit review" + ) + if callback.id in nodes: + assert isinstance( + nodes[callback.id], (ast.FunctionDef, ast.AsyncFunctionDef) + ) + else: + module = imports[callback.id] + assert module.startswith(_US_RUNTIME_IMPORT_PREFIX + ".") + modules.add(module.removeprefix(_US_RUNTIME_IMPORT_PREFIX + ".") + ".py") + # Local wrappers also delegate to imported helpers such as PUF aggregation. + # Their direct US implementation imports remain part of the raw scan. + modules.update(_imported_us_runtime_modules(source)) + return frozenset(modules) + + +def test_current_handler_registry_keeps_raw_population_operator_coverage(): + modules = _current_handler_modules((_US_RUNTIME / "source_runtime.py").read_text()) + assert not _unclassified_runtime_modules(set(modules)) + assert not { + name: findings + for name in sorted(modules) + if (findings := _source_spine_accesses((_US_RUNTIME / name).read_text())) + } + + +def test_reviewed_stage_contracts_bind_all_findings_callers_and_domains(): + """The reviewed fixture records syntax evidence; CI never regenerates it.""" + assert len(_STAGE_CONTRACTS["module_roles"]) == 107 + assert len(_STAGE_CONTRACTS["scopes"]) == 87 + assert ( + sum(len(row["findings"]) for row in _STAGE_CONTRACTS["scopes"].values()) == 227 + ) + assert all(_STAGE_CONTRACTS["module_roles"].values()) + assert not set(_STAGE_CONTRACTS["module_roles"]) & ( + set(_OTHER_US_RUNTIME_MODULES) | set(_SPINE_BLIND_OPERATOR_MODULES) + ) + assert not _stage_contract_failures(_stage_repository_sources(), _STAGE_CONTRACTS) + + +@pytest.fixture +def stage_contract_example(): + """Small repositories make mutations causal without repeating the full scan.""" + prefix = _US_RUNTIME_IMPORT_PREFIX + "." + owner, caller, domain = ( + prefix + name for name in ("stage_fixture", "stage_entry", "stage_domain") + ) + sources = { + owner: ( + "from .stage_domain import PROFILE\n" + "COLUMNS = PROFILE\n" + "def project(table):\n" + ' return table["person_support_channel"]\n' + ), + caller: ( + "from .stage_fixture import project as observed_projection\n" + "def prepare(table):\n" + " return observed_projection(table)\n" + ), + domain: 'PROFILE = ("employment_income",)\n', + } + scope = owner + "::project" + roots = [owner + "::@COLUMNS"] + dependencies = _stage_domain_dependencies(sources, roots) + targets = {scope, caller + "::prepare", *dependencies} + references = _stage_reference_inventory(sources, targets) + contracts = {} + for key in sorted(targets): + module, name = key.split("::", 1) + contracts[key] = { + "body_sha256": _stage_ast_sha(_stage_nodes(sources[module])[name]), + "references": references[key], + "basis": "Synthetic authenticated-source projection and its bounded entry.", + } + access = contracts.pop(scope) + access.update( + role="source_authentication", + access="provenance", + findings=[ + row for row in _stage_findings(sources[owner]) if row["scope"] == "project" + ], + ) + document = { + "schema_version": 1, + "module_roles": {"stage_fixture.py": "source_authentication"}, + "scopes": {scope: access}, + "bindings": contracts, + "selector_domain_roots": roots, + "selector_domain_bindings": sorted(dependencies), + "selector_domain_dependencies": dependencies, + } + assert not _stage_contract_failures(sources, document) + return sources, document, owner, caller, domain + + +@pytest.mark.parametrize( + "mutation", ["new_access", "new_function", "changed_body", "removed_scope"] +) +def test_stage_contract_rejects_changed_or_stale_access_scopes( + stage_contract_example, monkeypatch, mutation +): + sources, document, owner, _, _ = stage_contract_example + source = sources[owner] + if mutation == "new_access": + source = source.replace( + " return", " table.person_support_channel\n return" + ) + elif mutation == "new_function": + source += '\ndef other(table):\n return table["person_support_channel"]\n' + elif mutation == "changed_body": + source = source.replace(" return", " table = table.copy()\n return") + else: + source = source[: source.index("def project")] + monkeypatch.setitem(globals(), "_STAGE_CONTRACTS", document) + assert _non_owner_source_spine_accesses("stage_fixture.py", source) + + +@pytest.mark.parametrize( + "style", ["direct", "import_alias", "qualified", "assignment_alias", "named_alias"] +) +def test_stage_contract_rejects_new_supported_callers(stage_contract_example, style): + sources, document, owner, _, _ = stage_contract_example + forms = { + "direct": f"from {owner} import project\ndef operate(t):\n return project(t)\n", + "import_alias": f"from {owner} import project as borrowed\ndef operate(t):\n return borrowed(t)\n", + "qualified": f"import {owner} as source\ndef operate(t):\n return source.project(t)\n", + "assignment_alias": f"from {owner} import project\nborrowed = project\ndef operate(t):\n return borrowed(t)\n", + "named_alias": f"from {owner} import project\ndef operate(t):\n return (borrowed := project)(t)\n", + } + sources[_US_RUNTIME_IMPORT_PREFIX + ".future_operator"] = forms[style] + errors = _stage_contract_failures(sources, document) + assert "changed callers: " + owner + "::project" in errors + assert "unbound caller body: " + owner + "::project" in errors + + +@pytest.mark.parametrize("resolution", ["known", "unknown", "mixed"]) +def test_stage_reference_fallback_distinguishes_known_foreign_definitions( + stage_contract_example, resolution +): + sources, document, owner, _, _ = stage_contract_example + prefix = _US_RUNTIME_IMPORT_PREFIX + "." + sources[prefix + "stage_known"] = "def project(table):\n return table\n" + imports = [] + if resolution in {"known", "mixed"}: + imports.append("from .stage_known import project as candidate") + if resolution in {"unknown", "mixed"}: + imports.append("from .stage_unindexed import project as candidate") + sources[prefix + "stage_other_caller"] = ( + "\n".join(imports) + "\ndef operate(table):\n return candidate(table)\n" + ) + errors = _stage_contract_failures(sources, document) + if resolution == "known": + assert not errors + else: + assert "changed callers: " + owner + "::project" in errors + assert "unbound caller body: " + owner + "::project" in errors + + +def test_stage_contract_binds_control_flow_around_an_unchanged_call( + stage_contract_example, +): + sources, document, owner, caller, _ = stage_contract_example + sources[caller] = sources[caller].replace( + " return observed_projection(table)", + " if table.shape[0] > 1:\n return observed_projection(table)\n return table", + ) + errors = _stage_contract_failures(sources, document) + assert "changed body/domain: " + caller + "::prepare" in errors + assert "changed callers: " + owner + "::project" not in errors + + +@pytest.mark.parametrize( + "mutation", + ["protected_profile", "different_owner", "missing_domain", "duplicate_domain"], +) +def test_stage_contract_binds_imported_profile_definitions( + stage_contract_example, mutation +): + sources, document, owner, _, domain = stage_contract_example + if mutation == "protected_profile": + sources[domain] = sources[domain].replace( + "employment_income", "person_support_channel" + ) + expected = "changed body/domain: " + domain + "::@PROFILE" + elif mutation == "different_owner": + sources[domain + "_other"] = sources[domain] + sources[owner] = sources[owner].replace( + ".stage_domain import", ".stage_domain_other import" + ) + expected = "changed finite-domain owner/dependencies" + elif mutation == "missing_domain": + document["selector_domain_bindings"].remove(domain + "::@PROFILE") + expected = "changed finite-domain owner/dependencies" + else: + document["selector_domain_bindings"].append(domain + "::@PROFILE") + expected = "missing, repeated or unbound selector domains" + assert expected in _stage_contract_failures(sources, document) + + +def test_stage_selector_contract_cannot_grant_source_origin_authority( + stage_contract_example, monkeypatch +): + sources, document, owner, _, _ = stage_contract_example + document["scopes"][owner + "::project"]["access"] = "selector" + monkeypatch.setitem(globals(), "_STAGE_CONTRACTS", document) + assert any( + "cannot grant provenance" in error + for error in _non_owner_source_spine_accesses( + "stage_fixture.py", sources[owner] + ) + ) + + +def test_stage_contract_refuses_a_stale_finding_inventory(stage_contract_example): + sources, document, owner, _, _ = stage_contract_example + key = owner + "::project" + document["scopes"][key]["findings"] = [] + assert "changed or stale findings: " + key in _stage_contract_failures( + sources, document + ) + + +def test_stage_contract_refuses_an_orphan_supporting_binding(stage_contract_example): + sources, document, owner, _, _ = stage_contract_example + sources[owner] += "\ndef unused(table):\n return table\n" + key = owner + "::unused" + document["bindings"][key] = { + "body_sha256": _stage_ast_sha(_stage_nodes(sources[owner])["unused"]), + "references": [], + "basis": "A stale supporting declaration must not remain silently accepted.", + } + assert "orphan supporting contracts: " + key in _stage_contract_failures( + sources, document + ) + + +def test_population_operator_raw_guard_cannot_borrow_a_stage_exception( + stage_contract_example, monkeypatch +): + sources, document, owner, _, _ = stage_contract_example + monkeypatch.setitem(globals(), "_STAGE_CONTRACTS", document) + assert not _non_owner_source_spine_accesses("stage_fixture.py", sources[owner]) + assert _operator_source_channel_reads(sources[owner]) + + +def test_handler_registry_refuses_opaque_registration(): + source = "def us_source_operation_handlers():\n return {'future': lambda frame: frame}\n" + with pytest.raises(AssertionError, match="explicit review"): + _current_handler_modules(source) + + def test_registered_population_operators_do_not_read_any_source_channel() -> None: """The migrated operator surface may resolve clone roles, never sources.""" @@ -3403,14 +4250,15 @@ def test_pool_build_tool_import_graph_is_source_spine_blind() -> None: for tool in _SPINE_BLIND_BUILD_TOOLS: runtime_graph, missing_modules = _us_runtime_import_graph(tool) - assert len(runtime_graph) == 70, ( - f"{tool.name} must reach the pinned 70-module runtime graph; " - f"reached {len(runtime_graph)}" + runtime_names = {path.name for path in runtime_graph} + assert runtime_names == _EXPECTED_POOL_RUNTIME_MODULES, ( + f"{tool.name} must reach the exact reviewed 73-module runtime graph; " + f"added={sorted(runtime_names - _EXPECTED_POOL_RUNTIME_MODULES)}, " + f"missing={sorted(_EXPECTED_POOL_RUNTIME_MODULES - runtime_names)}" ) assert not missing_modules, ( f"{tool.name} imports unresolved US runtime modules: {missing_modules}" ) - runtime_names = {path.name for path in runtime_graph} missing_required = sorted(_REQUIRED_POOL_RUNTIME_MODULES - runtime_names) assert not missing_required, ( f"{tool.name} does not reach the canonical pool seam modules: " diff --git a/packages/microcosm-build/tests/test_us_survey_age_calibration_run.py b/packages/microcosm-build/tests/test_us_survey_age_calibration_run.py index 31adbd17b..e44ff9a4c 100644 --- a/packages/microcosm-build/tests/test_us_survey_age_calibration_run.py +++ b/packages/microcosm-build/tests/test_us_survey_age_calibration_run.py @@ -310,20 +310,18 @@ def test_actual_seven_node_terminal_manifest_mutation_refuses( """Expensive actual source fixture: requires a separately approved workload.""" from test_us_graph_survey_population import authenticated_arguments - actual_graph = stage.run_graph - actual_verify = stage.budgets.verify_survey_weight_only_successor + graph_code = stage.run_graph.__code__ + verify_code = stage.budgets.verify_survey_weight_only_successor.__code__ returned = [] changed = [] - def capture(compiled, **kwargs): - manifest = actual_graph(compiled, **kwargs) - if len(compiled.order) == 7: - returned.append(manifest) - return manifest - - def late(value): - actual_verify(value) - if returned: + def observe_return(frame, event, value): + if event != "return": + return + if frame.f_code is graph_code and type(value) is RunManifest: + if len(frame.f_locals["compiled"].order) == 7: + returned.append(value) + elif frame.f_code is verify_code and returned and not changed: row = returned[-1].node(stage.numerical.CALIBRATION_NODE) if field == "capabilities": object.__setattr__(row.capabilities, "consumes_se", True) @@ -338,10 +336,12 @@ def late(value): assert getattr(row, field) == value changed.append(True) - monkeypatch.setattr(stage, "run_graph", capture) - monkeypatch.setattr(stage.budgets, "verify_survey_weight_only_successor", late) arguments = authenticated_arguments(tmp_path, monkeypatch) arguments["seed_value"] = arguments.pop("seed") + # Observe the actual producers: substituting either callable changes the + # source-authority seal before the intended late manifest fault is reached. + previous_profile = sys.getprofile() + sys.setprofile(observe_return) try: with pytest.raises(ValueError, match="MANIFEST_NODE_STATE"): stage.run_survey_age_calibration( @@ -351,13 +351,14 @@ def late(value): learning_rate=0.1, ) finally: + sys.setprofile(previous_profile) if field == "capabilities" and returned: object.__setattr__( returned[-1].node(stage.numerical.CALIBRATION_NODE).capabilities, "consumes_se", False, ) - assert changed + assert len(returned) == 1 and changed == [True] def _portable_array_values(): diff --git a/packages/microcosm-build/tests/test_us_survey_calibration.py b/packages/microcosm-build/tests/test_us_survey_calibration.py index f05a3967f..732fbcada 100644 --- a/packages/microcosm-build/tests/test_us_survey_calibration.py +++ b/packages/microcosm-build/tests/test_us_survey_calibration.py @@ -10,6 +10,9 @@ import test_us_survey_age_artifact as age_fixture from microcosm.build.us_runtime import graph_survey_calibration as stage +from microcosm.build.us_runtime import ( + survey_calibration_diagnostics as diagnostic_check, +) from microcosm.graph import ArtifactValue, KernelContext from microcosm.graph.canonical import canonical_json from microcosm.graph.kernel import Numeric, NumericScope @@ -82,6 +85,67 @@ def test_real_solver_retains_zero_rows_and_returns_weights_only(): assert output.receipt["source_admission"] == "required_from_country_runner" +@pytest.mark.parametrize( + "changed_option", + [ + None, + ("gate_initialization_supplied", True), + ("budget_basis", "open_probability_mass"), + ("feasible_draw_pi_hi", 1.0), + ("budget_search", {}), + ], +) +def test_diagnostics_reconstruct_fixed_solver_options(changed_option): + value = context() + output = stage.SurveyAgeCalibrationKernel().run(value) + payload = output.artifacts["diagnostics"] + document = json.loads(payload) + fixed_options = { + "gate_initialization_supplied": False, + "budget_basis": "nonzero_count", + "feasible_draw_pi_hi": None, + "budget_search": None, + } + assert {name: document["options"][name] for name in fixed_options} == fixed_options + arguments = { + "counts_payload": value.artifacts["counts"].payload, + "bounds_payload": value.artifacts["bounds"].payload, + "weights": output.weights.values, + "registry": fixture.fixture_registry(), + "epochs": value.params["epochs"], + "learning_rate": value.params["learning_rate"], + "anchors": { + name: output.receipt[name] + for name in ( + "budget_sha256", + "numeric_bounds_sha256", + "counts_sha256", + "accepted_weight_sha256", + "constraint_digest", + "weight_anchor", + "cap_enforcement", + "fixed_zero_rows", + ) + }, + } + # First validate the real output in every case, so mutation refusals cannot + # pass merely because the checker rejects all current solver diagnostics. + accepted_weights = output.weights.values.tobytes() + checked = diagnostic_check.validate_survey_calibration_diagnostics( + payload, **arguments + ) + assert checked.pop("verification")["optimizer_rerun"] is False + assert canonical_json(checked) == payload + if changed_option is not None: + name, changed = changed_option + document["options"][name] = changed + with pytest.raises(ValueError, match="SURVEY_DIAGNOSTICS_RECOMPUTED_VALUES"): + diagnostic_check.validate_survey_calibration_diagnostics( + canonical_json(document), **arguments + ) + assert output.weights.values.tobytes() == accepted_weights + + @pytest.mark.parametrize( "change", [ From 051fb972b19d319d58277bd63306d0d0e0947ce2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mar=C3=ADa=20Juaristi?= <127882282+juaristi22@users.noreply.github.com> Date: Thu, 10 Sep 2026 22:52:44 +0200 Subject: [PATCH 8/8] Fix issues from review: restore PUF test monkeypatch without module reload --- .../tests/test_us_puf_support.py | 22 +++++++------------ 1 file changed, 8 insertions(+), 14 deletions(-) diff --git a/packages/microcosm-build/tests/test_us_puf_support.py b/packages/microcosm-build/tests/test_us_puf_support.py index 75e510b6e..580113e08 100644 --- a/packages/microcosm-build/tests/test_us_puf_support.py +++ b/packages/microcosm-build/tests/test_us_puf_support.py @@ -1,6 +1,5 @@ """US PUF support-channel expansion tests.""" -import importlib from collections.abc import Sequence import numpy as np @@ -960,17 +959,16 @@ def test_assert_formula_owned_blocklist_current_flags_stale_entries() -> None: assert_formula_owned_blocklist_current(drifted_engine) -def test_resolve_formula_owned_outputs_engine_none_falls_back_to_static() -> None: +def test_resolve_formula_owned_outputs_engine_none_falls_back_to_static( + monkeypatch, +) -> None: # With no engine passed and metadata unavailable, resolution falls back to # the static seed. The fallback is exercised by monkeypatching the lazy # engine resolver to report no engine, so the test is deterministic even # where policyengine_us is installed. - puf_support_module._formula_owned_engine = lambda: None - try: - requested = {"interest_deduction", "employment_income_before_lsr"} - assert resolve_formula_owned_outputs(requested) == {"interest_deduction"} - finally: - importlib.reload(puf_support_module) + monkeypatch.setattr(puf_support_module, "_formula_owned_engine", lambda: None) + requested = {"interest_deduction", "employment_income_before_lsr"} + assert resolve_formula_owned_outputs(requested) == {"interest_deduction"} class _ImportErrorEngine: @@ -1291,16 +1289,12 @@ def test_policyengine_broadcasts_annual_reported_enrollment_to_each_month( "spm_units": { "unit_100": { "members": ["person_1", "person_2"], - "receives_tanf": { - "2024": bool(spm_flags.loc[100, "receives_tanf"]) - }, + "receives_tanf": {"2024": bool(spm_flags.loc[100, "receives_tanf"])}, "receives_snap": {"2024": bool(spm_flags.loc[100, "receives_snap"])}, }, "unit_200": { "members": ["person_3"], - "receives_tanf": { - "2024": bool(spm_flags.loc[200, "receives_tanf"]) - }, + "receives_tanf": {"2024": bool(spm_flags.loc[200, "receives_tanf"])}, "receives_snap": {"2024": bool(spm_flags.loc[200, "receives_snap"])}, }, },