UK local candidate gate battery, release diagnostics, and scorer (#147 PR 1)

changelog.d/147-uk-local-gate-battery.added.md

@@ -0,0 +1 @@
+Add a signed six-gate battery and calibration diagnostics bundle for UK local candidates.

docs/gate-battery-contract.md

@@ -224,10 +224,10 @@ stretches on declaration alone:
 
 | | BE | UK |
 | --- | --- | --- |
-| phases | `terminal` | `preflight`, `terminal` |
-| entries | 9 | 13 |
+| phases | `terminal` | `preflight`, `assembled`, `transferred`, `terminal` |
+| entries | 9 | 43 (37 national-build entries plus 6 local-candidate entries) |
 | incumbent posture | none — incumbent-comparison gates deliberately not selected; external oracles replace self-parity | full — export/target parity against the pinned enhanced-FRS incumbent |
-| criticality mix | 8 blocking + 1 diagnostic | all blocking |
+| criticality mix | 8 blocking + 1 diagnostic | 39 blocking + 4 local diagnostics |
 
 The differences are entirely in the two country inputs — the spec file
 and the registry — which is the point. The UK national build is the
@@ -236,6 +236,14 @@ build and runs both phases under `BLOCKS_ARTIFACT` (preflight before the
 frame loads, terminal immediately before the staging writer), while BE
 remains spec-only.
 
+UK evaluates the declaration through four producer scopes: spine,
+calibration seam, national release cut, and local candidate. The first three
+compose the national release certification. The six local-candidate entries
+are explicitly classified as excluded from that certification until the
+local publication work in #146; they are evaluated in their own signed
+`local_candidate` report rather than silently disappearing from union
+accounting.
+
 ## The reference rule
 
 A parity gate is only as honest as its reference: comparing a calibrated

packages/microcosm-build/src/microcosm/build/__init__.py

@@ -81,6 +81,7 @@ def _assert_frame_compatible(version: str, required: tuple[int, int]) -> None:
     TargetCoverageRequirement,
     TargetFitRequirement,
     aggregate_admin_gate,
+    area_support_gate,
     column_implication_gate,
     default_valued_columns_gate,
     enum_domain_gate,
@@ -204,6 +205,7 @@ def _assert_frame_compatible(version: str, required: tuple[int, int]) -> None:
     "PreparedMonetaryMeasure",
     "add_ledger_artifact_args",
     "aggregate_admin_gate",
+    "area_support_gate",
     "column_implication_gate",
     "apply_ledger_target_profile",
     "bind_monetary_target",

packages/microcosm-build/src/microcosm/build/country_spec.py

@@ -111,6 +111,7 @@
 ALLOWED_GATE_FUNCTIONS = frozenset(
     {
         "aggregate_admin",
+        "area_support",
         "calibration_reference_coverage",
         "column_implication",
         "degenerate_release_surface",

packages/microcosm-build/src/microcosm/build/gates.py

@@ -57,6 +57,7 @@
     "GateResult",
     "GateReport",
     "FitWeightRecord",
+    "area_support_gate",
     "default_valued_columns_gate",
     "enum_domain_gate",
     "export_surface_gate",
@@ -2420,6 +2421,160 @@ def support_gate(
     )
 
 
+def area_support_gate(
+    area_support: pd.DataFrame,
+    *,
+    area_roster: Mapping[str, Iterable[str]],
+    geography_levels: Iterable[str],
+    minimum_rows: int,
+    minimum_effective_sample_size: float,
+    minimum_distinct_sources: int,
+) -> GateResult:
+    """Require adequate positive-weight support in every declared local area.
+
+    ``area_support`` is the long-form candidate receipt.  The independently
+    declared ``area_roster`` is load-bearing: a missing weak area cannot make
+    the minima look better by disappearing from the evidence table.
+    """
+
+    levels = tuple(str(level) for level in geography_levels)
+    if not levels or len(set(levels)) != len(levels):
+        raise ValueError("geography_levels must be non-empty and unique.")
+    if (
+        isinstance(minimum_rows, bool)
+        or not isinstance(minimum_rows, int)
+        or minimum_rows < 1
+    ):
+        raise ValueError("minimum_rows must be a positive integer.")
+    if (
+        isinstance(minimum_distinct_sources, bool)
+        or not isinstance(minimum_distinct_sources, int)
+        or minimum_distinct_sources < 1
+    ):
+        raise ValueError("minimum_distinct_sources must be a positive integer.")
+    if (
+        not math.isfinite(float(minimum_effective_sample_size))
+        or minimum_effective_sample_size <= 0
+    ):
+        raise ValueError("minimum_effective_sample_size must be positive and finite.")
+    if not isinstance(area_support, pd.DataFrame):
+        raise TypeError("area_support must be a pandas DataFrame.")
+    required_columns = {
+        "geography_level",
+        "area_code",
+        "nonzero_households",
+        "nonzero_source_households",
+        "effective_sample_size",
+    }
+    missing_columns = sorted(required_columns - set(area_support.columns))
+    if missing_columns:
+        raise ValueError(
+            f"area_support is missing required column(s): {missing_columns}."
+        )
+    unexpected_levels = sorted(
+        set(area_support["geography_level"].astype(str)) - set(levels)
+    )
+    if unexpected_levels:
+        raise ValueError(
+            "area_support contains geography level(s) outside the declared "
+            f"scope: {unexpected_levels}."
+        )
+
+    rows = area_support.copy()
+    rows["geography_level"] = rows["geography_level"].astype(str)
+    rows["area_code"] = rows["area_code"].astype(str)
+    duplicate_mask = rows.duplicated(["geography_level", "area_code"], keep=False)
+    if duplicate_mask.any():
+        duplicates = sorted(
+            {
+                f"{row.geography_level}/{row.area_code}"
+                for row in rows.loc[
+                    duplicate_mask, ["geography_level", "area_code"]
+                ].itertuples(index=False)
+            }
+        )
+        raise ValueError(f"area_support contains duplicate area rows: {duplicates}.")
+
+    expected_pairs: set[tuple[str, str]] = set()
+    for level in levels:
+        if level not in area_roster:
+            raise ValueError(f"area_roster is missing geography level {level!r}.")
+        codes = tuple(str(code) for code in area_roster[level])
+        if not codes or len(set(codes)) != len(codes):
+            raise ValueError(f"area_roster[{level!r}] must be non-empty and unique.")
+        expected_pairs.update((level, code) for code in codes)
+    actual_pairs = set(zip(rows["geography_level"], rows["area_code"], strict=True))
+    missing_areas = sorted(expected_pairs - actual_pairs)
+    extra_areas = sorted(actual_pairs - expected_pairs)
+    if missing_areas or extra_areas:
+        raise ValueError(
+            "area_support must exactly cover the declared roster; "
+            f"missing={missing_areas[:10]}, extra={extra_areas[:10]}."
+        )
+
+    numeric_columns = (
+        "nonzero_households",
+        "nonzero_source_households",
+        "effective_sample_size",
+    )
+    values = rows.loc[:, list(numeric_columns)].to_numpy(dtype=np.float64)
+    if not np.isfinite(values).all() or (values < 0).any():
+        raise ValueError("area_support contains invalid support values.")
+
+    failures: list[str] = []
+    for row in rows.sort_values(["geography_level", "area_code"]).itertuples(
+        index=False
+    ):
+        shortfalls = []
+        if row.nonzero_households < minimum_rows:
+            shortfalls.append(f"rows {int(row.nonzero_households)} < {minimum_rows}")
+        if row.effective_sample_size < minimum_effective_sample_size:
+            shortfalls.append(
+                "ESS "
+                f"{float(row.effective_sample_size):.6g} < "
+                f"{float(minimum_effective_sample_size):.6g}"
+            )
+        if row.nonzero_source_households < minimum_distinct_sources:
+            shortfalls.append(
+                "distinct sources "
+                f"{int(row.nonzero_source_households)} < "
+                f"{minimum_distinct_sources}"
+            )
+        if shortfalls:
+            failures.append(
+                f"{row.geography_level}/{row.area_code}: " + ", ".join(shortfalls)
+            )
+
+    by_level: dict[str, dict[str, object]] = {}
+    for level in levels:
+        level_rows = rows.loc[rows["geography_level"] == level]
+        by_level[level] = {
+            "areas_checked": int(len(level_rows)),
+            "minimum_rows": int(level_rows["nonzero_households"].min()),
+            "minimum_effective_sample_size": float(
+                level_rows["effective_sample_size"].min()
+            ),
+            "minimum_distinct_sources": int(
+                level_rows["nonzero_source_households"].min()
+            ),
+        }
+    return GateResult(
+        name="area_support",
+        passed=not failures,
+        failures=tuple(failures),
+        details={
+            "floors": {
+                "minimum_rows": minimum_rows,
+                "minimum_effective_sample_size": float(minimum_effective_sample_size),
+                "minimum_distinct_sources": minimum_distinct_sources,
+            },
+            "by_geography_level": by_level,
+            "areas_checked": len(expected_pairs),
+            "areas_failed": len(failures),
+        },
+    )
+
+
 def aggregate_admin_gate(
     aggregates: Mapping[str, float],
     anchors: Iterable[TargetSpec],

packages/microcosm-build/src/microcosm/build/uk/gates.json

@@ -712,6 +712,75 @@
         "min_nonzero_records": 104
       },
       "notes": "Weighted tail-concentration audit of the QRF-imputed output columns derived from the HMRC source manifest. Armed from the #686 L3 baselines per #757 B4, each threshold recording the run that measured it: the 47-column qrf_tail grids of uk_weighted_integrity_baselines_686.json (sha256 8b3f6c4e9c522445bf3e05d3451ee8557f4e14d25524ceac3af8bfe05401a146, licensed acceptance dir 686-spine-swap), measured on spine-a.h5 (sha256 a65f2132736d1dcecb91709a513a0d54a5887635ea03760629cc888d188ee306, 113,649 person rows, design weights) for the #609/#578 threshold adjudication. top_k 100 is the measurement grid anchor [10, 100, 500, 1000]; max_top_share is the exact measured maximum over the checked surface (hmrc_spi_other_social_security_income, 104 carriers), no headroom; min_nonzero_records is the thinnest measured column above the grid anchor (the same column) - the policy domain requires min_nonzero_records > top_k, so the three saturated sub-anchor columns (hmrc_spi_taxable_termination_pay 12, charitable_investment_gifts 22, sda_reported 24 carriers, each top-100 share 1.0) sit below it and go thin visibly in the signed details on every run, never a silent pass. The baselines are design-weight measurements and the terminal gate runs on the calibrated release frame; a calibrated-weight breach names its column and is a finding, not noise. The 12 household_qrf_tail grids (was_wealth surface) are measured in the same baselines file and not yet armed - a household-surface gate entry is a declared follow-up. Reviewed exclusions live in the named register resource of this package."
+    },
+    {
+      "id": "uk_local_geography_ladder_post_calibration",
+      "gate": "spine_agreement",
+      "phase": "terminal",
+      "criticality": "release_blocking",
+      "parameters": {},
+      "notes": "The existing UK geography-ladder evaluator is rerun on the locally calibrated candidate weights and adapted through UKGateBinding; no second ladder verdict implementation is introduced."
+    },
+    {
+      "id": "uk_local_area_support",
+      "gate": "area_support",
+      "phase": "terminal",
+      "criticality": "release_blocking",
+      "parameters": {
+        "crosswalk_resource": "local_area_crosswalk.json",
+        "geography_levels": [
+          "constituency",
+          "local_authority"
+        ],
+        "minimum_rows": 50,
+        "minimum_effective_sample_size": 50.0,
+        "minimum_distinct_sources": 50
+      },
+      "notes": "Both local grains are blocking from birth. Each declared area needs at least 50 positive-weight rows, Kish ESS 50, and 50 distinct positive-weight source households. The constituency K=4 receipt measured minima of 101 rows, ESS 86.4, and 97 sources; a local-authority miss reopens K or requires a separately reviewed exclusion."
+    },
+    {
+      "id": "uk_local_target_fit",
+      "gate": "target_fit",
+      "phase": "terminal",
+      "criticality": "diagnostic",
+      "parameters": {
+        "surface": "local_candidate",
+        "max_abs_relative_error": 0.25
+      },
+      "notes": "Report every bound local target outside 25% absolute relative error. Diagnostic until the first #762 measured-fit receipt arms it."
+    },
+    {
+      "id": "uk_local_per_family_fit",
+      "gate": "per_family_fit",
+      "phase": "terminal",
+      "criticality": "diagnostic",
+      "parameters": {
+        "within": 0.1,
+        "min_family_share": 0.5,
+        "hard_within": 0.25,
+        "min_hard_family_share": 0.5
+      },
+      "notes": "Report 10% fit shares per bound family and the 25% broad-fit share, so a weak family cannot hide in the aggregate. Diagnostic until the first #762 measured-fit receipt arms it."
+    },
+    {
+      "id": "uk_local_weight_ratio",
+      "gate": "weight_ratio",
+      "phase": "terminal",
+      "criticality": "diagnostic",
+      "parameters": {
+        "maximum_max_to_median_ratio": 100.0
+      },
+      "notes": "The reviewed local doctrine ratio is 100.0. It ships diagnostic until #762 records the first measured candidate and arms the threshold."
+    },
+    {
+      "id": "uk_local_weight_ess",
+      "gate": "weight_ess",
+      "phase": "terminal",
+      "criticality": "diagnostic",
+      "parameters": {
+        "minimum_ess_fraction": 0.01
+      },
+      "notes": "The local ESS-fraction diagnostic uses the national analogue's 0.01 floor and remains diagnostic until #762's measured arming review."
     }
   ]
 }

packages/microcosm-build/src/microcosm/build/uk_runtime/__init__.py

@@ -29,6 +29,8 @@
     UK_DIAGNOSTICS_SCHEMA_VERSION,
     UK_TARGET_GEOGRAPHY_LEVELS,
     uk_calibration_diagnostics_payload,
+    uk_weakest_areas_by_fit,
+    uk_weakest_families,
     uk_weight_summary,
     uk_zero_weight_strata,
     write_uk_calibration_diagnostics,
@@ -273,11 +275,14 @@
 )
 from microcosm.build.uk_runtime.local_rowwise import (
     UK_LOCAL_BINDING_ADJUDICATION_REGISTER_RESOURCE,
+    UK_LOCAL_HOLDOUT_FOLDS,
+    UK_LOCAL_HOLDOUT_SEED,
     UKRowwiseDoctrineSolve,
     UKRowwiseLocalMatrix,
     build_uk_rowwise_local_matrix,
     past_cap_census,
     require_adjudicated_uk_local_binding,
+    rotated_uk_local_holdout,
     rowwise_area_support_summary,
     rowwise_calibration_mass_reason,
     solve_uk_rowwise_weights_under_doctrine,
@@ -509,6 +514,8 @@
     "UK_FRAME_METADATA_KEY",
     "UK_LOCAL_MAX_WEIGHT_RATIO",
     "UK_LOCAL_BINDING_ADJUDICATION_REGISTER_RESOURCE",
+    "UK_LOCAL_HOLDOUT_FOLDS",
+    "UK_LOCAL_HOLDOUT_SEED",
     "UK_LOCAL_SOLVE_DOCTRINE",
     "UK_LOCAL_TARGET_LOSS_CAP",
     "UK_CGT_GAINS_AMOUNT_COLUMN",
@@ -521,6 +528,7 @@
     "materialize_uk_cgt_calibration_frame",
     "require_adjudicated_uk_local_binding",
     "rowwise_calibration_mass_reason",
+    "rotated_uk_local_holdout",
     "uk_cgt_annual_exempt_amount",
     "UK_CGT_REQUIRED_COLUMNS",
     "UK_CGT_TARGET_COVERAGE_REQUIREMENTS",
@@ -838,6 +846,8 @@
     "uk_release_input_coverage_reviewed_exclusions",
     "uk_stage_metadata",
     "uk_weight_summary",
+    "uk_weakest_areas_by_fit",
+    "uk_weakest_families",
     "uk_zero_weight_strata",
     "update_england_wales_lad_codes",
     "validate_uk_firm_population",