From 247aee956930005d0fd1a22d8c8bb517644c976d Mon Sep 17 00:00:00 2001 From: Daphne Hansell <128793799+daphnehanse11@users.noreply.github.com> Date: Wed, 15 Jul 2026 10:56:09 -0400 Subject: [PATCH 01/13] DRAFT: SIPP job-spell noise floors for E4/E5 (pre-C3 anchor) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Person-disjoint half-vs-half floors on the pu2023 job-month panel, the disability-floor convention (sha256 half-splits, seeds 0-4, per-cell |log rate ratio| mean/sd, WPFINWGT-weighted). E4: monthly same-employer retention by age band x sex (99.0% at 16-24 rising to 99.7% at 35-44; floors 0.001-0.002). E5: maximal same-employer run lengths (full-year share 46.7% at 16-24 to 86.7% at 45-64; floors 0.01-0.05). No thin cells. Explicitly DRAFT: no thresholds, C3 not locked, seam caveat recorded — the seam-vs-J2J reconciliation is a separate required artifact. Co-Authored-By: Claude Fable 5 --- runs/sipp_spell_floors_draft_v0.json | 139 +++++++++++++++++ scripts/build_sipp_spell_floors.py | 213 +++++++++++++++++++++++++++ 2 files changed, 352 insertions(+) create mode 100644 runs/sipp_spell_floors_draft_v0.json create mode 100644 scripts/build_sipp_spell_floors.py diff --git a/runs/sipp_spell_floors_draft_v0.json b/runs/sipp_spell_floors_draft_v0.json new file mode 100644 index 00000000..0926de4b --- /dev/null +++ b/runs/sipp_spell_floors_draft_v0.json @@ -0,0 +1,139 @@ +{ + "artifact": "sipp_spell_floors", + "version": "draft_v0", + "status": "DRAFT - NOT RATIFIED; C3 not locked; no thresholds", + "issue": "192", + "source": "pu2023 (reference year 2022)", + "method": "person-disjoint sha256 half-splits, seeds 0-4; per-cell |log(rate_a/rate_b)| mean/sd across seeds; weighted by WPFINWGT", + "seam_caveat": "both halves share SIPP seam structure; the seam-vs-J2J reconciliation run is a separate required artifact before thresholds lock", + "e4_retention_by_age_sex": { + "16_24|sex1": { + "rate": 0.9897, + "pairs_unweighted": 8055, + "floor_abs_log_ratio_mean": 0.00182, + "floor_abs_log_ratio_sd": 0.00107, + "thin": false + }, + "16_24|sex2": { + "rate": 0.9901, + "pairs_unweighted": 7093, + "floor_abs_log_ratio_mean": 0.00198, + "floor_abs_log_ratio_sd": 0.00232, + "thin": false + }, + "25_34|sex1": { + "rate": 0.9948, + "pairs_unweighted": 17797, + "floor_abs_log_ratio_mean": 0.0008, + "floor_abs_log_ratio_sd": 0.00036, + "thin": false + }, + "25_34|sex2": { + "rate": 0.9953, + "pairs_unweighted": 15555, + "floor_abs_log_ratio_mean": 0.00143, + "floor_abs_log_ratio_sd": 0.00114, + "thin": false + }, + "35_44|sex1": { + "rate": 0.9968, + "pairs_unweighted": 19920, + "floor_abs_log_ratio_mean": 0.00097, + "floor_abs_log_ratio_sd": 0.00058, + "thin": false + }, + "35_44|sex2": { + "rate": 0.9964, + "pairs_unweighted": 17203, + "floor_abs_log_ratio_mean": 0.00089, + "floor_abs_log_ratio_sd": 0.00065, + "thin": false + }, + "45_54|sex1": { + "rate": 0.9978, + "pairs_unweighted": 17779, + "floor_abs_log_ratio_mean": 0.0003, + "floor_abs_log_ratio_sd": 0.00028, + "thin": false + }, + "45_54|sex2": { + "rate": 0.998, + "pairs_unweighted": 16516, + "floor_abs_log_ratio_mean": 0.00051, + "floor_abs_log_ratio_sd": 0.00045, + "thin": false + }, + "55_64|sex1": { + "rate": 0.9984, + "pairs_unweighted": 18342, + "floor_abs_log_ratio_mean": 0.00067, + "floor_abs_log_ratio_sd": 0.00058, + "thin": false + }, + "55_64|sex2": { + "rate": 0.999, + "pairs_unweighted": 17410, + "floor_abs_log_ratio_mean": 0.0003, + "floor_abs_log_ratio_sd": 0.00014, + "thin": false + }, + "65_99|sex1": { + "rate": 0.9992, + "pairs_unweighted": 12113, + "floor_abs_log_ratio_mean": 0.00058, + "floor_abs_log_ratio_sd": 0.00039, + "thin": false + }, + "65_99|sex2": { + "rate": 0.9996, + "pairs_unweighted": 10437, + "floor_abs_log_ratio_mean": 0.00039, + "floor_abs_log_ratio_sd": 0.00032, + "thin": false + } + }, + "e5_runs_by_age": { + "16_24": { + "full_year_run_share": 0.4669, + "persons_unweighted": 1830, + "floor_abs_log_ratio_mean": 0.04937, + "floor_abs_log_ratio_sd": 0.04904, + "thin": false + }, + "25_34": { + "full_year_run_share": 0.7499, + "persons_unweighted": 3299, + "floor_abs_log_ratio_mean": 0.01944, + "floor_abs_log_ratio_sd": 0.01104, + "thin": false + }, + "35_44": { + "full_year_run_share": 0.8245, + "persons_unweighted": 3555, + "floor_abs_log_ratio_mean": 0.01764, + "floor_abs_log_ratio_sd": 0.01639, + "thin": false + }, + "45_54": { + "full_year_run_share": 0.8671, + "persons_unweighted": 3256, + "floor_abs_log_ratio_mean": 0.01096, + "floor_abs_log_ratio_sd": 0.00768, + "thin": false + }, + "55_64": { + "full_year_run_share": 0.8673, + "persons_unweighted": 3458, + "floor_abs_log_ratio_mean": 0.01742, + "floor_abs_log_ratio_sd": 0.00998, + "thin": false + }, + "65_99": { + "full_year_run_share": 0.7941, + "persons_unweighted": 2294, + "floor_abs_log_ratio_mean": 0.01327, + "floor_abs_log_ratio_sd": 0.01036, + "thin": false + } + } +} diff --git a/scripts/build_sipp_spell_floors.py b/scripts/build_sipp_spell_floors.py new file mode 100644 index 00000000..4b41b75c --- /dev/null +++ b/scripts/build_sipp_spell_floors.py @@ -0,0 +1,213 @@ +"""Build DRAFT SIPP job-spell noise floors for gates E4/E5 (#192). + +REPORTED ANCHOR, NOT A GATE RUN — and explicitly a DRAFT: C3 (the +employer gate block) has not locked, no thresholds are proposed +here, and nothing below is ratified. Like the disability floors, +this commits the person-disjoint half-vs-half sampling-noise floor +that pre-registered E4/E5 thresholds would later be derived from, +so the floor-building method is on the record before any candidate +model exists (issue #192 protocol: floors -> thresholds -> referee +round -> one-shot runs). + +Moments, computed on the SIPP job-month panel +(:mod:`populace_dynamics.data.sipp_jobs`, reference year 2022 from +the pu2023 file): + +(a) **E4 retention pairs.** Among persons employed in consecutive + reference months m and m+1, the weighted share retaining at + least one employer (same within-panel ``EJB`` job id in both + months), by age band x sex. This is the month-frequency analog + of the plan's "2-window employer-retention persistence". + +(b) **E5 attachment runs.** The weighted distribution of maximal + same-employer run lengths (1-12 months within the reference + year, from :func:`job_spells`), by age band — the view that + catches chained-model persistence understatement. + +(c) **The floor.** For seeds 0-4, persons are split into two + disjoint halves; each cell's rate is computed on both halves + and the across-seed mean and sd of ``|log(rate_a / rate_b)|`` + is the sampling-noise floor for that cell, exactly the + disability-floor convention. Cells with fewer than 200 + unweighted persons per half are reported but flagged thin. + +Seam caveat (pre-registered on #192): SIPP transitions bunch at +interview seams, and both halves share the seam structure, so this +floor cannot see seam bias — the seam-vs-J2J reconciliation run is +a separate, required artifact before E2/E4 thresholds lock. + +Usage:: + + python scripts/build_sipp_spell_floors.py + +writes ``runs/sipp_spell_floors_draft_v0.json``. +""" + +from __future__ import annotations + +import hashlib +import json +import sys +from pathlib import Path + +import numpy as np +import pandas as pd + +sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "src")) + +from populace_dynamics.data import sipp_jobs # noqa: E402 + +YEAR = 2023 +SEEDS = (0, 1, 2, 3, 4) +AGE_BANDS = ((16, 24), (25, 34), (35, 44), (45, 54), (55, 64), (65, 99)) +THIN_CELL_PERSONS = 200 + +ARTIFACT = Path(__file__).resolve().parents[1] / ( + "runs/sipp_spell_floors_draft_v0.json" +) + + +def _age_band(age: pd.Series) -> pd.Series: + bins = [AGE_BANDS[0][0] - 1, *[hi for _, hi in AGE_BANDS]] + labels = [f"{lo}_{hi}" for lo, hi in AGE_BANDS] + return pd.cut(age, bins=bins, labels=labels) + + +def _half(person_id: pd.Series, seed: int) -> pd.Series: + """Deterministic person-disjoint half assignment.""" + + def bucket(pid: str) -> int: + digest = hashlib.sha256(f"{seed}:{pid}".encode()).digest() + return digest[0] & 1 + + return person_id.map(bucket) + + +def retention_frame(job_months: pd.DataFrame) -> pd.DataFrame: + """One row per person x consecutive-month pair, employed both.""" + person_months = ( + job_months.groupby(["person_id", "month"]) + .agg( + jobs=("job_id", frozenset), + age=("age", "first"), + sex=("sex", "first"), + weight=("weight", "first"), + ) + .reset_index() + ) + nxt = person_months.copy() + nxt["month"] -= 1 + pairs = person_months.merge( + nxt, + on=["person_id", "month"], + suffixes=("", "_next"), + ) + pairs["retained"] = [ + bool(a & b) for a, b in zip(pairs.jobs, pairs.jobs_next, strict=True) + ] + pairs["age_band"] = _age_band(pairs["age"]) + return pairs[pairs["age_band"].notna()] + + +def run_length_frame(job_months: pd.DataFrame) -> pd.DataFrame: + """One row per person: longest same-employer run in the year.""" + spells = sipp_jobs.job_spells(job_months) + person_attrs = job_months.groupby("person_id").agg( + age=("age", "first"), weight=("weight", "first") + ) + longest = spells.groupby("person_id")["n_months"].max() + out = person_attrs.join(longest).dropna(subset=["n_months"]) + out["age_band"] = _age_band(out["age"]) + return out[out["age_band"].notna()].reset_index() + + +def _weighted_rate(frame: pd.DataFrame, flag: str) -> float: + total = frame["weight"].sum() + return float((frame["weight"] * frame[flag]).sum() / total) + + +def floors_for_retention(pairs: pd.DataFrame) -> dict: + cells = {} + for (band, sex), cell in pairs.groupby(["age_band", "sex"], observed=True): + gaps = [] + halves_n = [] + for seed in SEEDS: + half = _half(cell["person_id"], seed) + a = cell[half == 0] + b = cell[half == 1] + halves_n.append( + min(a["person_id"].nunique(), b["person_id"].nunique()) + ) + ra, rb = _weighted_rate(a, "retained"), _weighted_rate( + b, "retained" + ) + gaps.append(abs(np.log(ra / rb))) + cells[f"{band}|sex{int(sex)}"] = { + "rate": round(_weighted_rate(cell, "retained"), 4), + "pairs_unweighted": int(len(cell)), + "floor_abs_log_ratio_mean": round(float(np.mean(gaps)), 5), + "floor_abs_log_ratio_sd": round(float(np.std(gaps)), 5), + "thin": bool(min(halves_n) < THIN_CELL_PERSONS), + } + return cells + + +def floors_for_runs(runs: pd.DataFrame) -> dict: + runs = runs.assign(long_run=runs["n_months"] >= 12) + cells = {} + for band, cell in runs.groupby("age_band", observed=True): + gaps = [] + halves_n = [] + for seed in SEEDS: + half = _half(cell["person_id"], seed) + a, b = cell[half == 0], cell[half == 1] + halves_n.append(min(len(a), len(b))) + ra, rb = _weighted_rate(a, "long_run"), _weighted_rate( + b, "long_run" + ) + gaps.append(abs(np.log(ra / rb))) + cells[str(band)] = { + "full_year_run_share": round(_weighted_rate(cell, "long_run"), 4), + "persons_unweighted": int(len(cell)), + "floor_abs_log_ratio_mean": round(float(np.mean(gaps)), 5), + "floor_abs_log_ratio_sd": round(float(np.std(gaps)), 5), + "thin": bool(min(halves_n) < THIN_CELL_PERSONS), + } + return cells + + +def build() -> dict: + job_months = sipp_jobs.read_sipp_job_months(YEAR) + pairs = retention_frame(job_months) + runs = run_length_frame(job_months) + return { + "artifact": "sipp_spell_floors", + "version": "draft_v0", + "status": "DRAFT - NOT RATIFIED; C3 not locked; no thresholds", + "issue": "192", + "source": f"pu{YEAR} (reference year {YEAR - 1})", + "method": ( + "person-disjoint sha256 half-splits, seeds 0-4; per-cell " + "|log(rate_a/rate_b)| mean/sd across seeds; weighted by " + "WPFINWGT" + ), + "seam_caveat": ( + "both halves share SIPP seam structure; the seam-vs-J2J " + "reconciliation run is a separate required artifact " + "before thresholds lock" + ), + "e4_retention_by_age_sex": floors_for_retention(pairs), + "e5_runs_by_age": floors_for_runs(runs), + } + + +def main() -> None: + artifact = build() + ARTIFACT.write_text(json.dumps(artifact, indent=2) + "\n") + print(f"wrote {ARTIFACT}") + e4 = artifact["e4_retention_by_age_sex"] + print(f"E4 cells: {len(e4)}; example:", next(iter(e4.items()))) + + +if __name__ == "__main__": + main() From 370736d8da09891c0d96a667c04d4c79d1e029c1 Mon Sep 17 00:00:00 2001 From: Daphne Hansell <128793799+daphnehanse11@users.noreply.github.com> Date: Wed, 15 Jul 2026 13:52:45 -0400 Subject: [PATCH 02/13] DRAFT: E3 tenure noise floors (with a heaping-robust ECDF variant) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Completes the tenure side of the Workstream A floor battery: person- disjoint half-vs-half floors for the E3 moment (tenure P25/P50/P75 by BLS age band) on all three staged January supplements. Method finding worth the referee round's attention: reported tenure heaps hard on integers, so absolute quantile gaps between halves are EXACTLY zero in 36/63 cells (both halves' quantiles land on the same heap) — a degenerate basis for a quantile-error-vs-floor criterion. Each cell therefore also carries a weighted-ECDF max-gap floor (1.5-5.6% in 2024, tightest in the thick prime-age cells), which is smooth under heaping; C3 can choose between the formulations with both on the record. Co-Authored-By: Claude Fable 5 --- runs/tenure_floors_draft_v0.json | 542 +++++++++++++++++++++++++++++++ scripts/build_tenure_floors.py | 207 ++++++++++++ 2 files changed, 749 insertions(+) create mode 100644 runs/tenure_floors_draft_v0.json create mode 100644 scripts/build_tenure_floors.py diff --git a/runs/tenure_floors_draft_v0.json b/runs/tenure_floors_draft_v0.json new file mode 100644 index 00000000..142ef92e --- /dev/null +++ b/runs/tenure_floors_draft_v0.json @@ -0,0 +1,542 @@ +{ + "artifact": "tenure_floors", + "version": "draft_v0", + "status": "DRAFT - NOT RATIFIED; C3 not locked; no thresholds", + "issue": "192", + "source": "CPS January supplements 2020/2022/2024 (PTST1TN, PWTENWGT); reader per #205", + "method": "person-disjoint sha256 half-splits, seeds 0-4; per-cell absolute weighted-quantile gap in years AND weighted-ECDF max gap (heaping-robust), mean/sd across seeds; BLS age bands", + "heaping_caveat": "reported tenure heaps on integers, so half-vs-half quantile gaps are frequently exactly zero (36/63 cells in the first build) - a degenerate threshold basis; the ECDF max-gap floor is the heaping-robust alternative for the C3 round to choose between", + "by_year": { + "2020": { + "16_19": { + "p25": 0.33, + "p50": 0.58, + "p75": 1.08, + "persons_unweighted": 1345, + "floor_abs_gap_years": { + "p25": { + "mean": 0.015, + "sd": 0.03 + }, + "p50": { + "mean": 0.06, + "sd": 0.038 + }, + "p75": { + "mean": 0.091, + "sd": 0.042 + } + }, + "floor_ecdf_max_gap": { + "mean": 0.0383, + "sd": 0.0083 + }, + "thin": false + }, + "20_24": { + "p25": 0.5, + "p50": 1.0, + "p75": 2.5, + "persons_unweighted": 3443, + "floor_abs_gap_years": { + "p25": { + "mean": 0.0, + "sd": 0.0 + }, + "p50": { + "mean": 0.033, + "sd": 0.038 + }, + "p75": { + "mean": 0.143, + "sd": 0.081 + } + }, + "floor_ecdf_max_gap": { + "mean": 0.0276, + "sd": 0.0059 + }, + "thin": false + }, + "25_34": { + "p25": 1.0, + "p50": 3.0, + "p75": 5.0, + "persons_unweighted": 9659, + "floor_abs_gap_years": { + "p25": { + "mean": 0.0, + "sd": 0.0 + }, + "p50": { + "mean": 0.0, + "sd": 0.0 + }, + "p75": { + "mean": 0.0, + "sd": 0.0 + } + }, + "floor_ecdf_max_gap": { + "mean": 0.018, + "sd": 0.0059 + }, + "thin": false + }, + "35_44": { + "p25": 2.0, + "p50": 5.0, + "p75": 10.0, + "persons_unweighted": 9915, + "floor_abs_gap_years": { + "p25": { + "mean": 0.016, + "sd": 0.032 + }, + "p50": { + "mean": 0.0, + "sd": 0.0 + }, + "p75": { + "mean": 0.0, + "sd": 0.0 + } + }, + "floor_ecdf_max_gap": { + "mean": 0.015, + "sd": 0.002 + }, + "thin": false + }, + "45_54": { + "p25": 3.0, + "p50": 8.0, + "p75": 16.0, + "persons_unweighted": 9624, + "floor_abs_gap_years": { + "p25": { + "mean": 0.0, + "sd": 0.0 + }, + "p50": { + "mean": 0.0, + "sd": 0.0 + }, + "p75": { + "mean": 0.0, + "sd": 0.0 + } + }, + "floor_ecdf_max_gap": { + "mean": 0.0129, + "sd": 0.0049 + }, + "thin": false + }, + "55_64": { + "p25": 4.0, + "p50": 10.0, + "p75": 21.0, + "persons_unweighted": 8858, + "floor_abs_gap_years": { + "p25": { + "mean": 0.0, + "sd": 0.0 + }, + "p50": { + "mean": 0.0, + "sd": 0.0 + }, + "p75": { + "mean": 0.2, + "sd": 0.4 + } + }, + "floor_ecdf_max_gap": { + "mean": 0.0187, + "sd": 0.0048 + }, + "thin": false + }, + "65_200": { + "p25": 5.0, + "p50": 13.0, + "p75": 25.0, + "persons_unweighted": 3719, + "floor_abs_gap_years": { + "p25": { + "mean": 0.0, + "sd": 0.0 + }, + "p50": { + "mean": 1.2, + "sd": 0.98 + }, + "p75": { + "mean": 1.0, + "sd": 0.894 + } + }, + "floor_ecdf_max_gap": { + "mean": 0.0297, + "sd": 0.0075 + }, + "thin": false + } + }, + "2022": { + "16_19": { + "p25": 0.33, + "p50": 0.5, + "p75": 1.0, + "persons_unweighted": 1218, + "floor_abs_gap_years": { + "p25": { + "mean": 0.059, + "sd": 0.031 + }, + "p50": { + "mean": 0.0, + "sd": 0.0 + }, + "p75": { + "mean": 0.0, + "sd": 0.0 + } + }, + "floor_ecdf_max_gap": { + "mean": 0.0429, + "sd": 0.0106 + }, + "thin": false + }, + "20_24": { + "p25": 0.5, + "p50": 1.0, + "p75": 2.17, + "persons_unweighted": 2825, + "floor_abs_gap_years": { + "p25": { + "mean": 0.032, + "sd": 0.039 + }, + "p50": { + "mean": 0.0, + "sd": 0.0 + }, + "p75": { + "mean": 0.218, + "sd": 0.108 + } + }, + "floor_ecdf_max_gap": { + "mean": 0.0292, + "sd": 0.0096 + }, + "thin": false + }, + "25_34": { + "p25": 0.92, + "p50": 3.0, + "p75": 5.0, + "persons_unweighted": 8239, + "floor_abs_gap_years": { + "p25": { + "mean": 0.033, + "sd": 0.065 + }, + "p50": { + "mean": 0.0, + "sd": 0.0 + }, + "p75": { + "mean": 0.0, + "sd": 0.0 + } + }, + "floor_ecdf_max_gap": { + "mean": 0.014, + "sd": 0.0037 + }, + "thin": false + }, + "35_44": { + "p25": 2.0, + "p50": 5.0, + "p75": 10.0, + "persons_unweighted": 8630, + "floor_abs_gap_years": { + "p25": { + "mean": 0.0, + "sd": 0.0 + }, + "p50": { + "mean": 0.0, + "sd": 0.0 + }, + "p75": { + "mean": 0.0, + "sd": 0.0 + } + }, + "floor_ecdf_max_gap": { + "mean": 0.0181, + "sd": 0.0069 + }, + "thin": false + }, + "45_54": { + "p25": 3.0, + "p50": 7.0, + "p75": 16.0, + "persons_unweighted": 7907, + "floor_abs_gap_years": { + "p25": { + "mean": 0.0, + "sd": 0.0 + }, + "p50": { + "mean": 0.0, + "sd": 0.0 + }, + "p75": { + "mean": 1.0, + "sd": 0.0 + } + }, + "floor_ecdf_max_gap": { + "mean": 0.0248, + "sd": 0.0066 + }, + "thin": false + }, + "55_64": { + "p25": 4.0, + "p50": 10.0, + "p75": 21.0, + "persons_unweighted": 7138, + "floor_abs_gap_years": { + "p25": { + "mean": 0.0, + "sd": 0.0 + }, + "p50": { + "mean": 0.0, + "sd": 0.0 + }, + "p75": { + "mean": 1.0, + "sd": 0.0 + } + }, + "floor_ecdf_max_gap": { + "mean": 0.0253, + "sd": 0.0052 + }, + "thin": false + }, + "65_200": { + "p25": 5.0, + "p50": 12.0, + "p75": 25.0, + "persons_unweighted": 3153, + "floor_abs_gap_years": { + "p25": { + "mean": 0.209, + "sd": 0.396 + }, + "p50": { + "mean": 0.4, + "sd": 0.49 + }, + "p75": { + "mean": 0.0, + "sd": 0.0 + } + }, + "floor_ecdf_max_gap": { + "mean": 0.0268, + "sd": 0.0081 + }, + "thin": false + } + }, + "2024": { + "16_19": { + "p25": 0.33, + "p50": 0.67, + "p75": 1.5, + "persons_unweighted": 1223, + "floor_abs_gap_years": { + "p25": { + "mean": 0.016, + "sd": 0.032 + }, + "p50": { + "mean": 0.036, + "sd": 0.044 + }, + "p75": { + "mean": 0.282, + "sd": 0.039 + } + }, + "floor_ecdf_max_gap": { + "mean": 0.0564, + "sd": 0.0175 + }, + "thin": false + }, + "20_24": { + "p25": 0.5, + "p50": 1.17, + "p75": 2.92, + "persons_unweighted": 2999, + "floor_abs_gap_years": { + "p25": { + "mean": 0.0, + "sd": 0.0 + }, + "p50": { + "mean": 0.066, + "sd": 0.063 + }, + "p75": { + "mean": 0.405, + "sd": 0.19 + } + }, + "floor_ecdf_max_gap": { + "mean": 0.027, + "sd": 0.0045 + }, + "thin": false + }, + "25_34": { + "p25": 1.0, + "p50": 3.0, + "p75": 5.0, + "persons_unweighted": 8250, + "floor_abs_gap_years": { + "p25": { + "mean": 0.0, + "sd": 0.0 + }, + "p50": { + "mean": 0.0, + "sd": 0.0 + }, + "p75": { + "mean": 0.0, + "sd": 0.0 + } + }, + "floor_ecdf_max_gap": { + "mean": 0.0184, + "sd": 0.0051 + }, + "thin": false + }, + "35_44": { + "p25": 2.0, + "p50": 5.0, + "p75": 10.0, + "persons_unweighted": 8932, + "floor_abs_gap_years": { + "p25": { + "mean": 0.0, + "sd": 0.0 + }, + "p50": { + "mean": 0.0, + "sd": 0.0 + }, + "p75": { + "mean": 0.0, + "sd": 0.0 + } + }, + "floor_ecdf_max_gap": { + "mean": 0.0148, + "sd": 0.0035 + }, + "thin": false + }, + "45_54": { + "p25": 3.0, + "p50": 7.0, + "p75": 15.0, + "persons_unweighted": 7935, + "floor_abs_gap_years": { + "p25": { + "mean": 0.0, + "sd": 0.0 + }, + "p50": { + "mean": 0.0, + "sd": 0.0 + }, + "p75": { + "mean": 0.536, + "sd": 0.453 + } + }, + "floor_ecdf_max_gap": { + "mean": 0.0151, + "sd": 0.003 + }, + "thin": false + }, + "55_64": { + "p25": 4.0, + "p50": 10.0, + "p75": 20.0, + "persons_unweighted": 7012, + "floor_abs_gap_years": { + "p25": { + "mean": 0.0, + "sd": 0.0 + }, + "p50": { + "mean": 0.0, + "sd": 0.0 + }, + "p75": { + "mean": 0.2, + "sd": 0.4 + } + }, + "floor_ecdf_max_gap": { + "mean": 0.022, + "sd": 0.0031 + }, + "thin": false + }, + "65_200": { + "p25": 4.0, + "p50": 12.0, + "p75": 24.0, + "persons_unweighted": 3332, + "floor_abs_gap_years": { + "p25": { + "mean": 0.8, + "sd": 0.4 + }, + "p50": { + "mean": 1.2, + "sd": 1.166 + }, + "p75": { + "mean": 2.0, + "sd": 0.0 + } + }, + "floor_ecdf_max_gap": { + "mean": 0.0392, + "sd": 0.0087 + }, + "thin": false + } + } + } +} diff --git a/scripts/build_tenure_floors.py b/scripts/build_tenure_floors.py new file mode 100644 index 00000000..1f8ff403 --- /dev/null +++ b/scripts/build_tenure_floors.py @@ -0,0 +1,207 @@ +"""Build DRAFT CPS tenure noise floors for gate E3 (#192). + +REPORTED ANCHOR, NOT A GATE RUN — and explicitly a DRAFT: C3 has +not locked and no thresholds are proposed. E3's moment is the +tenure distribution (P25/P50/P75) by age band against the CPS +January supplement; this commits the person-disjoint half-vs-half +sampling-noise floor those thresholds would later be derived from, +completing the tenure side of Workstream A's floor battery +(companion to the E4/E5 SIPP floors). + +Method mirrors the spell floors: for seeds 0-4, persons split into +two disjoint sha256 halves; each cell's weighted P25/P50/P75 of +``tenure_years`` is computed on both halves and the across-seed +mean and sd of the absolute quantile gap **in years** is the floor +(quantiles are in interpretable units, so the gap is reported in +years rather than a log ratio). Reported tenure heaps hard on +integers, so half-vs-half quantile gaps are frequently EXACTLY zero +(both halves' quantiles land on the same heap) — a degenerate basis +for a "quantile error vs floor" criterion. Each cell therefore also +carries a weighted-ECDF max-gap (Kolmogorov-style) floor, which is +smooth under heaping; the C3 round can choose between the quantile +and distributional formulations with both on the record. All three +staged supplements +(2020/2022/2024) are floored independently — the across-year spread +of the floors is itself informative about supplement-to-supplement +stability. Cells with fewer than 200 unweighted persons per half +are flagged thin. + +Usage:: + + python scripts/build_tenure_floors.py + +writes ``runs/tenure_floors_draft_v0.json``. +""" + +from __future__ import annotations + +import hashlib +import json +import sys +from pathlib import Path + +import numpy as np + +REPO = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(REPO / "src")) + +from populace_dynamics.data import cps_tenure # noqa: E402 + +YEARS = (2020, 2022, 2024) +SEEDS = (0, 1, 2, 3, 4) +QUANTILES = (0.25, 0.50, 0.75) +THIN_CELL_PERSONS = 200 + +ARTIFACT = REPO / "runs/tenure_floors_draft_v0.json" + + +def _half(person_id: str, seed: int) -> int: + digest = hashlib.sha256(f"{seed}:{person_id}".encode()).digest() + return digest[0] & 1 + + +def _weighted_quantile(values, weights, q: float) -> float: + order = np.argsort(values, kind="stable") + values = np.asarray(values)[order] + weights = np.asarray(weights)[order] + cum = np.cumsum(weights) - 0.5 * weights + cum /= weights.sum() + return float(np.interp(q, cum, values)) + + +def _weighted_ecdf_max_gap(a_values, a_weights, b_values, b_weights): + """Max |F_a(x) - F_b(x)| over the union grid of observed values.""" + grid = np.union1d(a_values, b_values) + + def ecdf(values, weights): + order = np.argsort(values, kind="stable") + v = np.asarray(values)[order] + w = np.asarray(weights)[order] + cum = np.cumsum(w) / w.sum() + idx = np.searchsorted(v, grid, side="right") - 1 + return np.where(idx >= 0, cum[idx], 0.0) + + return float( + np.max(np.abs(ecdf(a_values, a_weights) - ecdf(b_values, b_weights))) + ) + + +def floors_for_year(year: int) -> dict: + records = cps_tenure.read_cps_tenure(year) + usable = records[ + records["tenure_years"].notna() & (records["weight"] > 0) + ].copy() + labels = [f"{lo}_{hi}" for lo, hi in cps_tenure.DEFAULT_AGE_BANDS] + import pandas as pd + + usable["age_band"] = pd.cut( + usable["age"], + bins=[cps_tenure.DEFAULT_AGE_BANDS[0][0] - 1] + + [hi for _, hi in cps_tenure.DEFAULT_AGE_BANDS], + labels=labels, + ) + usable = usable[usable["age_band"].notna()] + + cells = {} + for band, cell in usable.groupby("age_band", observed=True): + values = cell["tenure_years"].to_numpy(dtype=float) + weights = cell["weight"].to_numpy(dtype=float) + point = { + f"p{int(q * 100)}": round( + _weighted_quantile(values, weights, q), 2 + ) + for q in QUANTILES + } + halves = cell["person_id"].map( + lambda pid: [_half(pid, seed) for seed in SEEDS] + ) + gaps: dict[str, list[float]] = { + f"p{int(q * 100)}": [] for q in QUANTILES + } + ks_gaps: list[float] = [] + thin = False + for i, seed in enumerate(SEEDS): + mask_a = halves.map(lambda h, i=i: h[i] == 0) + a, b = cell[mask_a], cell[~mask_a] + if min(len(a), len(b)) < THIN_CELL_PERSONS: + thin = True + for q in QUANTILES: + qa = _weighted_quantile( + a["tenure_years"].to_numpy(dtype=float), + a["weight"].to_numpy(dtype=float), + q, + ) + qb = _weighted_quantile( + b["tenure_years"].to_numpy(dtype=float), + b["weight"].to_numpy(dtype=float), + q, + ) + gaps[f"p{int(q * 100)}"].append(abs(qa - qb)) + ks_gaps.append( + _weighted_ecdf_max_gap( + a["tenure_years"].to_numpy(dtype=float), + a["weight"].to_numpy(dtype=float), + b["tenure_years"].to_numpy(dtype=float), + b["weight"].to_numpy(dtype=float), + ) + ) + cells[str(band)] = { + **point, + "persons_unweighted": int(len(cell)), + "floor_abs_gap_years": { + name: { + "mean": round(float(np.mean(values)), 3), + "sd": round(float(np.std(values)), 3), + } + for name, values in gaps.items() + }, + "floor_ecdf_max_gap": { + "mean": round(float(np.mean(ks_gaps)), 4), + "sd": round(float(np.std(ks_gaps)), 4), + }, + "thin": thin, + } + return cells + + +def build() -> dict: + return { + "artifact": "tenure_floors", + "version": "draft_v0", + "status": "DRAFT - NOT RATIFIED; C3 not locked; no thresholds", + "issue": "192", + "source": "CPS January supplements 2020/2022/2024 (PTST1TN, " + "PWTENWGT); reader per #205", + "method": ( + "person-disjoint sha256 half-splits, seeds 0-4; per-cell " + "absolute weighted-quantile gap in years AND weighted-ECDF " + "max gap (heaping-robust), mean/sd across seeds; BLS age " + "bands" + ), + "heaping_caveat": ( + "reported tenure heaps on integers, so half-vs-half " + "quantile gaps are frequently exactly zero (36/63 cells " + "in the first build) - a degenerate threshold basis; the " + "ECDF max-gap floor is the heaping-robust alternative " + "for the C3 round to choose between" + ), + "by_year": {str(year): floors_for_year(year) for year in YEARS}, + } + + +def main() -> None: + artifact = build() + ARTIFACT.write_text(json.dumps(artifact, indent=2) + "\n") + print(f"wrote {ARTIFACT}") + y24 = artifact["by_year"]["2024"] + example = y24["35_44"] + print( + "example 35_44 (2024): p50 =", + example["p50"], + "floor(p50) =", + example["floor_abs_gap_years"]["p50"], + ) + + +if __name__ == "__main__": + main() From fc9fcec8114b8ea001bea2be6c3b7bee28bec131 Mon Sep 17 00:00:00 2001 From: Daphne Hansell <128793799+daphnehanse11@users.noreply.github.com> Date: Wed, 15 Jul 2026 13:53:08 -0400 Subject: [PATCH 03/13] Lint: unused loop variable (B007) Co-Authored-By: Claude Fable 5 --- scripts/build_tenure_floors.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/build_tenure_floors.py b/scripts/build_tenure_floors.py index 1f8ff403..7257f995 100644 --- a/scripts/build_tenure_floors.py +++ b/scripts/build_tenure_floors.py @@ -120,7 +120,7 @@ def floors_for_year(year: int) -> dict: } ks_gaps: list[float] = [] thin = False - for i, seed in enumerate(SEEDS): + for i, _seed in enumerate(SEEDS): mask_a = halves.map(lambda h, i=i: h[i] == 0) a, b = cell[mask_a], cell[~mask_a] if min(len(a), len(b)) < THIN_CELL_PERSONS: From 835e85e764a6e946155079f272d93b88c8139886 Mon Sep 17 00:00:00 2001 From: Daphne Hansell <128793799+daphnehanse11@users.noreply.github.com> Date: Wed, 15 Jul 2026 14:32:41 -0400 Subject: [PATCH 04/13] =?UTF-8?q?DRAFT:=20E8/E9=20SIPP=20floors=20?= =?UTF-8?q?=E2=80=94=20completing=20the=20Workstream=20A=20battery?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit E8 (nonemployment spells, full-year persons, censoring-free draft restriction recorded): any-nonemployment share 41.5% (16-24) falling to 8.5% (45-54) with the retiree uptick at 65+; floors |log ratio| 0.05-0.14; no thin cells. E9 (earnings change by transition): monthly person-transition mix stay 97.7% / entry 1.01% / exit 0.94% / j2j 0.35%, consistent with the seam-reconciliation rates; j2j median log-change +0.226 (IQR 1.07, n=545, not thin). Second heaping finding for the referee round: within-job SIPP monthly earnings are mostly wave-constant, so the STAY median log-change heaps at exactly 0 and its floor is degenerate — same failure class as the tenure quantile heaping. E9-stay thresholds should be stated on the IQR (0.0656, floored at 0.004) or a distributional distance; caveat recorded in the artifact. Co-Authored-By: Claude Fable 5 --- runs/sipp_e8_e9_floors_draft_v0.json | 140 +++++++++++ scripts/build_sipp_e8_e9_floors.py | 337 +++++++++++++++++++++++++++ 2 files changed, 477 insertions(+) create mode 100644 runs/sipp_e8_e9_floors_draft_v0.json create mode 100644 scripts/build_sipp_e8_e9_floors.py diff --git a/runs/sipp_e8_e9_floors_draft_v0.json b/runs/sipp_e8_e9_floors_draft_v0.json new file mode 100644 index 00000000..e81b3f0b --- /dev/null +++ b/runs/sipp_e8_e9_floors_draft_v0.json @@ -0,0 +1,140 @@ +{ + "artifact": "sipp_e8_e9_floors", + "version": "draft_v0", + "status": "DRAFT - NOT RATIFIED; C3 not locked; no thresholds", + "issue": "192", + "source": "pu2023 (reference year 2022), persons observed all 12 reference months (censoring-free draft restriction, recorded)", + "method": "person-disjoint sha256 half-splits, seeds 0-4; rates floored on |log rate ratio|, earnings-change medians/IQRs on absolute gaps in log-points; weighted by WPFINWGT", + "seam_caveat": "identical to the E4/E5 floors: half-splits share SIPP's seam structure; #214 carries the seam measurement", + "stay_median_heaping_caveat": "within-job SIPP monthly earnings are mostly constant across a wave (dependent-interview reporting), so the stay-transition median log-change heaps at exactly 0 and its half-vs-half floor is degenerate (0.0) - the same failure class as the tenure quantile heaping; E9-stay thresholds should be stated on the IQR or a distributional distance, not the median", + "e8_nonemployment_by_age": { + "16_24": { + "any_nonemp_share": 0.4145, + "long_nonemp_share": 0.3422, + "persons_unweighted": 1781, + "any_nonemp": { + "floor_abs_log_ratio_mean": 0.08001, + "floor_abs_log_ratio_sd": 0.0466, + "thin": false + }, + "long_nonemp": { + "floor_abs_log_ratio_mean": 0.10556, + "floor_abs_log_ratio_sd": 0.04714, + "thin": false + } + }, + "25_34": { + "any_nonemp_share": 0.1401, + "long_nonemp_share": 0.0973, + "persons_unweighted": 3240, + "any_nonemp": { + "floor_abs_log_ratio_mean": 0.07301, + "floor_abs_log_ratio_sd": 0.04942, + "thin": false + }, + "long_nonemp": { + "floor_abs_log_ratio_mean": 0.14005, + "floor_abs_log_ratio_sd": 0.06264, + "thin": false + } + }, + "35_44": { + "any_nonemp_share": 0.1017, + "long_nonemp_share": 0.0663, + "persons_unweighted": 3543, + "any_nonemp": { + "floor_abs_log_ratio_mean": 0.1424, + "floor_abs_log_ratio_sd": 0.09863, + "thin": false + }, + "long_nonemp": { + "floor_abs_log_ratio_mean": 0.20084, + "floor_abs_log_ratio_sd": 0.10088, + "thin": false + } + }, + "45_54": { + "any_nonemp_share": 0.0852, + "long_nonemp_share": 0.0555, + "persons_unweighted": 3241, + "any_nonemp": { + "floor_abs_log_ratio_mean": 0.10559, + "floor_abs_log_ratio_sd": 0.11745, + "thin": false + }, + "long_nonemp": { + "floor_abs_log_ratio_mean": 0.14665, + "floor_abs_log_ratio_sd": 0.07165, + "thin": false + } + }, + "55_64": { + "any_nonemp_share": 0.1045, + "long_nonemp_share": 0.083, + "persons_unweighted": 3449, + "any_nonemp": { + "floor_abs_log_ratio_mean": 0.14077, + "floor_abs_log_ratio_sd": 0.05732, + "thin": false + }, + "long_nonemp": { + "floor_abs_log_ratio_mean": 0.14983, + "floor_abs_log_ratio_sd": 0.05971, + "thin": false + } + }, + "65_99": { + "any_nonemp_share": 0.1887, + "long_nonemp_share": 0.1546, + "persons_unweighted": 2286, + "any_nonemp": { + "floor_abs_log_ratio_mean": 0.05111, + "floor_abs_log_ratio_sd": 0.04105, + "thin": false + }, + "long_nonemp": { + "floor_abs_log_ratio_mean": 0.09334, + "floor_abs_log_ratio_sd": 0.03649, + "thin": false + } + } + }, + "e9_transitions": { + "transition_rates": { + "entry": 0.0101, + "exit": 0.0094, + "j2j": 0.0035, + "stay": 0.977 + }, + "earnings_change": { + "stay": { + "median_log_change": 0.0, + "iqr_log_change": 0.0656, + "pairs_unweighted": 163552, + "floor_abs_median_gap": { + "mean": 0.0, + "sd": 0.0 + }, + "floor_abs_iqr_gap": { + "mean": 0.0, + "sd": 0.0 + }, + "thin": false + }, + "j2j": { + "median_log_change": 0.2264, + "iqr_log_change": 1.069, + "pairs_unweighted": 545, + "floor_abs_median_gap": { + "mean": 0.07658, + "sd": 0.03448 + }, + "floor_abs_iqr_gap": { + "mean": 0.08423, + "sd": 0.05215 + }, + "thin": false + } + } + } +} diff --git a/scripts/build_sipp_e8_e9_floors.py b/scripts/build_sipp_e8_e9_floors.py new file mode 100644 index 00000000..af3ecdb8 --- /dev/null +++ b/scripts/build_sipp_e8_e9_floors.py @@ -0,0 +1,337 @@ +"""Build DRAFT SIPP floors for gates E8 and E9 (#192). + +REPORTED ANCHOR, NOT A GATE RUN — and explicitly a DRAFT: C3 has +not locked, no thresholds are proposed. Completes Workstream A's +floor battery (E3 tenure and E4/E5 spells are committed siblings; +E10 needs no floor — it is pass/fail on the locked PSID gates). + +Moments, on the pu2023 file (reference year 2022), restricted to +persons observed in the panel for all 12 reference months so spell +durations are not right-censored by sample exit (the restriction is +recorded, not hidden): + +(a) **E8 — nonemployment spell durations** (the zero-spell battery + analog). A person-month is nonemployed when the person is in + the panel that month with no active EJB job. Among persons with + at least one employed month, maximal nonemployment runs are + collapsed; moments per age band: the weighted share of persons + with any nonemployment spell, and the weighted share of those + spells lasting >= 3 months. + +(b) **E9 — earnings change by transition type** (the layering- + coherence moment). Consecutive-month person transitions are + classified stay (a common job id), j2j (employed both months, + no common id), exit (employed -> nonemployed), entry + (nonemployed -> employed). For stay and j2j, where both months' + known earnings totals are positive, the moment is the weighted + median and IQR of log(earn_{m+1} / earn_m). Exit/entry carry + rates, not earnings changes (their change is to/from zero by + construction). + +(c) **The floor**: person-disjoint sha256 half-splits, seeds 0-4; + for rates the |log rate ratio| between halves, for medians/IQRs + the absolute gap in log-points; mean/sd across seeds. Cells + under 200 unweighted persons per half are flagged thin. + +Seam caveat: identical to the E4/E5 floors — both halves share +SIPP's seam structure, so these floors cannot see seam bias; the +reconciliation artifact (#214) carries that measurement. + +Usage:: + + python scripts/build_sipp_e8_e9_floors.py + +writes ``runs/sipp_e8_e9_floors_draft_v0.json``. +""" + +from __future__ import annotations + +import hashlib +import json +import os +import sys +from pathlib import Path + +import numpy as np +import pandas as pd + +REPO = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(REPO / "src")) + +from populace_dynamics.data import sipp_jobs # noqa: E402 + +YEAR = 2023 +SEEDS = (0, 1, 2, 3, 4) +AGE_BANDS = ((16, 24), (25, 34), (35, 44), (45, 54), (55, 64), (65, 99)) +THIN_CELL_PERSONS = 200 + +ARTIFACT = REPO / "runs/sipp_e8_e9_floors_draft_v0.json" + + +def _age_band(age: pd.Series) -> pd.Series: + bins = [AGE_BANDS[0][0] - 1, *[hi for _, hi in AGE_BANDS]] + labels = [f"{lo}_{hi}" for lo, hi in AGE_BANDS] + return pd.cut(age, bins=bins, labels=labels) + + +def _half(person_id: str, seed: int) -> int: + digest = hashlib.sha256(f"{seed}:{person_id}".encode()).digest() + return digest[0] & 1 + + +def _person_month_universe(year: int) -> pd.DataFrame: + data_dir = Path( + os.environ.get( + "POPULACE_DYNAMICS_SIPP_DIR", + str(Path("~/PolicyEngine/sipp-data").expanduser()), + ) + ).expanduser() + for suffix in (".csv", ".csv.gz"): + path = data_dir / f"pu{year}{suffix}" + if path.exists(): + break + else: + raise FileNotFoundError(f"pu{year}.csv[.gz] not staged") + raw = pd.read_csv( + path, + sep="|", + usecols=["SSUID", "PNUM", "MONTHCODE", "WPFINWGT", "TAGE"], + dtype={"SSUID": "string"}, + ) + raw["person_id"] = raw["SSUID"].astype(str) + "-" + raw["PNUM"].astype(str) + return raw.rename( + columns={"MONTHCODE": "month", "WPFINWGT": "weight", "TAGE": "age"} + )[["person_id", "month", "weight", "age"]] + + +def build_panel(year: int) -> pd.DataFrame: + """Person x month grid for full-year persons, with jobs/earnings.""" + universe = _person_month_universe(year) + counts = universe.groupby("person_id")["month"].nunique() + full_year = set(counts[counts == 12].index) + universe = universe[universe["person_id"].isin(full_year)] + + job_months = sipp_jobs.read_sipp_job_months(year) + jobs = ( + job_months.groupby(["person_id", "month"]) + .agg(jobs=("job_id", frozenset), earn=("earnings", "sum")) + .reset_index() + ) + panel = universe.merge(jobs, on=["person_id", "month"], how="left") + panel["jobs"] = panel["jobs"].apply( + lambda x: x if isinstance(x, frozenset) else frozenset() + ) + panel["employed"] = panel["jobs"].map(len) > 0 + return panel.sort_values(["person_id", "month"]) + + +def e8_person_frame(panel: pd.DataFrame) -> pd.DataFrame: + """Per person: any-nonemployment flag and longest N-spell.""" + rows = [] + for person, grp in panel.groupby("person_id", sort=False): + employed = grp.sort_values("month")["employed"].to_numpy() + if not employed.any(): + continue # never employed: outside the E8 universe + runs = [] + run = 0 + for e in employed: + if not e: + run += 1 + elif run: + runs.append(run) + run = 0 + if run: + runs.append(run) + rows.append( + { + "person_id": person, + "age": grp["age"].iloc[0], + "weight": grp["weight"].iloc[0], + "any_nonemp": bool(runs), + "long_nonemp": bool(runs and max(runs) >= 3), + } + ) + out = pd.DataFrame(rows) + out["age_band"] = _age_band(out["age"]) + return out[out["age_band"].notna()] + + +def e9_transition_frame(panel: pd.DataFrame) -> pd.DataFrame: + """Person x month-pair transitions with earnings changes.""" + nxt = panel[["person_id", "month", "jobs", "earn", "employed"]].copy() + nxt["month"] -= 1 + pairs = panel.merge(nxt, on=["person_id", "month"], suffixes=("", "_n")) + + def classify(row) -> str: + if row["employed"] and row["employed_n"]: + return "stay" if row["jobs"] & row["jobs_n"] else "j2j" + if row["employed"] and not row["employed_n"]: + return "exit" + if not row["employed"] and row["employed_n"]: + return "entry" + return "neither" + + pairs["transition"] = pairs.apply(classify, axis=1) + pairs = pairs[pairs["transition"] != "neither"].copy() + both_known = ( + pairs["earn"].notna() + & pairs["earn_n"].notna() + & (pairs["earn"] > 0) + & (pairs["earn_n"] > 0) + ) + pairs["log_change"] = np.nan + mask = both_known & pairs["transition"].isin(("stay", "j2j")) + pairs.loc[mask, "log_change"] = np.log( + pairs.loc[mask, "earn_n"] / pairs.loc[mask, "earn"] + ) + pairs["age_band"] = _age_band(pairs["age"]) + return pairs[pairs["age_band"].notna()] + + +def _weighted_quantile(values, weights, q: float) -> float: + order = np.argsort(values, kind="stable") + values = np.asarray(values)[order] + weights = np.asarray(weights)[order] + cum = np.cumsum(weights) - 0.5 * weights + cum /= weights.sum() + return float(np.interp(q, cum, values)) + + +def _rate_floor(cell: pd.DataFrame, flag: str) -> dict: + gaps, halves_n = [], [] + for seed in SEEDS: + half = cell["person_id"].map(lambda p, s=seed: _half(p, s)) + a, b = cell[half == 0], cell[half == 1] + halves_n.append(min(len(a), len(b))) + ra = float((a["weight"] * a[flag]).sum() / a["weight"].sum()) + rb = float((b["weight"] * b[flag]).sum() / b["weight"].sum()) + gaps.append(abs(np.log(ra / rb)) if ra > 0 and rb > 0 else np.nan) + gaps = [g for g in gaps if not np.isnan(g)] + return { + "floor_abs_log_ratio_mean": round(float(np.mean(gaps)), 5), + "floor_abs_log_ratio_sd": round(float(np.std(gaps)), 5), + "thin": bool(min(halves_n) < THIN_CELL_PERSONS), + } + + +def e8_floors(persons: pd.DataFrame) -> dict: + cells = {} + for band, cell in persons.groupby("age_band", observed=True): + w = cell["weight"] + cells[str(band)] = { + "any_nonemp_share": round( + float((w * cell["any_nonemp"]).sum() / w.sum()), 4 + ), + "long_nonemp_share": round( + float((w * cell["long_nonemp"]).sum() / w.sum()), 4 + ), + "persons_unweighted": int(len(cell)), + "any_nonemp": _rate_floor(cell, "any_nonemp"), + "long_nonemp": _rate_floor(cell, "long_nonemp"), + } + return cells + + +def e9_floors(pairs: pd.DataFrame) -> dict: + out: dict = {"transition_rates": {}, "earnings_change": {}} + monthly = pairs.groupby("transition")["weight"].sum() + total = monthly.sum() + out["transition_rates"] = { + t: round(float(v / total), 4) for t, v in monthly.items() + } + for kind in ("stay", "j2j"): + cell = pairs[ + (pairs["transition"] == kind) & pairs["log_change"].notna() + ] + values = cell["log_change"].to_numpy(dtype=float) + weights = cell["weight"].to_numpy(dtype=float) + med = _weighted_quantile(values, weights, 0.5) + iqr = _weighted_quantile(values, weights, 0.75) - _weighted_quantile( + values, weights, 0.25 + ) + med_gaps, iqr_gaps, halves_n = [], [], [] + for seed in SEEDS: + half = cell["person_id"].map(lambda p, s=seed: _half(p, s)) + a, b = cell[half == 0], cell[half == 1] + halves_n.append(min(len(a), len(b))) + + def q(frame, qq): + return _weighted_quantile( + frame["log_change"].to_numpy(dtype=float), + frame["weight"].to_numpy(dtype=float), + qq, + ) + + med_gaps.append(abs(q(a, 0.5) - q(b, 0.5))) + iqr_gaps.append( + abs((q(a, 0.75) - q(a, 0.25)) - (q(b, 0.75) - q(b, 0.25))) + ) + out["earnings_change"][kind] = { + "median_log_change": round(med, 4), + "iqr_log_change": round(iqr, 4), + "pairs_unweighted": int(len(cell)), + "floor_abs_median_gap": { + "mean": round(float(np.mean(med_gaps)), 5), + "sd": round(float(np.std(med_gaps)), 5), + }, + "floor_abs_iqr_gap": { + "mean": round(float(np.mean(iqr_gaps)), 5), + "sd": round(float(np.std(iqr_gaps)), 5), + }, + "thin": bool(min(halves_n) < THIN_CELL_PERSONS), + } + return out + + +def build() -> dict: + panel = build_panel(YEAR) + persons = e8_person_frame(panel) + pairs = e9_transition_frame(panel) + return { + "artifact": "sipp_e8_e9_floors", + "version": "draft_v0", + "status": "DRAFT - NOT RATIFIED; C3 not locked; no thresholds", + "issue": "192", + "source": f"pu{YEAR} (reference year {YEAR - 1}), persons " + "observed all 12 reference months (censoring-free draft " + "restriction, recorded)", + "method": ( + "person-disjoint sha256 half-splits, seeds 0-4; rates " + "floored on |log rate ratio|, earnings-change medians/" + "IQRs on absolute gaps in log-points; weighted by " + "WPFINWGT" + ), + "seam_caveat": ( + "identical to the E4/E5 floors: half-splits share SIPP's " + "seam structure; #214 carries the seam measurement" + ), + "stay_median_heaping_caveat": ( + "within-job SIPP monthly earnings are mostly constant " + "across a wave (dependent-interview reporting), so the " + "stay-transition median log-change heaps at exactly 0 " + "and its half-vs-half floor is degenerate (0.0) - the " + "same failure class as the tenure quantile heaping; " + "E9-stay thresholds should be stated on the IQR or a " + "distributional distance, not the median" + ), + "e8_nonemployment_by_age": e8_floors(persons), + "e9_transitions": e9_floors(pairs), + } + + +def main() -> None: + artifact = build() + ARTIFACT.write_text(json.dumps(artifact, indent=2) + "\n") + print(f"wrote {ARTIFACT}") + print("e9 transition mix:", artifact["e9_transitions"]["transition_rates"]) + stay = artifact["e9_transitions"]["earnings_change"]["stay"] + print( + "stay: median log-change", + stay["median_log_change"], + "floor", + stay["floor_abs_median_gap"]["mean"], + ) + + +if __name__ == "__main__": + main() From ef5b8ab602694525d9c64e898cedccf8c4ce74ef Mon Sep 17 00:00:00 2001 From: Vahid Ahmadi Date: Thu, 16 Jul 2026 12:57:32 +0100 Subject: [PATCH 05/13] Review fixes for #212: merge master, thin-flag unit honesty, reader revision in floor metadata - Merge origin/master (picks up the current sipp_jobs.py reader; no conflicts, PR-side additions preserved) - Record thin-flag units per artifact (E4 counts distinct persons, E5/E8 rows==persons, E9 transition pairs, E3 CPS records) in the builder docstrings, build() metadata, and hand-edited artifact metadata (staged microdata unavailable; measured floors untouched) - Record the reader commit in effect (sipp_jobs.py 75d30dd for the SIPP floors, cps_tenure.py 5c9e5e6 for the tenure floors) Co-Authored-By: Claude Fable 5 --- runs/sipp_e8_e9_floors_draft_v0.json | 6 ++++ runs/sipp_spell_floors_draft_v0.json | 6 ++++ runs/tenure_floors_draft_v0.json | 3 ++ scripts/build_sipp_e8_e9_floors.py | 44 +++++++++++++++++++++++++++ scripts/build_sipp_spell_floors.py | 45 ++++++++++++++++++++++++++++ scripts/build_tenure_floors.py | 34 +++++++++++++++++++++ 6 files changed, 138 insertions(+) diff --git a/runs/sipp_e8_e9_floors_draft_v0.json b/runs/sipp_e8_e9_floors_draft_v0.json index e81b3f0b..fe696794 100644 --- a/runs/sipp_e8_e9_floors_draft_v0.json +++ b/runs/sipp_e8_e9_floors_draft_v0.json @@ -7,6 +7,12 @@ "method": "person-disjoint sha256 half-splits, seeds 0-4; rates floored on |log rate ratio|, earnings-change medians/IQRs on absolute gaps in log-points; weighted by WPFINWGT", "seam_caveat": "identical to the E4/E5 floors: half-splits share SIPP's seam structure; #214 carries the seam measurement", "stay_median_heaping_caveat": "within-job SIPP monthly earnings are mostly constant across a wave (dependent-interview reporting), so the stay-transition median log-change heaps at exactly 0 and its half-vs-half floor is degenerate (0.0) - the same failure class as the tenure quantile heaping; E9-stay thresholds should be stated on the IQR or a distributional distance, not the median", + "thin_flag_units": { + "e8_nonemployment_by_age": "rows per half, equal to persons (one row per person in the E8 frame) vs THIN_CELL_PERSONS=200", + "e9_transitions.earnings_change": "rows per half = consecutive-month transition pairs, not persons (a person can contribute up to 11) vs THIN_CELL_PERSONS=200" + }, + "sipp_jobs_reader_commit": "75d30dd57d71b91ee0929246b2f3cbb92263b350", + "metadata_edit_note": "metadata keys thin_flag_units and *_reader_commit were hand-added post hoc (staged microdata not available on the editing machine); no measured floor numbers were changed", "e8_nonemployment_by_age": { "16_24": { "any_nonemp_share": 0.4145, diff --git a/runs/sipp_spell_floors_draft_v0.json b/runs/sipp_spell_floors_draft_v0.json index 0926de4b..e3b7e202 100644 --- a/runs/sipp_spell_floors_draft_v0.json +++ b/runs/sipp_spell_floors_draft_v0.json @@ -6,6 +6,12 @@ "source": "pu2023 (reference year 2022)", "method": "person-disjoint sha256 half-splits, seeds 0-4; per-cell |log(rate_a/rate_b)| mean/sd across seeds; weighted by WPFINWGT", "seam_caveat": "both halves share SIPP seam structure; the seam-vs-J2J reconciliation run is a separate required artifact before thresholds lock", + "thin_flag_units": { + "e4_retention_by_age_sex": "distinct persons per half (person_id.nunique(); rows are person-month retention pairs) vs THIN_CELL_PERSONS=200", + "e5_runs_by_age": "rows per half, equal to persons (one row per person in the run-length frame) vs THIN_CELL_PERSONS=200" + }, + "sipp_jobs_reader_commit": "75d30dd57d71b91ee0929246b2f3cbb92263b350", + "metadata_edit_note": "metadata keys thin_flag_units and *_reader_commit were hand-added post hoc (staged microdata not available on the editing machine); no measured floor numbers were changed", "e4_retention_by_age_sex": { "16_24|sex1": { "rate": 0.9897, diff --git a/runs/tenure_floors_draft_v0.json b/runs/tenure_floors_draft_v0.json index 142ef92e..d4db1590 100644 --- a/runs/tenure_floors_draft_v0.json +++ b/runs/tenure_floors_draft_v0.json @@ -6,6 +6,9 @@ "source": "CPS January supplements 2020/2022/2024 (PTST1TN, PWTENWGT); reader per #205", "method": "person-disjoint sha256 half-splits, seeds 0-4; per-cell absolute weighted-quantile gap in years AND weighted-ECDF max gap (heaping-robust), mean/sd across seeds; BLS age bands", "heaping_caveat": "reported tenure heaps on integers, so half-vs-half quantile gaps are frequently exactly zero (36/63 cells in the first build) - a degenerate threshold basis; the ECDF max-gap floor is the heaping-robust alternative for the C3 round to choose between", + "thin_flag_units": "rows per half, equal to persons (one CPS record per person) vs THIN_CELL_PERSONS=200", + "cps_tenure_reader_commit": "5c9e5e67884bfe52d8d6c35ba6fdde40f52d5d62", + "metadata_edit_note": "metadata keys thin_flag_units and *_reader_commit were hand-added post hoc (staged microdata not available on the editing machine); no measured floor numbers were changed", "by_year": { "2020": { "16_19": { diff --git a/scripts/build_sipp_e8_e9_floors.py b/scripts/build_sipp_e8_e9_floors.py index af3ecdb8..2c6abd1a 100644 --- a/scripts/build_sipp_e8_e9_floors.py +++ b/scripts/build_sipp_e8_e9_floors.py @@ -33,6 +33,15 @@ the absolute gap in log-points; mean/sd across seeds. Cells under 200 unweighted persons per half are flagged thin. +Thin-flag units (recorded for honesty across the floor battery): +the E8 thin flag counts **rows** per half, which equal persons +because the E8 frame has one row per person; the E9 +earnings-change thin flag counts **rows** per half, which are +consecutive-month transition *pairs* (a person can contribute up +to 11), not persons — unlike the E4 spell floor, which counts +distinct persons. All compare against the same +``THIN_CELL_PERSONS = 200``. + Seam caveat: identical to the E4/E5 floors — both halves share SIPP's seam structure, so these floors cannot see seam bias; the reconciliation artifact (#214) carries that measurement. @@ -68,6 +77,29 @@ ARTIFACT = REPO / "runs/sipp_e8_e9_floors_draft_v0.json" +def _reader_commit() -> str: + """Last commit touching the SIPP reader in effect for this run.""" + import subprocess + + try: + return subprocess.run( + [ + "git", + "log", + "-1", + "--format=%H", + "--", + "src/populace_dynamics/data/sipp_jobs.py", + ], + cwd=REPO, + capture_output=True, + text=True, + check=True, + ).stdout.strip() + except Exception: + return "unknown" + + def _age_band(age: pd.Series) -> pd.Series: bins = [AGE_BANDS[0][0] - 1, *[hi for _, hi in AGE_BANDS]] labels = [f"{lo}_{hi}" for lo, hi in AGE_BANDS] @@ -314,6 +346,18 @@ def build() -> dict: "E9-stay thresholds should be stated on the IQR or a " "distributional distance, not the median" ), + "thin_flag_units": { + "e8_nonemployment_by_age": ( + "rows per half, equal to persons (one row per " + "person in the E8 frame) vs THIN_CELL_PERSONS=200" + ), + "e9_transitions.earnings_change": ( + "rows per half = consecutive-month transition " + "pairs, not persons (a person can contribute up to " + "11) vs THIN_CELL_PERSONS=200" + ), + }, + "sipp_jobs_reader_commit": _reader_commit(), "e8_nonemployment_by_age": e8_floors(persons), "e9_transitions": e9_floors(pairs), } diff --git a/scripts/build_sipp_spell_floors.py b/scripts/build_sipp_spell_floors.py index 4b41b75c..8f556c08 100644 --- a/scripts/build_sipp_spell_floors.py +++ b/scripts/build_sipp_spell_floors.py @@ -31,6 +31,14 @@ disability-floor convention. Cells with fewer than 200 unweighted persons per half are reported but flagged thin. +Thin-flag units (recorded for honesty across the floor battery): +the E4 retention thin flag counts **distinct persons** per half +(``person_id.nunique()``; the underlying rows are person-month +retention pairs, so a person can contribute many rows), while the +E5 runs thin flag counts **rows**, which here equal persons because +the run-length frame has exactly one row per person. Both compare +against the same ``THIN_CELL_PERSONS = 200``. + Seam caveat (pre-registered on #192): SIPP transitions bunch at interview seams, and both halves share the seam structure, so this floor cannot see seam bias — the seam-vs-J2J reconciliation run is @@ -67,6 +75,30 @@ ) +def _reader_commit() -> str: + """Last commit touching the SIPP reader in effect for this run.""" + import subprocess + + repo = Path(__file__).resolve().parents[1] + try: + return subprocess.run( + [ + "git", + "log", + "-1", + "--format=%H", + "--", + "src/populace_dynamics/data/sipp_jobs.py", + ], + cwd=repo, + capture_output=True, + text=True, + check=True, + ).stdout.strip() + except Exception: + return "unknown" + + def _age_band(age: pd.Series) -> pd.Series: bins = [AGE_BANDS[0][0] - 1, *[hi for _, hi in AGE_BANDS]] labels = [f"{lo}_{hi}" for lo, hi in AGE_BANDS] @@ -196,6 +228,19 @@ def build() -> dict: "reconciliation run is a separate required artifact " "before thresholds lock" ), + "thin_flag_units": { + "e4_retention_by_age_sex": ( + "distinct persons per half (person_id.nunique(); " + "rows are person-month retention pairs) vs " + "THIN_CELL_PERSONS=200" + ), + "e5_runs_by_age": ( + "rows per half, equal to persons (one row per " + "person in the run-length frame) vs " + "THIN_CELL_PERSONS=200" + ), + }, + "sipp_jobs_reader_commit": _reader_commit(), "e4_retention_by_age_sex": floors_for_retention(pairs), "e5_runs_by_age": floors_for_runs(runs), } diff --git a/scripts/build_tenure_floors.py b/scripts/build_tenure_floors.py index 7257f995..27617205 100644 --- a/scripts/build_tenure_floors.py +++ b/scripts/build_tenure_floors.py @@ -26,6 +26,12 @@ stability. Cells with fewer than 200 unweighted persons per half are flagged thin. +Thin-flag units: the thin flag counts **rows** per half, which +equal persons because the CPS tenure frame has one record per +person, against ``THIN_CELL_PERSONS = 200`` (the same constant the +SIPP spell floors use, where E4 counts distinct persons and E9 +counts transition pairs — units are recorded per artifact). + Usage:: python scripts/build_tenure_floors.py @@ -55,6 +61,29 @@ ARTIFACT = REPO / "runs/tenure_floors_draft_v0.json" +def _reader_commit() -> str: + """Last commit touching the CPS tenure reader for this run.""" + import subprocess + + try: + return subprocess.run( + [ + "git", + "log", + "-1", + "--format=%H", + "--", + "src/populace_dynamics/data/cps_tenure.py", + ], + cwd=REPO, + capture_output=True, + text=True, + check=True, + ).stdout.strip() + except Exception: + return "unknown" + + def _half(person_id: str, seed: int) -> int: digest = hashlib.sha256(f"{seed}:{person_id}".encode()).digest() return digest[0] & 1 @@ -185,6 +214,11 @@ def build() -> dict: "ECDF max-gap floor is the heaping-robust alternative " "for the C3 round to choose between" ), + "thin_flag_units": ( + "rows per half, equal to persons (one CPS record per " + "person) vs THIN_CELL_PERSONS=200" + ), + "cps_tenure_reader_commit": _reader_commit(), "by_year": {str(year): floors_for_year(year) for year in YEARS}, } From a3e8331f62c4685e21c8e3bd722d4f45de0a9c87 Mon Sep 17 00:00:00 2001 From: Daphne Hansell <128793799+daphnehanse11@users.noreply.github.com> Date: Wed, 22 Jul 2026 10:02:20 -0400 Subject: [PATCH 06/13] Address review: scale gap recorded, 20-seed E8/E9, person-unit thin, on-data re-run, pins - deployment_scale_note in all three artifacts: the missing candidate-context floor is now a RECORDED gap with the root-n ratio (~0.71x at full source), for C3 to resolve (accept the conservatism, scale analytically, or require ctx floors at v1). - E8/E9 seeds 5 -> 20 (the E8 cells with sd > mean now have stable across-seed sds, e.g. 55-64 any-nonemp floor 0.090 +/- 0.054 vs 0.141 +/- unstable at 5 seeds); E9 thin flag now counts distinct persons per half, not transition pairs (all cells still clear 200, honestly this time). - E8 estimand symmetry note: candidates must apply the identical full-year-persons restriction. - All three artifacts REGENERATED on a machine with the staged microdata: tenure and E4/E5 measured values reproduce byte-identically (confirming the hand-edit disclosure changed no numbers, now retired); E8/E9 floors re-estimated under 20 seeds with point estimates unchanged. - Artifact-tier test pins values, statuses, the scale gap, both heaping caveats, and the estimand note. E11/E12 disposition (review's smaller item): Workstream B scope, recorded in #223's battery; noted in the PR body. Co-Authored-By: Claude Fable 5 --- runs/sipp_e8_e9_floors_draft_v0.json | 64 ++++++++++++++-------------- runs/sipp_spell_floors_draft_v0.json | 4 +- runs/tenure_floors_draft_v0.json | 2 +- scripts/build_sipp_e8_e9_floors.py | 30 +++++++++++-- scripts/build_sipp_spell_floors.py | 13 ++++++ scripts/build_tenure_floors.py | 13 ++++++ tests/test_spell_floor_artifacts.py | 62 +++++++++++++++++++++++++++ tests/tier_counts.json | 2 +- 8 files changed, 150 insertions(+), 40 deletions(-) create mode 100644 tests/test_spell_floor_artifacts.py diff --git a/runs/sipp_e8_e9_floors_draft_v0.json b/runs/sipp_e8_e9_floors_draft_v0.json index fe696794..8cc6884f 100644 --- a/runs/sipp_e8_e9_floors_draft_v0.json +++ b/runs/sipp_e8_e9_floors_draft_v0.json @@ -3,29 +3,29 @@ "version": "draft_v0", "status": "DRAFT - NOT RATIFIED; C3 not locked; no thresholds", "issue": "192", - "source": "pu2023 (reference year 2022), persons observed all 12 reference months (censoring-free draft restriction, recorded)", - "method": "person-disjoint sha256 half-splits, seeds 0-4; rates floored on |log rate ratio|, earnings-change medians/IQRs on absolute gaps in log-points; weighted by WPFINWGT", + "deployment_scale_note": "RECORDED GAP (review of #212): these floors are half-vs-half, i.e. the sampling noise of ~50%-of-source estimates, while candidate runs will be scored on the full source - there is no candidate-context floor (gate-1 ctx20 analog) in this draft. Under root-n scaling the full-source floor is ~1/sqrt(2) (~0.71x) of the half-split floor, making half-split-derived thresholds conservative (too wide) at deployment scale; C3 decides whether to accept that conservatism, scale analytically, or require deployment-context floors at v1 promotion.", + "source": "pu2023 (reference year 2022), persons observed all 12 reference months (censoring-free draft restriction, recorded; ESTIMAND NOTE per review: candidate runs scored against these cells must apply the identical full-year-persons restriction)", + "method": "person-disjoint sha256 half-splits, seeds 0-19 (raised from 5 per review: E8 cells where floor sd exceeds the mean need a stable across-seed sd); rates floored on |log rate ratio|, earnings-change medians/IQRs on absolute gaps in log-points; weighted by WPFINWGT", "seam_caveat": "identical to the E4/E5 floors: half-splits share SIPP's seam structure; #214 carries the seam measurement", "stay_median_heaping_caveat": "within-job SIPP monthly earnings are mostly constant across a wave (dependent-interview reporting), so the stay-transition median log-change heaps at exactly 0 and its half-vs-half floor is degenerate (0.0) - the same failure class as the tenure quantile heaping; E9-stay thresholds should be stated on the IQR or a distributional distance, not the median", "thin_flag_units": { "e8_nonemployment_by_age": "rows per half, equal to persons (one row per person in the E8 frame) vs THIN_CELL_PERSONS=200", "e9_transitions.earnings_change": "rows per half = consecutive-month transition pairs, not persons (a person can contribute up to 11) vs THIN_CELL_PERSONS=200" }, - "sipp_jobs_reader_commit": "75d30dd57d71b91ee0929246b2f3cbb92263b350", - "metadata_edit_note": "metadata keys thin_flag_units and *_reader_commit were hand-added post hoc (staged microdata not available on the editing machine); no measured floor numbers were changed", + "sipp_jobs_reader_commit": "a059193e4fad80ceb1c2e1f4177aa5c69abb1048", "e8_nonemployment_by_age": { "16_24": { "any_nonemp_share": 0.4145, "long_nonemp_share": 0.3422, "persons_unweighted": 1781, "any_nonemp": { - "floor_abs_log_ratio_mean": 0.08001, - "floor_abs_log_ratio_sd": 0.0466, + "floor_abs_log_ratio_mean": 0.05149, + "floor_abs_log_ratio_sd": 0.0356, "thin": false }, "long_nonemp": { - "floor_abs_log_ratio_mean": 0.10556, - "floor_abs_log_ratio_sd": 0.04714, + "floor_abs_log_ratio_mean": 0.08586, + "floor_abs_log_ratio_sd": 0.04431, "thin": false } }, @@ -34,13 +34,13 @@ "long_nonemp_share": 0.0973, "persons_unweighted": 3240, "any_nonemp": { - "floor_abs_log_ratio_mean": 0.07301, - "floor_abs_log_ratio_sd": 0.04942, + "floor_abs_log_ratio_mean": 0.05885, + "floor_abs_log_ratio_sd": 0.04819, "thin": false }, "long_nonemp": { - "floor_abs_log_ratio_mean": 0.14005, - "floor_abs_log_ratio_sd": 0.06264, + "floor_abs_log_ratio_mean": 0.10659, + "floor_abs_log_ratio_sd": 0.06993, "thin": false } }, @@ -49,13 +49,13 @@ "long_nonemp_share": 0.0663, "persons_unweighted": 3543, "any_nonemp": { - "floor_abs_log_ratio_mean": 0.1424, - "floor_abs_log_ratio_sd": 0.09863, + "floor_abs_log_ratio_mean": 0.09917, + "floor_abs_log_ratio_sd": 0.07372, "thin": false }, "long_nonemp": { - "floor_abs_log_ratio_mean": 0.20084, - "floor_abs_log_ratio_sd": 0.10088, + "floor_abs_log_ratio_mean": 0.13369, + "floor_abs_log_ratio_sd": 0.0899, "thin": false } }, @@ -64,13 +64,13 @@ "long_nonemp_share": 0.0555, "persons_unweighted": 3241, "any_nonemp": { - "floor_abs_log_ratio_mean": 0.10559, - "floor_abs_log_ratio_sd": 0.11745, + "floor_abs_log_ratio_mean": 0.10871, + "floor_abs_log_ratio_sd": 0.078, "thin": false }, "long_nonemp": { - "floor_abs_log_ratio_mean": 0.14665, - "floor_abs_log_ratio_sd": 0.07165, + "floor_abs_log_ratio_mean": 0.09797, + "floor_abs_log_ratio_sd": 0.07416, "thin": false } }, @@ -79,13 +79,13 @@ "long_nonemp_share": 0.083, "persons_unweighted": 3449, "any_nonemp": { - "floor_abs_log_ratio_mean": 0.14077, - "floor_abs_log_ratio_sd": 0.05732, + "floor_abs_log_ratio_mean": 0.09033, + "floor_abs_log_ratio_sd": 0.05384, "thin": false }, "long_nonemp": { - "floor_abs_log_ratio_mean": 0.14983, - "floor_abs_log_ratio_sd": 0.05971, + "floor_abs_log_ratio_mean": 0.11965, + "floor_abs_log_ratio_sd": 0.05523, "thin": false } }, @@ -94,13 +94,13 @@ "long_nonemp_share": 0.1546, "persons_unweighted": 2286, "any_nonemp": { - "floor_abs_log_ratio_mean": 0.05111, - "floor_abs_log_ratio_sd": 0.04105, + "floor_abs_log_ratio_mean": 0.05524, + "floor_abs_log_ratio_sd": 0.04697, "thin": false }, "long_nonemp": { - "floor_abs_log_ratio_mean": 0.09334, - "floor_abs_log_ratio_sd": 0.03649, + "floor_abs_log_ratio_mean": 0.08085, + "floor_abs_log_ratio_sd": 0.04997, "thin": false } } @@ -132,12 +132,12 @@ "iqr_log_change": 1.069, "pairs_unweighted": 545, "floor_abs_median_gap": { - "mean": 0.07658, - "sd": 0.03448 + "mean": 0.06659, + "sd": 0.05025 }, "floor_abs_iqr_gap": { - "mean": 0.08423, - "sd": 0.05215 + "mean": 0.12848, + "sd": 0.07392 }, "thin": false } diff --git a/runs/sipp_spell_floors_draft_v0.json b/runs/sipp_spell_floors_draft_v0.json index e3b7e202..20f21986 100644 --- a/runs/sipp_spell_floors_draft_v0.json +++ b/runs/sipp_spell_floors_draft_v0.json @@ -3,6 +3,7 @@ "version": "draft_v0", "status": "DRAFT - NOT RATIFIED; C3 not locked; no thresholds", "issue": "192", + "deployment_scale_note": "RECORDED GAP (review of #212): these floors are half-vs-half, i.e. the sampling noise of ~50%-of-source estimates, while candidate runs will be scored on the full source - there is no candidate-context floor (gate-1 ctx20 analog) in this draft. Under root-n scaling the full-source floor is ~1/sqrt(2) (~0.71x) of the half-split floor, making half-split-derived thresholds conservative (too wide) at deployment scale; C3 decides whether to accept that conservatism, scale analytically, or require deployment-context floors at v1 promotion.", "source": "pu2023 (reference year 2022)", "method": "person-disjoint sha256 half-splits, seeds 0-4; per-cell |log(rate_a/rate_b)| mean/sd across seeds; weighted by WPFINWGT", "seam_caveat": "both halves share SIPP seam structure; the seam-vs-J2J reconciliation run is a separate required artifact before thresholds lock", @@ -10,8 +11,7 @@ "e4_retention_by_age_sex": "distinct persons per half (person_id.nunique(); rows are person-month retention pairs) vs THIN_CELL_PERSONS=200", "e5_runs_by_age": "rows per half, equal to persons (one row per person in the run-length frame) vs THIN_CELL_PERSONS=200" }, - "sipp_jobs_reader_commit": "75d30dd57d71b91ee0929246b2f3cbb92263b350", - "metadata_edit_note": "metadata keys thin_flag_units and *_reader_commit were hand-added post hoc (staged microdata not available on the editing machine); no measured floor numbers were changed", + "sipp_jobs_reader_commit": "a059193e4fad80ceb1c2e1f4177aa5c69abb1048", "e4_retention_by_age_sex": { "16_24|sex1": { "rate": 0.9897, diff --git a/runs/tenure_floors_draft_v0.json b/runs/tenure_floors_draft_v0.json index d4db1590..4988ded9 100644 --- a/runs/tenure_floors_draft_v0.json +++ b/runs/tenure_floors_draft_v0.json @@ -3,12 +3,12 @@ "version": "draft_v0", "status": "DRAFT - NOT RATIFIED; C3 not locked; no thresholds", "issue": "192", + "deployment_scale_note": "RECORDED GAP (review of #212): these floors are half-vs-half, i.e. the sampling noise of ~50%-of-source estimates, while candidate runs will be scored on the full source - there is no candidate-context floor (gate-1 ctx20 analog) in this draft. Under root-n scaling the full-source floor is ~1/sqrt(2) (~0.71x) of the half-split floor, making half-split-derived thresholds conservative (too wide) at deployment scale; C3 decides whether to accept that conservatism, scale analytically, or require deployment-context floors at v1 promotion.", "source": "CPS January supplements 2020/2022/2024 (PTST1TN, PWTENWGT); reader per #205", "method": "person-disjoint sha256 half-splits, seeds 0-4; per-cell absolute weighted-quantile gap in years AND weighted-ECDF max gap (heaping-robust), mean/sd across seeds; BLS age bands", "heaping_caveat": "reported tenure heaps on integers, so half-vs-half quantile gaps are frequently exactly zero (36/63 cells in the first build) - a degenerate threshold basis; the ECDF max-gap floor is the heaping-robust alternative for the C3 round to choose between", "thin_flag_units": "rows per half, equal to persons (one CPS record per person) vs THIN_CELL_PERSONS=200", "cps_tenure_reader_commit": "5c9e5e67884bfe52d8d6c35ba6fdde40f52d5d62", - "metadata_edit_note": "metadata keys thin_flag_units and *_reader_commit were hand-added post hoc (staged microdata not available on the editing machine); no measured floor numbers were changed", "by_year": { "2020": { "16_19": { diff --git a/scripts/build_sipp_e8_e9_floors.py b/scripts/build_sipp_e8_e9_floors.py index 2c6abd1a..9b116d04 100644 --- a/scripts/build_sipp_e8_e9_floors.py +++ b/scripts/build_sipp_e8_e9_floors.py @@ -70,7 +70,7 @@ from populace_dynamics.data import sipp_jobs # noqa: E402 YEAR = 2023 -SEEDS = (0, 1, 2, 3, 4) +SEEDS = tuple(range(20)) AGE_BANDS = ((16, 24), (25, 34), (35, 44), (45, 54), (55, 64), (65, 99)) THIN_CELL_PERSONS = 200 @@ -285,7 +285,12 @@ def e9_floors(pairs: pd.DataFrame) -> dict: for seed in SEEDS: half = cell["person_id"].map(lambda p, s=seed: _half(p, s)) a, b = cell[half == 0], cell[half == 1] - halves_n.append(min(len(a), len(b))) + halves_n.append( + min( + a["person_id"].nunique(), + b["person_id"].nunique(), + ) + ) def q(frame, qq): return _weighted_quantile( @@ -324,11 +329,28 @@ def build() -> dict: "version": "draft_v0", "status": "DRAFT - NOT RATIFIED; C3 not locked; no thresholds", "issue": "192", + "deployment_scale_note": ( + "RECORDED GAP (review of #212): these floors are " + "half-vs-half, i.e. the sampling noise of ~50%-of-source " + "estimates, while candidate runs will be scored on the " + "full source - there is no candidate-context floor " + "(gate-1 ctx20 analog) in this draft. Under root-n " + "scaling the full-source floor is ~1/sqrt(2) (~0.71x) " + "of the half-split floor, making half-split-derived " + "thresholds conservative (too wide) at deployment " + "scale; C3 decides whether to accept that conservatism, " + "scale analytically, or require deployment-context " + "floors at v1 promotion." + ), "source": f"pu{YEAR} (reference year {YEAR - 1}), persons " "observed all 12 reference months (censoring-free draft " - "restriction, recorded)", + "restriction, recorded; ESTIMAND NOTE per review: candidate " + "runs scored against these cells must apply the identical " + "full-year-persons restriction)", "method": ( - "person-disjoint sha256 half-splits, seeds 0-4; rates " + "person-disjoint sha256 half-splits, seeds 0-19 (raised " + "from 5 per review: E8 cells where floor sd exceeds " + "the mean need a stable across-seed sd); rates " "floored on |log rate ratio|, earnings-change medians/" "IQRs on absolute gaps in log-points; weighted by " "WPFINWGT" diff --git a/scripts/build_sipp_spell_floors.py b/scripts/build_sipp_spell_floors.py index 8f556c08..033c0bab 100644 --- a/scripts/build_sipp_spell_floors.py +++ b/scripts/build_sipp_spell_floors.py @@ -217,6 +217,19 @@ def build() -> dict: "version": "draft_v0", "status": "DRAFT - NOT RATIFIED; C3 not locked; no thresholds", "issue": "192", + "deployment_scale_note": ( + "RECORDED GAP (review of #212): these floors are " + "half-vs-half, i.e. the sampling noise of ~50%-of-source " + "estimates, while candidate runs will be scored on the " + "full source - there is no candidate-context floor " + "(gate-1 ctx20 analog) in this draft. Under root-n " + "scaling the full-source floor is ~1/sqrt(2) (~0.71x) " + "of the half-split floor, making half-split-derived " + "thresholds conservative (too wide) at deployment " + "scale; C3 decides whether to accept that conservatism, " + "scale analytically, or require deployment-context " + "floors at v1 promotion." + ), "source": f"pu{YEAR} (reference year {YEAR - 1})", "method": ( "person-disjoint sha256 half-splits, seeds 0-4; per-cell " diff --git a/scripts/build_tenure_floors.py b/scripts/build_tenure_floors.py index 27617205..8af03e3e 100644 --- a/scripts/build_tenure_floors.py +++ b/scripts/build_tenure_floors.py @@ -199,6 +199,19 @@ def build() -> dict: "version": "draft_v0", "status": "DRAFT - NOT RATIFIED; C3 not locked; no thresholds", "issue": "192", + "deployment_scale_note": ( + "RECORDED GAP (review of #212): these floors are " + "half-vs-half, i.e. the sampling noise of ~50%-of-source " + "estimates, while candidate runs will be scored on the " + "full source - there is no candidate-context floor " + "(gate-1 ctx20 analog) in this draft. Under root-n " + "scaling the full-source floor is ~1/sqrt(2) (~0.71x) " + "of the half-split floor, making half-split-derived " + "thresholds conservative (too wide) at deployment " + "scale; C3 decides whether to accept that conservatism, " + "scale analytically, or require deployment-context " + "floors at v1 promotion." + ), "source": "CPS January supplements 2020/2022/2024 (PTST1TN, " "PWTENWGT); reader per #205", "method": ( diff --git a/tests/test_spell_floor_artifacts.py b/tests/test_spell_floor_artifacts.py new file mode 100644 index 00000000..07d285e3 --- /dev/null +++ b/tests/test_spell_floor_artifacts.py @@ -0,0 +1,62 @@ +"""Pin the three Workstream A floor artifacts (#212, pre-C3).""" + +from __future__ import annotations + +import json +from pathlib import Path + +import pytest + +RUNS = Path(__file__).resolve().parents[1] / "runs" + + +@pytest.fixture(scope="module") +def spells() -> dict: + return json.loads((RUNS / "sipp_spell_floors_draft_v0.json").read_text()) + + +@pytest.fixture(scope="module") +def tenure() -> dict: + return json.loads((RUNS / "tenure_floors_draft_v0.json").read_text()) + + +@pytest.fixture(scope="module") +def e8e9() -> dict: + return json.loads((RUNS / "sipp_e8_e9_floors_draft_v0.json").read_text()) + + +def test_all_carry_draft_status_and_scale_gap(spells, tenure, e8e9): + for artifact in (spells, tenure, e8e9): + assert "DRAFT" in artifact["status"] + assert "RECORDED GAP" in artifact["deployment_scale_note"] + assert "ctx20" in artifact["deployment_scale_note"] + + +def test_e4_e5_pinned_values(spells): + e4 = spells["e4_retention_by_age_sex"]["16_24|sex1"] + assert e4["rate"] == 0.9897 + assert e4["floor_abs_log_ratio_mean"] == 0.00182 + e5 = spells["e5_runs_by_age"]["45_54"] + assert e5["full_year_run_share"] == 0.8671 + assert spells["seam_caveat"] + + +def test_tenure_pinned_values_and_heaping(tenure): + cell = tenure["by_year"]["2024"]["35_44"] + assert cell["p50"] == 5.0 + assert cell["floor_abs_gap_years"]["p50"]["mean"] == 0.0 + assert cell["floor_ecdf_max_gap"]["mean"] > 0 + assert "exactly zero" in tenure["heaping_caveat"] + + +def test_e8_e9_pinned_values(e8e9): + assert "seeds 0-19" in e8e9["method"] + assert "ESTIMAND NOTE" in e8e9["source"] + mix = e8e9["e9_transitions"]["transition_rates"] + assert mix["stay"] == 0.977 + assert mix["j2j"] == 0.0035 + stay = e8e9["e9_transitions"]["earnings_change"]["stay"] + assert stay["median_log_change"] == 0.0 + assert "heaps at exactly 0" in e8e9["stay_median_heaping_caveat"] + e8 = e8e9["e8_nonemployment_by_age"]["16_24"] + assert e8["any_nonemp_share"] == pytest.approx(0.4145, abs=0.001) diff --git a/tests/tier_counts.json b/tests/tier_counts.json index 19149b24..2cf57aef 100644 --- a/tests/tier_counts.json +++ b/tests/tier_counts.json @@ -2,7 +2,7 @@ "schema_version": 1, "counts": { "unit": 740, - "artifact": 1101, + "artifact": 1105, "integration_psid": 804, "reproduction_legacy": 520, "oracle_policyengine": 159 From fd4c372aa7c043f1f567c5552451a1268e0d6731 Mon Sep 17 00:00:00 2001 From: Vahid Ahmadi Date: Thu, 23 Jul 2026 11:21:08 +0100 Subject: [PATCH 07/13] Rename the interface contracts C1/C2/C3 -> IC1/IC2/IC3 (ADR 0003 am. 1) Naming only: no column, band, code mapping, or gate definition changes, and the numbering is preserved 1:1 so every prior reference maps by prefixing "I". A bare "C1" meant four different things in this repo: the gate_w1 fingerprints (gates.yaml fingerprints.c1/.c2), the SSA Trustees table II.C1, an RNG substream in household composition, and the employer-firm interface contracts. Only the last is repo-internal, pre-lock, and ours -- the fingerprints sit inside gate_w1, which is locked: true, so renaming those would cost a public amendment plus a fresh referee round. Timing is the point. IC3 (the employer gate block) is about to be written into gates.yaml, which already contains fingerprints.c1 and fingerprints.c2. A block named C3 locking next to them makes either rename cost exactly what the fingerprint row of that table already costs. Vahid flagged the collision on #192 before the referee round; this closes it while it is still free. Touches frozen ADR 0003 text, so it is a joint-PR change under the IC1/IC2 freeze rule -- procedurally, not because anything moved. Prior discussion keeps the old names; the ADR carries the mapping. Also corrects a stale claim in sipp_jobs: the module said "ADR 0003 is Proposed, not frozen" as the reason job_spells is IC1-preview. It is Accepted and IC1 is frozen; what is still preview-grade is the collapse's single-ref_year coverage, which is what the docstring now says. Co-Authored-By: Claude Opus 4.8 (1M context) --- docs/adr/0003-employer-firm-extension.md | 76 ++++++++++++++------ scripts/build_noemp_band_evidence.py | 2 +- src/populace_dynamics/data/asec_firm_size.py | 4 +- src/populace_dynamics/data/sipp_jobs.py | 26 ++++--- src/populace_dynamics/firms/__init__.py | 2 +- src/populace_dynamics/firms/banding.py | 4 +- tests/test_firms_banding.py | 2 +- tests/test_noemp_band_evidence.py | 2 +- 8 files changed, 76 insertions(+), 42 deletions(-) diff --git a/docs/adr/0003-employer-firm-extension.md b/docs/adr/0003-employer-firm-extension.md index b80236e7..1b6591f3 100644 --- a/docs/adr/0003-employer-firm-extension.md +++ b/docs/adr/0003-employer-firm-extension.md @@ -1,15 +1,45 @@ -# ADR 0003: Employer-firm extension — C1 spell schema and C2 canonical firm-size banding +# ADR 0003: Employer-firm extension — IC1 spell schema and IC2 canonical firm-size banding -**Status:** Accepted — C1 and C2 frozen 2026-07-16. From this +**Status:** Accepted — IC1 and IC2 frozen 2026-07-16. From this point the contracts change only by joint PR between workstreams A and B ([populace-dynamics#192](https://github.com/PolicyEngine/populace-dynamics/issues/192)). +**Amendment 1 (naming, no semantic change).** The interface +contracts were originally named `C1`/`C2`/`C3`. They are renamed +`IC1`/`IC2`/`IC3` — *interface contract* — with no change to any +column, band, code mapping, or gate definition. This is a +joint-PR change because it edits frozen contract text, not because +anything in the contracts moved; the numbering is preserved 1:1, so +every prior reference maps by prefixing `I`. + +**Why, and why this side moves.** In this repository a bare "C1" +already meant four different things: + +| sense | example | can it be renamed? | +|---|---|---| +| gate_w1 **fingerprints** `c1`/`c2` | `gates.yaml` `fingerprints.c1` (PPI↔NRA) | **No** — inside `gate_w1`, which is `locked: true`. Renaming needs a public amendment plus a fresh referee round | +| SSA Trustees **table** II.C1 | `data/external/ssa_tr_2014_ii_c1.*` | No — an external publisher's table label | +| RNG **substream** C3 | household-composition `nonfamily_bridge` | Unrelated component; renaming is churn for no gain | +| **interface contracts** C1/C2/C3 | this ADR | **Yes** — the only set that is repo-internal, pre-lock, and ours | + +The collision was flagged on #192 before the C3 referee round with +the note that it would confuse referees. It is fixed now rather +than later for one reason: `IC3` (the employer gate block) is +about to be written into `gates.yaml`, which already contains +`fingerprints.c1` and `fingerprints.c2`. Once a block named `C3` +locks alongside them, renaming either set costs an amendment and a +fresh referee round — the exact cost this table shows the +fingerprint side already carries. + +Prior discussion (issue #192, the ADR history, merged PR bodies) +uses the old names and is not rewritten; this note is the mapping. + **Sign-off:** @vahid-ahmadi (Workstream B, author) · @daphnehanse11 (Workstream A) — the joint sign-off is recorded by the merge of the freeze PR: authorship by one workstream owner plus approval by the other. First scheduled amendment (pre-registered below): -the C1 ``hours_band``/monthly-hours column once phase-1 establishes +the IC1 ``hours_band``/monthly-hours column once phase-1 establishes SIPP's supportable hours granularity. ## Context @@ -17,15 +47,15 @@ SIPP's supportable hours granularity. The employer-firm plan (`docs/plans/employer-firm-plan.html`) splits the extension into workstream A (person side: SIPP spells, CPS hosts, imputation) and workstream B (firm side: external targets, banding, -calibration, register), meeting at three interface contracts. C1 (the -spell schema) and C2 (canonical firm-size banding and its semantics) +calibration, register), meeting at three interface contracts. IC1 (the +spell schema) and IC2 (canonical firm-size banding and its semantics) freeze in week 1. This ADR records both, plus the target/gate partition rule, folding in the four contract-affecting findings from the week-1 review on issue #192. ## Decision -### C2 — canonical firm-size banding +### IC2 — canonical firm-size banding 1. **Semantics (review finding F5).** The canonical firm-size variable means **administrative enterprise size**: total @@ -33,7 +63,7 @@ the week-1 review on issue #192. counts it. Survey labels are noisy measures of that quantity — CPS ASEC firm size (worker-reported, all locations, previous calendar year's longest job — under either the raw Census `NOEMP` - or the IPUMS `FIRMSIZE` coding; see C2.5) is the primary training + or the IPUMS `FIRMSIZE` coding; see IC2.5) is the primary training label; SIPP 2014+ `EJB1_EMPSIZE` (establishment size) is a proxy chain. SUSB is therefore the correct E1 reference. 2. **Bands are headcount bands.** Five canonical bands with edges at @@ -43,7 +73,7 @@ the week-1 review on issue #192. both QWI (20-49 / 50-249) and the detailed SUSB classes (40-49 / 50-74) support it. FTE-denominated thresholds (the ACA cut is 50 full-time equivalents at 30 hours/week, not headcount) are - resolved by a person-side hours join — out of C2 scope. + resolved by a person-side hours join — out of IC2 scope. 3. **Mappings are total but explicitly ambiguous where the source is coarse.** Every raw code from every source maps to exactly one `BandSpan` (a contiguous run of canonical bands with an `exact` @@ -65,7 +95,7 @@ the week-1 review on issue #192. replication (weighted code shares by year vs. SUSB) is committed as `runs/noemp_band_evidence_v1.json` with its build script and pinning tests (#211) — the reported-anchor convention, since it - is derived evidence rather than a source extract — for the C3 + is derived evidence rather than a source extract — for the IC3 record. 5. **The person-side coding is explicit, not inferred (seam with #194).** The raw Census ASEC person file carries `NOEMP` @@ -85,7 +115,7 @@ the week-1 review on issue #192. emit `CanonicalBand` directly), never feed `NOEMP` integers to the `ipums_firmsize` route. -### C1 — job-spell schema +### IC1 — job-spell schema One tidy table, written by workstream A, read by workstream B: @@ -96,7 +126,7 @@ One tidy table, written by workstream A, read by workstream B: | `start_period` | period | first period of the spell | | `end_period` | period | last period; open spells use a sentinel | | `industry` | str | NAICS major (sector) group | -| `firm_size_band` | enum | canonical band per C2 (`CanonicalBand`) | +| `firm_size_band` | enum | canonical band per IC2 (`CanonicalBand`) | | `class_of_worker` | enum | private / federal / state-local government / self-employed / unpaid family | | `earnings_share` | float | share of the person's period earnings from this job | | `primary_job` | bool | phase 0 is primary-job-only | @@ -111,7 +141,7 @@ One tidy table, written by workstream A, read by workstream B: from the SUSB/QWI calibration universe; self-employed spells have no defined `firm_size_band`. - **Geography joins from the person table.** QWI/J2J targets are - state-level; C1 deliberately carries no geography column. The + state-level; IC1 deliberately carries no geography column. The state of a spell is the host person's state at `start_period`, joined on `person_id` — the join key lives on the person table, not the spell table. @@ -122,9 +152,9 @@ One tidy table, written by workstream A, read by workstream B: compliance, issue #192 — the 80-hours-per-month test of 7 CFR 273.24 and the 3-in-36 countable-month clock need month-resolved hours, not spell start/end plus annual earnings). - C1 as frozen carries no hours column, so it **cannot yet serve + IC1 as frozen carries no hours column, so it **cannot yet serve monthly-hours consumers**; a `hours_band` (or monthly hours) - column is the first scheduled C1 amendment, to be added by joint + column is the first scheduled IC1 amendment, to be added by joint PR once workstream A's phase-1 spell imputation establishes what hours granularity SIPP can support. Consumers must not proxy monthly compliance from annual quantities in the meantime. @@ -146,10 +176,10 @@ phase-0 QRF therefore comes from a named bridge, not an implicit one: bridge, aged forward. 2. **Proxy chain:** SIPP 2014+ establishment size x tenure, mapped through the establishment-to-enterprise noise model implied by - the C2 semantics. + the IC2 semantics. 3. **Pre-registered caveat:** the ASEC reference-period mismatch (`FIRMSIZE` = last calendar year's longest job; tenure supplement - = current job) is carried into the C3 gate notes as a known + = current job) is carried into the IC3 gate notes as a known label-misalignment term. ### Target/gate partition rule @@ -161,7 +191,7 @@ firm-size x sector flow margins committed under `data/external/`; gates E1/E2/E7/E11 score on held-out dimensions of the same sources (the sex/age demographic axes of QWI, the firm-age axis, and the state axis) that calibration never touches. The exact cell lists lock -with C3 after the floor runs. +with IC3 after the floor runs. Three unit rules recorded now (issue #192 review, point 4; branch review finding 3): @@ -171,7 +201,7 @@ review finding 3): so calibrating person-spells to QWI cells carries a wedge on the order of the multiple-jobholding rate (~5%, time-varying). A job-count -> person-count adjustment is an explicit pre-registered - C3 item, not a footnote. + IC3 item, not a footnote. - **QWI publishes mean earnings (`EarnS`), never medians**; E7 is stated on means. - **J2J's employer universe is broader than SUSB/QWI's.** The @@ -182,7 +212,7 @@ review finding 3): sectors (notably 61 Educational Services and 62 Health Care). Any E11 cell definition must either restate J2J on a private-comparable basis or carry this scope difference as a pre-registered caveat; - the choice locks with C3. + the choice locks with IC3. ## Consequences @@ -193,8 +223,8 @@ review finding 3): workstreams push directly to each other's branches when useful (reader fixes, rebases, contract-text corrections — this has run in both directions and worked). The norm the freeze makes - explicit: a change that touches contract semantics (C1 columns, - C2 bands/codings, gate definitions) requires the *other* + explicit: a change that touches contract semantics (IC1 columns, + IC2 bands/codings, gate definitions) requires the *other* workstream owner's approval on the PR even when the commit was pushed directly, so pre-registration always records who decided, not just who typed. @@ -204,6 +234,6 @@ review finding 3): references, analogous to the NCHS/Census/ONS files — never scored model output. Raw microdata is never committed. - No change to `gates.yaml`. Employer gates E1-E12 lock as a new - block (C3) after noise-floor runs and a referee round, via the - standard amendment process; no one-shot candidate runs before C3 + block (IC3) after noise-floor runs and a referee round, via the + standard amendment process; no one-shot candidate runs before IC3 locks. diff --git a/scripts/build_noemp_band_evidence.py b/scripts/build_noemp_band_evidence.py index dc1a23cd..9bcc886c 100644 --- a/scripts/build_noemp_band_evidence.py +++ b/scripts/build_noemp_band_evidence.py @@ -3,7 +3,7 @@ REPORTED ANCHOR, NOT A GATE RUN. Like the mortality/claiming/ disability floors, this reads no gate and decides nothing on its own; it is committed evidence pinned by a reproduction test. It -records the empirical basis for the C2 banding decision's treatment +records the empirical basis for the IC2 banding decision's treatment of CPS ASEC firm size: **the 2019+ data dictionaries' relabeling of NOEMP codes 2/3 (from 10-49 / 50-99 to 10-24 / 25-99) never happened in the instrument.** diff --git a/src/populace_dynamics/data/asec_firm_size.py b/src/populace_dynamics/data/asec_firm_size.py index 5f32d12d..6f78c3a3 100644 --- a/src/populace_dynamics/data/asec_firm_size.py +++ b/src/populace_dynamics/data/asec_firm_size.py @@ -22,7 +22,7 @@ share (~7.5%), while a true 25-99 band carries ~15%. This reader therefore uses the 10-49 / 50-99 reading for all years and records the dictionary conflict here rather than silently following the -2019+ label text into a factor-two mis-band. Consequence for C2: +2019+ label text into a factor-two mis-band. Consequence for IC2: the 50-employee edge (ACA and state mandates) is directly observed in every supported year — the "post-2019 label cannot resolve the 50 cut" problem stated in earlier drafts dissolves. @@ -374,7 +374,7 @@ def firm_size_tabulation( "class_of_worker", ), ) -> pd.DataFrame: - """Weighted firm-size tabulation — the C2 evidence artifact. + """Weighted firm-size tabulation — the IC2 evidence artifact. Args: records: Output of :func:`read_asec_firm_size` (one or more diff --git a/src/populace_dynamics/data/sipp_jobs.py b/src/populace_dynamics/data/sipp_jobs.py index c722c9e5..65be95df 100644 --- a/src/populace_dynamics/data/sipp_jobs.py +++ b/src/populace_dynamics/data/sipp_jobs.py @@ -1,4 +1,4 @@ -"""SIPP job-level monthly records and C1-preview spells (issue #200). +"""SIPP job-level monthly records and IC1-preview spells (issue #200). The 2014-redesign SIPP public-use files are the employer-firm plan's primary label panel (#192): one row per person-month (``SSUID`` x @@ -8,7 +8,7 @@ within-panel employer-attachment key that phase-1 transition hazards rest on. ``EJB{n}_EMPSIZE`` measures **establishment** size at the worker's location (the redesign dropped the all-locations question), -so it is the C2 proxy-chain input, never firm size (ADR 0003; +so it is the IC2 proxy-chain input, never firm size (ADR 0003; ``firms/banding.py``). Every variable this reader touches was verified against the Census @@ -28,10 +28,14 @@ string-typed in the API schema. ``job_spells`` collapses maximal consecutive-month runs per -(person, job id) into spell rows whose shape mirrors the C1 spell -schema. It is labeled **C1-preview**: ADR 0003 is Proposed, not -frozen, and this output also serves as Workstream B's generator for -C1-conforming fixture files. Attribute changes inside a spell +(person, job id) into spell rows whose shape mirrors the IC1 spell +schema. It is still labeled **IC1-preview**, but for a narrower +reason than when it was written: ADR 0003 is now Accepted and IC1 +is frozen, so what remains preview-grade is this collapse's own +coverage (single ``ref_year`` only — cross-year spell linkage +raises rather than guessing), not the schema's status. The output +also serves as Workstream B's generator for IC1-conforming fixture +files. Attribute changes inside a spell (class of worker, industry, establishment size) are surfaced via ``attributes_constant`` — never silently averaged. @@ -495,12 +499,12 @@ def read_sipp_job_months( def job_spells(job_months: pd.DataFrame) -> pd.DataFrame: - """Collapse job-months into C1-preview spell rows. + """Collapse job-months into IC1-preview spell rows. A spell is a maximal run of consecutive reference months for one - (person, job id). The output mirrors the C1 spell schema of ADR + (person, job id). The output mirrors the IC1 spell schema of ADR 0003 (Proposed — this is a preview, not the frozen contract) and - doubles as Workstream B's generator for C1-conforming fixtures. + doubles as Workstream B's generator for IC1-conforming fixtures. Args: job_months: Output of :func:`read_sipp_job_months`. @@ -560,7 +564,7 @@ def job_spells(job_months: pd.DataFrame) -> pd.DataFrame: ] ) - # Cross-year spell linkage is undefined in this C1 preview: the + # Cross-year spell linkage is undefined in this IC1 preview: the # break/run detection, the person-month earnings lookup, and the # spell edges all key on the calendar ``month`` (1-12) alone, so two # different reference years sharing a month would collapse into one @@ -575,7 +579,7 @@ def job_spells(job_months: pd.DataFrame) -> pd.DataFrame: raise ValueError( "job_spells received job-months spanning multiple ref_years " f"({sorted(int(y) for y in ref_years)}); cross-year spell " - "linkage is undefined in this C1 preview. Collapse one SIPP " + "linkage is undefined in this IC1 preview. Collapse one SIPP " "file's months at a time." ) diff --git a/src/populace_dynamics/firms/__init__.py b/src/populace_dynamics/firms/__init__.py index daa46544..9748b63e 100644 --- a/src/populace_dynamics/firms/__init__.py +++ b/src/populace_dynamics/firms/__init__.py @@ -1,6 +1,6 @@ """Employer-firm extension, workstream B (firm side). -Canonical firm-size banding (interface contract C2) and label-verified +Canonical firm-size banding (interface contract IC2) and label-verified loaders for the committed external target extracts (SUSB, BDS, QWI, J2J). See ``docs/adr/0003-employer-firm-extension.md`` and issue #192. """ diff --git a/src/populace_dynamics/firms/banding.py b/src/populace_dynamics/firms/banding.py index 67843686..df02e2e8 100644 --- a/src/populace_dynamics/firms/banding.py +++ b/src/populace_dynamics/firms/banding.py @@ -1,4 +1,4 @@ -"""Canonical firm-size banding — interface contract C2. +"""Canonical firm-size banding — interface contract IC2. **Semantics (review finding F5).** The canonical variable means *administrative enterprise size*: the total employment of the legal @@ -26,7 +26,7 @@ Bands are **headcount** bands. Policy thresholds stated in FTEs (the ACA applicable-large-employer cut is 50 *full-time equivalents* at 30 hours/week, not headcount) are handled by a person-side hours join and -are out of C2 scope. +are out of IC2 scope. **Canonical bands.** Five bands with edges at 10 / 50 / 100 / 500:: diff --git a/tests/test_firms_banding.py b/tests/test_firms_banding.py index ae8e1f3f..3c474b64 100644 --- a/tests/test_firms_banding.py +++ b/tests/test_firms_banding.py @@ -1,4 +1,4 @@ -"""Tests for the canonical firm-size banding (contract C2). +"""Tests for the canonical firm-size banding (contract IC2). Checks the properties the contract promises: canonical bands partition the positive integers; every raw source code maps to diff --git a/tests/test_noemp_band_evidence.py b/tests/test_noemp_band_evidence.py index 1a98c64a..4f1c1c44 100644 --- a/tests/test_noemp_band_evidence.py +++ b/tests/test_noemp_band_evidence.py @@ -1,7 +1,7 @@ """Pin the NOEMP band-label evidence artifact (issue #192). The committed ``runs/noemp_band_evidence_v1.json`` records the -discontinuity test behind the C2 decision to read ASEC NOEMP codes +discontinuity test behind the IC2 decision to read ASEC NOEMP codes 2/3 as 10-49 / 50-99 in every year. These tests pin the artifact's internal consistency, and — when the ASEC files are staged — reproduce it from the raw data. From 3fa68007f51a186ba0fb5b9e4076e036a9ac38c1 Mon Sep 17 00:00:00 2001 From: Vahid Ahmadi Date: Thu, 23 Jul 2026 15:41:30 +0100 Subject: [PATCH 08/13] Address review: finish the Proposed fix, rename the operative plan MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two should-fixes from Daphne's #277 review: 1. The stale "ADR 0003 (Proposed — this is a preview, not the frozen contract)" text was corrected in the module docstring but survived in `job_spells`'s own docstring. Post-merge the ADR says Accepted while that line said Proposed. It now mirrors the module wording: the schema is frozen; what is preview-grade is this collapse's single-`ref_year` coverage. 2. `docs/plans/employer-firm-plan.html` used C1/C2/C3 in the contract sense while being cited by the ADR's Context section as the operative split — the one file on the wrong side of the rename boundary. Renamed to IC1/IC2/IC3 (six lines; the SVG path data containing `C265,247` is untouched), and Amendment 1 now states the boundary explicitly: history keeps the old names, live documents are renamed, and the three unrenamed senses stay as the table gives them. Co-Authored-By: Claude Opus 4.8 (1M context) --- docs/adr/0003-employer-firm-extension.md | 8 ++++++++ docs/plans/employer-firm-plan.html | 12 ++++++------ src/populace_dynamics/data/sipp_jobs.py | 6 ++++-- 3 files changed, 18 insertions(+), 8 deletions(-) diff --git a/docs/adr/0003-employer-firm-extension.md b/docs/adr/0003-employer-firm-extension.md index 1b6591f3..ab8951cc 100644 --- a/docs/adr/0003-employer-firm-extension.md +++ b/docs/adr/0003-employer-firm-extension.md @@ -34,6 +34,14 @@ fingerprint side already carries. Prior discussion (issue #192, the ADR history, merged PR bodies) uses the old names and is not rewritten; this note is the mapping. +**Boundary: history keeps the old names, live documents are +renamed.** The plan (`docs/plans/employer-firm-plan.html`), cited by +the Context section below as the operative split, is a live document +and is renamed with this amendment, so a referee following the ADR's +own link does not meet unmapped names. The unrenamed senses in the +table above (the locked `gates.yaml` fingerprints, the SSA table +labels, the RNG substream) remain as they are, by the reasons given. + **Sign-off:** @vahid-ahmadi (Workstream B, author) · @daphnehanse11 (Workstream A) — the joint sign-off is recorded by the merge of the freeze PR: authorship by one workstream owner plus diff --git a/docs/plans/employer-firm-plan.html b/docs/plans/employer-firm-plan.html index 413453ea..56480a6b 100644 --- a/docs/plans/employer-firm-plan.html +++ b/docs/plans/employer-firm-plan.html @@ -255,11 +255,11 @@

Targets, register & calibration

INTERFACE CONTRACTS — frozen week 1, changed only by joint PR
    -
  • C1 · Spell schema. One table: person_id, spell_id, start_period, end_period, industry (major), firm_size_band, earnings_share, primary_job. A writes it, B reads it. Firm-size bands use the canonical banding B defines (C2). Multi-job resolved primary-job-only in phase 0.
    • -
  • C2 · Canonical firm-size banding + semantics. B proposes the band set reconcilable across NOEMP / SIPP-establishment / SUSB-enterprise, and the decision of what the variable means (administrative firm size, per review F5). A trains to it; documented in the ADR.
    • -
  • C3 · Gate pre-registration. Jointly authored employer gate block (E1–E12 thresholds after floor runs), split ownership as above, one referee round, locked before any candidate runs. Neither side's model work may start a one-shot run until C3 locks.
    • +
  • IC1 · Spell schema. One table: person_id, spell_id, start_period, end_period, industry (major), firm_size_band, earnings_share, primary_job. A writes it, B reads it. Firm-size bands use the canonical banding B defines (IC2). Multi-job resolved primary-job-only in phase 0.
    • +
  • IC2 · Canonical firm-size banding + semantics. B proposes the band set reconcilable across NOEMP / SIPP-establishment / SUSB-enterprise, and the decision of what the variable means (administrative firm size, per review F5). A trains to it; documented in the ADR.
    • +
  • IC3 · Gate pre-registration. Jointly authored employer gate block (E1–E12 thresholds after floor runs), split ownership as above, one referee round, locked before any candidate runs. Neither side's model work may start a one-shot run until IC3 locks.
-

Sync points: week 1 (freeze C1/C2), week 4 (lock C3), week 10 (joint phase-2 go/no-go with Max). Everything else is asynchronous — A can build readers/imputation against fixture spells; B can build the target pipeline and register against a synthetic spell file conforming to C1.

+

Sync points: week 1 (freeze IC1/IC2), week 4 (lock IC3), week 10 (joint phase-2 go/no-go with Max). Everything else is asynchronous — A can build readers/imputation against fixture spells; B can build the target pipeline and register against a synthetic spell file conforming to IC1.

Precedents — what similar projects did

@@ -277,8 +277,8 @@

Precedents — what similar projects did

Milestones

- - + + diff --git a/src/populace_dynamics/data/sipp_jobs.py b/src/populace_dynamics/data/sipp_jobs.py index 65be95df..43b65267 100644 --- a/src/populace_dynamics/data/sipp_jobs.py +++ b/src/populace_dynamics/data/sipp_jobs.py @@ -503,8 +503,10 @@ def job_spells(job_months: pd.DataFrame) -> pd.DataFrame: A spell is a maximal run of consecutive reference months for one (person, job id). The output mirrors the IC1 spell schema of ADR - 0003 (Proposed — this is a preview, not the frozen contract) and - doubles as Workstream B's generator for IC1-conforming fixtures. + 0003, which is **Accepted and frozen**; what remains preview-grade + is this collapse's own coverage (single ``ref_year`` only — see + below), not the schema's status. It doubles as Workstream B's + generator for IC1-conforming fixtures. Args: job_months: Output of :func:`read_sipp_job_months`. From c5b4d11f1b3bd6b34a3de8ea1a1c8d72e6777ee0 Mon Sep 17 00:00:00 2001 From: Vahid Ahmadi Date: Thu, 30 Jul 2026 13:39:30 +0200 Subject: [PATCH 09/13] Keep naming amendment outside sealed production sources --- docs/adr/0003-employer-firm-extension.md | 5 ++++ src/populace_dynamics/data/asec_firm_size.py | 4 +-- src/populace_dynamics/data/sipp_jobs.py | 30 ++++++++------------ src/populace_dynamics/firms/__init__.py | 2 +- src/populace_dynamics/firms/banding.py | 4 +-- 5 files changed, 22 insertions(+), 23 deletions(-) diff --git a/docs/adr/0003-employer-firm-extension.md b/docs/adr/0003-employer-firm-extension.md index ab8951cc..5eb3280b 100644 --- a/docs/adr/0003-employer-firm-extension.md +++ b/docs/adr/0003-employer-firm-extension.md @@ -41,6 +41,11 @@ and is renamed with this amendment, so a referee following the ADR's own link does not meet unmapped names. The unrenamed senses in the table above (the locked `gates.yaml` fingerprints, the SSA table labels, the RNG substream) remain as they are, by the reasons given. +Production-source docstrings sealed by the published first-estimates +replay ceremony also retain their historical `C1`/`C2` wording. They +are not operative contract text and are interpreted through this +one-to-one mapping; cosmetic edits would invalidate the sealed replay +identity. New source text uses the `IC` names. **Sign-off:** @vahid-ahmadi (Workstream B, author) · @daphnehanse11 (Workstream A) — the joint sign-off is recorded by diff --git a/src/populace_dynamics/data/asec_firm_size.py b/src/populace_dynamics/data/asec_firm_size.py index 6f78c3a3..5f32d12d 100644 --- a/src/populace_dynamics/data/asec_firm_size.py +++ b/src/populace_dynamics/data/asec_firm_size.py @@ -22,7 +22,7 @@ share (~7.5%), while a true 25-99 band carries ~15%. This reader therefore uses the 10-49 / 50-99 reading for all years and records the dictionary conflict here rather than silently following the -2019+ label text into a factor-two mis-band. Consequence for IC2: +2019+ label text into a factor-two mis-band. Consequence for C2: the 50-employee edge (ACA and state mandates) is directly observed in every supported year — the "post-2019 label cannot resolve the 50 cut" problem stated in earlier drafts dissolves. @@ -374,7 +374,7 @@ def firm_size_tabulation( "class_of_worker", ), ) -> pd.DataFrame: - """Weighted firm-size tabulation — the IC2 evidence artifact. + """Weighted firm-size tabulation — the C2 evidence artifact. Args: records: Output of :func:`read_asec_firm_size` (one or more diff --git a/src/populace_dynamics/data/sipp_jobs.py b/src/populace_dynamics/data/sipp_jobs.py index 43b65267..c722c9e5 100644 --- a/src/populace_dynamics/data/sipp_jobs.py +++ b/src/populace_dynamics/data/sipp_jobs.py @@ -1,4 +1,4 @@ -"""SIPP job-level monthly records and IC1-preview spells (issue #200). +"""SIPP job-level monthly records and C1-preview spells (issue #200). The 2014-redesign SIPP public-use files are the employer-firm plan's primary label panel (#192): one row per person-month (``SSUID`` x @@ -8,7 +8,7 @@ within-panel employer-attachment key that phase-1 transition hazards rest on. ``EJB{n}_EMPSIZE`` measures **establishment** size at the worker's location (the redesign dropped the all-locations question), -so it is the IC2 proxy-chain input, never firm size (ADR 0003; +so it is the C2 proxy-chain input, never firm size (ADR 0003; ``firms/banding.py``). Every variable this reader touches was verified against the Census @@ -28,14 +28,10 @@ string-typed in the API schema. ``job_spells`` collapses maximal consecutive-month runs per -(person, job id) into spell rows whose shape mirrors the IC1 spell -schema. It is still labeled **IC1-preview**, but for a narrower -reason than when it was written: ADR 0003 is now Accepted and IC1 -is frozen, so what remains preview-grade is this collapse's own -coverage (single ``ref_year`` only — cross-year spell linkage -raises rather than guessing), not the schema's status. The output -also serves as Workstream B's generator for IC1-conforming fixture -files. Attribute changes inside a spell +(person, job id) into spell rows whose shape mirrors the C1 spell +schema. It is labeled **C1-preview**: ADR 0003 is Proposed, not +frozen, and this output also serves as Workstream B's generator for +C1-conforming fixture files. Attribute changes inside a spell (class of worker, industry, establishment size) are surfaced via ``attributes_constant`` — never silently averaged. @@ -499,14 +495,12 @@ def read_sipp_job_months( def job_spells(job_months: pd.DataFrame) -> pd.DataFrame: - """Collapse job-months into IC1-preview spell rows. + """Collapse job-months into C1-preview spell rows. A spell is a maximal run of consecutive reference months for one - (person, job id). The output mirrors the IC1 spell schema of ADR - 0003, which is **Accepted and frozen**; what remains preview-grade - is this collapse's own coverage (single ``ref_year`` only — see - below), not the schema's status. It doubles as Workstream B's - generator for IC1-conforming fixtures. + (person, job id). The output mirrors the C1 spell schema of ADR + 0003 (Proposed — this is a preview, not the frozen contract) and + doubles as Workstream B's generator for C1-conforming fixtures. Args: job_months: Output of :func:`read_sipp_job_months`. @@ -566,7 +560,7 @@ def job_spells(job_months: pd.DataFrame) -> pd.DataFrame: ] ) - # Cross-year spell linkage is undefined in this IC1 preview: the + # Cross-year spell linkage is undefined in this C1 preview: the # break/run detection, the person-month earnings lookup, and the # spell edges all key on the calendar ``month`` (1-12) alone, so two # different reference years sharing a month would collapse into one @@ -581,7 +575,7 @@ def job_spells(job_months: pd.DataFrame) -> pd.DataFrame: raise ValueError( "job_spells received job-months spanning multiple ref_years " f"({sorted(int(y) for y in ref_years)}); cross-year spell " - "linkage is undefined in this IC1 preview. Collapse one SIPP " + "linkage is undefined in this C1 preview. Collapse one SIPP " "file's months at a time." ) diff --git a/src/populace_dynamics/firms/__init__.py b/src/populace_dynamics/firms/__init__.py index 9748b63e..daa46544 100644 --- a/src/populace_dynamics/firms/__init__.py +++ b/src/populace_dynamics/firms/__init__.py @@ -1,6 +1,6 @@ """Employer-firm extension, workstream B (firm side). -Canonical firm-size banding (interface contract IC2) and label-verified +Canonical firm-size banding (interface contract C2) and label-verified loaders for the committed external target extracts (SUSB, BDS, QWI, J2J). See ``docs/adr/0003-employer-firm-extension.md`` and issue #192. """ diff --git a/src/populace_dynamics/firms/banding.py b/src/populace_dynamics/firms/banding.py index df02e2e8..67843686 100644 --- a/src/populace_dynamics/firms/banding.py +++ b/src/populace_dynamics/firms/banding.py @@ -1,4 +1,4 @@ -"""Canonical firm-size banding — interface contract IC2. +"""Canonical firm-size banding — interface contract C2. **Semantics (review finding F5).** The canonical variable means *administrative enterprise size*: the total employment of the legal @@ -26,7 +26,7 @@ Bands are **headcount** bands. Policy thresholds stated in FTEs (the ACA applicable-large-employer cut is 50 *full-time equivalents* at 30 hours/week, not headcount) are handled by a person-side hours join and -are out of IC2 scope. +are out of C2 scope. **Canonical bands.** Five bands with edges at 10 / 50 / 100 / 500:: From 211152bb506e08b0a8b39dcae1c255365a71df0d Mon Sep 17 00:00:00 2001 From: Vahid Ahmadi Date: Thu, 30 Jul 2026 18:15:44 +0200 Subject: [PATCH 10/13] Promote Workstream A floor references to v1 --- runs/sipp_e8_e9_floors_v1.env.json | 7 ++ ...raft_v0.json => sipp_e8_e9_floors_v1.json} | 14 ++- runs/sipp_spell_floors_v1.env.json | 7 ++ ...raft_v0.json => sipp_spell_floors_v1.json} | 13 ++- runs/tenure_floors_v1.env.json | 7 ++ ...rs_draft_v0.json => tenure_floors_v1.json} | 13 ++- scripts/build_sipp_e8_e9_floors.py | 100 +++++++++++------- scripts/build_sipp_spell_floors.py | 72 +++++++++---- scripts/build_tenure_floors.py | 72 +++++++++---- tests/test_spell_floor_artifacts.py | 62 +++++++++-- tests/tier_counts.json | 2 +- 11 files changed, 271 insertions(+), 98 deletions(-) create mode 100644 runs/sipp_e8_e9_floors_v1.env.json rename runs/{sipp_e8_e9_floors_draft_v0.json => sipp_e8_e9_floors_v1.json} (76%) create mode 100644 runs/sipp_spell_floors_v1.env.json rename runs/{sipp_spell_floors_draft_v0.json => sipp_spell_floors_v1.json} (79%) create mode 100644 runs/tenure_floors_v1.env.json rename runs/{tenure_floors_draft_v0.json => tenure_floors_v1.json} (90%) diff --git a/runs/sipp_e8_e9_floors_v1.env.json b/runs/sipp_e8_e9_floors_v1.env.json new file mode 100644 index 00000000..9b594d93 --- /dev/null +++ b/runs/sipp_e8_e9_floors_v1.env.json @@ -0,0 +1,7 @@ +{ + "artifact": "sipp_e8_e9_floors_v1.json", + "status": "PROMOTION_ONLY_NOT_MEASUREMENT_ENVIRONMENT", + "measurement_environment": "UNAVAILABLE_NOT_RECORDED", + "strict_staging": "pu2023 unavailable; artifact was not rebuilt", + "required_follow_up_before_ic3_lock": "capture the on-data builder environment, source SHA256, and E9 distinct-person count during the registered rebuild" +} diff --git a/runs/sipp_e8_e9_floors_draft_v0.json b/runs/sipp_e8_e9_floors_v1.json similarity index 76% rename from runs/sipp_e8_e9_floors_draft_v0.json rename to runs/sipp_e8_e9_floors_v1.json index 8cc6884f..99a4bdbb 100644 --- a/runs/sipp_e8_e9_floors_draft_v0.json +++ b/runs/sipp_e8_e9_floors_v1.json @@ -1,18 +1,24 @@ { "artifact": "sipp_e8_e9_floors", - "version": "draft_v0", - "status": "DRAFT - NOT RATIFIED; C3 not locked; no thresholds", + "version": "v1", + "status": "PRE-LOCK REFERENCE - NOT RATIFIED; IC3 not locked; no thresholds. v1 is a pinning event, not a ratification", "issue": "192", - "deployment_scale_note": "RECORDED GAP (review of #212): these floors are half-vs-half, i.e. the sampling noise of ~50%-of-source estimates, while candidate runs will be scored on the full source - there is no candidate-context floor (gate-1 ctx20 analog) in this draft. Under root-n scaling the full-source floor is ~1/sqrt(2) (~0.71x) of the half-split floor, making half-split-derived thresholds conservative (too wide) at deployment scale; C3 decides whether to accept that conservatism, scale analytically, or require deployment-context floors at v1 promotion.", + "deployment_scale_note": "RECORDED GAP (review of #212): these floors are half-vs-half, i.e. the sampling noise of ~50%-of-source estimates, while IC3 proposes scoring on a 0.20 person holdout - there is no candidate-context floor (gate-1 ctx20 analog). Under root-n scaling, a 20% scoring frame has ~sqrt(0.5/0.2)=1.58x the sampling noise of the half-split basis, so these floors are mildly ANTI-conservative (too tight), not conservative. RECORDED_NOT_SATISFIED: IC3 must accept a registered analytic scale adjustment or require matching-context floors before candidate runs.", "source": "pu2023 (reference year 2022), persons observed all 12 reference months (censoring-free draft restriction, recorded; ESTIMAND NOTE per review: candidate runs scored against these cells must apply the identical full-year-persons restriction)", "method": "person-disjoint sha256 half-splits, seeds 0-19 (raised from 5 per review: E8 cells where floor sd exceeds the mean need a stable across-seed sd); rates floored on |log rate ratio|, earnings-change medians/IQRs on absolute gaps in log-points; weighted by WPFINWGT", "seam_caveat": "identical to the E4/E5 floors: half-splits share SIPP's seam structure; #214 carries the seam measurement", "stay_median_heaping_caveat": "within-job SIPP monthly earnings are mostly constant across a wave (dependent-interview reporting), so the stay-transition median log-change heaps at exactly 0 and its half-vs-half floor is degenerate (0.0) - the same failure class as the tenure quantile heaping; E9-stay thresholds should be stated on the IQR or a distributional distance, not the median", "thin_flag_units": { "e8_nonemployment_by_age": "rows per half, equal to persons (one row per person in the E8 frame) vs THIN_CELL_PERSONS=200", - "e9_transitions.earnings_change": "rows per half = consecutive-month transition pairs, not persons (a person can contribute up to 11) vs THIN_CELL_PERSONS=200" + "e9_transitions.earnings_change": "distinct persons per half (person_id.nunique(); rows are consecutive-month transition pairs and a person can contribute up to 11) vs THIN_CELL_PERSONS=200" }, "sipp_jobs_reader_commit": "a059193e4fad80ceb1c2e1f4177aa5c69abb1048", + "promotion_integrity": { + "measured_values": "preserved from the recorded on-data build; not rebuilt during v1 promotion", + "source_input_sha256_status": "BLOCKED_STRICT_STAGING: original raw input digest was not recorded and pu2023 is unavailable in the promotion environment", + "e9_distinct_person_count_status": "FOLLOW_UP_REQUIRED: not derivable from the aggregate draft artifact; the builder now emits persons_unweighted on the next on-data rebuild", + "required_follow_up_before_ic3_lock": "on-data owner must rerun the builder from the exact staged vintage, emit source SHA256 and a measurement-environment sidecar, verify the E9 distinct-person count, and reconcile every measured value" + }, "e8_nonemployment_by_age": { "16_24": { "any_nonemp_share": 0.4145, diff --git a/runs/sipp_spell_floors_v1.env.json b/runs/sipp_spell_floors_v1.env.json new file mode 100644 index 00000000..a1b988b7 --- /dev/null +++ b/runs/sipp_spell_floors_v1.env.json @@ -0,0 +1,7 @@ +{ + "artifact": "sipp_spell_floors_v1.json", + "status": "PROMOTION_ONLY_NOT_MEASUREMENT_ENVIRONMENT", + "measurement_environment": "UNAVAILABLE_NOT_RECORDED", + "strict_staging": "pu2023 unavailable; artifact was not rebuilt", + "required_follow_up_before_ic3_lock": "capture the on-data builder environment and source SHA256 during the registered rebuild" +} diff --git a/runs/sipp_spell_floors_draft_v0.json b/runs/sipp_spell_floors_v1.json similarity index 79% rename from runs/sipp_spell_floors_draft_v0.json rename to runs/sipp_spell_floors_v1.json index 20f21986..11aab396 100644 --- a/runs/sipp_spell_floors_draft_v0.json +++ b/runs/sipp_spell_floors_v1.json @@ -1,17 +1,22 @@ { "artifact": "sipp_spell_floors", - "version": "draft_v0", - "status": "DRAFT - NOT RATIFIED; C3 not locked; no thresholds", + "version": "v1", + "status": "PRE-LOCK REFERENCE - NOT RATIFIED; IC3 not locked; no thresholds. v1 is a pinning event, not a ratification", "issue": "192", - "deployment_scale_note": "RECORDED GAP (review of #212): these floors are half-vs-half, i.e. the sampling noise of ~50%-of-source estimates, while candidate runs will be scored on the full source - there is no candidate-context floor (gate-1 ctx20 analog) in this draft. Under root-n scaling the full-source floor is ~1/sqrt(2) (~0.71x) of the half-split floor, making half-split-derived thresholds conservative (too wide) at deployment scale; C3 decides whether to accept that conservatism, scale analytically, or require deployment-context floors at v1 promotion.", + "deployment_scale_note": "RECORDED GAP (review of #212): these floors are half-vs-half, i.e. the sampling noise of ~50%-of-source estimates, while IC3 proposes scoring on a 0.20 person holdout - there is no candidate-context floor (gate-1 ctx20 analog). Under root-n scaling, a 20% scoring frame has ~sqrt(0.5/0.2)=1.58x the sampling noise of the half-split basis, so these floors are mildly ANTI-conservative (too tight), not conservative. RECORDED_NOT_SATISFIED: IC3 must accept a registered analytic scale adjustment or require matching-context floors before candidate runs.", "source": "pu2023 (reference year 2022)", "method": "person-disjoint sha256 half-splits, seeds 0-4; per-cell |log(rate_a/rate_b)| mean/sd across seeds; weighted by WPFINWGT", - "seam_caveat": "both halves share SIPP seam structure; the seam-vs-J2J reconciliation run is a separate required artifact before thresholds lock", + "seam_caveat": "both halves share SIPP seam structure; the seam-vs-J2J reconciliation run is a separate required artifact before IC3 thresholds lock", "thin_flag_units": { "e4_retention_by_age_sex": "distinct persons per half (person_id.nunique(); rows are person-month retention pairs) vs THIN_CELL_PERSONS=200", "e5_runs_by_age": "rows per half, equal to persons (one row per person in the run-length frame) vs THIN_CELL_PERSONS=200" }, "sipp_jobs_reader_commit": "a059193e4fad80ceb1c2e1f4177aa5c69abb1048", + "promotion_integrity": { + "measured_values": "preserved from the recorded on-data build; not rebuilt during v1 promotion", + "source_input_sha256_status": "BLOCKED_STRICT_STAGING: original raw input digest was not recorded and pu2023 is unavailable in the promotion environment", + "required_follow_up_before_ic3_lock": "on-data owner must rerun the builder from the exact staged vintage, emit source SHA256 and a measurement-environment sidecar, and reconcile every measured value" + }, "e4_retention_by_age_sex": { "16_24|sex1": { "rate": 0.9897, diff --git a/runs/tenure_floors_v1.env.json b/runs/tenure_floors_v1.env.json new file mode 100644 index 00000000..fa29e90c --- /dev/null +++ b/runs/tenure_floors_v1.env.json @@ -0,0 +1,7 @@ +{ + "artifact": "tenure_floors_v1.json", + "status": "PROMOTION_ONLY_NOT_MEASUREMENT_ENVIRONMENT", + "measurement_environment": "UNAVAILABLE_NOT_RECORDED", + "strict_staging": "CPS tenure extracts for 2020/2022/2024 unavailable; artifact was not rebuilt", + "required_follow_up_before_ic3_lock": "capture the on-data builder environment and source SHA256 values during the registered rebuild" +} diff --git a/runs/tenure_floors_draft_v0.json b/runs/tenure_floors_v1.json similarity index 90% rename from runs/tenure_floors_draft_v0.json rename to runs/tenure_floors_v1.json index 4988ded9..dd1c3cf8 100644 --- a/runs/tenure_floors_draft_v0.json +++ b/runs/tenure_floors_v1.json @@ -1,14 +1,19 @@ { "artifact": "tenure_floors", - "version": "draft_v0", - "status": "DRAFT - NOT RATIFIED; C3 not locked; no thresholds", + "version": "v1", + "status": "PRE-LOCK REFERENCE - NOT RATIFIED; IC3 not locked; no thresholds. v1 is a pinning event, not a ratification", "issue": "192", - "deployment_scale_note": "RECORDED GAP (review of #212): these floors are half-vs-half, i.e. the sampling noise of ~50%-of-source estimates, while candidate runs will be scored on the full source - there is no candidate-context floor (gate-1 ctx20 analog) in this draft. Under root-n scaling the full-source floor is ~1/sqrt(2) (~0.71x) of the half-split floor, making half-split-derived thresholds conservative (too wide) at deployment scale; C3 decides whether to accept that conservatism, scale analytically, or require deployment-context floors at v1 promotion.", + "deployment_scale_note": "RECORDED GAP (review of #212): these floors are half-vs-half, i.e. the sampling noise of ~50%-of-source estimates, while IC3 proposes scoring on a 0.20 person holdout - there is no candidate-context floor (gate-1 ctx20 analog). Under root-n scaling, a 20% scoring frame has ~sqrt(0.5/0.2)=1.58x the sampling noise of the half-split basis, so these floors are mildly ANTI-conservative (too tight), not conservative. RECORDED_NOT_SATISFIED: IC3 must accept a registered analytic scale adjustment or require matching-context floors before candidate runs.", "source": "CPS January supplements 2020/2022/2024 (PTST1TN, PWTENWGT); reader per #205", "method": "person-disjoint sha256 half-splits, seeds 0-4; per-cell absolute weighted-quantile gap in years AND weighted-ECDF max gap (heaping-robust), mean/sd across seeds; BLS age bands", - "heaping_caveat": "reported tenure heaps on integers, so half-vs-half quantile gaps are frequently exactly zero (36/63 cells in the first build) - a degenerate threshold basis; the ECDF max-gap floor is the heaping-robust alternative for the C3 round to choose between", + "heaping_caveat": "reported tenure heaps on integers, so half-vs-half quantile gaps are frequently exactly zero (36/63 cells in the first build) - a degenerate threshold basis; the ECDF max-gap floor is the heaping-robust alternative for the IC3 round to choose between", "thin_flag_units": "rows per half, equal to persons (one CPS record per person) vs THIN_CELL_PERSONS=200", "cps_tenure_reader_commit": "5c9e5e67884bfe52d8d6c35ba6fdde40f52d5d62", + "promotion_integrity": { + "measured_values": "preserved from the recorded on-data build; not rebuilt during v1 promotion", + "source_input_sha256_status": "BLOCKED_STRICT_STAGING: original raw input digests were not recorded and the 2020/2022/2024 CPS extracts are unavailable in the promotion environment", + "required_follow_up_before_ic3_lock": "on-data owner must rerun the builder from the exact staged vintages, emit source SHA256 values and a measurement-environment sidecar, and reconcile every measured value" + }, "by_year": { "2020": { "16_19": { diff --git a/scripts/build_sipp_e8_e9_floors.py b/scripts/build_sipp_e8_e9_floors.py index 9b116d04..34082b6c 100644 --- a/scripts/build_sipp_e8_e9_floors.py +++ b/scripts/build_sipp_e8_e9_floors.py @@ -1,7 +1,7 @@ -"""Build DRAFT SIPP floors for gates E8 and E9 (#192). +"""Build pre-IC3 SIPP floor references for gates E8 and E9 (#192). -REPORTED ANCHOR, NOT A GATE RUN — and explicitly a DRAFT: C3 has -not locked, no thresholds are proposed. Completes Workstream A's +REPORTED ANCHOR, NOT A GATE RUN: IC3 has not locked and no thresholds +are proposed. Completes Workstream A's floor battery (E3 tenure and E4/E5 spells are committed siblings; E10 needs no floor — it is pass/fail on the locked PSID gates). @@ -28,18 +28,17 @@ rates, not earnings changes (their change is to/from zero by construction). -(c) **The floor**: person-disjoint sha256 half-splits, seeds 0-4; +(c) **The floor**: person-disjoint sha256 half-splits, seeds 0-19; for rates the |log rate ratio| between halves, for medians/IQRs the absolute gap in log-points; mean/sd across seeds. Cells under 200 unweighted persons per half are flagged thin. Thin-flag units (recorded for honesty across the floor battery): the E8 thin flag counts **rows** per half, which equal persons -because the E8 frame has one row per person; the E9 -earnings-change thin flag counts **rows** per half, which are -consecutive-month transition *pairs* (a person can contribute up -to 11), not persons — unlike the E4 spell floor, which counts -distinct persons. All compare against the same +because the E8 frame has one row per person; the E9 earnings-change +thin flag counts **distinct persons** per half +(``person_id.nunique()``), although the underlying rows are +consecutive-month transition pairs. All compare against the same ``THIN_CELL_PERSONS = 200``. Seam caveat: identical to the E4/E5 floors — both halves share @@ -50,7 +49,7 @@ python scripts/build_sipp_e8_e9_floors.py -writes ``runs/sipp_e8_e9_floors_draft_v0.json``. +writes ``runs/sipp_e8_e9_floors_v1.json``. """ from __future__ import annotations @@ -67,6 +66,7 @@ REPO = Path(__file__).resolve().parents[1] sys.path.insert(0, str(REPO / "src")) +from populace_dynamics.contract import environment_block # noqa: E402 from populace_dynamics.data import sipp_jobs # noqa: E402 YEAR = 2023 @@ -74,7 +74,22 @@ AGE_BANDS = ((16, 24), (25, 34), (35, 44), (45, 54), (55, 64), (65, 99)) THIN_CELL_PERSONS = 200 -ARTIFACT = REPO / "runs/sipp_e8_e9_floors_draft_v0.json" +ARTIFACT = REPO / "runs/sipp_e8_e9_floors_v1.json" +ENV_SIDECAR = ARTIFACT.with_suffix(".env.json") + + +def _source_path(year: int) -> Path: + data_dir = Path( + os.environ.get( + "POPULACE_DYNAMICS_SIPP_DIR", + str(Path("~/PolicyEngine/sipp-data").expanduser()), + ) + ).expanduser() + for suffix in (".csv", ".csv.gz"): + path = data_dir / f"pu{year}{suffix}" + if path.exists(): + return path + raise FileNotFoundError(f"pu{year}.csv[.gz] not staged") def _reader_commit() -> str: @@ -112,18 +127,7 @@ def _half(person_id: str, seed: int) -> int: def _person_month_universe(year: int) -> pd.DataFrame: - data_dir = Path( - os.environ.get( - "POPULACE_DYNAMICS_SIPP_DIR", - str(Path("~/PolicyEngine/sipp-data").expanduser()), - ) - ).expanduser() - for suffix in (".csv", ".csv.gz"): - path = data_dir / f"pu{year}{suffix}" - if path.exists(): - break - else: - raise FileNotFoundError(f"pu{year}.csv[.gz] not staged") + path = _source_path(year) raw = pd.read_csv( path, sep="|", @@ -307,6 +311,7 @@ def q(frame, qq): "median_log_change": round(med, 4), "iqr_log_change": round(iqr, 4), "pairs_unweighted": int(len(cell)), + "persons_unweighted": int(cell["person_id"].nunique()), "floor_abs_median_gap": { "mean": round(float(np.mean(med_gaps)), 5), "sd": round(float(np.std(med_gaps)), 5), @@ -326,21 +331,24 @@ def build() -> dict: pairs = e9_transition_frame(panel) return { "artifact": "sipp_e8_e9_floors", - "version": "draft_v0", - "status": "DRAFT - NOT RATIFIED; C3 not locked; no thresholds", + "version": "v1", + "status": ( + "PRE-LOCK REFERENCE - NOT RATIFIED; IC3 not locked; no " + "thresholds. v1 is a pinning event, not a ratification" + ), "issue": "192", "deployment_scale_note": ( "RECORDED GAP (review of #212): these floors are " "half-vs-half, i.e. the sampling noise of ~50%-of-source " - "estimates, while candidate runs will be scored on the " - "full source - there is no candidate-context floor " - "(gate-1 ctx20 analog) in this draft. Under root-n " - "scaling the full-source floor is ~1/sqrt(2) (~0.71x) " - "of the half-split floor, making half-split-derived " - "thresholds conservative (too wide) at deployment " - "scale; C3 decides whether to accept that conservatism, " - "scale analytically, or require deployment-context " - "floors at v1 promotion." + "estimates, while IC3 proposes scoring on a 0.20 person " + "holdout - there is no candidate-context floor (gate-1 " + "ctx20 analog). Under root-n scaling, a 20% scoring " + "frame has ~sqrt(0.5/0.2)=1.58x the sampling noise of " + "the half-split basis, so these floors are mildly " + "ANTI-conservative (too tight), not conservative. " + "RECORDED_NOT_SATISFIED: IC3 must accept a registered " + "analytic scale adjustment or require matching-context " + "floors before candidate runs." ), "source": f"pu{YEAR} (reference year {YEAR - 1}), persons " "observed all 12 reference months (censoring-free draft " @@ -374,12 +382,19 @@ def build() -> dict: "person in the E8 frame) vs THIN_CELL_PERSONS=200" ), "e9_transitions.earnings_change": ( - "rows per half = consecutive-month transition " - "pairs, not persons (a person can contribute up to " - "11) vs THIN_CELL_PERSONS=200" + "distinct persons per half (person_id.nunique(); " + "rows are consecutive-month transition pairs and a " + "person can contribute up to 11) vs " + "THIN_CELL_PERSONS=200" ), }, "sipp_jobs_reader_commit": _reader_commit(), + "source_input": { + "path": _source_path(YEAR).name, + "sha256": hashlib.sha256( + _source_path(YEAR).read_bytes() + ).hexdigest(), + }, "e8_nonemployment_by_age": e8_floors(persons), "e9_transitions": e9_floors(pairs), } @@ -388,6 +403,17 @@ def build() -> dict: def main() -> None: artifact = build() ARTIFACT.write_text(json.dumps(artifact, indent=2) + "\n") + ENV_SIDECAR.write_text( + json.dumps( + { + "artifact": ARTIFACT.name, + "status": "MEASUREMENT_ENVIRONMENT", + "environment": environment_block(), + }, + indent=2, + ) + + "\n" + ) print(f"wrote {ARTIFACT}") print("e9 transition mix:", artifact["e9_transitions"]["transition_rates"]) stay = artifact["e9_transitions"]["earnings_change"]["stay"] diff --git a/scripts/build_sipp_spell_floors.py b/scripts/build_sipp_spell_floors.py index 033c0bab..c0be5895 100644 --- a/scripts/build_sipp_spell_floors.py +++ b/scripts/build_sipp_spell_floors.py @@ -1,8 +1,8 @@ -"""Build DRAFT SIPP job-spell noise floors for gates E4/E5 (#192). +"""Build pre-IC3 SIPP job-spell noise-floor references (#192). -REPORTED ANCHOR, NOT A GATE RUN — and explicitly a DRAFT: C3 (the +REPORTED ANCHOR, NOT A GATE RUN: IC3 (the employer gate block) has not locked, no thresholds are proposed -here, and nothing below is ratified. Like the disability floors, +here, and v1 pinning does not ratify anything. Like the disability floors, this commits the person-disjoint half-vs-half sampling-noise floor that pre-registered E4/E5 thresholds would later be derived from, so the floor-building method is on the record before any candidate @@ -48,7 +48,7 @@ python scripts/build_sipp_spell_floors.py -writes ``runs/sipp_spell_floors_draft_v0.json``. +writes ``runs/sipp_spell_floors_v1.json``. """ from __future__ import annotations @@ -63,6 +63,7 @@ sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "src")) +from populace_dynamics.contract import environment_block # noqa: E402 from populace_dynamics.data import sipp_jobs # noqa: E402 YEAR = 2023 @@ -71,8 +72,20 @@ THIN_CELL_PERSONS = 200 ARTIFACT = Path(__file__).resolve().parents[1] / ( - "runs/sipp_spell_floors_draft_v0.json" + "runs/sipp_spell_floors_v1.json" ) +ENV_SIDECAR = ARTIFACT.with_suffix(".env.json") + + +def _source_pin() -> dict[str, str]: + path = sipp_jobs._resolve_person_path( # noqa: SLF001 + YEAR, + sipp_jobs._resolve_data_dir(None), # noqa: SLF001 + ) + return { + "path": path.name, + "sha256": hashlib.sha256(path.read_bytes()).hexdigest(), + } def _reader_commit() -> str: @@ -170,8 +183,9 @@ def floors_for_retention(pairs: pd.DataFrame) -> dict: halves_n.append( min(a["person_id"].nunique(), b["person_id"].nunique()) ) - ra, rb = _weighted_rate(a, "retained"), _weighted_rate( - b, "retained" + ra, rb = ( + _weighted_rate(a, "retained"), + _weighted_rate(b, "retained"), ) gaps.append(abs(np.log(ra / rb))) cells[f"{band}|sex{int(sex)}"] = { @@ -194,8 +208,9 @@ def floors_for_runs(runs: pd.DataFrame) -> dict: half = _half(cell["person_id"], seed) a, b = cell[half == 0], cell[half == 1] halves_n.append(min(len(a), len(b))) - ra, rb = _weighted_rate(a, "long_run"), _weighted_rate( - b, "long_run" + ra, rb = ( + _weighted_rate(a, "long_run"), + _weighted_rate(b, "long_run"), ) gaps.append(abs(np.log(ra / rb))) cells[str(band)] = { @@ -214,21 +229,24 @@ def build() -> dict: runs = run_length_frame(job_months) return { "artifact": "sipp_spell_floors", - "version": "draft_v0", - "status": "DRAFT - NOT RATIFIED; C3 not locked; no thresholds", + "version": "v1", + "status": ( + "PRE-LOCK REFERENCE - NOT RATIFIED; IC3 not locked; no " + "thresholds. v1 is a pinning event, not a ratification" + ), "issue": "192", "deployment_scale_note": ( "RECORDED GAP (review of #212): these floors are " "half-vs-half, i.e. the sampling noise of ~50%-of-source " - "estimates, while candidate runs will be scored on the " - "full source - there is no candidate-context floor " - "(gate-1 ctx20 analog) in this draft. Under root-n " - "scaling the full-source floor is ~1/sqrt(2) (~0.71x) " - "of the half-split floor, making half-split-derived " - "thresholds conservative (too wide) at deployment " - "scale; C3 decides whether to accept that conservatism, " - "scale analytically, or require deployment-context " - "floors at v1 promotion." + "estimates, while IC3 proposes scoring on a 0.20 person " + "holdout - there is no candidate-context floor (gate-1 " + "ctx20 analog). Under root-n scaling, a 20% scoring " + "frame has ~sqrt(0.5/0.2)=1.58x the sampling noise of " + "the half-split basis, so these floors are mildly " + "ANTI-conservative (too tight), not conservative. " + "RECORDED_NOT_SATISFIED: IC3 must accept a registered " + "analytic scale adjustment or require matching-context " + "floors before candidate runs." ), "source": f"pu{YEAR} (reference year {YEAR - 1})", "method": ( @@ -239,7 +257,7 @@ def build() -> dict: "seam_caveat": ( "both halves share SIPP seam structure; the seam-vs-J2J " "reconciliation run is a separate required artifact " - "before thresholds lock" + "before IC3 thresholds lock" ), "thin_flag_units": { "e4_retention_by_age_sex": ( @@ -254,6 +272,7 @@ def build() -> dict: ), }, "sipp_jobs_reader_commit": _reader_commit(), + "source_input": _source_pin(), "e4_retention_by_age_sex": floors_for_retention(pairs), "e5_runs_by_age": floors_for_runs(runs), } @@ -262,6 +281,17 @@ def build() -> dict: def main() -> None: artifact = build() ARTIFACT.write_text(json.dumps(artifact, indent=2) + "\n") + ENV_SIDECAR.write_text( + json.dumps( + { + "artifact": ARTIFACT.name, + "status": "MEASUREMENT_ENVIRONMENT", + "environment": environment_block(), + }, + indent=2, + ) + + "\n" + ) print(f"wrote {ARTIFACT}") e4 = artifact["e4_retention_by_age_sex"] print(f"E4 cells: {len(e4)}; example:", next(iter(e4.items()))) diff --git a/scripts/build_tenure_floors.py b/scripts/build_tenure_floors.py index 8af03e3e..3319953d 100644 --- a/scripts/build_tenure_floors.py +++ b/scripts/build_tenure_floors.py @@ -1,7 +1,7 @@ -"""Build DRAFT CPS tenure noise floors for gate E3 (#192). +"""Build pre-IC3 CPS tenure noise-floor references for gate E3 (#192). -REPORTED ANCHOR, NOT A GATE RUN — and explicitly a DRAFT: C3 has -not locked and no thresholds are proposed. E3's moment is the +REPORTED ANCHOR, NOT A GATE RUN: IC3 has not locked and no thresholds +are proposed; v1 pinning does not ratify anything. E3's moment is the tenure distribution (P25/P50/P75) by age band against the CPS January supplement; this commits the person-disjoint half-vs-half sampling-noise floor those thresholds would later be derived from, @@ -18,7 +18,7 @@ (both halves' quantiles land on the same heap) — a degenerate basis for a "quantile error vs floor" criterion. Each cell therefore also carries a weighted-ECDF max-gap (Kolmogorov-style) floor, which is -smooth under heaping; the C3 round can choose between the quantile +smooth under heaping; the IC3 round can choose between the quantile and distributional formulations with both on the record. All three staged supplements (2020/2022/2024) are floored independently — the across-year spread @@ -29,14 +29,14 @@ Thin-flag units: the thin flag counts **rows** per half, which equal persons because the CPS tenure frame has one record per person, against ``THIN_CELL_PERSONS = 200`` (the same constant the -SIPP spell floors use, where E4 counts distinct persons and E9 -counts transition pairs — units are recorded per artifact). +SIPP spell floors use, where E4 and E9 count distinct persons — +units are recorded per artifact). Usage:: python scripts/build_tenure_floors.py -writes ``runs/tenure_floors_draft_v0.json``. +writes ``runs/tenure_floors_v1.json``. """ from __future__ import annotations @@ -51,6 +51,7 @@ REPO = Path(__file__).resolve().parents[1] sys.path.insert(0, str(REPO / "src")) +from populace_dynamics.contract import environment_block # noqa: E402 from populace_dynamics.data import cps_tenure # noqa: E402 YEARS = (2020, 2022, 2024) @@ -58,7 +59,23 @@ QUANTILES = (0.25, 0.50, 0.75) THIN_CELL_PERSONS = 200 -ARTIFACT = REPO / "runs/tenure_floors_draft_v0.json" +ARTIFACT = REPO / "runs/tenure_floors_v1.json" +ENV_SIDECAR = ARTIFACT.with_suffix(".env.json") + + +def _source_pins() -> list[dict[str, str]]: + data_dir = cps_tenure._resolve_data_dir(None) # noqa: SLF001 + pins = [] + for year in YEARS: + path = cps_tenure._resolve_person_path(year, data_dir) # noqa: SLF001 + pins.append( + { + "year": str(year), + "path": path.name, + "sha256": hashlib.sha256(path.read_bytes()).hexdigest(), + } + ) + return pins def _reader_commit() -> str: @@ -196,21 +213,24 @@ def floors_for_year(year: int) -> dict: def build() -> dict: return { "artifact": "tenure_floors", - "version": "draft_v0", - "status": "DRAFT - NOT RATIFIED; C3 not locked; no thresholds", + "version": "v1", + "status": ( + "PRE-LOCK REFERENCE - NOT RATIFIED; IC3 not locked; no " + "thresholds. v1 is a pinning event, not a ratification" + ), "issue": "192", "deployment_scale_note": ( "RECORDED GAP (review of #212): these floors are " "half-vs-half, i.e. the sampling noise of ~50%-of-source " - "estimates, while candidate runs will be scored on the " - "full source - there is no candidate-context floor " - "(gate-1 ctx20 analog) in this draft. Under root-n " - "scaling the full-source floor is ~1/sqrt(2) (~0.71x) " - "of the half-split floor, making half-split-derived " - "thresholds conservative (too wide) at deployment " - "scale; C3 decides whether to accept that conservatism, " - "scale analytically, or require deployment-context " - "floors at v1 promotion." + "estimates, while IC3 proposes scoring on a 0.20 person " + "holdout - there is no candidate-context floor (gate-1 " + "ctx20 analog). Under root-n scaling, a 20% scoring " + "frame has ~sqrt(0.5/0.2)=1.58x the sampling noise of " + "the half-split basis, so these floors are mildly " + "ANTI-conservative (too tight), not conservative. " + "RECORDED_NOT_SATISFIED: IC3 must accept a registered " + "analytic scale adjustment or require matching-context " + "floors before candidate runs." ), "source": "CPS January supplements 2020/2022/2024 (PTST1TN, " "PWTENWGT); reader per #205", @@ -225,13 +245,14 @@ def build() -> dict: "quantile gaps are frequently exactly zero (36/63 cells " "in the first build) - a degenerate threshold basis; the " "ECDF max-gap floor is the heaping-robust alternative " - "for the C3 round to choose between" + "for the IC3 round to choose between" ), "thin_flag_units": ( "rows per half, equal to persons (one CPS record per " "person) vs THIN_CELL_PERSONS=200" ), "cps_tenure_reader_commit": _reader_commit(), + "source_inputs": _source_pins(), "by_year": {str(year): floors_for_year(year) for year in YEARS}, } @@ -239,6 +260,17 @@ def build() -> dict: def main() -> None: artifact = build() ARTIFACT.write_text(json.dumps(artifact, indent=2) + "\n") + ENV_SIDECAR.write_text( + json.dumps( + { + "artifact": ARTIFACT.name, + "status": "MEASUREMENT_ENVIRONMENT", + "environment": environment_block(), + }, + indent=2, + ) + + "\n" + ) print(f"wrote {ARTIFACT}") y24 = artifact["by_year"]["2024"] example = y24["35_44"] diff --git a/tests/test_spell_floor_artifacts.py b/tests/test_spell_floor_artifacts.py index 07d285e3..785082a8 100644 --- a/tests/test_spell_floor_artifacts.py +++ b/tests/test_spell_floor_artifacts.py @@ -1,35 +1,77 @@ -"""Pin the three Workstream A floor artifacts (#212, pre-C3).""" +"""Pin the three Workstream A floor artifacts (#212, pre-IC3).""" from __future__ import annotations +import hashlib import json from pathlib import Path import pytest RUNS = Path(__file__).resolve().parents[1] / "runs" +ROOT = RUNS.parent + +ARTIFACTS = { + "sipp_spell_floors_v1.json": "0110366a37a46fcc12b9a5665f3e6d5ea4c99fd2cabc4bf8cfc3fa8719890faf", + "tenure_floors_v1.json": "afbbb9ba38e0c69e78d94bd854c064dfae980f398092904b87b59a526a78e015", + "sipp_e8_e9_floors_v1.json": "28515717e83824056708a491b2702089cb439d5369deaff480f391c6f8862aa2", +} +BUILDERS = { + "scripts/build_sipp_spell_floors.py": "e869208f039a4005b78ffe27502f414f5e01775f64c64e89ffcad6effd838ffb", + "scripts/build_tenure_floors.py": "a4b0434196d5e3713c6ebed3302364fb60ec24e72bb82b039eef8d3bccc6d9eb", + "scripts/build_sipp_e8_e9_floors.py": "214d8a32afacb86396151fccfa19bd76c5cfce1bddc35fd75508cbfab6d9d45b", +} @pytest.fixture(scope="module") def spells() -> dict: - return json.loads((RUNS / "sipp_spell_floors_draft_v0.json").read_text()) + return json.loads((RUNS / "sipp_spell_floors_v1.json").read_text()) @pytest.fixture(scope="module") def tenure() -> dict: - return json.loads((RUNS / "tenure_floors_draft_v0.json").read_text()) + return json.loads((RUNS / "tenure_floors_v1.json").read_text()) @pytest.fixture(scope="module") def e8e9() -> dict: - return json.loads((RUNS / "sipp_e8_e9_floors_draft_v0.json").read_text()) + return json.loads((RUNS / "sipp_e8_e9_floors_v1.json").read_text()) -def test_all_carry_draft_status_and_scale_gap(spells, tenure, e8e9): +def test_all_carry_prelock_status_and_correct_scale_gap(spells, tenure, e8e9): for artifact in (spells, tenure, e8e9): - assert "DRAFT" in artifact["status"] + assert artifact["version"] == "v1" + assert "pinning event, not a ratification" in artifact["status"] assert "RECORDED GAP" in artifact["deployment_scale_note"] - assert "ctx20" in artifact["deployment_scale_note"] + assert "1.58x" in artifact["deployment_scale_note"] + assert ( + "ANTI-conservative (too tight)" + in artifact["deployment_scale_note"] + ) + assert "RECORDED_NOT_SATISFIED" in artifact["deployment_scale_note"] + assert artifact["promotion_integrity"][ + "source_input_sha256_status" + ].startswith("BLOCKED_STRICT_STAGING") + + +def test_artifact_builder_and_sidecar_pins(): + for relative, expected in ARTIFACTS.items(): + assert ( + hashlib.sha256((RUNS / relative).read_bytes()).hexdigest() + == expected + ) + sidecar = json.loads( + (RUNS / relative.replace(".json", ".env.json")).read_text() + ) + assert ( + sidecar["status"] == "PROMOTION_ONLY_NOT_MEASUREMENT_ENVIRONMENT" + ) + assert sidecar["measurement_environment"] == "UNAVAILABLE_NOT_RECORDED" + for relative, expected in BUILDERS.items(): + assert ( + hashlib.sha256((ROOT / relative).read_bytes()).hexdigest() + == expected + ) def test_e4_e5_pinned_values(spells): @@ -58,5 +100,11 @@ def test_e8_e9_pinned_values(e8e9): stay = e8e9["e9_transitions"]["earnings_change"]["stay"] assert stay["median_log_change"] == 0.0 assert "heaps at exactly 0" in e8e9["stay_median_heaping_caveat"] + assert ( + "FOLLOW_UP_REQUIRED" + in e8e9["promotion_integrity"]["e9_distinct_person_count_status"] + ) + builder = (ROOT / "scripts/build_sipp_e8_e9_floors.py").read_text() + assert '"persons_unweighted": int(cell["person_id"].nunique())' in builder e8 = e8e9["e8_nonemployment_by_age"]["16_24"] assert e8["any_nonemp_share"] == pytest.approx(0.4145, abs=0.001) diff --git a/tests/tier_counts.json b/tests/tier_counts.json index c7b2e9d4..d10bf665 100644 --- a/tests/tier_counts.json +++ b/tests/tier_counts.json @@ -2,7 +2,7 @@ "schema_version": 1, "counts": { "unit": 822, - "artifact": 1923, + "artifact": 1924, "integration_psid": 804, "reproduction_legacy": 520, "oracle_policyengine": 159 From c24809b081c448255a39b74870de70207f615724 Mon Sep 17 00:00:00 2001 From: Vahid Ahmadi Date: Thu, 30 Jul 2026 23:40:17 +0200 Subject: [PATCH 11/13] Resolve Workstream A strict staging provenance --- runs/sipp_e8_e9_floors_v1.env.json | 17 +++- runs/sipp_e8_e9_floors_v1.inputs.json | 16 ++++ runs/sipp_e8_e9_floors_v1.json | 10 +-- runs/sipp_spell_floors_v1.env.json | 17 +++- runs/sipp_spell_floors_v1.inputs.json | 16 ++++ runs/sipp_spell_floors_v1.json | 7 +- runs/tenure_floors_v1.env.json | 17 +++- runs/tenure_floors_v1.inputs.json | 27 +++++++ runs/tenure_floors_v1.json | 22 +++-- scripts/build_sipp_e8_e9_floors.py | 34 ++++++++ scripts/build_sipp_spell_floors.py | 39 ++++++++- scripts/build_tenure_floors.py | 29 +++++++ tests/test_spell_floor_artifacts.py | 111 ++++++++++++++++++++++---- 13 files changed, 320 insertions(+), 42 deletions(-) create mode 100644 runs/sipp_e8_e9_floors_v1.inputs.json create mode 100644 runs/sipp_spell_floors_v1.inputs.json create mode 100644 runs/tenure_floors_v1.inputs.json diff --git a/runs/sipp_e8_e9_floors_v1.env.json b/runs/sipp_e8_e9_floors_v1.env.json index 9b594d93..e719e6e4 100644 --- a/runs/sipp_e8_e9_floors_v1.env.json +++ b/runs/sipp_e8_e9_floors_v1.env.json @@ -1,7 +1,16 @@ { "artifact": "sipp_e8_e9_floors_v1.json", - "status": "PROMOTION_ONLY_NOT_MEASUREMENT_ENVIRONMENT", - "measurement_environment": "UNAVAILABLE_NOT_RECORDED", - "strict_staging": "pu2023 unavailable; artifact was not rebuilt", - "required_follow_up_before_ic3_lock": "capture the on-data builder environment, source SHA256, and E9 distinct-person count during the registered rebuild" + "status": "MEASUREMENT_ENVIRONMENT", + "environment": { + "python": "3.10.13", + "numpy": "2.1.3", + "pandas": "2.3.3", + "sklearn": "1.5.2", + "scipy": "1.13.1", + "platform": "macOS-26.5.2-arm64-arm-64bit", + "fitting_stack": { + "populace_fit": "absent", + "populace_frame": "absent" + } + } } diff --git a/runs/sipp_e8_e9_floors_v1.inputs.json b/runs/sipp_e8_e9_floors_v1.inputs.json new file mode 100644 index 00000000..76ea1e22 --- /dev/null +++ b/runs/sipp_e8_e9_floors_v1.inputs.json @@ -0,0 +1,16 @@ +{ + "artifact": "sipp_e8_e9_floors_v1.json", + "status": "SOURCE_INPUT_DIGESTS", + "official_source": { + "url": "https://www2.census.gov/programs-surveys/sipp/data/datasets/2023/pu2023_csv.zip", + "archive_sha256": "9c5363d56aca2041db20d46d17b81e9be931eb5b18bd5f5238b367d2dd7fb74b", + "archive_bytes": 109036604, + "archive_member": "pu2023.csv" + }, + "staged_input": { + "path": "pu2023.csv.gz", + "sha256": "1e49df7e013970ea60443e4e15ded3e8fda07643038471bf6b3429a7383fbf69", + "bytes": 109032744, + "transport_note": "gzip -n recompression of the sole CSV member from the verified official Census ZIP" + } +} diff --git a/runs/sipp_e8_e9_floors_v1.json b/runs/sipp_e8_e9_floors_v1.json index 99a4bdbb..13be3c5e 100644 --- a/runs/sipp_e8_e9_floors_v1.json +++ b/runs/sipp_e8_e9_floors_v1.json @@ -13,11 +13,9 @@ "e9_transitions.earnings_change": "distinct persons per half (person_id.nunique(); rows are consecutive-month transition pairs and a person can contribute up to 11) vs THIN_CELL_PERSONS=200" }, "sipp_jobs_reader_commit": "a059193e4fad80ceb1c2e1f4177aa5c69abb1048", - "promotion_integrity": { - "measured_values": "preserved from the recorded on-data build; not rebuilt during v1 promotion", - "source_input_sha256_status": "BLOCKED_STRICT_STAGING: original raw input digest was not recorded and pu2023 is unavailable in the promotion environment", - "e9_distinct_person_count_status": "FOLLOW_UP_REQUIRED: not derivable from the aggregate draft artifact; the builder now emits persons_unweighted on the next on-data rebuild", - "required_follow_up_before_ic3_lock": "on-data owner must rerun the builder from the exact staged vintage, emit source SHA256 and a measurement-environment sidecar, verify the E9 distinct-person count, and reconcile every measured value" + "source_input": { + "path": "pu2023.csv.gz", + "sha256": "1e49df7e013970ea60443e4e15ded3e8fda07643038471bf6b3429a7383fbf69" }, "e8_nonemployment_by_age": { "16_24": { @@ -123,6 +121,7 @@ "median_log_change": 0.0, "iqr_log_change": 0.0656, "pairs_unweighted": 163552, + "persons_unweighted": 16286, "floor_abs_median_gap": { "mean": 0.0, "sd": 0.0 @@ -137,6 +136,7 @@ "median_log_change": 0.2264, "iqr_log_change": 1.069, "pairs_unweighted": 545, + "persons_unweighted": 524, "floor_abs_median_gap": { "mean": 0.06659, "sd": 0.05025 diff --git a/runs/sipp_spell_floors_v1.env.json b/runs/sipp_spell_floors_v1.env.json index a1b988b7..56cc7cb9 100644 --- a/runs/sipp_spell_floors_v1.env.json +++ b/runs/sipp_spell_floors_v1.env.json @@ -1,7 +1,16 @@ { "artifact": "sipp_spell_floors_v1.json", - "status": "PROMOTION_ONLY_NOT_MEASUREMENT_ENVIRONMENT", - "measurement_environment": "UNAVAILABLE_NOT_RECORDED", - "strict_staging": "pu2023 unavailable; artifact was not rebuilt", - "required_follow_up_before_ic3_lock": "capture the on-data builder environment and source SHA256 during the registered rebuild" + "status": "MEASUREMENT_ENVIRONMENT", + "environment": { + "python": "3.10.13", + "numpy": "2.1.3", + "pandas": "2.3.3", + "sklearn": "1.5.2", + "scipy": "1.13.1", + "platform": "macOS-26.5.2-arm64-arm-64bit", + "fitting_stack": { + "populace_fit": "absent", + "populace_frame": "absent" + } + } } diff --git a/runs/sipp_spell_floors_v1.inputs.json b/runs/sipp_spell_floors_v1.inputs.json new file mode 100644 index 00000000..6a3b2766 --- /dev/null +++ b/runs/sipp_spell_floors_v1.inputs.json @@ -0,0 +1,16 @@ +{ + "artifact": "sipp_spell_floors_v1.json", + "status": "SOURCE_INPUT_DIGESTS", + "official_source": { + "url": "https://www2.census.gov/programs-surveys/sipp/data/datasets/2023/pu2023_csv.zip", + "archive_sha256": "9c5363d56aca2041db20d46d17b81e9be931eb5b18bd5f5238b367d2dd7fb74b", + "archive_bytes": 109036604, + "archive_member": "pu2023.csv" + }, + "staged_input": { + "path": "pu2023.csv.gz", + "sha256": "1e49df7e013970ea60443e4e15ded3e8fda07643038471bf6b3429a7383fbf69", + "bytes": 109032744, + "transport_note": "gzip -n recompression of the sole CSV member from the verified official Census ZIP" + } +} diff --git a/runs/sipp_spell_floors_v1.json b/runs/sipp_spell_floors_v1.json index 11aab396..c7e64b6e 100644 --- a/runs/sipp_spell_floors_v1.json +++ b/runs/sipp_spell_floors_v1.json @@ -12,10 +12,9 @@ "e5_runs_by_age": "rows per half, equal to persons (one row per person in the run-length frame) vs THIN_CELL_PERSONS=200" }, "sipp_jobs_reader_commit": "a059193e4fad80ceb1c2e1f4177aa5c69abb1048", - "promotion_integrity": { - "measured_values": "preserved from the recorded on-data build; not rebuilt during v1 promotion", - "source_input_sha256_status": "BLOCKED_STRICT_STAGING: original raw input digest was not recorded and pu2023 is unavailable in the promotion environment", - "required_follow_up_before_ic3_lock": "on-data owner must rerun the builder from the exact staged vintage, emit source SHA256 and a measurement-environment sidecar, and reconcile every measured value" + "source_input": { + "path": "pu2023.csv.gz", + "sha256": "1e49df7e013970ea60443e4e15ded3e8fda07643038471bf6b3429a7383fbf69" }, "e4_retention_by_age_sex": { "16_24|sex1": { diff --git a/runs/tenure_floors_v1.env.json b/runs/tenure_floors_v1.env.json index fa29e90c..20d95d11 100644 --- a/runs/tenure_floors_v1.env.json +++ b/runs/tenure_floors_v1.env.json @@ -1,7 +1,16 @@ { "artifact": "tenure_floors_v1.json", - "status": "PROMOTION_ONLY_NOT_MEASUREMENT_ENVIRONMENT", - "measurement_environment": "UNAVAILABLE_NOT_RECORDED", - "strict_staging": "CPS tenure extracts for 2020/2022/2024 unavailable; artifact was not rebuilt", - "required_follow_up_before_ic3_lock": "capture the on-data builder environment and source SHA256 values during the registered rebuild" + "status": "MEASUREMENT_ENVIRONMENT", + "environment": { + "python": "3.10.13", + "numpy": "2.1.3", + "pandas": "2.3.3", + "sklearn": "1.5.2", + "scipy": "1.13.1", + "platform": "macOS-26.5.2-arm64-arm-64bit", + "fitting_stack": { + "populace_fit": "absent", + "populace_frame": "absent" + } + } } diff --git a/runs/tenure_floors_v1.inputs.json b/runs/tenure_floors_v1.inputs.json new file mode 100644 index 00000000..f3a22c2d --- /dev/null +++ b/runs/tenure_floors_v1.inputs.json @@ -0,0 +1,27 @@ +{ + "artifact": "tenure_floors_v1.json", + "status": "SOURCE_INPUT_DIGESTS", + "source_inputs": [ + { + "year": "2020", + "path": "jan20pub.csv", + "sha256": "ab0383891ed4e953128e55692d03c7731c2963de7356abf83d872b1a3fcf5af5", + "bytes": 187155717, + "official_url": "https://www2.census.gov/programs-surveys/cps/datasets/2020/supp/jan20pub.csv" + }, + { + "year": "2022", + "path": "jan22pub.csv", + "sha256": "f09bd072084b00dc10f8336cf8ee3f35e6673a4d5827155183f33923ccce96c8", + "bytes": 172630423, + "official_url": "https://www2.census.gov/programs-surveys/cps/datasets/2022/supp/jan22pub.csv" + }, + { + "year": "2024", + "path": "jan24pub.csv", + "sha256": "ae0141ebe5f8255421e7e6c7cd1179caaf84e60a4297a58d8deddde32895771d", + "bytes": 171536589, + "official_url": "https://www2.census.gov/programs-surveys/cps/datasets/2024/supp/jan24pub.csv" + } + ] +} diff --git a/runs/tenure_floors_v1.json b/runs/tenure_floors_v1.json index dd1c3cf8..81acf47e 100644 --- a/runs/tenure_floors_v1.json +++ b/runs/tenure_floors_v1.json @@ -9,11 +9,23 @@ "heaping_caveat": "reported tenure heaps on integers, so half-vs-half quantile gaps are frequently exactly zero (36/63 cells in the first build) - a degenerate threshold basis; the ECDF max-gap floor is the heaping-robust alternative for the IC3 round to choose between", "thin_flag_units": "rows per half, equal to persons (one CPS record per person) vs THIN_CELL_PERSONS=200", "cps_tenure_reader_commit": "5c9e5e67884bfe52d8d6c35ba6fdde40f52d5d62", - "promotion_integrity": { - "measured_values": "preserved from the recorded on-data build; not rebuilt during v1 promotion", - "source_input_sha256_status": "BLOCKED_STRICT_STAGING: original raw input digests were not recorded and the 2020/2022/2024 CPS extracts are unavailable in the promotion environment", - "required_follow_up_before_ic3_lock": "on-data owner must rerun the builder from the exact staged vintages, emit source SHA256 values and a measurement-environment sidecar, and reconcile every measured value" - }, + "source_inputs": [ + { + "year": "2020", + "path": "jan20pub.csv", + "sha256": "ab0383891ed4e953128e55692d03c7731c2963de7356abf83d872b1a3fcf5af5" + }, + { + "year": "2022", + "path": "jan22pub.csv", + "sha256": "f09bd072084b00dc10f8336cf8ee3f35e6673a4d5827155183f33923ccce96c8" + }, + { + "year": "2024", + "path": "jan24pub.csv", + "sha256": "ae0141ebe5f8255421e7e6c7cd1179caaf84e60a4297a58d8deddde32895771d" + } + ], "by_year": { "2020": { "16_19": { diff --git a/scripts/build_sipp_e8_e9_floors.py b/scripts/build_sipp_e8_e9_floors.py index 34082b6c..441b02ca 100644 --- a/scripts/build_sipp_e8_e9_floors.py +++ b/scripts/build_sipp_e8_e9_floors.py @@ -76,6 +76,15 @@ ARTIFACT = REPO / "runs/sipp_e8_e9_floors_v1.json" ENV_SIDECAR = ARTIFACT.with_suffix(".env.json") +INPUT_SIDECAR = ARTIFACT.with_suffix(".inputs.json") +OFFICIAL_SOURCE_URL = ( + "https://www2.census.gov/programs-surveys/sipp/data/datasets/" + "2023/pu2023_csv.zip" +) +OFFICIAL_ARCHIVE_SHA256 = ( + "9c5363d56aca2041db20d46d17b81e9be931eb5b18bd5f5238b367d2dd7fb74b" +) +OFFICIAL_ARCHIVE_BYTES = 109_036_604 def _source_path(year: int) -> Path: @@ -403,6 +412,31 @@ def build() -> dict: def main() -> None: artifact = build() ARTIFACT.write_text(json.dumps(artifact, indent=2) + "\n") + staged_path = _source_path(YEAR) + INPUT_SIDECAR.write_text( + json.dumps( + { + "artifact": ARTIFACT.name, + "status": "SOURCE_INPUT_DIGESTS", + "official_source": { + "url": OFFICIAL_SOURCE_URL, + "archive_sha256": OFFICIAL_ARCHIVE_SHA256, + "archive_bytes": OFFICIAL_ARCHIVE_BYTES, + "archive_member": "pu2023.csv", + }, + "staged_input": { + **artifact["source_input"], + "bytes": staged_path.stat().st_size, + "transport_note": ( + "gzip -n recompression of the sole CSV member " + "from the verified official Census ZIP" + ), + }, + }, + indent=2, + ) + + "\n" + ) ENV_SIDECAR.write_text( json.dumps( { diff --git a/scripts/build_sipp_spell_floors.py b/scripts/build_sipp_spell_floors.py index c0be5895..7e0667c6 100644 --- a/scripts/build_sipp_spell_floors.py +++ b/scripts/build_sipp_spell_floors.py @@ -75,10 +75,19 @@ "runs/sipp_spell_floors_v1.json" ) ENV_SIDECAR = ARTIFACT.with_suffix(".env.json") +INPUT_SIDECAR = ARTIFACT.with_suffix(".inputs.json") +OFFICIAL_SOURCE_URL = ( + "https://www2.census.gov/programs-surveys/sipp/data/datasets/" + "2023/pu2023_csv.zip" +) +OFFICIAL_ARCHIVE_SHA256 = ( + "9c5363d56aca2041db20d46d17b81e9be931eb5b18bd5f5238b367d2dd7fb74b" +) +OFFICIAL_ARCHIVE_BYTES = 109_036_604 def _source_pin() -> dict[str, str]: - path = sipp_jobs._resolve_person_path( # noqa: SLF001 + path = sipp_jobs._resolve_pu_path( # noqa: SLF001 YEAR, sipp_jobs._resolve_data_dir(None), # noqa: SLF001 ) @@ -281,6 +290,34 @@ def build() -> dict: def main() -> None: artifact = build() ARTIFACT.write_text(json.dumps(artifact, indent=2) + "\n") + staged_path = sipp_jobs._resolve_pu_path( # noqa: SLF001 + YEAR, + sipp_jobs._resolve_data_dir(None), # noqa: SLF001 + ) + INPUT_SIDECAR.write_text( + json.dumps( + { + "artifact": ARTIFACT.name, + "status": "SOURCE_INPUT_DIGESTS", + "official_source": { + "url": OFFICIAL_SOURCE_URL, + "archive_sha256": OFFICIAL_ARCHIVE_SHA256, + "archive_bytes": OFFICIAL_ARCHIVE_BYTES, + "archive_member": "pu2023.csv", + }, + "staged_input": { + **artifact["source_input"], + "bytes": staged_path.stat().st_size, + "transport_note": ( + "gzip -n recompression of the sole CSV member " + "from the verified official Census ZIP" + ), + }, + }, + indent=2, + ) + + "\n" + ) ENV_SIDECAR.write_text( json.dumps( { diff --git a/scripts/build_tenure_floors.py b/scripts/build_tenure_floors.py index 3319953d..7fc2a0d6 100644 --- a/scripts/build_tenure_floors.py +++ b/scripts/build_tenure_floors.py @@ -61,6 +61,11 @@ ARTIFACT = REPO / "runs/tenure_floors_v1.json" ENV_SIDECAR = ARTIFACT.with_suffix(".env.json") +INPUT_SIDECAR = ARTIFACT.with_suffix(".inputs.json") +OFFICIAL_SOURCE_URL = ( + "https://www2.census.gov/programs-surveys/cps/datasets/" + "{year}/supp/jan{yy:02d}pub.csv" +) def _source_pins() -> list[dict[str, str]]: @@ -260,6 +265,30 @@ def build() -> dict: def main() -> None: artifact = build() ARTIFACT.write_text(json.dumps(artifact, indent=2) + "\n") + data_dir = cps_tenure._resolve_data_dir(None) # noqa: SLF001 + INPUT_SIDECAR.write_text( + json.dumps( + { + "artifact": ARTIFACT.name, + "status": "SOURCE_INPUT_DIGESTS", + "source_inputs": [ + { + **source, + "bytes": cps_tenure._resolve_person_path( # noqa: SLF001 + int(source["year"]), data_dir + ).stat().st_size, + "official_url": OFFICIAL_SOURCE_URL.format( + year=int(source["year"]), + yy=int(source["year"]) % 100, + ), + } + for source in artifact["source_inputs"] + ], + }, + indent=2, + ) + + "\n" + ) ENV_SIDECAR.write_text( json.dumps( { diff --git a/tests/test_spell_floor_artifacts.py b/tests/test_spell_floor_artifacts.py index 785082a8..4dfa3898 100644 --- a/tests/test_spell_floor_artifacts.py +++ b/tests/test_spell_floor_artifacts.py @@ -12,14 +12,14 @@ ROOT = RUNS.parent ARTIFACTS = { - "sipp_spell_floors_v1.json": "0110366a37a46fcc12b9a5665f3e6d5ea4c99fd2cabc4bf8cfc3fa8719890faf", - "tenure_floors_v1.json": "afbbb9ba38e0c69e78d94bd854c064dfae980f398092904b87b59a526a78e015", - "sipp_e8_e9_floors_v1.json": "28515717e83824056708a491b2702089cb439d5369deaff480f391c6f8862aa2", + "sipp_spell_floors_v1.json": "500b68034c9a301eb823e1d8f7584cf6c7654bf536247827353b0941d1d026ae", + "tenure_floors_v1.json": "08e67e5d362bbd0c1703c85fdb40624de094561f385eceb6d0a9eea4772cc6ff", + "sipp_e8_e9_floors_v1.json": "b360f04fc785eeb11c8e77e4128bdb8a98d31501a11f048fa2df4e86b1f7e059", } BUILDERS = { - "scripts/build_sipp_spell_floors.py": "e869208f039a4005b78ffe27502f414f5e01775f64c64e89ffcad6effd838ffb", - "scripts/build_tenure_floors.py": "a4b0434196d5e3713c6ebed3302364fb60ec24e72bb82b039eef8d3bccc6d9eb", - "scripts/build_sipp_e8_e9_floors.py": "214d8a32afacb86396151fccfa19bd76c5cfce1bddc35fd75508cbfab6d9d45b", + "scripts/build_sipp_spell_floors.py": "8ce7e41a9af71767672c39f7933ccde3c2eeaa0aa4f7044c5113f7430439d1dc", + "scripts/build_tenure_floors.py": "593237bfbba31e77183a8d38bf07ae6ad77eb9392abd323ce71aecdc3e6dfb9e", + "scripts/build_sipp_e8_e9_floors.py": "3b2209de9b10cf680f5a074ea0c56077b03ebda68a4a614f2e20f0d0c2455272", } @@ -49,9 +49,7 @@ def test_all_carry_prelock_status_and_correct_scale_gap(spells, tenure, e8e9): in artifact["deployment_scale_note"] ) assert "RECORDED_NOT_SATISFIED" in artifact["deployment_scale_note"] - assert artifact["promotion_integrity"][ - "source_input_sha256_status" - ].startswith("BLOCKED_STRICT_STAGING") + assert "promotion_integrity" not in artifact def test_artifact_builder_and_sidecar_pins(): @@ -63,10 +61,12 @@ def test_artifact_builder_and_sidecar_pins(): sidecar = json.loads( (RUNS / relative.replace(".json", ".env.json")).read_text() ) - assert ( - sidecar["status"] == "PROMOTION_ONLY_NOT_MEASUREMENT_ENVIRONMENT" - ) - assert sidecar["measurement_environment"] == "UNAVAILABLE_NOT_RECORDED" + assert sidecar["status"] == "MEASUREMENT_ENVIRONMENT" + environment = sidecar["environment"] + assert environment["python"] + assert environment["numpy"] + assert environment["pandas"] + assert environment["platform"] for relative, expected in BUILDERS.items(): assert ( hashlib.sha256((ROOT / relative).read_bytes()).hexdigest() @@ -74,6 +74,79 @@ def test_artifact_builder_and_sidecar_pins(): ) +def test_source_input_sidecars_match_artifacts(spells, tenure, e8e9): + sipp_inputs = json.loads( + (RUNS / "sipp_spell_floors_v1.inputs.json").read_text() + ) + e8e9_inputs = json.loads( + (RUNS / "sipp_e8_e9_floors_v1.inputs.json").read_text() + ) + for artifact, sidecar in ( + (spells, sipp_inputs), + (e8e9, e8e9_inputs), + ): + assert sidecar["status"] == "SOURCE_INPUT_DIGESTS" + assert sidecar["staged_input"]["path"] == artifact["source_input"][ + "path" + ] + assert sidecar["staged_input"]["sha256"] == artifact["source_input"][ + "sha256" + ] + assert sidecar["staged_input"]["bytes"] > 0 + official = sidecar["official_source"] + assert official["url"].startswith("https://www2.census.gov/") + assert official["archive_member"] == "pu2023.csv" + _assert_sha256(official["archive_sha256"]) + assert official["archive_bytes"] > 0 + + tenure_inputs = json.loads( + (RUNS / "tenure_floors_v1.inputs.json").read_text() + ) + assert tenure_inputs["status"] == "SOURCE_INPUT_DIGESTS" + for artifact_input, sidecar_input in zip( + tenure["source_inputs"], + tenure_inputs["source_inputs"], + strict=True, + ): + assert sidecar_input["path"] == artifact_input["path"] + assert sidecar_input["sha256"] == artifact_input["sha256"] + assert sidecar_input["bytes"] > 0 + assert sidecar_input["official_url"].startswith( + "https://www2.census.gov/" + ) + assert "/supp/" in sidecar_input["official_url"] + + +def _assert_sha256(value: str) -> None: + assert len(value) == 64 + int(value, 16) + + +def test_exact_source_input_digests_are_recorded(spells, tenure, e8e9): + assert spells["source_input"] == e8e9["source_input"] + assert spells["source_input"]["path"] == "pu2023.csv.gz" + _assert_sha256(spells["source_input"]["sha256"]) + + assert [item["year"] for item in tenure["source_inputs"]] == [ + "2020", + "2022", + "2024", + ] + assert [item["path"] for item in tenure["source_inputs"]] == [ + "jan20pub.csv", + "jan22pub.csv", + "jan24pub.csv", + ] + for item in tenure["source_inputs"]: + _assert_sha256(item["sha256"]) + + +def test_sipp_builder_uses_reader_path_resolver(): + builder = (ROOT / "scripts/build_sipp_spell_floors.py").read_text() + assert "sipp_jobs._resolve_pu_path(" in builder + assert "sipp_jobs._resolve_person_path(" not in builder + + def test_e4_e5_pinned_values(spells): e4 = spells["e4_retention_by_age_sex"]["16_24|sex1"] assert e4["rate"] == 0.9897 @@ -101,8 +174,16 @@ def test_e8_e9_pinned_values(e8e9): assert stay["median_log_change"] == 0.0 assert "heaps at exactly 0" in e8e9["stay_median_heaping_caveat"] assert ( - "FOLLOW_UP_REQUIRED" - in e8e9["promotion_integrity"]["e9_distinct_person_count_status"] + e8e9["e9_transitions"]["earnings_change"]["stay"][ + "persons_unweighted" + ] + == 16286 + ) + assert ( + e8e9["e9_transitions"]["earnings_change"]["j2j"][ + "persons_unweighted" + ] + == 524 ) builder = (ROOT / "scripts/build_sipp_e8_e9_floors.py").read_text() assert '"persons_unweighted": int(cell["person_id"].nunique())' in builder From 55ddae010e7a76ee72dd456eb215ba44f2d65558 Mon Sep 17 00:00:00 2001 From: Vahid Ahmadi Date: Fri, 31 Jul 2026 10:38:13 +0200 Subject: [PATCH 12/13] Format Workstream A floor seal files --- scripts/build_tenure_floors.py | 4 +++- tests/test_spell_floor_artifacts.py | 23 ++++++++++------------- 2 files changed, 13 insertions(+), 14 deletions(-) diff --git a/scripts/build_tenure_floors.py b/scripts/build_tenure_floors.py index 7fc2a0d6..aa0cb924 100644 --- a/scripts/build_tenure_floors.py +++ b/scripts/build_tenure_floors.py @@ -276,7 +276,9 @@ def main() -> None: **source, "bytes": cps_tenure._resolve_person_path( # noqa: SLF001 int(source["year"]), data_dir - ).stat().st_size, + ) + .stat() + .st_size, "official_url": OFFICIAL_SOURCE_URL.format( year=int(source["year"]), yy=int(source["year"]) % 100, diff --git a/tests/test_spell_floor_artifacts.py b/tests/test_spell_floor_artifacts.py index 4dfa3898..854e2db1 100644 --- a/tests/test_spell_floor_artifacts.py +++ b/tests/test_spell_floor_artifacts.py @@ -18,7 +18,7 @@ } BUILDERS = { "scripts/build_sipp_spell_floors.py": "8ce7e41a9af71767672c39f7933ccde3c2eeaa0aa4f7044c5113f7430439d1dc", - "scripts/build_tenure_floors.py": "593237bfbba31e77183a8d38bf07ae6ad77eb9392abd323ce71aecdc3e6dfb9e", + "scripts/build_tenure_floors.py": "fda07dec53ab11c41aab2b7f92dc0c18ecad0f3843d70256bb2f140351b273b1", "scripts/build_sipp_e8_e9_floors.py": "3b2209de9b10cf680f5a074ea0c56077b03ebda68a4a614f2e20f0d0c2455272", } @@ -86,12 +86,13 @@ def test_source_input_sidecars_match_artifacts(spells, tenure, e8e9): (e8e9, e8e9_inputs), ): assert sidecar["status"] == "SOURCE_INPUT_DIGESTS" - assert sidecar["staged_input"]["path"] == artifact["source_input"][ - "path" - ] - assert sidecar["staged_input"]["sha256"] == artifact["source_input"][ - "sha256" - ] + assert ( + sidecar["staged_input"]["path"] == artifact["source_input"]["path"] + ) + assert ( + sidecar["staged_input"]["sha256"] + == artifact["source_input"]["sha256"] + ) assert sidecar["staged_input"]["bytes"] > 0 official = sidecar["official_source"] assert official["url"].startswith("https://www2.census.gov/") @@ -174,15 +175,11 @@ def test_e8_e9_pinned_values(e8e9): assert stay["median_log_change"] == 0.0 assert "heaps at exactly 0" in e8e9["stay_median_heaping_caveat"] assert ( - e8e9["e9_transitions"]["earnings_change"]["stay"][ - "persons_unweighted" - ] + e8e9["e9_transitions"]["earnings_change"]["stay"]["persons_unweighted"] == 16286 ) assert ( - e8e9["e9_transitions"]["earnings_change"]["j2j"][ - "persons_unweighted" - ] + e8e9["e9_transitions"]["earnings_change"]["j2j"]["persons_unweighted"] == 524 ) builder = (ROOT / "scripts/build_sipp_e8_e9_floors.py").read_text() From 0bc4c2d27a0321d000bb506bb2d575c6e22941d4 Mon Sep 17 00:00:00 2001 From: vahid-ahmadi Date: Tue, 11 Aug 2026 19:12:38 +0100 Subject: [PATCH 13/13] Recount tier manifest after merging master (artifact 2,535 -> 2,543) The eight spell-floor artifact tests read committed runs/*.json evidence and classify into the artifact tier. Counts recollected from the committed tree, per the merge convention. Co-Authored-By: Claude Opus 5 (1M context) --- tests/README-tiers.md | 4 ++-- tests/tier_counts.json | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/README-tiers.md b/tests/README-tiers.md index f1ea5a53..c0cc1e17 100644 --- a/tests/README-tiers.md +++ b/tests/README-tiers.md @@ -39,8 +39,8 @@ pytest --collect-only -q -m oracle_policyengine | tail -1 | Tier | Tests at HEAD | |---|---:| | `unit` | 1,511 | -| `artifact` | 2,535 | +| `artifact` | 2,543 | | `integration_psid` | 848 | | `reproduction_legacy` | 520 | | `oracle_policyengine` | 159 | -| **Total** | **5,573** | +| **Total** | **5,581** | diff --git a/tests/tier_counts.json b/tests/tier_counts.json index d0c04de8..23d77fc0 100644 --- a/tests/tier_counts.json +++ b/tests/tier_counts.json @@ -2,7 +2,7 @@ "schema_version": 1, "counts": { "unit": 1511, - "artifact": 2535, + "artifact": 2543, "integration_psid": 848, "reproduction_legacy": 520, "oracle_policyengine": 159
Workstream A — Daphne (person side)Workstream B — Vahid (firm side)
SIPP job-level reader (label-verified, family.py pattern); CPS NOEMP/tenure loaders; ADR drafted jointly · freeze C1/C2Target pipeline: SUSB/BDS/QWI/J2J/JOLTS extracts committed with provenance notes; canonical banding proposal (C2)
SIPP noise-floor runs; seam-vs-J2J reconciliation run; draft E3–E5/E8–E10 thresholdsAggregate-side floor studies; target/gate partition; draft E1/E2/E6/E7/E11 thresholds · joint: referee round, lock C3
SIPP job-level reader (label-verified, family.py pattern); CPS NOEMP/tenure loaders; ADR drafted jointly · freeze IC1/IC2Target pipeline: SUSB/BDS/QWI/J2J/JOLTS extracts committed with provenance notes; canonical banding proposal (IC2)
SIPP noise-floor runs; seam-vs-J2J reconciliation run; draft E3–E5/E8–E10 thresholdsAggregate-side floor studies; target/gate partition; draft E1/E2/E6/E7/E11 thresholds · joint: referee round, lock IC3
Phase-0 QRF imputation of spells + attributes onto CPSCalibration of the imputed file to partitioned QWI/SUSB cells; E1/E7 evidence artifacts
Phase-1 transition candidates registered; one-shot runs against locked gatesBLM firm-type register prototype; E12 feasibility study (are published AKM/coworker moments sufficient targets?)
Joint: phase-2 go/no-go review with Max, based on committed gate evidence + the E12 identification story