diff --git a/runs/sipp_e8_e9_floors_v1.env.json b/runs/sipp_e8_e9_floors_v1.env.json new file mode 100644 index 00000000..e719e6e4 --- /dev/null +++ b/runs/sipp_e8_e9_floors_v1.env.json @@ -0,0 +1,16 @@ +{ + "artifact": "sipp_e8_e9_floors_v1.json", + "status": "MEASUREMENT_ENVIRONMENT", + "environment": { + "python": "3.10.13", + "numpy": "2.1.3", + "pandas": "2.3.3", + "sklearn": "1.5.2", + "scipy": "1.13.1", + "platform": "macOS-26.5.2-arm64-arm-64bit", + "fitting_stack": { + "populace_fit": "absent", + "populace_frame": "absent" + } + } +} diff --git a/runs/sipp_e8_e9_floors_v1.inputs.json b/runs/sipp_e8_e9_floors_v1.inputs.json new file mode 100644 index 00000000..76ea1e22 --- /dev/null +++ b/runs/sipp_e8_e9_floors_v1.inputs.json @@ -0,0 +1,16 @@ +{ + "artifact": "sipp_e8_e9_floors_v1.json", + "status": "SOURCE_INPUT_DIGESTS", + "official_source": { + "url": "https://www2.census.gov/programs-surveys/sipp/data/datasets/2023/pu2023_csv.zip", + "archive_sha256": "9c5363d56aca2041db20d46d17b81e9be931eb5b18bd5f5238b367d2dd7fb74b", + "archive_bytes": 109036604, + "archive_member": "pu2023.csv" + }, + "staged_input": { + "path": "pu2023.csv.gz", + "sha256": "1e49df7e013970ea60443e4e15ded3e8fda07643038471bf6b3429a7383fbf69", + "bytes": 109032744, + "transport_note": "gzip -n recompression of the sole CSV member from the verified official Census ZIP" + } +} diff --git a/runs/sipp_e8_e9_floors_v1.json b/runs/sipp_e8_e9_floors_v1.json new file mode 100644 index 00000000..13be3c5e --- /dev/null +++ b/runs/sipp_e8_e9_floors_v1.json @@ -0,0 +1,152 @@ +{ + "artifact": "sipp_e8_e9_floors", + "version": "v1", + "status": "PRE-LOCK REFERENCE - NOT RATIFIED; IC3 not locked; no thresholds. v1 is a pinning event, not a ratification", + "issue": "192", + "deployment_scale_note": "RECORDED GAP (review of #212): these floors are half-vs-half, i.e. the sampling noise of ~50%-of-source estimates, while IC3 proposes scoring on a 0.20 person holdout - there is no candidate-context floor (gate-1 ctx20 analog). Under root-n scaling, a 20% scoring frame has ~sqrt(0.5/0.2)=1.58x the sampling noise of the half-split basis, so these floors are mildly ANTI-conservative (too tight), not conservative. RECORDED_NOT_SATISFIED: IC3 must accept a registered analytic scale adjustment or require matching-context floors before candidate runs.", + "source": "pu2023 (reference year 2022), persons observed all 12 reference months (censoring-free draft restriction, recorded; ESTIMAND NOTE per review: candidate runs scored against these cells must apply the identical full-year-persons restriction)", + "method": "person-disjoint sha256 half-splits, seeds 0-19 (raised from 5 per review: E8 cells where floor sd exceeds the mean need a stable across-seed sd); rates floored on |log rate ratio|, earnings-change medians/IQRs on absolute gaps in log-points; weighted by WPFINWGT", + "seam_caveat": "identical to the E4/E5 floors: half-splits share SIPP's seam structure; #214 carries the seam measurement", + "stay_median_heaping_caveat": "within-job SIPP monthly earnings are mostly constant across a wave (dependent-interview reporting), so the stay-transition median log-change heaps at exactly 0 and its half-vs-half floor is degenerate (0.0) - the same failure class as the tenure quantile heaping; E9-stay thresholds should be stated on the IQR or a distributional distance, not the median", + "thin_flag_units": { + "e8_nonemployment_by_age": "rows per half, equal to persons (one row per person in the E8 frame) vs THIN_CELL_PERSONS=200", + "e9_transitions.earnings_change": "distinct persons per half (person_id.nunique(); rows are consecutive-month transition pairs and a person can contribute up to 11) vs THIN_CELL_PERSONS=200" + }, + "sipp_jobs_reader_commit": "a059193e4fad80ceb1c2e1f4177aa5c69abb1048", + "source_input": { + "path": "pu2023.csv.gz", + "sha256": "1e49df7e013970ea60443e4e15ded3e8fda07643038471bf6b3429a7383fbf69" + }, + "e8_nonemployment_by_age": { + "16_24": { + "any_nonemp_share": 0.4145, + "long_nonemp_share": 0.3422, + "persons_unweighted": 1781, + "any_nonemp": { + "floor_abs_log_ratio_mean": 0.05149, + "floor_abs_log_ratio_sd": 0.0356, + "thin": false + }, + "long_nonemp": { + "floor_abs_log_ratio_mean": 0.08586, + "floor_abs_log_ratio_sd": 0.04431, + "thin": false + } + }, + "25_34": { + "any_nonemp_share": 0.1401, + "long_nonemp_share": 0.0973, + "persons_unweighted": 3240, + "any_nonemp": { + "floor_abs_log_ratio_mean": 0.05885, + "floor_abs_log_ratio_sd": 0.04819, + "thin": false + }, + "long_nonemp": { + "floor_abs_log_ratio_mean": 0.10659, + "floor_abs_log_ratio_sd": 0.06993, + "thin": false + } + }, + "35_44": { + "any_nonemp_share": 0.1017, + "long_nonemp_share": 0.0663, + "persons_unweighted": 3543, + "any_nonemp": { + "floor_abs_log_ratio_mean": 0.09917, + "floor_abs_log_ratio_sd": 0.07372, + "thin": false + }, + "long_nonemp": { + "floor_abs_log_ratio_mean": 0.13369, + "floor_abs_log_ratio_sd": 0.0899, + "thin": false + } + }, + "45_54": { + "any_nonemp_share": 0.0852, + "long_nonemp_share": 0.0555, + "persons_unweighted": 3241, + "any_nonemp": { + "floor_abs_log_ratio_mean": 0.10871, + "floor_abs_log_ratio_sd": 0.078, + "thin": false + }, + "long_nonemp": { + "floor_abs_log_ratio_mean": 0.09797, + "floor_abs_log_ratio_sd": 0.07416, + "thin": false + } + }, + "55_64": { + "any_nonemp_share": 0.1045, + "long_nonemp_share": 0.083, + "persons_unweighted": 3449, + "any_nonemp": { + "floor_abs_log_ratio_mean": 0.09033, + "floor_abs_log_ratio_sd": 0.05384, + "thin": false + }, + "long_nonemp": { + "floor_abs_log_ratio_mean": 0.11965, + "floor_abs_log_ratio_sd": 0.05523, + "thin": false + } + }, + "65_99": { + "any_nonemp_share": 0.1887, + "long_nonemp_share": 0.1546, + "persons_unweighted": 2286, + "any_nonemp": { + "floor_abs_log_ratio_mean": 0.05524, + "floor_abs_log_ratio_sd": 0.04697, + "thin": false + }, + "long_nonemp": { + "floor_abs_log_ratio_mean": 0.08085, + "floor_abs_log_ratio_sd": 0.04997, + "thin": false + } + } + }, + "e9_transitions": { + "transition_rates": { + "entry": 0.0101, + "exit": 0.0094, + "j2j": 0.0035, + "stay": 0.977 + }, + "earnings_change": { + "stay": { + "median_log_change": 0.0, + "iqr_log_change": 0.0656, + "pairs_unweighted": 163552, + "persons_unweighted": 16286, + "floor_abs_median_gap": { + "mean": 0.0, + "sd": 0.0 + }, + "floor_abs_iqr_gap": { + "mean": 0.0, + "sd": 0.0 + }, + "thin": false + }, + "j2j": { + "median_log_change": 0.2264, + "iqr_log_change": 1.069, + "pairs_unweighted": 545, + "persons_unweighted": 524, + "floor_abs_median_gap": { + "mean": 0.06659, + "sd": 0.05025 + }, + "floor_abs_iqr_gap": { + "mean": 0.12848, + "sd": 0.07392 + }, + "thin": false + } + } + } +} diff --git a/runs/sipp_spell_floors_v1.env.json b/runs/sipp_spell_floors_v1.env.json new file mode 100644 index 00000000..56cc7cb9 --- /dev/null +++ b/runs/sipp_spell_floors_v1.env.json @@ -0,0 +1,16 @@ +{ + "artifact": "sipp_spell_floors_v1.json", + "status": "MEASUREMENT_ENVIRONMENT", + "environment": { + "python": "3.10.13", + "numpy": "2.1.3", + "pandas": "2.3.3", + "sklearn": "1.5.2", + "scipy": "1.13.1", + "platform": "macOS-26.5.2-arm64-arm-64bit", + "fitting_stack": { + "populace_fit": "absent", + "populace_frame": "absent" + } + } +} diff --git a/runs/sipp_spell_floors_v1.inputs.json b/runs/sipp_spell_floors_v1.inputs.json new file mode 100644 index 00000000..6a3b2766 --- /dev/null +++ b/runs/sipp_spell_floors_v1.inputs.json @@ -0,0 +1,16 @@ +{ + "artifact": "sipp_spell_floors_v1.json", + "status": "SOURCE_INPUT_DIGESTS", + "official_source": { + "url": "https://www2.census.gov/programs-surveys/sipp/data/datasets/2023/pu2023_csv.zip", + "archive_sha256": "9c5363d56aca2041db20d46d17b81e9be931eb5b18bd5f5238b367d2dd7fb74b", + "archive_bytes": 109036604, + "archive_member": "pu2023.csv" + }, + "staged_input": { + "path": "pu2023.csv.gz", + "sha256": "1e49df7e013970ea60443e4e15ded3e8fda07643038471bf6b3429a7383fbf69", + "bytes": 109032744, + "transport_note": "gzip -n recompression of the sole CSV member from the verified official Census ZIP" + } +} diff --git a/runs/sipp_spell_floors_v1.json b/runs/sipp_spell_floors_v1.json new file mode 100644 index 00000000..c7e64b6e --- /dev/null +++ b/runs/sipp_spell_floors_v1.json @@ -0,0 +1,149 @@ +{ + "artifact": "sipp_spell_floors", + "version": "v1", + "status": "PRE-LOCK REFERENCE - NOT RATIFIED; IC3 not locked; no thresholds. v1 is a pinning event, not a ratification", + "issue": "192", + "deployment_scale_note": "RECORDED GAP (review of #212): these floors are half-vs-half, i.e. the sampling noise of ~50%-of-source estimates, while IC3 proposes scoring on a 0.20 person holdout - there is no candidate-context floor (gate-1 ctx20 analog). Under root-n scaling, a 20% scoring frame has ~sqrt(0.5/0.2)=1.58x the sampling noise of the half-split basis, so these floors are mildly ANTI-conservative (too tight), not conservative. RECORDED_NOT_SATISFIED: IC3 must accept a registered analytic scale adjustment or require matching-context floors before candidate runs.", + "source": "pu2023 (reference year 2022)", + "method": "person-disjoint sha256 half-splits, seeds 0-4; per-cell |log(rate_a/rate_b)| mean/sd across seeds; weighted by WPFINWGT", + "seam_caveat": "both halves share SIPP seam structure; the seam-vs-J2J reconciliation run is a separate required artifact before IC3 thresholds lock", + "thin_flag_units": { + "e4_retention_by_age_sex": "distinct persons per half (person_id.nunique(); rows are person-month retention pairs) vs THIN_CELL_PERSONS=200", + "e5_runs_by_age": "rows per half, equal to persons (one row per person in the run-length frame) vs THIN_CELL_PERSONS=200" + }, + "sipp_jobs_reader_commit": "a059193e4fad80ceb1c2e1f4177aa5c69abb1048", + "source_input": { + "path": "pu2023.csv.gz", + "sha256": "1e49df7e013970ea60443e4e15ded3e8fda07643038471bf6b3429a7383fbf69" + }, + "e4_retention_by_age_sex": { + "16_24|sex1": { + "rate": 0.9897, + "pairs_unweighted": 8055, + "floor_abs_log_ratio_mean": 0.00182, + "floor_abs_log_ratio_sd": 0.00107, + "thin": false + }, + "16_24|sex2": { + "rate": 0.9901, + "pairs_unweighted": 7093, + "floor_abs_log_ratio_mean": 0.00198, + "floor_abs_log_ratio_sd": 0.00232, + "thin": false + }, + "25_34|sex1": { + "rate": 0.9948, + "pairs_unweighted": 17797, + "floor_abs_log_ratio_mean": 0.0008, + "floor_abs_log_ratio_sd": 0.00036, + "thin": false + }, + "25_34|sex2": { + "rate": 0.9953, + "pairs_unweighted": 15555, + "floor_abs_log_ratio_mean": 0.00143, + "floor_abs_log_ratio_sd": 0.00114, + "thin": false + }, + "35_44|sex1": { + "rate": 0.9968, + "pairs_unweighted": 19920, + "floor_abs_log_ratio_mean": 0.00097, + "floor_abs_log_ratio_sd": 0.00058, + "thin": false + }, + "35_44|sex2": { + "rate": 0.9964, + "pairs_unweighted": 17203, + "floor_abs_log_ratio_mean": 0.00089, + "floor_abs_log_ratio_sd": 0.00065, + "thin": false + }, + "45_54|sex1": { + "rate": 0.9978, + "pairs_unweighted": 17779, + "floor_abs_log_ratio_mean": 0.0003, + "floor_abs_log_ratio_sd": 0.00028, + "thin": false + }, + "45_54|sex2": { + "rate": 0.998, + "pairs_unweighted": 16516, + "floor_abs_log_ratio_mean": 0.00051, + "floor_abs_log_ratio_sd": 0.00045, + "thin": false + }, + "55_64|sex1": { + "rate": 0.9984, + "pairs_unweighted": 18342, + "floor_abs_log_ratio_mean": 0.00067, + "floor_abs_log_ratio_sd": 0.00058, + "thin": false + }, + "55_64|sex2": { + "rate": 0.999, + "pairs_unweighted": 17410, + "floor_abs_log_ratio_mean": 0.0003, + "floor_abs_log_ratio_sd": 0.00014, + "thin": false + }, + "65_99|sex1": { + "rate": 0.9992, + "pairs_unweighted": 12113, + "floor_abs_log_ratio_mean": 0.00058, + "floor_abs_log_ratio_sd": 0.00039, + "thin": false + }, + "65_99|sex2": { + "rate": 0.9996, + "pairs_unweighted": 10437, + "floor_abs_log_ratio_mean": 0.00039, + "floor_abs_log_ratio_sd": 0.00032, + "thin": false + } + }, + "e5_runs_by_age": { + "16_24": { + "full_year_run_share": 0.4669, + "persons_unweighted": 1830, + "floor_abs_log_ratio_mean": 0.04937, + "floor_abs_log_ratio_sd": 0.04904, + "thin": false + }, + "25_34": { + "full_year_run_share": 0.7499, + "persons_unweighted": 3299, + "floor_abs_log_ratio_mean": 0.01944, + "floor_abs_log_ratio_sd": 0.01104, + "thin": false + }, + "35_44": { + "full_year_run_share": 0.8245, + "persons_unweighted": 3555, + "floor_abs_log_ratio_mean": 0.01764, + "floor_abs_log_ratio_sd": 0.01639, + "thin": false + }, + "45_54": { + "full_year_run_share": 0.8671, + "persons_unweighted": 3256, + "floor_abs_log_ratio_mean": 0.01096, + "floor_abs_log_ratio_sd": 0.00768, + "thin": false + }, + "55_64": { + "full_year_run_share": 0.8673, + "persons_unweighted": 3458, + "floor_abs_log_ratio_mean": 0.01742, + "floor_abs_log_ratio_sd": 0.00998, + "thin": false + }, + "65_99": { + "full_year_run_share": 0.7941, + "persons_unweighted": 2294, + "floor_abs_log_ratio_mean": 0.01327, + "floor_abs_log_ratio_sd": 0.01036, + "thin": false + } + } +} diff --git a/runs/tenure_floors_v1.env.json b/runs/tenure_floors_v1.env.json new file mode 100644 index 00000000..20d95d11 --- /dev/null +++ b/runs/tenure_floors_v1.env.json @@ -0,0 +1,16 @@ +{ + "artifact": "tenure_floors_v1.json", + "status": "MEASUREMENT_ENVIRONMENT", + "environment": { + "python": "3.10.13", + "numpy": "2.1.3", + "pandas": "2.3.3", + "sklearn": "1.5.2", + "scipy": "1.13.1", + "platform": "macOS-26.5.2-arm64-arm-64bit", + "fitting_stack": { + "populace_fit": "absent", + "populace_frame": "absent" + } + } +} diff --git a/runs/tenure_floors_v1.inputs.json b/runs/tenure_floors_v1.inputs.json new file mode 100644 index 00000000..f3a22c2d --- /dev/null +++ b/runs/tenure_floors_v1.inputs.json @@ -0,0 +1,27 @@ +{ + "artifact": "tenure_floors_v1.json", + "status": "SOURCE_INPUT_DIGESTS", + "source_inputs": [ + { + "year": "2020", + "path": "jan20pub.csv", + "sha256": "ab0383891ed4e953128e55692d03c7731c2963de7356abf83d872b1a3fcf5af5", + "bytes": 187155717, + "official_url": "https://www2.census.gov/programs-surveys/cps/datasets/2020/supp/jan20pub.csv" + }, + { + "year": "2022", + "path": "jan22pub.csv", + "sha256": "f09bd072084b00dc10f8336cf8ee3f35e6673a4d5827155183f33923ccce96c8", + "bytes": 172630423, + "official_url": "https://www2.census.gov/programs-surveys/cps/datasets/2022/supp/jan22pub.csv" + }, + { + "year": "2024", + "path": "jan24pub.csv", + "sha256": "ae0141ebe5f8255421e7e6c7cd1179caaf84e60a4297a58d8deddde32895771d", + "bytes": 171536589, + "official_url": "https://www2.census.gov/programs-surveys/cps/datasets/2024/supp/jan24pub.csv" + } + ] +} diff --git a/runs/tenure_floors_v1.json b/runs/tenure_floors_v1.json new file mode 100644 index 00000000..81acf47e --- /dev/null +++ b/runs/tenure_floors_v1.json @@ -0,0 +1,562 @@ +{ + "artifact": "tenure_floors", + "version": "v1", + "status": "PRE-LOCK REFERENCE - NOT RATIFIED; IC3 not locked; no thresholds. v1 is a pinning event, not a ratification", + "issue": "192", + "deployment_scale_note": "RECORDED GAP (review of #212): these floors are half-vs-half, i.e. the sampling noise of ~50%-of-source estimates, while IC3 proposes scoring on a 0.20 person holdout - there is no candidate-context floor (gate-1 ctx20 analog). Under root-n scaling, a 20% scoring frame has ~sqrt(0.5/0.2)=1.58x the sampling noise of the half-split basis, so these floors are mildly ANTI-conservative (too tight), not conservative. RECORDED_NOT_SATISFIED: IC3 must accept a registered analytic scale adjustment or require matching-context floors before candidate runs.", + "source": "CPS January supplements 2020/2022/2024 (PTST1TN, PWTENWGT); reader per #205", + "method": "person-disjoint sha256 half-splits, seeds 0-4; per-cell absolute weighted-quantile gap in years AND weighted-ECDF max gap (heaping-robust), mean/sd across seeds; BLS age bands", + "heaping_caveat": "reported tenure heaps on integers, so half-vs-half quantile gaps are frequently exactly zero (36/63 cells in the first build) - a degenerate threshold basis; the ECDF max-gap floor is the heaping-robust alternative for the IC3 round to choose between", + "thin_flag_units": "rows per half, equal to persons (one CPS record per person) vs THIN_CELL_PERSONS=200", + "cps_tenure_reader_commit": "5c9e5e67884bfe52d8d6c35ba6fdde40f52d5d62", + "source_inputs": [ + { + "year": "2020", + "path": "jan20pub.csv", + "sha256": "ab0383891ed4e953128e55692d03c7731c2963de7356abf83d872b1a3fcf5af5" + }, + { + "year": "2022", + "path": "jan22pub.csv", + "sha256": "f09bd072084b00dc10f8336cf8ee3f35e6673a4d5827155183f33923ccce96c8" + }, + { + "year": "2024", + "path": "jan24pub.csv", + "sha256": "ae0141ebe5f8255421e7e6c7cd1179caaf84e60a4297a58d8deddde32895771d" + } + ], + "by_year": { + "2020": { + "16_19": { + "p25": 0.33, + "p50": 0.58, + "p75": 1.08, + "persons_unweighted": 1345, + "floor_abs_gap_years": { + "p25": { + "mean": 0.015, + "sd": 0.03 + }, + "p50": { + "mean": 0.06, + "sd": 0.038 + }, + "p75": { + "mean": 0.091, + "sd": 0.042 + } + }, + "floor_ecdf_max_gap": { + "mean": 0.0383, + "sd": 0.0083 + }, + "thin": false + }, + "20_24": { + "p25": 0.5, + "p50": 1.0, + "p75": 2.5, + "persons_unweighted": 3443, + "floor_abs_gap_years": { + "p25": { + "mean": 0.0, + "sd": 0.0 + }, + "p50": { + "mean": 0.033, + "sd": 0.038 + }, + "p75": { + "mean": 0.143, + "sd": 0.081 + } + }, + "floor_ecdf_max_gap": { + "mean": 0.0276, + "sd": 0.0059 + }, + "thin": false + }, + "25_34": { + "p25": 1.0, + "p50": 3.0, + "p75": 5.0, + "persons_unweighted": 9659, + "floor_abs_gap_years": { + "p25": { + "mean": 0.0, + "sd": 0.0 + }, + "p50": { + "mean": 0.0, + "sd": 0.0 + }, + "p75": { + "mean": 0.0, + "sd": 0.0 + } + }, + "floor_ecdf_max_gap": { + "mean": 0.018, + "sd": 0.0059 + }, + "thin": false + }, + "35_44": { + "p25": 2.0, + "p50": 5.0, + "p75": 10.0, + "persons_unweighted": 9915, + "floor_abs_gap_years": { + "p25": { + "mean": 0.016, + "sd": 0.032 + }, + "p50": { + "mean": 0.0, + "sd": 0.0 + }, + "p75": { + "mean": 0.0, + "sd": 0.0 + } + }, + "floor_ecdf_max_gap": { + "mean": 0.015, + "sd": 0.002 + }, + "thin": false + }, + "45_54": { + "p25": 3.0, + "p50": 8.0, + "p75": 16.0, + "persons_unweighted": 9624, + "floor_abs_gap_years": { + "p25": { + "mean": 0.0, + "sd": 0.0 + }, + "p50": { + "mean": 0.0, + "sd": 0.0 + }, + "p75": { + "mean": 0.0, + "sd": 0.0 + } + }, + "floor_ecdf_max_gap": { + "mean": 0.0129, + "sd": 0.0049 + }, + "thin": false + }, + "55_64": { + "p25": 4.0, + "p50": 10.0, + "p75": 21.0, + "persons_unweighted": 8858, + "floor_abs_gap_years": { + "p25": { + "mean": 0.0, + "sd": 0.0 + }, + "p50": { + "mean": 0.0, + "sd": 0.0 + }, + "p75": { + "mean": 0.2, + "sd": 0.4 + } + }, + "floor_ecdf_max_gap": { + "mean": 0.0187, + "sd": 0.0048 + }, + "thin": false + }, + "65_200": { + "p25": 5.0, + "p50": 13.0, + "p75": 25.0, + "persons_unweighted": 3719, + "floor_abs_gap_years": { + "p25": { + "mean": 0.0, + "sd": 0.0 + }, + "p50": { + "mean": 1.2, + "sd": 0.98 + }, + "p75": { + "mean": 1.0, + "sd": 0.894 + } + }, + "floor_ecdf_max_gap": { + "mean": 0.0297, + "sd": 0.0075 + }, + "thin": false + } + }, + "2022": { + "16_19": { + "p25": 0.33, + "p50": 0.5, + "p75": 1.0, + "persons_unweighted": 1218, + "floor_abs_gap_years": { + "p25": { + "mean": 0.059, + "sd": 0.031 + }, + "p50": { + "mean": 0.0, + "sd": 0.0 + }, + "p75": { + "mean": 0.0, + "sd": 0.0 + } + }, + "floor_ecdf_max_gap": { + "mean": 0.0429, + "sd": 0.0106 + }, + "thin": false + }, + "20_24": { + "p25": 0.5, + "p50": 1.0, + "p75": 2.17, + "persons_unweighted": 2825, + "floor_abs_gap_years": { + "p25": { + "mean": 0.032, + "sd": 0.039 + }, + "p50": { + "mean": 0.0, + "sd": 0.0 + }, + "p75": { + "mean": 0.218, + "sd": 0.108 + } + }, + "floor_ecdf_max_gap": { + "mean": 0.0292, + "sd": 0.0096 + }, + "thin": false + }, + "25_34": { + "p25": 0.92, + "p50": 3.0, + "p75": 5.0, + "persons_unweighted": 8239, + "floor_abs_gap_years": { + "p25": { + "mean": 0.033, + "sd": 0.065 + }, + "p50": { + "mean": 0.0, + "sd": 0.0 + }, + "p75": { + "mean": 0.0, + "sd": 0.0 + } + }, + "floor_ecdf_max_gap": { + "mean": 0.014, + "sd": 0.0037 + }, + "thin": false + }, + "35_44": { + "p25": 2.0, + "p50": 5.0, + "p75": 10.0, + "persons_unweighted": 8630, + "floor_abs_gap_years": { + "p25": { + "mean": 0.0, + "sd": 0.0 + }, + "p50": { + "mean": 0.0, + "sd": 0.0 + }, + "p75": { + "mean": 0.0, + "sd": 0.0 + } + }, + "floor_ecdf_max_gap": { + "mean": 0.0181, + "sd": 0.0069 + }, + "thin": false + }, + "45_54": { + "p25": 3.0, + "p50": 7.0, + "p75": 16.0, + "persons_unweighted": 7907, + "floor_abs_gap_years": { + "p25": { + "mean": 0.0, + "sd": 0.0 + }, + "p50": { + "mean": 0.0, + "sd": 0.0 + }, + "p75": { + "mean": 1.0, + "sd": 0.0 + } + }, + "floor_ecdf_max_gap": { + "mean": 0.0248, + "sd": 0.0066 + }, + "thin": false + }, + "55_64": { + "p25": 4.0, + "p50": 10.0, + "p75": 21.0, + "persons_unweighted": 7138, + "floor_abs_gap_years": { + "p25": { + "mean": 0.0, + "sd": 0.0 + }, + "p50": { + "mean": 0.0, + "sd": 0.0 + }, + "p75": { + "mean": 1.0, + "sd": 0.0 + } + }, + "floor_ecdf_max_gap": { + "mean": 0.0253, + "sd": 0.0052 + }, + "thin": false + }, + "65_200": { + "p25": 5.0, + "p50": 12.0, + "p75": 25.0, + "persons_unweighted": 3153, + "floor_abs_gap_years": { + "p25": { + "mean": 0.209, + "sd": 0.396 + }, + "p50": { + "mean": 0.4, + "sd": 0.49 + }, + "p75": { + "mean": 0.0, + "sd": 0.0 + } + }, + "floor_ecdf_max_gap": { + "mean": 0.0268, + "sd": 0.0081 + }, + "thin": false + } + }, + "2024": { + "16_19": { + "p25": 0.33, + "p50": 0.67, + "p75": 1.5, + "persons_unweighted": 1223, + "floor_abs_gap_years": { + "p25": { + "mean": 0.016, + "sd": 0.032 + }, + "p50": { + "mean": 0.036, + "sd": 0.044 + }, + "p75": { + "mean": 0.282, + "sd": 0.039 + } + }, + "floor_ecdf_max_gap": { + "mean": 0.0564, + "sd": 0.0175 + }, + "thin": false + }, + "20_24": { + "p25": 0.5, + "p50": 1.17, + "p75": 2.92, + "persons_unweighted": 2999, + "floor_abs_gap_years": { + "p25": { + "mean": 0.0, + "sd": 0.0 + }, + "p50": { + "mean": 0.066, + "sd": 0.063 + }, + "p75": { + "mean": 0.405, + "sd": 0.19 + } + }, + "floor_ecdf_max_gap": { + "mean": 0.027, + "sd": 0.0045 + }, + "thin": false + }, + "25_34": { + "p25": 1.0, + "p50": 3.0, + "p75": 5.0, + "persons_unweighted": 8250, + "floor_abs_gap_years": { + "p25": { + "mean": 0.0, + "sd": 0.0 + }, + "p50": { + "mean": 0.0, + "sd": 0.0 + }, + "p75": { + "mean": 0.0, + "sd": 0.0 + } + }, + "floor_ecdf_max_gap": { + "mean": 0.0184, + "sd": 0.0051 + }, + "thin": false + }, + "35_44": { + "p25": 2.0, + "p50": 5.0, + "p75": 10.0, + "persons_unweighted": 8932, + "floor_abs_gap_years": { + "p25": { + "mean": 0.0, + "sd": 0.0 + }, + "p50": { + "mean": 0.0, + "sd": 0.0 + }, + "p75": { + "mean": 0.0, + "sd": 0.0 + } + }, + "floor_ecdf_max_gap": { + "mean": 0.0148, + "sd": 0.0035 + }, + "thin": false + }, + "45_54": { + "p25": 3.0, + "p50": 7.0, + "p75": 15.0, + "persons_unweighted": 7935, + "floor_abs_gap_years": { + "p25": { + "mean": 0.0, + "sd": 0.0 + }, + "p50": { + "mean": 0.0, + "sd": 0.0 + }, + "p75": { + "mean": 0.536, + "sd": 0.453 + } + }, + "floor_ecdf_max_gap": { + "mean": 0.0151, + "sd": 0.003 + }, + "thin": false + }, + "55_64": { + "p25": 4.0, + "p50": 10.0, + "p75": 20.0, + "persons_unweighted": 7012, + "floor_abs_gap_years": { + "p25": { + "mean": 0.0, + "sd": 0.0 + }, + "p50": { + "mean": 0.0, + "sd": 0.0 + }, + "p75": { + "mean": 0.2, + "sd": 0.4 + } + }, + "floor_ecdf_max_gap": { + "mean": 0.022, + "sd": 0.0031 + }, + "thin": false + }, + "65_200": { + "p25": 4.0, + "p50": 12.0, + "p75": 24.0, + "persons_unweighted": 3332, + "floor_abs_gap_years": { + "p25": { + "mean": 0.8, + "sd": 0.4 + }, + "p50": { + "mean": 1.2, + "sd": 1.166 + }, + "p75": { + "mean": 2.0, + "sd": 0.0 + } + }, + "floor_ecdf_max_gap": { + "mean": 0.0392, + "sd": 0.0087 + }, + "thin": false + } + } + } +} diff --git a/scripts/build_sipp_e8_e9_floors.py b/scripts/build_sipp_e8_e9_floors.py new file mode 100644 index 00000000..441b02ca --- /dev/null +++ b/scripts/build_sipp_e8_e9_floors.py @@ -0,0 +1,463 @@ +"""Build pre-IC3 SIPP floor references for gates E8 and E9 (#192). + +REPORTED ANCHOR, NOT A GATE RUN: IC3 has not locked and no thresholds +are proposed. Completes Workstream A's +floor battery (E3 tenure and E4/E5 spells are committed siblings; +E10 needs no floor — it is pass/fail on the locked PSID gates). + +Moments, on the pu2023 file (reference year 2022), restricted to +persons observed in the panel for all 12 reference months so spell +durations are not right-censored by sample exit (the restriction is +recorded, not hidden): + +(a) **E8 — nonemployment spell durations** (the zero-spell battery + analog). A person-month is nonemployed when the person is in + the panel that month with no active EJB job. Among persons with + at least one employed month, maximal nonemployment runs are + collapsed; moments per age band: the weighted share of persons + with any nonemployment spell, and the weighted share of those + spells lasting >= 3 months. + +(b) **E9 — earnings change by transition type** (the layering- + coherence moment). Consecutive-month person transitions are + classified stay (a common job id), j2j (employed both months, + no common id), exit (employed -> nonemployed), entry + (nonemployed -> employed). For stay and j2j, where both months' + known earnings totals are positive, the moment is the weighted + median and IQR of log(earn_{m+1} / earn_m). Exit/entry carry + rates, not earnings changes (their change is to/from zero by + construction). + +(c) **The floor**: person-disjoint sha256 half-splits, seeds 0-19; + for rates the |log rate ratio| between halves, for medians/IQRs + the absolute gap in log-points; mean/sd across seeds. Cells + under 200 unweighted persons per half are flagged thin. + +Thin-flag units (recorded for honesty across the floor battery): +the E8 thin flag counts **rows** per half, which equal persons +because the E8 frame has one row per person; the E9 earnings-change +thin flag counts **distinct persons** per half +(``person_id.nunique()``), although the underlying rows are +consecutive-month transition pairs. All compare against the same +``THIN_CELL_PERSONS = 200``. + +Seam caveat: identical to the E4/E5 floors — both halves share +SIPP's seam structure, so these floors cannot see seam bias; the +reconciliation artifact (#214) carries that measurement. + +Usage:: + + python scripts/build_sipp_e8_e9_floors.py + +writes ``runs/sipp_e8_e9_floors_v1.json``. +""" + +from __future__ import annotations + +import hashlib +import json +import os +import sys +from pathlib import Path + +import numpy as np +import pandas as pd + +REPO = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(REPO / "src")) + +from populace_dynamics.contract import environment_block # noqa: E402 +from populace_dynamics.data import sipp_jobs # noqa: E402 + +YEAR = 2023 +SEEDS = tuple(range(20)) +AGE_BANDS = ((16, 24), (25, 34), (35, 44), (45, 54), (55, 64), (65, 99)) +THIN_CELL_PERSONS = 200 + +ARTIFACT = REPO / "runs/sipp_e8_e9_floors_v1.json" +ENV_SIDECAR = ARTIFACT.with_suffix(".env.json") +INPUT_SIDECAR = ARTIFACT.with_suffix(".inputs.json") +OFFICIAL_SOURCE_URL = ( + "https://www2.census.gov/programs-surveys/sipp/data/datasets/" + "2023/pu2023_csv.zip" +) +OFFICIAL_ARCHIVE_SHA256 = ( + "9c5363d56aca2041db20d46d17b81e9be931eb5b18bd5f5238b367d2dd7fb74b" +) +OFFICIAL_ARCHIVE_BYTES = 109_036_604 + + +def _source_path(year: int) -> Path: + data_dir = Path( + os.environ.get( + "POPULACE_DYNAMICS_SIPP_DIR", + str(Path("~/PolicyEngine/sipp-data").expanduser()), + ) + ).expanduser() + for suffix in (".csv", ".csv.gz"): + path = data_dir / f"pu{year}{suffix}" + if path.exists(): + return path + raise FileNotFoundError(f"pu{year}.csv[.gz] not staged") + + +def _reader_commit() -> str: + """Last commit touching the SIPP reader in effect for this run.""" + import subprocess + + try: + return subprocess.run( + [ + "git", + "log", + "-1", + "--format=%H", + "--", + "src/populace_dynamics/data/sipp_jobs.py", + ], + cwd=REPO, + capture_output=True, + text=True, + check=True, + ).stdout.strip() + except Exception: + return "unknown" + + +def _age_band(age: pd.Series) -> pd.Series: + bins = [AGE_BANDS[0][0] - 1, *[hi for _, hi in AGE_BANDS]] + labels = [f"{lo}_{hi}" for lo, hi in AGE_BANDS] + return pd.cut(age, bins=bins, labels=labels) + + +def _half(person_id: str, seed: int) -> int: + digest = hashlib.sha256(f"{seed}:{person_id}".encode()).digest() + return digest[0] & 1 + + +def _person_month_universe(year: int) -> pd.DataFrame: + path = _source_path(year) + raw = pd.read_csv( + path, + sep="|", + usecols=["SSUID", "PNUM", "MONTHCODE", "WPFINWGT", "TAGE"], + dtype={"SSUID": "string"}, + ) + raw["person_id"] = raw["SSUID"].astype(str) + "-" + raw["PNUM"].astype(str) + return raw.rename( + columns={"MONTHCODE": "month", "WPFINWGT": "weight", "TAGE": "age"} + )[["person_id", "month", "weight", "age"]] + + +def build_panel(year: int) -> pd.DataFrame: + """Person x month grid for full-year persons, with jobs/earnings.""" + universe = _person_month_universe(year) + counts = universe.groupby("person_id")["month"].nunique() + full_year = set(counts[counts == 12].index) + universe = universe[universe["person_id"].isin(full_year)] + + job_months = sipp_jobs.read_sipp_job_months(year) + jobs = ( + job_months.groupby(["person_id", "month"]) + .agg(jobs=("job_id", frozenset), earn=("earnings", "sum")) + .reset_index() + ) + panel = universe.merge(jobs, on=["person_id", "month"], how="left") + panel["jobs"] = panel["jobs"].apply( + lambda x: x if isinstance(x, frozenset) else frozenset() + ) + panel["employed"] = panel["jobs"].map(len) > 0 + return panel.sort_values(["person_id", "month"]) + + +def e8_person_frame(panel: pd.DataFrame) -> pd.DataFrame: + """Per person: any-nonemployment flag and longest N-spell.""" + rows = [] + for person, grp in panel.groupby("person_id", sort=False): + employed = grp.sort_values("month")["employed"].to_numpy() + if not employed.any(): + continue # never employed: outside the E8 universe + runs = [] + run = 0 + for e in employed: + if not e: + run += 1 + elif run: + runs.append(run) + run = 0 + if run: + runs.append(run) + rows.append( + { + "person_id": person, + "age": grp["age"].iloc[0], + "weight": grp["weight"].iloc[0], + "any_nonemp": bool(runs), + "long_nonemp": bool(runs and max(runs) >= 3), + } + ) + out = pd.DataFrame(rows) + out["age_band"] = _age_band(out["age"]) + return out[out["age_band"].notna()] + + +def e9_transition_frame(panel: pd.DataFrame) -> pd.DataFrame: + """Person x month-pair transitions with earnings changes.""" + nxt = panel[["person_id", "month", "jobs", "earn", "employed"]].copy() + nxt["month"] -= 1 + pairs = panel.merge(nxt, on=["person_id", "month"], suffixes=("", "_n")) + + def classify(row) -> str: + if row["employed"] and row["employed_n"]: + return "stay" if row["jobs"] & row["jobs_n"] else "j2j" + if row["employed"] and not row["employed_n"]: + return "exit" + if not row["employed"] and row["employed_n"]: + return "entry" + return "neither" + + pairs["transition"] = pairs.apply(classify, axis=1) + pairs = pairs[pairs["transition"] != "neither"].copy() + both_known = ( + pairs["earn"].notna() + & pairs["earn_n"].notna() + & (pairs["earn"] > 0) + & (pairs["earn_n"] > 0) + ) + pairs["log_change"] = np.nan + mask = both_known & pairs["transition"].isin(("stay", "j2j")) + pairs.loc[mask, "log_change"] = np.log( + pairs.loc[mask, "earn_n"] / pairs.loc[mask, "earn"] + ) + pairs["age_band"] = _age_band(pairs["age"]) + return pairs[pairs["age_band"].notna()] + + +def _weighted_quantile(values, weights, q: float) -> float: + order = np.argsort(values, kind="stable") + values = np.asarray(values)[order] + weights = np.asarray(weights)[order] + cum = np.cumsum(weights) - 0.5 * weights + cum /= weights.sum() + return float(np.interp(q, cum, values)) + + +def _rate_floor(cell: pd.DataFrame, flag: str) -> dict: + gaps, halves_n = [], [] + for seed in SEEDS: + half = cell["person_id"].map(lambda p, s=seed: _half(p, s)) + a, b = cell[half == 0], cell[half == 1] + halves_n.append(min(len(a), len(b))) + ra = float((a["weight"] * a[flag]).sum() / a["weight"].sum()) + rb = float((b["weight"] * b[flag]).sum() / b["weight"].sum()) + gaps.append(abs(np.log(ra / rb)) if ra > 0 and rb > 0 else np.nan) + gaps = [g for g in gaps if not np.isnan(g)] + return { + "floor_abs_log_ratio_mean": round(float(np.mean(gaps)), 5), + "floor_abs_log_ratio_sd": round(float(np.std(gaps)), 5), + "thin": bool(min(halves_n) < THIN_CELL_PERSONS), + } + + +def e8_floors(persons: pd.DataFrame) -> dict: + cells = {} + for band, cell in persons.groupby("age_band", observed=True): + w = cell["weight"] + cells[str(band)] = { + "any_nonemp_share": round( + float((w * cell["any_nonemp"]).sum() / w.sum()), 4 + ), + "long_nonemp_share": round( + float((w * cell["long_nonemp"]).sum() / w.sum()), 4 + ), + "persons_unweighted": int(len(cell)), + "any_nonemp": _rate_floor(cell, "any_nonemp"), + "long_nonemp": _rate_floor(cell, "long_nonemp"), + } + return cells + + +def e9_floors(pairs: pd.DataFrame) -> dict: + out: dict = {"transition_rates": {}, "earnings_change": {}} + monthly = pairs.groupby("transition")["weight"].sum() + total = monthly.sum() + out["transition_rates"] = { + t: round(float(v / total), 4) for t, v in monthly.items() + } + for kind in ("stay", "j2j"): + cell = pairs[ + (pairs["transition"] == kind) & pairs["log_change"].notna() + ] + values = cell["log_change"].to_numpy(dtype=float) + weights = cell["weight"].to_numpy(dtype=float) + med = _weighted_quantile(values, weights, 0.5) + iqr = _weighted_quantile(values, weights, 0.75) - _weighted_quantile( + values, weights, 0.25 + ) + med_gaps, iqr_gaps, halves_n = [], [], [] + for seed in SEEDS: + half = cell["person_id"].map(lambda p, s=seed: _half(p, s)) + a, b = cell[half == 0], cell[half == 1] + halves_n.append( + min( + a["person_id"].nunique(), + b["person_id"].nunique(), + ) + ) + + def q(frame, qq): + return _weighted_quantile( + frame["log_change"].to_numpy(dtype=float), + frame["weight"].to_numpy(dtype=float), + qq, + ) + + med_gaps.append(abs(q(a, 0.5) - q(b, 0.5))) + iqr_gaps.append( + abs((q(a, 0.75) - q(a, 0.25)) - (q(b, 0.75) - q(b, 0.25))) + ) + out["earnings_change"][kind] = { + "median_log_change": round(med, 4), + "iqr_log_change": round(iqr, 4), + "pairs_unweighted": int(len(cell)), + "persons_unweighted": int(cell["person_id"].nunique()), + "floor_abs_median_gap": { + "mean": round(float(np.mean(med_gaps)), 5), + "sd": round(float(np.std(med_gaps)), 5), + }, + "floor_abs_iqr_gap": { + "mean": round(float(np.mean(iqr_gaps)), 5), + "sd": round(float(np.std(iqr_gaps)), 5), + }, + "thin": bool(min(halves_n) < THIN_CELL_PERSONS), + } + return out + + +def build() -> dict: + panel = build_panel(YEAR) + persons = e8_person_frame(panel) + pairs = e9_transition_frame(panel) + return { + "artifact": "sipp_e8_e9_floors", + "version": "v1", + "status": ( + "PRE-LOCK REFERENCE - NOT RATIFIED; IC3 not locked; no " + "thresholds. v1 is a pinning event, not a ratification" + ), + "issue": "192", + "deployment_scale_note": ( + "RECORDED GAP (review of #212): these floors are " + "half-vs-half, i.e. the sampling noise of ~50%-of-source " + "estimates, while IC3 proposes scoring on a 0.20 person " + "holdout - there is no candidate-context floor (gate-1 " + "ctx20 analog). Under root-n scaling, a 20% scoring " + "frame has ~sqrt(0.5/0.2)=1.58x the sampling noise of " + "the half-split basis, so these floors are mildly " + "ANTI-conservative (too tight), not conservative. " + "RECORDED_NOT_SATISFIED: IC3 must accept a registered " + "analytic scale adjustment or require matching-context " + "floors before candidate runs." + ), + "source": f"pu{YEAR} (reference year {YEAR - 1}), persons " + "observed all 12 reference months (censoring-free draft " + "restriction, recorded; ESTIMAND NOTE per review: candidate " + "runs scored against these cells must apply the identical " + "full-year-persons restriction)", + "method": ( + "person-disjoint sha256 half-splits, seeds 0-19 (raised " + "from 5 per review: E8 cells where floor sd exceeds " + "the mean need a stable across-seed sd); rates " + "floored on |log rate ratio|, earnings-change medians/" + "IQRs on absolute gaps in log-points; weighted by " + "WPFINWGT" + ), + "seam_caveat": ( + "identical to the E4/E5 floors: half-splits share SIPP's " + "seam structure; #214 carries the seam measurement" + ), + "stay_median_heaping_caveat": ( + "within-job SIPP monthly earnings are mostly constant " + "across a wave (dependent-interview reporting), so the " + "stay-transition median log-change heaps at exactly 0 " + "and its half-vs-half floor is degenerate (0.0) - the " + "same failure class as the tenure quantile heaping; " + "E9-stay thresholds should be stated on the IQR or a " + "distributional distance, not the median" + ), + "thin_flag_units": { + "e8_nonemployment_by_age": ( + "rows per half, equal to persons (one row per " + "person in the E8 frame) vs THIN_CELL_PERSONS=200" + ), + "e9_transitions.earnings_change": ( + "distinct persons per half (person_id.nunique(); " + "rows are consecutive-month transition pairs and a " + "person can contribute up to 11) vs " + "THIN_CELL_PERSONS=200" + ), + }, + "sipp_jobs_reader_commit": _reader_commit(), + "source_input": { + "path": _source_path(YEAR).name, + "sha256": hashlib.sha256( + _source_path(YEAR).read_bytes() + ).hexdigest(), + }, + "e8_nonemployment_by_age": e8_floors(persons), + "e9_transitions": e9_floors(pairs), + } + + +def main() -> None: + artifact = build() + ARTIFACT.write_text(json.dumps(artifact, indent=2) + "\n") + staged_path = _source_path(YEAR) + INPUT_SIDECAR.write_text( + json.dumps( + { + "artifact": ARTIFACT.name, + "status": "SOURCE_INPUT_DIGESTS", + "official_source": { + "url": OFFICIAL_SOURCE_URL, + "archive_sha256": OFFICIAL_ARCHIVE_SHA256, + "archive_bytes": OFFICIAL_ARCHIVE_BYTES, + "archive_member": "pu2023.csv", + }, + "staged_input": { + **artifact["source_input"], + "bytes": staged_path.stat().st_size, + "transport_note": ( + "gzip -n recompression of the sole CSV member " + "from the verified official Census ZIP" + ), + }, + }, + indent=2, + ) + + "\n" + ) + ENV_SIDECAR.write_text( + json.dumps( + { + "artifact": ARTIFACT.name, + "status": "MEASUREMENT_ENVIRONMENT", + "environment": environment_block(), + }, + indent=2, + ) + + "\n" + ) + print(f"wrote {ARTIFACT}") + print("e9 transition mix:", artifact["e9_transitions"]["transition_rates"]) + stay = artifact["e9_transitions"]["earnings_change"]["stay"] + print( + "stay: median log-change", + stay["median_log_change"], + "floor", + stay["floor_abs_median_gap"]["mean"], + ) + + +if __name__ == "__main__": + main() diff --git a/scripts/build_sipp_spell_floors.py b/scripts/build_sipp_spell_floors.py new file mode 100644 index 00000000..7e0667c6 --- /dev/null +++ b/scripts/build_sipp_spell_floors.py @@ -0,0 +1,338 @@ +"""Build pre-IC3 SIPP job-spell noise-floor references (#192). + +REPORTED ANCHOR, NOT A GATE RUN: IC3 (the +employer gate block) has not locked, no thresholds are proposed +here, and v1 pinning does not ratify anything. Like the disability floors, +this commits the person-disjoint half-vs-half sampling-noise floor +that pre-registered E4/E5 thresholds would later be derived from, +so the floor-building method is on the record before any candidate +model exists (issue #192 protocol: floors -> thresholds -> referee +round -> one-shot runs). + +Moments, computed on the SIPP job-month panel +(:mod:`populace_dynamics.data.sipp_jobs`, reference year 2022 from +the pu2023 file): + +(a) **E4 retention pairs.** Among persons employed in consecutive + reference months m and m+1, the weighted share retaining at + least one employer (same within-panel ``EJB`` job id in both + months), by age band x sex. This is the month-frequency analog + of the plan's "2-window employer-retention persistence". + +(b) **E5 attachment runs.** The weighted distribution of maximal + same-employer run lengths (1-12 months within the reference + year, from :func:`job_spells`), by age band — the view that + catches chained-model persistence understatement. + +(c) **The floor.** For seeds 0-4, persons are split into two + disjoint halves; each cell's rate is computed on both halves + and the across-seed mean and sd of ``|log(rate_a / rate_b)|`` + is the sampling-noise floor for that cell, exactly the + disability-floor convention. Cells with fewer than 200 + unweighted persons per half are reported but flagged thin. + +Thin-flag units (recorded for honesty across the floor battery): +the E4 retention thin flag counts **distinct persons** per half +(``person_id.nunique()``; the underlying rows are person-month +retention pairs, so a person can contribute many rows), while the +E5 runs thin flag counts **rows**, which here equal persons because +the run-length frame has exactly one row per person. Both compare +against the same ``THIN_CELL_PERSONS = 200``. + +Seam caveat (pre-registered on #192): SIPP transitions bunch at +interview seams, and both halves share the seam structure, so this +floor cannot see seam bias — the seam-vs-J2J reconciliation run is +a separate, required artifact before E2/E4 thresholds lock. + +Usage:: + + python scripts/build_sipp_spell_floors.py + +writes ``runs/sipp_spell_floors_v1.json``. +""" + +from __future__ import annotations + +import hashlib +import json +import sys +from pathlib import Path + +import numpy as np +import pandas as pd + +sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "src")) + +from populace_dynamics.contract import environment_block # noqa: E402 +from populace_dynamics.data import sipp_jobs # noqa: E402 + +YEAR = 2023 +SEEDS = (0, 1, 2, 3, 4) +AGE_BANDS = ((16, 24), (25, 34), (35, 44), (45, 54), (55, 64), (65, 99)) +THIN_CELL_PERSONS = 200 + +ARTIFACT = Path(__file__).resolve().parents[1] / ( + "runs/sipp_spell_floors_v1.json" +) +ENV_SIDECAR = ARTIFACT.with_suffix(".env.json") +INPUT_SIDECAR = ARTIFACT.with_suffix(".inputs.json") +OFFICIAL_SOURCE_URL = ( + "https://www2.census.gov/programs-surveys/sipp/data/datasets/" + "2023/pu2023_csv.zip" +) +OFFICIAL_ARCHIVE_SHA256 = ( + "9c5363d56aca2041db20d46d17b81e9be931eb5b18bd5f5238b367d2dd7fb74b" +) +OFFICIAL_ARCHIVE_BYTES = 109_036_604 + + +def _source_pin() -> dict[str, str]: + path = sipp_jobs._resolve_pu_path( # noqa: SLF001 + YEAR, + sipp_jobs._resolve_data_dir(None), # noqa: SLF001 + ) + return { + "path": path.name, + "sha256": hashlib.sha256(path.read_bytes()).hexdigest(), + } + + +def _reader_commit() -> str: + """Last commit touching the SIPP reader in effect for this run.""" + import subprocess + + repo = Path(__file__).resolve().parents[1] + try: + return subprocess.run( + [ + "git", + "log", + "-1", + "--format=%H", + "--", + "src/populace_dynamics/data/sipp_jobs.py", + ], + cwd=repo, + capture_output=True, + text=True, + check=True, + ).stdout.strip() + except Exception: + return "unknown" + + +def _age_band(age: pd.Series) -> pd.Series: + bins = [AGE_BANDS[0][0] - 1, *[hi for _, hi in AGE_BANDS]] + labels = [f"{lo}_{hi}" for lo, hi in AGE_BANDS] + return pd.cut(age, bins=bins, labels=labels) + + +def _half(person_id: pd.Series, seed: int) -> pd.Series: + """Deterministic person-disjoint half assignment.""" + + def bucket(pid: str) -> int: + digest = hashlib.sha256(f"{seed}:{pid}".encode()).digest() + return digest[0] & 1 + + return person_id.map(bucket) + + +def retention_frame(job_months: pd.DataFrame) -> pd.DataFrame: + """One row per person x consecutive-month pair, employed both.""" + person_months = ( + job_months.groupby(["person_id", "month"]) + .agg( + jobs=("job_id", frozenset), + age=("age", "first"), + sex=("sex", "first"), + weight=("weight", "first"), + ) + .reset_index() + ) + nxt = person_months.copy() + nxt["month"] -= 1 + pairs = person_months.merge( + nxt, + on=["person_id", "month"], + suffixes=("", "_next"), + ) + pairs["retained"] = [ + bool(a & b) for a, b in zip(pairs.jobs, pairs.jobs_next, strict=True) + ] + pairs["age_band"] = _age_band(pairs["age"]) + return pairs[pairs["age_band"].notna()] + + +def run_length_frame(job_months: pd.DataFrame) -> pd.DataFrame: + """One row per person: longest same-employer run in the year.""" + spells = sipp_jobs.job_spells(job_months) + person_attrs = job_months.groupby("person_id").agg( + age=("age", "first"), weight=("weight", "first") + ) + longest = spells.groupby("person_id")["n_months"].max() + out = person_attrs.join(longest).dropna(subset=["n_months"]) + out["age_band"] = _age_band(out["age"]) + return out[out["age_band"].notna()].reset_index() + + +def _weighted_rate(frame: pd.DataFrame, flag: str) -> float: + total = frame["weight"].sum() + return float((frame["weight"] * frame[flag]).sum() / total) + + +def floors_for_retention(pairs: pd.DataFrame) -> dict: + cells = {} + for (band, sex), cell in pairs.groupby(["age_band", "sex"], observed=True): + gaps = [] + halves_n = [] + for seed in SEEDS: + half = _half(cell["person_id"], seed) + a = cell[half == 0] + b = cell[half == 1] + halves_n.append( + min(a["person_id"].nunique(), b["person_id"].nunique()) + ) + ra, rb = ( + _weighted_rate(a, "retained"), + _weighted_rate(b, "retained"), + ) + gaps.append(abs(np.log(ra / rb))) + cells[f"{band}|sex{int(sex)}"] = { + "rate": round(_weighted_rate(cell, "retained"), 4), + "pairs_unweighted": int(len(cell)), + "floor_abs_log_ratio_mean": round(float(np.mean(gaps)), 5), + "floor_abs_log_ratio_sd": round(float(np.std(gaps)), 5), + "thin": bool(min(halves_n) < THIN_CELL_PERSONS), + } + return cells + + +def floors_for_runs(runs: pd.DataFrame) -> dict: + runs = runs.assign(long_run=runs["n_months"] >= 12) + cells = {} + for band, cell in runs.groupby("age_band", observed=True): + gaps = [] + halves_n = [] + for seed in SEEDS: + half = _half(cell["person_id"], seed) + a, b = cell[half == 0], cell[half == 1] + halves_n.append(min(len(a), len(b))) + ra, rb = ( + _weighted_rate(a, "long_run"), + _weighted_rate(b, "long_run"), + ) + gaps.append(abs(np.log(ra / rb))) + cells[str(band)] = { + "full_year_run_share": round(_weighted_rate(cell, "long_run"), 4), + "persons_unweighted": int(len(cell)), + "floor_abs_log_ratio_mean": round(float(np.mean(gaps)), 5), + "floor_abs_log_ratio_sd": round(float(np.std(gaps)), 5), + "thin": bool(min(halves_n) < THIN_CELL_PERSONS), + } + return cells + + +def build() -> dict: + job_months = sipp_jobs.read_sipp_job_months(YEAR) + pairs = retention_frame(job_months) + runs = run_length_frame(job_months) + return { + "artifact": "sipp_spell_floors", + "version": "v1", + "status": ( + "PRE-LOCK REFERENCE - NOT RATIFIED; IC3 not locked; no " + "thresholds. v1 is a pinning event, not a ratification" + ), + "issue": "192", + "deployment_scale_note": ( + "RECORDED GAP (review of #212): these floors are " + "half-vs-half, i.e. the sampling noise of ~50%-of-source " + "estimates, while IC3 proposes scoring on a 0.20 person " + "holdout - there is no candidate-context floor (gate-1 " + "ctx20 analog). Under root-n scaling, a 20% scoring " + "frame has ~sqrt(0.5/0.2)=1.58x the sampling noise of " + "the half-split basis, so these floors are mildly " + "ANTI-conservative (too tight), not conservative. " + "RECORDED_NOT_SATISFIED: IC3 must accept a registered " + "analytic scale adjustment or require matching-context " + "floors before candidate runs." + ), + "source": f"pu{YEAR} (reference year {YEAR - 1})", + "method": ( + "person-disjoint sha256 half-splits, seeds 0-4; per-cell " + "|log(rate_a/rate_b)| mean/sd across seeds; weighted by " + "WPFINWGT" + ), + "seam_caveat": ( + "both halves share SIPP seam structure; the seam-vs-J2J " + "reconciliation run is a separate required artifact " + "before IC3 thresholds lock" + ), + "thin_flag_units": { + "e4_retention_by_age_sex": ( + "distinct persons per half (person_id.nunique(); " + "rows are person-month retention pairs) vs " + "THIN_CELL_PERSONS=200" + ), + "e5_runs_by_age": ( + "rows per half, equal to persons (one row per " + "person in the run-length frame) vs " + "THIN_CELL_PERSONS=200" + ), + }, + "sipp_jobs_reader_commit": _reader_commit(), + "source_input": _source_pin(), + "e4_retention_by_age_sex": floors_for_retention(pairs), + "e5_runs_by_age": floors_for_runs(runs), + } + + +def main() -> None: + artifact = build() + ARTIFACT.write_text(json.dumps(artifact, indent=2) + "\n") + staged_path = sipp_jobs._resolve_pu_path( # noqa: SLF001 + YEAR, + sipp_jobs._resolve_data_dir(None), # noqa: SLF001 + ) + INPUT_SIDECAR.write_text( + json.dumps( + { + "artifact": ARTIFACT.name, + "status": "SOURCE_INPUT_DIGESTS", + "official_source": { + "url": OFFICIAL_SOURCE_URL, + "archive_sha256": OFFICIAL_ARCHIVE_SHA256, + "archive_bytes": OFFICIAL_ARCHIVE_BYTES, + "archive_member": "pu2023.csv", + }, + "staged_input": { + **artifact["source_input"], + "bytes": staged_path.stat().st_size, + "transport_note": ( + "gzip -n recompression of the sole CSV member " + "from the verified official Census ZIP" + ), + }, + }, + indent=2, + ) + + "\n" + ) + ENV_SIDECAR.write_text( + json.dumps( + { + "artifact": ARTIFACT.name, + "status": "MEASUREMENT_ENVIRONMENT", + "environment": environment_block(), + }, + indent=2, + ) + + "\n" + ) + print(f"wrote {ARTIFACT}") + e4 = artifact["e4_retention_by_age_sex"] + print(f"E4 cells: {len(e4)}; example:", next(iter(e4.items()))) + + +if __name__ == "__main__": + main() diff --git a/scripts/build_tenure_floors.py b/scripts/build_tenure_floors.py new file mode 100644 index 00000000..aa0cb924 --- /dev/null +++ b/scripts/build_tenure_floors.py @@ -0,0 +1,317 @@ +"""Build pre-IC3 CPS tenure noise-floor references for gate E3 (#192). + +REPORTED ANCHOR, NOT A GATE RUN: IC3 has not locked and no thresholds +are proposed; v1 pinning does not ratify anything. E3's moment is the +tenure distribution (P25/P50/P75) by age band against the CPS +January supplement; this commits the person-disjoint half-vs-half +sampling-noise floor those thresholds would later be derived from, +completing the tenure side of Workstream A's floor battery +(companion to the E4/E5 SIPP floors). + +Method mirrors the spell floors: for seeds 0-4, persons split into +two disjoint sha256 halves; each cell's weighted P25/P50/P75 of +``tenure_years`` is computed on both halves and the across-seed +mean and sd of the absolute quantile gap **in years** is the floor +(quantiles are in interpretable units, so the gap is reported in +years rather than a log ratio). Reported tenure heaps hard on +integers, so half-vs-half quantile gaps are frequently EXACTLY zero +(both halves' quantiles land on the same heap) — a degenerate basis +for a "quantile error vs floor" criterion. Each cell therefore also +carries a weighted-ECDF max-gap (Kolmogorov-style) floor, which is +smooth under heaping; the IC3 round can choose between the quantile +and distributional formulations with both on the record. All three +staged supplements +(2020/2022/2024) are floored independently — the across-year spread +of the floors is itself informative about supplement-to-supplement +stability. Cells with fewer than 200 unweighted persons per half +are flagged thin. + +Thin-flag units: the thin flag counts **rows** per half, which +equal persons because the CPS tenure frame has one record per +person, against ``THIN_CELL_PERSONS = 200`` (the same constant the +SIPP spell floors use, where E4 and E9 count distinct persons — +units are recorded per artifact). + +Usage:: + + python scripts/build_tenure_floors.py + +writes ``runs/tenure_floors_v1.json``. +""" + +from __future__ import annotations + +import hashlib +import json +import sys +from pathlib import Path + +import numpy as np + +REPO = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(REPO / "src")) + +from populace_dynamics.contract import environment_block # noqa: E402 +from populace_dynamics.data import cps_tenure # noqa: E402 + +YEARS = (2020, 2022, 2024) +SEEDS = (0, 1, 2, 3, 4) +QUANTILES = (0.25, 0.50, 0.75) +THIN_CELL_PERSONS = 200 + +ARTIFACT = REPO / "runs/tenure_floors_v1.json" +ENV_SIDECAR = ARTIFACT.with_suffix(".env.json") +INPUT_SIDECAR = ARTIFACT.with_suffix(".inputs.json") +OFFICIAL_SOURCE_URL = ( + "https://www2.census.gov/programs-surveys/cps/datasets/" + "{year}/supp/jan{yy:02d}pub.csv" +) + + +def _source_pins() -> list[dict[str, str]]: + data_dir = cps_tenure._resolve_data_dir(None) # noqa: SLF001 + pins = [] + for year in YEARS: + path = cps_tenure._resolve_person_path(year, data_dir) # noqa: SLF001 + pins.append( + { + "year": str(year), + "path": path.name, + "sha256": hashlib.sha256(path.read_bytes()).hexdigest(), + } + ) + return pins + + +def _reader_commit() -> str: + """Last commit touching the CPS tenure reader for this run.""" + import subprocess + + try: + return subprocess.run( + [ + "git", + "log", + "-1", + "--format=%H", + "--", + "src/populace_dynamics/data/cps_tenure.py", + ], + cwd=REPO, + capture_output=True, + text=True, + check=True, + ).stdout.strip() + except Exception: + return "unknown" + + +def _half(person_id: str, seed: int) -> int: + digest = hashlib.sha256(f"{seed}:{person_id}".encode()).digest() + return digest[0] & 1 + + +def _weighted_quantile(values, weights, q: float) -> float: + order = np.argsort(values, kind="stable") + values = np.asarray(values)[order] + weights = np.asarray(weights)[order] + cum = np.cumsum(weights) - 0.5 * weights + cum /= weights.sum() + return float(np.interp(q, cum, values)) + + +def _weighted_ecdf_max_gap(a_values, a_weights, b_values, b_weights): + """Max |F_a(x) - F_b(x)| over the union grid of observed values.""" + grid = np.union1d(a_values, b_values) + + def ecdf(values, weights): + order = np.argsort(values, kind="stable") + v = np.asarray(values)[order] + w = np.asarray(weights)[order] + cum = np.cumsum(w) / w.sum() + idx = np.searchsorted(v, grid, side="right") - 1 + return np.where(idx >= 0, cum[idx], 0.0) + + return float( + np.max(np.abs(ecdf(a_values, a_weights) - ecdf(b_values, b_weights))) + ) + + +def floors_for_year(year: int) -> dict: + records = cps_tenure.read_cps_tenure(year) + usable = records[ + records["tenure_years"].notna() & (records["weight"] > 0) + ].copy() + labels = [f"{lo}_{hi}" for lo, hi in cps_tenure.DEFAULT_AGE_BANDS] + import pandas as pd + + usable["age_band"] = pd.cut( + usable["age"], + bins=[cps_tenure.DEFAULT_AGE_BANDS[0][0] - 1] + + [hi for _, hi in cps_tenure.DEFAULT_AGE_BANDS], + labels=labels, + ) + usable = usable[usable["age_band"].notna()] + + cells = {} + for band, cell in usable.groupby("age_band", observed=True): + values = cell["tenure_years"].to_numpy(dtype=float) + weights = cell["weight"].to_numpy(dtype=float) + point = { + f"p{int(q * 100)}": round( + _weighted_quantile(values, weights, q), 2 + ) + for q in QUANTILES + } + halves = cell["person_id"].map( + lambda pid: [_half(pid, seed) for seed in SEEDS] + ) + gaps: dict[str, list[float]] = { + f"p{int(q * 100)}": [] for q in QUANTILES + } + ks_gaps: list[float] = [] + thin = False + for i, _seed in enumerate(SEEDS): + mask_a = halves.map(lambda h, i=i: h[i] == 0) + a, b = cell[mask_a], cell[~mask_a] + if min(len(a), len(b)) < THIN_CELL_PERSONS: + thin = True + for q in QUANTILES: + qa = _weighted_quantile( + a["tenure_years"].to_numpy(dtype=float), + a["weight"].to_numpy(dtype=float), + q, + ) + qb = _weighted_quantile( + b["tenure_years"].to_numpy(dtype=float), + b["weight"].to_numpy(dtype=float), + q, + ) + gaps[f"p{int(q * 100)}"].append(abs(qa - qb)) + ks_gaps.append( + _weighted_ecdf_max_gap( + a["tenure_years"].to_numpy(dtype=float), + a["weight"].to_numpy(dtype=float), + b["tenure_years"].to_numpy(dtype=float), + b["weight"].to_numpy(dtype=float), + ) + ) + cells[str(band)] = { + **point, + "persons_unweighted": int(len(cell)), + "floor_abs_gap_years": { + name: { + "mean": round(float(np.mean(values)), 3), + "sd": round(float(np.std(values)), 3), + } + for name, values in gaps.items() + }, + "floor_ecdf_max_gap": { + "mean": round(float(np.mean(ks_gaps)), 4), + "sd": round(float(np.std(ks_gaps)), 4), + }, + "thin": thin, + } + return cells + + +def build() -> dict: + return { + "artifact": "tenure_floors", + "version": "v1", + "status": ( + "PRE-LOCK REFERENCE - NOT RATIFIED; IC3 not locked; no " + "thresholds. v1 is a pinning event, not a ratification" + ), + "issue": "192", + "deployment_scale_note": ( + "RECORDED GAP (review of #212): these floors are " + "half-vs-half, i.e. the sampling noise of ~50%-of-source " + "estimates, while IC3 proposes scoring on a 0.20 person " + "holdout - there is no candidate-context floor (gate-1 " + "ctx20 analog). Under root-n scaling, a 20% scoring " + "frame has ~sqrt(0.5/0.2)=1.58x the sampling noise of " + "the half-split basis, so these floors are mildly " + "ANTI-conservative (too tight), not conservative. " + "RECORDED_NOT_SATISFIED: IC3 must accept a registered " + "analytic scale adjustment or require matching-context " + "floors before candidate runs." + ), + "source": "CPS January supplements 2020/2022/2024 (PTST1TN, " + "PWTENWGT); reader per #205", + "method": ( + "person-disjoint sha256 half-splits, seeds 0-4; per-cell " + "absolute weighted-quantile gap in years AND weighted-ECDF " + "max gap (heaping-robust), mean/sd across seeds; BLS age " + "bands" + ), + "heaping_caveat": ( + "reported tenure heaps on integers, so half-vs-half " + "quantile gaps are frequently exactly zero (36/63 cells " + "in the first build) - a degenerate threshold basis; the " + "ECDF max-gap floor is the heaping-robust alternative " + "for the IC3 round to choose between" + ), + "thin_flag_units": ( + "rows per half, equal to persons (one CPS record per " + "person) vs THIN_CELL_PERSONS=200" + ), + "cps_tenure_reader_commit": _reader_commit(), + "source_inputs": _source_pins(), + "by_year": {str(year): floors_for_year(year) for year in YEARS}, + } + + +def main() -> None: + artifact = build() + ARTIFACT.write_text(json.dumps(artifact, indent=2) + "\n") + data_dir = cps_tenure._resolve_data_dir(None) # noqa: SLF001 + INPUT_SIDECAR.write_text( + json.dumps( + { + "artifact": ARTIFACT.name, + "status": "SOURCE_INPUT_DIGESTS", + "source_inputs": [ + { + **source, + "bytes": cps_tenure._resolve_person_path( # noqa: SLF001 + int(source["year"]), data_dir + ) + .stat() + .st_size, + "official_url": OFFICIAL_SOURCE_URL.format( + year=int(source["year"]), + yy=int(source["year"]) % 100, + ), + } + for source in artifact["source_inputs"] + ], + }, + indent=2, + ) + + "\n" + ) + ENV_SIDECAR.write_text( + json.dumps( + { + "artifact": ARTIFACT.name, + "status": "MEASUREMENT_ENVIRONMENT", + "environment": environment_block(), + }, + indent=2, + ) + + "\n" + ) + print(f"wrote {ARTIFACT}") + y24 = artifact["by_year"]["2024"] + example = y24["35_44"] + print( + "example 35_44 (2024): p50 =", + example["p50"], + "floor(p50) =", + example["floor_abs_gap_years"]["p50"], + ) + + +if __name__ == "__main__": + main() diff --git a/tests/README-tiers.md b/tests/README-tiers.md index f1ea5a53..c0cc1e17 100644 --- a/tests/README-tiers.md +++ b/tests/README-tiers.md @@ -39,8 +39,8 @@ pytest --collect-only -q -m oracle_policyengine | tail -1 | Tier | Tests at HEAD | |---|---:| | `unit` | 1,511 | -| `artifact` | 2,535 | +| `artifact` | 2,543 | | `integration_psid` | 848 | | `reproduction_legacy` | 520 | | `oracle_policyengine` | 159 | -| **Total** | **5,573** | +| **Total** | **5,581** | diff --git a/tests/test_spell_floor_artifacts.py b/tests/test_spell_floor_artifacts.py new file mode 100644 index 00000000..854e2db1 --- /dev/null +++ b/tests/test_spell_floor_artifacts.py @@ -0,0 +1,188 @@ +"""Pin the three Workstream A floor artifacts (#212, pre-IC3).""" + +from __future__ import annotations + +import hashlib +import json +from pathlib import Path + +import pytest + +RUNS = Path(__file__).resolve().parents[1] / "runs" +ROOT = RUNS.parent + +ARTIFACTS = { + "sipp_spell_floors_v1.json": "500b68034c9a301eb823e1d8f7584cf6c7654bf536247827353b0941d1d026ae", + "tenure_floors_v1.json": "08e67e5d362bbd0c1703c85fdb40624de094561f385eceb6d0a9eea4772cc6ff", + "sipp_e8_e9_floors_v1.json": "b360f04fc785eeb11c8e77e4128bdb8a98d31501a11f048fa2df4e86b1f7e059", +} +BUILDERS = { + "scripts/build_sipp_spell_floors.py": "8ce7e41a9af71767672c39f7933ccde3c2eeaa0aa4f7044c5113f7430439d1dc", + "scripts/build_tenure_floors.py": "fda07dec53ab11c41aab2b7f92dc0c18ecad0f3843d70256bb2f140351b273b1", + "scripts/build_sipp_e8_e9_floors.py": "3b2209de9b10cf680f5a074ea0c56077b03ebda68a4a614f2e20f0d0c2455272", +} + + +@pytest.fixture(scope="module") +def spells() -> dict: + return json.loads((RUNS / "sipp_spell_floors_v1.json").read_text()) + + +@pytest.fixture(scope="module") +def tenure() -> dict: + return json.loads((RUNS / "tenure_floors_v1.json").read_text()) + + +@pytest.fixture(scope="module") +def e8e9() -> dict: + return json.loads((RUNS / "sipp_e8_e9_floors_v1.json").read_text()) + + +def test_all_carry_prelock_status_and_correct_scale_gap(spells, tenure, e8e9): + for artifact in (spells, tenure, e8e9): + assert artifact["version"] == "v1" + assert "pinning event, not a ratification" in artifact["status"] + assert "RECORDED GAP" in artifact["deployment_scale_note"] + assert "1.58x" in artifact["deployment_scale_note"] + assert ( + "ANTI-conservative (too tight)" + in artifact["deployment_scale_note"] + ) + assert "RECORDED_NOT_SATISFIED" in artifact["deployment_scale_note"] + assert "promotion_integrity" not in artifact + + +def test_artifact_builder_and_sidecar_pins(): + for relative, expected in ARTIFACTS.items(): + assert ( + hashlib.sha256((RUNS / relative).read_bytes()).hexdigest() + == expected + ) + sidecar = json.loads( + (RUNS / relative.replace(".json", ".env.json")).read_text() + ) + assert sidecar["status"] == "MEASUREMENT_ENVIRONMENT" + environment = sidecar["environment"] + assert environment["python"] + assert environment["numpy"] + assert environment["pandas"] + assert environment["platform"] + for relative, expected in BUILDERS.items(): + assert ( + hashlib.sha256((ROOT / relative).read_bytes()).hexdigest() + == expected + ) + + +def test_source_input_sidecars_match_artifacts(spells, tenure, e8e9): + sipp_inputs = json.loads( + (RUNS / "sipp_spell_floors_v1.inputs.json").read_text() + ) + e8e9_inputs = json.loads( + (RUNS / "sipp_e8_e9_floors_v1.inputs.json").read_text() + ) + for artifact, sidecar in ( + (spells, sipp_inputs), + (e8e9, e8e9_inputs), + ): + assert sidecar["status"] == "SOURCE_INPUT_DIGESTS" + assert ( + sidecar["staged_input"]["path"] == artifact["source_input"]["path"] + ) + assert ( + sidecar["staged_input"]["sha256"] + == artifact["source_input"]["sha256"] + ) + assert sidecar["staged_input"]["bytes"] > 0 + official = sidecar["official_source"] + assert official["url"].startswith("https://www2.census.gov/") + assert official["archive_member"] == "pu2023.csv" + _assert_sha256(official["archive_sha256"]) + assert official["archive_bytes"] > 0 + + tenure_inputs = json.loads( + (RUNS / "tenure_floors_v1.inputs.json").read_text() + ) + assert tenure_inputs["status"] == "SOURCE_INPUT_DIGESTS" + for artifact_input, sidecar_input in zip( + tenure["source_inputs"], + tenure_inputs["source_inputs"], + strict=True, + ): + assert sidecar_input["path"] == artifact_input["path"] + assert sidecar_input["sha256"] == artifact_input["sha256"] + assert sidecar_input["bytes"] > 0 + assert sidecar_input["official_url"].startswith( + "https://www2.census.gov/" + ) + assert "/supp/" in sidecar_input["official_url"] + + +def _assert_sha256(value: str) -> None: + assert len(value) == 64 + int(value, 16) + + +def test_exact_source_input_digests_are_recorded(spells, tenure, e8e9): + assert spells["source_input"] == e8e9["source_input"] + assert spells["source_input"]["path"] == "pu2023.csv.gz" + _assert_sha256(spells["source_input"]["sha256"]) + + assert [item["year"] for item in tenure["source_inputs"]] == [ + "2020", + "2022", + "2024", + ] + assert [item["path"] for item in tenure["source_inputs"]] == [ + "jan20pub.csv", + "jan22pub.csv", + "jan24pub.csv", + ] + for item in tenure["source_inputs"]: + _assert_sha256(item["sha256"]) + + +def test_sipp_builder_uses_reader_path_resolver(): + builder = (ROOT / "scripts/build_sipp_spell_floors.py").read_text() + assert "sipp_jobs._resolve_pu_path(" in builder + assert "sipp_jobs._resolve_person_path(" not in builder + + +def test_e4_e5_pinned_values(spells): + e4 = spells["e4_retention_by_age_sex"]["16_24|sex1"] + assert e4["rate"] == 0.9897 + assert e4["floor_abs_log_ratio_mean"] == 0.00182 + e5 = spells["e5_runs_by_age"]["45_54"] + assert e5["full_year_run_share"] == 0.8671 + assert spells["seam_caveat"] + + +def test_tenure_pinned_values_and_heaping(tenure): + cell = tenure["by_year"]["2024"]["35_44"] + assert cell["p50"] == 5.0 + assert cell["floor_abs_gap_years"]["p50"]["mean"] == 0.0 + assert cell["floor_ecdf_max_gap"]["mean"] > 0 + assert "exactly zero" in tenure["heaping_caveat"] + + +def test_e8_e9_pinned_values(e8e9): + assert "seeds 0-19" in e8e9["method"] + assert "ESTIMAND NOTE" in e8e9["source"] + mix = e8e9["e9_transitions"]["transition_rates"] + assert mix["stay"] == 0.977 + assert mix["j2j"] == 0.0035 + stay = e8e9["e9_transitions"]["earnings_change"]["stay"] + assert stay["median_log_change"] == 0.0 + assert "heaps at exactly 0" in e8e9["stay_median_heaping_caveat"] + assert ( + e8e9["e9_transitions"]["earnings_change"]["stay"]["persons_unweighted"] + == 16286 + ) + assert ( + e8e9["e9_transitions"]["earnings_change"]["j2j"]["persons_unweighted"] + == 524 + ) + builder = (ROOT / "scripts/build_sipp_e8_e9_floors.py").read_text() + assert '"persons_unweighted": int(cell["person_id"].nunique())' in builder + e8 = e8e9["e8_nonemployment_by_age"]["16_24"] + assert e8["any_nonemp_share"] == pytest.approx(0.4145, abs=0.001) diff --git a/tests/tier_counts.json b/tests/tier_counts.json index d0c04de8..23d77fc0 100644 --- a/tests/tier_counts.json +++ b/tests/tier_counts.json @@ -2,7 +2,7 @@ "schema_version": 1, "counts": { "unit": 1511, - "artifact": 2535, + "artifact": 2543, "integration_psid": 848, "reproduction_legacy": 520, "oracle_policyengine": 159