From afdbcbd0621baf5710daab367b21083b696bb9e3 Mon Sep 17 00:00:00 2001 From: Daphne Hansell <128793799+daphnehanse11@users.noreply.github.com> Date: Tue, 14 Jul 2026 10:24:28 -0400 Subject: [PATCH 1/9] Add label-verified CPS ASEC firm-size (NOEMP) reader (#193) Workstream A week-1 piece of the employer-firm plan (#192): a regime-aware reader for the ASEC firm-size label. NOEMP keeps the same 0:6 code domain every year while codes 2-3 change meaning (10-49/50-99 in 2011-2018, 10-24/25-99 in 2019+, verified against all fifteen Census data dictionaries 2011-2025), so the reader hard-codes the per-year map, refuses unverified years and path/year mismatches, and enforces the WKSWORK>0 universe and code domains at read time. Records carry LJCW class of worker, longest-job industry, the I_NOEMP allocation flag, and MARSUPWT; firm_size_tabulation emits the weighted band evidence the C2 banding decision consumes. Staging follows the PSID pattern (~/PolicyEngine/asec-data, POPULACE_DYNAMICS_ASEC_DIR override). Co-Authored-By: Claude Fable 5 --- src/populace_dynamics/data/asec_firm_size.py | 333 +++++++++++++++++++ tests/data/test_asec_firm_size.py | 189 +++++++++++ tests/tier_counts.json | 2 +- 3 files changed, 523 insertions(+), 1 deletion(-) create mode 100644 src/populace_dynamics/data/asec_firm_size.py create mode 100644 tests/data/test_asec_firm_size.py diff --git a/src/populace_dynamics/data/asec_firm_size.py b/src/populace_dynamics/data/asec_firm_size.py new file mode 100644 index 00000000..d938401d --- /dev/null +++ b/src/populace_dynamics/data/asec_firm_size.py @@ -0,0 +1,333 @@ +"""CPS ASEC firm-size (NOEMP) records for the employer extension. + +NOEMP is the worker-reported count of employees "at all locations +where this employer operates" for the **longest job held last +calendar year** (universe ``WKSWORK > 0``), and is the plan's +designated firm-size training label (issue #192; loader scope in +issue #193). The variable keeps the identical code domain ``0:6`` in +every year while codes 2 and 3 silently change meaning across two +band regimes, so a year-blind read mis-bands those codes with no +error. This reader hard-codes the dictionary-adjudicated map per +year and refuses years it has not verified. + +Band regimes, verified against every year's Census public-use data +dictionary at www2.census.gov/programs-surveys/cps/datasets +(read 2026-07-14): + +* 2011-2018 (``asec2011_pubuse.dd.txt`` ... ``08ASEC2018_Data_ + Dict_Full.txt``): 1 under 10, **2 = 10-49, 3 = 50-99**, + 4 = 100-499, 5 = 500-999, 6 = 1000+. +* 2019-2025 (``06_ASEC_2019-Data_Dictionary_Full.pdf`` ... + ``asec2025_ddl_pub_full.pdf``): 1 under 10, **2 = 10-24, + 3 = 25-99**, 4 = 100-499, 5 = 500-999, 6 = 1000+. + +Note the 2019+ regime cannot resolve a 50-employee cut (it falls +inside 25-99), while 2011-2018 can — the C2 banding decision on +issue #192 consumes the tabulations this module emits. + +Alongside the band, each record carries the fields the calibration +side needs to reason about universes: ``LJCW`` (longest-job class of +worker — SUSB/QWI targets exclude government and self-employment), +longest-job industry (detailed ``INDUSTRY`` and major-group +``WEIND``), ``WKSWORK``, the ``I_NOEMP`` allocation flag, and the +ASEC supplement weight ``MARSUPWT``. + +Staging: like the PSID readers, raw microdata stays out of the +repository. Person files are staged as CSVs with original Census +variable names (the published ``asecpub{yy}csv.zip`` person file for +2017+; earlier years converted from the fixed-width ``.dat`` with the +year's ``.dd`` layout) under ``~/PolicyEngine/asec-data`` as +``pppub{yy}.csv`` (or ``.csv.gz``), overridable via the +``POPULACE_DYNAMICS_ASEC_DIR`` environment variable. +""" + +from __future__ import annotations + +import os +import re +from pathlib import Path + +import pandas as pd + +__all__ = [ + "ASEC_FIRM_SIZE_YEARS", + "CLASS_OF_WORKER_LABELS", + "NOEMP_BANDS_2011_2018", + "NOEMP_BANDS_2019_PLUS", + "band_regime", + "firm_size_tabulation", + "noemp_band_map", + "read_asec_firm_size", +] + +#: NOEMP code -> band label, 2011-2018 dictionaries (code 0 is NIU). +NOEMP_BANDS_2011_2018: dict[int, str] = { + 1: "under_10", + 2: "10_49", + 3: "50_99", + 4: "100_499", + 5: "500_999", + 6: "1000_plus", +} + +#: NOEMP code -> band label, 2019-2025 dictionaries (code 0 is NIU). +NOEMP_BANDS_2019_PLUS: dict[int, str] = { + 1: "under_10", + 2: "10_24", + 3: "25_99", + 4: "100_499", + 5: "500_999", + 6: "1000_plus", +} + +#: Survey years whose dictionaries the band maps were verified +#: against; ``noemp_band_map`` refuses anything else. +ASEC_FIRM_SIZE_YEARS: tuple[int, ...] = tuple(range(2011, 2026)) + +#: LJCW code -> label, from the 2024 dictionary ("longest job class +#: of worker"; 5/6 are self-employed incorporated yes/no-or-farm). +CLASS_OF_WORKER_LABELS: dict[int, str] = { + 1: "private", + 2: "federal", + 3: "state", + 4: "local", + 5: "self_employed_incorporated", + 6: "self_employed_unincorporated", + 7: "without_pay", +} + +#: Raw person-file columns the reader requires. +_REQUIRED_COLUMNS = ( + "PERIDNUM", + "NOEMP", + "I_NOEMP", + "LJCW", + "INDUSTRY", + "WEIND", + "WKSWORK", + "MARSUPWT", +) + +_DATA_DIR_ENV = "POPULACE_DYNAMICS_ASEC_DIR" +_DEFAULT_DATA_DIR = Path("~/PolicyEngine/asec-data").expanduser() + +_README_POINTER = ( + "stage Census ASEC person files as pppub{yy}.csv[.gz] under " + "~/PolicyEngine/asec-data (or POPULACE_DYNAMICS_ASEC_DIR)" +) + +#: A pppub filename encodes its survey year as two digits; used to +#: refuse a path/year mismatch instead of silently mis-banding. +_PPPUB_YEAR_RE = re.compile(r"pppub(\d{2})\.csv(\.gz)?$", re.I) + + +def band_regime(year: int) -> str: + """Return the band-regime key (``"2011_2018"``/``"2019_plus"``). + + Raises: + ValueError: If ``year`` has no dictionary-verified band map. + """ + if year not in ASEC_FIRM_SIZE_YEARS: + raise ValueError( + f"No dictionary-verified NOEMP band map for ASEC {year}; " + f"supported years are {ASEC_FIRM_SIZE_YEARS[0]}-" + f"{ASEC_FIRM_SIZE_YEARS[-1]}. Extend the module only " + "with that year's Census data dictionary in hand." + ) + return "2011_2018" if year <= 2018 else "2019_plus" + + +def noemp_band_map(year: int) -> dict[int, str]: + """Return the NOEMP code -> band label map for a survey year.""" + if band_regime(year) == "2011_2018": + return dict(NOEMP_BANDS_2011_2018) + return dict(NOEMP_BANDS_2019_PLUS) + + +def _resolve_data_dir(data_dir: Path | None) -> Path: + """Resolve the ASEC data directory from arg, env var, default.""" + if data_dir is not None: + return Path(data_dir).expanduser() + env_value = os.environ.get(_DATA_DIR_ENV) + if env_value: + return Path(env_value).expanduser() + return _DEFAULT_DATA_DIR + + +def _resolve_person_path(year: int, data_dir: Path) -> Path: + """Locate the staged person file for ``year`` under ``data_dir``.""" + stem = f"pppub{year % 100:02d}" + for suffix in (".csv", ".csv.gz"): + candidate = data_dir / f"{stem}{suffix}" + if candidate.exists(): + return candidate + raise FileNotFoundError( + f"No {stem}.csv[.gz] under {data_dir}; {_README_POINTER}." + ) + + +def _check_path_year(path: Path, year: int) -> None: + """Refuse a pppub filename whose year digits contradict ``year``.""" + match = _PPPUB_YEAR_RE.search(path.name) + if match is None: + return + file_year = 2000 + int(match.group(1)) + if file_year != year: + raise ValueError( + f"{path.name} is an ASEC {file_year} person file but " + f"year={year} was requested; the NOEMP band regimes " + "differ across years, so the mismatch would silently " + "mis-band codes 2-3." + ) + + +def _domain_error(year: int, column: str, bad: pd.Series) -> ValueError: + values = ", ".join(str(v) for v in sorted(bad.unique())[:8]) + return ValueError( + f"ASEC {year} {column} contains out-of-dictionary value(s) " + f"[{values}] on {len(bad)} row(s); refusing to band a file " + "that does not match the year's data dictionary." + ) + + +def read_asec_firm_size( + year: int, + *, + path: str | Path | None = None, + data_dir: str | Path | None = None, +) -> pd.DataFrame: + """Read one ASEC year's longest-job firm-size records. + + Args: + year: ASEC survey year (the job attributes describe the + longest job of calendar year ``year - 1``). + path: Explicit person-file CSV. When the filename carries + pppub year digits they must agree with ``year``. + data_dir: Staging directory (default resolution: explicit + argument, then ``POPULACE_DYNAMICS_ASEC_DIR``, then + ``~/PolicyEngine/asec-data``). + + Returns: + One row per person in the NOEMP universe (``WKSWORK > 0``, + i.e. worked last calendar year), with columns ``person_id``, + ``year``, ``income_year``, ``band_regime``, ``noemp``, + ``firm_size_band``, ``noemp_allocated``, ``ljcw``, + ``class_of_worker``, ``industry_major``, + ``industry_detailed``, ``wkswork``, and ``weight`` + (``MARSUPWT``, persons). + + Raises: + ValueError: On an unsupported year, a path/year mismatch, + missing required columns, or any value outside the + year's dictionary domain (NOEMP outside 0-6, LJCW + outside 0-7, or a nonzero NOEMP off the ``WKSWORK > 0`` + universe). + FileNotFoundError: If no staged person file can be found. + """ + bands = noemp_band_map(year) + if path is not None: + person_path = Path(path).expanduser() + if not person_path.exists(): + raise FileNotFoundError( + f"ASEC person file not found: {person_path}" + ) + else: + person_path = _resolve_person_path(year, _resolve_data_dir(data_dir)) + _check_path_year(person_path, year) + + raw = pd.read_csv(person_path, usecols=None, low_memory=False) + missing = sorted(set(_REQUIRED_COLUMNS) - set(raw.columns)) + if missing: + raise ValueError( + f"{person_path.name} is missing required column(s) " + f"{missing}; expected original Census ASEC person-file " + "variable names." + ) + raw = raw.loc[:, list(_REQUIRED_COLUMNS)].copy() + + for column, low, high in ( + ("NOEMP", 0, 6), + ("LJCW", 0, 7), + ("WKSWORK", 0, 52), + ): + values = pd.to_numeric(raw[column], errors="coerce") + bad = raw[column][values.isna() | (values < low) | (values > high)] + if len(bad): + raise _domain_error(year, column, bad) + raw[column] = values.astype(int) + + off_universe = raw[(raw["NOEMP"] > 0) & (raw["WKSWORK"] == 0)] + if len(off_universe): + raise ValueError( + f"ASEC {year}: {len(off_universe)} row(s) carry a nonzero " + "NOEMP outside the WKSWORK > 0 universe; the file does " + "not match the dictionary's universe statement." + ) + + universe = raw[raw["WKSWORK"] > 0].reset_index(drop=True) + return pd.DataFrame( + { + "person_id": universe["PERIDNUM"].astype(str), + "year": year, + "income_year": year - 1, + "band_regime": band_regime(year), + "noemp": universe["NOEMP"], + "firm_size_band": universe["NOEMP"].map( + lambda code: bands.get(int(code), "niu") + ), + "noemp_allocated": pd.to_numeric(universe["I_NOEMP"]) > 0, + "ljcw": universe["LJCW"], + "class_of_worker": universe["LJCW"].map( + lambda code: CLASS_OF_WORKER_LABELS.get(int(code), "niu") + ), + "industry_major": pd.to_numeric(universe["WEIND"]).astype(int), + "industry_detailed": pd.to_numeric(universe["INDUSTRY"]).astype( + int + ), + "wkswork": universe["WKSWORK"], + "weight": pd.to_numeric(universe["MARSUPWT"]).astype(float), + } + ) + + +def firm_size_tabulation( + records: pd.DataFrame, + by: tuple[str, ...] = ( + "year", + "band_regime", + "firm_size_band", + "class_of_worker", + ), +) -> pd.DataFrame: + """Weighted firm-size tabulation — the C2 evidence artifact. + + Args: + records: Output of :func:`read_asec_firm_size` (one or more + years concatenated). + by: Grouping columns. + + Returns: + One row per group with ``weighted_persons`` (MARSUPWT sum), + ``unweighted_n``, and ``allocated_share`` (weighted share of + the group whose NOEMP was allocated/edited), sorted by the + grouping columns. + """ + missing = sorted( + (set(by) | {"weight", "noemp_allocated"}) - set(records.columns) + ) + if missing: + raise ValueError( + f"records is missing column(s) {missing}; pass the " + "output of read_asec_firm_size." + ) + working = records.assign( + _allocated_weight=records["weight"] * records["noemp_allocated"] + ) + grouped = working.groupby(list(by), sort=True) + out = grouped.agg( + weighted_persons=("weight", "sum"), + unweighted_n=("weight", "size"), + _allocated_weight=("_allocated_weight", "sum"), + ).reset_index() + out["allocated_share"] = out["_allocated_weight"] / out["weighted_persons"] + return out.drop(columns="_allocated_weight") diff --git a/tests/data/test_asec_firm_size.py b/tests/data/test_asec_firm_size.py new file mode 100644 index 00000000..1900115c --- /dev/null +++ b/tests/data/test_asec_firm_size.py @@ -0,0 +1,189 @@ +"""Tests for the ASEC firm-size (NOEMP) reader and tabulation.""" + +from __future__ import annotations + +from pathlib import Path + +import pandas as pd +import pytest + +from populace_dynamics.data import asec_firm_size + +REAL_DATA = Path("~/PolicyEngine/asec-data").expanduser() +needs_real_asec = pytest.mark.skipif( + not REAL_DATA.is_dir(), + reason="ASEC person files not staged", +) + + +def _write_person_file( + directory: Path, + year: int, + rows: list[dict], + filename: str | None = None, +) -> Path: + """Write a fixture Census-style ASEC person CSV. + + Each row dict may override any raw column; defaults describe a + private-sector worker with an unallocated NOEMP of 2 who worked + all year, so tests only state what they exercise. + """ + directory.mkdir(parents=True, exist_ok=True) + defaults = { + "PERIDNUM": 0, + "NOEMP": 2, + "I_NOEMP": 0, + "LJCW": 1, + "INDUSTRY": 770, + "WEIND": 4, + "WKSWORK": 52, + "MARSUPWT": 1000.0, + } + frame = pd.DataFrame( + [ + {**defaults, "PERIDNUM": 10_000 + i, **row} + for i, row in enumerate(rows) + ] + ) + path = directory / (filename or f"pppub{year % 100:02d}.csv") + frame.to_csv(path, index=False) + return path + + +class TestBandMaps: + def test_2016_maps_code_2_to_10_49(self): + assert asec_firm_size.noemp_band_map(2016)[2] == "10_49" + assert asec_firm_size.noemp_band_map(2016)[3] == "50_99" + + def test_2024_maps_code_2_to_10_24(self): + assert asec_firm_size.noemp_band_map(2024)[2] == "10_24" + assert asec_firm_size.noemp_band_map(2024)[3] == "25_99" + + def test_regime_boundaries(self): + assert asec_firm_size.band_regime(2011) == "2011_2018" + assert asec_firm_size.band_regime(2018) == "2011_2018" + assert asec_firm_size.band_regime(2019) == "2019_plus" + assert asec_firm_size.band_regime(2025) == "2019_plus" + + def test_unverified_year_raises(self): + with pytest.raises(ValueError, match="dictionary-verified"): + asec_firm_size.noemp_band_map(2010) + with pytest.raises(ValueError, match="dictionary-verified"): + asec_firm_size.noemp_band_map(2026) + + +class TestReadAsecFirmSize: + def test_bands_by_regime(self, tmp_path): + rows = [{"NOEMP": 2}, {"NOEMP": 3}] + for year, expected in ( + (2016, ["10_49", "50_99"]), + (2024, ["10_24", "25_99"]), + ): + path = _write_person_file(tmp_path / str(year), year, rows) + out = asec_firm_size.read_asec_firm_size(year, path=path) + assert list(out["firm_size_band"]) == expected + assert ( + out["band_regime"] == asec_firm_size.band_regime(year) + ).all() + assert (out["income_year"] == year - 1).all() + + def test_year_and_filename_must_agree(self, tmp_path): + path = _write_person_file(tmp_path, 2016, [{}]) + with pytest.raises(ValueError, match="mis-band"): + asec_firm_size.read_asec_firm_size(2024, path=path) + + def test_resolves_staged_directory(self, tmp_path): + _write_person_file(tmp_path, 2021, [{"NOEMP": 3}]) + out = asec_firm_size.read_asec_firm_size(2021, data_dir=tmp_path) + assert list(out["firm_size_band"]) == ["25_99"] + + def test_missing_staged_file_raises(self, tmp_path): + with pytest.raises(FileNotFoundError, match="pppub21"): + asec_firm_size.read_asec_firm_size(2021, data_dir=tmp_path) + + def test_missing_column_raises(self, tmp_path): + path = _write_person_file(tmp_path, 2021, [{}]) + frame = pd.read_csv(path).drop(columns="LJCW") + frame.to_csv(path, index=False) + with pytest.raises(ValueError, match=r"\['LJCW'\]"): + asec_firm_size.read_asec_firm_size(2021, path=path) + + def test_out_of_dictionary_noemp_raises(self, tmp_path): + path = _write_person_file(tmp_path, 2021, [{"NOEMP": 7}]) + with pytest.raises(ValueError, match="out-of-dictionary"): + asec_firm_size.read_asec_firm_size(2021, path=path) + + def test_noemp_outside_universe_raises(self, tmp_path): + path = _write_person_file(tmp_path, 2021, [{"NOEMP": 2, "WKSWORK": 0}]) + with pytest.raises(ValueError, match="universe"): + asec_firm_size.read_asec_firm_size(2021, path=path) + + def test_universe_and_flags(self, tmp_path): + rows = [ + {"NOEMP": 1, "I_NOEMP": 1, "LJCW": 6}, + {"NOEMP": 0, "WKSWORK": 0, "LJCW": 0}, # NIU, dropped + {"NOEMP": 4, "LJCW": 3}, + ] + path = _write_person_file(tmp_path, 2021, rows) + out = asec_firm_size.read_asec_firm_size(2021, path=path) + assert len(out) == 2 + assert list(out["noemp_allocated"]) == [True, False] + assert list(out["class_of_worker"]) == [ + "self_employed_unincorporated", + "state", + ] + + def test_env_var_staging(self, tmp_path, monkeypatch): + _write_person_file(tmp_path, 2021, [{}]) + monkeypatch.setenv("POPULACE_DYNAMICS_ASEC_DIR", str(tmp_path)) + out = asec_firm_size.read_asec_firm_size(2021) + assert len(out) == 1 + + +class TestFirmSizeTabulation: + def test_weighted_counts_and_allocated_share(self, tmp_path): + rows = [ + {"NOEMP": 2, "MARSUPWT": 1000.0, "I_NOEMP": 1}, + {"NOEMP": 2, "MARSUPWT": 3000.0, "I_NOEMP": 0}, + {"NOEMP": 5, "MARSUPWT": 500.0, "I_NOEMP": 0, "LJCW": 2}, + ] + path = _write_person_file(tmp_path, 2024, rows) + records = asec_firm_size.read_asec_firm_size(2024, path=path) + out = asec_firm_size.firm_size_tabulation(records) + ten_24 = out[out["firm_size_band"] == "10_24"].iloc[0] + assert ten_24["weighted_persons"] == 4000.0 + assert ten_24["unweighted_n"] == 2 + assert ten_24["allocated_share"] == 0.25 + federal = out[out["class_of_worker"] == "federal"].iloc[0] + assert federal["weighted_persons"] == 500.0 + assert federal["firm_size_band"] == "500_999" + + def test_regime_break_is_visible_across_years(self, tmp_path): + code_2 = [{"NOEMP": 2}] + frames = [ + asec_firm_size.read_asec_firm_size( + year, + path=_write_person_file(tmp_path / str(year), year, code_2), + ) + for year in (2018, 2019) + ] + out = asec_firm_size.firm_size_tabulation(pd.concat(frames)) + assert set(out["firm_size_band"]) == {"10_49", "10_24"} + + def test_wrong_frame_raises(self): + with pytest.raises(ValueError, match="read_asec_firm_size"): + asec_firm_size.firm_size_tabulation(pd.DataFrame({"x": [1]})) + + +@needs_real_asec +class TestRealData: + def test_reads_any_staged_year(self): + staged = sorted(REAL_DATA.glob("pppub*.csv*")) + if not staged: + pytest.skip("no pppub files staged") + year = 2000 + int(staged[-1].name[5:7]) + out = asec_firm_size.read_asec_firm_size(year, path=staged[-1]) + assert len(out) > 10_000 + assert set(out["firm_size_band"]) <= set( + asec_firm_size.noemp_band_map(year).values() + ) diff --git a/tests/tier_counts.json b/tests/tier_counts.json index b298cf6d..3c011a8f 100644 --- a/tests/tier_counts.json +++ b/tests/tier_counts.json @@ -1,7 +1,7 @@ { "schema_version": 1, "counts": { - "unit": 420, + "unit": 450, "artifact": 1011, "integration_psid": 800, "reproduction_legacy": 520, From 288bae1ba165b05ab6237386fa86ad548a4be0f1 Mon Sep 17 00:00:00 2001 From: Daphne Hansell <128793799+daphnehanse11@users.noreply.github.com> Date: Tue, 14 Jul 2026 11:08:06 -0400 Subject: [PATCH 2/9] Address review: string PERIDNUM, symmetric universe checks, total error paths - Read PERIDNUM as string: 22-digit ids overflow int64 and a float64 fallback rounds distinct persons together before astype(str); adds a last-digit-apart fixture test and a uniqueness assertion to the real-data pass. - Enforce the shared NOEMP/LJCW universe symmetrically (nonzero exactly where WKSWORK > 0); the 2011-2018 dictionaries state it as WORKYN = 1 and 2019+ as WKSWORK > 0, which coincide. - Make _domain_error total on mixed-type columns (sorted(map(str, ...))). - Validate I_NOEMP (0-9) and MARSUPWT (non-negative) domains. - Read only the eight required columns (usecols) instead of all ~800. Co-Authored-By: Claude Fable 5 --- src/populace_dynamics/data/asec_firm_size.py | 67 +++++++++++++------- tests/data/test_asec_firm_size.py | 37 +++++++++++ 2 files changed, 82 insertions(+), 22 deletions(-) diff --git a/src/populace_dynamics/data/asec_firm_size.py b/src/populace_dynamics/data/asec_firm_size.py index d938401d..a4258c73 100644 --- a/src/populace_dynamics/data/asec_firm_size.py +++ b/src/populace_dynamics/data/asec_firm_size.py @@ -182,7 +182,7 @@ def _check_path_year(path: Path, year: int) -> None: def _domain_error(year: int, column: str, bad: pd.Series) -> ValueError: - values = ", ".join(str(v) for v in sorted(bad.unique())[:8]) + values = ", ".join(sorted(map(str, bad.unique()))[:8]) return ValueError( f"ASEC {year} {column} contains out-of-dictionary value(s) " f"[{values}] on {len(bad)} row(s); refusing to band a file " @@ -218,10 +218,13 @@ def read_asec_firm_size( Raises: ValueError: On an unsupported year, a path/year mismatch, - missing required columns, or any value outside the - year's dictionary domain (NOEMP outside 0-6, LJCW - outside 0-7, or a nonzero NOEMP off the ``WKSWORK > 0`` - universe). + missing required columns, any value outside the year's + dictionary domain (NOEMP outside 0-6, LJCW outside 0-7, + a negative weight), or a universe violation: NOEMP and + LJCW share the longest-job universe in every dictionary + (stated as ``WORKYN = 1`` in 2011-2018 and + ``WKSWORK > 0`` in 2019+, which coincide), so each must + be nonzero exactly on the ``WKSWORK > 0`` rows. FileNotFoundError: If no staged person file can be found. """ bands = noemp_band_map(year) @@ -235,8 +238,16 @@ def read_asec_firm_size( person_path = _resolve_person_path(year, _resolve_data_dir(data_dir)) _check_path_year(person_path, year) - raw = pd.read_csv(person_path, usecols=None, low_memory=False) - missing = sorted(set(_REQUIRED_COLUMNS) - set(raw.columns)) + # PERIDNUM is a 22-digit identifier: wider than int64, and a + # float64 fallback would round distinct persons together, so it + # must never pass through a numeric dtype. + required = set(_REQUIRED_COLUMNS) + raw = pd.read_csv( + person_path, + usecols=lambda column: column in required, + dtype={"PERIDNUM": "string"}, + ) + missing = sorted(required - set(raw.columns)) if missing: raise ValueError( f"{person_path.name} is missing required column(s) " @@ -245,24 +256,36 @@ def read_asec_firm_size( ) raw = raw.loc[:, list(_REQUIRED_COLUMNS)].copy() - for column, low, high in ( - ("NOEMP", 0, 6), - ("LJCW", 0, 7), - ("WKSWORK", 0, 52), + for column, low, high, cast in ( + ("NOEMP", 0, 6, int), + ("LJCW", 0, 7, int), + ("WKSWORK", 0, 52, int), + ("I_NOEMP", 0, 9, int), + ("MARSUPWT", 0, None, float), ): values = pd.to_numeric(raw[column], errors="coerce") - bad = raw[column][values.isna() | (values < low) | (values > high)] + out_of_domain = values.isna() | (values < low) + if high is not None: + out_of_domain |= values > high + bad = raw[column][out_of_domain] if len(bad): raise _domain_error(year, column, bad) - raw[column] = values.astype(int) - - off_universe = raw[(raw["NOEMP"] > 0) & (raw["WKSWORK"] == 0)] - if len(off_universe): - raise ValueError( - f"ASEC {year}: {len(off_universe)} row(s) carry a nonzero " - "NOEMP outside the WKSWORK > 0 universe; the file does " - "not match the dictionary's universe statement." - ) + raw[column] = values.astype(cast) + + # NOEMP and LJCW share the longest-job universe in every + # dictionary year, so a zero inside WKSWORK > 0 (or a nonzero + # outside it) means the file does not match its dictionary. + for column in ("NOEMP", "LJCW"): + mismatch = raw[(raw[column] > 0) != (raw["WKSWORK"] > 0)] + if len(mismatch): + raise ValueError( + f"ASEC {year}: {len(mismatch)} row(s) violate the " + f"{column} universe ({column} must be nonzero " + "exactly where WKSWORK > 0); the file does not " + "match the dictionary's universe statement — " + "re-adjudicate against that year's dictionary " + "before extending the reader." + ) universe = raw[raw["WKSWORK"] > 0].reset_index(drop=True) return pd.DataFrame( @@ -275,7 +298,7 @@ def read_asec_firm_size( "firm_size_band": universe["NOEMP"].map( lambda code: bands.get(int(code), "niu") ), - "noemp_allocated": pd.to_numeric(universe["I_NOEMP"]) > 0, + "noemp_allocated": universe["I_NOEMP"] > 0, "ljcw": universe["LJCW"], "class_of_worker": universe["LJCW"].map( lambda code: CLASS_OF_WORKER_LABELS.get(int(code), "niu") diff --git a/tests/data/test_asec_firm_size.py b/tests/data/test_asec_firm_size.py index 1900115c..09f234ea 100644 --- a/tests/data/test_asec_firm_size.py +++ b/tests/data/test_asec_firm_size.py @@ -118,6 +118,42 @@ def test_noemp_outside_universe_raises(self, tmp_path): with pytest.raises(ValueError, match="universe"): asec_firm_size.read_asec_firm_size(2021, path=path) + def test_zero_noemp_inside_universe_raises(self, tmp_path): + path = _write_person_file(tmp_path, 2021, [{"NOEMP": 0}]) + with pytest.raises(ValueError, match="NOEMP universe"): + asec_firm_size.read_asec_firm_size(2021, path=path) + + def test_zero_ljcw_inside_universe_raises(self, tmp_path): + path = _write_person_file(tmp_path, 2021, [{"LJCW": 0}]) + with pytest.raises(ValueError, match="LJCW universe"): + asec_firm_size.read_asec_firm_size(2021, path=path) + + def test_mixed_type_domain_violation_stays_a_value_error(self, tmp_path): + rows = [{"NOEMP": "A"}, {"NOEMP": 7}] + path = _write_person_file(tmp_path, 2021, rows) + with pytest.raises(ValueError, match="out-of-dictionary"): + asec_firm_size.read_asec_firm_size(2021, path=path) + + def test_negative_weight_raises(self, tmp_path): + path = _write_person_file(tmp_path, 2021, [{"MARSUPWT": -1.0}]) + with pytest.raises(ValueError, match="MARSUPWT"): + asec_firm_size.read_asec_firm_size(2021, path=path) + + def test_wide_person_ids_survive_exactly(self, tmp_path): + # 22-digit PERIDNUMs differing only in the last digit round + # together under any numeric read (int64 tops out at 19 + # digits; float64 keeps ~15-16), so exact string survival + # is the regression test for the dtype pin. + ids = [ + "8812345678901234567891", + "8812345678901234567892", + ] + path = _write_person_file( + tmp_path, 2021, [{"PERIDNUM": i} for i in ids] + ) + out = asec_firm_size.read_asec_firm_size(2021, path=path) + assert list(out["person_id"]) == ids + def test_universe_and_flags(self, tmp_path): rows = [ {"NOEMP": 1, "I_NOEMP": 1, "LJCW": 6}, @@ -184,6 +220,7 @@ def test_reads_any_staged_year(self): year = 2000 + int(staged[-1].name[5:7]) out = asec_firm_size.read_asec_firm_size(year, path=staged[-1]) assert len(out) > 10_000 + assert out["person_id"].is_unique assert set(out["firm_size_band"]) <= set( asec_firm_size.noemp_band_map(year).values() ) From 9e0754e6b13afe86fcd06b053c6a45d8c568c492 Mon Sep 17 00:00:00 2001 From: Daphne Hansell <128793799+daphnehanse11@users.noreply.github.com> Date: Tue, 14 Jul 2026 13:10:10 -0400 Subject: [PATCH 3/9] Harden the loader against silent-wrong-output seams (round-2 review) - Refuse blank or duplicated PERIDNUM (the join key now gets the same refuse-on-mismatch treatment as every coded column; NaN ids vanish silently from downstream groupbys). - Reject non-integral code values (2.9 truncated to band 10_24 with no error) and non-finite weights. - Move WEIND/INDUSTRY into the domain loop: friendly errors instead of raw pandas cast failures, and validation no longer depends on the universe filter. - firm_size_tabulation: keep NaN group keys (dropna=False), NaN allocated_share for zero-weight groups, documented. - Friendly message for an empty staged file. Co-Authored-By: Claude Fable 5 --- src/populace_dynamics/data/asec_firm_size.py | 65 +++++++++++++++----- tests/data/test_asec_firm_size.py | 46 ++++++++++++++ 2 files changed, 96 insertions(+), 15 deletions(-) diff --git a/src/populace_dynamics/data/asec_firm_size.py b/src/populace_dynamics/data/asec_firm_size.py index a4258c73..e4ce25d1 100644 --- a/src/populace_dynamics/data/asec_firm_size.py +++ b/src/populace_dynamics/data/asec_firm_size.py @@ -47,6 +47,7 @@ import re from pathlib import Path +import numpy as np import pandas as pd __all__ = [ @@ -242,11 +243,16 @@ def read_asec_firm_size( # float64 fallback would round distinct persons together, so it # must never pass through a numeric dtype. required = set(_REQUIRED_COLUMNS) - raw = pd.read_csv( - person_path, - usecols=lambda column: column in required, - dtype={"PERIDNUM": "string"}, - ) + try: + raw = pd.read_csv( + person_path, + usecols=lambda column: column in required, + dtype={"PERIDNUM": "string"}, + ) + except pd.errors.EmptyDataError: + raise ValueError( + f"{person_path.name} is empty; {_README_POINTER}." + ) from None missing = sorted(required - set(raw.columns)) if missing: raise ValueError( @@ -256,17 +262,43 @@ def read_asec_firm_size( ) raw = raw.loc[:, list(_REQUIRED_COLUMNS)].copy() + # The join key gets the same refuse-on-mismatch treatment as + # every coded column: a blank or duplicated PERIDNUM is exactly + # what a botched fixed-width conversion produces, and NaN ids + # vanish silently from downstream groupbys. + ids = raw["PERIDNUM"] + blank = ids.isna() | (ids.astype(str).str.strip() == "") + if blank.any(): + raise ValueError( + f"ASEC {year}: {int(blank.sum())} row(s) have a blank " + "PERIDNUM; refusing a file with unusable person ids." + ) + duplicated = ids[ids.duplicated()] + if len(duplicated): + raise ValueError( + f"ASEC {year}: PERIDNUM is not unique " + f"({len(duplicated)} duplicated id(s), e.g. " + f"{duplicated.iloc[0]!r}); refusing a file with a " + "corrupted person-id column." + ) + for column, low, high, cast in ( ("NOEMP", 0, 6, int), ("LJCW", 0, 7, int), ("WKSWORK", 0, 52, int), ("I_NOEMP", 0, 9, int), + ("WEIND", 0, 23, int), + ("INDUSTRY", 0, 9999, int), ("MARSUPWT", 0, None, float), ): values = pd.to_numeric(raw[column], errors="coerce") - out_of_domain = values.isna() | (values < low) + out_of_domain = values.isna() | (values < low) | ~np.isfinite(values) if high is not None: out_of_domain |= values > high + if cast is int: + # Dictionary codes are integers; 2.9 is not a code and + # must not silently truncate into one. + out_of_domain |= values != values.round() bad = raw[column][out_of_domain] if len(bad): raise _domain_error(year, column, bad) @@ -303,12 +335,10 @@ def read_asec_firm_size( "class_of_worker": universe["LJCW"].map( lambda code: CLASS_OF_WORKER_LABELS.get(int(code), "niu") ), - "industry_major": pd.to_numeric(universe["WEIND"]).astype(int), - "industry_detailed": pd.to_numeric(universe["INDUSTRY"]).astype( - int - ), + "industry_major": universe["WEIND"], + "industry_detailed": universe["INDUSTRY"], "wkswork": universe["WKSWORK"], - "weight": pd.to_numeric(universe["MARSUPWT"]).astype(float), + "weight": universe["MARSUPWT"], } ) @@ -332,8 +362,9 @@ def firm_size_tabulation( Returns: One row per group with ``weighted_persons`` (MARSUPWT sum), ``unweighted_n``, and ``allocated_share`` (weighted share of - the group whose NOEMP was allocated/edited), sorted by the - grouping columns. + the group whose NOEMP was allocated/edited; NaN for a group + whose weights sum to zero), sorted by the grouping columns. + NaN group keys are kept, never silently dropped. """ missing = sorted( (set(by) | {"weight", "noemp_allocated"}) - set(records.columns) @@ -346,11 +377,15 @@ def firm_size_tabulation( working = records.assign( _allocated_weight=records["weight"] * records["noemp_allocated"] ) - grouped = working.groupby(list(by), sort=True) + grouped = working.groupby(list(by), sort=True, dropna=False) out = grouped.agg( weighted_persons=("weight", "sum"), unweighted_n=("weight", "size"), _allocated_weight=("_allocated_weight", "sum"), ).reset_index() - out["allocated_share"] = out["_allocated_weight"] / out["weighted_persons"] + out["allocated_share"] = np.where( + out["weighted_persons"] > 0, + out["_allocated_weight"] / out["weighted_persons"], + np.nan, + ) return out.drop(columns="_allocated_weight") diff --git a/tests/data/test_asec_firm_size.py b/tests/data/test_asec_firm_size.py index 09f234ea..99377518 100644 --- a/tests/data/test_asec_firm_size.py +++ b/tests/data/test_asec_firm_size.py @@ -139,6 +139,38 @@ def test_negative_weight_raises(self, tmp_path): with pytest.raises(ValueError, match="MARSUPWT"): asec_firm_size.read_asec_firm_size(2021, path=path) + def test_blank_person_id_raises(self, tmp_path): + path = _write_person_file(tmp_path, 2021, [{"PERIDNUM": ""}]) + with pytest.raises(ValueError, match="blank PERIDNUM"): + asec_firm_size.read_asec_firm_size(2021, path=path) + + def test_duplicate_person_id_raises(self, tmp_path): + rows = [{"PERIDNUM": 77}, {"PERIDNUM": 77}] + path = _write_person_file(tmp_path, 2021, rows) + with pytest.raises(ValueError, match="not unique"): + asec_firm_size.read_asec_firm_size(2021, path=path) + + def test_fractional_code_raises(self, tmp_path): + path = _write_person_file(tmp_path, 2021, [{"NOEMP": 2.9}]) + with pytest.raises(ValueError, match="NOEMP"): + asec_firm_size.read_asec_firm_size(2021, path=path) + + def test_industry_garbage_gets_friendly_error(self, tmp_path): + path = _write_person_file(tmp_path, 2021, [{"INDUSTRY": "XX"}]) + with pytest.raises(ValueError, match="INDUSTRY"): + asec_firm_size.read_asec_firm_size(2021, path=path) + + def test_infinite_weight_raises(self, tmp_path): + path = _write_person_file(tmp_path, 2021, [{"MARSUPWT": "inf"}]) + with pytest.raises(ValueError, match="MARSUPWT"): + asec_firm_size.read_asec_firm_size(2021, path=path) + + def test_empty_file_gets_friendly_error(self, tmp_path): + path = tmp_path / "pppub21.csv" + path.write_text("") + with pytest.raises(ValueError, match="empty"): + asec_firm_size.read_asec_firm_size(2021, path=path) + def test_wide_person_ids_survive_exactly(self, tmp_path): # 22-digit PERIDNUMs differing only in the last digit round # together under any numeric read (int64 tops out at 19 @@ -206,6 +238,20 @@ def test_regime_break_is_visible_across_years(self, tmp_path): out = asec_firm_size.firm_size_tabulation(pd.concat(frames)) assert set(out["firm_size_band"]) == {"10_49", "10_24"} + def test_zero_weight_group_share_is_nan(self, tmp_path): + path = _write_person_file(tmp_path, 2024, [{"MARSUPWT": 0.0}]) + records = asec_firm_size.read_asec_firm_size(2024, path=path) + out = asec_firm_size.firm_size_tabulation(records) + assert out.iloc[0]["weighted_persons"] == 0.0 + assert pd.isna(out.iloc[0]["allocated_share"]) + + def test_nan_group_keys_are_kept(self, tmp_path): + path = _write_person_file(tmp_path, 2024, [{}, {}]) + records = asec_firm_size.read_asec_firm_size(2024, path=path) + records.loc[0, "class_of_worker"] = None + out = asec_firm_size.firm_size_tabulation(records) + assert out["weighted_persons"].sum() == records["weight"].sum() + def test_wrong_frame_raises(self): with pytest.raises(ValueError, match="read_asec_firm_size"): asec_firm_size.firm_size_tabulation(pd.DataFrame({"x": [1]})) From 4aead2ffa5c428350f346d0ce7ca382a959aea7a Mon Sep 17 00:00:00 2001 From: Vahid Ahmadi Date: Wed, 15 Jul 2026 12:26:19 +0100 Subject: [PATCH 4/9] Re-export the ASEC firm-size reader from data/__init__.py Addresses the #204 review (non-blocking note 1): every sibling reader is re-exported and named in the package docstring; asec_firm_size was neither. Adds read_asec_firm_size / firm_size_tabulation / band_regime / noemp_band_map and the band constants to the package API, and names the reader in the module docstring, for API consistency and the downstream C2 consumer. Co-Authored-By: Claude Opus 4.8 (1M context) --- src/populace_dynamics/data/__init__.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/src/populace_dynamics/data/__init__.py b/src/populace_dynamics/data/__init__.py index 0f14300c..712144b4 100644 --- a/src/populace_dynamics/data/__init__.py +++ b/src/populace_dynamics/data/__init__.py @@ -7,10 +7,22 @@ readers -- marriage (:mod:`~populace_dynamics.data.marriage`), childbirth and adoption (:mod:`~populace_dynamics.data.births`), and the family relationship matrix (:mod:`~populace_dynamics.data.relmap`). + +Also exposes the label-verified CPS ASEC firm-size (NOEMP) reader +(:mod:`~populace_dynamics.data.asec_firm_size`). """ from __future__ import annotations +from populace_dynamics.data.asec_firm_size import ( + ASEC_FIRM_SIZE_YEARS, + NOEMP_BANDS_2011_2018, + NOEMP_BANDS_2019_PLUS, + band_regime, + firm_size_tabulation, + noemp_band_map, + read_asec_firm_size, +) from populace_dynamics.data.births import birth_events, birth_history from populace_dynamics.data.deaths import ( decode_death_code, @@ -48,6 +60,13 @@ ) __all__ = [ + "ASEC_FIRM_SIZE_YEARS", + "NOEMP_BANDS_2011_2018", + "NOEMP_BANDS_2019_PLUS", + "band_regime", + "firm_size_tabulation", + "noemp_band_map", + "read_asec_firm_size", "decode_death_code", "read_death_records", "FAMILY_WAVES", From bdb91606fa04e98f29a0a29b07e1ef54219c2347 Mon Sep 17 00:00:00 2001 From: Daphne Hansell <128793799+daphnehanse11@users.noreply.github.com> Date: Wed, 15 Jul 2026 09:28:57 -0400 Subject: [PATCH 5/9] Descale MARSUPWT (two implied decimals) and close review round 3 - weight = MARSUPWT / 100 (raw 158007 = 1580.07 persons), matching the tenure reader's PWTENWGT handling; regression fixture with a realistic raw value plus a 1e8-3e8 magnitude tripwire on the real-data pass (unscaled reads land near 17 billion). - I_NOEMP tightened to its exact dictionary set {0, 1, 9}. - The 2011-2018 WORKYN=1 vs 2019+ WKSWORK>0 universe coincidence is now asserted at read time, not assumed in prose. - Dead niu fallbacks replaced with total .map()s; domain-error values sort numerically. Co-Authored-By: Claude Fable 5 --- src/populace_dynamics/data/asec_firm_size.py | 78 ++++++++++++++------ tests/data/test_asec_firm_size.py | 47 +++++++++--- tests/tier_counts.json | 2 +- 3 files changed, 92 insertions(+), 35 deletions(-) diff --git a/src/populace_dynamics/data/asec_firm_size.py b/src/populace_dynamics/data/asec_firm_size.py index e4ce25d1..5cfd6c1f 100644 --- a/src/populace_dynamics/data/asec_firm_size.py +++ b/src/populace_dynamics/data/asec_firm_size.py @@ -30,7 +30,10 @@ worker — SUSB/QWI targets exclude government and self-employment), longest-job industry (detailed ``INDUSTRY`` and major-group ``WEIND``), ``WKSWORK``, the ``I_NOEMP`` allocation flag, and the -ASEC supplement weight ``MARSUPWT``. +ASEC supplement weight ``MARSUPWT`` — which carries **two implied +decimals** (raw ``158007`` = 1580.07 persons) and is descaled to +persons at read time, the same Census convention the tenure reader +descales for ``PWTENWGT``. Staging: like the PSID readers, raw microdata stays out of the repository. Person files are staged as CSVs with original Census @@ -106,6 +109,7 @@ "INDUSTRY", "WEIND", "WKSWORK", + "WORKYN", "MARSUPWT", ) @@ -183,7 +187,12 @@ def _check_path_year(path: Path, year: int) -> None: def _domain_error(year: int, column: str, bad: pd.Series) -> ValueError: - values = ", ".join(sorted(map(str, bad.unique()))[:8]) + unique = list(bad.unique()) + try: + unique = sorted(unique) + except TypeError: + unique = sorted(map(str, unique)) + values = ", ".join(str(v) for v in unique[:8]) return ValueError( f"ASEC {year} {column} contains out-of-dictionary value(s) " f"[{values}] on {len(bad)} row(s); refusing to band a file " @@ -215,17 +224,20 @@ def read_asec_firm_size( ``firm_size_band``, ``noemp_allocated``, ``ljcw``, ``class_of_worker``, ``industry_major``, ``industry_detailed``, ``wkswork``, and ``weight`` - (``MARSUPWT``, persons). + (``MARSUPWT / 100`` — the raw column carries two implied + decimals — in persons). Raises: ValueError: On an unsupported year, a path/year mismatch, missing required columns, any value outside the year's dictionary domain (NOEMP outside 0-6, LJCW outside 0-7, - a negative weight), or a universe violation: NOEMP and - LJCW share the longest-job universe in every dictionary - (stated as ``WORKYN = 1`` in 2011-2018 and - ``WKSWORK > 0`` in 2019+, which coincide), so each must - be nonzero exactly on the ``WKSWORK > 0`` rows. + I_NOEMP outside {0, 1, 9}, a negative weight), or a + universe violation: NOEMP and LJCW share the longest-job + universe in every dictionary (stated as ``WORKYN = 1`` + in 2011-2018 and ``WKSWORK > 0`` in 2019+ — a + coincidence this reader asserts at read time rather than + assuming), so each must be nonzero exactly on the + ``WKSWORK > 0`` rows. FileNotFoundError: If no staged person file can be found. """ bands = noemp_band_map(year) @@ -282,19 +294,25 @@ def read_asec_firm_size( "corrupted person-id column." ) - for column, low, high, cast in ( - ("NOEMP", 0, 6, int), - ("LJCW", 0, 7, int), - ("WKSWORK", 0, 52, int), - ("I_NOEMP", 0, 9, int), - ("WEIND", 0, 23, int), - ("INDUSTRY", 0, 9999, int), - ("MARSUPWT", 0, None, float), + # I_NOEMP's dictionary domain is the exact set {0 no change, + # 1 allocated, 9 full imputation} — a stray 2-8 is corruption, + # not a flag state. + for column, low, high, allowed, cast in ( + ("NOEMP", 0, 6, None, int), + ("LJCW", 0, 7, None, int), + ("WKSWORK", 0, 52, None, int), + ("WORKYN", 0, 2, None, int), + ("I_NOEMP", 0, 9, (0, 1, 9), int), + ("WEIND", 0, 23, None, int), + ("INDUSTRY", 0, 9999, None, int), + ("MARSUPWT", 0, None, None, float), ): values = pd.to_numeric(raw[column], errors="coerce") out_of_domain = values.isna() | (values < low) | ~np.isfinite(values) if high is not None: out_of_domain |= values > high + if allowed is not None: + out_of_domain |= ~values.isin(allowed) if cast is int: # Dictionary codes are integers; 2.9 is not a code and # must not silently truncate into one. @@ -304,6 +322,19 @@ def read_asec_firm_size( raise _domain_error(year, column, bad) raw[column] = values.astype(cast) + # The 2011-2018 dictionaries state the longest-job universe as + # WORKYN = 1 and the 2019+ dictionaries as WKSWORK > 0; the code + # keys on WKSWORK, so their coincidence is asserted here rather + # than assumed in prose. + workyn_mismatch = raw[(raw["WORKYN"] == 1) != (raw["WKSWORK"] > 0)] + if len(workyn_mismatch): + raise ValueError( + f"ASEC {year}: {len(workyn_mismatch)} row(s) have " + "WORKYN = 1 without WKSWORK > 0 (or vice versa); the " + "two universe statements no longer coincide — " + "re-adjudicate against that year's dictionary." + ) + # NOEMP and LJCW share the longest-job universe in every # dictionary year, so a zero inside WKSWORK > 0 (or a nonzero # outside it) means the file does not match its dictionary. @@ -327,18 +358,19 @@ def read_asec_firm_size( "income_year": year - 1, "band_regime": band_regime(year), "noemp": universe["NOEMP"], - "firm_size_band": universe["NOEMP"].map( - lambda code: bands.get(int(code), "niu") - ), + # Total mappings: the domain + universe checks guarantee + # NOEMP in 1-6 and LJCW in 1-7 here, so no fallback. + "firm_size_band": universe["NOEMP"].map(bands), "noemp_allocated": universe["I_NOEMP"] > 0, "ljcw": universe["LJCW"], - "class_of_worker": universe["LJCW"].map( - lambda code: CLASS_OF_WORKER_LABELS.get(int(code), "niu") - ), + "class_of_worker": universe["LJCW"].map(CLASS_OF_WORKER_LABELS), "industry_major": universe["WEIND"], "industry_detailed": universe["INDUSTRY"], "wkswork": universe["WKSWORK"], - "weight": universe["MARSUPWT"], + # MARSUPWT carries two implied decimals (raw 158007 is + # 1580.07 persons), the same Census convention the + # tenure reader descales for PWTENWGT. + "weight": universe["MARSUPWT"] / 100.0, } ) diff --git a/tests/data/test_asec_firm_size.py b/tests/data/test_asec_firm_size.py index 99377518..04572530 100644 --- a/tests/data/test_asec_firm_size.py +++ b/tests/data/test_asec_firm_size.py @@ -26,7 +26,9 @@ def _write_person_file( Each row dict may override any raw column; defaults describe a private-sector worker with an unallocated NOEMP of 2 who worked - all year, so tests only state what they exercise. + all year at raw weight 100000 (MARSUPWT carries two implied + decimals, so that is 1000.00 persons). WORKYN defaults to match + the row's WKSWORK unless overridden. """ directory.mkdir(parents=True, exist_ok=True) defaults = { @@ -37,14 +39,16 @@ def _write_person_file( "INDUSTRY": 770, "WEIND": 4, "WKSWORK": 52, - "MARSUPWT": 1000.0, + "MARSUPWT": 100_000, } - frame = pd.DataFrame( - [ - {**defaults, "PERIDNUM": 10_000 + i, **row} - for i, row in enumerate(rows) - ] - ) + records = [] + for i, row in enumerate(rows): + record = {**defaults, "PERIDNUM": 10_000 + i, **row} + record.setdefault( + "WORKYN", 1 if record["WKSWORK"] not in (0, "0") else 0 + ) + records.append(record) + frame = pd.DataFrame(records) path = directory / (filename or f"pppub{year % 100:02d}.csv") frame.to_csv(path, index=False) return path @@ -211,9 +215,9 @@ def test_env_var_staging(self, tmp_path, monkeypatch): class TestFirmSizeTabulation: def test_weighted_counts_and_allocated_share(self, tmp_path): rows = [ - {"NOEMP": 2, "MARSUPWT": 1000.0, "I_NOEMP": 1}, - {"NOEMP": 2, "MARSUPWT": 3000.0, "I_NOEMP": 0}, - {"NOEMP": 5, "MARSUPWT": 500.0, "I_NOEMP": 0, "LJCW": 2}, + {"NOEMP": 2, "MARSUPWT": 100_000, "I_NOEMP": 1}, + {"NOEMP": 2, "MARSUPWT": 300_000, "I_NOEMP": 0}, + {"NOEMP": 5, "MARSUPWT": 50_000, "I_NOEMP": 0, "LJCW": 2}, ] path = _write_person_file(tmp_path, 2024, rows) records = asec_firm_size.read_asec_firm_size(2024, path=path) @@ -257,6 +261,23 @@ def test_wrong_frame_raises(self): asec_firm_size.firm_size_tabulation(pd.DataFrame({"x": [1]})) +class TestWeightScaling: + def test_raw_weight_has_two_implied_decimals(self, tmp_path): + path = _write_person_file(tmp_path, 2024, [{"MARSUPWT": "158007"}]) + out = asec_firm_size.read_asec_firm_size(2024, path=path) + assert list(out["weight"]) == [1580.07] + + def test_stray_allocation_flag_code_raises(self, tmp_path): + path = _write_person_file(tmp_path, 2024, [{"I_NOEMP": 5}]) + with pytest.raises(ValueError, match="I_NOEMP"): + asec_firm_size.read_asec_firm_size(2024, path=path) + + def test_workyn_wkswork_mismatch_raises(self, tmp_path): + path = _write_person_file(tmp_path, 2024, [{"WORKYN": 2}]) + with pytest.raises(ValueError, match="WORKYN"): + asec_firm_size.read_asec_firm_size(2024, path=path) + + @needs_real_asec class TestRealData: def test_reads_any_staged_year(self): @@ -270,3 +291,7 @@ def test_reads_any_staged_year(self): assert set(out["firm_size_band"]) <= set( asec_firm_size.noemp_band_map(year).values() ) + # Magnitude tripwire for the two-implied-decimals weight + # convention: the worked-last-year universe is ~170 million + # persons; an unscaled read lands near 17 billion. + assert 1.0e8 < out["weight"].sum() < 3.0e8 diff --git a/tests/tier_counts.json b/tests/tier_counts.json index 7c4c6487..0631c0bb 100644 --- a/tests/tier_counts.json +++ b/tests/tier_counts.json @@ -1,7 +1,7 @@ { "schema_version": 1, "counts": { - "unit": 464, + "unit": 467, "artifact": 1013, "integration_psid": 802, "reproduction_legacy": 520, From 90eec6eed1689cfaca88a41cf9358d4693ccf9b4 Mon Sep 17 00:00:00 2001 From: Daphne Hansell <128793799+daphnehanse11@users.noreply.github.com> Date: Wed, 15 Jul 2026 09:49:48 -0400 Subject: [PATCH 6/9] Scope the WORKYN coincidence to 2011-2018 (real-data finding) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit First run against a real file (2024 pppub, 144,265 rows): NOEMP and LJCW track the WKSWORK universe with zero violations and the scaled weights sum to 173.2M persons in the worked-last-year universe — but 572 rows (0.4%) carry WORKYN = 2 beside a fully edited work block. The 2019+ dictionaries state the universe as WKSWORK > 0 directly, so the WORKYN = 1 coincidence is now enforced only for 2011-2018 files, where it is the stated universe. Real-data test passes end to end. Co-Authored-By: Claude Fable 5 --- src/populace_dynamics/data/asec_firm_size.py | 30 +++++++++++++------- tests/data/test_asec_firm_size.py | 18 ++++++++++-- tests/tier_counts.json | 2 +- 3 files changed, 35 insertions(+), 15 deletions(-) diff --git a/src/populace_dynamics/data/asec_firm_size.py b/src/populace_dynamics/data/asec_firm_size.py index 5cfd6c1f..b2b7594c 100644 --- a/src/populace_dynamics/data/asec_firm_size.py +++ b/src/populace_dynamics/data/asec_firm_size.py @@ -323,17 +323,25 @@ def read_asec_firm_size( raw[column] = values.astype(cast) # The 2011-2018 dictionaries state the longest-job universe as - # WORKYN = 1 and the 2019+ dictionaries as WKSWORK > 0; the code - # keys on WKSWORK, so their coincidence is asserted here rather - # than assumed in prose. - workyn_mismatch = raw[(raw["WORKYN"] == 1) != (raw["WKSWORK"] > 0)] - if len(workyn_mismatch): - raise ValueError( - f"ASEC {year}: {len(workyn_mismatch)} row(s) have " - "WORKYN = 1 without WKSWORK > 0 (or vice versa); the " - "two universe statements no longer coincide — " - "re-adjudicate against that year's dictionary." - ) + # WORKYN = 1; the code keys on WKSWORK, so for those years the + # coincidence is asserted rather than assumed. The 2019+ + # dictionaries state the universe as WKSWORK > 0 directly, and + # the coincidence genuinely fails there on real data: the 2024 + # file carries 572 rows (0.4%) with WORKYN = 2 beside a fully + # edited work block (most not even allocated) while NOEMP and + # LJCW track WKSWORK exactly (0 violations on 144,265 rows) — + # so for 2019+ WORKYN is domain-checked but not required to + # coincide. + if band_regime(year) == "2011_2018": + workyn_mismatch = raw[(raw["WORKYN"] == 1) != (raw["WKSWORK"] > 0)] + if len(workyn_mismatch): + raise ValueError( + f"ASEC {year}: {len(workyn_mismatch)} row(s) have " + "WORKYN = 1 without WKSWORK > 0 (or vice versa); " + "the stated WORKYN = 1 universe no longer matches " + "the WKSWORK key this reader uses — re-adjudicate " + "against that year's dictionary." + ) # NOEMP and LJCW share the longest-job universe in every # dictionary year, so a zero inside WKSWORK > 0 (or a nonzero diff --git a/tests/data/test_asec_firm_size.py b/tests/data/test_asec_firm_size.py index 04572530..e39481cf 100644 --- a/tests/data/test_asec_firm_size.py +++ b/tests/data/test_asec_firm_size.py @@ -272,10 +272,22 @@ def test_stray_allocation_flag_code_raises(self, tmp_path): with pytest.raises(ValueError, match="I_NOEMP"): asec_firm_size.read_asec_firm_size(2024, path=path) - def test_workyn_wkswork_mismatch_raises(self, tmp_path): - path = _write_person_file(tmp_path, 2024, [{"WORKYN": 2}]) + def test_workyn_wkswork_mismatch_raises_pre_2019(self, tmp_path): + # WORKYN = 1 is the stated universe in the 2011-2018 + # dictionaries, so the coincidence with WKSWORK is enforced + # there. + path = _write_person_file(tmp_path, 2016, [{"WORKYN": 2}]) with pytest.raises(ValueError, match="WORKYN"): - asec_firm_size.read_asec_firm_size(2024, path=path) + asec_firm_size.read_asec_firm_size(2016, path=path) + + def test_workyn_mismatch_tolerated_2019_plus(self, tmp_path): + # 2019+ dictionaries state the universe as WKSWORK > 0 + # directly; real 2024 data carries ~0.4% WORKYN = 2 rows + # beside a fully edited work block, so no coincidence + # requirement there. + path = _write_person_file(tmp_path, 2024, [{"WORKYN": 2}]) + out = asec_firm_size.read_asec_firm_size(2024, path=path) + assert len(out) == 1 @needs_real_asec diff --git a/tests/tier_counts.json b/tests/tier_counts.json index 0631c0bb..4d4e097b 100644 --- a/tests/tier_counts.json +++ b/tests/tier_counts.json @@ -1,7 +1,7 @@ { "schema_version": 1, "counts": { - "unit": 467, + "unit": 468, "artifact": 1013, "integration_psid": 802, "reproduction_legacy": 520, From c89e011f250e8e468c74ba77779ca48daf994fe9 Mon Sep 17 00:00:00 2001 From: Daphne Hansell <128793799+daphnehanse11@users.noreply.github.com> Date: Wed, 15 Jul 2026 10:18:27 -0400 Subject: [PATCH 7/9] One band map for all years: the 2019 dictionary relabeling is phantom MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Real-data finding, established with a discontinuity test across the supposed 2018/2019 band revert. Weighted NOEMP code shares among WKSWORK>0 workers are continuous (code 2: 14.9% in 2017, 14.4% in 2019, 14.1% in 2024; code 3: 7.0%/6.9%/6.7%), where a genuine re-bin of 10-49 into 10-24 would have roughly halved code 2 and doubled code 3. SUSB corroborates: code 3's ~7% matches the 50-99 administrative employment share (~7.5%); a true 25-99 band carries ~15%. The 2019+ dictionaries (and IPUMS FIRMSIZE, which inherits them) mislabel codes 2/3; the instrument has collected 10-49/50-99 continuously since 2011. Consequences: a single NOEMP_BANDS map (codes 2/3 = 10_49/50_99 in every supported year), the band_regime machinery removed, and the C2 story inverts — the 50-employee edge is directly observed in 2019+ data after all. Evidence documented in the module docstring; flagged on #192 for the C2 contract and #195's IPUMS-keyed maps. Co-Authored-By: Claude Fable 5 --- src/populace_dynamics/data/__init__.py | 8 +- src/populace_dynamics/data/asec_firm_size.py | 95 +++++++++----------- tests/data/test_asec_firm_size.py | 53 +++++------ tests/tier_counts.json | 2 +- 4 files changed, 71 insertions(+), 87 deletions(-) diff --git a/src/populace_dynamics/data/__init__.py b/src/populace_dynamics/data/__init__.py index 712144b4..34ef5a6b 100644 --- a/src/populace_dynamics/data/__init__.py +++ b/src/populace_dynamics/data/__init__.py @@ -16,9 +16,7 @@ from populace_dynamics.data.asec_firm_size import ( ASEC_FIRM_SIZE_YEARS, - NOEMP_BANDS_2011_2018, - NOEMP_BANDS_2019_PLUS, - band_regime, + NOEMP_BANDS, firm_size_tabulation, noemp_band_map, read_asec_firm_size, @@ -61,9 +59,7 @@ __all__ = [ "ASEC_FIRM_SIZE_YEARS", - "NOEMP_BANDS_2011_2018", - "NOEMP_BANDS_2019_PLUS", - "band_regime", + "NOEMP_BANDS", "firm_size_tabulation", "noemp_band_map", "read_asec_firm_size", diff --git a/src/populace_dynamics/data/asec_firm_size.py b/src/populace_dynamics/data/asec_firm_size.py index b2b7594c..2d504567 100644 --- a/src/populace_dynamics/data/asec_firm_size.py +++ b/src/populace_dynamics/data/asec_firm_size.py @@ -4,26 +4,28 @@ where this employer operates" for the **longest job held last calendar year** (universe ``WKSWORK > 0``), and is the plan's designated firm-size training label (issue #192; loader scope in -issue #193). The variable keeps the identical code domain ``0:6`` in -every year while codes 2 and 3 silently change meaning across two -band regimes, so a year-blind read mis-bands those codes with no -error. This reader hard-codes the dictionary-adjudicated map per -year and refuses years it has not verified. - -Band regimes, verified against every year's Census public-use data -dictionary at www2.census.gov/programs-surveys/cps/datasets -(read 2026-07-14): - -* 2011-2018 (``asec2011_pubuse.dd.txt`` ... ``08ASEC2018_Data_ - Dict_Full.txt``): 1 under 10, **2 = 10-49, 3 = 50-99**, - 4 = 100-499, 5 = 500-999, 6 = 1000+. -* 2019-2025 (``06_ASEC_2019-Data_Dictionary_Full.pdf`` ... - ``asec2025_ddl_pub_full.pdf``): 1 under 10, **2 = 10-24, - 3 = 25-99**, 4 = 100-499, 5 = 500-999, 6 = 1000+. - -Note the 2019+ regime cannot resolve a 50-employee cut (it falls -inside 25-99), while 2011-2018 can — the C2 banding decision on -issue #192 consumes the tabulations this module emits. +issue #193). One band map covers every supported year: **1 under +10, 2 = 10-49, 3 = 50-99, 4 = 100-499, 5 = 500-999, 6 = 1000+**. + +**The phantom 2019 relabeling.** The published data dictionaries +disagree across years: 2011-2018 label codes 2/3 as 10-49 / 50-99, +while the 2019-2025 dictionaries (and IPUMS FIRMSIZE, which +inherits them) relabel the same codes 10-24 / 25-99. The data show +the relabeling never happened in the instrument. Weighted code +shares among ``WKSWORK > 0`` workers are continuous across the +supposed break (code 2: 14.9% in 2017, 14.4% in 2019, 14.1% in +2024; code 3: 7.0% / 6.9% / 6.7% — measured from the published +public-use files, 2026-07-15), where a genuine re-binning of a +39-integer band into a 15-integer band would have roughly halved +code 2 and doubled code 3. SUSB's administrative distribution +corroborates: code 3's ~7% share matches SUSB's 50-99 employment +share (~7.5%), while a true 25-99 band carries ~15%. This reader +therefore uses the 10-49 / 50-99 reading for all years and records +the dictionary conflict here rather than silently following the +2019+ label text into a factor-two mis-band. Consequence for C2: +the 50-employee edge (ACA and state mandates) is directly observed +in every supported year — the "post-2019 label cannot resolve the +50 cut" problem stated in earlier drafts dissolves. Alongside the band, each record carries the fields the calibration side needs to reason about universes: ``LJCW`` (longest-job class of @@ -56,16 +58,17 @@ __all__ = [ "ASEC_FIRM_SIZE_YEARS", "CLASS_OF_WORKER_LABELS", - "NOEMP_BANDS_2011_2018", - "NOEMP_BANDS_2019_PLUS", - "band_regime", + "NOEMP_BANDS", "firm_size_tabulation", "noemp_band_map", "read_asec_firm_size", ] -#: NOEMP code -> band label, 2011-2018 dictionaries (code 0 is NIU). -NOEMP_BANDS_2011_2018: dict[int, str] = { +#: NOEMP code -> band label for every supported year (code 0 is +#: NIU). The 2019-2025 dictionaries relabel codes 2/3 as 10-24 / +#: 25-99, but the instrument never changed — see "The phantom 2019 +#: relabeling" in the module docstring for the evidence. +NOEMP_BANDS: dict[int, str] = { 1: "under_10", 2: "10_49", 3: "50_99", @@ -74,16 +77,6 @@ 6: "1000_plus", } -#: NOEMP code -> band label, 2019-2025 dictionaries (code 0 is NIU). -NOEMP_BANDS_2019_PLUS: dict[int, str] = { - 1: "under_10", - 2: "10_24", - 3: "25_99", - 4: "100_499", - 5: "500_999", - 6: "1000_plus", -} - #: Survey years whose dictionaries the band maps were verified #: against; ``noemp_band_map`` refuses anything else. ASEC_FIRM_SIZE_YEARS: tuple[int, ...] = tuple(range(2011, 2026)) @@ -126,27 +119,29 @@ _PPPUB_YEAR_RE = re.compile(r"pppub(\d{2})\.csv(\.gz)?$", re.I) -def band_regime(year: int) -> str: - """Return the band-regime key (``"2011_2018"``/``"2019_plus"``). - - Raises: - ValueError: If ``year`` has no dictionary-verified band map. - """ +def _check_supported_year(year: int) -> None: if year not in ASEC_FIRM_SIZE_YEARS: raise ValueError( f"No dictionary-verified NOEMP band map for ASEC {year}; " f"supported years are {ASEC_FIRM_SIZE_YEARS[0]}-" f"{ASEC_FIRM_SIZE_YEARS[-1]}. Extend the module only " - "with that year's Census data dictionary in hand." + "with that year's Census data dictionary in hand — and " + "check the empirical code distribution before trusting " + "the dictionary's band labels (module docstring)." ) - return "2011_2018" if year <= 2018 else "2019_plus" def noemp_band_map(year: int) -> dict[int, str]: - """Return the NOEMP code -> band label map for a survey year.""" - if band_regime(year) == "2011_2018": - return dict(NOEMP_BANDS_2011_2018) - return dict(NOEMP_BANDS_2019_PLUS) + """Return the NOEMP code -> band label map for a survey year. + + One map covers every supported year: the 2019+ dictionaries' + relabeling of codes 2/3 is documentary only (module docstring). + + Raises: + ValueError: If ``year`` is outside the verified range. + """ + _check_supported_year(year) + return dict(NOEMP_BANDS) def _resolve_data_dir(data_dir: Path | None) -> Path: @@ -220,7 +215,7 @@ def read_asec_firm_size( Returns: One row per person in the NOEMP universe (``WKSWORK > 0``, i.e. worked last calendar year), with columns ``person_id``, - ``year``, ``income_year``, ``band_regime``, ``noemp``, + ``year``, ``income_year``, ``noemp``, ``firm_size_band``, ``noemp_allocated``, ``ljcw``, ``class_of_worker``, ``industry_major``, ``industry_detailed``, ``wkswork``, and ``weight`` @@ -332,7 +327,7 @@ def read_asec_firm_size( # LJCW track WKSWORK exactly (0 violations on 144,265 rows) — # so for 2019+ WORKYN is domain-checked but not required to # coincide. - if band_regime(year) == "2011_2018": + if year <= 2018: workyn_mismatch = raw[(raw["WORKYN"] == 1) != (raw["WKSWORK"] > 0)] if len(workyn_mismatch): raise ValueError( @@ -364,7 +359,6 @@ def read_asec_firm_size( "person_id": universe["PERIDNUM"].astype(str), "year": year, "income_year": year - 1, - "band_regime": band_regime(year), "noemp": universe["NOEMP"], # Total mappings: the domain + universe checks guarantee # NOEMP in 1-6 and LJCW in 1-7 here, so no fallback. @@ -387,7 +381,6 @@ def firm_size_tabulation( records: pd.DataFrame, by: tuple[str, ...] = ( "year", - "band_regime", "firm_size_band", "class_of_worker", ), diff --git a/tests/data/test_asec_firm_size.py b/tests/data/test_asec_firm_size.py index e39481cf..2613cf08 100644 --- a/tests/data/test_asec_firm_size.py +++ b/tests/data/test_asec_firm_size.py @@ -55,19 +55,16 @@ def _write_person_file( class TestBandMaps: - def test_2016_maps_code_2_to_10_49(self): - assert asec_firm_size.noemp_band_map(2016)[2] == "10_49" - assert asec_firm_size.noemp_band_map(2016)[3] == "50_99" - - def test_2024_maps_code_2_to_10_24(self): - assert asec_firm_size.noemp_band_map(2024)[2] == "10_24" - assert asec_firm_size.noemp_band_map(2024)[3] == "25_99" - - def test_regime_boundaries(self): - assert asec_firm_size.band_regime(2011) == "2011_2018" - assert asec_firm_size.band_regime(2018) == "2011_2018" - assert asec_firm_size.band_regime(2019) == "2019_plus" - assert asec_firm_size.band_regime(2025) == "2019_plus" + def test_one_map_for_all_years(self): + # The 2019+ dictionaries relabel codes 2/3 as 10-24/25-99, + # but the instrument never changed (weighted code shares are + # continuous 2017 -> 2019 -> 2024 and match SUSB only under + # the 10-49/50-99 reading — module docstring). One map. + for year in (2011, 2016, 2018, 2019, 2024, 2025): + bands = asec_firm_size.noemp_band_map(year) + assert bands[2] == "10_49" + assert bands[3] == "50_99" + assert bands == asec_firm_size.NOEMP_BANDS def test_unverified_year_raises(self): with pytest.raises(ValueError, match="dictionary-verified"): @@ -77,18 +74,12 @@ def test_unverified_year_raises(self): class TestReadAsecFirmSize: - def test_bands_by_regime(self, tmp_path): + def test_bands_stable_across_years(self, tmp_path): rows = [{"NOEMP": 2}, {"NOEMP": 3}] - for year, expected in ( - (2016, ["10_49", "50_99"]), - (2024, ["10_24", "25_99"]), - ): + for year in (2016, 2024): path = _write_person_file(tmp_path / str(year), year, rows) out = asec_firm_size.read_asec_firm_size(year, path=path) - assert list(out["firm_size_band"]) == expected - assert ( - out["band_regime"] == asec_firm_size.band_regime(year) - ).all() + assert list(out["firm_size_band"]) == ["10_49", "50_99"] assert (out["income_year"] == year - 1).all() def test_year_and_filename_must_agree(self, tmp_path): @@ -99,7 +90,7 @@ def test_year_and_filename_must_agree(self, tmp_path): def test_resolves_staged_directory(self, tmp_path): _write_person_file(tmp_path, 2021, [{"NOEMP": 3}]) out = asec_firm_size.read_asec_firm_size(2021, data_dir=tmp_path) - assert list(out["firm_size_band"]) == ["25_99"] + assert list(out["firm_size_band"]) == ["50_99"] def test_missing_staged_file_raises(self, tmp_path): with pytest.raises(FileNotFoundError, match="pppub21"): @@ -222,15 +213,18 @@ def test_weighted_counts_and_allocated_share(self, tmp_path): path = _write_person_file(tmp_path, 2024, rows) records = asec_firm_size.read_asec_firm_size(2024, path=path) out = asec_firm_size.firm_size_tabulation(records) - ten_24 = out[out["firm_size_band"] == "10_24"].iloc[0] - assert ten_24["weighted_persons"] == 4000.0 - assert ten_24["unweighted_n"] == 2 - assert ten_24["allocated_share"] == 0.25 + band = out[out["firm_size_band"] == "10_49"].iloc[0] + assert band["weighted_persons"] == 4000.0 + assert band["unweighted_n"] == 2 + assert band["allocated_share"] == 0.25 federal = out[out["class_of_worker"] == "federal"].iloc[0] assert federal["weighted_persons"] == 500.0 assert federal["firm_size_band"] == "500_999" - def test_regime_break_is_visible_across_years(self, tmp_path): + def test_bands_pool_cleanly_across_years(self, tmp_path): + # Same code, same band on both sides of the documentary + # 2018/2019 dictionary relabeling — pooled years share one + # band vocabulary. code_2 = [{"NOEMP": 2}] frames = [ asec_firm_size.read_asec_firm_size( @@ -240,7 +234,8 @@ def test_regime_break_is_visible_across_years(self, tmp_path): for year in (2018, 2019) ] out = asec_firm_size.firm_size_tabulation(pd.concat(frames)) - assert set(out["firm_size_band"]) == {"10_49", "10_24"} + assert set(out["firm_size_band"]) == {"10_49"} + assert set(out["year"]) == {2018, 2019} def test_zero_weight_group_share_is_nan(self, tmp_path): path = _write_person_file(tmp_path, 2024, [{"MARSUPWT": 0.0}]) diff --git a/tests/tier_counts.json b/tests/tier_counts.json index 2dcde623..38fb4de0 100644 --- a/tests/tier_counts.json +++ b/tests/tier_counts.json @@ -1,7 +1,7 @@ { "schema_version": 1, "counts": { - "unit": 471, + "unit": 469, "artifact": 1014, "integration_psid": 802, "reproduction_legacy": 520, From 0eb9529326d5376ada8c6052dd77522b6e8356ae Mon Sep 17 00:00:00 2001 From: Daphne Hansell <128793799+daphnehanse11@users.noreply.github.com> Date: Wed, 15 Jul 2026 10:42:18 -0400 Subject: [PATCH 8/9] Drop the WORKYN coincidence check in both regimes (2017 real-data finding) Staging the 2017 file (converted from the fixed-width .dat per the documented path) shows the WORKYN=2-beside-edited-work-block inconsistency is a stable ~0.4% property of BOTH regimes (2017: 821 of 185,914 rows; 2024: 572 of 144,265; never the reverse direction), while NOEMP and LJCW track WKSWORK with zero violations in both files. WKSWORK is the operative universe key in every year; WORKYN stays domain-checked. Multi-year read now validates 2017/2019/2024 end to end (164.8M/167.7M/173.2M weighted workers) with smooth band shares across the documentary break. Co-Authored-By: Claude Fable 5 --- src/populace_dynamics/data/asec_firm_size.py | 29 ++++++------------ tests/data/test_asec_firm_size.py | 31 ++++++++++---------- tests/tier_counts.json | 2 +- 3 files changed, 26 insertions(+), 36 deletions(-) diff --git a/src/populace_dynamics/data/asec_firm_size.py b/src/populace_dynamics/data/asec_firm_size.py index 2d504567..5f32d12d 100644 --- a/src/populace_dynamics/data/asec_firm_size.py +++ b/src/populace_dynamics/data/asec_firm_size.py @@ -317,26 +317,15 @@ def read_asec_firm_size( raise _domain_error(year, column, bad) raw[column] = values.astype(cast) - # The 2011-2018 dictionaries state the longest-job universe as - # WORKYN = 1; the code keys on WKSWORK, so for those years the - # coincidence is asserted rather than assumed. The 2019+ - # dictionaries state the universe as WKSWORK > 0 directly, and - # the coincidence genuinely fails there on real data: the 2024 - # file carries 572 rows (0.4%) with WORKYN = 2 beside a fully - # edited work block (most not even allocated) while NOEMP and - # LJCW track WKSWORK exactly (0 violations on 144,265 rows) — - # so for 2019+ WORKYN is domain-checked but not required to - # coincide. - if year <= 2018: - workyn_mismatch = raw[(raw["WORKYN"] == 1) != (raw["WKSWORK"] > 0)] - if len(workyn_mismatch): - raise ValueError( - f"ASEC {year}: {len(workyn_mismatch)} row(s) have " - "WORKYN = 1 without WKSWORK > 0 (or vice versa); " - "the stated WORKYN = 1 universe no longer matches " - "the WKSWORK key this reader uses — re-adjudicate " - "against that year's dictionary." - ) + # The dictionaries state the longest-job universe as WORKYN = 1 + # (2011-2018) or WKSWORK > 0 (2019+), but the two do NOT + # coincide exactly on real files in either regime: a stable + # ~0.4% of rows carry WORKYN = 2 beside a fully edited work + # block (2017: 821 of 185,914; 2024: 572 of 144,265 — never the + # reverse direction), while NOEMP and LJCW track WKSWORK with + # zero violations in both files. WKSWORK is therefore the + # operative universe key in every year; WORKYN stays + # domain-checked but is not required to coincide. # NOEMP and LJCW share the longest-job universe in every # dictionary year, so a zero inside WKSWORK > 0 (or a nonzero diff --git a/tests/data/test_asec_firm_size.py b/tests/data/test_asec_firm_size.py index 2613cf08..3bc1e77c 100644 --- a/tests/data/test_asec_firm_size.py +++ b/tests/data/test_asec_firm_size.py @@ -267,22 +267,23 @@ def test_stray_allocation_flag_code_raises(self, tmp_path): with pytest.raises(ValueError, match="I_NOEMP"): asec_firm_size.read_asec_firm_size(2024, path=path) - def test_workyn_wkswork_mismatch_raises_pre_2019(self, tmp_path): - # WORKYN = 1 is the stated universe in the 2011-2018 - # dictionaries, so the coincidence with WKSWORK is enforced - # there. - path = _write_person_file(tmp_path, 2016, [{"WORKYN": 2}]) + def test_workyn_mismatch_tolerated_every_year(self, tmp_path): + # A stable ~0.4% of real rows in BOTH regimes carry + # WORKYN = 2 beside a fully edited work block (2017: 821 + # rows; 2024: 572), while NOEMP/LJCW track WKSWORK exactly — + # WKSWORK is the operative universe key, so the WORKYN + # mismatch is tolerated, never refused. + for year in (2016, 2024): + path = _write_person_file( + tmp_path / str(year), year, [{"WORKYN": 2}] + ) + out = asec_firm_size.read_asec_firm_size(year, path=path) + assert len(out) == 1 + + def test_workyn_out_of_domain_still_raises(self, tmp_path): + path = _write_person_file(tmp_path, 2024, [{"WORKYN": 3}]) with pytest.raises(ValueError, match="WORKYN"): - asec_firm_size.read_asec_firm_size(2016, path=path) - - def test_workyn_mismatch_tolerated_2019_plus(self, tmp_path): - # 2019+ dictionaries state the universe as WKSWORK > 0 - # directly; real 2024 data carries ~0.4% WORKYN = 2 rows - # beside a fully edited work block, so no coincidence - # requirement there. - path = _write_person_file(tmp_path, 2024, [{"WORKYN": 2}]) - out = asec_firm_size.read_asec_firm_size(2024, path=path) - assert len(out) == 1 + asec_firm_size.read_asec_firm_size(2024, path=path) @needs_real_asec diff --git a/tests/tier_counts.json b/tests/tier_counts.json index 38fb4de0..8a07c357 100644 --- a/tests/tier_counts.json +++ b/tests/tier_counts.json @@ -1,7 +1,7 @@ { "schema_version": 1, "counts": { - "unit": 469, + "unit": 470, "artifact": 1014, "integration_psid": 802, "reproduction_legacy": 520, From 6f08e203793de789298bd983a0f65ef90374a3c9 Mon Sep 17 00:00:00 2001 From: Vahid Ahmadi Date: Wed, 15 Jul 2026 16:23:56 +0100 Subject: [PATCH 9/9] Fix tier_counts unit 470 -> 469 (WORKYN-drop removed a test) The band-map unification bumped the collection to 470, but dropping the WORKYN coincidence check in 0eb9529 removed its test, returning the unit tier to 469. The manifest was left at 470, failing test_tier_policy. Verified against the full collection: 469 + 1014 + 802 + 520 + 159 = 2964 = total collected. Co-Authored-By: Claude Opus 4.8 (1M context) --- tests/tier_counts.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/tier_counts.json b/tests/tier_counts.json index 8a07c357..38fb4de0 100644 --- a/tests/tier_counts.json +++ b/tests/tier_counts.json @@ -1,7 +1,7 @@ { "schema_version": 1, "counts": { - "unit": 470, + "unit": 469, "artifact": 1014, "integration_psid": 802, "reproduction_legacy": 520,