diff --git a/data/external/kauffman_firm_survey_public_use.md b/data/external/kauffman_firm_survey_public_use.md new file mode 100644 index 00000000..cf79a3a6 --- /dev/null +++ b/data/external/kauffman_firm_survey_public_use.md @@ -0,0 +1,64 @@ +# Kauffman Firm Survey public-use microdata + +The Kauffman Firm Survey (KFS) is an observed-firm panel of 4,928 +anonymized businesses founded in 2004 and followed through 2011. It is +not synthetic data. It is also not a current cross-section of all US +employers and contains neither employee rosters nor observed links to +SIPP or CPS respondents. + +Raw files are staged outside this repository and must not be committed +or redistributed without explicit permission. The publisher describes +the files as public-use and permits download, but supplies no explicit +KFS redistribution license. + +## Staged input + +- Publisher: Ewing Marion Kauffman Foundation +- Landing page: +- Archive: +- Retrieved: 2026-07-30 +- Archive bytes: 11,118,022 +- Archive SHA-256: + `2428d3197405a2436f49cadd03492057cf8b51fd9f8ee7be9a73d16156136bde` +- Long-file SHA-256: + `a6042ad2be946b836e4f36a5e04d9d1fe5790387b252c7a256d0dc8a08140f8a` +- Default local long-file path: + `~/PolicyEngine/kfs-data/logically-imputed/Public_Use_LI_Long.dta` + +The archive's workbook and readme document logical imputation, soft and +hard missing values, renamed variables, newly constructed variables, +and conversions of range variables. The project reader deliberately +retains employment ranges as intervals instead of replacing them with +midpoints. + +## Permitted project use before IC3 lock + +KFS may support descriptive profiling of observed young-firm +trajectories and development of firm-record handling methods. It is not +an IC3 target, a substitute for SUSB/QWI/BDS/J2J margins, or an input to +candidate fitting. Any later calibration or model use requires a +post-lock design decision. + +## Initial descriptive profile + +The staged long file contains 39,424 unique firm-year rows: 4,928 +anonymized firms in each of eight years. Employment is available as an +exact count or interval on 24,139 rows; 23,569 are exact and 570 are +interval- or top-coded. Completed records decline from 4,928 in 2004 to +2,007 in 2011 as the cohort exits, merges, pauses, or stops responding. + +| Year | Firm rows | Complete records | Employment available | Exact zero employees | +|---:|---:|---:|---:|---:| +| 2004 | 4,928 | 4,928 | 4,823 | 2,838 | +| 2005 | 4,928 | 3,998 | 3,952 | 1,633 | +| 2006 | 4,928 | 3,390 | 3,353 | 1,267 | +| 2007 | 4,928 | 2,915 | 2,890 | 1,299 | +| 2008 | 4,928 | 2,606 | 2,602 | 1,170 | +| 2009 | 4,928 | 2,408 | 2,398 | 1,158 | +| 2010 | 4,928 | 2,126 | 2,121 | 1,044 | +| 2011 | 4,928 | 2,007 | 2,000 | 969 | + +These values describe a startup cohort, not national firm counts. +Cross-sectional survey weights sum to approximately 73,278 cohort firms +in every year, but must not be interpreted as the number of all US +firms. diff --git a/scripts/first_estimates_birth_evidence.py b/scripts/first_estimates_birth_evidence.py index af2b51ad..8378d727 100644 --- a/scripts/first_estimates_birth_evidence.py +++ b/scripts/first_estimates_birth_evidence.py @@ -134,6 +134,7 @@ ) POST_REVIEW_SOURCE_EXCLUSIONS = ( Path("src/populace_dynamics/artifacts.py"), + Path("src/populace_dynamics/data/kauffman_firms.py"), Path("src/populace_dynamics/firms/targets.py"), # Entry-11 PSID data-layer additions are downstream source readers and # registries. They are outside the reviewed birth-evidence projection diff --git a/src/populace_dynamics/data/kauffman_firms.py b/src/populace_dynamics/data/kauffman_firms.py new file mode 100644 index 00000000..2d12adeb --- /dev/null +++ b/src/populace_dynamics/data/kauffman_firms.py @@ -0,0 +1,257 @@ +"""Observed young-firm records from the Kauffman Firm Survey (KFS). + +KFS follows 4,928 anonymized businesses founded in 2004 through 2011. +These are observed survey respondents, not synthetic firms. They are +useful for descriptive young-firm trajectories and method development, +but they are not a current cross-section of all US employers and contain +no employee roster or observed link to SIPP/CPS workers. + +Raw public-use files remain outside the repository. By default this +reader expects the publisher's logically imputed long Stata file at +``~/PolicyEngine/kfs-data/logically-imputed/Public_Use_LI_Long.dta``. +Set ``POPULACE_DYNAMICS_KFS_DIR`` or pass an explicit path to override +that location. +""" + +from __future__ import annotations + +import os +import re +from pathlib import Path + +import numpy as np +import pandas as pd + +__all__ = ["KFS_YEARS", "kfs_profile", "read_kauffman_firms"] + +KFS_YEARS: tuple[int, ...] = tuple(range(2004, 2012)) + +_DATA_DIR_ENV = "POPULACE_DYNAMICS_KFS_DIR" +_DEFAULT_DATA_DIR = Path( + "~/PolicyEngine/kfs-data/logically-imputed" +).expanduser() +_LONG_FILENAME = "Public_Use_LI_Long.dta" + +_REQUIRED_COLUMNS = ( + "mprid", + "year", + "status", + "c5_num_employees", + "c6_num_ft_employees", + "c7_num_pt_employees", + "naics_code", + "cswgt_final", +) + +_COUNT_INTERVAL_RE = re.compile( + r"^(?P\d+)$|^(?P\d+)\+$|" + r"^(?P\d+)-(?P\d+)$" +) +_MISSING_COUNT_CODES = frozenset({"", ".a"}) + + +def _resolve_path( + path: str | Path | None, + data_dir: str | Path | None, +) -> Path: + if path is not None: + return Path(path).expanduser() + if data_dir is not None: + directory = Path(data_dir).expanduser() + elif os.environ.get(_DATA_DIR_ENV): + directory = Path(os.environ[_DATA_DIR_ENV]).expanduser() + else: + directory = _DEFAULT_DATA_DIR + candidate = directory / _LONG_FILENAME + if not candidate.exists(): + raise FileNotFoundError( + f"No {_LONG_FILENAME} under {directory}; download the KFS " + "logically imputed public-use archive and stage its contents " + "outside git, or set POPULACE_DYNAMICS_KFS_DIR." + ) + return candidate + + +def _count_intervals( + values: pd.Series, + *, + column: str, +) -> pd.DataFrame: + """Parse exact and top-/interval-coded employee counts.""" + text = values.fillna("").astype("string").str.strip() + missing = text.isin(_MISSING_COUNT_CODES) + parsed = text.where(~missing).str.extract(_COUNT_INTERVAL_RE) + invalid = ~missing & parsed.isna().all(axis=1) + if invalid.any(): + examples = sorted(text[invalid].unique().tolist())[:8] + raise ValueError( + f"KFS {column} contains unsupported count code(s) {examples}; " + "refusing to invent interval semantics." + ) + + exact = pd.to_numeric(parsed["exact"], errors="coerce") + lower = exact.fillna( + pd.to_numeric(parsed["lower"], errors="coerce") + ).fillna(pd.to_numeric(parsed["range_lower"], errors="coerce")) + upper = exact.fillna(pd.to_numeric(parsed["range_upper"], errors="coerce")) + bad_range = lower.notna() & upper.notna() & (upper < lower) + if bad_range.any(): + raise ValueError(f"KFS {column} contains a decreasing count interval.") + + return pd.DataFrame( + { + f"{column}_raw": text.mask(missing, pd.NA), + f"{column}_lower": lower.astype("Int64"), + f"{column}_upper": upper.astype("Int64"), + f"{column}_exact": exact.astype("Int64"), + }, + index=values.index, + ) + + +def read_kauffman_firms( + *, + path: str | Path | None = None, + data_dir: str | Path | None = None, +) -> pd.DataFrame: + """Read the KFS logically imputed public-use firm-year file. + + Returns one row per anonymized KFS firm and survey year. Employee + counts retain their publisher-supplied raw code and expose lower, + upper, and exact values separately; open-ended and interval-coded + counts are never silently replaced by midpoints. + """ + source = _resolve_path(path, data_dir) + if not source.exists(): + raise FileNotFoundError( + f"KFS public-use file does not exist: {source}" + ) + + try: + labels = pd.io.stata.StataReader(source).variable_labels() + except (OSError, ValueError) as error: + raise ValueError(f"KFS Stata file is unreadable: {source}") from error + missing_columns = sorted(set(_REQUIRED_COLUMNS) - set(labels)) + if missing_columns: + raise ValueError( + f"KFS file is missing required columns {missing_columns}." + ) + + raw = pd.read_stata( + source, + columns=list(_REQUIRED_COLUMNS), + convert_categoricals=False, + ) + if raw.empty: + raise ValueError("KFS public-use file contains no firm-year rows.") + + firm_number = pd.to_numeric(raw["mprid"], errors="coerce") + bad_id = ( + firm_number.isna() + | ~np.isfinite(firm_number) + | (firm_number <= 0) + | (firm_number % 1 != 0) + ) + if bad_id.any(): + raise ValueError( + "KFS mprid contains blank or non-integral identifiers." + ) + firm_id = firm_number.astype("int64").astype("string") + + year_number = pd.to_numeric(raw["year"], errors="coerce") + bad_year = ( + year_number.isna() + | (year_number % 1 != 0) + | ~year_number.isin(KFS_YEARS) + ) + if bad_year.any(): + values = sorted(raw.loc[bad_year, "year"].astype(str).unique())[:8] + raise ValueError(f"KFS year contains unsupported value(s) {values}.") + year = year_number.astype("int16") + + keys = pd.DataFrame({"firm_id": firm_id, "year": year}) + if keys.duplicated().any(): + raise ValueError("KFS mprid/year is not unique.") + + status = raw["status"].fillna("").astype("string").str.strip() + if status.eq("").any(): + raise ValueError("KFS status is blank on one or more firm-year rows.") + + industry = pd.to_numeric(raw["naics_code"], errors="coerce") + bad_industry = industry.notna() & ( + ~np.isfinite(industry) + | (industry % 1 != 0) + | ~industry.between(11, 99) + ) + if bad_industry.any(): + values = sorted(raw.loc[bad_industry, "naics_code"].unique())[:8] + raise ValueError(f"KFS naics_code contains invalid value(s) {values}.") + + weight = pd.to_numeric(raw["cswgt_final"], errors="coerce") + bad_weight = weight.isna() | ~np.isfinite(weight) | (weight < 0) + if bad_weight.any(): + raise ValueError( + "KFS cswgt_final contains missing, negative, or infinite weights." + ) + + count_frames = [ + _count_intervals(raw[source_name], column=output_name) + for source_name, output_name in ( + ("c5_num_employees", "employees"), + ("c6_num_ft_employees", "full_time_employees"), + ("c7_num_pt_employees", "part_time_employees"), + ) + ] + result = pd.concat( + [ + keys, + pd.DataFrame( + { + "status": status, + "industry_major": industry.astype("Int64"), + "cross_sectional_weight": weight.astype("float64"), + } + ), + *count_frames, + ], + axis=1, + ) + return result.reset_index(drop=True) + + +def kfs_profile(firms: pd.DataFrame) -> pd.DataFrame: + """Return descriptive coverage statistics without altering input.""" + required = { + "firm_id", + "year", + "status", + "employees_lower", + "employees_exact", + "cross_sectional_weight", + } + missing = sorted(required - set(firms)) + if missing: + raise ValueError(f"KFS profile input is missing columns {missing}.") + if firms.duplicated(["firm_id", "year"]).any(): + raise ValueError( + "KFS profile input contains duplicate firm-year rows." + ) + + rows = [] + for year, group in firms.groupby("year", sort=True): + available = group["employees_lower"].notna() + rows.append( + { + "year": int(year), + "firm_records": int(len(group)), + "complete_records": int(group["status"].eq("Complete").sum()), + "employee_count_available": int(available.sum()), + "zero_employee_records": int( + group["employees_exact"].eq(0).fillna(False).sum() + ), + "weighted_cohort_firms": float( + group["cross_sectional_weight"].sum() + ), + } + ) + return pd.DataFrame(rows) diff --git a/tests/README-tiers.md b/tests/README-tiers.md index 0d121c95..d2cda7cb 100644 --- a/tests/README-tiers.md +++ b/tests/README-tiers.md @@ -38,9 +38,9 @@ pytest --collect-only -q -m oracle_policyengine | tail -1 | Tier | Tests at HEAD | |---|---:| -| `unit` | 834 | +| `unit` | 852 | | `artifact` | 2,001 | | `integration_psid` | 812 | | `reproduction_legacy` | 520 | | `oracle_policyengine` | 159 | -| **Total** | **4,326** | +| **Total** | **4,344** | diff --git a/tests/data/test_kauffman_firms.py b/tests/data/test_kauffman_firms.py new file mode 100644 index 00000000..276f3df5 --- /dev/null +++ b/tests/data/test_kauffman_firms.py @@ -0,0 +1,182 @@ +"""Tests for the observed-firm Kauffman Firm Survey reader.""" + +from __future__ import annotations + +from pathlib import Path + +import numpy as np +import pandas as pd +import pytest + +from populace_dynamics.data import kauffman_firms + +REAL_KFS = Path( + "~/PolicyEngine/kfs-data/logically-imputed/Public_Use_LI_Long.dta" +).expanduser() +needs_real_kfs = pytest.mark.skipif( + not REAL_KFS.exists(), + reason="KFS public-use long file is not staged", +) + + +def _write_kfs(directory: Path, rows: list[dict]) -> Path: + directory.mkdir(parents=True, exist_ok=True) + defaults = { + "mprid": 10_000_016, + "year": 2004, + "status": "Complete", + "c5_num_employees": "04", + "c6_num_ft_employees": "03", + "c7_num_pt_employees": "01", + "naics_code": 54, + "cswgt_final": 2.5, + } + frame = pd.DataFrame([{**defaults, **row} for row in rows]) + path = directory / "Public_Use_LI_Long.dta" + frame.to_stata(path, write_index=False, version=118) + return path + + +def test_reads_firm_year_rows_and_count_intervals(tmp_path): + path = _write_kfs( + tmp_path, + [ + {}, + { + "mprid": 10_000_090, + "year": 2005, + "c5_num_employees": "26-60", + "c6_num_ft_employees": "25+", + "c7_num_pt_employees": ".a", + }, + ], + ) + firms = kauffman_firms.read_kauffman_firms(path=path) + + assert list(firms["firm_id"]) == ["10000016", "10000090"] + assert list(firms["year"]) == [2004, 2005] + assert firms.loc[0, "employees_exact"] == 4 + assert firms.loc[1, "employees_lower"] == 26 + assert firms.loc[1, "employees_upper"] == 60 + assert pd.isna(firms.loc[1, "full_time_employees_upper"]) + assert pd.isna(firms.loc[1, "part_time_employees_lower"]) + + +def test_resolves_staged_directory(tmp_path): + _write_kfs(tmp_path, [{}]) + firms = kauffman_firms.read_kauffman_firms(data_dir=tmp_path) + assert len(firms) == 1 + + +def test_resolves_environment_directory(tmp_path, monkeypatch): + _write_kfs(tmp_path, [{}]) + monkeypatch.setenv("POPULACE_DYNAMICS_KFS_DIR", str(tmp_path)) + firms = kauffman_firms.read_kauffman_firms() + assert len(firms) == 1 + + +def test_missing_staged_file_raises(tmp_path): + with pytest.raises(FileNotFoundError, match="outside git"): + kauffman_firms.read_kauffman_firms(data_dir=tmp_path) + + +def test_missing_required_column_raises(tmp_path): + path = _write_kfs(tmp_path, [{}]) + frame = pd.read_stata(path).drop(columns="status") + frame.to_stata(path, write_index=False, version=118) + with pytest.raises(ValueError, match="status"): + kauffman_firms.read_kauffman_firms(path=path) + + +@pytest.mark.parametrize("year", [2003, 2012, 2004.5]) +def test_unsupported_year_raises(tmp_path, year): + path = _write_kfs(tmp_path, [{"year": year}]) + with pytest.raises(ValueError, match="unsupported"): + kauffman_firms.read_kauffman_firms(path=path) + + +def test_duplicate_firm_year_raises(tmp_path): + path = _write_kfs(tmp_path, [{}, {}]) + with pytest.raises(ValueError, match="not unique"): + kauffman_firms.read_kauffman_firms(path=path) + + +@pytest.mark.parametrize("firm_id", ["", -1, 10.5]) +def test_invalid_firm_identifier_raises(tmp_path, firm_id): + path = _write_kfs(tmp_path, [{"mprid": firm_id}]) + with pytest.raises(ValueError, match="identifiers"): + kauffman_firms.read_kauffman_firms(path=path) + + +@pytest.mark.parametrize("weight", [-1, np.nan]) +def test_invalid_weight_raises(tmp_path, weight): + path = _write_kfs(tmp_path, [{"cswgt_final": weight}]) + with pytest.raises(ValueError, match="weights"): + kauffman_firms.read_kauffman_firms(path=path) + + +def test_unknown_employee_count_code_fails_closed(tmp_path): + path = _write_kfs(tmp_path, [{"c5_num_employees": "about five"}]) + with pytest.raises(ValueError, match="unsupported count"): + kauffman_firms.read_kauffman_firms(path=path) + + +def test_profile_reports_coverage_without_mutating_input(tmp_path): + path = _write_kfs( + tmp_path, + [ + {}, + { + "mprid": 10_000_090, + "c5_num_employees": "00", + "cswgt_final": 3.5, + }, + { + "mprid": 10_000_320, + "year": 2005, + "status": "Out of Business", + "c5_num_employees": ".a", + "cswgt_final": 1.0, + }, + ], + ) + firms = kauffman_firms.read_kauffman_firms(path=path) + before = firms.copy(deep=True) + profile = kauffman_firms.kfs_profile(firms) + + row_2004 = profile.loc[profile["year"] == 2004].iloc[0] + assert row_2004["firm_records"] == 2 + assert row_2004["complete_records"] == 2 + assert row_2004["employee_count_available"] == 2 + assert row_2004["zero_employee_records"] == 1 + assert row_2004["weighted_cohort_firms"] == 6.0 + pd.testing.assert_frame_equal(firms, before) + + +def test_profile_rejects_duplicate_keys(tmp_path): + firms = kauffman_firms.read_kauffman_firms(path=_write_kfs(tmp_path, [{}])) + duplicated = pd.concat([firms, firms], ignore_index=True) + with pytest.raises(ValueError, match="duplicate"): + kauffman_firms.kfs_profile(duplicated) + + +@needs_real_kfs +def test_staged_public_use_file_has_documented_panel_shape(): + firms = kauffman_firms.read_kauffman_firms(path=REAL_KFS) + assert len(firms) == 39_424 + assert firms["firm_id"].nunique() == 4_928 + assert set(firms["year"]) == set(kauffman_firms.KFS_YEARS) + assert not firms.duplicated(["firm_id", "year"]).any() + profile = kauffman_firms.kfs_profile(firms) + assert list(profile["complete_records"]) == [ + 4_928, + 3_998, + 3_390, + 2_915, + 2_606, + 2_408, + 2_126, + 2_007, + ] + assert int(firms["employees_lower"].notna().sum()) == 24_139 + assert int(firms["employees_exact"].notna().sum()) == 23_569 diff --git a/tests/estimates/test_birth_evidence_artifact.py b/tests/estimates/test_birth_evidence_artifact.py index eb328b0a..e8cb2e70 100644 --- a/tests/estimates/test_birth_evidence_artifact.py +++ b/tests/estimates/test_birth_evidence_artifact.py @@ -67,6 +67,7 @@ def test_reducer_input_identity_matches_reviewed_branch(): def test_context_report_sources_are_outside_historical_reducer_identity(): assert reducer.POST_REVIEW_SOURCE_EXCLUSIONS == ( Path("src/populace_dynamics/artifacts.py"), + Path("src/populace_dynamics/data/kauffman_firms.py"), Path("src/populace_dynamics/firms/targets.py"), Path("src/populace_dynamics/data/psid_covered_earnings_registry.py"), Path("src/populace_dynamics/data/psid_job_context.py"), diff --git a/tests/tier_counts.json b/tests/tier_counts.json index 37ae0b3b..344afead 100644 --- a/tests/tier_counts.json +++ b/tests/tier_counts.json @@ -1,7 +1,7 @@ { "schema_version": 1, "counts": { - "unit": 834, + "unit": 852, "artifact": 2001, "integration_psid": 812, "reproduction_legacy": 520,