diff --git a/.gitattributes b/.gitattributes index 1bac0365..bfade738 100644 --- a/.gitattributes +++ b/.gitattributes @@ -1,2 +1,3 @@ # Keep pinned source artifacts byte-for-byte identical to their publishers. db/data/dfe/funded_early_education_childcare_2026/headline_figures_feeac_2011_2026.csv -diff +db/data/dwp/uc_households_family_type_april_december_2025/uc_households_by_family_type_april_december_2023_2025.csv -diff diff --git a/chronicle/source_package.py b/chronicle/source_package.py index 5f6a5709..74ff5884 100644 --- a/chronicle/source_package.py +++ b/chronicle/source_package.py @@ -1765,6 +1765,10 @@ def _row_from_mapping(payload: dict[str, Any], *, year: int) -> SourceRecordSetR _constraint_from_mapping(constraint, year=year) for constraint in payload.get("constraints", ()) ), + source_row_dimensions={ + str(key): _render_value(value, year=year) + for key, value in payload.get("source_row_dimensions", {}).items() + }, value_scale=_render_value(payload.get("value_scale", 1), year=year), source_row_id=payload.get("source_row_id"), table_record_kind=payload.get("table_record_kind", "detail"), diff --git a/chronicle/sources/specs.py b/chronicle/sources/specs.py index f87cfa9e..6732bdc8 100644 --- a/chronicle/sources/specs.py +++ b/chronicle/sources/specs.py @@ -143,6 +143,7 @@ class SourceRecordSetRow: geography_vintage: str | None = None filters: dict[str, Scalar] = field(default_factory=dict) constraints: tuple[AggregateConstraint, ...] = () + source_row_dimensions: dict[str, Scalar] = field(default_factory=dict) value_scale: int | float = 1 source_row_id: str | None = None table_record_kind: str = "detail" @@ -818,6 +819,8 @@ def _record_set_spec_hash(spec: SourceRecordSetSpec) -> str: row.pop("expected_row_header", None) if row.get("expected_row_header_column") is None: row.pop("expected_row_header_column", None) + if not row.get("source_row_dimensions"): + row.pop("source_row_dimensions", None) if not row.get("guard_cells"): row.pop("guard_cells", None) else: diff --git a/chronicle/suite.py b/chronicle/suite.py index 157b7420..3482a785 100644 --- a/chronicle/suite.py +++ b/chronicle/suite.py @@ -395,6 +395,9 @@ def build_source_suite( source_column_dimensions_by_record_id=( _source_column_dimensions_by_record_id(source_record_set_specs) ), + source_row_dimensions_by_record_id=( + _source_row_dimensions_by_record_id(source_record_set_specs) + ), selected_only_source_parse=( bool(source_package) and source_package.artifact.parser == "delimited_text_selected_rows" @@ -581,6 +584,7 @@ def build_agent_acceptance_report( concept_alignments: ConceptAlignmentReport, require_axiom_validation: bool = False, source_column_dimensions_by_record_id: dict[str, dict[str, Any]] | None = None, + source_row_dimensions_by_record_id: dict[str, dict[str, Any]] | None = None, selected_only_source_parse: bool = False, ) -> AgentAcceptanceReport: """Build the stricter report agents should satisfy before review.""" @@ -599,6 +603,7 @@ def build_agent_acceptance_report( for cell in cells } source_column_dimensions_by_record_id = source_column_dimensions_by_record_id or {} + source_row_dimensions_by_record_id = source_row_dimensions_by_record_id or {} raw_r2_link_count = 0 if not cells and not rows: @@ -714,6 +719,12 @@ def build_agent_acceptance_report( {}, ) ), + source_row_dimensions=( + source_row_dimensions_by_record_id.get( + fact.source_record_id or "", + {}, + ) + ), ): row_semantic_error_count += 1 errors.append(issue) @@ -971,9 +982,11 @@ def _row_semantic_evidence_issues( cells: list[SourceCell], *, source_column_dimensions: dict[str, Any] | None = None, + source_row_dimensions: dict[str, Any] | None = None, ) -> list[AgentAcceptanceIssue]: issues: list[AgentAcceptanceIssue] = [] source_column_dimensions = source_column_dimensions or {} + source_row_dimensions = source_row_dimensions or {} fact_key = build_fact_key(fact) period_values = _source_row_values(rows, "period") for value in period_values: @@ -993,7 +1006,7 @@ def _row_semantic_evidence_issues( for variable, value in fact.filters.items(): if value is None: continue - if value == "all" and not source_column_dimensions: + if value == "all" and not (source_column_dimensions or source_row_dimensions): continue matched_values = _source_row_values(rows, variable) if not matched_values: @@ -1005,6 +1018,12 @@ def _row_semantic_evidence_issues( value, ): continue + if _declared_dimension_evidences( + source_row_dimensions, + variable, + value, + ): + continue issues.append( AgentAcceptanceIssue( code="row_filter_not_evidenced", @@ -1042,6 +1061,11 @@ def _row_semantic_evidence_issues( constraint, ): continue + if _declared_constraint_evidenced( + source_row_dimensions, + constraint, + ): + continue matched_values = _source_row_values(rows, constraint.variable) if not matched_values: issues.append( @@ -1079,20 +1103,43 @@ def _wide_table_filter_evidenced_by_source_column( expected: Any, ) -> bool: """Accept an explicitly declared dimension of a guarded source column.""" - if variable not in source_column_dimensions: + return _declared_dimension_evidences( + source_column_dimensions, + variable, + expected, + ) + + +def _declared_dimension_evidences( + dimensions: dict[str, Any], + variable: str, + expected: Any, +) -> bool: + """Accept an explicitly declared semantic dimension of a source axis.""" + if variable not in dimensions: return False - declared = source_column_dimensions[variable] + declared = dimensions[variable] return type(declared) is type(expected) and declared == expected def _wide_table_constraint_evidenced_by_source_column( source_column_dimensions: dict[str, Any], constraint: Any, +) -> bool: + return _declared_constraint_evidenced( + source_column_dimensions, + constraint, + ) + + +def _declared_constraint_evidenced( + dimensions: dict[str, Any], + constraint: Any, ) -> bool: if constraint.operator != "==": return False - return _wide_table_filter_evidenced_by_source_column( - source_column_dimensions, + return _declared_dimension_evidences( + dimensions, str(constraint.variable), constraint.value, ) @@ -1113,6 +1160,21 @@ def _source_column_dimensions_by_record_id( } +def _source_row_dimensions_by_record_id( + record_sets: list[SourceRecordSetSpec], +) -> dict[str, dict[str, Any]]: + """Index explicit row dimensions for row-header tables.""" + return { + f"{record_set.source_record_id_prefix}.{row.value_id}.{measure.measure_id}": ( + dict(row.source_row_dimensions) + ) + for record_set in record_sets + for row in record_set.rows + for measure in record_set.measures + if row.source_row_dimensions + } + + def _constraint_evidenced_by_source_cells( cells: list[SourceCell], constraint: Any, diff --git a/db/data/dwp/uc_households_family_type_april_december_2025/manifest.yaml b/db/data/dwp/uc_households_family_type_april_december_2025/manifest.yaml index de516312..82ffc25e 100644 --- a/db/data/dwp/uc_households_family_type_april_december_2025/manifest.yaml +++ b/db/data/dwp/uc_households_family_type_april_december_2025/manifest.yaml @@ -2,17 +2,18 @@ source_id: dwp package_id: dwp-uc-households-family-type-april-december-2025 dataset: dwp_dwp-uc-households-family-type-april-december-2025 source_page: https://stat-xplore.dwp.gov.uk/ -table: Households on Universal Credit by family type, April to December 2025 +table: Households on Universal Credit by family type, April to December 2023 through + 2025 files: 2025: - filename: uc_households_by_family_type_april_december_2025.json - source_url: https://stat-xplore.dwp.gov.uk/webapi/rest/v1/table - sha256: 44d67348da8c34465611ac00efdfcb80a69ad0f52e788d53f82517b34594377f - size_bytes: 6719 - fetched_at: '2026-08-21T11:45:49Z' + filename: uc_households_by_family_type_april_december_2023_2025.csv + source_url: https://stat-xplore.dwp.gov.uk/webapi/jsf/tableView/tableView.xhtml + sha256: 3783b64b9b0b5d35fb8ce7488f1e0238db89d67e6da96563140a151dfa4e5764 + size_bytes: 4343 + fetched_at: '2026-09-07T07:38:34Z' storage: r2: provider: r2 bucket: ledger-raw - key: raw/dwp/dwp-uc-households-family-type-april-december-2025/2025/44d67348da8c34465611ac00efdfcb80a69ad0f52e788d53f82517b34594377f/uc_households_by_family_type_april_december_2025.json - uri: r2://ledger-raw/raw/dwp/dwp-uc-households-family-type-april-december-2025/2025/44d67348da8c34465611ac00efdfcb80a69ad0f52e788d53f82517b34594377f/uc_households_by_family_type_april_december_2025.json + key: raw/uk/dwp/dwp-uc-households-family-type-april-december-2025/2025/3783b64b9b0b5d35fb8ce7488f1e0238db89d67e6da96563140a151dfa4e5764/uc_households_by_family_type_april_december_2023_2025.csv + uri: r2://ledger-raw/raw/uk/dwp/dwp-uc-households-family-type-april-december-2025/2025/3783b64b9b0b5d35fb8ce7488f1e0238db89d67e6da96563140a151dfa4e5764/uc_households_by_family_type_april_december_2023_2025.csv diff --git a/db/data/dwp/uc_households_family_type_april_december_2025/uc_households_by_family_type_april_december_2023_2025.csv b/db/data/dwp/uc_households_family_type_april_december_2025/uc_households_by_family_type_april_december_2023_2025.csv new file mode 100644 index 00000000..f97cc792 --- /dev/null +++ b/db/data/dwp/uc_households_family_type_april_december_2025/uc_households_by_family_type_april_december_2023_2025.csv @@ -0,0 +1,32 @@ +Stat-Xplore + +"Households on Universal Credit (I II III IV z)" +"Family Type by Month" +"Counting: Households on Universal Credit" + +Filters: +"Default Summation","Households on Universal Credit" + +"Month","April 2024","April 2024 - Annotations","May 2024","May 2024 - Annotations","June 2024 (r)","June 2024 - Annotations","July 2024 (r)","July 2024 - Annotations","August 2024 (r)","August 2024 - Annotations","September 2024 (r)","September 2024 - Annotations","October 2024 (r)","October 2024 - Annotations","November 2024 (r)","November 2024 - Annotations","December 2024 (r)","December 2024 - Annotations","April 2023","April 2023 - Annotations","May 2023","May 2023 - Annotations","June 2023","June 2023 - Annotations","July 2023","July 2023 - Annotations","August 2023","August 2023 - Annotations","September 2023","September 2023 - Annotations","October 2023","October 2023 - Annotations","November 2023","November 2023 - Annotations","December 2023","December 2023 - Annotations","April 2025 (r)","April 2025 - Annotations","May 2025 (r)","May 2025 - Annotations","June 2025 (r)","June 2025 - Annotations","July 2025 (r)","July 2025 - Annotations","August 2025 (r)","August 2025 - Annotations","September 2025 (r)","September 2025 - Annotations","October 2025 (r)","October 2025 - Annotations","November 2025 (r)","November 2025 - Annotations","December 2025 (r)","December 2025 - Annotations", +"Family Type", +"Single, no children",2706575,,2712106,,2723460,,2743844,,2766134,,2798252,,2823250,,2858415,,2896968,,2473370,,2481406,,2493624,,2515891,,2534581,,2556314,,2578790,,2604763,,2634407,,3134001,,3193287,,3282370,,3362245,,3460441,,3550437,,3623576,,3690997,,3725304,, +"Single, with children",1964494,,1998179,,2027919,,2066556,,2104721,,2125571,,2144071,,2170644,,2184171,,1733859,,1746189,,1759681,,1778847,,1795426,,1825682,,1856372,,1881018,,1904309,,2207559,,2212526,,2219462,,2225365,,2228790,,2231333,,2234296,,2237019,,2239630,, +"Couple, no children",217781,,219594,,219799,,221981,,225284,,231939,,233971,,237209,,240452,,198021,,198387,,199033,,199840,,200649,,201264,,202372,,203938,,205801,,260740,,266117,,274424,,280070,,286102,,291802,,296932,,300679,,302244,, +"Couple, with children",795356,,816067,,826955,,830896,,842927,,875196,,885411,,898910,,906064,,657801,,659255,,661324,,663134,,664168,,666793,,671892,,678520,,688185,,915707,,911319,,907142,,902225,,897867,,895566,,892226,,888199,,885561,, +"Unknown or missing family type",1787,,1795,,1612,,1618,,1530,,1282,,1253,,1227,,1241,,2322,,2387,,2367,,2412,,2541,,2796,,2928,,2973,,3034,,1193,,1193,,1217,,1259,,1223,,1176,,1197,,1261,,1301,, +"Total",5685995,,5747730,,5799748,,5864899,,5940590,,6032248,,6087962,,6166404,,6228895,,5065371,,5087622,,5116034,,5160124,,5197374,,5252845,,5312353,,5371220,,5435736,,6519195,,6584436,,6684615,,6771172,,6874424,,6970315,,7048227,,7118153,,7154045,, + + +"INFO","Statistical disclosure control has been applied to this table to avoid the release of confidential data. Totals may not sum due to the disclosure control applied." + + +Symbol,Description +"I","A limited test of the full Universal Credit service was launched in Sutton, South London on 26th November 2014, and has expanded to other areas since. This publication now includes statistics covering both the live and full Universal Credit service. Please see https://www.gov.uk/government/publications/universal-credit-transition-to-full-service for details of the areas in which the full Universal Credit service is operating." +"II","Data for more recent months will be subject to a higher degree of revision. Universal Credit award amounts may be retrospectively revised." +"III","Monthly award amounts include any awards due to entitlement such as the standard allowance or housing entitlement plus any advance payments. Advance payments will normally be recovered during subsequent assessment periods." +"IV","Award, entitlement and payment information may be missing for a very small number of households on Universal Credit, where more limited information is entered onto IT systems." +"z",""".."" denotes a nil or negligible number of claimants or award amount based on a nil or negligible number of claimants." +"r","Figures marked ""r"" have been revised since the previous release." + + +Source: Department for Work and Pensions diff --git a/db/data/dwp/uc_households_family_type_april_december_2025/uc_households_by_family_type_april_december_2025.json b/db/data/dwp/uc_households_family_type_april_december_2025/uc_households_by_family_type_april_december_2025.json deleted file mode 100644 index 4046bcbf..00000000 --- a/db/data/dwp/uc_households_family_type_april_december_2025/uc_households_by_family_type_april_december_2025.json +++ /dev/null @@ -1,117 +0,0 @@ -{ - "query" : { - "database" : "str:database:UC_Households", - "measures" : [ "str:count:UC_Households:V_F_UC_HOUSEHOLDS" ], - "recodes" : { - "str:field:UC_Households:F_UC_HH_DATE:DATE_NAME" : { - "map" : [ [ "str:value:UC_Households:F_UC_HH_DATE:DATE_NAME:C_UC_HH_DATE:202504" ], [ "str:value:UC_Households:F_UC_HH_DATE:DATE_NAME:C_UC_HH_DATE:202505" ], [ "str:value:UC_Households:F_UC_HH_DATE:DATE_NAME:C_UC_HH_DATE:202506" ], [ "str:value:UC_Households:F_UC_HH_DATE:DATE_NAME:C_UC_HH_DATE:202507" ], [ "str:value:UC_Households:F_UC_HH_DATE:DATE_NAME:C_UC_HH_DATE:202508" ], [ "str:value:UC_Households:F_UC_HH_DATE:DATE_NAME:C_UC_HH_DATE:202509" ], [ "str:value:UC_Households:F_UC_HH_DATE:DATE_NAME:C_UC_HH_DATE:202510" ], [ "str:value:UC_Households:F_UC_HH_DATE:DATE_NAME:C_UC_HH_DATE:202511" ], [ "str:value:UC_Households:F_UC_HH_DATE:DATE_NAME:C_UC_HH_DATE:202512" ] ], - "total" : false - }, - "str:field:UC_Households:V_F_UC_HOUSEHOLDS:hnfamily_type" : { - "map" : [ [ "str:value:UC_Households:V_F_UC_HOUSEHOLDS:hnfamily_type:C_UC_FAMILY_TYPE:1" ], [ "str:value:UC_Households:V_F_UC_HOUSEHOLDS:hnfamily_type:C_UC_FAMILY_TYPE:2" ], [ "str:value:UC_Households:V_F_UC_HOUSEHOLDS:hnfamily_type:C_UC_FAMILY_TYPE:3" ], [ "str:value:UC_Households:V_F_UC_HOUSEHOLDS:hnfamily_type:C_UC_FAMILY_TYPE:4" ], [ "str:value:UC_Households:V_F_UC_HOUSEHOLDS:hnfamily_type:C_UC_FAMILY_TYPE:99" ] ], - "total" : false - } - }, - "dimensions" : [ [ "str:field:UC_Households:F_UC_HH_DATE:DATE_NAME" ], [ "str:field:UC_Households:V_F_UC_HOUSEHOLDS:hnfamily_type" ] ] - }, - "database" : { - "id" : "UC_Households", - "uri" : "str:database:UC_Households", - "label" : "Households on Universal Credit", - "annotationKeys" : [ "I", "II", "III", "IV", "z" ] - }, - "measures" : [ { - "uri" : "str:count:UC_Households:V_F_UC_HOUSEHOLDS", - "label" : "Households on Universal Credit", - "measure" : "str:count:UC_Households:V_F_UC_HOUSEHOLDS", - "function" : "COUNT" - } ], - "fields" : [ { - "uri" : "str:field:UC_Households:F_UC_HH_DATE:DATE_NAME", - "label" : "Month", - "items" : [ { - "type" : "RecodeItem", - "uris" : [ "str:value:UC_Households:F_UC_HH_DATE:DATE_NAME:C_UC_HH_DATE:202504" ], - "labels" : [ "April 2025" ], - "annotationKeys" : [ "r" ] - }, { - "type" : "RecodeItem", - "uris" : [ "str:value:UC_Households:F_UC_HH_DATE:DATE_NAME:C_UC_HH_DATE:202505" ], - "labels" : [ "May 2025" ], - "annotationKeys" : [ "r" ] - }, { - "type" : "RecodeItem", - "uris" : [ "str:value:UC_Households:F_UC_HH_DATE:DATE_NAME:C_UC_HH_DATE:202506" ], - "labels" : [ "June 2025" ], - "annotationKeys" : [ "r" ] - }, { - "type" : "RecodeItem", - "uris" : [ "str:value:UC_Households:F_UC_HH_DATE:DATE_NAME:C_UC_HH_DATE:202507" ], - "labels" : [ "July 2025" ], - "annotationKeys" : [ "r" ] - }, { - "type" : "RecodeItem", - "uris" : [ "str:value:UC_Households:F_UC_HH_DATE:DATE_NAME:C_UC_HH_DATE:202508" ], - "labels" : [ "August 2025" ], - "annotationKeys" : [ "r" ] - }, { - "type" : "RecodeItem", - "uris" : [ "str:value:UC_Households:F_UC_HH_DATE:DATE_NAME:C_UC_HH_DATE:202509" ], - "labels" : [ "September 2025" ], - "annotationKeys" : [ "r" ] - }, { - "type" : "RecodeItem", - "uris" : [ "str:value:UC_Households:F_UC_HH_DATE:DATE_NAME:C_UC_HH_DATE:202510" ], - "labels" : [ "October 2025" ], - "annotationKeys" : [ "r" ] - }, { - "type" : "RecodeItem", - "uris" : [ "str:value:UC_Households:F_UC_HH_DATE:DATE_NAME:C_UC_HH_DATE:202511" ], - "labels" : [ "November 2025" ], - "annotationKeys" : [ "r" ] - }, { - "type" : "RecodeItem", - "uris" : [ "str:value:UC_Households:F_UC_HH_DATE:DATE_NAME:C_UC_HH_DATE:202512" ], - "labels" : [ "December 2025" ], - "annotationKeys" : [ "r" ] - } ] - }, { - "uri" : "str:field:UC_Households:V_F_UC_HOUSEHOLDS:hnfamily_type", - "label" : "Family Type", - "items" : [ { - "type" : "RecodeItem", - "uris" : [ "str:value:UC_Households:V_F_UC_HOUSEHOLDS:hnfamily_type:C_UC_FAMILY_TYPE:1" ], - "labels" : [ "Single, no children" ] - }, { - "type" : "RecodeItem", - "uris" : [ "str:value:UC_Households:V_F_UC_HOUSEHOLDS:hnfamily_type:C_UC_FAMILY_TYPE:2" ], - "labels" : [ "Single, with children" ] - }, { - "type" : "RecodeItem", - "uris" : [ "str:value:UC_Households:V_F_UC_HOUSEHOLDS:hnfamily_type:C_UC_FAMILY_TYPE:3" ], - "labels" : [ "Couple, no children" ] - }, { - "type" : "RecodeItem", - "uris" : [ "str:value:UC_Households:V_F_UC_HOUSEHOLDS:hnfamily_type:C_UC_FAMILY_TYPE:4" ], - "labels" : [ "Couple, with children" ] - }, { - "type" : "RecodeItem", - "uris" : [ "str:value:UC_Households:V_F_UC_HOUSEHOLDS:hnfamily_type:C_UC_FAMILY_TYPE:99" ], - "labels" : [ "Unknown or missing family type" ] - } ] - } ], - "cubes" : { - "str:count:UC_Households:V_F_UC_HOUSEHOLDS" : { - "values" : [ [ 3134001, 2207559, 260740, 915707, 1193 ], [ 3193287, 2212526, 266117, 911319, 1193 ], [ 3282370, 2219462, 274424, 907142, 1217 ], [ 3362245, 2225365, 280070, 902225, 1259 ], [ 3460441, 2228790, 286102, 897867, 1223 ], [ 3550437, 2231333, 291802, 895566, 1176 ], [ 3623576, 2234296, 296932, 892226, 1197 ], [ 3690997, 2237019, 300679, 888199, 1261 ], [ 3725304, 2239630, 302244, 885561, 1301 ] ], - "precision" : 0 - } - }, - "annotationMap" : { - "I" : "A limited test of the full Universal Credit service was launched in Sutton, South London on 26th November 2014, and has expanded to other areas since. This publication now includes statistics covering both the live and full Universal Credit service. Please see https://www.gov.uk/government/publications/universal-credit-transition-to-full-service for details of the areas in which the full Universal Credit service is operating.", - "II" : "Data for more recent months will be subject to a higher degree of revision. Universal Credit award amounts may be retrospectively revised.", - "III" : "Monthly award amounts include any awards due to entitlement such as the standard allowance or housing entitlement plus any advance payments. Advance payments will normally be recovered during subsequent assessment periods.", - "IV" : "Award, entitlement and payment information may be missing for a very small number of households on Universal Credit, where more limited information is entered onto IT systems.", - "z" : "\"..\" denotes a nil or negligible number of claimants or award amount based on a nil or negligible number of claimants.", - "r" : "Figures marked \"r\" have been revised since the previous release." - } -} \ No newline at end of file diff --git a/docs/pe-uk-source-checklist.md b/docs/pe-uk-source-checklist.md index 3a3f3bd0..2b4504ac 100644 --- a/docs/pe-uk-source-checklist.md +++ b/docs/pe-uk-source-checklist.md @@ -66,7 +66,7 @@ cannot be obtained. Nothing in uk-data is absent from this table. | Public sector employment ([`ons_public_sector_employment.py`](https://github.com/PolicyEngine/policyengine-uk-data/blob/ebf733c/policyengine_uk_data/targets/sources/ons_public_sector_employment.py)) | 1 `ons/public_sector_employment` (5.90m 2023 / 5.94m 2024) | **ported** | `ons-public-sector-employment-2026` (12 facts, Table 1 headcount by sector, Dec-2023 + Dec-2024) | [#141](https://github.com/PolicyEngine/chronicle/pull/141) | **Re-pinned to dataset — vintage revised by ONS.** uk-data hand-transcribes headline headcounts from the [PSE bulletin `/latest`](https://www.ons.gov.uk/employmentandlabourmarket/peopleinwork/publicsectorpersonnel/bulletins/publicsectoremployment/latest) (no fixed vintage); Chronicle pins the March-2026 [reference table](https://www.ons.gov.uk/employmentandlabourmarket/peopleinwork/publicsectorpersonnel/datasets/publicsectoremploymentreferencetable), where reclassifications have revised the series to 6,089k (Dec 2023) / 6,146k (Dec 2024) total public sector. The pre-revision vintage stays portable via the backfill lane. | | UC payment distribution ([`dwp.py`](https://github.com/PolicyEngine/policyengine-uk-data/blob/ebf733c/policyengine_uk_data/targets/sources/dwp.py) → `utils/uc_data.py`) | 104 `dwp/uc_payment_dist/*` | **ported** | `dwp-uc-payment-distribution-april-december-2025` (1,215 facts, 27 bands × 5 family types × 9 months, Stat-Xplore; the `£1500.01 or over` band applies to months up to August 2022 only and is not ported; the May-2025 snapshot package is retired) | [#141](https://github.com/PolicyEngine/chronicle/pull/141), [#234](https://github.com/PolicyEngine/chronicle/issues/234) | Monthly history covers the April–December 2025 caseload averaging window, including the `No payment` (nil-award) band per family type. uk-data parses a committed provisional export with no API provenance; its open `£2500.01 or over` band parses to NaN (4 impossible `nan_to_nan` targets) — here it is an ordinary category. Band identities are exact publisher labels; the ×12 annualization is populace-side. | | UC claimants by children ([`dwp.py`](https://github.com/PolicyEngine/policyengine-uk-data/blob/ebf733c/policyengine_uk_data/targets/sources/dwp.py)) | 5 `dwp/uc/claimants_with_{n}_children` | **ported** | `dwp-uc-households-children-april-december-2025` (72 facts, Stat-Xplore) | [#141](https://github.com/PolicyEngine/chronicle/pull/141), [#187](https://github.com/PolicyEngine/chronicle/issues/187) | Monthly history now covers the April–December 2025 caseload averaging window. Digit categories carry integer `number_of_children` constraints; `5 or more` and the unknown tail stay categorical (uk-data merged the tail into its `5+`). Raw revised publisher cells are preserved. | -| UC claimants by family type ([`dwp.py`](https://github.com/PolicyEngine/policyengine-uk-data/blob/ebf733c/policyengine_uk_data/targets/sources/dwp.py)) | 4 `dwp/uc/claimants_{family_type}` | **ported** | `dwp-uc-households-family-type-april-december-2025` (45 facts, Stat-Xplore); [#233](https://github.com/PolicyEngine/chronicle/issues/233) crosses: `dwp-uc-households-family-type-child-entitlement-april-december-2025` (90 facts, family type × child element in the award), `dwp-uc-households-children-child-entitlement-april-december-2025` (144 facts, number of children × child element), `dwp-uc-households-family-type-payment-indicator-april-december-2025` (90 facts, family type × in payment / nil award) | [#141](https://github.com/PolicyEngine/chronicle/pull/141), [#187](https://github.com/PolicyEngine/chronicle/issues/187), [#233](https://github.com/PolicyEngine/chronicle/issues/233) | Monthly history now covers the April–December 2025 caseload averaging window. DWP's family-type 'with children' counts any verified child or young person under 20 in the household; the Child Entitlement cross carries the child-element concept and the Payment Indicator cross the nil-award split on the same months, both as published. Raw publisher counts only — uk-data's `undercount_relative` inflation is **excluded (computed)** and stays populace-side. | +| UC claimants by family type ([`dwp.py`](https://github.com/PolicyEngine/policyengine-uk-data/blob/ebf733c/policyengine_uk_data/targets/sources/dwp.py)) | 4 `dwp/uc/claimants_{family_type}` | **ported** | `dwp-uc-households-family-type-april-december-2025` (162 facts, Stat-Xplore: five family types plus the separately published Total for April–December 2023, 2024, and 2025); [#233](https://github.com/PolicyEngine/chronicle/issues/233) crosses: `dwp-uc-households-family-type-child-entitlement-april-december-2025` (90 facts, family type × child element in the award), `dwp-uc-households-children-child-entitlement-april-december-2025` (144 facts, number of children × child element), `dwp-uc-households-family-type-payment-indicator-april-december-2025` (90 facts, family type × in payment / nil award) | [#141](https://github.com/PolicyEngine/chronicle/pull/141), [#187](https://github.com/PolicyEngine/chronicle/issues/187), [#233](https://github.com/PolicyEngine/chronicle/issues/233), [#247](https://github.com/PolicyEngine/chronicle/issues/247) | Family-type history covers the April–December windows from 2023 onward through the latest complete window, with the DWP Total kept as `total_benefit_units` rather than derived from detail cells. DWP's family-type 'with children' counts any verified child or young person under 20 in the household; the Child Entitlement cross carries the child-element concept and the Payment Indicator cross the nil-award split on the 2025 months, both as published. Raw publisher counts only — uk-data's `undercount_relative` inflation is **excluded (computed)** and stays populace-side. | | UC element caseloads | Childcare, housing, LCWRA and carer benefit units | **ported** | `dwp-uc-childcare-element-march-2021-august-2025` (54 facts) plus three April–December 2025 Stat-Xplore entitlement packages (9 facts each) | [#187](https://github.com/PolicyEngine/chronicle/issues/187) | DWP's “household” is normalized to the UC benefit unit. Housing is a publisher-side grouped API cell across social, private and other tenure categories; no Chronicle-side summation or annual averaging is introduced. Coverage is Great Britain (`K03000001`), matching the source. | | Childcare programme anchors | Tax-Free Childcare annual top-up and annual/monthly child activity; funded early-education registered and eligible child measures | **ported; downstream activity and mutually exclusive target arithmetic excluded (computed)** | `hmrc-tax-free-childcare-march-2026` (126 facts: 108 monthly used-account child counts, 9 annual-unique child counts, 9 annual top-up flows); `dfe-funded-early-education-childcare-2026` (770 national child headline facts, 2011–2026) | [#219](https://github.com/PolicyEngine/chronicle/issues/219) | **Publisher cells only.** HMRC's April-2025 used-account definition break is an exact filter and constraint on every monthly fact; consumers combining months must select one `definition_regime`. Annual figures use the revised treatment from FY2024-25. DfE facts are third-week-of-January England stocks: raw entitlement and age labels are row constraints, while registration/eligibility and nursery/reception identities use explicit guarded source-column dimension declarations rather than column-name inference. Total and component columns have distinct concepts, and the release vintage is pinned in provenance. The TFC duration proxy, `379,029 + 242,453`, and `778,327 - 361,790` remain consumer computations; overlapping universal and working-parent universes are not reconciled. | | Scotland UC households, child under 1 ([`dwp.py`](https://github.com/PolicyEngine/policyengine-uk-data/blob/ebf733c/policyengine_uk_data/targets/sources/dwp.py)) | 1 `dwp/scotland_uc_households_child_under_1` (14,000) | **ported** | `dwp-uc-scotland-youngest-child-april-december-2025` (207 facts, single year of age × 9 months, Stat-Xplore; the May-2025 snapshot package is retired) | [#141](https://github.com/PolicyEngine/chronicle/pull/141), [#234](https://github.com/PolicyEngine/chronicle/issues/234) | The age-0 row (14,412 in the retired May snapshot; 14,448 in the revised May 2025 cell of the history) is the precise value behind uk-data's rounded 14,000. Single-year cells differ from the published 0–4 band by 4 households — Stat-Xplore disclosure-control rounding, both publisher-emitted. | diff --git a/packages/dwp/uc_households_family_type_april_december_2025/source_package.yaml b/packages/dwp/uc_households_family_type_april_december_2025/source_package.yaml index 4c9c9199..fa6d82dc 100644 --- a/packages/dwp/uc_households_family_type_april_december_2025/source_package.yaml +++ b/packages/dwp/uc_households_family_type_april_december_2025/source_package.yaml @@ -1,31 +1,47 @@ # DWP's "household" is the Universal Credit unit of assessment (benefit unit), -# not an ONS household. This package preserves each monthly, publisher-emitted -# cube cell; DWP's source adjustment means independently queried cubes need not -# reconcile exactly. +# not an ONS household. The five family-type cells and Total below are separate +# publisher-emitted facts. DWP applies statistical disclosure control, so the +# published Total may differ slightly from the sum of the five detail cells. schema_version: ledger.source_package.v1 package_id: dwp-uc-households-family-type-april-december-2025 -label: 'DWP Stat-Xplore: Universal Credit benefit units by family type, Great Britain, April to December 2025' +label: 'DWP Stat-Xplore: Universal Credit benefit units by family type and published total, Great Britain, April to December 2023 through 2025' artifact: source_name: dwp - source_table: Households on Universal Credit by family type, April to December 2025 + source_table: Households on Universal Credit by family type, April to December 2023 through 2025 resource_package: db resource_directory: data/dwp/uc_households_family_type_april_december_2025 manifest: manifest.yaml - vintage: stat_xplore_2026_08_21 - extracted_at: '2026-08-21' - extraction_method: Stat-Xplore Open Data API table response parsed and unpivoted to one row per cube cell - parser: statxplore_table_json_rows - sheet_name: api_response + vintage: stat_xplore_2026_09_07 + extracted_at: '2026-09-07' + extraction_method: >- + Stat-Xplore publisher CSV download from table path "Households on Universal + Credit / Family Type by Month" in database URI + str:database:UC_Households, using measure URI + str:count:UC_Households:V_F_UC_HOUSEHOLDS and dimension field URIs + str:field:UC_Households:F_UC_HH_DATE:DATE_NAME and + str:field:UC_Households:V_F_UC_HOUSEHOLDS:hnfamily_type; preserves five + family-type rows, the publisher-emitted Total row, revision annotations, + and the disclosure-control notice + parser: delimited_text_full_rows + header_row: 10 + selected_rows: + - {Month: 'Single, no children'} + - {Month: 'Single, with children'} + - {Month: 'Couple, no children'} + - {Month: 'Couple, with children'} + - {Month: 'Unknown or missing family type'} + - {Month: Total} + sheet_name: stat_xplore_family_type artifact_year: 2025 record_sets: -- &monthly - record_set_id: dwp.statx.uc_benefit_units.family_type.month2025_04 +- &detail + record_set_id: dwp.statx.uc_benefit_units.family_type.month2023_04 provenance_class: administrative - record_set_spec_id: dwp.statx.uc_benefit_units.family_type.month2025_04.v1 - source_record_id_prefix: dwp.statx.uc_benefit_units.family_type.month2025_04 - sheet_name: api_response + record_set_spec_id: dwp.statx.uc_benefit_units.family_type.month2023_04.v1 + source_record_id_prefix: dwp.statx.uc_benefit_units.family_type.month2023_04 + sheet_name: stat_xplore_family_type period_type: month - period: 2025-04 + period: 2023-04 geography_id: K03000001 geography_level: country geography_name: Great Britain @@ -41,69 +57,70 @@ record_sets: label: Single, no children ordinal: 0 row_number: 2 - expected_row_header_column: E + expected_row_header_column: A expected_row_header: Single, no children table_record_kind: detail filters: {family_type: 'Single, no children'} + source_row_dimensions: {family_type: 'Single, no children'} constraints: - {variable: family_type, operator: '==', value: 'Single, no children', label: family type} - guard_cells: &april_guard [{column: C, row: 2, expected_value: 'April 2025', label: month}] - &single_with_children value_id: single_with_children label: Single, with children ordinal: 1 row_number: 3 - expected_row_header_column: E + expected_row_header_column: A expected_row_header: Single, with children table_record_kind: detail filters: {family_type: 'Single, with children'} + source_row_dimensions: {family_type: 'Single, with children'} constraints: - {variable: family_type, operator: '==', value: 'Single, with children', label: family type} - guard_cells: [{column: C, row: 3, expected_value: 'April 2025', label: month}] - &couple_no_children value_id: couple_no_children label: Couple, no children ordinal: 2 row_number: 4 - expected_row_header_column: E + expected_row_header_column: A expected_row_header: Couple, no children table_record_kind: detail filters: {family_type: 'Couple, no children'} + source_row_dimensions: {family_type: 'Couple, no children'} constraints: - {variable: family_type, operator: '==', value: 'Couple, no children', label: family type} - guard_cells: [{column: C, row: 4, expected_value: 'April 2025', label: month}] - &couple_with_children value_id: couple_with_children label: Couple, with children ordinal: 3 row_number: 5 - expected_row_header_column: E + expected_row_header_column: A expected_row_header: Couple, with children table_record_kind: detail filters: {family_type: 'Couple, with children'} + source_row_dimensions: {family_type: 'Couple, with children'} constraints: - {variable: family_type, operator: '==', value: 'Couple, with children', label: family type} - guard_cells: [{column: C, row: 5, expected_value: 'April 2025', label: month}] - &unknown_family value_id: unknown_or_missing_family_type label: Unknown or missing family type ordinal: 4 row_number: 6 - expected_row_header_column: E + expected_row_header_column: A expected_row_header: Unknown or missing family type table_record_kind: detail filters: {family_type: 'Unknown or missing family type'} + source_row_dimensions: {family_type: 'Unknown or missing family type'} constraints: - {variable: family_type, operator: '==', value: 'Unknown or missing family type', label: family type} - guard_cells: [{column: C, row: 6, expected_value: 'April 2025', label: month}] - measures: &measures - - measure_id: benefit_units + measures: + - &benefit_units + measure_id: benefit_units label: Universal Credit benefit units ordinal: 0 - column: B - source_column_id: value + column: T + source_column_id: april_2023 expected_column_header_row: 1 - expected_column_header: value + expected_column_header: April 2023 concept: dwp.uc_benefit_units source_concept: dwp.uc_households concept_relation: source_label @@ -113,99 +130,418 @@ record_sets: unit: count aggregation: sum expected_cell_type: number -- <<: *monthly - record_set_id: dwp.statx.uc_benefit_units.family_type.month2025_05 +- &total + record_set_id: dwp.statx.uc_benefit_units.family_type.month2023_04.total + provenance_class: administrative + record_set_spec_id: dwp.statx.uc_benefit_units.family_type.month2023_04.total.v1 + source_record_id_prefix: dwp.statx.uc_benefit_units.family_type.month2023_04.total + sheet_name: stat_xplore_family_type + period_type: month + period: 2023-04 + geography_id: K03000001 + geography_level: country + geography_name: Great Britain + geography_vintage: current + entity: benefit_unit + entity_role: universal_credit_unit_of_assessment + domain: universal_credit + groupby_dimension: dwp.uc_family_type + assertion: observation + rows: + - &all_family_types + value_id: all_family_types + label: Total + ordinal: 0 + row_number: 7 + expected_row_header_column: A + expected_row_header: Total + table_record_kind: total + filters: {family_type: all} + source_row_dimensions: {family_type: all} + constraints: + - {variable: family_type, operator: '==', value: all, label: family type} + measures: + - &total_benefit_units + measure_id: total_benefit_units + label: Universal Credit benefit units, published total + ordinal: 0 + column: T + source_column_id: april_2023 + expected_column_header_row: 1 + expected_column_header: April 2023 + concept: dwp.uc_benefit_units + source_concept: dwp.uc_households + concept_relation: source_label + concept_authority: dwp + concept_evidence_url: https://stat-xplore.dwp.gov.uk/ + concept_evidence_notes: DWP publishes this all-family-types Total directly; Chronicle does not derive it from the five displayed family-type cells. + unit: count + aggregation: sum + expected_cell_type: number + +- <<: *detail + provenance_class: administrative + record_set_id: dwp.statx.uc_benefit_units.family_type.month2023_05 + record_set_spec_id: dwp.statx.uc_benefit_units.family_type.month2023_05.v1 + source_record_id_prefix: dwp.statx.uc_benefit_units.family_type.month2023_05 + period: 2023-05 + measures: [{<<: *benefit_units, column: V, source_column_id: may_2023, expected_column_header: May 2023}] +- <<: *total + provenance_class: administrative + record_set_id: dwp.statx.uc_benefit_units.family_type.month2023_05.total + record_set_spec_id: dwp.statx.uc_benefit_units.family_type.month2023_05.total.v1 + source_record_id_prefix: dwp.statx.uc_benefit_units.family_type.month2023_05.total + period: 2023-05 + measures: [{<<: *total_benefit_units, column: V, source_column_id: may_2023, expected_column_header: May 2023}] +- <<: *detail + provenance_class: administrative + record_set_id: dwp.statx.uc_benefit_units.family_type.month2023_06 + record_set_spec_id: dwp.statx.uc_benefit_units.family_type.month2023_06.v1 + source_record_id_prefix: dwp.statx.uc_benefit_units.family_type.month2023_06 + period: 2023-06 + measures: [{<<: *benefit_units, column: X, source_column_id: june_2023, expected_column_header: June 2023}] +- <<: *total + provenance_class: administrative + record_set_id: dwp.statx.uc_benefit_units.family_type.month2023_06.total + record_set_spec_id: dwp.statx.uc_benefit_units.family_type.month2023_06.total.v1 + source_record_id_prefix: dwp.statx.uc_benefit_units.family_type.month2023_06.total + period: 2023-06 + measures: [{<<: *total_benefit_units, column: X, source_column_id: june_2023, expected_column_header: June 2023}] +- <<: *detail + provenance_class: administrative + record_set_id: dwp.statx.uc_benefit_units.family_type.month2023_07 + record_set_spec_id: dwp.statx.uc_benefit_units.family_type.month2023_07.v1 + source_record_id_prefix: dwp.statx.uc_benefit_units.family_type.month2023_07 + period: 2023-07 + measures: [{<<: *benefit_units, column: Z, source_column_id: july_2023, expected_column_header: July 2023}] +- <<: *total + provenance_class: administrative + record_set_id: dwp.statx.uc_benefit_units.family_type.month2023_07.total + record_set_spec_id: dwp.statx.uc_benefit_units.family_type.month2023_07.total.v1 + source_record_id_prefix: dwp.statx.uc_benefit_units.family_type.month2023_07.total + period: 2023-07 + measures: [{<<: *total_benefit_units, column: Z, source_column_id: july_2023, expected_column_header: July 2023}] +- <<: *detail + provenance_class: administrative + record_set_id: dwp.statx.uc_benefit_units.family_type.month2023_08 + record_set_spec_id: dwp.statx.uc_benefit_units.family_type.month2023_08.v1 + source_record_id_prefix: dwp.statx.uc_benefit_units.family_type.month2023_08 + period: 2023-08 + measures: [{<<: *benefit_units, column: AB, source_column_id: august_2023, expected_column_header: August 2023}] +- <<: *total + provenance_class: administrative + record_set_id: dwp.statx.uc_benefit_units.family_type.month2023_08.total + record_set_spec_id: dwp.statx.uc_benefit_units.family_type.month2023_08.total.v1 + source_record_id_prefix: dwp.statx.uc_benefit_units.family_type.month2023_08.total + period: 2023-08 + measures: [{<<: *total_benefit_units, column: AB, source_column_id: august_2023, expected_column_header: August 2023}] +- <<: *detail + provenance_class: administrative + record_set_id: dwp.statx.uc_benefit_units.family_type.month2023_09 + record_set_spec_id: dwp.statx.uc_benefit_units.family_type.month2023_09.v1 + source_record_id_prefix: dwp.statx.uc_benefit_units.family_type.month2023_09 + period: 2023-09 + measures: [{<<: *benefit_units, column: AD, source_column_id: september_2023, expected_column_header: September 2023}] +- <<: *total + provenance_class: administrative + record_set_id: dwp.statx.uc_benefit_units.family_type.month2023_09.total + record_set_spec_id: dwp.statx.uc_benefit_units.family_type.month2023_09.total.v1 + source_record_id_prefix: dwp.statx.uc_benefit_units.family_type.month2023_09.total + period: 2023-09 + measures: [{<<: *total_benefit_units, column: AD, source_column_id: september_2023, expected_column_header: September 2023}] +- <<: *detail + provenance_class: administrative + record_set_id: dwp.statx.uc_benefit_units.family_type.month2023_10 + record_set_spec_id: dwp.statx.uc_benefit_units.family_type.month2023_10.v1 + source_record_id_prefix: dwp.statx.uc_benefit_units.family_type.month2023_10 + period: 2023-10 + measures: [{<<: *benefit_units, column: AF, source_column_id: october_2023, expected_column_header: October 2023}] +- <<: *total + provenance_class: administrative + record_set_id: dwp.statx.uc_benefit_units.family_type.month2023_10.total + record_set_spec_id: dwp.statx.uc_benefit_units.family_type.month2023_10.total.v1 + source_record_id_prefix: dwp.statx.uc_benefit_units.family_type.month2023_10.total + period: 2023-10 + measures: [{<<: *total_benefit_units, column: AF, source_column_id: october_2023, expected_column_header: October 2023}] +- <<: *detail + provenance_class: administrative + record_set_id: dwp.statx.uc_benefit_units.family_type.month2023_11 + record_set_spec_id: dwp.statx.uc_benefit_units.family_type.month2023_11.v1 + source_record_id_prefix: dwp.statx.uc_benefit_units.family_type.month2023_11 + period: 2023-11 + measures: [{<<: *benefit_units, column: AH, source_column_id: november_2023, expected_column_header: November 2023}] +- <<: *total + provenance_class: administrative + record_set_id: dwp.statx.uc_benefit_units.family_type.month2023_11.total + record_set_spec_id: dwp.statx.uc_benefit_units.family_type.month2023_11.total.v1 + source_record_id_prefix: dwp.statx.uc_benefit_units.family_type.month2023_11.total + period: 2023-11 + measures: [{<<: *total_benefit_units, column: AH, source_column_id: november_2023, expected_column_header: November 2023}] +- <<: *detail + provenance_class: administrative + record_set_id: dwp.statx.uc_benefit_units.family_type.month2023_12 + record_set_spec_id: dwp.statx.uc_benefit_units.family_type.month2023_12.v1 + source_record_id_prefix: dwp.statx.uc_benefit_units.family_type.month2023_12 + period: 2023-12 + measures: [{<<: *benefit_units, column: AJ, source_column_id: december_2023, expected_column_header: December 2023}] +- <<: *total + provenance_class: administrative + record_set_id: dwp.statx.uc_benefit_units.family_type.month2023_12.total + record_set_spec_id: dwp.statx.uc_benefit_units.family_type.month2023_12.total.v1 + source_record_id_prefix: dwp.statx.uc_benefit_units.family_type.month2023_12.total + period: 2023-12 + measures: [{<<: *total_benefit_units, column: AJ, source_column_id: december_2023, expected_column_header: December 2023}] + +- <<: *detail + provenance_class: administrative + record_set_id: dwp.statx.uc_benefit_units.family_type.month2024_04 + record_set_spec_id: dwp.statx.uc_benefit_units.family_type.month2024_04.v1 + source_record_id_prefix: dwp.statx.uc_benefit_units.family_type.month2024_04 + period: 2024-04 + measures: [{<<: *benefit_units, column: B, source_column_id: april_2024, expected_column_header: April 2024}] +- <<: *total + provenance_class: administrative + record_set_id: dwp.statx.uc_benefit_units.family_type.month2024_04.total + record_set_spec_id: dwp.statx.uc_benefit_units.family_type.month2024_04.total.v1 + source_record_id_prefix: dwp.statx.uc_benefit_units.family_type.month2024_04.total + period: 2024-04 + measures: [{<<: *total_benefit_units, column: B, source_column_id: april_2024, expected_column_header: April 2024}] +- <<: *detail + provenance_class: administrative + record_set_id: dwp.statx.uc_benefit_units.family_type.month2024_05 + record_set_spec_id: dwp.statx.uc_benefit_units.family_type.month2024_05.v1 + source_record_id_prefix: dwp.statx.uc_benefit_units.family_type.month2024_05 + period: 2024-05 + measures: [{<<: *benefit_units, column: D, source_column_id: may_2024, expected_column_header: May 2024}] +- <<: *total + provenance_class: administrative + record_set_id: dwp.statx.uc_benefit_units.family_type.month2024_05.total + record_set_spec_id: dwp.statx.uc_benefit_units.family_type.month2024_05.total.v1 + source_record_id_prefix: dwp.statx.uc_benefit_units.family_type.month2024_05.total + period: 2024-05 + measures: [{<<: *total_benefit_units, column: D, source_column_id: may_2024, expected_column_header: May 2024}] +- <<: *detail + provenance_class: administrative + record_set_id: dwp.statx.uc_benefit_units.family_type.month2024_06 + record_set_spec_id: dwp.statx.uc_benefit_units.family_type.month2024_06.v1 + source_record_id_prefix: dwp.statx.uc_benefit_units.family_type.month2024_06 + period: 2024-06 + measures: [{<<: *benefit_units, column: F, source_column_id: june_2024, expected_column_header: 'June 2024 (r)'}] +- <<: *total + provenance_class: administrative + record_set_id: dwp.statx.uc_benefit_units.family_type.month2024_06.total + record_set_spec_id: dwp.statx.uc_benefit_units.family_type.month2024_06.total.v1 + source_record_id_prefix: dwp.statx.uc_benefit_units.family_type.month2024_06.total + period: 2024-06 + measures: [{<<: *total_benefit_units, column: F, source_column_id: june_2024, expected_column_header: 'June 2024 (r)'}] +- <<: *detail + provenance_class: administrative + record_set_id: dwp.statx.uc_benefit_units.family_type.month2024_07 + record_set_spec_id: dwp.statx.uc_benefit_units.family_type.month2024_07.v1 + source_record_id_prefix: dwp.statx.uc_benefit_units.family_type.month2024_07 + period: 2024-07 + measures: [{<<: *benefit_units, column: H, source_column_id: july_2024, expected_column_header: 'July 2024 (r)'}] +- <<: *total + provenance_class: administrative + record_set_id: dwp.statx.uc_benefit_units.family_type.month2024_07.total + record_set_spec_id: dwp.statx.uc_benefit_units.family_type.month2024_07.total.v1 + source_record_id_prefix: dwp.statx.uc_benefit_units.family_type.month2024_07.total + period: 2024-07 + measures: [{<<: *total_benefit_units, column: H, source_column_id: july_2024, expected_column_header: 'July 2024 (r)'}] +- <<: *detail + provenance_class: administrative + record_set_id: dwp.statx.uc_benefit_units.family_type.month2024_08 + record_set_spec_id: dwp.statx.uc_benefit_units.family_type.month2024_08.v1 + source_record_id_prefix: dwp.statx.uc_benefit_units.family_type.month2024_08 + period: 2024-08 + measures: [{<<: *benefit_units, column: J, source_column_id: august_2024, expected_column_header: 'August 2024 (r)'}] +- <<: *total provenance_class: administrative + record_set_id: dwp.statx.uc_benefit_units.family_type.month2024_08.total + record_set_spec_id: dwp.statx.uc_benefit_units.family_type.month2024_08.total.v1 + source_record_id_prefix: dwp.statx.uc_benefit_units.family_type.month2024_08.total + period: 2024-08 + measures: [{<<: *total_benefit_units, column: J, source_column_id: august_2024, expected_column_header: 'August 2024 (r)'}] +- <<: *detail + provenance_class: administrative + record_set_id: dwp.statx.uc_benefit_units.family_type.month2024_09 + record_set_spec_id: dwp.statx.uc_benefit_units.family_type.month2024_09.v1 + source_record_id_prefix: dwp.statx.uc_benefit_units.family_type.month2024_09 + period: 2024-09 + measures: [{<<: *benefit_units, column: L, source_column_id: september_2024, expected_column_header: 'September 2024 (r)'}] +- <<: *total + provenance_class: administrative + record_set_id: dwp.statx.uc_benefit_units.family_type.month2024_09.total + record_set_spec_id: dwp.statx.uc_benefit_units.family_type.month2024_09.total.v1 + source_record_id_prefix: dwp.statx.uc_benefit_units.family_type.month2024_09.total + period: 2024-09 + measures: [{<<: *total_benefit_units, column: L, source_column_id: september_2024, expected_column_header: 'September 2024 (r)'}] +- <<: *detail + provenance_class: administrative + record_set_id: dwp.statx.uc_benefit_units.family_type.month2024_10 + record_set_spec_id: dwp.statx.uc_benefit_units.family_type.month2024_10.v1 + source_record_id_prefix: dwp.statx.uc_benefit_units.family_type.month2024_10 + period: 2024-10 + measures: [{<<: *benefit_units, column: N, source_column_id: october_2024, expected_column_header: 'October 2024 (r)'}] +- <<: *total + provenance_class: administrative + record_set_id: dwp.statx.uc_benefit_units.family_type.month2024_10.total + record_set_spec_id: dwp.statx.uc_benefit_units.family_type.month2024_10.total.v1 + source_record_id_prefix: dwp.statx.uc_benefit_units.family_type.month2024_10.total + period: 2024-10 + measures: [{<<: *total_benefit_units, column: N, source_column_id: october_2024, expected_column_header: 'October 2024 (r)'}] +- <<: *detail + provenance_class: administrative + record_set_id: dwp.statx.uc_benefit_units.family_type.month2024_11 + record_set_spec_id: dwp.statx.uc_benefit_units.family_type.month2024_11.v1 + source_record_id_prefix: dwp.statx.uc_benefit_units.family_type.month2024_11 + period: 2024-11 + measures: [{<<: *benefit_units, column: P, source_column_id: november_2024, expected_column_header: 'November 2024 (r)'}] +- <<: *total + provenance_class: administrative + record_set_id: dwp.statx.uc_benefit_units.family_type.month2024_11.total + record_set_spec_id: dwp.statx.uc_benefit_units.family_type.month2024_11.total.v1 + source_record_id_prefix: dwp.statx.uc_benefit_units.family_type.month2024_11.total + period: 2024-11 + measures: [{<<: *total_benefit_units, column: P, source_column_id: november_2024, expected_column_header: 'November 2024 (r)'}] +- <<: *detail + provenance_class: administrative + record_set_id: dwp.statx.uc_benefit_units.family_type.month2024_12 + record_set_spec_id: dwp.statx.uc_benefit_units.family_type.month2024_12.v1 + source_record_id_prefix: dwp.statx.uc_benefit_units.family_type.month2024_12 + period: 2024-12 + measures: [{<<: *benefit_units, column: R, source_column_id: december_2024, expected_column_header: 'December 2024 (r)'}] +- <<: *total + provenance_class: administrative + record_set_id: dwp.statx.uc_benefit_units.family_type.month2024_12.total + record_set_spec_id: dwp.statx.uc_benefit_units.family_type.month2024_12.total.v1 + source_record_id_prefix: dwp.statx.uc_benefit_units.family_type.month2024_12.total + period: 2024-12 + measures: [{<<: *total_benefit_units, column: R, source_column_id: december_2024, expected_column_header: 'December 2024 (r)'}] + +- <<: *detail + provenance_class: administrative + record_set_id: dwp.statx.uc_benefit_units.family_type.month2025_04 + record_set_spec_id: dwp.statx.uc_benefit_units.family_type.month2025_04.v1 + source_record_id_prefix: dwp.statx.uc_benefit_units.family_type.month2025_04 + period: 2025-04 + measures: [{<<: *benefit_units, column: AL, source_column_id: april_2025, expected_column_header: 'April 2025 (r)'}] +- <<: *total + provenance_class: administrative + record_set_id: dwp.statx.uc_benefit_units.family_type.month2025_04.total + record_set_spec_id: dwp.statx.uc_benefit_units.family_type.month2025_04.total.v1 + source_record_id_prefix: dwp.statx.uc_benefit_units.family_type.month2025_04.total + period: 2025-04 + measures: [{<<: *total_benefit_units, column: AL, source_column_id: april_2025, expected_column_header: 'April 2025 (r)'}] +- <<: *detail + provenance_class: administrative + record_set_id: dwp.statx.uc_benefit_units.family_type.month2025_05 record_set_spec_id: dwp.statx.uc_benefit_units.family_type.month2025_05.v1 source_record_id_prefix: dwp.statx.uc_benefit_units.family_type.month2025_05 period: 2025-05 - rows: &may_rows - - {<<: *single_no_children, row_number: 7, guard_cells: [{column: C, row: 7, expected_value: 'May 2025', label: month}]} - - {<<: *single_with_children, row_number: 8, guard_cells: [{column: C, row: 8, expected_value: 'May 2025', label: month}]} - - {<<: *couple_no_children, row_number: 9, guard_cells: [{column: C, row: 9, expected_value: 'May 2025', label: month}]} - - {<<: *couple_with_children, row_number: 10, guard_cells: [{column: C, row: 10, expected_value: 'May 2025', label: month}]} - - {<<: *unknown_family, row_number: 11, guard_cells: [{column: C, row: 11, expected_value: 'May 2025', label: month}]} -- <<: *monthly - record_set_id: dwp.statx.uc_benefit_units.family_type.month2025_06 + measures: [{<<: *benefit_units, column: AN, source_column_id: may_2025, expected_column_header: 'May 2025 (r)'}] +- <<: *total provenance_class: administrative + record_set_id: dwp.statx.uc_benefit_units.family_type.month2025_05.total + record_set_spec_id: dwp.statx.uc_benefit_units.family_type.month2025_05.total.v1 + source_record_id_prefix: dwp.statx.uc_benefit_units.family_type.month2025_05.total + period: 2025-05 + measures: [{<<: *total_benefit_units, column: AN, source_column_id: may_2025, expected_column_header: 'May 2025 (r)'}] +- <<: *detail + provenance_class: administrative + record_set_id: dwp.statx.uc_benefit_units.family_type.month2025_06 record_set_spec_id: dwp.statx.uc_benefit_units.family_type.month2025_06.v1 source_record_id_prefix: dwp.statx.uc_benefit_units.family_type.month2025_06 period: 2025-06 - rows: - - {<<: *single_no_children, row_number: 12, guard_cells: [{column: C, row: 12, expected_value: 'June 2025', label: month}]} - - {<<: *single_with_children, row_number: 13, guard_cells: [{column: C, row: 13, expected_value: 'June 2025', label: month}]} - - {<<: *couple_no_children, row_number: 14, guard_cells: [{column: C, row: 14, expected_value: 'June 2025', label: month}]} - - {<<: *couple_with_children, row_number: 15, guard_cells: [{column: C, row: 15, expected_value: 'June 2025', label: month}]} - - {<<: *unknown_family, row_number: 16, guard_cells: [{column: C, row: 16, expected_value: 'June 2025', label: month}]} -- <<: *monthly - record_set_id: dwp.statx.uc_benefit_units.family_type.month2025_07 + measures: [{<<: *benefit_units, column: AP, source_column_id: june_2025, expected_column_header: 'June 2025 (r)'}] +- <<: *total + provenance_class: administrative + record_set_id: dwp.statx.uc_benefit_units.family_type.month2025_06.total + record_set_spec_id: dwp.statx.uc_benefit_units.family_type.month2025_06.total.v1 + source_record_id_prefix: dwp.statx.uc_benefit_units.family_type.month2025_06.total + period: 2025-06 + measures: [{<<: *total_benefit_units, column: AP, source_column_id: june_2025, expected_column_header: 'June 2025 (r)'}] +- <<: *detail provenance_class: administrative + record_set_id: dwp.statx.uc_benefit_units.family_type.month2025_07 record_set_spec_id: dwp.statx.uc_benefit_units.family_type.month2025_07.v1 source_record_id_prefix: dwp.statx.uc_benefit_units.family_type.month2025_07 period: 2025-07 - rows: - - {<<: *single_no_children, row_number: 17, guard_cells: [{column: C, row: 17, expected_value: 'July 2025', label: month}]} - - {<<: *single_with_children, row_number: 18, guard_cells: [{column: C, row: 18, expected_value: 'July 2025', label: month}]} - - {<<: *couple_no_children, row_number: 19, guard_cells: [{column: C, row: 19, expected_value: 'July 2025', label: month}]} - - {<<: *couple_with_children, row_number: 20, guard_cells: [{column: C, row: 20, expected_value: 'July 2025', label: month}]} - - {<<: *unknown_family, row_number: 21, guard_cells: [{column: C, row: 21, expected_value: 'July 2025', label: month}]} -- <<: *monthly - record_set_id: dwp.statx.uc_benefit_units.family_type.month2025_08 + measures: [{<<: *benefit_units, column: AR, source_column_id: july_2025, expected_column_header: 'July 2025 (r)'}] +- <<: *total + provenance_class: administrative + record_set_id: dwp.statx.uc_benefit_units.family_type.month2025_07.total + record_set_spec_id: dwp.statx.uc_benefit_units.family_type.month2025_07.total.v1 + source_record_id_prefix: dwp.statx.uc_benefit_units.family_type.month2025_07.total + period: 2025-07 + measures: [{<<: *total_benefit_units, column: AR, source_column_id: july_2025, expected_column_header: 'July 2025 (r)'}] +- <<: *detail provenance_class: administrative + record_set_id: dwp.statx.uc_benefit_units.family_type.month2025_08 record_set_spec_id: dwp.statx.uc_benefit_units.family_type.month2025_08.v1 source_record_id_prefix: dwp.statx.uc_benefit_units.family_type.month2025_08 period: 2025-08 - rows: - - {<<: *single_no_children, row_number: 22, guard_cells: [{column: C, row: 22, expected_value: 'August 2025', label: month}]} - - {<<: *single_with_children, row_number: 23, guard_cells: [{column: C, row: 23, expected_value: 'August 2025', label: month}]} - - {<<: *couple_no_children, row_number: 24, guard_cells: [{column: C, row: 24, expected_value: 'August 2025', label: month}]} - - {<<: *couple_with_children, row_number: 25, guard_cells: [{column: C, row: 25, expected_value: 'August 2025', label: month}]} - - {<<: *unknown_family, row_number: 26, guard_cells: [{column: C, row: 26, expected_value: 'August 2025', label: month}]} -- <<: *monthly - record_set_id: dwp.statx.uc_benefit_units.family_type.month2025_09 + measures: [{<<: *benefit_units, column: AT, source_column_id: august_2025, expected_column_header: 'August 2025 (r)'}] +- <<: *total + provenance_class: administrative + record_set_id: dwp.statx.uc_benefit_units.family_type.month2025_08.total + record_set_spec_id: dwp.statx.uc_benefit_units.family_type.month2025_08.total.v1 + source_record_id_prefix: dwp.statx.uc_benefit_units.family_type.month2025_08.total + period: 2025-08 + measures: [{<<: *total_benefit_units, column: AT, source_column_id: august_2025, expected_column_header: 'August 2025 (r)'}] +- <<: *detail provenance_class: administrative + record_set_id: dwp.statx.uc_benefit_units.family_type.month2025_09 record_set_spec_id: dwp.statx.uc_benefit_units.family_type.month2025_09.v1 source_record_id_prefix: dwp.statx.uc_benefit_units.family_type.month2025_09 period: 2025-09 - rows: - - {<<: *single_no_children, row_number: 27, guard_cells: [{column: C, row: 27, expected_value: 'September 2025', label: month}]} - - {<<: *single_with_children, row_number: 28, guard_cells: [{column: C, row: 28, expected_value: 'September 2025', label: month}]} - - {<<: *couple_no_children, row_number: 29, guard_cells: [{column: C, row: 29, expected_value: 'September 2025', label: month}]} - - {<<: *couple_with_children, row_number: 30, guard_cells: [{column: C, row: 30, expected_value: 'September 2025', label: month}]} - - {<<: *unknown_family, row_number: 31, guard_cells: [{column: C, row: 31, expected_value: 'September 2025', label: month}]} -- <<: *monthly - record_set_id: dwp.statx.uc_benefit_units.family_type.month2025_10 + measures: [{<<: *benefit_units, column: AV, source_column_id: september_2025, expected_column_header: 'September 2025 (r)'}] +- <<: *total provenance_class: administrative + record_set_id: dwp.statx.uc_benefit_units.family_type.month2025_09.total + record_set_spec_id: dwp.statx.uc_benefit_units.family_type.month2025_09.total.v1 + source_record_id_prefix: dwp.statx.uc_benefit_units.family_type.month2025_09.total + period: 2025-09 + measures: [{<<: *total_benefit_units, column: AV, source_column_id: september_2025, expected_column_header: 'September 2025 (r)'}] +- <<: *detail + provenance_class: administrative + record_set_id: dwp.statx.uc_benefit_units.family_type.month2025_10 record_set_spec_id: dwp.statx.uc_benefit_units.family_type.month2025_10.v1 source_record_id_prefix: dwp.statx.uc_benefit_units.family_type.month2025_10 period: 2025-10 - rows: - - {<<: *single_no_children, row_number: 32, guard_cells: [{column: C, row: 32, expected_value: 'October 2025', label: month}]} - - {<<: *single_with_children, row_number: 33, guard_cells: [{column: C, row: 33, expected_value: 'October 2025', label: month}]} - - {<<: *couple_no_children, row_number: 34, guard_cells: [{column: C, row: 34, expected_value: 'October 2025', label: month}]} - - {<<: *couple_with_children, row_number: 35, guard_cells: [{column: C, row: 35, expected_value: 'October 2025', label: month}]} - - {<<: *unknown_family, row_number: 36, guard_cells: [{column: C, row: 36, expected_value: 'October 2025', label: month}]} -- <<: *monthly - record_set_id: dwp.statx.uc_benefit_units.family_type.month2025_11 + measures: [{<<: *benefit_units, column: AX, source_column_id: october_2025, expected_column_header: 'October 2025 (r)'}] +- <<: *total + provenance_class: administrative + record_set_id: dwp.statx.uc_benefit_units.family_type.month2025_10.total + record_set_spec_id: dwp.statx.uc_benefit_units.family_type.month2025_10.total.v1 + source_record_id_prefix: dwp.statx.uc_benefit_units.family_type.month2025_10.total + period: 2025-10 + measures: [{<<: *total_benefit_units, column: AX, source_column_id: october_2025, expected_column_header: 'October 2025 (r)'}] +- <<: *detail provenance_class: administrative + record_set_id: dwp.statx.uc_benefit_units.family_type.month2025_11 record_set_spec_id: dwp.statx.uc_benefit_units.family_type.month2025_11.v1 source_record_id_prefix: dwp.statx.uc_benefit_units.family_type.month2025_11 period: 2025-11 - rows: - - {<<: *single_no_children, row_number: 37, guard_cells: [{column: C, row: 37, expected_value: 'November 2025', label: month}]} - - {<<: *single_with_children, row_number: 38, guard_cells: [{column: C, row: 38, expected_value: 'November 2025', label: month}]} - - {<<: *couple_no_children, row_number: 39, guard_cells: [{column: C, row: 39, expected_value: 'November 2025', label: month}]} - - {<<: *couple_with_children, row_number: 40, guard_cells: [{column: C, row: 40, expected_value: 'November 2025', label: month}]} - - {<<: *unknown_family, row_number: 41, guard_cells: [{column: C, row: 41, expected_value: 'November 2025', label: month}]} -- <<: *monthly - record_set_id: dwp.statx.uc_benefit_units.family_type.month2025_12 + measures: [{<<: *benefit_units, column: AZ, source_column_id: november_2025, expected_column_header: 'November 2025 (r)'}] +- <<: *total provenance_class: administrative + record_set_id: dwp.statx.uc_benefit_units.family_type.month2025_11.total + record_set_spec_id: dwp.statx.uc_benefit_units.family_type.month2025_11.total.v1 + source_record_id_prefix: dwp.statx.uc_benefit_units.family_type.month2025_11.total + period: 2025-11 + measures: [{<<: *total_benefit_units, column: AZ, source_column_id: november_2025, expected_column_header: 'November 2025 (r)'}] +- <<: *detail + provenance_class: administrative + record_set_id: dwp.statx.uc_benefit_units.family_type.month2025_12 record_set_spec_id: dwp.statx.uc_benefit_units.family_type.month2025_12.v1 source_record_id_prefix: dwp.statx.uc_benefit_units.family_type.month2025_12 period: 2025-12 - rows: - - {<<: *single_no_children, row_number: 42, guard_cells: [{column: C, row: 42, expected_value: 'December 2025', label: month}]} - - {<<: *single_with_children, row_number: 43, guard_cells: [{column: C, row: 43, expected_value: 'December 2025', label: month}]} - - {<<: *couple_no_children, row_number: 44, guard_cells: [{column: C, row: 44, expected_value: 'December 2025', label: month}]} - - {<<: *couple_with_children, row_number: 45, guard_cells: [{column: C, row: 45, expected_value: 'December 2025', label: month}]} - - {<<: *unknown_family, row_number: 46, guard_cells: [{column: C, row: 46, expected_value: 'December 2025', label: month}]} + measures: [{<<: *benefit_units, column: BB, source_column_id: december_2025, expected_column_header: 'December 2025 (r)'}] +- <<: *total + provenance_class: administrative + record_set_id: dwp.statx.uc_benefit_units.family_type.month2025_12.total + record_set_spec_id: dwp.statx.uc_benefit_units.family_type.month2025_12.total.v1 + source_record_id_prefix: dwp.statx.uc_benefit_units.family_type.month2025_12.total + period: 2025-12 + measures: [{<<: *total_benefit_units, column: BB, source_column_id: december_2025, expected_column_header: 'December 2025 (r)'}] diff --git a/tests/test_chronicle_bundle.py b/tests/test_chronicle_bundle.py index 737a7433..fb41d32c 100644 --- a/tests/test_chronicle_bundle.py +++ b/tests/test_chronicle_bundle.py @@ -129,7 +129,7 @@ def test_build_bundle_writes_merged_consumer_contract(tmp_path): "aggregate_duplicate_key_count": 0, "entity_count": 12, "error_count": 0, - "fact_count": 192460, + "fact_count": 192577, "geography_count": 12539, "period_count": 269, "semantic_duplicate_key_count": 177, @@ -138,7 +138,7 @@ def test_build_bundle_writes_merged_consumer_contract(tmp_path): "source_package_count": 167, "warning_count": 1, } - assert len(rows) == 192460 + assert len(rows) == 192577 assert {row["provenance_class"] for row in rows} <= { "administrative", "census", @@ -170,7 +170,7 @@ def test_build_bundle_writes_merged_consumer_contract(tmp_path): "jct-obbba-revenue-estimates-2025", "jct-tax-expenditures-2024", ] - assert coverage["fact_count"] == 192460 + assert coverage["fact_count"] == 192577 assert coverage["counts"]["by_source"] == { "bea": 445, "bfp_economic_outlook": 5, @@ -185,7 +185,7 @@ def test_build_bundle_writes_merged_consumer_contract(tmp_path): "dfe": 770, "dfc_ni": 1189, "dft": 233, - "dwp": 8130, + "dwp": 8247, "eurostat": 207, "federal_reserve": 1, "fpb_economic_outlook": 1000, @@ -306,9 +306,10 @@ def test_build_bundle_writes_merged_consumer_contract(tmp_path): ) assert ( table_counts[ - "dwp:Households on Universal Credit by family type, April to December 2025" + "dwp:Households on Universal Credit by family type, " + "April to December 2023 through 2025" ] - == 45 + == 162 ) assert ( table_counts[ @@ -756,39 +757,39 @@ def test_build_bundle_writes_merged_consumer_contract(tmp_path): "month:2023-01": 380, "month:2023-02": 379, "month:2023-03": 387, - "month:2023-04": 379, - "month:2023-05": 400, - "month:2023-06": 387, - "month:2023-07": 379, - "month:2023-08": 402, - "month:2023-09": 387, - "month:2023-10": 379, - "month:2023-11": 379, - "month:2023-12": 393, + "month:2023-04": 385, + "month:2023-05": 406, + "month:2023-06": 393, + "month:2023-07": 385, + "month:2023-08": 408, + "month:2023-09": 393, + "month:2023-10": 385, + "month:2023-11": 385, + "month:2023-12": 399, "month:2024-01": 380, "month:2024-02": 379, "month:2024-03": 387, - "month:2024-04": 379, - "month:2024-05": 400, - "month:2024-06": 387, - "month:2024-07": 379, - "month:2024-08": 402, - "month:2024-09": 387, - "month:2024-10": 485, - "month:2024-11": 485, - "month:2024-12": 763, + "month:2024-04": 385, + "month:2024-05": 406, + "month:2024-06": 393, + "month:2024-07": 385, + "month:2024-08": 408, + "month:2024-09": 393, + "month:2024-10": 491, + "month:2024-11": 491, + "month:2024-12": 769, "month:2025-01": 487, "month:2025-02": 485, "month:2025-03": 615, - "month:2025-04": 684, - "month:2025-05": 6651, - "month:2025-06": 600, - "month:2025-07": 592, - "month:2025-08": 1029, - "month:2025-09": 608, - "month:2025-10": 591, - "month:2025-11": 606, - "month:2025-12": 855, + "month:2025-04": 685, + "month:2025-05": 6652, + "month:2025-06": 601, + "month:2025-07": 593, + "month:2025-08": 1030, + "month:2025-09": 609, + "month:2025-10": 592, + "month:2025-11": 607, + "month:2025-12": 856, "month:2026-01": 381, "month:2026-02": 385, "month:2026-03": 386, @@ -877,10 +878,10 @@ def test_build_bundle_writes_merged_consumer_contract(tmp_path): ) assert coverage["counts"]["by_geography"]["country:K02000001"] == 6373 assert coverage["counts"]["by_geography"]["country:E92000001"] == 1437 - assert coverage["counts"]["by_geography"]["country:K03000001"] == 1975 + assert coverage["counts"]["by_geography"]["country:K03000001"] == 2092 assert len(coverage["counts"]["by_geography"]) == 12539 assert coverage["counts"]["by_entity"] == { - "benefit_unit": 1979, + "benefit_unit": 2096, "dwelling": 27041, "family": 1299, "firm": 1439, diff --git a/tests/test_chronicle_source_package.py b/tests/test_chronicle_source_package.py index ae6f0166..94873d65 100644 --- a/tests/test_chronicle_source_package.py +++ b/tests/test_chronicle_source_package.py @@ -879,22 +879,14 @@ def test_dwp_uc_element_packages_emit_one_benefit_unit_fact_per_month( assert validate_consumer_fact_contract(facts).valid -@pytest.mark.parametrize( - ("alias", "facts_per_month"), - [ - ("dwp-uc-households-family-type-april-december-2025", 5), - ("dwp-uc-households-children-april-december-2025", 8), - ], -) -def test_dwp_uc_composition_packages_cover_the_caseload_months( - alias, - facts_per_month, -): - facts = load_source_package(alias).build_facts(2025) +def test_dwp_uc_children_composition_package_covers_the_caseload_months(): + facts = load_source_package( + "dwp-uc-households-children-april-december-2025" + ).build_facts(2025) consumer_rows = consumer_fact_rows(facts) periods = [f"2025-{month:02d}" for month in range(4, 13)] - assert len(facts) == facts_per_month * len(periods) + assert len(facts) == 8 * len(periods) assert {fact.period.value for fact in facts} == set(periods) assert all(fact.measure.concept == "dwp.uc_benefit_units" for fact in facts) assert all(fact.period.type == "month" for fact in facts) @@ -912,6 +904,70 @@ def test_dwp_uc_composition_packages_cover_the_caseload_months( } +def test_dwp_uc_family_type_package_preserves_published_totals_from_2023(tmp_path): + facts = load_source_package( + "dwp-uc-households-family-type-april-december-2025" + ).build_facts(2025) + consumer_rows = consumer_fact_rows(facts) + periods = { + f"{year}-{month:02d}" for year in range(2023, 2026) for month in range(4, 13) + } + + assert len(facts) == 162 + assert {fact.period.value for fact in facts} == periods + assert all(fact.measure.concept == "dwp.uc_benefit_units" for fact in facts) + assert all(fact.period.type == "month" for fact in facts) + assert all(fact.entity.name == "benefit_unit" for fact in facts) + assert all(fact.geography.id == "K03000001" for fact in facts) + assert all(fact.assertion == "observation" for fact in facts) + assert all(fact.provenance_class == "administrative" for fact in facts) + assert all(fact.source_row_keys for fact in facts) + assert validate_consumer_fact_contract(facts).valid + + detail_rows = [ + row + for row in consumer_rows + if row["observed_measure"]["source_measure_id"] == "benefit_units" + ] + total_rows = [ + row + for row in consumer_rows + if row["observed_measure"]["source_measure_id"] == "total_benefit_units" + ] + assert len(detail_rows) == 135 + assert len(total_rows) == 27 + assert all(row["layout"]["table_record_kind"] == "detail" for row in detail_rows) + assert all(row["layout"]["table_record_kind"] == "total" for row in total_rows) + assert all(row["dimensions"] == {"family_type": "all"} for row in total_rows) + assert {row["observed_measure"]["source_concept"] for row in consumer_rows} == { + "dwp.uc_households" + } + + for period in sorted(periods): + detail_sum = sum( + row["value"] for row in detail_rows if row["period"]["value"] == period + ) + published_total = next( + row["value"] for row in total_rows if row["period"]["value"] == period + ) + # DWP warns that disclosure control can keep displayed totals from + # summing exactly; the archived table differs by at most 11 units. + assert abs(published_total - detail_sum) <= 15 + + published_totals = {row["period"]["value"]: row["value"] for row in total_rows} + assert published_totals["2023-04"] == 5_065_371 + assert published_totals["2024-04"] == 5_685_995 + assert published_totals["2025-12"] == 7_154_045 + + suite = build_source_suite( + "dwp-uc-households-family-type-april-december-2025", + tmp_path / "dwp-uc-households-family-type-april-december-2025", + year=2025, + ) + assert suite.agent_acceptance.valid + assert suite.agent_acceptance.counts["row_semantic_error_count"] == 0 + + def test_source_package_alias_compiles_soi_table_1_1_specs(): package = load_source_package("soi-table-1-1") record_set = package.build_source_record_set_specs(2023)[0]