Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions .gitattributes
Original file line number Diff line number Diff line change
@@ -1,2 +1,3 @@
# Keep pinned source artifacts byte-for-byte identical to their publishers.
db/data/dfe/funded_early_education_childcare_2026/headline_figures_feeac_2011_2026.csv -diff
db/data/dwp/uc_households_family_type_april_december_2025/uc_households_by_family_type_april_december_2023_2025.csv -diff
4 changes: 4 additions & 0 deletions chronicle/source_package.py
Original file line number Diff line number Diff line change
Expand Up @@ -1765,6 +1765,10 @@ def _row_from_mapping(payload: dict[str, Any], *, year: int) -> SourceRecordSetR
_constraint_from_mapping(constraint, year=year)
for constraint in payload.get("constraints", ())
),
source_row_dimensions={
str(key): _render_value(value, year=year)
for key, value in payload.get("source_row_dimensions", {}).items()
},
value_scale=_render_value(payload.get("value_scale", 1), year=year),
source_row_id=payload.get("source_row_id"),
table_record_kind=payload.get("table_record_kind", "detail"),
Expand Down
3 changes: 3 additions & 0 deletions chronicle/sources/specs.py
Original file line number Diff line number Diff line change
Expand Up @@ -143,6 +143,7 @@ class SourceRecordSetRow:
geography_vintage: str | None = None
filters: dict[str, Scalar] = field(default_factory=dict)
constraints: tuple[AggregateConstraint, ...] = ()
source_row_dimensions: dict[str, Scalar] = field(default_factory=dict)
value_scale: int | float = 1
source_row_id: str | None = None
table_record_kind: str = "detail"
Expand Down Expand Up @@ -818,6 +819,8 @@ def _record_set_spec_hash(spec: SourceRecordSetSpec) -> str:
row.pop("expected_row_header", None)
if row.get("expected_row_header_column") is None:
row.pop("expected_row_header_column", None)
if not row.get("source_row_dimensions"):
row.pop("source_row_dimensions", None)
if not row.get("guard_cells"):
row.pop("guard_cells", None)
else:
Expand Down
72 changes: 67 additions & 5 deletions chronicle/suite.py
Original file line number Diff line number Diff line change
Expand Up @@ -395,6 +395,9 @@ def build_source_suite(
source_column_dimensions_by_record_id=(
_source_column_dimensions_by_record_id(source_record_set_specs)
),
source_row_dimensions_by_record_id=(
_source_row_dimensions_by_record_id(source_record_set_specs)
),
selected_only_source_parse=(
bool(source_package)
and source_package.artifact.parser == "delimited_text_selected_rows"
Expand Down Expand Up @@ -581,6 +584,7 @@ def build_agent_acceptance_report(
concept_alignments: ConceptAlignmentReport,
require_axiom_validation: bool = False,
source_column_dimensions_by_record_id: dict[str, dict[str, Any]] | None = None,
source_row_dimensions_by_record_id: dict[str, dict[str, Any]] | None = None,
selected_only_source_parse: bool = False,
) -> AgentAcceptanceReport:
"""Build the stricter report agents should satisfy before review."""
Expand All @@ -599,6 +603,7 @@ def build_agent_acceptance_report(
for cell in cells
}
source_column_dimensions_by_record_id = source_column_dimensions_by_record_id or {}
source_row_dimensions_by_record_id = source_row_dimensions_by_record_id or {}
raw_r2_link_count = 0

if not cells and not rows:
Expand Down Expand Up @@ -714,6 +719,12 @@ def build_agent_acceptance_report(
{},
)
),
source_row_dimensions=(
source_row_dimensions_by_record_id.get(
fact.source_record_id or "",
{},
)
),
):
row_semantic_error_count += 1
errors.append(issue)
Expand Down Expand Up @@ -971,9 +982,11 @@ def _row_semantic_evidence_issues(
cells: list[SourceCell],
*,
source_column_dimensions: dict[str, Any] | None = None,
source_row_dimensions: dict[str, Any] | None = None,
) -> list[AgentAcceptanceIssue]:
issues: list[AgentAcceptanceIssue] = []
source_column_dimensions = source_column_dimensions or {}
source_row_dimensions = source_row_dimensions or {}
fact_key = build_fact_key(fact)
period_values = _source_row_values(rows, "period")
for value in period_values:
Expand All @@ -993,7 +1006,7 @@ def _row_semantic_evidence_issues(
for variable, value in fact.filters.items():
if value is None:
continue
if value == "all" and not source_column_dimensions:
if value == "all" and not (source_column_dimensions or source_row_dimensions):
continue
matched_values = _source_row_values(rows, variable)
if not matched_values:
Expand All @@ -1005,6 +1018,12 @@ def _row_semantic_evidence_issues(
value,
):
continue
if _declared_dimension_evidences(
source_row_dimensions,
variable,
value,
):
continue
issues.append(
AgentAcceptanceIssue(
code="row_filter_not_evidenced",
Expand Down Expand Up @@ -1042,6 +1061,11 @@ def _row_semantic_evidence_issues(
constraint,
):
continue
if _declared_constraint_evidenced(
source_row_dimensions,
constraint,
):
continue
matched_values = _source_row_values(rows, constraint.variable)
if not matched_values:
issues.append(
Expand Down Expand Up @@ -1079,20 +1103,43 @@ def _wide_table_filter_evidenced_by_source_column(
expected: Any,
) -> bool:
"""Accept an explicitly declared dimension of a guarded source column."""
if variable not in source_column_dimensions:
return _declared_dimension_evidences(
source_column_dimensions,
variable,
expected,
)


def _declared_dimension_evidences(
dimensions: dict[str, Any],
variable: str,
expected: Any,
) -> bool:
"""Accept an explicitly declared semantic dimension of a source axis."""
if variable not in dimensions:
return False
declared = source_column_dimensions[variable]
declared = dimensions[variable]
return type(declared) is type(expected) and declared == expected


def _wide_table_constraint_evidenced_by_source_column(
source_column_dimensions: dict[str, Any],
constraint: Any,
) -> bool:
return _declared_constraint_evidenced(
source_column_dimensions,
constraint,
)


def _declared_constraint_evidenced(
dimensions: dict[str, Any],
constraint: Any,
) -> bool:
if constraint.operator != "==":
return False
return _wide_table_filter_evidenced_by_source_column(
source_column_dimensions,
return _declared_dimension_evidences(
dimensions,
str(constraint.variable),
constraint.value,
)
Expand All @@ -1113,6 +1160,21 @@ def _source_column_dimensions_by_record_id(
}


def _source_row_dimensions_by_record_id(
record_sets: list[SourceRecordSetSpec],
) -> dict[str, dict[str, Any]]:
"""Index explicit row dimensions for row-header tables."""
return {
f"{record_set.source_record_id_prefix}.{row.value_id}.{measure.measure_id}": (
dict(row.source_row_dimensions)
)
for record_set in record_sets
for row in record_set.rows
for measure in record_set.measures
if row.source_row_dimensions
}


def _constraint_evidenced_by_source_cells(
cells: list[SourceCell],
constraint: Any,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2,17 +2,18 @@ source_id: dwp
package_id: dwp-uc-households-family-type-april-december-2025
dataset: dwp_dwp-uc-households-family-type-april-december-2025
source_page: https://stat-xplore.dwp.gov.uk/
table: Households on Universal Credit by family type, April to December 2025
table: Households on Universal Credit by family type, April to December 2023 through
2025
files:
2025:
filename: uc_households_by_family_type_april_december_2025.json
source_url: https://stat-xplore.dwp.gov.uk/webapi/rest/v1/table
sha256: 44d67348da8c34465611ac00efdfcb80a69ad0f52e788d53f82517b34594377f
size_bytes: 6719
fetched_at: '2026-08-21T11:45:49Z'
filename: uc_households_by_family_type_april_december_2023_2025.csv
source_url: https://stat-xplore.dwp.gov.uk/webapi/jsf/tableView/tableView.xhtml
sha256: 3783b64b9b0b5d35fb8ce7488f1e0238db89d67e6da96563140a151dfa4e5764
size_bytes: 4343
fetched_at: '2026-09-07T07:38:34Z'
storage:
r2:
provider: r2
bucket: ledger-raw
key: raw/dwp/dwp-uc-households-family-type-april-december-2025/2025/44d67348da8c34465611ac00efdfcb80a69ad0f52e788d53f82517b34594377f/uc_households_by_family_type_april_december_2025.json
uri: r2://ledger-raw/raw/dwp/dwp-uc-households-family-type-april-december-2025/2025/44d67348da8c34465611ac00efdfcb80a69ad0f52e788d53f82517b34594377f/uc_households_by_family_type_april_december_2025.json
key: raw/uk/dwp/dwp-uc-households-family-type-april-december-2025/2025/3783b64b9b0b5d35fb8ce7488f1e0238db89d67e6da96563140a151dfa4e5764/uc_households_by_family_type_april_december_2023_2025.csv
uri: r2://ledger-raw/raw/uk/dwp/dwp-uc-households-family-type-april-december-2025/2025/3783b64b9b0b5d35fb8ce7488f1e0238db89d67e6da96563140a151dfa4e5764/uc_households_by_family_type_april_december_2023_2025.csv
Original file line number Diff line number Diff line change
@@ -0,0 +1,32 @@
Stat-Xplore

"Households on Universal Credit (I II III IV z)"
"Family Type by Month"
"Counting: Households on Universal Credit"

Filters:
"Default Summation","Households on Universal Credit"

"Month","April 2024","April 2024 - Annotations","May 2024","May 2024 - Annotations","June 2024 (r)","June 2024 - Annotations","July 2024 (r)","July 2024 - Annotations","August 2024 (r)","August 2024 - Annotations","September 2024 (r)","September 2024 - Annotations","October 2024 (r)","October 2024 - Annotations","November 2024 (r)","November 2024 - Annotations","December 2024 (r)","December 2024 - Annotations","April 2023","April 2023 - Annotations","May 2023","May 2023 - Annotations","June 2023","June 2023 - Annotations","July 2023","July 2023 - Annotations","August 2023","August 2023 - Annotations","September 2023","September 2023 - Annotations","October 2023","October 2023 - Annotations","November 2023","November 2023 - Annotations","December 2023","December 2023 - Annotations","April 2025 (r)","April 2025 - Annotations","May 2025 (r)","May 2025 - Annotations","June 2025 (r)","June 2025 - Annotations","July 2025 (r)","July 2025 - Annotations","August 2025 (r)","August 2025 - Annotations","September 2025 (r)","September 2025 - Annotations","October 2025 (r)","October 2025 - Annotations","November 2025 (r)","November 2025 - Annotations","December 2025 (r)","December 2025 - Annotations",
"Family Type",
"Single, no children",2706575,,2712106,,2723460,,2743844,,2766134,,2798252,,2823250,,2858415,,2896968,,2473370,,2481406,,2493624,,2515891,,2534581,,2556314,,2578790,,2604763,,2634407,,3134001,,3193287,,3282370,,3362245,,3460441,,3550437,,3623576,,3690997,,3725304,,
"Single, with children",1964494,,1998179,,2027919,,2066556,,2104721,,2125571,,2144071,,2170644,,2184171,,1733859,,1746189,,1759681,,1778847,,1795426,,1825682,,1856372,,1881018,,1904309,,2207559,,2212526,,2219462,,2225365,,2228790,,2231333,,2234296,,2237019,,2239630,,
"Couple, no children",217781,,219594,,219799,,221981,,225284,,231939,,233971,,237209,,240452,,198021,,198387,,199033,,199840,,200649,,201264,,202372,,203938,,205801,,260740,,266117,,274424,,280070,,286102,,291802,,296932,,300679,,302244,,
"Couple, with children",795356,,816067,,826955,,830896,,842927,,875196,,885411,,898910,,906064,,657801,,659255,,661324,,663134,,664168,,666793,,671892,,678520,,688185,,915707,,911319,,907142,,902225,,897867,,895566,,892226,,888199,,885561,,
"Unknown or missing family type",1787,,1795,,1612,,1618,,1530,,1282,,1253,,1227,,1241,,2322,,2387,,2367,,2412,,2541,,2796,,2928,,2973,,3034,,1193,,1193,,1217,,1259,,1223,,1176,,1197,,1261,,1301,,
"Total",5685995,,5747730,,5799748,,5864899,,5940590,,6032248,,6087962,,6166404,,6228895,,5065371,,5087622,,5116034,,5160124,,5197374,,5252845,,5312353,,5371220,,5435736,,6519195,,6584436,,6684615,,6771172,,6874424,,6970315,,7048227,,7118153,,7154045,,


"INFO","Statistical disclosure control has been applied to this table to avoid the release of confidential data. Totals may not sum due to the disclosure control applied."


Symbol,Description
"I","A limited test of the full Universal Credit service was launched in Sutton, South London on 26th November 2014, and has expanded to other areas since. This publication now includes statistics covering both the live and full Universal Credit service. Please see https://www.gov.uk/government/publications/universal-credit-transition-to-full-service for details of the areas in which the full Universal Credit service is operating."
"II","Data for more recent months will be subject to a higher degree of revision. Universal Credit award amounts may be retrospectively revised."
"III","Monthly award amounts include any awards due to entitlement such as the standard allowance or housing entitlement plus any advance payments. Advance payments will normally be recovered during subsequent assessment periods."
"IV","Award, entitlement and payment information may be missing for a very small number of households on Universal Credit, where more limited information is entered onto IT systems."
"z",""".."" denotes a nil or negligible number of claimants or award amount based on a nil or negligible number of claimants."
"r","Figures marked ""r"" have been revised since the previous release."


Source: Department for Work and Pensions
Loading
Loading