Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions .gitattributes
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
# Keep pinned source artifacts byte-for-byte identical to their publishers.
db/data/dfe/funded_early_education_childcare_2026/headline_figures_feeac_2011_2026.csv -diff
3 changes: 3 additions & 0 deletions chronicle/bundle.py
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@
BUNDLE_SOURCES_SCHEMA_VERSION = "ledger.bundle_sources.v1"
DEFAULT_BUNDLE_SOURCES = tuple(sorted(SOURCE_PACKAGE_ALIASES))
UK_BUNDLE_SOURCE_PREFIXES = (
"dfe",
"dft",
"dwp",
"hmrc",
Expand All @@ -36,6 +37,7 @@
"welshgov",
)
UK_BUNDLE_SOURCES = (
"dfe-funded-early-education-childcare-2026",
"dft-nts-vehicle-ownership-2024",
"dwp-benefit-cap-november-2025",
"dwp-benefit-statistics-february-2026",
Expand All @@ -59,6 +61,7 @@
"hmrc-salary-sacrifice-relief-2024-25",
"hmrc-spi-income-bands-2023-24",
"hmrc-spi-income-by-area-2023-24",
"hmrc-tax-free-childcare-march-2026",
"hmrc-vat-firm-sector-targets-2024-25",
"hmrc-vat-firm-targets-2024-25",
"isc-annual-census-2023",
Expand Down
9 changes: 5 additions & 4 deletions chronicle/core.py
Original file line number Diff line number Diff line change
Expand Up @@ -575,9 +575,7 @@ def fact_counts(facts: list[AggregateFact]) -> dict[str, dict[str, int]]:
f"{fact.period.type}:{fact.period.value}" for fact in facts
),
"by_assertion": _counter_dict(fact.assertion for fact in facts),
"by_provenance_class": _counter_dict(
fact.provenance_class for fact in facts
),
"by_provenance_class": _counter_dict(fact.provenance_class for fact in facts),
"missing_labels": {"count": sum(1 for fact in facts if not fact.label)},
"missing_provenance": {
"count": sum(1 for fact in facts if _has_missing_provenance(fact))
Expand Down Expand Up @@ -651,7 +649,10 @@ def _validate_provenance_class(
return

if provenance_class == "survey_aggregate":
if type(fact.survey_instrument) is not str or not fact.survey_instrument.strip():
if (
type(fact.survey_instrument) is not str
or not fact.survey_instrument.strip()
):
errors.append(
_issue(
"missing_survey_instrument",
Expand Down
8 changes: 8 additions & 0 deletions chronicle/source_package.py
Original file line number Diff line number Diff line change
Expand Up @@ -93,6 +93,7 @@
"hmrc-salary-sacrifice-reform-2029-headcounts": Path(
"hmrc/salary_sacrifice_reform_2029_headcounts"
),
"hmrc-tax-free-childcare-march-2026": Path("hmrc/tax_free_childcare_march_2026"),
"ici-fact-book-table-30": Path("ici/fact_book_table_30"),
"isc-annual-census-2023": Path("isc/annual_census_2023"),
"isc-annual-census-2024": Path("isc/annual_census_2024"),
Expand Down Expand Up @@ -129,6 +130,9 @@
"fpb-economic-outlook-2026-2031-june-2026": Path(
"fpb/economic_outlook_2026_2031_june_2026"
),
"dfe-funded-early-education-childcare-2026": Path(
"dfe/funded_early_education_childcare_2026"
),
"dft-nts-vehicle-ownership-2024": Path("dft/nts_vehicle_ownership_2024"),
"dwp-benefit-cap-november-2025": Path("dwp/benefit_cap_november_2025"),
"dwp-benefit-statistics-february-2026": Path(
Expand Down Expand Up @@ -1872,6 +1876,10 @@ def _measure_from_mapping(
if payload.get("source_column_id") is not None
else None
),
source_column_dimensions={
str(key): _render_value(value, year=year)
for key, value in payload.get("source_column_dimensions", {}).items()
},
expected_cell_type=payload.get("expected_cell_type", "number"),
expected_column_header_row=(
int(payload["expected_column_header_row"])
Expand Down
3 changes: 3 additions & 0 deletions chronicle/sources/specs.py
Original file line number Diff line number Diff line change
Expand Up @@ -167,6 +167,7 @@ class SourceRecordSetMeasure:
divisor_column: str | None = None
round_to: int | float | None = None
source_column_id: str | None = None
source_column_dimensions: dict[str, Scalar] = field(default_factory=dict)
expected_cell_type: str = "number"
expected_column_header_row: int | None = None
expected_column_header: Scalar = None
Expand Down Expand Up @@ -846,6 +847,8 @@ def _record_set_spec_hash(spec: SourceRecordSetSpec) -> str:
measure.pop("divisor_column", None)
if measure.get("round_to") is None:
measure.pop("round_to", None)
if not measure.get("source_column_dimensions"):
measure.pop("source_column_dimensions", None)
if measure.get("expected_column_header") is None:
measure.pop("expected_column_header", None)
if measure.get("expected_column_header_row") is None:
Expand Down
84 changes: 78 additions & 6 deletions chronicle/suite.py
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,7 @@
)
from chronicle.sources.specs import (
SourceRecordSpec,
SourceRecordSetSpec,
SourceRegionSpec,
build_cells_by_sheet_address,
resolve_source_record,
Expand Down Expand Up @@ -320,12 +321,16 @@ def build_source_suite(
source_region_report.to_dict(),
)

source_record_set_specs = (
source_package.build_source_record_set_specs(year) if source_package else []
)
source_record_specs = (
source_package.build_source_record_specs(year)
if source_package
else build_source_record_specs(source, year=year)
)
source_record_report = validate_source_record_specs(
(
source_package.build_source_record_specs(year)
if source_package
else build_source_record_specs(source, year=year)
),
source_record_specs,
cells,
)
_write_report(
Expand Down Expand Up @@ -386,6 +391,9 @@ def build_source_suite(
fact_report=fact_report,
concept_alignments=concept_report,
require_axiom_validation=require_axiom_validation,
source_column_dimensions_by_record_id=(
_source_column_dimensions_by_record_id(source_record_set_specs)
),
selected_only_source_parse=(
bool(source_package)
and source_package.artifact.parser == "delimited_text_selected_rows"
Expand Down Expand Up @@ -558,6 +566,7 @@ def build_agent_acceptance_report(
fact_report: ValidationReport,
concept_alignments: ConceptAlignmentReport,
require_axiom_validation: bool = False,
source_column_dimensions_by_record_id: dict[str, dict[str, Any]] | None = None,
selected_only_source_parse: bool = False,
) -> AgentAcceptanceReport:
"""Build the stricter report agents should satisfy before review."""
Expand All @@ -570,6 +579,7 @@ def build_agent_acceptance_report(
source_rows_by_key = {build_source_row_key(row): row for row in rows}
source_row_keys = set(source_rows_by_key)
source_cells_by_key = {build_source_cell_key(cell): cell for cell in cells}
source_column_dimensions_by_record_id = source_column_dimensions_by_record_id or {}
raw_r2_link_count = 0

if not cells and not rows:
Expand Down Expand Up @@ -669,6 +679,12 @@ def build_agent_acceptance_report(
for key in fact.source_cell_keys
if key in source_cells_by_key
],
source_column_dimensions=(
source_column_dimensions_by_record_id.get(
fact.source_record_id or "",
{},
)
),
):
row_semantic_error_count += 1
errors.append(issue)
Expand Down Expand Up @@ -924,8 +940,11 @@ def _row_semantic_evidence_issues(
fact: AggregateFact,
rows: list[SourceRow],
cells: list[SourceCell],
*,
source_column_dimensions: dict[str, Any] | None = None,
) -> list[AgentAcceptanceIssue]:
issues: list[AgentAcceptanceIssue] = []
source_column_dimensions = source_column_dimensions or {}
fact_key = build_fact_key(fact)
period_values = _source_row_values(rows, "period")
for value in period_values:
Expand All @@ -943,12 +962,20 @@ def _row_semantic_evidence_issues(
)

for variable, value in fact.filters.items():
if value in (None, "all"):
if value is None:
continue
if value == "all" and not source_column_dimensions:
continue
matched_values = _source_row_values(rows, variable)
if not matched_values:
if _filter_evidenced_by_source_cells(cells, variable, value):
continue
if _wide_table_filter_evidenced_by_source_column(
source_column_dimensions,
variable,
value,
):
continue
issues.append(
AgentAcceptanceIssue(
code="row_filter_not_evidenced",
Expand Down Expand Up @@ -981,6 +1008,11 @@ def _row_semantic_evidence_issues(
continue
if _constraint_evidenced_by_source_cells(cells, constraint):
continue
if _wide_table_constraint_evidenced_by_source_column(
source_column_dimensions,
constraint,
):
continue
matched_values = _source_row_values(rows, constraint.variable)
if not matched_values:
issues.append(
Expand Down Expand Up @@ -1012,6 +1044,46 @@ def _row_semantic_evidence_issues(
return issues


def _wide_table_filter_evidenced_by_source_column(
source_column_dimensions: dict[str, Any],
variable: str,
expected: Any,
) -> bool:
"""Accept an explicitly declared dimension of a guarded source column."""
if variable not in source_column_dimensions:
return False
declared = source_column_dimensions[variable]
return type(declared) is type(expected) and declared == expected


def _wide_table_constraint_evidenced_by_source_column(
source_column_dimensions: dict[str, Any],
constraint: Any,
) -> bool:
if constraint.operator != "==":
return False
return _wide_table_filter_evidenced_by_source_column(
source_column_dimensions,
str(constraint.variable),
constraint.value,
)


def _source_column_dimensions_by_record_id(
record_sets: list[SourceRecordSetSpec],
) -> dict[str, dict[str, Any]]:
"""Index explicit wide-column dimensions without changing fact payloads."""
return {
f"{record_set.source_record_id_prefix}.{row.value_id}.{measure.measure_id}": (
dict(measure.source_column_dimensions)
)
for record_set in record_sets
for row in record_set.rows
for measure in record_set.measures
if measure.source_column_dimensions
}


def _constraint_evidenced_by_source_cells(
cells: list[SourceCell],
constraint: Any,
Expand Down
Loading
Loading