Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
20 changes: 20 additions & 0 deletions chronicle/bundle.py
Original file line number Diff line number Diff line change
Expand Up @@ -25,8 +25,10 @@
BUNDLE_SOURCES_SCHEMA_VERSION = schema_id("bundle_sources")
DEFAULT_BUNDLE_SOURCES = tuple(sorted(SOURCE_PACKAGE_ALIASES))
UK_BUNDLE_SOURCE_PREFIXES = (
"desnz",
"dfc_ni",
"dfe",
"dfi_ni",
"dft",
"dwp",
"hmrc",
Expand All @@ -35,19 +37,27 @@
"nisra",
"nrs",
"obr",
"ofgem",
"ons",
"orr",
"scotgov",
"slc",
"voa",
"welshgov",
)
UK_BUNDLE_SOURCES = (
"desnz-monthly-annual-road-fuel-prices-august-2026",
"desnz-need-england-wales-2023",
"desnz-need-scotland-2023",
"desnz-weekly-road-fuel-prices-september-2026",
"dfc-ni-uc-statistics-may-2026",
"dfe-funded-early-education-childcare-2026",
"dfi-ni-public-transport-statistics-2024-25",
"dft-bus0415-fares-index-2026",
"dft-bus05i-revenue-support-2025",
"dft-nts-vehicle-ownership-2024",
"dft-nts0705-local-bus-trips-2024",
"dft-veh1103-cars-fuel-type-2025",
"dwp-benefit-cap-november-2025",
"dwp-benefit-statistics-february-2026",
"dwp-pip-daily-living-foi-2025",
Expand Down Expand Up @@ -75,6 +85,7 @@
"hmrc-cgt-size-of-gain-2026",
"hmrc-cgt-statistics-2026",
"hmrc-child-benefit-august-2025",
"hmrc-hydrocarbon-oils-quantities-june-2026",
"hmrc-salary-sacrifice-reform-2029-headcounts",
"hmrc-salary-sacrifice-relief-2024-25",
"hmrc-spi-income-bands-2023-24",
Expand All @@ -101,10 +112,13 @@
"obr-efo-economy-march-2026",
"obr-efo-expenditure-march-2026",
"obr-efo-receipts-march-2026",
"obr-fuel-duty-receipts-by-vehicle-april-2024",
"ofgem-energy-price-cap-q1-2024",
"ons-census2021-ts003-household-composition-country",
"ons-census2021-ts041-households-lad",
"ons-census2021-ts041-households-pcon24",
"ons-census2021-ts054-tenure-lad",
"ons-consumer-trends-current-price-2026",
"ons-families-households-2025",
"ons-households-by-type-country-2025",
"ons-lad-population-by-age-2024",
Expand All @@ -122,8 +136,13 @@
"ons-uk-business-firm-sector-targets-2025",
"ons-uk-business-firm-targets-2025",
"ons-uk-population-projections-2024",
"orr-government-support-7270-2024-25",
"orr-government-support-7271-2024-25",
"orr-rail-fares-7180-2026",
"orr-rail-finance-7223-2024-25",
"scotgov-band-d-council-tax-rates-2026-27",
"scotgov-band-d-equivalents-2025",
"scotgov-bus-coach-statistics-2023-24",
"scotgov-council-tax-bands-2025",
"scotgov-council-tax-collection-2024-25",
"scotgov-council-tax-collection-2025-26",
Expand All @@ -137,6 +156,7 @@
"slc-student-support-england-2025",
"voa-council-tax-bands-2025",
"voa-council-tax-stock-by-lad-2025",
"welshgov-bus-statistics-2024-25",
"welshgov-council-tax-collection-2024-25",
"welshgov-council-tax-collection-2025-26",
"welshgov-council-tax-levels-2026-27",
Expand Down
5 changes: 5 additions & 0 deletions chronicle/core.py
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,9 @@
# New Zealand labels its April-March income tax year by the ending year, so the
# publisher's "2024 tax year" is tax_year 2024 with PeriodCoverage 2023-04-01
# through 2024-03-31 and the publisher label recorded in source_period_label.
# Quarter values use YYYY-QN and ISO week values use YYYY-Www; exact coverage
# dates remain explicit in PeriodCoverage so source observation dates are not
# inferred from the compact period identity.
# Caution: the EES helper _academic_year_end (chronicle/sources/rows.py) names
# value COLUMNS
# by the academic year's END year ("2024/25" -> 2025); that is source-layout
Expand All @@ -37,6 +40,8 @@
"fiscal_year",
"academic_year",
"month",
"quarter",
"week",
}
ALLOWED_GEOGRAPHY_LEVELS = {
"country",
Expand Down
30 changes: 30 additions & 0 deletions chronicle/source_package.py
Original file line number Diff line number Diff line change
Expand Up @@ -141,9 +141,21 @@
),
"dft-bus0415-fares-index-2026": Path("dft/bus0415_fares_index_2026"),
"dft-bus05i-revenue-support-2025": Path("dft/bus05i_revenue_support_2025"),
"dft-veh1103-cars-fuel-type-2025": Path("dft/veh1103_cars_fuel_type_2025"),
"dft-nts0705-local-bus-trips-2024": Path("dft/nts0705_local_bus_trips_2024"),
"dft-nts-vehicle-ownership-2024": Path("dft/nts_vehicle_ownership_2024"),
"dfc-ni-uc-statistics-may-2026": Path("dfc_ni/uc_statistics_may_2026"),
"dfi-ni-public-transport-statistics-2024-25": Path(
"dfi_ni/public_transport_statistics_2024_25"
),
"desnz-monthly-annual-road-fuel-prices-august-2026": Path(
"desnz/monthly_annual_road_fuel_prices_august_2026"
),
"desnz-need-england-wales-2023": Path("desnz/need_england_wales_2023"),
"desnz-need-scotland-2023": Path("desnz/need_scotland_2023"),
"desnz-weekly-road-fuel-prices-september-2026": Path(
"desnz/weekly_road_fuel_prices_september_2026"
),
"dwp-benefit-cap-november-2025": Path("dwp/benefit_cap_november_2025"),
"dwp-benefit-statistics-february-2026": Path(
"dwp/benefit_statistics_february_2026"
Expand Down Expand Up @@ -307,6 +319,9 @@
"hmrc-vat-firm-sector-targets-2024-25": Path(
"hmrc/vat_firm_sector_targets_2024_25"
),
"hmrc-hydrocarbon-oils-quantities-june-2026": Path(
"hmrc/hydrocarbon_oils_quantities_june_2026"
),
"eurostat-gov-10a-taxag": Path("eurostat/gov_10a_taxag"),
"eurostat-gov-10a-taxag-2025": Path("eurostat/gov_10a_taxag_2025"),
"eurostat-spr-exp-func": Path("eurostat/spr_exp_func"),
Expand Down Expand Up @@ -335,6 +350,9 @@
"ons/subnational_dwellings_by_tenure_2024"
),
"ons-pipr-rents-by-area-june-2026": Path("ons/pipr_rents_by_area_june_2026"),
"ons-consumer-trends-current-price-2026": Path(
"ons/consumer_trends_current_price_2026"
),
"nrs-census2022-households-ukpc24": Path("nrs/census2022_households_ukpc24"),
"nrs-pcon24-population-by-age-2024": Path("nrs/pcon24_population_by_age_2024"),
"nrs-census2022-uv113-household-composition-country": Path(
Expand Down Expand Up @@ -366,6 +384,9 @@
"scotgov-scottish-budget-social-security-assistance-2026": Path(
"scotgov/scottish_budget_social_security_assistance_2026"
),
"scotgov-bus-coach-statistics-2023-24": Path(
"scotgov/bus_coach_statistics_2023_24"
),
"welshgov-council-tax-levels-2026-27": Path("welshgov/council_tax_levels_2026_27"),
"welshgov-council-tax-collection-2024-25": Path(
"welshgov/council_tax_collection_2024_25"
Expand All @@ -375,12 +396,21 @@
),
"welshgov-ctrs-annual-report-2024-25": Path("welshgov/ctrs_annual_report_2024_25"),
"welshgov-ctrs-annual-report-2025-26": Path("welshgov/ctrs_annual_report_2025_26"),
"welshgov-bus-statistics-2024-25": Path("welshgov/bus_statistics_2024_25"),
"voa-council-tax-bands-2025": Path("voa/council_tax_bands_2025"),
"voa-council-tax-stock-by-lad-2025": Path("voa/council_tax_stock_by_lad_2025"),
"obr-efo-receipts-march-2026": Path("obr/efo_receipts_march_2026"),
"obr-efo-expenditure-march-2026": Path("obr/efo_expenditure_march_2026"),
"obr-efo-economy-march-2026": Path("obr/efo_economy_march_2026"),
"obr-efo-aggregates-march-2026": Path("obr/efo_aggregates_march_2026"),
"obr-fuel-duty-receipts-by-vehicle-april-2024": Path(
"obr/fuel_duty_receipts_by_vehicle_april_2024"
),
"ofgem-energy-price-cap-q1-2024": Path("ofgem/energy_price_cap_q1_2024"),
"orr-government-support-7270-2024-25": Path("orr/government_support_7270_2024_25"),
"orr-government-support-7271-2024-25": Path("orr/government_support_7271_2024_25"),
"orr-rail-fares-7180-2026": Path("orr/rail_fares_7180_2026"),
"orr-rail-finance-7223-2024-25": Path("orr/rail_finance_7223_2024_25"),
"ons-national-balance-sheet-land-2025": Path(
"ons/national_balance_sheet_land_2025"
),
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,18 @@
source_id: desnz
package_id: desnz-monthly-annual-road-fuel-prices-august-2026
dataset: desnz_desnz-monthly-annual-road-fuel-prices-august-2026
source_page: https://www.gov.uk/government/statistical-data-sets/oil-and-petroleum-products-monthly-statistics
table: Digest of UK Energy Statistics tables 4.1.1 to 4.1.3 workbook; emitted sheets 4.1.1 and 4.1.2
files:
2026:
filename: table_411_413.xlsx
source_url: https://assets.publishing.service.gov.uk/media/6a8d7214f5b35599aec18cfe/table_411_413.xlsx
sha256: 30b19fdfda95adcf73ef4cdb5ab9993223595ebd3c0284d76a86210264dd358f
size_bytes: 542455
fetched_at: '2026-09-09T00:00:00+00:00'
storage:
r2:
provider: r2
bucket: ledger-raw
key: raw/desnz/desnz-monthly-annual-road-fuel-prices-august-2026/2026/30b19fdfda95adcf73ef4cdb5ab9993223595ebd3c0284d76a86210264dd358f/table_411_413.xlsx
uri: r2://ledger-raw/raw/desnz/desnz-monthly-annual-road-fuel-prices-august-2026/2026/30b19fdfda95adcf73ef4cdb5ab9993223595ebd3c0284d76a86210264dd358f/table_411_413.xlsx
Binary file not shown.
18 changes: 18 additions & 0 deletions db/data/desnz/need_england_wales_2023/manifest.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,18 @@
source_id: desnz
package_id: desnz-need-england-wales-2023
dataset: desnz_desnz-need-england-wales-2023
source_page: https://www.gov.uk/government/statistics/national-energy-efficiency-data-framework-need-report-summary-of-analysis-2025
table: NEED consumption headline tables, England and Wales, 2023
files:
2023:
filename: need-ew-2023.xlsx
source_url: https://assets.publishing.service.gov.uk/media/685bd3947d72089d199760d5/Consumption_headline_EW_2023.xlsx
sha256: b46fa7bb9e9bc4e048077b6e4ed432bc53dfaa1f02152fa8cbb29f3c9a99f0f9
size_bytes: 508380
fetched_at: '2026-09-09T00:00:00+00:00'
storage:
r2:
provider: r2
bucket: ledger-raw
key: raw/desnz/desnz-need-england-wales-2023/2023/b46fa7bb9e9bc4e048077b6e4ed432bc53dfaa1f02152fa8cbb29f3c9a99f0f9/need-ew-2023.xlsx
uri: r2://ledger-raw/raw/desnz/desnz-need-england-wales-2023/2023/b46fa7bb9e9bc4e048077b6e4ed432bc53dfaa1f02152fa8cbb29f3c9a99f0f9/need-ew-2023.xlsx
Binary file not shown.
18 changes: 18 additions & 0 deletions db/data/desnz/need_scotland_2023/manifest.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,18 @@
source_id: desnz
package_id: desnz-need-scotland-2023
dataset: desnz_desnz-need-scotland-2023
source_page: https://www.gov.uk/government/statistics/national-energy-efficiency-data-framework-need-report-summary-of-analysis-2025
table: NEED consumption headline tables, Scotland, 2023
files:
2023:
filename: need-scotland-2023.xlsx
source_url: https://assets.publishing.service.gov.uk/media/685bd3acc07c71e5a87097de/Consumption_headline_Scotland_2023.xlsx
sha256: d1956714384789c7626efe1ffb0a1e05bb5064c8c52c139fcd4a7c222807377d
size_bytes: 361542
fetched_at: '2026-09-09T00:00:00+00:00'
storage:
r2:
provider: r2
bucket: ledger-raw
key: raw/desnz/desnz-need-scotland-2023/2023/d1956714384789c7626efe1ffb0a1e05bb5064c8c52c139fcd4a7c222807377d/need-scotland-2023.xlsx
uri: r2://ledger-raw/raw/desnz/desnz-need-scotland-2023/2023/d1956714384789c7626efe1ffb0a1e05bb5064c8c52c139fcd4a7c222807377d/need-scotland-2023.xlsx
Binary file not shown.
Binary file not shown.
18 changes: 18 additions & 0 deletions db/data/desnz/weekly_road_fuel_prices_september_2026/manifest.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,18 @@
source_id: desnz
package_id: desnz-weekly-road-fuel-prices-september-2026
dataset: desnz_desnz-weekly-road-fuel-prices-september-2026
source_page: https://www.gov.uk/government/statistical-data-sets/oil-and-petroleum-products-weekly-statistics
table: Weekly road fuel prices
files:
2026:
filename: Weekly_Fuel_Prices_07-09-26.xlsx
source_url: https://assets.publishing.service.gov.uk/media/6a9ec4436beb5206555555d1/Weekly_Fuel_Prices_07-09-26.xlsx
sha256: 65cfb716a6934ae6df82c3213c4f6f9dfa551ecb7075400e1f7eeb343eeb0ad7
size_bytes: 260185
fetched_at: '2026-09-09T00:00:00+00:00'
storage:
r2:
provider: r2
bucket: ledger-raw
key: raw/desnz/desnz-weekly-road-fuel-prices-september-2026/2026/65cfb716a6934ae6df82c3213c4f6f9dfa551ecb7075400e1f7eeb343eeb0ad7/Weekly_Fuel_Prices_07-09-26.xlsx
uri: r2://ledger-raw/raw/desnz/desnz-weekly-road-fuel-prices-september-2026/2026/65cfb716a6934ae6df82c3213c4f6f9dfa551ecb7075400e1f7eeb343eeb0ad7/Weekly_Fuel_Prices_07-09-26.xlsx
18 changes: 18 additions & 0 deletions db/data/dfi_ni/public_transport_statistics_2024_25/manifest.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,18 @@
source_id: dfi_ni
package_id: dfi-ni-public-transport-statistics-2024-25
dataset: dfi_ni_dfi-ni-public-transport-statistics-2024-25
source_page: https://www.infrastructure-ni.gov.uk/publications/public-transport-statistics-northern-ireland-2024-25
table: Public Transport Statistics Northern Ireland 2024-25 figures 3 and 4
files:
2025:
filename: public-transport-statistics-2024-25.xlsx
source_url: https://www.infrastructure-ni.gov.uk/system/files/2025-11/public-transport-statistics-2024-2025-tables-and-figures.xlsx
sha256: 8e0a0f14410a25f16a13b26d312cfd4e3dac55129ed0c541165679aaa9fcd6ca
size_bytes: 1099593
fetched_at: '2026-09-09T00:00:00+00:00'
storage:
r2:
provider: r2
bucket: ledger-raw
key: raw/dfi_ni/dfi-ni-public-transport-statistics-2024-25/2025/8e0a0f14410a25f16a13b26d312cfd4e3dac55129ed0c541165679aaa9fcd6ca/public-transport-statistics-2024-25.xlsx
uri: r2://ledger-raw/raw/dfi_ni/dfi-ni-public-transport-statistics-2024-25/2025/8e0a0f14410a25f16a13b26d312cfd4e3dac55129ed0c541165679aaa9fcd6ca/public-transport-statistics-2024-25.xlsx
Binary file not shown.
18 changes: 18 additions & 0 deletions db/data/dft/veh1103_cars_fuel_type_2025/manifest.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,18 @@
source_id: dft
package_id: dft-veh1103-cars-fuel-type-2025
dataset: dft_dft-veh1103-cars-fuel-type-2025
source_page: https://www.gov.uk/government/statistical-data-sets/vehicle-licensing-statistics-data-tables
table: VEH1103a licensed vehicles by body type and fuel type
files:
2025:
filename: veh1103.ods
source_url: https://assets.publishing.service.gov.uk/media/6a5613d531fb6daf31413625/veh1103.ods
sha256: 2e1107fd9ea55f213ee8c207018cb445b8217932211c65136dd05a715e244849
size_bytes: 634787
fetched_at: '2026-09-09T00:00:00+00:00'
storage:
r2:
provider: r2
bucket: ledger-raw
key: raw/dft/dft-veh1103-cars-fuel-type-2025/2025/2e1107fd9ea55f213ee8c207018cb445b8217932211c65136dd05a715e244849/veh1103.ods
uri: r2://ledger-raw/raw/dft/dft-veh1103-cars-fuel-type-2025/2025/2e1107fd9ea55f213ee8c207018cb445b8217932211c65136dd05a715e244849/veh1103.ods
Binary file not shown.
Binary file not shown.
18 changes: 18 additions & 0 deletions db/data/hmrc/hydrocarbon_oils_quantities_june_2026/manifest.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,18 @@
source_id: hmrc
package_id: hmrc-hydrocarbon-oils-quantities-june-2026
dataset: hmrc_hmrc-hydrocarbon-oils-quantities-june-2026
source_page: https://www.gov.uk/government/statistics/hydrocarbon-oils-bulletin
table: Hydrocarbon Oils Bulletin tables 2a and 2c
files:
2026:
filename: Oils_Tab_Jun_26.ods
source_url: https://assets.publishing.service.gov.uk/media/6a68808f83f71b8684d020c6/Oils_Tab_Jun_26.ods
sha256: a8c237fb4adc7a0b9bcb76c1c1737deb567a288c0bad56c2ffb8c35e9c861b97
size_bytes: 216510
fetched_at: '2026-09-09T00:00:00+00:00'
storage:
r2:
provider: r2
bucket: ledger-raw
key: raw/hmrc/hmrc-hydrocarbon-oils-quantities-june-2026/2026/a8c237fb4adc7a0b9bcb76c1c1737deb567a288c0bad56c2ffb8c35e9c861b97/Oils_Tab_Jun_26.ods
uri: r2://ledger-raw/raw/hmrc/hmrc-hydrocarbon-oils-quantities-june-2026/2026/a8c237fb4adc7a0b9bcb76c1c1737deb567a288c0bad56c2ffb8c35e9c861b97/Oils_Tab_Jun_26.ods
Binary file not shown.
18 changes: 18 additions & 0 deletions db/data/obr/fuel_duty_receipts_by_vehicle_april_2024/manifest.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,18 @@
source_id: obr
package_id: obr-fuel-duty-receipts-by-vehicle-april-2024
dataset: obr_obr-fuel-duty-receipts-by-vehicle-april-2024
source_page: https://obr.uk/efo/economic-and-fiscal-outlook-march-2024/
table: 'Fuel duty supplementary release: receipts by vehicle type'
files:
2024:
filename: fuel-duty-receipts-by-vehicle-type.pdf
source_url: https://obr.uk/docs/dlm_uploads/Fuel-duty-supplementary-release_receipts-by-vehicle-type.pdf
sha256: 4c5a65745550fbc62fb281c43dbde602a64f465b6095e3383e0473d66b31f018
size_bytes: 110064
fetched_at: '2026-09-09T00:00:00+00:00'
storage:
r2:
provider: r2
bucket: ledger-raw
key: raw/obr/obr-fuel-duty-receipts-by-vehicle-april-2024/2024/4c5a65745550fbc62fb281c43dbde602a64f465b6095e3383e0473d66b31f018/fuel-duty-receipts-by-vehicle-type.pdf
uri: r2://ledger-raw/raw/obr/obr-fuel-duty-receipts-by-vehicle-april-2024/2024/4c5a65745550fbc62fb281c43dbde602a64f465b6095e3383e0473d66b31f018/fuel-duty-receipts-by-vehicle-type.pdf
18 changes: 18 additions & 0 deletions db/data/ofgem/energy_price_cap_q1_2024/manifest.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,18 @@
source_id: ofgem
package_id: ofgem-energy-price-cap-q1-2024
dataset: ofgem_ofgem-energy-price-cap-q1-2024
source_page: https://www.ofgem.gov.uk/news/new-energy-price-cap-level-january-march-2024-starts-today
table: New energy price cap level for January to March 2024
files:
2024:
filename: ofgem-price-cap-q1-2024.html
source_url: https://www.ofgem.gov.uk/news/new-energy-price-cap-level-january-march-2024-starts-today
sha256: 07eb8e53a5023e70878828dd3cdecf95863253e30ce0d3c2255017be26fe0db7
size_bytes: 202876
fetched_at: '2026-09-09T00:00:00+00:00'
storage:
r2:
provider: r2
bucket: ledger-raw
key: raw/ofgem/ofgem-energy-price-cap-q1-2024/2024/07eb8e53a5023e70878828dd3cdecf95863253e30ce0d3c2255017be26fe0db7/ofgem-price-cap-q1-2024.html
uri: r2://ledger-raw/raw/ofgem/ofgem-energy-price-cap-q1-2024/2024/07eb8e53a5023e70878828dd3cdecf95863253e30ce0d3c2255017be26fe0db7/ofgem-price-cap-q1-2024.html
Loading
Loading