diff --git a/.gitattributes b/.gitattributes index 2e52db14e..c41f1e96c 100644 --- a/.gitattributes +++ b/.gitattributes @@ -1,13 +1,25 @@ # Binary assets. Marked so git does not attempt line-ending conversion or # textual diffs on them. # -# Not tracked in Git LFS. As of the first committed image (the nowcast progress -# report figures, ~120 KB at REPORT_DPI) the repo holds no other binaries and -# packs to ~19 MB, so LFS would add a required tool for every contributor and -# every CI job to save a fraction of a percent. Revisit if these figures start -# churning per-commit rather than per-milestone: an LFS migration is easy going -# forward, it is only the already-committed blobs that stay in history. -*.png binary +# PNG figures are stored in Git LFS. The original call was to keep them as +# ordinary blobs -- at the first committed image the repo packed to ~19 MB and +# LFS would have added a required tool for every contributor and every CI job +# to save a fraction of a percent -- and it named the condition for revisiting: +# "if these figures start churning per-commit rather than per-milestone". That +# is what happened. The report's 30 comparison figures are re-rendered whenever +# the colour scheme or the tables underneath them move, 1.16 MB of +# incompressible blobs each time, and that ran twice in three days. +# +# Two consequences, both deliberate. Working on this repo now needs git-lfs +# installed. And blobs committed before this change stay in history as blobs: +# only new content becomes a pointer, so the repo does not get smaller, it +# stops growing at this rate. Converting the history as well would rewrite +# main and every open branch, which is a separate decision and not this one. +# +# The nine committed .parquet fixtures are 3.1 MB and are still ordinary +# blobs. They are static reference artifacts rather than regenerated output, +# so they do not churn; move them here if that stops being true. +*.png filter=lfs diff=lfs merge=lfs -text *.jpg binary *.jpeg binary *.gif binary diff --git a/bedrock/analysis/electricity/historical/original_vs_eia_anchored_deck/figures/v0.2_original_electricity_disagg_D.png b/bedrock/analysis/electricity/historical/original_vs_eia_anchored_deck/figures/v0.2_original_electricity_disagg_D.png index 5fb001bbb..c22476135 100644 Binary files a/bedrock/analysis/electricity/historical/original_vs_eia_anchored_deck/figures/v0.2_original_electricity_disagg_D.png and b/bedrock/analysis/electricity/historical/original_vs_eia_anchored_deck/figures/v0.2_original_electricity_disagg_D.png differ diff --git a/bedrock/analysis/electricity/historical/original_vs_eia_anchored_deck/figures/v0.2_original_electricity_disagg_N.png b/bedrock/analysis/electricity/historical/original_vs_eia_anchored_deck/figures/v0.2_original_electricity_disagg_N.png index 92f0b6ff4..ce95d2d3e 100644 Binary files a/bedrock/analysis/electricity/historical/original_vs_eia_anchored_deck/figures/v0.2_original_electricity_disagg_N.png and b/bedrock/analysis/electricity/historical/original_vs_eia_anchored_deck/figures/v0.2_original_electricity_disagg_N.png differ diff --git a/bedrock/analysis/electricity/historical/original_vs_eia_anchored_deck/figures/v0.3_eia_gtd_pre_mecs_D.png b/bedrock/analysis/electricity/historical/original_vs_eia_anchored_deck/figures/v0.3_eia_gtd_pre_mecs_D.png index c98a85f0b..2e14cc49c 100644 Binary files a/bedrock/analysis/electricity/historical/original_vs_eia_anchored_deck/figures/v0.3_eia_gtd_pre_mecs_D.png and b/bedrock/analysis/electricity/historical/original_vs_eia_anchored_deck/figures/v0.3_eia_gtd_pre_mecs_D.png differ diff --git a/bedrock/analysis/electricity/historical/original_vs_eia_anchored_deck/figures/v0.3_eia_gtd_pre_mecs_N.png b/bedrock/analysis/electricity/historical/original_vs_eia_anchored_deck/figures/v0.3_eia_gtd_pre_mecs_N.png index 1b9df52c4..eb317b69e 100644 Binary files a/bedrock/analysis/electricity/historical/original_vs_eia_anchored_deck/figures/v0.3_eia_gtd_pre_mecs_N.png and b/bedrock/analysis/electricity/historical/original_vs_eia_anchored_deck/figures/v0.3_eia_gtd_pre_mecs_N.png differ diff --git a/bedrock/analysis/nowcasting/images/final_demand_provenance_2017.png b/bedrock/analysis/nowcasting/images/final_demand_provenance_2017.png index e2cd417b5..4976cae1a 100644 Binary files a/bedrock/analysis/nowcasting/images/final_demand_provenance_2017.png and b/bedrock/analysis/nowcasting/images/final_demand_provenance_2017.png differ diff --git a/bedrock/analysis/nowcasting/images/go_vs_nowcast_mut.png b/bedrock/analysis/nowcasting/images/go_vs_nowcast_mut.png index 828b4c159..f5f1f2c80 100644 Binary files a/bedrock/analysis/nowcasting/images/go_vs_nowcast_mut.png and b/bedrock/analysis/nowcasting/images/go_vs_nowcast_mut.png differ diff --git a/bedrock/analysis/nowcasting/images/intermediate_seed_coverage_2021.png b/bedrock/analysis/nowcasting/images/intermediate_seed_coverage_2021.png index 212fd3c7e..5d29889b3 100644 Binary files a/bedrock/analysis/nowcasting/images/intermediate_seed_coverage_2021.png and b/bedrock/analysis/nowcasting/images/intermediate_seed_coverage_2021.png differ diff --git a/bedrock/analysis/nowcasting/images/intermediate_seed_coverage_2022.png b/bedrock/analysis/nowcasting/images/intermediate_seed_coverage_2022.png index 8d1a9d6e6..c93704a8b 100644 Binary files a/bedrock/analysis/nowcasting/images/intermediate_seed_coverage_2022.png and b/bedrock/analysis/nowcasting/images/intermediate_seed_coverage_2022.png differ diff --git a/bedrock/analysis/nowcasting/images/make_after_redef_detail_mut_2017.png b/bedrock/analysis/nowcasting/images/make_after_redef_detail_mut_2017.png index 3a34955d0..51d532882 100644 Binary files a/bedrock/analysis/nowcasting/images/make_after_redef_detail_mut_2017.png and b/bedrock/analysis/nowcasting/images/make_after_redef_detail_mut_2017.png differ diff --git a/bedrock/analysis/nowcasting/images/supply_bridge_detail_sut_2017.png b/bedrock/analysis/nowcasting/images/supply_bridge_detail_sut_2017.png index ba4229cd8..f2bc117b5 100644 Binary files a/bedrock/analysis/nowcasting/images/supply_bridge_detail_sut_2017.png and b/bedrock/analysis/nowcasting/images/supply_bridge_detail_sut_2017.png differ diff --git a/bedrock/analysis/nowcasting/images/supply_bridge_detail_sut_seed_2017.png b/bedrock/analysis/nowcasting/images/supply_bridge_detail_sut_seed_2017.png index 9b7efdb3a..7d06c23cb 100644 Binary files a/bedrock/analysis/nowcasting/images/supply_bridge_detail_sut_seed_2017.png and b/bedrock/analysis/nowcasting/images/supply_bridge_detail_sut_seed_2017.png differ diff --git a/bedrock/analysis/nowcasting/images/supply_bridge_provenance_2017.png b/bedrock/analysis/nowcasting/images/supply_bridge_provenance_2017.png index dd5e432f2..20b6c529e 100644 Binary files a/bedrock/analysis/nowcasting/images/supply_bridge_provenance_2017.png and b/bedrock/analysis/nowcasting/images/supply_bridge_provenance_2017.png differ diff --git a/bedrock/analysis/nowcasting/images/supply_output_detail_sut_2017.png b/bedrock/analysis/nowcasting/images/supply_output_detail_sut_2017.png index 9f564b095..ce43a8f6c 100644 Binary files a/bedrock/analysis/nowcasting/images/supply_output_detail_sut_2017.png and b/bedrock/analysis/nowcasting/images/supply_output_detail_sut_2017.png differ diff --git a/bedrock/analysis/nowcasting/images/supply_output_detail_sut_seed_2017.png b/bedrock/analysis/nowcasting/images/supply_output_detail_sut_seed_2017.png index 8f3f5fb05..2b4f5f156 100644 Binary files a/bedrock/analysis/nowcasting/images/supply_output_detail_sut_seed_2017.png and b/bedrock/analysis/nowcasting/images/supply_output_detail_sut_seed_2017.png differ diff --git a/bedrock/analysis/nowcasting/images/supply_summary_sut_2017.png b/bedrock/analysis/nowcasting/images/supply_summary_sut_2017.png index c57f2c971..2858b2626 100644 Binary files a/bedrock/analysis/nowcasting/images/supply_summary_sut_2017.png and b/bedrock/analysis/nowcasting/images/supply_summary_sut_2017.png differ diff --git a/bedrock/analysis/nowcasting/images/supply_summary_sut_2018.png b/bedrock/analysis/nowcasting/images/supply_summary_sut_2018.png index 8b8d0c534..d43544b46 100644 Binary files a/bedrock/analysis/nowcasting/images/supply_summary_sut_2018.png and b/bedrock/analysis/nowcasting/images/supply_summary_sut_2018.png differ diff --git a/bedrock/analysis/nowcasting/images/supply_summary_sut_2019.png b/bedrock/analysis/nowcasting/images/supply_summary_sut_2019.png index 0a4ba691f..1b652fcd3 100644 Binary files a/bedrock/analysis/nowcasting/images/supply_summary_sut_2019.png and b/bedrock/analysis/nowcasting/images/supply_summary_sut_2019.png differ diff --git a/bedrock/analysis/nowcasting/images/supply_summary_sut_2020.png b/bedrock/analysis/nowcasting/images/supply_summary_sut_2020.png index c0b67e8b5..217428596 100644 Binary files a/bedrock/analysis/nowcasting/images/supply_summary_sut_2020.png and b/bedrock/analysis/nowcasting/images/supply_summary_sut_2020.png differ diff --git a/bedrock/analysis/nowcasting/images/supply_summary_sut_2021.png b/bedrock/analysis/nowcasting/images/supply_summary_sut_2021.png index 3f6d69700..690411970 100644 Binary files a/bedrock/analysis/nowcasting/images/supply_summary_sut_2021.png and b/bedrock/analysis/nowcasting/images/supply_summary_sut_2021.png differ diff --git a/bedrock/analysis/nowcasting/images/supply_summary_sut_2022.png b/bedrock/analysis/nowcasting/images/supply_summary_sut_2022.png index 8263e8ef7..1fa00a42e 100644 Binary files a/bedrock/analysis/nowcasting/images/supply_summary_sut_2022.png and b/bedrock/analysis/nowcasting/images/supply_summary_sut_2022.png differ diff --git a/bedrock/analysis/nowcasting/images/supply_summary_sut_2023.png b/bedrock/analysis/nowcasting/images/supply_summary_sut_2023.png index d62144dbd..9b6841eb9 100644 Binary files a/bedrock/analysis/nowcasting/images/supply_summary_sut_2023.png and b/bedrock/analysis/nowcasting/images/supply_summary_sut_2023.png differ diff --git a/bedrock/analysis/nowcasting/images/supply_summary_sut_2024.png b/bedrock/analysis/nowcasting/images/supply_summary_sut_2024.png index d5842812a..c5a3dab92 100644 Binary files a/bedrock/analysis/nowcasting/images/supply_summary_sut_2024.png and b/bedrock/analysis/nowcasting/images/supply_summary_sut_2024.png differ diff --git a/bedrock/analysis/nowcasting/images/supply_table_provenance_2017.png b/bedrock/analysis/nowcasting/images/supply_table_provenance_2017.png index 4ba96e74e..be8fa1bb6 100644 Binary files a/bedrock/analysis/nowcasting/images/supply_table_provenance_2017.png and b/bedrock/analysis/nowcasting/images/supply_table_provenance_2017.png differ diff --git a/bedrock/analysis/nowcasting/images/uimp_after_redef_detail_mut_2017.png b/bedrock/analysis/nowcasting/images/uimp_after_redef_detail_mut_2017.png index 309af6770..cc0d6eaee 100644 Binary files a/bedrock/analysis/nowcasting/images/uimp_after_redef_detail_mut_2017.png and b/bedrock/analysis/nowcasting/images/uimp_after_redef_detail_mut_2017.png differ diff --git a/bedrock/analysis/nowcasting/images/use_after_redef_detail_mut_2017.png b/bedrock/analysis/nowcasting/images/use_after_redef_detail_mut_2017.png index ff8d789ee..cc8c84e47 100644 Binary files a/bedrock/analysis/nowcasting/images/use_after_redef_detail_mut_2017.png and b/bedrock/analysis/nowcasting/images/use_after_redef_detail_mut_2017.png differ diff --git a/bedrock/analysis/nowcasting/images/use_fd_detail_sut_2017.png b/bedrock/analysis/nowcasting/images/use_fd_detail_sut_2017.png index a8a05b7b6..bdb6bd019 100644 Binary files a/bedrock/analysis/nowcasting/images/use_fd_detail_sut_2017.png and b/bedrock/analysis/nowcasting/images/use_fd_detail_sut_2017.png differ diff --git a/bedrock/analysis/nowcasting/images/use_fd_detail_sut_seed_2017.png b/bedrock/analysis/nowcasting/images/use_fd_detail_sut_seed_2017.png index 65f3cfccf..c73d4e1a4 100644 Binary files a/bedrock/analysis/nowcasting/images/use_fd_detail_sut_seed_2017.png and b/bedrock/analysis/nowcasting/images/use_fd_detail_sut_seed_2017.png differ diff --git a/bedrock/analysis/nowcasting/images/use_intermediate_detail_sut_2017.png b/bedrock/analysis/nowcasting/images/use_intermediate_detail_sut_2017.png index bef193af6..efbd94b47 100644 Binary files a/bedrock/analysis/nowcasting/images/use_intermediate_detail_sut_2017.png and b/bedrock/analysis/nowcasting/images/use_intermediate_detail_sut_2017.png differ diff --git a/bedrock/analysis/nowcasting/images/use_intermediate_detail_sut_seed_2017.png b/bedrock/analysis/nowcasting/images/use_intermediate_detail_sut_seed_2017.png index ca6fe478d..5f59049ba 100644 Binary files a/bedrock/analysis/nowcasting/images/use_intermediate_detail_sut_seed_2017.png and b/bedrock/analysis/nowcasting/images/use_intermediate_detail_sut_seed_2017.png differ diff --git a/bedrock/analysis/nowcasting/images/use_summary_sut_2017.png b/bedrock/analysis/nowcasting/images/use_summary_sut_2017.png index ede974319..dc4a9eaa5 100644 Binary files a/bedrock/analysis/nowcasting/images/use_summary_sut_2017.png and b/bedrock/analysis/nowcasting/images/use_summary_sut_2017.png differ diff --git a/bedrock/analysis/nowcasting/images/use_summary_sut_2018.png b/bedrock/analysis/nowcasting/images/use_summary_sut_2018.png index 8f49c630f..8ee742b07 100644 Binary files a/bedrock/analysis/nowcasting/images/use_summary_sut_2018.png and b/bedrock/analysis/nowcasting/images/use_summary_sut_2018.png differ diff --git a/bedrock/analysis/nowcasting/images/use_summary_sut_2019.png b/bedrock/analysis/nowcasting/images/use_summary_sut_2019.png index 5cca7cbdb..bd1e1cd5a 100644 Binary files a/bedrock/analysis/nowcasting/images/use_summary_sut_2019.png and b/bedrock/analysis/nowcasting/images/use_summary_sut_2019.png differ diff --git a/bedrock/analysis/nowcasting/images/use_summary_sut_2020.png b/bedrock/analysis/nowcasting/images/use_summary_sut_2020.png index 443822b2c..c90d2fe1a 100644 Binary files a/bedrock/analysis/nowcasting/images/use_summary_sut_2020.png and b/bedrock/analysis/nowcasting/images/use_summary_sut_2020.png differ diff --git a/bedrock/analysis/nowcasting/images/use_summary_sut_2021.png b/bedrock/analysis/nowcasting/images/use_summary_sut_2021.png index b38e04383..c1633b681 100644 Binary files a/bedrock/analysis/nowcasting/images/use_summary_sut_2021.png and b/bedrock/analysis/nowcasting/images/use_summary_sut_2021.png differ diff --git a/bedrock/analysis/nowcasting/images/use_summary_sut_2022.png b/bedrock/analysis/nowcasting/images/use_summary_sut_2022.png index 0d24227be..d517e9c15 100644 Binary files a/bedrock/analysis/nowcasting/images/use_summary_sut_2022.png and b/bedrock/analysis/nowcasting/images/use_summary_sut_2022.png differ diff --git a/bedrock/analysis/nowcasting/images/use_summary_sut_2023.png b/bedrock/analysis/nowcasting/images/use_summary_sut_2023.png index bd3c1ae2d..010899962 100644 Binary files a/bedrock/analysis/nowcasting/images/use_summary_sut_2023.png and b/bedrock/analysis/nowcasting/images/use_summary_sut_2023.png differ diff --git a/bedrock/analysis/nowcasting/images/use_summary_sut_2024.png b/bedrock/analysis/nowcasting/images/use_summary_sut_2024.png index eec9c1d9c..093546385 100644 Binary files a/bedrock/analysis/nowcasting/images/use_summary_sut_2024.png and b/bedrock/analysis/nowcasting/images/use_summary_sut_2024.png differ diff --git a/bedrock/analysis/nowcasting/images/use_table_provenance_2017.png b/bedrock/analysis/nowcasting/images/use_table_provenance_2017.png index 13707c6a1..67075dfa2 100644 Binary files a/bedrock/analysis/nowcasting/images/use_table_provenance_2017.png and b/bedrock/analysis/nowcasting/images/use_table_provenance_2017.png differ diff --git a/bedrock/analysis/nowcasting/images/use_va_detail_sut_2017.png b/bedrock/analysis/nowcasting/images/use_va_detail_sut_2017.png index f5f87eac0..c84bb4b9f 100644 Binary files a/bedrock/analysis/nowcasting/images/use_va_detail_sut_2017.png and b/bedrock/analysis/nowcasting/images/use_va_detail_sut_2017.png differ diff --git a/bedrock/analysis/nowcasting/images/use_va_detail_sut_seed_2017.png b/bedrock/analysis/nowcasting/images/use_va_detail_sut_seed_2017.png index 085e3ca15..c25692b85 100644 Binary files a/bedrock/analysis/nowcasting/images/use_va_detail_sut_seed_2017.png and b/bedrock/analysis/nowcasting/images/use_va_detail_sut_seed_2017.png differ diff --git a/bedrock/analysis/nowcasting/images/va_after_redef_detail_mut_2017.png b/bedrock/analysis/nowcasting/images/va_after_redef_detail_mut_2017.png index 42b285c72..af763f401 100644 Binary files a/bedrock/analysis/nowcasting/images/va_after_redef_detail_mut_2017.png and b/bedrock/analysis/nowcasting/images/va_after_redef_detail_mut_2017.png differ diff --git a/bedrock/analysis/nowcasting/images/value_added_provenance_2017.png b/bedrock/analysis/nowcasting/images/value_added_provenance_2017.png index 4c8e1addb..92df024f2 100644 Binary files a/bedrock/analysis/nowcasting/images/value_added_provenance_2017.png and b/bedrock/analysis/nowcasting/images/value_added_provenance_2017.png differ diff --git a/bedrock/analysis/nowcasting/plots.py b/bedrock/analysis/nowcasting/plots.py index 8205dadc2..1736e0806 100644 --- a/bedrock/analysis/nowcasting/plots.py +++ b/bedrock/analysis/nowcasting/plots.py @@ -4,7 +4,7 @@ as a strip down the right edge and the column totals as a strip along the bottom, on the same colour scale. The margins are not decoration -- ``T014`` nets to ~1 economy-wide and redefinition preserves every total, so a green -interior above a yellow column strip localises an error that the grand total +interior above a red column strip localises an error that the grand total cannot see. The interior is drawn with a single ``imshow`` of a pre-built RGB array rather @@ -14,30 +14,35 @@ Colour ------ -============ ========================================================== -white ``ABSENT`` -- neither side has a value -green ``MATCH`` -- both present, within tolerance -yellow-amber ``PARTIAL`` -- both present, shaded by how far off -purple ``MISS`` -- the reference has a value, we produced none -blue ``EXTRA`` -- we produced a value the reference does not have -============ ========================================================== - -The yellow ramp runs from the tolerance boundary (severity 0) to -``Tolerance.ramp`` (severity 1), so the shading has a stated scale and the -colour bar can print it. - -These five anchors were checked, not assumed, against simulated protanopia, -deuteranopia and tritanopia: every category pair separates by at least -``dE 27`` (CIE76) under all four vision models. :func:`palette_separation` -re-runs that check on whatever the palette currently is, so an edit to it can -be re-verified rather than argued about -- ``--check-palette`` on the CLI. - -The binding constraint is not the one people expect. Green against yellow is -the notorious pair, and it is handled by lightness: the match green is dark and -the whole yellow ramp is light. The closest pair in the palette is actually -*white against the palest yellow* under tritanopia, which is why the ramp -starts at a saturated ``#ffd84d`` rather than the near-white a "shade of -yellow" would suggest. +=========== ========================================================== +white ``ABSENT`` -- neither side has a value +green-red both sides have a value, coloured by relative difference +purple ``MISS`` -- the reference has a value, we produced none +blue ``EXTRA`` -- we produced a value the reference does not have +=========== ========================================================== + +Where both sides have a value the cell sits on one continuous ramp: green at a +relative difference of 0, red at 1.0 and beyond. There is no threshold in the +picture. A cell 0.9% off and a cell 1.1% off are drawn almost identically, +because that is what they are; the earlier within-tolerance / outside-tolerance +split drew them as two different colours, which put a step in the picture where +the data has none. ``Tolerance`` still exists in :mod:`~..table_match`, where +the pass/fail gates need a boundary -- it just no longer decides a colour. + +Lightness falls monotonically along the ramp, light green through amber to a +dark red. That is what keeps it readable without hue discrimination: under +protanopia and deuteranopia the two ends converge in hue, so position along the +ramp has to be carried by something else, and lightness is the only channel +left. A green-to-red ramp built on hue alone is the one gradient those viewers +cannot read at all. + +The anchors were checked, not assumed, against simulated protanopia, +deuteranopia and tritanopia. :func:`palette_separation` re-runs the check on +whatever the palette currently is, so an edit to it can be re-verified rather +than argued about -- ``--check-palette`` on the CLI. Ramp-against-ramp pairs +are exempt from the floor, since adjacent points on a continuous scale are +meant to be close, but the two ends are held to it: if 0% and 100% are not +separable the ramp is not carrying its own scale. CLI:: @@ -70,23 +75,42 @@ OUTPUT_DIR = Path(__file__).parent / 'output' -#: Flat category colours. ``PARTIAL`` is a ramp, see :data:`PARTIAL_RAMP`. +#: Flat colours for the three presence states. Cells both sides populate are +#: not in here; they are drawn from :data:`DIFF_RAMP`. PALETTE: dict[CellStatus, str] = { CellStatus.ABSENT: '#ffffff', - CellStatus.MATCH: '#166534', CellStatus.MISS: '#6a3d9a', CellStatus.EXTRA: '#67a9cf', } -#: ``(severity 0, severity 1)``. The light end is deliberately a saturated -#: yellow rather than a near-white one: a cell just outside tolerance has to -#: stay distinguishable from an empty cell, including under tritanopia, where -#: white and pale yellow are the closest pair in the whole palette. -PARTIAL_RAMP: tuple[str, str] = ('#ffd84d', '#b87700') +#: ``(position, colour)`` from a relative difference of 0 to :data:`DIFF_MAX`. +#: Lightness falls monotonically along it -- ``L*`` 81, 73, 64, 48, 33 -- which +#: is the property that carries the scale for viewers who cannot separate the +#: two hues. The steps are kept roughly even rather than merely monotonic: +#: most cells in a Use table sit in the first quarter of this scale, so a ramp +#: that is flat there has thrown away the channel where it needs it most. +#: +#: These anchors were searched, not chosen: the green end is squeezed between +#: white (``ABSENT``) above it and the blue of ``EXTRA`` below it, and under +#: tritanopia both of those sit close to a light green. The best available +#: worst-pair separation is ``dE 26.0`` against a floor of +#: :data:`MIN_SEPARATION`, binding on white against the green end -- the same +#: pair that bound the previous palette. +DIFF_RAMP: tuple[tuple[float, str], ...] = ( + (0.00, '#a5d96a'), + (0.25, '#a8b84e'), + (0.50, '#c9902e'), + (0.75, '#b85526'), + (1.00, '#94201f'), +) + +#: Relative difference at which the ramp saturates. The legend states it, so a +#: saturated cell reads as "100% off or worse" rather than "exactly 100%". +DIFF_MAX = 1.0 -#: A severity that could not be computed still has to be drawn as ``PARTIAL``; +#: A relative difference that could not be computed still has to be drawn; #: mid-ramp is the honest placeholder. -DEFAULT_SEVERITY = 0.5 +DEFAULT_REL = 0.5 LABEL_AXIS_MAX = 60 # tick labels stop being legible somewhere around here TICK_FONTSIZE = 7 @@ -107,26 +131,38 @@ def _hex_to_rgb(value: str) -> np.ndarray: return np.array([int(value[i : i + 2], 16) / 255 for i in (0, 2, 4)]) -def partial_rgb(severity: np.ndarray) -> np.ndarray: - """Interpolate the yellow ramp at ``severity`` (0-1), shape ``(..., 3)``.""" - lo, hi = (_hex_to_rgb(c) for c in PARTIAL_RAMP) - t = np.clip(np.nan_to_num(severity, nan=DEFAULT_SEVERITY), 0.0, 1.0)[..., None] - return lo * (1 - t) + hi * t +def diff_rgb(rel: np.ndarray) -> np.ndarray: + """Interpolate :data:`DIFF_RAMP` at relative difference ``rel``. + + ``rel`` is a fraction, not a percentage, and is clipped to + ``[0, DIFF_MAX]``: everything at or past the top of the scale is the same + red, so one cell 40x out does not decide how the rest of the table reads. + """ + positions = np.array([p for p, _ in DIFF_RAMP]) * DIFF_MAX + colours = np.stack([_hex_to_rgb(c) for _, c in DIFF_RAMP]) + t = np.clip( + np.nan_to_num(np.asarray(rel, dtype=float), nan=DEFAULT_REL), 0.0, DIFF_MAX + ) + return np.stack([np.interp(t, positions, colours[:, i]) for i in range(3)], axis=-1) -def status_rgb(status: np.ndarray, severity: np.ndarray) -> np.ndarray: - """Build the RGB raster for a status/severity pair, shape ``(..., 3)``. +def status_rgb(status: np.ndarray, rel: np.ndarray) -> np.ndarray: + """Build the RGB raster for a status/relative-difference pair, ``(..., 3)``. Kept separate from any figure so it can be asserted on directly, and so the margin strips and the interior are coloured by exactly one function. + + ``MATCH`` and ``PARTIAL`` are both "the two sides have a value here" and + are coloured identically, by their difference. The distinction between + them survives only in the gates. """ status = np.asarray(status) rgb = np.zeros((*status.shape, 3), dtype=float) for code, hexval in PALETTE.items(): rgb[status == int(code)] = _hex_to_rgb(hexval) - partial = status == int(CellStatus.PARTIAL) - if partial.any(): - rgb[partial] = partial_rgb(np.asarray(severity, dtype=float)[partial]) + both = (status == int(CellStatus.MATCH)) | (status == int(CellStatus.PARTIAL)) + if both.any(): + rgb[both] = diff_rgb(np.asarray(rel, dtype=float)[both]) return rgb @@ -152,13 +188,13 @@ def tick_labels(axis: pd.Index, names: Mapping[str, str] | None) -> list[str] | def _draw( ax: Axes, status: np.ndarray, - severity: np.ndarray, + rel: np.ndarray, *, row_labels: list[str] | None = None, col_labels: list[str] | None = None, strip: str = '', ) -> None: - rgb = status_rgb(status, severity) + rgb = status_rgb(status, rel) if rgb.ndim == 2: # a margin strip arrives 1-D rgb = rgb[None, :, :] if strip == 'column' else rgb[:, None, :] ax.imshow(rgb, aspect='auto', interpolation='nearest', origin='upper') @@ -233,9 +269,11 @@ def _to_lab(rgb: np.ndarray) -> np.ndarray: def palette_separation() -> pd.DataFrame: """CIE76 distance between every category pair, under four vision models. - The ramp is sampled at both ends and its middle, and ramp-vs-ramp pairs are - skipped: shading within ``PARTIAL`` is a magnitude cue, not a category - boundary, so those are meant to be close. + The ramp is sampled at each of its anchors. Ramp-vs-ramp pairs are skipped + -- shading within the ramp is a magnitude cue, not a category boundary, so + neighbouring samples are meant to be close -- with one exception: the two + *ends* are held to the floor like any other pair, because a ramp whose ends + are not separable is not carrying a scale. :return: ``vision``, ``a``, ``b``, ``delta_e``, ``delta_l``, worst first reversed -- sort ascending and read the top row for the binding pair. @@ -245,8 +283,9 @@ def palette_separation() -> pd.DataFrame: anchors: dict[str, np.ndarray] = { STATUS_NAMES[code]: _hex_to_rgb(value) for code, value in PALETTE.items() } - for t in (0.0, 0.5, 1.0): - anchors[f'partial@{t:.1f}'] = partial_rgb(np.array(t)) + for position, _ in DIFF_RAMP: + anchors[f'diff@{position:.2f}'] = diff_rgb(np.array(position)) + ends = {f'diff@{DIFF_RAMP[0][0]:.2f}', f'diff@{DIFF_RAMP[-1][0]:.2f}'} rows = [] for vision, matrix in _CVD_MATRICES.items(): @@ -255,7 +294,8 @@ def palette_separation() -> pd.DataFrame: for name, rgb in anchors.items() } for a, b in itertools.combinations(seen, 2): - if a.startswith('partial@') and b.startswith('partial@'): + ramp_pair = a.startswith('diff@') and b.startswith('diff@') + if ramp_pair and {a, b} != ends: continue rows.append( { @@ -276,9 +316,9 @@ def _legend_handles() -> list[matplotlib.patches.Patch]: Patch( facecolor=PALETTE[CellStatus.ABSENT], edgecolor='#999999', label='absent' ), - Patch(facecolor=PALETTE[CellStatus.MATCH], label='match'), - Patch(facecolor=PARTIAL_RAMP[0], label='partial (at tolerance)'), - Patch(facecolor=PARTIAL_RAMP[1], label='partial (at ramp)'), + Patch(facecolor=DIFF_RAMP[0][1], label='0% different'), + Patch(facecolor=DIFF_RAMP[2][1], label='50% different'), + Patch(facecolor=DIFF_RAMP[-1][1], label=f'{DIFF_MAX:.0%} or more'), Patch(facecolor=PALETTE[CellStatus.MISS], label='miss (reference only)'), Patch(facecolor=PALETTE[CellStatus.EXTRA], label='extra (ours only)'), ] @@ -366,7 +406,7 @@ def plot_match( _draw( interior, match.status.to_numpy(), - match.severity.to_numpy(), + match.rel_error.to_numpy(), row_labels=row_ticks, ) interior.set_ylabel(str(match.status.index.name or ''), fontsize=9) @@ -378,7 +418,7 @@ def plot_match( _draw( right, match.row_totals.status.to_numpy(), - match.row_totals.severity.to_numpy(), + match.row_totals.rel_error.to_numpy(), strip='row', ) right.set_title('row\ntotals', fontsize=8, pad=4) @@ -387,7 +427,7 @@ def plot_match( _draw( bottom, match.col_totals.status.to_numpy(), - match.col_totals.severity.to_numpy(), + match.col_totals.rel_error.to_numpy(), col_labels=col_ticks, strip='column', ) @@ -398,8 +438,8 @@ def plot_match( # check that passes on broken data. corner = fig.add_subplot(grid[1, 1]) gt = match.grand_total - gt_status, _, gt_sev = _classify_scalar(match, gt) - _draw(corner, np.array([[gt_status]]), np.array([[gt_sev]])) + gt_status, gt_rel, _ = _classify_scalar(match, gt) + _draw(corner, np.array([[gt_status]]), np.array([[gt_rel]])) corner.set_title('grand\ntotal', fontsize=7, pad=2, y=-0.9) if chrome: @@ -444,12 +484,16 @@ def _subtitle(match: TableMatch, section: Section | None, width: float) -> str: import textwrap # noqa: PLC0415 n = match.counts().loc['cells'] - counts = ' '.join( - f'{name} {int(n[name]):,}' for name in STATUS_NAMES.values() if name != 'absent' + # 'match' and 'partial' are the two halves of "both sides have a value"; + # they are one thing here, because the picture no longer splits them. + counts = ( + f'both {int(n["match"]) + int(n["partial"]):,} ' + f'miss {int(n["miss"]):,} extra {int(n["extra"]):,}' ) lines = [ - f'tolerance {match.tolerance.describe()} | ' - f'coverage {match.coverage:.1%} | accuracy {match.accuracy:.1%}', + f'coverage {match.coverage:.1%} | median difference ' + f'{match.median_rel_error:.1%} | value-weighted ' + f'{match.weighted_rel_error:.1%}', f'cells: {counts} | grand total off by ' f'{match.grand_total["rel_error"]:.2%}', ] diff --git a/bedrock/analysis/nowcasting/progress_report.md b/bedrock/analysis/nowcasting/progress_report.md index e6476c505..2f6390ef1 100644 --- a/bedrock/analysis/nowcasting/progress_report.md +++ b/bedrock/analysis/nowcasting/progress_report.md @@ -1,6 +1,6 @@ # Nowcasting Phase 1 Report -*Updated 3 September 2026* +*Updated 7 September 2026* The nowcast builds annual US input-output tables — Supply and Use tables, then Make-Use tables before and after redefinitions — for 2017 through 2024 @@ -20,20 +20,31 @@ tables, and the published industry output series. 4. [Conversion to Make-Use Tables](#4-conversion-to-make-use-tables) 5. [Redefinitions: Before to After](#5-redefinitions-before-to-after) 6. [Seed Data Provenance and Quality](#6-seed-data-provenance-and-quality) +6a. [The 7 September 2026 rebuild](#6a-the-7-september-2026-rebuild) 7. [Conclusions and Next Steps](#7-conclusions-and-next-steps) **How to read the comparison figures.** Every comparison figure in sections -1 and 2 uses one visual language. Each cell of the table under comparison is -coloured by its match status against the published reference: **green** -lands within tolerance (1% of the cell, or 1.3% for final demand, with an -absolute floor of $0.5M — half BEA's publication grain); **amber** is -outside tolerance, darkening with severity; **purple** cells are populated +1 and 2 uses one visual language. Where our table and the published reference +both have a value, the cell is coloured by how far apart the two are, on one +continuous ramp: **green** at no difference, through amber, to **red** at a +difference of 100% or more of the published cell. There is no threshold in +the picture — a cell 0.9% off and a cell 1.1% off are drawn almost +identically, because that is what they are. **Purple** cells are populated only in the published table; **blue** cells only in ours; white cells are empty on both sides. The strip on the right is the row totals, the strip along the bottom the column totals, and the corner square the grand total — drawn smallest because it is the check that passes on broken data. -**Coverage** is the share of reference-populated cells we also populate; -**accuracy** is the share of populated cells landing within tolerance. + +**Two numbers accompany each figure, and they answer different questions.** +*Coverage* is the share of reference-populated cells we also populate — how +much of this table we have, asked before how good it is. *Median difference* +is the middle relative difference across the cells both sides populate. +*Value-weighted* is the total absolute difference over the total published +value of those same cells. They are quoted together on purpose: the median +says what a typical cell does and counts a $0.6M cell and a $50bn cell +alike, while the value-weighted figure says what share of the compared +dollars sits in the wrong place and can only be moved by cells that are +large. Where they diverge, the gap is the finding. ## 1. Comparison to BEA 2017 Benchmark SUTs @@ -47,17 +58,18 @@ and the supply bridge, each built from its own primary sources. The balancing step imposed the accounting identities and observed aggregates. Table 1 summarises both states. -*Table 1. The five SUT blocks against the published 2017 detail benchmark, -in the seed state and the balanced state. Coverage / accuracy over populated -cells; total difference is the grand total against published.* +*Table 1. The five SUT blocks against the published 2017 detail +benchmark, in the seed state and the balanced state. Coverage, then the +median and value-weighted relative difference over the cells both sides +populate; total difference is the grand total against published.* -| block | seed cov. / acc. | balanced cov. / acc. | balanced total diff | -|---|---:|---:|---:| -| Final demand (402 × 19) | 90.0% / 66.0% | 89.9% / 50.9% | 0.071% | -| Value added (6 × 402) | 99.9% / 79.4% | 99.9% / 67.7% | 0.000% | -| Intermediate interior (402 × 402) | 100.0% / 100.0% | 100.0% / 57.2% | 0.063% | -| Domestic output interior (402 × 402) | 100.0% / 99.6% | 100.0% / 74.5% | 0.009% | -| Supply bridge (402 × 12) | 99.5% / 76.1% | 99.4% / 49.5% | 0.018% | +| block | seed cov. | seed med. / wtd. | balanced cov. | balanced med. / wtd. | balanced total diff | +|---|---:|---:|---:|---:|---:| +| Final demand (402 × 19) | 90.0% | <0.01% / 1.5% | 89.9% | 0.98% / 1.8% | 0.071% | +| Value added (6 × 402) | 99.9% | <0.01% / 2.1% | 99.9% | 0.93% / 2.2% | 0.000% | +| Intermediate interior (402 × 402) | 100.0% | <0.01% / <0.01% | 100.0% | 2.1% / 2.9% | 0.063% | +| Domestic output interior (402 × 402) | 100.0% | <0.01% / <0.01% | 100.0% | 0.96% / 0.81% | 0.009% | +| Supply bridge (402 × 12) | 99.5% | 0.02% / 0.36% | 99.4% | 1.1% / 1.1% | 0.018% | ⚠️ **The two columns answer different questions, and the drop between them is not degradation.** The seed interiors reproduce published 2017 because at @@ -79,82 +91,90 @@ certify only that the build is unbroken. *Figure 1. Seed final-demand columns (commodity × final-demand code, purchaser price) against the published 2017 detail Use table. Coverage -90.0%, accuracy 66.0%. The inventory-change and export columns carry nearly -all misses; the twelve government columns land cell for cell.* +90.0%, median difference <0.01%, value-weighted 1.5%. The +inventory-change and export columns carry nearly all misses; the twelve +government columns land cell for cell.* ![Seed value added](images/use_va_detail_sut_seed_2017.png) *Figure 2. Seed value-added rows (six rows × 402 industries) against the -published 2017 detail Use table. Coverage 99.9%, accuracy 79.4%. All row -totals match; the shortfall is one row — taxes on products, whose industry -split places tax with producers where the published row places it with -sellers (wholesale and retail).* +published 2017 detail Use table. Coverage 99.9%, median difference +<0.01%, value-weighted 2.1%. All row totals match; the shortfall is one +row — taxes on products, whose industry split places tax with producers +where the published row places it with sellers (wholesale and retail).* ![Seed intermediate interior](images/use_intermediate_detail_sut_seed_2017.png) *Figure 3. Seed intermediate interior (402 commodities × 402 industries) -against the published 2017 detail Use table. Coverage and accuracy 100.0% — -a plumbing certification, since the 2017 seed carries the published interior -by construction. The estimation content of this block is scored in section -6, where 35.7% of its dollars are observed by an annual source.* +against the published 2017 detail Use table. Coverage 100.0%, median +difference <0.01%, value-weighted <0.01% — a plumbing certification, +since the 2017 seed carries the published interior by construction. The +estimation content of this block is scored in section 6, where 35.7% of +its dollars are observed by an annual source.* ![Seed domestic output](images/supply_output_detail_sut_seed_2017.png) -*Figure 4. Seed domestic output block (402 × 402, basic price) against the -published 2017 detail Supply table. Coverage 100.0%, accuracy 99.6%. Only -~5,000 of 161,604 cells are populated on either side — an industry makes a -handful of commodities — and both-empty cells count as absent, not matches. +*Figure 4. Seed domestic output block (402 × 402, basic price) against +the published 2017 detail Supply table. Coverage 100.0%, median +difference <0.01%, value-weighted <0.01%. Only ~5,000 of 161,604 cells +are populated on either side — an industry makes a handful of +commodities — and both-empty cells count as absent, not matches. Near-circular at 2017: the same detail mix appears on both sides.* ![Seed supply bridge](images/supply_bridge_detail_sut_seed_2017.png) -*Figure 5. Seed supply bridge (402 commodities × 12 bridge codes: imports, -margins, taxes and their subtotals) against the published 2017 detail Supply -table. Coverage 99.5%, accuracy 76.1%. The two margin columns net to -exactly zero by construction and cannot be scored by their column totals; -the import column's remaining error is concentrated in ~50 crosswalk -decisions on goods commodities.* +*Figure 5. Seed supply bridge (402 commodities × 12 bridge codes: +imports, margins, taxes and their subtotals) against the published 2017 +detail Supply table. Coverage 99.5%, median difference 0.02%, +value-weighted 0.36%. The two margin columns net to exactly zero by +construction and cannot be scored by their column totals; the import +column's remaining error is concentrated in ~50 crosswalk decisions on +goods commodities.* ### 1.2 Balanced RAS Tables Figures 6-10 show the same five blocks in the final balanced state — the tables the rest of the pipeline consumes. Totals and identities are exact -everywhere; the amber that appears relative to the seed figures marks where -the balance placed the reconciliation. +everywhere; the warmer cells relative to the seed figures mark where the +balance placed the reconciliation. ![Balanced final demand](images/use_fd_detail_sut_2017.png) -*Figure 6. Balanced final-demand columns against the published 2017 detail -Use table. Coverage 89.9%, accuracy 50.9%. Compare Figure 1: the balance -spread part of the inventory and export reconciliation across otherwise -matching columns while closing every imposed column total exactly.* +*Figure 6. Balanced final-demand columns against the published 2017 +detail Use table. Coverage 89.9%, median difference 0.98%, +value-weighted 1.8%. Compare Figure 1: the balance spread part of the +inventory and export reconciliation across otherwise matching columns +while closing every imposed column total exactly.* ![Balanced value added](images/use_va_detail_sut_2017.png) -*Figure 7. Balanced value-added rows against the published 2017 detail Use -table. Coverage 99.9%, accuracy 67.7%; the grand total agrees to 0.000%. -Compare Figure 2.* +*Figure 7. Balanced value-added rows against the published 2017 detail +Use table. Coverage 99.9%, median difference 0.93%, value-weighted 2.2%; +the grand total agrees to 0.000%. Compare Figure 2.* ![Balanced intermediate interior](images/use_intermediate_detail_sut_2017.png) -*Figure 8. Balanced intermediate interior against the published 2017 detail -Use table. Coverage 100.0%, accuracy 57.2%. The drop from Figure 3's 100% -is the balance absorbing the seed's supply-use gap — a fitted 2.3% before -balancing — into the interior while closing the identity exactly for every +*Figure 8. Balanced intermediate interior against the published 2017 +detail Use table. Coverage 100.0%, median difference 2.1%, +value-weighted 2.9%. The drop from Figure 3's 100% is the balance +absorbing the seed's supply-use gap — a fitted 2.3% before balancing — +into the interior while closing the identity exactly for every commodity.* ![Balanced domestic output](images/supply_output_detail_sut_2017.png) -*Figure 9. Balanced domestic output block against the published 2017 detail -Supply table. Coverage 100.0%, accuracy 74.5%; the grand total agrees to -0.009%. Compare Figure 4.* +*Figure 9. Balanced domestic output block against the published 2017 +detail Supply table. Coverage 100.0%, median difference 0.96%, +value-weighted 0.81%; the grand total agrees to 0.009%. Compare Figure +4.* ![Balanced supply bridge](images/supply_bridge_detail_sut_2017.png) -*Figure 10. Balanced supply bridge against the published 2017 detail Supply -table. Coverage 99.4%, accuracy 49.5%; the grand total agrees to 0.018%. -The trade-margin rows dominate the amber: the balance moved margin mass -between commodities to close the purchaser-value identities.* +*Figure 10. Balanced supply bridge against the published 2017 detail +Supply table. Coverage 99.4%, median difference 1.1%, value-weighted +1.1%; the grand total agrees to 0.018%. The trade-margin rows dominate +the divergence: the balance moved margin mass between commodities to +close the purchaser-value identities.* ## 2. Annual Comparison to Published Summary Tables @@ -168,20 +188,21 @@ detail mix deliberately follows the 2022 Economic Census where it disagrees with the published tables. Table 2 summarises the span; Figures 11-24 show each year. -*Table 2. The balanced tables aggregated to summary against the published -summary tables, per year. Accuracy is the share of populated cells within -1%; total difference is the grand total against published.* +*Table 2. The balanced tables aggregated to summary against the +published summary tables, per year. Median and value-weighted are the +relative difference over the cells both sides populate; total difference +is the grand total against published.* -| year | Supply cov. | Supply acc. | Use cov. | Use acc. | Supply total diff | Use total diff | +| year | Supply cov. | Supply med. / wtd. | Use cov. | Use med. / wtd. | Supply total diff | Use total diff | |---:|---:|---:|---:|---:|---:|---:| -| 2017 | 99.2% | 55.5% | 99.6% | 41.5% | 0.012% | 0.012% | -| 2018 | 99.0% | 50.0% | 99.5% | 15.6% | 0.011% | 0.009% | -| 2019 | 98.7% | 42.2% | 99.5% | 12.6% | 0.058% | 0.008% | -| 2020 | 98.4% | 25.0% | 99.5% | 7.8% | 0.067% | 0.185% | -| 2021 | 98.1% | 17.5% | 99.1% | 6.9% | 0.340% | 0.586% | -| 2022 | 98.4% | 12.0% | 98.7% | 6.3% | **1.033%** | **1.038%** | -| 2023 | 98.4% | 35.9% | 98.6% | 6.3% | 0.017% | 0.016% | -| 2024 | 98.4% | 36.3% | 98.7% | 6.2% | 0.017% | 0.015% | +| 2017 | 99.2% | 1.1% / 0.69% | 99.6% | 1.3% / 1.8% | 0.012% | 0.012% | +| 2018 | 99.0% | 1.5% / 0.87% | 99.4% | 6.4% / 3.2% | 0.011% | 0.013% | +| 2019 | 98.7% | 2.3% / 1.0% | 99.4% | 9.3% / 3.8% | 0.059% | 0.013% | +| 2020 | 98.4% | 3.6% / 1.6% | 99.3% | 14.5% / 5.1% | 0.067% | 0.119% | +| 2021 | 98.1% | 5.4% / 1.9% | 99.0% | 15.5% / 5.9% | 0.357% | 0.408% | +| 2022 | 98.4% | 14.0% / 4.0% | 98.8% | 20.2% / 8.7% | **1.045%** | **1.039%** | +| 2023 | 98.3% | 3.4% / 1.5% | 98.6% | 21.6% / 6.8% | 0.017% | 0.021% | +| 2024 | 98.4% | 3.0% / 1.4% | 98.6% | 22.0% / 6.7% | 0.019% | 0.022% | Three things the span says. **Coverage holds at ~99% every year** — the nowcast populates what the published tables populate. **Cell-level @@ -197,83 +218,84 @@ than 2022 and lands at a sixtieth of its total difference. ![Supply summary 2017](images/supply_summary_sut_2017.png) *Figure 11. Balanced Supply aggregated to summary vs published, 2017. -Coverage 99.2%, accuracy 55.5%.* +Coverage 99.2%, median difference 1.1%, value-weighted 0.69%.* ![Use summary 2017](images/use_summary_sut_2017.png) *Figure 12. Balanced Use aggregated to summary vs published, 2017. -Coverage 99.6%, accuracy 41.5%.* +Coverage 99.6%, median difference 1.3%, value-weighted 1.8%.* ![Supply summary 2018](images/supply_summary_sut_2018.png) -*Figure 13. Supply at summary vs published, 2018. Coverage 99.0%, accuracy -50.0%.* +*Figure 13. Supply at summary vs published, 2018. Coverage 99.0%, median +difference 1.5%, value-weighted 0.87%.* ![Use summary 2018](images/use_summary_sut_2018.png) -*Figure 14. Use at summary vs published, 2018. Coverage 99.5%, accuracy -15.6%.* +*Figure 14. Use at summary vs published, 2018. Coverage 99.4%, median +difference 6.4%, value-weighted 3.2%.* ![Supply summary 2019](images/supply_summary_sut_2019.png) -*Figure 15. Supply at summary vs published, 2019. Coverage 98.7%, accuracy -42.2%.* +*Figure 15. Supply at summary vs published, 2019. Coverage 98.7%, median +difference 2.3%, value-weighted 1.0%.* ![Use summary 2019](images/use_summary_sut_2019.png) -*Figure 16. Use at summary vs published, 2019. Coverage 99.5%, accuracy -12.6%.* +*Figure 16. Use at summary vs published, 2019. Coverage 99.4%, median +difference 9.3%, value-weighted 3.8%.* ![Supply summary 2020](images/supply_summary_sut_2020.png) -*Figure 17. Supply at summary vs published, 2020. Coverage 98.4%, accuracy -25.0%.* +*Figure 17. Supply at summary vs published, 2020. Coverage 98.4%, median +difference 3.6%, value-weighted 1.6%.* ![Use summary 2020](images/use_summary_sut_2020.png) -*Figure 18. Use at summary vs published, 2020. Coverage 99.5%, accuracy -7.8%.* +*Figure 18. Use at summary vs published, 2020. Coverage 99.3%, median +difference 14.5%, value-weighted 5.1%.* ![Supply summary 2021](images/supply_summary_sut_2021.png) -*Figure 19. Supply at summary vs published, 2021. Coverage 98.1%, accuracy -17.5%.* +*Figure 19. Supply at summary vs published, 2021. Coverage 98.1%, median +difference 5.4%, value-weighted 1.9%.* ![Use summary 2021](images/use_summary_sut_2021.png) -*Figure 20. Use at summary vs published, 2021. Coverage 99.1%, accuracy -6.9%.* +*Figure 20. Use at summary vs published, 2021. Coverage 99.0%, median +difference 15.5%, value-weighted 5.9%.* ![Supply summary 2022](images/supply_summary_sut_2022.png) -*Figure 21. Supply at summary vs published, 2022 — the census-conditioning -year, and the widest divergence of the span. Coverage 98.4%, accuracy -12.0%.* +*Figure 21. Supply at summary vs published, 2022 — the +census-conditioning year, and the widest divergence of the span. +Coverage 98.4%, median difference 14.0%, value-weighted 4.0%.* ![Use summary 2022](images/use_summary_sut_2022.png) -*Figure 22. Use at summary vs published, 2022. Coverage 98.7%, accuracy -6.3%.* +*Figure 22. Use at summary vs published, 2022. Coverage 98.8%, median +difference 20.2%, value-weighted 8.7%.* ![Supply summary 2023](images/supply_summary_sut_2023.png) -*Figure 23. Supply at summary vs published, 2023. Coverage 98.4%, accuracy -35.9%.* +*Figure 23. Supply at summary vs published, 2023. Coverage 98.3%, median +difference 3.4%, value-weighted 1.5%.* ![Use summary 2023](images/use_summary_sut_2023.png) -*Figure 24. Use at summary vs published, 2023. Coverage 98.6%, accuracy -6.3%.* +*Figure 24. Use at summary vs published, 2023. Coverage 98.6%, median +difference 21.6%, value-weighted 6.8%.* ![Supply summary 2024](images/supply_summary_sut_2024.png) -*Figure 25. Supply at summary vs published, 2024 — the first year built on -the 2024 Annual Integrated Economic Survey. Coverage 98.4%, accuracy 36.3%.* +*Figure 25. Supply at summary vs published, 2024 — the first year built +on the 2024 Annual Integrated Economic Survey. Coverage 98.4%, median +difference 3.0%, value-weighted 1.4%.* ![Use summary 2024](images/use_summary_sut_2024.png) -*Figure 26. Use at summary vs published, 2024. Coverage 98.7%, accuracy -6.2%.* +*Figure 26. Use at summary vs published, 2024. Coverage 98.6%, median +difference 22.0%, value-weighted 6.7%.* ## 3. Industry Output Against the Published Series @@ -349,7 +371,7 @@ on every year's build and refuse to save on breach.* | check | result | |---|---| | Producer-price Use replay, 2017 | interior and final demand reproduce published to $0M per cell | -| Imports column rule (goods, adjustment, duties with the customs credit) | every commodity within tolerance at 2017 and 2012 | +| Imports column rule (goods, adjustment, duties with the customs credit) | every commodity reproduces the published `F05000` to $0.5M — half BEA's publication grain — at 2017 and 2012 | | Import matrix row control | exact at both published benchmarks | | Margins layered identities (basic → producer → purchaser) | close at $0.00 per cell at 2017, 2012, and 2012 on the 2017 anchor | | Cost of the frozen 2017 margin rates, measured at 2012 | 2.06% of goods mass; 25.0% of margin-row placement | @@ -424,6 +446,106 @@ than unattempted work. A match score of 100% can coexist with one source datum spread over 400 cells; reliability and specificity are scored separately for exactly that reason. +## 6a. The 7 September 2026 rebuild + +Every table in this report was rebuilt on 7 September from cleared +artifacts, on methods that changed in seven places since the 3 September +run. The rebuild is the reason the figures moved, and what moved is +worth stating plainly. + +**Nothing moved by accident.** Grand totals are preserved: the Use SUT +grand total differs from the previous build by 3 million USD at 2017 on +56.7 trillion, and by 0.007% at worst across the span. Every change in +this cycle is a *redistribution* — trade mass across commodities, value +added across industries, gross output across two aerospace codes — so +the interior moves and the totals do not. + +**How the movement is measured.** Each figure below is the *gross* +cell-by-cell movement between the previous build and this one. The two +tables are aligned on their common cells and the absolute differences are +summed — the sum of |new − old| over all 406 × 421 = 170,926 cells of the +balanced Use SUT — and the percentage divides that by the previous build's +grand total for the same cells. Two properties follow, and both matter for +reading the table: + +- The metric is **gross, not net**. Offsetting moves add rather than + cancel. That is the point: the row and column controls hold, so the net + movement is near zero by construction and would show nothing. +- It therefore **counts every relocated dollar twice**, once where it + leaves and once where it arrives. The 762,736 million USD at 2017 is + roughly 381 billion of value actually changing position. + +**The measure covers the whole balanced Use SUT**, not only the +commodity-by-industry interior: the value-added rows and the final-demand +columns are in it too. Their shares are not stable across the span, so +they are broken out here rather than left implied. + +| year | Use SUT, gross \|Δ\| | intermediate | value added | final demand | Supply moved | MUT Use moved | +|---|---:|---:|---:|---:|---:|---:| +| 2017 | 762,736 $M (1.3%) | 59% | 28% | 13% | 120,033 $M (0.3%) | 1.4% | +| 2018 | 1,688,403 $M (2.8%) | 52% | 40% | 8% | 164,282 $M (0.4%) | 2.8% | +| 2019 | 1,965,366 $M (3.2%) | 50% | 42% | 7% | 181,344 $M (0.4%) | 3.2% | +| 2020 | 3,637,855 $M (6.0%) | 49% | 43% | 8% | 615,046 $M (1.6%) | 5.9% | +| 2021 | 4,781,009 $M (7.0%) | 48% | 42% | 10% | 1,296,471 $M (2.9%) | 6.8% | +| 2022 | 5,423,197 $M (7.1%) | 50% | 44% | 7% | 648,615 $M (1.3%) | 7.0% | +| 2023 | 5,409,242 $M (6.8%) | 50% | 45% | 5% | 336,857 $M (0.6%) | 6.6% | +| 2024 | 6,131,087 $M (7.3%) | 50% | 45% | 5% | 423,500 $M (0.8%) | 7.2% | + +The three middle columns are shares of that year's gross movement and +round to 100%. The value-added share nearly doubles across the span, from +28% at 2017 to 45% at 2024 — 2,766,328 million USD of the 2024 total. That +block is the value-added reconciliation, and its growth is the same effect +the next paragraph describes, visible directly rather than inferred. + +⚠️ **The movement grows with distance from the benchmark, and that is the +expected shape.** 2017 is anchored, so only the trade re-splits and the +aerospace conditioning can move it. Later years additionally carry the +value-added reconciliation, whose gap against BEA's published series grows +across the span. A rebuild that moved 2024 as little as 2017 would mean the +new methods were not doing anything. + +**All eight years balance exactly.** `T11` maximum absolute residual is +**0.0 $M** in every year, at 20 outer iterations, with nothing skipped. + +⚠️ **Seven of the eight years did not balance at first, and the reason was +latent rather than new.** The value-added reconciliation routes each +industry's residual to gross operating surplus; private households +(`814000`) have no surplus row — BEA publishes `V00300` of exactly zero +for them in 2007, 2012 *and* 2017, with `VAPRO` equal to compensation to +the dollar, because the sector's output *is* the compensation it pays. The +balance mask holds that cell as a structural zero and refused the seed. The +balanced products on disk had been built four days *before* the +reconciliation landed, so it had never been exercised on 2018-2024. The +residual now goes to the row each industry actually carries; exactly two +industries take that exception, `814000` and `4200ID` customs duties. + +**The rebuild carried through to Step 7.** Step 6 produces the tables +*before* redefinitions; leaving the after-redefinitions quartet on the 2 +September tables would have left the pipeline internally inconsistent, +because the model build consumes the after-redefinitions products. The +2017 replay gate was run first and held — maximum Make cell gap 19.0 +million USD, Use interior 6.1 million, value added 15.9 million — and the +redefinitions were then applied across the span: 8 years, Make 402x402, +Use 405x422, Import 402x422, and roughly 51,700 margin rows a year. Steps +5, 6 and 7 together wrote 160 files to cloud storage. + +**What the artifact check now sees.** `stale_artifacts` reported 10 stale +artifacts before this cycle and 290 after being taught to see a *method* +change — the class where nothing upstream moves, the cache key does not +move, and a rerun silently returns the previous parquet. It also reports +artifacts that carry no lineage at all rather than passing them, which is +what the Step 5 and Step 6 products do. After the rebuild it reports 386, and +**none of them is a product built in this cycle**: 108 are superseded +earlier versions of the nowcast quartet still sitting on disk, and the +rest are upstream extract artifacts that were deliberately left alone, +because regenerating the source panels was not part of this rebuild. + +⚠️ **A real gap remains there.** Those products record a builder but no +sources, so rebuilding an input still cannot flag them. This cycle was +correct by construction — everything downstream was cleared and rebuilt in +order — but the next one will not be unless `save_balance` and `save_mut` +start recording what they consumed. + ## 7. Conclusions and Next Steps **The pipeline is complete and its products exist.** Annual detail Supply @@ -451,9 +573,10 @@ transport margin and the change-in-inventories column were waiting on. Taking it was not a drop-in — Census had retired the API path the extracts used, for every year, and moved the gross-margin measure to a different dataset — but the year it produced is continuous with its predecessors on -every diagnostic in this report: coverage and accuracy within a few tenths -of 2023, grand totals within 0.02%, and an industry-output gap that repeats -2022's census correction rather than growing. +every diagnostic in this report: coverage, median and value-weighted +difference all within half a point of 2023 on both tables, grand totals +within 0.03%, and an industry-output gap that repeats 2022's census +correction rather than growing. **Next steps.** diff --git a/bedrock/analysis/nowcasting/sections.py b/bedrock/analysis/nowcasting/sections.py index 8a32cff69..b2cc78adb 100644 --- a/bedrock/analysis/nowcasting/sections.py +++ b/bedrock/analysis/nowcasting/sections.py @@ -938,10 +938,10 @@ def initial_Y_pur_exported_candidate(year: int) -> pd.DataFrame: 'the seed view. ' 'The Supply interior, 402 x 402 and 96.9% structurally empty: 5,059 ' 'cells of 161,604 are present, because an industry makes a handful of ' - 'commodities, not 402. Quote a match rate only over present cells. ' - '2017 runs at 100.0% coverage and 99.6% accuracy - 5,059 match, 21 ' - 'partial, no misses and no extras, of 5,080 present cells ' - '(re-measured 2026-08-28). ' + 'commodities, not 402. Quote a difference only over present cells. ' + '2017 runs at 99.98% coverage, a median difference of 0.96% and 0.81% ' + 'value-weighted - 5,079 cells present on both sides, 1 miss and no ' + 'extras, of 5,080 (re-measured 2026-09-08). ' 'Candidate is the Detail_Supply_Mix_ FBS, ' 'which disaggregates the published summary domestic-output block onto ' 'the 2017 detail mix; from 2022 the mix itself moves on Economic ' diff --git a/bedrock/analysis/nowcasting/table_match.py b/bedrock/analysis/nowcasting/table_match.py index 20867bed6..d3e02484d 100644 --- a/bedrock/analysis/nowcasting/table_match.py +++ b/bedrock/analysis/nowcasting/table_match.py @@ -92,20 +92,26 @@ class CellStatus(IntEnum): @dataclass(frozen=True) class Tolerance: - """The green/yellow boundary, and the scale of the yellow ramp. + """The pass/fail boundary for the gates, and the scale of :attr:`severity`. One instance governs a whole comparison -- every cell, every row total, - every column total -- so that a shade means the same thing wherever it + every column total -- so that a verdict means the same thing wherever it appears. A cell is :attr:`~CellStatus.MATCH` when ``|candidate - reference| <= atol + rtol * |reference|``. + This is a *gate* boundary, and only that. It no longer decides any colour: + the figures colour a cell by its relative difference on a continuous ramp, + so that where a cell falls is an observation rather than an artefact of + where the boundary was put. What still needs a boundary is :meth:`check`, + which has to answer pass or fail. + :param rtol: relative tolerance as a fraction (``0.013`` for the ~1.3% PCE bar, ``0.0022`` for PEQ, ``0.0`` for an identity that must hold exactly) :param atol: absolute tolerance in the table's own units, which keeps cells with a near-zero reference from failing on rounding alone - :param ramp: relative error at which the yellow ramp saturates. Severity is - ``0`` at the tolerance boundary and ``1`` at ``ramp`` and beyond, so the - shading has a stated scale that reports can quote. + :param ramp: relative error at which :attr:`severity` saturates. Severity + is ``0`` at the boundary and ``1`` at ``ramp`` and beyond. It is kept + because the gate reports quote it; the pictures do not use it. :param presence: ``|value| <= presence`` counts as *no data on that side*. The default treats an exact zero as absent, which is what a zero means in a published BEA cell. @@ -227,6 +233,10 @@ def status(self) -> pd.Series: def severity(self) -> pd.Series: return self.table['severity'] + @property + def rel_error(self) -> pd.Series: + return self.table['rel_error'] + def counts(self) -> pd.Series: return _counts(self.table['status'].to_numpy()) @@ -346,11 +356,52 @@ def coverage(self) -> float: @property def accuracy(self) -> float: - """Share of the cells we do populate that land within tolerance.""" + """Share of the cells we do populate that land within tolerance. + + A threshold statistic, kept for the gates in :meth:`check`, which are + pass/fail and therefore need a boundary. The figures and the standing + report quote :attr:`median_rel_error` and :attr:`weighted_rel_error` + instead: where a cell falls on a continuous scale is an observation, + where it falls relative to 1% is an artefact of picking 1%. + """ n = self.counts().loc['cells'] have = n['match'] + n['partial'] return float(n['match'] / have) if have else float('nan') + def _both_present(self) -> np.ndarray: + """Mask of the cells both sides populate -- ``MATCH`` or ``PARTIAL``.""" + s = self.status.to_numpy() + return (s == int(CellStatus.MATCH)) | (s == int(CellStatus.PARTIAL)) + + @property + def median_rel_error(self) -> float: + """Median relative difference across the cells both sides populate. + + The counterpart to :attr:`coverage`: coverage says how much of the + reference we have, this says what a typical cell we do have is worth. + Every cell counts once, so a $0.6M cell weighs as much as a $50bn one; + :attr:`weighted_rel_error` is the other half of the picture. + """ + rel = self.rel_error.to_numpy()[self._both_present()] + rel = rel[np.isfinite(rel)] + return float(np.median(rel)) if rel.size else float('nan') + + @property + def weighted_rel_error(self) -> float: + """Total absolute difference over total reference value, where both agree + a cell exists. + + The dollar-weighted counterpart to :attr:`median_rel_error`: the share + of the compared mass sitting in the wrong cell. A handful of large + cells can move it and four hundred tiny ones cannot, which is the + failure mode a count-based statistic has. + """ + both = self._both_present() + c = self.candidate.to_numpy()[both] + r = self.reference.to_numpy()[both] + denom = float(np.nansum(np.abs(r))) + return float(np.nansum(np.abs(c - r)) / denom) if denom else float('nan') + def summary(self) -> dict[str, float | str]: """A flat, machine-readable digest -- what CI records and compares.""" n = self.counts() @@ -360,6 +411,8 @@ def summary(self) -> dict[str, float | str]: out[f'{scope}.{name}'] = int(n.loc[scope, name]) out['coverage'] = self.coverage out['accuracy'] = self.accuracy + out['median_rel_error'] = self.median_rel_error + out['weighted_rel_error'] = self.weighted_rel_error gt = self.grand_total out['grand_total.candidate'] = float(gt['candidate']) out['grand_total.reference'] = float(gt['reference']) diff --git a/bedrock/transform/iot/nowcast_mut.py b/bedrock/transform/iot/nowcast_mut.py index 8c6837bfd..c97124dc4 100644 --- a/bedrock/transform/iot/nowcast_mut.py +++ b/bedrock/transform/iot/nowcast_mut.py @@ -86,6 +86,7 @@ PKG_VERSION_NUMBER, ) from bedrock.utils.economic.units import MILLION_CURRENCY_TO_CURRENCY +from bedrock.utils.metadata.metadata import source_lineage from bedrock.utils.taxonomy.bea.v2017_commodity import USA_2017_COMMODITY_CODES from bedrock.utils.taxonomy.bea.v2017_final_demand import SUT_FINAL_DEMAND_CODES from bedrock.utils.taxonomy.bea.v2017_industry import USA_2017_INDUSTRY_CODES @@ -410,6 +411,7 @@ def save_mut( 'branch': GIT_BRANCH, 'commit': GIT_HASH_LONG, 'balanced_inputs': list(tables.sources), + 'primary_source_meta': source_lineage(tables.sources), 'margin_anchor': ( '2017 rate panel and placement shares, tax-inclusive ' '(BEA definition); the nowcast-year fiscal split is #823' diff --git a/bedrock/transform/iot/nowcast_redefinitions.py b/bedrock/transform/iot/nowcast_redefinitions.py index 6d4e42fcc..f85216c03 100644 --- a/bedrock/transform/iot/nowcast_redefinitions.py +++ b/bedrock/transform/iot/nowcast_redefinitions.py @@ -72,6 +72,7 @@ PKG_VERSION_NUMBER, ) from bedrock.utils.economic.units import MILLION_CURRENCY_TO_CURRENCY +from bedrock.utils.metadata.metadata import source_lineage from bedrock.utils.taxonomy.bea.v2017_commodity import USA_2017_COMMODITY_CODES from bedrock.utils.taxonomy.bea.v2017_industry import USA_2017_INDUSTRY_CODES from bedrock.utils.taxonomy.bea.v2017_value_added import USA_2017_VALUE_ADDED_CODES @@ -548,6 +549,7 @@ def save_after_redef( 'branch': GIT_BRANCH, 'commit': GIT_HASH_LONG, 'before_inputs': before_inputs or [], + 'primary_source_meta': source_lineage(before_inputs or [], directory), 'method': ( 'Make: 2017 detail movement pattern on the year\'s own ' 'cells; Use: 2017 cell-ratio carry with commodity-row ' diff --git a/bedrock/utils/metadata/metadata.py b/bedrock/utils/metadata/metadata.py index adbfb473d..ea51f20d3 100644 --- a/bedrock/utils/metadata/metadata.py +++ b/bedrock/utils/metadata/metadata.py @@ -8,6 +8,7 @@ import json import os +from collections.abc import Iterable from pathlib import Path from typing import Any @@ -446,3 +447,43 @@ def getMetadata( meta = {'source_meta': f'No metadata found for {name}'} return meta + + +def source_lineage( + stems: Iterable[str], directory: Path | str | None = None +) -> dict[str, Any]: + """Sidecar metadata for artifacts named by *stems*, keyed for lineage. + + The bespoke savers in Step 5 through Step 7 write their products directly + rather than through the FBS framework, so nothing populates + ``primary_source_meta`` for them. Without it the staleness check has no + way to know an input moved: it reported "no stale cached artifacts" about + balanced SUTs sitting on trade inputs that had been rebuilt hours earlier. + Passing what a saver actually read through this function gives those + products the same lineage shape an FBS product carries, which is what + ``stale_artifacts._walk_sources`` already knows how to follow. + + :param stems: artifact stems without the ``.parquet`` suffix, as + ``load_balanced_sut`` returns them + :param directory: where the sidecars live; defaults to :data:`FBS_DIR` + :return: ``{name: metadata}``, keyed by the sidecar's own ``name_data`` + where it has one and by the stem otherwise. A stem with no readable + sidecar is recorded with a ``lineage_error`` rather than dropped -- a + silently missing source reads as "no inputs moved", which is the + failure this function exists to stop. + """ + where = Path(directory) if directory is not None else Path(FBS_DIR) + out: dict[str, Any] = {} + for stem in stems: + # NOT Path(stem).stem: these names carry a dotted version, so + # ``..._v0.3.0_d2e2112`` would lose its tail to a phantom suffix. + base = str(stem).removesuffix('.parquet') + path = where / f'{base}_metadata.json' + try: + meta = json.loads(path.read_text(encoding='utf-8')) + except (OSError, json.JSONDecodeError) as exc: + out[base] = {'lineage_error': f'no readable sidecar at {path}: {exc}'} + continue + key = meta.get('name_data') or base + out[str(key)] = meta + return out