diff --git a/tests/utils/test_archive.py b/tests/utils/test_archive.py index b3d041fd8..2114d78e0 100644 --- a/tests/utils/test_archive.py +++ b/tests/utils/test_archive.py @@ -13,6 +13,7 @@ from __future__ import annotations +import hashlib import os import tarfile from collections import Counter @@ -494,3 +495,212 @@ def test_create_snapshots_only_first_cycle_skips_runtime_files(tmp_path): # Runtime files kept loose for the next solve assert (tmp_path / 'spider_mesh.dat').exists() assert (tmp_path / 'spider_eos' / 'table.dat').exists() + + +# --------------------------------------------------------------------------- +# End-to-end lifecycle inflation +# +# The tests above exercise archive.* in isolation. The two below replay the +# full archive lifecycle from src/proteus/proteus.py on a realistic mini +# output/data/ directory and measure tar inflation by *content*, not just by +# name. This is what the production profiling exposed: a per-run data.tar that +# stored far more bytes than it extracted to, because tarfile append never +# deduplicates (archive.py:183-187). +# +# The proteus.py sequence being mirrored: +# * rolling, in the coupling loop (proteus.py:1188-1198): +# archive.update(data, remove_files=False, snapshots_only=True) +# archive.remove_old(data, before=Time*0.99) +# * final, at run end (proteus.py:1269-1272): +# archive.update(data, remove_files=True) # snapshots_only defaults False +# --------------------------------------------------------------------------- + +# Distinct simulated times [yr] for the rolling cycles. The ratio between +# successive times exceeds 1/0.99, so remove_old(before=t*0.99) prunes the +# previous cycle's snapshot only after the current cycle has already +# re-appended it -- the mechanism that duplicates every snapshot. +_CYCLE_TIMES = [100, 1000, 10000, 100000] + + +def _tar_content_stats(tar_path) -> dict: + """Inspect a tar by member *content*, not just by name. + + Returns a dict with: + ``gross`` total bytes over every file member (duplicates + counted every time they appear); + ``distinct`` total bytes counting each unique content once + (keyed by SHA-256 of the member bytes); + ``counts`` Counter of member name -> number of members; + ``distinct_by_name`` name -> number of *distinct* content hashes seen + for that name. + + ``inflation = gross / distinct``; 1.0 means no byte-identical duplication. + Measuring by content is what separates true waste -- the same bytes stored + twice, e.g. an unchanged EOS table re-appended every cycle -- from a + fixed-name file whose content legitimately evolved between cycles + (``distinct_by_name`` > 1), which must be preserved and is not waste. + """ + gross = 0 + size_by_hash: dict[str, int] = {} + counts: Counter = Counter() + hashes_by_name: dict[str, set[str]] = {} + with tarfile.open(tar_path, 'r') as tar: + for member in tar.getmembers(): + counts[member.name] += 1 + if not member.isfile(): + continue + data = tar.extractfile(member).read() + digest = hashlib.sha256(data).hexdigest() + gross += len(data) + size_by_hash.setdefault(digest, len(data)) + hashes_by_name.setdefault(member.name, set()).add(digest) + return { + 'gross': gross, + 'distinct': sum(size_by_hash.values()), + 'counts': counts, + 'distinct_by_name': {n: len(h) for n, h in hashes_by_name.items()}, + } + + +def _build_data_dir(root): + """Create a mini output/data/ mirroring the production layout. + + A sizeable, *unchanging* EOS lookup table (the payload the profiling + found duplicated ~200x), the fixed-name runtime hand-off files the + interior modules re-read between structure re-solves, and no snapshots + yet -- those are written per cycle by :func:`_replay_lifecycle`. + """ + root.mkdir() + (root / 'spider_eos').mkdir() + # Unchanging reference data: the same bytes for the whole run. + (root / 'spider_eos' / 'table.dat').write_bytes(b'eos-table' * 512) + (root / 'spider_mesh.dat').write_bytes(b'mesh' * 64) + (root / 'zalmoxis_output.dat').write_bytes(b'zalmoxis-v0' * 64) + return root + + +def _replay_lifecycle(data_dir, *, rolling_snapshots_only): + """Replay the proteus.py archive lifecycle over :data:`_CYCLE_TIMES`. + + Each cycle writes the interior+atmosphere snapshots for the current time + (content depends only on the time, so a given snapshot's bytes never + change once written -- re-appending it is byte-identical waste), mutates + the fixed-name ``zalmoxis_output.dat`` to a new content (as a dynamic run + does), rolls the directory into the tar, then prunes old snapshots. After + the loop the end-of-run full archive packs whatever is still loose. + """ + for i, t in enumerate(_CYCLE_TIMES): + (data_dir / f'{t}_int.nc').write_bytes((b'int-%d' % t) * 64) + (data_dir / f'{t}_atm.nc').write_bytes((b'atm-%d' % t) * 64) + # Fixed-name file whose content legitimately evolves each cycle. + (data_dir / 'zalmoxis_output.dat').write_bytes((b'zalmoxis-v%d' % i) * 64) + # Rolling in-loop archive (proteus.py:1188-1198). + archive_mod.update( + str(data_dir), remove_files=False, snapshots_only=rolling_snapshots_only + ) + archive_mod.remove_old(str(data_dir), before=t * 0.99) + # End-of-run full archive (proteus.py:1269-1272): snapshots_only defaults + # to False, so this packs the still-loose snapshots plus every fixed-name + # file and the EOS directory in one pass. + archive_mod.update(str(data_dir), remove_files=True) + + +def test_lifecycle_final_pass_stores_reference_data_once(tmp_path): + """Over the *full* proteus.py lifecycle the static EOS table and the + fixed-name runtime files end up in data.tar exactly once. + + Scope vs the existing suite: that snapshots_only=True keeps these files + out of the per-cycle *rolling* growth is already covered by + ``test_update_snapshots_only_bounds_tar_growth_across_cycles`` (added in + PROTEUS #706), which stops after the rolling loop and checks name counts. + This test extends that coverage in two ways it does not reach: + 1. it also runs the end-of-run full archive + (``update(remove_files=True)``, proteus.py:1269-1272), so the + assertion is "exactly once across the whole lifecycle" rather than + "zero during rolling" -- the final pass is where these files + legitimately enter the tar; + 2. it measures by member *content*, not just by name. + + Content-level discrimination against the pre-#706 behaviour: replaying + the identical directory with snapshots_only=False (which emulates the + frozen release that produced the profiled dataset) re-appends the EOS + table every cycle. By member *content* those repeats are one distinct + payload duplicated len(_CYCLE_TIMES)+1 times -- pure waste -- whereas the + evolving zalmoxis_output.dat yields one distinct content per cycle, which + must be preserved rather than counted as waste. A name-only count cannot + tell these apart; the assertions below check both directions so the guard + fails on the unfiltered path and is not trivially satisfied. + """ + n_cycles = len(_CYCLE_TIMES) + + fixed = _build_data_dir(tmp_path / 'fixed') + _replay_lifecycle(fixed, rolling_snapshots_only=True) + fixed_stats = _tar_content_stats(fixed / 'fixed.tar') + + # The fix: reference data and fixed-name files enter the tar exactly once. + assert fixed_stats['counts']['spider_eos/table.dat'] == 1 + assert fixed_stats['counts']['spider_mesh.dat'] == 1 + assert fixed_stats['counts']['zalmoxis_output.dat'] == 1 + # ... and that one EOS copy is on the reference table, not duplicated bytes. + assert fixed_stats['distinct_by_name']['spider_eos/table.dat'] == 1 + + unfixed = _build_data_dir(tmp_path / 'unfixed') + _replay_lifecycle(unfixed, rolling_snapshots_only=False) + unfixed_stats = _tar_content_stats(unfixed / 'unfixed.tar') + + # Discrimination: without the filter the unchanging EOS table is re-added + # every rolling cycle plus once by the final pass -- many members, one + # distinct content. This is exactly the storage waste the profiling found, + # and proves the guard above would fail on the pre-#706 code path. + assert unfixed_stats['counts']['spider_eos/table.dat'] == n_cycles + 1 + assert unfixed_stats['distinct_by_name']['spider_eos/table.dat'] == 1 + # But zalmoxis_output.dat genuinely changed each cycle: content hashing + # keeps all n_cycles distinct versions and must not treat them as waste. + assert unfixed_stats['distinct_by_name']['zalmoxis_output.dat'] == n_cycles + + +@pytest.mark.xfail( + reason=( + 'Snapshot .nc/.json members are re-appended across rolling cycles ' + '(the in-loop archive uses remove_files=False, so a snapshot stays ' + 'loose and is appended again on the next cycle before remove_old ' + 'prunes it) and once more by the final full-archive pass. PROTEUS ' + '#706 fixed the EOS/fixed-file inflation via snapshots_only=True but ' + 'not this snapshot re-append: on main every snapshot lands in ' + 'data.tar ~2x (observed inflation ~1.45x on this fixture). Expected ' + 'to pass once the Phase 2 archive redesign stores each distinct ' + 'content once.' + ), + strict=True, +) +def test_lifecycle_snapshot_reappend_no_inflation(tmp_path): + """The archive should store each snapshot once: replaying the proteus.py + lifecycle must leave every timestamped snapshot in data.tar exactly once + and an overall content-inflation factor of ~1.0. + + This encodes the *target* contract, which main does not yet meet -- hence + the strict xfail. Verified behaviour on main (commit reachable from this + branch, run with the snapshots_only=True rolling config, i.e. the current + fix): each snapshot appears twice and gross/distinct is ~1.45 on this + fixture. strict=True means the test turns into a hard failure the moment + the redesign removes the duplication, forcing this marker to be dropped. + """ + data_dir = _build_data_dir(tmp_path / 'run') + _replay_lifecycle(data_dir, rolling_snapshots_only=True) + stats = _tar_content_stats(data_dir / 'run.tar') + + # Primary contract: no snapshot is stored more than once. + snapshot_names = [ + name + for name in stats['counts'] + if archive_mod._snapshot_time(os.path.basename(name)) is not None + ] + # Discrimination: the fixture really does produce snapshots to check, so a + # regression that archived nothing could not make this pass vacuously. + assert len(snapshot_names) == 2 * len(_CYCLE_TIMES) # int + atm per cycle + for name in snapshot_names: + assert stats['counts'][name] == 1 + + # Overall: bytes stored should equal bytes of distinct content (no waste). + inflation = stats['gross'] / stats['distinct'] + assert inflation == pytest.approx(1.0, abs=0.01)