Test: reproduce/guard data.tar inflation over the archive lifecycle

tests/utils/test_archive.py

@@ -13,6 +13,7 @@
 
 from __future__ import annotations
 
+import hashlib
 import os
 import tarfile
 from collections import Counter
@@ -494,3 +495,212 @@ def test_create_snapshots_only_first_cycle_skips_runtime_files(tmp_path):
     # Runtime files kept loose for the next solve
     assert (tmp_path / 'spider_mesh.dat').exists()
     assert (tmp_path / 'spider_eos' / 'table.dat').exists()
+
+
+# ---------------------------------------------------------------------------
+# End-to-end lifecycle inflation
+#
+# The tests above exercise archive.* in isolation. The two below replay the
+# full archive lifecycle from src/proteus/proteus.py on a realistic mini
+# output/data/ directory and measure tar inflation by *content*, not just by
+# name. This is what the production profiling exposed: a per-run data.tar that
+# stored far more bytes than it extracted to, because tarfile append never
+# deduplicates (archive.py:183-187).
+#
+# The proteus.py sequence being mirrored:
+#   * rolling, in the coupling loop (proteus.py:1188-1198):
+#       archive.update(data, remove_files=False, snapshots_only=True)
+#       archive.remove_old(data, before=Time*0.99)
+#   * final, at run end (proteus.py:1269-1272):
+#       archive.update(data, remove_files=True)   # snapshots_only defaults False
+# ---------------------------------------------------------------------------
+
+# Distinct simulated times [yr] for the rolling cycles. The ratio between
+# successive times exceeds 1/0.99, so remove_old(before=t*0.99) prunes the
+# previous cycle's snapshot only after the current cycle has already
+# re-appended it -- the mechanism that duplicates every snapshot.
+_CYCLE_TIMES = [100, 1000, 10000, 100000]
+
+
+def _tar_content_stats(tar_path) -> dict:
+    """Inspect a tar by member *content*, not just by name.
+
+    Returns a dict with:
+      ``gross``            total bytes over every file member (duplicates
+                           counted every time they appear);
+      ``distinct``         total bytes counting each unique content once
+                           (keyed by SHA-256 of the member bytes);
+      ``counts``           Counter of member name -> number of members;
+      ``distinct_by_name`` name -> number of *distinct* content hashes seen
+                           for that name.
+
+    ``inflation = gross / distinct``; 1.0 means no byte-identical duplication.
+    Measuring by content is what separates true waste -- the same bytes stored
+    twice, e.g. an unchanged EOS table re-appended every cycle -- from a
+    fixed-name file whose content legitimately evolved between cycles
+    (``distinct_by_name`` > 1), which must be preserved and is not waste.
+    """
+    gross = 0
+    size_by_hash: dict[str, int] = {}
+    counts: Counter = Counter()
+    hashes_by_name: dict[str, set[str]] = {}
+    with tarfile.open(tar_path, 'r') as tar:
+        for member in tar.getmembers():
+            counts[member.name] += 1
+            if not member.isfile():
+                continue
+            data = tar.extractfile(member).read()
+            digest = hashlib.sha256(data).hexdigest()
+            gross += len(data)
+            size_by_hash.setdefault(digest, len(data))
+            hashes_by_name.setdefault(member.name, set()).add(digest)
+    return {
+        'gross': gross,
+        'distinct': sum(size_by_hash.values()),
+        'counts': counts,
+        'distinct_by_name': {n: len(h) for n, h in hashes_by_name.items()},
+    }
+
+
+def _build_data_dir(root):
+    """Create a mini output/data/ mirroring the production layout.
+
+    A sizeable, *unchanging* EOS lookup table (the payload the profiling
+    found duplicated ~200x), the fixed-name runtime hand-off files the
+    interior modules re-read between structure re-solves, and no snapshots
+    yet -- those are written per cycle by :func:`_replay_lifecycle`.
+    """
+    root.mkdir()
+    (root / 'spider_eos').mkdir()
+    # Unchanging reference data: the same bytes for the whole run.
+    (root / 'spider_eos' / 'table.dat').write_bytes(b'eos-table' * 512)
+    (root / 'spider_mesh.dat').write_bytes(b'mesh' * 64)
+    (root / 'zalmoxis_output.dat').write_bytes(b'zalmoxis-v0' * 64)
+    return root
+
+
+def _replay_lifecycle(data_dir, *, rolling_snapshots_only):
+    """Replay the proteus.py archive lifecycle over :data:`_CYCLE_TIMES`.
+
+    Each cycle writes the interior+atmosphere snapshots for the current time
+    (content depends only on the time, so a given snapshot's bytes never
+    change once written -- re-appending it is byte-identical waste), mutates
+    the fixed-name ``zalmoxis_output.dat`` to a new content (as a dynamic run
+    does), rolls the directory into the tar, then prunes old snapshots. After
+    the loop the end-of-run full archive packs whatever is still loose.
+    """
+    for i, t in enumerate(_CYCLE_TIMES):
+        (data_dir / f'{t}_int.nc').write_bytes((b'int-%d' % t) * 64)
+        (data_dir / f'{t}_atm.nc').write_bytes((b'atm-%d' % t) * 64)
+        # Fixed-name file whose content legitimately evolves each cycle.
+        (data_dir / 'zalmoxis_output.dat').write_bytes((b'zalmoxis-v%d' % i) * 64)
+        # Rolling in-loop archive (proteus.py:1188-1198).
+        archive_mod.update(
+            str(data_dir), remove_files=False, snapshots_only=rolling_snapshots_only
+        )
+        archive_mod.remove_old(str(data_dir), before=t * 0.99)
+    # End-of-run full archive (proteus.py:1269-1272): snapshots_only defaults
+    # to False, so this packs the still-loose snapshots plus every fixed-name
+    # file and the EOS directory in one pass.
+    archive_mod.update(str(data_dir), remove_files=True)
+
+
+def test_lifecycle_final_pass_stores_reference_data_once(tmp_path):
+    """Over the *full* proteus.py lifecycle the static EOS table and the
+    fixed-name runtime files end up in data.tar exactly once.
+
+    Scope vs the existing suite: that snapshots_only=True keeps these files
+    out of the per-cycle *rolling* growth is already covered by
+    ``test_update_snapshots_only_bounds_tar_growth_across_cycles`` (added in
+    PROTEUS #706), which stops after the rolling loop and checks name counts.
+    This test extends that coverage in two ways it does not reach:
+      1. it also runs the end-of-run full archive
+         (``update(remove_files=True)``, proteus.py:1269-1272), so the
+         assertion is "exactly once across the whole lifecycle" rather than
+         "zero during rolling" -- the final pass is where these files
+         legitimately enter the tar;
+      2. it measures by member *content*, not just by name.
+
+    Content-level discrimination against the pre-#706 behaviour: replaying
+    the identical directory with snapshots_only=False (which emulates the
+    frozen release that produced the profiled dataset) re-appends the EOS
+    table every cycle. By member *content* those repeats are one distinct
+    payload duplicated len(_CYCLE_TIMES)+1 times -- pure waste -- whereas the
+    evolving zalmoxis_output.dat yields one distinct content per cycle, which
+    must be preserved rather than counted as waste. A name-only count cannot
+    tell these apart; the assertions below check both directions so the guard
+    fails on the unfiltered path and is not trivially satisfied.
+    """
+    n_cycles = len(_CYCLE_TIMES)
+
+    fixed = _build_data_dir(tmp_path / 'fixed')
+    _replay_lifecycle(fixed, rolling_snapshots_only=True)
+    fixed_stats = _tar_content_stats(fixed / 'fixed.tar')
+
+    # The fix: reference data and fixed-name files enter the tar exactly once.
+    assert fixed_stats['counts']['spider_eos/table.dat'] == 1
+    assert fixed_stats['counts']['spider_mesh.dat'] == 1
+    assert fixed_stats['counts']['zalmoxis_output.dat'] == 1
+    # ... and that one EOS copy is on the reference table, not duplicated bytes.
+    assert fixed_stats['distinct_by_name']['spider_eos/table.dat'] == 1
+
+    unfixed = _build_data_dir(tmp_path / 'unfixed')
+    _replay_lifecycle(unfixed, rolling_snapshots_only=False)
+    unfixed_stats = _tar_content_stats(unfixed / 'unfixed.tar')
+
+    # Discrimination: without the filter the unchanging EOS table is re-added
+    # every rolling cycle plus once by the final pass -- many members, one
+    # distinct content. This is exactly the storage waste the profiling found,
+    # and proves the guard above would fail on the pre-#706 code path.
+    assert unfixed_stats['counts']['spider_eos/table.dat'] == n_cycles + 1
+    assert unfixed_stats['distinct_by_name']['spider_eos/table.dat'] == 1
+    # But zalmoxis_output.dat genuinely changed each cycle: content hashing
+    # keeps all n_cycles distinct versions and must not treat them as waste.
+    assert unfixed_stats['distinct_by_name']['zalmoxis_output.dat'] == n_cycles
+
+
+@pytest.mark.xfail(
+    reason=(
+        'Snapshot .nc/.json members are re-appended across rolling cycles '
+        '(the in-loop archive uses remove_files=False, so a snapshot stays '
+        'loose and is appended again on the next cycle before remove_old '
+        'prunes it) and once more by the final full-archive pass. PROTEUS '
+        '#706 fixed the EOS/fixed-file inflation via snapshots_only=True but '
+        'not this snapshot re-append: on main every snapshot lands in '
+        'data.tar ~2x (observed inflation ~1.45x on this fixture). Expected '
+        'to pass once the Phase 2 archive redesign stores each distinct '
+        'content once.'
+    ),
+    strict=True,
+)
+def test_lifecycle_snapshot_reappend_no_inflation(tmp_path):
+    """The archive should store each snapshot once: replaying the proteus.py
+    lifecycle must leave every timestamped snapshot in data.tar exactly once
+    and an overall content-inflation factor of ~1.0.
+
+    This encodes the *target* contract, which main does not yet meet -- hence
+    the strict xfail. Verified behaviour on main (commit reachable from this
+    branch, run with the snapshots_only=True rolling config, i.e. the current
+    fix): each snapshot appears twice and gross/distinct is ~1.45 on this
+    fixture. strict=True means the test turns into a hard failure the moment
+    the redesign removes the duplication, forcing this marker to be dropped.
+    """
+    data_dir = _build_data_dir(tmp_path / 'run')
+    _replay_lifecycle(data_dir, rolling_snapshots_only=True)
+    stats = _tar_content_stats(data_dir / 'run.tar')
+
+    # Primary contract: no snapshot is stored more than once.
+    snapshot_names = [
+        name
+        for name in stats['counts']
+        if archive_mod._snapshot_time(os.path.basename(name)) is not None
+    ]
+    # Discrimination: the fixture really does produce snapshots to check, so a
+    # regression that archived nothing could not make this pass vacuously.
+    assert len(snapshot_names) == 2 * len(_CYCLE_TIMES)  # int + atm per cycle
+    for name in snapshot_names:
+        assert stats['counts'][name] == 1
+
+    # Overall: bytes stored should equal bytes of distinct content (no waste).
+    inflation = stats['gross'] / stats['distinct']
+    assert inflation == pytest.approx(1.0, abs=0.01)