diff --git a/PROGRESS.md b/PROGRESS.md index e528270f7..53a600861 100644 --- a/PROGRESS.md +++ b/PROGRESS.md @@ -1,3 +1,68 @@ +# Issue #907 — population `_storage_parts` hashes object dtype by pointer + +Lane: `fix-907-population-stamp-object-storage`, branched from +`origin/main` at `295130c9f901e08db11457f16dbdee4e2349c5ba` on 2026-09-11. +Report: `/Users/maxghenis/PolicyEngine/_recovered/scratch-backup/893/lanes/out-907-build-r1.md`. + +## State + +Implementation complete on `fix-907-population-stamp-object-storage`; ten +commits, nothing pushed, no new branch, no stash. `decl.py`, `kernel.py`, +`docs/graph-interface.lock`, every `test_acceptance_*`, `uv.lock`, spec pins +and evidence JSON are untouched — `shasum -a 256` on `decl.py`/`kernel.py` +still matches the lock byte for byte. + +## Done + +- Read `CLAUDE.md` and `docs/shared-constants.md`; confirmed the lane rules. +- Reproduced #907 directly, and end to end: two independently constructed + equal object-dtype Series give different `_storage_parts` value bytes + (PyObject addresses), which on `origin/main` surfaces as a **spurious** + `PopulationError: Structural node 'n1' changed carried storage in + household.tenure_type` for content that did not change. +- Landed the red regression in `packages/microcosm-graph/tests/` + (`test_graph_population.py`, an already-tracked file, so + `tools/ci_test_groups.py` needed no change). Final tests against the pre-fix + source: 34 failed, 24 passed, 46 deselected; direct exit 1. +- Fixed `_storage_parts` by routing any materialized object array through a + length-prefixed encoding whose body is `store._encode_object_scalar` — the + graph package's existing object-leaf codec, the one `ContentStore` writes + and reads back. A second, parallel vocabulary would have left a column + unequal to its own persisted-and-reloaded self for `pd.NA`, `pd.NaT` and + NumPy scalars. Unsupported leaves raise `PopulationError` under the static + code `storage-object-leaf` instead of being `repr()`-ed. +- Kept the masked, numeric and `StringDtype` branches byte-identical, verified + by a pre-fix/post-fix byte diff over 20 dtypes and pinned by hex-literal + characterization tests. Only object, Categorical, DatetimeTZ, Period and + Interval move — every one of them a dtype whose old bytes were addresses. +- Documented at `storage_equal` what the parts do and do not seal, with + regressions pinning the masked half, the numpy-normalization, the new + refusals, and a real `ContentStore` frame round trip. +- Adversarial review (three lenses) found two real defects, both fixed: the + refusal missed `UnicodeEncodeError` from a lone-surrogate `str`, and two + deliberate normalizations were undocumented. +- Green: `pytest packages/microcosm-graph` 399 passed; the nine graph-adjacent + build/frame/fit/calibrate files 578 passed; `test_graph_population.py` 119 + passed; `ruff check .` 0; `ruff format --check` on both changed files 0; + `tools/ci_test_groups.py --verify` 0; `tools/spec_engine_coverage.py + --check` 0 (42156/42156 fields, 41/41 inventory); `tools/graph_acceptance_ + burndown.py --verify` 0 (green 41, red 0, missing 0). + +## Next + +- Human review. One item this lane could not complete: the issue asked for a + note at `_population_stamp` in `us_runtime/survey_atomic_geography.py`, a + module that exists only at the tip of the unmerged + `origin/microcosm-us-launch-integration-20260909`. The verified wording — + corrected, because the issue's own phrasing was incomplete — is in the lane + report for whoever owns that branch. +- No pinned digest moves. The three `*_population_sha256` stamps on that + branch will take new values once this merges; their old values were process + addresses, they are recomputed on both sides of every comparison, and + `survey_origin_budget` already excludes them from the persisted identity. + +--- + # F1 portable worker identity — CI crawl fix ## State diff --git a/changelog.d/907-object-storage-hashing.fixed.md b/changelog.d/907-object-storage-hashing.fixed.md new file mode 100644 index 000000000..a35cad16c --- /dev/null +++ b/changelog.d/907-object-storage-hashing.fixed.md @@ -0,0 +1 @@ +Hash object-dtype graph storage by content instead of PyObject pointers, so a population column stays storage-equal to its own persisted-and-reloaded self. diff --git a/packages/microcosm-graph/src/microcosm/graph/population.py b/packages/microcosm-graph/src/microcosm/graph/population.py index b5561f1d1..bdeef2b50 100644 --- a/packages/microcosm-graph/src/microcosm/graph/population.py +++ b/packages/microcosm-graph/src/microcosm/graph/population.py @@ -26,6 +26,7 @@ StructuralDelta, ) from .kernel import KernelResult +from .store import _encode_object_scalar __all__ = [ "MassRecord", @@ -1036,7 +1037,26 @@ def storage_equal( right: pd.Series, positions: np.ndarray | pd.Series | None = None, ) -> bool: - """Compare physical values and nullable masks exactly, including float bits.""" + """Compare physical values and nullable masks exactly, including float bits. + + Dense, string and object leaves are compared by content, so a column stays + equal to its own persisted-and-reloaded self. Object leaves are compared + at the ContentStore's own leaf resolution, which is what makes that round + trip a fixed point: a NumPy floating leaf compares at ``float64`` width and + a NumPy integer leaf by value, because that is what the store decodes back. + A leaf outside that vocabulary raises :class:`PopulationError` rather than + comparing addresses, so for object columns this predicate is not total. + + Masked storage is not compared by content: it reads ``_data`` under the + null mask, where pandas leaves whatever the construction route happened to + put, so two content-equal ``Int64`` columns built by different routes can + legitimately differ. A digest folded from these parts is therefore an + in-process seal — a statement about what the digest can identify, not about + where it may be stored. Build a cross-reconstruction pin from a content + identity of the values instead, together with whatever population-level + state the pin has to cover; a frame identity alone does not carry the + version, owners, weight kinds, mass ledger or design weights. + """ if left.dtype != right.dtype or len(left) != len(right): return False @@ -2671,6 +2691,51 @@ def _rebuild_frame(frame: Frame, tables: Mapping[str, pd.DataFrame]) -> Frame: ) +def _object_storage_values(values: np.ndarray) -> bytes: + """Encode an object array as length-prefixed, type-tagged content bytes. + + An object array holds PyObject pointers, so ``tobytes()`` on one + serializes addresses: equal content re-materialized in a second array + hashes differently, and a comparison across a store round trip or a + rebuilt frame can never agree. Each leaf therefore contributes an 8-byte + little-endian body length followed by the ContentStore's own object-scalar + body, mirroring the ``StringDtype`` framing above. Reusing that encoder is + what keeps a column equal to its own persisted-and-reloaded self: it is the + store's definition of an object leaf's bytes, and its tags keep ``1``, + ``1.0``, ``True``, ``"1"`` and ``b"1"`` distinct. (The executor's + kernel-context digest keeps a separate leaf vocabulary in + ``executor._update_scalar``, with a ``repr()`` fallback; storage hashing + deliberately does not share it.) Every body + carries a tag, so a body is never empty and a zero length stays reserved. + + What it deliberately does not keep distinct is a NumPy integer or float + scalar from its Python counterpart: the encoder normalizes ``np.int32(1)`` + to ``1`` and ``np.float32(1.0)`` to ``1.0`` because the store's decoder + hands back the Python form, so a leaf-type-only difference inside an + object column is not a storage change. Refusing it would mean a column + could never equal its own persisted-and-reloaded self, which is the defect + being fixed. ``np.timedelta64`` and ``np.datetime64`` leaves are refused + outright: timedelta64 is a signedinteger subclass at runtime, and encoding + it as an integer would drop the unit and collide with a plain int. + """ + + payload = bytearray() + for value in values: + try: + body = _encode_object_scalar(value) + except (TypeError, UnicodeEncodeError) as error: + # Both ways that encoder declines a leaf: TypeError for a type + # outside its vocabulary, UnicodeEncodeError for a str that is not + # encodable (a lone surrogate). Fail closed rather than repr() an + # unvetted leaf: a repr is neither guaranteed injective nor + # guaranteed stable across reconstructions, and calling it can + # itself raise. + raise PopulationError(f"storage-object-leaf: {error}") from error + payload.extend(len(body).to_bytes(8, "little")) + payload.extend(body) + return bytes(payload) + + def _storage_parts(series: pd.Series, selected: np.ndarray) -> tuple[bytes, bytes]: nulls = series.isna().to_numpy(dtype=np.bool_, copy=False)[selected] array = series.array @@ -2693,6 +2758,11 @@ def _storage_parts(series: pd.Series, selected: np.ndarray) -> tuple[bytes, byte payload.extend(encoded) return bytes(payload), np.ascontiguousarray(nulls).tobytes() values = series.to_numpy(copy=False)[selected] + if values.dtype == object: + return ( + _object_storage_values(values), + np.ascontiguousarray(nulls).tobytes(), + ) return ( np.ascontiguousarray(values).tobytes(), np.ascontiguousarray(nulls).tobytes(), diff --git a/packages/microcosm-graph/src/microcosm/graph/store.py b/packages/microcosm-graph/src/microcosm/graph/store.py index 6f93b8b65..2d943d4a4 100644 --- a/packages/microcosm-graph/src/microcosm/graph/store.py +++ b/packages/microcosm-graph/src/microcosm/graph/store.py @@ -329,6 +329,13 @@ def _encode_object_scalar(value: object) -> bytes: return bytes([_TAG_PD_NAT]) if isinstance(value, (bool, np.bool_)): return bytes([_TAG_TRUE if bool(value) else _TAG_FALSE]) + if isinstance(value, (np.timedelta64, np.datetime64)): + # timedelta64 subclasses signedinteger at runtime; letting it reach the + # integer branch would drop the unit and collide with a plain int. + raise TypeError( + "Object columns may not carry numpy datetime64 or timedelta64 leaves; " + f"found {type(value).__name__}." + ) if isinstance(value, (int, np.integer)): return bytes([_TAG_INTEGER]) + str(int(value)).encode("ascii") if isinstance(value, (float, np.floating)): diff --git a/packages/microcosm-graph/tests/test_graph_population.py b/packages/microcosm-graph/tests/test_graph_population.py index e08aac0f8..3735edda9 100644 --- a/packages/microcosm-graph/tests/test_graph_population.py +++ b/packages/microcosm-graph/tests/test_graph_population.py @@ -2,6 +2,10 @@ from __future__ import annotations +from datetime import date +from decimal import Decimal +from pathlib import Path + import numpy as np import pandas as pd import pytest @@ -19,6 +23,7 @@ from microcosm.graph.population import ( Population, PopulationError, + _storage_parts, dtype_for_token, dtype_matches, entrant_strata_receipt, @@ -30,6 +35,7 @@ token_for_dtype, weight_cap_receipt, ) +from microcosm.graph.store import ContentStore, _encode_object_scalar def _frame() -> Frame: @@ -1436,6 +1442,438 @@ def test_reweight_can_synthesize_frame_but_must_not_change_ids() -> None: patch(population, node, KernelResult(frame=reordered)) +# --------------------------------------------------------------------------- +# Object-dtype storage hashing (issue #907) +# --------------------------------------------------------------------------- + + +def _fresh_str(value: str) -> str: + """Return an equal ``str`` that is a distinct object from every literal.""" + + return "".join(list(value)) + + +def _object_series(values: list[object]) -> pd.Series: + return pd.Series(values, dtype=object) + + +_ALL_ROWS = np.ones(3, dtype=np.bool_) + + +def test_object_storage_hashes_content_not_pyobject_pointers() -> None: + left = _object_series([_fresh_str("alpha"), _fresh_str("beta"), None]) + right = _object_series([_fresh_str("alpha"), _fresh_str("beta"), None]) + + assert left.dtype == object + assert [id(value) for value in left.to_numpy()[:2]] != [ + id(value) for value in right.to_numpy()[:2] + ] + assert _storage_parts(left, _ALL_ROWS) == _storage_parts(right, _ALL_ROWS) + assert storage_equal(left, right) + + +def test_object_storage_separates_differing_content() -> None: + left = _object_series([_fresh_str("alpha"), _fresh_str("beta"), None]) + right = _object_series([_fresh_str("alpha"), _fresh_str("gamma"), None]) + + assert _storage_parts(left, _ALL_ROWS) != _storage_parts(right, _ALL_ROWS) + assert not storage_equal(left, right) + + +def test_object_storage_keeps_the_null_bitmap_separate_from_values() -> None: + series = _object_series([_fresh_str("alpha"), None, _fresh_str("beta")]) + + values, bitmap = _storage_parts(series, _ALL_ROWS) + + assert bitmap == np.array([False, True, False]).tobytes() + assert ( + values + == _storage_parts( + _object_series([_fresh_str("alpha"), None, _fresh_str("beta")]), _ALL_ROWS + )[0] + ) + # A moved null is a different column even though the value bytes of the + # surviving labels are unchanged, exactly as in the StringDtype branch. + moved = _object_series([None, _fresh_str("alpha"), _fresh_str("beta")]) + assert _storage_parts(series, _ALL_ROWS) != _storage_parts(moved, _ALL_ROWS) + + +def test_object_storage_respects_the_selection_mask() -> None: + left = _object_series([_fresh_str("alpha"), _fresh_str("beta"), None]) + right = _object_series([_fresh_str("alpha"), _fresh_str("zeta"), None]) + selected = np.array([True, False, True]) + + assert _storage_parts(left, selected) == _storage_parts(right, selected) + assert storage_equal(left, right, selected) + assert not storage_equal(left, right) + + +@pytest.mark.parametrize( + ("left_leaf", "right_leaf"), + [ + ("1", 1), + ("1", b"1"), + (1, b"1"), + (1, True), + (1, 1.0), + (True, 1.0), + ("", None), + (b"", None), + (0.0, -0.0), + ("a", "a\x00"), + ("ab", "a"), + ], +) +def test_object_storage_does_not_conflate_distinct_leaves( + left_leaf: object, right_leaf: object +) -> None: + left = _object_series([left_leaf, None, None]) + right = _object_series([right_leaf, None, None]) + + assert _storage_parts(left, _ALL_ROWS) != _storage_parts(right, _ALL_ROWS) + + +@pytest.mark.parametrize( + "leaf", + [ + _fresh_str("label"), + b"label", + 7, + -(2**70), + 0, + 1.5, + -0.0, + True, + False, + None, + pd.NA, + pd.NaT, + np.int64(3), + np.bool_(True), + np.float64(1.5), + np.bytes_(b"label"), + np.str_("label"), + ], +) +def test_object_storage_accepts_every_supported_leaf(leaf: object) -> None: + left = _object_series([leaf, None, _fresh_str("tail")]) + right = _object_series([leaf, None, _fresh_str("tail")]) + + assert _storage_parts(left, _ALL_ROWS) == _storage_parts(right, _ALL_ROWS) + + +def test_object_storage_preserves_negative_zero_and_nan_payloads() -> None: + nan_payload = np.frombuffer( + np.uint64(0x7FF8_0000_0000_0001).tobytes(), dtype=np.float64 + )[0] + quiet_nan = float("nan") + + assert _storage_parts( + _object_series([-0.0, None, None]), _ALL_ROWS + ) != _storage_parts(_object_series([0.0, None, None]), _ALL_ROWS) + assert _storage_parts( + _object_series([float(nan_payload), None, None]), _ALL_ROWS + ) != _storage_parts(_object_series([quiet_nan, None, None]), _ALL_ROWS) + + +@pytest.mark.parametrize( + "leaf", + [ + object(), + ("tuple",), + ["list"], + {"set"}, + {"dict": 1}, + bytearray(b"mutable"), + complex(1, 2), + np.datetime64("2020-01-01"), + np.timedelta64(1, "D"), + np.timedelta64(1, "ns"), + np.timedelta64(1, "M"), + np.timedelta64(1, "Y"), + Decimal("1.5"), + date(2020, 1, 1), + ], +) +def test_object_storage_refuses_unsupported_leaves(leaf: object) -> None: + series = _object_series([leaf, None, None]) + + with pytest.raises(PopulationError, match="storage-object-leaf"): + _storage_parts(series, _ALL_ROWS) + + +@pytest.mark.parametrize( + "leaf", + [ + np.timedelta64(1, "ns"), + np.timedelta64(1, "M"), + np.timedelta64(1, "Y"), + np.datetime64("2020-01-01"), + ], +) +def test_object_storage_refuses_numpy_datetimes_by_type_not_by_unit( + leaf: object, +) -> None: + """``np.timedelta64`` subclasses ``np.signedinteger`` at runtime. + + A unit ``int()`` converts (ns, M, Y) would otherwise reach the integer + branch, drop the unit, and collide with the plain ``1``; the codec must + refuse the type before that branch, so the refusal never depends on which + unit happens to fail ``int()``. + """ + + from microcosm.graph.store import _encode_object_scalar + + assert _encode_object_scalar(1) == _encode_object_scalar(np.int64(1)) + with pytest.raises(TypeError, match="datetime64 or timedelta64"): + _encode_object_scalar(leaf) + series = _object_series([leaf, None, None]) + with pytest.raises(PopulationError, match="storage-object-leaf"): + _storage_parts(series, _ALL_ROWS) + + +def test_object_storage_refuses_an_unencodable_string_leaf() -> None: + """A lone surrogate makes the encoder raise UnicodeEncodeError, not TypeError.""" + + series = _object_series(["\ud800", None, None]) + + with pytest.raises(PopulationError, match="storage-object-leaf"): + _storage_parts(series, _ALL_ROWS) + + +def test_object_storage_refuses_an_unsupported_leaf_compared_with_itself() -> None: + """The one place the refusal is new rather than sharper. + + Positional copies preserve PyObject identity, so the pre-fix pointer + comparison answered True for a column compared against itself no matter + what it held. It now refuses, matching ContentStore, which will not + persist such a column, and token_for_dtype, which will not declare it. + """ + + series = _object_series([date(2020, 1, 1), None, None]) + + with pytest.raises(PopulationError, match="storage-object-leaf"): + storage_equal(series, series) + + +def test_object_storage_refusal_message_excludes_the_value_repr() -> None: + class _Loud: + def __repr__(self) -> str: # pragma: no cover - must never be called + raise AssertionError("storage refusal must not repr the leaf") + + series = _object_series([_Loud(), None, None]) + + with pytest.raises(PopulationError, match="storage-object-leaf") as excinfo: + _storage_parts(series, _ALL_ROWS) + assert "_Loud" in str(excinfo.value) + + +@pytest.mark.parametrize( + ("numpy_leaf", "python_leaf"), + [ + (np.int32(1), 1), + (np.int64(1), 1), + (np.uint64(2**64 - 1), 2**64 - 1), + (np.float32(1.0), 1.0), + (np.float64(1.5), 1.5), + (np.bool_(True), True), + (np.str_("a"), "a"), + (np.bytes_(b"a"), b"a"), + ], +) +def test_object_storage_normalizes_numpy_scalars_like_the_store_decoder( + numpy_leaf: object, python_leaf: object +) -> None: + """A deliberate widening, and the reason the store round trip is a fixed point. + + ``_decode_object_chunks`` hands back the Python form, so refusing to call + these equal would mean a column could never equal its own reloaded self. + """ + + assert storage_equal( + _object_series([numpy_leaf, None, None]), + _object_series([python_leaf, None, None]), + ) + + +def test_object_storage_uses_the_content_store_leaf_encoding() -> None: + """The object encoding is the ContentStore's, not a second definition.""" + + leaves = [_fresh_str("alpha"), 7, None] + series = _object_series(leaves) + + payload = bytearray() + for leaf in leaves: + body = _encode_object_scalar(leaf) + payload.extend(len(body).to_bytes(8, "little")) + payload.extend(body) + + assert _storage_parts(series, _ALL_ROWS)[0] == bytes(payload) + + +@pytest.mark.parametrize( + ("left_leaf", "right_leaf"), + [(None, pd.NA), (None, pd.NaT), (pd.NA, pd.NaT), (None, float("nan"))], +) +def test_object_storage_keeps_distinct_null_sentinels_apart( + left_leaf: object, right_leaf: object +) -> None: + """A null bitmap alone would collapse these; the value bytes must not.""" + + left = _object_series([left_leaf, _fresh_str("tail"), None]) + right = _object_series([right_leaf, _fresh_str("tail"), None]) + + assert left.isna().tolist() == right.isna().tolist() + assert _storage_parts(left, _ALL_ROWS)[1] == _storage_parts(right, _ALL_ROWS)[1] + assert _storage_parts(left, _ALL_ROWS) != _storage_parts(right, _ALL_ROWS) + + +def test_object_column_survives_a_content_store_round_trip(tmp_path: Path) -> None: + """The point of the fix: a reloaded column equals the one that was stored.""" + + schema = EntitySchema(group_entities=("household",)) + person = pd.DataFrame( + { + "person_id": np.asarray([1, 2, 3], dtype=np.int64), + "person_household_id": np.asarray([10, 10, 20], dtype=np.int64), + "tenure": pd.Series( + [_fresh_str("OWNED"), None, _fresh_str("RENTED")], dtype=object + ), + } + ) + household = pd.DataFrame({"household_id": np.asarray([10, 20], dtype=np.int64)}) + frame = Frame( + {"person": person, "household": household}, + schema, + {"household": Weights(np.asarray([1.0, 2.0]), WeightKind.DESIGN)}, + pd.Series(["a", "a", "b"], name="stratum", dtype=object), + ) + key = "b" * 64 + store = ContentStore(tmp_path / "store") + store.put_frame(key, frame, node_key="c" * 64) + reloaded = store.load_frame(key, node_key="c" * 64) + + original = frame.table("person")["tenure"] + restored = reloaded.table("person")["tenure"] + assert restored.dtype == object + assert [id(value) for value in restored.to_numpy()] != [ + id(value) for value in original.to_numpy() + ] + assert storage_equal(original, restored) + assert storage_equal(frame.strata, reloaded.strata) + + +def test_object_storage_flows_through_storage_equal_dtype_guard() -> None: + objects = _object_series([_fresh_str("alpha"), _fresh_str("beta"), None]) + strings = pd.Series( + ["alpha", "beta", None], + dtype=pd.StringDtype(storage="python", na_value=pd.NA), + ) + + # Different dtypes short-circuit before any encoding is compared, so the + # object encoding never has to agree byte-for-byte with the string branch. + assert not storage_equal(objects, strings) + + +@pytest.mark.parametrize( + ("dtype", "values", "expected_values", "expected_bitmap"), + [ + ( + "Int64", + [1, 2, 3, pd.NA], + "010000000000000002000000000000000100000000000000", + "000001", + ), + ("boolean", [True, False, pd.NA, True], "010001", "000000"), + ( + "float64", + [-0.0, 1.5, 2.0, float("nan")], + "0000000000000080000000000000f83f000000000000f87f", + "000001", + ), + ( + "int64", + [1, 2, 3, 4], + "010000000000000002000000000000000400000000000000", + "000000", + ), + ("bool", [True, False, True, False], "010000", "000000"), + ], +) +def test_masked_and_numeric_storage_encodings_stay_byte_identical( + dtype: str, values: list[object], expected_values: str, expected_bitmap: str +) -> None: + series = pd.Series(values, dtype=dtype) + selected = np.array([True, True, False, True]) + + payload, bitmap = _storage_parts(series, selected) + + assert payload.hex() == expected_values + assert bitmap.hex() == expected_bitmap + + +@pytest.mark.parametrize( + ("label", "values"), + [ + ("datetimetz", pd.to_datetime(["2020-01-01"] * 3, utc=True)), + ("period", pd.period_range("2020-01", "2020-03", freq="M")), + ("interval", pd.interval_range(0, 3)), + ], +) +def test_object_backed_extension_dtypes_now_fail_closed( + label: str, values: object +) -> None: + """These materialize as object arrays, so they were pointer-hashed too. + + ``ContentStore`` already refuses to persist them (store.py rejects + CategoricalDtype, DatetimeTZDtype and every other extension dtype), and + ``token_for_dtype`` refuses to declare them, so an explicit refusal is the + consistent outcome — silently comparing their addresses was not. + """ + + series = pd.Series(values) + + assert series.to_numpy(copy=False).dtype == object + with pytest.raises(PopulationError, match="storage-object-leaf"): + _storage_parts(series, _ALL_ROWS) + + +def test_categorical_storage_compares_category_values_not_addresses() -> None: + left = pd.Series(pd.Categorical([_fresh_str("a"), _fresh_str("b"), None])) + right = pd.Series(pd.Categorical([_fresh_str("a"), _fresh_str("b"), None])) + + assert left.dtype == right.dtype + assert left.to_numpy(copy=False).dtype == object + assert storage_equal(left, right) + assert not storage_equal( + left, pd.Series(pd.Categorical([_fresh_str("a"), _fresh_str("b"), "c"])) + ) + + +def test_masked_storage_still_compares_bytes_beneath_the_null_mask() -> None: + """Pins the docstring claim that storage_equal stays an in-process seal.""" + + direct = pd.Series([1, 2, pd.NA], dtype="Int64") + masked = pd.Series([1, 2, 7], dtype="Int64").mask(pd.Series([False, False, True])) + + assert direct.tolist() == masked.tolist() + assert direct.isna().tolist() == masked.isna().tolist() + assert not storage_equal(direct, masked) + + +def test_string_storage_encoding_stays_byte_identical() -> None: + series = pd.Series( + ["a", "bb", "c", None], + dtype=pd.StringDtype(storage="python", na_value=pd.NA), + ) + selected = np.array([True, True, False, True]) + + payload, bitmap = _storage_parts(series, selected) + + assert payload.hex() == ("020000000000000061030000000000000062620000000000000000") + assert bitmap.hex() == "000001" + + def test_new_dense_column_on_a_filtered_population_keeps_its_dtype() -> None: # A filtered population (a sampled spine rung) carries a non-contiguous # table index. The zero-filled placeholder for a new dense column is