From 9bfe855a6cfd6985073fd5e4e5b5be027628eca8 Mon Sep 17 00:00:00 2001 From: Davis Bennett Date: Tue, 14 Jul 2026 13:25:16 +0200 Subject: [PATCH 1/3] fix: byte-order handling for structured dtypes in the bytes codec (#220) * fix: byte-order handling for structured dtypes in the bytes codec The bytes codec neither byte-swapped structured-dtype fields to its configured endian on encode (numpy reports byteorder '|' for void dtypes, so the top-level byteorder comparison never detected a mismatch) nor honored its endian when decoding, silently corrupting any structured data whose field byte order differed from the stored one (e.g. virtual references to external big-endian data). Encode now detects byte-order mismatches by comparing full dtypes via newbyteorder, and decode reinterprets raw bytes in the stored byte order before converting to the data type's declared byte order, so the stored layout (codec state) and the in-memory layout (array data type) are independent. Closes #4141 Assisted-by: ClaudeCode:claude-fable-5 * test: fold structured byte-order cases into existing bytes codec tests Extend test_endian's parametrization with structured dtypes and test_bytes_codec_sync_roundtrip with endian/dtype parametrization plus stored-layout and decoded-dtype assertions, instead of adding parallel test functions for the same properties. Assisted-by: ClaudeCode:claude-fable-5 * refactor: rename stored_dtype to view_dtype in BytesCodec decode The variable is the dtype used to view the raw chunk bytes (byte order from the codec's endian configuration), not a property of the stored data or of the returned buffer, which always carries the array's declared dtype. Assisted-by: ClaudeCode:claude-fable-5 * docs: note that the decode-side byte-order conversion copies the chunk Assisted-by: ClaudeCode:claude-fable-5 --- changes/4141.bugfix.md | 1 + src/zarr/codecs/bytes.py | 33 +++++++++++----- tests/test_codecs/test_bytes.py | 69 ++++++++++++++++++++++++++------- 3 files changed, 80 insertions(+), 23 deletions(-) create mode 100644 changes/4141.bugfix.md diff --git a/changes/4141.bugfix.md b/changes/4141.bugfix.md new file mode 100644 index 0000000000..6a132da3f5 --- /dev/null +++ b/changes/4141.bugfix.md @@ -0,0 +1 @@ +Fix silent byte-order corruption for structured dtypes with the `bytes` codec: multi-byte fields are now byte-swapped to the codec's configured `endian` on write and decoded honoring it on read, so non-native-endian structured data (e.g. big-endian fields, as produced by virtual references to external data) round-trips correctly. diff --git a/src/zarr/codecs/bytes.py b/src/zarr/codecs/bytes.py index 240c077627..fae762fd08 100644 --- a/src/zarr/codecs/bytes.py +++ b/src/zarr/codecs/bytes.py @@ -100,14 +100,28 @@ def _decode_sync( chunk_spec: ArraySpec, ) -> NDBuffer: endian_str = self.endian + dtype = chunk_spec.dtype.to_native_dtype() + # The byte order of the stored data is set by this codec's `endian` + # configuration; the byte order of the decoded array is set by the array's + # data type. The two are independent: the raw bytes are viewed with a dtype + # in the stored byte order, then converted to the declared dtype if needed. if isinstance(chunk_spec.dtype, HasEndianness): - dtype = replace(chunk_spec.dtype, endianness=endian_str).to_native_dtype() # type: ignore[call-arg] + view_dtype = replace(chunk_spec.dtype, endianness=endian_str).to_native_dtype() # type: ignore[call-arg] + elif isinstance(chunk_spec.dtype, Struct) and endian_str is not None: + # Per the struct data type spec, all multi-byte fields are stored in the + # byte order configured on this codec. + view_dtype = dtype.newbyteorder(endian_str) else: - dtype = chunk_spec.dtype.to_native_dtype() + view_dtype = dtype as_array_like = chunk_bytes.as_array_like() chunk_array = chunk_spec.prototype.nd_buffer.from_ndarray_like( - as_array_like.view(dtype=dtype) # type: ignore[attr-defined] + as_array_like.view(dtype=view_dtype) # type: ignore[attr-defined] ) + if view_dtype != dtype: + # This byte-swapping conversion copies the chunk. The dtype inequality + # guard keeps the common case, where the stored and declared byte orders + # already match, on the zero-copy view path above. + chunk_array = chunk_array.astype(dtype) # ensure correct chunk shape if chunk_array.shape != chunk_spec.shape: @@ -129,13 +143,14 @@ def _encode_sync( chunk_spec: ArraySpec, ) -> Buffer | None: assert isinstance(chunk_array, NDBuffer) - if ( - chunk_array.dtype.itemsize > 1 - and self.endian is not None - and self.endian != chunk_array.byteorder - ): + if chunk_array.dtype.itemsize > 1 and self.endian is not None: + # Compare full dtypes rather than the top-level byteorder: numpy reports + # byteorder '|' for structured dtypes even when their fields are + # byte-order-sensitive, so newbyteorder is the only reliable way to + # detect (and normalize) a byte-order mismatch. new_dtype = chunk_array.dtype.newbyteorder(self.endian) - chunk_array = chunk_array.astype(new_dtype) + if new_dtype != chunk_array.dtype: + chunk_array = chunk_array.astype(new_dtype) nd_array = chunk_array.as_ndarray_like() # Flatten the nd-array (only copy if needed) and reinterpret as bytes diff --git a/tests/test_codecs/test_bytes.py b/tests/test_codecs/test_bytes.py index 03dd0b40c6..ead778f526 100644 --- a/tests/test_codecs/test_bytes.py +++ b/tests/test_codecs/test_bytes.py @@ -33,29 +33,50 @@ @pytest.mark.parametrize("store", ["local", "memory"], indirect=["store"]) -@pytest.mark.parametrize("input_dtype", [">u2", "u2", + "f4"), ("mask", ">i4")], + [("flux", "u2", " None: """ The `bytes` codec stores multi-byte data in the byte order configured on the codec, regardless of the input array's byte order, and reads it back to the - original values. The input-dtype/store-endian cross-product exercises the - encode-side byteswap (input byte order != store byte order) and the no-op - case alike. Compression is disabled so the stored chunk is the codec's raw - output and its byte layout can be asserted directly. + original values. For structured dtypes this applies to every multi-byte + field, per the `struct` data type spec; the struct cases guard against the + endianness bugs from + https://github.com/zarr-developers/zarr-python/issues/4141, where the + encode path never byte-swapped struct fields (numpy reports byteorder '|' + for void dtypes) and the decode path ignored the codec's endian entirely. + The input-dtype/store-endian cross-product exercises the encode-side + byteswap (input byte order != store byte order) and the no-op case alike. + Compression is disabled so the stored chunk is the codec's raw output and + its byte layout can be asserted directly. """ - data = np.arange(0, 256, dtype=input_dtype).reshape((16, 16)) + dtype = np.dtype(input_dtype) + if dtype.fields is None: + data = np.arange(0, 256, dtype=dtype).reshape((16, 16)) + else: + data = np.zeros((16, 16), dtype=dtype) + data["flux"] = np.arange(0, 256).reshape((16, 16)) + data["mask"] = np.arange(256, 512).reshape((16, 16)) path = "endian" spath = StorePath(store, path) a = await zarr.api.asynchronous.create_array( spath, shape=data.shape, chunks=(16, 16), - dtype="uint16", + dtype=dtype, fill_value=0, compressors=None, serializer=BytesCodec(endian=store_endian), @@ -66,8 +87,7 @@ async def test_endian( # The stored chunk is laid out in the byte order configured on the codec. stored = await store.get(f"{path}/c/0/0", prototype=default_buffer_prototype()) assert stored is not None - expected_dtype = ">u2" if store_endian == "big" else " None: assert isinstance(BytesCodec(), SupportsSyncCodec) -def test_bytes_codec_sync_roundtrip() -> None: - codec = BytesCodec() - arr = np.arange(100, dtype="float64") +@pytest.mark.parametrize("endian", ENDIAN) +@pytest.mark.parametrize( + "native_dtype", + [np.dtype("float64"), np.dtype(">u2"), np.dtype([("a", ">f4"), ("b", " None: + """ + The synchronous encode/decode path round-trips data, and the two byte + orders involved are independent: the codec's `endian` configuration governs + only the stored byte layout (every multi-byte value, including struct + fields, is laid out in the codec's byte order regardless of the input + array's byte order), while the decoded buffer's byte order is governed by + the array's data type regardless of the codec's. The mixed-endian struct + case pins that per-field byte order of the in-memory dtype survives a + roundtrip through a single stored byte order. + """ + if native_dtype.fields is None: + arr = np.arange(100, dtype=native_dtype) + else: + arr = np.array([(1.5, 2), (3.5, 4), (5.5, 6), (7.5, 8)], dtype=native_dtype) zdtype = get_data_type_from_native_dtype(arr.dtype) spec = ArraySpec( shape=arr.shape, @@ -91,11 +129,14 @@ def test_bytes_codec_sync_roundtrip() -> None: ) nd_buf: NDBuffer = default_buffer_prototype().nd_buffer.from_numpy_array(arr) - codec = codec.evolve_from_array_spec(spec) + codec = BytesCodec(endian=endian).evolve_from_array_spec(spec) encoded = codec._encode_sync(nd_buf, spec) assert encoded is not None + assert encoded.to_bytes() == arr.astype(native_dtype.newbyteorder(endian)).tobytes() + decoded = codec._decode_sync(encoded, spec) + assert decoded.dtype == zdtype.to_native_dtype() np.testing.assert_array_equal(arr, decoded.as_numpy_array()) From 28645a435105f4d0f9b0b6bef4382835be61fd41 Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Thu, 16 Jul 2026 23:00:39 +0200 Subject: [PATCH 2/3] fix(codecs): don't skip outer AA/BB codecs in FusedCodecPipeline partial IO Partial IO delegates the whole chunk to the array->bytes codec, so it is only sound when that codec is the entire pipeline. BatchedCodecPipeline has always required empty AA/BB codec lists for this reason; FusedCodecPipeline passed require_no_aa_bb=False and so took the partial path even with an outer codec wrapping a sharding codec, silently dropping that codec. With filters=[TransposeCodec(order=(1,0))] and a ShardingCodec serializer, the fused pipeline drops the transpose on both encode and decode. It is self-consistent, so a fused-only roundtrip looks correct while the stored bytes do not match the metadata: batched, and any other Zarr implementation, reads that array back transposed. Since require_no_aa_bb now has a single value at every call site, the parameter is gone rather than left as an attractive nuisance. This costs nothing for layouts create_array produces: filters, order= and compressors all nest inside the shard, leaving the outer codec lists empty, so partial IO stays enabled. Only an outer AA/BB codec around a partial-IO-capable codec falls back to the general path, and that layout was returning wrong data. Assisted-by: ClaudeCode:claude-opus-4.8 --- src/zarr/core/codec_pipeline.py | 34 ++++++++----------------- tests/test_pipeline_parity.py | 45 +++++++++++++++++++++++++++++++++ 2 files changed, 56 insertions(+), 23 deletions(-) diff --git a/src/zarr/core/codec_pipeline.py b/src/zarr/core/codec_pipeline.py index 2e3f1ed122..bea0eef7d8 100644 --- a/src/zarr/core/codec_pipeline.py +++ b/src/zarr/core/codec_pipeline.py @@ -132,23 +132,18 @@ def pipeline_supports_partial_decode( *, array_array_codecs: tuple[ArrayArrayCodec, ...], bytes_bytes_codecs: tuple[BytesBytesCodec, ...], - require_no_aa_bb: bool, ) -> bool: """Whether a codec pipeline can decode a partial selection without a full read. Requires the array->bytes codec to implement - ``ArrayBytesCodecPartialDecodeMixin``. When ``require_no_aa_bb`` is True it - additionally requires no array->array / bytes->bytes codecs, because those - can change the slice<->byte-range correspondence (an AA codec can make the - selection non-contiguous, a BB codec can rewrite the bytes), making partial - decode infeasible. - - NOTE: the two pipelines currently pass different ``require_no_aa_bb`` values - (Batched: True; Fused: False). That divergence is intentional-for-now and - tracked separately; this function centralizes the predicate without changing - either pipeline's behavior. + `ArrayBytesCodecPartialDecodeMixin`, and requires no array->array / + bytes->bytes codecs. Partial IO delegates the whole chunk to the + array->bytes codec, so it is only sound when that codec is the entire + pipeline: an outer AA codec can make the selection non-contiguous and a BB + codec can rewrite the bytes, so neither the slice<->byte-range + correspondence nor the outer codec itself survives the delegation. """ - if require_no_aa_bb and (len(array_array_codecs) + len(bytes_bytes_codecs)) != 0: + if (len(array_array_codecs) + len(bytes_bytes_codecs)) != 0: return False return isinstance(array_bytes_codec, ArrayBytesCodecPartialDecodeMixin) @@ -158,14 +153,12 @@ def pipeline_supports_partial_encode( *, array_array_codecs: tuple[ArrayArrayCodec, ...], bytes_bytes_codecs: tuple[BytesBytesCodec, ...], - require_no_aa_bb: bool, ) -> bool: """Whether a codec pipeline can encode a partial selection without a full rewrite. - Mirror of ``pipeline_supports_partial_decode`` for encoding. See its note re: - the per-pipeline ``require_no_aa_bb`` divergence. + Mirror of `pipeline_supports_partial_decode` for encoding. """ - if require_no_aa_bb and (len(array_array_codecs) + len(bytes_bytes_codecs)) != 0: + if (len(array_array_codecs) + len(bytes_bytes_codecs)) != 0: return False return isinstance(array_bytes_codec, ArrayBytesCodecPartialEncodeMixin) @@ -554,7 +547,6 @@ def supports_partial_decode(self) -> bool: self.array_bytes_codec, array_array_codecs=self.array_array_codecs, bytes_bytes_codecs=self.bytes_bytes_codecs, - require_no_aa_bb=True, ) @property @@ -563,7 +555,6 @@ def supports_partial_encode(self) -> bool: self.array_bytes_codec, array_array_codecs=self.array_array_codecs, bytes_bytes_codecs=self.bytes_bytes_codecs, - require_no_aa_bb=True, ) def __iter__(self) -> Iterator[Codec]: @@ -934,14 +925,12 @@ def __iter__(self) -> Iterator[Codec]: @property def supports_partial_decode(self) -> bool: - # NOTE: unlike BatchedCodecPipeline this does NOT require the AA/BB codec - # lists to be empty (require_no_aa_bb=False). That divergence is tracked - # separately; see pipeline_supports_partial_decode. + # Only a single ArrayBytesCodec that supports partial decoding, and no + # AA/BB codecs (they break the slice<->byte-range correspondence). return pipeline_supports_partial_decode( self.array_bytes_codec, array_array_codecs=self.array_array_codecs, bytes_bytes_codecs=self.bytes_bytes_codecs, - require_no_aa_bb=False, ) @property @@ -950,7 +939,6 @@ def supports_partial_encode(self) -> bool: self.array_bytes_codec, array_array_codecs=self.array_array_codecs, bytes_bytes_codecs=self.bytes_bytes_codecs, - require_no_aa_bb=False, ) def validate( diff --git a/tests/test_pipeline_parity.py b/tests/test_pipeline_parity.py index 717f0f48f1..0534505cd2 100644 --- a/tests/test_pipeline_parity.py +++ b/tests/test_pipeline_parity.py @@ -48,6 +48,7 @@ ShardingCodec, SubchunkWriteOrder, ) +from zarr.codecs.transpose import TransposeCodec from zarr.core.config import config as zarr_config from zarr.storage import MemoryStore @@ -332,6 +333,50 @@ def test_pipeline_parity( ) +# --------------------------------------------------------------------------- +# Outer array->array codec wrapping a sharding codec +# --------------------------------------------------------------------------- + + +@pytest.mark.filterwarnings("ignore:Combining a `sharding_indexed` codec:UserWarning") +@pytest.mark.parametrize("write_pipeline", [_BATCHED, _FUSED]) +@pytest.mark.parametrize("read_pipeline", [_BATCHED, _FUSED]) +def test_outer_array_array_codec_around_sharding(write_pipeline: str, read_pipeline: str) -> None: + """An array->array codec *outside* a sharding codec must not be skipped. + + Partial IO hands the chunk straight to the array->bytes codec, which is only + sound when that codec is the whole pipeline. With an outer transpose the + partial paths must be declined, or the transpose is silently dropped. + + `filters` are applied outside the serializer, so `create_array` reaches this + layout — no low-level `codecs=` needed. Every write/read pipeline pairing is + checked because each pipeline is self-consistent on its own: it drops the + transpose symmetrically on write and read, so the corruption is only visible + when the data crosses pipelines (or implementations). + """ + shape, shard_shape, inner_chunk = (20, 20), (10, 10), (5, 5) + ref = np.arange(int(np.prod(shape)), dtype="int32").reshape(shape) + + store = MemoryStore() + with zarr_config.set({"codec_pipeline.path": write_pipeline}): + arr = zarr.create_array( + store=store, + shape=shape, + chunks=shard_shape, + dtype="int32", + filters=[TransposeCodec(order=(1, 0))], + serializer=ShardingCodec(chunk_shape=inner_chunk), + compressors=None, + ) + arr[:] = ref + + np.testing.assert_array_equal( + _read_under_pipeline(read_pipeline, store), + ref, + err_msg=f"outer TransposeCodec dropped (write={write_pipeline}, read={read_pipeline})", + ) + + # --------------------------------------------------------------------------- # Partial-read parity across subchunk write orders # --------------------------------------------------------------------------- From ea02850ef3709a175008f6e90d390b7e655b74c3 Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Thu, 16 Jul 2026 23:01:15 +0200 Subject: [PATCH 3/3] docs: add changelog entry for fused pipeline outer codec fix Assisted-by: ClaudeCode:claude-opus-4.8 --- changes/225.bugfix.md | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) create mode 100644 changes/225.bugfix.md diff --git a/changes/225.bugfix.md b/changes/225.bugfix.md new file mode 100644 index 0000000000..509978ef75 --- /dev/null +++ b/changes/225.bugfix.md @@ -0,0 +1,16 @@ +`FusedCodecPipeline` no longer skips array->array and bytes->bytes codecs that +wrap a codec capable of partial IO, such as a `TransposeCodec` around a +`ShardingCodec`. Partial IO delegates the whole chunk to the array->bytes codec, +which is only correct when that codec is the entire pipeline, so the pipeline +now declines partial IO when outer codecs are present — matching +`BatchedCodecPipeline`. + +Previously the outer codec was dropped on both encode and decode. Because the +pipeline was self-consistent, a roundtrip under `FusedCodecPipeline` alone +looked correct while the stored bytes did not match the array metadata, so +`BatchedCodecPipeline` and other Zarr implementations read such arrays back +incorrectly. + +Arrays created by `create_array` are unaffected: `filters`, `order` and +`compressors` are nested inside the shard, so partial IO remains enabled for +them.