Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion .pre-commit-config.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@ default_language_version:

repos:
- repo: https://github.com/astral-sh/ruff-pre-commit
rev: v0.15.20
rev: v0.16.0
hooks:
- id: ruff-check
args: ["--fix", "--show-fixes"]
Expand Down
1 change: 1 addition & 0 deletions changes/4213.misc.md
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
Updated ruff to 0.16.0 and fixed the violations surfaced by its expanded default rule set: narrowed a blind `except Exception` in `StorePath.__eq__` to `AttributeError`, removed unnecessary `global` declarations in `zarr.core.sync`, made `subprocess.run` calls in tests pass `check=False` explicitly, and applied automatic fixes (`None` moved to the end of type unions, unused `noqa` directives removed).
67 changes: 38 additions & 29 deletions design/chunk-grid.md
Original file line number Diff line number Diff line change
Expand Up @@ -162,9 +162,9 @@ class DimensionGrid(Protocol):
@property
def extent(self) -> int: ...
def index_to_chunk(self, idx: int) -> int: ...
def chunk_offset(self, chunk_ix: int) -> int: ... # raises IndexError if OOB
def chunk_size(self, chunk_ix: int) -> int: ... # raises IndexError if OOB
def data_size(self, chunk_ix: int) -> int: ... # raises IndexError if OOB
def chunk_offset(self, chunk_ix: int) -> int: ... # raises IndexError if OOB
def chunk_size(self, chunk_ix: int) -> int: ... # raises IndexError if OOB
def data_size(self, chunk_ix: int) -> int: ... # raises IndexError if OOB
def indices_to_chunks(self, indices: NDArray[np.intp]) -> NDArray[np.intp]: ...
@property
def unique_edge_lengths(self) -> Iterable[int]: ...
Expand All @@ -181,8 +181,8 @@ The protocol is `@runtime_checkable`, enabling polymorphic handling of both dime
```python
@dataclass(frozen=True)
class ChunkSpec:
slices: tuple[slice, ...] # valid data region in array coordinates
codec_shape: tuple[int, ...] # buffer shape for codec processing
slices: tuple[slice, ...] # valid data region in array coordinates
codec_shape: tuple[int, ...] # buffer shape for codec processing

@property
def shape(self) -> tuple[int, ...]:
Expand All @@ -199,34 +199,34 @@ For interior chunks, `shape == codec_shape`. For boundary chunks of a regular gr

```python
# Creating arrays
arr = zarr.create_array(shape=(100, 200), chunks=(10, 20)) # regular
arr = zarr.create_array(shape=(60, 100), chunks=[[10, 20, 30], [25, 25, 25, 25]]) # rectilinear
arr = zarr.create_array(shape=(100, 200), chunks=(10, 20)) # regular
arr = zarr.create_array(shape=(60, 100), chunks=[[10, 20, 30], [25, 25, 25, 25]]) # rectilinear

# ChunkGrid as a collection
grid = arr._chunk_grid # ChunkGrid (bound to array shape)
grid.grid_shape # (10, 10) — number of chunks per dimension
grid.ndim # 2
grid.is_regular # True if all dimensions are Fixed
grid = arr._chunk_grid # ChunkGrid (bound to array shape)
grid.grid_shape # (10, 10) — number of chunks per dimension
grid.ndim # 2
grid.is_regular # True if all dimensions are Fixed

spec = grid[0, 1] # ChunkSpec for chunk at grid position (0, 1)
spec.slices # (slice(0, 10), slice(20, 40))
spec.shape # (10, 20) — data shape
spec.codec_shape # (10, 20) — same for interior chunks
spec = grid[0, 1] # ChunkSpec for chunk at grid position (0, 1)
spec.slices # (slice(0, 10), slice(20, 40))
spec.shape # (10, 20) — data shape
spec.codec_shape # (10, 20) — same for interior chunks

boundary = grid[9, 0] # boundary chunk (extent=100, size=10)
boundary.shape # (10, 20) — data shape
boundary.codec_shape # (10, 20) — codec sees full buffer
boundary = grid[9, 0] # boundary chunk (extent=100, size=10)
boundary.shape # (10, 20) — data shape
boundary.codec_shape # (10, 20) — codec sees full buffer

grid[99, 99] # None — out of bounds
grid[99, 99] # None — out of bounds

for spec in grid: # iterate all chunks
for spec in grid: # iterate all chunks
...

# .chunks property: retained for regular grids, raises NotImplementedError for rectilinear
arr.chunks # (10, 20)
arr.chunks # (10, 20)

# .read_chunk_sizes / .write_chunk_sizes: works for all grids (dask-style)
arr.write_chunk_sizes # ((10, 10, ..., 10), (20, 20, ..., 20))
arr.write_chunk_sizes # ((10, 10, ..., 10), (20, 20, ..., 20))
```

`ChunkGrid.__getitem__` constructs `ChunkSpec` using `chunk_size` for `codec_shape` and `data_size` for `slices`:
Expand Down Expand Up @@ -274,7 +274,10 @@ When `extent < sum(edges)`, the dimension is always stored as `VaryingDimension`
{"name": "regular", "configuration": {"chunk_shape": [10, 20]}}

# Rectilinear grid (with RLE compression and "kind" field):
{"name": "rectilinear", "configuration": {"kind": "inline", "chunk_shapes": [[10, 20, 30], [[25, 4]]]}}
{
"name": "rectilinear",
"configuration": {"kind": "inline", "chunk_shapes": [[10, 20, 30], [[25, 4]]]},
}
```

Both names deserialize to the same `ChunkGrid` class. The serialized form does not include the array extent — that comes from `shape` in array metadata and is combined with the chunk grid when constructing a `ChunkGrid` via `ChunkGrid.from_metadata()`.
Expand Down Expand Up @@ -324,9 +327,9 @@ The underlying `ChunkGrid.chunk_sizes` property (on the grid, not the array) ret
#### Resize

```python
arr.resize((80, 100)) # re-binds extent; FixedDimension stays fixed
arr.resize((200, 100)) # VaryingDimension grows by appending a new chunk
arr.resize((30, 100)) # VaryingDimension shrinks: preserves all edges, re-binds extent
arr.resize((80, 100)) # re-binds extent; FixedDimension stays fixed
arr.resize((200, 100)) # VaryingDimension grows by appending a new chunk
arr.resize((30, 100)) # VaryingDimension shrinks: preserves all edges, re-binds extent
```

Resize uses `ChunkGrid.update_shape(new_shape)`, which delegates to each dimension's `.resize()` method:
Expand Down Expand Up @@ -363,8 +366,8 @@ When `chunks="keep"`, the logic checks `data._chunk_grid.is_regular`:
The indexing pipeline is coupled to regular grid assumptions — every per-dimension indexer takes a scalar `dim_chunk_len: int` and uses `//` and `*`:

```python
dim_chunk_ix = self.dim_sel // self.dim_chunk_len # IntDimIndexer
dim_offset = dim_chunk_ix * self.dim_chunk_len # SliceDimIndexer
dim_chunk_ix = self.dim_sel // self.dim_chunk_len # IntDimIndexer
dim_offset = dim_chunk_ix * self.dim_chunk_len # SliceDimIndexer
```

Replace `dim_chunk_len: int` with the dimension object (`FixedDimension | VaryingDimension`). The shared interface means the indexer code structure stays the same — `dim_sel // dim_chunk_len` becomes `dim_grid.index_to_chunk(dim_sel)`. O(1) for regular, binary search for varying.
Expand Down Expand Up @@ -590,14 +593,18 @@ If cubed needs to support both old and new zarr-python:
def _create_zarr_indexer(selection, shape, chunks):
if zarr.__version__[0] == "3":
from zarr.core.indexing import OrthogonalIndexer

try:
from zarr.core.chunk_grids import ChunkGrid

return OrthogonalIndexer(selection, shape, ChunkGrid.from_sizes(shape, chunks))
except ImportError:
from zarr.core.chunk_grids import RegularChunkGrid

return OrthogonalIndexer(selection, shape, RegularChunkGrid(chunk_shape=chunks))
else:
from zarr.indexing import OrthogonalIndexer

return OrthogonalIndexer(selection, ZarrArrayIndexingAdaptor(shape, chunks))
```

Expand Down Expand Up @@ -625,13 +632,15 @@ def _resolve_chunk_grid(chunk_grid, shape):
"""Coerce ChunkGridMetadata to runtime ChunkGrid if needed."""
from zarr.core.chunk_grids import ChunkGrid as _ChunkGrid
from zarr.core.metadata.v3 import ChunkGridMetadata

if isinstance(chunk_grid, _ChunkGrid):
return chunk_grid
if isinstance(chunk_grid, ChunkGridMetadata):
warnings.warn(
"Passing ChunkGridMetadata to indexers is deprecated. "
"Use ChunkGrid.from_sizes() instead.",
DeprecationWarning, stacklevel=2,
DeprecationWarning,
stacklevel=2,
)
if hasattr(chunk_grid, "chunk_shape"):
return _ChunkGrid.from_sizes(shape, tuple(chunk_grid.chunk_shape))
Expand Down
2 changes: 1 addition & 1 deletion pyproject.toml
Original file line number Diff line number Diff line change
Expand Up @@ -129,7 +129,7 @@ docs = [
"mkdocs-redirects==1.2.3",
"markdown-exec[ansi]==1.12.1",
"griffe-inherited-docstrings==1.1.3",
"ruff==0.15.20",
"ruff==0.16.0",
# Changelog generation
{include-group = "release"},
# Optional dependencies to run examples
Expand Down
2 changes: 1 addition & 1 deletion src/zarr/api/synchronous.py
Original file line number Diff line number Diff line change
Expand Up @@ -998,7 +998,7 @@ def from_array(
write_data: bool = True,
name: str | None = None,
chunks: ChunksLike | Literal["auto", "keep"] = "keep",
shards: ShardsLike | None | Literal["keep"] = "keep",
shards: ShardsLike | Literal["keep"] | None = "keep",
filters: FiltersLike | Literal["keep"] = "keep",
compressors: CompressorsLike | Literal["keep"] = "keep",
serializer: SerializerLike | Literal["keep"] = "keep",
Expand Down
2 changes: 1 addition & 1 deletion src/zarr/codecs/sharding.py
Original file line number Diff line number Diff line change
Expand Up @@ -841,7 +841,7 @@ def _encode_partial_sync(
# `_sentinel` distinguishes "not computed yet" from a memoized `None`
# (an empty chunk).
_sentinel = object()
scalar_complete_result: Buffer | None | object = _sentinel
scalar_complete_result: Buffer | object | None = _sentinel

for chunk_coords, chunk_sel, out_sel, is_complete_chunk in indexer:
if is_scalar and is_complete_chunk:
Expand Down
6 changes: 3 additions & 3 deletions src/zarr/core/array.py
Original file line number Diff line number Diff line change
Expand Up @@ -2175,7 +2175,7 @@ def filters(self) -> tuple[Numcodec, ...] | tuple[ArrayArrayCodec, ...]:
return self.async_array.filters

@property
def serializer(self) -> None | ArrayBytesCodec:
def serializer(self) -> ArrayBytesCodec | None:
"""
Array-to-bytes codec to use for serializing the chunks into bytes.
"""
Expand Down Expand Up @@ -4066,7 +4066,7 @@ async def from_array(
write_data: bool = True,
name: str | None = None,
chunks: ChunksLike | Literal["auto", "keep"] = "keep",
shards: ShardsLike | None | Literal["keep"] = "keep",
shards: ShardsLike | Literal["keep"] | None = "keep",
filters: FiltersLike | Literal["keep"] = "keep",
compressors: CompressorsLike | Literal["keep"] = "keep",
serializer: SerializerLike | Literal["keep"] = "keep",
Expand Down Expand Up @@ -4757,7 +4757,7 @@ async def create_array(
def _parse_keep_array_attr(
data: AnyArray | npt.ArrayLike,
chunks: ChunksLike | Literal["auto", "keep"],
shards: ShardsLike | None | Literal["keep"],
shards: ShardsLike | Literal["keep"] | None,
filters: FiltersLike | Literal["keep"],
compressors: CompressorsLike | Literal["keep"],
serializer: SerializerLike | Literal["keep"],
Expand Down
2 changes: 1 addition & 1 deletion src/zarr/core/dtype/common.py
Original file line number Diff line number Diff line change
Expand Up @@ -52,7 +52,7 @@
DTypeName_V2 = StructuredName_V2 | str


class DTypeConfig_V2[TDTypeNameV2: DTypeName_V2, TObjectCodecID: None | str](TypedDict):
class DTypeConfig_V2[TDTypeNameV2: DTypeName_V2, TObjectCodecID: str | None](TypedDict):
name: ReadOnly[TDTypeNameV2]
object_codec_id: ReadOnly[TObjectCodecID]

Expand Down
6 changes: 4 additions & 2 deletions src/zarr/core/sync.py
Original file line number Diff line number Diff line change
Expand Up @@ -90,7 +90,9 @@ def reset_resources_after_fork() -> None:
Ensure that global resources are reset after a fork. Without this function,
forked processes will retain invalid references to the parent process's resources.
"""
global loop, iothread, _executor
# `loop` and `iothread` are mutated in place rather than rebound, so only
# `_executor` needs the global declaration.
global _executor
# These lines are excluded from coverage because this function only runs in a child process,
# which is not observed by the test coverage instrumentation. Despite the apparent lack of
# test coverage, this function should be adequately tested by any test that uses Zarr IO with
Expand All @@ -112,7 +114,7 @@ async def _runner[T](coro: Coroutine[Any, Any, T]) -> T | BaseException:
"""
try:
return await coro
except Exception as ex:
except Exception as ex: # noqa: BLE001 -- the caller re-raises the returned exception
return ex


Expand Down
5 changes: 2 additions & 3 deletions src/zarr/storage/_common.py
Original file line number Diff line number Diff line change
Expand Up @@ -297,9 +297,8 @@ def __eq__(self, other: object) -> bool:
"""
try:
return self.store == other.store and self.path == other.path # type: ignore[attr-defined, no-any-return]
except Exception:
pass
return False
except AttributeError:
return False


type StoreLike = Store | StorePath | FSMap | Path | str | dict[str, Buffer]
Expand Down
2 changes: 1 addition & 1 deletion src/zarr/testing/store.py
Original file line number Diff line number Diff line change
Expand Up @@ -113,7 +113,7 @@ def test_store_type(self, store: S) -> None:

def test_store_eq(self, store: S, store_kwargs: dict[str, Any]) -> None:
# check self equality
assert store == store
assert store == store # noqa: PLR0124 -- self-equality is the property under test

# check store equality with same inputs
# asserting this is important for being able to compare (de)serialized stores
Expand Down
6 changes: 3 additions & 3 deletions src/zarr/testing/strategies.py
Original file line number Diff line number Diff line change
Expand Up @@ -132,7 +132,7 @@ def clear_store(x: Store) -> Store:


@st.composite
def dimension_names(draw: st.DrawFn, *, ndim: int | None = None) -> list[None | str] | None:
def dimension_names(draw: st.DrawFn, *, ndim: int | None = None) -> list[str | None] | None:
simple_text = st.text(zarr_key_chars, min_size=0)
return draw(st.none() | st.lists(st.none() | simple_text, min_size=ndim, max_size=ndim)) # type: ignore[arg-type]

Expand Down Expand Up @@ -292,7 +292,7 @@ def arrays(
if arrays is None:
arrays = numpy_arrays(shapes=shapes)
nparray = draw(arrays, label="array data")
dim_names: None | list[str | None] = None
dim_names: list[str | None] | None = None
serializer: SerializerLike = "auto"
compressors_unsearched: CompressorsLike = "auto"

Expand Down Expand Up @@ -328,7 +328,7 @@ def arrays(
else:
chunks_param = draw(chunk_shapes(shape=nparray.shape), label="chunk shape")

if all(s > c and c > 1 for s, c in zip(nparray.shape, chunks_param, strict=True)):
if all(s > c > 1 for s, c in zip(nparray.shape, chunks_param, strict=True)):
shard_shape = draw(
st.none() | shard_shapes(shape=nparray.shape, chunk_shape=chunks_param),
label="shard shape",
Expand Down
2 changes: 1 addition & 1 deletion tests/test_api.py
Original file line number Diff line number Diff line change
Expand Up @@ -419,7 +419,7 @@ async def test_open_group_unspecified_version(tmp_path: Path, zarr_format: ZarrF
@pytest.mark.parametrize("n_args", [10, 1, 0])
@pytest.mark.parametrize("n_kwargs", [10, 1, 0])
@pytest.mark.parametrize("path", [None, "some_path"])
def test_save(store: Store, n_args: int, n_kwargs: int, path: None | str) -> None:
def test_save(store: Store, n_args: int, n_kwargs: int, path: str | None) -> None:
data = np.arange(10)
args = [np.arange(10) for _ in range(n_args)]
kwargs = {f"arg_{i}": data for i in range(n_kwargs)}
Expand Down
4 changes: 2 additions & 2 deletions tests/test_codecs/test_blosc.py
Original file line number Diff line number Diff line change
Expand Up @@ -74,7 +74,7 @@ async def test_blosc_evolve(dtype: str) -> None:
@pytest.mark.parametrize("shuffle", [None, "bitshuffle", "legacy-enum"])
@pytest.mark.parametrize("typesize", [None, 1, 2])
def test_tunable_attrs_param(
shuffle: None | BloscShuffleLiteral | str, typesize: None | int
shuffle: BloscShuffleLiteral | str | None, typesize: int | None
) -> None:
"""
Test that the tunable_attrs parameter is set as expected when creating a BloscCodec.
Expand All @@ -83,7 +83,7 @@ def test_tunable_attrs_param(
# contaminating the BloscCodec construction below with that warning.
if shuffle == "legacy-enum":
with pytest.warns(DeprecationWarning, match="BloscShuffle.shuffle"):
shuffle_arg: None | BloscShuffleLiteral | str = BloscShuffle.shuffle
shuffle_arg: BloscShuffleLiteral | str | None = BloscShuffle.shuffle
else:
shuffle_arg = shuffle

Expand Down
2 changes: 1 addition & 1 deletion tests/test_dtype_registry.py
Original file line number Diff line number Diff line change
Expand Up @@ -170,7 +170,7 @@ def test_entrypoint_dtype(zarr_format: ZarrFormat) -> None:
)
def test_parse_data_type(
data_type: ZDType[Any, Any],
json_style: tuple[ZarrFormat, None | Literal["internal", "metadata"]],
json_style: tuple[ZarrFormat, Literal["internal", "metadata"] | None],
dtype_parser_func: Any,
) -> None:
"""
Expand Down
2 changes: 1 addition & 1 deletion tests/test_examples.py
Original file line number Diff line number Diff line change
Expand Up @@ -80,7 +80,7 @@ def test_scripts_can_run(script_path: Path, tmp_path: Path) -> None:
# This allows the example to be useful to users who don't have Zarr installed, but also testable.
resave_script(script_path, dest_path)
result = subprocess.run(
["uv", "run", "--refresh", str(dest_path)], capture_output=True, text=True
["uv", "run", "--refresh", str(dest_path)], capture_output=True, text=True, check=False
)
assert result.returncode == 0, (
f"Script at {script_path} failed to run. Output: {result.stdout} Error: {result.stderr}"
Expand Down
2 changes: 2 additions & 0 deletions tests/test_regression/test_v2_dtype_regression.py
Original file line number Diff line number Diff line change
Expand Up @@ -215,6 +215,7 @@ def test_roundtrip_v2(source_array_v2: ArrayV2, tmp_path: Path, script_path: Pat
],
capture_output=True,
text=True,
check=False,
)
assert copy_op.returncode == 0, f"stdout {copy_op.stdout}\n stderr{copy_op.stderr}"
out_array = zarr.open_array(store=out_path, mode="r", zarr_format=2)
Expand All @@ -240,6 +241,7 @@ def test_roundtrip_v3(source_array_v3: ArrayV3, tmp_path: Path) -> None:
],
capture_output=True,
text=True,
check=False,
)
assert copy_op.returncode == 0
out_array = zarr.open_array(store=out_path, mode="r", zarr_format=3)
Expand Down
2 changes: 1 addition & 1 deletion tests/test_store/test_core.py
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,7 @@
)
def store_like(
request: pytest.FixtureRequest,
) -> Generator[None | str | Path | StorePath | MemoryStore | dict[Any, Any], None, None]:
) -> Generator[str | Path | StorePath | MemoryStore | dict[Any, Any] | None, None, None]:
if request.param == "none":
yield None
elif request.param == "temp_dir_str":
Expand Down
1 change: 0 additions & 1 deletion tests/test_store/test_object.py
Original file line number Diff line number Diff line change
@@ -1,4 +1,3 @@
# ruff: noqa: E402
import re
from pathlib import Path
from typing import TypedDict
Expand Down
4 changes: 2 additions & 2 deletions tests/test_unified_chunk_grid.py
Original file line number Diff line number Diff line change
Expand Up @@ -2784,8 +2784,8 @@ def test_rectilinear_roundtrip(json_input: RectilinearChunkGridMetadataJSON) ->

pytest.importorskip("hypothesis")

import hypothesis.strategies as st # noqa: E402
from hypothesis import event, given, settings # noqa: E402
import hypothesis.strategies as st
from hypothesis import event, given, settings


@st.composite
Expand Down
2 changes: 1 addition & 1 deletion tests/test_v2.py
Original file line number Diff line number Diff line change
Expand Up @@ -294,7 +294,7 @@ def test_parse_structured_fill_value_valid(


@pytest.mark.parametrize("fill_value", [None, b"x"], ids=["no_fill", "fill"])
def test_other_dtype_roundtrip(fill_value: None | bytes, tmp_path: Path) -> None:
def test_other_dtype_roundtrip(fill_value: bytes | None, tmp_path: Path) -> None:
a = np.array([b"a\0\0", b"bb", b"ccc"], dtype="V7")
array_path = tmp_path / "data.zarr"
za = zarr.create(
Expand Down
Loading
Loading