From 88f746d4145645bde78078c84197187a2894fa14 Mon Sep 17 00:00:00 2001 From: Agisilaos Kounelis Date: Sun, 28 Jun 2026 14:47:10 +0300 Subject: [PATCH 1/4] Support NumPy 2.5.0 Replace the now-deprecated `ndarray.dtype`/`shape` attribute assignments with `view()`/`reshape()`, compute datetime dimension bounds in int64 to avoid the new overflow errors, and add 2.5.0 to the nightly matrix. --- .github/workflows/daily-test-build-numpy.yml | 6 ++--- tiledb/array.py | 2 +- tiledb/core.cc | 4 ++-- tiledb/dataframe_.py | 17 +++++++++---- tiledb/dense_array.py | 14 +++++------ tiledb/multirange_indexing.py | 4 ++-- tiledb/npbuffer.cc | 25 ++++++++------------ tiledb/sparse_array.py | 7 +++--- tiledb/tests/test_core.py | 2 +- tiledb/tests/test_libtiledb.py | 18 +++++++++++--- tiledb/tests/test_subarray.py | 4 ++-- 11 files changed, 58 insertions(+), 45 deletions(-) diff --git a/.github/workflows/daily-test-build-numpy.yml b/.github/workflows/daily-test-build-numpy.yml index 0cc133df33..854b6dfa4f 100644 --- a/.github/workflows/daily-test-build-numpy.yml +++ b/.github/workflows/daily-test-build-numpy.yml @@ -30,10 +30,10 @@ jobs: - { python-version: "3.11", numpy-version: "1.25.2" } - { python-version: "3.11", numpy-version: "2.3.4" } - { python-version: "3.12", numpy-version: "1.26.4" } - - { python-version: "3.12", numpy-version: "2.3.4" } + - { python-version: "3.12", numpy-version: "2.5.0" } - { python-version: "3.13", numpy-version: "2.1.3" } - - { python-version: "3.13", numpy-version: "2.3.4" } - - { python-version: "3.14", numpy-version: "2.3.4" } + - { python-version: "3.13", numpy-version: "2.5.0" } + - { python-version: "3.14", numpy-version: "2.5.0" } fail-fast: false env: TILEDB_VERSION: ${{ inputs.libtiledb_version }} diff --git a/tiledb/array.py b/tiledb/array.py index cd9b3c72b4..734fd9b605 100644 --- a/tiledb/array.py +++ b/tiledb/array.py @@ -1188,7 +1188,7 @@ def set_query(self, serialized_query): out = OrderedDict() for name in results.keys(): arr = results[name][0] - arr.dtype = q.buffer_dtype(name) + arr = arr.view(q.buffer_dtype(name)) out[name] = arr return out diff --git a/tiledb/core.cc b/tiledb/core.cc index 05887b3004..f9d374d689 100644 --- a/tiledb/core.cc +++ b/tiledb/core.cc @@ -1447,8 +1447,8 @@ class PyQuery { auto arr = py::array(py::dtype("int8"), size, data_ptr); o = py::memoryview(arr); } else { - o = py::array(py::dtype("uint8"), size, data_ptr); - o.attr("dtype") = dtype; + // `size` is the cell's byte length, a multiple of the itemsize. + o = py::array(dtype, size / dtype.itemsize(), data_ptr); } result_p[i - 1] = o; diff --git a/tiledb/dataframe_.py b/tiledb/dataframe_.py index 50755eccad..d7983853b9 100644 --- a/tiledb/dataframe_.py +++ b/tiledb/dataframe_.py @@ -372,9 +372,13 @@ def create_dim(dtype, values, full_domain, tile, **kwargs): if np.issubdtype(dtype, np.datetime64): date_unit = np.datetime_data(dtype)[0] dim_min = np.datetime64(dtype_min, date_unit) - tile_max = np.iinfo(np.uint64).max - tile - if np.uint64(dtype_max - dtype_min) > tile_max: - dim_max = np.datetime64(dtype_max - tile, date_unit) + # Subtract in Python ints; a full-domain datetime64 difference + # overflows int64. + dtype_min_i = int(dtype_min.astype(np.int64)) + dtype_max_i = int(dtype_max.astype(np.int64)) + tile_max = int(np.iinfo(np.uint64).max) - int(tile) + if (dtype_max_i - dtype_min_i) > tile_max: + dim_max = np.datetime64(dtype_max_i - int(tile), date_unit) elif np.issubdtype(dtype, np.integer): tile_max = np.iinfo(np.uint64).max - tile if np.uint64(dtype_max - dtype_min) > tile_max: @@ -383,9 +387,14 @@ def create_dim(dtype, values, full_domain, tile, **kwargs): dim_min, dim_max = None, None # TODO: simplify this logic and/or move to DataType.cast_tile_extent - if np.issubdtype(dtype, np.integer) or np.issubdtype(dtype, np.datetime64): + if np.issubdtype(dtype, np.integer): # we can't make a tile larger than the dimension range or lower than 1 tile = max(1, min(tile, 1 + np.uint64(dim_max - dim_min))) + elif np.issubdtype(dtype, np.datetime64): + # Clamp as for integers, computing the range in Python ints since a + # full-domain datetime64 difference overflows int64. + dim_range = int(dim_max.astype(np.int64)) - int(dim_min.astype(np.int64)) + tile = max(1, min(tile, 1 + dim_range)) elif np.issubdtype(dtype, np.floating): # this difference can be inf with np.errstate(over="ignore"): diff --git a/tiledb/dense_array.py b/tiledb/dense_array.py index 71e086aaa4..7f7015715c 100644 --- a/tiledb/dense_array.py +++ b/tiledb/dense_array.py @@ -345,7 +345,7 @@ def _read_dense_subarray( # Note: fixed blobs are always 1 byte per cell arr = arr.view("S1") else: - arr.dtype = dtype + arr = arr.view(dtype) if len(arr) == 0: # special case: the C API returns 0 len for blank arrays arr = np.zeros(output_shape, dtype=dtype) @@ -357,13 +357,11 @@ def _read_dense_subarray( ) if layout == lt.LayoutType.ROW_MAJOR: - arr.shape = output_shape - arr = np.require(arr, requirements="C") + arr = np.require(arr.reshape(output_shape), requirements="C") elif layout == lt.LayoutType.COL_MAJOR: - arr.shape = output_shape - arr = np.require(arr, requirements="F") + arr = np.require(arr.reshape(output_shape), requirements="F") else: - arr.shape = np.prod(output_shape) + arr = arr.reshape(np.prod(output_shape)) out[name] = arr @@ -838,12 +836,12 @@ def read_subarray(self, subarray): arr = pyquery.unpack_buffer(name, item[0], item[1]) else: arr = item[0] - arr.dtype = ( + arr = arr.view( self.schema.attr_or_dim_dtype(name) if not self.schema.has_dim_label(name) else self.schema.dim_label(name).dtype ) - arr.shape = result_shape + arr = arr.reshape(result_shape) result_dict[name if name != "__attr" else ""] = arr return result_dict diff --git a/tiledb/multirange_indexing.py b/tiledb/multirange_indexing.py index 537caf22dd..09cb7aeaec 100644 --- a/tiledb/multirange_indexing.py +++ b/tiledb/multirange_indexing.py @@ -409,7 +409,7 @@ def _run_query(self) -> Dict[str, np.ndarray]: for name, arr in result_dict.items(): # TODO check/test layout if not self.array.schema.has_dim_label(name): - arr.shape = self.result_shape + result_dict[name] = arr.reshape(self.result_shape) return result_dict @@ -826,7 +826,7 @@ def _get_pyquery_results(pyquery: PyQuery, array: Array) -> Dict[str, np.ndarray arr = pyquery.unpack_buffer(name, item[0], item[1]) else: arr = item[0] - arr.dtype = ( + arr = arr.view( schema.attr_or_dim_dtype(name) if not schema.has_dim_label(name) else schema.dim_label(name).dtype diff --git a/tiledb/npbuffer.cc b/tiledb/npbuffer.cc index 0ee7c39bab..65cd21eca2 100644 --- a/tiledb/npbuffer.cc +++ b/tiledb/npbuffer.cc @@ -471,22 +471,17 @@ class NumpyConvert { NumpyConvert(py::array input) { // require a flat buffer if (input.ndim() != 1) { - // try to take a 1D view on the input - auto v = input.attr("view")(); - // this will throw if the shape cannot be modified zero-copy, - // which is what we want - try { - v.attr("shape") = py::int_(input.size()); - } catch (py::error_already_set& e) { - if (e.matches(PyExc_AttributeError)) { - use_iter_ = true; - } else { - throw; - } - } catch (std::exception& e) { - std::cout << e.what() << std::endl; + // reshape(-1) yields a zero-copy view when the array can be + // flattened in place and a fresh copy otherwise; the bulk paths + // need the view, so compare buffer addresses and fall back to the + // element-wise iterator (which handles strided input) on a copy. + py::array flat = input.attr("reshape")(py::int_(-1)); + if (flat.data() == input.data()) { + input_ = flat; + } else { + use_iter_ = true; + input_ = input; } - input_ = v; } else { input_ = input; } diff --git a/tiledb/sparse_array.py b/tiledb/sparse_array.py index 70be2c86a8..9aac098b25 100644 --- a/tiledb/sparse_array.py +++ b/tiledb/sparse_array.py @@ -440,7 +440,7 @@ def read_subarray(self, subarray): arr = pyquery.unpack_buffer(name, item[0], item[1]) else: arr = item[0] - arr.dtype = ( + arr = arr.view( self.schema.attr_or_dim_dtype(name) if not self.schema.has_dim_label(name) else self.schema.dim_label(name).dtype @@ -598,7 +598,7 @@ def _read_sparse_subarray(self, subarray, attr_names: list, cond, layout): arr = q.unpack_buffer(name, results[name][0], results[name][1]) else: arr = results[name][0] - arr.dtype = self.schema.attr_or_dim_dtype(name) + arr = arr.view(self.schema.attr_or_dim_dtype(name)) out[final_name] = arr else: arr = results[name][0] @@ -623,8 +623,7 @@ def _read_sparse_subarray(self, subarray, attr_names: list, cond, layout): elif el_dtype == np.dtype("U0"): out[final_name] = "" else: - arr.dtype = el_dtype - out[final_name] = arr + out[final_name] = arr.view(el_dtype) if self.schema.has_attr(final_name) and self.attr(final_name).isnullable: out[final_name] = np.ma.array( diff --git a/tiledb/tests/test_core.py b/tiledb/tests/test_core.py index b236a0f879..72fe6932ad 100644 --- a/tiledb/tests/test_core.py +++ b/tiledb/tests/test_core.py @@ -38,7 +38,7 @@ def test_pyquery_basic(self): q2.set_subarray(subarray) q2.submit() res = q2.results()[""][0] - res.dtype = np.double + res = res.view(np.double) assert_array_equal(res, a[:]) def test_pyquery_init(self): diff --git a/tiledb/tests/test_libtiledb.py b/tiledb/tests/test_libtiledb.py index 07d852ba5b..06fa2f0107 100644 --- a/tiledb/tests/test_libtiledb.py +++ b/tiledb/tests/test_libtiledb.py @@ -2697,7 +2697,11 @@ def test_dense_datetime_vector(self): (np.datetime64("2010-11-01") - start) / np.timedelta64(1, "D") ) read_ndays = int( - (np.datetime64("2011-01-31") - np.datetime64("2010-11-01") + 1) + ( + np.datetime64("2011-01-31") + - np.datetime64("2010-11-01") + + np.timedelta64(1, "D") + ) / np.timedelta64(1, "D") ) expected = a1_vals[read_offset : read_offset + read_ndays] @@ -2712,7 +2716,11 @@ def test_dense_datetime_vector(self): (np.datetime64("2010-01-01") - start) / np.timedelta64(1, "D") ) read_ndays = int( - (np.datetime64("2011-01-01") - np.datetime64("2010-01-01") + 1) + ( + np.datetime64("2011-01-01") + - np.datetime64("2010-01-01") + + np.timedelta64(1, "D") + ) / np.timedelta64(1, "D") ) expected = a1_vals[read_offset : read_offset + read_ndays] @@ -2725,7 +2733,11 @@ def test_dense_datetime_vector(self): (np.datetime64("2010-01-01") - start) / np.timedelta64(1, "D") ) read_ndays = int( - (np.datetime64("2011-01-31") - np.datetime64("2010-01-01") + 1) + ( + np.datetime64("2011-01-31") + - np.datetime64("2010-01-01") + + np.timedelta64(1, "D") + ) / np.timedelta64(1, "D") ) expected = a1_vals[read_offset : read_offset + read_ndays] diff --git a/tiledb/tests/test_subarray.py b/tiledb/tests/test_subarray.py index 42fa2d2349..02a360ae4e 100644 --- a/tiledb/tests/test_subarray.py +++ b/tiledb/tests/test_subarray.py @@ -203,7 +203,7 @@ def test_add_fixed_sized_point_ranges(self): results = q.results() arr = results["a"][0] - arr.dtype = np.int32 + arr = arr.view(np.int32) assert arr.shape == (2,) assert np.array_equal(arr, np.array([10, 30], dtype=np.int32)) @@ -235,6 +235,6 @@ def test_add_var_sized_point_ranges(self): results = q.results() arr = results["a"][0] - arr.dtype = np.int32 + arr = arr.view(np.int32) assert arr.shape == (2,) assert np.array_equal(arr, np.array([10, 30], dtype=np.int32)) From 2ca39c2e7b15c6075b268ea43d823d03afd4a3d2 Mon Sep 17 00:00:00 2001 From: Agisilaos Kounelis Date: Tue, 30 Jun 2026 11:49:23 +0300 Subject: [PATCH 2/4] Validate cell byte length and clarify the flatten fallback Raise instead of silently truncating when a cell's byte length is not a whole number of dtype elements, and reword the npbuffer comment to match what the copy branch actually does. --- tiledb/core.cc | 8 +++++++- tiledb/npbuffer.cc | 9 +++++---- 2 files changed, 12 insertions(+), 5 deletions(-) diff --git a/tiledb/core.cc b/tiledb/core.cc index f9d374d689..a7c2152b2f 100644 --- a/tiledb/core.cc +++ b/tiledb/core.cc @@ -1447,7 +1447,13 @@ class PyQuery { auto arr = py::array(py::dtype("int8"), size, data_ptr); o = py::memoryview(arr); } else { - // `size` is the cell's byte length, a multiple of the itemsize. + // `size` is the cell's byte length and must be a whole number + // of dtype elements; a partial element means corrupt offsets. + if (size % dtype.itemsize() != 0) { + TPY_ERROR_LOC( + "internal error: buffer size is not a multiple of the " + "attribute itemsize"); + } o = py::array(dtype, size / dtype.itemsize(), data_ptr); } diff --git a/tiledb/npbuffer.cc b/tiledb/npbuffer.cc index 65cd21eca2..0b6b30bc76 100644 --- a/tiledb/npbuffer.cc +++ b/tiledb/npbuffer.cc @@ -471,10 +471,11 @@ class NumpyConvert { NumpyConvert(py::array input) { // require a flat buffer if (input.ndim() != 1) { - // reshape(-1) yields a zero-copy view when the array can be - // flattened in place and a fresh copy otherwise; the bulk paths - // need the view, so compare buffer addresses and fall back to the - // element-wise iterator (which handles strided input) on a copy. + // reshape(-1) returns a zero-copy view when the array can be + // flattened in place; the bulk paths need that view, so use it + // when the buffer address is unchanged. Otherwise the array is + // not contiguous: discard the copy and iterate the original + // element-wise instead. py::array flat = input.attr("reshape")(py::int_(-1)); if (flat.data() == input.data()) { input_ = flat; From 67b739f93da38d89da0788835c5c63f979e04487 Mon Sep 17 00:00:00 2001 From: Agisilaos Kounelis Date: Tue, 30 Jun 2026 12:00:36 +0300 Subject: [PATCH 3/4] Test first and last supported numpy per Python in the daily matrix 3.11 dropped out of numpy 2.5, so its newest supported numpy is 2.4.6, and 3.14 was missing its oldest supported numpy (2.3.x). --- .github/workflows/daily-test-build-numpy.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/daily-test-build-numpy.yml b/.github/workflows/daily-test-build-numpy.yml index 854b6dfa4f..e3056647b7 100644 --- a/.github/workflows/daily-test-build-numpy.yml +++ b/.github/workflows/daily-test-build-numpy.yml @@ -28,11 +28,12 @@ jobs: - { python-version: "3.10", numpy-version: "1.25.2" } - { python-version: "3.10", numpy-version: "2.2.6" } - { python-version: "3.11", numpy-version: "1.25.2" } - - { python-version: "3.11", numpy-version: "2.3.4" } + - { python-version: "3.11", numpy-version: "2.4.6" } - { python-version: "3.12", numpy-version: "1.26.4" } - { python-version: "3.12", numpy-version: "2.5.0" } - { python-version: "3.13", numpy-version: "2.1.3" } - { python-version: "3.13", numpy-version: "2.5.0" } + - { python-version: "3.14", numpy-version: "2.3.5" } - { python-version: "3.14", numpy-version: "2.5.0" } fail-fast: false env: From 395cbff79cb9690e5350216a4d570fad52a07671 Mon Sep 17 00:00:00 2001 From: Agisilaos Kounelis Date: Tue, 30 Jun 2026 12:04:46 +0300 Subject: [PATCH 4/4] Drop redundant intermediate assignments around view() view() returns a new array, so the temporary `arr = results[...][0]` line that the old in-place `arr.dtype =` needed is no longer necessary. --- tiledb/array.py | 4 +--- tiledb/dense_array.py | 3 +-- tiledb/multirange_indexing.py | 3 +-- tiledb/sparse_array.py | 6 ++---- tiledb/tests/test_core.py | 3 +-- tiledb/tests/test_subarray.py | 6 ++---- 6 files changed, 8 insertions(+), 17 deletions(-) diff --git a/tiledb/array.py b/tiledb/array.py index 734fd9b605..d235ebb2c9 100644 --- a/tiledb/array.py +++ b/tiledb/array.py @@ -1187,9 +1187,7 @@ def set_query(self, serialized_query): out = OrderedDict() for name in results.keys(): - arr = results[name][0] - arr = arr.view(q.buffer_dtype(name)) - out[name] = arr + out[name] = results[name][0].view(q.buffer_dtype(name)) return out # pickling support: this is a lightweight pickle for distributed use. diff --git a/tiledb/dense_array.py b/tiledb/dense_array.py index 7f7015715c..76ea6fef2b 100644 --- a/tiledb/dense_array.py +++ b/tiledb/dense_array.py @@ -835,8 +835,7 @@ def read_subarray(self, subarray): if len(item[1]) > 0: arr = pyquery.unpack_buffer(name, item[0], item[1]) else: - arr = item[0] - arr = arr.view( + arr = item[0].view( self.schema.attr_or_dim_dtype(name) if not self.schema.has_dim_label(name) else self.schema.dim_label(name).dtype diff --git a/tiledb/multirange_indexing.py b/tiledb/multirange_indexing.py index 09cb7aeaec..b4004cefaf 100644 --- a/tiledb/multirange_indexing.py +++ b/tiledb/multirange_indexing.py @@ -825,8 +825,7 @@ def _get_pyquery_results(pyquery: PyQuery, array: Array) -> Dict[str, np.ndarray if len(item[1]) > 0: arr = pyquery.unpack_buffer(name, item[0], item[1]) else: - arr = item[0] - arr = arr.view( + arr = item[0].view( schema.attr_or_dim_dtype(name) if not schema.has_dim_label(name) else schema.dim_label(name).dtype diff --git a/tiledb/sparse_array.py b/tiledb/sparse_array.py index 9aac098b25..fba1c66fb9 100644 --- a/tiledb/sparse_array.py +++ b/tiledb/sparse_array.py @@ -439,8 +439,7 @@ def read_subarray(self, subarray): if len(item[1]) > 0: arr = pyquery.unpack_buffer(name, item[0], item[1]) else: - arr = item[0] - arr = arr.view( + arr = item[0].view( self.schema.attr_or_dim_dtype(name) if not self.schema.has_dim_label(name) else self.schema.dim_label(name).dtype @@ -597,8 +596,7 @@ def _read_sparse_subarray(self, subarray, attr_names: list, cond, layout): if len(results[name][1]) > 0: # note: len(offsets) > 0 arr = q.unpack_buffer(name, results[name][0], results[name][1]) else: - arr = results[name][0] - arr = arr.view(self.schema.attr_or_dim_dtype(name)) + arr = results[name][0].view(self.schema.attr_or_dim_dtype(name)) out[final_name] = arr else: arr = results[name][0] diff --git a/tiledb/tests/test_core.py b/tiledb/tests/test_core.py index 72fe6932ad..88d9b6ba4e 100644 --- a/tiledb/tests/test_core.py +++ b/tiledb/tests/test_core.py @@ -37,8 +37,7 @@ def test_pyquery_basic(self): subarray.add_ranges([[(0, 3)]]) q2.set_subarray(subarray) q2.submit() - res = q2.results()[""][0] - res = res.view(np.double) + res = q2.results()[""][0].view(np.double) assert_array_equal(res, a[:]) def test_pyquery_init(self): diff --git a/tiledb/tests/test_subarray.py b/tiledb/tests/test_subarray.py index 02a360ae4e..5e6455e1f9 100644 --- a/tiledb/tests/test_subarray.py +++ b/tiledb/tests/test_subarray.py @@ -202,8 +202,7 @@ def test_add_fixed_sized_point_ranges(self): q.submit() results = q.results() - arr = results["a"][0] - arr = arr.view(np.int32) + arr = results["a"][0].view(np.int32) assert arr.shape == (2,) assert np.array_equal(arr, np.array([10, 30], dtype=np.int32)) @@ -234,7 +233,6 @@ def test_add_var_sized_point_ranges(self): q.submit() results = q.results() - arr = results["a"][0] - arr = arr.view(np.int32) + arr = results["a"][0].view(np.int32) assert arr.shape == (2,) assert np.array_equal(arr, np.array([10, 30], dtype=np.int32))