From 0e01707336be8e1bd2b6e706e00b20d469231d14 Mon Sep 17 00:00:00 2001 From: Vecko <36369090+VeckoTheGecko@users.noreply.github.com> Date: Wed, 27 May 2026 15:20:47 +0200 Subject: [PATCH 01/11] Add datasets_sgrid["2d_left_rotated"] --- src/parcels/_datasets/structured/generic.py | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/src/parcels/_datasets/structured/generic.py b/src/parcels/_datasets/structured/generic.py index 5d2da3d63..9731c45ec 100644 --- a/src/parcels/_datasets/structured/generic.py +++ b/src/parcels/_datasets/structured/generic.py @@ -325,4 +325,24 @@ def _unrolled_cone_curvilinear_grid(): "lat": (["node_dimension2"], np.linspace(0, 1, 10)), }, ), + "2d_left_rotated": ( + datasets_comodo["2d_left_rotated"] + .pipe( + sgrid._attach_sgrid_metadata, + sgrid.SGrid2DMetadata( + cf_role="grid_topology", + topology_dimension=2, + node_dimensions=("XG", "YG"), + face_dimensions=( + sgrid.FaceNodePadding("XC", "XG", sgrid.Padding.HIGH), + sgrid.FaceNodePadding("YC", "YG", sgrid.Padding.HIGH), + ), + node_coordinates=("lon", "lat"), + vertical_dimensions=(sgrid.FaceNodePadding("ZC", "ZG", sgrid.Padding.HIGH),), + ), + ) + .sgrid.rename( + _COMODO_TO_2D_SGRID, + ) + ), } From 284185c50497f348f00c67e2291705dfec830e1c Mon Sep 17 00:00:00 2001 From: Vecko <36369090+VeckoTheGecko@users.noreply.github.com> Date: Wed, 27 May 2026 16:11:33 +0200 Subject: [PATCH 02/11] Add SGRID metadata to normal datasets --- src/parcels/_core/xgrid.py | 2 +- src/parcels/_datasets/structured/generic.py | 94 ++++++++++----------- tests/test_fieldset.py | 9 +- 3 files changed, 48 insertions(+), 57 deletions(-) diff --git a/src/parcels/_core/xgrid.py b/src/parcels/_core/xgrid.py index 3edd0875c..62234150b 100644 --- a/src/parcels/_core/xgrid.py +++ b/src/parcels/_core/xgrid.py @@ -40,7 +40,7 @@ def _drop_field_data(ds: xr.Dataset) -> xr.Dataset: when passed to the XGCM grid, the object only functions as an in memory representation of the grid. """ - return ds.drop_vars(ds.data_vars) + return ds.drop_vars(set(ds.data_vars) - {"grid"}) # don't drop sgrid metadata def assert_all_field_dims_have_axis(da: xr.DataArray, xgcm_grid: xgcm.Grid) -> None: diff --git a/src/parcels/_datasets/structured/generic.py b/src/parcels/_datasets/structured/generic.py index 9731c45ec..ce905b1fd 100644 --- a/src/parcels/_datasets/structured/generic.py +++ b/src/parcels/_datasets/structured/generic.py @@ -140,7 +140,20 @@ def _unrolled_cone_curvilinear_grid(): datasets_comodo = { - "2d_left_rotated": _rotated_curvilinear_grid(), + "2d_left_rotated": _rotated_curvilinear_grid().pipe( + sgrid._attach_sgrid_metadata, + sgrid.SGrid2DMetadata( + cf_role="grid_topology", + topology_dimension=2, + node_dimensions=("XG", "YG"), + face_dimensions=( + sgrid.FaceNodePadding("XC", "XG", sgrid.Padding.HIGH), + sgrid.FaceNodePadding("YC", "YG", sgrid.Padding.HIGH), + ), + node_coordinates=("lon", "lat"), + vertical_dimensions=(sgrid.FaceNodePadding("ZC", "ZG", sgrid.Padding.HIGH),), + ), + ), "ds_2d_left": xr.Dataset( # MITgcm indexing style { "data_g": (["time", "ZG", "YG", "XG"], np.random.rand(T, Z, Y, X)), @@ -182,6 +195,19 @@ def _unrolled_cone_curvilinear_grid(): "depth": (["ZG"], np.arange(Z)), "time": (["time"], TIME, {"axis": "T"}), }, + ).pipe( + sgrid._attach_sgrid_metadata, + sgrid.SGrid2DMetadata( + cf_role="grid_topology", + topology_dimension=2, + node_dimensions=("XG", "YG"), + face_dimensions=( + sgrid.FaceNodePadding("XC", "XG", sgrid.Padding.HIGH), + sgrid.FaceNodePadding("YC", "YG", sgrid.Padding.HIGH), + ), + node_coordinates=("lon", "lat"), + vertical_dimensions=(sgrid.FaceNodePadding("ZC", "ZG", sgrid.Padding.HIGH),), + ), ), "ds_2d_right": xr.Dataset( # NEMO indexing style { @@ -224,6 +250,19 @@ def _unrolled_cone_curvilinear_grid(): "depth": (["ZG"], np.arange(Z)), "time": (["time"], TIME, {"axis": "T"}), }, + ).pipe( + sgrid._attach_sgrid_metadata, + sgrid.SGrid2DMetadata( + cf_role="grid_topology", + topology_dimension=2, + node_dimensions=("XG", "YG"), + face_dimensions=( + sgrid.FaceNodePadding("XC", "XG", sgrid.Padding.LOW), + sgrid.FaceNodePadding("YC", "YG", sgrid.Padding.LOW), + ), + node_coordinates=("lon", "lat"), + vertical_dimensions=(sgrid.FaceNodePadding("ZC", "ZG", sgrid.Padding.LOW),), + ), ), "2d_left_unrolled_cone": _unrolled_cone_curvilinear_grid(), } @@ -236,44 +275,16 @@ def _unrolled_cone_curvilinear_grid(): "ZG": "vertical_dimensions_dim1", "ZC": "vertical_dimensions_dim2", } + + datasets_sgrid = { "ds_2d_padded_high": ( - datasets_comodo["ds_2d_left"] - .pipe( - sgrid._attach_sgrid_metadata, - sgrid.SGrid2DMetadata( - cf_role="grid_topology", - topology_dimension=2, - node_dimensions=("XG", "YG"), - face_dimensions=( - sgrid.FaceNodePadding("XC", "XG", sgrid.Padding.HIGH), - sgrid.FaceNodePadding("YC", "YG", sgrid.Padding.HIGH), - ), - node_coordinates=("lon", "lat"), - vertical_dimensions=(sgrid.FaceNodePadding("ZC", "ZG", sgrid.Padding.HIGH),), - ), - ) - .sgrid.rename( + datasets_comodo["ds_2d_left"].sgrid.rename( _COMODO_TO_2D_SGRID, ) ), "ds_2d_padded_low": ( - datasets_comodo["ds_2d_right"] - .pipe( - sgrid._attach_sgrid_metadata, - sgrid.SGrid2DMetadata( - cf_role="grid_topology", - topology_dimension=2, - node_dimensions=("XG", "YG"), - face_dimensions=( - sgrid.FaceNodePadding("XC", "XG", sgrid.Padding.LOW), - sgrid.FaceNodePadding("YC", "YG", sgrid.Padding.LOW), - ), - node_coordinates=("lon", "lat"), - vertical_dimensions=(sgrid.FaceNodePadding("ZC", "ZG", sgrid.Padding.LOW),), - ), - ) - .sgrid.rename( + datasets_comodo["ds_2d_right"].sgrid.rename( _COMODO_TO_2D_SGRID, ) ), @@ -326,22 +337,7 @@ def _unrolled_cone_curvilinear_grid(): }, ), "2d_left_rotated": ( - datasets_comodo["2d_left_rotated"] - .pipe( - sgrid._attach_sgrid_metadata, - sgrid.SGrid2DMetadata( - cf_role="grid_topology", - topology_dimension=2, - node_dimensions=("XG", "YG"), - face_dimensions=( - sgrid.FaceNodePadding("XC", "XG", sgrid.Padding.HIGH), - sgrid.FaceNodePadding("YC", "YG", sgrid.Padding.HIGH), - ), - node_coordinates=("lon", "lat"), - vertical_dimensions=(sgrid.FaceNodePadding("ZC", "ZG", sgrid.Padding.HIGH),), - ), - ) - .sgrid.rename( + datasets_comodo["2d_left_rotated"].sgrid.rename( _COMODO_TO_2D_SGRID, ) ), diff --git a/tests/test_fieldset.py b/tests/test_fieldset.py index 198e1e54b..40e7641c4 100644 --- a/tests/test_fieldset.py +++ b/tests/test_fieldset.py @@ -113,14 +113,9 @@ def SampleP(particles, fieldset): @pytest.mark.parametrize("ds", [pytest.param(ds, id=k) for k, ds in datasets_structured.items()]) def test_fieldset_from_structured_generic_datasets(ds): - grid = XGrid.from_dataset(ds, mesh="flat") - fields = [] - for var in ds.data_vars: - fields.append(Field(var, ds[var], grid, interp_method=XLinear)) - - fieldset = FieldSet(fields) + fieldset = FieldSet.from_sgrid_conventions(ds, mesh="flat") - assert len(fieldset.fields) == len(ds.data_vars) + assert len(fieldset.fields) == len(ds.data_vars) - 1 # `-1` for the SGRID metadata for field in fieldset.fields.values(): utils.assert_valid_field_data(field.data, field.grid) From f569149cc7c2bfd8b1d3b31653e11aa52e78c20b Mon Sep 17 00:00:00 2001 From: Vecko <36369090+VeckoTheGecko@users.noreply.github.com> Date: Thu, 28 May 2026 13:52:23 +0200 Subject: [PATCH 03/11] Disable setting of "SGRID" in "Conventions" field At the moment XGrid.from_dataset uses xgcm internals to set the time dimension. Once the `from_dataset` code path is dropped, this can be enabled again --- src/parcels/_sgrid/core.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/parcels/_sgrid/core.py b/src/parcels/_sgrid/core.py index 05bec18ad..2dec2c3ec 100644 --- a/src/parcels/_sgrid/core.py +++ b/src/parcels/_sgrid/core.py @@ -695,7 +695,7 @@ def _attach_sgrid_metadata(ds: xr.Dataset, grid: SGrid2DMetadata | SGrid3DMetada 0, grid.to_attrs(), ) - ds.attrs["Conventions"] = "SGRID" + # ds.attrs["Conventions"] = "SGRID" # TODO: re-enable once XGrid.from_dataset is gone return ds From 3393372b569a0871de3f5aace147e1a4fff7f802 Mon Sep 17 00:00:00 2001 From: Vecko <36369090+VeckoTheGecko@users.noreply.github.com> Date: Wed, 27 May 2026 16:33:49 +0200 Subject: [PATCH 04/11] Set Conventions attr --- src/parcels/_datasets/structured/generic.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/parcels/_datasets/structured/generic.py b/src/parcels/_datasets/structured/generic.py index ce905b1fd..b79725922 100644 --- a/src/parcels/_datasets/structured/generic.py +++ b/src/parcels/_datasets/structured/generic.py @@ -311,6 +311,7 @@ def _unrolled_cone_curvilinear_grid(): "lon": (["node_dimension1"], np.linspace(0, 1, 10)), "lat": (["node_dimension2"], np.linspace(0, 1, 10)), }, + attrs={"Conventions": "SGRID"}, ), "ds_2d_padded_both": xr.Dataset( { @@ -335,6 +336,7 @@ def _unrolled_cone_curvilinear_grid(): "lon": (["node_dimension1"], np.linspace(0, 1, 10)), "lat": (["node_dimension2"], np.linspace(0, 1, 10)), }, + attrs={"Conventions": "SGRID"}, ), "2d_left_rotated": ( datasets_comodo["2d_left_rotated"].sgrid.rename( From 010d6dbbd5c30cf50273f6560f3dc8c178546a9e Mon Sep 17 00:00:00 2001 From: Vecko <36369090+VeckoTheGecko@users.noreply.github.com> Date: Wed, 27 May 2026 17:12:35 +0200 Subject: [PATCH 05/11] Add ds_2d_outer and ds_2d_inner --- src/parcels/_datasets/structured/generic.py | 110 ++++++++++++++++++++ 1 file changed, 110 insertions(+) diff --git a/src/parcels/_datasets/structured/generic.py b/src/parcels/_datasets/structured/generic.py index b79725922..934382d29 100644 --- a/src/parcels/_datasets/structured/generic.py +++ b/src/parcels/_datasets/structured/generic.py @@ -264,6 +264,116 @@ def _unrolled_cone_curvilinear_grid(): vertical_dimensions=(sgrid.FaceNodePadding("ZC", "ZG", sgrid.Padding.LOW),), ), ), + "ds_2d_inner": xr.Dataset( + { + "data_g": (["time", "ZG", "YG", "XG"], np.random.rand(T, Z, Y, X)), + "data_c": (["time", "ZC", "YC", "XC"], np.random.rand(T, Z - 1, Y - 1, X - 1)), + "U_A_grid": (["time", "ZG", "YG", "XG"], np.random.rand(T, Z, Y, X)), + "V_A_grid": (["time", "ZG", "YG", "XG"], np.random.rand(T, Z, Y, X)), + "U_C_grid": (["time", "ZG", "YC", "XG"], np.random.rand(T, Z, Y - 1, X)), + "V_C_grid": (["time", "ZG", "YG", "XC"], np.random.rand(T, Z, Y, X - 1)), + }, + coords={ + "XG": ( + ["XG"], + 2 * np.pi / X * np.arange(0, X), + {"axis": "X", "c_grid_axis_shift": 0.5}, + ), + "XC": (["XC"], 2 * np.pi / X * (np.arange(0, X - 1) - 0.5), {"axis": "X"}), + "YG": ( + ["YG"], + 2 * np.pi / (Y) * np.arange(0, Y), + {"axis": "Y", "c_grid_axis_shift": 0.5}, + ), + "YC": ( + ["YC"], + 2 * np.pi / (Y) * (np.arange(0, Y - 1) - 0.5), + {"axis": "Y"}, + ), + "ZG": ( + ["ZG"], + np.arange(Z), + {"axis": "Z", "c_grid_axis_shift": 0.5}, + ), + "ZC": ( + ["ZC"], + np.arange(Z - 1) - 0.5, + {"axis": "Z"}, + ), + "lon": (["XG"], 2 * np.pi / X * np.arange(0, X)), + "lat": (["YG"], 2 * np.pi / (Y) * np.arange(0, Y)), + "depth": (["ZG"], np.arange(Z)), + "time": (["time"], TIME, {"axis": "T"}), + }, + ).pipe( + sgrid._attach_sgrid_metadata, + sgrid.SGrid2DMetadata( + cf_role="grid_topology", + topology_dimension=2, + node_dimensions=("XG", "YG"), + face_dimensions=( + sgrid.FaceNodePadding("XC", "XG", sgrid.Padding.BOTH), + sgrid.FaceNodePadding("YC", "YG", sgrid.Padding.BOTH), + ), + node_coordinates=("lon", "lat"), + vertical_dimensions=(sgrid.FaceNodePadding("ZC", "ZG", sgrid.Padding.BOTH),), + ), + ), + "ds_2d_outer": xr.Dataset( + { + "data_g": (["time", "ZG", "YG", "XG"], np.random.rand(T, Z, Y, X)), + "data_c": (["time", "ZC", "YC", "XC"], np.random.rand(T, Z + 1, Y + 1, X + 1)), + "U_A_grid": (["time", "ZG", "YG", "XG"], np.random.rand(T, Z, Y, X)), + "V_A_grid": (["time", "ZG", "YG", "XG"], np.random.rand(T, Z, Y, X)), + "U_C_grid": (["time", "ZG", "YC", "XG"], np.random.rand(T, Z, Y + 1, X)), + "V_C_grid": (["time", "ZG", "YG", "XC"], np.random.rand(T, Z, Y, X + 1)), + }, + coords={ + "XG": ( + ["XG"], + 2 * np.pi / X * np.arange(0, X), + {"axis": "X", "c_grid_axis_shift": -0.5}, + ), + "XC": (["XC"], 2 * np.pi / X * (np.arange(0, X + 1) + 0.5), {"axis": "X"}), + "YG": ( + ["YG"], + 2 * np.pi / (Y) * np.arange(0, Y), + {"axis": "Y", "c_grid_axis_shift": -0.5}, + ), + "YC": ( + ["YC"], + 2 * np.pi / (Y) * (np.arange(0, Y + 1) + 0.5), + {"axis": "Y"}, + ), + "ZG": ( + ["ZG"], + np.arange(Z), + {"axis": "Z", "c_grid_axis_shift": -0.5}, + ), + "ZC": ( + ["ZC"], + np.arange(Z + 1) + 0.5, + {"axis": "Z"}, + ), + "lon": (["XG"], 2 * np.pi / X * np.arange(0, X)), + "lat": (["YG"], 2 * np.pi / (Y) * np.arange(0, Y)), + "depth": (["ZG"], np.arange(Z)), + "time": (["time"], TIME, {"axis": "T"}), + }, + ).pipe( + sgrid._attach_sgrid_metadata, + sgrid.SGrid2DMetadata( + cf_role="grid_topology", + topology_dimension=2, + node_dimensions=("XG", "YG"), + face_dimensions=( + sgrid.FaceNodePadding("XC", "XG", sgrid.Padding.NONE), + sgrid.FaceNodePadding("YC", "YG", sgrid.Padding.NONE), + ), + node_coordinates=("lon", "lat"), + vertical_dimensions=(sgrid.FaceNodePadding("ZC", "ZG", sgrid.Padding.NONE),), + ), + ), "2d_left_unrolled_cone": _unrolled_cone_curvilinear_grid(), } From a8709e3258ec36a29e8272c64916b4738582c6ca Mon Sep 17 00:00:00 2001 From: Vecko <36369090+VeckoTheGecko@users.noreply.github.com> Date: Thu, 28 May 2026 14:03:47 +0200 Subject: [PATCH 06/11] Rename `datasets_comodo` to `datasets` --- src/parcels/_datasets/structured/generic.py | 10 ++--- tests/datasets/test_structured.py | 6 +-- tests/datasets/test_utils.py | 4 +- tests/test_field.py | 2 +- tests/test_fieldset.py | 2 +- tests/test_index_search.py | 4 +- tests/test_kernel.py | 2 +- tests/test_particlefile.py | 4 +- tests/test_particleset.py | 2 +- tests/test_particleset_execute.py | 2 +- tests/test_particlesetview.py | 2 +- tests/test_spatialhash.py | 8 ++-- tests/test_xgrid.py | 50 ++++++++++----------- 13 files changed, 49 insertions(+), 49 deletions(-) diff --git a/src/parcels/_datasets/structured/generic.py b/src/parcels/_datasets/structured/generic.py index 934382d29..b94d623d4 100644 --- a/src/parcels/_datasets/structured/generic.py +++ b/src/parcels/_datasets/structured/generic.py @@ -5,7 +5,7 @@ from . import T, X, Y, Z -__all__ = ["T", "X", "Y", "Z", "datasets_comodo", "datasets_sgrid"] +__all__ = ["T", "X", "Y", "Z", "datasets", "datasets_sgrid"] TIME = xr.date_range("2000", "2001", T) @@ -139,7 +139,7 @@ def _unrolled_cone_curvilinear_grid(): ) -datasets_comodo = { +datasets = { "2d_left_rotated": _rotated_curvilinear_grid().pipe( sgrid._attach_sgrid_metadata, sgrid.SGrid2DMetadata( @@ -389,12 +389,12 @@ def _unrolled_cone_curvilinear_grid(): datasets_sgrid = { "ds_2d_padded_high": ( - datasets_comodo["ds_2d_left"].sgrid.rename( + datasets["ds_2d_left"].sgrid.rename( _COMODO_TO_2D_SGRID, ) ), "ds_2d_padded_low": ( - datasets_comodo["ds_2d_right"].sgrid.rename( + datasets["ds_2d_right"].sgrid.rename( _COMODO_TO_2D_SGRID, ) ), @@ -449,7 +449,7 @@ def _unrolled_cone_curvilinear_grid(): attrs={"Conventions": "SGRID"}, ), "2d_left_rotated": ( - datasets_comodo["2d_left_rotated"].sgrid.rename( + datasets["2d_left_rotated"].sgrid.rename( _COMODO_TO_2D_SGRID, ) ), diff --git a/tests/datasets/test_structured.py b/tests/datasets/test_structured.py index 9ca6195f1..2750583f5 100644 --- a/tests/datasets/test_structured.py +++ b/tests/datasets/test_structured.py @@ -1,12 +1,12 @@ import xgcm from parcels._core.xgrid import _DEFAULT_XGCM_KWARGS -from parcels._datasets.structured.generic import datasets_comodo +from parcels._datasets.structured.generic import datasets def test_left_indexed_dataset(): """Checks that 'ds_2d_left' is right indexed on all variables.""" - ds = datasets_comodo["ds_2d_left"] + ds = datasets["ds_2d_left"] grid = xgcm.Grid(ds, **_DEFAULT_XGCM_KWARGS) for _axis_name, axis in grid.axes.items(): @@ -16,7 +16,7 @@ def test_left_indexed_dataset(): def test_right_indexed_dataset(): """Checks that 'ds_2d_right' is right indexed on all variables.""" - ds = datasets_comodo["ds_2d_right"] + ds = datasets["ds_2d_right"] grid = xgcm.Grid(ds, **_DEFAULT_XGCM_KWARGS) for _axis_name, axis in grid.axes.items(): for pos, _dim_name in axis.coords.items(): diff --git a/tests/datasets/test_utils.py b/tests/datasets/test_utils.py index 914e4055b..b376283e3 100644 --- a/tests/datasets/test_utils.py +++ b/tests/datasets/test_utils.py @@ -3,7 +3,7 @@ import xarray as xr from parcels._datasets import utils -from parcels._datasets.structured.generic import datasets_comodo +from parcels._datasets.structured.generic import datasets @pytest.fixture @@ -28,7 +28,7 @@ def nonzero_ds(): ) -@pytest.mark.parametrize("ds", [pytest.param(v, id=k) for k, v in datasets_comodo.items()]) +@pytest.mark.parametrize("ds", [pytest.param(v, id=k) for k, v in datasets.items()]) @pytest.mark.parametrize("except_for", [None, "coords"]) def test_replace_arrays_with_zeros(ds, except_for): # make sure doesn't error with range of datasets diff --git a/tests/test_field.py b/tests/test_field.py index 70fc6a753..cc264beae 100644 --- a/tests/test_field.py +++ b/tests/test_field.py @@ -7,7 +7,7 @@ from parcels import Field, UxGrid, VectorField, XGrid from parcels._datasets.structured.generic import T as T_structured -from parcels._datasets.structured.generic import datasets_comodo as datasets_structured +from parcels._datasets.structured.generic import datasets as datasets_structured from parcels._datasets.unstructured.generic import datasets as datasets_unstructured from parcels.interpolators import ( UxConstantFaceConstantZC, diff --git a/tests/test_fieldset.py b/tests/test_fieldset.py index 40e7641c4..abcb08df4 100644 --- a/tests/test_fieldset.py +++ b/tests/test_fieldset.py @@ -10,7 +10,7 @@ from parcels import Field, ParticleFile, ParticleSet, VectorField, XGrid, convert from parcels._core.fieldset import CalendarError, FieldSet, _datetime_to_msg from parcels._datasets.structured.generic import T as T_structured -from parcels._datasets.structured.generic import datasets_comodo as datasets_structured +from parcels._datasets.structured.generic import datasets as datasets_structured from parcels._datasets.structured.generic import datasets_sgrid from parcels._datasets.unstructured.generic import datasets as datasets_unstructured from parcels.interpolators import XLinear, XLinear_Velocity diff --git a/tests/test_index_search.py b/tests/test_index_search.py index 153eafafe..b85e6e3b6 100644 --- a/tests/test_index_search.py +++ b/tests/test_index_search.py @@ -5,13 +5,13 @@ import parcels.tutorial from parcels import Field, XGrid from parcels._core.index_search import _latlon_rad_to_xyz, _search_indices_curvilinear_2d -from parcels._datasets.structured.generic import datasets_comodo +from parcels._datasets.structured.generic import datasets from parcels.interpolators import XLinear @pytest.fixture def field_cone(): - ds = datasets_comodo["2d_left_unrolled_cone"] + ds = datasets["2d_left_unrolled_cone"] grid = XGrid.from_dataset(ds, mesh="flat") field = Field( name="test_field", diff --git a/tests/test_kernel.py b/tests/test_kernel.py index 691e88938..b4a3de922 100644 --- a/tests/test_kernel.py +++ b/tests/test_kernel.py @@ -8,7 +8,7 @@ XGrid, ) from parcels._core.kernel import Kernel -from parcels._datasets.structured.generic import datasets_comodo as datasets_structured +from parcels._datasets.structured.generic import datasets as datasets_structured from parcels.interpolators import XLinear from parcels.kernels import AdvectionRK4, AdvectionRK45 from tests.common_kernels import MoveEast, MoveNorth diff --git a/tests/test_particlefile.py b/tests/test_particlefile.py index 233545559..6aa5a90ca 100755 --- a/tests/test_particlefile.py +++ b/tests/test_particlefile.py @@ -26,7 +26,7 @@ from parcels._core.particlefile import _get_schema from parcels._core.utils.time import TimeInterval, timedelta_to_float from parcels._datasets.structured.generated import peninsula_dataset -from parcels._datasets.structured.generic import datasets_comodo +from parcels._datasets.structured.generic import datasets from parcels.convert import copernicusmarine_to_sgrid from parcels.interpolators import XLinear, XLinear_Velocity from parcels.kernels import AdvectionRK4 @@ -36,7 +36,7 @@ @pytest.fixture def fieldset() -> FieldSet: # TODO v4: Move into a `conftest.py` file and remove duplicates """Fixture to create a FieldSet object for testing.""" - ds = datasets_comodo["ds_2d_left"] + ds = datasets["ds_2d_left"] grid = XGrid.from_dataset(ds, mesh="flat") U = Field("U", ds["U_A_grid"], grid, XLinear) V = Field("V", ds["V_A_grid"], grid, XLinear) diff --git a/tests/test_particleset.py b/tests/test_particleset.py index 565655039..5c66341b5 100644 --- a/tests/test_particleset.py +++ b/tests/test_particleset.py @@ -15,7 +15,7 @@ Variable, XGrid, ) -from parcels._datasets.structured.generic import datasets_comodo as datasets_structured +from parcels._datasets.structured.generic import datasets as datasets_structured from parcels.interpolators import XLinear from tests.common_kernels import DoNothing from tests.utils import round_and_hash_float_array diff --git a/tests/test_particleset_execute.py b/tests/test_particleset_execute.py index 012d14f2c..87a6fabd1 100644 --- a/tests/test_particleset_execute.py +++ b/tests/test_particleset_execute.py @@ -21,7 +21,7 @@ ) from parcels._core.utils.time import timedelta_to_float from parcels._datasets.structured.generated import simple_UV_dataset -from parcels._datasets.structured.generic import datasets_comodo as datasets_structured +from parcels._datasets.structured.generic import datasets as datasets_structured from parcels._datasets.unstructured.generic import datasets as datasets_unstructured from parcels.interpolators import ( Ux_Velocity, diff --git a/tests/test_particlesetview.py b/tests/test_particlesetview.py index fb28f1e05..2f0532543 100644 --- a/tests/test_particlesetview.py +++ b/tests/test_particlesetview.py @@ -3,7 +3,7 @@ from parcels import Field, FieldSet, Particle, ParticleSet, Variable, VectorField, XGrid from parcels._core.statuscodes import StatusCode -from parcels._datasets.structured.generic import datasets_comodo as datasets_structured +from parcels._datasets.structured.generic import datasets as datasets_structured from parcels.interpolators import XLinear, XLinear_Velocity diff --git a/tests/test_spatialhash.py b/tests/test_spatialhash.py index 4d87882cb..6beedfbf6 100644 --- a/tests/test_spatialhash.py +++ b/tests/test_spatialhash.py @@ -1,18 +1,18 @@ import numpy as np from parcels import XGrid -from parcels._datasets.structured.generic import datasets_comodo +from parcels._datasets.structured.generic import datasets def test_spatialhash_init(): - ds = datasets_comodo["2d_left_rotated"] + ds = datasets["2d_left_rotated"] grid = XGrid.from_dataset(ds, mesh="flat") spatialhash = grid.get_spatial_hash() assert spatialhash is not None def test_invalid_positions(): - ds = datasets_comodo["2d_left_rotated"] + ds = datasets["2d_left_rotated"] grid = XGrid.from_dataset(ds, mesh="flat") j, i, _ = grid.get_spatial_hash().query([np.nan, np.inf], [np.nan, np.inf]) @@ -21,7 +21,7 @@ def test_invalid_positions(): def test_mixed_positions(): - ds = datasets_comodo["2d_left_rotated"] + ds = datasets["2d_left_rotated"] grid = XGrid.from_dataset(ds, mesh="flat") lat = grid.lat.mean() lon = grid.lon.mean() diff --git a/tests/test_xgrid.py b/tests/test_xgrid.py index c9e3881b6..53b0124e8 100644 --- a/tests/test_xgrid.py +++ b/tests/test_xgrid.py @@ -17,24 +17,24 @@ XGrid, _transpose_xfield_data_to_tzyx, ) -from parcels._datasets.structured.generic import X, Y, Z, datasets_comodo, datasets_sgrid +from parcels._datasets.structured.generic import X, Y, Z, datasets, datasets_sgrid from parcels.interpolators import XLinear from tests import utils GridTestCase = namedtuple("GridTestCase", ["ds", "attr", "expected"]) test_cases = [ - GridTestCase(datasets_comodo["ds_2d_left"], "lon", datasets_comodo["ds_2d_left"].XG.values), - GridTestCase(datasets_comodo["ds_2d_left"], "lat", datasets_comodo["ds_2d_left"].YG.values), - GridTestCase(datasets_comodo["ds_2d_left"], "depth", datasets_comodo["ds_2d_left"].ZG.values), + GridTestCase(datasets["ds_2d_left"], "lon", datasets["ds_2d_left"].XG.values), + GridTestCase(datasets["ds_2d_left"], "lat", datasets["ds_2d_left"].YG.values), + GridTestCase(datasets["ds_2d_left"], "depth", datasets["ds_2d_left"].ZG.values), GridTestCase( - datasets_comodo["ds_2d_left"], + datasets["ds_2d_left"], "time", - datasets_comodo["ds_2d_left"].time.values.astype(np.float64) / 1e9, + datasets["ds_2d_left"].time.values.astype(np.float64) / 1e9, ), - GridTestCase(datasets_comodo["ds_2d_left"], "xdim", X - 1), - GridTestCase(datasets_comodo["ds_2d_left"], "ydim", Y - 1), - GridTestCase(datasets_comodo["ds_2d_left"], "zdim", Z - 1), + GridTestCase(datasets["ds_2d_left"], "xdim", X - 1), + GridTestCase(datasets["ds_2d_left"], "ydim", Y - 1), + GridTestCase(datasets["ds_2d_left"], "zdim", Z - 1), ] @@ -48,7 +48,7 @@ def assert_equal(actual, expected): assert_allclose(actual, expected) -@pytest.mark.parametrize("ds", [datasets_comodo["ds_2d_left"]]) +@pytest.mark.parametrize("ds", [datasets["ds_2d_left"]]) def test_grid_init_param_types(ds): with pytest.raises(ValueError, match="Invalid value 'invalid'. Valid options are.*"): XGrid.from_dataset(ds, mesh="invalid") @@ -61,25 +61,25 @@ def test_xgrid_properties_ground_truth(ds, attr, expected): assert_equal(actual, expected) -@pytest.mark.parametrize("ds", [pytest.param(ds, id=key) for key, ds in datasets_comodo.items()]) +@pytest.mark.parametrize("ds", [pytest.param(ds, id=key) for key, ds in datasets.items()]) def test_xgrid_from_dataset_on_generic_datasets(ds): XGrid.from_dataset(ds, mesh="flat") -@pytest.mark.parametrize("ds", [datasets_comodo["ds_2d_left"]]) +@pytest.mark.parametrize("ds", [datasets["ds_2d_left"]]) def test_xgrid_axes(ds): grid = XGrid.from_dataset(ds, mesh="flat") assert grid.axes == ["Z", "Y", "X"] -@pytest.mark.parametrize("ds", [datasets_comodo["ds_2d_left"]]) +@pytest.mark.parametrize("ds", [datasets["ds_2d_left"]]) @pytest.mark.parametrize("mesh", ["flat", "spherical"]) def test_uxgrid_mesh(ds, mesh): grid = XGrid.from_dataset(ds, mesh=mesh) assert grid._mesh == mesh -@pytest.mark.parametrize("ds", [datasets_comodo["ds_2d_left"]]) +@pytest.mark.parametrize("ds", [datasets["ds_2d_left"]]) def test_transpose_xfield_data_to_tzyx(ds): da = ds["data_g"] grid = XGrid.from_dataset(ds, mesh="flat") @@ -93,7 +93,7 @@ def test_transpose_xfield_data_to_tzyx(ds): utils.assert_valid_field_data(da_test, grid) -@pytest.mark.parametrize("ds", [datasets_comodo["ds_2d_left"]]) +@pytest.mark.parametrize("ds", [datasets["ds_2d_left"]]) def test_xgrid_get_axis_dim(ds): grid = XGrid.from_dataset(ds, mesh="flat") assert grid.get_axis_dim("Z") == Z - 1 @@ -108,7 +108,7 @@ def test_invalid_xgrid_field_array(): def test_invalid_lon_lat(): """Stress test the grid initialiser by creating incompatible datasets that test the edge cases""" - ds = datasets_comodo["ds_2d_left"].copy() + ds = datasets["ds_2d_left"].copy() ds["lon"], ds["lat"] = xr.broadcast(ds["YC"], ds["XC"]) with pytest.raises( @@ -117,7 +117,7 @@ def test_invalid_lon_lat(): ): XGrid.from_dataset(ds, mesh="flat") - ds = datasets_comodo["ds_2d_left"].copy() + ds = datasets["ds_2d_left"].copy() ds["lon"], _ = xr.broadcast(ds["YG"], ds["XG"]) with pytest.raises( ValueError, @@ -125,7 +125,7 @@ def test_invalid_lon_lat(): ): XGrid.from_dataset(ds, mesh="flat") - ds = datasets_comodo["ds_2d_left"].copy() + ds = datasets["ds_2d_left"].copy() ds["lon"], ds["lat"] = xr.broadcast(ds["YG"], ds["XG"]) ds["lon"], ds["lat"] = ds["lon"].transpose(), ds["lat"].transpose() @@ -137,7 +137,7 @@ def test_invalid_lon_lat(): def test_invalid_depth(): - ds = datasets_comodo["ds_2d_left"].copy() + ds = datasets["ds_2d_left"].copy() ds = ds.reindex({"ZG": ds.ZG[::-1]}) with pytest.raises(ValueError, match="Depth DataArray .* must be strictly increasing*"): @@ -202,8 +202,8 @@ def test_time1D_field(): @pytest.mark.parametrize( "ds", [ - pytest.param(datasets_comodo["ds_2d_left"], id="1D lon/lat"), - pytest.param(datasets_comodo["2d_left_rotated"], id="2D lon/lat"), + pytest.param(datasets["ds_2d_left"], id="1D lon/lat"), + pytest.param(datasets["2d_left_rotated"], id="2D lon/lat"), ], ) # for key, ds in datasets.items()]) def test_xgrid_search_cpoints(ds): @@ -282,7 +282,7 @@ def test_search_1d_array_some_out_of_bounds(array, x, expected_xi): "ds, da_name, expected", [ pytest.param( - datasets_comodo["ds_2d_left"], + datasets["ds_2d_left"], "U_C_grid", { "XG": (np.int64(0), np.float64(0.0)), @@ -292,7 +292,7 @@ def test_search_1d_array_some_out_of_bounds(array, x, expected_xi): id="MITgcm indexing style U_C_grid", ), pytest.param( - datasets_comodo["ds_2d_left"], + datasets["ds_2d_left"], "V_C_grid", { "XC": (np.int64(-1), np.float64(0.5)), @@ -302,7 +302,7 @@ def test_search_1d_array_some_out_of_bounds(array, x, expected_xi): id="MITgcm indexing style V_C_grid", ), pytest.param( - datasets_comodo["ds_2d_right"], + datasets["ds_2d_right"], "U_C_grid", { "XG": (np.int64(0), np.float64(0.0)), @@ -312,7 +312,7 @@ def test_search_1d_array_some_out_of_bounds(array, x, expected_xi): id="NEMO indexing style U_C_grid", ), pytest.param( - datasets_comodo["ds_2d_right"], + datasets["ds_2d_right"], "V_C_grid", { "XC": (np.int64(0), np.float64(0.5)), From adc806967494c56ca352d7ec83ce5d73c3f09eac Mon Sep 17 00:00:00 2001 From: Vecko <36369090+VeckoTheGecko@users.noreply.github.com> Date: Thu, 28 May 2026 14:04:31 +0200 Subject: [PATCH 07/11] Update datasets_sgrid definitions Update ds_2d_padded_both and ds_2d_padded_none --- src/parcels/_datasets/structured/generic.py | 52 ++------------------- 1 file changed, 4 insertions(+), 48 deletions(-) diff --git a/src/parcels/_datasets/structured/generic.py b/src/parcels/_datasets/structured/generic.py index b94d623d4..4d81bd688 100644 --- a/src/parcels/_datasets/structured/generic.py +++ b/src/parcels/_datasets/structured/generic.py @@ -398,55 +398,11 @@ def _unrolled_cone_curvilinear_grid(): _COMODO_TO_2D_SGRID, ) ), - "ds_2d_padded_none": xr.Dataset( - { - "data_g": (["node_dimension1", "node_dimension2"], np.random.rand(10, 10)), - "data_c": (["face_dimension1", "face_dimension2"], np.random.rand(9, 9)), - "grid": ( - [], - np.array(0), - sgrid.SGrid2DMetadata( - cf_role="grid_topology", - topology_dimension=2, - node_dimensions=("node_dimension1", "node_dimension2"), - face_dimensions=( - sgrid.FaceNodePadding("face_dimension1", "node_dimension1", sgrid.Padding.NONE), - sgrid.FaceNodePadding("face_dimension2", "node_dimension2", sgrid.Padding.NONE), - ), - node_coordinates=("lon", "lat"), - ).to_attrs(), - ), - }, - coords={ - "lon": (["node_dimension1"], np.linspace(0, 1, 10)), - "lat": (["node_dimension2"], np.linspace(0, 1, 10)), - }, - attrs={"Conventions": "SGRID"}, + "ds_2d_padded_none": datasets["ds_2d_outer"].sgrid.rename( + _COMODO_TO_2D_SGRID, ), - "ds_2d_padded_both": xr.Dataset( - { - "data_g": (["node_dimension1", "node_dimension2"], np.random.rand(10, 10)), - "data_c": (["face_dimension1", "face_dimension2"], np.random.rand(11, 11)), - "grid": ( - [], - np.array(0), - sgrid.SGrid2DMetadata( - cf_role="grid_topology", - topology_dimension=2, - node_dimensions=("node_dimension1", "node_dimension2"), - face_dimensions=( - sgrid.FaceNodePadding("face_dimension1", "node_dimension1", sgrid.Padding.BOTH), - sgrid.FaceNodePadding("face_dimension2", "node_dimension2", sgrid.Padding.BOTH), - ), - node_coordinates=("lon", "lat"), - ).to_attrs(), - ), - }, - coords={ - "lon": (["node_dimension1"], np.linspace(0, 1, 10)), - "lat": (["node_dimension2"], np.linspace(0, 1, 10)), - }, - attrs={"Conventions": "SGRID"}, + "ds_2d_padded_both": datasets["ds_2d_inner"].sgrid.rename( + _COMODO_TO_2D_SGRID, ), "2d_left_rotated": ( datasets["2d_left_rotated"].sgrid.rename( From fdd6a710a6dd04bfc2a71a7012ec94f877dd6a16 Mon Sep 17 00:00:00 2001 From: Vecko <36369090+VeckoTheGecko@users.noreply.github.com> Date: Thu, 28 May 2026 14:11:20 +0200 Subject: [PATCH 08/11] Refactor datasets_sgrid --- src/parcels/_datasets/structured/generic.py | 39 ++++++++++----------- 1 file changed, 18 insertions(+), 21 deletions(-) diff --git a/src/parcels/_datasets/structured/generic.py b/src/parcels/_datasets/structured/generic.py index 4d81bd688..43687af5b 100644 --- a/src/parcels/_datasets/structured/generic.py +++ b/src/parcels/_datasets/structured/generic.py @@ -386,27 +386,24 @@ def _unrolled_cone_curvilinear_grid(): "ZC": "vertical_dimensions_dim2", } +_DATASET_NAME_TO_SGRID_NAME = { + "ds_2d_left": "ds_2d_padded_high", + "ds_2d_right": "ds_2d_padded_low", + "ds_2d_outer": "ds_2d_padded_none", + "ds_2d_inner": "ds_2d_padded_both", + "2d_left_rotated": "2d_left_rotated", +} -datasets_sgrid = { - "ds_2d_padded_high": ( - datasets["ds_2d_left"].sgrid.rename( - _COMODO_TO_2D_SGRID, - ) - ), - "ds_2d_padded_low": ( - datasets["ds_2d_right"].sgrid.rename( - _COMODO_TO_2D_SGRID, - ) - ), - "ds_2d_padded_none": datasets["ds_2d_outer"].sgrid.rename( - _COMODO_TO_2D_SGRID, - ), - "ds_2d_padded_both": datasets["ds_2d_inner"].sgrid.rename( - _COMODO_TO_2D_SGRID, - ), - "2d_left_rotated": ( - datasets["2d_left_rotated"].sgrid.rename( + +def create_datasets_sgrid(datasets_: dict[str, xr.Dataset]) -> dict[str, xr.Dataset]: + ret = {} + for name, ds in datasets_.items(): + if name not in _DATASET_NAME_TO_SGRID_NAME: + continue + ret[_DATASET_NAME_TO_SGRID_NAME[name]] = ds.sgrid.rename( _COMODO_TO_2D_SGRID, ) - ), -} + return ret + + +datasets_sgrid = create_datasets_sgrid(datasets) From 018ccc96e2521f8b2558308cf22332dc869ccef3 Mon Sep 17 00:00:00 2001 From: Vecko <36369090+VeckoTheGecko@users.noreply.github.com> Date: Thu, 28 May 2026 14:29:52 +0200 Subject: [PATCH 09/11] Add SGRID metadata to 2d_left_unrolled_cone --- src/parcels/_datasets/structured/generic.py | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/src/parcels/_datasets/structured/generic.py b/src/parcels/_datasets/structured/generic.py index 43687af5b..dd2ea37b4 100644 --- a/src/parcels/_datasets/structured/generic.py +++ b/src/parcels/_datasets/structured/generic.py @@ -374,7 +374,20 @@ def _unrolled_cone_curvilinear_grid(): vertical_dimensions=(sgrid.FaceNodePadding("ZC", "ZG", sgrid.Padding.NONE),), ), ), - "2d_left_unrolled_cone": _unrolled_cone_curvilinear_grid(), + "2d_left_unrolled_cone": _unrolled_cone_curvilinear_grid().pipe( + sgrid._attach_sgrid_metadata, + sgrid.SGrid2DMetadata( + cf_role="grid_topology", + topology_dimension=2, + node_dimensions=("XG", "YG"), + face_dimensions=( + sgrid.FaceNodePadding("XC", "XG", sgrid.Padding.HIGH), + sgrid.FaceNodePadding("YC", "YG", sgrid.Padding.HIGH), + ), + node_coordinates=("lon", "lat"), + vertical_dimensions=(sgrid.FaceNodePadding("ZC", "ZG", sgrid.Padding.HIGH),), + ), + ), } _COMODO_TO_2D_SGRID = { # Note "2D SGRID" here is meant in the context of SGRID convention (i.e., 1D depth) From e76e8b81c4915d12ae7a8da48210adf8c5778f6e Mon Sep 17 00:00:00 2001 From: Vecko <36369090+VeckoTheGecko@users.noreply.github.com> Date: Thu, 28 May 2026 14:34:03 +0200 Subject: [PATCH 10/11] Remove 2d_left_rotated from datasets_sgrid Now that the datasets themselves have sgrid metadata, theres no need to have this extra dataset --- src/parcels/_datasets/structured/generic.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/parcels/_datasets/structured/generic.py b/src/parcels/_datasets/structured/generic.py index dd2ea37b4..e8d283f72 100644 --- a/src/parcels/_datasets/structured/generic.py +++ b/src/parcels/_datasets/structured/generic.py @@ -404,7 +404,6 @@ def _unrolled_cone_curvilinear_grid(): "ds_2d_right": "ds_2d_padded_low", "ds_2d_outer": "ds_2d_padded_none", "ds_2d_inner": "ds_2d_padded_both", - "2d_left_rotated": "2d_left_rotated", } From ad0d84acc74f11760c36a69c3648c2e8e89cfa80 Mon Sep 17 00:00:00 2001 From: Vecko <36369090+VeckoTheGecko@users.noreply.github.com> Date: Thu, 28 May 2026 15:02:58 +0200 Subject: [PATCH 11/11] Fix docs --- docs/development/posting-issues.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/development/posting-issues.md b/docs/development/posting-issues.md index ad29ed1ef..0d47aea8a 100644 --- a/docs/development/posting-issues.md +++ b/docs/development/posting-issues.md @@ -46,8 +46,8 @@ As a user with access to your dataset, you would do: # Generate an example dataset to zip. The user would use their own. import xarray as xr -from parcels._datasets.structured.generic import datasets_comodo -datasets_comodo['ds_2d_left'].to_netcdf("my_dataset.nc") +from parcels._datasets.structured.generic import datasets +datasets['ds_2d_left'].to_netcdf("my_dataset.nc") ``` ```{code-cell}