From 8a556672ce86cbd4258d773cb47a3675ed4c079b Mon Sep 17 00:00:00 2001 From: Andre Lobato <255781626+AndreLobatoSeequent@users.noreply.github.com> Date: Mon, 31 Aug 2026 16:01:35 +1200 Subject: [PATCH 1/8] Add column groups support to block-models --- packages/evo-blockmodels/pyproject.toml | 2 +- .../src/evo/blockmodels/client.py | 167 +++++++++++- .../src/evo/blockmodels/data.py | 130 +++++++++ .../tests/test_group_helpers.py | 127 +++++++++ packages/evo-blockmodels/tests/test_update.py | 256 +++++++++++++++++- 5 files changed, 665 insertions(+), 17 deletions(-) create mode 100644 packages/evo-blockmodels/tests/test_group_helpers.py diff --git a/packages/evo-blockmodels/pyproject.toml b/packages/evo-blockmodels/pyproject.toml index ee2f5e3d..2ddc1561 100644 --- a/packages/evo-blockmodels/pyproject.toml +++ b/packages/evo-blockmodels/pyproject.toml @@ -1,7 +1,7 @@ [project] name = "evo-blockmodels" description = "Python SDK for using the Seequent Evo Geoscience Block Model API" -version = "0.6.0" +version = "0.7.0" requires-python = ">=3.10" license-files = ["LICENSE.md"] dynamic = ["readme"] diff --git a/packages/evo-blockmodels/src/evo/blockmodels/client.py b/packages/evo-blockmodels/src/evo/blockmodels/client.py index 32dec066..3b0d0e32 100644 --- a/packages/evo-blockmodels/src/evo/blockmodels/client.py +++ b/packages/evo-blockmodels/src/evo/blockmodels/client.py @@ -31,6 +31,8 @@ ColumnMetadataUpdate, FlexibleGridDefinition, FullySubBlockedGridDefinition, + GroupDefinition, + GroupMetadataUpdate, ListingVersion, OctreeGridDefinition, RegularGridDefinition, @@ -136,6 +138,39 @@ def _version_listing_from_model(version: models.ListingVersion) -> ListingVersio } +def _group_lite_from_definition(definition: GroupDefinition) -> models.GroupLite: + """Convert a public :class:`GroupDefinition` to the generated title-addressed ``GroupLite``.""" + return models.GroupLite(**definition.model_dump(exclude_unset=True)) + + +def _group_values_from_update(update: GroupMetadataUpdate) -> models.GroupUpdateMetadataValuesLite: + """Convert a public :class:`GroupMetadataUpdate` to the generated ``GroupUpdateMetadataValuesLite``. + + Only fields the caller explicitly set are forwarded, so untouched fields are omitted on the wire. + The public ``new_title`` is mapped onto the wire field ``title`` (a group rename). + """ + values = update.model_dump(exclude_unset=True) + if "new_title" in values: + values["title"] = values.pop("new_title") + return models.GroupUpdateMetadataValuesLite(**values) + + +def _build_update_groups_lite( + new: list[GroupDefinition] | None, + update: dict[str, GroupMetadataUpdate] | None, + delete: list[str] | None, +) -> models.UpdateGroupsLite: + """Build the title-addressed ``UpdateGroupsLite`` payload from the public group arguments.""" + return models.UpdateGroupsLite( + new=[_group_lite_from_definition(definition) for definition in (new or [])], + update_metadata=[ + models.GroupUpdateMetadataLite(title=title, values=_group_values_from_update(values)) + for title, values in (update or {}).items() + ], + delete=list(delete or []), + ) + + class BlockModelAPIClient(BaseAPIClient): def __init__( self, @@ -399,21 +434,24 @@ async def _upload_data(self, bm_id: uuid.UUID, job_id: uuid.UUID, upload_url: st return await self.upload_block_model(bm_id, job_id, upload_url, cache_location) async def _update_model_no_data( - self, bm_id: UUID, columns: models.UpdateColumnsLite, comment: str | None = None + self, + bm_id: UUID, + columns: models.UpdateColumnsLite, + comment: str | None = None, + groups: models.UpdateGroupsLite | None = None, ) -> Version: """Helper to apply an UpdateColumnsLite and return the resulting Version. This is for column operations where new data is not required. """ + # Only set ``groups`` when provided so it stays unset (and off the wire) otherwise. + update_data = models.UpdateDataLite1(columns=columns, comment=comment) + if groups is not None: + update_data.groups = groups update_response = await self._column_operations_api.update_block_model_from_latest_version( org_id=str(self._environment.org_id), workspace_id=str(self._environment.workspace_id), bm_id=str(bm_id), - update_data_lite=models.UpdateDataLite( - models.UpdateDataLite1( - columns=columns, - comment=comment, - ) - ), + update_data_lite=models.UpdateDataLite(update_data), additional_headers=self._preview_headers(), ) @@ -605,6 +643,7 @@ async def create_block_model( initial_data: Table | None = None, units: dict[str, str] | None = None, tags: dict[str, dict[str, Any]] | None = None, + column_groups: dict[str, str] | None = None, comment: str | None = None, fill_subblocks: bool = False, ) -> tuple[BlockModel, Version]: @@ -631,6 +670,9 @@ async def create_block_model( :param units: A dictionary mapping column names within `initial_data` to units. :param tags: A dictionary mapping column names within `initial_data` to their tags object. Column tags are a preview feature; the client must be constructed with ``preview=True`` to use them. + :param column_groups: A dictionary mapping column names within `initial_data` to the qualified title of the + group the column should be placed in. Column groups are a preview feature; the client must be constructed + with ``preview=True`` to use them. :param comment: An optional comment describing the initial data. :param fill_subblocks: Sets the default fill_subblocks behaviour for this block model. If ``True``, updates to a fully sub-blocked model with ``update_type``=``merge`` and ``geometry_change``=``True`` will fill any missing @@ -641,6 +683,8 @@ async def create_block_model( raise ValueError("units can only be provided if initial_data is provided") if tags is not None and initial_data is None: raise ValueError("tags can only be provided if initial_data is provided") + if column_groups is not None and initial_data is None: + raise ValueError("column_groups can only be provided if initial_data is provided") if initial_data is not None and self._cache is None: raise CacheNotConfiguredException( "Cache must be configured to use this method. Please set the 'cache' parameter in the constructor." @@ -661,7 +705,9 @@ async def create_block_model( geometry_change = True else: geometry_change = None - version = await self._add_new_columns(create_result.bm_uuid, initial_data, units, geometry_change, tags) + version = await self._add_new_columns( + create_result.bm_uuid, initial_data, units, geometry_change, tags, column_groups + ) return self._bm_from_model(create_result), version async def add_new_subblocked_columns( @@ -670,6 +716,7 @@ async def add_new_subblocked_columns( data: Table, units: dict[str, str] | None = None, tags: dict[str, dict[str, Any]] | None = None, + column_groups: dict[str, str] | None = None, ) -> Version: """Add new columns to an existing sub-blocked block model. This will not change the sub-blocking structure, thus the provided data must match existing sub-blocks in the model. @@ -682,10 +729,15 @@ async def add_new_subblocked_columns( :param units: A dictionary mapping column names within `data` to units. :param tags: A dictionary mapping column names within `data` to their tags object. Column tags are a preview feature; the client must be constructed with ``preview=True`` to use them. + :param column_groups: A dictionary mapping column names within `data` to the qualified title of the group the + column should be placed in. Column groups are a preview feature; the client must be constructed with + ``preview=True`` to use them. :raises CacheNotConfiguredException: If the cache is not configured. :return: The new version of the block model with the added columns. """ - return await self._add_new_columns(bm_id, data, units, geometry_change=False, tags=tags) + return await self._add_new_columns( + bm_id, data, units, geometry_change=False, tags=tags, column_groups=column_groups + ) async def _add_new_columns( self, @@ -694,6 +746,7 @@ async def _add_new_columns( units: dict[str, str] | None = None, geometry_change: bool | None = None, tags: dict[str, dict[str, Any]] | None = None, + column_groups: dict[str, str] | None = None, ) -> Version: """Add new columns to an existing block model. @@ -709,6 +762,9 @@ async def _add_new_columns( :param geometry_change: Whether the geometry of the block model is changing. :param tags: A dictionary mapping column names within `data` to their tags object. Column tags are a preview feature; the client must be constructed with ``preview=True`` to use them. + :param column_groups: A dictionary mapping column names within `data` to the qualified title of the group the + column should be placed in. Column groups are a preview feature; the client must be constructed with + ``preview=True`` to use them. :raises CacheNotConfiguredException: If the cache is not configured. :return: The new version of the block model with the added columns. """ @@ -722,6 +778,8 @@ async def _add_new_columns( units = {} if tags is None: tags = {} + if column_groups is None: + column_groups = {} new_column_names = {name for name in schema.names if name not in _GEOMETRY_COLUMNS} unknown_unit_columns = set(units) - new_column_names if unknown_unit_columns: @@ -729,6 +787,11 @@ async def _add_new_columns( unknown_tag_columns = set(tags) - new_column_names if unknown_tag_columns: raise MissingColumnInTable(f"tags reference columns that are not being added: {unknown_tag_columns}") + unknown_group_columns = set(column_groups) - new_column_names + if unknown_group_columns: + raise MissingColumnInTable( + f"column_groups reference columns that are not being added: {unknown_group_columns}" + ) columns = models.UpdateColumnsLite( new=[ models.ColumnLite( @@ -736,6 +799,7 @@ async def _add_new_columns( data_type=convert_dtype(data_type), unit_id=units.get(name), **({"tags": tags[name]} if name in tags else {}), + **({"group": column_groups[name]} if name in column_groups else {}), ) for name, data_type in zip(schema.names, schema.types) if name not in _GEOMETRY_COLUMNS @@ -765,6 +829,7 @@ async def add_new_columns( data: Table, units: dict[str, str] | None = None, tags: dict[str, dict[str, Any]] | None = None, + column_groups: dict[str, str] | None = None, ) -> Version: """Add new columns to an existing regular block model. @@ -777,10 +842,15 @@ async def add_new_columns( :param units: A dictionary mapping column names within `data` to units. :param tags: A dictionary mapping column names within `data` to their tags object. Column tags are a preview feature; the client must be constructed with ``preview=True`` to use them. + :param column_groups: A dictionary mapping column names within `data` to the qualified title of the group the + column should be placed in. Column groups are a preview feature; the client must be constructed with + ``preview=True`` to use them. :raises CacheNotConfiguredException: If the cache is not configured. :return: The new version of the block model with the added columns. """ - return await self._add_new_columns(bm_id, data, units, geometry_change=None, tags=tags) + return await self._add_new_columns( + bm_id, data, units, geometry_change=None, tags=tags, column_groups=column_groups + ) async def _update_columns( self, @@ -793,6 +863,7 @@ async def _update_columns( geometry_change: bool | None = None, fill_subblocks: bool | None = None, tags: dict[str, dict[str, Any]] | None = None, + column_groups: dict[str, str] | None = None, update_type: models.UpdateType = models.UpdateType.replace, ) -> Version: if self._cache is None: @@ -805,6 +876,8 @@ async def _update_columns( units = {} if tags is None: tags = {} + if column_groups is None: + column_groups = {} data_type_map = {name: data_type for name, data_type in zip(schema.names, schema.types)} if update_columns is None: @@ -832,6 +905,13 @@ async def _update_columns( "To tag existing columns, use update_column_metadata." ) + unknown_group_columns = set(column_groups) - set(new_columns) + if unknown_group_columns: + raise MissingColumnInTable( + f"column_groups reference columns that are not in new_columns: {unknown_group_columns}. " + "To move existing columns to a group, use update_column_metadata." + ) + columns = models.UpdateColumnsLite( new=[ models.ColumnLite( @@ -839,6 +919,7 @@ async def _update_columns( data_type=convert_dtype(data_type_map[new_column]), unit_id=units.get(new_column), **({"tags": tags[new_column]} if new_column in tags else {}), + **({"group": column_groups[new_column]} if new_column in column_groups else {}), ) for new_column in new_columns ], @@ -871,6 +952,7 @@ async def update_block_model_columns( delete_columns: set[str] | None = None, units: dict[str, str] | None = None, tags: dict[str, dict[str, Any]] | None = None, + column_groups: dict[str, str] | None = None, update_type: models.UpdateType = models.UpdateType.replace, ) -> Version: """Add, update, or delete regular block model columns. @@ -887,6 +969,9 @@ async def update_block_model_columns( :param units: A dictionary mapping column names within `data` to units. :param tags: A dictionary mapping new column names to their tags object. Column tags are a preview feature; the client must be constructed with ``preview=True`` to use them. + :param column_groups: A dictionary mapping new column names to the qualified title of the group the column + should be placed in. To move an existing column to a group, use :meth:`update_column_metadata`. Column + groups are a preview feature; the client must be constructed with ``preview=True`` to use them. :param: update_type: Provide the type of update. Either 'replace' or 'merge' (default: replace) :raises CacheNotConfiguredException: If the cache is not configured. :return: The new version of the block model with the added columns. @@ -900,6 +985,7 @@ async def update_block_model_columns( units, geometry_change=None, tags=tags, + column_groups=column_groups, update_type=update_type, ) @@ -914,6 +1000,7 @@ async def update_subblocked_columns( geometry_change: bool = False, fill_subblocks: bool | None = None, tags: dict[str, dict[str, Any]] | None = None, + column_groups: dict[str, str] | None = None, update_type: models.UpdateType = models.UpdateType.replace, ) -> Version: """Add, update, or delete sub-blocked block model columns. @@ -939,6 +1026,9 @@ async def update_subblocked_columns( the block model's own ``fill_subblocks`` setting is used. :param tags: A dictionary mapping new column names to their tags object. Column tags are a preview feature; the client must be constructed with ``preview=True`` to use them. + :param column_groups: A dictionary mapping new column names to the qualified title of the group the column + should be placed in. To move an existing column to a group, use :meth:`update_column_metadata`. Column + groups are a preview feature; the client must be constructed with ``preview=True`` to use them. :param: update_type: Provide the type of update. Either 'replace' or 'merge' (default: replace) """ return await self._update_columns( @@ -951,6 +1041,7 @@ async def update_subblocked_columns( geometry_change=geometry_change, fill_subblocks=fill_subblocks, tags=tags, + column_groups=column_groups, update_type=update_type, ) @@ -960,7 +1051,7 @@ async def update_column_metadata( column_updates: dict[str, str | None | ColumnMetadataUpdate], comment: str | None = None, ) -> Version: - """Update metadata (e.g., units and tags) for existing block model columns. + """Update metadata (e.g., units, tags and group) for existing block model columns. This method updates column properties without requiring data upload or cache configuration. @@ -968,16 +1059,17 @@ async def update_column_metadata( - A ``str`` sets the column's unit ID. - ``None`` clears the column's unit ID. - - A :class:`ColumnMetadataUpdate` sets any combination of unit ID and/or tags. Only the + - A :class:`ColumnMetadataUpdate` sets any combination of unit ID, tags and/or group. Only the fields explicitly set on the object are sent; unset fields are left untouched. Set - ``tags={}`` to clear a column's tags, or ``unit_id=None`` to clear its unit. + ``tags={}`` to clear a column's tags, ``unit_id=None`` to clear its unit, ``group=""`` to + ungroup the column, or ``group=""`` to move it to a different group. - Column tags are a preview feature; the client must be constructed with ``preview=True`` to use them. + Column tags and groups are a preview feature; the client must be constructed with ``preview=True`` to use them. :param bm_id: The ID of the block model to update. :param column_updates: A dictionary mapping column titles to their metadata update. Example: {"Cu": "%[mass]", "Au": None, - "Ag": ColumnMetadataUpdate(tags={"source": "assay"})} + "Ag": ColumnMetadataUpdate(tags={"source": "assay"}, group="Assays")} :param comment: An optional comment describing the metadata changes. This is max 250 characters. :return: The new version of the block model with updated metadata. """ @@ -1003,6 +1095,51 @@ def _to_values(value: str | None | ColumnMetadataUpdate) -> models.UpdateMetadat return await self._update_model_no_data(bm_id, columns, comment=comment) + async def update_groups( + self, + bm_id: UUID, + *, + new: list[GroupDefinition] | None = None, + update: dict[str, GroupMetadataUpdate] | None = None, + delete: list[str] | None = None, + comment: str | None = None, + ) -> Version: + """Create, update, and/or delete column groups on a block model. + + This method manages group definitions without requiring data upload or cache configuration. Any + combination of ``new``, ``update`` and ``delete`` can be supplied in a single call. + + Groups are addressed by their qualified title (a bare title for a top-level group, or segments + joined by ``▸`` for a nested group). To assign columns to a group, use the ``column_groups`` + parameter on the column methods (for new columns) or :meth:`update_column_metadata` (to move an + existing column). To resolve a written group back to its server-assigned UUID and resolved policy, + use the helpers on the returned :class:`~evo.blockmodels.data.Version`, e.g. + :meth:`~evo.blockmodels.data.Version.group_by_qualified_title`. + + Column groups are a preview feature; the client must be constructed with ``preview=True`` to use them. + + :param bm_id: The ID of the block model to update. + :param new: Definitions of new groups to create. + :param update: A dictionary mapping the qualified title of an existing group to the metadata update + to apply to it. Use :class:`GroupMetadataUpdate` to rename, re-parent, change the missing-column + policy, replace tags, or toggle the hidden flag. + :param delete: Qualified titles of groups to delete. + :param comment: An optional comment describing the changes. This is max 250 characters. + :return: The new version of the block model with the updated groups. + """ + if not new and not update and not delete: + raise ValueError("At least one of 'new', 'update' or 'delete' must be provided.") + + columns = models.UpdateColumnsLite( + new=[], + update=[], + delete=[], + rename=[], + ) + groups = _build_update_groups_lite(new, update, delete) + + return await self._update_model_no_data(bm_id, columns, comment=comment, groups=groups) + async def rename_block_model_columns( self, bm_id: UUID, diff --git a/packages/evo-blockmodels/src/evo/blockmodels/data.py b/packages/evo-blockmodels/src/evo/blockmodels/data.py index c09e9473..29727255 100644 --- a/packages/evo-blockmodels/src/evo/blockmodels/data.py +++ b/packages/evo-blockmodels/src/evo/blockmodels/data.py @@ -24,26 +24,34 @@ Column, ListingColumn, ListingGroup, + MissingColumnPolicy, ResolvedGroup, RotationAxis, ) __all__ = [ + "QUALIFIED_TITLE_SEPARATOR", "BaseGridDefinition", "BlockModel", "Column", "ColumnMetadataUpdate", "FlexibleGridDefinition", "FullySubBlockedGridDefinition", + "GroupDefinition", + "GroupMetadataUpdate", "ListingColumn", "ListingGroup", "ListingVersion", + "MissingColumnPolicy", "OctreeGridDefinition", "RegularGridDefinition", "ResolvedGroup", "Version", ] +QUALIFIED_TITLE_SEPARATOR = "\u25b8" +"""Default single-character separator (``▸``) used to build and parse qualified group titles.""" + class ColumnMetadataUpdate(CustomBaseModel): """A per-column metadata update for :meth:`BlockModelAPIClient.update_column_metadata`. @@ -60,6 +68,66 @@ class ColumnMetadataUpdate(CustomBaseModel): Column tags are a preview feature; the client must be constructed with ``preview=True`` to use them.""" + group: str | None = None + """Move the column to a different group, identified by its qualified group title (a bare title for a + top-level group, or segments joined by ``▸`` for a nested group). Send an empty string (``""``) to + ungroup the column. Omit this field to leave the column's group unchanged. + + Column groups are a preview feature; the client must be constructed with ``preview=True`` to use them.""" + + +class GroupDefinition(CustomBaseModel): + """Definition of a new column group to create via :meth:`BlockModelAPIClient.update_groups`. + + Column groups are a preview feature; the client must be constructed with ``preview=True`` to use them. + """ + + title: str + """Human-readable label for the group, unique across its siblings. Must not contain the qualified + title separator ``▸``.""" + + parent_group: str | None = None + """Qualified title of the parent group (a bare title for a top-level parent, or segments joined by + ``▸`` for a nested parent). ``None`` (the default) makes this a top-level group.""" + + missing_column_policy: MissingColumnPolicy | None = None + """Policy for columns in this group's zone that are absent from an update. ``None`` (the default) + lets the service apply its default of ``INHERIT``.""" + + tags: dict[str, Any] | None = None + """Publisher-supplied free-form metadata for the group.""" + + is_hidden: bool = False + """When ``True``, the group's direct member columns are excluded from wildcard queries unless + ``include_hidden`` is set.""" + + +class GroupMetadataUpdate(CustomBaseModel): + """A metadata update for an existing column group, used by :meth:`BlockModelAPIClient.update_groups`. + + Only the fields you explicitly set are sent to the service; unset fields are left unchanged. + + Column groups are a preview feature; the client must be constructed with ``preview=True`` to use them. + """ + + new_title: str | None = None + """Rename the group to this title. Omit to leave the title unchanged.""" + + parent_group: str | None = None + """Re-parent the group. Provide the qualified title of the new parent group, or an empty string + (``""``) to make it a top-level group. Omit to leave the parent unchanged.""" + + missing_column_policy: MissingColumnPolicy | None = None + """Set the group's missing-column policy. Use ``INHERIT`` to adopt the parent chain's policy zone. + Omit to leave the policy unchanged.""" + + tags: dict[str, Any] | None = None + """Replacement tags for the group. Send a populated object to replace the group's tags wholesale, + or ``{}`` to clear them. Omit to leave the existing tags untouched.""" + + is_hidden: bool | None = None + """Set the group's hidden flag. Omit to leave it unchanged.""" + @dataclass(frozen=True, kw_only=True) class BaseGridDefinition: @@ -279,6 +347,68 @@ def __repr__(self) -> str: f"columns={col_names})" ) + def group_by_uuid(self, group_uuid: UUID) -> "ResolvedGroup | ListingGroup | None": + """Return the group on this version with the given UUID, or ``None`` if there is no match.""" + for group in self.groups: + if group.group_uuid == group_uuid: + return group + return None + + def group_for_column(self, column: "Column | ListingColumn | str") -> "ResolvedGroup | ListingGroup | None": + """Resolve the group a column belongs to. + + This bridges the read path (columns reference their group by UUID) so a caller can get the + group's title, parent and resolved missing-column policy without hand-rolling a lookup. + + :param column: A column from this version, or a column title. + :return: The column's group, or ``None`` if the column is ungrouped or unknown. + """ + if isinstance(column, str): + column = next((c for c in self.columns if c.title == column), None) + if column is None: + return None + if column.group_uuid is None: + return None + return self.group_by_uuid(column.group_uuid) + + def qualified_group_title( + self, group: "ResolvedGroup | ListingGroup", separator: str = QUALIFIED_TITLE_SEPARATOR + ) -> str: + """Build the qualified (``▸``-joined) title of a group by walking its parent chain. + + :param group: A group from this version. + :param separator: Separator used to join the path segments. + :return: The fully-qualified group title. + """ + titles = [group.title] + seen = {group.group_uuid} + parent_uuid = group.parent_group_uuid + while parent_uuid is not None and parent_uuid not in seen: + parent = self.group_by_uuid(parent_uuid) + if parent is None: + break + titles.append(parent.title) + seen.add(parent.group_uuid) + parent_uuid = parent.parent_group_uuid + return separator.join(reversed(titles)) + + def group_by_qualified_title( + self, qualified_title: str, separator: str = QUALIFIED_TITLE_SEPARATOR + ) -> "ResolvedGroup | ListingGroup | None": + """Find a group by the qualified title it was created with. + + This bridges the write/read asymmetry: groups are written by title but read back by UUID, so a + caller who created a group by title can find it again on the returned version. + + :param qualified_title: A bare title for a top-level group, or ``▸``-joined segments for a nested group. + :param separator: Separator used to parse and rebuild qualified titles. + :return: The matching group, or ``None`` if there is no match. + """ + for group in self.groups: + if self.qualified_group_title(group, separator) == qualified_title: + return group + return None + @dataclass(frozen=True, kw_only=True, repr=False) class Version(_VersionBase): diff --git a/packages/evo-blockmodels/tests/test_group_helpers.py b/packages/evo-blockmodels/tests/test_group_helpers.py new file mode 100644 index 00000000..8c64b049 --- /dev/null +++ b/packages/evo-blockmodels/tests/test_group_helpers.py @@ -0,0 +1,127 @@ +# Copyright © 2025 Bentley Systems, Incorporated +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import unittest +import uuid +from datetime import datetime, timezone + +from evo.blockmodels.data import QUALIFIED_TITLE_SEPARATOR, MissingColumnPolicy, Version +from evo.blockmodels.endpoints import models +from evo.common import ServiceUser + +DATE = datetime(2021, 1, 1, tzinfo=timezone.utc) +USER = ServiceUser.from_model(models.IMSUserInfo(email="test@test.com", name="Test User", id=uuid.uuid4())) + +PARENT_UUID = uuid.uuid4() +CHILD_UUID = uuid.uuid4() +COL_IN_CHILD = uuid.uuid4() + + +def _resolved_group(group_uuid, title, parent_group_uuid=None): + return models.ResolvedGroup( + group_uuid=group_uuid, + title=title, + parent_group_uuid=parent_group_uuid, + missing_column_policy=MissingColumnPolicy.SET_NULL, + resolved_missing_column_policy=MissingColumnPolicy.SET_NULL, + tags={"source": "assay"}, + ) + + +def _make_version() -> Version: + return Version( + bm_uuid=uuid.uuid4(), + version_id=2, + version_uuid=uuid.uuid4(), + parent_version_id=1, + base_version_id=1, + geoscience_version_id="3", + created_at=DATE, + created_by=USER, + comment="", + columns=[ + models.Column( + col_id=str(COL_IN_CHILD), title="Cu", data_type=models.DataType.Float64, group_uuid=CHILD_UUID + ), + models.Column(col_id=str(uuid.uuid4()), title="Au", data_type=models.DataType.Float64), + ], + groups=[ + _resolved_group(PARENT_UUID, "Assays"), + _resolved_group(CHILD_UUID, "Primary", parent_group_uuid=PARENT_UUID), + ], + ) + + +class TestVersionGroupHelpers(unittest.TestCase): + def setUp(self) -> None: + self.version = _make_version() + + def test_group_by_uuid(self) -> None: + group = self.version.group_by_uuid(CHILD_UUID) + self.assertIsNotNone(group) + self.assertEqual(group.title, "Primary") + self.assertIsNone(self.version.group_by_uuid(uuid.uuid4())) + + def test_qualified_group_title_nested(self) -> None: + child = self.version.group_by_uuid(CHILD_UUID) + self.assertEqual( + self.version.qualified_group_title(child), + f"Assays{QUALIFIED_TITLE_SEPARATOR}Primary", + ) + + def test_qualified_group_title_top_level(self) -> None: + parent = self.version.group_by_uuid(PARENT_UUID) + self.assertEqual(self.version.qualified_group_title(parent), "Assays") + + def test_qualified_group_title_custom_separator(self) -> None: + child = self.version.group_by_uuid(CHILD_UUID) + self.assertEqual(self.version.qualified_group_title(child, separator="/"), "Assays/Primary") + + def test_group_by_qualified_title_roundtrip(self) -> None: + # A caller who created "Assays▸Primary" by title finds it again by its qualified title. + group = self.version.group_by_qualified_title(f"Assays{QUALIFIED_TITLE_SEPARATOR}Primary") + self.assertIsNotNone(group) + self.assertEqual(group.group_uuid, CHILD_UUID) + self.assertIsNone(self.version.group_by_qualified_title("Missing")) + + def test_group_for_column_by_object(self) -> None: + column = self.version.columns[0] + group = self.version.group_for_column(column) + self.assertIsNotNone(group) + self.assertEqual(group.group_uuid, CHILD_UUID) + self.assertEqual(group.resolved_missing_column_policy, MissingColumnPolicy.SET_NULL) + + def test_group_for_column_by_title(self) -> None: + group = self.version.group_for_column("Cu") + self.assertIsNotNone(group) + self.assertEqual(group.group_uuid, CHILD_UUID) + + def test_group_for_ungrouped_column(self) -> None: + self.assertIsNone(self.version.group_for_column("Au")) + + def test_group_for_unknown_column(self) -> None: + self.assertIsNone(self.version.group_for_column("does_not_exist")) + + def test_qualified_group_title_tolerates_broken_parent_chain(self) -> None: + # A dangling parent reference must not loop or raise; it just stops walking. + orphan = models.ResolvedGroup( + group_uuid=uuid.uuid4(), + title="Orphan", + parent_group_uuid=uuid.uuid4(), + missing_column_policy=MissingColumnPolicy.INHERIT, + resolved_missing_column_policy=MissingColumnPolicy.USE_PREVIOUS, + ) + object.__setattr__(self.version, "groups", [*self.version.groups, orphan]) + self.assertEqual(self.version.qualified_group_title(orphan), "Orphan") + + +if __name__ == "__main__": + unittest.main() diff --git a/packages/evo-blockmodels/tests/test_update.py b/packages/evo-blockmodels/tests/test_update.py index d7f42670..33b87902 100644 --- a/packages/evo-blockmodels/tests/test_update.py +++ b/packages/evo-blockmodels/tests/test_update.py @@ -19,7 +19,7 @@ from parameterized import parameterized from evo.blockmodels import BlockModelAPIClient -from evo.blockmodels.data import ColumnMetadataUpdate +from evo.blockmodels.data import ColumnMetadataUpdate, GroupDefinition, GroupMetadataUpdate, MissingColumnPolicy from evo.blockmodels.endpoints import models from evo.blockmodels.endpoints.models import JobResponse, JobStatus from evo.blockmodels.exceptions import CacheNotConfiguredException, JobFailedException, MissingColumnInTable @@ -141,6 +141,12 @@ def setUp(self) -> None: environment=self.environment, preview=True, ) + self.preview_client = BlockModelAPIClient( + connector=self.connector, + environment=self.environment, + cache=self.cache, + preview=True, + ) self.setup_universal_headers(get_header_metadata(BlockModelAPIClient.__module__)) @property @@ -1038,3 +1044,251 @@ async def test_delete_block_model_columns_job_failed(self) -> None: ) with self.assertRaises(JobFailedException): await self.bms_client_without_cache.delete_block_model_columns(BM_UUID, ["col1"]) + + async def test_add_new_columns_with_column_groups(self) -> None: + """New columns can be assigned to a group via ``column_groups``.""" + self.transport.set_request_handler( + UpdateRequestHandler( + update_result=UPDATE_RESULT, + job_response=JobResponse(job_status=JobStatus.COMPLETE, payload=UPDATED_VERSION), + ) + ) + with mock.patch("evo.common.io.upload.StorageDestination") as mock_destination: + mock_destination.upload_file = mock.AsyncMock() + await self.preview_client.add_new_columns( + BM_UUID, + REGULAR_DATA, + column_groups={"col2": "Assays"}, + ) + mock_destination.upload_file.assert_called_once() + + # col2 is placed in the "Assays" group; col1 stays ungrouped so has no group field on the wire. + expected_update_body = models.UpdateDataLite1( + columns=models.UpdateColumnsLite( + new=[ + models.ColumnLite(title="col1", data_type=models.DataType.Utf8, unit_id=None), + models.ColumnLite( + title="col2", data_type=models.DataType.Float64, unit_id=None, group="Assays" + ), + ], + update=[], + rename=[], + delete=[], + ), + update_type=models.UpdateType.replace, + geometry_change=None, + ) + self.assert_any_request_made( + method=RequestMethod.PATCH, + path=f"{self.base_path}/block-models/{BM_UUID}/blocks", + body=expected_update_body.model_dump(mode="json", exclude_unset=True), + headers=DEFAULT_EXPECTED_HEADERS | {"API-Preview": "opt-in"}, + ) + + async def test_add_new_columns_with_unknown_group_column(self) -> None: + with self.assertRaises(MissingColumnInTable): + await self.bms_client.add_new_columns( + BM_UUID, + REGULAR_DATA, + column_groups={"does_not_exist": "Assays"}, + ) + + async def test_update_block_model_columns_with_column_groups(self) -> None: + """New columns added through update_block_model_columns can be grouped.""" + self.transport.set_request_handler( + UpdateRequestHandler( + update_result=UPDATE_RESULT, + job_response=JobResponse(job_status=JobStatus.COMPLETE, payload=UPDATED_VERSION), + ) + ) + with mock.patch("evo.common.io.upload.StorageDestination") as mock_destination: + mock_destination.upload_file = mock.AsyncMock() + await self.bms_client.update_block_model_columns( + BM_UUID, + REGULAR_DATA, + new_columns=["col1", "col2"], + column_groups={"col1": "Assays\u25b8Primary"}, + ) + + expected_update_body = models.UpdateDataLite1( + columns=models.UpdateColumnsLite( + new=[ + models.ColumnLite( + title="col1", + data_type=models.DataType.Utf8, + unit_id=None, + group="Assays\u25b8Primary", + ), + models.ColumnLite(title="col2", data_type=models.DataType.Float64, unit_id=None), + ], + update=[], + rename=[], + delete=[], + ), + update_type=models.UpdateType.replace, + geometry_change=None, + fill_subblocks=None, + ) + self.assert_any_request_made( + method=RequestMethod.PATCH, + path=f"{self.base_path}/block-models/{BM_UUID}/blocks", + body=expected_update_body.model_dump(mode="json", exclude_unset=True), + headers=DEFAULT_EXPECTED_HEADERS, + ) + + async def test_update_columns_with_unknown_group_column(self) -> None: + with self.assertRaises(MissingColumnInTable): + await self.bms_client.update_block_model_columns( + BM_UUID, + REGULAR_DATA, + new_columns=["col1"], + column_groups={"col2": "Assays"}, # col2 not in new_columns + ) + + async def test_update_column_metadata_moves_column_group(self) -> None: + """An existing column can be moved to another group (or ungrouped) via update_column_metadata.""" + self.transport.set_request_handler( + UpdateRequestHandler( + update_result=UPDATE_RESULT, + job_response=JobResponse(job_status=JobStatus.COMPLETE, payload=UPDATED_VERSION), + ) + ) + await self.preview_client_without_cache.update_column_metadata( + BM_UUID, + column_updates={ + "col1": ColumnMetadataUpdate(group="Assays"), # move to a group + "col2": ColumnMetadataUpdate(group=""), # ungroup + }, + ) + + expected_update_body = models.UpdateDataLite1( + columns=models.UpdateColumnsLite( + new=[], + update=[], + rename=[], + delete=[], + update_metadata=[ + models.UpdateMetadataLite(title="col1", values=models.UpdateMetadataValuesLite(group="Assays")), + models.UpdateMetadataLite(title="col2", values=models.UpdateMetadataValuesLite(group="")), + ], + ), + comment=None, + ) + self.assert_any_request_made( + method=RequestMethod.PATCH, + path=f"{self.base_path}/block-models/{BM_UUID}/blocks", + body=expected_update_body.model_dump(mode="json", exclude_unset=True), + headers=DEFAULT_EXPECTED_HEADERS | {"API-Preview": "opt-in"}, + ) + + async def test_update_groups_create(self) -> None: + self.transport.set_request_handler( + UpdateRequestHandler( + update_result=UPDATE_RESULT, + job_response=JobResponse(job_status=JobStatus.COMPLETE, payload=UPDATED_VERSION), + ) + ) + version = await self.preview_client_without_cache.update_groups( + BM_UUID, + new=[ + GroupDefinition(title="Assays", missing_column_policy=MissingColumnPolicy.SET_NULL), + GroupDefinition(title="Primary", parent_group="Assays", is_hidden=True), + ], + ) + + expected_update_body = models.UpdateDataLite1( + columns=models.UpdateColumnsLite(new=[], update=[], delete=[], rename=[]), + comment=None, + ) + expected_update_body.groups = models.UpdateGroupsLite( + new=[ + models.GroupLite(title="Assays", missing_column_policy=MissingColumnPolicy.SET_NULL), + models.GroupLite(title="Primary", parent_group="Assays", is_hidden=True), + ], + update_metadata=[], + delete=[], + ) + self.assert_any_request_made( + method=RequestMethod.PATCH, + path=f"{self.base_path}/block-models/{BM_UUID}/blocks", + body=expected_update_body.model_dump(mode="json", exclude_unset=True), + headers=DEFAULT_EXPECTED_HEADERS | {"API-Preview": "opt-in"}, + ) + self.assertEqual(version.bm_uuid, BM_UUID) + self.assertEqual(version.version_id, 2) + + async def test_update_groups_update_metadata_and_delete(self) -> None: + self.transport.set_request_handler( + UpdateRequestHandler( + update_result=UPDATE_RESULT, + job_response=JobResponse(job_status=JobStatus.COMPLETE, payload=UPDATED_VERSION), + ) + ) + await self.preview_client_without_cache.update_groups( + BM_UUID, + update={ + "Assays": GroupMetadataUpdate( + new_title="Assay Results", missing_column_policy=MissingColumnPolicy.INHERIT + ), + "Waste": GroupMetadataUpdate(parent_group="", tags={}), + }, + delete=["Old Group"], + comment="Reorganise groups", + ) + + expected_update_body = models.UpdateDataLite1( + columns=models.UpdateColumnsLite(new=[], update=[], delete=[], rename=[]), + comment="Reorganise groups", + ) + expected_update_body.groups = models.UpdateGroupsLite( + new=[], + update_metadata=[ + models.GroupUpdateMetadataLite( + title="Assays", + values=models.GroupUpdateMetadataValuesLite( + title="Assay Results", missing_column_policy=MissingColumnPolicy.INHERIT + ), + ), + models.GroupUpdateMetadataLite( + title="Waste", + values=models.GroupUpdateMetadataValuesLite(parent_group="", tags={}), + ), + ], + delete=["Old Group"], + ) + self.assert_any_request_made( + method=RequestMethod.PATCH, + path=f"{self.base_path}/block-models/{BM_UUID}/blocks", + body=expected_update_body.model_dump(mode="json", exclude_unset=True), + headers=DEFAULT_EXPECTED_HEADERS | {"API-Preview": "opt-in"}, + ) + + async def test_update_groups_requires_an_operation(self) -> None: + with self.assertRaises(ValueError): + await self.preview_client_without_cache.update_groups(BM_UUID) + + async def test_update_groups_job_failed(self) -> None: + self.transport.set_request_handler( + UpdateRequestHandler( + update_result=UPDATE_RESULT, + job_response=JobResponse( + job_status=JobStatus.FAILED, + payload=models.JobErrorPayload( + detail="Group update failed", + status=500, + title="Group update failed", + type="https://seequent.com/error-codes/block-model-service/job/internal-error", + ), + ), + ) + ) + with self.assertRaises(JobFailedException): + await self.preview_client_without_cache.update_groups(BM_UUID, delete=["Assays"]) + + async def test_create_block_model_column_groups_requires_initial_data(self) -> None: + with self.assertRaises(ValueError): + await self.bms_client.create_block_model( + name="test", + grid_definition=mock.MagicMock(), + column_groups={"col1": "Assays"}, + ) From 5b4f0e7d2efb26946c2e3de1a9bbfbff4ee21a00 Mon Sep 17 00:00:00 2001 From: Andre Lobato <255781626+AndreLobatoSeequent@users.noreply.github.com> Date: Mon, 31 Aug 2026 16:10:58 +1200 Subject: [PATCH 2/8] Fix arg order, removed create_block_models with groups --- .../src/evo/blockmodels/client.py | 19 ++++++++---------- .../src/evo/blockmodels/data.py | 6 ++++++ .../tests/test_group_helpers.py | 20 +++++++++++++++++++ packages/evo-blockmodels/tests/test_update.py | 8 -------- 4 files changed, 34 insertions(+), 19 deletions(-) diff --git a/packages/evo-blockmodels/src/evo/blockmodels/client.py b/packages/evo-blockmodels/src/evo/blockmodels/client.py index 3b0d0e32..346c887c 100644 --- a/packages/evo-blockmodels/src/evo/blockmodels/client.py +++ b/packages/evo-blockmodels/src/evo/blockmodels/client.py @@ -643,7 +643,6 @@ async def create_block_model( initial_data: Table | None = None, units: dict[str, str] | None = None, tags: dict[str, dict[str, Any]] | None = None, - column_groups: dict[str, str] | None = None, comment: str | None = None, fill_subblocks: bool = False, ) -> tuple[BlockModel, Version]: @@ -670,21 +669,21 @@ async def create_block_model( :param units: A dictionary mapping column names within `initial_data` to units. :param tags: A dictionary mapping column names within `initial_data` to their tags object. Column tags are a preview feature; the client must be constructed with ``preview=True`` to use them. - :param column_groups: A dictionary mapping column names within `initial_data` to the qualified title of the - group the column should be placed in. Column groups are a preview feature; the client must be constructed - with ``preview=True`` to use them. :param comment: An optional comment describing the initial data. :param fill_subblocks: Sets the default fill_subblocks behaviour for this block model. If ``True``, updates to a fully sub-blocked model with ``update_type``=``merge`` and ``geometry_change``=``True`` will fill any missing sub-blocks with data from the parent block. Defaults to ``False``. :return: A tuple containing the created block model and the version of the block model. + + .. note:: + To place columns in a group, first create the model, define the groups with :meth:`update_groups`, + then add the columns with ``column_groups`` on :meth:`add_new_columns` / + :meth:`update_block_model_columns`. Groups cannot be referenced during creation because none exist yet. """ if units is not None and initial_data is None: raise ValueError("units can only be provided if initial_data is provided") if tags is not None and initial_data is None: raise ValueError("tags can only be provided if initial_data is provided") - if column_groups is not None and initial_data is None: - raise ValueError("column_groups can only be provided if initial_data is provided") if initial_data is not None and self._cache is None: raise CacheNotConfiguredException( "Cache must be configured to use this method. Please set the 'cache' parameter in the constructor." @@ -705,9 +704,7 @@ async def create_block_model( geometry_change = True else: geometry_change = None - version = await self._add_new_columns( - create_result.bm_uuid, initial_data, units, geometry_change, tags, column_groups - ) + version = await self._add_new_columns(create_result.bm_uuid, initial_data, units, geometry_change, tags) return self._bm_from_model(create_result), version async def add_new_subblocked_columns( @@ -952,8 +949,8 @@ async def update_block_model_columns( delete_columns: set[str] | None = None, units: dict[str, str] | None = None, tags: dict[str, dict[str, Any]] | None = None, - column_groups: dict[str, str] | None = None, update_type: models.UpdateType = models.UpdateType.replace, + column_groups: dict[str, str] | None = None, ) -> Version: """Add, update, or delete regular block model columns. @@ -1000,8 +997,8 @@ async def update_subblocked_columns( geometry_change: bool = False, fill_subblocks: bool | None = None, tags: dict[str, dict[str, Any]] | None = None, - column_groups: dict[str, str] | None = None, update_type: models.UpdateType = models.UpdateType.replace, + column_groups: dict[str, str] | None = None, ) -> Version: """Add, update, or delete sub-blocked block model columns. diff --git a/packages/evo-blockmodels/src/evo/blockmodels/data.py b/packages/evo-blockmodels/src/evo/blockmodels/data.py index 29727255..63e12a31 100644 --- a/packages/evo-blockmodels/src/evo/blockmodels/data.py +++ b/packages/evo-blockmodels/src/evo/blockmodels/data.py @@ -14,6 +14,8 @@ from typing import Any from uuid import UUID +from pydantic import ConfigDict + from evo.common import ResourceMetadata from evo.workspaces import ServiceUser @@ -82,6 +84,8 @@ class GroupDefinition(CustomBaseModel): Column groups are a preview feature; the client must be constructed with ``preview=True`` to use them. """ + model_config = ConfigDict(extra="forbid", protected_namespaces=()) + title: str """Human-readable label for the group, unique across its siblings. Must not contain the qualified title separator ``▸``.""" @@ -110,6 +114,8 @@ class GroupMetadataUpdate(CustomBaseModel): Column groups are a preview feature; the client must be constructed with ``preview=True`` to use them. """ + model_config = ConfigDict(extra="forbid", protected_namespaces=()) + new_title: str | None = None """Rename the group to this title. Omit to leave the title unchanged.""" diff --git a/packages/evo-blockmodels/tests/test_group_helpers.py b/packages/evo-blockmodels/tests/test_group_helpers.py index 8c64b049..6cf220c9 100644 --- a/packages/evo-blockmodels/tests/test_group_helpers.py +++ b/packages/evo-blockmodels/tests/test_group_helpers.py @@ -125,3 +125,23 @@ def test_qualified_group_title_tolerates_broken_parent_chain(self) -> None: if __name__ == "__main__": unittest.main() + + +class TestGroupInputModelsRejectExtras(unittest.TestCase): + def test_group_definition_forbids_unknown_fields(self) -> None: + from pydantic import ValidationError + + from evo.blockmodels.data import GroupDefinition + + with self.assertRaises(ValidationError): + GroupDefinition(title="Assays", parnet_group="typo") + + def test_group_metadata_update_forbids_title_field(self) -> None: + # The wire rename field is ``new_title``; passing ``title`` (the wire name) must be rejected + # so it can't silently collide with the new_title -> title remap. + from pydantic import ValidationError + + from evo.blockmodels.data import GroupMetadataUpdate + + with self.assertRaises(ValidationError): + GroupMetadataUpdate(title="X") diff --git a/packages/evo-blockmodels/tests/test_update.py b/packages/evo-blockmodels/tests/test_update.py index 33b87902..3cedfb44 100644 --- a/packages/evo-blockmodels/tests/test_update.py +++ b/packages/evo-blockmodels/tests/test_update.py @@ -1284,11 +1284,3 @@ async def test_update_groups_job_failed(self) -> None: ) with self.assertRaises(JobFailedException): await self.preview_client_without_cache.update_groups(BM_UUID, delete=["Assays"]) - - async def test_create_block_model_column_groups_requires_initial_data(self) -> None: - with self.assertRaises(ValueError): - await self.bms_client.create_block_model( - name="test", - grid_definition=mock.MagicMock(), - column_groups={"col1": "Assays"}, - ) From 21bd593f11712d548ca09c4315d118359fe4c82f Mon Sep 17 00:00:00 2001 From: Andre Lobato <255781626+AndreLobatoSeequent@users.noreply.github.com> Date: Tue, 1 Sep 2026 11:37:20 +1200 Subject: [PATCH 3/8] Groups moving requires data upload, add qualified titles columns helper --- .../src/evo/blockmodels/client.py | 90 +++++++++--- .../src/evo/blockmodels/data.py | 9 +- .../tests/test_group_helpers.py | 10 ++ packages/evo-blockmodels/tests/test_update.py | 136 ++++++++++++++---- 4 files changed, 191 insertions(+), 54 deletions(-) diff --git a/packages/evo-blockmodels/src/evo/blockmodels/client.py b/packages/evo-blockmodels/src/evo/blockmodels/client.py index 346c887c..6cf652ef 100644 --- a/packages/evo-blockmodels/src/evo/blockmodels/client.py +++ b/packages/evo-blockmodels/src/evo/blockmodels/client.py @@ -25,6 +25,9 @@ from ._types import Table from ._utils import convert_dtype, extract_payload +from .data import ( + QUALIFIED_TITLE_SEPARATOR as _QUALIFIED_TITLE_SEPARATOR, +) from .data import ( BaseGridDefinition, BlockModel, @@ -171,6 +174,33 @@ def _build_update_groups_lite( ) +def _qualified_heading(group: str | None, title: str) -> str: + """Return the parquet upload heading the lite ingest expects for a column. + + A grouped column must be uploaded under its fully-qualified title (``group▸…▸leaf``); an + ungrouped column (no group, or ``group == ""``) keeps its bare title. The service binds + uploaded data to a column by matching this heading, so it must be applied to the data table + before upload or the column's data is silently discarded. + """ + if group: + return f"{group}{_QUALIFIED_TITLE_SEPARATOR}{title}" + return title + + +def _apply_group_headings(data: Table, column_groups: dict[str, str]) -> Table: + """Rename grouped columns in ``data`` to the qualified upload headings the service expects. + + ``column_groups`` maps a bare column title to the qualified title of the group the column is + being placed in (or ``""`` to ungroup). Columns absent from the mapping are left untouched. + """ + if not column_groups: + return data + new_names = [ + _qualified_heading(column_groups[name], name) if name in column_groups else name for name in data.schema.names + ] + return data.rename_columns(new_names) + + class BlockModelAPIClient(BaseAPIClient): def __init__( self, @@ -805,6 +835,8 @@ async def _add_new_columns( delete=[], rename=[], ) + # Grouped columns must be uploaded under their qualified heading, or the service drops their data. + data = _apply_group_headings(data, column_groups) update_response = await self._column_operations_api.update_block_model_from_latest_version( org_id=str(self._environment.org_id), workspace_id=str(self._environment.workspace_id), @@ -902,13 +934,25 @@ async def _update_columns( "To tag existing columns, use update_column_metadata." ) - unknown_group_columns = set(column_groups) - set(new_columns) + # A group assignment requires the column's data to be (re-)uploaded in this same request, so + # column_groups may only reference new columns or existing columns being updated. Moving a + # column without its data is rejected by the service, so there is no metadata-only path. + movable_columns = set(new_columns) | update_columns + unknown_group_columns = set(column_groups) - movable_columns if unknown_group_columns: raise MissingColumnInTable( - f"column_groups reference columns that are not in new_columns: {unknown_group_columns}. " - "To move existing columns to a group, use update_column_metadata." + f"column_groups reference columns that are neither new nor being updated: {unknown_group_columns}. " + "A column's group can only be changed when its data is re-uploaded, so the column must be listed " + "in new_columns or update_columns." ) + # Existing columns are moved by pairing their re-uploaded data with an update_metadata group change. + update_metadata_entries = [ + models.UpdateMetadataLite(title=column, values=models.UpdateMetadataValuesLite(group=group)) + for column, group in column_groups.items() + if column in update_columns + ] + columns = models.UpdateColumnsLite( new=[ models.ColumnLite( @@ -924,6 +968,10 @@ async def _update_columns( delete=list(delete_columns), rename=[], ) + if update_metadata_entries: + columns.update_metadata = update_metadata_entries + # Grouped columns must be uploaded under their qualified heading, or the service drops their data. + data = _apply_group_headings(data, column_groups) update_response = await self._column_operations_api.update_block_model_from_latest_version( org_id=str(self._environment.org_id), workspace_id=str(self._environment.workspace_id), @@ -966,9 +1014,11 @@ async def update_block_model_columns( :param units: A dictionary mapping column names within `data` to units. :param tags: A dictionary mapping new column names to their tags object. Column tags are a preview feature; the client must be constructed with ``preview=True`` to use them. - :param column_groups: A dictionary mapping new column names to the qualified title of the group the column - should be placed in. To move an existing column to a group, use :meth:`update_column_metadata`. Column - groups are a preview feature; the client must be constructed with ``preview=True`` to use them. + :param column_groups: A dictionary mapping column names to the qualified title of the group the column + should be placed in (or ``""`` to move it out of any group). New columns are grouped as they are added. + To move an *existing* column, it must also be listed in ``update_columns`` so its data is re-uploaded in + the same request. Column groups are a preview feature; the client must be constructed with + ``preview=True`` to use them. :param: update_type: Provide the type of update. Either 'replace' or 'merge' (default: replace) :raises CacheNotConfiguredException: If the cache is not configured. :return: The new version of the block model with the added columns. @@ -1023,9 +1073,11 @@ async def update_subblocked_columns( the block model's own ``fill_subblocks`` setting is used. :param tags: A dictionary mapping new column names to their tags object. Column tags are a preview feature; the client must be constructed with ``preview=True`` to use them. - :param column_groups: A dictionary mapping new column names to the qualified title of the group the column - should be placed in. To move an existing column to a group, use :meth:`update_column_metadata`. Column - groups are a preview feature; the client must be constructed with ``preview=True`` to use them. + :param column_groups: A dictionary mapping column names to the qualified title of the group the column + should be placed in (or ``""`` to move it out of any group). New columns are grouped as they are added. + To move an *existing* column, it must also be listed in ``update_columns`` so its data is re-uploaded in + the same request. Column groups are a preview feature; the client must be constructed with + ``preview=True`` to use them. :param: update_type: Provide the type of update. Either 'replace' or 'merge' (default: replace) """ return await self._update_columns( @@ -1048,7 +1100,7 @@ async def update_column_metadata( column_updates: dict[str, str | None | ColumnMetadataUpdate], comment: str | None = None, ) -> Version: - """Update metadata (e.g., units, tags and group) for existing block model columns. + """Update metadata (e.g., units and tags) for existing block model columns. This method updates column properties without requiring data upload or cache configuration. @@ -1056,17 +1108,21 @@ async def update_column_metadata( - A ``str`` sets the column's unit ID. - ``None`` clears the column's unit ID. - - A :class:`ColumnMetadataUpdate` sets any combination of unit ID, tags and/or group. Only the + - A :class:`ColumnMetadataUpdate` sets any combination of unit ID and/or tags. Only the fields explicitly set on the object are sent; unset fields are left untouched. Set - ``tags={}`` to clear a column's tags, ``unit_id=None`` to clear its unit, ``group=""`` to - ungroup the column, or ``group=""`` to move it to a different group. + ``tags={}`` to clear a column's tags or ``unit_id=None`` to clear its unit. + + A column's group cannot be changed here: the service requires the column's data to be re-uploaded + when its group membership changes. Use the ``column_groups`` parameter on + :meth:`update_block_model_columns` / :meth:`update_subblocked_columns` (listing the column in + ``update_columns``) to move an existing column. - Column tags and groups are a preview feature; the client must be constructed with ``preview=True`` to use them. + Column tags are a preview feature; the client must be constructed with ``preview=True`` to use them. :param bm_id: The ID of the block model to update. :param column_updates: A dictionary mapping column titles to their metadata update. Example: {"Cu": "%[mass]", "Au": None, - "Ag": ColumnMetadataUpdate(tags={"source": "assay"}, group="Assays")} + "Ag": ColumnMetadataUpdate(tags={"source": "assay"})} :param comment: An optional comment describing the metadata changes. This is max 250 characters. :return: The new version of the block model with updated metadata. """ @@ -1108,8 +1164,8 @@ async def update_groups( Groups are addressed by their qualified title (a bare title for a top-level group, or segments joined by ``▸`` for a nested group). To assign columns to a group, use the ``column_groups`` - parameter on the column methods (for new columns) or :meth:`update_column_metadata` (to move an - existing column). To resolve a written group back to its server-assigned UUID and resolved policy, + parameter on the column methods (for new columns, or existing columns whose data is re-uploaded + in the same call). To resolve a written group back to its server-assigned UUID and resolved policy, use the helpers on the returned :class:`~evo.blockmodels.data.Version`, e.g. :meth:`~evo.blockmodels.data.Version.group_by_qualified_title`. diff --git a/packages/evo-blockmodels/src/evo/blockmodels/data.py b/packages/evo-blockmodels/src/evo/blockmodels/data.py index 63e12a31..d968ec00 100644 --- a/packages/evo-blockmodels/src/evo/blockmodels/data.py +++ b/packages/evo-blockmodels/src/evo/blockmodels/data.py @@ -61,6 +61,8 @@ class ColumnMetadataUpdate(CustomBaseModel): Only the fields you explicitly set are sent to the service; unset fields are left untouched. """ + model_config = ConfigDict(extra="forbid", protected_namespaces=()) + unit_id: str | None = None """The new unit ID for the column.""" @@ -70,13 +72,6 @@ class ColumnMetadataUpdate(CustomBaseModel): Column tags are a preview feature; the client must be constructed with ``preview=True`` to use them.""" - group: str | None = None - """Move the column to a different group, identified by its qualified group title (a bare title for a - top-level group, or segments joined by ``▸`` for a nested group). Send an empty string (``""``) to - ungroup the column. Omit this field to leave the column's group unchanged. - - Column groups are a preview feature; the client must be constructed with ``preview=True`` to use them.""" - class GroupDefinition(CustomBaseModel): """Definition of a new column group to create via :meth:`BlockModelAPIClient.update_groups`. diff --git a/packages/evo-blockmodels/tests/test_group_helpers.py b/packages/evo-blockmodels/tests/test_group_helpers.py index 6cf220c9..fe5fc575 100644 --- a/packages/evo-blockmodels/tests/test_group_helpers.py +++ b/packages/evo-blockmodels/tests/test_group_helpers.py @@ -145,3 +145,13 @@ def test_group_metadata_update_forbids_title_field(self) -> None: with self.assertRaises(ValidationError): GroupMetadataUpdate(title="X") + + def test_column_metadata_update_forbids_group_field(self) -> None: + # A column's group can only change when its data is re-uploaded, so metadata-only group moves + # are not supported. ``group`` must be rejected rather than silently forwarded onto the wire. + from pydantic import ValidationError + + from evo.blockmodels.data import ColumnMetadataUpdate + + with self.assertRaises(ValidationError): + ColumnMetadataUpdate(group="Assays") diff --git a/packages/evo-blockmodels/tests/test_update.py b/packages/evo-blockmodels/tests/test_update.py index 3cedfb44..1980a307 100644 --- a/packages/evo-blockmodels/tests/test_update.py +++ b/packages/evo-blockmodels/tests/test_update.py @@ -16,6 +16,7 @@ from unittest import mock import pyarrow +import pyarrow.parquet from parameterized import parameterized from evo.blockmodels import BlockModelAPIClient @@ -1053,7 +1054,10 @@ async def test_add_new_columns_with_column_groups(self) -> None: job_response=JobResponse(job_status=JobStatus.COMPLETE, payload=UPDATED_VERSION), ) ) - with mock.patch("evo.common.io.upload.StorageDestination") as mock_destination: + with ( + mock.patch("evo.common.io.upload.StorageDestination") as mock_destination, + mock.patch("pyarrow.parquet.write_table", wraps=pyarrow.parquet.write_table) as mock_write, + ): mock_destination.upload_file = mock.AsyncMock() await self.preview_client.add_new_columns( BM_UUID, @@ -1085,6 +1089,11 @@ async def test_add_new_columns_with_column_groups(self) -> None: headers=DEFAULT_EXPECTED_HEADERS | {"API-Preview": "opt-in"}, ) + # The uploaded data must carry the grouped column under its qualified heading, or the service + # would silently drop its data. Ungrouped columns keep their bare heading. + uploaded_table = mock_write.call_args.args[0] + self.assertEqual(uploaded_table.schema.names, ["i", "j", "k", "col1", "Assays\u25b8col2"]) + async def test_add_new_columns_with_unknown_group_column(self) -> None: with self.assertRaises(MissingColumnInTable): await self.bms_client.add_new_columns( @@ -1101,7 +1110,10 @@ async def test_update_block_model_columns_with_column_groups(self) -> None: job_response=JobResponse(job_status=JobStatus.COMPLETE, payload=UPDATED_VERSION), ) ) - with mock.patch("evo.common.io.upload.StorageDestination") as mock_destination: + with ( + mock.patch("evo.common.io.upload.StorageDestination") as mock_destination, + mock.patch("pyarrow.parquet.write_table", wraps=pyarrow.parquet.write_table) as mock_write, + ): mock_destination.upload_file = mock.AsyncMock() await self.bms_client.update_block_model_columns( BM_UUID, @@ -1136,50 +1148,114 @@ async def test_update_block_model_columns_with_column_groups(self) -> None: headers=DEFAULT_EXPECTED_HEADERS, ) + uploaded_table = mock_write.call_args.args[0] + self.assertEqual(uploaded_table.schema.names, ["i", "j", "k", "Assays\u25b8Primary\u25b8col1", "col2"]) + async def test_update_columns_with_unknown_group_column(self) -> None: with self.assertRaises(MissingColumnInTable): await self.bms_client.update_block_model_columns( BM_UUID, REGULAR_DATA, new_columns=["col1"], - column_groups={"col2": "Assays"}, # col2 not in new_columns + column_groups={"col2": "Assays"}, # col2 neither new nor updated ) - async def test_update_column_metadata_moves_column_group(self) -> None: - """An existing column can be moved to another group (or ungrouped) via update_column_metadata.""" + async def test_update_block_model_columns_moves_existing_column_group(self) -> None: + """An existing column is moved to another group by re-uploading its data with a group change.""" self.transport.set_request_handler( UpdateRequestHandler( update_result=UPDATE_RESULT, job_response=JobResponse(job_status=JobStatus.COMPLETE, payload=UPDATED_VERSION), ) ) - await self.preview_client_without_cache.update_column_metadata( - BM_UUID, - column_updates={ - "col1": ColumnMetadataUpdate(group="Assays"), # move to a group - "col2": ColumnMetadataUpdate(group=""), # ungroup - }, - ) + with ( + mock.patch("evo.common.io.upload.StorageDestination") as mock_destination, + mock.patch("pyarrow.parquet.write_table", wraps=pyarrow.parquet.write_table) as mock_write, + ): + mock_destination.upload_file = mock.AsyncMock() + await self.bms_client.update_block_model_columns( + BM_UUID, + REGULAR_DATA, + new_columns=[], + update_columns={"col1"}, + column_groups={"col1": "Assays"}, # move existing col1 into a group + ) - expected_update_body = models.UpdateDataLite1( - columns=models.UpdateColumnsLite( - new=[], - update=[], - rename=[], - delete=[], - update_metadata=[ - models.UpdateMetadataLite(title="col1", values=models.UpdateMetadataValuesLite(group="Assays")), - models.UpdateMetadataLite(title="col2", values=models.UpdateMetadataValuesLite(group="")), - ], - ), - comment=None, - ) - self.assert_any_request_made( - method=RequestMethod.PATCH, - path=f"{self.base_path}/block-models/{BM_UUID}/blocks", - body=expected_update_body.model_dump(mode="json", exclude_unset=True), - headers=DEFAULT_EXPECTED_HEADERS | {"API-Preview": "opt-in"}, + expected_update_body = models.UpdateDataLite1( + columns=models.UpdateColumnsLite( + new=[], + update=["col1"], + rename=[], + delete=[], + update_metadata=[ + models.UpdateMetadataLite(title="col1", values=models.UpdateMetadataValuesLite(group="Assays")), + ], + ), + update_type=models.UpdateType.replace, + geometry_change=None, + fill_subblocks=None, + ) + self.assert_any_request_made( + method=RequestMethod.PATCH, + path=f"{self.base_path}/block-models/{BM_UUID}/blocks", + body=expected_update_body.model_dump(mode="json", exclude_unset=True), + headers=DEFAULT_EXPECTED_HEADERS, + ) + + uploaded_table = mock_write.call_args.args[0] + # col1 is uploaded under its new qualified heading so the service binds its re-uploaded data. + self.assertEqual(uploaded_table.schema.names, ["i", "j", "k", "Assays\u25b8col1", "col2"]) + + async def test_update_block_model_columns_ungroups_existing_column(self) -> None: + """An existing grouped column is moved out of its group by re-uploading its data with ``group=""``.""" + self.transport.set_request_handler( + UpdateRequestHandler( + update_result=UPDATE_RESULT, + job_response=JobResponse(job_status=JobStatus.COMPLETE, payload=UPDATED_VERSION), + ) ) + with ( + mock.patch("evo.common.io.upload.StorageDestination") as mock_destination, + mock.patch("pyarrow.parquet.write_table", wraps=pyarrow.parquet.write_table) as mock_write, + ): + mock_destination.upload_file = mock.AsyncMock() + await self.bms_client.update_block_model_columns( + BM_UUID, + REGULAR_DATA, + new_columns=[], + update_columns={"col2"}, + column_groups={"col2": ""}, # ungroup existing col2 + ) + + expected_update_body = models.UpdateDataLite1( + columns=models.UpdateColumnsLite( + new=[], + update=["col2"], + rename=[], + delete=[], + update_metadata=[ + models.UpdateMetadataLite(title="col2", values=models.UpdateMetadataValuesLite(group="")), + ], + ), + update_type=models.UpdateType.replace, + geometry_change=None, + fill_subblocks=None, + ) + self.assert_any_request_made( + method=RequestMethod.PATCH, + path=f"{self.base_path}/block-models/{BM_UUID}/blocks", + body=expected_update_body.model_dump(mode="json", exclude_unset=True), + headers=DEFAULT_EXPECTED_HEADERS, + ) + + uploaded_table = mock_write.call_args.args[0] + # Ungrouped column keeps its bare heading. + self.assertEqual(uploaded_table.schema.names, ["i", "j", "k", "col1", "col2"]) + + async def test_column_metadata_update_rejects_group(self) -> None: + """Group membership can no longer be changed through metadata; the service requires re-uploaded data.""" + with self.assertRaises(ValueError): + ColumnMetadataUpdate(group="Assays") async def test_update_groups_create(self) -> None: self.transport.set_request_handler( From 3248f85868ce5e10cf3172ba688492daca2a5dbc Mon Sep 17 00:00:00 2001 From: Andre Lobato <255781626+AndreLobatoSeequent@users.noreply.github.com> Date: Tue, 1 Sep 2026 13:45:38 +1200 Subject: [PATCH 4/8] Mirror service contract for column-group writes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Make the block model column-group API reflect the service instead of adding a bare-title abstraction layer: - Callers key `data` by each column's exact upload heading (bare for ungrouped, qualified `group▸leaf` for grouped); the SDK uploads the table verbatim and never renames columns. - `column_groups` maps a new column's qualified heading → group, or an existing column's current qualified title → new group (`""` ungroups). - `update_columns` / `delete_columns` use the column's current stored title (qualified if grouped, bare otherwise). Removed the hidden latest-version fetch that translated bare → qualified titles. - Add opt-in `qualified_heading` and `qualify_headings` helpers (and a `Table.rename_columns` protocol method) to build qualified headings from bare-titled data, re-exported from the package root. - Update docstrings and rewrite/extend tests for the new contract. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../src/evo/blockmodels/__init__.py | 4 + .../src/evo/blockmodels/_types.py | 4 + .../src/evo/blockmodels/client.py | 150 ++++++++------ .../src/evo/blockmodels/data.py | 56 ++++++ .../tests/test_group_helpers.py | 52 +++++ packages/evo-blockmodels/tests/test_update.py | 185 ++++++++++++++++-- 6 files changed, 375 insertions(+), 76 deletions(-) diff --git a/packages/evo-blockmodels/src/evo/blockmodels/__init__.py b/packages/evo-blockmodels/src/evo/blockmodels/__init__.py index d0d4d103..6906aaec 100644 --- a/packages/evo-blockmodels/src/evo/blockmodels/__init__.py +++ b/packages/evo-blockmodels/src/evo/blockmodels/__init__.py @@ -10,7 +10,11 @@ # limitations under the License. from .client import BlockModelAPIClient +from .data import QUALIFIED_TITLE_SEPARATOR, qualified_heading, qualify_headings __all__ = [ + "QUALIFIED_TITLE_SEPARATOR", "BlockModelAPIClient", + "qualified_heading", + "qualify_headings", ] diff --git a/packages/evo-blockmodels/src/evo/blockmodels/_types.py b/packages/evo-blockmodels/src/evo/blockmodels/_types.py index 00e5cc55..6a13687e 100644 --- a/packages/evo-blockmodels/src/evo/blockmodels/_types.py +++ b/packages/evo-blockmodels/src/evo/blockmodels/_types.py @@ -74,6 +74,10 @@ def num_rows(self) -> int: def to_pandas(self) -> DataFrame: """Convert to a pandas-compatible NumPy array or DataFrame, as appropriate""" + def rename_columns(self, names: list[str]) -> "Table": + """Return a copy of this table with its columns renamed to ``names`` (positional).""" + ... + class DataFrame(Protocol): """Pandas DataFrame. diff --git a/packages/evo-blockmodels/src/evo/blockmodels/client.py b/packages/evo-blockmodels/src/evo/blockmodels/client.py index 6cf652ef..f5024d32 100644 --- a/packages/evo-blockmodels/src/evo/blockmodels/client.py +++ b/packages/evo-blockmodels/src/evo/blockmodels/client.py @@ -42,6 +42,9 @@ SubBlockedGridDefinition, Version, ) +from .data import ( + qualified_heading as _qualified_heading, +) from .endpoints import models from .endpoints.api import ColumnOperationsApi, JobsApi, MetadataApi, OperationsApi, ReportsApi, VersionsApi from .endpoints.models import ( @@ -174,31 +177,22 @@ def _build_update_groups_lite( ) -def _qualified_heading(group: str | None, title: str) -> str: - """Return the parquet upload heading the lite ingest expects for a column. - - A grouped column must be uploaded under its fully-qualified title (``group▸…▸leaf``); an - ungrouped column (no group, or ``group == ""``) keeps its bare title. The service binds - uploaded data to a column by matching this heading, so it must be applied to the data table - before upload or the column's data is silently discarded. - """ - if group: - return f"{group}{_QUALIFIED_TITLE_SEPARATOR}{title}" - return title - - -def _apply_group_headings(data: Table, column_groups: dict[str, str]) -> Table: - """Rename grouped columns in ``data`` to the qualified upload headings the service expects. +def _leaf_title(heading: str, group: str | None) -> str: + """Recover a column's bare (leaf) title from its qualified upload heading. - ``column_groups`` maps a bare column title to the qualified title of the group the column is - being placed in (or ``""`` to ungroup). Columns absent from the mapping are left untouched. + ``heading`` is the fully-qualified upload heading (e.g. ``Assays▸Cu``); ``group`` is the qualified + group path it should belong to (e.g. ``Assays``). Stripping the ``group▸`` prefix yields the leaf + title the service stores. An ungrouped column (no group) has a bare heading, so it is returned as-is. """ - if not column_groups: - return data - new_names = [ - _qualified_heading(column_groups[name], name) if name in column_groups else name for name in data.schema.names - ] - return data.rename_columns(new_names) + if not group: + return heading + prefix = f"{group}{_QUALIFIED_TITLE_SEPARATOR}" + if not heading.startswith(prefix): + raise MissingColumnInTable( + f"column '{heading}' is declared in group '{group}' but its data heading is not the qualified " + f"title '{prefix}'. Key the data by its exact upload heading (see qualify_headings)." + ) + return heading[len(prefix) :] class BlockModelAPIClient(BaseAPIClient): @@ -752,12 +746,16 @@ async def add_new_subblocked_columns( This method requires the `pyarrow` package to be installed, and the 'cache' parameter to be set in the constructor. :param bm_id: The ID of the block model to add columns to. - :param data: The data containing the new columns to add. + :param data: The data containing the new columns to add, keyed by each column's exact upload heading + (a bare title for an ungrouped column, or the qualified ``group▸leaf`` title for a grouped one). :param units: A dictionary mapping column names within `data` to units. :param tags: A dictionary mapping column names within `data` to their tags object. Column tags are a preview feature; the client must be constructed with ``preview=True`` to use them. - :param column_groups: A dictionary mapping column names within `data` to the qualified title of the group the - column should be placed in. Column groups are a preview feature; the client must be constructed with + :param column_groups: A dictionary mapping a grouped column's qualified upload heading (its key in + `data`, e.g. ``"Assays▸Cu"``) to the qualified title of the group it belongs to (e.g. ``"Assays"``). + Ungrouped columns are keyed by their bare title in `data` and omitted here. `data` must be keyed by each + column's exact upload heading; :func:`~evo.blockmodels.data.qualify_headings` can build that from + bare-titled data. Column groups are a preview feature; the client must be constructed with ``preview=True`` to use them. :raises CacheNotConfiguredException: If the cache is not configured. :return: The new version of the block model with the added columns. @@ -784,13 +782,17 @@ async def _add_new_columns( This method requires the `pyarrow` package to be installed, and the 'cache' parameter to be set in the constructor. :param bm_id: The ID of the block model to add columns to. - :param data: The data containing the new columns to add. + :param data: The data containing the new columns to add, keyed by each column's exact upload heading + (a bare title for an ungrouped column, or the qualified ``group▸leaf`` title for a grouped one). :param units: A dictionary mapping column names within `data` to units. :param geometry_change: Whether the geometry of the block model is changing. :param tags: A dictionary mapping column names within `data` to their tags object. Column tags are a preview feature; the client must be constructed with ``preview=True`` to use them. - :param column_groups: A dictionary mapping column names within `data` to the qualified title of the group the - column should be placed in. Column groups are a preview feature; the client must be constructed with + :param column_groups: A dictionary mapping a grouped column's qualified upload heading (its key in + `data`, e.g. ``"Assays▸Cu"``) to the qualified title of the group it belongs to (e.g. ``"Assays"``). + Ungrouped columns are keyed by their bare title in `data` and omitted here. `data` must be keyed by each + column's exact upload heading; :func:`~evo.blockmodels.data.qualify_headings` can build that from + bare-titled data. Column groups are a preview feature; the client must be constructed with ``preview=True`` to use them. :raises CacheNotConfiguredException: If the cache is not configured. :return: The new version of the block model with the added columns. @@ -822,7 +824,7 @@ async def _add_new_columns( columns = models.UpdateColumnsLite( new=[ models.ColumnLite( - title=name, + title=_leaf_title(name, column_groups.get(name)), data_type=convert_dtype(data_type), unit_id=units.get(name), **({"tags": tags[name]} if name in tags else {}), @@ -835,8 +837,6 @@ async def _add_new_columns( delete=[], rename=[], ) - # Grouped columns must be uploaded under their qualified heading, or the service drops their data. - data = _apply_group_headings(data, column_groups) update_response = await self._column_operations_api.update_block_model_from_latest_version( org_id=str(self._environment.org_id), workspace_id=str(self._environment.workspace_id), @@ -867,12 +867,16 @@ async def add_new_columns( This method requires the `pyarrow` package to be installed, and the 'cache' parameter to be set in the constructor. :param bm_id: The ID of the block model to add columns to. - :param data: The data containing the new columns to add. + :param data: The data containing the new columns to add, keyed by each column's exact upload heading + (a bare title for an ungrouped column, or the qualified ``group▸leaf`` title for a grouped one). :param units: A dictionary mapping column names within `data` to units. :param tags: A dictionary mapping column names within `data` to their tags object. Column tags are a preview feature; the client must be constructed with ``preview=True`` to use them. - :param column_groups: A dictionary mapping column names within `data` to the qualified title of the group the - column should be placed in. Column groups are a preview feature; the client must be constructed with + :param column_groups: A dictionary mapping a grouped column's qualified upload heading (its key in + `data`, e.g. ``"Assays▸Cu"``) to the qualified title of the group it belongs to (e.g. ``"Assays"``). + Ungrouped columns are keyed by their bare title in `data` and omitted here. `data` must be keyed by each + column's exact upload heading; :func:`~evo.blockmodels.data.qualify_headings` can build that from + bare-titled data. Column groups are a preview feature; the client must be constructed with ``preview=True`` to use them. :raises CacheNotConfiguredException: If the cache is not configured. :return: The new version of the block model with the added columns. @@ -915,10 +919,25 @@ async def _update_columns( if delete_columns is None: delete_columns = set() - # Check for any new or updated columns that are not in the data - missing = (set(new_columns) | update_columns) - data_type_map.keys() + # Existing columns are addressed by the exact title the service stores: a currently-grouped + # column by its qualified title (``group▸leaf``), an ungrouped column by its bare title. New + # columns are keyed in ``data`` by their upload heading. Work out the data heading each declared + # column should be found under, so we can validate the table without renaming it. + def _expected_data_heading(column: str) -> str: + if column in update_columns and column in column_groups: + # A move re-uploads the column's data under its NEW heading (new group + current leaf). + leaf = column.rsplit(_QUALIFIED_TITLE_SEPARATOR, 1)[-1] + return _qualified_heading(column_groups[column], leaf) + # New columns and plain data updates are uploaded under their own title/heading. + return column + + expected_headings = {_expected_data_heading(column) for column in (set(new_columns) | update_columns)} + missing = expected_headings - data_type_map.keys() if missing: - raise MissingColumnInTable(f"Columns {missing} are not present in the provided table.") + raise MissingColumnInTable( + f"Columns {missing} are not present in the provided table. Key the data by each column's " + "exact upload heading (qualified 'group▸leaf' for grouped columns, bare otherwise)." + ) unknown_unit_columns = set(units) - set(new_columns) if unknown_unit_columns: @@ -943,10 +962,11 @@ async def _update_columns( raise MissingColumnInTable( f"column_groups reference columns that are neither new nor being updated: {unknown_group_columns}. " "A column's group can only be changed when its data is re-uploaded, so the column must be listed " - "in new_columns or update_columns." + "in new_columns (as its qualified heading) or update_columns (as its current qualified title)." ) - # Existing columns are moved by pairing their re-uploaded data with an update_metadata group change. + # An existing column is moved by pairing its re-uploaded data with an update_metadata group change, + # addressed by the column's CURRENT qualified title (the same reference used in columns.update). update_metadata_entries = [ models.UpdateMetadataLite(title=column, values=models.UpdateMetadataValuesLite(group=group)) for column, group in column_groups.items() @@ -956,7 +976,7 @@ async def _update_columns( columns = models.UpdateColumnsLite( new=[ models.ColumnLite( - title=new_column, + title=_leaf_title(new_column, column_groups.get(new_column)), data_type=convert_dtype(data_type_map[new_column]), unit_id=units.get(new_column), **({"tags": tags[new_column]} if new_column in tags else {}), @@ -970,8 +990,6 @@ async def _update_columns( ) if update_metadata_entries: columns.update_metadata = update_metadata_entries - # Grouped columns must be uploaded under their qualified heading, or the service drops their data. - data = _apply_group_headings(data, column_groups) update_response = await self._column_operations_api.update_block_model_from_latest_version( org_id=str(self._environment.org_id), workspace_id=str(self._environment.workspace_id), @@ -1007,18 +1025,23 @@ async def update_block_model_columns( This method requires the `pyarrow` package to be installed, and the 'cache' parameter to be set in the constructor. :param bm_id: The ID of the block model to add columns to. - :param data: The data containing the new columns to add. - :param new_columns: A list of new column names to add to the block model. - :param update_columns: A set of column names to update in the block model. - :param delete_columns: A set of column names to delete from the block model. + :param data: The data containing the affected columns, keyed by each column's exact upload heading + (a bare title for an ungrouped column, or the qualified ``group▸leaf`` title for a grouped one). + :func:`~evo.blockmodels.data.qualify_headings` can build these headings from bare-titled data. + :param new_columns: A list of new columns to add, named by their upload heading in `data` (qualified + ``group▸leaf`` for a grouped column, bare otherwise). + :param update_columns: A set of existing columns to re-upload, each identified by the title the service + currently stores it under: its qualified title (``group▸leaf``) if grouped, or its bare title if not. + :param delete_columns: A set of existing columns to delete, identified the same way as ``update_columns`` + (qualified title if grouped, bare otherwise). :param units: A dictionary mapping column names within `data` to units. :param tags: A dictionary mapping new column names to their tags object. Column tags are a preview feature; the client must be constructed with ``preview=True`` to use them. - :param column_groups: A dictionary mapping column names to the qualified title of the group the column - should be placed in (or ``""`` to move it out of any group). New columns are grouped as they are added. - To move an *existing* column, it must also be listed in ``update_columns`` so its data is re-uploaded in - the same request. Column groups are a preview feature; the client must be constructed with - ``preview=True`` to use them. + :param column_groups: A dictionary assigning columns to groups. For a **new** column, map its qualified + upload heading (its key in `data`) to the group it belongs to. To **move** an *existing* column, map its + **current** qualified title to the new group (or ``""`` to ungroup); the column must also be listed in + ``update_columns`` and its data supplied under the **new** heading. Column groups are a preview feature; + the client must be constructed with ``preview=True`` to use them. :param: update_type: Provide the type of update. Either 'replace' or 'merge' (default: replace) :raises CacheNotConfiguredException: If the cache is not configured. :return: The new version of the block model with the added columns. @@ -1062,10 +1085,15 @@ async def update_subblocked_columns( This method requires the `pyarrow` package to be installed, and the 'cache' parameter to be set in the constructor. :param bm_id: The ID of the block model to add columns to. - :param data: The data containing the new columns to add. - :param new_columns: A list of new column names to add to the block model. - :param update_columns: A set of column names to update in the block model. - :param delete_columns: A set of column names to delete from the block model. + :param data: The data containing the affected columns, keyed by each column's exact upload heading + (a bare title for an ungrouped column, or the qualified ``group▸leaf`` title for a grouped one). + :func:`~evo.blockmodels.data.qualify_headings` can build these headings from bare-titled data. + :param new_columns: A list of new columns to add, named by their upload heading in `data` (qualified + ``group▸leaf`` for a grouped column, bare otherwise). + :param update_columns: A set of existing columns to re-upload, each identified by the title the service + currently stores it under: its qualified title (``group▸leaf``) if grouped, or its bare title if not. + :param delete_columns: A set of existing columns to delete, identified the same way as ``update_columns`` + (qualified title if grouped, bare otherwise). :param units: A dictionary mapping column names within `data` to units. :param geometry_change: Whether the geometry of the sub-blocked model changes. :param fill_subblocks: If ``True``, any missing sub-blocks will be filled with data from the parent block. @@ -1073,11 +1101,11 @@ async def update_subblocked_columns( the block model's own ``fill_subblocks`` setting is used. :param tags: A dictionary mapping new column names to their tags object. Column tags are a preview feature; the client must be constructed with ``preview=True`` to use them. - :param column_groups: A dictionary mapping column names to the qualified title of the group the column - should be placed in (or ``""`` to move it out of any group). New columns are grouped as they are added. - To move an *existing* column, it must also be listed in ``update_columns`` so its data is re-uploaded in - the same request. Column groups are a preview feature; the client must be constructed with - ``preview=True`` to use them. + :param column_groups: A dictionary assigning columns to groups. For a **new** column, map its qualified + upload heading (its key in `data`) to the group it belongs to. To **move** an *existing* column, map its + **current** qualified title to the new group (or ``""`` to ungroup); the column must also be listed in + ``update_columns`` and its data supplied under the **new** heading. Column groups are a preview feature; + the client must be constructed with ``preview=True`` to use them. :param: update_type: Provide the type of update. Either 'replace' or 'merge' (default: replace) """ return await self._update_columns( diff --git a/packages/evo-blockmodels/src/evo/blockmodels/data.py b/packages/evo-blockmodels/src/evo/blockmodels/data.py index d968ec00..82a221f9 100644 --- a/packages/evo-blockmodels/src/evo/blockmodels/data.py +++ b/packages/evo-blockmodels/src/evo/blockmodels/data.py @@ -20,6 +20,7 @@ from evo.workspaces import ServiceUser from ._model_config import CustomBaseModel +from ._types import Table from .endpoints.models import ( BBox, BBoxXYZ, @@ -49,12 +50,67 @@ "RegularGridDefinition", "ResolvedGroup", "Version", + "qualified_heading", + "qualify_headings", ] QUALIFIED_TITLE_SEPARATOR = "\u25b8" """Default single-character separator (``▸``) used to build and parse qualified group titles.""" +def qualified_heading(group: str | None, title: str, separator: str = QUALIFIED_TITLE_SEPARATOR) -> str: + """Build the upload heading the block model service expects for a column. + + A grouped column must be uploaded under its fully-qualified title (``group▸…▸leaf``); an + ungrouped column (no group, or ``group == ""``) keeps its bare title. + + :param group: The qualified title (path) of the column's group, or ``None``/``""`` if ungrouped. + :param title: The column's bare (leaf) title. + :param separator: Separator used to join the path segments. + :return: The qualified heading (``group▸title``) if grouped, otherwise ``title``. + """ + if group: + return f"{group}{separator}{title}" + return title + + +def qualify_headings( + data: Table, groups: dict[str, str], separator: str = QUALIFIED_TITLE_SEPARATOR +) -> tuple[Table, dict[str, str]]: + """Rename a table's columns to the qualified upload headings the service expects. + + The column methods on :class:`~evo.blockmodels.client.BlockModelAPIClient` expect ``data`` to be + keyed by each column's exact upload heading — a qualified ``group▸leaf`` title for a grouped column, + or the bare title for an ungrouped one — and do not rename anything for you. This opt-in helper + performs that shift: pass a table keyed by bare (leaf) titles plus a mapping of the columns you want + grouped, and it returns the renamed table alongside the ``column_groups`` mapping to hand back to the + client. + + :param data: A table keyed by bare (leaf) column titles. + :param groups: A mapping of a column's bare title to the qualified title of the group it should be + placed in. Columns absent from the mapping (or mapped to ``""``) are treated as ungrouped and + left with their bare heading. + :param separator: Separator used to build qualified titles. + :return: A ``(table, column_groups)`` pair: the table with grouped columns renamed to their qualified + heading, and a ``{qualified_heading: group}`` mapping to pass as ``column_groups``. + :raises KeyError: If ``groups`` references a column that is not present in ``data``. + """ + existing = set(data.schema.names) + unknown = set(groups) - existing + if unknown: + raise KeyError(f"groups reference columns that are not present in the table: {unknown}") + + new_names: list[str] = [] + column_groups: dict[str, str] = {} + for name in data.schema.names: + group = groups.get(name) + heading = qualified_heading(group, name, separator) + new_names.append(heading) + if group: + column_groups[heading] = group + return data.rename_columns(new_names), column_groups + + class ColumnMetadataUpdate(CustomBaseModel): """A per-column metadata update for :meth:`BlockModelAPIClient.update_column_metadata`. diff --git a/packages/evo-blockmodels/tests/test_group_helpers.py b/packages/evo-blockmodels/tests/test_group_helpers.py index fe5fc575..3d2cc1bb 100644 --- a/packages/evo-blockmodels/tests/test_group_helpers.py +++ b/packages/evo-blockmodels/tests/test_group_helpers.py @@ -155,3 +155,55 @@ def test_column_metadata_update_forbids_group_field(self) -> None: with self.assertRaises(ValidationError): ColumnMetadataUpdate(group="Assays") + + +class TestQualifyHeadings(unittest.TestCase): + def test_qualified_heading_builds_qualified_and_bare_titles(self) -> None: + from evo.blockmodels.data import qualified_heading + + self.assertEqual(qualified_heading("Assays", "Cu"), f"Assays{QUALIFIED_TITLE_SEPARATOR}Cu") + self.assertEqual( + qualified_heading("Assays\u25b8Primary", "Cu"), f"Assays\u25b8Primary{QUALIFIED_TITLE_SEPARATOR}Cu" + ) + # An ungrouped column keeps its bare title. + self.assertEqual(qualified_heading(None, "Cu"), "Cu") + self.assertEqual(qualified_heading("", "Cu"), "Cu") + + def test_qualify_headings_renames_and_builds_column_groups(self) -> None: + import pyarrow + + from evo.blockmodels.data import qualify_headings + + data = pyarrow.table({"i": [1], "Cu": [2.0], "Au": [3.0], "rock": ["x"]}) + renamed, column_groups = qualify_headings(data, {"Cu": "Assays", "Au": "Assays"}) + + self.assertEqual( + renamed.schema.names, + ["i", f"Assays{QUALIFIED_TITLE_SEPARATOR}Cu", f"Assays{QUALIFIED_TITLE_SEPARATOR}Au", "rock"], + ) + self.assertEqual( + column_groups, + {f"Assays{QUALIFIED_TITLE_SEPARATOR}Cu": "Assays", f"Assays{QUALIFIED_TITLE_SEPARATOR}Au": "Assays"}, + ) + # Untouched columns keep their bare heading and are absent from column_groups. + self.assertEqual(renamed.column("i").to_pylist(), [1]) + + def test_qualify_headings_treats_empty_group_as_ungrouped(self) -> None: + import pyarrow + + from evo.blockmodels.data import qualify_headings + + data = pyarrow.table({"Cu": [2.0]}) + renamed, column_groups = qualify_headings(data, {"Cu": ""}) + + self.assertEqual(renamed.schema.names, ["Cu"]) + self.assertEqual(column_groups, {}) + + def test_qualify_headings_rejects_unknown_columns(self) -> None: + import pyarrow + + from evo.blockmodels.data import qualify_headings + + data = pyarrow.table({"Cu": [2.0]}) + with self.assertRaises(KeyError): + qualify_headings(data, {"Au": "Assays"}) diff --git a/packages/evo-blockmodels/tests/test_update.py b/packages/evo-blockmodels/tests/test_update.py index 1980a307..d41a8017 100644 --- a/packages/evo-blockmodels/tests/test_update.py +++ b/packages/evo-blockmodels/tests/test_update.py @@ -1059,10 +1059,20 @@ async def test_add_new_columns_with_column_groups(self) -> None: mock.patch("pyarrow.parquet.write_table", wraps=pyarrow.parquet.write_table) as mock_write, ): mock_destination.upload_file = mock.AsyncMock() + # The caller keys the grouped column by its qualified upload heading; the SDK never renames data. + data = pyarrow.table( + { + "i": [1, 2, 3], + "j": [4, 5, 6], + "k": [7, 8, 9], + "col1": ["A", "B", "B"], + "Assays\u25b8col2": [4.5, 5.3, 6.2], + } + ) await self.preview_client.add_new_columns( BM_UUID, - REGULAR_DATA, - column_groups={"col2": "Assays"}, + data, + column_groups={"Assays\u25b8col2": "Assays"}, ) mock_destination.upload_file.assert_called_once() @@ -1089,8 +1099,8 @@ async def test_add_new_columns_with_column_groups(self) -> None: headers=DEFAULT_EXPECTED_HEADERS | {"API-Preview": "opt-in"}, ) - # The uploaded data must carry the grouped column under its qualified heading, or the service - # would silently drop its data. Ungrouped columns keep their bare heading. + # The uploaded data is untouched: the caller already keyed the grouped column by its + # qualified heading, and the ungrouped column keeps its bare heading. uploaded_table = mock_write.call_args.args[0] self.assertEqual(uploaded_table.schema.names, ["i", "j", "k", "col1", "Assays\u25b8col2"]) @@ -1115,11 +1125,21 @@ async def test_update_block_model_columns_with_column_groups(self) -> None: mock.patch("pyarrow.parquet.write_table", wraps=pyarrow.parquet.write_table) as mock_write, ): mock_destination.upload_file = mock.AsyncMock() + # New columns are named by their upload heading; the grouped one is qualified. + data = pyarrow.table( + { + "i": [1, 2, 3], + "j": [4, 5, 6], + "k": [7, 8, 9], + "Assays\u25b8Primary\u25b8col1": ["A", "B", "B"], + "col2": [4.5, 5.3, 6.2], + } + ) await self.bms_client.update_block_model_columns( BM_UUID, - REGULAR_DATA, - new_columns=["col1", "col2"], - column_groups={"col1": "Assays\u25b8Primary"}, + data, + new_columns=["Assays\u25b8Primary\u25b8col1", "col2"], + column_groups={"Assays\u25b8Primary\u25b8col1": "Assays\u25b8Primary"}, ) expected_update_body = models.UpdateDataLite1( @@ -1149,6 +1169,7 @@ async def test_update_block_model_columns_with_column_groups(self) -> None: ) uploaded_table = mock_write.call_args.args[0] + # Data is uploaded exactly as provided (already keyed by the qualified heading). self.assertEqual(uploaded_table.schema.names, ["i", "j", "k", "Assays\u25b8Primary\u25b8col1", "col2"]) async def test_update_columns_with_unknown_group_column(self) -> None: @@ -1173,11 +1194,21 @@ async def test_update_block_model_columns_moves_existing_column_group(self) -> N mock.patch("pyarrow.parquet.write_table", wraps=pyarrow.parquet.write_table) as mock_write, ): mock_destination.upload_file = mock.AsyncMock() + # col1 is currently ungrouped; its data is re-uploaded under its NEW qualified heading. + data = pyarrow.table( + { + "i": [1, 2, 3], + "j": [4, 5, 6], + "k": [7, 8, 9], + "Assays\u25b8col1": ["A", "B", "B"], + "col2": [4.5, 5.3, 6.2], + } + ) await self.bms_client.update_block_model_columns( BM_UUID, - REGULAR_DATA, + data, new_columns=[], - update_columns={"col1"}, + update_columns={"col1"}, # current (bare) title of the ungrouped column column_groups={"col1": "Assays"}, # move existing col1 into a group ) @@ -1219,22 +1250,35 @@ async def test_update_block_model_columns_ungroups_existing_column(self) -> None mock.patch("pyarrow.parquet.write_table", wraps=pyarrow.parquet.write_table) as mock_write, ): mock_destination.upload_file = mock.AsyncMock() + # col2 currently lives in "Assays"; ungrouping re-uploads its data under its NEW bare heading. + data = pyarrow.table( + { + "i": [1, 2, 3], + "j": [4, 5, 6], + "k": [7, 8, 9], + "col1": ["A", "B", "B"], + "col2": [4.5, 5.3, 6.2], + } + ) await self.bms_client.update_block_model_columns( BM_UUID, - REGULAR_DATA, + data, new_columns=[], - update_columns={"col2"}, - column_groups={"col2": ""}, # ungroup existing col2 + update_columns={"Assays\u25b8col2"}, # current qualified title of the grouped column + column_groups={"Assays\u25b8col2": ""}, # ungroup existing col2 ) + # col2 is currently grouped, so the service identifies it by its current qualified title. expected_update_body = models.UpdateDataLite1( columns=models.UpdateColumnsLite( new=[], - update=["col2"], + update=["Assays\u25b8col2"], rename=[], delete=[], update_metadata=[ - models.UpdateMetadataLite(title="col2", values=models.UpdateMetadataValuesLite(group="")), + models.UpdateMetadataLite( + title="Assays\u25b8col2", values=models.UpdateMetadataValuesLite(group="") + ), ], ), update_type=models.UpdateType.replace, @@ -1249,9 +1293,120 @@ async def test_update_block_model_columns_ungroups_existing_column(self) -> None ) uploaded_table = mock_write.call_args.args[0] - # Ungrouped column keeps its bare heading. + # The now-ungrouped column is uploaded under its new bare heading. self.assertEqual(uploaded_table.schema.names, ["i", "j", "k", "col1", "col2"]) + async def test_update_block_model_columns_moves_currently_grouped_column(self) -> None: + """An already-grouped column is moved to another group, referenced by its current qualified title.""" + self.transport.set_request_handler( + UpdateRequestHandler( + update_result=UPDATE_RESULT, + job_response=JobResponse(job_status=JobStatus.COMPLETE, payload=UPDATED_VERSION), + ) + ) + with ( + mock.patch("evo.common.io.upload.StorageDestination") as mock_destination, + mock.patch("pyarrow.parquet.write_table", wraps=pyarrow.parquet.write_table) as mock_write, + ): + mock_destination.upload_file = mock.AsyncMock() + # col1 currently lives in "Assays"; moving it re-uploads its data under its NEW qualified heading. + data = pyarrow.table( + { + "i": [1, 2, 3], + "j": [4, 5, 6], + "k": [7, 8, 9], + "Geology\u25b8col1": ["A", "B", "B"], + "col2": [4.5, 5.3, 6.2], + } + ) + await self.bms_client.update_block_model_columns( + BM_UUID, + data, + new_columns=[], + update_columns={"Assays\u25b8col1"}, # current qualified title + column_groups={"Assays\u25b8col1": "Geology"}, # move col1 from "Assays" to "Geology" + ) + + expected_update_body = models.UpdateDataLite1( + columns=models.UpdateColumnsLite( + new=[], + update=["Assays\u25b8col1"], # current reference + rename=[], + delete=[], + update_metadata=[ + models.UpdateMetadataLite( + title="Assays\u25b8col1", values=models.UpdateMetadataValuesLite(group="Geology") + ), + ], + ), + update_type=models.UpdateType.replace, + geometry_change=None, + fill_subblocks=None, + ) + self.assert_any_request_made( + method=RequestMethod.PATCH, + path=f"{self.base_path}/block-models/{BM_UUID}/blocks", + body=expected_update_body.model_dump(mode="json", exclude_unset=True), + headers=DEFAULT_EXPECTED_HEADERS, + ) + + uploaded_table = mock_write.call_args.args[0] + # The moved column is uploaded under its NEW qualified heading. + self.assertEqual(uploaded_table.schema.names, ["i", "j", "k", "Geology\u25b8col1", "col2"]) + + async def test_update_block_model_columns_data_only_update_of_grouped_column(self) -> None: + """A plain data update of an already-grouped column references it by its current qualified title.""" + self.transport.set_request_handler( + UpdateRequestHandler( + update_result=UPDATE_RESULT, + job_response=JobResponse(job_status=JobStatus.COMPLETE, payload=UPDATED_VERSION), + ) + ) + with ( + mock.patch("evo.common.io.upload.StorageDestination") as mock_destination, + mock.patch("pyarrow.parquet.write_table", wraps=pyarrow.parquet.write_table) as mock_write, + ): + mock_destination.upload_file = mock.AsyncMock() + # A data-only update keeps the grouped column under its current qualified heading. + data = pyarrow.table( + { + "i": [1, 2, 3], + "j": [4, 5, 6], + "k": [7, 8, 9], + "Assays\u25b8col1": ["A", "B", "B"], + "col2": [4.5, 5.3, 6.2], + } + ) + await self.bms_client.update_block_model_columns( + BM_UUID, + data, + new_columns=[], + update_columns={"Assays\u25b8col1"}, # current qualified title, no group change + ) + + # No group change, but the grouped column is still referenced by its current qualified title. + expected_update_body = models.UpdateDataLite1( + columns=models.UpdateColumnsLite( + new=[], + update=["Assays\u25b8col1"], + rename=[], + delete=[], + ), + update_type=models.UpdateType.replace, + geometry_change=None, + fill_subblocks=None, + ) + self.assert_any_request_made( + method=RequestMethod.PATCH, + path=f"{self.base_path}/block-models/{BM_UUID}/blocks", + body=expected_update_body.model_dump(mode="json", exclude_unset=True), + headers=DEFAULT_EXPECTED_HEADERS, + ) + + uploaded_table = mock_write.call_args.args[0] + # The grouped column keeps its current qualified heading so its data binds correctly. + self.assertEqual(uploaded_table.schema.names, ["i", "j", "k", "Assays\u25b8col1", "col2"]) + async def test_column_metadata_update_rejects_group(self) -> None: """Group membership can no longer be changed through metadata; the service requires re-uploaded data.""" with self.assertRaises(ValueError): From df01e5ba30d90d5ba3b02d70595482b68a2059b4 Mon Sep 17 00:00:00 2001 From: Andre Lobato <255781626+AndreLobatoSeequent@users.noreply.github.com> Date: Tue, 1 Sep 2026 14:28:32 +1200 Subject: [PATCH 5/8] rename qualify title helper functions --- .../src/evo/blockmodels/__init__.py | 6 ++-- .../src/evo/blockmodels/client.py | 16 +++++----- .../src/evo/blockmodels/data.py | 12 ++++---- .../tests/test_group_helpers.py | 30 +++++++++---------- 4 files changed, 32 insertions(+), 32 deletions(-) diff --git a/packages/evo-blockmodels/src/evo/blockmodels/__init__.py b/packages/evo-blockmodels/src/evo/blockmodels/__init__.py index 6906aaec..9a0f5fa9 100644 --- a/packages/evo-blockmodels/src/evo/blockmodels/__init__.py +++ b/packages/evo-blockmodels/src/evo/blockmodels/__init__.py @@ -10,11 +10,11 @@ # limitations under the License. from .client import BlockModelAPIClient -from .data import QUALIFIED_TITLE_SEPARATOR, qualified_heading, qualify_headings +from .data import QUALIFIED_TITLE_SEPARATOR, get_qualified_title, qualify_column_titles __all__ = [ "QUALIFIED_TITLE_SEPARATOR", "BlockModelAPIClient", - "qualified_heading", - "qualify_headings", + "get_qualified_title", + "qualify_column_titles", ] diff --git a/packages/evo-blockmodels/src/evo/blockmodels/client.py b/packages/evo-blockmodels/src/evo/blockmodels/client.py index f5024d32..7c022231 100644 --- a/packages/evo-blockmodels/src/evo/blockmodels/client.py +++ b/packages/evo-blockmodels/src/evo/blockmodels/client.py @@ -43,7 +43,7 @@ Version, ) from .data import ( - qualified_heading as _qualified_heading, + get_qualified_title as _get_qualified_title, ) from .endpoints import models from .endpoints.api import ColumnOperationsApi, JobsApi, MetadataApi, OperationsApi, ReportsApi, VersionsApi @@ -190,7 +190,7 @@ def _leaf_title(heading: str, group: str | None) -> str: if not heading.startswith(prefix): raise MissingColumnInTable( f"column '{heading}' is declared in group '{group}' but its data heading is not the qualified " - f"title '{prefix}'. Key the data by its exact upload heading (see qualify_headings)." + f"title '{prefix}'. Key the data by its exact upload heading (see qualify_column_titles)." ) return heading[len(prefix) :] @@ -754,7 +754,7 @@ async def add_new_subblocked_columns( :param column_groups: A dictionary mapping a grouped column's qualified upload heading (its key in `data`, e.g. ``"Assays▸Cu"``) to the qualified title of the group it belongs to (e.g. ``"Assays"``). Ungrouped columns are keyed by their bare title in `data` and omitted here. `data` must be keyed by each - column's exact upload heading; :func:`~evo.blockmodels.data.qualify_headings` can build that from + column's exact upload heading; :func:`~evo.blockmodels.data.qualify_column_titles` can build that from bare-titled data. Column groups are a preview feature; the client must be constructed with ``preview=True`` to use them. :raises CacheNotConfiguredException: If the cache is not configured. @@ -791,7 +791,7 @@ async def _add_new_columns( :param column_groups: A dictionary mapping a grouped column's qualified upload heading (its key in `data`, e.g. ``"Assays▸Cu"``) to the qualified title of the group it belongs to (e.g. ``"Assays"``). Ungrouped columns are keyed by their bare title in `data` and omitted here. `data` must be keyed by each - column's exact upload heading; :func:`~evo.blockmodels.data.qualify_headings` can build that from + column's exact upload heading; :func:`~evo.blockmodels.data.qualify_column_titles` can build that from bare-titled data. Column groups are a preview feature; the client must be constructed with ``preview=True`` to use them. :raises CacheNotConfiguredException: If the cache is not configured. @@ -875,7 +875,7 @@ async def add_new_columns( :param column_groups: A dictionary mapping a grouped column's qualified upload heading (its key in `data`, e.g. ``"Assays▸Cu"``) to the qualified title of the group it belongs to (e.g. ``"Assays"``). Ungrouped columns are keyed by their bare title in `data` and omitted here. `data` must be keyed by each - column's exact upload heading; :func:`~evo.blockmodels.data.qualify_headings` can build that from + column's exact upload heading; :func:`~evo.blockmodels.data.qualify_column_titles` can build that from bare-titled data. Column groups are a preview feature; the client must be constructed with ``preview=True`` to use them. :raises CacheNotConfiguredException: If the cache is not configured. @@ -927,7 +927,7 @@ def _expected_data_heading(column: str) -> str: if column in update_columns and column in column_groups: # A move re-uploads the column's data under its NEW heading (new group + current leaf). leaf = column.rsplit(_QUALIFIED_TITLE_SEPARATOR, 1)[-1] - return _qualified_heading(column_groups[column], leaf) + return _get_qualified_title(column_groups[column], leaf) # New columns and plain data updates are uploaded under their own title/heading. return column @@ -1027,7 +1027,7 @@ async def update_block_model_columns( :param bm_id: The ID of the block model to add columns to. :param data: The data containing the affected columns, keyed by each column's exact upload heading (a bare title for an ungrouped column, or the qualified ``group▸leaf`` title for a grouped one). - :func:`~evo.blockmodels.data.qualify_headings` can build these headings from bare-titled data. + :func:`~evo.blockmodels.data.qualify_column_titles` can build these headings from bare-titled data. :param new_columns: A list of new columns to add, named by their upload heading in `data` (qualified ``group▸leaf`` for a grouped column, bare otherwise). :param update_columns: A set of existing columns to re-upload, each identified by the title the service @@ -1087,7 +1087,7 @@ async def update_subblocked_columns( :param bm_id: The ID of the block model to add columns to. :param data: The data containing the affected columns, keyed by each column's exact upload heading (a bare title for an ungrouped column, or the qualified ``group▸leaf`` title for a grouped one). - :func:`~evo.blockmodels.data.qualify_headings` can build these headings from bare-titled data. + :func:`~evo.blockmodels.data.qualify_column_titles` can build these headings from bare-titled data. :param new_columns: A list of new columns to add, named by their upload heading in `data` (qualified ``group▸leaf`` for a grouped column, bare otherwise). :param update_columns: A set of existing columns to re-upload, each identified by the title the service diff --git a/packages/evo-blockmodels/src/evo/blockmodels/data.py b/packages/evo-blockmodels/src/evo/blockmodels/data.py index 82a221f9..184da7b7 100644 --- a/packages/evo-blockmodels/src/evo/blockmodels/data.py +++ b/packages/evo-blockmodels/src/evo/blockmodels/data.py @@ -50,15 +50,15 @@ "RegularGridDefinition", "ResolvedGroup", "Version", - "qualified_heading", - "qualify_headings", + "get_qualified_title", + "qualify_column_titles", ] QUALIFIED_TITLE_SEPARATOR = "\u25b8" """Default single-character separator (``▸``) used to build and parse qualified group titles.""" -def qualified_heading(group: str | None, title: str, separator: str = QUALIFIED_TITLE_SEPARATOR) -> str: +def get_qualified_title(group: str | None, title: str, separator: str = QUALIFIED_TITLE_SEPARATOR) -> str: """Build the upload heading the block model service expects for a column. A grouped column must be uploaded under its fully-qualified title (``group▸…▸leaf``); an @@ -74,7 +74,7 @@ def qualified_heading(group: str | None, title: str, separator: str = QUALIFIED_ return title -def qualify_headings( +def qualify_column_titles( data: Table, groups: dict[str, str], separator: str = QUALIFIED_TITLE_SEPARATOR ) -> tuple[Table, dict[str, str]]: """Rename a table's columns to the qualified upload headings the service expects. @@ -92,7 +92,7 @@ def qualify_headings( left with their bare heading. :param separator: Separator used to build qualified titles. :return: A ``(table, column_groups)`` pair: the table with grouped columns renamed to their qualified - heading, and a ``{qualified_heading: group}`` mapping to pass as ``column_groups``. + heading, and a ``{qualified_title: group}`` mapping to pass as ``column_groups``. :raises KeyError: If ``groups`` references a column that is not present in ``data``. """ existing = set(data.schema.names) @@ -104,7 +104,7 @@ def qualify_headings( column_groups: dict[str, str] = {} for name in data.schema.names: group = groups.get(name) - heading = qualified_heading(group, name, separator) + heading = get_qualified_title(group, name, separator) new_names.append(heading) if group: column_groups[heading] = group diff --git a/packages/evo-blockmodels/tests/test_group_helpers.py b/packages/evo-blockmodels/tests/test_group_helpers.py index 3d2cc1bb..7f6b858b 100644 --- a/packages/evo-blockmodels/tests/test_group_helpers.py +++ b/packages/evo-blockmodels/tests/test_group_helpers.py @@ -158,24 +158,24 @@ def test_column_metadata_update_forbids_group_field(self) -> None: class TestQualifyHeadings(unittest.TestCase): - def test_qualified_heading_builds_qualified_and_bare_titles(self) -> None: - from evo.blockmodels.data import qualified_heading + def test_get_qualified_title_builds_qualified_and_bare_titles(self) -> None: + from evo.blockmodels.data import get_qualified_title - self.assertEqual(qualified_heading("Assays", "Cu"), f"Assays{QUALIFIED_TITLE_SEPARATOR}Cu") + self.assertEqual(get_qualified_title("Assays", "Cu"), f"Assays{QUALIFIED_TITLE_SEPARATOR}Cu") self.assertEqual( - qualified_heading("Assays\u25b8Primary", "Cu"), f"Assays\u25b8Primary{QUALIFIED_TITLE_SEPARATOR}Cu" + get_qualified_title("Assays\u25b8Primary", "Cu"), f"Assays\u25b8Primary{QUALIFIED_TITLE_SEPARATOR}Cu" ) # An ungrouped column keeps its bare title. - self.assertEqual(qualified_heading(None, "Cu"), "Cu") - self.assertEqual(qualified_heading("", "Cu"), "Cu") + self.assertEqual(get_qualified_title(None, "Cu"), "Cu") + self.assertEqual(get_qualified_title("", "Cu"), "Cu") - def test_qualify_headings_renames_and_builds_column_groups(self) -> None: + def test_qualify_column_titles_renames_and_builds_column_groups(self) -> None: import pyarrow - from evo.blockmodels.data import qualify_headings + from evo.blockmodels.data import qualify_column_titles data = pyarrow.table({"i": [1], "Cu": [2.0], "Au": [3.0], "rock": ["x"]}) - renamed, column_groups = qualify_headings(data, {"Cu": "Assays", "Au": "Assays"}) + renamed, column_groups = qualify_column_titles(data, {"Cu": "Assays", "Au": "Assays"}) self.assertEqual( renamed.schema.names, @@ -188,22 +188,22 @@ def test_qualify_headings_renames_and_builds_column_groups(self) -> None: # Untouched columns keep their bare heading and are absent from column_groups. self.assertEqual(renamed.column("i").to_pylist(), [1]) - def test_qualify_headings_treats_empty_group_as_ungrouped(self) -> None: + def test_qualify_column_titles_treats_empty_group_as_ungrouped(self) -> None: import pyarrow - from evo.blockmodels.data import qualify_headings + from evo.blockmodels.data import qualify_column_titles data = pyarrow.table({"Cu": [2.0]}) - renamed, column_groups = qualify_headings(data, {"Cu": ""}) + renamed, column_groups = qualify_column_titles(data, {"Cu": ""}) self.assertEqual(renamed.schema.names, ["Cu"]) self.assertEqual(column_groups, {}) - def test_qualify_headings_rejects_unknown_columns(self) -> None: + def test_qualify_column_titles_rejects_unknown_columns(self) -> None: import pyarrow - from evo.blockmodels.data import qualify_headings + from evo.blockmodels.data import qualify_column_titles data = pyarrow.table({"Cu": [2.0]}) with self.assertRaises(KeyError): - qualify_headings(data, {"Au": "Assays"}) + qualify_column_titles(data, {"Au": "Assays"}) From 49fb50cb09a6b28f1f6cdceed80ae1c87f83a91a Mon Sep 17 00:00:00 2001 From: Andre Lobato <255781626+AndreLobatoSeequent@users.noreply.github.com> Date: Wed, 2 Sep 2026 09:57:14 +1200 Subject: [PATCH 6/8] Improve wording to reflect nominal terminology --- .../src/evo/blockmodels/client.py | 119 +++++++++--------- .../src/evo/blockmodels/data.py | 37 +++--- .../tests/test_group_helpers.py | 4 +- packages/evo-blockmodels/tests/test_update.py | 24 ++-- 4 files changed, 92 insertions(+), 92 deletions(-) diff --git a/packages/evo-blockmodels/src/evo/blockmodels/client.py b/packages/evo-blockmodels/src/evo/blockmodels/client.py index 7c022231..684f1375 100644 --- a/packages/evo-blockmodels/src/evo/blockmodels/client.py +++ b/packages/evo-blockmodels/src/evo/blockmodels/client.py @@ -177,22 +177,23 @@ def _build_update_groups_lite( ) -def _leaf_title(heading: str, group: str | None) -> str: - """Recover a column's bare (leaf) title from its qualified upload heading. +def _title_from_column_title(column_title: str, group: str | None) -> str: + """Recover a column's plain title from its (possibly qualified) column title. - ``heading`` is the fully-qualified upload heading (e.g. ``Assays▸Cu``); ``group`` is the qualified - group path it should belong to (e.g. ``Assays``). Stripping the ``group▸`` prefix yields the leaf - title the service stores. An ungrouped column (no group) has a bare heading, so it is returned as-is. + ``column_title`` is the column's title in the data table (e.g. ``Assays▸Cu``); ``group`` is the + qualified group path it should belong to (e.g. ``Assays``). Stripping the ``group▸`` prefix yields + the title the service stores. An ungrouped column (no group) keeps its plain title, so it is + returned as-is. """ if not group: - return heading + return column_title prefix = f"{group}{_QUALIFIED_TITLE_SEPARATOR}" - if not heading.startswith(prefix): + if not column_title.startswith(prefix): raise MissingColumnInTable( - f"column '{heading}' is declared in group '{group}' but its data heading is not the qualified " - f"title '{prefix}'. Key the data by its exact upload heading (see qualify_column_titles)." + f"column '{column_title}' is declared in group '{group}' but its column title is not the qualified " + f"title '{prefix}'. Key the data by each column's exact title (see qualify_column_titles)." ) - return heading[len(prefix) :] + return column_title.removeprefix(prefix) class BlockModelAPIClient(BaseAPIClient): @@ -746,16 +747,16 @@ async def add_new_subblocked_columns( This method requires the `pyarrow` package to be installed, and the 'cache' parameter to be set in the constructor. :param bm_id: The ID of the block model to add columns to. - :param data: The data containing the new columns to add, keyed by each column's exact upload heading - (a bare title for an ungrouped column, or the qualified ``group▸leaf`` title for a grouped one). + :param data: The data containing the new columns to add, keyed by each column's title + (a plain title for an ungrouped column, or the qualified ``group▸title`` for a grouped one). :param units: A dictionary mapping column names within `data` to units. :param tags: A dictionary mapping column names within `data` to their tags object. Column tags are a preview feature; the client must be constructed with ``preview=True`` to use them. - :param column_groups: A dictionary mapping a grouped column's qualified upload heading (its key in + :param column_groups: A dictionary mapping a grouped column's qualified title (its key in `data`, e.g. ``"Assays▸Cu"``) to the qualified title of the group it belongs to (e.g. ``"Assays"``). - Ungrouped columns are keyed by their bare title in `data` and omitted here. `data` must be keyed by each - column's exact upload heading; :func:`~evo.blockmodels.data.qualify_column_titles` can build that from - bare-titled data. Column groups are a preview feature; the client must be constructed with + Ungrouped columns are keyed by their plain title in `data` and omitted here. `data` must be keyed by each + column's exact title; :func:`~evo.blockmodels.data.qualify_column_titles` can build that from + plain-titled data. Column groups are a preview feature; the client must be constructed with ``preview=True`` to use them. :raises CacheNotConfiguredException: If the cache is not configured. :return: The new version of the block model with the added columns. @@ -782,17 +783,17 @@ async def _add_new_columns( This method requires the `pyarrow` package to be installed, and the 'cache' parameter to be set in the constructor. :param bm_id: The ID of the block model to add columns to. - :param data: The data containing the new columns to add, keyed by each column's exact upload heading - (a bare title for an ungrouped column, or the qualified ``group▸leaf`` title for a grouped one). + :param data: The data containing the new columns to add, keyed by each column's title + (a plain title for an ungrouped column, or the qualified ``group▸title`` for a grouped one). :param units: A dictionary mapping column names within `data` to units. :param geometry_change: Whether the geometry of the block model is changing. :param tags: A dictionary mapping column names within `data` to their tags object. Column tags are a preview feature; the client must be constructed with ``preview=True`` to use them. - :param column_groups: A dictionary mapping a grouped column's qualified upload heading (its key in + :param column_groups: A dictionary mapping a grouped column's qualified title (its key in `data`, e.g. ``"Assays▸Cu"``) to the qualified title of the group it belongs to (e.g. ``"Assays"``). - Ungrouped columns are keyed by their bare title in `data` and omitted here. `data` must be keyed by each - column's exact upload heading; :func:`~evo.blockmodels.data.qualify_column_titles` can build that from - bare-titled data. Column groups are a preview feature; the client must be constructed with + Ungrouped columns are keyed by their plain title in `data` and omitted here. `data` must be keyed by each + column's exact title; :func:`~evo.blockmodels.data.qualify_column_titles` can build that from + plain-titled data. Column groups are a preview feature; the client must be constructed with ``preview=True`` to use them. :raises CacheNotConfiguredException: If the cache is not configured. :return: The new version of the block model with the added columns. @@ -824,7 +825,7 @@ async def _add_new_columns( columns = models.UpdateColumnsLite( new=[ models.ColumnLite( - title=_leaf_title(name, column_groups.get(name)), + title=_title_from_column_title(name, column_groups.get(name)), data_type=convert_dtype(data_type), unit_id=units.get(name), **({"tags": tags[name]} if name in tags else {}), @@ -867,16 +868,16 @@ async def add_new_columns( This method requires the `pyarrow` package to be installed, and the 'cache' parameter to be set in the constructor. :param bm_id: The ID of the block model to add columns to. - :param data: The data containing the new columns to add, keyed by each column's exact upload heading - (a bare title for an ungrouped column, or the qualified ``group▸leaf`` title for a grouped one). + :param data: The data containing the new columns to add, keyed by each column's title + (a plain title for an ungrouped column, or the qualified ``group▸title`` for a grouped one). :param units: A dictionary mapping column names within `data` to units. :param tags: A dictionary mapping column names within `data` to their tags object. Column tags are a preview feature; the client must be constructed with ``preview=True`` to use them. - :param column_groups: A dictionary mapping a grouped column's qualified upload heading (its key in + :param column_groups: A dictionary mapping a grouped column's qualified title (its key in `data`, e.g. ``"Assays▸Cu"``) to the qualified title of the group it belongs to (e.g. ``"Assays"``). - Ungrouped columns are keyed by their bare title in `data` and omitted here. `data` must be keyed by each - column's exact upload heading; :func:`~evo.blockmodels.data.qualify_column_titles` can build that from - bare-titled data. Column groups are a preview feature; the client must be constructed with + Ungrouped columns are keyed by their plain title in `data` and omitted here. `data` must be keyed by each + column's exact title; :func:`~evo.blockmodels.data.qualify_column_titles` can build that from + plain-titled data. Column groups are a preview feature; the client must be constructed with ``preview=True`` to use them. :raises CacheNotConfiguredException: If the cache is not configured. :return: The new version of the block model with the added columns. @@ -920,23 +921,23 @@ async def _update_columns( delete_columns = set() # Existing columns are addressed by the exact title the service stores: a currently-grouped - # column by its qualified title (``group▸leaf``), an ungrouped column by its bare title. New - # columns are keyed in ``data`` by their upload heading. Work out the data heading each declared + # column by its qualified title (``group▸title``), an ungrouped column by its plain title. New + # columns are keyed in ``data`` by their column title. Work out the column title each declared # column should be found under, so we can validate the table without renaming it. - def _expected_data_heading(column: str) -> str: + def _expected_column_title(column: str) -> str: if column in update_columns and column in column_groups: - # A move re-uploads the column's data under its NEW heading (new group + current leaf). - leaf = column.rsplit(_QUALIFIED_TITLE_SEPARATOR, 1)[-1] - return _get_qualified_title(column_groups[column], leaf) - # New columns and plain data updates are uploaded under their own title/heading. + # A move re-uploads the column's data under its NEW column title (new group + current title). + title = column.rsplit(_QUALIFIED_TITLE_SEPARATOR, 1)[-1] + return _get_qualified_title(column_groups[column], title) + # New columns and plain data updates are uploaded under their own column title. return column - expected_headings = {_expected_data_heading(column) for column in (set(new_columns) | update_columns)} - missing = expected_headings - data_type_map.keys() + expected_column_titles = {_expected_column_title(column) for column in (set(new_columns) | update_columns)} + missing = expected_column_titles - data_type_map.keys() if missing: raise MissingColumnInTable( f"Columns {missing} are not present in the provided table. Key the data by each column's " - "exact upload heading (qualified 'group▸leaf' for grouped columns, bare otherwise)." + "exact title (qualified 'group▸title' for grouped columns, plain otherwise)." ) unknown_unit_columns = set(units) - set(new_columns) @@ -962,7 +963,7 @@ def _expected_data_heading(column: str) -> str: raise MissingColumnInTable( f"column_groups reference columns that are neither new nor being updated: {unknown_group_columns}. " "A column's group can only be changed when its data is re-uploaded, so the column must be listed " - "in new_columns (as its qualified heading) or update_columns (as its current qualified title)." + "in new_columns (as its qualified title) or update_columns (as its current qualified title)." ) # An existing column is moved by pairing its re-uploaded data with an update_metadata group change, @@ -976,7 +977,7 @@ def _expected_data_heading(column: str) -> str: columns = models.UpdateColumnsLite( new=[ models.ColumnLite( - title=_leaf_title(new_column, column_groups.get(new_column)), + title=_title_from_column_title(new_column, column_groups.get(new_column)), data_type=convert_dtype(data_type_map[new_column]), unit_id=units.get(new_column), **({"tags": tags[new_column]} if new_column in tags else {}), @@ -1025,22 +1026,22 @@ async def update_block_model_columns( This method requires the `pyarrow` package to be installed, and the 'cache' parameter to be set in the constructor. :param bm_id: The ID of the block model to add columns to. - :param data: The data containing the affected columns, keyed by each column's exact upload heading - (a bare title for an ungrouped column, or the qualified ``group▸leaf`` title for a grouped one). - :func:`~evo.blockmodels.data.qualify_column_titles` can build these headings from bare-titled data. - :param new_columns: A list of new columns to add, named by their upload heading in `data` (qualified - ``group▸leaf`` for a grouped column, bare otherwise). + :param data: The data containing the affected columns, keyed by each column's title + (a plain title for an ungrouped column, or the qualified ``group▸title`` for a grouped one). + :func:`~evo.blockmodels.data.qualify_column_titles` can build these titles from plain-titled data. + :param new_columns: A list of new columns to add, named by their title in `data` (qualified + ``group▸title`` for a grouped column, plain otherwise). :param update_columns: A set of existing columns to re-upload, each identified by the title the service - currently stores it under: its qualified title (``group▸leaf``) if grouped, or its bare title if not. + currently stores it under: its qualified title (``group▸title``) if grouped, or its plain title if not. :param delete_columns: A set of existing columns to delete, identified the same way as ``update_columns`` - (qualified title if grouped, bare otherwise). + (qualified title if grouped, plain otherwise). :param units: A dictionary mapping column names within `data` to units. :param tags: A dictionary mapping new column names to their tags object. Column tags are a preview feature; the client must be constructed with ``preview=True`` to use them. :param column_groups: A dictionary assigning columns to groups. For a **new** column, map its qualified - upload heading (its key in `data`) to the group it belongs to. To **move** an *existing* column, map its + title (its key in `data`) to the group it belongs to. To **move** an *existing* column, map its **current** qualified title to the new group (or ``""`` to ungroup); the column must also be listed in - ``update_columns`` and its data supplied under the **new** heading. Column groups are a preview feature; + ``update_columns`` and its data supplied under the **new** title. Column groups are a preview feature; the client must be constructed with ``preview=True`` to use them. :param: update_type: Provide the type of update. Either 'replace' or 'merge' (default: replace) :raises CacheNotConfiguredException: If the cache is not configured. @@ -1085,15 +1086,15 @@ async def update_subblocked_columns( This method requires the `pyarrow` package to be installed, and the 'cache' parameter to be set in the constructor. :param bm_id: The ID of the block model to add columns to. - :param data: The data containing the affected columns, keyed by each column's exact upload heading - (a bare title for an ungrouped column, or the qualified ``group▸leaf`` title for a grouped one). - :func:`~evo.blockmodels.data.qualify_column_titles` can build these headings from bare-titled data. - :param new_columns: A list of new columns to add, named by their upload heading in `data` (qualified - ``group▸leaf`` for a grouped column, bare otherwise). + :param data: The data containing the affected columns, keyed by each column's title + (a plain title for an ungrouped column, or the qualified ``group▸title`` for a grouped one). + :func:`~evo.blockmodels.data.qualify_column_titles` can build these titles from plain-titled data. + :param new_columns: A list of new columns to add, named by their title in `data` (qualified + ``group▸title`` for a grouped column, plain otherwise). :param update_columns: A set of existing columns to re-upload, each identified by the title the service - currently stores it under: its qualified title (``group▸leaf``) if grouped, or its bare title if not. + currently stores it under: its qualified title (``group▸title``) if grouped, or its plain title if not. :param delete_columns: A set of existing columns to delete, identified the same way as ``update_columns`` - (qualified title if grouped, bare otherwise). + (qualified title if grouped, plain otherwise). :param units: A dictionary mapping column names within `data` to units. :param geometry_change: Whether the geometry of the sub-blocked model changes. :param fill_subblocks: If ``True``, any missing sub-blocks will be filled with data from the parent block. @@ -1102,9 +1103,9 @@ async def update_subblocked_columns( :param tags: A dictionary mapping new column names to their tags object. Column tags are a preview feature; the client must be constructed with ``preview=True`` to use them. :param column_groups: A dictionary assigning columns to groups. For a **new** column, map its qualified - upload heading (its key in `data`) to the group it belongs to. To **move** an *existing* column, map its + title (its key in `data`) to the group it belongs to. To **move** an *existing* column, map its **current** qualified title to the new group (or ``""`` to ungroup); the column must also be listed in - ``update_columns`` and its data supplied under the **new** heading. Column groups are a preview feature; + ``update_columns`` and its data supplied under the **new** title. Column groups are a preview feature; the client must be constructed with ``preview=True`` to use them. :param: update_type: Provide the type of update. Either 'replace' or 'merge' (default: replace) """ diff --git a/packages/evo-blockmodels/src/evo/blockmodels/data.py b/packages/evo-blockmodels/src/evo/blockmodels/data.py index 184da7b7..941d8316 100644 --- a/packages/evo-blockmodels/src/evo/blockmodels/data.py +++ b/packages/evo-blockmodels/src/evo/blockmodels/data.py @@ -59,15 +59,15 @@ def get_qualified_title(group: str | None, title: str, separator: str = QUALIFIED_TITLE_SEPARATOR) -> str: - """Build the upload heading the block model service expects for a column. + """Build the column title the block model service expects for a column. - A grouped column must be uploaded under its fully-qualified title (``group▸…▸leaf``); an - ungrouped column (no group, or ``group == ""``) keeps its bare title. + A grouped column must be uploaded under its qualified title (``group▸…▸title``); an + ungrouped column (no group, or ``group == ""``) keeps its plain title. :param group: The qualified title (path) of the column's group, or ``None``/``""`` if ungrouped. - :param title: The column's bare (leaf) title. + :param title: The column's title. :param separator: Separator used to join the path segments. - :return: The qualified heading (``group▸title``) if grouped, otherwise ``title``. + :return: The qualified title (``group▸title``) if grouped, otherwise ``title``. """ if group: return f"{group}{separator}{title}" @@ -77,22 +77,21 @@ def get_qualified_title(group: str | None, title: str, separator: str = QUALIFIE def qualify_column_titles( data: Table, groups: dict[str, str], separator: str = QUALIFIED_TITLE_SEPARATOR ) -> tuple[Table, dict[str, str]]: - """Rename a table's columns to the qualified upload headings the service expects. + """Rename a table's columns to the qualified titles the service expects. The column methods on :class:`~evo.blockmodels.client.BlockModelAPIClient` expect ``data`` to be - keyed by each column's exact upload heading — a qualified ``group▸leaf`` title for a grouped column, - or the bare title for an ungrouped one — and do not rename anything for you. This opt-in helper - performs that shift: pass a table keyed by bare (leaf) titles plus a mapping of the columns you want - grouped, and it returns the renamed table alongside the ``column_groups`` mapping to hand back to the - client. - - :param data: A table keyed by bare (leaf) column titles. - :param groups: A mapping of a column's bare title to the qualified title of the group it should be + keyed by each column's title — a qualified ``group▸title`` for a grouped column, or the plain title + for an ungrouped one — and do not rename anything for you. This opt-in helper performs that shift: + pass a table keyed by plain titles plus a mapping of the columns you want grouped, and it returns the + renamed table alongside the ``column_groups`` mapping to hand back to the client. + + :param data: A table keyed by plain column titles. + :param groups: A mapping of a column's title to the qualified title of the group it should be placed in. Columns absent from the mapping (or mapped to ``""``) are treated as ungrouped and - left with their bare heading. + left with their plain title. :param separator: Separator used to build qualified titles. :return: A ``(table, column_groups)`` pair: the table with grouped columns renamed to their qualified - heading, and a ``{qualified_title: group}`` mapping to pass as ``column_groups``. + title, and a ``{qualified_title: group}`` mapping to pass as ``column_groups``. :raises KeyError: If ``groups`` references a column that is not present in ``data``. """ existing = set(data.schema.names) @@ -104,10 +103,10 @@ def qualify_column_titles( column_groups: dict[str, str] = {} for name in data.schema.names: group = groups.get(name) - heading = get_qualified_title(group, name, separator) - new_names.append(heading) + column_title = get_qualified_title(group, name, separator) + new_names.append(column_title) if group: - column_groups[heading] = group + column_groups[column_title] = group return data.rename_columns(new_names), column_groups diff --git a/packages/evo-blockmodels/tests/test_group_helpers.py b/packages/evo-blockmodels/tests/test_group_helpers.py index 7f6b858b..24aeb2e4 100644 --- a/packages/evo-blockmodels/tests/test_group_helpers.py +++ b/packages/evo-blockmodels/tests/test_group_helpers.py @@ -157,7 +157,7 @@ def test_column_metadata_update_forbids_group_field(self) -> None: ColumnMetadataUpdate(group="Assays") -class TestQualifyHeadings(unittest.TestCase): +class TestQualifyColumnTitles(unittest.TestCase): def test_get_qualified_title_builds_qualified_and_bare_titles(self) -> None: from evo.blockmodels.data import get_qualified_title @@ -185,7 +185,7 @@ def test_qualify_column_titles_renames_and_builds_column_groups(self) -> None: column_groups, {f"Assays{QUALIFIED_TITLE_SEPARATOR}Cu": "Assays", f"Assays{QUALIFIED_TITLE_SEPARATOR}Au": "Assays"}, ) - # Untouched columns keep their bare heading and are absent from column_groups. + # Untouched columns keep their plain title and are absent from column_groups. self.assertEqual(renamed.column("i").to_pylist(), [1]) def test_qualify_column_titles_treats_empty_group_as_ungrouped(self) -> None: diff --git a/packages/evo-blockmodels/tests/test_update.py b/packages/evo-blockmodels/tests/test_update.py index d41a8017..9efaf83c 100644 --- a/packages/evo-blockmodels/tests/test_update.py +++ b/packages/evo-blockmodels/tests/test_update.py @@ -1059,7 +1059,7 @@ async def test_add_new_columns_with_column_groups(self) -> None: mock.patch("pyarrow.parquet.write_table", wraps=pyarrow.parquet.write_table) as mock_write, ): mock_destination.upload_file = mock.AsyncMock() - # The caller keys the grouped column by its qualified upload heading; the SDK never renames data. + # The caller keys the grouped column by its qualified title; the SDK never renames data. data = pyarrow.table( { "i": [1, 2, 3], @@ -1100,7 +1100,7 @@ async def test_add_new_columns_with_column_groups(self) -> None: ) # The uploaded data is untouched: the caller already keyed the grouped column by its - # qualified heading, and the ungrouped column keeps its bare heading. + # qualified title, and the ungrouped column keeps its plain title. uploaded_table = mock_write.call_args.args[0] self.assertEqual(uploaded_table.schema.names, ["i", "j", "k", "col1", "Assays\u25b8col2"]) @@ -1125,7 +1125,7 @@ async def test_update_block_model_columns_with_column_groups(self) -> None: mock.patch("pyarrow.parquet.write_table", wraps=pyarrow.parquet.write_table) as mock_write, ): mock_destination.upload_file = mock.AsyncMock() - # New columns are named by their upload heading; the grouped one is qualified. + # New columns are named by their title; the grouped one is qualified. data = pyarrow.table( { "i": [1, 2, 3], @@ -1169,7 +1169,7 @@ async def test_update_block_model_columns_with_column_groups(self) -> None: ) uploaded_table = mock_write.call_args.args[0] - # Data is uploaded exactly as provided (already keyed by the qualified heading). + # Data is uploaded exactly as provided (already keyed by the qualified title). self.assertEqual(uploaded_table.schema.names, ["i", "j", "k", "Assays\u25b8Primary\u25b8col1", "col2"]) async def test_update_columns_with_unknown_group_column(self) -> None: @@ -1194,7 +1194,7 @@ async def test_update_block_model_columns_moves_existing_column_group(self) -> N mock.patch("pyarrow.parquet.write_table", wraps=pyarrow.parquet.write_table) as mock_write, ): mock_destination.upload_file = mock.AsyncMock() - # col1 is currently ungrouped; its data is re-uploaded under its NEW qualified heading. + # col1 is currently ungrouped; its data is re-uploaded under its NEW qualified title. data = pyarrow.table( { "i": [1, 2, 3], @@ -1234,7 +1234,7 @@ async def test_update_block_model_columns_moves_existing_column_group(self) -> N ) uploaded_table = mock_write.call_args.args[0] - # col1 is uploaded under its new qualified heading so the service binds its re-uploaded data. + # col1 is uploaded under its new qualified title so the service binds its re-uploaded data. self.assertEqual(uploaded_table.schema.names, ["i", "j", "k", "Assays\u25b8col1", "col2"]) async def test_update_block_model_columns_ungroups_existing_column(self) -> None: @@ -1250,7 +1250,7 @@ async def test_update_block_model_columns_ungroups_existing_column(self) -> None mock.patch("pyarrow.parquet.write_table", wraps=pyarrow.parquet.write_table) as mock_write, ): mock_destination.upload_file = mock.AsyncMock() - # col2 currently lives in "Assays"; ungrouping re-uploads its data under its NEW bare heading. + # col2 currently lives in "Assays"; ungrouping re-uploads its data under its NEW plain title. data = pyarrow.table( { "i": [1, 2, 3], @@ -1293,7 +1293,7 @@ async def test_update_block_model_columns_ungroups_existing_column(self) -> None ) uploaded_table = mock_write.call_args.args[0] - # The now-ungrouped column is uploaded under its new bare heading. + # The now-ungrouped column is uploaded under its new plain title. self.assertEqual(uploaded_table.schema.names, ["i", "j", "k", "col1", "col2"]) async def test_update_block_model_columns_moves_currently_grouped_column(self) -> None: @@ -1309,7 +1309,7 @@ async def test_update_block_model_columns_moves_currently_grouped_column(self) - mock.patch("pyarrow.parquet.write_table", wraps=pyarrow.parquet.write_table) as mock_write, ): mock_destination.upload_file = mock.AsyncMock() - # col1 currently lives in "Assays"; moving it re-uploads its data under its NEW qualified heading. + # col1 currently lives in "Assays"; moving it re-uploads its data under its NEW qualified title. data = pyarrow.table( { "i": [1, 2, 3], @@ -1351,7 +1351,7 @@ async def test_update_block_model_columns_moves_currently_grouped_column(self) - ) uploaded_table = mock_write.call_args.args[0] - # The moved column is uploaded under its NEW qualified heading. + # The moved column is uploaded under its NEW qualified title. self.assertEqual(uploaded_table.schema.names, ["i", "j", "k", "Geology\u25b8col1", "col2"]) async def test_update_block_model_columns_data_only_update_of_grouped_column(self) -> None: @@ -1367,7 +1367,7 @@ async def test_update_block_model_columns_data_only_update_of_grouped_column(sel mock.patch("pyarrow.parquet.write_table", wraps=pyarrow.parquet.write_table) as mock_write, ): mock_destination.upload_file = mock.AsyncMock() - # A data-only update keeps the grouped column under its current qualified heading. + # A data-only update keeps the grouped column under its current qualified title. data = pyarrow.table( { "i": [1, 2, 3], @@ -1404,7 +1404,7 @@ async def test_update_block_model_columns_data_only_update_of_grouped_column(sel ) uploaded_table = mock_write.call_args.args[0] - # The grouped column keeps its current qualified heading so its data binds correctly. + # The grouped column keeps its current qualified title so its data binds correctly. self.assertEqual(uploaded_table.schema.names, ["i", "j", "k", "Assays\u25b8col1", "col2"]) async def test_column_metadata_update_rejects_group(self) -> None: From 6c0f93edf97312d851eebcc75edaf5b99f3b880f Mon Sep 17 00:00:00 2001 From: Andre Lobato <255781626+AndreLobatoSeequent@users.noreply.github.com> Date: Wed, 2 Sep 2026 10:11:04 +1200 Subject: [PATCH 7/8] Update group_for_column function to deal to potential title ambiguity --- .../src/evo/blockmodels/data.py | 14 +++--- .../tests/test_group_helpers.py | 46 +++++++++++++++---- 2 files changed, 45 insertions(+), 15 deletions(-) diff --git a/packages/evo-blockmodels/src/evo/blockmodels/data.py b/packages/evo-blockmodels/src/evo/blockmodels/data.py index 941d8316..68204688 100644 --- a/packages/evo-blockmodels/src/evo/blockmodels/data.py +++ b/packages/evo-blockmodels/src/evo/blockmodels/data.py @@ -410,19 +410,19 @@ def group_by_uuid(self, group_uuid: UUID) -> "ResolvedGroup | ListingGroup | Non return group return None - def group_for_column(self, column: "Column | ListingColumn | str") -> "ResolvedGroup | ListingGroup | None": + def group_for_column(self, column: "Column | ListingColumn") -> "ResolvedGroup | ListingGroup | None": """Resolve the group a column belongs to. This bridges the read path (columns reference their group by UUID) so a caller can get the group's title, parent and resolved missing-column policy without hand-rolling a lookup. - :param column: A column from this version, or a column title. - :return: The column's group, or ``None`` if the column is ungrouped or unknown. + A column object is required rather than a title: titles are only unique within a group, so a + title can be ambiguous across groups. The column carries its group reference unambiguously. + + :param column: A column from this version. + :return: The column's group, or ``None`` if the column is ungrouped or its group is not on + this version. """ - if isinstance(column, str): - column = next((c for c in self.columns if c.title == column), None) - if column is None: - return None if column.group_uuid is None: return None return self.group_by_uuid(column.group_uuid) diff --git a/packages/evo-blockmodels/tests/test_group_helpers.py b/packages/evo-blockmodels/tests/test_group_helpers.py index 24aeb2e4..7fac67b7 100644 --- a/packages/evo-blockmodels/tests/test_group_helpers.py +++ b/packages/evo-blockmodels/tests/test_group_helpers.py @@ -99,16 +99,46 @@ def test_group_for_column_by_object(self) -> None: self.assertEqual(group.group_uuid, CHILD_UUID) self.assertEqual(group.resolved_missing_column_policy, MissingColumnPolicy.SET_NULL) - def test_group_for_column_by_title(self) -> None: - group = self.version.group_for_column("Cu") - self.assertIsNotNone(group) - self.assertEqual(group.group_uuid, CHILD_UUID) - def test_group_for_ungrouped_column(self) -> None: - self.assertIsNone(self.version.group_for_column("Au")) + # columns[1] ("Au") has no group_uuid. + self.assertIsNone(self.version.group_for_column(self.version.columns[1])) - def test_group_for_unknown_column(self) -> None: - self.assertIsNone(self.version.group_for_column("does_not_exist")) + def test_group_for_column_with_dangling_group(self) -> None: + # A column referencing a group that is not on this version resolves to None rather than guessing. + orphan_column = models.Column( + col_id=str(uuid.uuid4()), title="Cu", data_type=models.DataType.Float64, group_uuid=uuid.uuid4() + ) + self.assertIsNone(self.version.group_for_column(orphan_column)) + + def test_group_for_column_is_unambiguous_across_groups(self) -> None: + # Two columns share the title "Cu" but live in different groups; each resolves to its own group + # because the lookup uses the column's group_uuid, not its (non-unique) title. + other_uuid = uuid.uuid4() + version = Version( + bm_uuid=uuid.uuid4(), + version_id=2, + version_uuid=uuid.uuid4(), + parent_version_id=1, + base_version_id=1, + geoscience_version_id="3", + created_at=DATE, + created_by=USER, + comment="", + columns=[ + models.Column( + col_id=str(uuid.uuid4()), title="Cu", data_type=models.DataType.Float64, group_uuid=CHILD_UUID + ), + models.Column( + col_id=str(uuid.uuid4()), title="Cu", data_type=models.DataType.Float64, group_uuid=other_uuid + ), + ], + groups=[ + _resolved_group(CHILD_UUID, "Primary"), + _resolved_group(other_uuid, "Secondary"), + ], + ) + self.assertEqual(version.group_for_column(version.columns[0]).group_uuid, CHILD_UUID) + self.assertEqual(version.group_for_column(version.columns[1]).group_uuid, other_uuid) def test_qualified_group_title_tolerates_broken_parent_chain(self) -> None: # A dangling parent reference must not loop or raise; it just stops walking. From aeb4f0cececf44dbefb1a983df438c1c8083ad73 Mon Sep 17 00:00:00 2001 From: Andre Lobato <255781626+AndreLobatoSeequent@users.noreply.github.com> Date: Fri, 4 Sep 2026 13:04:16 +1200 Subject: [PATCH 8/8] remove requirement of group moves to require data upload --- .../src/evo/blockmodels/client.py | 88 +++---- .../src/evo/blockmodels/data.py | 7 + .../tests/test_group_helpers.py | 18 +- packages/evo-blockmodels/tests/test_update.py | 249 +++++------------- 4 files changed, 124 insertions(+), 238 deletions(-) diff --git a/packages/evo-blockmodels/src/evo/blockmodels/client.py b/packages/evo-blockmodels/src/evo/blockmodels/client.py index 684f1375..7e5f0d79 100644 --- a/packages/evo-blockmodels/src/evo/blockmodels/client.py +++ b/packages/evo-blockmodels/src/evo/blockmodels/client.py @@ -42,9 +42,6 @@ SubBlockedGridDefinition, Version, ) -from .data import ( - get_qualified_title as _get_qualified_title, -) from .endpoints import models from .endpoints.api import ColumnOperationsApi, JobsApi, MetadataApi, OperationsApi, ReportsApi, VersionsApi from .endpoints.models import ( @@ -920,19 +917,11 @@ async def _update_columns( if delete_columns is None: delete_columns = set() - # Existing columns are addressed by the exact title the service stores: a currently-grouped - # column by its qualified title (``group▸title``), an ungrouped column by its plain title. New - # columns are keyed in ``data`` by their column title. Work out the column title each declared - # column should be found under, so we can validate the table without renaming it. - def _expected_column_title(column: str) -> str: - if column in update_columns and column in column_groups: - # A move re-uploads the column's data under its NEW column title (new group + current title). - title = column.rsplit(_QUALIFIED_TITLE_SEPARATOR, 1)[-1] - return _get_qualified_title(column_groups[column], title) - # New columns and plain data updates are uploaded under their own column title. - return column - - expected_column_titles = {_expected_column_title(column) for column in (set(new_columns) | update_columns)} + # Every declared column is uploaded under its own title: new columns by their title in ``data``, + # existing data updates by the title the service currently stores them under (a qualified + # ``group▸title`` if grouped, a plain title if not). Data is never renamed, so validate the table + # directly against those titles. + expected_column_titles = set(new_columns) | update_columns missing = expected_column_titles - data_type_map.keys() if missing: raise MissingColumnInTable( @@ -954,26 +943,16 @@ def _expected_column_title(column: str) -> str: "To tag existing columns, use update_column_metadata." ) - # A group assignment requires the column's data to be (re-)uploaded in this same request, so - # column_groups may only reference new columns or existing columns being updated. Moving a - # column without its data is rejected by the service, so there is no metadata-only path. - movable_columns = set(new_columns) | update_columns - unknown_group_columns = set(column_groups) - movable_columns + # ``column_groups`` here only assigns *new* columns to a group as they are added. Moving or + # ungrouping an existing column is a metadata-only operation; use update_column_metadata. + unknown_group_columns = set(column_groups) - set(new_columns) if unknown_group_columns: raise MissingColumnInTable( - f"column_groups reference columns that are neither new nor being updated: {unknown_group_columns}. " - "A column's group can only be changed when its data is re-uploaded, so the column must be listed " - "in new_columns (as its qualified title) or update_columns (as its current qualified title)." + f"column_groups reference columns that are not in new_columns: {unknown_group_columns}. " + "column_groups only groups new columns; to move or ungroup an existing column use " + "update_column_metadata." ) - # An existing column is moved by pairing its re-uploaded data with an update_metadata group change, - # addressed by the column's CURRENT qualified title (the same reference used in columns.update). - update_metadata_entries = [ - models.UpdateMetadataLite(title=column, values=models.UpdateMetadataValuesLite(group=group)) - for column, group in column_groups.items() - if column in update_columns - ] - columns = models.UpdateColumnsLite( new=[ models.ColumnLite( @@ -989,8 +968,6 @@ def _expected_column_title(column: str) -> str: delete=list(delete_columns), rename=[], ) - if update_metadata_entries: - columns.update_metadata = update_metadata_entries update_response = await self._column_operations_api.update_block_model_from_latest_version( org_id=str(self._environment.org_id), workspace_id=str(self._environment.workspace_id), @@ -1038,11 +1015,11 @@ async def update_block_model_columns( :param units: A dictionary mapping column names within `data` to units. :param tags: A dictionary mapping new column names to their tags object. Column tags are a preview feature; the client must be constructed with ``preview=True`` to use them. - :param column_groups: A dictionary assigning columns to groups. For a **new** column, map its qualified - title (its key in `data`) to the group it belongs to. To **move** an *existing* column, map its - **current** qualified title to the new group (or ``""`` to ungroup); the column must also be listed in - ``update_columns`` and its data supplied under the **new** title. Column groups are a preview feature; - the client must be constructed with ``preview=True`` to use them. + :param column_groups: A dictionary assigning **new** columns to groups: map a new column's qualified + title (its key in `data`, e.g. ``"Assays▸Cu"``) to the qualified title of the group it belongs to. + To move or ungroup an *existing* column, use :meth:`update_column_metadata` instead — a group change is + metadata-only and does not require re-uploading data. Column groups are a preview feature; the client + must be constructed with ``preview=True`` to use them. :param: update_type: Provide the type of update. Either 'replace' or 'merge' (default: replace) :raises CacheNotConfiguredException: If the cache is not configured. :return: The new version of the block model with the added columns. @@ -1102,11 +1079,11 @@ async def update_subblocked_columns( the block model's own ``fill_subblocks`` setting is used. :param tags: A dictionary mapping new column names to their tags object. Column tags are a preview feature; the client must be constructed with ``preview=True`` to use them. - :param column_groups: A dictionary assigning columns to groups. For a **new** column, map its qualified - title (its key in `data`) to the group it belongs to. To **move** an *existing* column, map its - **current** qualified title to the new group (or ``""`` to ungroup); the column must also be listed in - ``update_columns`` and its data supplied under the **new** title. Column groups are a preview feature; - the client must be constructed with ``preview=True`` to use them. + :param column_groups: A dictionary assigning **new** columns to groups: map a new column's qualified + title (its key in `data`, e.g. ``"Assays▸Cu"``) to the qualified title of the group it belongs to. + To move or ungroup an *existing* column, use :meth:`update_column_metadata` instead — a group change is + metadata-only and does not require re-uploading data. Column groups are a preview feature; the client + must be constructed with ``preview=True`` to use them. :param: update_type: Provide the type of update. Either 'replace' or 'merge' (default: replace) """ return await self._update_columns( @@ -1137,21 +1114,22 @@ async def update_column_metadata( - A ``str`` sets the column's unit ID. - ``None`` clears the column's unit ID. - - A :class:`ColumnMetadataUpdate` sets any combination of unit ID and/or tags. Only the + - A :class:`ColumnMetadataUpdate` sets any combination of unit ID, tags and/or group. Only the fields explicitly set on the object are sent; unset fields are left untouched. Set - ``tags={}`` to clear a column's tags or ``unit_id=None`` to clear its unit. + ``tags={}`` to clear a column's tags, ``unit_id=None`` to clear its unit, or ``group=""`` to + move the column out of any group. - A column's group cannot be changed here: the service requires the column's data to be re-uploaded - when its group membership changes. Use the ``column_groups`` parameter on - :meth:`update_block_model_columns` / :meth:`update_subblocked_columns` (listing the column in - ``update_columns``) to move an existing column. + A column's group is metadata, so it can be moved (or ungrouped) here without re-uploading its + data. Address the column by the title the service currently stores it under: its qualified title + (``group▸title``) if it is currently grouped, or its plain title if it is not. Set + ``ColumnMetadataUpdate(group=...)`` to the target group's qualified title (or ``""`` to ungroup). Column tags are a preview feature; the client must be constructed with ``preview=True`` to use them. :param bm_id: The ID of the block model to update. :param column_updates: A dictionary mapping column titles to their metadata update. Example: {"Cu": "%[mass]", "Au": None, - "Ag": ColumnMetadataUpdate(tags={"source": "assay"})} + "Assays▸Ag": ColumnMetadataUpdate(group="Geology")} :param comment: An optional comment describing the metadata changes. This is max 250 characters. :return: The new version of the block model with updated metadata. """ @@ -1192,10 +1170,10 @@ async def update_groups( combination of ``new``, ``update`` and ``delete`` can be supplied in a single call. Groups are addressed by their qualified title (a bare title for a top-level group, or segments - joined by ``▸`` for a nested group). To assign columns to a group, use the ``column_groups`` - parameter on the column methods (for new columns, or existing columns whose data is re-uploaded - in the same call). To resolve a written group back to its server-assigned UUID and resolved policy, - use the helpers on the returned :class:`~evo.blockmodels.data.Version`, e.g. + joined by ``▸`` for a nested group). To assign a *new* column to a group, use the ``column_groups`` + parameter on the column methods; to move or ungroup an *existing* column, use + :meth:`update_column_metadata`. To resolve a written group back to its server-assigned UUID and + resolved policy, use the helpers on the returned :class:`~evo.blockmodels.data.Version`, e.g. :meth:`~evo.blockmodels.data.Version.group_by_qualified_title`. Column groups are a preview feature; the client must be constructed with ``preview=True`` to use them. diff --git a/packages/evo-blockmodels/src/evo/blockmodels/data.py b/packages/evo-blockmodels/src/evo/blockmodels/data.py index 68204688..ee7f66cf 100644 --- a/packages/evo-blockmodels/src/evo/blockmodels/data.py +++ b/packages/evo-blockmodels/src/evo/blockmodels/data.py @@ -121,6 +121,13 @@ class ColumnMetadataUpdate(CustomBaseModel): unit_id: str | None = None """The new unit ID for the column.""" + group: str | None = None + """The qualified title of the new group for the column (a bare title for a top-level group, or + segments joined by ``▸`` for a nested group). Send ``""`` to move the column out of any group. + Omit this field to leave the column's current group unchanged. + + Column groups are a preview feature; the client must be constructed with ``preview=True`` to use them.""" + tags: dict[str, Any] | None = None """Replacement tags for the column. Send a populated object to replace the column's tags wholesale, or ``{}`` to clear them. Omit this field to leave the existing tags untouched. diff --git a/packages/evo-blockmodels/tests/test_group_helpers.py b/packages/evo-blockmodels/tests/test_group_helpers.py index 7fac67b7..e404db1b 100644 --- a/packages/evo-blockmodels/tests/test_group_helpers.py +++ b/packages/evo-blockmodels/tests/test_group_helpers.py @@ -176,15 +176,25 @@ def test_group_metadata_update_forbids_title_field(self) -> None: with self.assertRaises(ValidationError): GroupMetadataUpdate(title="X") - def test_column_metadata_update_forbids_group_field(self) -> None: - # A column's group can only change when its data is re-uploaded, so metadata-only group moves - # are not supported. ``group`` must be rejected rather than silently forwarded onto the wire. + def test_column_metadata_update_accepts_group_field(self) -> None: + # A column's group is metadata and can be changed without re-uploading data, so ``group`` is a + # supported field. Only fields explicitly set are forwarded onto the wire. + from evo.blockmodels.data import ColumnMetadataUpdate + + update = ColumnMetadataUpdate(group="Assays") + self.assertEqual(update.group, "Assays") + self.assertEqual(update.model_dump(exclude_unset=True), {"group": "Assays"}) + + # An empty string ungroups the column. + self.assertEqual(ColumnMetadataUpdate(group="").model_dump(exclude_unset=True), {"group": ""}) + + def test_column_metadata_update_forbids_unknown_field(self) -> None: from pydantic import ValidationError from evo.blockmodels.data import ColumnMetadataUpdate with self.assertRaises(ValidationError): - ColumnMetadataUpdate(group="Assays") + ColumnMetadataUpdate(not_a_field="x") class TestQualifyColumnTitles(unittest.TestCase): diff --git a/packages/evo-blockmodels/tests/test_update.py b/packages/evo-blockmodels/tests/test_update.py index 9efaf83c..13697a44 100644 --- a/packages/evo-blockmodels/tests/test_update.py +++ b/packages/evo-blockmodels/tests/test_update.py @@ -898,6 +898,75 @@ async def test_update_column_metadata_with_preview_sends_header(self) -> None: ) self.assertEqual(version.version_id, 2) + async def test_update_column_metadata_moves_column_between_groups(self) -> None: + """An existing grouped column is moved to another group as a metadata-only update (no data).""" + self.transport.set_request_handler( + UpdateRequestHandler( + update_result=UPDATE_RESULT, + job_response=JobResponse(job_status=JobStatus.COMPLETE, payload=UPDATED_VERSION), + ) + ) + await self.preview_client_without_cache.update_column_metadata( + BM_UUID, + # The column is addressed by its current qualified title; the group is set to the new path. + column_updates={"Assays\u25b8col1": ColumnMetadataUpdate(group="Geology")}, + ) + + expected_update_body = models.UpdateDataLite1( + columns=models.UpdateColumnsLite( + new=[], + update=[], + rename=[], + delete=[], + update_metadata=[ + models.UpdateMetadataLite( + title="Assays\u25b8col1", values=models.UpdateMetadataValuesLite(group="Geology") + ), + ], + ), + comment=None, + ) + self.assert_any_request_made( + method=RequestMethod.PATCH, + path=f"{self.base_path}/block-models/{BM_UUID}/blocks", + body=expected_update_body.model_dump(mode="json", exclude_unset=True), + headers=DEFAULT_EXPECTED_HEADERS | {"API-Preview": "opt-in"}, + ) + + async def test_update_column_metadata_ungroups_column(self) -> None: + """An existing grouped column is ungrouped as a metadata-only update by sending ``group=""``.""" + self.transport.set_request_handler( + UpdateRequestHandler( + update_result=UPDATE_RESULT, + job_response=JobResponse(job_status=JobStatus.COMPLETE, payload=UPDATED_VERSION), + ) + ) + await self.bms_client_without_cache.update_column_metadata( + BM_UUID, + column_updates={"Assays\u25b8col2": ColumnMetadataUpdate(group="")}, + ) + + expected_update_body = models.UpdateDataLite1( + columns=models.UpdateColumnsLite( + new=[], + update=[], + rename=[], + delete=[], + update_metadata=[ + models.UpdateMetadataLite( + title="Assays\u25b8col2", values=models.UpdateMetadataValuesLite(group="") + ), + ], + ), + comment=None, + ) + self.assert_any_request_made( + method=RequestMethod.PATCH, + path=f"{self.base_path}/block-models/{BM_UUID}/blocks", + body=expected_update_body.model_dump(mode="json", exclude_unset=True), + headers=DEFAULT_EXPECTED_HEADERS, + ) + async def test_rename_block_model_columns(self) -> None: self.transport.set_request_handler( UpdateRequestHandler( @@ -1178,182 +1247,9 @@ async def test_update_columns_with_unknown_group_column(self) -> None: BM_UUID, REGULAR_DATA, new_columns=["col1"], - column_groups={"col2": "Assays"}, # col2 neither new nor updated - ) - - async def test_update_block_model_columns_moves_existing_column_group(self) -> None: - """An existing column is moved to another group by re-uploading its data with a group change.""" - self.transport.set_request_handler( - UpdateRequestHandler( - update_result=UPDATE_RESULT, - job_response=JobResponse(job_status=JobStatus.COMPLETE, payload=UPDATED_VERSION), - ) - ) - with ( - mock.patch("evo.common.io.upload.StorageDestination") as mock_destination, - mock.patch("pyarrow.parquet.write_table", wraps=pyarrow.parquet.write_table) as mock_write, - ): - mock_destination.upload_file = mock.AsyncMock() - # col1 is currently ungrouped; its data is re-uploaded under its NEW qualified title. - data = pyarrow.table( - { - "i": [1, 2, 3], - "j": [4, 5, 6], - "k": [7, 8, 9], - "Assays\u25b8col1": ["A", "B", "B"], - "col2": [4.5, 5.3, 6.2], - } - ) - await self.bms_client.update_block_model_columns( - BM_UUID, - data, - new_columns=[], - update_columns={"col1"}, # current (bare) title of the ungrouped column - column_groups={"col1": "Assays"}, # move existing col1 into a group - ) - - expected_update_body = models.UpdateDataLite1( - columns=models.UpdateColumnsLite( - new=[], - update=["col1"], - rename=[], - delete=[], - update_metadata=[ - models.UpdateMetadataLite(title="col1", values=models.UpdateMetadataValuesLite(group="Assays")), - ], - ), - update_type=models.UpdateType.replace, - geometry_change=None, - fill_subblocks=None, - ) - self.assert_any_request_made( - method=RequestMethod.PATCH, - path=f"{self.base_path}/block-models/{BM_UUID}/blocks", - body=expected_update_body.model_dump(mode="json", exclude_unset=True), - headers=DEFAULT_EXPECTED_HEADERS, - ) - - uploaded_table = mock_write.call_args.args[0] - # col1 is uploaded under its new qualified title so the service binds its re-uploaded data. - self.assertEqual(uploaded_table.schema.names, ["i", "j", "k", "Assays\u25b8col1", "col2"]) - - async def test_update_block_model_columns_ungroups_existing_column(self) -> None: - """An existing grouped column is moved out of its group by re-uploading its data with ``group=""``.""" - self.transport.set_request_handler( - UpdateRequestHandler( - update_result=UPDATE_RESULT, - job_response=JobResponse(job_status=JobStatus.COMPLETE, payload=UPDATED_VERSION), - ) - ) - with ( - mock.patch("evo.common.io.upload.StorageDestination") as mock_destination, - mock.patch("pyarrow.parquet.write_table", wraps=pyarrow.parquet.write_table) as mock_write, - ): - mock_destination.upload_file = mock.AsyncMock() - # col2 currently lives in "Assays"; ungrouping re-uploads its data under its NEW plain title. - data = pyarrow.table( - { - "i": [1, 2, 3], - "j": [4, 5, 6], - "k": [7, 8, 9], - "col1": ["A", "B", "B"], - "col2": [4.5, 5.3, 6.2], - } - ) - await self.bms_client.update_block_model_columns( - BM_UUID, - data, - new_columns=[], - update_columns={"Assays\u25b8col2"}, # current qualified title of the grouped column - column_groups={"Assays\u25b8col2": ""}, # ungroup existing col2 - ) - - # col2 is currently grouped, so the service identifies it by its current qualified title. - expected_update_body = models.UpdateDataLite1( - columns=models.UpdateColumnsLite( - new=[], - update=["Assays\u25b8col2"], - rename=[], - delete=[], - update_metadata=[ - models.UpdateMetadataLite( - title="Assays\u25b8col2", values=models.UpdateMetadataValuesLite(group="") - ), - ], - ), - update_type=models.UpdateType.replace, - geometry_change=None, - fill_subblocks=None, - ) - self.assert_any_request_made( - method=RequestMethod.PATCH, - path=f"{self.base_path}/block-models/{BM_UUID}/blocks", - body=expected_update_body.model_dump(mode="json", exclude_unset=True), - headers=DEFAULT_EXPECTED_HEADERS, + column_groups={"col2": "Assays"}, # col2 is not a new column ) - uploaded_table = mock_write.call_args.args[0] - # The now-ungrouped column is uploaded under its new plain title. - self.assertEqual(uploaded_table.schema.names, ["i", "j", "k", "col1", "col2"]) - - async def test_update_block_model_columns_moves_currently_grouped_column(self) -> None: - """An already-grouped column is moved to another group, referenced by its current qualified title.""" - self.transport.set_request_handler( - UpdateRequestHandler( - update_result=UPDATE_RESULT, - job_response=JobResponse(job_status=JobStatus.COMPLETE, payload=UPDATED_VERSION), - ) - ) - with ( - mock.patch("evo.common.io.upload.StorageDestination") as mock_destination, - mock.patch("pyarrow.parquet.write_table", wraps=pyarrow.parquet.write_table) as mock_write, - ): - mock_destination.upload_file = mock.AsyncMock() - # col1 currently lives in "Assays"; moving it re-uploads its data under its NEW qualified title. - data = pyarrow.table( - { - "i": [1, 2, 3], - "j": [4, 5, 6], - "k": [7, 8, 9], - "Geology\u25b8col1": ["A", "B", "B"], - "col2": [4.5, 5.3, 6.2], - } - ) - await self.bms_client.update_block_model_columns( - BM_UUID, - data, - new_columns=[], - update_columns={"Assays\u25b8col1"}, # current qualified title - column_groups={"Assays\u25b8col1": "Geology"}, # move col1 from "Assays" to "Geology" - ) - - expected_update_body = models.UpdateDataLite1( - columns=models.UpdateColumnsLite( - new=[], - update=["Assays\u25b8col1"], # current reference - rename=[], - delete=[], - update_metadata=[ - models.UpdateMetadataLite( - title="Assays\u25b8col1", values=models.UpdateMetadataValuesLite(group="Geology") - ), - ], - ), - update_type=models.UpdateType.replace, - geometry_change=None, - fill_subblocks=None, - ) - self.assert_any_request_made( - method=RequestMethod.PATCH, - path=f"{self.base_path}/block-models/{BM_UUID}/blocks", - body=expected_update_body.model_dump(mode="json", exclude_unset=True), - headers=DEFAULT_EXPECTED_HEADERS, - ) - - uploaded_table = mock_write.call_args.args[0] - # The moved column is uploaded under its NEW qualified title. - self.assertEqual(uploaded_table.schema.names, ["i", "j", "k", "Geology\u25b8col1", "col2"]) - async def test_update_block_model_columns_data_only_update_of_grouped_column(self) -> None: """A plain data update of an already-grouped column references it by its current qualified title.""" self.transport.set_request_handler( @@ -1407,11 +1303,6 @@ async def test_update_block_model_columns_data_only_update_of_grouped_column(sel # The grouped column keeps its current qualified title so its data binds correctly. self.assertEqual(uploaded_table.schema.names, ["i", "j", "k", "Assays\u25b8col1", "col2"]) - async def test_column_metadata_update_rejects_group(self) -> None: - """Group membership can no longer be changed through metadata; the service requires re-uploaded data.""" - with self.assertRaises(ValueError): - ColumnMetadataUpdate(group="Assays") - async def test_update_groups_create(self) -> None: self.transport.set_request_handler( UpdateRequestHandler(