diff --git a/AGENTS.md b/AGENTS.md index 6c30c82..cfbea45 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -39,7 +39,7 @@ The DataHub REST API this SDK targets is a separate Spring Boot project; the HTT This crate is a thin async HTTP SDK around a DataHub-style REST API. Entry point is `create_api_service()` in `src/lib.rs`, which returns an `Arc` built with `Arc::new_cyclic` so each subservice holds a `Weak` back-reference. Subservices are fields on `ApiService`: -- `time_series` (`src/timeseries/`) — `TimeSeries` + datapoint ingestion/retrieval +- `time_series` (`src/timeseries/`) — `TimeSeries` + datapoint ingestion/retrieval. Neither `TimeSeries` nor `TimeSeriesUpdate` has **`securityCategories`**: it was stored, writable and returned, but nothing ever read it — no part in access control (dataset grants are Keycloak organization groups), no query filtering on it, and the backend silently dropped any id that did not already exist, so the field never round-tripped. It has been removed server-side along with its join table, and the api reads request bodies strictly, so sending it is now a 400. Files keep their own `securityCategories` (`INode` in `src/generic.rs`) — separate entity, separate question. `ListFieldU64` went with it: it was the only field of that type, so the Python wrapper class is gone too (`ListFieldStr` and `ListFieldIdCollection` remain). - `units` (`src/unit/`) - `events` (`src/events/`) — event CRUD, filter/search, plus the vocabulary endpoints (`list_types`/`search_types` and the same pair for sub-types, statuses and sources, over `EventDimension`). Those answer "what values does this tenant actually use" for the four categorical fields and back filter dropdowns; they read small server-side dimension tables rather than scanning events, so they are cheap but *eventually consistent* with the events. Note the route asymmetry the SDK hides: `/events/list/{plural}` but `/events/search/{singular}`. `EventUpdate` has **no `event_time`**: an event's time is immutable after creation — the events table is partitioned by it, so ClickHouse refuses the mutation outright, and the api used to validate the field, echo the new value back with a 200 and then fail to apply it. It has been dropped from the update form, so sending it is now a 400. Record a corrected time as a new event. - `resources` (`src/resources/`) — hierarchical asset-like entities; relationship edges live in `src/relations/` (`EdgeProxy`, `RelForm`, `RelatedNode`) diff --git a/datahub_python_bindings/python/intellistream_datahub_sdk/__init__.pyi b/datahub_python_bindings/python/intellistream_datahub_sdk/__init__.pyi index 509c2b3..855a56d 100644 --- a/datahub_python_bindings/python/intellistream_datahub_sdk/__init__.pyi +++ b/datahub_python_bindings/python/intellistream_datahub_sdk/__init__.pyi @@ -299,13 +299,6 @@ class FieldGeoJson: # An update is either a replace (`set`) or a delta (`add`/`remove`), never both. The two # constructors make the illegal mix unrepresentable; there is no bare initializer. -class ListFieldU64: - @classmethod - def set(cls, values: list[int]) -> ListFieldU64: ... - @classmethod - def delta(cls, add: list[int] | None = None, remove: list[int] | None = None) -> ListFieldU64: ... - - class ListFieldStr: @classmethod def set(cls, values: list[str]) -> ListFieldStr: ... @@ -344,7 +337,6 @@ class TimeSeries: unit_external_id: str | None = None, description: str | None = None, metadata: dict[str, str] | None = None, - security_categories: list[int] | None = None, data_set_id: int | None = None, id: int | None = None, related_resources: list[RelatedNode] | None = None, @@ -385,10 +377,6 @@ class TimeSeries: @metadata.setter def metadata(self, value: dict[str, str] | None) -> None: ... @property - def security_categories(self) -> list[int] | None: ... - @security_categories.setter - def security_categories(self, value: list[int] | None) -> None: ... - @property def data_set_id(self) -> int | None: ... @data_set_id.setter def data_set_id(self, value: int | None) -> None: ... @@ -452,7 +440,6 @@ class TimeSeriesUpdate: unit: FieldStr | None = None, description: FieldStr | None = None, unit_external_id: FieldStr | None = None, - security_categories: ListFieldU64 | None = None, data_set_id: FieldU64 | None = None, source: FieldStr | None = None, ) -> None: ... @@ -473,8 +460,6 @@ class TimeSeriesUpdate: @property def unit_external_id(self) -> FieldStr: ... @property - def security_categories(self) -> ListFieldU64: ... - @property def data_set_id(self) -> FieldU64: ... @property def source(self) -> FieldStr: ... diff --git a/datahub_python_bindings/src/events/mod.rs b/datahub_python_bindings/src/events/mod.rs index 1257433..263ffa7 100644 --- a/datahub_python_bindings/src/events/mod.rs +++ b/datahub_python_bindings/src/events/mod.rs @@ -353,8 +353,8 @@ impl From for EventIdCollection { /// One event's update for `events.update`. Target the event by an `Event`, its UUID `id`, or its /// `external_id`; every field is optional and uses the same wrappers as the other services -/// (`FieldStr`/`FieldU64` for scalars, `ListFieldU64`/`ListFieldStr` for the related-resource -/// lists, `MapField` for metadata). Mirrors `ResourceUpdate`. +/// (`FieldStr`/`FieldU64` for scalars, `ListFieldIdCollection` for the related-resource list, +/// `MapField` for metadata). Mirrors `ResourceUpdate`. #[pyclass(module = "intellistream_datahub_sdk", name = "EventUpdate")] #[derive(Clone)] pub struct PyEventUpdate { diff --git a/datahub_python_bindings/src/lib.rs b/datahub_python_bindings/src/lib.rs index 571eb85..61dcea7 100644 --- a/datahub_python_bindings/src/lib.rs +++ b/datahub_python_bindings/src/lib.rs @@ -954,33 +954,6 @@ impl DatahubIdentity for Identifiable { } } -#[pyclass(module = "intellistream_datahub_sdk", name = "ListFieldU64")] -#[derive(Clone, Debug)] -pub struct PyListFieldU64(ListField); -impl From> for PyListFieldU64 { - fn from(ts: ListField) -> Self { - Self(ts) - } -} -impl From for ListField { - fn from(ts: PyListFieldU64) -> Self { - ts.0 - } -} -#[pymethods] -impl PyListFieldU64 { - /// Replace the whole list. - #[classmethod] - pub fn set(_cls: Py, values: Vec) -> Self { - Self(ListField::set(values)) - } - /// Add and/or remove entries, keeping the rest. Pass `add`, `remove`, or both. - #[classmethod] - #[pyo3(signature=(add=None, remove=None))] - pub fn delta(_cls: Py, add: Option>, remove: Option>) -> Self { - Self(ListField::delta(add, remove)) - } -} #[pyclass(module = "intellistream_datahub_sdk", name = "ListFieldStr")] #[derive(Clone, Debug)] pub struct PyListFieldStr(ListField); @@ -1241,7 +1214,6 @@ fn _core(m: &Bound<'_, PyModule>) -> PyResult<()> { m.add_class::()?; m.add_class::()?; m.add_class::()?; - m.add_class::()?; m.add_class::()?; m.add_class::()?; m.add_class::()?; diff --git a/datahub_python_bindings/src/timeseries/construction.rs b/datahub_python_bindings/src/timeseries/construction.rs index c7cc269..393987c 100644 --- a/datahub_python_bindings/src/timeseries/construction.rs +++ b/datahub_python_bindings/src/timeseries/construction.rs @@ -35,7 +35,6 @@ impl PyTimeSeries { description = None, unit = None, unit_external_id = None, - security_categories = None, data_set_id = None, related_resources = None, source = None @@ -49,7 +48,6 @@ impl PyTimeSeries { description: Option, unit: Option, unit_external_id: Option, - security_categories: Option>, data_set_id: Option, related_resources: Option>, source: Option, @@ -75,7 +73,6 @@ impl PyTimeSeries { unit, description, unit_external_id, - security_categories, data_set_id, value_type: value_type.to_string(), source, diff --git a/datahub_python_bindings/src/timeseries/general.rs b/datahub_python_bindings/src/timeseries/general.rs index 7c3c743..640564f 100644 --- a/datahub_python_bindings/src/timeseries/general.rs +++ b/datahub_python_bindings/src/timeseries/general.rs @@ -62,14 +62,6 @@ impl PyTimeSeries { self.inner.unit_external_id = value; } #[getter] - pub fn security_categories(&self) -> Option<&Vec> { - self.inner.security_categories.as_ref() - } - #[setter] - pub fn set_security_categories(&mut self, value: Option>) { - self.inner.security_categories = value; - } - #[getter] pub fn data_set_id(&self) -> Option { self.inner.data_set_id } diff --git a/datahub_python_bindings/src/timeseries/mod.rs b/datahub_python_bindings/src/timeseries/mod.rs index 3b863b0..b2971f9 100644 --- a/datahub_python_bindings/src/timeseries/mod.rs +++ b/datahub_python_bindings/src/timeseries/mod.rs @@ -7,7 +7,7 @@ use crate::timeseries::datapoints::{ }; use crate::timeseries::sync_service::PyTimeSeriesServiceSync; use crate::{ - DatahubIdentity, Identifiable, PyFieldStr, PyFieldU64, PyIdCollection, PyListFieldU64, + DatahubIdentity, Identifiable, PyFieldStr, PyFieldU64, PyIdCollection, PyMapField, }; use chrono::{DateTime, Utc}; @@ -69,9 +69,6 @@ pub mod sync_service; /// External id for the units of the timeseries this is used to connecto to datahub units system. /// The units system will allow you to convert between units and easily convert between units-systems for unified storage. /// -/// security_categories: list[int], default None -/// Currenty not supported used. -/// /// data_set_id: int /// the id of the datasets this timeseries belongs to /// related_resources: list[RelatedNode] @@ -256,7 +253,6 @@ impl PyTimeSeriesUpdate { unit=None, description=None, unit_external_id=None, - security_categories=None, data_set_id=None, source=None, ))] @@ -269,7 +265,6 @@ impl PyTimeSeriesUpdate { unit: Option, description: Option, unit_external_id: Option, - security_categories: Option, data_set_id: Option, source: Option, ) -> PyResult { @@ -281,7 +276,6 @@ impl PyTimeSeriesUpdate { unit: unit.map(|s| s.0).unwrap_or_default(), description: description.map(|s| s.0).unwrap_or_default(), unit_external_id: unit_external_id.map(|s| s.0).unwrap_or_default(), - security_categories: security_categories.map(|s| s.0).unwrap_or_default(), data_set_id: data_set_id.map(|s| s.0).unwrap_or_default(), source: source.map(|s| s.0).unwrap_or_default(), }; @@ -326,10 +320,6 @@ impl PyTimeSeriesUpdate { self.inner.update.unit_external_id.clone().into() } #[getter] - fn security_categories(&self) -> PyListFieldU64 { - self.inner.update.security_categories.clone().into() - } - #[getter] fn data_set_id(&self) -> PyFieldU64 { self.inner.update.data_set_id.clone().into() } diff --git a/python_tests/test_entity_fields.py b/python_tests/test_entity_fields.py index 01b3850..8cdebc1 100644 --- a/python_tests/test_entity_fields.py +++ b/python_tests/test_entity_fields.py @@ -41,7 +41,6 @@ def test_minimal_constructor_and_defaults(self): assert ts.unit_external_id is None assert ts.description is None assert ts.metadata is None - assert ts.security_categories is None assert ts.data_set_id is None assert ts.source is None @@ -54,7 +53,6 @@ def test_full_constructor_round_trips_through_getters(self): unit_external_id="u-ext", description="a description", metadata={"k": "v"}, - security_categories=[1, 2], data_set_id=99, source="sap_pi", ) @@ -65,7 +63,6 @@ def test_full_constructor_round_trips_through_getters(self): assert ts.unit_external_id == "u-ext" assert ts.description == "a description" assert ts.metadata == {"k": "v"} - assert ts.security_categories == [1, 2] assert ts.data_set_id == 99 assert ts.source == "sap_pi" @@ -96,7 +93,6 @@ def test_setters_update_every_field(self): ts.unit_external_id = "u-ext" ts.description = "a description" ts.metadata = {"k": "v"} - ts.security_categories = [1, 2, 3] ts.data_set_id = 42 ts.source = "sap_pi" @@ -106,7 +102,6 @@ def test_setters_update_every_field(self): assert ts.unit_external_id == "u-ext" assert ts.description == "a description" assert ts.metadata == {"k": "v"} - assert ts.security_categories == [1, 2, 3] assert ts.data_set_id == 42 assert ts.source == "sap_pi" @@ -126,12 +121,10 @@ def test_optional_setters_accept_none(self): ts.data_set_id = None ts.description = None ts.unit_external_id = None - ts.security_categories = None ts.source = None assert ts.unit is None assert ts.metadata is None assert ts.data_set_id is None - assert ts.security_categories is None assert ts.source is None def test_invalid_value_type_raises(self): diff --git a/python_tests/test_resource_label_updates.py b/python_tests/test_resource_label_updates.py index 27337b1..09fb0d6 100644 --- a/python_tests/test_resource_label_updates.py +++ b/python_tests/test_resource_label_updates.py @@ -12,7 +12,7 @@ import time import pytest -from intellistream_datahub_sdk import ListFieldStr, ListFieldU64, MapField, Resource, ResourceUpdate +from intellistream_datahub_sdk import ListFieldIdCollection, ListFieldStr, MapField, Resource, ResourceUpdate from fixtures import async_client, make_resource, sync_client, unique_id @@ -47,7 +47,7 @@ def test_resource_update_requires_a_target(): def test_field_update_is_set_or_delta_only(): # `set` and `delta` are the only constructors; there is no bare initializer, so a replace can # never be built carrying an add/remove delta — the illegal mix is simply not expressible. - for wrapper in (ListFieldStr, ListFieldU64, MapField): + for wrapper in (ListFieldStr, ListFieldIdCollection, MapField): with pytest.raises(TypeError): wrapper() # no __init__ diff --git a/python_tests/test_timeseries_crud.py b/python_tests/test_timeseries_crud.py index a8aaaa9..916066b 100644 --- a/python_tests/test_timeseries_crud.py +++ b/python_tests/test_timeseries_crud.py @@ -6,7 +6,6 @@ * FieldStr / FieldU64 scalar fields -> set-value and set_null * MapField (metadata) -> add (merge), set (replace), remove (by key) - * ListFieldU64 (security_categories) -> add, set (replace), remove (by value) * value_type re-typing * targeting an update by created-object / external-id string / numeric id * multi-field updates, batch updates, and no-op updates @@ -304,73 +303,6 @@ def test_update_metadata_cleared_by_an_empty_set(sync_client, make_ts): assert not (sync_client.timeseries.update([update])[0].metadata or {}) -# --------------------------------------------------------------------------- # -# UPDATE — ListFieldU64 (security_categories): add / set / remove -# -# These exercise the three ListFieldU64 serialisation paths (set/add/remove). -# They are xfail because the backend silently drops arbitrary security-category -# ids — verified by creating a series with security_categories=[1, 2] and getting -# back securityCategories=[]. Real categories aren't creatable through this SDK, -# so persistence can't be asserted; strict=False surfaces an xpass if the backend -# starts honouring them. -# --------------------------------------------------------------------------- # - -_SEC_CAT_XFAIL = pytest.mark.xfail( - reason="backend does not persist arbitrary security-category ids " - "(needs pre-existing categories not creatable via this SDK)", - strict=False, -) - - -@_SEC_CAT_XFAIL -def test_update_security_categories_set(sync_client, make_ts): - ts = make_ts(security_categories=[1, 2]) - - update = intellistream_datahub_sdk.TimeSeriesUpdate( - ts, security_categories=intellistream_datahub_sdk.ListFieldU64.set([3, 4]) - ) - updated = sync_client.timeseries.update([update])[0] - assert sorted(updated.security_categories or []) == [3, 4] - - -@_SEC_CAT_XFAIL -def test_update_security_categories_add(sync_client, make_ts): - ts = make_ts(security_categories=[1, 2]) - - update = intellistream_datahub_sdk.TimeSeriesUpdate( - ts, security_categories=intellistream_datahub_sdk.ListFieldU64.delta(add=[3]) - ) - updated = sync_client.timeseries.update([update])[0] - assert set(updated.security_categories or []) >= {1, 2, 3} - - -@_SEC_CAT_XFAIL -def test_update_security_categories_remove(sync_client, make_ts): - ts = make_ts(security_categories=[1, 2, 3]) - - update = intellistream_datahub_sdk.TimeSeriesUpdate( - ts, security_categories=intellistream_datahub_sdk.ListFieldU64.delta(remove=[2]) - ) - updated = sync_client.timeseries.update([update])[0] - cats = set(updated.security_categories or []) - assert 2 not in cats - assert {1, 3} <= cats - - -def test_update_security_categories_cleared_by_an_empty_set(sync_client, make_ts): - """``ListFieldU64`` has no ``setNull`` either; an empty ``set`` empties the list. - - Not xfail: the backend drops arbitrary category ids on the way in, so the list is already - empty — clearing it is the one security-category assertion that holds either way. - """ - ts = make_ts(security_categories=[1, 2]) - - update = intellistream_datahub_sdk.TimeSeriesUpdate( - ts, security_categories=intellistream_datahub_sdk.ListFieldU64.set([]) - ) - assert not (sync_client.timeseries.update([update])[0].security_categories or []) - - # --------------------------------------------------------------------------- # # UPDATE — data_set_id (FieldU64): set value and clear # --------------------------------------------------------------------------- # diff --git a/src/timeseries/mod.rs b/src/timeseries/mod.rs index 278b13f..e1cbf03 100644 --- a/src/timeseries/mod.rs +++ b/src/timeseries/mod.rs @@ -2,7 +2,7 @@ mod test; use crate::buffer::DurableSpool; use crate::datahub::DataHubConfig; -use crate::fields::{Field, ListField, MapField}; +use crate::fields::{Field, MapField}; use crate::generic::{ ApiServiceProvider, DataWrapper, Datapoint, DatapointString, DatapointsCollection, DeleteFilter, IdAndExtId, RetrieveFilter, SearchAndFilterForm, @@ -580,8 +580,6 @@ pub struct TimeSeries { pub description: Option, #[serde(rename = "unitExternalId")] pub unit_external_id: Option, - #[serde(rename = "securityCategories")] - pub security_categories: Option>, #[serde(rename = "dataSetId")] #[serde(default, with = "crate::serde_helper::opt_string_id")] pub data_set_id: Option, @@ -613,7 +611,6 @@ impl TimeSeries { unit: None, description: None, unit_external_id: None, - security_categories: None, data_set_id: None, value_type: "float".to_string(), source: None, @@ -633,9 +630,6 @@ impl TimeSeries { unit: dict.get("units").map(|v| v.to_string()), description: dict.get("description").map(|v| v.to_string()), unit_external_id: dict.get("unitExternalId").map(|v| v.to_string()), - security_categories: dict - .get("securityCategories") - .map(|v| serde_json::from_str(v).unwrap()), data_set_id: dict.get("dataSetId").map(|v| v.parse::().unwrap()), value_type: dict.get("valueType").unwrap().to_string(), source: dict.get("source").map(|v| v.to_string()), @@ -679,11 +673,6 @@ impl TimeSeries { self } - pub fn set_security_categories(&mut self, security_categories: Vec) -> &mut TimeSeries { - self.security_categories = Some(security_categories); - self - } - pub fn set_data_set_id(&mut self, data_set_id: u64) -> &mut TimeSeries { self.data_set_id = Some(data_set_id); self @@ -725,8 +714,6 @@ pub struct TimeSeriesUpdateFields { pub description: Field, #[serde(rename = "unitExternalId")] pub unit_external_id: Field, - #[serde(rename = "securityCategories")] - pub security_categories: ListField, #[serde(rename = "dataSetId")] pub data_set_id: Field, pub source: Field, @@ -741,7 +728,6 @@ impl TimeSeriesUpdateFields { unit: Field::default(), description: Field::default(), unit_external_id: Field::default(), - security_categories: ListField::default(), data_set_id: Field::default(), source: Field::default(), }