From aa67a6f9b2ff5fdb3337b3a3665b91457c0954f4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Erik=20Bj=C3=A4reholt?= Date: Sat, 26 Sep 2026 23:39:09 +0200 Subject: [PATCH 1/2] fix(transform): don't clear the data of period_union's input events period_union cleared .data on the events it returned, and events that weren't merged with another were the input objects themselves. Queries share event objects between variables (filter_keyvals_regex returns the same objects), so e.g. aw-client's always_active_pattern query not_treat_as_afk = filter_keyvals_regex(events, "app", "Slack"); not_afk = period_union(not_afk, not_treat_as_afk); events = filter_period_intersect(events, not_afk); wiped app/title from the Slack window events in 'events'. aw-server-rust doesn't mutate its inputs. Return new events instead. --- aw_transform/filter_period_intersect.py | 11 +++++++---- tests/test_transforms.py | 15 +++++++++++++++ 2 files changed, 22 insertions(+), 4 deletions(-) diff --git a/aw_transform/filter_period_intersect.py b/aw_transform/filter_period_intersect.py index 8c8c5e9..4e5a204 100644 --- a/aw_transform/filter_period_intersect.py +++ b/aw_transform/filter_period_intersect.py @@ -126,10 +126,13 @@ def period_union(events1: List[Event], events2: List[Event]) -> List[Event]: merged_events[-1] = _replace_event_period(last_event, new_period) else: merged_events.append(e) - for event in merged_events: - # Clear data - event.data = {} - return merged_events + # Return new events without data. Clearing .data in place would also wipe + # the data of the input events it kept as-is, which the caller may still + # use (e.g. filter_keyvals_regex(events, ...) returns the same objects). + return [ + Event(id=e.id, timestamp=e.timestamp, duration=e.duration, data={}) + for e in merged_events + ] def union(events1: List[Event], events2: List[Event]) -> List[Event]: diff --git a/tests/test_transforms.py b/tests/test_transforms.py index b1855f6..b0bc621 100644 --- a/tests/test_transforms.py +++ b/tests/test_transforms.py @@ -206,6 +206,21 @@ def test_filter_period_intersect_zero_duration(): ) +def test_period_union_does_not_modify_inputs(): + """The input events keep their data (they're often shared with other variables)""" + now = datetime(2026, 1, 5, 9, 0, tzinfo=timezone.utc) + window = [ + Event(timestamp=now, duration=30, data={"app": "Slack"}), + Event(timestamp=now + timedelta(minutes=5), duration=10, data={"app": "Code"}), + ] + not_afk = [Event(timestamp=now + timedelta(minutes=5), duration=5, data={})] + slack = [e for e in window if e.data["app"] == "Slack"] # same objects + result = period_union(not_afk, slack) + assert all(e.data == {} for e in result) + assert [e.data for e in window] == [{"app": "Slack"}, {"app": "Code"}] + assert filter_period_intersect(window, result)[0].data == {"app": "Slack"} + + def test_period_union(): now = datetime.now(timezone.utc) From 05719b89d0d8df7be9081a38804053a3b8b4768b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Erik=20Bj=C3=A4reholt?= Date: Sat, 26 Sep 2026 23:42:10 +0200 Subject: [PATCH 2/2] fix(query): don't let categorize, tag and split_url_events modify other variables' events Query variables share Event objects (`other = events;`, or a filter that returns the same objects), and categorize, tag and split_url_events set keys in event.data in place. So in aw-webui's multidevice query not_afk = events; events = categorize(events, ...); the Android events in not_afk got $category as well. aw-server-rust has value semantics. The query wrappers now pass copies (a new Event and data dict per event, which is all these transforms modify). --- aw_query/functions.py | 20 +++++++++++++++++--- tests/test_query2.py | 41 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 58 insertions(+), 3 deletions(-) diff --git a/aw_query/functions.py b/aw_query/functions.py index 00a85fc..7c2ad39 100644 --- a/aw_query/functions.py +++ b/aw_query/functions.py @@ -64,6 +64,20 @@ def _verify_variable_is_type(variable, t): functions: Dict[str, TQueryFunction] = {} +def _copy_events(events: List[Event]) -> List[Event]: + """Copies of events for transforms that set keys in ``data`` in place. + + Query variables share event objects (``b = a;``, or a filter returning the + same objects), so modifying them in place would change other variables + too. aw-server-rust has value semantics. Only ``data`` is copied, one level + deep, which is all these transforms modify. + """ + return [ + Event(id=e.id, timestamp=e.timestamp, duration=e.duration, data=dict(e.data)) + for e in events + ] + + def q2_function(transform_func=None): """ Decorator used to register query functions. @@ -304,7 +318,7 @@ def q2_flood(events: list, pulsetime: float = 5) -> List[Event]: @q2_function(split_url_events) @q2_typecheck def q2_split_url_events(events: list) -> List[Event]: - return split_url_events(events) + return split_url_events(_copy_events(events)) @q2_function(simplify_string) @@ -337,7 +351,7 @@ def q2_categorize(events: list, classes: list): classes = [(_cls, Rule(rule_dict)) for _cls, rule_dict in classes] except ValueError as exc: raise QueryFunctionException(str(exc)) from None - return categorize(events, classes) + return categorize(_copy_events(events), classes) @q2_function(tag) @@ -347,4 +361,4 @@ def q2_tag(events: list, classes: list): classes = [(_cls, Rule(rule_dict)) for _cls, rule_dict in classes] except ValueError as exc: raise QueryFunctionException(str(exc)) from None - return tag(events, classes) + return tag(_copy_events(events), classes) diff --git a/tests/test_query2.py b/tests/test_query2.py index 21e536b..70b8964 100644 --- a/tests/test_query2.py +++ b/tests/test_query2.py @@ -351,6 +351,47 @@ def test_query2_categorize_invalid_priority(): query(qname, example_query, starttime, endtime, ds) +@pytest.mark.parametrize("datastore", param_datastore_objects()) +def test_query2_transforms_dont_modify_other_variables(datastore): + """Query variables have value semantics, like in aw-server-rust""" + starttime = iso8601.parse_date("1970-01-01") + endtime = iso8601.parse_date("1970-01-02") + with TempTestBucket(datastore) as bucket: + bucket.insert( + Event( + data={"app": "Slack", "url": "https://x.org/a"}, + timestamp=starttime, + duration=timedelta(seconds=10), + ) + ) + bid = bucket.bucket_id + rule = '{"type": "regex", "regex": "Slack"}' + for transform in [ + f'categorize(events, [[["Comms"], {rule}]])', + f'tag(events, [["Comms", {rule}]])', + "split_url_events(events)", + ]: + q = f""" + events = query_bucket("{bid}"); + other = events; + events = {transform}; + RETURN = other; + """ + result = query("test", q, starttime, endtime, datastore) + assert result[0]["data"] == {"app": "Slack", "url": "https://x.org/a"} + + # aw-client's always_active_pattern: period_union must not clear the + # data of the window events it was given + q = f""" + events = query_bucket("{bid}"); + slack = filter_keyvals_regex(events, "app", "Slack"); + not_afk = period_union([], slack); + RETURN = filter_period_intersect(events, not_afk); + """ + result = query("test", q, starttime, endtime, datastore) + assert result[0]["data"]["app"] == "Slack" + + @pytest.mark.parametrize("datastore", param_datastore_objects()) def test_query2_function_in_function(datastore): qname = "asd"