fix: Iceberg lookup double-encodes user_meta (0.19.1)

CHANGELOG.md

@@ -5,6 +5,12 @@ All notable changes to this project will be documented in this file.
 
 ## [Unreleased]
 
+## [0.19.1] - 2026-07-14
+
+### Fixed
+
+- Bucketless Iceberg lookup no longer double-encodes `user_meta`. The query projected `json_format(CAST(m.metadata AS JSON))`, but Trino's VARCHAR→JSON cast wraps the JSON document string as a JSON *string value* instead of parsing it — so every returned row logged "Athena metadata JSON was not an object" and the result's metadata silently degraded to the matched `{key: value}` fallback (canvas rendering was unaffected). The projection now selects the column raw, matching the legacy `_packages-view` path, and `_parse_user_meta` defensively decodes a double-encoded string (#399)
+
 ## [0.19.0] - 2026-07-03
 
 ### Added

docker/app-manifest.yaml

@@ -2,7 +2,7 @@ manifestVersion: 1
 info:
   name: nightly-quilttest-com
   description: Packaging Benchling Notebooks as Quilt packages
-  version: 0.19.0
+  version: 0.19.1
 features:
   - name: Quilt Connector
     id: quilt_entry

docker/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "benchling-quilt-integration"
-version = "0.19.0"
+version = "0.19.1"
 description = "Benchling-Quilt Integration Webhook Service"
 license = {text = "Apache-2.0"}
 authors = [

docker/src/package_query.py

@@ -232,6 +232,13 @@ def _parse_user_meta(self, raw_meta: Optional[str], *, pkg_name: str, key: str,
 
         try:
             parsed = json.loads(raw_meta)
+            if isinstance(parsed, str):
+                # Double-encoded (e.g. a JSON string wrapping the document, as
+                # produced by Trino's VARCHAR→JSON cast, #399): decode once more.
+                try:
+                    parsed = json.loads(parsed)
+                except json.JSONDecodeError:
+                    pass  # genuinely a string value; falls to the warning below
             if isinstance(parsed, dict):
                 return parsed
             self.logger.warning(
@@ -490,6 +497,13 @@ def _build_iceberg_union_query(self, buckets: List[str], key: str, value: str) -
         ``TYPE_MISMATCH: Expression m.metadata is not of type ROW``, so we use
         json_extract_scalar just like the parquet _packages-view path.
 
+        The projection selects the column raw (``m.metadata AS user_meta``),
+        also like the parquet path. Do NOT wrap it in
+        ``json_format(CAST(m.metadata AS JSON))``: Trino's VARCHAR→JSON cast
+        does not parse the string — it wraps it as a JSON *string value* —
+        so json_format double-encodes and json.loads yields a str, not a
+        dict (#399).
+
         Args:
             buckets: List of bucket names to search
             key: Metadata key to filter on (JSON path field name)
@@ -507,7 +521,7 @@ def _build_iceberg_union_query(self, buckets: List[str], key: str, value: str) -
                 r.pkg_name,
                 r.timestamp,
                 m.message,
-                json_format(CAST(m.metadata AS JSON)) AS user_meta,
+                m.metadata AS user_meta,
                 '{b}' AS _src_bucket
             FROM "{idb}"."{b}_package_revision" r
             JOIN "{idb}"."{b}_package_manifest" m ON r.top_hash = m.top_hash

docker/tests/test_package_query.py

@@ -1,5 +1,6 @@
 """Tests for package_query module."""
 
+import json
 from unittest.mock import Mock, patch
 
 import pytest
@@ -417,8 +418,11 @@ def test_iceberg_build_union_query(self, mock_role_manager_class):
         # Should reference both buckets
         assert "bucket-a" in sql
         assert "bucket-b" in sql
-        # Should serialize metadata deterministically for Python parsing
-        assert "json_format(CAST(m.metadata AS JSON)) AS user_meta" in sql
+        # metadata is already a JSON document string; project it raw. Wrapping
+        # it in json_format(CAST(... AS JSON)) double-encodes, because Trino's
+        # VARCHAR→JSON cast wraps the string instead of parsing it (#399).
+        assert "m.metadata AS user_meta" in sql
+        assert "json_format" not in sql
         # metadata is a JSON string (from user_meta), not a native STRUCT, so
         # it must be filtered via json_extract_scalar to avoid TYPE_MISMATCH.
         assert "json_extract_scalar(m.metadata, '$.experiment_id')" in sql
@@ -506,6 +510,52 @@ def test_iceberg_non_object_metadata_keeps_package_match(self, mock_role_manager
             "experiment_id": "EXP-1",
         }
 
+    @patch("src.package_query.RoleManager")
+    def test_iceberg_double_encoded_metadata_parses(self, mock_role_manager_class):
+        """Double-encoded user_meta (a JSON string wrapping the document, as
+        produced by Trino's VARCHAR→JSON cast, #399) must round-trip to the
+        full metadata dict, not the {key: value} fallback."""
+        mock_athena = Mock()
+        mock_glue = Mock()
+
+        mock_role_manager = Mock()
+        mock_session = Mock()
+        mock_session.client.side_effect = lambda service, **kw: {
+            "athena": mock_athena,
+            "glue": mock_glue,
+        }[service]
+        mock_role_manager._get_or_create_session.return_value = (mock_session, None)
+        mock_role_manager.role_arn = None
+        mock_role_manager._session = None
+        mock_role_manager._expires_at = None
+        mock_role_manager_class.return_value = mock_role_manager
+
+        query = PackageQuery(
+            bucket="",
+            catalog_url="catalog.example.com",
+            database="test_db",
+            region="us-west-2",
+            iceberg_database="iceberg_db",
+        )
+
+        metadata = {"experiment_id": "EXP-1", "entry_id": "etr_123", "canvas_id": "cnvs_abc"}
+        query._list_iceberg_manifest_buckets = Mock(return_value=["bucket-a"])
+        query._execute_query = Mock(
+            return_value=[
+                {
+                    "pkg_name": "benchling/pkg-a",
+                    "timestamp": "latest",
+                    "message": "A",
+                    "user_meta": json.dumps(json.dumps(metadata)),
+                    "_src_bucket": "bucket-a",
+                }
+            ]
+        )
+
+        result = query.find_unique_packages("experiment_id", "EXP-1")
+
+        assert result["results"]["package_info"]["bucket-a/benchling/pkg-a"]["metadata"] == metadata
+
     @patch("src.package_query.RoleManager")
     def test_iceberg_glue_access_denied_reports_glue_database(self, mock_role_manager_class):
         """Glue table-list permission failures should not be reported as Athena DB failures."""

package.json

@@ -1,6 +1,6 @@
 {
   "name": "@quiltdata/benchling-webhook",
-  "version": "0.19.0",
+  "version": "0.19.1",
   "description": "AWS CDK deployment for Benchling webhook processing using Fargate - Deploy directly with npx",
   "main": "dist/lib/index.js",
   "types": "dist/lib/index.d.ts",