Add WSI support

Makefile

@@ -42,7 +42,7 @@ test:
 	pytest -s tests  --import-mode=importlib -W ignore::DeprecationWarning
 
 style:
-	pre-commit run --all-files
+	PATH="$(CURDIR)/.venv/bin:$$PATH" .venv/bin/pre-commit run --all-files
 
 check: style test
 

notebooks/wsi/01-ingest-wsi.py

@@ -0,0 +1,144 @@
+# Databricks notebook source
+# /// script
+# [tool.databricks.environment]
+# environment_version = "5"
+# ///
+# DBTITLE 1,WSI Ingest — Overview
+# MAGIC %md
+# MAGIC # Whole Slide Image (WSI) Ingest Pipeline
+# MAGIC
+# MAGIC Unified ingest for **all OpenSlide-supported formats**: Aperio SVS, Hamamatsu NDPI, Leica SCN, MIRAX MRXS, Philips TIFF, Ventana BIF/TIF, Sakura SVSLIDE, generic TIFF.
+# MAGIC
+# MAGIC **Pipeline**
+# MAGIC 1. Install dependencies (`openslide-python`, `openslide-bin`, `tifffile`)
+# MAGIC 2. Add `src/` to path for editable development
+# MAGIC 3. Configure source paths and target Delta table
+# MAGIC 4. `WSICatalog.catalog()` — discover all WSI files (multi-extension)
+# MAGIC 5. `WSIMetaExtractor.transform()` — extract metadata into `meta VARIANT`
+# MAGIC 6. Save enriched catalog to Delta
+# MAGIC 7. Verify and explore results
+
+# COMMAND ----------
+
+# DBTITLE 1,Install dependencies
+# MAGIC %pip install openslide-python openslide-bin tifffile imagecodecs Pillow -q
+
+# COMMAND ----------
+
+# DBTITLE 1,Add src/ to sys.path
+import sys
+
+SRC_PATH = "/Workspace/Users/douglas.moore@databricks.com/pixels-tiff/src"
+if SRC_PATH not in sys.path:
+    sys.path.insert(0, SRC_PATH)
+    print(f"Added to sys.path: {SRC_PATH}")
+else:
+    print(f"Already on sys.path: {SRC_PATH}")
+
+# COMMAND ----------
+
+# DBTITLE 1,Configuration
+# --- Source paths (all UC Volume locations with WSI files) ---
+SOURCE_PATHS = [
+    "/Volumes/hls_radiology_east/orthanc_demo/raw_images/Aperio",
+    "/Volumes/hls_radiology_east/orthanc_demo/raw_images/Generic-TIFF",
+    "/Volumes/hls_radiology_east/orthanc_demo/raw_images/Philips-TIFF",
+    "/Volumes/hls_radiology_east/osuwmc/sample",
+    "/Volumes/hls_radiology_east/jsl/samples",
+]
+
+# --- Target Delta table and volume ---
+TABLE  = "dmoore.wsi.object_catalog"
+VOLUME = "dmoore.wsi.wsi_volume"
+
+print(f"Source paths: {len(SOURCE_PATHS)}")
+print(f"Target table: {TABLE}")
+print(f"Volume:       {VOLUME}")
+
+# COMMAND ----------
+
+# DBTITLE 1,Initialize WSICatalog
+from dbx.pixels.wsi import WSICatalog, WSIMetaExtractor, OPENSLIDE_PATTERNS
+
+catalog = WSICatalog(spark, table=TABLE, volume=VOLUME)
+print(f"Catalog: {catalog}")
+print(f"WSI patterns: {OPENSLIDE_PATTERNS}")
+
+# COMMAND ----------
+
+# DBTITLE 1,Discover & catalog all WSI files
+from functools import reduce
+from pyspark.sql import DataFrame
+
+# Catalog each source path and union results
+dfs = []
+for path in SOURCE_PATHS:
+    try:
+        df = catalog.catalog(path=path)  # Uses all WSI patterns by default
+        dfs.append(df)
+        print(f"  ✓ {path}")
+    except Exception as e:
+        print(f"  ✗ {path}: {e}")
+
+if dfs:
+    catalog_df = reduce(DataFrame.union, dfs).dropDuplicates(["path"]).repartition(8)
+    print(f"\nTotal WSI files discovered: {catalog_df.count()}")
+    display(catalog_df.select("path", "modificationTime", "length", "extension"))
+else:
+    raise ValueError("No WSI files found in any source path")
+
+# COMMAND ----------
+
+# DBTITLE 1,Extract WSI metadata
+# WSIMetaExtractor adds a `meta` VARIANT column with:
+# - All OpenSlide properties (vendor-specific: aperio.*, hamamatsu.*, philips.*, etc.)
+# - Derived fields: _wsi_vendor, _wsi_format, _wsi_width, _wsi_height, _wsi_level_count,
+#   _wsi_mpp_x, _wsi_mpp_y, _wsi_objective_power, _wsi_has_label, _wsi_has_macro
+# - phi_tag_report: PHI classification of all properties
+
+extractor = WSIMetaExtractor(catalog, inputCol="local_path", outputCol="meta")
+enriched_df = extractor.transform(catalog_df)
+
+print(f"Schema: {[f.name for f in enriched_df.schema.fields]}")
+display(enriched_df.select("local_path", "extension", "meta"))
+
+# COMMAND ----------
+
+# DBTITLE 1,Save to Delta table
+# Save the enriched catalog to Delta
+catalog.save(enriched_df, mode="overwrite")
+
+print(f"✓ Saved to {TABLE}")
+print(f"  Rows: {spark.table(TABLE).count()}")
+
+# COMMAND ----------
+
+# DBTITLE 1,Explore metadata — vendor breakdown
+# MAGIC %sql
+# MAGIC -- Vendor breakdown from extracted metadata
+# MAGIC SELECT 
+# MAGIC   meta:_wsi_vendor::STRING AS vendor,
+# MAGIC   meta:_wsi_format::STRING AS format,
+# MAGIC   meta:_wsi_backend::STRING AS backend,
+# MAGIC   COUNT(*) AS file_count,
+# MAGIC   AVG(meta:_wsi_width::INT) AS avg_width,
+# MAGIC   AVG(meta:_wsi_height::INT) AS avg_height,
+# MAGIC   AVG(meta:_wsi_mpp_x::DOUBLE) AS avg_mpp_x
+# MAGIC FROM dmoore.wsi.object_catalog
+# MAGIC GROUP BY 1, 2, 3
+# MAGIC ORDER BY file_count DESC
+
+# COMMAND ----------
+
+# DBTITLE 1,Explore metadata — PHI exposure
+# MAGIC %sql
+# MAGIC -- PHI tag exposure summary
+# MAGIC SELECT
+# MAGIC   local_path,
+# MAGIC   meta:_wsi_vendor::STRING AS vendor,
+# MAGIC   meta:_wsi_has_label::BOOLEAN AS has_label,
+# MAGIC   meta:_wsi_has_macro::BOOLEAN AS has_macro,
+# MAGIC   SIZE(CAST(meta:phi_tag_report AS ARRAY<STRING>)) AS phi_tag_count
+# MAGIC FROM dmoore.wsi.object_catalog
+# MAGIC WHERE SIZE(CAST(meta:phi_tag_report AS ARRAY<STRING>)) > 0
+# MAGIC ORDER BY phi_tag_count DESC

notebooks/wsi/02-phi-pixel-detection.py

@@ -0,0 +1,112 @@
+# Databricks notebook source
+# /// script
+# [tool.databricks.environment]
+# environment_version = "5"
+# ///
+# DBTITLE 1,PHI Pixel Detection — Overview
+# MAGIC %md
+# MAGIC # WSI Pixel-Level PHI Detection
+# MAGIC
+# MAGIC Uses `WSIVLMPhiDetector` (OpenSlide-backed) to detect Protected Health Information
+# MAGIC in the **label** and **macro** associated images of Whole Slide Image files.
+# MAGIC
+# MAGIC OpenSlide extracts the sub-image (no full-resolution data loaded), then a Databricks
+# MAGIC VLM serving endpoint identifies PHI entities (patient names, DOBs, accession numbers, barcodes).
+# MAGIC
+# MAGIC **Prerequisite**: Run `01-ingest-wsi` first to populate `dmoore.wsi.object_catalog`.
+
+# COMMAND ----------
+
+# DBTITLE 1,Install dependencies
+# MAGIC %pip install openslide-python openslide-bin tifffile imagecodecs Pillow mlflow openai -q
+
+# COMMAND ----------
+
+# DBTITLE 1,Add src/ to sys.path
+import sys
+
+SRC_PATH = "/Workspace/Users/douglas.moore@databricks.com/pixels-tiff/src"
+if SRC_PATH not in sys.path:
+    sys.path.insert(0, SRC_PATH)
+    print(f"Added to sys.path: {SRC_PATH}")
+else:
+    print(f"Already on sys.path: {SRC_PATH}")
+
+# COMMAND ----------
+
+# DBTITLE 1,Configuration
+# --- Source: enriched metadata from 01-ingest-wsi ---
+TABLE = "dmoore.wsi.object_catalog"
+
+# --- VLM endpoint for PHI detection ---
+VLM_ENDPOINT = "phi-detection"
+
+print(f"Source table: {TABLE}")
+print(f"VLM endpoint: {VLM_ENDPOINT}")
+
+# COMMAND ----------
+
+# DBTITLE 1,VLM PHI detection — label images
+from pyspark.sql.functions import lit
+from dbx.pixels.wsi import WSIVLMPhiDetector
+
+# Run VLM PHI detection on all three sub-image series:
+#   label  — printed patient name, barcode, accession (primary PHI target)
+#   macro  — overview image (may contain handwritten labels)
+#   tissue — smallest pyramid level (rarely contains PHI, but checks embedded text)
+#
+# wsi_to_image returns None (recorded as error, no VLM call) when a series is absent.
+
+base_df = spark.sql(f"""
+    SELECT local_path, extension,
+           meta:_wsi_vendor::STRING AS vendor,
+           meta:_wsi_format::STRING AS format
+    FROM {TABLE}
+""").repartition(8)
+
+print(f"Total WSI files: {base_df.count()}")
+
+results = []
+for series in ["label", "macro", "tissue"]:
+    detector = WSIVLMPhiDetector(
+        endpoint=VLM_ENDPOINT,
+        inputCol="local_path",
+        outputCol="vlm_phi",
+        series=series,
+        max_width=768,
+        temperature=0.0,
+    )
+    series_df = detector.transform(base_df).withColumn("series", lit(series))
+    results.append(series_df)
+
+phi_df = results[0].unionByName(results[1]).unionByName(results[2])
+
+print(f"Total results (files × series): {phi_df.count()}")
+display(phi_df.select("local_path", "vendor", "format", "series", "vlm_phi.*"))
+
+# COMMAND ----------
+
+# DBTITLE 1,PHI detection summary
+from pyspark.sql.functions import col, size
+
+results = phi_df.select(
+    "local_path",
+    "vendor",
+    "series",
+    col("vlm_phi.content").alias("phi_entities"),
+    col("vlm_phi.error").alias("error"),
+    col("vlm_phi.total_tokens").alias("tokens"),
+)
+
+total = results.count()
+with_phi = results.filter(size("phi_entities") > 0).count()
+with_error = results.filter(col("error").isNotNull()).count()
+
+print("=== VLM PHI Detection Summary ===")
+print(f"Total scans (files × series): {total}")
+print(f"Scans with PHI detected:      {with_phi}")
+print(f"Scans with errors:            {with_error}")
+print()
+
+# Show files/series where PHI was found
+display(results.filter(size("phi_entities") > 0))

src/dbx/pixels/dicom/cache/bot_cache_builder.py

@@ -655,7 +655,8 @@ def get_preload_list(
         hl_used = self.half_lives["last_used"] * 3600.0  # hours → seconds
         hl_ins = self.half_lives["inserted"] * 3600.0
 
-        query = sql.SQL("""
+        query = sql.SQL(
+            """
             SELECT
                 filename,
                 COUNT(*)                             AS frame_count,
@@ -685,7 +686,8 @@ def get_preload_list(
             GROUP BY filename
             ORDER BY priority_score DESC
             LIMIT %s
-            """).format(
+            """
+        ).format(
             table=sql.Identifier(self.lb.schema, "dicom_frames"),
             w_used=sql.Literal(w_used),
             w_ins=sql.Literal(w_ins),

src/dbx/pixels/lakebase.py

@@ -799,7 +799,8 @@ def get_preload_priority_list(
         # avoids the json_agg / OID-114 deserialization issue where psycopg2
         # may return the json type as Python None if the JSON adapter is not
         # registered in the connection context.
-        query = sql.SQL("""
+        query = sql.SQL(
+            """
             SELECT
                 filename,
                 COUNT(*)                             AS frame_count,
@@ -829,7 +830,8 @@ def get_preload_priority_list(
             GROUP BY filename
             ORDER BY priority_score DESC
             LIMIT %s
-            """).format(table=sql.Identifier(self.schema, table))
+            """
+        ).format(table=sql.Identifier(self.schema, table))
 
         rows = self.execute_and_fetch_query(query, (uc_table_name, limit))
         results = []

src/dbx/pixels/wsi/__init__.py

@@ -0,0 +1,64 @@
+"""dbx.pixels.wsi — Unified Whole Slide Image handler for databricks-pixels.
+
+Supports all OpenSlide-recognized WSI formats:
+- Aperio SVS
+- Hamamatsu NDPI / VMS / VMU
+- Leica SCN
+- MIRAX MRXS
+- Philips TIFF
+- Sakura SVSLIDE
+- Trestle TIF
+- Ventana BIF / TIF
+- Generic TIFF
+
+Install alongside ``databricks-pixels`` and extend the ``dbx.pixels`` namespace::
+
+    import sys
+    SRC_PATH = "/Workspace/Users/<you>/pixels-tiff/src"
+    if SRC_PATH not in sys.path:
+        sys.path.insert(0, SRC_PATH)
+
+    from dbx.pixels.wsi import WSICatalog, WSIMetaExtractor, WSIVLMPhiDetector, wsi_to_image
+"""
+
+from dbx.pixels.wsi.catalog import WSICatalog
+from dbx.pixels.wsi.wsi_meta_extractor import WSIMetaExtractor
+from dbx.pixels.wsi.wsi_phi_tags import (
+    LARGE_TAGS,
+    NOT_PHI_TAGS,
+    OPENSLIDE_PATTERNS,
+    PHI_TAGS,
+    QUESTIONABLE_TAGS,
+    SUPPORTED_EXTENSIONS,
+    classify_tag,
+    classify_tags,
+    scrub_image_description,
+)
+from dbx.pixels.wsi.wsi_utils import (
+    wsi_detect_format,
+    wsi_get_properties,
+    wsi_to_image,
+)
+from dbx.pixels.wsi.wsi_vlm_phi_detector import WSIVLMPhiDetector
+
+__all__ = [
+    # Classes
+    "WSICatalog",
+    "WSIMetaExtractor",
+    "WSIVLMPhiDetector",
+    # Utils
+    "wsi_to_image",
+    "wsi_detect_format",
+    "wsi_get_properties",
+    # PHI tags
+    "classify_tag",
+    "classify_tags",
+    "scrub_image_description",
+    "PHI_TAGS",
+    "QUESTIONABLE_TAGS",
+    "NOT_PHI_TAGS",
+    "LARGE_TAGS",
+    # Constants
+    "SUPPORTED_EXTENSIONS",
+    "OPENSLIDE_PATTERNS",
+]