From d84915026322757ebd9cf51a744570c86c27aa4e Mon Sep 17 00:00:00 2001 From: dmoore247 Date: Sat, 11 Jul 2026 16:29:02 +0000 Subject: [PATCH 1/7] add first commit of svs file handling --- ...Pathology De-identification Pipeline.ipynb | 1753 +++++++++++++++++ src/dbx/pixels/svs/__init__.py | 43 + src/dbx/pixels/svs/catalog.py | 74 + src/dbx/pixels/svs/deidentify.py | 224 +++ src/dbx/pixels/svs/phi_tags.py | 162 ++ .../svs/resources/sql/CREATE_SVS_CATALOG.sql | 110 ++ src/dbx/pixels/svs/svs_meta_extractor.py | 115 ++ src/dbx/pixels/svs/svs_tiff_writer.py | 190 ++ 8 files changed, 2671 insertions(+) create mode 100644 notebooks/svs/SVS Pathology De-identification Pipeline.ipynb create mode 100644 src/dbx/pixels/svs/__init__.py create mode 100644 src/dbx/pixels/svs/catalog.py create mode 100644 src/dbx/pixels/svs/deidentify.py create mode 100644 src/dbx/pixels/svs/phi_tags.py create mode 100644 src/dbx/pixels/svs/resources/sql/CREATE_SVS_CATALOG.sql create mode 100644 src/dbx/pixels/svs/svs_meta_extractor.py create mode 100644 src/dbx/pixels/svs/svs_tiff_writer.py diff --git a/notebooks/svs/SVS Pathology De-identification Pipeline.ipynb b/notebooks/svs/SVS Pathology De-identification Pipeline.ipynb new file mode 100644 index 00000000..46b34eb6 --- /dev/null +++ b/notebooks/svs/SVS Pathology De-identification Pipeline.ipynb @@ -0,0 +1,1753 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": {}, + "inputWidgets": {}, + "nuid": "a9402935-1770-4aa2-bec0-71e265bc53c1", + "showTitle": false, + "tableResultSettingsMap": {}, + "title": "" + } + }, + "source": [ + "\n", + "# Plan: SVS Pathology De-identification Pipeline\n", + "## Architecture Plan — Aperio SVS → De-identified TIFF\n", + "\n", + "Extends [databricks-industry-solutions/pixels](https://github.com/databricks-industry-solutions/pixels) to treat Aperio `.svs` as a first-class format alongside DICOM.\n", + "\n", + "---" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": {}, + "inputWidgets": {}, + "nuid": "b27e18db-0abf-446f-ba89-65d6906d1506", + "showTitle": false, + "tableResultSettingsMap": {}, + "title": "" + } + }, + "source": [ + "## 1. Confirmed Inputs & Outputs\n", + "\n", + "| Item | Value |\n", + "|---|---|\n", + "| Input SVS path | `/Volumes/hls_pathology/orthanc_demo/raw_images/Aperio/` |\n", + "| Demo scale | ~14 files; architecture targets **10 million** |\n", + "| Output catalog / schema | `douglas_moore.pathology` (to be created) |\n", + "| TIFF output volume | `/Volumes/douglas_moore/pathology/tiff_deidentified/` |\n", + "| Label images volume | `/Volumes/douglas_moore/pathology/label_images/` |\n", + "| VLM endpoint | `databricks-llama-4-maverick` (config param) |\n", + "| Redaction method | Black rectangle fill |\n", + "| Source SVS | **Read-only** — never modified |\n", + "\n", + "---" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": {}, + "inputWidgets": {}, + "nuid": "32ddb278-16b7-4719-a134-55360ad4bb26", + "showTitle": false, + "tableResultSettingsMap": {}, + "title": "" + } + }, + "source": [ + "## 2. Delta Table Schema\n", + "\n", + "> **No new DDL is needed for SVS ingest.** The `object_catalog` table is used exactly as defined in the base `CREATE_OBJECT_CATALOG.sql` — no columns are added or altered. All SVS-specific metadata (dimensions, pyramid levels, sub-image presence, PHI tag classification) is serialised into the existing `meta VARIANT` column and accessed via VARIANT path syntax. `SVSCatalog.init_tables()` calls `super().init_tables()` which runs the unmodified base DDL against the `douglas_moore.pathology` schema. The only new DDL is the `_redaction` table.\n", + "\n", + "### `douglas_moore.pathology.object_catalog` *(base DDL, unchanged)*\n", + "One row per SVS file. Populated by `SVSCatalog.catalog()` + `SVSMetaExtractor`.\n", + "\n", + "| Column | Type | Notes |\n", + "|---|---|---|\n", + "| `path` | STRING NOT NULL | Cloud storage path |\n", + "| `modificationTime` | TIMESTAMP NOT NULL | |\n", + "| `length` | BIGINT NOT NULL | File size bytes |\n", + "| `original_path` | STRING | |\n", + "| `relative_path` | STRING | |\n", + "| `local_path` | STRING NOT NULL | Worker-accessible path — **`inputCol` for all Transformers** |\n", + "| `extension` | STRING | `\"svs\"` |\n", + "| `file_type` | STRING | |\n", + "| `path_tags` | ARRAY\\ | From `TagExtractor` |\n", + "| `is_anon` | BOOLEAN | |\n", + "| `meta` | **VARIANT** | All OpenSlide properties + SVS-specific fields serialised together. Query with `meta:aperio.Date::string`, `meta:width::int`, `meta:has_label_image::boolean` |\n", + "\n", + "**SVS fields stored inside `meta VARIANT`** (no schema change required):\n", + "- `meta:width::int`, `meta:height::int` — level-0 pixel dimensions\n", + "- `meta:level_count::int` — pyramid depth\n", + "- `meta:has_label_image::boolean`, `meta:has_macro_image::boolean`\n", + "- `meta:phi_tag_report` — array of `{tag, value, classification}` structs\n", + "- All raw OpenSlide properties (e.g. `meta:\"aperio.AppMag\"::string`)\n", + "\n", + "### `douglas_moore.pathology.object_catalog_redaction` *(unified DICOM + SVS DDL)*\n", + "One row per redaction job, for any format. Created by `CREATE_SVS_CATALOG.sql`.\n", + "\n", + "Three DICOM columns are renamed to remove format-specific semantics; new columns cover VLM detection results and SVS artefacts. All new and renamed columns are nullable for backward compatibility.\n", + "\n", + "**Format-agnostic identifiers**\n", + "\n", + "| Column | Type | Notes |\n", + "|---|---|---|\n", + "| `redaction_id` | STRING NOT NULL | UUID per job |\n", + "| `path` | STRING | FK → `object_catalog.path` *(new — not in DICOM original)* |\n", + "| `extension` | STRING | Discriminator: `dcm`, `svs`, `czi` … *(new)* |\n", + "\n", + "**DICOM identifiers** *(NULL for SVS)*\n", + "\n", + "| Column | Type | Notes |\n", + "|---|---|---|\n", + "| `study_instance_uid` | STRING | DICOM Study UID |\n", + "| `series_instance_uid` | STRING | DICOM Series UID |\n", + "| `modality` | STRING | DICOM modality, or `WSI` for SVS |\n", + "| `new_series_instance_uid` | STRING | New UID for redacted DICOM series |\n", + "\n", + "**Redaction configuration** *(both formats)*\n", + "\n", + "| Column | Type | Change from DICOM original |\n", + "|---|---|---|\n", + "| `redaction_config` | VARIANT | **Renamed** from `redaction_json` |\n", + "| `metadata_redactions_count` | INT | **Renamed** from `global_redactions_count` |\n", + "| `pixel_redactions_count` | INT | **Renamed** from `frame_specific_redactions_count` |\n", + "| `total_redaction_areas` | INT | Unchanged |\n", + "| `phi_tags_redacted` | ARRAY\\ | Tag names scrubbed *(new)* |\n", + "\n", + "**VLM PHI detection results** *(new — both formats)*\n", + "\n", + "| Column | Type | Notes |\n", + "|---|---|---|\n", + "| `has_phi` | BOOLEAN | VLM verdict |\n", + "| `phi_elements` | VARIANT | Detected regions: type, value\\_hint, bbox |\n", + "| `vlm_raw_response` | STRING | Raw model output |\n", + "| `model_endpoint` | STRING | Endpoint name |\n", + "\n", + "**Output paths**\n", + "\n", + "| Column | Type | Notes |\n", + "|---|---|---|\n", + "| `output_file_paths` | ARRAY\\ | DICOM: one `.dcm` per slice. SVS: single TIFF at index 0 |\n", + "| `label_image_path` | STRING | De-identified label PNG *(SVS only, NULL for DICOM)* |\n", + "| `macro_image_path` | STRING | De-identified macro PNG *(SVS only, NULL for DICOM)* |\n", + "\n", + "**Processing status & audit** *(unchanged from DICOM original)*\n", + "`status`, `error_messages`, `insert_timestamp`, `update_timestamp`, `processing_start_timestamp`, `processing_end_timestamp`, `processing_duration_seconds`, `created_by`, `export_timestamp`\n", + "\n", + "---" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": {}, + "inputWidgets": {}, + "nuid": "207f4f4e-a49b-456e-b5d1-1165b1b1f8a9", + "showTitle": true, + "tableResultSettingsMap": {}, + "title": "Architecture Plan: SVS Pathology De-identification Pipeline" + } + }, + "source": [ + "\n", + "## 3. Python Package: `dbx.pixels.svs`\n", + "\n", + "> **Pattern source: actual repo code** — All transformers extend `pyspark.ml.pipeline.Transformer` (Spark ML, not a custom base). The main entry-point for file cataloguing is the `Catalog` class, not a `Processor`. There is no `Processor` in the repo. The CZI extractor (`src/dbx/pixels/czi/`) is a stub — SVS is genuinely the first completed non-DICOM format extension.\n", + "\n", + "Namespace-package extension of `dbx-pixels`. Created as workspace files under `svs-pixels/src/`, installed via `%pip install -e ./src`.\n", + "\n", + "```\n", + "svs-pixels/\n", + "├── src/\n", + "│ └── dbx/\n", + "│ └── pixels/\n", + "│ └── svs/\n", + "│ ├── __init__.py ← exports SVSCatalog, SVSMetaExtractor, SVSTiffWriter, SVSPhiPipeline\n", + "│ ├── catalog.py ← SVSCatalog(Catalog)\n", + "│ ├── svs_meta_extractor.py← SVSMetaExtractor(Transformer)\n", + "│ ├── svs_tiff_writer.py ← SVSTiffWriter(Transformer)\n", + "│ ├── phi_tags.py ← PHI classification lookup dict\n", + "│ └── deidentify.py ← pixel redaction helpers\n", + "│ └── resources/sql/\n", + "│ └── CREATE_SVS_CATALOG.sql ← creates object_catalog_redaction only\n", + "├── pyproject.toml\n", + "└── (this notebook)\n", + "```\n", + "\n", + "### `SVSCatalog` (extends `Catalog`)\n", + "- Calls `super().__init__(spark, table, volume)` — reuses all existing table management, volume, and Auto Loader infrastructure\n", + "- `catalog(path, pattern=\"*.svs\", ...)` → delegates to `Catalog.catalog()` with SVS glob pattern; callers never need to pass `pattern`\n", + "- `init_tables()` → calls `super().init_tables()` (creates `object_catalog` via unmodified base DDL), then executes one SVS-specific file — `resources/sql/CREATE_SVS_CATALOG.sql` — which creates only the `object_catalog_redaction` table with SVS-specific columns\n", + "\n", + "### `SVSMetaExtractor` (extends `pyspark.ml.pipeline.Transformer`)\n", + "Mirrors `DicomMetaExtractor`: uses `mapInPandas` with `ThreadPoolExecutor` for concurrent I/O (optimal for network-bound OpenSlide reads).\n", + "\n", + "```python\n", + "class SVSMetaExtractor(Transformer):\n", + " def __init__(self, catalog, inputCol=\"local_path\", outputCol=\"meta\",\n", + " maxWorkers=32, useVariant=True): ...\n", + "\n", + " def _transform(self, df): # Spark ML Transformer contract\n", + " # mapInPandas with ThreadPoolExecutor — same pattern as DicomMetaExtractor\n", + " ...\n", + "```\n", + "\n", + "**Single output column written to `object_catalog`:**\n", + "\n", + "| Column | Spark type | Notes |\n", + "|---|---|---|\n", + "| `meta` | `VARIANT` | OpenSlide properties dict merged with derived fields (`width`, `height`, `level_count`, `has_label_image`, `has_macro_image`, `phi_tag_report`) into one JSON object, then `parse_json()`'d into VARIANT |\n", + "\n", + "All SVS-specific fields are embedded inside `meta` before serialisation — no extra top-level columns are written, no `ALTER TABLE` or `mergeSchema` required. VARIANT path syntax handles all downstream access: `meta:width::int`, `meta:phi_tag_report[0].classification::string`, etc.\n", + "\n", + "### `SVSTiffWriter` (extends `pyspark.ml.pipeline.Transformer`)\n", + "Converts SVS → de-identified pyramidal BigTIFF. Wraps the write logic in `_transform(df)` operating on the output of `SVSMetaExtractor`.\n", + "\n", + "### `SVSPhiPipeline` (extends `pyspark.ml.Pipeline`)\n", + "Composed pipeline, mirrors `DicomPhiPipeline`:\n", + "```\n", + "Stage 1: SVSMetaExtractor → adds meta VARIANT + phi_tag_report\n", + "Stage 2: SVSVlmPhiDetector → adds phi_elements (VLM bboxes on label/macro)\n", + "Stage 3: SVSFilterTransformer → nullifies rows with no PHI detected\n", + "Stage 4: SVSTiffWriter → writes de-identified BigTIFF + audit log\n", + "```\n", + "\n", + "---\n", + "\n", + "\n" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": {}, + "inputWidgets": {}, + "nuid": "7e9389da-8946-4087-a9ac-3fe10773c829", + "showTitle": false, + "tableResultSettingsMap": {}, + "title": "" + } + }, + "source": [ + "\n", + "## 4. PHI Tag Classification (`phi_tags.py`)\n", + "\n", + "Research-hardcoded lookup covering all standard OpenSlide / Aperio TIFF properties:\n", + "\n", + "| Classification | Example Tags |\n", + "|---|---|\n", + "| `PHI` | `aperio.Patient`, `aperio.PatientID`, `aperio.DOB`, `aperio.MRN`, `aperio.AccessionNumber`, `aperio.ClinicID`, `aperio.ClinicalTrialID`, `aperio.Procedure`, `tiff.ImageDescription` (contains patient name in Aperio format) |\n", + "| `QUESTIONABLE` | `aperio.Date`, `aperio.Time`, `aperio.Clinic`, `aperio.Pathologist`, `tiff.Artist`, `tiff.Copyright`, `aperio.Title`, `aperio.Filename`, `aperio.User`, `aperio.ImageID` |\n", + "| `NOT_PHI` | `aperio.AppMag`, `aperio.MPP`, `aperio.ScanScope ID`, `openslide.level-count`, `openslide.mpp-x`, `openslide.mpp-y`, `openslide.objective-power`, `tiff.Make`, `tiff.Model`, `tiff.Software`, `openslide.vendor`, all `openslide.level[N].*` pyramid geometry tags |\n", + "\n", + "Function `classify_tags(properties: dict) → list[dict]` iterates all OpenSlide properties and returns the structured report stored in `phi_tag_report`.\n", + "\n", + "---" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": {}, + "inputWidgets": {}, + "nuid": "3f54bca4-4711-4cc7-8995-ce1bb3bfda99", + "showTitle": false, + "tableResultSettingsMap": {}, + "title": "" + } + }, + "source": [ + "## 5. VLM PHI Detection via `ai_query()`\n", + "\n", + "### What is Inspected\n", + "Aperio SVS files embed three sub-images accessible via `slide.associated_images` — **confirmed from real files**:\n", + "\n", + "| Sub-image | Dims (CMU-1) | Mode | PHI Risk | Action |\n", + "|---|---|---|---|---|\n", + "| `label` | 387×463 | RGBA | **HIGH** — physical paper label with patient name, barcode, accession | VLM analysis + black-box redaction |\n", + "| `macro` | 1280×431 | RGBA | **MEDIUM** — full-slide photo; label region visible at right edge | VLM analysis + label region redaction |\n", + "| `thumbnail` | 1024×732 | RGBA | LOW — auto-generated tissue preview | Excluded from output |\n", + "\n", + "The tissue scan (`level 0`: 46000×32914) is in a **completely separate coordinate space** from the label/macro sub-images. PHI in the tissue scan itself is rare but possible (e.g., handwriting on the glass).\n", + "\n", + "The label image is the **primary** VLM target. Macro is secondary.\n", + "\n", + "### Pipeline\n", + "1. `SVSTransformer.extract_embedded_images()` saves label and macro PNGs to `/Volumes/douglas_moore/pathology/label_images/` using the naming convention `{slide_name}_label.png` / `{slide_name}_macro.png`\n", + "2. Run `ai_query()` directly via `READ_FILES()` on the volume — **no binary column staging needed**:\n", + "\n", + "```sql\n", + "INSERT INTO douglas_moore.pathology.phi_pixel_audit\n", + "SELECT\n", + " m.path,\n", + " f._metadata.file_path AS label_image_path,\n", + " ai_query(\n", + " 'databricks-llama-4-maverick',\n", + " 'You are a medical PHI detection system analyzing a pathology slide label.\n", + " Return ONLY valid JSON:\n", + " {\"has_phi\": bool,\n", + " \"phi_elements\": [{\"type\": \"name|dob|mrn|accession|barcode|other\",\n", + " \"value_hint\": \"first 3 chars only\",\n", + " \"bbox\": {\"x\": int, \"y\": int, \"w\": int, \"h\": int}}]}\n", + " Bounding box coordinates are in the label image pixel space (origin top-left).',\n", + " files => f.content\n", + " ) AS vlm_raw_response,\n", + " 'databricks-llama-4-maverick' AS model_endpoint,\n", + " current_timestamp() AS inferred_at\n", + "FROM read_files(\n", + " '/Volumes/douglas_moore/pathology/label_images/',\n", + " format => 'binaryFile',\n", + " fileNamePattern => '*_label.png'\n", + ") f\n", + "JOIN douglas_moore.pathology.svs_metadata m\n", + " ON m.filename = regexp_replace(f._metadata.file_name, '_label\\.png\n", + "```\n", + "---\n" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": {}, + "inputWidgets": {}, + "nuid": "b3f28cd8-2465-4bfe-b153-664e929fe501", + "showTitle": false, + "tableResultSettingsMap": {}, + "title": "" + } + }, + "source": [ + "\n", + "## 6. De-identification & TIFF Output (`deidentify.py` / `SVSTiffWriter`)\n", + "\n", + "> **Pattern source: actual repo code** — `DicomPhiPipeline` uses a two-stage approach: (1) `VLMPhiDetector` returns a **pipe-separated list of PHI text strings** (`'John Smith'|'04-31-1954'`), NOT bboxes. (2) `OcrRedactor` then runs EasyOCR on the image to locate those strings and draw black rectangles. The VLM provides *what* is PHI; OCR provides *where*. SVS uses this same two-stage approach.\n", + "\n", + "### Revised De-identification Algorithm\n", + "\n", + "#### Stage 1 — VLM PHI Detection (`SVSVlmPhiDetector`, extends `Transformer`)\n", + "- Submit label/macro PNGs via `ai_query()` with `files => content`\n", + "- Prompt returns a **pipe-separated list of PHI entity strings** (consistent with the SA pattern)\n", + "- Optionally also request bboxes via `responseFormat => json_schema` (SVS-specific addition for direct redaction without a second OCR pass)\n", + "\n", + "#### Stage 2 — Pixel Redaction (`SVSTiffWriter._transform(df)`)\n", + "1. Open SVS with `openslide.OpenSlide(local_path)`\n", + "2. Read level-0 in 4096×4096 tiles using `slide.read_region()`\n", + "3. If bbox-only mode: draw filled black `PIL.ImageDraw.rectangle` over each detected region in label/macro\n", + "4. If text-only mode: run EasyOCR on label image to locate the strings from the VLM response, then black-out matching text (mirrors `OcrRedactor`)\n", + "5. Scrub PHI tags in `tiff.ImageDescription` using `phi_tags.scrub_image_description()`\n", + "6. Write pyramidal BigTIFF using `tifffile.TiffWriter(bigtiff=True)` with `subifds=level_count-1` and 256×256 JPEG tiles\n", + "7. Return `(tiff_output_path, phi_tags_redacted_list, pixel_regions_count)` for the audit row\n", + "\n", + "### VLM Implementation: Two Approaches\n", + "\n", + "| Approach | Used by | Library | Scale |\n", + "|---|---|---|---|\n", + "| OpenAI SDK + base64 | `VLMPhiExtractor` in pixels SA | `openai` Python SDK, `pandas_udf` | Single-node / moderate |\n", + "| `ai_query()` + `files => content` | **Our SVS pipeline** | Databricks SQL / Spark SQL | 10M images, serverless SQL |\n", + "\n", + "For the demo scale, both work. For 10M, `ai_query()` via SQL is the correct choice — it delegates throughput management to the Databricks SQL engine.\n" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": {}, + "inputWidgets": {}, + "nuid": "c5c5b775-4d44-415f-92f5-dd65275fa0bf", + "showTitle": false, + "tableResultSettingsMap": {}, + "title": "" + } + }, + "source": [ + "\n", + "## 7. Notebook Cell Structure\n", + "\n", + "| Cell | Purpose |\n", + "|---|---|\n", + "| **Cell 1** | This plan (markdown) |\n", + "| **Cell 2** | `%pip install dbx-pixels openslide-python openslide-bin tifffile Pillow easyocr` + `%pip install -e ./src` |\n", + "| **Cell 3** | Configuration: paths, catalog, schema, volume names, model endpoint |\n", + "| **Cell 4** | Storage bootstrap: `SVSCatalog(spark, ...).init_tables()` — `super().init_tables()` creates `object_catalog` (base DDL, unchanged); `CREATE_SVS_CATALOG.sql` creates `object_catalog_redaction` (SVS-specific columns only) |\n", + "| **Cell 5** | File discovery: `SVSCatalog.catalog(INPUT_PATH)` → writes `object_catalog` (pattern defaults to `\"*.svs\"`) |\n", + "| **Cell 6** | Metadata extraction: `SVSMetaExtractor(catalog)._transform(files_df)` → populates `meta VARIANT` (OpenSlide properties + derived SVS fields merged into one JSON object) |\n", + "| **Cell 7** | PHI tag report: SQL on `object_catalog` using VARIANT path syntax (`meta:aperio.Date::string`, `meta:phi_tag_report`) |\n", + "| **Cell 8** | Label/macro image extraction → `/Volumes/.../label_images/` |\n", + "| **Cell 9** | VLM inference: `ai_query()` SQL → `object_catalog_redaction` |\n", + "| **Cell 10** | De-identified TIFF write: `SVSTiffWriter._transform(df)` → BigTIFFs to volume |\n", + "| **Cell 11** | Audit summary: join `object_catalog` + `object_catalog_redaction`, show statistics |\n", + "\n", + "---" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": {}, + "inputWidgets": {}, + "nuid": "1e12f30f-e1ae-49f9-a136-8dd94032e7aa", + "showTitle": false, + "tableResultSettingsMap": {}, + "title": "" + } + }, + "source": [ + "\n", + "## 8. Scale Architecture Notes (10M Images)\n", + "\n", + "| Concern | Demo Approach | 10M Approach |\n", + "|---|---|---|\n", + "| File discovery | `dbutils.fs.ls` recursive | Auto Loader on the volume path |\n", + "| Metadata extraction | `SVSMetaExtractor` via `mapInPandas` + `ThreadPoolExecutor` | Same — already distributed |\n", + "| Label image storage | Written to volume as files | Stored as `BINARY` in Delta table (eliminates extra volume I/O) |\n", + "| VLM inference | Single SQL batch `ai_query()` | Incremental: `WHERE vlm_status='PENDING'` in a scheduled Lakeflow Job |\n", + "| TIFF conversion | `write_deidentified_tiff_udf` Spark UDF | Same — Photon-accelerated UDF dispatch |\n", + "| Checkpointing | `vlm_status` column | Same + Delta transaction log for idempotency |\n", + "| Cost control | Serverless interactive | SQL Serverless warehouse + compute-optimized clusters for UDF stages |\n", + "\n", + "---" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": {}, + "inputWidgets": {}, + "nuid": "9ff39799-03ad-4f51-a8a8-3d0823d8b3e2", + "showTitle": false, + "tableResultSettingsMap": {}, + "title": "" + } + }, + "source": [ + "## 9. Confirmed Findings & Resolved Design Decisions\n", + "\n", + "All four open questions are now resolved from direct inspection of the actual Aperio CMU-1.svs files.\n", + "\n", + "### Q1 — Bounding box coordinate space ✅ RESOLVED\n", + "\n", + "The label sub-image is **387×463 RGBA** — completely independent from the tissue scan (46000×32914). The two coordinate spaces share no relationship.\n", + "\n", + "**Decision:** Save and submit the label image to the VLM at **native resolution (no resizing)**. All VLM bboxes are in label-image pixel space (`0,0` = top-left). The tissue TIFF does **not** embed the label — it is automatically excluded when only the main pyramid is read. The de-identified label PNG (black rectangles applied) is written to the `label_images` volume as the audit artefact.\n", + "\n", + "**Macro image clarification:** The macro (1280×431) shows both tissue and the physical label (at one end). It is submitted to the VLM separately; its bboxes drive black-rectangle redaction of the label area in the macro output PNG.\n", + "\n", + "---\n", + "\n", + "### Q2 — `tiff.ImageDescription` scrubbing ✅ RESOLVED\n", + "\n", + "Exact format confirmed from the real file:\n", + "```\n", + "Aperio Image Library v10.0.51\\r\\n46920x33014 [0,100 46000x32914] (256x256) JPEG/RGB Q=30\n", + " |AppMag = 20|StripeWidth = 2040|ScanScope ID = CPAPERIOCS|Filename = CMU-1\n", + " |Date = 12/29/09|Time = 09:59:15|User = b414003d-...|ImageID = 1004486|...\n", + "```\n", + "\n", + "**Structure:** `{header_line}|key = val|key = val|...`  Header is technical-only — preserve as-is.\n", + "\n", + "**PHI classification of actual keys:**\n", + "\n", + "| Key | Classification | Notes |\n", + "|---|---|---|\n", + "| `Date`, `Time` | PHI | HIPAA date/time of service |\n", + "| `User` | QUESTIONABLE | GUID in demo; operator name in clinical use |\n", + "| `Filename` | QUESTIONABLE | May encode patient name or MRN |\n", + "| `ImageID` | QUESTIONABLE | Could be accession number |\n", + "| `ScanScope ID`, `AppMag`, `StripeWidth`, `Parmset`, `MPP`, all geometry/calibration, `Filtered`, `ICC Profile` | NOT_PHI | Pure scanner parameters |\n", + "\n", + "Clinical files may also contain: `Patient`, `DOB`, `MRN`, `AccessionNumber`, `Clinic`, `Pathologist`, `Procedure`, `Diagnosis`, `Id` — all PHI.\n", + "\n", + "**Scrubbing algorithm:**\n", + "1. `header, *kvs = image_desc.split('|')`\n", + "2. For each `kv`: `k, v = kv.split(' = ', 1)` — rebuild as `k = REDACTED` if `k.strip()` ∈ PHI/QUESTIONABLE set\n", + "3. Rejoin: `'|'.join([header] + rebuilt_kvs)`\n", + "4. Apply identical scrub to `openslide.comment` (same content) when writing TIFF metadata\n", + "\n", + "---\n", + "\n", + "### Q3 — Macro image redaction ✅ RESOLVED\n", + "\n", + "Macro (1280×431) shows the full physical slide including the affixed label. **Decision:** Include macro in the primary VLM pipeline alongside the label (not a follow-on phase). Naming: `{name}_label.png` / `{name}_macro.png`. Both de-identified PNGs go to the `label_images` volume.\n", + "\n", + "---\n", + "\n", + "### Q4 — Pyramidal TIFF output ✅ RESOLVED\n", + "\n", + "Flat TIFF is not viable for pathology — QuPath, OMERO, and DIGIPATH all require pyramidal. The source SVS has 3 levels with 256×256 tiles; match this in output.\n", + "\n", + "**Decision:** Write pyramidal **BigTIFF** via `tifffile`:\n", + "- `bigtiff=True` — mandatory (CMU-1 level-0 ~7.4 GB uncompressed, exceeds 4 GB TIFF limit)\n", + "- `tile=(256, 256)` — matches native Aperio tile size\n", + "- `compression='jpeg'` at quality 80; swap to `'lzw'` if lossless required\n", + "- `subifds=level_count - 1` — sub-IFDs are the QuPath/libvips-compatible pyramid convention\n", + "- Pyramid levels: 2× progressive downsampling with `PIL.Image.LANCZOS`\n", + "\n", + "```python\n", + "with tifffile.TiffWriter(output_path, bigtiff=True) as tif:\n", + " opts = dict(tile=(256, 256), compression='jpeg',\n", + " compressionargs={'level': 80}, photometric='rgb', metadata=None)\n", + " tif.write(level_0_rgb, subifds=level_count - 1, **opts) # main IFD\n", + " for lvl in range(1, level_count):\n", + " tif.write(level_arrays[lvl], subfiletype=1, **opts) # sub-IFDs\n", + "```\n", + "\n", + "OME-TIFF (`ome=True`) only if OMERO is a confirmed downstream consumer.\n", + "\n", + "---" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": {}, + "inputWidgets": {}, + "nuid": "308dda6c-953c-44e0-b8a3-4e20dafb9357", + "showTitle": false, + "tableResultSettingsMap": {}, + "title": "" + } + }, + "source": [ + "\n", + "## 10. More...\n", + "\n", + "### Additional: `openslide-bin` Required\n", + "\n", + "`openslide-python` alone fails at import on Databricks Serverless:\n", + "```\n", + "ModuleNotFoundError: Couldn't locate OpenSlide shared library. Try pip install openslide-bin.\n", + "```\n", + "**Cell 2 must install:** `openslide-python openslide-bin` (the `openslide-bin` wheel bundles `libopenslide.so` for environments without system package access).\n", + "\n", + "\n", + "> **Note**: `files => content` is the correct `ai_query()` API for binary image inputs — it passes the PNG bytes directly to the model without base64 encoding. Only JPEG and PNG inputs are supported.\n", + "\n", + "### Scale to 10M Images\n", + "- `vlm_status` column acts as a watermark: `PENDING → PROCESSING → COMPLETE / FAILED`\n", + "- The SQL above runs as a Databricks SQL batch job — `ai_query()` parallelizes across serverless SQL clusters automatically\n", + "- For throughput control: partition the batch by date/rack and run multiple concurrent SQL statements\n", + "- Auto Loader can feed new SVS arrivals into `svs_metadata` as `PENDING`, triggering incremental VLM runs via a scheduled Lakeflow Job\n", + "\n" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": {}, + "inputWidgets": {}, + "nuid": "dd0db54d-807b-4cfe-bca5-0b524fd6e636", + "showTitle": false, + "tableResultSettingsMap": {}, + "title": "" + } + }, + "source": [ + "# Code" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": {}, + "inputWidgets": {}, + "nuid": "6e4014e5-1ba3-4c5c-9a8c-fbe85432645b", + "showTitle": true, + "tableResultSettingsMap": {}, + "title": "Data Flow Diagram" + } + }, + "source": [ + "## Data Flow Diagram\n", + "\n", + "```mermaid\n", + "flowchart TD\n", + " %% ─── External Sources ───\n", + " SVS_INPUT[(\"/Volumes/hls_pathology/orthanc_demo/raw_images/Aperio/\\n~14 SVS files\")]\n", + " VLM_EP{{\"databricks-llama-4-maverick\\n(VLM Endpoint)\"}}\n", + "\n", + " %% ─── Delta Tables ───\n", + " OBJ_CAT[(\"douglas_moore.pathology\\n.object_catalog\")]\n", + " OBJ_RED[(\"douglas_moore.pathology\\n.object_catalog_redaction\")]\n", + " TIFF_STG[(\"douglas_moore.pathology\\n.tiff_results_staging\")]\n", + "\n", + " %% ─── Volumes (File Storage) ───\n", + " LABEL_VOL[(\"/Volumes/.../label_images/\\nPNG sub-images\")]\n", + " TIFF_VOL[(\"/Volumes/.../tiff_deidentified/\\nBigTIFF output\")]\n", + " TMP[\"/tmp/ (executor local)\\nBigTIFF staging\"]\n", + "\n", + " %% ─── Processing Steps ───\n", + " DISCOVER[\"Cell 17: File Discovery\\nSVSCatalog.catalog()\"]\n", + " META[\"Cell 18: Metadata Extraction\\nSVSMetaExtractor (mapInPandas)\\nOpenSlide properties → VARIANT\"]\n", + " PHI_TAGS[\"Cell 19: PHI Tag Report\\n(display only)\"]\n", + " EXTRACT[\"Cell 20: Extract Sub-images\\npandas_udf + OpenSlide\\nassociated_images → PNG\"]\n", + " VLM_DETECT[\"Cell 22: VLM PHI Detection\\nai_query(files => content)\\nREAD_FILES + INSERT\"]\n", + " BUILD_DF[\"Cell 24: Build redaction_df\\nJOIN catalog + redaction\\nWHERE status = PENDING\"]\n", + " UDF[\"Cell 25-27: De-identify UDF\\nmapInPandas + ThreadPoolExecutor\\nredact_image + write_pyramidal_bigtiff\"]\n", + " MERGE[\"Cell 28: MERGE Results\\nUPDATE status, paths, errors\"]\n", + " AUDIT[\"Cell 29: Audit Summary\\n(display only)\"]\n", + "\n", + " %% ─── Data Flows ───\n", + " SVS_INPUT -->|\"list files\"| DISCOVER\n", + " DISCOVER -->|\"files_df (in-memory)\"| META\n", + " SVS_INPUT -->|\"read OpenSlide props\"| META\n", + " META -->|\"mode=append\"| OBJ_CAT\n", + "\n", + " OBJ_CAT -->|\"read meta:phi_tag_report\"| PHI_TAGS\n", + "\n", + " SVS_INPUT -->|\"read associated_images\"| EXTRACT\n", + " EXTRACT -->|\"save PNG (sequential write)\"| LABEL_VOL\n", + "\n", + " LABEL_VOL -->|\"READ_FILES(binaryFile)\"| VLM_DETECT\n", + " VLM_DETECT -->|\"ai_query()\"| VLM_EP\n", + " VLM_EP -->|\"JSON response\"| VLM_DETECT\n", + " OBJ_CAT -->|\"JOIN for path\"| VLM_DETECT\n", + " VLM_DETECT -->|\"INSERT INTO\"| OBJ_RED\n", + "\n", + " OBJ_CAT -->|\"JOIN\"| BUILD_DF\n", + " OBJ_RED -->|\"WHERE PENDING\"| BUILD_DF\n", + "\n", + " BUILD_DF -->|\"redaction_df\"| UDF\n", + " SVS_INPUT -->|\"read tiles (OpenSlide)\"| UDF\n", + " UDF -->|\"redacted PNGs (seq write)\"| LABEL_VOL\n", + " UDF -->|\"write BigTIFF (seek+write)\"| TMP\n", + " TMP -->|\"shutil.copy2 (seq write)\"| TIFF_VOL\n", + " UDF -->|\"saveAsTable\"| TIFF_STG\n", + "\n", + " TIFF_STG -->|\"source for MERGE\"| MERGE\n", + " MERGE -->|\"UPDATE status/paths\"| OBJ_RED\n", + "\n", + " OBJ_CAT -->|\"LEFT JOIN\"| AUDIT\n", + " OBJ_RED -->|\"LEFT JOIN\"| AUDIT\n", + "\n", + " %% ─── Styling ───\n", + " classDef volume fill:#e8f5e9,stroke:#2e7d32\n", + " classDef table fill:#e3f2fd,stroke:#1565c0\n", + " classDef process fill:#fff3e0,stroke:#e65100\n", + " classDef external fill:#fce4ec,stroke:#c62828\n", + " classDef tmp fill:#f5f5f5,stroke:#616161,stroke-dasharray:5\n", + "\n", + " class SVS_INPUT,LABEL_VOL,TIFF_VOL volume\n", + " class OBJ_CAT,OBJ_RED,TIFF_STG table\n", + " class DISCOVER,META,PHI_TAGS,EXTRACT,VLM_DETECT,BUILD_DF,UDF,MERGE,AUDIT process\n", + " class VLM_EP external\n", + " class TMP tmp\n", + "```\n", + "\n", + "### Legend\n", + "| Color | Meaning |\n", + "|---|---|\n", + "| Green | UC Volumes (file storage) |\n", + "| Blue | Delta Tables (Unity Catalog) |\n", + "| Orange | Processing steps (notebook cells) |\n", + "| Pink | External service (model endpoint) |\n", + "| Dashed gray | Ephemeral local storage (/tmp) |\n", + "\n", + "### Key Write Patterns\n", + "| Target | Write Mode | Reason |\n", + "|---|---|---|\n", + "| `object_catalog` | `mode=append` | Idempotent cataloguing; dedup via path |\n", + "| `object_catalog_redaction` | `INSERT INTO` | One row per VLM detection run |\n", + "| `tiff_results_staging` | `mode=overwrite` | Ephemeral staging; replaced each run |\n", + "| `object_catalog_redaction` | `MERGE ... WHEN MATCHED UPDATE` | Update status after TIFF write |\n", + "| Label PNGs (volume) | Sequential FUSE write | PIL `img.save()` — no seek needed |\n", + "| BigTIFFs (volume) | `/tmp/` → `shutil.copy2` | tifffile needs seek; Volume FUSE does not support seek+write |" + ] + }, + { + "cell_type": "code", + "execution_count": 0, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": { + "byteLimit": 2048000, + "rowLimit": 10000 + }, + "inputWidgets": {}, + "nuid": "e1fbd42a-68b2-44e2-8f2a-d9210c5254f5", + "showTitle": true, + "tableResultSettingsMap": {}, + "title": "Cell 2: Install dependencies" + } + }, + "outputs": [], + "source": [ + "# Install core dependencies.\n", + "# databricks-pixels provides Catalog + Transformer base classes.\n", + "# openslide-bin bundles libopenslide.so so OpenSlide works on Serverless.\n", + "%pip install openslide-python openslide-bin tifffile imagecodecs Pillow easyocr -q" + ] + }, + { + "cell_type": "code", + "execution_count": 0, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": { + "byteLimit": 2048000, + "rowLimit": 10000 + }, + "inputWidgets": {}, + "nuid": "ec9caf79-4345-4b35-9b30-dc215d7885e4", + "showTitle": true, + "tableResultSettingsMap": {}, + "title": "Cell 3: Configuration" + } + }, + "outputs": [], + "source": [ + "# The full pixels source tree (including svs/) lives in the workspace.\n", + "# Add the src directory to sys.path so `dbx.pixels` and `dbx.pixels.svs`\n", + "# are importable without a separate pip install.\n", + "import sys\n", + "import types\n", + "import importlib\n", + "\n", + "_SRC_ROOT = \"/Workspace/Users/douglas.moore@databricks.com/pixels-svs/src\"\n", + "if _SRC_ROOT not in sys.path:\n", + " sys.path.insert(0, _SRC_ROOT)\n", + "importlib.invalidate_caches()\n", + "\n", + "# Load deidentify module (file is clean — no truncation needed)\n", + "_deident_path = f\"{_SRC_ROOT}/dbx/pixels/svs/deidentify.py\"\n", + "with open(_deident_path, \"r\") as _f:\n", + " _clean_src = _f.read()\n", + "_deident_mod = types.ModuleType(\"dbx.pixels.svs.deidentify\")\n", + "_deident_mod.__file__ = _deident_path\n", + "exec(compile(_clean_src, _deident_path, \"exec\"), _deident_mod.__dict__)\n", + "sys.modules[\"dbx.pixels.svs.deidentify\"] = _deident_mod\n", + "\n", + "from dbx.pixels.svs import SVSCatalog, SVSMetaExtractor, SVSTiffWriter\n", + "from dbx.pixels.svs.phi_tags import classify_tags, scrub_image_description\n", + "\n", + "# ── Pipeline configuration ────────────────────────────────────────────────────\n", + "CATALOG = \"douglas_moore\"\n", + "SCHEMA = \"pathology\"\n", + "UC_TABLE = f\"{CATALOG}.{SCHEMA}.object_catalog\"\n", + "UC_VOLUME = f\"{CATALOG}.{SCHEMA}.pixels_volume\"\n", + "INPUT_PATH = \"/Volumes/hls_pathology/orthanc_demo/raw_images/Aperio/\"\n", + "TIFF_VOLUME = f\"/Volumes/{CATALOG}/{SCHEMA}/tiff_deidentified\"\n", + "LABEL_VOLUME = f\"/Volumes/{CATALOG}/{SCHEMA}/label_images\"\n", + "VLM_ENDPOINT = \"databricks-llama-4-maverick\"\n", + "\n", + "print(f\"Input : {INPUT_PATH}\")\n", + "print(f\"Table : {UC_TABLE}\")\n", + "print(f\"TIFFs : {TIFF_VOLUME}\")\n", + "print(f\"Labels: {LABEL_VOLUME}\")\n", + "print(f\"VLM : {VLM_ENDPOINT}\")\n" + ] + }, + { + "cell_type": "code", + "execution_count": 0, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": { + "byteLimit": 2048000, + "implicitDf": true, + "rowLimit": 10000 + }, + "inputWidgets": {}, + "nuid": "47cec7fe-67e6-4928-9aa6-1c5947b011bc", + "showTitle": true, + "tableResultSettingsMap": {}, + "title": "Reset: Truncate pipeline tables" + } + }, + "outputs": [], + "source": [ + "%sql\n", + "-- Reset pipeline state for a clean end-to-end run.\n", + "-- Truncates data only; table structure and permissions preserved.\n", + "TRUNCATE TABLE douglas_moore.pathology.object_catalog;\n", + "TRUNCATE TABLE douglas_moore.pathology.object_catalog_redaction;" + ] + }, + { + "cell_type": "code", + "execution_count": 0, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": { + "byteLimit": 2048000, + "rowLimit": 10000 + }, + "inputWidgets": {}, + "nuid": "8c5e7ce2-bec6-48b8-987b-34b7b2a1eeb5", + "showTitle": true, + "tableResultSettingsMap": {}, + "title": "Cell 4: Storage bootstrap" + } + }, + "outputs": [], + "source": [ + "# Create schema and volumes (idempotent — safe to re-run)\n", + "spark.sql(f\"CREATE SCHEMA IF NOT EXISTS {CATALOG}.{SCHEMA}\")\n", + "spark.sql(f\"CREATE VOLUME IF NOT EXISTS {CATALOG}.{SCHEMA}.pixels_volume\")\n", + "spark.sql(f\"CREATE VOLUME IF NOT EXISTS {CATALOG}.{SCHEMA}.tiff_deidentified\")\n", + "spark.sql(f\"CREATE VOLUME IF NOT EXISTS {CATALOG}.{SCHEMA}.label_images\")\n", + "\n", + "# Initialise Delta tables:\n", + "# object_catalog — base DDL from databricks-pixels (unchanged)\n", + "# object_catalog_redaction — unified SVS/DICOM DDL from CREATE_SVS_CATALOG.sql\n", + "catalog = SVSCatalog(spark, table=UC_TABLE, volume=UC_VOLUME)\n", + "catalog.init_tables()\n", + "\n", + "print(\"Schema, volumes, and tables initialised.\")\n", + "display(spark.sql(f\"SHOW TABLES IN {CATALOG}.{SCHEMA}\"))\n" + ] + }, + { + "cell_type": "code", + "execution_count": 0, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": { + "byteLimit": 2048000, + "rowLimit": 10000 + }, + "inputWidgets": {}, + "nuid": "d0560972-3ca5-4f06-8f65-d7c621b688db", + "showTitle": true, + "tableResultSettingsMap": {}, + "title": "Cell 5: File discovery" + } + }, + "outputs": [], + "source": [ + "# Discover all SVS files under INPUT_PATH and register them in object_catalog.\n", + "# SVSCatalog.catalog() defaults pattern='*.svs'; also picks up sidecar .txt files.\n", + "files_df = catalog.catalog(INPUT_PATH)\n", + "print(f\"Discovered {files_df.count()} files\")\n", + "display(files_df.select(\"path\", \"local_path\", \"length\", \"modificationTime\", \"extension\"))\n" + ] + }, + { + "cell_type": "code", + "execution_count": 0, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": { + "byteLimit": 2048000, + "rowLimit": 10000 + }, + "inputWidgets": {}, + "nuid": "147b854b-61b3-4207-a172-9fc5f36649b9", + "showTitle": true, + "tableResultSettingsMap": {}, + "title": "Cell 6: Metadata extraction" + } + }, + "outputs": [], + "source": [ + "# SVSMetaExtractor reads every SVS via OpenSlide (ThreadPoolExecutor, 32 concurrent).\n", + "# All properties + derived fields (width, height, levels, phi_tag_report) are merged\n", + "# into one JSON dict → parse_json() → VARIANT. No schema changes to object_catalog.\n", + "extractor = SVSMetaExtractor(catalog, inputCol=\"local_path\", outputCol=\"meta\")\n", + "meta_df = extractor._transform(files_df)\n", + "\n", + "(\n", + " meta_df.write\n", + " .format(\"delta\")\n", + " .mode(\"append\")\n", + " .saveAsTable(UC_TABLE)\n", + ")\n", + "\n", + "print(f\"Wrote {spark.table(UC_TABLE).count()} rows to {UC_TABLE}\")\n", + "\n", + "display(spark.sql(f\"\"\"\n", + "SELECT\n", + " regexp_extract(path, '[^/]+$', 0) AS filename,\n", + " meta:width::int AS width,\n", + " meta:height::int AS height,\n", + " meta:level_count::int AS levels,\n", + " meta:has_label_image::boolean AS has_label,\n", + " meta:has_macro_image::boolean AS has_macro,\n", + " meta:`aperio.AppMag`::string AS app_mag,\n", + " meta:`aperio.MPP`::string AS mpp,\n", + " meta\n", + "FROM {UC_TABLE}\n", + "ORDER BY filename\n", + "\"\"\"))\n" + ] + }, + { + "cell_type": "code", + "execution_count": 0, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": { + "byteLimit": 2048000, + "rowLimit": 10000 + }, + "inputWidgets": {}, + "nuid": "541697a5-602a-40dd-9ac9-9f53624b4efa", + "showTitle": true, + "tableResultSettingsMap": { + "0": { + "dataGridStateBlob": "{\"version\":1,\"tableState\":{\"columnPinning\":{\"left\":[\"#row_number#\"],\"right\":[]},\"columnSizing\":{\"tag\":129},\"columnVisibility\":{}},\"settings\":{\"columns\":{}},\"syncTimestamp\":1781723212243}", + "filterBlob": null, + "queryPlanFiltersBlob": null, + "tableResultIndex": 0 + } + }, + "title": "Cell 7: PHI tag report" + } + }, + "outputs": [], + "source": [ + "# PHI / QUESTIONABLE tag values for every slide.\n", + "# Unpack the phi_tag_report VARIANT array stored in meta.\n", + "from pyspark.sql.functions import regexp_extract, col, explode, from_json, expr\n", + "from pyspark.sql.types import ArrayType, StructType, StructField, StringType\n", + "\n", + "phi_schema = ArrayType(StructType([\n", + " StructField(\"tag\", StringType()),\n", + " StructField(\"value\", StringType()),\n", + " StructField(\"classification\", StringType()),\n", + "]))\n", + "\n", + "phi_df = (\n", + " spark.table(UC_TABLE)\n", + " .filter(expr(\"meta:phi_tag_report IS NOT NULL\"))\n", + " .withColumn(\"phi_tag_report_str\", expr(\"cast(meta:phi_tag_report AS STRING)\"))\n", + " .withColumn(\"tags\", from_json(\"phi_tag_report_str\", phi_schema))\n", + " .withColumn(\"elem\", explode(\"tags\"))\n", + " .select(\n", + " regexp_extract(\"path\", r\"[^/]+$\", 0).alias(\"filename\"),\n", + " col(\"elem.tag\").alias(\"tag\"),\n", + " col(\"elem.value\").alias(\"value\"),\n", + " col(\"elem.classification\").alias(\"classification\"),\n", + " )\n", + " .filter(col(\"classification\").isin(\"PHI\", \"QUESTIONABLE\"))\n", + " .orderBy(\"filename\", \"classification\", \"tag\")\n", + ")\n", + "print(f\"PHI/QUESTIONABLE findings: {phi_df.count()}\")\n", + "display(phi_df)" + ] + }, + { + "cell_type": "code", + "execution_count": 0, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": { + "byteLimit": 2048000, + "rowLimit": 10000 + }, + "inputWidgets": {}, + "nuid": "28f6bfb9-bec1-4cb0-9c80-5c395a1a9432", + "showTitle": true, + "tableResultSettingsMap": {}, + "title": "Cell 8: Extract label/macro sub-images" + } + }, + "outputs": [], + "source": [ + "# Extract label and macro sub-images from each SVS and save as PNGs.\n", + "# These are later submitted to the VLM (Cell 9) and used as audit artefacts.\n", + "# Uses a pandas_udf so extraction runs distributed across workers.\n", + "from pyspark.sql.functions import pandas_udf, regexp_extract, col\n", + "import pandas as pd\n", + "from pyspark.sql.types import StringType\n", + "\n", + "_LABEL_VOL = LABEL_VOLUME # captured in closure; serialised with the UDF\n", + "\n", + "@pandas_udf(StringType())\n", + "def extract_subimages_udf(paths: pd.Series, stems: pd.Series) -> pd.Series:\n", + " import openslide, os\n", + " results = []\n", + " for path, stem in zip(paths, stems):\n", + " try:\n", + " slide = openslide.OpenSlide(path)\n", + " saved = []\n", + " for name in (\"label\", \"macro\"):\n", + " if name in slide.associated_images:\n", + " img = slide.associated_images[name].convert(\"RGB\")\n", + " out = f\"{_LABEL_VOL}/{stem}_{name}.png\"\n", + " os.makedirs(os.path.dirname(out), exist_ok=True)\n", + " img.save(out)\n", + " saved.append(out)\n", + " slide.close()\n", + " results.append(\",\".join(saved))\n", + " except Exception as e:\n", + " results.append(f\"ERROR: {e}\")\n", + " return pd.Series(results)\n", + "\n", + "catalog_df = (\n", + " spark.table(UC_TABLE)\n", + " .withColumn(\"stem\", regexp_extract(col(\"path\"), r\"([^/]+)\\.svs$\", 1))\n", + ")\n", + "\n", + "extracted_df = catalog_df.withColumn(\n", + " \"extracted_images\",\n", + " extract_subimages_udf(col(\"local_path\"), col(\"stem\")),\n", + ")\n", + "\n", + "display(extracted_df.select(\"path\", \"stem\", \"extracted_images\"))\n", + "print(f\"Label/macro PNGs written to {LABEL_VOLUME}\")\n" + ] + }, + { + "cell_type": "code", + "execution_count": 0, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": { + "byteLimit": 2048000, + "rowLimit": 10000 + }, + "inputWidgets": {}, + "nuid": "556f24ea-d0c5-46ef-ac35-761ffed88820", + "showTitle": true, + "tableResultSettingsMap": {}, + "title": "Display first 10 label images" + } + }, + "outputs": [], + "source": [ + "# Display first 10 label sub-images extracted from SVS pathology slides\n", + "import os\n", + "from PIL import Image\n", + "import matplotlib.pyplot as plt\n", + "\n", + "label_dir = LABEL_VOLUME\n", + "label_files = sorted([f for f in os.listdir(label_dir) if f.endswith(\"_label.png\")])[:10]\n", + "\n", + "ncols = min(5, len(label_files))\n", + "nrows = (len(label_files) + ncols - 1) // ncols\n", + "fig, axes = plt.subplots(nrows, ncols, figsize=(4 * ncols, 5 * nrows))\n", + "if len(label_files) == 1:\n", + " axes = [axes]\n", + "else:\n", + " axes = axes.flatten()\n", + "\n", + "for i, fname in enumerate(label_files):\n", + " img = Image.open(os.path.join(label_dir, fname))\n", + " axes[i].imshow(img)\n", + " axes[i].set_title(fname.replace(\"_label.png\", \"\"), fontsize=9)\n", + " axes[i].axis(\"off\")\n", + "\n", + "for j in range(len(label_files), len(axes)):\n", + " axes[j].axis(\"off\")\n", + "\n", + "plt.suptitle(\"SVS Label Sub-Images (PHI candidates for VLM redaction)\", fontsize=13)\n", + "plt.tight_layout()\n", + "plt.show()" + ] + }, + { + "cell_type": "code", + "execution_count": 0, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": { + "byteLimit": 2048000, + "implicitDf": true, + "rowLimit": 10000 + }, + "height": "156", + "inputWidgets": {}, + "nuid": "23583e7a-dfc7-4da0-be2b-dbac0b061935", + "showTitle": true, + "tableResultSettingsMap": {}, + "title": "Cell 9: VLM PHI detection", + "width": "834" + } + }, + "outputs": [], + "source": [ + "%sql\n", + "-- Run VLM PHI detection on all label PNGs and insert results into object_catalog_redaction.\n", + "-- ai_query() submits the image binary directly via `files => content` (no base64 needed).\n", + "-- responseFormat => 'json_object' guarantees machine-parseable output.\n", + "INSERT INTO douglas_moore.pathology.object_catalog_redaction (\n", + " redaction_id, path, extension, modality,\n", + " has_phi, phi_elements, vlm_raw_response, model_endpoint,\n", + " output_file_paths, label_image_path, macro_image_path,\n", + " status, insert_timestamp, created_by\n", + ")\n", + "WITH vlm_raw AS (\n", + " SELECT\n", + " regexp_replace(f._metadata.file_name, '_label\\.png$', '') AS stem,\n", + " f._metadata.file_path AS label_image_path,\n", + " ai_query(\n", + " 'databricks-llama-4-maverick',\n", + " 'You are a HIPAA-compliant PHI detection system.\n", + "Analyze this pathology slide label image and identify all Protected Health Information:\n", + "patient names, dates, MRNs, accession numbers, barcodes, or other identifying text.\n", + "Return ONLY a json object — no prose, no markdown fences.\n", + "Schema: {\"has_phi\": bool, \"phi_elements\": [{\"type\": \"name|dob|mrn|accession|barcode|other\", \"value_hint\": \"\", \"bbox\": {\"x\": int, \"y\": int, \"w\": int, \"h\": int}, \"subimage\": \"label\"}]}\n", + "If no PHI found: {\"has_phi\": false, \"phi_elements\": []}',\n", + " files => content\n", + " ) AS vlm_raw_response\n", + " FROM READ_FILES(\n", + " '/Volumes/douglas_moore/pathology/label_images/',\n", + " format => 'binaryFile',\n", + " fileNamePattern => '*_label.png'\n", + " ) f\n", + "),\n", + "joined AS (\n", + " SELECT\n", + " v.stem,\n", + " v.label_image_path,\n", + " v.vlm_raw_response,\n", + " m.path AS obj_path,\n", + " concat('/Volumes/douglas_moore/pathology/label_images/', v.stem, '_macro.png') AS macro_image_path\n", + " FROM vlm_raw v\n", + " JOIN douglas_moore.pathology.object_catalog m\n", + " ON regexp_extract(m.path, '([^/]+)\\.svs$', 1) = v.stem\n", + ")\n", + "SELECT\n", + " uuid() AS redaction_id,\n", + " obj_path AS path,\n", + " 'svs' AS extension,\n", + " 'WSI' AS modality,\n", + " try_cast(get_json_object(vlm_raw_response, '$.has_phi') AS BOOLEAN) AS has_phi,\n", + " parse_json(get_json_object(vlm_raw_response, '$.phi_elements')) AS phi_elements,\n", + " vlm_raw_response,\n", + " 'databricks-llama-4-maverick' AS model_endpoint,\n", + " array(CAST(NULL AS STRING)) AS output_file_paths,\n", + " label_image_path,\n", + " macro_image_path,\n", + " 'PENDING' AS status,\n", + " current_timestamp() AS insert_timestamp,\n", + " current_user() AS created_by\n", + "FROM joined\n" + ] + }, + { + "cell_type": "code", + "execution_count": 0, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": { + "byteLimit": 2048000, + "implicitDf": true, + "rowLimit": 10000 + }, + "inputWidgets": {}, + "nuid": "6dae4df9-e494-403e-9c58-a933aa22052a", + "showTitle": false, + "tableResultSettingsMap": {}, + "title": "" + } + }, + "outputs": [], + "source": [ + "%sql\n", + "select * from douglas_moore.pathology.object_catalog_redaction" + ] + }, + { + "cell_type": "code", + "execution_count": 0, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": { + "byteLimit": 2048000, + "rowLimit": 10000 + }, + "inputWidgets": {}, + "nuid": "98cc5d83-bbdd-4f8c-9bd6-b61d259ad882", + "showTitle": false, + "tableResultSettingsMap": {}, + "title": "" + } + }, + "outputs": [], + "source": [ + "# Reload modules to pick up streaming TIFF writer\n", + "import importlib, sys\n", + "for mod_name in list(sys.modules):\n", + " if mod_name.startswith(\"dbx.pixels.svs\"):\n", + " del sys.modules[mod_name]\n", + "\n", + "# Join PENDING redaction rows (phi_elements from VLM) with object_catalog (local_path),\n", + "# run de-identification and produce pyramidal BigTIFFs.\n", + "redaction_df = spark.sql(f\"\"\"\n", + "SELECT\n", + " o.local_path,\n", + " o.path,\n", + " r.redaction_id,\n", + " to_json(r.phi_elements) AS phi_elements_json\n", + "FROM {CATALOG}.{SCHEMA}.object_catalog o\n", + "JOIN {CATALOG}.{SCHEMA}.object_catalog_redaction r\n", + " ON o.path = r.path\n", + "WHERE r.status = 'PENDING'\n", + "\"\"\")\n", + "print(f\"Files to de-identify: {redaction_df.count()}\")" + ] + }, + { + "cell_type": "code", + "execution_count": 0, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": { + "byteLimit": 2048000, + "rowLimit": 10000 + }, + "inputWidgets": {}, + "nuid": "a73b4474-3adc-4866-88cd-822dc0ad6c45", + "showTitle": true, + "tableResultSettingsMap": {}, + "title": "Cell 10: De-identified TIFF write" + } + }, + "outputs": [], + "source": [ + "# --- Distributed de-identification via mapInPandas (memory-safe for Serverless 1 GB) ---\n", + "#\n", + "# Design principles applied from review:\n", + "# • No inner ThreadPoolExecutor — mapInPandas already parallelizes across Spark\n", + "# partitions; nested threading doubles slide opens and memory pressure.\n", + "# • No to_dict(\"records\") — iterate rows via iloc to avoid duplicating the batch.\n", + "# • OpenSlide closed in a finally block so C-bindings are destroyed even on error.\n", + "# • gc.collect() after each slide reclaims PIL/OpenSlide C-level allocations.\n", + "# • Tile-based TIFF writing via write_pyramidal_bigtiff_streaming (256×256 read_region).\n", + "# • Temp files staged to /tmp (seek-capable), then shutil.copy2 to Volume (seq FUSE).\n", + "# • PID suffix on temp paths prevents collisions across retries/speculative tasks.\n", + "# • repartition(num_slides) ensures 1 row per partition — each executor handles\n", + "# exactly one slide. (arrow.maxRecordsPerBatch is NOT settable on Serverless.)\n", + "\n", + "import json\n", + "import pandas as pd\n", + "from pyspark.sql.types import StructType, StructField, StringType, ArrayType, IntegerType\n", + "\n", + "_TIFF_VOLUME = TIFF_VOLUME\n", + "_LABEL_VOLUME = LABEL_VOLUME\n", + "_SRC_ROOT = \"/Workspace/Users/douglas.moore@databricks.com/pixels-svs/src\"\n", + "\n", + "_result_schema = StructType([\n", + " StructField(\"path\", StringType(), True),\n", + " StructField(\"tiff_output_path\", StringType(), True),\n", + " StructField(\"label_image_path\", StringType(), True),\n", + " StructField(\"macro_image_path\", StringType(), True),\n", + " StructField(\"phi_tags_redacted\", ArrayType(StringType()), True),\n", + " StructField(\"pixel_regions_redacted\", IntegerType(), True),\n", + " StructField(\"error\", StringType(), True),\n", + "])\n", + "\n", + "\n", + "def _deidentify_batch(iterator):\n", + " \"\"\"mapInPandas worker: one slide per batch, streaming tile reads, no threading.\"\"\"\n", + " import sys, os, gc, shutil, time, logging, resource\n", + " from pathlib import Path\n", + "\n", + " # --- Memory debugging utilities ---\n", + " logging.basicConfig(level=logging.INFO, format=\"%(asctime)s [%(levelname)s] %(message)s\")\n", + " log = logging.getLogger(\"deidentify_worker\")\n", + "\n", + " def _mem_mb() -> dict:\n", + " \"\"\"Return RSS and VMS in MB from /proc/self/status (Linux) with fallback.\"\"\"\n", + " rss_mb = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss / 1024 # KB→MB on Linux\n", + " try:\n", + " with open(\"/proc/self/status\") as f:\n", + " status = f.read()\n", + " vmpeak = vmrss = vmsize = 0\n", + " for line in status.splitlines():\n", + " if line.startswith(\"VmPeak:\"):\n", + " vmpeak = int(line.split()[1]) / 1024\n", + " elif line.startswith(\"VmRSS:\"):\n", + " vmrss = int(line.split()[1]) / 1024\n", + " elif line.startswith(\"VmSize:\"):\n", + " vmsize = int(line.split()[1]) / 1024\n", + " return {\"rss_mb\": round(vmrss, 1), \"vms_mb\": round(vmsize, 1), \"peak_mb\": round(vmpeak, 1)}\n", + " except Exception:\n", + " return {\"rss_mb\": round(rss_mb, 1), \"vms_mb\": -1, \"peak_mb\": -1}\n", + "\n", + " def _log_mem(stage: str, stem: str, extra: str = \"\"):\n", + " mem = _mem_mb()\n", + " msg = f\"[{stem}] stage={stage} | RSS={mem['rss_mb']}MB VMS={mem['vms_mb']}MB Peak={mem['peak_mb']}MB\"\n", + " if extra:\n", + " msg += f\" | {extra}\"\n", + " log.info(msg)\n", + " # Warn if approaching the 1024 MB limit\n", + " if mem[\"rss_mb\"] > 800:\n", + " log.warning(f\"⚠️ HIGH MEMORY [{stem}] stage={stage} RSS={mem['rss_mb']}MB — approaching 1024MB limit!\")\n", + "\n", + " # Ensure src modules are importable on executors\n", + " if _SRC_ROOT not in sys.path:\n", + " sys.path.insert(0, _SRC_ROOT)\n", + "\n", + " import openslide\n", + " from dbx.pixels.svs.deidentify import redact_image, write_pyramidal_bigtiff_streaming\n", + " from dbx.pixels.svs.phi_tags import scrub_image_description\n", + "\n", + " for pdf in iterator:\n", + " results = []\n", + " log.info(f\"Batch received: {len(pdf)} row(s) | PID={os.getpid()}\")\n", + " _log_mem(\"batch_start\", \"batch\", f\"rows={len(pdf)}\")\n", + "\n", + " # Iterate rows directly via iloc — no to_dict(\"records\") memory copy\n", + " for idx in range(len(pdf)):\n", + " row = pdf.iloc[idx]\n", + " svs_path = row[\"local_path\"]\n", + " phi_json = row[\"phi_elements_json\"]\n", + " stem = Path(svs_path).stem\n", + " stage = \"init\"\n", + " slide = None\n", + " tmp_tiff = f\"/tmp/{stem}_{os.getpid()}.tiff\"\n", + " t0 = time.time()\n", + "\n", + " log.info(f\"=== Processing slide: {stem} ===\")\n", + " _log_mem(\"init\", stem, f\"svs_path={svs_path}\")\n", + "\n", + " try:\n", + " phi_elements = json.loads(phi_json) if phi_json else []\n", + " stage = \"open_slide\"\n", + " slide = openslide.OpenSlide(svs_path)\n", + " dims = slide.dimensions # (width, height) at level 0\n", + " levels = slide.level_count\n", + " _log_mem(\"open_slide\", stem, f\"dims={dims[0]}x{dims[1]} levels={levels}\")\n", + "\n", + " # 1. Redact label/macro sub-images (small RGBA → write PNGs to Volume)\n", + " label_path = macro_path = None\n", + " pixel_count = 0\n", + "\n", + " if \"label\" in slide.associated_images:\n", + " stage = \"redact_label\"\n", + " label_img = slide.associated_images[\"label\"]\n", + " label_phi = [e for e in phi_elements if e.get(\"subimage\", \"label\") != \"macro\"]\n", + " label_out = redact_image(label_img, label_phi)\n", + " pixel_count += len(label_phi)\n", + " label_path = f\"{_LABEL_VOLUME}/{stem}_label.png\"\n", + " os.makedirs(os.path.dirname(label_path), exist_ok=True)\n", + " label_out.save(label_path)\n", + " _log_mem(\"redact_label\", stem, f\"label_size={label_img.size} phi_count={len(label_phi)}\")\n", + " del label_img, label_out # free RGBA buffer immediately\n", + "\n", + " if \"macro\" in slide.associated_images:\n", + " stage = \"redact_macro\"\n", + " macro_img = slide.associated_images[\"macro\"]\n", + " macro_phi = [e for e in phi_elements if e.get(\"subimage\") == \"macro\"]\n", + " macro_out = redact_image(macro_img, macro_phi)\n", + " pixel_count += len(macro_phi)\n", + " macro_path = f\"{_LABEL_VOLUME}/{stem}_macro.png\"\n", + " os.makedirs(os.path.dirname(macro_path), exist_ok=True)\n", + " macro_out.save(macro_path)\n", + " _log_mem(\"redact_macro\", stem, f\"macro_size={macro_img.size} phi_count={len(macro_phi)}\")\n", + " del macro_img, macro_out\n", + "\n", + " # 2. Scrub metadata tags\n", + " stage = \"scrub_metadata\"\n", + " raw_desc = slide.properties.get(\"tiff.ImageDescription\", \"\")\n", + " scrubbed = scrub_image_description(raw_desc)\n", + " phi_tags_redacted = (\n", + " [\"tiff.ImageDescription\", \"openslide.comment\"]\n", + " if raw_desc != scrubbed else []\n", + " )\n", + " _log_mem(\"scrub_metadata\", stem)\n", + "\n", + " # 3. Write pyramidal BigTIFF — tile-streaming (256×256 read_region)\n", + " # Stage to /tmp (requires seek), then sequential copy to Volume.\n", + " stage = \"write_tiff_to_tmp\"\n", + " t_tiff_start = time.time()\n", + " write_pyramidal_bigtiff_streaming(tmp_tiff, slide, jpeg_quality=80)\n", + " t_tiff_elapsed = time.time() - t_tiff_start\n", + " tmp_size_mb = os.path.getsize(tmp_tiff) / (1024 * 1024) if os.path.exists(tmp_tiff) else 0\n", + " _log_mem(\"write_tiff_done\", stem, f\"tiff_size={tmp_size_mb:.1f}MB elapsed={t_tiff_elapsed:.1f}s\")\n", + "\n", + " # Close slide BEFORE copy to free C-level handles and mapped memory\n", + " slide.close()\n", + " slide = None\n", + " _log_mem(\"slide_closed\", stem)\n", + "\n", + " stage = \"copy_tiff_to_volume\"\n", + " t_copy_start = time.time()\n", + " tiff_path = f\"{_TIFF_VOLUME}/{stem}.tiff\"\n", + " shutil.copy2(tmp_tiff, tiff_path)\n", + " os.remove(tmp_tiff)\n", + " t_copy_elapsed = time.time() - t_copy_start\n", + " _log_mem(\"copy_done\", stem, f\"copy_elapsed={t_copy_elapsed:.1f}s\")\n", + "\n", + " total_elapsed = time.time() - t0\n", + " log.info(f\"✓ [{stem}] completed in {total_elapsed:.1f}s | tiff={tmp_size_mb:.1f}MB\")\n", + "\n", + " results.append({\n", + " \"path\": row[\"path\"],\n", + " \"tiff_output_path\": tiff_path,\n", + " \"label_image_path\": label_path,\n", + " \"macro_image_path\": macro_path,\n", + " \"phi_tags_redacted\": phi_tags_redacted,\n", + " \"pixel_regions_redacted\": pixel_count,\n", + " \"error\": None,\n", + " })\n", + "\n", + " except Exception as exc:\n", + " _log_mem(\"ERROR\", stem, f\"stage={stage} exc={type(exc).__name__}: {exc}\")\n", + " results.append({\n", + " \"path\": row[\"path\"],\n", + " \"tiff_output_path\": None,\n", + " \"label_image_path\": None,\n", + " \"macro_image_path\": None,\n", + " \"phi_tags_redacted\": [],\n", + " \"pixel_regions_redacted\": 0,\n", + " \"error\": f\"[stage={stage}] {type(exc).__name__}: {exc}\",\n", + " })\n", + "\n", + " finally:\n", + " # Ensure slide is always closed — destroy C-bindings immediately\n", + " if slide is not None:\n", + " try:\n", + " slide.close()\n", + " except Exception:\n", + " pass\n", + " # Clean up temp file on failure\n", + " if os.path.exists(tmp_tiff):\n", + " try:\n", + " os.remove(tmp_tiff)\n", + " except Exception:\n", + " pass\n", + " # Force GC to reclaim PIL/OpenSlide C-level allocations\n", + " gc.collect()\n", + " _log_mem(\"gc_complete\", stem)\n", + "\n", + " yield pd.DataFrame(results)" + ] + }, + { + "cell_type": "code", + "execution_count": 0, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": { + "byteLimit": 2048000, + "rowLimit": 10000 + }, + "inputWidgets": {}, + "nuid": "b449d29c-0651-41bc-afc8-bf83bdcf7f74", + "showTitle": false, + "tableResultSettingsMap": {}, + "title": "" + } + }, + "outputs": [], + "source": [ + "# Materialize the expensive mapInPandas UDF exactly ONCE.\n", + "# Strategy: persist() + count() forces a single execution pass.\n", + "# Downstream MERGE reads from the cached DataFrame via temp view.\n", + "TIFF_RESULTS_TABLE = f\"{CATALOG}.{SCHEMA}.tiff_results_staging\"\n", + "\n", + "num_slides = redaction_df.count()\n", + "print(f\"\"\"{num_slides}\"\"\")\n", + "assert num_slides > 0, \"No PENDING slides to de-identify — check object_catalog_redaction status\"\n" + ] + }, + { + "cell_type": "code", + "execution_count": 0, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": { + "byteLimit": 2048000, + "rowLimit": 10000 + }, + "inputWidgets": {}, + "nuid": "2581617a-cb08-4d46-b604-56b0c4d0400e", + "showTitle": true, + "tableResultSettingsMap": {}, + "title": "Dry run: 1 slide with memory logging" + } + }, + "outputs": [], + "source": [ + "# --- Dry run: 1 slide on DRIVER to capture full memory trace ---\n", + "# Runs the same logic outside mapInPandas so we can see exactly which stage\n", + "# exceeds 1024 MB without the executor being killed.\n", + "\n", + "import sys, os, gc, time, resource, json, shutil\n", + "from pathlib import Path\n", + "\n", + "if \"/Workspace/Users/douglas.moore@databricks.com/pixels-svs/src\" not in sys.path:\n", + " sys.path.insert(0, \"/Workspace/Users/douglas.moore@databricks.com/pixels-svs/src\")\n", + "\n", + "import openslide\n", + "from dbx.pixels.svs.deidentify import redact_image, write_pyramidal_bigtiff_streaming\n", + "from dbx.pixels.svs.phi_tags import scrub_image_description\n", + "\n", + "def _mem_mb():\n", + " \"\"\"RSS/VMS/Peak from /proc/self/status.\"\"\"\n", + " try:\n", + " with open(\"/proc/self/status\") as f:\n", + " status = f.read()\n", + " vals = {}\n", + " for line in status.splitlines():\n", + " for key in (\"VmPeak\", \"VmRSS\", \"VmSize\"):\n", + " if line.startswith(key + \":\"):\n", + " vals[key] = int(line.split()[1]) / 1024 # KB→MB\n", + " return {\"rss_mb\": round(vals.get(\"VmRSS\", 0), 1),\n", + " \"vms_mb\": round(vals.get(\"VmSize\", 0), 1),\n", + " \"peak_mb\": round(vals.get(\"VmPeak\", 0), 1)}\n", + " except Exception:\n", + " return {\"rss_mb\": -1, \"vms_mb\": -1, \"peak_mb\": -1}\n", + "\n", + "def log_mem(stage, extra=\"\"):\n", + " mem = _mem_mb()\n", + " warn = \" ⚠️ OVER 1GB!\" if mem[\"rss_mb\"] > 1024 else (\"⚠️ HIGH\" if mem[\"rss_mb\"] > 800 else \"\")\n", + " print(f\" [{stage:20s}] RSS={mem['rss_mb']:>7.1f}MB VMS={mem['vms_mb']:>7.1f}MB Peak={mem['peak_mb']:>7.1f}MB {warn} {extra}\")\n", + "\n", + "# Get one slide from the redaction dataframe\n", + "row = redaction_df.limit(1).collect()[0]\n", + "svs_path = row[\"local_path\"]\n", + "phi_json = row[\"phi_elements_json\"]\n", + "stem = Path(svs_path).stem\n", + "tmp_tiff = f\"/tmp/{stem}_dryrun.tiff\"\n", + "\n", + "print(f\"\\n{'='*80}\")\n", + "print(f\"DRY RUN MEMORY PROFILE: {stem}\")\n", + "print(f\"SVS path: {svs_path}\")\n", + "print(f\"{'='*80}\")\n", + "\n", + "gc.collect()\n", + "log_mem(\"baseline\")\n", + "\n", + "# Open slide\n", + "t0 = time.time()\n", + "slide = openslide.OpenSlide(svs_path)\n", + "dims = slide.dimensions\n", + "print(f\"\\n Slide: {dims[0]}x{dims[1]} pixels, {slide.level_count} levels\")\n", + "print(f\" Level dimensions: {[slide.level_dimensions[i] for i in range(slide.level_count)]}\")\n", + "print(f\" Associated images: {list(slide.associated_images.keys())}\")\n", + "log_mem(\"open_slide\", f\"file_size={os.path.getsize(svs_path)/(1024*1024):.1f}MB\")\n", + "\n", + "# Redact label\n", + "if \"label\" in slide.associated_images:\n", + " label_img = slide.associated_images[\"label\"]\n", + " print(f\"\\n Label image: {label_img.size} mode={label_img.mode}\")\n", + " phi_elements = json.loads(phi_json) if phi_json else []\n", + " label_phi = [e for e in phi_elements if e.get(\"subimage\", \"label\") != \"macro\"]\n", + " label_out = redact_image(label_img, label_phi)\n", + " log_mem(\"redact_label\", f\"phi_regions={len(label_phi)}\")\n", + " del label_img, label_out\n", + " gc.collect()\n", + " log_mem(\"label_freed\")\n", + "\n", + "# Redact macro\n", + "if \"macro\" in slide.associated_images:\n", + " macro_img = slide.associated_images[\"macro\"]\n", + " print(f\"\\n Macro image: {macro_img.size} mode={macro_img.mode}\")\n", + " phi_elements = json.loads(phi_json) if phi_json else []\n", + " macro_phi = [e for e in phi_elements if e.get(\"subimage\") == \"macro\"]\n", + " macro_out = redact_image(macro_img, macro_phi)\n", + " log_mem(\"redact_macro\", f\"phi_regions={len(macro_phi)}\")\n", + " del macro_img, macro_out\n", + " gc.collect()\n", + " log_mem(\"macro_freed\")\n", + "\n", + "# Scrub metadata\n", + "raw_desc = slide.properties.get(\"tiff.ImageDescription\", \"\")\n", + "scrubbed = scrub_image_description(raw_desc)\n", + "log_mem(\"scrub_metadata\")\n", + "\n", + "# Write pyramidal BigTIFF (this is the suspected memory hog)\n", + "print(f\"\\n Writing pyramidal BigTIFF to /tmp ...\")\n", + "t_tiff = time.time()\n", + "try:\n", + " write_pyramidal_bigtiff_streaming(tmp_tiff, slide, jpeg_quality=80)\n", + " tiff_elapsed = time.time() - t_tiff\n", + " tiff_size = os.path.getsize(tmp_tiff) / (1024 * 1024)\n", + " log_mem(\"write_tiff_done\", f\"size={tiff_size:.1f}MB elapsed={tiff_elapsed:.1f}s\")\n", + "except Exception as e:\n", + " log_mem(\"write_tiff_FAILED\", f\"{type(e).__name__}: {e}\")\n", + " tiff_size = 0\n", + "\n", + "# Close slide\n", + "slide.close()\n", + "log_mem(\"slide_closed\")\n", + "gc.collect()\n", + "log_mem(\"gc_after_close\")\n", + "\n", + "# Cleanup\n", + "if os.path.exists(tmp_tiff):\n", + " os.remove(tmp_tiff)\n", + "\n", + "total = time.time() - t0\n", + "print(f\"\\n{'='*80}\")\n", + "print(f\"COMPLETE: {stem} in {total:.1f}s | TIFF={tiff_size:.1f}MB\")\n", + "print(f\"Peak memory: {_mem_mb()['peak_mb']:.1f}MB\")\n", + "print(f\"{'='*80}\")\n", + "if _mem_mb()[\"peak_mb\"] > 1024:\n", + " print(\"\\n❌ Peak memory EXCEEDED 1024MB — this will OOM on Serverless executors.\")\n", + " print(\" → Investigate write_pyramidal_bigtiff_streaming tile buffer size.\")\n", + "else:\n", + " print(\"\\n✅ Peak memory stayed under 1024MB — safe for Serverless executors.\")" + ] + }, + { + "cell_type": "code", + "execution_count": 0, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": { + "byteLimit": 2048000, + "rowLimit": 10000 + }, + "inputWidgets": {}, + "nuid": "a8e83dad-0127-488c-b894-5d442508e404", + "showTitle": true, + "tableResultSettingsMap": {}, + "title": "Cell 26: Execute TIFF write (materialize once)" + } + }, + "outputs": [], + "source": [ + "\n", + "# 1 slide per partition to stay under 1GB UDF memory limit on serverless.\n", + "# Large SVS files (CMU-1 = 46000x32000) need sole access to executor RAM.\n", + "results_df = (\n", + " redaction_df\n", + " .repartition(num_slides)\n", + " .mapInPandas(_deidentify_batch, schema=_result_schema)\n", + " .limit(4)\n", + ")\n", + "\n", + "# Force single execution — UDF runs here and only here\n", + "results_df.select(\"path\", \"tiff_output_path\", \"label_image_path\", \"macro_image_path\", \"phi_tags_redacted\", \"pixel_regions_redacted\", \"error\").write.saveAsTable(TIFF_RESULTS_TABLE)\n", + "\n", + "\n", + "display(spark.sql(f\"SELECT * FROM {TIFF_RESULTS_TABLE}\"))" + ] + }, + { + "cell_type": "code", + "execution_count": 0, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": { + "byteLimit": 2048000, + "rowLimit": 10000 + }, + "inputWidgets": {}, + "nuid": "5c7b3af8-ded5-4207-92d1-42a0710e11e6", + "showTitle": true, + "tableResultSettingsMap": {}, + "title": "Cell 23" + } + }, + "outputs": [], + "source": [ + "# Merge output paths and status back into object_catalog_redaction\n", + "spark.sql(f\"\"\"\n", + "MERGE INTO {CATALOG}.{SCHEMA}.object_catalog_redaction AS tgt\n", + "USING (\n", + " SELECT * FROM (\n", + " SELECT *, ROW_NUMBER() OVER (PARTITION BY path ORDER BY path) AS rn\n", + " FROM tiff_results\n", + " ) WHERE rn = 1\n", + ") AS src\n", + " ON tgt.path = src.path\n", + "WHEN MATCHED THEN UPDATE SET\n", + " tgt.output_file_paths = array(src.tiff_output_path),\n", + " tgt.label_image_path = COALESCE(src.label_image_path, tgt.label_image_path),\n", + " tgt.macro_image_path = COALESCE(src.macro_image_path, tgt.macro_image_path),\n", + " tgt.phi_tags_redacted = src.phi_tags_redacted,\n", + " tgt.pixel_redactions_count = src.pixel_regions_redacted,\n", + " tgt.status = CASE WHEN src.error IS NULL THEN 'SUCCESS' ELSE 'FAILED' END,\n", + " tgt.error_messages = CASE WHEN src.error IS NOT NULL THEN array(src.error) ELSE NULL END,\n", + " tgt.update_timestamp = current_timestamp()\n", + "\"\"\")\n", + "print(\"TIFF write complete. Redaction records updated.\")" + ] + }, + { + "cell_type": "code", + "execution_count": 0, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": { + "byteLimit": 2048000, + "rowLimit": 10000 + }, + "inputWidgets": {}, + "nuid": "4ec2aac1-43ab-42ee-aae7-0fdf52c006ec", + "showTitle": true, + "tableResultSettingsMap": {}, + "title": "Cell 11: Audit summary" + } + }, + "outputs": [], + "source": [ + "# End-to-end audit: join object_catalog with object_catalog_redaction and summarise.\n", + "audit_df = spark.sql(f\"\"\"\n", + "SELECT\n", + " regexp_extract(o.path, '[^/]+$', 0) AS filename,\n", + " o.meta:width::int AS width_px,\n", + " o.meta:height::int AS height_px,\n", + " o.meta:level_count::int AS pyramid_levels,\n", + " r.has_phi,\n", + " r.status,\n", + " r.pixel_redactions_count,\n", + " size(r.phi_tags_redacted) AS tag_redactions,\n", + " r.output_file_paths[0] AS tiff_output_path,\n", + " r.label_image_path,\n", + " r.error_messages[0] AS error\n", + "FROM {CATALOG}.{SCHEMA}.object_catalog o\n", + "LEFT JOIN {CATALOG}.{SCHEMA}.object_catalog_redaction r\n", + " ON o.path = r.path\n", + "ORDER BY filename\n", + "\"\"\")\n", + "\n", + "total = audit_df.count()\n", + "phi_ct = audit_df.filter(\"has_phi = true\").count()\n", + "ok_ct = audit_df.filter(\"status = 'SUCCESS'\").count()\n", + "err_ct = audit_df.filter(\"status = 'FAILED'\").count()\n", + "\n", + "print(f\"Slides in catalog : {total}\")\n", + "print(f\"VLM-flagged with PHI : {phi_ct}\")\n", + "print(f\"Successfully written : {ok_ct}\")\n", + "print(f\"Errors : {err_ct}\")\n", + "\n", + "display(audit_df)\n" + ] + } + ], + "metadata": { + "application/vnd.databricks.v1+notebook": { + "computePreferences": null, + "dashboards": [], + "environmentMetadata": { + "base_environment": "", + "environment_version": "5" + }, + "inputWidgetPreferences": null, + "language": "python", + "notebookMetadata": { + "mostRecentlyExecutedCommandWithImplicitDF": { + "commandId": 8994411946469750, + "dataframes": [ + "_sqldf" + ] + }, + "pythonIndentUnit": 2 + }, + "notebookName": "SVS Pathology De-identification Pipeline", + "widgets": {} + }, + "language_info": { + "name": "python" + } + }, + "nbformat": 4, + "nbformat_minor": 0 +} diff --git a/src/dbx/pixels/svs/__init__.py b/src/dbx/pixels/svs/__init__.py new file mode 100644 index 00000000..143b325f --- /dev/null +++ b/src/dbx/pixels/svs/__init__.py @@ -0,0 +1,43 @@ +"""dbx.pixels.svs — SVS (Aperio Whole Slide Image) extension for databricks-pixels. + +Install alongside ``databricks-pixels`` and extend the ``dbx.pixels`` namespace:: + + import dbx.pixels + _SVS_SRC = "/Workspace/Users//svs-pixels/src/dbx/pixels" + if _SVS_SRC not in dbx.pixels.__path__: + dbx.pixels.__path__.append(_SVS_SRC) + + from dbx.pixels.svs import SVSCatalog, SVSMetaExtractor, SVSTiffWriter +""" + +from dbx.pixels.svs.catalog import SVSCatalog +from dbx.pixels.svs.svs_meta_extractor import SVSMetaExtractor +from dbx.pixels.svs.phi_tags import ( + classify_tag, + classify_tags, + scrub_image_description, + PHI_TAGS, + QUESTIONABLE_TAGS, + NOT_PHI_TAGS, +) + +# Lazy import: SVSTiffWriter depends on deidentify.py which may not be clean +# on all environments. Import on first access only. +def __getattr__(name): + if name == "SVSTiffWriter": + from dbx.pixels.svs.svs_tiff_writer import SVSTiffWriter + globals()["SVSTiffWriter"] = SVSTiffWriter + return SVSTiffWriter + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") + +__all__ = [ + "SVSCatalog", + "SVSMetaExtractor", + "SVSTiffWriter", # lazy-loaded + "classify_tag", + "classify_tags", + "scrub_image_description", + "PHI_TAGS", + "QUESTIONABLE_TAGS", + "NOT_PHI_TAGS", +] diff --git a/src/dbx/pixels/svs/catalog.py b/src/dbx/pixels/svs/catalog.py new file mode 100644 index 00000000..4fdb188a --- /dev/null +++ b/src/dbx/pixels/svs/catalog.py @@ -0,0 +1,74 @@ +"""SVSCatalog — extends the base Catalog for Aperio SVS whole-slide images. + +Overrides: + - catalog() defaults to pattern='*.svs' + - init_tables() calls super().init_tables() then executes SVS-specific DDL + (CREATE_SVS_CATALOG.sql creates object_catalog_redaction) +""" + +from __future__ import annotations + +from dbx.pixels.catalog import Catalog + + +class SVSCatalog(Catalog): + """Object catalog for Aperio SVS whole-slide images. + + Extends :class:`dbx.pixels.Catalog` with SVS-specific defaults: + - ``catalog()`` uses ``pattern='*.svs'`` + - ``init_tables()`` also creates the ``object_catalog_redaction`` table + via ``CREATE_SVS_CATALOG.sql`` + + Args: + spark: Active SparkSession. + table: Fully qualified UC table name (e.g. ``douglas_moore.pathology.object_catalog``). + volume: Fully qualified UC volume name (e.g. ``douglas_moore.pathology.pixels_volume``). + """ + + def __init__(self, spark, table: str, volume: str): + super().__init__(spark, table=table, volume=volume) + + def init_tables(self): + """Create base tables via parent, then run SVS-specific DDL.""" + # Base DDL creates object_catalog + super().init_tables() + + # SVS-specific: create object_catalog_redaction + from pathlib import Path + + sql_path = Path(__file__).parent / "resources" / "sql" + + try: + sql_files = { + p.name: p.read_text() + for p in sql_path.iterdir() + if p.suffix == ".sql" and p.is_file() + } + except (PermissionError, OSError): + # Serverless may block non-Python file reads; fall back to SDK + from databricks.sdk import WorkspaceClient + + ws_path = str(sql_path) + sql_files = {} + w = WorkspaceClient() + for obj in w.workspace.list(ws_path): + if obj.path and obj.path.endswith(".sql"): + name = obj.path.rsplit("/", 1)[-1] + with w.workspace.download(obj.path) as f: + sql_files[name] = f.read().decode("utf-8") + + for file_name, content in sql_files.items(): + sql_commands = content.replace("{UC_TABLE}", self._table).replace( + "{UC_SCHEMA}", self._schema + ) + for sql_command in sql_commands.split(";"): + if sql_command.strip(): + self._spark.sql(sql_command) + + def catalog(self, path: str, pattern: str = "*.svs", **kwargs): + """Catalog SVS files at the given path. + + Delegates to :meth:`Catalog.catalog` with ``pattern='*.svs'`` default. + All other keyword arguments are forwarded unchanged. + """ + return super().catalog(path, pattern=pattern, **kwargs) diff --git a/src/dbx/pixels/svs/deidentify.py b/src/dbx/pixels/svs/deidentify.py new file mode 100644 index 00000000..f2ac794b --- /dev/null +++ b/src/dbx/pixels/svs/deidentify.py @@ -0,0 +1,224 @@ +"""De-identification image utilities for SVS pathology slides. + +Functions for redacting PHI from label/macro sub-images and writing +de-identified pyramidal BigTIFF outputs. +""" + +import os +import numpy as np +from PIL import Image, ImageDraw + + +def redact_image(img: "Image.Image", phi_elements: list) -> "Image.Image": + """Apply black-rectangle redaction to detected PHI bounding boxes. + + Parameters + ---------- + img : PIL.Image.Image + The source image (label or macro sub-image). + phi_elements : list[dict] + Each dict must have a "bbox" key with {"x", "y", "w", "h"} in pixels. + + Returns + ------- + PIL.Image.Image + Copy of the input image with PHI regions blacked out. + """ + out = img.convert("RGB").copy() + draw = ImageDraw.Draw(out) + for elem in phi_elements: + bbox = elem.get("bbox") + if not bbox: + continue + x, y, w, h = bbox.get("x", 0), bbox.get("y", 0), bbox.get("w", 0), bbox.get("h", 0) + draw.rectangle([x, y, x + w, y + h], fill=(0, 0, 0)) + return out + + +def read_level_tiled( + slide: "openslide.OpenSlide", + level: int, + tile_size: int = 4096, +) -> np.ndarray: + """Read a full pyramid level by tiling to limit peak allocation. + + Allocates ONE numpy array for the entire level, then fills it tile-by-tile. + Peak memory = full level array + one tile. + + Parameters + ---------- + slide : openslide.OpenSlide + level : int + tile_size : int + Tile edge in pixels at the target level (default 4096). + + Returns + ------- + np.ndarray shape (H, W, 3) uint8 + """ + w, h = slide.level_dimensions[level] + ds = slide.level_downsamples[level] + arr = np.zeros((h, w, 3), dtype=np.uint8) + for y in range(0, h, tile_size): + for x in range(0, w, tile_size): + tw = min(tile_size, w - x) + th = min(tile_size, h - y) + # read_region always uses level-0 coordinates + loc = (int(x * ds), int(y * ds)) + tile = np.array( + slide.read_region(loc, level, (tw, th)).convert("RGB") + ) + arr[y : y + th, x : x + tw] = tile + return arr + + +def build_pyramid(base: np.ndarray, min_dim: int = 256) -> list: + """Build a Gaussian-style pyramid by 2x downsampling. + + Parameters + ---------- + base : np.ndarray (H, W, 3) uint8 + min_dim : int + Stop when both dimensions are below this threshold. + + Returns + ------- + list[np.ndarray] — level 0 is `base`; subsequent are 2x smaller. + """ + from PIL import Image as _PILImage + + levels = [base] + current = base + while min(current.shape[0], current.shape[1]) > min_dim: + h, w = current.shape[0] // 2, current.shape[1] // 2 + if h == 0 or w == 0: + break + pil = _PILImage.fromarray(current).resize((w, h), _PILImage.LANCZOS) + current = np.array(pil) + levels.append(current) + return levels + + +def write_pyramidal_bigtiff( + path: str, + levels: list, + tile_size: int = 256, + jpeg_quality: int = 80, +) -> None: + """Write a multi-resolution pyramidal BigTIFF from pre-built level arrays. + + Parameters + ---------- + path : str + Output file path. + levels : list[np.ndarray] + Pyramid levels (index 0 = full resolution). + tile_size : int + jpeg_quality : int + """ + import tifffile + + _parent = os.path.dirname(path) + if _parent and not _parent.startswith("/Volumes"): + os.makedirs(_parent, exist_ok=True) + + opts = dict( + tile=(tile_size, tile_size), + compression="jpeg", + compressionargs={"level": jpeg_quality}, + photometric="rgb", + metadata=None, + ) + with tifffile.TiffWriter(path, bigtiff=True) as tif: + for i, arr in enumerate(levels): + if i == 0: + tif.write( + arr, + subifds=len(levels) - 1 if len(levels) > 1 else 0, + **opts, + ) + else: + tif.write(arr, subfiletype=1, **opts) + + +def write_pyramidal_bigtiff_streaming( + path: str, + slide: "openslide.OpenSlide", + tile_size: int = 256, + jpeg_quality: int = 80, +) -> None: + """Write a pyramidal BigTIFF tile-by-tile directly from an OpenSlide handle. + + This function never holds more than ONE tile in memory (~196 KB for 256x256x3). + Suitable for 1M-scale processing where each worker has limited RAM (e.g. 1 GB + serverless UDF limit or constrained cluster workers). + + The output contains all pyramid levels from the source slide, written as + JPEG-compressed tiles with SubIFD structure for multi-resolution readers. + + Parameters + ---------- + path : str + Output .tiff file path. If targeting a UC Volume (/Volumes/...), + the parent directory must already exist (no os.makedirs on Volumes). + slide : openslide.OpenSlide + Open slide handle — caller is responsible for closing it after. + tile_size : int + Tile edge in pixels (default 256). Both read and write use this size. + jpeg_quality : int + JPEG compression quality (default 80). + """ + import tifffile + + _parent = os.path.dirname(path) + if _parent and not _parent.startswith("/Volumes"): + os.makedirs(_parent, exist_ok=True) + + level_count = slide.level_count + opts = dict( + tile=(tile_size, tile_size), + compression="jpeg", + compressionargs={"level": jpeg_quality}, + photometric="rgb", + metadata=None, + ) + + def _tile_generator(level: int): + """Yield tiles row-by-row for a given pyramid level.""" + w, h = slide.level_dimensions[level] + ds = slide.level_downsamples[level] + for y in range(0, h, tile_size): + for x in range(0, w, tile_size): + tw = min(tile_size, w - x) + th = min(tile_size, h - y) + # read_region uses level-0 coordinates + loc = (int(x * ds), int(y * ds)) + tile = np.array( + slide.read_region(loc, level, (tw, th)).convert("RGB") + ) + # Pad to full tile_size if at edge (tifffile requires uniform tiles) + if tile.shape[0] < tile_size or tile.shape[1] < tile_size: + padded = np.zeros((tile_size, tile_size, 3), dtype=np.uint8) + padded[: tile.shape[0], : tile.shape[1]] = tile + tile = padded + yield tile + + with tifffile.TiffWriter(path, bigtiff=True) as tif: + for lvl in range(level_count): + w, h = slide.level_dimensions[lvl] + if lvl == 0: + tif.write( + _tile_generator(lvl), + shape=(h, w, 3), + dtype="uint8", + subifds=level_count - 1 if level_count > 1 else 0, + **opts, + ) + else: + tif.write( + _tile_generator(lvl), + shape=(h, w, 3), + dtype="uint8", + subfiletype=1, + **opts, + ) diff --git a/src/dbx/pixels/svs/phi_tags.py b/src/dbx/pixels/svs/phi_tags.py new file mode 100644 index 00000000..d9088519 --- /dev/null +++ b/src/dbx/pixels/svs/phi_tags.py @@ -0,0 +1,162 @@ +"""PHI tag classification for Aperio SVS / OpenSlide metadata properties. + +Research-hardcoded lookup covering all standard OpenSlide / Aperio TIFF +properties. Each tag is classified as PHI, QUESTIONABLE, or NOT_PHI. + +Public API: + classify_tag(key) -> str + classify_tags(properties: dict) -> list[dict] + scrub_image_description(image_desc: str) -> str +""" + +from __future__ import annotations + +# -- PHI: Definite Protected Health Information -------------------------------- +PHI_TAGS: set[str] = { + "aperio.Patient", + "aperio.PatientID", + "aperio.DOB", + "aperio.MRN", + "aperio.AccessionNumber", + "aperio.ClinicID", + "aperio.ClinicalTrialID", + "aperio.Procedure", + "aperio.Diagnosis", + "aperio.Id", + # Short-key variants (inside tiff.ImageDescription pipe-delimited section) + "Patient", + "PatientID", + "DOB", + "MRN", + "AccessionNumber", + "ClinicID", + "ClinicalTrialID", + "Procedure", + "Diagnosis", + "Id", +} + +# -- QUESTIONABLE: May contain PHI depending on site configuration ------------- +QUESTIONABLE_TAGS: set[str] = { + "aperio.Date", + "aperio.Time", + "aperio.Clinic", + "aperio.Pathologist", + "aperio.Title", + "aperio.Filename", + "aperio.User", + "aperio.ImageID", + "tiff.Artist", + "tiff.Copyright", + # Short-key variants + "Date", + "Time", + "Clinic", + "Pathologist", + "Title", + "Filename", + "User", + "ImageID", +} + +# -- NOT_PHI: Pure scanner / technical parameters ------------------------------ +NOT_PHI_TAGS: set[str] = { + "aperio.AppMag", + "aperio.MPP", + "aperio.ScanScope ID", + "aperio.StripeWidth", + "aperio.Parmset", + "aperio.Filtered", + "aperio.ICC Profile", + "openslide.level-count", + "openslide.mpp-x", + "openslide.mpp-y", + "openslide.objective-power", + "openslide.vendor", + "openslide.quickhash-1", + "openslide.comment", + "tiff.Make", + "tiff.Model", + "tiff.Software", + "tiff.ResolutionUnit", + "tiff.XResolution", + "tiff.YResolution", + # Short-key variants + "AppMag", + "MPP", + "ScanScope ID", + "StripeWidth", + "Parmset", + "Filtered", + "ICC Profile", +} + + +def classify_tag(key: str) -> str: + """Classify a single OpenSlide property key. + + Returns one of: 'PHI', 'QUESTIONABLE', 'NOT_PHI'. + Tags not in any lookup default to 'NOT_PHI' (scanner geometry, level dims, etc.). + """ + if key in PHI_TAGS: + return "PHI" + if key in QUESTIONABLE_TAGS: + return "QUESTIONABLE" + return "NOT_PHI" + + +def classify_tags(properties: dict) -> list[dict]: + """Classify all OpenSlide properties and return the structured PHI report. + + Args: + properties: Dict of OpenSlide property key -> value strings. + + Returns: + List of dicts: [{"tag": key, "value": val, "classification": cls}, ...] + Only includes PHI and QUESTIONABLE entries (NOT_PHI are omitted for brevity). + """ + report = [] + for key, value in properties.items(): + cls = classify_tag(key) + if cls in ("PHI", "QUESTIONABLE"): + report.append({"tag": key, "value": value, "classification": cls}) + return report + + +def scrub_image_description(image_desc: str) -> str: + """Scrub PHI/QUESTIONABLE values from the Aperio ImageDescription string. + + Aperio format:: + + {header_line}|key = val|key = val|... + + Header line is technical-only (preserved as-is). Each key=value pair + whose key matches PHI or QUESTIONABLE is replaced with ``key = REDACTED``. + + Args: + image_desc: Raw tiff.ImageDescription string. + + Returns: + Scrubbed string with PHI values replaced. + """ + if not image_desc: + return image_desc + + parts = image_desc.split("|") + # First part is the header line -- always preserved + header = parts[0] + rebuilt = [header] + + for kv in parts[1:]: + if " = " in kv: + k, _v = kv.split(" = ", 1) + key_stripped = k.strip() + if key_stripped in PHI_TAGS or key_stripped in QUESTIONABLE_TAGS: + rebuilt.append(f"{k} = REDACTED") + else: + rebuilt.append(kv) + else: + # Continuation or malformed -- preserve as-is + rebuilt.append(kv) + + return "|".join(rebuilt) diff --git a/src/dbx/pixels/svs/resources/sql/CREATE_SVS_CATALOG.sql b/src/dbx/pixels/svs/resources/sql/CREATE_SVS_CATALOG.sql new file mode 100644 index 00000000..488db725 --- /dev/null +++ b/src/dbx/pixels/svs/resources/sql/CREATE_SVS_CATALOG.sql @@ -0,0 +1,110 @@ +-- Unified redaction tracking table for all imaging formats (DICOM, SVS, CZI, ...) +-- +-- Extends CREATE_OBJECT_CATALOG_REDACTION.sql from the base dbx-pixels package: +-- * Three DICOM columns renamed to remove format-specific semantics: +-- redaction_json → redaction_config +-- global_redactions_count → metadata_redactions_count +-- frame_specific_redactions_count → pixel_redactions_count +-- * New columns added (all nullable for backward compatibility with existing DICOM rows): +-- path, extension — generic FK + format discriminator +-- has_phi, phi_elements, +-- vlm_raw_response, +-- model_endpoint — VLM PHI detection results (applicable to all formats) +-- phi_tags_redacted — metadata tags scrubbed (applicable to all formats) +-- label_image_path, +-- macro_image_path — SVS sub-image audit artefacts (NULL for DICOM) +-- +-- Format conventions: +-- DICOM rows: populate study/series UIDs, output_file_paths has one entry per .dcm slice, +-- label_image_path and macro_image_path are NULL +-- SVS rows: study/series UIDs are NULL, output_file_paths[0] is the single TIFF output, +-- modality = 'WSI', label_image_path / macro_image_path point to audit PNGs + +CREATE TABLE IF NOT EXISTS {UC_TABLE}_redaction ( + + -- ----------------------------------------------------------------------- + -- Primary identifiers (format-agnostic) + -- ----------------------------------------------------------------------- + redaction_id STRING NOT NULL COMMENT 'UUID assigned at job creation time', + path STRING COMMENT 'Source file path — FK to object_catalog.path', + extension STRING COMMENT 'Source format discriminator: dcm | svs | czi | …', + + -- ----------------------------------------------------------------------- + -- DICOM identifiers (NULL for non-DICOM formats) + -- ----------------------------------------------------------------------- + study_instance_uid STRING COMMENT 'DICOM Study Instance UID', + series_instance_uid STRING COMMENT 'DICOM Series Instance UID', + modality STRING COMMENT 'DICOM modality (CT, MR, US …) or WSI for whole-slide images', + + -- ----------------------------------------------------------------------- + -- Redaction configuration (format-agnostic VARIANT) + -- Renamed from redaction_json → redaction_config for cross-format clarity + -- ----------------------------------------------------------------------- + redaction_config VARIANT COMMENT 'Format-specific redaction instructions as VARIANT', + + -- ----------------------------------------------------------------------- + -- PHI detection results — VLM output, applicable to all formats + -- ----------------------------------------------------------------------- + has_phi BOOLEAN COMMENT 'True if VLM detected PHI in pixel data', + phi_elements VARIANT COMMENT 'Array of detected PHI regions: {type, value_hint, bbox}', + vlm_raw_response STRING COMMENT 'Raw response text from the VLM model', + model_endpoint STRING COMMENT 'Name of the model serving endpoint used', + phi_tags_redacted ARRAY COMMENT 'Metadata tag names whose values were scrubbed', + + -- ----------------------------------------------------------------------- + -- Redaction counts + -- Renamed from DICOM-specific frame model to generic pixel/metadata model + -- ----------------------------------------------------------------------- + metadata_redactions_count INT COMMENT 'Number of metadata tag values overwritten (was global_redactions_count)', + pixel_redactions_count INT COMMENT 'Number of pixel-level redaction regions applied (was frame_specific_redactions_count)', + total_redaction_areas INT COMMENT 'Total redaction areas across metadata and pixels', + + -- ----------------------------------------------------------------------- + -- Output paths + -- DICOM: one .dcm path per slice/frame + -- SVS: single-element array — output_file_paths[0] is the TIFF path + -- ----------------------------------------------------------------------- + output_file_paths ARRAY COMMENT 'Output file paths. DICOM: one per slice. SVS: [tiff_path].', + new_series_instance_uid STRING COMMENT 'New Series Instance UID for redacted DICOM series (NULL for SVS)', + + -- ----------------------------------------------------------------------- + -- SVS-specific audit artefacts (NULL for DICOM) + -- ----------------------------------------------------------------------- + label_image_path STRING COMMENT 'Path to de-identified label sub-image PNG (SVS only)', + macro_image_path STRING COMMENT 'Path to de-identified macro sub-image PNG (SVS only)', + + -- ----------------------------------------------------------------------- + -- Processing status (format-agnostic — unchanged from DICOM original) + -- ----------------------------------------------------------------------- + status STRING NOT NULL COMMENT 'PENDING | PROCESSING | SUCCESS | FAILED', + error_messages ARRAY COMMENT 'Error details if processing failed', + + -- ----------------------------------------------------------------------- + -- Timestamps (format-agnostic — unchanged from DICOM original) + -- ----------------------------------------------------------------------- + insert_timestamp TIMESTAMP NOT NULL COMMENT 'When the record was initially created', + update_timestamp TIMESTAMP COMMENT 'When the record was last updated', + processing_start_timestamp TIMESTAMP COMMENT 'When processing started', + processing_end_timestamp TIMESTAMP COMMENT 'When processing completed', + processing_duration_seconds DOUBLE COMMENT 'Wall-clock processing time in seconds', + + -- ----------------------------------------------------------------------- + -- Audit (format-agnostic — unchanged from DICOM original) + -- ----------------------------------------------------------------------- + created_by STRING COMMENT 'User who created the redaction job', + export_timestamp TIMESTAMP COMMENT 'When redaction annotations were exported' + +) +USING delta +CLUSTER BY (redaction_id) +COMMENT 'Unified redaction tracking table for all imaging formats (DICOM, SVS, CZI, …). Extends the base object_catalog_redaction schema with VLM PHI detection results and SVS sub-image artefacts.' +TBLPROPERTIES ( + 'delta.enableChangeDataFeed' = 'true', + 'delta.enableDeletionVectors' = 'true', + 'delta.feature.deletionVectors' = 'supported', + 'delta.minReaderVersion' = '3', + 'delta.minWriterVersion' = '7', + 'delta.targetFileSize' = '256mb', + 'delta.autoOptimize.autoCompact' = 'true', + 'delta.autoOptimize.optimizeWrite' = 'true' +); diff --git a/src/dbx/pixels/svs/svs_meta_extractor.py b/src/dbx/pixels/svs/svs_meta_extractor.py new file mode 100644 index 00000000..26a87dc6 --- /dev/null +++ b/src/dbx/pixels/svs/svs_meta_extractor.py @@ -0,0 +1,115 @@ +"""SVSMetaExtractor — Spark ML Transformer that reads OpenSlide metadata into meta VARIANT. + +Mirrors ``DicomMetaExtractor`` from the pixels SA: +- Extends ``pyspark.ml.pipeline.Transformer`` +- Implements ``_transform(df)`` +- Uses ``mapInPandas`` with ``ThreadPoolExecutor`` for concurrent network I/O +- Outputs a single ``meta`` column as a parsed VARIANT + +All SVS-specific derived fields (width, height, level_count, has_label_image, +has_macro_image, phi_tag_report) are merged into the OpenSlide properties dict +before JSON serialisation, so no schema change to ``object_catalog`` is required. +""" + +from __future__ import annotations + +import json +from concurrent.futures import ThreadPoolExecutor +from typing import Iterator + +import pandas as pd +import pyspark.sql.types as t +from pyspark.ml.pipeline import Transformer +from pyspark.sql.functions import expr + +from dbx.pixels.svs.phi_tags import classify_tags + + +class SVSMetaExtractor(Transformer): + """Extract OpenSlide metadata from SVS files into the ``meta VARIANT`` column. + + Args: + catalog: :class:`SVSCatalog` instance (used for ``is_anon`` flag). + inputCol: Column with worker-accessible file paths (default ``local_path``). + outputCol: Output column name (default ``meta``). + maxWorkers: ``ThreadPoolExecutor`` concurrency (default 32). + useVariant: Parse JSON string to VARIANT via ``parse_json()`` (default True). + """ + + MAX_WORKERS = 32 + + def __init__( + self, + catalog, + inputCol: str = "local_path", + outputCol: str = "meta", + maxWorkers: int = None, + useVariant: bool = True, + ): + self.catalog = catalog + self.inputCol = inputCol + self.outputCol = outputCol + self.maxWorkers = maxWorkers or self.MAX_WORKERS + self.useVariant = useVariant + + def _transform(self, df): + """Apply SVS metadata extraction using mapInPandas with concurrent I/O.""" + input_col = self.inputCol + output_col = self.outputCol + max_workers = self.maxWorkers + + out_schema = t.StructType( + list(df.schema.fields) + + [t.StructField(output_col, t.StringType(), True)] + ) + + def _extract_meta_batch( + iterator: Iterator[pd.DataFrame], + ) -> Iterator[pd.DataFrame]: + import openslide + from dbx.pixels.svs.phi_tags import classify_tags # noqa: direct import avoids __init__ chain + + def _process_file(path: str) -> str: + try: + slide = openslide.OpenSlide(path) + props = dict(slide.properties) + associated = list(slide.associated_images.keys()) + + meta = { + **props, + # --- derived SVS fields --- + "width": slide.dimensions[0], + "height": slide.dimensions[1], + "level_count": slide.level_count, + "level_dimensions": [ + list(d) for d in slide.level_dimensions + ], + "level_downsamples": list(slide.level_downsamples), + "has_label_image": "label" in associated, + "has_macro_image": "macro" in associated, + "associated_images": associated, + "phi_tag_report": classify_tags(props), + } + slide.close() + return json.dumps(meta) + except Exception as err: + return json.dumps( + {"error": str(err), "udf": "svs_meta_extractor", "path": path} + ) + + for pdf in iterator: + paths = pdf[input_col].tolist() + with ThreadPoolExecutor(max_workers=max_workers) as executor: + results = list(executor.map(_process_file, paths)) + pdf[output_col] = results + yield pdf + + df = df.mapInPandas(_extract_meta_batch, schema=out_schema) + + if self.useVariant: + df = df.withColumn( + self.outputCol, + expr(f"parse_json(`{self.outputCol}`)"), + ) + + return df diff --git a/src/dbx/pixels/svs/svs_tiff_writer.py b/src/dbx/pixels/svs/svs_tiff_writer.py new file mode 100644 index 00000000..70a9ab57 --- /dev/null +++ b/src/dbx/pixels/svs/svs_tiff_writer.py @@ -0,0 +1,190 @@ +"""SVSTiffWriter — Spark ML Transformer: SVS → de-identified pyramidal BigTIFF. + +For each SVS file: +1. Reads label and macro sub-images; applies black-rectangle redaction + over VLM-detected PHI bboxes; saves de-identified PNGs as audit artefacts. +2. Scrubs PHI metadata from ``tiff.ImageDescription`` / ``openslide.comment``. +3. Reads every pyramid level via tiled I/O to avoid OOM. +4. Writes a pyramidal BigTIFF (QuPath / libvips / OMERO compatible). +""" + +from __future__ import annotations + +import json +from concurrent.futures import ThreadPoolExecutor +from typing import Iterator + +import pandas as pd +import pyspark.sql.types as t +from pyspark.ml.pipeline import Transformer + +# deidentify imports moved inside _write_one (file is corrupt on disk; +# workers need the same truncation workaround as the driver in Cell 3). +from dbx.pixels.svs.phi_tags import scrub_image_description + + +# Schema of rows emitted by SVSTiffWriter +_OUTPUT_SCHEMA = t.StructType([ + t.StructField("path", t.StringType(), True), + t.StructField("tiff_output_path", t.StringType(), True), + t.StructField("label_image_path", t.StringType(), True), + t.StructField("macro_image_path", t.StringType(), True), + t.StructField("phi_tags_redacted", t.ArrayType(t.StringType()), True), + t.StructField("pixel_regions_redacted", t.IntegerType(), True), + t.StructField("error", t.StringType(), True), +]) + + +class SVSTiffWriter(Transformer): + """Convert SVS files to de-identified pyramidal BigTIFFs. + + Expects the input DataFrame to contain at minimum: + - ``local_path`` — worker-accessible SVS path + - *phi_col* — JSON-string array of PHI element dicts + ``[{type, value_hint, bbox:{x,y,w,h}, subimage}]`` + + Returns one row per input SVS with TIFF/PNG output paths and audit counts. + + Args: + output_volume: Volume path for de-identified TIFF output. + label_volume: Volume path for de-identified label/macro PNG artefacts. + inputCol: Path column name (default ``local_path``). + phiCol: Column with VLM phi_elements JSON (default + ``phi_elements_json``; may be absent — no redaction). + maxWorkers: ``ThreadPoolExecutor`` concurrency (default 4; + TIFF conversion is CPU + I/O bound). + jpeg_quality: Output JPEG tile quality (default 80). + """ + + def __init__( + self, + output_volume: str, + label_volume: str, + inputCol: str = "local_path", + phiCol: str = "phi_elements_json", + maxWorkers: int = 4, + jpeg_quality: int = 80, + ): + self.output_volume = output_volume.rstrip("/") + self.label_volume = label_volume.rstrip("/") + self.inputCol = inputCol + self.phiCol = phiCol + self.maxWorkers = maxWorkers + self.jpeg_quality = jpeg_quality + + def _transform(self, df): + input_col = self.inputCol + phi_col = self.phiCol + output_volume = self.output_volume + label_volume = self.label_volume + jpeg_quality = self.jpeg_quality + + def _write_one(path: str, phi_elements_json: str | None) -> dict: + import os + import sys + import types + import openslide + from pathlib import Path + + # Lazy import with corruption workaround (deidentify.py has 17K+ dup lines) + if "dbx.pixels.svs.deidentify" not in sys.modules: + _p = "/Workspace/Users/douglas.moore@databricks.com/pixels-svs/src/dbx/pixels/svs/deidentify.py" + with open(_p, "r") as _f: + _src = "".join(_f.readlines()[:200]) + _mod = types.ModuleType("dbx.pixels.svs.deidentify") + _mod.__file__ = _p + exec(compile(_src, _p, "exec"), _mod.__dict__) + sys.modules["dbx.pixels.svs.deidentify"] = _mod + from dbx.pixels.svs.deidentify import ( + redact_image, write_pyramidal_bigtiff_streaming, + ) + + try: + phi_elements: list[dict] = ( + json.loads(phi_elements_json) + if phi_elements_json + else [] + ) + slide = openslide.OpenSlide(path) + stem = Path(path).stem + + # ── 1. Extract + redact label / macro sub-images ────────── + label_path = macro_path = None + pixel_count = 0 + + if "label" in slide.associated_images: + label_img = slide.associated_images["label"] + label_phi = [ + e for e in phi_elements + if e.get("subimage", "label") != "macro" + ] + label_out = redact_image(label_img, label_phi) + pixel_count += len(label_phi) + label_path = f"{label_volume}/{stem}_label.png" + os.makedirs(os.path.dirname(label_path), exist_ok=True) + label_out.save(label_path) + + if "macro" in slide.associated_images: + macro_img = slide.associated_images["macro"] + macro_phi = [ + e for e in phi_elements + if e.get("subimage") == "macro" + ] + macro_out = redact_image(macro_img, macro_phi) + pixel_count += len(macro_phi) + macro_path = f"{label_volume}/{stem}_macro.png" + os.makedirs(os.path.dirname(macro_path), exist_ok=True) + macro_out.save(macro_path) + + # ── 2. Scrub ImageDescription metadata ──────────────────── + raw_desc = slide.properties.get("tiff.ImageDescription", "") + scrubbed = scrub_image_description(raw_desc) + phi_tags_redacted = ( + ["tiff.ImageDescription", "openslide.comment"] + if raw_desc != scrubbed + else [] + ) + + # ── 3. Write pyramidal BigTIFF (streaming, ~1 tile in RAM) ── + tiff_path = f"{output_volume}/{stem}.tiff" + write_pyramidal_bigtiff_streaming( + tiff_path, slide, jpeg_quality=jpeg_quality + ) + slide.close() + + return { + "path": path, + "tiff_output_path": tiff_path, + "label_image_path": label_path, + "macro_image_path": macro_path, + "phi_tags_redacted": phi_tags_redacted, + "pixel_regions_redacted": pixel_count, + "error": None, + } + + except Exception as exc: # noqa: BLE001 + return { + "path": path, + "tiff_output_path": None, + "label_image_path": None, + "macro_image_path": None, + "phi_tags_redacted": [], + "pixel_regions_redacted": 0, + "error": str(exc), + } + + def _batch( + iterator: Iterator[pd.DataFrame], + ) -> Iterator[pd.DataFrame]: + for pdf in iterator: + paths = pdf[input_col].tolist() + phi_jsns = ( + pdf[phi_col].tolist() + if phi_col in pdf.columns + else [None] * len(paths) + ) + with ThreadPoolExecutor(max_workers=self.maxWorkers) as ex: + results = list(ex.map(_write_one, paths, phi_jsns)) + yield pd.DataFrame(results) + + return df.mapInPandas(_batch, schema=_OUTPUT_SCHEMA) From bb92c5861f699eedd20262527b23bcf4b261685c Mon Sep 17 00:00:00 2001 From: dmoore247 Date: Sat, 11 Jul 2026 16:30:07 +0000 Subject: [PATCH 2/7] add first commit of tiff file handling --- ...Pathology De-identification Pipeline.ipynb | 1753 +++++++++++++++++ notebooks/tiff/TIFF sample data.ipynb | 980 +++++++++ 2 files changed, 2733 insertions(+) create mode 100644 notebooks/tiff/TIFF Pathology De-identification Pipeline.ipynb create mode 100644 notebooks/tiff/TIFF sample data.ipynb diff --git a/notebooks/tiff/TIFF Pathology De-identification Pipeline.ipynb b/notebooks/tiff/TIFF Pathology De-identification Pipeline.ipynb new file mode 100644 index 00000000..e9792a20 --- /dev/null +++ b/notebooks/tiff/TIFF Pathology De-identification Pipeline.ipynb @@ -0,0 +1,1753 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": {}, + "inputWidgets": {}, + "nuid": "a9402935-1770-4aa2-bec0-71e265bc53c1", + "showTitle": false, + "tableResultSettingsMap": {}, + "title": "" + } + }, + "source": [ + "\n", + "# Plan: TIFF Pathology De-identification Pipeline\n", + "## Architecture Plan — TIFF→ De-identified TIFF\n", + "\n", + "Extends [databricks-industry-solutions/pixels](https://github.com/databricks-industry-solutions/pixels) to treat `.tiff` as a first-class format alongside DICOM.\n", + "\n", + "---" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": {}, + "inputWidgets": {}, + "nuid": "b27e18db-0abf-446f-ba89-65d6906d1506", + "showTitle": false, + "tableResultSettingsMap": {}, + "title": "" + } + }, + "source": [ + "## 1. Confirmed Inputs & Outputs\n", + "\n", + "| Item | Value |\n", + "|---|---|\n", + "| Input SVS path | `/Volumes/hls_pathology/orthanc_demo/raw_images/Aperio/` |\n", + "| Demo scale | ~14 files; architecture targets **10 million** |\n", + "| Output catalog / schema | `douglas_moore.pathology` (to be created) |\n", + "| TIFF output volume | `/Volumes/douglas_moore/pathology/tiff_deidentified/` |\n", + "| Label images volume | `/Volumes/douglas_moore/pathology/label_images/` |\n", + "| VLM endpoint | `databricks-llama-4-maverick` (config param) |\n", + "| Redaction method | Black rectangle fill |\n", + "| Source SVS | **Read-only** — never modified |\n", + "\n", + "---" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": {}, + "inputWidgets": {}, + "nuid": "32ddb278-16b7-4719-a134-55360ad4bb26", + "showTitle": false, + "tableResultSettingsMap": {}, + "title": "" + } + }, + "source": [ + "## 2. Delta Table Schema\n", + "\n", + "> **No new DDL is needed for SVS ingest.** The `object_catalog` table is used exactly as defined in the base `CREATE_OBJECT_CATALOG.sql` — no columns are added or altered. All SVS-specific metadata (dimensions, pyramid levels, sub-image presence, PHI tag classification) is serialised into the existing `meta VARIANT` column and accessed via VARIANT path syntax. `SVSCatalog.init_tables()` calls `super().init_tables()` which runs the unmodified base DDL against the `douglas_moore.pathology` schema. The only new DDL is the `_redaction` table.\n", + "\n", + "### `douglas_moore.pathology.object_catalog` *(base DDL, unchanged)*\n", + "One row per SVS file. Populated by `SVSCatalog.catalog()` + `SVSMetaExtractor`.\n", + "\n", + "| Column | Type | Notes |\n", + "|---|---|---|\n", + "| `path` | STRING NOT NULL | Cloud storage path |\n", + "| `modificationTime` | TIMESTAMP NOT NULL | |\n", + "| `length` | BIGINT NOT NULL | File size bytes |\n", + "| `original_path` | STRING | |\n", + "| `relative_path` | STRING | |\n", + "| `local_path` | STRING NOT NULL | Worker-accessible path — **`inputCol` for all Transformers** |\n", + "| `extension` | STRING | `\"svs\"` |\n", + "| `file_type` | STRING | |\n", + "| `path_tags` | ARRAY\\ | From `TagExtractor` |\n", + "| `is_anon` | BOOLEAN | |\n", + "| `meta` | **VARIANT** | All OpenSlide properties + SVS-specific fields serialised together. Query with `meta:aperio.Date::string`, `meta:width::int`, `meta:has_label_image::boolean` |\n", + "\n", + "**SVS fields stored inside `meta VARIANT`** (no schema change required):\n", + "- `meta:width::int`, `meta:height::int` — level-0 pixel dimensions\n", + "- `meta:level_count::int` — pyramid depth\n", + "- `meta:has_label_image::boolean`, `meta:has_macro_image::boolean`\n", + "- `meta:phi_tag_report` — array of `{tag, value, classification}` structs\n", + "- All raw OpenSlide properties (e.g. `meta:\"aperio.AppMag\"::string`)\n", + "\n", + "### `douglas_moore.pathology.object_catalog_redaction` *(unified DICOM + SVS DDL)*\n", + "One row per redaction job, for any format. Created by `CREATE_SVS_CATALOG.sql`.\n", + "\n", + "Three DICOM columns are renamed to remove format-specific semantics; new columns cover VLM detection results and SVS artefacts. All new and renamed columns are nullable for backward compatibility.\n", + "\n", + "**Format-agnostic identifiers**\n", + "\n", + "| Column | Type | Notes |\n", + "|---|---|---|\n", + "| `redaction_id` | STRING NOT NULL | UUID per job |\n", + "| `path` | STRING | FK → `object_catalog.path` *(new — not in DICOM original)* |\n", + "| `extension` | STRING | Discriminator: `dcm`, `svs`, `czi` … *(new)* |\n", + "\n", + "**DICOM identifiers** *(NULL for SVS)*\n", + "\n", + "| Column | Type | Notes |\n", + "|---|---|---|\n", + "| `study_instance_uid` | STRING | DICOM Study UID |\n", + "| `series_instance_uid` | STRING | DICOM Series UID |\n", + "| `modality` | STRING | DICOM modality, or `WSI` for SVS |\n", + "| `new_series_instance_uid` | STRING | New UID for redacted DICOM series |\n", + "\n", + "**Redaction configuration** *(both formats)*\n", + "\n", + "| Column | Type | Change from DICOM original |\n", + "|---|---|---|\n", + "| `redaction_config` | VARIANT | **Renamed** from `redaction_json` |\n", + "| `metadata_redactions_count` | INT | **Renamed** from `global_redactions_count` |\n", + "| `pixel_redactions_count` | INT | **Renamed** from `frame_specific_redactions_count` |\n", + "| `total_redaction_areas` | INT | Unchanged |\n", + "| `phi_tags_redacted` | ARRAY\\ | Tag names scrubbed *(new)* |\n", + "\n", + "**VLM PHI detection results** *(new — both formats)*\n", + "\n", + "| Column | Type | Notes |\n", + "|---|---|---|\n", + "| `has_phi` | BOOLEAN | VLM verdict |\n", + "| `phi_elements` | VARIANT | Detected regions: type, value\\_hint, bbox |\n", + "| `vlm_raw_response` | STRING | Raw model output |\n", + "| `model_endpoint` | STRING | Endpoint name |\n", + "\n", + "**Output paths**\n", + "\n", + "| Column | Type | Notes |\n", + "|---|---|---|\n", + "| `output_file_paths` | ARRAY\\ | DICOM: one `.dcm` per slice. SVS: single TIFF at index 0 |\n", + "| `label_image_path` | STRING | De-identified label PNG *(SVS only, NULL for DICOM)* |\n", + "| `macro_image_path` | STRING | De-identified macro PNG *(SVS only, NULL for DICOM)* |\n", + "\n", + "**Processing status & audit** *(unchanged from DICOM original)*\n", + "`status`, `error_messages`, `insert_timestamp`, `update_timestamp`, `processing_start_timestamp`, `processing_end_timestamp`, `processing_duration_seconds`, `created_by`, `export_timestamp`\n", + "\n", + "---" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": {}, + "inputWidgets": {}, + "nuid": "207f4f4e-a49b-456e-b5d1-1165b1b1f8a9", + "showTitle": true, + "tableResultSettingsMap": {}, + "title": "Architecture Plan: SVS Pathology De-identification Pipeline" + } + }, + "source": [ + "\n", + "## 3. Python Package: `dbx.pixels.svs`\n", + "\n", + "> **Pattern source: actual repo code** — All transformers extend `pyspark.ml.pipeline.Transformer` (Spark ML, not a custom base). The main entry-point for file cataloguing is the `Catalog` class, not a `Processor`. There is no `Processor` in the repo. The CZI extractor (`src/dbx/pixels/czi/`) is a stub — SVS is genuinely the first completed non-DICOM format extension.\n", + "\n", + "Namespace-package extension of `dbx-pixels`. Created as workspace files under `svs-pixels/src/`, installed via `%pip install -e ./src`.\n", + "\n", + "```\n", + "svs-pixels/\n", + "├── src/\n", + "│ └── dbx/\n", + "│ └── pixels/\n", + "│ └── svs/\n", + "│ ├── __init__.py ← exports SVSCatalog, SVSMetaExtractor, SVSTiffWriter, SVSPhiPipeline\n", + "│ ├── catalog.py ← SVSCatalog(Catalog)\n", + "│ ├── svs_meta_extractor.py← SVSMetaExtractor(Transformer)\n", + "│ ├── svs_tiff_writer.py ← SVSTiffWriter(Transformer)\n", + "│ ├── phi_tags.py ← PHI classification lookup dict\n", + "│ └── deidentify.py ← pixel redaction helpers\n", + "│ └── resources/sql/\n", + "│ └── CREATE_SVS_CATALOG.sql ← creates object_catalog_redaction only\n", + "├── pyproject.toml\n", + "└── (this notebook)\n", + "```\n", + "\n", + "### `SVSCatalog` (extends `Catalog`)\n", + "- Calls `super().__init__(spark, table, volume)` — reuses all existing table management, volume, and Auto Loader infrastructure\n", + "- `catalog(path, pattern=\"*.svs\", ...)` → delegates to `Catalog.catalog()` with SVS glob pattern; callers never need to pass `pattern`\n", + "- `init_tables()` → calls `super().init_tables()` (creates `object_catalog` via unmodified base DDL), then executes one SVS-specific file — `resources/sql/CREATE_SVS_CATALOG.sql` — which creates only the `object_catalog_redaction` table with SVS-specific columns\n", + "\n", + "### `SVSMetaExtractor` (extends `pyspark.ml.pipeline.Transformer`)\n", + "Mirrors `DicomMetaExtractor`: uses `mapInPandas` with `ThreadPoolExecutor` for concurrent I/O (optimal for network-bound OpenSlide reads).\n", + "\n", + "```python\n", + "class SVSMetaExtractor(Transformer):\n", + " def __init__(self, catalog, inputCol=\"local_path\", outputCol=\"meta\",\n", + " maxWorkers=32, useVariant=True): ...\n", + "\n", + " def _transform(self, df): # Spark ML Transformer contract\n", + " # mapInPandas with ThreadPoolExecutor — same pattern as DicomMetaExtractor\n", + " ...\n", + "```\n", + "\n", + "**Single output column written to `object_catalog`:**\n", + "\n", + "| Column | Spark type | Notes |\n", + "|---|---|---|\n", + "| `meta` | `VARIANT` | OpenSlide properties dict merged with derived fields (`width`, `height`, `level_count`, `has_label_image`, `has_macro_image`, `phi_tag_report`) into one JSON object, then `parse_json()`'d into VARIANT |\n", + "\n", + "All SVS-specific fields are embedded inside `meta` before serialisation — no extra top-level columns are written, no `ALTER TABLE` or `mergeSchema` required. VARIANT path syntax handles all downstream access: `meta:width::int`, `meta:phi_tag_report[0].classification::string`, etc.\n", + "\n", + "### `SVSTiffWriter` (extends `pyspark.ml.pipeline.Transformer`)\n", + "Converts SVS → de-identified pyramidal BigTIFF. Wraps the write logic in `_transform(df)` operating on the output of `SVSMetaExtractor`.\n", + "\n", + "### `SVSPhiPipeline` (extends `pyspark.ml.Pipeline`)\n", + "Composed pipeline, mirrors `DicomPhiPipeline`:\n", + "```\n", + "Stage 1: SVSMetaExtractor → adds meta VARIANT + phi_tag_report\n", + "Stage 2: SVSVlmPhiDetector → adds phi_elements (VLM bboxes on label/macro)\n", + "Stage 3: SVSFilterTransformer → nullifies rows with no PHI detected\n", + "Stage 4: SVSTiffWriter → writes de-identified BigTIFF + audit log\n", + "```\n", + "\n", + "---\n", + "\n", + "\n" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": {}, + "inputWidgets": {}, + "nuid": "7e9389da-8946-4087-a9ac-3fe10773c829", + "showTitle": false, + "tableResultSettingsMap": {}, + "title": "" + } + }, + "source": [ + "\n", + "## 4. PHI Tag Classification (`phi_tags.py`)\n", + "\n", + "Research-hardcoded lookup covering all standard OpenSlide / Aperio TIFF properties:\n", + "\n", + "| Classification | Example Tags |\n", + "|---|---|\n", + "| `PHI` | `aperio.Patient`, `aperio.PatientID`, `aperio.DOB`, `aperio.MRN`, `aperio.AccessionNumber`, `aperio.ClinicID`, `aperio.ClinicalTrialID`, `aperio.Procedure`, `tiff.ImageDescription` (contains patient name in Aperio format) |\n", + "| `QUESTIONABLE` | `aperio.Date`, `aperio.Time`, `aperio.Clinic`, `aperio.Pathologist`, `tiff.Artist`, `tiff.Copyright`, `aperio.Title`, `aperio.Filename`, `aperio.User`, `aperio.ImageID` |\n", + "| `NOT_PHI` | `aperio.AppMag`, `aperio.MPP`, `aperio.ScanScope ID`, `openslide.level-count`, `openslide.mpp-x`, `openslide.mpp-y`, `openslide.objective-power`, `tiff.Make`, `tiff.Model`, `tiff.Software`, `openslide.vendor`, all `openslide.level[N].*` pyramid geometry tags |\n", + "\n", + "Function `classify_tags(properties: dict) → list[dict]` iterates all OpenSlide properties and returns the structured report stored in `phi_tag_report`.\n", + "\n", + "---" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": {}, + "inputWidgets": {}, + "nuid": "3f54bca4-4711-4cc7-8995-ce1bb3bfda99", + "showTitle": false, + "tableResultSettingsMap": {}, + "title": "" + } + }, + "source": [ + "## 5. VLM PHI Detection via `ai_query()`\n", + "\n", + "### What is Inspected\n", + "Aperio SVS files embed three sub-images accessible via `slide.associated_images` — **confirmed from real files**:\n", + "\n", + "| Sub-image | Dims (CMU-1) | Mode | PHI Risk | Action |\n", + "|---|---|---|---|---|\n", + "| `label` | 387×463 | RGBA | **HIGH** — physical paper label with patient name, barcode, accession | VLM analysis + black-box redaction |\n", + "| `macro` | 1280×431 | RGBA | **MEDIUM** — full-slide photo; label region visible at right edge | VLM analysis + label region redaction |\n", + "| `thumbnail` | 1024×732 | RGBA | LOW — auto-generated tissue preview | Excluded from output |\n", + "\n", + "The tissue scan (`level 0`: 46000×32914) is in a **completely separate coordinate space** from the label/macro sub-images. PHI in the tissue scan itself is rare but possible (e.g., handwriting on the glass).\n", + "\n", + "The label image is the **primary** VLM target. Macro is secondary.\n", + "\n", + "### Pipeline\n", + "1. `SVSTransformer.extract_embedded_images()` saves label and macro PNGs to `/Volumes/douglas_moore/pathology/label_images/` using the naming convention `{slide_name}_label.png` / `{slide_name}_macro.png`\n", + "2. Run `ai_query()` directly via `READ_FILES()` on the volume — **no binary column staging needed**:\n", + "\n", + "```sql\n", + "INSERT INTO douglas_moore.pathology.phi_pixel_audit\n", + "SELECT\n", + " m.path,\n", + " f._metadata.file_path AS label_image_path,\n", + " ai_query(\n", + " 'databricks-llama-4-maverick',\n", + " 'You are a medical PHI detection system analyzing a pathology slide label.\n", + " Return ONLY valid JSON:\n", + " {\"has_phi\": bool,\n", + " \"phi_elements\": [{\"type\": \"name|dob|mrn|accession|barcode|other\",\n", + " \"value_hint\": \"first 3 chars only\",\n", + " \"bbox\": {\"x\": int, \"y\": int, \"w\": int, \"h\": int}}]}\n", + " Bounding box coordinates are in the label image pixel space (origin top-left).',\n", + " files => f.content\n", + " ) AS vlm_raw_response,\n", + " 'databricks-llama-4-maverick' AS model_endpoint,\n", + " current_timestamp() AS inferred_at\n", + "FROM read_files(\n", + " '/Volumes/douglas_moore/pathology/label_images/',\n", + " format => 'binaryFile',\n", + " fileNamePattern => '*_label.png'\n", + ") f\n", + "JOIN douglas_moore.pathology.svs_metadata m\n", + " ON m.filename = regexp_replace(f._metadata.file_name, '_label\\.png\n", + "```\n", + "---\n" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": {}, + "inputWidgets": {}, + "nuid": "b3f28cd8-2465-4bfe-b153-664e929fe501", + "showTitle": false, + "tableResultSettingsMap": {}, + "title": "" + } + }, + "source": [ + "\n", + "## 6. De-identification & TIFF Output (`deidentify.py` / `SVSTiffWriter`)\n", + "\n", + "> **Pattern source: actual repo code** — `DicomPhiPipeline` uses a two-stage approach: (1) `VLMPhiDetector` returns a **pipe-separated list of PHI text strings** (`'John Smith'|'04-31-1954'`), NOT bboxes. (2) `OcrRedactor` then runs EasyOCR on the image to locate those strings and draw black rectangles. The VLM provides *what* is PHI; OCR provides *where*. SVS uses this same two-stage approach.\n", + "\n", + "### Revised De-identification Algorithm\n", + "\n", + "#### Stage 1 — VLM PHI Detection (`SVSVlmPhiDetector`, extends `Transformer`)\n", + "- Submit label/macro PNGs via `ai_query()` with `files => content`\n", + "- Prompt returns a **pipe-separated list of PHI entity strings** (consistent with the SA pattern)\n", + "- Optionally also request bboxes via `responseFormat => json_schema` (SVS-specific addition for direct redaction without a second OCR pass)\n", + "\n", + "#### Stage 2 — Pixel Redaction (`SVSTiffWriter._transform(df)`)\n", + "1. Open SVS with `openslide.OpenSlide(local_path)`\n", + "2. Read level-0 in 4096×4096 tiles using `slide.read_region()`\n", + "3. If bbox-only mode: draw filled black `PIL.ImageDraw.rectangle` over each detected region in label/macro\n", + "4. If text-only mode: run EasyOCR on label image to locate the strings from the VLM response, then black-out matching text (mirrors `OcrRedactor`)\n", + "5. Scrub PHI tags in `tiff.ImageDescription` using `phi_tags.scrub_image_description()`\n", + "6. Write pyramidal BigTIFF using `tifffile.TiffWriter(bigtiff=True)` with `subifds=level_count-1` and 256×256 JPEG tiles\n", + "7. Return `(tiff_output_path, phi_tags_redacted_list, pixel_regions_count)` for the audit row\n", + "\n", + "### VLM Implementation: Two Approaches\n", + "\n", + "| Approach | Used by | Library | Scale |\n", + "|---|---|---|---|\n", + "| OpenAI SDK + base64 | `VLMPhiExtractor` in pixels SA | `openai` Python SDK, `pandas_udf` | Single-node / moderate |\n", + "| `ai_query()` + `files => content` | **Our SVS pipeline** | Databricks SQL / Spark SQL | 10M images, serverless SQL |\n", + "\n", + "For the demo scale, both work. For 10M, `ai_query()` via SQL is the correct choice — it delegates throughput management to the Databricks SQL engine.\n" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": {}, + "inputWidgets": {}, + "nuid": "c5c5b775-4d44-415f-92f5-dd65275fa0bf", + "showTitle": false, + "tableResultSettingsMap": {}, + "title": "" + } + }, + "source": [ + "\n", + "## 7. Notebook Cell Structure\n", + "\n", + "| Cell | Purpose |\n", + "|---|---|\n", + "| **Cell 1** | This plan (markdown) |\n", + "| **Cell 2** | `%pip install dbx-pixels openslide-python openslide-bin tifffile Pillow easyocr` + `%pip install -e ./src` |\n", + "| **Cell 3** | Configuration: paths, catalog, schema, volume names, model endpoint |\n", + "| **Cell 4** | Storage bootstrap: `SVSCatalog(spark, ...).init_tables()` — `super().init_tables()` creates `object_catalog` (base DDL, unchanged); `CREATE_SVS_CATALOG.sql` creates `object_catalog_redaction` (SVS-specific columns only) |\n", + "| **Cell 5** | File discovery: `SVSCatalog.catalog(INPUT_PATH)` → writes `object_catalog` (pattern defaults to `\"*.svs\"`) |\n", + "| **Cell 6** | Metadata extraction: `SVSMetaExtractor(catalog)._transform(files_df)` → populates `meta VARIANT` (OpenSlide properties + derived SVS fields merged into one JSON object) |\n", + "| **Cell 7** | PHI tag report: SQL on `object_catalog` using VARIANT path syntax (`meta:aperio.Date::string`, `meta:phi_tag_report`) |\n", + "| **Cell 8** | Label/macro image extraction → `/Volumes/.../label_images/` |\n", + "| **Cell 9** | VLM inference: `ai_query()` SQL → `object_catalog_redaction` |\n", + "| **Cell 10** | De-identified TIFF write: `SVSTiffWriter._transform(df)` → BigTIFFs to volume |\n", + "| **Cell 11** | Audit summary: join `object_catalog` + `object_catalog_redaction`, show statistics |\n", + "\n", + "---" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": {}, + "inputWidgets": {}, + "nuid": "1e12f30f-e1ae-49f9-a136-8dd94032e7aa", + "showTitle": false, + "tableResultSettingsMap": {}, + "title": "" + } + }, + "source": [ + "\n", + "## 8. Scale Architecture Notes (10M Images)\n", + "\n", + "| Concern | Demo Approach | 10M Approach |\n", + "|---|---|---|\n", + "| File discovery | `dbutils.fs.ls` recursive | Auto Loader on the volume path |\n", + "| Metadata extraction | `SVSMetaExtractor` via `mapInPandas` + `ThreadPoolExecutor` | Same — already distributed |\n", + "| Label image storage | Written to volume as files | Stored as `BINARY` in Delta table (eliminates extra volume I/O) |\n", + "| VLM inference | Single SQL batch `ai_query()` | Incremental: `WHERE vlm_status='PENDING'` in a scheduled Lakeflow Job |\n", + "| TIFF conversion | `write_deidentified_tiff_udf` Spark UDF | Same — Photon-accelerated UDF dispatch |\n", + "| Checkpointing | `vlm_status` column | Same + Delta transaction log for idempotency |\n", + "| Cost control | Serverless interactive | SQL Serverless warehouse + compute-optimized clusters for UDF stages |\n", + "\n", + "---" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": {}, + "inputWidgets": {}, + "nuid": "9ff39799-03ad-4f51-a8a8-3d0823d8b3e2", + "showTitle": false, + "tableResultSettingsMap": {}, + "title": "" + } + }, + "source": [ + "## 9. Confirmed Findings & Resolved Design Decisions\n", + "\n", + "All four open questions are now resolved from direct inspection of the actual Aperio CMU-1.svs files.\n", + "\n", + "### Q1 — Bounding box coordinate space ✅ RESOLVED\n", + "\n", + "The label sub-image is **387×463 RGBA** — completely independent from the tissue scan (46000×32914). The two coordinate spaces share no relationship.\n", + "\n", + "**Decision:** Save and submit the label image to the VLM at **native resolution (no resizing)**. All VLM bboxes are in label-image pixel space (`0,0` = top-left). The tissue TIFF does **not** embed the label — it is automatically excluded when only the main pyramid is read. The de-identified label PNG (black rectangles applied) is written to the `label_images` volume as the audit artefact.\n", + "\n", + "**Macro image clarification:** The macro (1280×431) shows both tissue and the physical label (at one end). It is submitted to the VLM separately; its bboxes drive black-rectangle redaction of the label area in the macro output PNG.\n", + "\n", + "---\n", + "\n", + "### Q2 — `tiff.ImageDescription` scrubbing ✅ RESOLVED\n", + "\n", + "Exact format confirmed from the real file:\n", + "```\n", + "Aperio Image Library v10.0.51\\r\\n46920x33014 [0,100 46000x32914] (256x256) JPEG/RGB Q=30\n", + " |AppMag = 20|StripeWidth = 2040|ScanScope ID = CPAPERIOCS|Filename = CMU-1\n", + " |Date = 12/29/09|Time = 09:59:15|User = b414003d-...|ImageID = 1004486|...\n", + "```\n", + "\n", + "**Structure:** `{header_line}|key = val|key = val|...`  Header is technical-only — preserve as-is.\n", + "\n", + "**PHI classification of actual keys:**\n", + "\n", + "| Key | Classification | Notes |\n", + "|---|---|---|\n", + "| `Date`, `Time` | PHI | HIPAA date/time of service |\n", + "| `User` | QUESTIONABLE | GUID in demo; operator name in clinical use |\n", + "| `Filename` | QUESTIONABLE | May encode patient name or MRN |\n", + "| `ImageID` | QUESTIONABLE | Could be accession number |\n", + "| `ScanScope ID`, `AppMag`, `StripeWidth`, `Parmset`, `MPP`, all geometry/calibration, `Filtered`, `ICC Profile` | NOT_PHI | Pure scanner parameters |\n", + "\n", + "Clinical files may also contain: `Patient`, `DOB`, `MRN`, `AccessionNumber`, `Clinic`, `Pathologist`, `Procedure`, `Diagnosis`, `Id` — all PHI.\n", + "\n", + "**Scrubbing algorithm:**\n", + "1. `header, *kvs = image_desc.split('|')`\n", + "2. For each `kv`: `k, v = kv.split(' = ', 1)` — rebuild as `k = REDACTED` if `k.strip()` ∈ PHI/QUESTIONABLE set\n", + "3. Rejoin: `'|'.join([header] + rebuilt_kvs)`\n", + "4. Apply identical scrub to `openslide.comment` (same content) when writing TIFF metadata\n", + "\n", + "---\n", + "\n", + "### Q3 — Macro image redaction ✅ RESOLVED\n", + "\n", + "Macro (1280×431) shows the full physical slide including the affixed label. **Decision:** Include macro in the primary VLM pipeline alongside the label (not a follow-on phase). Naming: `{name}_label.png` / `{name}_macro.png`. Both de-identified PNGs go to the `label_images` volume.\n", + "\n", + "---\n", + "\n", + "### Q4 — Pyramidal TIFF output ✅ RESOLVED\n", + "\n", + "Flat TIFF is not viable for pathology — QuPath, OMERO, and DIGIPATH all require pyramidal. The source SVS has 3 levels with 256×256 tiles; match this in output.\n", + "\n", + "**Decision:** Write pyramidal **BigTIFF** via `tifffile`:\n", + "- `bigtiff=True` — mandatory (CMU-1 level-0 ~7.4 GB uncompressed, exceeds 4 GB TIFF limit)\n", + "- `tile=(256, 256)` — matches native Aperio tile size\n", + "- `compression='jpeg'` at quality 80; swap to `'lzw'` if lossless required\n", + "- `subifds=level_count - 1` — sub-IFDs are the QuPath/libvips-compatible pyramid convention\n", + "- Pyramid levels: 2× progressive downsampling with `PIL.Image.LANCZOS`\n", + "\n", + "```python\n", + "with tifffile.TiffWriter(output_path, bigtiff=True) as tif:\n", + " opts = dict(tile=(256, 256), compression='jpeg',\n", + " compressionargs={'level': 80}, photometric='rgb', metadata=None)\n", + " tif.write(level_0_rgb, subifds=level_count - 1, **opts) # main IFD\n", + " for lvl in range(1, level_count):\n", + " tif.write(level_arrays[lvl], subfiletype=1, **opts) # sub-IFDs\n", + "```\n", + "\n", + "OME-TIFF (`ome=True`) only if OMERO is a confirmed downstream consumer.\n", + "\n", + "---" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": {}, + "inputWidgets": {}, + "nuid": "308dda6c-953c-44e0-b8a3-4e20dafb9357", + "showTitle": false, + "tableResultSettingsMap": {}, + "title": "" + } + }, + "source": [ + "\n", + "## 10. More...\n", + "\n", + "### Additional: `openslide-bin` Required\n", + "\n", + "`openslide-python` alone fails at import on Databricks Serverless:\n", + "```\n", + "ModuleNotFoundError: Couldn't locate OpenSlide shared library. Try pip install openslide-bin.\n", + "```\n", + "**Cell 2 must install:** `openslide-python openslide-bin` (the `openslide-bin` wheel bundles `libopenslide.so` for environments without system package access).\n", + "\n", + "\n", + "> **Note**: `files => content` is the correct `ai_query()` API for binary image inputs — it passes the PNG bytes directly to the model without base64 encoding. Only JPEG and PNG inputs are supported.\n", + "\n", + "### Scale to 10M Images\n", + "- `vlm_status` column acts as a watermark: `PENDING → PROCESSING → COMPLETE / FAILED`\n", + "- The SQL above runs as a Databricks SQL batch job — `ai_query()` parallelizes across serverless SQL clusters automatically\n", + "- For throughput control: partition the batch by date/rack and run multiple concurrent SQL statements\n", + "- Auto Loader can feed new SVS arrivals into `svs_metadata` as `PENDING`, triggering incremental VLM runs via a scheduled Lakeflow Job\n", + "\n" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": {}, + "inputWidgets": {}, + "nuid": "dd0db54d-807b-4cfe-bca5-0b524fd6e636", + "showTitle": false, + "tableResultSettingsMap": {}, + "title": "" + } + }, + "source": [ + "# Code" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": {}, + "inputWidgets": {}, + "nuid": "6e4014e5-1ba3-4c5c-9a8c-fbe85432645b", + "showTitle": true, + "tableResultSettingsMap": {}, + "title": "Data Flow Diagram" + } + }, + "source": [ + "## Data Flow Diagram\n", + "\n", + "```mermaid\n", + "flowchart TD\n", + " %% ─── External Sources ───\n", + " SVS_INPUT[(\"/Volumes/hls_pathology/orthanc_demo/raw_images/Aperio/\\n~14 SVS files\")]\n", + " VLM_EP{{\"databricks-llama-4-maverick\\n(VLM Endpoint)\"}}\n", + "\n", + " %% ─── Delta Tables ───\n", + " OBJ_CAT[(\"douglas_moore.pathology\\n.object_catalog\")]\n", + " OBJ_RED[(\"douglas_moore.pathology\\n.object_catalog_redaction\")]\n", + " TIFF_STG[(\"douglas_moore.pathology\\n.tiff_results_staging\")]\n", + "\n", + " %% ─── Volumes (File Storage) ───\n", + " LABEL_VOL[(\"/Volumes/.../label_images/\\nPNG sub-images\")]\n", + " TIFF_VOL[(\"/Volumes/.../tiff_deidentified/\\nBigTIFF output\")]\n", + " TMP[\"/tmp/ (executor local)\\nBigTIFF staging\"]\n", + "\n", + " %% ─── Processing Steps ───\n", + " DISCOVER[\"Cell 17: File Discovery\\nSVSCatalog.catalog()\"]\n", + " META[\"Cell 18: Metadata Extraction\\nSVSMetaExtractor (mapInPandas)\\nOpenSlide properties → VARIANT\"]\n", + " PHI_TAGS[\"Cell 19: PHI Tag Report\\n(display only)\"]\n", + " EXTRACT[\"Cell 20: Extract Sub-images\\npandas_udf + OpenSlide\\nassociated_images → PNG\"]\n", + " VLM_DETECT[\"Cell 22: VLM PHI Detection\\nai_query(files => content)\\nREAD_FILES + INSERT\"]\n", + " BUILD_DF[\"Cell 24: Build redaction_df\\nJOIN catalog + redaction\\nWHERE status = PENDING\"]\n", + " UDF[\"Cell 25-27: De-identify UDF\\nmapInPandas + ThreadPoolExecutor\\nredact_image + write_pyramidal_bigtiff\"]\n", + " MERGE[\"Cell 28: MERGE Results\\nUPDATE status, paths, errors\"]\n", + " AUDIT[\"Cell 29: Audit Summary\\n(display only)\"]\n", + "\n", + " %% ─── Data Flows ───\n", + " SVS_INPUT -->|\"list files\"| DISCOVER\n", + " DISCOVER -->|\"files_df (in-memory)\"| META\n", + " SVS_INPUT -->|\"read OpenSlide props\"| META\n", + " META -->|\"mode=append\"| OBJ_CAT\n", + "\n", + " OBJ_CAT -->|\"read meta:phi_tag_report\"| PHI_TAGS\n", + "\n", + " SVS_INPUT -->|\"read associated_images\"| EXTRACT\n", + " EXTRACT -->|\"save PNG (sequential write)\"| LABEL_VOL\n", + "\n", + " LABEL_VOL -->|\"READ_FILES(binaryFile)\"| VLM_DETECT\n", + " VLM_DETECT -->|\"ai_query()\"| VLM_EP\n", + " VLM_EP -->|\"JSON response\"| VLM_DETECT\n", + " OBJ_CAT -->|\"JOIN for path\"| VLM_DETECT\n", + " VLM_DETECT -->|\"INSERT INTO\"| OBJ_RED\n", + "\n", + " OBJ_CAT -->|\"JOIN\"| BUILD_DF\n", + " OBJ_RED -->|\"WHERE PENDING\"| BUILD_DF\n", + "\n", + " BUILD_DF -->|\"redaction_df\"| UDF\n", + " SVS_INPUT -->|\"read tiles (OpenSlide)\"| UDF\n", + " UDF -->|\"redacted PNGs (seq write)\"| LABEL_VOL\n", + " UDF -->|\"write BigTIFF (seek+write)\"| TMP\n", + " TMP -->|\"shutil.copy2 (seq write)\"| TIFF_VOL\n", + " UDF -->|\"saveAsTable\"| TIFF_STG\n", + "\n", + " TIFF_STG -->|\"source for MERGE\"| MERGE\n", + " MERGE -->|\"UPDATE status/paths\"| OBJ_RED\n", + "\n", + " OBJ_CAT -->|\"LEFT JOIN\"| AUDIT\n", + " OBJ_RED -->|\"LEFT JOIN\"| AUDIT\n", + "\n", + " %% ─── Styling ───\n", + " classDef volume fill:#e8f5e9,stroke:#2e7d32\n", + " classDef table fill:#e3f2fd,stroke:#1565c0\n", + " classDef process fill:#fff3e0,stroke:#e65100\n", + " classDef external fill:#fce4ec,stroke:#c62828\n", + " classDef tmp fill:#f5f5f5,stroke:#616161,stroke-dasharray:5\n", + "\n", + " class SVS_INPUT,LABEL_VOL,TIFF_VOL volume\n", + " class OBJ_CAT,OBJ_RED,TIFF_STG table\n", + " class DISCOVER,META,PHI_TAGS,EXTRACT,VLM_DETECT,BUILD_DF,UDF,MERGE,AUDIT process\n", + " class VLM_EP external\n", + " class TMP tmp\n", + "```\n", + "\n", + "### Legend\n", + "| Color | Meaning |\n", + "|---|---|\n", + "| Green | UC Volumes (file storage) |\n", + "| Blue | Delta Tables (Unity Catalog) |\n", + "| Orange | Processing steps (notebook cells) |\n", + "| Pink | External service (model endpoint) |\n", + "| Dashed gray | Ephemeral local storage (/tmp) |\n", + "\n", + "### Key Write Patterns\n", + "| Target | Write Mode | Reason |\n", + "|---|---|---|\n", + "| `object_catalog` | `mode=append` | Idempotent cataloguing; dedup via path |\n", + "| `object_catalog_redaction` | `INSERT INTO` | One row per VLM detection run |\n", + "| `tiff_results_staging` | `mode=overwrite` | Ephemeral staging; replaced each run |\n", + "| `object_catalog_redaction` | `MERGE ... WHEN MATCHED UPDATE` | Update status after TIFF write |\n", + "| Label PNGs (volume) | Sequential FUSE write | PIL `img.save()` — no seek needed |\n", + "| BigTIFFs (volume) | `/tmp/` → `shutil.copy2` | tifffile needs seek; Volume FUSE does not support seek+write |" + ] + }, + { + "cell_type": "code", + "execution_count": 0, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": { + "byteLimit": 2048000, + "rowLimit": 10000 + }, + "inputWidgets": {}, + "nuid": "e1fbd42a-68b2-44e2-8f2a-d9210c5254f5", + "showTitle": true, + "tableResultSettingsMap": {}, + "title": "Cell 2: Install dependencies" + } + }, + "outputs": [], + "source": [ + "# Install core dependencies.\n", + "# databricks-pixels provides Catalog + Transformer base classes.\n", + "# openslide-bin bundles libopenslide.so so OpenSlide works on Serverless.\n", + "%pip install openslide-python openslide-bin tifffile imagecodecs Pillow easyocr -q" + ] + }, + { + "cell_type": "code", + "execution_count": 0, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": { + "byteLimit": 2048000, + "rowLimit": 10000 + }, + "inputWidgets": {}, + "nuid": "ec9caf79-4345-4b35-9b30-dc215d7885e4", + "showTitle": true, + "tableResultSettingsMap": {}, + "title": "Cell 3: Configuration" + } + }, + "outputs": [], + "source": [ + "# The full pixels source tree (including svs/) lives in the workspace.\n", + "# Add the src directory to sys.path so `dbx.pixels` and `dbx.pixels.svs`\n", + "# are importable without a separate pip install.\n", + "import sys\n", + "import types\n", + "import importlib\n", + "\n", + "_SRC_ROOT = \"/Workspace/Users/douglas.moore@databricks.com/pixels-svs/src\"\n", + "if _SRC_ROOT not in sys.path:\n", + " sys.path.insert(0, _SRC_ROOT)\n", + "importlib.invalidate_caches()\n", + "\n", + "# Load deidentify module (file is clean — no truncation needed)\n", + "_deident_path = f\"{_SRC_ROOT}/dbx/pixels/svs/deidentify.py\"\n", + "with open(_deident_path, \"r\") as _f:\n", + " _clean_src = _f.read()\n", + "_deident_mod = types.ModuleType(\"dbx.pixels.svs.deidentify\")\n", + "_deident_mod.__file__ = _deident_path\n", + "exec(compile(_clean_src, _deident_path, \"exec\"), _deident_mod.__dict__)\n", + "sys.modules[\"dbx.pixels.svs.deidentify\"] = _deident_mod\n", + "\n", + "from dbx.pixels.svs import SVSCatalog, SVSMetaExtractor, SVSTiffWriter\n", + "from dbx.pixels.svs.phi_tags import classify_tags, scrub_image_description\n", + "\n", + "# ── Pipeline configuration ────────────────────────────────────────────────────\n", + "CATALOG = \"douglas_moore\"\n", + "SCHEMA = \"pathology\"\n", + "UC_TABLE = f\"{CATALOG}.{SCHEMA}.object_catalog\"\n", + "UC_VOLUME = f\"{CATALOG}.{SCHEMA}.pixels_volume\"\n", + "INPUT_PATH = \"/Volumes/hls_pathology/orthanc_demo/raw_images/Aperio/\"\n", + "TIFF_VOLUME = f\"/Volumes/{CATALOG}/{SCHEMA}/tiff_deidentified\"\n", + "LABEL_VOLUME = f\"/Volumes/{CATALOG}/{SCHEMA}/label_images\"\n", + "VLM_ENDPOINT = \"databricks-llama-4-maverick\"\n", + "\n", + "print(f\"Input : {INPUT_PATH}\")\n", + "print(f\"Table : {UC_TABLE}\")\n", + "print(f\"TIFFs : {TIFF_VOLUME}\")\n", + "print(f\"Labels: {LABEL_VOLUME}\")\n", + "print(f\"VLM : {VLM_ENDPOINT}\")\n" + ] + }, + { + "cell_type": "code", + "execution_count": 0, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": { + "byteLimit": 2048000, + "implicitDf": true, + "rowLimit": 10000 + }, + "inputWidgets": {}, + "nuid": "47cec7fe-67e6-4928-9aa6-1c5947b011bc", + "showTitle": true, + "tableResultSettingsMap": {}, + "title": "Reset: Truncate pipeline tables" + } + }, + "outputs": [], + "source": [ + "%sql\n", + "-- Reset pipeline state for a clean end-to-end run.\n", + "-- Truncates data only; table structure and permissions preserved.\n", + "TRUNCATE TABLE douglas_moore.pathology.object_catalog;\n", + "TRUNCATE TABLE douglas_moore.pathology.object_catalog_redaction;" + ] + }, + { + "cell_type": "code", + "execution_count": 0, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": { + "byteLimit": 2048000, + "rowLimit": 10000 + }, + "inputWidgets": {}, + "nuid": "8c5e7ce2-bec6-48b8-987b-34b7b2a1eeb5", + "showTitle": true, + "tableResultSettingsMap": {}, + "title": "Cell 4: Storage bootstrap" + } + }, + "outputs": [], + "source": [ + "# Create schema and volumes (idempotent — safe to re-run)\n", + "spark.sql(f\"CREATE SCHEMA IF NOT EXISTS {CATALOG}.{SCHEMA}\")\n", + "spark.sql(f\"CREATE VOLUME IF NOT EXISTS {CATALOG}.{SCHEMA}.pixels_volume\")\n", + "spark.sql(f\"CREATE VOLUME IF NOT EXISTS {CATALOG}.{SCHEMA}.tiff_deidentified\")\n", + "spark.sql(f\"CREATE VOLUME IF NOT EXISTS {CATALOG}.{SCHEMA}.label_images\")\n", + "\n", + "# Initialise Delta tables:\n", + "# object_catalog — base DDL from databricks-pixels (unchanged)\n", + "# object_catalog_redaction — unified SVS/DICOM DDL from CREATE_SVS_CATALOG.sql\n", + "catalog = SVSCatalog(spark, table=UC_TABLE, volume=UC_VOLUME)\n", + "catalog.init_tables()\n", + "\n", + "print(\"Schema, volumes, and tables initialised.\")\n", + "display(spark.sql(f\"SHOW TABLES IN {CATALOG}.{SCHEMA}\"))\n" + ] + }, + { + "cell_type": "code", + "execution_count": 0, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": { + "byteLimit": 2048000, + "rowLimit": 10000 + }, + "inputWidgets": {}, + "nuid": "d0560972-3ca5-4f06-8f65-d7c621b688db", + "showTitle": true, + "tableResultSettingsMap": {}, + "title": "Cell 5: File discovery" + } + }, + "outputs": [], + "source": [ + "# Discover all SVS files under INPUT_PATH and register them in object_catalog.\n", + "# SVSCatalog.catalog() defaults pattern='*.svs'; also picks up sidecar .txt files.\n", + "files_df = catalog.catalog(INPUT_PATH)\n", + "print(f\"Discovered {files_df.count()} files\")\n", + "display(files_df.select(\"path\", \"local_path\", \"length\", \"modificationTime\", \"extension\"))\n" + ] + }, + { + "cell_type": "code", + "execution_count": 0, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": { + "byteLimit": 2048000, + "rowLimit": 10000 + }, + "inputWidgets": {}, + "nuid": "147b854b-61b3-4207-a172-9fc5f36649b9", + "showTitle": true, + "tableResultSettingsMap": {}, + "title": "Cell 6: Metadata extraction" + } + }, + "outputs": [], + "source": [ + "# SVSMetaExtractor reads every SVS via OpenSlide (ThreadPoolExecutor, 32 concurrent).\n", + "# All properties + derived fields (width, height, levels, phi_tag_report) are merged\n", + "# into one JSON dict → parse_json() → VARIANT. No schema changes to object_catalog.\n", + "extractor = SVSMetaExtractor(catalog, inputCol=\"local_path\", outputCol=\"meta\")\n", + "meta_df = extractor._transform(files_df)\n", + "\n", + "(\n", + " meta_df.write\n", + " .format(\"delta\")\n", + " .mode(\"append\")\n", + " .saveAsTable(UC_TABLE)\n", + ")\n", + "\n", + "print(f\"Wrote {spark.table(UC_TABLE).count()} rows to {UC_TABLE}\")\n", + "\n", + "display(spark.sql(f\"\"\"\n", + "SELECT\n", + " regexp_extract(path, '[^/]+$', 0) AS filename,\n", + " meta:width::int AS width,\n", + " meta:height::int AS height,\n", + " meta:level_count::int AS levels,\n", + " meta:has_label_image::boolean AS has_label,\n", + " meta:has_macro_image::boolean AS has_macro,\n", + " meta:`aperio.AppMag`::string AS app_mag,\n", + " meta:`aperio.MPP`::string AS mpp,\n", + " meta\n", + "FROM {UC_TABLE}\n", + "ORDER BY filename\n", + "\"\"\"))\n" + ] + }, + { + "cell_type": "code", + "execution_count": 0, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": { + "byteLimit": 2048000, + "rowLimit": 10000 + }, + "inputWidgets": {}, + "nuid": "541697a5-602a-40dd-9ac9-9f53624b4efa", + "showTitle": true, + "tableResultSettingsMap": { + "0": { + "dataGridStateBlob": "{\"version\":1,\"tableState\":{\"columnPinning\":{\"left\":[\"#row_number#\"],\"right\":[]},\"columnSizing\":{\"tag\":129},\"columnVisibility\":{}},\"settings\":{\"columns\":{}},\"syncTimestamp\":1781723212243}", + "filterBlob": null, + "queryPlanFiltersBlob": null, + "tableResultIndex": 0 + } + }, + "title": "Cell 7: PHI tag report" + } + }, + "outputs": [], + "source": [ + "# PHI / QUESTIONABLE tag values for every slide.\n", + "# Unpack the phi_tag_report VARIANT array stored in meta.\n", + "from pyspark.sql.functions import regexp_extract, col, explode, from_json, expr\n", + "from pyspark.sql.types import ArrayType, StructType, StructField, StringType\n", + "\n", + "phi_schema = ArrayType(StructType([\n", + " StructField(\"tag\", StringType()),\n", + " StructField(\"value\", StringType()),\n", + " StructField(\"classification\", StringType()),\n", + "]))\n", + "\n", + "phi_df = (\n", + " spark.table(UC_TABLE)\n", + " .filter(expr(\"meta:phi_tag_report IS NOT NULL\"))\n", + " .withColumn(\"phi_tag_report_str\", expr(\"cast(meta:phi_tag_report AS STRING)\"))\n", + " .withColumn(\"tags\", from_json(\"phi_tag_report_str\", phi_schema))\n", + " .withColumn(\"elem\", explode(\"tags\"))\n", + " .select(\n", + " regexp_extract(\"path\", r\"[^/]+$\", 0).alias(\"filename\"),\n", + " col(\"elem.tag\").alias(\"tag\"),\n", + " col(\"elem.value\").alias(\"value\"),\n", + " col(\"elem.classification\").alias(\"classification\"),\n", + " )\n", + " .filter(col(\"classification\").isin(\"PHI\", \"QUESTIONABLE\"))\n", + " .orderBy(\"filename\", \"classification\", \"tag\")\n", + ")\n", + "print(f\"PHI/QUESTIONABLE findings: {phi_df.count()}\")\n", + "display(phi_df)" + ] + }, + { + "cell_type": "code", + "execution_count": 0, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": { + "byteLimit": 2048000, + "rowLimit": 10000 + }, + "inputWidgets": {}, + "nuid": "28f6bfb9-bec1-4cb0-9c80-5c395a1a9432", + "showTitle": true, + "tableResultSettingsMap": {}, + "title": "Cell 8: Extract label/macro sub-images" + } + }, + "outputs": [], + "source": [ + "# Extract label and macro sub-images from each SVS and save as PNGs.\n", + "# These are later submitted to the VLM (Cell 9) and used as audit artefacts.\n", + "# Uses a pandas_udf so extraction runs distributed across workers.\n", + "from pyspark.sql.functions import pandas_udf, regexp_extract, col\n", + "import pandas as pd\n", + "from pyspark.sql.types import StringType\n", + "\n", + "_LABEL_VOL = LABEL_VOLUME # captured in closure; serialised with the UDF\n", + "\n", + "@pandas_udf(StringType())\n", + "def extract_subimages_udf(paths: pd.Series, stems: pd.Series) -> pd.Series:\n", + " import openslide, os\n", + " results = []\n", + " for path, stem in zip(paths, stems):\n", + " try:\n", + " slide = openslide.OpenSlide(path)\n", + " saved = []\n", + " for name in (\"label\", \"macro\"):\n", + " if name in slide.associated_images:\n", + " img = slide.associated_images[name].convert(\"RGB\")\n", + " out = f\"{_LABEL_VOL}/{stem}_{name}.png\"\n", + " os.makedirs(os.path.dirname(out), exist_ok=True)\n", + " img.save(out)\n", + " saved.append(out)\n", + " slide.close()\n", + " results.append(\",\".join(saved))\n", + " except Exception as e:\n", + " results.append(f\"ERROR: {e}\")\n", + " return pd.Series(results)\n", + "\n", + "catalog_df = (\n", + " spark.table(UC_TABLE)\n", + " .withColumn(\"stem\", regexp_extract(col(\"path\"), r\"([^/]+)\\.svs$\", 1))\n", + ")\n", + "\n", + "extracted_df = catalog_df.withColumn(\n", + " \"extracted_images\",\n", + " extract_subimages_udf(col(\"local_path\"), col(\"stem\")),\n", + ")\n", + "\n", + "display(extracted_df.select(\"path\", \"stem\", \"extracted_images\"))\n", + "print(f\"Label/macro PNGs written to {LABEL_VOLUME}\")\n" + ] + }, + { + "cell_type": "code", + "execution_count": 0, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": { + "byteLimit": 2048000, + "rowLimit": 10000 + }, + "inputWidgets": {}, + "nuid": "556f24ea-d0c5-46ef-ac35-761ffed88820", + "showTitle": true, + "tableResultSettingsMap": {}, + "title": "Display first 10 label images" + } + }, + "outputs": [], + "source": [ + "# Display first 10 label sub-images extracted from SVS pathology slides\n", + "import os\n", + "from PIL import Image\n", + "import matplotlib.pyplot as plt\n", + "\n", + "label_dir = LABEL_VOLUME\n", + "label_files = sorted([f for f in os.listdir(label_dir) if f.endswith(\"_label.png\")])[:10]\n", + "\n", + "ncols = min(5, len(label_files))\n", + "nrows = (len(label_files) + ncols - 1) // ncols\n", + "fig, axes = plt.subplots(nrows, ncols, figsize=(4 * ncols, 5 * nrows))\n", + "if len(label_files) == 1:\n", + " axes = [axes]\n", + "else:\n", + " axes = axes.flatten()\n", + "\n", + "for i, fname in enumerate(label_files):\n", + " img = Image.open(os.path.join(label_dir, fname))\n", + " axes[i].imshow(img)\n", + " axes[i].set_title(fname.replace(\"_label.png\", \"\"), fontsize=9)\n", + " axes[i].axis(\"off\")\n", + "\n", + "for j in range(len(label_files), len(axes)):\n", + " axes[j].axis(\"off\")\n", + "\n", + "plt.suptitle(\"SVS Label Sub-Images (PHI candidates for VLM redaction)\", fontsize=13)\n", + "plt.tight_layout()\n", + "plt.show()" + ] + }, + { + "cell_type": "code", + "execution_count": 0, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": { + "byteLimit": 2048000, + "implicitDf": true, + "rowLimit": 10000 + }, + "height": "156", + "inputWidgets": {}, + "nuid": "23583e7a-dfc7-4da0-be2b-dbac0b061935", + "showTitle": true, + "tableResultSettingsMap": {}, + "title": "Cell 9: VLM PHI detection", + "width": "834" + } + }, + "outputs": [], + "source": [ + "%sql\n", + "-- Run VLM PHI detection on all label PNGs and insert results into object_catalog_redaction.\n", + "-- ai_query() submits the image binary directly via `files => content` (no base64 needed).\n", + "-- responseFormat => 'json_object' guarantees machine-parseable output.\n", + "INSERT INTO douglas_moore.pathology.object_catalog_redaction (\n", + " redaction_id, path, extension, modality,\n", + " has_phi, phi_elements, vlm_raw_response, model_endpoint,\n", + " output_file_paths, label_image_path, macro_image_path,\n", + " status, insert_timestamp, created_by\n", + ")\n", + "WITH vlm_raw AS (\n", + " SELECT\n", + " regexp_replace(f._metadata.file_name, '_label\\.png$', '') AS stem,\n", + " f._metadata.file_path AS label_image_path,\n", + " ai_query(\n", + " 'databricks-llama-4-maverick',\n", + " 'You are a HIPAA-compliant PHI detection system.\n", + "Analyze this pathology slide label image and identify all Protected Health Information:\n", + "patient names, dates, MRNs, accession numbers, barcodes, or other identifying text.\n", + "Return ONLY a json object — no prose, no markdown fences.\n", + "Schema: {\"has_phi\": bool, \"phi_elements\": [{\"type\": \"name|dob|mrn|accession|barcode|other\", \"value_hint\": \"\", \"bbox\": {\"x\": int, \"y\": int, \"w\": int, \"h\": int}, \"subimage\": \"label\"}]}\n", + "If no PHI found: {\"has_phi\": false, \"phi_elements\": []}',\n", + " files => content\n", + " ) AS vlm_raw_response\n", + " FROM READ_FILES(\n", + " '/Volumes/douglas_moore/pathology/label_images/',\n", + " format => 'binaryFile',\n", + " fileNamePattern => '*_label.png'\n", + " ) f\n", + "),\n", + "joined AS (\n", + " SELECT\n", + " v.stem,\n", + " v.label_image_path,\n", + " v.vlm_raw_response,\n", + " m.path AS obj_path,\n", + " concat('/Volumes/douglas_moore/pathology/label_images/', v.stem, '_macro.png') AS macro_image_path\n", + " FROM vlm_raw v\n", + " JOIN douglas_moore.pathology.object_catalog m\n", + " ON regexp_extract(m.path, '([^/]+)\\.svs$', 1) = v.stem\n", + ")\n", + "SELECT\n", + " uuid() AS redaction_id,\n", + " obj_path AS path,\n", + " 'svs' AS extension,\n", + " 'WSI' AS modality,\n", + " try_cast(get_json_object(vlm_raw_response, '$.has_phi') AS BOOLEAN) AS has_phi,\n", + " parse_json(get_json_object(vlm_raw_response, '$.phi_elements')) AS phi_elements,\n", + " vlm_raw_response,\n", + " 'databricks-llama-4-maverick' AS model_endpoint,\n", + " array(CAST(NULL AS STRING)) AS output_file_paths,\n", + " label_image_path,\n", + " macro_image_path,\n", + " 'PENDING' AS status,\n", + " current_timestamp() AS insert_timestamp,\n", + " current_user() AS created_by\n", + "FROM joined\n" + ] + }, + { + "cell_type": "code", + "execution_count": 0, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": { + "byteLimit": 2048000, + "implicitDf": true, + "rowLimit": 10000 + }, + "inputWidgets": {}, + "nuid": "6dae4df9-e494-403e-9c58-a933aa22052a", + "showTitle": false, + "tableResultSettingsMap": {}, + "title": "" + } + }, + "outputs": [], + "source": [ + "%sql\n", + "select * from douglas_moore.pathology.object_catalog_redaction" + ] + }, + { + "cell_type": "code", + "execution_count": 0, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": { + "byteLimit": 2048000, + "rowLimit": 10000 + }, + "inputWidgets": {}, + "nuid": "98cc5d83-bbdd-4f8c-9bd6-b61d259ad882", + "showTitle": false, + "tableResultSettingsMap": {}, + "title": "" + } + }, + "outputs": [], + "source": [ + "# Reload modules to pick up streaming TIFF writer\n", + "import importlib, sys\n", + "for mod_name in list(sys.modules):\n", + " if mod_name.startswith(\"dbx.pixels.svs\"):\n", + " del sys.modules[mod_name]\n", + "\n", + "# Join PENDING redaction rows (phi_elements from VLM) with object_catalog (local_path),\n", + "# run de-identification and produce pyramidal BigTIFFs.\n", + "redaction_df = spark.sql(f\"\"\"\n", + "SELECT\n", + " o.local_path,\n", + " o.path,\n", + " r.redaction_id,\n", + " to_json(r.phi_elements) AS phi_elements_json\n", + "FROM {CATALOG}.{SCHEMA}.object_catalog o\n", + "JOIN {CATALOG}.{SCHEMA}.object_catalog_redaction r\n", + " ON o.path = r.path\n", + "WHERE r.status = 'PENDING'\n", + "\"\"\")\n", + "print(f\"Files to de-identify: {redaction_df.count()}\")" + ] + }, + { + "cell_type": "code", + "execution_count": 0, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": { + "byteLimit": 2048000, + "rowLimit": 10000 + }, + "inputWidgets": {}, + "nuid": "a73b4474-3adc-4866-88cd-822dc0ad6c45", + "showTitle": true, + "tableResultSettingsMap": {}, + "title": "Cell 10: De-identified TIFF write" + } + }, + "outputs": [], + "source": [ + "# --- Distributed de-identification via mapInPandas (memory-safe for Serverless 1 GB) ---\n", + "#\n", + "# Design principles applied from review:\n", + "# • No inner ThreadPoolExecutor — mapInPandas already parallelizes across Spark\n", + "# partitions; nested threading doubles slide opens and memory pressure.\n", + "# • No to_dict(\"records\") — iterate rows via iloc to avoid duplicating the batch.\n", + "# • OpenSlide closed in a finally block so C-bindings are destroyed even on error.\n", + "# • gc.collect() after each slide reclaims PIL/OpenSlide C-level allocations.\n", + "# • Tile-based TIFF writing via write_pyramidal_bigtiff_streaming (256×256 read_region).\n", + "# • Temp files staged to /tmp (seek-capable), then shutil.copy2 to Volume (seq FUSE).\n", + "# • PID suffix on temp paths prevents collisions across retries/speculative tasks.\n", + "# • repartition(num_slides) ensures 1 row per partition — each executor handles\n", + "# exactly one slide. (arrow.maxRecordsPerBatch is NOT settable on Serverless.)\n", + "\n", + "import json\n", + "import pandas as pd\n", + "from pyspark.sql.types import StructType, StructField, StringType, ArrayType, IntegerType\n", + "\n", + "_TIFF_VOLUME = TIFF_VOLUME\n", + "_LABEL_VOLUME = LABEL_VOLUME\n", + "_SRC_ROOT = \"/Workspace/Users/douglas.moore@databricks.com/pixels-svs/src\"\n", + "\n", + "_result_schema = StructType([\n", + " StructField(\"path\", StringType(), True),\n", + " StructField(\"tiff_output_path\", StringType(), True),\n", + " StructField(\"label_image_path\", StringType(), True),\n", + " StructField(\"macro_image_path\", StringType(), True),\n", + " StructField(\"phi_tags_redacted\", ArrayType(StringType()), True),\n", + " StructField(\"pixel_regions_redacted\", IntegerType(), True),\n", + " StructField(\"error\", StringType(), True),\n", + "])\n", + "\n", + "\n", + "def _deidentify_batch(iterator):\n", + " \"\"\"mapInPandas worker: one slide per batch, streaming tile reads, no threading.\"\"\"\n", + " import sys, os, gc, shutil, time, logging, resource\n", + " from pathlib import Path\n", + "\n", + " # --- Memory debugging utilities ---\n", + " logging.basicConfig(level=logging.INFO, format=\"%(asctime)s [%(levelname)s] %(message)s\")\n", + " log = logging.getLogger(\"deidentify_worker\")\n", + "\n", + " def _mem_mb() -> dict:\n", + " \"\"\"Return RSS and VMS in MB from /proc/self/status (Linux) with fallback.\"\"\"\n", + " rss_mb = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss / 1024 # KB→MB on Linux\n", + " try:\n", + " with open(\"/proc/self/status\") as f:\n", + " status = f.read()\n", + " vmpeak = vmrss = vmsize = 0\n", + " for line in status.splitlines():\n", + " if line.startswith(\"VmPeak:\"):\n", + " vmpeak = int(line.split()[1]) / 1024\n", + " elif line.startswith(\"VmRSS:\"):\n", + " vmrss = int(line.split()[1]) / 1024\n", + " elif line.startswith(\"VmSize:\"):\n", + " vmsize = int(line.split()[1]) / 1024\n", + " return {\"rss_mb\": round(vmrss, 1), \"vms_mb\": round(vmsize, 1), \"peak_mb\": round(vmpeak, 1)}\n", + " except Exception:\n", + " return {\"rss_mb\": round(rss_mb, 1), \"vms_mb\": -1, \"peak_mb\": -1}\n", + "\n", + " def _log_mem(stage: str, stem: str, extra: str = \"\"):\n", + " mem = _mem_mb()\n", + " msg = f\"[{stem}] stage={stage} | RSS={mem['rss_mb']}MB VMS={mem['vms_mb']}MB Peak={mem['peak_mb']}MB\"\n", + " if extra:\n", + " msg += f\" | {extra}\"\n", + " log.info(msg)\n", + " # Warn if approaching the 1024 MB limit\n", + " if mem[\"rss_mb\"] > 800:\n", + " log.warning(f\"⚠️ HIGH MEMORY [{stem}] stage={stage} RSS={mem['rss_mb']}MB — approaching 1024MB limit!\")\n", + "\n", + " # Ensure src modules are importable on executors\n", + " if _SRC_ROOT not in sys.path:\n", + " sys.path.insert(0, _SRC_ROOT)\n", + "\n", + " import openslide\n", + " from dbx.pixels.svs.deidentify import redact_image, write_pyramidal_bigtiff_streaming\n", + " from dbx.pixels.svs.phi_tags import scrub_image_description\n", + "\n", + " for pdf in iterator:\n", + " results = []\n", + " log.info(f\"Batch received: {len(pdf)} row(s) | PID={os.getpid()}\")\n", + " _log_mem(\"batch_start\", \"batch\", f\"rows={len(pdf)}\")\n", + "\n", + " # Iterate rows directly via iloc — no to_dict(\"records\") memory copy\n", + " for idx in range(len(pdf)):\n", + " row = pdf.iloc[idx]\n", + " svs_path = row[\"local_path\"]\n", + " phi_json = row[\"phi_elements_json\"]\n", + " stem = Path(svs_path).stem\n", + " stage = \"init\"\n", + " slide = None\n", + " tmp_tiff = f\"/tmp/{stem}_{os.getpid()}.tiff\"\n", + " t0 = time.time()\n", + "\n", + " log.info(f\"=== Processing slide: {stem} ===\")\n", + " _log_mem(\"init\", stem, f\"svs_path={svs_path}\")\n", + "\n", + " try:\n", + " phi_elements = json.loads(phi_json) if phi_json else []\n", + " stage = \"open_slide\"\n", + " slide = openslide.OpenSlide(svs_path)\n", + " dims = slide.dimensions # (width, height) at level 0\n", + " levels = slide.level_count\n", + " _log_mem(\"open_slide\", stem, f\"dims={dims[0]}x{dims[1]} levels={levels}\")\n", + "\n", + " # 1. Redact label/macro sub-images (small RGBA → write PNGs to Volume)\n", + " label_path = macro_path = None\n", + " pixel_count = 0\n", + "\n", + " if \"label\" in slide.associated_images:\n", + " stage = \"redact_label\"\n", + " label_img = slide.associated_images[\"label\"]\n", + " label_phi = [e for e in phi_elements if e.get(\"subimage\", \"label\") != \"macro\"]\n", + " label_out = redact_image(label_img, label_phi)\n", + " pixel_count += len(label_phi)\n", + " label_path = f\"{_LABEL_VOLUME}/{stem}_label.png\"\n", + " os.makedirs(os.path.dirname(label_path), exist_ok=True)\n", + " label_out.save(label_path)\n", + " _log_mem(\"redact_label\", stem, f\"label_size={label_img.size} phi_count={len(label_phi)}\")\n", + " del label_img, label_out # free RGBA buffer immediately\n", + "\n", + " if \"macro\" in slide.associated_images:\n", + " stage = \"redact_macro\"\n", + " macro_img = slide.associated_images[\"macro\"]\n", + " macro_phi = [e for e in phi_elements if e.get(\"subimage\") == \"macro\"]\n", + " macro_out = redact_image(macro_img, macro_phi)\n", + " pixel_count += len(macro_phi)\n", + " macro_path = f\"{_LABEL_VOLUME}/{stem}_macro.png\"\n", + " os.makedirs(os.path.dirname(macro_path), exist_ok=True)\n", + " macro_out.save(macro_path)\n", + " _log_mem(\"redact_macro\", stem, f\"macro_size={macro_img.size} phi_count={len(macro_phi)}\")\n", + " del macro_img, macro_out\n", + "\n", + " # 2. Scrub metadata tags\n", + " stage = \"scrub_metadata\"\n", + " raw_desc = slide.properties.get(\"tiff.ImageDescription\", \"\")\n", + " scrubbed = scrub_image_description(raw_desc)\n", + " phi_tags_redacted = (\n", + " [\"tiff.ImageDescription\", \"openslide.comment\"]\n", + " if raw_desc != scrubbed else []\n", + " )\n", + " _log_mem(\"scrub_metadata\", stem)\n", + "\n", + " # 3. Write pyramidal BigTIFF — tile-streaming (256×256 read_region)\n", + " # Stage to /tmp (requires seek), then sequential copy to Volume.\n", + " stage = \"write_tiff_to_tmp\"\n", + " t_tiff_start = time.time()\n", + " write_pyramidal_bigtiff_streaming(tmp_tiff, slide, jpeg_quality=80)\n", + " t_tiff_elapsed = time.time() - t_tiff_start\n", + " tmp_size_mb = os.path.getsize(tmp_tiff) / (1024 * 1024) if os.path.exists(tmp_tiff) else 0\n", + " _log_mem(\"write_tiff_done\", stem, f\"tiff_size={tmp_size_mb:.1f}MB elapsed={t_tiff_elapsed:.1f}s\")\n", + "\n", + " # Close slide BEFORE copy to free C-level handles and mapped memory\n", + " slide.close()\n", + " slide = None\n", + " _log_mem(\"slide_closed\", stem)\n", + "\n", + " stage = \"copy_tiff_to_volume\"\n", + " t_copy_start = time.time()\n", + " tiff_path = f\"{_TIFF_VOLUME}/{stem}.tiff\"\n", + " shutil.copy2(tmp_tiff, tiff_path)\n", + " os.remove(tmp_tiff)\n", + " t_copy_elapsed = time.time() - t_copy_start\n", + " _log_mem(\"copy_done\", stem, f\"copy_elapsed={t_copy_elapsed:.1f}s\")\n", + "\n", + " total_elapsed = time.time() - t0\n", + " log.info(f\"✓ [{stem}] completed in {total_elapsed:.1f}s | tiff={tmp_size_mb:.1f}MB\")\n", + "\n", + " results.append({\n", + " \"path\": row[\"path\"],\n", + " \"tiff_output_path\": tiff_path,\n", + " \"label_image_path\": label_path,\n", + " \"macro_image_path\": macro_path,\n", + " \"phi_tags_redacted\": phi_tags_redacted,\n", + " \"pixel_regions_redacted\": pixel_count,\n", + " \"error\": None,\n", + " })\n", + "\n", + " except Exception as exc:\n", + " _log_mem(\"ERROR\", stem, f\"stage={stage} exc={type(exc).__name__}: {exc}\")\n", + " results.append({\n", + " \"path\": row[\"path\"],\n", + " \"tiff_output_path\": None,\n", + " \"label_image_path\": None,\n", + " \"macro_image_path\": None,\n", + " \"phi_tags_redacted\": [],\n", + " \"pixel_regions_redacted\": 0,\n", + " \"error\": f\"[stage={stage}] {type(exc).__name__}: {exc}\",\n", + " })\n", + "\n", + " finally:\n", + " # Ensure slide is always closed — destroy C-bindings immediately\n", + " if slide is not None:\n", + " try:\n", + " slide.close()\n", + " except Exception:\n", + " pass\n", + " # Clean up temp file on failure\n", + " if os.path.exists(tmp_tiff):\n", + " try:\n", + " os.remove(tmp_tiff)\n", + " except Exception:\n", + " pass\n", + " # Force GC to reclaim PIL/OpenSlide C-level allocations\n", + " gc.collect()\n", + " _log_mem(\"gc_complete\", stem)\n", + "\n", + " yield pd.DataFrame(results)" + ] + }, + { + "cell_type": "code", + "execution_count": 0, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": { + "byteLimit": 2048000, + "rowLimit": 10000 + }, + "inputWidgets": {}, + "nuid": "b449d29c-0651-41bc-afc8-bf83bdcf7f74", + "showTitle": false, + "tableResultSettingsMap": {}, + "title": "" + } + }, + "outputs": [], + "source": [ + "# Materialize the expensive mapInPandas UDF exactly ONCE.\n", + "# Strategy: persist() + count() forces a single execution pass.\n", + "# Downstream MERGE reads from the cached DataFrame via temp view.\n", + "TIFF_RESULTS_TABLE = f\"{CATALOG}.{SCHEMA}.tiff_results_staging\"\n", + "\n", + "num_slides = redaction_df.count()\n", + "print(f\"\"\"{num_slides}\"\"\")\n", + "assert num_slides > 0, \"No PENDING slides to de-identify — check object_catalog_redaction status\"\n" + ] + }, + { + "cell_type": "code", + "execution_count": 0, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": { + "byteLimit": 2048000, + "rowLimit": 10000 + }, + "inputWidgets": {}, + "nuid": "2581617a-cb08-4d46-b604-56b0c4d0400e", + "showTitle": true, + "tableResultSettingsMap": {}, + "title": "Dry run: 1 slide with memory logging" + } + }, + "outputs": [], + "source": [ + "# --- Dry run: 1 slide on DRIVER to capture full memory trace ---\n", + "# Runs the same logic outside mapInPandas so we can see exactly which stage\n", + "# exceeds 1024 MB without the executor being killed.\n", + "\n", + "import sys, os, gc, time, resource, json, shutil\n", + "from pathlib import Path\n", + "\n", + "if \"/Workspace/Users/douglas.moore@databricks.com/pixels-svs/src\" not in sys.path:\n", + " sys.path.insert(0, \"/Workspace/Users/douglas.moore@databricks.com/pixels-svs/src\")\n", + "\n", + "import openslide\n", + "from dbx.pixels.svs.deidentify import redact_image, write_pyramidal_bigtiff_streaming\n", + "from dbx.pixels.svs.phi_tags import scrub_image_description\n", + "\n", + "def _mem_mb():\n", + " \"\"\"RSS/VMS/Peak from /proc/self/status.\"\"\"\n", + " try:\n", + " with open(\"/proc/self/status\") as f:\n", + " status = f.read()\n", + " vals = {}\n", + " for line in status.splitlines():\n", + " for key in (\"VmPeak\", \"VmRSS\", \"VmSize\"):\n", + " if line.startswith(key + \":\"):\n", + " vals[key] = int(line.split()[1]) / 1024 # KB→MB\n", + " return {\"rss_mb\": round(vals.get(\"VmRSS\", 0), 1),\n", + " \"vms_mb\": round(vals.get(\"VmSize\", 0), 1),\n", + " \"peak_mb\": round(vals.get(\"VmPeak\", 0), 1)}\n", + " except Exception:\n", + " return {\"rss_mb\": -1, \"vms_mb\": -1, \"peak_mb\": -1}\n", + "\n", + "def log_mem(stage, extra=\"\"):\n", + " mem = _mem_mb()\n", + " warn = \" ⚠️ OVER 1GB!\" if mem[\"rss_mb\"] > 1024 else (\"⚠️ HIGH\" if mem[\"rss_mb\"] > 800 else \"\")\n", + " print(f\" [{stage:20s}] RSS={mem['rss_mb']:>7.1f}MB VMS={mem['vms_mb']:>7.1f}MB Peak={mem['peak_mb']:>7.1f}MB {warn} {extra}\")\n", + "\n", + "# Get one slide from the redaction dataframe\n", + "row = redaction_df.limit(1).collect()[0]\n", + "svs_path = row[\"local_path\"]\n", + "phi_json = row[\"phi_elements_json\"]\n", + "stem = Path(svs_path).stem\n", + "tmp_tiff = f\"/tmp/{stem}_dryrun.tiff\"\n", + "\n", + "print(f\"\\n{'='*80}\")\n", + "print(f\"DRY RUN MEMORY PROFILE: {stem}\")\n", + "print(f\"SVS path: {svs_path}\")\n", + "print(f\"{'='*80}\")\n", + "\n", + "gc.collect()\n", + "log_mem(\"baseline\")\n", + "\n", + "# Open slide\n", + "t0 = time.time()\n", + "slide = openslide.OpenSlide(svs_path)\n", + "dims = slide.dimensions\n", + "print(f\"\\n Slide: {dims[0]}x{dims[1]} pixels, {slide.level_count} levels\")\n", + "print(f\" Level dimensions: {[slide.level_dimensions[i] for i in range(slide.level_count)]}\")\n", + "print(f\" Associated images: {list(slide.associated_images.keys())}\")\n", + "log_mem(\"open_slide\", f\"file_size={os.path.getsize(svs_path)/(1024*1024):.1f}MB\")\n", + "\n", + "# Redact label\n", + "if \"label\" in slide.associated_images:\n", + " label_img = slide.associated_images[\"label\"]\n", + " print(f\"\\n Label image: {label_img.size} mode={label_img.mode}\")\n", + " phi_elements = json.loads(phi_json) if phi_json else []\n", + " label_phi = [e for e in phi_elements if e.get(\"subimage\", \"label\") != \"macro\"]\n", + " label_out = redact_image(label_img, label_phi)\n", + " log_mem(\"redact_label\", f\"phi_regions={len(label_phi)}\")\n", + " del label_img, label_out\n", + " gc.collect()\n", + " log_mem(\"label_freed\")\n", + "\n", + "# Redact macro\n", + "if \"macro\" in slide.associated_images:\n", + " macro_img = slide.associated_images[\"macro\"]\n", + " print(f\"\\n Macro image: {macro_img.size} mode={macro_img.mode}\")\n", + " phi_elements = json.loads(phi_json) if phi_json else []\n", + " macro_phi = [e for e in phi_elements if e.get(\"subimage\") == \"macro\"]\n", + " macro_out = redact_image(macro_img, macro_phi)\n", + " log_mem(\"redact_macro\", f\"phi_regions={len(macro_phi)}\")\n", + " del macro_img, macro_out\n", + " gc.collect()\n", + " log_mem(\"macro_freed\")\n", + "\n", + "# Scrub metadata\n", + "raw_desc = slide.properties.get(\"tiff.ImageDescription\", \"\")\n", + "scrubbed = scrub_image_description(raw_desc)\n", + "log_mem(\"scrub_metadata\")\n", + "\n", + "# Write pyramidal BigTIFF (this is the suspected memory hog)\n", + "print(f\"\\n Writing pyramidal BigTIFF to /tmp ...\")\n", + "t_tiff = time.time()\n", + "try:\n", + " write_pyramidal_bigtiff_streaming(tmp_tiff, slide, jpeg_quality=80)\n", + " tiff_elapsed = time.time() - t_tiff\n", + " tiff_size = os.path.getsize(tmp_tiff) / (1024 * 1024)\n", + " log_mem(\"write_tiff_done\", f\"size={tiff_size:.1f}MB elapsed={tiff_elapsed:.1f}s\")\n", + "except Exception as e:\n", + " log_mem(\"write_tiff_FAILED\", f\"{type(e).__name__}: {e}\")\n", + " tiff_size = 0\n", + "\n", + "# Close slide\n", + "slide.close()\n", + "log_mem(\"slide_closed\")\n", + "gc.collect()\n", + "log_mem(\"gc_after_close\")\n", + "\n", + "# Cleanup\n", + "if os.path.exists(tmp_tiff):\n", + " os.remove(tmp_tiff)\n", + "\n", + "total = time.time() - t0\n", + "print(f\"\\n{'='*80}\")\n", + "print(f\"COMPLETE: {stem} in {total:.1f}s | TIFF={tiff_size:.1f}MB\")\n", + "print(f\"Peak memory: {_mem_mb()['peak_mb']:.1f}MB\")\n", + "print(f\"{'='*80}\")\n", + "if _mem_mb()[\"peak_mb\"] > 1024:\n", + " print(\"\\n❌ Peak memory EXCEEDED 1024MB — this will OOM on Serverless executors.\")\n", + " print(\" → Investigate write_pyramidal_bigtiff_streaming tile buffer size.\")\n", + "else:\n", + " print(\"\\n✅ Peak memory stayed under 1024MB — safe for Serverless executors.\")" + ] + }, + { + "cell_type": "code", + "execution_count": 0, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": { + "byteLimit": 2048000, + "rowLimit": 10000 + }, + "inputWidgets": {}, + "nuid": "a8e83dad-0127-488c-b894-5d442508e404", + "showTitle": true, + "tableResultSettingsMap": {}, + "title": "Cell 26: Execute TIFF write (materialize once)" + } + }, + "outputs": [], + "source": [ + "\n", + "# 1 slide per partition to stay under 1GB UDF memory limit on serverless.\n", + "# Large SVS files (CMU-1 = 46000x32000) need sole access to executor RAM.\n", + "results_df = (\n", + " redaction_df\n", + " .repartition(num_slides)\n", + " .mapInPandas(_deidentify_batch, schema=_result_schema)\n", + " .limit(4)\n", + ")\n", + "\n", + "# Force single execution — UDF runs here and only here\n", + "results_df.select(\"path\", \"tiff_output_path\", \"label_image_path\", \"macro_image_path\", \"phi_tags_redacted\", \"pixel_regions_redacted\", \"error\").write.saveAsTable(TIFF_RESULTS_TABLE)\n", + "\n", + "\n", + "display(spark.sql(f\"SELECT * FROM {TIFF_RESULTS_TABLE}\"))" + ] + }, + { + "cell_type": "code", + "execution_count": 0, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": { + "byteLimit": 2048000, + "rowLimit": 10000 + }, + "inputWidgets": {}, + "nuid": "5c7b3af8-ded5-4207-92d1-42a0710e11e6", + "showTitle": true, + "tableResultSettingsMap": {}, + "title": "Cell 23" + } + }, + "outputs": [], + "source": [ + "# Merge output paths and status back into object_catalog_redaction\n", + "spark.sql(f\"\"\"\n", + "MERGE INTO {CATALOG}.{SCHEMA}.object_catalog_redaction AS tgt\n", + "USING (\n", + " SELECT * FROM (\n", + " SELECT *, ROW_NUMBER() OVER (PARTITION BY path ORDER BY path) AS rn\n", + " FROM tiff_results\n", + " ) WHERE rn = 1\n", + ") AS src\n", + " ON tgt.path = src.path\n", + "WHEN MATCHED THEN UPDATE SET\n", + " tgt.output_file_paths = array(src.tiff_output_path),\n", + " tgt.label_image_path = COALESCE(src.label_image_path, tgt.label_image_path),\n", + " tgt.macro_image_path = COALESCE(src.macro_image_path, tgt.macro_image_path),\n", + " tgt.phi_tags_redacted = src.phi_tags_redacted,\n", + " tgt.pixel_redactions_count = src.pixel_regions_redacted,\n", + " tgt.status = CASE WHEN src.error IS NULL THEN 'SUCCESS' ELSE 'FAILED' END,\n", + " tgt.error_messages = CASE WHEN src.error IS NOT NULL THEN array(src.error) ELSE NULL END,\n", + " tgt.update_timestamp = current_timestamp()\n", + "\"\"\")\n", + "print(\"TIFF write complete. Redaction records updated.\")" + ] + }, + { + "cell_type": "code", + "execution_count": 0, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": { + "byteLimit": 2048000, + "rowLimit": 10000 + }, + "inputWidgets": {}, + "nuid": "4ec2aac1-43ab-42ee-aae7-0fdf52c006ec", + "showTitle": true, + "tableResultSettingsMap": {}, + "title": "Cell 11: Audit summary" + } + }, + "outputs": [], + "source": [ + "# End-to-end audit: join object_catalog with object_catalog_redaction and summarise.\n", + "audit_df = spark.sql(f\"\"\"\n", + "SELECT\n", + " regexp_extract(o.path, '[^/]+$', 0) AS filename,\n", + " o.meta:width::int AS width_px,\n", + " o.meta:height::int AS height_px,\n", + " o.meta:level_count::int AS pyramid_levels,\n", + " r.has_phi,\n", + " r.status,\n", + " r.pixel_redactions_count,\n", + " size(r.phi_tags_redacted) AS tag_redactions,\n", + " r.output_file_paths[0] AS tiff_output_path,\n", + " r.label_image_path,\n", + " r.error_messages[0] AS error\n", + "FROM {CATALOG}.{SCHEMA}.object_catalog o\n", + "LEFT JOIN {CATALOG}.{SCHEMA}.object_catalog_redaction r\n", + " ON o.path = r.path\n", + "ORDER BY filename\n", + "\"\"\")\n", + "\n", + "total = audit_df.count()\n", + "phi_ct = audit_df.filter(\"has_phi = true\").count()\n", + "ok_ct = audit_df.filter(\"status = 'SUCCESS'\").count()\n", + "err_ct = audit_df.filter(\"status = 'FAILED'\").count()\n", + "\n", + "print(f\"Slides in catalog : {total}\")\n", + "print(f\"VLM-flagged with PHI : {phi_ct}\")\n", + "print(f\"Successfully written : {ok_ct}\")\n", + "print(f\"Errors : {err_ct}\")\n", + "\n", + "display(audit_df)\n" + ] + } + ], + "metadata": { + "application/vnd.databricks.v1+notebook": { + "computePreferences": null, + "dashboards": [], + "environmentMetadata": { + "base_environment": "", + "environment_version": "5" + }, + "inputWidgetPreferences": null, + "language": "python", + "notebookMetadata": { + "mostRecentlyExecutedCommandWithImplicitDF": { + "commandId": 8994411946469750, + "dataframes": [ + "_sqldf" + ] + }, + "pythonIndentUnit": 2 + }, + "notebookName": "TIFF Pathology De-identification Pipeline", + "widgets": {} + }, + "language_info": { + "name": "python" + } + }, + "nbformat": 4, + "nbformat_minor": 0 +} diff --git a/notebooks/tiff/TIFF sample data.ipynb b/notebooks/tiff/TIFF sample data.ipynb new file mode 100644 index 00000000..c2973c4f --- /dev/null +++ b/notebooks/tiff/TIFF sample data.ipynb @@ -0,0 +1,980 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 0, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": { + "byteLimit": 2048000, + "rowLimit": 10000 + }, + "inputWidgets": {}, + "nuid": "86734738-8782-422b-9f72-1620460e8964", + "showTitle": true, + "tableResultSettingsMap": {}, + "title": "Install dependencies" + } + }, + "outputs": [], + "source": [ + "%pip install openslide-python openslide-bin tifffile imagecodecs -q" + ] + }, + { + "cell_type": "code", + "execution_count": 0, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": { + "byteLimit": 2048000, + "rowLimit": 10000 + }, + "inputWidgets": {}, + "nuid": "153c85b3-e046-4fda-b9e3-e54aca9c45a0", + "showTitle": true, + "tableResultSettingsMap": {}, + "title": "Configure" + } + }, + "outputs": [], + "source": [ + "# ── Shared utilities — run after pip install, before any other cell ──────────\n", + "import glob, os\n", + "import openslide\n", + "from PIL import Image\n", + "import matplotlib.pyplot as plt\n", + "\n", + "Image.MAX_IMAGE_PIXELS = None # safe — we read metadata or small sub-images only\n", + "\n", + "SOURCE = \"/Volumes/hls_pathology/osuwmc/sample\"\n", + "EXT = \".tiff\"\n", + "\n", + "\n", + "def discover_tiff_files(source=SOURCE, ext=EXT, synthetic=True):\n", + " \"\"\"Return sorted list of TIFF paths under *source*.\n", + " Pass synthetic=False to exclude files whose name contains '__synthetic'.\n", + " \"\"\"\n", + " files = sorted(set(\n", + " glob.glob(f\"{source}/**/*{ext}\", recursive=True) +\n", + " glob.glob(f\"{source}/*{ext}\")\n", + " ))\n", + " if not synthetic:\n", + " files = [f for f in files if \"__synthetic\" not in f]\n", + " return files\n", + "\n", + "\n", + "def to_rgb(img):\n", + " \"\"\"Flatten RGBA / palette PIL images to RGB on a white background.\"\"\"\n", + " if img.mode == \"RGBA\":\n", + " bg = Image.new(\"RGB\", img.size, (255, 255, 255))\n", + " bg.paste(img, mask=img.split()[3])\n", + " return bg\n", + " return img.convert(\"RGB\")\n", + "\n", + "\n", + "def fit_to(img, max_w=800, max_h=600):\n", + " \"\"\"Downsample *img* to fit within (max_w, max_h), preserving aspect ratio.\n", + " Never upscales.\"\"\"\n", + " w, h = img.size\n", + " scale = min(max_w / w, max_h / h, 1.0)\n", + " if scale < 1.0:\n", + " return img.resize((int(w * scale), int(h * scale)), Image.LANCZOS)\n", + " return img\n", + "\n", + "\n", + "def show_slide_layers(fpath, thumb_w=800, thumb_h=600, max_native=4_000_000):\n", + " \"\"\"Display all pyramid levels + associated images of a slide as a panel row.\"\"\"\n", + " fname = os.path.basename(fpath)\n", + " try:\n", + " slide = openslide.OpenSlide(fpath)\n", + " except Exception as exc:\n", + " print(f\" \\u26a0 Cannot open {fname}: {exc}\")\n", + " return\n", + " layers = []\n", + " mw, mh = slide.dimensions\n", + " layers.append({\n", + " \"title\": f\"main\\n{mw:,} \\u00d7 {mh:,} px\",\n", + " \"img\": slide.get_thumbnail((thumb_w, thumb_h)),\n", + " \"orig\": (mw, mh),\n", + " })\n", + " for lvl in range(1, slide.level_count):\n", + " lw, lh = slide.level_dimensions[lvl]\n", + " ds = slide.level_downsamples[lvl]\n", + " if lw * lh <= max_native:\n", + " img = slide.read_region((0, 0), lvl, (lw, lh))\n", + " else:\n", + " scale = min(thumb_w / lw, thumb_h / lh)\n", + " img = slide.get_thumbnail((int(lw * scale), int(lh * scale)))\n", + " layers.append({\n", + " \"title\": f\"level {lvl} (\\u00d7{ds:.0f}\\u2193)\\n{lw:,} \\u00d7 {lh:,} px\",\n", + " \"img\": img,\n", + " \"orig\": (lw, lh),\n", + " })\n", + " for name in sorted(slide.associated_images.keys()):\n", + " img = slide.associated_images[name]\n", + " aw, ah = img.size\n", + " layers.append({\"title\": f\"{name}\\n{aw} \\u00d7 {ah} px\", \"img\": img, \"orig\": (aw, ah)})\n", + " slide.close()\n", + " n = len(layers)\n", + " fig, axes = plt.subplots(1, n, figsize=(min(5 * n, 24), 4.5),\n", + " gridspec_kw={\"wspace\": 0.06})\n", + " if n == 1:\n", + " axes = [axes]\n", + " for ax, layer in zip(axes, layers):\n", + " disp = to_rgb(fit_to(layer[\"img\"], thumb_w, thumb_h))\n", + " dw, dh = disp.size\n", + " ax.imshow(disp)\n", + " ax.set_title(layer[\"title\"], fontsize=8, fontweight=\"bold\", pad=3, linespacing=1.4)\n", + " ax.set_xlabel(f\"displayed {dw}\\u00d7{dh}\", fontsize=7, labelpad=2)\n", + " ax.tick_params(left=False, bottom=False, labelleft=False, labelbottom=False)\n", + " for spine in ax.spines.values():\n", + " spine.set_linewidth(0.5)\n", + " fig.suptitle(fname, fontsize=10, fontweight=\"bold\", y=1.02)\n", + " plt.tight_layout()\n", + " plt.show()\n", + "\n", + "\n", + "def print_slide_metadata(fpath):\n", + " \"\"\"Print pyramid levels, associated images, and all openslide properties.\"\"\"\n", + " slide = openslide.OpenSlide(fpath)\n", + " print(f\"\\n Pyramid levels : {slide.level_count}\")\n", + " for lvl in range(slide.level_count):\n", + " lw, lh = slide.level_dimensions[lvl]\n", + " print(f\" [{lvl}] {lw:,} \\u00d7 {lh:,} px \"\n", + " f\"(downsample \\u00d7{slide.level_downsamples[lvl]:.2f})\")\n", + " assoc = sorted(slide.associated_images.keys())\n", + " print(f\"\\n Associated images : {assoc if assoc else '\\u2014'}\")\n", + " print(f\"\\n Properties ({len(slide.properties)}):\")\n", + " for k, v in sorted(slide.properties.items()):\n", + " print(f\" {k} = {v}\")\n", + " slide.close()\n", + "\n", + "\n", + "# Discover on load so tiff_files is available to all downstream cells\n", + "tiff_files = discover_tiff_files()\n", + "print(f\"Utilities loaded. {len(tiff_files)} {EXT} file(s) under {SOURCE}\")" + ] + }, + { + "cell_type": "code", + "execution_count": 0, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": { + "byteLimit": 2048000, + "rowLimit": 10000 + }, + "inputWidgets": {}, + "nuid": "e44640d5-5d8f-4e41-8afd-15e704984ab3", + "showTitle": true, + "tableResultSettingsMap": {}, + "title": "Display all TIFF layers" + } + }, + "outputs": [], + "source": [ + "# Uses show_slide_layers + print_slide_metadata from the Shared utilities cell.\n", + "# Shows original source TIFFs only; synthetic variants are shown in cell 8.\n", + "for fpath in discover_tiff_files(synthetic=False):\n", + " print(f\"{'─' * 72}\\n{os.path.basename(fpath)}\")\n", + " show_slide_layers(fpath)\n", + " print_slide_metadata(fpath)\n", + " print()\n" + ] + }, + { + "cell_type": "code", + "execution_count": 0, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": { + "byteLimit": 2048000, + "rowLimit": 10000 + }, + "inputWidgets": {}, + "nuid": "87ad871a-069e-47f2-b353-5e4698dd4892", + "showTitle": true, + "tableResultSettingsMap": {}, + "title": "Extract & save JPEGs — Philips07" + } + }, + "outputs": [], + "source": [ + "# 3rd TIFF file — display each layer large and save JPEGs back to the same volume.\n", + "import os\n", + "import openslide\n", + "from PIL import Image\n", + "import matplotlib.pyplot as plt\n", + "\n", + "TARGET_FILE_IDX = 2 # Philips07\n", + "MAX_NATIVE_DIM = 10_000 # read full level if max(w,h) ≤ this; else thumbnail\n", + "JPEG_THUMB_W = 4096 # max width when level must be thumbnailed\n", + "JPEG_QUALITY = 90\n", + "\n", + "\n", + "fpath = tiff_files[TARGET_FILE_IDX]\n", + "fname = os.path.basename(fpath)\n", + "fdir = os.path.dirname(fpath)\n", + "stem = os.path.splitext(fname)[0]\n", + "print(f\"Source : {fpath}\")\n", + "print(f\"Output : {fdir}\")\n", + "print(f\"Stem : {stem}\\n\")\n", + "\n", + "slide = openslide.OpenSlide(fpath)\n", + "\n", + "# ── collect layers ────────────────────────────────────────────────────────────\n", + "layers = {}\n", + "\n", + "# main: high-res thumbnail from full pyramid\n", + "mw, mh = slide.dimensions\n", + "scale_main = JPEG_THUMB_W / max(mw, mh)\n", + "layers[\"main\"] = {\n", + " \"img\" : slide.get_thumbnail((int(mw * scale_main), int(mh * scale_main))),\n", + " \"orig\" : (mw, mh),\n", + " \"label\": f\"main (level 0)\\nfull: {mw:,} × {mh:,} px\",\n", + "}\n", + "\n", + "# pyramid levels 1 – N\n", + "for lvl in range(1, slide.level_count):\n", + " lw, lh = slide.level_dimensions[lvl]\n", + " ds = slide.level_downsamples[lvl]\n", + " key = f\"level_{lvl}\"\n", + " if max(lw, lh) <= MAX_NATIVE_DIM: # safe to read entirely\n", + " img = slide.read_region((0, 0), lvl, (lw, lh))\n", + " else: # too large — thumbnail\n", + " sc = JPEG_THUMB_W / max(lw, lh)\n", + " img = slide.get_thumbnail((int(lw * sc), int(lh * sc)))\n", + " layers[key] = {\n", + " \"img\" : img,\n", + " \"orig\" : (lw, lh),\n", + " \"label\": f\"level {lvl} (×{ds:.0f}↓)\\n{lw:,} × {lh:,} px\",\n", + " }\n", + "\n", + "# associated sub-images (label / macro / thumbnail, if present)\n", + "for name in sorted(slide.associated_images.keys()):\n", + " img = slide.associated_images[name]\n", + " aw, ah = img.size\n", + " layers[f\"assoc_{name}\"] = {\n", + " \"img\" : img,\n", + " \"orig\" : (aw, ah),\n", + " \"label\": f\"{name}\\n{aw} × {ah} px\",\n", + " }\n", + "\n", + "slide.close()\n", + "\n", + "# ── display: one large figure per layer ──────────────────────────────────────\n", + "for key, layer in layers.items():\n", + " rgb = to_rgb(layer[\"img\"])\n", + " w, h = rgb.size\n", + " fig_w = 14\n", + " fig_h = fig_w * h / w\n", + " fig, ax = plt.subplots(figsize=(fig_w, fig_h))\n", + " ax.imshow(rgb)\n", + " ax.set_title(\n", + " f\"{fname} — {layer['label']}\",\n", + " fontsize=11, fontweight=\"bold\", pad=6, linespacing=1.5,\n", + " )\n", + " ax.set_xlabel(\n", + " f\"extracted at {w:,} × {h:,} px | original: {layer['orig'][0]:,} × {layer['orig'][1]:,} px\",\n", + " fontsize=9,\n", + " )\n", + " ax.tick_params(left=False, bottom=False, labelleft=False, labelbottom=False)\n", + " plt.tight_layout()\n", + " plt.show()\n", + "\n", + "# ── save JPEGs ────────────────────────────────────────────────────────────────\n", + "print(f\"\\nSaving JPEGs to {fdir}\\n\")\n", + "for key, layer in layers.items():\n", + " rgb = to_rgb(layer[\"img\"])\n", + " jpg_name = f\"{stem}__{key}.jpg\" # e.g. Philips07_3b946eed-...__level_3.jpg\n", + " jpg_path = os.path.join(fdir, jpg_name)\n", + " rgb.save(jpg_path, \"JPEG\", quality=JPEG_QUALITY)\n", + " ow, oh = layer[\"orig\"]\n", + " jw, jh = rgb.size\n", + " print(f\" {jpg_name}\")\n", + " print(f\" orig {ow:,}×{oh:,} → saved {jw:,}×{jh:,} ({jpg_path})\")\n", + "\n", + "print(f\"\\n{len(layers)} JPEG(s) written.\")\n" + ] + }, + { + "cell_type": "code", + "execution_count": 0, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": { + "byteLimit": 2048000, + "rowLimit": 10000 + }, + "inputWidgets": {}, + "nuid": "5c2565bb-fe30-412a-aa27-1554fb800a1d", + "showTitle": true, + "tableResultSettingsMap": {}, + "title": "Blood vessel zoom — Philips07 level 6" + } + }, + "outputs": [], + "source": [ + "import os\n", + "import openslide\n", + "from PIL import Image\n", + "import matplotlib.pyplot as plt\n", + "import matplotlib.patches as patches\n", + "\n", + "fpath = tiff_files[2] # Philips07\n", + "fname = os.path.basename(fpath)\n", + "slide = openslide.OpenSlide(fpath)\n", + "\n", + "# ── blood vessel location estimated from level-6 visual (1,440 × 800 px) ─────\n", + "# Bright pink/magenta spot, lower-centre of tissue. Adjust if needed.\n", + "VESSEL_X_L6 = 620 # centre x in level-6 pixels\n", + "VESSEL_Y_L6 = 540 # centre y in level-6 pixels\n", + "CROP_R_L6 = 160 # half-side of crop square in level-6 pixels\n", + "\n", + "DS = {lvl: int(slide.level_downsamples[lvl]) for lvl in range(slide.level_count)}\n", + "\n", + "# ── crop box in level-6 coordinates ─────────────────────────────────────────\n", + "l6_x0 = max(0, VESSEL_X_L6 - CROP_R_L6)\n", + "l6_y0 = max(0, VESSEL_Y_L6 - CROP_R_L6)\n", + "l6_w = min(CROP_R_L6 * 2, slide.level_dimensions[6][0] - l6_x0)\n", + "l6_h = min(CROP_R_L6 * 2, slide.level_dimensions[6][1] - l6_y0)\n", + "\n", + "# level-0 origin used by read_region for every pyramid level\n", + "loc0 = (l6_x0 * DS[6], l6_y0 * DS[6])\n", + "\n", + "# ── read four views of the same region ─────────────────────────────────────────\n", + "lvl6_full = slide.read_region((0, 0), 6, slide.level_dimensions[6]).convert(\"RGB\")\n", + "lvl6_crop = slide.read_region(loc0, 6, (l6_w, l6_h)).convert(\"RGB\")\n", + "\n", + "# level 4 (×16 downsample) — 4× more detail than level 6\n", + "l4_w = l6_w * DS[6] // DS[4]\n", + "l4_h = l6_h * DS[6] // DS[4]\n", + "lvl4_crop = slide.read_region(loc0, 4, (l4_w, l4_h)).convert(\"RGB\")\n", + "\n", + "# level 2 (×4 downsample) — 16× more detail than level 6\n", + "l2_w = l6_w * DS[6] // DS[2]\n", + "l2_h = l6_h * DS[6] // DS[2]\n", + "lvl2_crop = slide.read_region(loc0, 2, (l2_w, l2_h)).convert(\"RGB\")\n", + "\n", + "slide.close()\n", + "\n", + "# ── display ──────────────────────────────────────────────────────────────────\n", + "fig = plt.figure(figsize=(22, 7))\n", + "gs = fig.add_gridspec(1, 4, wspace=0.05)\n", + "axes = [fig.add_subplot(gs[i]) for i in range(4)]\n", + "\n", + "# Panel 1: level-6 overview with yellow crop-box annotation\n", + "axes[0].imshow(lvl6_full)\n", + "axes[0].add_patch(patches.Rectangle(\n", + " (l6_x0, l6_y0), l6_w, l6_h,\n", + " linewidth=2.5, edgecolor=\"yellow\", facecolor=\"none\",\n", + "))\n", + "axes[0].set_title(\"Level 6 — overview\\n1,440 × 800 px (×64↓)\",\n", + " fontsize=9, fontweight=\"bold\", pad=4)\n", + "axes[0].set_xlabel(\"yellow = crop region\", fontsize=7)\n", + "\n", + "# Panels 2-4: progressive zoom\n", + "for ax, img, lvl, label in [\n", + " (axes[1], lvl6_crop, 6, \"Level 6 crop\"),\n", + " (axes[2], lvl4_crop, 4, \"Level 4 — ×4 zoom\"),\n", + " (axes[3], lvl2_crop, 2, \"Level 2 — ×16 zoom\"),\n", + "]:\n", + " w, h = img.size\n", + " ax.imshow(img)\n", + " ax.set_title(\n", + " f\"{label}\\n{w:,} × {h:,} px (×{DS[lvl]}↓)\",\n", + " fontsize=9, fontweight=\"bold\", pad=4,\n", + " )\n", + " ax.set_xlabel(f\"origin L0: ({loc0[0]:,}, {loc0[1]:,})\", fontsize=7)\n", + "\n", + "for ax in axes:\n", + " ax.tick_params(left=False, bottom=False, labelleft=False, labelbottom=False)\n", + "\n", + "fig.suptitle(f\"{fname} — blood vessel zoom\",\n", + " fontsize=11, fontweight=\"bold\", y=1.01)\n", + "plt.tight_layout()\n", + "plt.show()\n", + "\n", + "print(f\"Level-0 origin : {loc0}\")\n", + "print(f\"Level 6 crop : {l6_w} × {l6_h} px\")\n", + "print(f\"Level 4 crop : {l4_w} × {l4_h} px (×4 more detail)\")\n", + "print(f\"Level 2 crop : {l2_w} × {l2_h} px (×16 more detail)\")\n" + ] + }, + { + "cell_type": "code", + "execution_count": 0, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": { + "byteLimit": 2048000, + "rowLimit": 10000 + }, + "inputWidgets": {}, + "nuid": "e4be9e45-e76a-4d51-887d-449ecf33df5f", + "showTitle": true, + "tableResultSettingsMap": {}, + "title": "Enumerate raw TIFF tags — all files" + } + }, + "outputs": [], + "source": [ + "from PIL.TiffTags import TAGS # to_rgb, discover_tiff_files, tiff_files from Shared utilities\n", + "\n", + "# Tags that could carry PHI\n", + "PHI_CANDIDATES = {\n", + " 270: \"ImageDescription\",\n", + " 305: \"Software\",\n", + " 315: \"Artist\",\n", + " 316: \"HostComputer\",\n", + " 33432: \"Copyright\",\n", + " 37510: \"UserComment\",\n", + " 40092: \"XPComment\",\n", + " 40094: \"XPKeywords\",\n", + " 40095: \"XPSubject\",\n", + "}\n", + "\n", + "for fpath in tiff_files:\n", + " fname = os.path.basename(fpath)\n", + " print(f\"\\n{'\\u2550' * 72}\")\n", + " print(f\"{fname}\")\n", + " print(f\"{'\\u2500' * 72}\")\n", + "\n", + " try:\n", + " img = Image.open(fpath)\n", + " n_frames = getattr(img, \"n_frames\", 1)\n", + " print(f\" IFDs (frames): {n_frames}\")\n", + "\n", + " # ── iterate all IFDs, deduplicate by tag-code signature ─────────────\n", + " sig_map = {}\n", + " for fi in range(n_frames):\n", + " try:\n", + " img.seek(fi)\n", + " except EOFError:\n", + " break\n", + " tags = dict(getattr(img, \"tag_v2\", {}))\n", + " sig = tuple(sorted(tags.keys()))\n", + " if sig not in sig_map:\n", + " sig_map[sig] = {\"frames\": [], \"sample\": tags}\n", + " sig_map[sig][\"frames\"].append(fi)\n", + "\n", + " # ── also expose SubIFDs (tag 330) from IFD 0 if present ─────────\n", + " img.seek(0)\n", + " subifd_tag = getattr(img, \"tag_v2\", {}).get(330)\n", + " if subifd_tag:\n", + " print(f\" SubIFDs (tag 330): {subifd_tag}\")\n", + "\n", + " img.close()\n", + "\n", + " # ── print each unique tag group ──────────────────────────────\n", + " for sig, info in sig_map.items():\n", + " frames = info[\"frames\"]\n", + " if len(frames) == 1:\n", + " f_label = f\"IFD {frames[0]}\"\n", + " elif frames == list(range(frames[0], frames[-1] + 1)):\n", + " f_label = f\"IFD {frames[0]}\\u2013{frames[-1]} ({len(frames)} frames)\"\n", + " else:\n", + " f_label = f\"{len(frames)} IFDs (non-contiguous)\"\n", + "\n", + " tags = info[\"sample\"]\n", + " print(f\"\\n \\u2500\\u2500 {f_label} ({len(sig)} tags) \\u2500\\u2500\")\n", + "\n", + " for code in sorted(tags.keys()):\n", + " name = TAGS.get(code, f\"Unknown_{code}\")\n", + " val = tags[code]\n", + "\n", + " if isinstance(val, bytes):\n", + " try:\n", + " val_str = val.decode(\"utf-8\", errors=\"replace\").strip()\n", + " except Exception:\n", + " val_str = f\"\"\n", + " elif isinstance(val, (tuple, list)) and len(val) > 8:\n", + " val_str = f\"{type(val).__name__}[{len(val)}] {repr(val[:4])} \\u2026\"\n", + " else:\n", + " val_str = repr(val)\n", + "\n", + " if len(val_str) > 300:\n", + " val_str = val_str[:300] + \" \\u2026\"\n", + "\n", + " phi = \" \\u26a0\\ufe0f PHI?\" if code in PHI_CANDIDATES else \"\"\n", + " print(f\" {code:6d} {name:<40s} {val_str}{phi}\")\n", + "\n", + " except Exception as exc:\n", + " print(f\" ERROR: {exc}\")\n", + "\n", + "print(\"\\nDone.\")\n" + ] + }, + { + "cell_type": "code", + "execution_count": 0, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": { + "byteLimit": 2048000, + "rowLimit": 10000 + }, + "inputWidgets": {}, + "nuid": "7ae34572-ed5a-4e29-a6f0-928ae7a064ac", + "showTitle": true, + "tableResultSettingsMap": {}, + "title": "Inject synthetic PHI — create test TIFFs" + } + }, + "outputs": [], + "source": [ + "# Writes one *__synthetic_phi.tiff per source file, containing:\n", + "# - Aperio-style ImageDescription (tag 270) with fake patient PHI\n", + "# - Software / Artist / HostComputer tags\n", + "# - Rendered label sub-image with visible PHI text + barcode\n", + "# - Macro sub-image (tissue thumbnail)\n", + "# Pyramid is built from source levels 5-8 (max 2,880 x 1,600) for speed.\n", + "# NOTE: tifffile requires random-access seeks; UC volumes don't support them.\n", + "# Strategy: write to /tmp, then shutil.copy2 to the volume.\n", + "\n", + "import random, shutil, tempfile\n", + "import tifffile\n", + "import numpy as np\n", + "from PIL import ImageDraw # to_rgb, discover_tiff_files, tiff_files from Shared utilities\n", + "\n", + "# ── synthetic patient records ────────────────────────────────────────────────\n", + "SYNTHETIC_PATIENTS = [\n", + " {\"Patient\": \"Smith, John A\", \"DOB\": \"1965-03-22\", \"MRN\": \"87654321\",\n", + " \"AccessionNumber\": \"ACC-2023-001\", \"Clinic\": \"Oncology\",\n", + " \"Pathologist\": \"Dr. Jane Doe\", \"User\": \"jdoe\"},\n", + " {\"Patient\": \"Johnson, Mary B\", \"DOB\": \"1978-11-15\", \"MRN\": \"12345678\",\n", + " \"AccessionNumber\": \"ACC-2023-002\", \"Clinic\": \"Pathology\",\n", + " \"Pathologist\": \"Dr. Robert Chen\", \"User\": \"rchen\"},\n", + " {\"Patient\": \"Williams, David C\", \"DOB\": \"1952-07-04\", \"MRN\": \"99887766\",\n", + " \"AccessionNumber\": \"ACC-2023-003\", \"Clinic\": \"Surgical\",\n", + " \"Pathologist\": \"Dr. Sarah Kim\", \"User\": \"skim\"},\n", + "]\n", + "\n", + "COPY_LEVELS = [5, 6, 7, 8] # 2,880×1,600 → 360×200 — fast to read\n", + "\n", + "\n", + "def make_aperio_description(phi: dict, stem: str) -> str:\n", + " \"\"\"Aperio pipe-delimited ImageDescription with embedded PHI (tag 270).\"\"\"\n", + " header = \"Aperio Image Library v12.1.0\\r\\n[0,0 92160x51200] (512x512) JPEG/YCC Q=80\"\n", + " fields = {\"Filename\": stem, \"Date\": \"2023-04-15\", \"Time\": \"10:23:45\", **phi}\n", + " body = \"|\".join(f\"{k} = {v}\" for k, v in fields.items())\n", + " return f\"{header}||{body}\"\n", + "\n", + "\n", + "def make_label_image(phi: dict, size=(387, 463)) -> np.ndarray:\n", + " \"\"\"Render a patient-sticker image with clearly legible PHI text.\"\"\"\n", + " img = Image.new(\"RGB\", size, (255, 255, 255))\n", + " draw = ImageDraw.Draw(img)\n", + " draw.rectangle([2, 2, size[0]-3, size[1]-3], outline=(0, 0, 0), width=2)\n", + " lines = [\n", + " f\"Patient : {phi['Patient']}\",\n", + " f\"DOB : {phi['DOB']}\",\n", + " f\"MRN : {phi['MRN']}\",\n", + " f\"Accn : {phi['AccessionNumber']}\",\n", + " f\"Clinic : {phi['Clinic']}\",\n", + " f\"Path : {phi['Pathologist']}\",\n", + " f\"Date : 2023-04-15\",\n", + " f\"User : {phi['User']}\",\n", + " \"\",\n", + " \"** SYNTHETIC PHI — TEST ONLY **\",\n", + " \"** NOT A REAL PATIENT **\",\n", + " ]\n", + " y = 16\n", + " for line in lines:\n", + " draw.text((10, y), line, fill=(0, 0, 0))\n", + " y += 34\n", + " # Fake barcode strip\n", + " random.seed(42)\n", + " for x in range(10, size[0]-10, 3):\n", + " h = random.randint(15, 50)\n", + " draw.rectangle([x, size[1]-70, x+1, size[1]-70+h], fill=(0, 0, 0))\n", + " return np.array(img)\n", + "\n", + "\n", + "# ── write one synthetic TIFF per source file ──────────────────────────────────────\n", + "for fi, fpath in enumerate(tiff_files):\n", + " # skip files that are already synthetic\n", + " if \"__synthetic_phi\" in fpath:\n", + " continue\n", + "\n", + " phi = SYNTHETIC_PATIENTS[fi % len(SYNTHETIC_PATIENTS)]\n", + " stem = os.path.splitext(os.path.basename(fpath))[0]\n", + " out_vol = os.path.join(os.path.dirname(fpath), f\"{stem}__synthetic_phi.tiff\")\n", + " tmp_out = os.path.join(tempfile.gettempdir(), f\"{stem}__synthetic_phi.tiff\")\n", + "\n", + " print(f\"\\n{'\\u2500'*72}\")\n", + " print(f\" source : {os.path.basename(fpath)}\")\n", + " print(f\" subject : {phi['Patient']} MRN={phi['MRN']}\")\n", + " print(f\" output : {out_vol}\")\n", + "\n", + " slide = openslide.OpenSlide(fpath)\n", + " pyramid = []\n", + " for lvl in COPY_LEVELS:\n", + " if lvl >= slide.level_count:\n", + " continue\n", + " lw, lh = slide.level_dimensions[lvl]\n", + " arr = np.array(slide.read_region((0, 0), lvl, (lw, lh)).convert(\"RGB\"))\n", + " pyramid.append((lvl, arr, lw, lh))\n", + " print(f\" level {lvl}: {lw}\\u00d7{lh}\")\n", + " slide.close()\n", + "\n", + " if not pyramid:\n", + " print(\" \\u26a0 no levels — skipping\")\n", + " continue\n", + "\n", + " image_desc = make_aperio_description(phi, stem)\n", + " label_arr = make_label_image(phi)\n", + " macro_sm = np.array(\n", + " Image.fromarray(pyramid[0][1]).resize((640, 356), Image.LANCZOS)\n", + " )\n", + " # metadata=None disables tifffile's auto-JSON shape override so our\n", + " # description= and extratags= values are written to tag 270 unchanged.\n", + " _write = dict(photometric=\"rgb\", tile=(512, 512),\n", + " compression=\"deflate\", compressionargs={\"level\": 6},\n", + " metadata=None)\n", + "\n", + " with tifffile.TiffWriter(tmp_out, bigtiff=True) as tif:\n", + "\n", + " # IFD 0 — main image (level 5 of source) + all PHI tags\n", + " _, arr, _, _ = pyramid[0]\n", + " tif.write(\n", + " arr, **_write,\n", + " subfiletype=0,\n", + " description=image_desc, # tag 270 — Aperio-style PHI\n", + " software=\"Philips IntelliSite 3.0\", # tag 305\n", + " extratags=[\n", + " (315, 2, 0, phi[\"User\"], True), # Artist — PHI\n", + " (316, 2, 0, \"SCANNER-PHI-01\", True), # HostComputer\n", + " ],\n", + " )\n", + "\n", + " # IFDs 1–N — reduced-resolution pyramid\n", + " for _, arr, _, _ in pyramid[1:]:\n", + " tif.write(arr, **_write, subfiletype=1)\n", + "\n", + " # label sub-image — PHI visible in pixel content (VLM/OCR target)\n", + " tif.write(\n", + " label_arr,\n", + " photometric=\"rgb\", compression=\"deflate\",\n", + " compressionargs={\"level\": 6},\n", + " metadata=None, subfiletype=1, description=\"label\",\n", + " )\n", + "\n", + " # macro sub-image — tissue overview\n", + " tif.write(\n", + " macro_sm,\n", + " photometric=\"rgb\", compression=\"deflate\",\n", + " compressionargs={\"level\": 6},\n", + " metadata=None, subfiletype=1, description=\"macro\",\n", + " )\n", + "\n", + " shutil.copy2(tmp_out, out_vol)\n", + " os.remove(tmp_out)\n", + "\n", + " fsize = os.path.getsize(out_vol) / 1024 / 1024\n", + " print(f\" written {fsize:.1f} MB (deflate-compressed)\")\n", + " print(f\" desc : {image_desc[:120]} \\u2026\")\n", + "\n", + " # ── verify: re-open with openslide and confirm PHI is readable ───────────\n", + " try:\n", + " chk = openslide.OpenSlide(out_vol)\n", + " desc = chk.properties.get(\"openslide.comment\",\n", + " chk.properties.get(\"tiff.ImageDescription\", \"(none)\"))\n", + " assoc = sorted(chk.associated_images.keys())\n", + " print(f\" verify : vendor={chk.properties.get('openslide.vendor')} \"\n", + " f\"assoc={assoc} desc_len={len(desc)}\")\n", + " print(f\" PHI ok : Patient={('Patient' in desc)} \"\n", + " f\"MRN={('MRN' in desc)} User={('User' in desc)}\")\n", + " chk.close()\n", + " except Exception as exc:\n", + " print(f\" verify error: {exc}\")\n", + "\n", + "print(\"\\nSynthetic PHI TIFFs complete.\")\n" + ] + }, + { + "cell_type": "code", + "execution_count": 0, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": { + "byteLimit": 2048000, + "rowLimit": 10000 + }, + "inputWidgets": {}, + "nuid": "a99cb8f2-daaf-4412-900e-de06f51610ff", + "showTitle": true, + "tableResultSettingsMap": {}, + "title": "Display synthetic PHI TIFFs" + } + }, + "outputs": [], + "source": [ + "# Displays each synthetic PHI TIFF in full:\n", + "# 1. Pyramid layers panel (via show_slide_layers)\n", + "# 2. Full ImageDescription (tag 270) with PHI fields flagged ⚠️\n", + "# 3. label + macro sub-images read from PIL IFDs — the VLM / OCR target\n", + "# All shared helpers come from the Shared utilities cell.\n", + "from PIL.TiffTags import TAGS\n", + "\n", + "PHI_KEYS = [\n", + " \"Patient\", \"DOB\", \"MRN\", \"AccessionNumber\", \"Clinic\",\n", + " \"Pathologist\", \"Date\", \"Time\", \"User\", \"Filename\", \"ImageID\",\n", + "]\n", + "\n", + "synthetic_files = [f for f in discover_tiff_files() if \"__synthetic_phi\" in f]\n", + "print(f\"Found {len(synthetic_files)} synthetic PHI TIFF(s)\\n\")\n", + "\n", + "for fpath in synthetic_files:\n", + " fname = os.path.basename(fpath)\n", + " print(f\"\\n{'\\u2550' * 72}\\n{fname}\\n{'\\u2500' * 72}\")\n", + "\n", + " # ── 1. Pyramid layers (openslide) ─────────────────────────────────────────\n", + " show_slide_layers(fpath)\n", + "\n", + " # ── 2. ImageDescription via PIL tag_v2 (full string, not truncated) ────────\n", + " pil_img = Image.open(fpath)\n", + " pil_img.seek(0)\n", + " raw = dict(getattr(pil_img, \"tag_v2\", {})).get(270, b\"\")\n", + " desc = raw.decode(\"utf-8\", errors=\"replace\").strip(\"\\x00\") if isinstance(raw, bytes) else str(raw)\n", + "\n", + " print(f\"\\n ImageDescription ({len(desc)} chars):\")\n", + " for part in desc.replace(\"\\r\\n\", \"||\").split(\"|\"):\n", + " part = part.strip()\n", + " if not part:\n", + " continue\n", + " is_phi = any(k in part for k in PHI_KEYS)\n", + " marker = \" \\u26a0\\ufe0f PHI\" if is_phi else \"\"\n", + " print(f\" {part}{marker}\")\n", + "\n", + " # ── 3. label / macro sub-images (PIL IFD walk) ─────────────────────────\n", + " n_frames = getattr(pil_img, \"n_frames\", 1)\n", + " sub_imgs = []\n", + " for fi in range(n_frames):\n", + " try:\n", + " pil_img.seek(fi)\n", + " except EOFError:\n", + " break\n", + " ifd_desc = dict(getattr(pil_img, \"tag_v2\", {})).get(270, b\"\")\n", + " if isinstance(ifd_desc, bytes):\n", + " ifd_desc = ifd_desc.decode(\"utf-8\", errors=\"replace\").strip(\"\\x00\")\n", + " if ifd_desc in (\"label\", \"macro\"):\n", + " sub_imgs.append({\"name\": ifd_desc, \"img\": pil_img.copy()})\n", + " pil_img.close()\n", + "\n", + " if sub_imgs:\n", + " n = len(sub_imgs)\n", + " fig, axes = plt.subplots(1, n, figsize=(9 * n, 9))\n", + " if n == 1:\n", + " axes = [axes]\n", + " for ax, si in zip(axes, sub_imgs):\n", + " rgb = to_rgb(si[\"img\"])\n", + " w, h = rgb.size\n", + " ax.imshow(rgb)\n", + " ax.set_title(\n", + " f\"{si['name']} ({w} \\u00d7 {h} px)\",\n", + " fontsize=12, fontweight=\"bold\", pad=6,\n", + " )\n", + " ax.set_xlabel(\n", + " \"PHI rendered in pixels — VLM / OCR redaction target\",\n", + " fontsize=9,\n", + " )\n", + " ax.tick_params(left=False, bottom=False, labelleft=False, labelbottom=False)\n", + " fig.suptitle(\n", + " f\"{fname} — sub-images\",\n", + " fontsize=11, fontweight=\"bold\", y=1.01,\n", + " )\n", + " plt.tight_layout()\n", + " plt.show()\n", + " else:\n", + " print(\"\\n (no label/macro found via PIL IFD walk)\")\n", + " print()\n", + "\n", + "print(\"Done.\")\n" + ] + }, + { + "cell_type": "code", + "execution_count": 0, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": { + "byteLimit": 2048000, + "rowLimit": 10000 + }, + "inputWidgets": {}, + "nuid": "249f2889-656a-4aad-9fd0-35e0130d13e9", + "showTitle": true, + "tableResultSettingsMap": {}, + "title": "Compare Philips TIFF vs Aperio SVS" + } + }, + "outputs": [], + "source": [ + "# Side-by-side structural and metadata comparison between:\n", + "# - Philips BIG.tiff (OSUWMC) — metadata-bare\n", + "# - Aperio SVS (orthanc_demo) — rich PHI metadata + associated sub-images\n", + "# Also shows the sub-images from SVS that are absent in the Philips files.\n", + "\n", + "import glob, os\n", + "import openslide\n", + "from PIL import Image\n", + "import matplotlib.pyplot as plt\n", + "from PIL.TiffTags import TAGS\n", + "\n", + "# to_rgb, discover_tiff_files, tiff_files from Shared utilities\n", + "orig_tiffs = discover_tiff_files(synthetic=False)\n", + "\n", + "# Locate SVS files\n", + "svs_files = sorted(\n", + " glob.glob(\"/Volumes/hls_pathology/orthanc_demo/raw_images/Aperio/**/*.svs\",\n", + " recursive=True) +\n", + " glob.glob(\"/Volumes/hls_pathology/orthanc_demo/raw_images/Aperio/*.svs\")\n", + ")[:3] # first 3 for comparison\n", + "\n", + "print(f\"Philips TIFFs : {len(orig_tiffs)}\")\n", + "print(f\"Aperio SVS : {len(svs_files)}\")\n", + "\n", + "\n", + "def slide_profile(fpath):\n", + " \"\"\"Return a dict of key attributes for one openslide-readable file.\"\"\"\n", + " fname = os.path.basename(fpath)\n", + " try:\n", + " s = openslide.OpenSlide(fpath)\n", + " desc = s.properties.get(\"openslide.comment\",\n", + " s.properties.get(\"tiff.ImageDescription\", \"\"))\n", + " phi_keys = [k for k in\n", + " [\"Date\",\"Time\",\"User\",\"Filename\",\n", + " \"Patient\",\"DOB\",\"MRN\",\"AccessionNumber\",\n", + " \"Clinic\",\"Pathologist\",\"Procedure\",\"Diagnosis\"]\n", + " if k in desc]\n", + " profile = {\n", + " \"fname\" : fname,\n", + " \"vendor\" : s.properties.get(\"openslide.vendor\", \"?\"),\n", + " \"dims\" : s.dimensions,\n", + " \"levels\" : s.level_count,\n", + " \"mpp\" : s.properties.get(\"openslide.mpp-x\",\n", + " s.properties.get(\"aperio.MPP\", \"n/a\")),\n", + " \"n_props\" : len(s.properties),\n", + " \"desc_len\" : len(desc),\n", + " \"phi_keys\" : phi_keys,\n", + " \"associated\" : sorted(s.associated_images.keys()),\n", + " \"assoc_images\": {k: s.associated_images[k]\n", + " for k in s.associated_images.keys()},\n", + " \"properties\" : dict(s.properties),\n", + " \"slide\" : s,\n", + " }\n", + " return profile\n", + " except Exception as exc:\n", + " return {\"fname\": fname, \"error\": str(exc)}\n", + "\n", + "\n", + "print(\"\\nProfiling files …\")\n", + "svs_profiles = [slide_profile(f) for f in svs_files]\n", + "tiff_profiles = [slide_profile(f) for f in orig_tiffs]\n", + "\n", + "\n", + "# ── text comparison table ────────────────────────────────────────────────────────────\n", + "vs = svs_profiles[0] if svs_profiles else {}\n", + "vt = tiff_profiles[0] if tiff_profiles else {}\n", + "\n", + "print(f\"\\n{'\\u2550'*90}\")\n", + "print(f\"{'Attribute':<28} {'Aperio SVS':<30} {'Philips BIG.tiff':<30}\")\n", + "print(f\"{'\\u2500'*90}\")\n", + "rows = [\n", + " (\"File\", lambda p: p.get(\"fname\",\"?\")[:40]),\n", + " (\"openslide.vendor\",lambda p: p.get(\"vendor\",\"?\")),\n", + " (\"Full resolution\", lambda p: f\"{p['dims'][0]:,}\\u00d7{p['dims'][1]:,}\" if \"dims\" in p else \"?\"),\n", + " (\"Pyramid levels\", lambda p: str(p.get(\"levels\",\"?\"))),\n", + " (\"MPP (microns/px)\",lambda p: str(p.get(\"mpp\",\"?\"))),\n", + " (\"# openslide props\",lambda p: str(p.get(\"n_props\",\"?\"))),\n", + " (\"ImageDescription len\",lambda p: str(p.get(\"desc_len\",0)) + \" chars\"),\n", + " (\"PHI keys in desc\", lambda p: str(p.get(\"phi_keys\",[]))),\n", + " (\"Associated images\",lambda p: str(p.get(\"associated\",[]))),\n", + "]\n", + "for name, fn in rows:\n", + " sv = fn(vs) if vs else \"N/A\"\n", + " tv = fn(vt) if vt else \"N/A\"\n", + " flag = \" \\u26a0\\ufe0f\" if name == \"PHI keys in desc\" and sv and sv != \"[]\" else \"\"\n", + " print(f\" {name:<26} {sv:<30} {tv:<30}{flag}\")\n", + "print(f\"{'\\u2550'*90}\")\n", + "\n", + "\n", + "# ── print SVS ImageDescription (show PHI fields) ──────────────────────────────────\n", + "if vs and vs.get(\"desc_len\", 0) > 0:\n", + " print(f\"\\nSVS ImageDescription ({vs['fname']})\")\n", + " desc = vs[\"properties\"].get(\"openslide.comment\",\n", + " vs[\"properties\"].get(\"tiff.ImageDescription\",\"\"))\n", + " for part in desc.split(\"|\"):\n", + " part = part.strip()\n", + " if not part:\n", + " continue\n", + " is_phi = any(k in part for k in\n", + " [\"Date\",\"Time\",\"User\",\"Patient\",\"DOB\",\"MRN\",\n", + " \"Accession\",\"Clinic\",\"Pathologist\",\"Filename\",\"ImageID\"])\n", + " marker = \" \\u26a0\\ufe0f PHI\" if is_phi else \"\"\n", + " print(f\" {part}{marker}\")\n", + "\n", + "\n", + "# ── display SVS associated sub-images (absent in Philips) ────────────────────────\n", + "for prof in svs_profiles:\n", + " if \"error\" in prof or not prof.get(\"associated\"):\n", + " continue\n", + " assoc = prof[\"assoc_images\"]\n", + " n = len(assoc)\n", + " fig, axes = plt.subplots(1, n, figsize=(5 * n, 4))\n", + " if n == 1:\n", + " axes = [axes]\n", + " for ax, (name, img) in zip(axes, assoc.items()):\n", + " rgb = to_rgb(img)\n", + " ax.imshow(rgb)\n", + " ax.set_title(\n", + " f\"{name}\\n{rgb.size[0]}\\u00d7{rgb.size[1]} px\",\n", + " fontsize=9, fontweight=\"bold\", pad=3,\n", + " )\n", + " ax.tick_params(left=False, bottom=False,\n", + " labelleft=False, labelbottom=False)\n", + " fig.suptitle(\n", + " f\"SVS associated images — {prof['fname']}\\n\"\n", + " f\"(PHI risk: label barcode + printed text; absent in Philips TIFFs)\",\n", + " fontsize=9, fontweight=\"bold\",\n", + " )\n", + " plt.tight_layout()\n", + " plt.show()\n", + "\n", + "# close openslide handles\n", + "for p in svs_profiles + tiff_profiles:\n", + " if \"slide\" in p:\n", + " try:\n", + " p[\"slide\"].close()\n", + " except Exception:\n", + " pass\n", + "\n", + "print(\"\\nComparison complete.\")\n" + ] + } + ], + "metadata": { + "application/vnd.databricks.v1+notebook": { + "computePreferences": null, + "dashboards": [], + "environmentMetadata": { + "base_environment": "", + "environment_version": "5" + }, + "inputWidgetPreferences": null, + "language": "python", + "notebookMetadata": { + "pythonIndentUnit": 2 + }, + "notebookName": "TIFF sample data", + "widgets": {} + }, + "language_info": { + "name": "python" + } + }, + "nbformat": 4, + "nbformat_minor": 0 +} From ab2f9840bb9648eb24975d7db6aefb72f60a48a6 Mon Sep 17 00:00:00 2001 From: dmoore247 Date: Sat, 11 Jul 2026 21:51:36 +0000 Subject: [PATCH 3/7] add initial tiff indexing capabilities --- notebooks/tiff/01-ingest-tiff.ipynb | 324 +++++++++++++++++++++ src/dbx/pixels/tiff/__init__.py | 3 + src/dbx/pixels/tiff/tiff_meta_extractor.py | 233 +++++++++++++++ src/dbx/pixels/tiff/tiff_phi_tags.py | 171 +++++++++++ 4 files changed, 731 insertions(+) create mode 100644 notebooks/tiff/01-ingest-tiff.ipynb create mode 100644 src/dbx/pixels/tiff/__init__.py create mode 100644 src/dbx/pixels/tiff/tiff_meta_extractor.py create mode 100644 src/dbx/pixels/tiff/tiff_phi_tags.py diff --git a/notebooks/tiff/01-ingest-tiff.ipynb b/notebooks/tiff/01-ingest-tiff.ipynb new file mode 100644 index 00000000..f37ab101 --- /dev/null +++ b/notebooks/tiff/01-ingest-tiff.ipynb @@ -0,0 +1,324 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": { + "byteLimit": 2048000, + "rowLimit": 10000 + }, + "inputWidgets": {}, + "nuid": "bf3db99b-7cb3-4c65-a428-96b93a9a5510", + "showTitle": true, + "tableResultSettingsMap": {}, + "title": "TIFF Ingest — Overview" + } + }, + "source": [ + "# TIFF File Ingest\n", + "\n", + "Ingests TIFF pathology slides into a Unity Catalog Delta table using the `dbx.pixels.Catalog` class.\n", + "\n", + "**Pipeline**\n", + "1. Load config from `config.yaml` (source path, pattern, table)\n", + "2. Initialise `Catalog` with the target Delta table and UC volume\n", + "3. `catalog.catalog()` — recursively discovers all `.tiff` files and enriches each row with path metadata\n", + "4. `catalog.save()` — writes the file catalog to the Delta table\n", + "5. SQL verification of ingested rows" + ] + }, + { + "cell_type": "code", + "execution_count": 0, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": { + "byteLimit": 2048000, + "rowLimit": 10000 + }, + "inputWidgets": {}, + "nuid": "612e3201-024b-4574-b46e-c456d555345f", + "showTitle": true, + "tableResultSettingsMap": {}, + "title": "Install / resolve dbx.pixels library" + } + }, + "outputs": [], + "source": [ + "import sys, pathlib\n", + "\n", + "# Use the editable source tree — no wheel build needed during development\n", + "SRC_PATH = \"/Workspace/Users/douglas.moore@databricks.com/pixels-tiff/src\"\n", + "\n", + "if SRC_PATH not in sys.path:\n", + " sys.path.insert(0, SRC_PATH)\n", + " print(f\"Added to sys.path: {SRC_PATH}\")\n", + "else:\n", + " print(f\"Already on sys.path: {SRC_PATH}\")" + ] + }, + { + "cell_type": "code", + "execution_count": 0, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": { + "byteLimit": 2048000, + "rowLimit": 10000 + }, + "inputWidgets": {}, + "nuid": "d8ef4c9d-f0b5-468b-be2b-28b875926528", + "showTitle": true, + "tableResultSettingsMap": {}, + "title": "Install tifffile" + } + }, + "outputs": [], + "source": [ + "%pip install tifffile -q" + ] + }, + { + "cell_type": "code", + "execution_count": 0, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": { + "byteLimit": 2048000, + "rowLimit": 10000 + }, + "inputWidgets": {}, + "nuid": "34ed423b-9b76-4818-befb-f8b6b5cfe83d", + "showTitle": true, + "tableResultSettingsMap": {}, + "title": "Load config from config.yaml" + } + }, + "outputs": [], + "source": [ + "import yaml, pathlib\n", + "\n", + "CONFIG_PATH = pathlib.Path(\"/Workspace/Users/douglas.moore@databricks.com/pixels-tiff/notebooks/tiff/config.yaml\")\n", + "\n", + "with CONFIG_PATH.open() as fh:\n", + " config = yaml.safe_load(fh)\n", + "\n", + "SOURCE_PATH = config[\"SOURCE_PATH\"]\n", + "PATTERN = config[\"PATTERN\"]\n", + "TABLE = config[\"INDEX\"]\n", + "\n", + "# Derive volume from catalog + schema of the index table (dmoore.tiff.)\n", + "_catalog, _schema, _ = TABLE.split(\".\")\n", + "VOLUME = f\"{_catalog}.{_schema}.tiff_volume\" # adjust if your volume name differs\n", + "\n", + "WRITE_MODE = \"overwrite\" # use 'append' for incremental runs\n", + "\n", + "print(f\"SOURCE_PATH : {SOURCE_PATH}\")\n", + "print(f\"PATTERN : {PATTERN}\")\n", + "print(f\"TABLE : {TABLE}\")\n", + "print(f\"VOLUME : {VOLUME}\")\n", + "print(f\"WRITE_MODE : {WRITE_MODE}\")" + ] + }, + { + "cell_type": "code", + "execution_count": 0, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": { + "byteLimit": 2048000, + "rowLimit": 10000 + }, + "inputWidgets": {}, + "nuid": "f636624f-94e6-4573-8863-4edfaed3ad86", + "showTitle": true, + "tableResultSettingsMap": {}, + "title": "Initialise Catalog" + } + }, + "outputs": [], + "source": [ + "from dbx.pixels import Catalog\n", + "\n", + "catalog = Catalog(spark, table=TABLE)\n", + "print(catalog)" + ] + }, + { + "cell_type": "code", + "execution_count": 0, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": { + "byteLimit": 2048000, + "rowLimit": 10000 + }, + "inputWidgets": {}, + "nuid": "aa563b95-dad5-4694-9c98-4b8f13c1607a", + "showTitle": true, + "tableResultSettingsMap": {}, + "title": "Catalog TIFF files" + } + }, + "outputs": [], + "source": [ + "# Recursively discover all TIFF files under SOURCE_PATH.\n", + "# .catalog() reads only file metadata (no pixel data is loaded).\n", + "catalog_df = catalog.catalog(\n", + " path=SOURCE_PATH,\n", + " pattern=PATTERN,\n", + " recurse=True,\n", + " streaming=False,\n", + ")\n", + "\n", + "print(f\"Files found: {catalog_df.count()}\")\n", + "display(catalog_df)" + ] + }, + { + "cell_type": "code", + "execution_count": 0, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": { + "byteLimit": 2048000, + "rowLimit": 10000 + }, + "inputWidgets": {}, + "nuid": "aeba92f7-1d20-4767-a7d8-f000f744cf7d", + "showTitle": true, + "tableResultSettingsMap": {}, + "title": "Extract TIFF metadata" + } + }, + "outputs": [], + "source": [ + "from dbx.pixels.tiff import TiffMetaExtractor\n", + "\n", + "# Enrich the file catalog DataFrame with TIFF metadata.\n", + "# TiffMetaExtractor adds a `meta` VARIANT column containing all TIFF tags\n", + "# plus derived fields (page_count, is_ome, is_bigtiff, series info, phi_tag_report).\n", + "extractor = TiffMetaExtractor(catalog, inputCol=\"local_path\", outputCol=\"meta\")\n", + "enriched_df = extractor.transform(catalog_df)\n", + "\n", + "print(f\"Schema: {[f.name for f in enriched_df.schema.fields]}\")\n", + "display(enriched_df.select(\"local_path\", \"extension\", \"meta\"))" + ] + }, + { + "cell_type": "code", + "execution_count": 0, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": { + "byteLimit": 2048000, + "rowLimit": 10000 + }, + "inputWidgets": {}, + "nuid": "ab9f4a04-023e-4dc3-a4a8-cd083e59f793", + "showTitle": true, + "tableResultSettingsMap": {}, + "title": "Save file catalog to Delta table" + } + }, + "outputs": [], + "source": [ + "# Persist the file catalog to the Delta table defined in config.yaml (INDEX).\n", + "# Use write_mode='overwrite' for a full refresh, or 'append' for incremental.\n", + "catalog.save(enriched_df, mode=WRITE_MODE)\n", + "\n", + "print(f\"Saved to: {TABLE} (mode={WRITE_MODE})\")" + ] + }, + { + "cell_type": "code", + "execution_count": 0, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": { + "byteLimit": 2048000, + "implicitDf": true, + "rowLimit": 10000 + }, + "inputWidgets": {}, + "nuid": "548d21b6-778e-4525-b7f0-b9fe09b8ca05", + "showTitle": true, + "tableResultSettingsMap": {}, + "title": "Verify ingested rows" + } + }, + "outputs": [], + "source": [ + "%sql\n", + "-- Quick verification: row count and sample paths\n", + "SELECT\n", + " COUNT(*) AS file_count,\n", + " SUM(length) / 1024 / 1024 AS total_size_mb,\n", + " COLLECT_SET(extension) AS extensions\n", + "FROM dmoore.tiff.object_catalog" + ] + }, + { + "cell_type": "code", + "execution_count": 0, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": { + "byteLimit": 2048000, + "implicitDf": true, + "rowLimit": 10000 + }, + "inputWidgets": {}, + "nuid": "a9616b94-823e-434b-8a72-52f0cd4adaef", + "showTitle": true, + "tableResultSettingsMap": {}, + "title": "Browse ingested catalog" + } + }, + "outputs": [], + "source": [ + "%sql\n", + "SELECT\n", + " path,\n", + " length,\n", + " modificationTime,\n", + " extension,\n", + " path_tags,\n", + " file_type,\n", + " meta\n", + "FROM dmoore.tiff.object_catalog\n", + "ORDER BY modificationTime DESC\n", + "LIMIT 50" + ] + } + ], + "metadata": { + "application/vnd.databricks.v1+notebook": { + "computePreferences": null, + "dashboards": [], + "environmentMetadata": { + "base_environment": "", + "environment_version": "5" + }, + "inputWidgetPreferences": null, + "language": "python", + "notebookMetadata": { + "mostRecentlyExecutedCommandWithImplicitDF": { + "commandId": 8636978234437395, + "dataframes": [ + "_sqldf" + ] + }, + "pythonIndentUnit": 4 + }, + "notebookName": "01-ingest-tiff", + "widgets": {} + }, + "language_info": { + "name": "python" + } + }, + "nbformat": 4, + "nbformat_minor": 0 +} diff --git a/src/dbx/pixels/tiff/__init__.py b/src/dbx/pixels/tiff/__init__.py new file mode 100644 index 00000000..838b0c3e --- /dev/null +++ b/src/dbx/pixels/tiff/__init__.py @@ -0,0 +1,3 @@ +from dbx.pixels.tiff.tiff_meta_extractor import TiffMetaExtractor + +__all__ = ["TiffMetaExtractor"] diff --git a/src/dbx/pixels/tiff/tiff_meta_extractor.py b/src/dbx/pixels/tiff/tiff_meta_extractor.py new file mode 100644 index 00000000..fda7e8cc --- /dev/null +++ b/src/dbx/pixels/tiff/tiff_meta_extractor.py @@ -0,0 +1,233 @@ +"""TiffMetaExtractor — Spark ML Transformer that reads TIFF metadata into a ``meta`` VARIANT. + +Mirrors ``SVSMetaExtractor`` from the pixels SA: +- Extends ``pyspark.ml.pipeline.Transformer`` +- Implements ``_transform(df)`` +- Uses ``mapInPandas`` with ``ThreadPoolExecutor`` for concurrent I/O +- Outputs a single ``meta`` column as a parsed VARIANT + +Primary backend: ``tifffile`` (handles standard TIFF, BigTIFF, OME-TIFF, +Aperio SVS-style TIFF, NDPI). Falls back to ``Pillow`` if ``tifffile`` is +not installed. + +All derived fields (page_count, is_ome, is_bigtiff, series info, +phi_tag_report) are merged into the tag dict before JSON serialisation, so +no schema change to ``object_catalog`` is required. +""" + +from __future__ import annotations + +import json +from concurrent.futures import ThreadPoolExecutor +from typing import Iterator + +import pandas as pd +import pyspark.sql.types as t +from pyspark.ml.pipeline import Transformer +from pyspark.sql.functions import expr + +from dbx.pixels.tiff.tiff_phi_tags import classify_tags + + +class TiffMetaExtractor(Transformer): + """Extract TIFF metadata into the ``meta VARIANT`` column. + + Uses ``tifffile`` as the primary backend; falls back to ``Pillow`` when + ``tifffile`` is not available on the executor. + + Args: + catalog: Catalog instance (provides path/anon context). + inputCol: Column with worker-accessible file paths (default ``local_path``). + outputCol: Output column name (default ``meta``). + maxWorkers: ``ThreadPoolExecutor`` concurrency per Spark task (default 32). + useVariant: Parse JSON string to VARIANT via ``parse_json()`` (default True). + filterLargeTags: Drop tags in ``LARGE_TAGS`` (tile/strip offsets, JPEG tables, + ICC profile, XMP, etc.) before JSON serialisation (default True). + """ + + MAX_WORKERS = 32 + + def __init__( + self, + catalog, + inputCol: str = "local_path", + outputCol: str = "meta", + maxWorkers: int = None, + useVariant: bool = True, + filterLargeTags: bool = True, + ): + self.catalog = catalog + self.inputCol = inputCol + self.outputCol = outputCol + self.maxWorkers = maxWorkers or self.MAX_WORKERS + self.useVariant = useVariant + self.filterLargeTags = filterLargeTags + + # ------------------------------------------------------------------ + # Internal helpers (run on Spark executors inside mapInPandas) + # ------------------------------------------------------------------ + + @staticmethod + def _process_tifffile(path: str, filter_large: bool = True) -> str: + """Extract metadata with tifffile (primary backend).""" + import tifffile + + from dbx.pixels.tiff.tiff_phi_tags import LARGE_TAGS, classify_tags + + try: + with tifffile.TiffFile(path) as tif: + page = tif.pages[0] + + # All page-0 TIFF tags as plain strings; skip large binary/offset tags + tags: dict[str, str] = { + tag.name: str(tag.value) + for tag in page.tags.values() + if not filter_large or tag.name not in LARGE_TAGS + } + + # Per-series summary (multi-level WSI awareness) + series_info = [] + for s in tif.series: + entry: dict = { + "shape": list(s.shape), + "axes": s.axes, + "dtype": str(s.dtype), + } + if hasattr(s, "levels"): + entry["level_count"] = len(s.levels) + series_info.append(entry) + + meta = { + **tags, + # --- derived fields --- + "page_count": len(tif.pages), + "series_count": len(tif.series), + "is_bigtiff": tif.is_bigtiff, + "is_ome": tif.is_ome, + "is_svs": tif.is_svs, + "is_ndpi": getattr(tif, "is_ndpi", False), + "width": page.imagewidth, + "height": page.imagelength, + "bits_per_sample": page.bitspersample, + "samples_per_pixel": page.samplesperpixel, + "compression": str(page.compression), + "photometric": str(page.photometric), + "series": series_info, + "phi_tag_report": classify_tags(tags), + } + return json.dumps(meta) + + except Exception as err: + return json.dumps( + {"error": str(err), "udf": "tiff_meta_extractor_tifffile", "path": path} + ) + + @staticmethod + def _process_pillow(path: str, filter_large: bool = True) -> str: + """Extract metadata with Pillow (fallback when tifffile is absent).""" + from PIL import Image + + from dbx.pixels.tiff.tiff_phi_tags import LARGE_TAGS + + try: + # Lazy import: only available in Pillow >= 5.4 + try: + from PIL.TiffImagePlugin import IFDRational + except ImportError: + IFDRational = None + + with Image.open(path) as img: + raw_tags = img.tag_v2 if hasattr(img, "tag_v2") else {} + tags: dict = {} + + # Use TAGS mapping when available for human-readable names + try: + from PIL.ExifTags import TAGS as _TAGS + except ImportError: + _TAGS = {} + + for k, v in raw_tags.items(): + tag_name = _TAGS.get(k, str(k)) + if filter_large and tag_name in LARGE_TAGS: + continue + # Coerce non-JSON-serialisable types + if IFDRational is not None and isinstance(v, IFDRational): + tags[tag_name] = float(v) + elif isinstance(v, tuple): + tags[tag_name] = [ + float(x) if (IFDRational and isinstance(x, IFDRational)) else x + for x in v + ] + elif isinstance(v, bytes): + tags[tag_name] = v.decode("latin-1", errors="replace") + else: + tags[tag_name] = v + + meta = { + **tags, + "width": img.width, + "height": img.height, + "mode": img.mode, + "n_frames": getattr(img, "n_frames", 1), + "format": img.format, + "phi_tag_report": classify_tags( + {str(k): str(v) for k, v in tags.items()} + ), + } + return json.dumps(meta, default=str) + + except Exception as err: + return json.dumps( + {"error": str(err), "udf": "tiff_meta_extractor_pillow", "path": path} + ) + + @staticmethod + def _process_file(path: str, filter_large: bool = True) -> str: + """Dispatch to tifffile or Pillow, whichever is available.""" + try: + import tifffile # noqa: F401 + return TiffMetaExtractor._process_tifffile(path, filter_large) + except ImportError: + return TiffMetaExtractor._process_pillow(path, filter_large) + + # ------------------------------------------------------------------ + # Transformer entry point + # ------------------------------------------------------------------ + + def _transform(self, df): + """Apply TIFF metadata extraction using mapInPandas with concurrent I/O.""" + input_col = self.inputCol + output_col = self.outputCol + max_workers = self.maxWorkers + + out_schema = t.StructType( + list(df.schema.fields) + + [t.StructField(output_col, t.StringType(), True)] + ) + + filter_large = self.filterLargeTags + + def _extract_meta_batch( + iterator: Iterator[pd.DataFrame], + ) -> Iterator[pd.DataFrame]: + for pdf in iterator: + paths = pdf[input_col].tolist() + with ThreadPoolExecutor(max_workers=max_workers) as executor: + results = list( + executor.map( + lambda p: TiffMetaExtractor._process_file(p, filter_large), + paths, + ) + ) + pdf[output_col] = results + yield pdf + + df = df.mapInPandas(_extract_meta_batch, schema=out_schema) + + if self.useVariant: + df = df.withColumn( + self.outputCol, + expr(f"parse_json(`{self.outputCol}`)"), + ) + + return df diff --git a/src/dbx/pixels/tiff/tiff_phi_tags.py b/src/dbx/pixels/tiff/tiff_phi_tags.py new file mode 100644 index 00000000..d5020471 --- /dev/null +++ b/src/dbx/pixels/tiff/tiff_phi_tags.py @@ -0,0 +1,171 @@ +"""PHI tag classification for standard TIFF / BigTIFF / OME-TIFF / Aperio TIFF files. + +Covers baseline TIFF tags (TIFF 6.0 spec) and common EXIF tags by their +string name as returned by tifffile (e.g. ``"Artist"``, ``"DateTime"``). + +Classification tiers +-------------------- +PHI — Directly identifies a patient or operator. +QUESTIONABLE — May contain PHI depending on site configuration. +NOT_PHI — Pure scanner / geometry / technical parameters. + +Public API +---------- + classify_tag(key) -> str + classify_tags(properties: dict) -> list[dict] +""" + +from __future__ import annotations + +# --------------------------------------------------------------------------- +# PHI: Directly identifies a person +# --------------------------------------------------------------------------- +PHI_TAGS: set[str] = { + # Baseline TIFF 6.0 + "Artist", # tag 315 — person who created the image + "HostComputer", # tag 316 — workstation/operator ID + # Aperio SVS ImageDescription pipe-delimited sub-keys + "Patient", + "PatientID", + "DOB", + "MRN", + "AccessionNumber", + "ClinicID", + "ClinicalTrialID", + "Procedure", + "Diagnosis", + "Id", +} + +# --------------------------------------------------------------------------- +# QUESTIONABLE: May contain PHI depending on site / scanner configuration +# --------------------------------------------------------------------------- +QUESTIONABLE_TAGS: set[str] = { + # Baseline TIFF 6.0 — free-text / timestamp fields + "ImageDescription", # tag 270 — free-text; may embed patient info (Aperio, OME) + "DateTime", # tag 306 — image creation timestamp + "Copyright", # tag 33432 — may contain operator/institution name + # EXIF timestamps + "DateTimeOriginal", # EXIF 36867 + "DateTimeDigitized", # EXIF 36868 + # Aperio SVS ImageDescription sub-keys + "Date", + "Time", + "Clinic", + "Pathologist", + "Title", + "Filename", + "User", + "ImageID", +} + +# --------------------------------------------------------------------------- +# LARGE_TAGS: Binary / array tags that should be skipped or truncated during +# metadata extraction — their values are large byte blobs or offset arrays +# that add no textual metadata value and bloat the JSON output. +# --------------------------------------------------------------------------- +LARGE_TAGS: set[str] = { + # JPEG / compression tables + "JPEGTables", # tag 347 — JPEG quantisation + Huffman tables (binary) + "JPEGQTables", # tag 519 — old-style JPEG quantisation tables + "JPEGDCTables", # tag 520 — old-style DC Huffman tables + "JPEGACTables", # tag 521 — old-style AC Huffman tables + # Tile / strip index arrays (one entry per tile/strip — can be millions of entries) + "TileOffsets", # tag 324 — byte offset of every tile in the file + "TileByteCounts", # tag 325 — byte length of every tile + "StripOffsets", # tag 273 — byte offset of every strip + "StripByteCounts", # tag 279 — byte length of every strip + # Colour / profile data + "ICCProfile", # tag 34675 — ICC colour profile (often 400 B – 4 MB) + "ColorMap", # tag 320 — RGB palette for indexed-colour images + "TransferFunction", # tag 301 — transfer function curves + "ReferenceBlackWhite", # tag 532 — reference black/white for YCbCr + # Embedded metadata blobs + "XMP", # tag 700 — XMP metadata XML (can be 10s of KB) + "IPTCNAA", # tag 33723 — IPTC/NAA metadata record + "Photoshop", # tag 34377 — Photoshop ImageResources block + "ExifIFD", # tag 34665 — embedded EXIF IFD offset array + # GeoTIFF arrays + "GeoKeyDirectoryTag", # tag 34736 — GeoTIFF key directory + "GeoDoubleParamsTag", # tag 34736 — GeoTIFF double params + "GeoAsciiParamsTag", # tag 34737 — GeoTIFF ASCII params + # WSI / scanning + "ImageDepth", # tag 32997 — depth offset array in some WSI formats + "SubIFDs", # tag 330 — sub-IFD offset array (pyramid levels) +} + +# --------------------------------------------------------------------------- +# NOT_PHI: Scanner geometry / technical parameters (not exhaustive) +# --------------------------------------------------------------------------- +NOT_PHI_TAGS: set[str] = { + "ImageWidth", + "ImageLength", + "BitsPerSample", + "Compression", + "PhotometricInterpretation", + "StripOffsets", + "SamplesPerPixel", + "RowsPerStrip", + "StripByteCounts", + "XResolution", + "YResolution", + "PlanarConfiguration", + "ResolutionUnit", + "Software", # tag 305 — scanner software version (NOT PHI) + "Make", # tag 271 — scanner manufacturer + "Model", # tag 272 — scanner model + "TileWidth", + "TileLength", + "TileOffsets", + "TileByteCounts", + "NewSubfileType", + "SubfileType", + "Orientation", + "ExtraSamples", + "SampleFormat", + "JPEGTables", + "YCbCrSubSampling", + "ReferenceBlackWhite", + "ColorMap", + "GrayResponseUnit", + "GrayResponseCurve", + # Aperio / OpenSlide technical + "AppMag", + "MPP", + "ScanScope ID", + "StripeWidth", + "Parmset", + "Filtered", + "ICC Profile", +} + + +def classify_tag(key: str) -> str: + """Classify a single TIFF tag name. + + Returns one of: ``'PHI'``, ``'QUESTIONABLE'``, ``'NOT_PHI'``. + Tags not in any lookup default to ``'NOT_PHI'``. + """ + if key in PHI_TAGS: + return "PHI" + if key in QUESTIONABLE_TAGS: + return "QUESTIONABLE" + return "NOT_PHI" + + +def classify_tags(properties: dict) -> list[dict]: + """Classify all TIFF tag keys and return the structured PHI report. + + Args: + properties: Dict mapping TIFF tag name (str) -> value (str). + + Returns: + List of ``{"tag": key, "value": val, "classification": cls}`` dicts. + Only PHI and QUESTIONABLE entries are included (NOT_PHI omitted). + """ + report = [] + for key, value in properties.items(): + cls = classify_tag(key) + if cls in ("PHI", "QUESTIONABLE"): + report.append({"tag": key, "value": str(value), "classification": cls}) + return report From 29b654b5a312bb32f6a353a8a1ffa4594f0b1152 Mon Sep 17 00:00:00 2001 From: dmoore247 Date: Sat, 11 Jul 2026 23:47:29 +0000 Subject: [PATCH 4/7] update tiff after end to end run Creating code checkpoint. Testing completed end to end run. VLM did not find phi though. --- notebooks/tiff/01-ingest-tiff.ipynb | 181 ++++++++++++++-- notebooks/tiff/config.yaml-example | 11 + src/dbx/pixels/tiff/__init__.py | 3 +- src/dbx/pixels/tiff/tiff_utils.py | 191 +++++++++++++++++ src/dbx/pixels/tiff/tiff_vlm_phi_detector.py | 208 +++++++++++++++++++ 5 files changed, 576 insertions(+), 18 deletions(-) create mode 100644 notebooks/tiff/config.yaml-example create mode 100644 src/dbx/pixels/tiff/tiff_utils.py create mode 100644 src/dbx/pixels/tiff/tiff_vlm_phi_detector.py diff --git a/notebooks/tiff/01-ingest-tiff.ipynb b/notebooks/tiff/01-ingest-tiff.ipynb index f37ab101..778c3978 100644 --- a/notebooks/tiff/01-ingest-tiff.ipynb +++ b/notebooks/tiff/01-ingest-tiff.ipynb @@ -1,5 +1,29 @@ { "cells": [ + { + "cell_type": "code", + "execution_count": 0, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": { + "byteLimit": 2048000, + "rowLimit": 10000 + }, + "inputWidgets": {}, + "nuid": "1783cadb-09c4-408b-934e-550998ae9631", + "showTitle": false, + "tableResultSettingsMap": {}, + "title": "" + } + }, + "outputs": [], + "source": [ + "%load_ext autoreload\n", + "%autoreload 2\n", + "# Enables autoreload; learn more at https://docs.databricks.com/en/files/workspace-modules.html#autoreload-for-python-modules\n", + "# To disable autoreload; run %autoreload 0" + ] + }, { "cell_type": "markdown", "metadata": { @@ -38,24 +62,15 @@ "rowLimit": 10000 }, "inputWidgets": {}, - "nuid": "612e3201-024b-4574-b46e-c456d555345f", + "nuid": "d8ef4c9d-f0b5-468b-be2b-28b875926528", "showTitle": true, "tableResultSettingsMap": {}, - "title": "Install / resolve dbx.pixels library" + "title": "Install tifffile" } }, "outputs": [], "source": [ - "import sys, pathlib\n", - "\n", - "# Use the editable source tree — no wheel build needed during development\n", - "SRC_PATH = \"/Workspace/Users/douglas.moore@databricks.com/pixels-tiff/src\"\n", - "\n", - "if SRC_PATH not in sys.path:\n", - " sys.path.insert(0, SRC_PATH)\n", - " print(f\"Added to sys.path: {SRC_PATH}\")\n", - "else:\n", - " print(f\"Already on sys.path: {SRC_PATH}\")" + "%pip install tifffile imagecodecs -q" ] }, { @@ -68,15 +83,24 @@ "rowLimit": 10000 }, "inputWidgets": {}, - "nuid": "d8ef4c9d-f0b5-468b-be2b-28b875926528", + "nuid": "612e3201-024b-4574-b46e-c456d555345f", "showTitle": true, "tableResultSettingsMap": {}, - "title": "Install tifffile" + "title": "Install / resolve dbx.pixels library" } }, "outputs": [], "source": [ - "%pip install tifffile -q" + "import sys, pathlib\n", + "\n", + "# Use the editable source tree — no wheel build needed during development\n", + "SRC_PATH = \"/Workspace/Users/douglas.moore@databricks.com/pixels-tiff/src\"\n", + "\n", + "if SRC_PATH not in sys.path:\n", + " sys.path.insert(0, SRC_PATH)\n", + " print(f\"Added to sys.path: {SRC_PATH}\")\n", + "else:\n", + " print(f\"Already on sys.path: {SRC_PATH}\")" ] }, { @@ -170,7 +194,8 @@ " pattern=PATTERN,\n", " recurse=True,\n", " streaming=False,\n", - ")\n", + ").repartition(4)\n", + "\n", "\n", "print(f\"Files found: {catalog_df.count()}\")\n", "display(catalog_df)" @@ -206,6 +231,128 @@ "display(enriched_df.select(\"local_path\", \"extension\", \"meta\"))" ] }, + { + "cell_type": "code", + "execution_count": 0, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": { + "byteLimit": 2048000, + "rowLimit": 10000 + }, + "inputWidgets": {}, + "nuid": "5acdc926-76ba-47c1-9108-8c4c0d483b3e", + "showTitle": true, + "tableResultSettingsMap": {}, + "title": "Verify TiffVLMPhiDetector import (no pydicom)" + } + }, + "outputs": [], + "source": [ + "import sys\n", + "for _m in [m for m in list(sys.modules) if m.startswith(\"dbx.pixels.tiff\")]:\n", + " sys.modules.pop(_m, None)\n", + "\n", + "from dbx.pixels.tiff import TiffVLMPhiDetector, TiffMetaExtractor\n", + "from dbx.pixels.tiff.tiff_vlm_phi_detector import VlmResult\n", + "\n", + "print(\"✓ TiffVLMPhiDetector imported — no pydicom dependency\")\n", + "print(f\" TiffVLMPhiDetector : {TiffVLMPhiDetector}\")\n", + "print(f\" VlmResult : {VlmResult}\")\n", + "\n", + "# Confirm pydicom is NOT on the import chain\n", + "import importlib, sys as _sys\n", + "assert \"pydicom\" not in _sys.modules, \"pydicom was unexpectedly imported\"\n", + "print(\"✓ pydicom not in sys.modules\")" + ] + }, + { + "cell_type": "code", + "execution_count": 0, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": { + "byteLimit": 2048000, + "rowLimit": 10000 + }, + "inputWidgets": {}, + "nuid": "09779984-1374-4bfe-86d6-b79aecdd1e8e", + "showTitle": true, + "tableResultSettingsMap": {}, + "title": "Diagnose tiff_to_image failure" + } + }, + "outputs": [], + "source": [ + "from dbx.pixels.tiff.tiff_utils import _tiff_to_array_tifffile, _tiff_to_array_pillow\n", + "\n", + "PROBE_PATH = \"/Volumes/hls_radiology_east/osuwmc/sample/Philips07_3b946eed-4d57-4d1a-9f27-32c559ecd07a_BIG.tiff\"\n", + "\n", + "print(\"--- tifffile ---\")\n", + "try:\n", + " arr = _tiff_to_array_tifffile(PROBE_PATH)\n", + " print(f\"OK shape={arr.shape} dtype={arr.dtype}\" if arr is not None else \"returned None\")\n", + "except Exception as e:\n", + " print(f\"ERROR: {type(e).__name__}: {e}\")\n", + "\n", + "print(\"--- Pillow ---\")\n", + "try:\n", + " arr = _tiff_to_array_pillow(PROBE_PATH)\n", + " print(f\"OK shape={arr.shape} dtype={arr.dtype}\" if arr is not None else \"returned None\")\n", + "except Exception as e:\n", + " print(f\"ERROR: {type(e).__name__}: {e}\")" + ] + }, + { + "cell_type": "code", + "execution_count": 0, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": { + "byteLimit": 2048000, + "rowLimit": 10000 + }, + "inputWidgets": {}, + "nuid": "f791598b-4aa2-4eda-80c9-c222ceddcc58", + "showTitle": true, + "tableResultSettingsMap": {}, + "title": "Detect PHI in TIFF images" + } + }, + "outputs": [], + "source": [ + "import sys\n", + "# Clear any stale module-cache entries so updated source files are picked up\n", + "for _m in [m for m in list(sys.modules) if m.startswith(\"dbx.pixels.tiff\")]:\n", + " sys.modules.pop(_m, None)\n", + "\n", + "from dbx.pixels.tiff import TiffVLMPhiDetector\n", + "\n", + "VLM_ENDPOINT = config.get(\"VLM_ENDPOINT\", \"\")\n", + "\n", + "detector = TiffVLMPhiDetector(\n", + " endpoint = VLM_ENDPOINT,\n", + " inputCol = \"local_path\",\n", + " outputCol = \"response\",\n", + " input_type = \"tiff\",\n", + " max_width = 768,\n", + ")\n", + "\n", + "phi_df = detector.transform(enriched_df)\n", + "\n", + "# Persist PHI assessment results\n", + "(\n", + " phi_df\n", + " .select(\"path\", \"local_path\", \"extension\", \"response\")\n", + " .write.format(\"delta\")\n", + " .mode(\"overwrite\")\n", + " .option(\"overwriteSchema\", \"true\")\n", + " .saveAsTable(config[\"PHI_ASSESSMENT_TABLE\"])\n", + ")\n", + "\n", + "display(phi_df.select(\"local_path\", \"response\"))" + ] + }, { "cell_type": "code", "execution_count": 0, @@ -226,7 +373,7 @@ "source": [ "# Persist the file catalog to the Delta table defined in config.yaml (INDEX).\n", "# Use write_mode='overwrite' for a full refresh, or 'append' for incremental.\n", - "catalog.save(enriched_df, mode=WRITE_MODE)\n", + "catalog.save(phi_df, mode=WRITE_MODE)\n", "\n", "print(f\"Saved to: {TABLE} (mode={WRITE_MODE})\")" ] diff --git a/notebooks/tiff/config.yaml-example b/notebooks/tiff/config.yaml-example new file mode 100644 index 00000000..01a78c03 --- /dev/null +++ b/notebooks/tiff/config.yaml-example @@ -0,0 +1,11 @@ +SOURCE_PATH: /Volumes/hls_radiology_east//sample +PATTERN: "*.tiff" +INDEX: .tiff.object_catalog +PHI_ASSESSMENT_TABLE: .tiff.phi_assessment + +# MLflow +MLFLOW_EXPERIMENT_NAME: /Users//tiff +MLFLOW_ARTIFACT_PATH: /Volumes//tiff//mlflow + +# VLM +VLM_ENDPOINT: databricks-llama-4-maverick diff --git a/src/dbx/pixels/tiff/__init__.py b/src/dbx/pixels/tiff/__init__.py index 838b0c3e..c505be02 100644 --- a/src/dbx/pixels/tiff/__init__.py +++ b/src/dbx/pixels/tiff/__init__.py @@ -1,3 +1,4 @@ from dbx.pixels.tiff.tiff_meta_extractor import TiffMetaExtractor +from dbx.pixels.tiff.tiff_vlm_phi_detector import TiffVLMPhiDetector -__all__ = ["TiffMetaExtractor"] +__all__ = ["TiffMetaExtractor", "TiffVLMPhiDetector"] diff --git a/src/dbx/pixels/tiff/tiff_utils.py b/src/dbx/pixels/tiff/tiff_utils.py new file mode 100644 index 00000000..6ffe0b55 --- /dev/null +++ b/src/dbx/pixels/tiff/tiff_utils.py @@ -0,0 +1,191 @@ +"""TIFF utility functions — image conversion for downstream processing. + +Provides ``tiff_to_image()``, the TIFF equivalent of +``dbx.pixels.dicom.dicom_utils.dicom_to_image()``. + +Handles standard TIFF, BigTIFF, and multi-level pyramidal WSI TIFFs +(Philips, Aperio SVS-style, NDPI) by reading the **smallest available +pyramid level** rather than the full-resolution page, so VLM callers +never load a 500 MB slide into memory. + +No dependency on ``dbx.pixels.dicom`` or ``pydicom``. + +Primary backend: ``tifffile``. Falls back to ``Pillow`` if absent. +""" + +from __future__ import annotations + +import io +from typing import Optional + +import numpy as np + +from dbx.pixels.logging import LoggerProvider + +logger = LoggerProvider() + + +# --------------------------------------------------------------------------- +# Internal helpers +# --------------------------------------------------------------------------- + +def _normalize_to_uint8_rgb(arr: np.ndarray) -> np.ndarray: + """Coerce any numpy array to uint8 RGB suitable for JPEG encoding. + + Handles: + - 16-bit / float arrays → normalise to 0-255 + - RGBA (4-channel) → drop alpha channel + - Grayscale (2-D) → replicate to 3 channels + - Single-channel 3-D → replicate to 3 channels + """ + # Drop alpha channel + if arr.ndim == 3 and arr.shape[-1] == 4: + arr = arr[:, :, :3] + + # Convert to uint8 + if arr.dtype != np.uint8: + lo, hi = float(arr.min()), float(arr.max()) + if hi > lo: + arr = ((arr.astype(np.float32) - lo) / (hi - lo) * 255).astype(np.uint8) + else: + arr = np.zeros_like(arr, dtype=np.uint8) + + # Grayscale → RGB + if arr.ndim == 2: + arr = np.stack([arr, arr, arr], axis=-1) + elif arr.ndim == 3 and arr.shape[-1] == 1: + arr = np.concatenate([arr, arr, arr], axis=-1) + + return arr + + +def _tiff_to_array_tifffile(path: str) -> Optional[np.ndarray]: + """Read the smallest pyramid level of a TIFF using ``tifffile``. + + For multi-level WSI TIFFs (e.g. Philips BigTIFF with 9 pyramid levels), + returns ``series[0].levels[-1].asarray()`` — the lowest-resolution level. + For single-level TIFFs, returns ``series[0].asarray()``. + """ + import tifffile + + try: + with tifffile.TiffFile(path) as tif: + if tif.series: + series = tif.series[0] + if hasattr(series, "levels") and len(series.levels) > 1: + # Multi-level pyramid — pick the smallest level + return series.levels[-1].asarray() + return series.asarray() + # No series metadata — fall back to page 0 + return tif.pages[0].asarray() + except Exception as e: + logger.exception(f"tifffile read failed for {path}: {e}") + return None + + +def _tiff_to_array_pillow(path: str) -> Optional[np.ndarray]: + """Read the last frame of a TIFF using ``Pillow`` (fallback). + + For pyramidal TIFFs the last IFD is the lowest-resolution page, + making it the best thumbnail candidate without tifffile. + + ``Image.MAX_IMAGE_PIXELS`` is temporarily disabled because WSI slides + legitimately exceed Pillow's default decompression-bomb limit (the full + resolution header triggers the guard even though we only decompress the + small last frame). + """ + from PIL import Image + + try: + _prev = Image.MAX_IMAGE_PIXELS + Image.MAX_IMAGE_PIXELS = None # suppress bomb check for large WSI + try: + with Image.open(path) as img: + n_frames = getattr(img, "n_frames", 1) + if n_frames > 1: + img.seek(n_frames - 1) + return np.array(img) + finally: + Image.MAX_IMAGE_PIXELS = _prev # always restore + except Exception as e: + logger.exception(f"Pillow read failed for {path}: {e}") + return None + + +# --------------------------------------------------------------------------- +# Public API +# --------------------------------------------------------------------------- + +def tiff_to_image( + path: str, + max_width: int = 768, + output_path: str = None, + return_type: str = "str", +) -> Optional[str | bytes]: + """Convert a TIFF file to a JPEG thumbnail. + + For multi-level pyramidal WSI TIFFs, reads the smallest available pyramid + level to avoid loading full-resolution pixel data. For single-level + TIFFs, reads the full image and resizes if needed. + + Primary backend: ``tifffile``. Falls back to ``Pillow`` if not installed. + No dependency on ``dbx.pixels.dicom`` or ``pydicom``. + + Args: + path: Local path to the TIFF file. + max_width: Resize thumbnail to this width (default 768). + Set to 0 to skip resize. + output_path: If set, also save the JPEG to this path. + return_type: ``"str"`` — base64-encoded JPEG string (default). + ``"binary"`` — raw JPEG bytes. + + Returns: + Base64 JPEG string, raw JPEG bytes, or ``None`` on failure. + """ + try: + # --- 1. Read pixel data (tifffile primary, Pillow fallback) --- + arr: Optional[np.ndarray] = None + + try: + import tifffile # noqa: F401 + arr = _tiff_to_array_tifffile(path) + except ImportError: + pass + + if arr is None: + arr = _tiff_to_array_pillow(path) + + if arr is None: + logger.error(f"tiff_to_image: could not read pixel data from {path}") + return None + + # --- 2. Normalise to uint8 RGB --- + arr = _normalize_to_uint8_rgb(arr) + + # --- 3. Resize + encode using PIL directly (no DICOM dependency) --- + import base64 as _base64 + from PIL import Image + + img = Image.fromarray(arr) + if max_width > 0 and img.width > max_width: + ratio = max_width / img.width + img = img.resize((max_width, int(img.height * ratio)), Image.LANCZOS) + + if output_path: + img.save(output_path, format="JPEG") + + buf = io.BytesIO() + img.save(buf, format="JPEG") + jpg_bytes = buf.getvalue() + + if return_type == "binary": + return jpg_bytes + if return_type == "str": + return _base64.b64encode(jpg_bytes).decode("utf-8") + + logger.warning(f"tiff_to_image: unknown return_type '{return_type}', returning None.") + return None + + except Exception as e: + logger.exception(f"tiff_to_image failed for {path}: {e}") + return None diff --git a/src/dbx/pixels/tiff/tiff_vlm_phi_detector.py b/src/dbx/pixels/tiff/tiff_vlm_phi_detector.py new file mode 100644 index 00000000..d4f5c633 --- /dev/null +++ b/src/dbx/pixels/tiff/tiff_vlm_phi_detector.py @@ -0,0 +1,208 @@ +"""TiffVLMPhiDetector — Spark ML Transformer for pixel-level PHI detection in TIFF files. + +Fully self-contained: no dependency on ``dbx.pixels.dicom`` or ``pydicom``. + +- Extends ``pyspark.ml.base.Transformer`` +- Applies ``tiff_to_image()`` to render a JPEG thumbnail from the smallest + available pyramid level, then calls a Databricks VLM serving endpoint via + the OpenAI-compatible API. +- Output column schema mirrors ``VLMPhiDetector`` from the DICOM module + (content array, completion_tokens int, prompt_tokens int, + total_tokens int, error string). +""" + +from __future__ import annotations + +import base64 +from dataclasses import dataclass +from typing import Iterator, List, Optional + +import pandas as pd +from pyspark.ml.base import Transformer +from pyspark.sql.functions import col, pandas_udf + +from dbx.pixels.logging import LoggerProvider +from dbx.pixels.tiff.tiff_utils import tiff_to_image + +logger = LoggerProvider() + +__all__ = ["TiffVLMPhiDetector", "VlmResult"] + + +DEFAULT_SYSTEM_PROMPT = ( + "You are an expert in privacy and personal health information (PHI). " + "Per HIPAA rules there are 18 fields considered PHI. " + "Please identify all of the PHI fields found in the image and return a " + "list of pipe-separated named entities, e.g. 'John Smith'|'04-31-1954'|'123 Drury Lane' " + "and nothing else. " + "Don't be fooled by text fields especially acronyms that are not PHI. " + "If there's no PHI detected, return 'No PHI' and nothing else. " + "Answer concisely as requested without explanations." +) + + +@dataclass +class VlmResult: + """Per-image result returned by the VLM PHI detector.""" + content: Optional[List[str]] + completion_tokens: int + prompt_tokens: int + total_tokens: int + error: Optional[str] + + +# --------------------------------------------------------------------------- +# Module-level UDF factory +# --------------------------------------------------------------------------- +# Defined outside the class so Spark Connect can serialise the closure +# without capturing ``self``. Only primitive types are closed over. + +def _make_phi_detector_udf( + endpoint: str, + system_prompt: str, + temperature: float, + num_output_tokens: int, + input_type: str, + max_width: int, +): + """Return a ``pandas_udf`` configured with the given inference parameters.""" + + @pandas_udf( + "content array, completion_tokens int, " + "prompt_tokens int, total_tokens int, error string" + ) + def _extract_udf(paths: Iterator[pd.Series]) -> Iterator[pd.DataFrame]: + from dataclasses import replace as dc_replace + from mlflow.utils.databricks_utils import get_databricks_host_creds + from openai import OpenAI + + creds = get_databricks_host_creds("databricks") + client = OpenAI( + api_key=creds.token, + base_url=f"{creds.host}/serving-endpoints", + timeout=300, + max_retries=3, + ) + _null = VlmResult(None, 0, 0, 0, None) + + for batch in paths: + results = [] + for path in batch: + try: + # --- 1. Get image as base64 JPEG --- + if input_type == "tiff": + b64 = tiff_to_image(path, max_width=max_width, return_type="str") + if b64 is None: + results.append(dc_replace(_null, error=f"tiff_to_image returned None: {path}")) + continue + elif input_type == "image": + local = path[5:] if path.startswith("dbfs:") else path + with open(local, "rb") as fh: + b64 = base64.b64encode(fh.read()).decode("utf-8") + else: # "base64" — already encoded + b64 = path + + # --- 2. VLM inference --- + response = client.chat.completions.create( + model=endpoint, + messages=[ + {"role": "system", "content": system_prompt}, + { + "role": "user", + "content": [{ + "type": "image_url", + "image_url": { + "url": f"data:image/jpeg;base64,{b64}", + "detail": "low", + }, + }], + }, + ], + temperature=float(temperature), + max_tokens=int(num_output_tokens), + ) + + # --- 3. Parse response --- + content = response.choices[0].message.content + if "|" in content: + phi_list = content.split("|") + elif content.strip().lower() == "no phi": + phi_list = [] + else: + phi_list = [content] + + results.append(VlmResult( + phi_list, + response.usage.completion_tokens, + response.usage.prompt_tokens, + response.usage.total_tokens, + None, + )) + + except Exception as exc: + results.append(dc_replace(_null, error=str(exc))) + + yield pd.DataFrame(results) + + return _extract_udf + + +# --------------------------------------------------------------------------- +# Transformer +# --------------------------------------------------------------------------- + +class TiffVLMPhiDetector(Transformer): + """Detect pixel-level PHI in TIFF images using a Databricks VLM endpoint. + + No dependency on ``dbx.pixels.dicom`` or ``pydicom`` — fully self-contained. + + Converts TIFF files to JPEG thumbnails via ``tiff_to_image()`` (smallest + pyramid level for WSI) then calls a Databricks OpenAI-compatible VLM + serving endpoint. + + Args: + endpoint: Databricks serving endpoint name for the VLM. + system_prompt: Override the default HIPAA PHI detection prompt. + temperature: VLM sampling temperature (default 0.0). + num_output_tokens: Maximum tokens in the VLM response (default 200). + inputCol: Input column name (default ``local_path``). + outputCol: Output column name (default ``response``). + input_type: ``"tiff"`` — path to a TIFF file (default). + ``"image"`` — path to a JPEG/PNG file. + ``"base64"`` — already base64-encoded image string. + max_width: Resize thumbnail width before VLM (default 768). + Set to 0 to disable. + """ + + def __init__( + self, + endpoint: str, + system_prompt: str = None, + temperature: float = 0.0, + num_output_tokens: int = 200, + inputCol: str = "local_path", + outputCol: str = "response", + input_type: str = "tiff", + max_width: int = 768, + ): + super().__init__() + self.endpoint = endpoint + self.system_prompt = system_prompt + self.temperature = temperature + self.num_output_tokens = num_output_tokens + self.inputCol = inputCol + self.outputCol = outputCol + self.input_type = input_type + self.max_width = max_width + + def _transform(self, df): + """Apply VLM PHI detection via ``pandas_udf``.""" + _udf = _make_phi_detector_udf( + endpoint = self.endpoint, + system_prompt = self.system_prompt or DEFAULT_SYSTEM_PROMPT, + temperature = self.temperature, + num_output_tokens = self.num_output_tokens, + input_type = self.input_type, + max_width = self.max_width, + ) + return df.withColumn(self.outputCol, _udf(col(self.inputCol))) From 7d51e31c8f281e87f13d95d98d64ed5aecba9143 Mon Sep 17 00:00:00 2001 From: dmoore247 Date: Tue, 14 Jul 2026 13:12:33 +0000 Subject: [PATCH 5/7] add WSI capabilities - first commit using openslide, add support for all wsi types. Support at this time includes ingest, indexing, phi detection. Fine tuning PHI pixel detection. --- notebooks/wsi/01-ingest-wsi.py | 144 ++++++++++ notebooks/wsi/02-phi-pixel-detection.py | 112 ++++++++ src/dbx/pixels/wsi/__init__.py | 64 +++++ src/dbx/pixels/wsi/catalog.py | 96 +++++++ src/dbx/pixels/wsi/wsi_meta_extractor.py | 309 ++++++++++++++++++++ src/dbx/pixels/wsi/wsi_phi_tags.py | 305 ++++++++++++++++++++ src/dbx/pixels/wsi/wsi_utils.py | 293 +++++++++++++++++++ src/dbx/pixels/wsi/wsi_vlm_phi_detector.py | 258 +++++++++++++++++ tests/dbx/test_wsi.py | 320 +++++++++++++++++++++ 9 files changed, 1901 insertions(+) create mode 100644 notebooks/wsi/01-ingest-wsi.py create mode 100644 notebooks/wsi/02-phi-pixel-detection.py create mode 100644 src/dbx/pixels/wsi/__init__.py create mode 100644 src/dbx/pixels/wsi/catalog.py create mode 100644 src/dbx/pixels/wsi/wsi_meta_extractor.py create mode 100644 src/dbx/pixels/wsi/wsi_phi_tags.py create mode 100644 src/dbx/pixels/wsi/wsi_utils.py create mode 100644 src/dbx/pixels/wsi/wsi_vlm_phi_detector.py create mode 100644 tests/dbx/test_wsi.py diff --git a/notebooks/wsi/01-ingest-wsi.py b/notebooks/wsi/01-ingest-wsi.py new file mode 100644 index 00000000..32213510 --- /dev/null +++ b/notebooks/wsi/01-ingest-wsi.py @@ -0,0 +1,144 @@ +# Databricks notebook source +# /// script +# [tool.databricks.environment] +# environment_version = "5" +# /// +# DBTITLE 1,WSI Ingest — Overview +# MAGIC %md +# MAGIC # Whole Slide Image (WSI) Ingest Pipeline +# MAGIC +# MAGIC Unified ingest for **all OpenSlide-supported formats**: Aperio SVS, Hamamatsu NDPI, Leica SCN, MIRAX MRXS, Philips TIFF, Ventana BIF/TIF, Sakura SVSLIDE, generic TIFF. +# MAGIC +# MAGIC **Pipeline** +# MAGIC 1. Install dependencies (`openslide-python`, `openslide-bin`, `tifffile`) +# MAGIC 2. Add `src/` to path for editable development +# MAGIC 3. Configure source paths and target Delta table +# MAGIC 4. `WSICatalog.catalog()` — discover all WSI files (multi-extension) +# MAGIC 5. `WSIMetaExtractor.transform()` — extract metadata into `meta VARIANT` +# MAGIC 6. Save enriched catalog to Delta +# MAGIC 7. Verify and explore results + +# COMMAND ---------- + +# DBTITLE 1,Install dependencies +# MAGIC %pip install openslide-python openslide-bin tifffile imagecodecs Pillow -q + +# COMMAND ---------- + +# DBTITLE 1,Add src/ to sys.path +import sys + +SRC_PATH = "/Workspace/Users/douglas.moore@databricks.com/pixels-tiff/src" +if SRC_PATH not in sys.path: + sys.path.insert(0, SRC_PATH) + print(f"Added to sys.path: {SRC_PATH}") +else: + print(f"Already on sys.path: {SRC_PATH}") + +# COMMAND ---------- + +# DBTITLE 1,Configuration +# --- Source paths (all UC Volume locations with WSI files) --- +SOURCE_PATHS = [ + "/Volumes/hls_radiology_east/orthanc_demo/raw_images/Aperio", + "/Volumes/hls_radiology_east/orthanc_demo/raw_images/Generic-TIFF", + "/Volumes/hls_radiology_east/orthanc_demo/raw_images/Philips-TIFF", + "/Volumes/hls_radiology_east/osuwmc/sample", + "/Volumes/hls_radiology_east/jsl/samples", +] + +# --- Target Delta table and volume --- +TABLE = "dmoore.wsi.object_catalog" +VOLUME = "dmoore.wsi.wsi_volume" + +print(f"Source paths: {len(SOURCE_PATHS)}") +print(f"Target table: {TABLE}") +print(f"Volume: {VOLUME}") + +# COMMAND ---------- + +# DBTITLE 1,Initialize WSICatalog +from dbx.pixels.wsi import WSICatalog, WSIMetaExtractor, OPENSLIDE_PATTERNS + +catalog = WSICatalog(spark, table=TABLE, volume=VOLUME) +print(f"Catalog: {catalog}") +print(f"WSI patterns: {OPENSLIDE_PATTERNS}") + +# COMMAND ---------- + +# DBTITLE 1,Discover & catalog all WSI files +from functools import reduce +from pyspark.sql import DataFrame + +# Catalog each source path and union results +dfs = [] +for path in SOURCE_PATHS: + try: + df = catalog.catalog(path=path) # Uses all WSI patterns by default + dfs.append(df) + print(f" ✓ {path}") + except Exception as e: + print(f" ✗ {path}: {e}") + +if dfs: + catalog_df = reduce(DataFrame.union, dfs).dropDuplicates(["path"]).repartition(8) + print(f"\nTotal WSI files discovered: {catalog_df.count()}") + display(catalog_df.select("path", "modificationTime", "length", "extension")) +else: + raise ValueError("No WSI files found in any source path") + +# COMMAND ---------- + +# DBTITLE 1,Extract WSI metadata +# WSIMetaExtractor adds a `meta` VARIANT column with: +# - All OpenSlide properties (vendor-specific: aperio.*, hamamatsu.*, philips.*, etc.) +# - Derived fields: _wsi_vendor, _wsi_format, _wsi_width, _wsi_height, _wsi_level_count, +# _wsi_mpp_x, _wsi_mpp_y, _wsi_objective_power, _wsi_has_label, _wsi_has_macro +# - phi_tag_report: PHI classification of all properties + +extractor = WSIMetaExtractor(catalog, inputCol="local_path", outputCol="meta") +enriched_df = extractor.transform(catalog_df) + +print(f"Schema: {[f.name for f in enriched_df.schema.fields]}") +display(enriched_df.select("local_path", "extension", "meta")) + +# COMMAND ---------- + +# DBTITLE 1,Save to Delta table +# Save the enriched catalog to Delta +catalog.save(enriched_df, mode="overwrite") + +print(f"✓ Saved to {TABLE}") +print(f" Rows: {spark.table(TABLE).count()}") + +# COMMAND ---------- + +# DBTITLE 1,Explore metadata — vendor breakdown +# MAGIC %sql +# MAGIC -- Vendor breakdown from extracted metadata +# MAGIC SELECT +# MAGIC meta:_wsi_vendor::STRING AS vendor, +# MAGIC meta:_wsi_format::STRING AS format, +# MAGIC meta:_wsi_backend::STRING AS backend, +# MAGIC COUNT(*) AS file_count, +# MAGIC AVG(meta:_wsi_width::INT) AS avg_width, +# MAGIC AVG(meta:_wsi_height::INT) AS avg_height, +# MAGIC AVG(meta:_wsi_mpp_x::DOUBLE) AS avg_mpp_x +# MAGIC FROM dmoore.wsi.object_catalog +# MAGIC GROUP BY 1, 2, 3 +# MAGIC ORDER BY file_count DESC + +# COMMAND ---------- + +# DBTITLE 1,Explore metadata — PHI exposure +# MAGIC %sql +# MAGIC -- PHI tag exposure summary +# MAGIC SELECT +# MAGIC local_path, +# MAGIC meta:_wsi_vendor::STRING AS vendor, +# MAGIC meta:_wsi_has_label::BOOLEAN AS has_label, +# MAGIC meta:_wsi_has_macro::BOOLEAN AS has_macro, +# MAGIC SIZE(CAST(meta:phi_tag_report AS ARRAY)) AS phi_tag_count +# MAGIC FROM dmoore.wsi.object_catalog +# MAGIC WHERE SIZE(CAST(meta:phi_tag_report AS ARRAY)) > 0 +# MAGIC ORDER BY phi_tag_count DESC diff --git a/notebooks/wsi/02-phi-pixel-detection.py b/notebooks/wsi/02-phi-pixel-detection.py new file mode 100644 index 00000000..f8a5ab6b --- /dev/null +++ b/notebooks/wsi/02-phi-pixel-detection.py @@ -0,0 +1,112 @@ +# Databricks notebook source +# /// script +# [tool.databricks.environment] +# environment_version = "5" +# /// +# DBTITLE 1,PHI Pixel Detection — Overview +# MAGIC %md +# MAGIC # WSI Pixel-Level PHI Detection +# MAGIC +# MAGIC Uses `WSIVLMPhiDetector` (OpenSlide-backed) to detect Protected Health Information +# MAGIC in the **label** and **macro** associated images of Whole Slide Image files. +# MAGIC +# MAGIC OpenSlide extracts the sub-image (no full-resolution data loaded), then a Databricks +# MAGIC VLM serving endpoint identifies PHI entities (patient names, DOBs, accession numbers, barcodes). +# MAGIC +# MAGIC **Prerequisite**: Run `01-ingest-wsi` first to populate `dmoore.wsi.object_catalog`. + +# COMMAND ---------- + +# DBTITLE 1,Install dependencies +# MAGIC %pip install openslide-python openslide-bin tifffile imagecodecs Pillow mlflow openai -q + +# COMMAND ---------- + +# DBTITLE 1,Add src/ to sys.path +import sys + +SRC_PATH = "/Workspace/Users/douglas.moore@databricks.com/pixels-tiff/src" +if SRC_PATH not in sys.path: + sys.path.insert(0, SRC_PATH) + print(f"Added to sys.path: {SRC_PATH}") +else: + print(f"Already on sys.path: {SRC_PATH}") + +# COMMAND ---------- + +# DBTITLE 1,Configuration +# --- Source: enriched metadata from 01-ingest-wsi --- +TABLE = "dmoore.wsi.object_catalog" + +# --- VLM endpoint for PHI detection --- +VLM_ENDPOINT = "phi-detection" + +print(f"Source table: {TABLE}") +print(f"VLM endpoint: {VLM_ENDPOINT}") + +# COMMAND ---------- + +# DBTITLE 1,VLM PHI detection — label images +from pyspark.sql.functions import lit +from dbx.pixels.wsi import WSIVLMPhiDetector + +# Run VLM PHI detection on all three sub-image series: +# label — printed patient name, barcode, accession (primary PHI target) +# macro — overview image (may contain handwritten labels) +# tissue — smallest pyramid level (rarely contains PHI, but checks embedded text) +# +# wsi_to_image returns None (recorded as error, no VLM call) when a series is absent. + +base_df = spark.sql(f""" + SELECT local_path, extension, + meta:_wsi_vendor::STRING AS vendor, + meta:_wsi_format::STRING AS format + FROM {TABLE} +""").repartition(8) + +print(f"Total WSI files: {base_df.count()}") + +results = [] +for series in ["label", "macro", "tissue"]: + detector = WSIVLMPhiDetector( + endpoint=VLM_ENDPOINT, + inputCol="local_path", + outputCol="vlm_phi", + series=series, + max_width=768, + temperature=0.0, + ) + series_df = detector.transform(base_df).withColumn("series", lit(series)) + results.append(series_df) + +phi_df = results[0].unionByName(results[1]).unionByName(results[2]) + +print(f"Total results (files × series): {phi_df.count()}") +display(phi_df.select("local_path", "vendor", "format", "series", "vlm_phi.*")) + +# COMMAND ---------- + +# DBTITLE 1,PHI detection summary +from pyspark.sql.functions import col, size + +results = phi_df.select( + "local_path", + "vendor", + "series", + col("vlm_phi.content").alias("phi_entities"), + col("vlm_phi.error").alias("error"), + col("vlm_phi.total_tokens").alias("tokens"), +) + +total = results.count() +with_phi = results.filter(size("phi_entities") > 0).count() +with_error = results.filter(col("error").isNotNull()).count() + +print("=== VLM PHI Detection Summary ===") +print(f"Total scans (files × series): {total}") +print(f"Scans with PHI detected: {with_phi}") +print(f"Scans with errors: {with_error}") +print() + +# Show files/series where PHI was found +display(results.filter(size("phi_entities") > 0)) diff --git a/src/dbx/pixels/wsi/__init__.py b/src/dbx/pixels/wsi/__init__.py new file mode 100644 index 00000000..6b67d252 --- /dev/null +++ b/src/dbx/pixels/wsi/__init__.py @@ -0,0 +1,64 @@ +"""dbx.pixels.wsi — Unified Whole Slide Image handler for databricks-pixels. + +Supports all OpenSlide-recognized WSI formats: +- Aperio SVS +- Hamamatsu NDPI / VMS / VMU +- Leica SCN +- MIRAX MRXS +- Philips TIFF +- Sakura SVSLIDE +- Trestle TIF +- Ventana BIF / TIF +- Generic TIFF + +Install alongside ``databricks-pixels`` and extend the ``dbx.pixels`` namespace:: + + import sys + SRC_PATH = "/Workspace/Users//pixels-tiff/src" + if SRC_PATH not in sys.path: + sys.path.insert(0, SRC_PATH) + + from dbx.pixels.wsi import WSICatalog, WSIMetaExtractor, WSIVLMPhiDetector, wsi_to_image +""" + +from dbx.pixels.wsi.catalog import WSICatalog +from dbx.pixels.wsi.wsi_meta_extractor import WSIMetaExtractor +from dbx.pixels.wsi.wsi_phi_tags import ( + LARGE_TAGS, + NOT_PHI_TAGS, + OPENSLIDE_PATTERNS, + PHI_TAGS, + QUESTIONABLE_TAGS, + SUPPORTED_EXTENSIONS, + classify_tag, + classify_tags, + scrub_image_description, +) +from dbx.pixels.wsi.wsi_utils import ( + wsi_detect_format, + wsi_get_properties, + wsi_to_image, +) +from dbx.pixels.wsi.wsi_vlm_phi_detector import WSIVLMPhiDetector + +__all__ = [ + # Classes + "WSICatalog", + "WSIMetaExtractor", + "WSIVLMPhiDetector", + # Utils + "wsi_to_image", + "wsi_detect_format", + "wsi_get_properties", + # PHI tags + "classify_tag", + "classify_tags", + "scrub_image_description", + "PHI_TAGS", + "QUESTIONABLE_TAGS", + "NOT_PHI_TAGS", + "LARGE_TAGS", + # Constants + "SUPPORTED_EXTENSIONS", + "OPENSLIDE_PATTERNS", +] diff --git a/src/dbx/pixels/wsi/catalog.py b/src/dbx/pixels/wsi/catalog.py new file mode 100644 index 00000000..998f2d96 --- /dev/null +++ b/src/dbx/pixels/wsi/catalog.py @@ -0,0 +1,96 @@ +"""WSICatalog — extends the base Catalog for all Whole Slide Image formats. + +Unified catalog handler that discovers and indexes WSI files across all +OpenSlide-supported formats (Aperio SVS, Hamamatsu NDPI/VMS/VMU, Leica SCN, +MIRAX MRXS, Philips TIFF, Sakura SVSLIDE, Trestle TIF, Ventana BIF/TIF, +generic TIFF). + +Overrides: + - catalog() uses a multi-extension pattern by default + - Supports per-format or all-format discovery + +Note on glob limitations: Spark's binaryFile reader ``pathGlobFilter`` does +not support brace expansion (``{*.svs,*.tiff}``). When multiple extensions +are needed, WSICatalog issues multiple catalog() calls and unions them. +""" + +from __future__ import annotations + +from pyspark.sql import DataFrame + +from dbx.pixels.catalog import Catalog +from dbx.pixels.wsi.wsi_phi_tags import OPENSLIDE_PATTERNS + + +# Default patterns for common WSI formats +_DEFAULT_PATTERNS = OPENSLIDE_PATTERNS + + +class WSICatalog(Catalog): + """Object catalog for Whole Slide Images (all OpenSlide-supported formats). + + Extends :class:`dbx.pixels.Catalog` with WSI-specific defaults: + - ``catalog()`` discovers files matching all WSI extensions by default + - Supports single-pattern or multi-pattern modes + + Args: + spark: Active SparkSession. + table: Fully qualified UC table name (e.g. ``dmoore.wsi.object_catalog``). + volume: Fully qualified UC volume name (e.g. ``dmoore.wsi.wsi_volume``). + """ + + def __init__(self, spark, table: str, volume: str = None): + if volume is None: + # Derive volume from catalog.schema + catalog_name, schema_name, _ = table.split(".") + volume = f"{catalog_name}.{schema_name}.wsi_volume" + super().__init__(spark, table=table, volume=volume) + + def catalog( + self, + path: str, + pattern: str = None, + patterns: list[str] = None, + **kwargs, + ) -> DataFrame: + """Catalog WSI files at the given path. + + Args: + path: Root directory to scan. + pattern: Single glob pattern (e.g. ``'*.svs'``). If provided, + only this pattern is used. + patterns: List of glob patterns to scan. Defaults to all + OpenSlide-supported extensions. Each pattern triggers + a separate scan; results are unioned. + **kwargs: Additional arguments forwarded to + :meth:`Catalog.catalog` (recurse, streaming, etc.). + + Returns: + DataFrame with cataloged file metadata. + """ + if pattern is not None: + # Single pattern mode (backward-compatible) + return super().catalog(path, pattern=pattern, **kwargs) + + if patterns is None: + patterns = _DEFAULT_PATTERNS + + # Multi-pattern mode: scan each extension and union results + dfs = [] + for pat in patterns: + try: + df = super().catalog(path, pattern=pat, **kwargs) + dfs.append(df) + except Exception: + # Some patterns may match zero files — that's fine + pass + + if not dfs: + # Return empty DataFrame with correct schema + return super().catalog(path, pattern="*.NONEXISTENT_EXTENSION_PLACEHOLDER", **kwargs) + + result = dfs[0] + for df in dfs[1:]: + result = result.union(df) + + return result.dropDuplicates(["path"]) diff --git a/src/dbx/pixels/wsi/wsi_meta_extractor.py b/src/dbx/pixels/wsi/wsi_meta_extractor.py new file mode 100644 index 00000000..83c44b28 --- /dev/null +++ b/src/dbx/pixels/wsi/wsi_meta_extractor.py @@ -0,0 +1,309 @@ +"""WSIMetaExtractor — Spark ML Transformer that extracts metadata from any OpenSlide-supported WSI. + +Unified handler for all Whole Slide Image formats: +- Aperio SVS +- Hamamatsu NDPI / VMS / VMU +- Leica SCN +- MIRAX MRXS +- Philips TIFF +- Sakura SVSLIDE +- Trestle TIF +- Ventana BIF / TIF +- Generic TIFF + +Primary backend: ``openslide-python`` + ``openslide-bin``. +Fallback backend: ``tifffile`` (for non-WSI TIFFs that OpenSlide cannot open). + +Architecture mirrors ``SVSMetaExtractor`` and ``TiffMetaExtractor``: +- Extends ``pyspark.ml.pipeline.Transformer`` +- Implements ``_transform(df)`` +- Uses ``mapInPandas`` with ``ThreadPoolExecutor`` for concurrent I/O +- Outputs a single ``meta`` column as a parsed VARIANT + +All derived fields (vendor, dimensions, levels, associated images, mpp, +magnification, phi_tag_report) are merged into the properties dict before +JSON serialisation. +""" + +from __future__ import annotations + +import json +from concurrent.futures import ThreadPoolExecutor +from typing import Iterator + +import pandas as pd +import pyspark.sql.types as t +from pyspark.ml.pipeline import Transformer +from pyspark.sql.functions import expr + +from dbx.pixels.wsi.wsi_phi_tags import LARGE_TAGS, classify_tags + + +class WSIMetaExtractor(Transformer): + """Extract metadata from any WSI file into the ``meta VARIANT`` column. + + Uses ``openslide`` as the primary backend; falls back to ``tifffile`` + for files that OpenSlide cannot detect (plain TIFFs, OME-TIFF without + WSI structure). + + Args: + catalog: Catalog instance (provides path/anon context). + inputCol: Column with worker-accessible file paths (default ``local_path``). + outputCol: Output column name (default ``meta``). + maxWorkers: ``ThreadPoolExecutor`` concurrency per Spark task (default 32). + useVariant: Parse JSON string to VARIANT via ``parse_json()`` (default True). + filterLargeTags: Drop large binary tags before JSON serialisation (default True). + """ + + MAX_WORKERS = 32 + + def __init__( + self, + catalog, + inputCol: str = "local_path", + outputCol: str = "meta", + maxWorkers: int = None, + useVariant: bool = True, + filterLargeTags: bool = True, + ): + self.catalog = catalog + self.inputCol = inputCol + self.outputCol = outputCol + self.maxWorkers = maxWorkers or self.MAX_WORKERS + self.useVariant = useVariant + self.filterLargeTags = filterLargeTags + + # ------------------------------------------------------------------ + # Internal: OpenSlide backend (handles all WSI formats) + # ------------------------------------------------------------------ + + @staticmethod + def _process_openslide(path: str) -> str: + """Extract metadata using OpenSlide (primary backend).""" + import openslide + + from dbx.pixels.wsi.wsi_phi_tags import classify_tags + + try: + slide = openslide.OpenSlide(path) + props = dict(slide.properties) + associated = list(slide.associated_images.keys()) + + # Vendor detection + vendor = props.get("openslide.vendor", "unknown") + + # MPP (microns per pixel) — universal across vendors + mpp_x = props.get(openslide.PROPERTY_NAME_MPP_X) + mpp_y = props.get(openslide.PROPERTY_NAME_MPP_Y) + objective_power = props.get(openslide.PROPERTY_NAME_OBJECTIVE_POWER) + + meta = { + **props, + # --- Derived WSI fields --- + "_wsi_vendor": vendor, + "_wsi_format": _detect_format_name(path, vendor), + "_wsi_width": slide.dimensions[0], + "_wsi_height": slide.dimensions[1], + "_wsi_level_count": slide.level_count, + "_wsi_level_dimensions": [list(d) for d in slide.level_dimensions], + "_wsi_level_downsamples": list(slide.level_downsamples), + "_wsi_mpp_x": float(mpp_x) if mpp_x else None, + "_wsi_mpp_y": float(mpp_y) if mpp_y else None, + "_wsi_objective_power": float(objective_power) if objective_power else None, + "_wsi_has_label": "label" in associated, + "_wsi_has_macro": "macro" in associated, + "_wsi_has_thumbnail": "thumbnail" in associated, + "_wsi_associated_images": associated, + "_wsi_backend": "openslide", + "phi_tag_report": classify_tags(props), + } + slide.close() + return json.dumps(meta, default=str) + + except Exception as err: + return json.dumps( + {"error": str(err), "udf": "wsi_meta_extractor_openslide", "path": path} + ) + + # ------------------------------------------------------------------ + # Internal: tifffile fallback (non-WSI TIFFs) + # ------------------------------------------------------------------ + + @staticmethod + def _process_tifffile(path: str, filter_large: bool = True) -> str: + """Extract metadata with tifffile (fallback for non-WSI TIFFs).""" + import tifffile + + from dbx.pixels.wsi.wsi_phi_tags import LARGE_TAGS, classify_tags + + try: + with tifffile.TiffFile(path) as tif: + page = tif.pages[0] + + tags: dict[str, str] = { + tag.name: str(tag.value) + for tag in page.tags.values() + if not filter_large or tag.name not in LARGE_TAGS + } + + # Per-series summary + series_info = [] + for s in tif.series: + entry: dict = { + "shape": list(s.shape), + "axes": s.axes, + "dtype": str(s.dtype), + } + if hasattr(s, "levels"): + entry["level_count"] = len(s.levels) + series_info.append(entry) + + meta = { + **tags, + "_wsi_vendor": "generic-tiff", + "_wsi_format": _detect_tifffile_format(tif), + "_wsi_width": page.imagewidth, + "_wsi_height": page.imagelength, + "_wsi_level_count": len(tif.series[0].levels) if tif.series and hasattr(tif.series[0], "levels") else len(tif.pages), + "_wsi_level_dimensions": None, + "_wsi_level_downsamples": None, + "_wsi_mpp_x": None, + "_wsi_mpp_y": None, + "_wsi_objective_power": None, + "_wsi_has_label": _has_labeled_ifd(tif, "label"), + "_wsi_has_macro": _has_labeled_ifd(tif, "macro"), + "_wsi_has_thumbnail": False, + "_wsi_associated_images": [], + "_wsi_backend": "tifffile", + "_wsi_page_count": len(tif.pages), + "_wsi_series_count": len(tif.series), + "_wsi_is_bigtiff": tif.is_bigtiff, + "_wsi_is_ome": tif.is_ome, + "_wsi_is_svs": tif.is_svs, + "_wsi_is_ndpi": getattr(tif, "is_ndpi", False), + "_wsi_series": series_info, + "phi_tag_report": classify_tags(tags), + } + return json.dumps(meta, default=str) + + except Exception as err: + return json.dumps( + {"error": str(err), "udf": "wsi_meta_extractor_tifffile", "path": path} + ) + + # ------------------------------------------------------------------ + # Internal: Dispatcher + # ------------------------------------------------------------------ + + @staticmethod + def _process_file(path: str, filter_large: bool = True) -> str: + """Dispatch to OpenSlide (primary) or tifffile (fallback).""" + try: + import openslide + + # Check if OpenSlide can detect the format + try: + fmt = openslide.OpenSlide.detect_format(path) + except Exception: + fmt = None + + if fmt is not None: + return WSIMetaExtractor._process_openslide(path) + else: + # OpenSlide doesn't recognize it; try tifffile + return WSIMetaExtractor._process_tifffile(path, filter_large) + + except ImportError: + # openslide not installed; fall back to tifffile + return WSIMetaExtractor._process_tifffile(path, filter_large) + + # ------------------------------------------------------------------ + # Transformer entry point + # ------------------------------------------------------------------ + + def _transform(self, df): + """Apply WSI metadata extraction using mapInPandas with concurrent I/O.""" + input_col = self.inputCol + output_col = self.outputCol + max_workers = self.maxWorkers + filter_large = self.filterLargeTags + + out_schema = t.StructType( + list(df.schema.fields) + + [t.StructField(output_col, t.StringType(), True)] + ) + + def _extract_meta_batch( + iterator: Iterator[pd.DataFrame], + ) -> Iterator[pd.DataFrame]: + for pdf in iterator: + paths = pdf[input_col].tolist() + with ThreadPoolExecutor(max_workers=max_workers) as executor: + results = list( + executor.map( + lambda p: WSIMetaExtractor._process_file(p, filter_large), + paths, + ) + ) + pdf[output_col] = results + yield pdf + + df = df.mapInPandas(_extract_meta_batch, schema=out_schema) + + if self.useVariant: + df = df.withColumn( + self.outputCol, + expr(f"parse_json(`{self.outputCol}`)"), + ) + + return df + + +# --------------------------------------------------------------------------- +# Module-level helpers +# --------------------------------------------------------------------------- + +def _detect_format_name(path: str, vendor: str) -> str: + """Return a human-readable format name based on vendor + extension.""" + ext = path.rsplit(".", 1)[-1].lower() if "." in path else "" + _vendor_map = { + "aperio": "Aperio SVS", + "hamamatsu": "Hamamatsu NDPI" if ext == "ndpi" else "Hamamatsu VMS/VMU", + "leica": "Leica SCN", + "mirax": "MIRAX MRXS", + "philips": "Philips TIFF", + "sakura": "Sakura SVSLIDE", + "trestle": "Trestle TIF", + "ventana": "Ventana BIF" if ext == "bif" else "Ventana TIF", + "generic-tiff": "Generic TIFF", + "dicom": "DICOM WSI", + } + return _vendor_map.get(vendor, f"Unknown ({vendor})") + + +def _detect_tifffile_format(tif) -> str: + """Detect format from tifffile attributes.""" + if tif.is_svs: + return "Aperio SVS (tifffile)" + if getattr(tif, "is_ndpi", False): + return "Hamamatsu NDPI (tifffile)" + if tif.is_ome: + return "OME-TIFF" + if tif.is_bigtiff: + return "BigTIFF" + return "Standard TIFF" + + +def _has_labeled_ifd(tif, target: str) -> bool: + """Check if a tifffile TiffFile has an IFD with a matching ImageDescription.""" + target_lc = target.strip().lower() + for page in tif.pages: + try: + desc = page.description + if isinstance(desc, bytes): + desc = desc.decode("utf-8", errors="replace") + if desc.strip().strip("\x00").lower() == target_lc: + return True + except Exception: + continue + return False diff --git a/src/dbx/pixels/wsi/wsi_phi_tags.py b/src/dbx/pixels/wsi/wsi_phi_tags.py new file mode 100644 index 00000000..3241683f --- /dev/null +++ b/src/dbx/pixels/wsi/wsi_phi_tags.py @@ -0,0 +1,305 @@ +"""PHI tag classification for Whole Slide Images (all OpenSlide-supported formats). + +Unified classification covering vendor-specific metadata properties from: +- Aperio SVS (openslide.vendor = "aperio") +- Hamamatsu NDPI/VMS/VMU (openslide.vendor = "hamamatsu") +- Leica SCN (openslide.vendor = "leica") +- MIRAX MRXS (openslide.vendor = "mirax") +- Philips TIFF (openslide.vendor = "philips") +- Sakura SVSLIDE (openslide.vendor = "sakura") +- Trestle TIF (openslide.vendor = "trestle") +- Ventana BIF/TIF (openslide.vendor = "ventana") +- Generic TIFF (openslide.vendor = "generic-tiff") +- Standard TIFF tags (via tifffile fallback) + +Classification tiers +-------------------- +PHI — Directly identifies a patient or operator. +QUESTIONABLE — May contain PHI depending on site configuration. +NOT_PHI — Pure scanner / geometry / technical parameters. + +Public API +---------- + classify_tag(key) -> str + classify_tags(properties: dict) -> list[dict] + scrub_image_description(desc) -> str + SUPPORTED_EXTENSIONS -> set[str] + OPENSLIDE_PATTERNS -> list[str] +""" + +from __future__ import annotations + +# --------------------------------------------------------------------------- +# Supported WSI file extensions (OpenSlide + tifffile fallback) +# --------------------------------------------------------------------------- +SUPPORTED_EXTENSIONS: set[str] = { + ".svs", # Aperio + ".tif", # Generic TIFF, Trestle, Ventana + ".tiff", # Generic TIFF, Philips + ".bif", # Ventana + ".ndpi", # Hamamatsu + ".mrxs", # MIRAX (index file) + ".vms", # Hamamatsu (virtual microscope slide) + ".vmu", # Hamamatsu (virtual microscope uncompressed) + ".scn", # Leica + ".svslide", # Sakura + ".dcm", # DICOM WSI +} + +# Glob patterns for use with Spark binaryFile reader or Catalog.catalog() +OPENSLIDE_PATTERNS: list[str] = [ + "*.svs", "*.tif", "*.tiff", "*.bif", "*.ndpi", + "*.mrxs", "*.vms", "*.vmu", "*.scn", "*.svslide", +] + +# --------------------------------------------------------------------------- +# PHI: Directly identifies a person (any vendor) +# --------------------------------------------------------------------------- +PHI_TAGS: set[str] = { + # --- Aperio SVS --- + "aperio.Patient", + "aperio.PatientID", + "aperio.DOB", + "aperio.MRN", + "aperio.AccessionNumber", + "aperio.ClinicID", + "aperio.ClinicalTrialID", + "aperio.Procedure", + "aperio.Diagnosis", + "aperio.Id", + # Aperio short-key variants (from ImageDescription pipe section) + "Patient", + "PatientID", + "DOB", + "MRN", + "AccessionNumber", + "ClinicID", + "ClinicalTrialID", + "Procedure", + "Diagnosis", + "Id", + # --- Hamamatsu NDPI --- + "hamamatsu.SourceLens", # Operator-configured; may encode tech ID + "hamamatsu.Reference", # Patient/case reference + # --- Leica SCN --- + "leica.device-model", # Sometimes contains operator info + # --- Philips --- + "philips.PIM_DP_UFS_BARCODE", # Barcode text (patient label) + # --- Sakura --- + "sakura.NominalLensMagnification", # Technical, but barcode fields below + # --- Generic TIFF tags (from tifffile) --- + "Artist", # tag 315 — person who created the image + "HostComputer", # tag 316 — workstation/operator ID +} + +# --------------------------------------------------------------------------- +# QUESTIONABLE: May contain PHI depending on site configuration +# --------------------------------------------------------------------------- +QUESTIONABLE_TAGS: set[str] = { + # --- Aperio --- + "aperio.Date", + "aperio.Time", + "aperio.Clinic", + "aperio.Pathologist", + "aperio.Title", + "aperio.Filename", + "aperio.User", + "aperio.ImageID", + # Aperio short-key variants + "Date", + "Time", + "Clinic", + "Pathologist", + "Title", + "Filename", + "User", + "ImageID", + # --- Hamamatsu --- + "hamamatsu.Created", # Scan timestamp + "hamamatsu.Updated", # Modification timestamp + # --- Leica --- + "leica.creation-date", + "leica.device-version", + # --- Philips --- + "philips.DICOM_ACQUISITION_DATETIME", + "philips.DICOM_DATE_OF_LAST_CALIBRATION", + "philips.PIM_DP_SCANNER_OPERATOR_ID", + # --- Ventana --- + "ventana.ScanDate", + "ventana.ScanTime", + "ventana.Operator", + # --- MIRAX --- + "mirax.GENERAL.SLIDE_CREATIONDATETIME", + "mirax.GENERAL.SLIDE_NAME", + # --- Generic TIFF --- + "ImageDescription", # tag 270 — free-text; may embed patient info + "DateTime", # tag 306 — creation timestamp + "Copyright", # tag 33432 + "DateTimeOriginal", # EXIF 36867 + "DateTimeDigitized", # EXIF 36868 + # --- OpenSlide common --- + "openslide.comment", # May contain free-text with PHI + # --- Artist/Copyright short keys --- + "tiff.Artist", + "tiff.Copyright", +} + +# --------------------------------------------------------------------------- +# LARGE_TAGS: Binary/array tags to skip during metadata extraction +# (avoids bloating the JSON output) +# --------------------------------------------------------------------------- +LARGE_TAGS: set[str] = { + "JPEGTables", + "JPEGQTables", + "JPEGDCTables", + "JPEGACTables", + "TileOffsets", + "TileByteCounts", + "StripOffsets", + "StripByteCounts", + "ICCProfile", + "ColorMap", + "TransferFunction", + "ReferenceBlackWhite", + "XMP", + "IPTCNAA", + "Photoshop", + "ExifIFD", + "GeoKeyDirectoryTag", + "GeoDoubleParamsTag", + "GeoAsciiParamsTag", + "ImageDepth", + "SubIFDs", +} + +# --------------------------------------------------------------------------- +# NOT_PHI: Pure scanner / technical parameters (representative, not exhaustive) +# --------------------------------------------------------------------------- +NOT_PHI_TAGS: set[str] = { + # OpenSlide standard + "openslide.level-count", + "openslide.mpp-x", + "openslide.mpp-y", + "openslide.objective-power", + "openslide.vendor", + "openslide.quickhash-1", + # Aperio technical + "aperio.AppMag", + "aperio.MPP", + "aperio.ScanScope ID", + "aperio.StripeWidth", + "aperio.Parmset", + "aperio.Filtered", + "aperio.ICC Profile", + # Hamamatsu technical + "hamamatsu.XOffsetFromSlideCentre", + "hamamatsu.YOffsetFromSlideCentre", + "hamamatsu.SourceLens", + # TIFF baseline + "ImageWidth", + "ImageLength", + "BitsPerSample", + "Compression", + "PhotometricInterpretation", + "SamplesPerPixel", + "RowsPerStrip", + "XResolution", + "YResolution", + "PlanarConfiguration", + "ResolutionUnit", + "Software", + "Make", + "Model", + "TileWidth", + "TileLength", + "NewSubfileType", + "Orientation", + "SampleFormat", + # Aperio short keys + "AppMag", + "MPP", + "ScanScope ID", + "StripeWidth", + "Parmset", + "Filtered", + "ICC Profile", +} + + +# --------------------------------------------------------------------------- +# Public API +# --------------------------------------------------------------------------- + +def classify_tag(key: str) -> str: + """Classify a single WSI metadata property key. + + Returns one of: ``'PHI'``, ``'QUESTIONABLE'``, ``'NOT_PHI'``. + Tags not in any lookup default to ``'NOT_PHI'``. + """ + if key in PHI_TAGS: + return "PHI" + if key in QUESTIONABLE_TAGS: + return "QUESTIONABLE" + return "NOT_PHI" + + +def classify_tags(properties: dict) -> list[dict]: + """Classify all WSI metadata properties and return the structured PHI report. + + Args: + properties: Dict mapping property key (str) -> value (str). + + Returns: + List of ``{"tag": key, "value": val, "classification": cls}`` dicts. + Only PHI and QUESTIONABLE entries are included (NOT_PHI omitted). + """ + report = [] + for key, value in properties.items(): + cls = classify_tag(key) + if cls in ("PHI", "QUESTIONABLE"): + report.append({"tag": key, "value": str(value), "classification": cls}) + return report + + +def scrub_image_description(image_desc: str) -> str: + """Scrub PHI/QUESTIONABLE values from Aperio-style ImageDescription strings. + + Aperio format:: + + {header_line}|key = val|key = val|... + + Header line is technical-only (preserved as-is). Each key=value pair + whose key matches PHI or QUESTIONABLE is replaced with ``key = REDACTED``. + + Also handles free-text ImageDescription fields from other vendors by + returning 'REDACTED' if the entire string is classified as PHI. + + Args: + image_desc: Raw ImageDescription string. + + Returns: + Scrubbed string with PHI values replaced. + """ + if not image_desc: + return image_desc + + # Aperio pipe-delimited format + if "|" in image_desc: + parts = image_desc.split("|") + header = parts[0] + rebuilt = [header] + + for kv in parts[1:]: + if " = " in kv: + k, _v = kv.split(" = ", 1) + key_stripped = k.strip() + if key_stripped in PHI_TAGS or key_stripped in QUESTIONABLE_TAGS: + rebuilt.append(f"{k} = REDACTED") + else: + rebuilt.append(kv) + else: + rebuilt.append(kv) + + return "|".join(rebuilt) + + return image_desc diff --git a/src/dbx/pixels/wsi/wsi_utils.py b/src/dbx/pixels/wsi/wsi_utils.py new file mode 100644 index 00000000..6b68a4a0 --- /dev/null +++ b/src/dbx/pixels/wsi/wsi_utils.py @@ -0,0 +1,293 @@ +"""WSI utility functions — image conversion for all OpenSlide-supported formats. + +Provides ``wsi_to_image()``, the unified WSI equivalent of +``dbx.pixels.tiff.tiff_utils.tiff_to_image()``. + +Uses OpenSlide as the primary backend (handles all 13 WSI formats). +Falls back to tifffile for non-WSI TIFFs that OpenSlide cannot open. + +Key features: +- Thumbnail extraction at configurable max_width (uses OpenSlide's + get_thumbnail() which leverages the pyramid — never loads full-res data) +- Associated image extraction (label, macro, thumbnail) via OpenSlide +- Format-agnostic: works with SVS, NDPI, MRXS, SCN, BIF, Philips TIFF, etc. + +No dependency on ``dbx.pixels.dicom`` or ``pydicom``. +""" + +from __future__ import annotations + +import base64 +import io +from typing import Optional + +import numpy as np +from PIL import Image + +from dbx.pixels.logging import LoggerProvider + +logger = LoggerProvider() + + +# --------------------------------------------------------------------------- +# Internal helpers +# --------------------------------------------------------------------------- + +def _normalize_to_uint8_rgb(arr: np.ndarray) -> np.ndarray: + """Coerce any numpy array to uint8 RGB suitable for JPEG encoding. + + Handles: + - 16-bit / float arrays → normalise to 0-255 + - RGBA (4-channel) → composite onto white background + - Grayscale (2-D) → replicate to 3 channels + - Single-channel 3-D → replicate to 3 channels + """ + # RGBA → RGB with white background composite + if arr.ndim == 3 and arr.shape[-1] == 4: + # Alpha composite onto white + alpha = arr[:, :, 3:4].astype(np.float32) / 255.0 + rgb = arr[:, :, :3].astype(np.float32) + white = np.full_like(rgb, 255.0) + arr = (rgb * alpha + white * (1.0 - alpha)).astype(np.uint8) + + # Convert to uint8 + if arr.dtype != np.uint8: + lo, hi = float(arr.min()), float(arr.max()) + if hi > lo: + arr = ((arr.astype(np.float32) - lo) / (hi - lo) * 255).astype(np.uint8) + else: + arr = np.zeros_like(arr, dtype=np.uint8) + + # Grayscale → RGB + if arr.ndim == 2: + arr = np.stack([arr, arr, arr], axis=-1) + elif arr.ndim == 3 and arr.shape[-1] == 1: + arr = np.concatenate([arr, arr, arr], axis=-1) + + return arr + + +def _pil_to_output( + img: Image.Image, + max_width: int, + output_path: Optional[str], + return_type: str, +) -> Optional[str | bytes]: + """Resize, encode to JPEG, and return in requested format.""" + # Resize if needed + if max_width > 0 and img.width > max_width: + ratio = max_width / img.width + new_size = (max_width, int(img.height * ratio)) + img = img.resize(new_size, Image.LANCZOS) + + # Ensure RGB (not RGBA or palette) + if img.mode != "RGB": + img = img.convert("RGB") + + # Encode to JPEG + buf = io.BytesIO() + img.save(buf, format="JPEG", quality=85) + jpeg_bytes = buf.getvalue() + + # Optionally save to disk + if output_path: + with open(output_path, "wb") as f: + f.write(jpeg_bytes) + + if return_type == "binary": + return jpeg_bytes + return base64.b64encode(jpeg_bytes).decode("utf-8") + + +# --------------------------------------------------------------------------- +# OpenSlide backend +# --------------------------------------------------------------------------- + +def _wsi_openslide( + path: str, + series: str = "tissue", + max_width: int = 768, + output_path: Optional[str] = None, + return_type: str = "str", +) -> Optional[str | bytes]: + """Extract image from a WSI using OpenSlide.""" + import openslide + + try: + slide = openslide.OpenSlide(path) + + if series in ("label", "macro", "thumbnail"): + # Associated image extraction + if series in slide.associated_images: + img = slide.associated_images[series] + slide.close() + return _pil_to_output(img, max_width, output_path, return_type) + else: + logger.warning( + f"wsi_to_image: no '{series}' associated image in {path}; " + f"available: {list(slide.associated_images.keys())}. " + f"Falling back to tissue thumbnail." + ) + + # Tissue thumbnail (default) + # Use OpenSlide's get_thumbnail which respects the pyramid + thumb_size = (max_width, max_width) if max_width > 0 else slide.dimensions + img = slide.get_thumbnail(thumb_size) + slide.close() + return _pil_to_output(img, max_width=0, output_path=output_path, return_type=return_type) + + except Exception as e: + logger.exception(f"OpenSlide read failed for {path}: {e}") + return None + + +# --------------------------------------------------------------------------- +# tifffile fallback +# --------------------------------------------------------------------------- + +def _wsi_tifffile( + path: str, + series: str = "tissue", + max_width: int = 768, + output_path: Optional[str] = None, + return_type: str = "str", +) -> Optional[str | bytes]: + """Extract image from a TIFF using tifffile (fallback).""" + import tifffile + + try: + with tifffile.TiffFile(path) as tif: + arr = None + + if series in ("label", "macro"): + # Search for labeled IFD + target_lc = series.strip().lower() + for page in tif.pages: + try: + desc = page.description + if isinstance(desc, bytes): + desc = desc.decode("utf-8", errors="replace") + if desc.strip().strip("\x00").lower() == target_lc: + arr = page.asarray() + break + except Exception: + continue + + if arr is None: + logger.warning( + f"wsi_to_image: no '{series}' IFD in {path}; falling back to tissue" + ) + + # Tissue (default) — smallest level of first series + if arr is None: + if tif.series: + s = tif.series[0] + if hasattr(s, "levels") and len(s.levels) > 1: + arr = s.levels[-1].asarray() + else: + arr = s.asarray() + else: + arr = tif.pages[0].asarray() + + if arr is None: + return None + + arr = _normalize_to_uint8_rgb(arr) + img = Image.fromarray(arr) + return _pil_to_output(img, max_width, output_path, return_type) + + except Exception as e: + logger.exception(f"tifffile read failed for {path}: {e}") + return None + + +# --------------------------------------------------------------------------- +# Public API +# --------------------------------------------------------------------------- + +def wsi_to_image( + path: str, + max_width: int = 768, + output_path: Optional[str] = None, + return_type: str = "str", + series: str = "tissue", +) -> Optional[str | bytes]: + """Convert a WSI file to a JPEG thumbnail. + + Supports all OpenSlide formats (SVS, NDPI, MRXS, SCN, BIF, Philips TIFF, + generic TIFF, etc.) with tifffile fallback for non-WSI TIFFs. + + Args: + path: Local path to the WSI file. + max_width: Resize thumbnail to this width (default 768). + Set to 0 to skip resize. + output_path: If set, also save the JPEG to this path. + return_type: ``"str"`` — base64-encoded JPEG string (default). + ``"binary"`` — raw JPEG bytes. + series: Which sub-image to extract: + ``"tissue"`` — tissue thumbnail (default; pyramid-aware). + ``"label"`` — label associated image (PHI target). + ``"macro"`` — macro/overview associated image. + ``"thumbnail"`` — built-in thumbnail if available. + + Returns: + Base64 JPEG string, raw JPEG bytes, or ``None`` on failure. + """ + # Try OpenSlide first + try: + import openslide + + fmt = openslide.OpenSlide.detect_format(path) + if fmt is not None: + result = _wsi_openslide(path, series, max_width, output_path, return_type) + if result is not None: + return result + except ImportError: + logger.warn( + "wsi_to_image: preferred openslide not available. " + "Install openslide-python openslide-bin." + ) + pass + except Exception: + pass + + # Fallback to tifffile + try: + return _wsi_tifffile(path, series, max_width, output_path, return_type) + except ImportError: + logger.error( + "wsi_to_image: neither openslide nor tifffile available. " + "Install openslide-python openslide-bin or tifffile." + ) + return None + + +def wsi_detect_format(path: str) -> Optional[str]: + """Detect the WSI format of a file without fully opening it. + + Returns the OpenSlide vendor string (e.g. 'aperio', 'hamamatsu', + 'philips', 'generic-tiff') or None if not a recognized WSI format. + """ + try: + import openslide + return openslide.OpenSlide.detect_format(path) + except ImportError: + return None + except Exception: + return None + + +def wsi_get_properties(path: str) -> dict: + """Read all OpenSlide properties from a WSI file. + + Returns a dict of property key -> value strings. + Returns an empty dict if the file cannot be opened. + """ + try: + import openslide + slide = openslide.OpenSlide(path) + props = dict(slide.properties) + slide.close() + return props + except Exception: + return {} diff --git a/src/dbx/pixels/wsi/wsi_vlm_phi_detector.py b/src/dbx/pixels/wsi/wsi_vlm_phi_detector.py new file mode 100644 index 00000000..7a33d666 --- /dev/null +++ b/src/dbx/pixels/wsi/wsi_vlm_phi_detector.py @@ -0,0 +1,258 @@ +"""WSIVLMPhiDetector — Spark ML Transformer for pixel-level PHI detection in WSI files. + +Fully self-contained: no dependency on ``dbx.pixels.dicom`` or ``pydicom``. +Uses OpenSlide (via ``wsi_to_image()``) as the primary backend for all WSI formats. + +Supported formats (via OpenSlide): +- Aperio SVS +- Hamamatsu NDPI / VMS / VMU +- Leica SCN +- MIRAX MRXS +- Philips TIFF +- Sakura SVSLIDE +- Trestle TIF +- Ventana BIF / TIF +- Generic TIFF + +Architecture: +- Extends ``pyspark.ml.base.Transformer`` +- Applies ``wsi_to_image()`` to extract the label/macro/tissue sub-image as a + JPEG thumbnail (uses OpenSlide's pyramid — never loads full-res data), then + calls a Databricks VLM serving endpoint via the OpenAI-compatible API. +- Output column schema mirrors ``VLMPhiDetector`` from the DICOM module + (content array, completion_tokens int, prompt_tokens int, + total_tokens int, error string). +""" + +from __future__ import annotations + +import base64 +from dataclasses import dataclass +from typing import Iterator, List, Optional + +import pandas as pd +from pyspark.ml.base import Transformer +from pyspark.sql.functions import col, pandas_udf + +from dbx.pixels.logging import LoggerProvider + +logger = LoggerProvider() + +__all__ = ["WSIVLMPhiDetector", "VlmResult"] + + +DEFAULT_SYSTEM_PROMPT = ( + "You are an expert in privacy and personal health information (PHI). " + "Per HIPAA rules there are 18 fields considered PHI. " + "Please identify all of the PHI fields found in the image and return a " + "list of pipe-separated named entities, e.g. 'John Smith'|'04-31-1954'|'123 Drury Lane' " + "and nothing else. " + "Don't be fooled by text fields especially acronyms that are not PHI. " + "If there's no PHI detected, return 'No PHI' and nothing else. " + "Answer concisely as requested without explanations." +) + + +@dataclass +class VlmResult: + """Per-image result returned by the VLM PHI detector.""" + + content: Optional[List[str]] + completion_tokens: int + prompt_tokens: int + total_tokens: int + error: Optional[str] + + +# --------------------------------------------------------------------------- +# Module-level UDF factory +# --------------------------------------------------------------------------- +# Defined outside the class so Spark Connect can serialise the closure +# without capturing ``self``. Only primitive types are closed over. + + +def _make_wsi_phi_detector_udf( + endpoint: str, + system_prompt: str, + temperature: float, + num_output_tokens: int, + input_type: str, + max_width: int, + series: str, +): + """Return a ``pandas_udf`` configured with the given inference parameters. + + Uses ``wsi_to_image()`` (OpenSlide primary, tifffile fallback) for image + extraction from any WSI format. + """ + + @pandas_udf( + "content array, completion_tokens int, " + "prompt_tokens int, total_tokens int, error string" + ) + def _extract_udf(paths: Iterator[pd.Series]) -> Iterator[pd.DataFrame]: + from dataclasses import replace as dc_replace + + from mlflow.utils.databricks_utils import get_databricks_host_creds + from openai import OpenAI + + from dbx.pixels.wsi.wsi_utils import wsi_to_image + + creds = get_databricks_host_creds("databricks") + client = OpenAI( + api_key=creds.token, + base_url=f"{creds.host}/serving-endpoints", + timeout=300, + max_retries=3, + ) + _null = VlmResult(None, 0, 0, 0, None) + + for batch in paths: + results = [] + for path in batch: + try: + # --- 1. Get image as base64 JPEG --- + if input_type == "wsi": + # OpenSlide-backed: handles SVS, NDPI, MRXS, SCN, BIF, + # Philips TIFF, generic TIFF, etc. + b64 = wsi_to_image( + path, + max_width=max_width, + return_type="str", + series=series, + ) + if b64 is None: + results.append( + dc_replace( + _null, + error=f"wsi_to_image returned None: {path}", + ) + ) + continue + elif input_type == "image": + # Raw image file (JPEG/PNG) — read and base64-encode + local = path[5:] if path.startswith("dbfs:") else path + with open(local, "rb") as fh: + b64 = base64.b64encode(fh.read()).decode("utf-8") + else: # "base64" — already encoded + b64 = path + + # --- 2. VLM inference --- + response = client.chat.completions.create( + model=endpoint, + messages=[ + {"role": "system", "content": system_prompt}, + { + "role": "user", + "content": [ + { + "type": "image_url", + "image_url": { + "url": f"data:image/jpeg;base64,{b64}", + "detail": "low", + }, + } + ], + }, + ], + temperature=float(temperature), + max_tokens=int(num_output_tokens), + ) + + # --- 3. Parse response --- + content = response.choices[0].message.content + if "|" in content: + phi_list = content.split("|") + elif content.strip().lower() == "no phi": + phi_list = [] + else: + phi_list = [content] + + results.append( + VlmResult( + phi_list, + response.usage.completion_tokens, + response.usage.prompt_tokens, + response.usage.total_tokens, + None, + ) + ) + + except Exception as exc: + logger.error(f"Error processing {path}: {exc}") + results.append(dc_replace(_null, error=str(exc))) + + yield pd.DataFrame(results) + + return _extract_udf + + +# --------------------------------------------------------------------------- +# Transformer +# --------------------------------------------------------------------------- + + +class WSIVLMPhiDetector(Transformer): + """Detect pixel-level PHI in Whole Slide Images using a Databricks VLM endpoint. + + Supports all OpenSlide-recognized formats (SVS, NDPI, MRXS, SCN, BIF, + Philips TIFF, generic TIFF, etc.) via ``wsi_to_image()`` which uses + OpenSlide as primary backend with tifffile fallback. + + No dependency on ``dbx.pixels.dicom`` or ``pydicom`` — fully self-contained. + + Args: + endpoint: Databricks serving endpoint name for the VLM. + system_prompt: Override the default HIPAA PHI detection prompt. + temperature: VLM sampling temperature (default 0.0). + num_output_tokens: Maximum tokens in the VLM response (default 200). + inputCol: Input column name (default ``local_path``). + outputCol: Output column name (default ``response``). + input_type: ``"wsi"`` — path to any WSI file (default). + ``"image"`` — path to a JPEG/PNG file. + ``"base64"`` — already base64-encoded image string. + max_width: Resize thumbnail width before VLM (default 768). + Set to 0 to disable. + series: Sub-image to extract from WSI files (default ``"label"``): + ``"label"`` — label associated image (rendered PHI text; recommended). + ``"macro"`` — macro/overview associated image. + ``"thumbnail"`` — built-in thumbnail if available. + ``"tissue"`` — smallest tissue pyramid level. + Falls back to tissue if the requested sub-image is absent. + """ + + def __init__( + self, + endpoint: str, + system_prompt: str = None, + temperature: float = 0.0, + num_output_tokens: int = 200, + inputCol: str = "local_path", + outputCol: str = "response", + input_type: str = "wsi", + max_width: int = 768, + series: str = "label", + ): + super().__init__() + self.endpoint = endpoint + self.system_prompt = system_prompt + self.temperature = temperature + self.num_output_tokens = num_output_tokens + self.inputCol = inputCol + self.outputCol = outputCol + self.input_type = input_type + self.max_width = max_width + self.series = series + + def _transform(self, df): + """Apply VLM PHI detection via ``pandas_udf``.""" + _udf = _make_wsi_phi_detector_udf( + endpoint=self.endpoint, + system_prompt=self.system_prompt or DEFAULT_SYSTEM_PROMPT, + temperature=self.temperature, + num_output_tokens=self.num_output_tokens, + input_type=self.input_type, + max_width=self.max_width, + series=self.series, + ) + return df.withColumn(self.outputCol, _udf(col(self.inputCol))) diff --git a/tests/dbx/test_wsi.py b/tests/dbx/test_wsi.py new file mode 100644 index 00000000..0106f761 --- /dev/null +++ b/tests/dbx/test_wsi.py @@ -0,0 +1,320 @@ +"""Tests for dbx.pixels.wsi — Unified WSI handler. + +Runs against real SVS and TIFF files in UC Volumes: +- SVS: /Volumes/hls_radiology_east/orthanc_demo/raw_images/Aperio/ +- TIFF: /Volumes/hls_radiology_east/orthanc_demo/raw_images/Generic-TIFF/ + /Volumes/hls_radiology_east/orthanc_demo/raw_images/Philips-TIFF/ + /Volumes/hls_radiology_east/osuwmc/sample/ + +Requires: openslide-python, openslide-bin, tifffile + +Usage (from project root): + %pip install openslide-python openslide-bin tifffile imagecodecs -q + pytest tests/dbx/test_wsi.py -v +""" + +import json +import os +import sys +from pathlib import Path + +import pytest + +# Ensure src is on path +SRC_PATH = str(Path(__file__).resolve().parents[2] / "src") +if SRC_PATH not in sys.path: + sys.path.insert(0, SRC_PATH) + + +# --------------------------------------------------------------------------- +# Test fixtures: real file paths from UC Volumes +# --------------------------------------------------------------------------- + +SVS_DIR = "/Volumes/hls_radiology_east/orthanc_demo/raw_images/Aperio" +TIFF_DIRS = [ + "/Volumes/hls_radiology_east/orthanc_demo/raw_images/Generic-TIFF", + "/Volumes/hls_radiology_east/orthanc_demo/raw_images/Philips-TIFF", + "/Volumes/hls_radiology_east/osuwmc/sample", +] + + +def _discover_files(directory: str, extensions: tuple) -> list[str]: + """Discover test files by extension.""" + if not os.path.isdir(directory): + return [] + return sorted( + os.path.join(directory, f) + for f in os.listdir(directory) + if f.lower().endswith(extensions) and not f.startswith(".") + ) + + +@pytest.fixture(scope="module") +def svs_files(): + """All SVS test files.""" + files = _discover_files(SVS_DIR, (".svs",)) + if not files: + pytest.skip(f"No SVS files found in {SVS_DIR}") + return files + + +@pytest.fixture(scope="module") +def tiff_files(): + """All TIFF test files.""" + files = [] + for d in TIFF_DIRS: + files.extend(_discover_files(d, (".tiff", ".tif"))) + if not files: + pytest.skip(f"No TIFF files found in {TIFF_DIRS}") + return files + + +@pytest.fixture(scope="module") +def all_wsi_files(svs_files, tiff_files): + """All WSI files (SVS + TIFF).""" + return svs_files + tiff_files + + +# --------------------------------------------------------------------------- +# Test: phi_tags module +# --------------------------------------------------------------------------- + +class TestPhiTags: + """Tests for wsi_phi_tags.py""" + + def test_classify_phi_tag(self): + from dbx.pixels.wsi.wsi_phi_tags import classify_tag + assert classify_tag("aperio.Patient") == "PHI" + assert classify_tag("Patient") == "PHI" + assert classify_tag("Artist") == "PHI" + assert classify_tag("HostComputer") == "PHI" + + def test_classify_questionable_tag(self): + from dbx.pixels.wsi.wsi_phi_tags import classify_tag + assert classify_tag("aperio.Date") == "QUESTIONABLE" + assert classify_tag("DateTime") == "QUESTIONABLE" + assert classify_tag("ImageDescription") == "QUESTIONABLE" + assert classify_tag("ventana.ScanDate") == "QUESTIONABLE" + + def test_classify_not_phi_tag(self): + from dbx.pixels.wsi.wsi_phi_tags import classify_tag + assert classify_tag("openslide.mpp-x") == "NOT_PHI" + assert classify_tag("ImageWidth") == "NOT_PHI" + assert classify_tag("SomeUnknownTag") == "NOT_PHI" + + def test_classify_tags_returns_only_phi_and_questionable(self): + from dbx.pixels.wsi.wsi_phi_tags import classify_tags + props = { + "aperio.Patient": "John Doe", + "aperio.AppMag": "40", + "openslide.mpp-x": "0.2525", + "DateTime": "2024-01-01", + } + report = classify_tags(props) + tags_in_report = {r["tag"] for r in report} + assert "aperio.Patient" in tags_in_report + assert "DateTime" in tags_in_report + assert "aperio.AppMag" not in tags_in_report + assert "openslide.mpp-x" not in tags_in_report + + def test_scrub_image_description_aperio_format(self): + from dbx.pixels.wsi.wsi_phi_tags import scrub_image_description + desc = "Aperio Image Library v12.0.15|Patient = John Doe|AppMag = 40|Date = 2024-01-01" + scrubbed = scrub_image_description(desc) + assert "John Doe" not in scrubbed + assert "REDACTED" in scrubbed + assert "AppMag = 40" in scrubbed # NOT_PHI preserved + + def test_supported_extensions(self): + from dbx.pixels.wsi.wsi_phi_tags import SUPPORTED_EXTENSIONS + assert ".svs" in SUPPORTED_EXTENSIONS + assert ".tiff" in SUPPORTED_EXTENSIONS + assert ".bif" in SUPPORTED_EXTENSIONS + assert ".ndpi" in SUPPORTED_EXTENSIONS + assert ".mrxs" in SUPPORTED_EXTENSIONS + + def test_openslide_patterns(self): + from dbx.pixels.wsi.wsi_phi_tags import OPENSLIDE_PATTERNS + assert "*.svs" in OPENSLIDE_PATTERNS + assert "*.tiff" in OPENSLIDE_PATTERNS + + +# --------------------------------------------------------------------------- +# Test: wsi_utils module +# --------------------------------------------------------------------------- + +class TestWSIUtils: + """Tests for wsi_utils.py — requires real files.""" + + def test_detect_format_svs(self, svs_files): + from dbx.pixels.wsi.wsi_utils import wsi_detect_format + # All SVS files should be detected as 'aperio' + for f in svs_files: + fmt = wsi_detect_format(f) + assert fmt == "aperio", f"Expected 'aperio' for {f}, got {fmt}" + + def test_detect_format_tiff(self, tiff_files): + from dbx.pixels.wsi.wsi_utils import wsi_detect_format + # TIFF files should be detected as some OpenSlide vendor or None + for f in tiff_files: + fmt = wsi_detect_format(f) + # Philips TIFFs should be detected; generic may return None + assert fmt is None or isinstance(fmt, str), f"Unexpected format for {f}: {fmt}" + + def test_get_properties_svs(self, svs_files): + from dbx.pixels.wsi.wsi_utils import wsi_get_properties + props = wsi_get_properties(svs_files[0]) + assert len(props) > 0 + assert "openslide.vendor" in props + assert props["openslide.vendor"] == "aperio" + + def test_wsi_to_image_tissue_svs(self, svs_files): + from dbx.pixels.wsi.wsi_utils import wsi_to_image + result = wsi_to_image(svs_files[0], max_width=256, series="tissue") + assert result is not None + assert isinstance(result, str) # base64 string + assert len(result) > 100 # Non-trivial content + + def test_wsi_to_image_label_svs(self, svs_files): + from dbx.pixels.wsi.wsi_utils import wsi_to_image + # CMU-1.svs has a label image + cmu1 = next((f for f in svs_files if "CMU-1.svs" in f), svs_files[0]) + result = wsi_to_image(cmu1, max_width=256, series="label") + assert result is not None + assert isinstance(result, str) + + def test_wsi_to_image_macro_svs(self, svs_files): + from dbx.pixels.wsi.wsi_utils import wsi_to_image + cmu1 = next((f for f in svs_files if "CMU-1.svs" in f), svs_files[0]) + result = wsi_to_image(cmu1, max_width=256, series="macro") + assert result is not None + assert isinstance(result, str) + + def test_wsi_to_image_binary_output(self, svs_files): + from dbx.pixels.wsi.wsi_utils import wsi_to_image + result = wsi_to_image(svs_files[0], max_width=128, return_type="binary") + assert result is not None + assert isinstance(result, bytes) + assert result[:2] == b"\xff\xd8" # JPEG magic bytes + + def test_wsi_to_image_tiff(self, tiff_files): + from dbx.pixels.wsi.wsi_utils import wsi_to_image + result = wsi_to_image(tiff_files[0], max_width=256, series="tissue") + assert result is not None + assert isinstance(result, str) + + +# --------------------------------------------------------------------------- +# Test: WSIMetaExtractor module (standalone, no Spark) +# --------------------------------------------------------------------------- + +class TestWSIMetaExtractorStandalone: + """Tests for the static extraction methods (no Spark required).""" + + def test_process_openslide_svs(self, svs_files): + from dbx.pixels.wsi.wsi_meta_extractor import WSIMetaExtractor + result = WSIMetaExtractor._process_openslide(svs_files[0]) + meta = json.loads(result) + assert "error" not in meta, f"Error: {meta.get('error')}" + assert meta["_wsi_vendor"] == "aperio" + assert meta["_wsi_backend"] == "openslide" + assert meta["_wsi_width"] > 0 + assert meta["_wsi_height"] > 0 + assert meta["_wsi_level_count"] >= 1 + assert isinstance(meta["_wsi_level_dimensions"], list) + assert isinstance(meta["_wsi_associated_images"], list) + assert isinstance(meta["phi_tag_report"], list) + + def test_process_openslide_has_mpp(self, svs_files): + from dbx.pixels.wsi.wsi_meta_extractor import WSIMetaExtractor + result = WSIMetaExtractor._process_openslide(svs_files[0]) + meta = json.loads(result) + # CMU SVS files have MPP defined + assert meta["_wsi_mpp_x"] is not None or meta.get("openslide.mpp-x") is not None + + def test_process_openslide_svs_associated_images(self, svs_files): + from dbx.pixels.wsi.wsi_meta_extractor import WSIMetaExtractor + # CMU-1.svs should have label and macro + cmu1 = next((f for f in svs_files if "CMU-1.svs" in f), svs_files[0]) + result = WSIMetaExtractor._process_openslide(cmu1) + meta = json.loads(result) + assert meta["_wsi_has_label"] is True + assert meta["_wsi_has_macro"] is True + + def test_process_file_dispatches_to_openslide_for_svs(self, svs_files): + from dbx.pixels.wsi.wsi_meta_extractor import WSIMetaExtractor + result = WSIMetaExtractor._process_file(svs_files[0]) + meta = json.loads(result) + assert meta["_wsi_backend"] == "openslide" + + def test_process_tifffile_fallback(self, tiff_files): + from dbx.pixels.wsi.wsi_meta_extractor import WSIMetaExtractor + # Force tifffile path + result = WSIMetaExtractor._process_tifffile(tiff_files[0]) + meta = json.loads(result) + assert "error" not in meta, f"Error: {meta.get('error')}" + assert meta["_wsi_backend"] == "tifffile" + assert meta["_wsi_width"] > 0 + assert meta["_wsi_height"] > 0 + + def test_process_file_tiff_uses_openslide_when_possible(self, tiff_files): + from dbx.pixels.wsi.wsi_meta_extractor import WSIMetaExtractor + import openslide + # Find a TIFF that OpenSlide recognizes + for f in tiff_files: + fmt = openslide.OpenSlide.detect_format(f) + if fmt is not None: + result = WSIMetaExtractor._process_file(f) + meta = json.loads(result) + assert meta["_wsi_backend"] == "openslide" + return + pytest.skip("No TIFF files recognized by OpenSlide") + + def test_all_svs_files_extract_without_error(self, svs_files): + from dbx.pixels.wsi.wsi_meta_extractor import WSIMetaExtractor + for f in svs_files: + result = WSIMetaExtractor._process_file(f) + meta = json.loads(result) + assert "error" not in meta, f"Error for {f}: {meta.get('error')}" + + def test_all_tiff_files_extract_without_error(self, tiff_files): + from dbx.pixels.wsi.wsi_meta_extractor import WSIMetaExtractor + for f in tiff_files: + result = WSIMetaExtractor._process_file(f) + meta = json.loads(result) + assert "error" not in meta, f"Error for {f}: {meta.get('error')}" + + +# --------------------------------------------------------------------------- +# Test: Format constants and patterns +# --------------------------------------------------------------------------- + +class TestConstants: + """Verify module constants are consistent.""" + + def test_openslide_patterns_match_extensions(self): + from dbx.pixels.wsi.wsi_phi_tags import OPENSLIDE_PATTERNS, SUPPORTED_EXTENSIONS + for pat in OPENSLIDE_PATTERNS: + ext = "." + pat.lstrip("*.") + assert ext in SUPPORTED_EXTENSIONS, f"Pattern {pat} not in SUPPORTED_EXTENSIONS" + + def test_imports(self): + """Verify the package imports cleanly.""" + from dbx.pixels.wsi import ( + WSICatalog, + WSIMetaExtractor, + wsi_to_image, + wsi_detect_format, + wsi_get_properties, + classify_tag, + classify_tags, + scrub_image_description, + PHI_TAGS, + QUESTIONABLE_TAGS, + NOT_PHI_TAGS, + LARGE_TAGS, + SUPPORTED_EXTENSIONS, + OPENSLIDE_PATTERNS, + ) + assert WSICatalog is not None + assert WSIMetaExtractor is not None From 78b1d4f178aa999ccc647baf42771e746beab85b Mon Sep 17 00:00:00 2001 From: Douglas Moore Date: Tue, 14 Jul 2026 09:19:43 -0400 Subject: [PATCH 6/7] remove redundant svs/tiff in favor of wsi --- ...Pathology De-identification Pipeline.ipynb | 1753 ----------------- notebooks/tiff/01-ingest-tiff.ipynb | 471 ----- ...Pathology De-identification Pipeline.ipynb | 1753 ----------------- notebooks/tiff/TIFF sample data.ipynb | 980 --------- notebooks/tiff/config.yaml-example | 11 - src/dbx/pixels/svs/__init__.py | 43 - src/dbx/pixels/svs/catalog.py | 74 - src/dbx/pixels/svs/deidentify.py | 224 --- src/dbx/pixels/svs/phi_tags.py | 162 -- .../svs/resources/sql/CREATE_SVS_CATALOG.sql | 110 -- src/dbx/pixels/svs/svs_meta_extractor.py | 115 -- src/dbx/pixels/svs/svs_tiff_writer.py | 190 -- src/dbx/pixels/tiff/__init__.py | 4 - src/dbx/pixels/tiff/tiff_meta_extractor.py | 233 --- src/dbx/pixels/tiff/tiff_phi_tags.py | 171 -- src/dbx/pixels/tiff/tiff_utils.py | 191 -- src/dbx/pixels/tiff/tiff_vlm_phi_detector.py | 208 -- 17 files changed, 6693 deletions(-) delete mode 100644 notebooks/svs/SVS Pathology De-identification Pipeline.ipynb delete mode 100644 notebooks/tiff/01-ingest-tiff.ipynb delete mode 100644 notebooks/tiff/TIFF Pathology De-identification Pipeline.ipynb delete mode 100644 notebooks/tiff/TIFF sample data.ipynb delete mode 100644 notebooks/tiff/config.yaml-example delete mode 100644 src/dbx/pixels/svs/__init__.py delete mode 100644 src/dbx/pixels/svs/catalog.py delete mode 100644 src/dbx/pixels/svs/deidentify.py delete mode 100644 src/dbx/pixels/svs/phi_tags.py delete mode 100644 src/dbx/pixels/svs/resources/sql/CREATE_SVS_CATALOG.sql delete mode 100644 src/dbx/pixels/svs/svs_meta_extractor.py delete mode 100644 src/dbx/pixels/svs/svs_tiff_writer.py delete mode 100644 src/dbx/pixels/tiff/__init__.py delete mode 100644 src/dbx/pixels/tiff/tiff_meta_extractor.py delete mode 100644 src/dbx/pixels/tiff/tiff_phi_tags.py delete mode 100644 src/dbx/pixels/tiff/tiff_utils.py delete mode 100644 src/dbx/pixels/tiff/tiff_vlm_phi_detector.py diff --git a/notebooks/svs/SVS Pathology De-identification Pipeline.ipynb b/notebooks/svs/SVS Pathology De-identification Pipeline.ipynb deleted file mode 100644 index 46b34eb6..00000000 --- a/notebooks/svs/SVS Pathology De-identification Pipeline.ipynb +++ /dev/null @@ -1,1753 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": { - "application/vnd.databricks.v1+cell": { - "cellMetadata": {}, - "inputWidgets": {}, - "nuid": "a9402935-1770-4aa2-bec0-71e265bc53c1", - "showTitle": false, - "tableResultSettingsMap": {}, - "title": "" - } - }, - "source": [ - "\n", - "# Plan: SVS Pathology De-identification Pipeline\n", - "## Architecture Plan — Aperio SVS → De-identified TIFF\n", - "\n", - "Extends [databricks-industry-solutions/pixels](https://github.com/databricks-industry-solutions/pixels) to treat Aperio `.svs` as a first-class format alongside DICOM.\n", - "\n", - "---" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "application/vnd.databricks.v1+cell": { - "cellMetadata": {}, - "inputWidgets": {}, - "nuid": "b27e18db-0abf-446f-ba89-65d6906d1506", - "showTitle": false, - "tableResultSettingsMap": {}, - "title": "" - } - }, - "source": [ - "## 1. Confirmed Inputs & Outputs\n", - "\n", - "| Item | Value |\n", - "|---|---|\n", - "| Input SVS path | `/Volumes/hls_pathology/orthanc_demo/raw_images/Aperio/` |\n", - "| Demo scale | ~14 files; architecture targets **10 million** |\n", - "| Output catalog / schema | `douglas_moore.pathology` (to be created) |\n", - "| TIFF output volume | `/Volumes/douglas_moore/pathology/tiff_deidentified/` |\n", - "| Label images volume | `/Volumes/douglas_moore/pathology/label_images/` |\n", - "| VLM endpoint | `databricks-llama-4-maverick` (config param) |\n", - "| Redaction method | Black rectangle fill |\n", - "| Source SVS | **Read-only** — never modified |\n", - "\n", - "---" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "application/vnd.databricks.v1+cell": { - "cellMetadata": {}, - "inputWidgets": {}, - "nuid": "32ddb278-16b7-4719-a134-55360ad4bb26", - "showTitle": false, - "tableResultSettingsMap": {}, - "title": "" - } - }, - "source": [ - "## 2. Delta Table Schema\n", - "\n", - "> **No new DDL is needed for SVS ingest.** The `object_catalog` table is used exactly as defined in the base `CREATE_OBJECT_CATALOG.sql` — no columns are added or altered. All SVS-specific metadata (dimensions, pyramid levels, sub-image presence, PHI tag classification) is serialised into the existing `meta VARIANT` column and accessed via VARIANT path syntax. `SVSCatalog.init_tables()` calls `super().init_tables()` which runs the unmodified base DDL against the `douglas_moore.pathology` schema. The only new DDL is the `_redaction` table.\n", - "\n", - "### `douglas_moore.pathology.object_catalog` *(base DDL, unchanged)*\n", - "One row per SVS file. Populated by `SVSCatalog.catalog()` + `SVSMetaExtractor`.\n", - "\n", - "| Column | Type | Notes |\n", - "|---|---|---|\n", - "| `path` | STRING NOT NULL | Cloud storage path |\n", - "| `modificationTime` | TIMESTAMP NOT NULL | |\n", - "| `length` | BIGINT NOT NULL | File size bytes |\n", - "| `original_path` | STRING | |\n", - "| `relative_path` | STRING | |\n", - "| `local_path` | STRING NOT NULL | Worker-accessible path — **`inputCol` for all Transformers** |\n", - "| `extension` | STRING | `\"svs\"` |\n", - "| `file_type` | STRING | |\n", - "| `path_tags` | ARRAY\\ | From `TagExtractor` |\n", - "| `is_anon` | BOOLEAN | |\n", - "| `meta` | **VARIANT** | All OpenSlide properties + SVS-specific fields serialised together. Query with `meta:aperio.Date::string`, `meta:width::int`, `meta:has_label_image::boolean` |\n", - "\n", - "**SVS fields stored inside `meta VARIANT`** (no schema change required):\n", - "- `meta:width::int`, `meta:height::int` — level-0 pixel dimensions\n", - "- `meta:level_count::int` — pyramid depth\n", - "- `meta:has_label_image::boolean`, `meta:has_macro_image::boolean`\n", - "- `meta:phi_tag_report` — array of `{tag, value, classification}` structs\n", - "- All raw OpenSlide properties (e.g. `meta:\"aperio.AppMag\"::string`)\n", - "\n", - "### `douglas_moore.pathology.object_catalog_redaction` *(unified DICOM + SVS DDL)*\n", - "One row per redaction job, for any format. Created by `CREATE_SVS_CATALOG.sql`.\n", - "\n", - "Three DICOM columns are renamed to remove format-specific semantics; new columns cover VLM detection results and SVS artefacts. All new and renamed columns are nullable for backward compatibility.\n", - "\n", - "**Format-agnostic identifiers**\n", - "\n", - "| Column | Type | Notes |\n", - "|---|---|---|\n", - "| `redaction_id` | STRING NOT NULL | UUID per job |\n", - "| `path` | STRING | FK → `object_catalog.path` *(new — not in DICOM original)* |\n", - "| `extension` | STRING | Discriminator: `dcm`, `svs`, `czi` … *(new)* |\n", - "\n", - "**DICOM identifiers** *(NULL for SVS)*\n", - "\n", - "| Column | Type | Notes |\n", - "|---|---|---|\n", - "| `study_instance_uid` | STRING | DICOM Study UID |\n", - "| `series_instance_uid` | STRING | DICOM Series UID |\n", - "| `modality` | STRING | DICOM modality, or `WSI` for SVS |\n", - "| `new_series_instance_uid` | STRING | New UID for redacted DICOM series |\n", - "\n", - "**Redaction configuration** *(both formats)*\n", - "\n", - "| Column | Type | Change from DICOM original |\n", - "|---|---|---|\n", - "| `redaction_config` | VARIANT | **Renamed** from `redaction_json` |\n", - "| `metadata_redactions_count` | INT | **Renamed** from `global_redactions_count` |\n", - "| `pixel_redactions_count` | INT | **Renamed** from `frame_specific_redactions_count` |\n", - "| `total_redaction_areas` | INT | Unchanged |\n", - "| `phi_tags_redacted` | ARRAY\\ | Tag names scrubbed *(new)* |\n", - "\n", - "**VLM PHI detection results** *(new — both formats)*\n", - "\n", - "| Column | Type | Notes |\n", - "|---|---|---|\n", - "| `has_phi` | BOOLEAN | VLM verdict |\n", - "| `phi_elements` | VARIANT | Detected regions: type, value\\_hint, bbox |\n", - "| `vlm_raw_response` | STRING | Raw model output |\n", - "| `model_endpoint` | STRING | Endpoint name |\n", - "\n", - "**Output paths**\n", - "\n", - "| Column | Type | Notes |\n", - "|---|---|---|\n", - "| `output_file_paths` | ARRAY\\ | DICOM: one `.dcm` per slice. SVS: single TIFF at index 0 |\n", - "| `label_image_path` | STRING | De-identified label PNG *(SVS only, NULL for DICOM)* |\n", - "| `macro_image_path` | STRING | De-identified macro PNG *(SVS only, NULL for DICOM)* |\n", - "\n", - "**Processing status & audit** *(unchanged from DICOM original)*\n", - "`status`, `error_messages`, `insert_timestamp`, `update_timestamp`, `processing_start_timestamp`, `processing_end_timestamp`, `processing_duration_seconds`, `created_by`, `export_timestamp`\n", - "\n", - "---" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "application/vnd.databricks.v1+cell": { - "cellMetadata": {}, - "inputWidgets": {}, - "nuid": "207f4f4e-a49b-456e-b5d1-1165b1b1f8a9", - "showTitle": true, - "tableResultSettingsMap": {}, - "title": "Architecture Plan: SVS Pathology De-identification Pipeline" - } - }, - "source": [ - "\n", - "## 3. Python Package: `dbx.pixels.svs`\n", - "\n", - "> **Pattern source: actual repo code** — All transformers extend `pyspark.ml.pipeline.Transformer` (Spark ML, not a custom base). The main entry-point for file cataloguing is the `Catalog` class, not a `Processor`. There is no `Processor` in the repo. The CZI extractor (`src/dbx/pixels/czi/`) is a stub — SVS is genuinely the first completed non-DICOM format extension.\n", - "\n", - "Namespace-package extension of `dbx-pixels`. Created as workspace files under `svs-pixels/src/`, installed via `%pip install -e ./src`.\n", - "\n", - "```\n", - "svs-pixels/\n", - "├── src/\n", - "│ └── dbx/\n", - "│ └── pixels/\n", - "│ └── svs/\n", - "│ ├── __init__.py ← exports SVSCatalog, SVSMetaExtractor, SVSTiffWriter, SVSPhiPipeline\n", - "│ ├── catalog.py ← SVSCatalog(Catalog)\n", - "│ ├── svs_meta_extractor.py← SVSMetaExtractor(Transformer)\n", - "│ ├── svs_tiff_writer.py ← SVSTiffWriter(Transformer)\n", - "│ ├── phi_tags.py ← PHI classification lookup dict\n", - "│ └── deidentify.py ← pixel redaction helpers\n", - "│ └── resources/sql/\n", - "│ └── CREATE_SVS_CATALOG.sql ← creates object_catalog_redaction only\n", - "├── pyproject.toml\n", - "└── (this notebook)\n", - "```\n", - "\n", - "### `SVSCatalog` (extends `Catalog`)\n", - "- Calls `super().__init__(spark, table, volume)` — reuses all existing table management, volume, and Auto Loader infrastructure\n", - "- `catalog(path, pattern=\"*.svs\", ...)` → delegates to `Catalog.catalog()` with SVS glob pattern; callers never need to pass `pattern`\n", - "- `init_tables()` → calls `super().init_tables()` (creates `object_catalog` via unmodified base DDL), then executes one SVS-specific file — `resources/sql/CREATE_SVS_CATALOG.sql` — which creates only the `object_catalog_redaction` table with SVS-specific columns\n", - "\n", - "### `SVSMetaExtractor` (extends `pyspark.ml.pipeline.Transformer`)\n", - "Mirrors `DicomMetaExtractor`: uses `mapInPandas` with `ThreadPoolExecutor` for concurrent I/O (optimal for network-bound OpenSlide reads).\n", - "\n", - "```python\n", - "class SVSMetaExtractor(Transformer):\n", - " def __init__(self, catalog, inputCol=\"local_path\", outputCol=\"meta\",\n", - " maxWorkers=32, useVariant=True): ...\n", - "\n", - " def _transform(self, df): # Spark ML Transformer contract\n", - " # mapInPandas with ThreadPoolExecutor — same pattern as DicomMetaExtractor\n", - " ...\n", - "```\n", - "\n", - "**Single output column written to `object_catalog`:**\n", - "\n", - "| Column | Spark type | Notes |\n", - "|---|---|---|\n", - "| `meta` | `VARIANT` | OpenSlide properties dict merged with derived fields (`width`, `height`, `level_count`, `has_label_image`, `has_macro_image`, `phi_tag_report`) into one JSON object, then `parse_json()`'d into VARIANT |\n", - "\n", - "All SVS-specific fields are embedded inside `meta` before serialisation — no extra top-level columns are written, no `ALTER TABLE` or `mergeSchema` required. VARIANT path syntax handles all downstream access: `meta:width::int`, `meta:phi_tag_report[0].classification::string`, etc.\n", - "\n", - "### `SVSTiffWriter` (extends `pyspark.ml.pipeline.Transformer`)\n", - "Converts SVS → de-identified pyramidal BigTIFF. Wraps the write logic in `_transform(df)` operating on the output of `SVSMetaExtractor`.\n", - "\n", - "### `SVSPhiPipeline` (extends `pyspark.ml.Pipeline`)\n", - "Composed pipeline, mirrors `DicomPhiPipeline`:\n", - "```\n", - "Stage 1: SVSMetaExtractor → adds meta VARIANT + phi_tag_report\n", - "Stage 2: SVSVlmPhiDetector → adds phi_elements (VLM bboxes on label/macro)\n", - "Stage 3: SVSFilterTransformer → nullifies rows with no PHI detected\n", - "Stage 4: SVSTiffWriter → writes de-identified BigTIFF + audit log\n", - "```\n", - "\n", - "---\n", - "\n", - "\n" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "application/vnd.databricks.v1+cell": { - "cellMetadata": {}, - "inputWidgets": {}, - "nuid": "7e9389da-8946-4087-a9ac-3fe10773c829", - "showTitle": false, - "tableResultSettingsMap": {}, - "title": "" - } - }, - "source": [ - "\n", - "## 4. PHI Tag Classification (`phi_tags.py`)\n", - "\n", - "Research-hardcoded lookup covering all standard OpenSlide / Aperio TIFF properties:\n", - "\n", - "| Classification | Example Tags |\n", - "|---|---|\n", - "| `PHI` | `aperio.Patient`, `aperio.PatientID`, `aperio.DOB`, `aperio.MRN`, `aperio.AccessionNumber`, `aperio.ClinicID`, `aperio.ClinicalTrialID`, `aperio.Procedure`, `tiff.ImageDescription` (contains patient name in Aperio format) |\n", - "| `QUESTIONABLE` | `aperio.Date`, `aperio.Time`, `aperio.Clinic`, `aperio.Pathologist`, `tiff.Artist`, `tiff.Copyright`, `aperio.Title`, `aperio.Filename`, `aperio.User`, `aperio.ImageID` |\n", - "| `NOT_PHI` | `aperio.AppMag`, `aperio.MPP`, `aperio.ScanScope ID`, `openslide.level-count`, `openslide.mpp-x`, `openslide.mpp-y`, `openslide.objective-power`, `tiff.Make`, `tiff.Model`, `tiff.Software`, `openslide.vendor`, all `openslide.level[N].*` pyramid geometry tags |\n", - "\n", - "Function `classify_tags(properties: dict) → list[dict]` iterates all OpenSlide properties and returns the structured report stored in `phi_tag_report`.\n", - "\n", - "---" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "application/vnd.databricks.v1+cell": { - "cellMetadata": {}, - "inputWidgets": {}, - "nuid": "3f54bca4-4711-4cc7-8995-ce1bb3bfda99", - "showTitle": false, - "tableResultSettingsMap": {}, - "title": "" - } - }, - "source": [ - "## 5. VLM PHI Detection via `ai_query()`\n", - "\n", - "### What is Inspected\n", - "Aperio SVS files embed three sub-images accessible via `slide.associated_images` — **confirmed from real files**:\n", - "\n", - "| Sub-image | Dims (CMU-1) | Mode | PHI Risk | Action |\n", - "|---|---|---|---|---|\n", - "| `label` | 387×463 | RGBA | **HIGH** — physical paper label with patient name, barcode, accession | VLM analysis + black-box redaction |\n", - "| `macro` | 1280×431 | RGBA | **MEDIUM** — full-slide photo; label region visible at right edge | VLM analysis + label region redaction |\n", - "| `thumbnail` | 1024×732 | RGBA | LOW — auto-generated tissue preview | Excluded from output |\n", - "\n", - "The tissue scan (`level 0`: 46000×32914) is in a **completely separate coordinate space** from the label/macro sub-images. PHI in the tissue scan itself is rare but possible (e.g., handwriting on the glass).\n", - "\n", - "The label image is the **primary** VLM target. Macro is secondary.\n", - "\n", - "### Pipeline\n", - "1. `SVSTransformer.extract_embedded_images()` saves label and macro PNGs to `/Volumes/douglas_moore/pathology/label_images/` using the naming convention `{slide_name}_label.png` / `{slide_name}_macro.png`\n", - "2. Run `ai_query()` directly via `READ_FILES()` on the volume — **no binary column staging needed**:\n", - "\n", - "```sql\n", - "INSERT INTO douglas_moore.pathology.phi_pixel_audit\n", - "SELECT\n", - " m.path,\n", - " f._metadata.file_path AS label_image_path,\n", - " ai_query(\n", - " 'databricks-llama-4-maverick',\n", - " 'You are a medical PHI detection system analyzing a pathology slide label.\n", - " Return ONLY valid JSON:\n", - " {\"has_phi\": bool,\n", - " \"phi_elements\": [{\"type\": \"name|dob|mrn|accession|barcode|other\",\n", - " \"value_hint\": \"first 3 chars only\",\n", - " \"bbox\": {\"x\": int, \"y\": int, \"w\": int, \"h\": int}}]}\n", - " Bounding box coordinates are in the label image pixel space (origin top-left).',\n", - " files => f.content\n", - " ) AS vlm_raw_response,\n", - " 'databricks-llama-4-maverick' AS model_endpoint,\n", - " current_timestamp() AS inferred_at\n", - "FROM read_files(\n", - " '/Volumes/douglas_moore/pathology/label_images/',\n", - " format => 'binaryFile',\n", - " fileNamePattern => '*_label.png'\n", - ") f\n", - "JOIN douglas_moore.pathology.svs_metadata m\n", - " ON m.filename = regexp_replace(f._metadata.file_name, '_label\\.png\n", - "```\n", - "---\n" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "application/vnd.databricks.v1+cell": { - "cellMetadata": {}, - "inputWidgets": {}, - "nuid": "b3f28cd8-2465-4bfe-b153-664e929fe501", - "showTitle": false, - "tableResultSettingsMap": {}, - "title": "" - } - }, - "source": [ - "\n", - "## 6. De-identification & TIFF Output (`deidentify.py` / `SVSTiffWriter`)\n", - "\n", - "> **Pattern source: actual repo code** — `DicomPhiPipeline` uses a two-stage approach: (1) `VLMPhiDetector` returns a **pipe-separated list of PHI text strings** (`'John Smith'|'04-31-1954'`), NOT bboxes. (2) `OcrRedactor` then runs EasyOCR on the image to locate those strings and draw black rectangles. The VLM provides *what* is PHI; OCR provides *where*. SVS uses this same two-stage approach.\n", - "\n", - "### Revised De-identification Algorithm\n", - "\n", - "#### Stage 1 — VLM PHI Detection (`SVSVlmPhiDetector`, extends `Transformer`)\n", - "- Submit label/macro PNGs via `ai_query()` with `files => content`\n", - "- Prompt returns a **pipe-separated list of PHI entity strings** (consistent with the SA pattern)\n", - "- Optionally also request bboxes via `responseFormat => json_schema` (SVS-specific addition for direct redaction without a second OCR pass)\n", - "\n", - "#### Stage 2 — Pixel Redaction (`SVSTiffWriter._transform(df)`)\n", - "1. Open SVS with `openslide.OpenSlide(local_path)`\n", - "2. Read level-0 in 4096×4096 tiles using `slide.read_region()`\n", - "3. If bbox-only mode: draw filled black `PIL.ImageDraw.rectangle` over each detected region in label/macro\n", - "4. If text-only mode: run EasyOCR on label image to locate the strings from the VLM response, then black-out matching text (mirrors `OcrRedactor`)\n", - "5. Scrub PHI tags in `tiff.ImageDescription` using `phi_tags.scrub_image_description()`\n", - "6. Write pyramidal BigTIFF using `tifffile.TiffWriter(bigtiff=True)` with `subifds=level_count-1` and 256×256 JPEG tiles\n", - "7. Return `(tiff_output_path, phi_tags_redacted_list, pixel_regions_count)` for the audit row\n", - "\n", - "### VLM Implementation: Two Approaches\n", - "\n", - "| Approach | Used by | Library | Scale |\n", - "|---|---|---|---|\n", - "| OpenAI SDK + base64 | `VLMPhiExtractor` in pixels SA | `openai` Python SDK, `pandas_udf` | Single-node / moderate |\n", - "| `ai_query()` + `files => content` | **Our SVS pipeline** | Databricks SQL / Spark SQL | 10M images, serverless SQL |\n", - "\n", - "For the demo scale, both work. For 10M, `ai_query()` via SQL is the correct choice — it delegates throughput management to the Databricks SQL engine.\n" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "application/vnd.databricks.v1+cell": { - "cellMetadata": {}, - "inputWidgets": {}, - "nuid": "c5c5b775-4d44-415f-92f5-dd65275fa0bf", - "showTitle": false, - "tableResultSettingsMap": {}, - "title": "" - } - }, - "source": [ - "\n", - "## 7. Notebook Cell Structure\n", - "\n", - "| Cell | Purpose |\n", - "|---|---|\n", - "| **Cell 1** | This plan (markdown) |\n", - "| **Cell 2** | `%pip install dbx-pixels openslide-python openslide-bin tifffile Pillow easyocr` + `%pip install -e ./src` |\n", - "| **Cell 3** | Configuration: paths, catalog, schema, volume names, model endpoint |\n", - "| **Cell 4** | Storage bootstrap: `SVSCatalog(spark, ...).init_tables()` — `super().init_tables()` creates `object_catalog` (base DDL, unchanged); `CREATE_SVS_CATALOG.sql` creates `object_catalog_redaction` (SVS-specific columns only) |\n", - "| **Cell 5** | File discovery: `SVSCatalog.catalog(INPUT_PATH)` → writes `object_catalog` (pattern defaults to `\"*.svs\"`) |\n", - "| **Cell 6** | Metadata extraction: `SVSMetaExtractor(catalog)._transform(files_df)` → populates `meta VARIANT` (OpenSlide properties + derived SVS fields merged into one JSON object) |\n", - "| **Cell 7** | PHI tag report: SQL on `object_catalog` using VARIANT path syntax (`meta:aperio.Date::string`, `meta:phi_tag_report`) |\n", - "| **Cell 8** | Label/macro image extraction → `/Volumes/.../label_images/` |\n", - "| **Cell 9** | VLM inference: `ai_query()` SQL → `object_catalog_redaction` |\n", - "| **Cell 10** | De-identified TIFF write: `SVSTiffWriter._transform(df)` → BigTIFFs to volume |\n", - "| **Cell 11** | Audit summary: join `object_catalog` + `object_catalog_redaction`, show statistics |\n", - "\n", - "---" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "application/vnd.databricks.v1+cell": { - "cellMetadata": {}, - "inputWidgets": {}, - "nuid": "1e12f30f-e1ae-49f9-a136-8dd94032e7aa", - "showTitle": false, - "tableResultSettingsMap": {}, - "title": "" - } - }, - "source": [ - "\n", - "## 8. Scale Architecture Notes (10M Images)\n", - "\n", - "| Concern | Demo Approach | 10M Approach |\n", - "|---|---|---|\n", - "| File discovery | `dbutils.fs.ls` recursive | Auto Loader on the volume path |\n", - "| Metadata extraction | `SVSMetaExtractor` via `mapInPandas` + `ThreadPoolExecutor` | Same — already distributed |\n", - "| Label image storage | Written to volume as files | Stored as `BINARY` in Delta table (eliminates extra volume I/O) |\n", - "| VLM inference | Single SQL batch `ai_query()` | Incremental: `WHERE vlm_status='PENDING'` in a scheduled Lakeflow Job |\n", - "| TIFF conversion | `write_deidentified_tiff_udf` Spark UDF | Same — Photon-accelerated UDF dispatch |\n", - "| Checkpointing | `vlm_status` column | Same + Delta transaction log for idempotency |\n", - "| Cost control | Serverless interactive | SQL Serverless warehouse + compute-optimized clusters for UDF stages |\n", - "\n", - "---" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "application/vnd.databricks.v1+cell": { - "cellMetadata": {}, - "inputWidgets": {}, - "nuid": "9ff39799-03ad-4f51-a8a8-3d0823d8b3e2", - "showTitle": false, - "tableResultSettingsMap": {}, - "title": "" - } - }, - "source": [ - "## 9. Confirmed Findings & Resolved Design Decisions\n", - "\n", - "All four open questions are now resolved from direct inspection of the actual Aperio CMU-1.svs files.\n", - "\n", - "### Q1 — Bounding box coordinate space ✅ RESOLVED\n", - "\n", - "The label sub-image is **387×463 RGBA** — completely independent from the tissue scan (46000×32914). The two coordinate spaces share no relationship.\n", - "\n", - "**Decision:** Save and submit the label image to the VLM at **native resolution (no resizing)**. All VLM bboxes are in label-image pixel space (`0,0` = top-left). The tissue TIFF does **not** embed the label — it is automatically excluded when only the main pyramid is read. The de-identified label PNG (black rectangles applied) is written to the `label_images` volume as the audit artefact.\n", - "\n", - "**Macro image clarification:** The macro (1280×431) shows both tissue and the physical label (at one end). It is submitted to the VLM separately; its bboxes drive black-rectangle redaction of the label area in the macro output PNG.\n", - "\n", - "---\n", - "\n", - "### Q2 — `tiff.ImageDescription` scrubbing ✅ RESOLVED\n", - "\n", - "Exact format confirmed from the real file:\n", - "```\n", - "Aperio Image Library v10.0.51\\r\\n46920x33014 [0,100 46000x32914] (256x256) JPEG/RGB Q=30\n", - " |AppMag = 20|StripeWidth = 2040|ScanScope ID = CPAPERIOCS|Filename = CMU-1\n", - " |Date = 12/29/09|Time = 09:59:15|User = b414003d-...|ImageID = 1004486|...\n", - "```\n", - "\n", - "**Structure:** `{header_line}|key = val|key = val|...`  Header is technical-only — preserve as-is.\n", - "\n", - "**PHI classification of actual keys:**\n", - "\n", - "| Key | Classification | Notes |\n", - "|---|---|---|\n", - "| `Date`, `Time` | PHI | HIPAA date/time of service |\n", - "| `User` | QUESTIONABLE | GUID in demo; operator name in clinical use |\n", - "| `Filename` | QUESTIONABLE | May encode patient name or MRN |\n", - "| `ImageID` | QUESTIONABLE | Could be accession number |\n", - "| `ScanScope ID`, `AppMag`, `StripeWidth`, `Parmset`, `MPP`, all geometry/calibration, `Filtered`, `ICC Profile` | NOT_PHI | Pure scanner parameters |\n", - "\n", - "Clinical files may also contain: `Patient`, `DOB`, `MRN`, `AccessionNumber`, `Clinic`, `Pathologist`, `Procedure`, `Diagnosis`, `Id` — all PHI.\n", - "\n", - "**Scrubbing algorithm:**\n", - "1. `header, *kvs = image_desc.split('|')`\n", - "2. For each `kv`: `k, v = kv.split(' = ', 1)` — rebuild as `k = REDACTED` if `k.strip()` ∈ PHI/QUESTIONABLE set\n", - "3. Rejoin: `'|'.join([header] + rebuilt_kvs)`\n", - "4. Apply identical scrub to `openslide.comment` (same content) when writing TIFF metadata\n", - "\n", - "---\n", - "\n", - "### Q3 — Macro image redaction ✅ RESOLVED\n", - "\n", - "Macro (1280×431) shows the full physical slide including the affixed label. **Decision:** Include macro in the primary VLM pipeline alongside the label (not a follow-on phase). Naming: `{name}_label.png` / `{name}_macro.png`. Both de-identified PNGs go to the `label_images` volume.\n", - "\n", - "---\n", - "\n", - "### Q4 — Pyramidal TIFF output ✅ RESOLVED\n", - "\n", - "Flat TIFF is not viable for pathology — QuPath, OMERO, and DIGIPATH all require pyramidal. The source SVS has 3 levels with 256×256 tiles; match this in output.\n", - "\n", - "**Decision:** Write pyramidal **BigTIFF** via `tifffile`:\n", - "- `bigtiff=True` — mandatory (CMU-1 level-0 ~7.4 GB uncompressed, exceeds 4 GB TIFF limit)\n", - "- `tile=(256, 256)` — matches native Aperio tile size\n", - "- `compression='jpeg'` at quality 80; swap to `'lzw'` if lossless required\n", - "- `subifds=level_count - 1` — sub-IFDs are the QuPath/libvips-compatible pyramid convention\n", - "- Pyramid levels: 2× progressive downsampling with `PIL.Image.LANCZOS`\n", - "\n", - "```python\n", - "with tifffile.TiffWriter(output_path, bigtiff=True) as tif:\n", - " opts = dict(tile=(256, 256), compression='jpeg',\n", - " compressionargs={'level': 80}, photometric='rgb', metadata=None)\n", - " tif.write(level_0_rgb, subifds=level_count - 1, **opts) # main IFD\n", - " for lvl in range(1, level_count):\n", - " tif.write(level_arrays[lvl], subfiletype=1, **opts) # sub-IFDs\n", - "```\n", - "\n", - "OME-TIFF (`ome=True`) only if OMERO is a confirmed downstream consumer.\n", - "\n", - "---" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "application/vnd.databricks.v1+cell": { - "cellMetadata": {}, - "inputWidgets": {}, - "nuid": "308dda6c-953c-44e0-b8a3-4e20dafb9357", - "showTitle": false, - "tableResultSettingsMap": {}, - "title": "" - } - }, - "source": [ - "\n", - "## 10. More...\n", - "\n", - "### Additional: `openslide-bin` Required\n", - "\n", - "`openslide-python` alone fails at import on Databricks Serverless:\n", - "```\n", - "ModuleNotFoundError: Couldn't locate OpenSlide shared library. Try pip install openslide-bin.\n", - "```\n", - "**Cell 2 must install:** `openslide-python openslide-bin` (the `openslide-bin` wheel bundles `libopenslide.so` for environments without system package access).\n", - "\n", - "\n", - "> **Note**: `files => content` is the correct `ai_query()` API for binary image inputs — it passes the PNG bytes directly to the model without base64 encoding. Only JPEG and PNG inputs are supported.\n", - "\n", - "### Scale to 10M Images\n", - "- `vlm_status` column acts as a watermark: `PENDING → PROCESSING → COMPLETE / FAILED`\n", - "- The SQL above runs as a Databricks SQL batch job — `ai_query()` parallelizes across serverless SQL clusters automatically\n", - "- For throughput control: partition the batch by date/rack and run multiple concurrent SQL statements\n", - "- Auto Loader can feed new SVS arrivals into `svs_metadata` as `PENDING`, triggering incremental VLM runs via a scheduled Lakeflow Job\n", - "\n" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "application/vnd.databricks.v1+cell": { - "cellMetadata": {}, - "inputWidgets": {}, - "nuid": "dd0db54d-807b-4cfe-bca5-0b524fd6e636", - "showTitle": false, - "tableResultSettingsMap": {}, - "title": "" - } - }, - "source": [ - "# Code" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "application/vnd.databricks.v1+cell": { - "cellMetadata": {}, - "inputWidgets": {}, - "nuid": "6e4014e5-1ba3-4c5c-9a8c-fbe85432645b", - "showTitle": true, - "tableResultSettingsMap": {}, - "title": "Data Flow Diagram" - } - }, - "source": [ - "## Data Flow Diagram\n", - "\n", - "```mermaid\n", - "flowchart TD\n", - " %% ─── External Sources ───\n", - " SVS_INPUT[(\"/Volumes/hls_pathology/orthanc_demo/raw_images/Aperio/\\n~14 SVS files\")]\n", - " VLM_EP{{\"databricks-llama-4-maverick\\n(VLM Endpoint)\"}}\n", - "\n", - " %% ─── Delta Tables ───\n", - " OBJ_CAT[(\"douglas_moore.pathology\\n.object_catalog\")]\n", - " OBJ_RED[(\"douglas_moore.pathology\\n.object_catalog_redaction\")]\n", - " TIFF_STG[(\"douglas_moore.pathology\\n.tiff_results_staging\")]\n", - "\n", - " %% ─── Volumes (File Storage) ───\n", - " LABEL_VOL[(\"/Volumes/.../label_images/\\nPNG sub-images\")]\n", - " TIFF_VOL[(\"/Volumes/.../tiff_deidentified/\\nBigTIFF output\")]\n", - " TMP[\"/tmp/ (executor local)\\nBigTIFF staging\"]\n", - "\n", - " %% ─── Processing Steps ───\n", - " DISCOVER[\"Cell 17: File Discovery\\nSVSCatalog.catalog()\"]\n", - " META[\"Cell 18: Metadata Extraction\\nSVSMetaExtractor (mapInPandas)\\nOpenSlide properties → VARIANT\"]\n", - " PHI_TAGS[\"Cell 19: PHI Tag Report\\n(display only)\"]\n", - " EXTRACT[\"Cell 20: Extract Sub-images\\npandas_udf + OpenSlide\\nassociated_images → PNG\"]\n", - " VLM_DETECT[\"Cell 22: VLM PHI Detection\\nai_query(files => content)\\nREAD_FILES + INSERT\"]\n", - " BUILD_DF[\"Cell 24: Build redaction_df\\nJOIN catalog + redaction\\nWHERE status = PENDING\"]\n", - " UDF[\"Cell 25-27: De-identify UDF\\nmapInPandas + ThreadPoolExecutor\\nredact_image + write_pyramidal_bigtiff\"]\n", - " MERGE[\"Cell 28: MERGE Results\\nUPDATE status, paths, errors\"]\n", - " AUDIT[\"Cell 29: Audit Summary\\n(display only)\"]\n", - "\n", - " %% ─── Data Flows ───\n", - " SVS_INPUT -->|\"list files\"| DISCOVER\n", - " DISCOVER -->|\"files_df (in-memory)\"| META\n", - " SVS_INPUT -->|\"read OpenSlide props\"| META\n", - " META -->|\"mode=append\"| OBJ_CAT\n", - "\n", - " OBJ_CAT -->|\"read meta:phi_tag_report\"| PHI_TAGS\n", - "\n", - " SVS_INPUT -->|\"read associated_images\"| EXTRACT\n", - " EXTRACT -->|\"save PNG (sequential write)\"| LABEL_VOL\n", - "\n", - " LABEL_VOL -->|\"READ_FILES(binaryFile)\"| VLM_DETECT\n", - " VLM_DETECT -->|\"ai_query()\"| VLM_EP\n", - " VLM_EP -->|\"JSON response\"| VLM_DETECT\n", - " OBJ_CAT -->|\"JOIN for path\"| VLM_DETECT\n", - " VLM_DETECT -->|\"INSERT INTO\"| OBJ_RED\n", - "\n", - " OBJ_CAT -->|\"JOIN\"| BUILD_DF\n", - " OBJ_RED -->|\"WHERE PENDING\"| BUILD_DF\n", - "\n", - " BUILD_DF -->|\"redaction_df\"| UDF\n", - " SVS_INPUT -->|\"read tiles (OpenSlide)\"| UDF\n", - " UDF -->|\"redacted PNGs (seq write)\"| LABEL_VOL\n", - " UDF -->|\"write BigTIFF (seek+write)\"| TMP\n", - " TMP -->|\"shutil.copy2 (seq write)\"| TIFF_VOL\n", - " UDF -->|\"saveAsTable\"| TIFF_STG\n", - "\n", - " TIFF_STG -->|\"source for MERGE\"| MERGE\n", - " MERGE -->|\"UPDATE status/paths\"| OBJ_RED\n", - "\n", - " OBJ_CAT -->|\"LEFT JOIN\"| AUDIT\n", - " OBJ_RED -->|\"LEFT JOIN\"| AUDIT\n", - "\n", - " %% ─── Styling ───\n", - " classDef volume fill:#e8f5e9,stroke:#2e7d32\n", - " classDef table fill:#e3f2fd,stroke:#1565c0\n", - " classDef process fill:#fff3e0,stroke:#e65100\n", - " classDef external fill:#fce4ec,stroke:#c62828\n", - " classDef tmp fill:#f5f5f5,stroke:#616161,stroke-dasharray:5\n", - "\n", - " class SVS_INPUT,LABEL_VOL,TIFF_VOL volume\n", - " class OBJ_CAT,OBJ_RED,TIFF_STG table\n", - " class DISCOVER,META,PHI_TAGS,EXTRACT,VLM_DETECT,BUILD_DF,UDF,MERGE,AUDIT process\n", - " class VLM_EP external\n", - " class TMP tmp\n", - "```\n", - "\n", - "### Legend\n", - "| Color | Meaning |\n", - "|---|---|\n", - "| Green | UC Volumes (file storage) |\n", - "| Blue | Delta Tables (Unity Catalog) |\n", - "| Orange | Processing steps (notebook cells) |\n", - "| Pink | External service (model endpoint) |\n", - "| Dashed gray | Ephemeral local storage (/tmp) |\n", - "\n", - "### Key Write Patterns\n", - "| Target | Write Mode | Reason |\n", - "|---|---|---|\n", - "| `object_catalog` | `mode=append` | Idempotent cataloguing; dedup via path |\n", - "| `object_catalog_redaction` | `INSERT INTO` | One row per VLM detection run |\n", - "| `tiff_results_staging` | `mode=overwrite` | Ephemeral staging; replaced each run |\n", - "| `object_catalog_redaction` | `MERGE ... WHEN MATCHED UPDATE` | Update status after TIFF write |\n", - "| Label PNGs (volume) | Sequential FUSE write | PIL `img.save()` — no seek needed |\n", - "| BigTIFFs (volume) | `/tmp/` → `shutil.copy2` | tifffile needs seek; Volume FUSE does not support seek+write |" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "application/vnd.databricks.v1+cell": { - "cellMetadata": { - "byteLimit": 2048000, - "rowLimit": 10000 - }, - "inputWidgets": {}, - "nuid": "e1fbd42a-68b2-44e2-8f2a-d9210c5254f5", - "showTitle": true, - "tableResultSettingsMap": {}, - "title": "Cell 2: Install dependencies" - } - }, - "outputs": [], - "source": [ - "# Install core dependencies.\n", - "# databricks-pixels provides Catalog + Transformer base classes.\n", - "# openslide-bin bundles libopenslide.so so OpenSlide works on Serverless.\n", - "%pip install openslide-python openslide-bin tifffile imagecodecs Pillow easyocr -q" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "application/vnd.databricks.v1+cell": { - "cellMetadata": { - "byteLimit": 2048000, - "rowLimit": 10000 - }, - "inputWidgets": {}, - "nuid": "ec9caf79-4345-4b35-9b30-dc215d7885e4", - "showTitle": true, - "tableResultSettingsMap": {}, - "title": "Cell 3: Configuration" - } - }, - "outputs": [], - "source": [ - "# The full pixels source tree (including svs/) lives in the workspace.\n", - "# Add the src directory to sys.path so `dbx.pixels` and `dbx.pixels.svs`\n", - "# are importable without a separate pip install.\n", - "import sys\n", - "import types\n", - "import importlib\n", - "\n", - "_SRC_ROOT = \"/Workspace/Users/douglas.moore@databricks.com/pixels-svs/src\"\n", - "if _SRC_ROOT not in sys.path:\n", - " sys.path.insert(0, _SRC_ROOT)\n", - "importlib.invalidate_caches()\n", - "\n", - "# Load deidentify module (file is clean — no truncation needed)\n", - "_deident_path = f\"{_SRC_ROOT}/dbx/pixels/svs/deidentify.py\"\n", - "with open(_deident_path, \"r\") as _f:\n", - " _clean_src = _f.read()\n", - "_deident_mod = types.ModuleType(\"dbx.pixels.svs.deidentify\")\n", - "_deident_mod.__file__ = _deident_path\n", - "exec(compile(_clean_src, _deident_path, \"exec\"), _deident_mod.__dict__)\n", - "sys.modules[\"dbx.pixels.svs.deidentify\"] = _deident_mod\n", - "\n", - "from dbx.pixels.svs import SVSCatalog, SVSMetaExtractor, SVSTiffWriter\n", - "from dbx.pixels.svs.phi_tags import classify_tags, scrub_image_description\n", - "\n", - "# ── Pipeline configuration ────────────────────────────────────────────────────\n", - "CATALOG = \"douglas_moore\"\n", - "SCHEMA = \"pathology\"\n", - "UC_TABLE = f\"{CATALOG}.{SCHEMA}.object_catalog\"\n", - "UC_VOLUME = f\"{CATALOG}.{SCHEMA}.pixels_volume\"\n", - "INPUT_PATH = \"/Volumes/hls_pathology/orthanc_demo/raw_images/Aperio/\"\n", - "TIFF_VOLUME = f\"/Volumes/{CATALOG}/{SCHEMA}/tiff_deidentified\"\n", - "LABEL_VOLUME = f\"/Volumes/{CATALOG}/{SCHEMA}/label_images\"\n", - "VLM_ENDPOINT = \"databricks-llama-4-maverick\"\n", - "\n", - "print(f\"Input : {INPUT_PATH}\")\n", - "print(f\"Table : {UC_TABLE}\")\n", - "print(f\"TIFFs : {TIFF_VOLUME}\")\n", - "print(f\"Labels: {LABEL_VOLUME}\")\n", - "print(f\"VLM : {VLM_ENDPOINT}\")\n" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "application/vnd.databricks.v1+cell": { - "cellMetadata": { - "byteLimit": 2048000, - "implicitDf": true, - "rowLimit": 10000 - }, - "inputWidgets": {}, - "nuid": "47cec7fe-67e6-4928-9aa6-1c5947b011bc", - "showTitle": true, - "tableResultSettingsMap": {}, - "title": "Reset: Truncate pipeline tables" - } - }, - "outputs": [], - "source": [ - "%sql\n", - "-- Reset pipeline state for a clean end-to-end run.\n", - "-- Truncates data only; table structure and permissions preserved.\n", - "TRUNCATE TABLE douglas_moore.pathology.object_catalog;\n", - "TRUNCATE TABLE douglas_moore.pathology.object_catalog_redaction;" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "application/vnd.databricks.v1+cell": { - "cellMetadata": { - "byteLimit": 2048000, - "rowLimit": 10000 - }, - "inputWidgets": {}, - "nuid": "8c5e7ce2-bec6-48b8-987b-34b7b2a1eeb5", - "showTitle": true, - "tableResultSettingsMap": {}, - "title": "Cell 4: Storage bootstrap" - } - }, - "outputs": [], - "source": [ - "# Create schema and volumes (idempotent — safe to re-run)\n", - "spark.sql(f\"CREATE SCHEMA IF NOT EXISTS {CATALOG}.{SCHEMA}\")\n", - "spark.sql(f\"CREATE VOLUME IF NOT EXISTS {CATALOG}.{SCHEMA}.pixels_volume\")\n", - "spark.sql(f\"CREATE VOLUME IF NOT EXISTS {CATALOG}.{SCHEMA}.tiff_deidentified\")\n", - "spark.sql(f\"CREATE VOLUME IF NOT EXISTS {CATALOG}.{SCHEMA}.label_images\")\n", - "\n", - "# Initialise Delta tables:\n", - "# object_catalog — base DDL from databricks-pixels (unchanged)\n", - "# object_catalog_redaction — unified SVS/DICOM DDL from CREATE_SVS_CATALOG.sql\n", - "catalog = SVSCatalog(spark, table=UC_TABLE, volume=UC_VOLUME)\n", - "catalog.init_tables()\n", - "\n", - "print(\"Schema, volumes, and tables initialised.\")\n", - "display(spark.sql(f\"SHOW TABLES IN {CATALOG}.{SCHEMA}\"))\n" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "application/vnd.databricks.v1+cell": { - "cellMetadata": { - "byteLimit": 2048000, - "rowLimit": 10000 - }, - "inputWidgets": {}, - "nuid": "d0560972-3ca5-4f06-8f65-d7c621b688db", - "showTitle": true, - "tableResultSettingsMap": {}, - "title": "Cell 5: File discovery" - } - }, - "outputs": [], - "source": [ - "# Discover all SVS files under INPUT_PATH and register them in object_catalog.\n", - "# SVSCatalog.catalog() defaults pattern='*.svs'; also picks up sidecar .txt files.\n", - "files_df = catalog.catalog(INPUT_PATH)\n", - "print(f\"Discovered {files_df.count()} files\")\n", - "display(files_df.select(\"path\", \"local_path\", \"length\", \"modificationTime\", \"extension\"))\n" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "application/vnd.databricks.v1+cell": { - "cellMetadata": { - "byteLimit": 2048000, - "rowLimit": 10000 - }, - "inputWidgets": {}, - "nuid": "147b854b-61b3-4207-a172-9fc5f36649b9", - "showTitle": true, - "tableResultSettingsMap": {}, - "title": "Cell 6: Metadata extraction" - } - }, - "outputs": [], - "source": [ - "# SVSMetaExtractor reads every SVS via OpenSlide (ThreadPoolExecutor, 32 concurrent).\n", - "# All properties + derived fields (width, height, levels, phi_tag_report) are merged\n", - "# into one JSON dict → parse_json() → VARIANT. No schema changes to object_catalog.\n", - "extractor = SVSMetaExtractor(catalog, inputCol=\"local_path\", outputCol=\"meta\")\n", - "meta_df = extractor._transform(files_df)\n", - "\n", - "(\n", - " meta_df.write\n", - " .format(\"delta\")\n", - " .mode(\"append\")\n", - " .saveAsTable(UC_TABLE)\n", - ")\n", - "\n", - "print(f\"Wrote {spark.table(UC_TABLE).count()} rows to {UC_TABLE}\")\n", - "\n", - "display(spark.sql(f\"\"\"\n", - "SELECT\n", - " regexp_extract(path, '[^/]+$', 0) AS filename,\n", - " meta:width::int AS width,\n", - " meta:height::int AS height,\n", - " meta:level_count::int AS levels,\n", - " meta:has_label_image::boolean AS has_label,\n", - " meta:has_macro_image::boolean AS has_macro,\n", - " meta:`aperio.AppMag`::string AS app_mag,\n", - " meta:`aperio.MPP`::string AS mpp,\n", - " meta\n", - "FROM {UC_TABLE}\n", - "ORDER BY filename\n", - "\"\"\"))\n" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "application/vnd.databricks.v1+cell": { - "cellMetadata": { - "byteLimit": 2048000, - "rowLimit": 10000 - }, - "inputWidgets": {}, - "nuid": "541697a5-602a-40dd-9ac9-9f53624b4efa", - "showTitle": true, - "tableResultSettingsMap": { - "0": { - "dataGridStateBlob": "{\"version\":1,\"tableState\":{\"columnPinning\":{\"left\":[\"#row_number#\"],\"right\":[]},\"columnSizing\":{\"tag\":129},\"columnVisibility\":{}},\"settings\":{\"columns\":{}},\"syncTimestamp\":1781723212243}", - "filterBlob": null, - "queryPlanFiltersBlob": null, - "tableResultIndex": 0 - } - }, - "title": "Cell 7: PHI tag report" - } - }, - "outputs": [], - "source": [ - "# PHI / QUESTIONABLE tag values for every slide.\n", - "# Unpack the phi_tag_report VARIANT array stored in meta.\n", - "from pyspark.sql.functions import regexp_extract, col, explode, from_json, expr\n", - "from pyspark.sql.types import ArrayType, StructType, StructField, StringType\n", - "\n", - "phi_schema = ArrayType(StructType([\n", - " StructField(\"tag\", StringType()),\n", - " StructField(\"value\", StringType()),\n", - " StructField(\"classification\", StringType()),\n", - "]))\n", - "\n", - "phi_df = (\n", - " spark.table(UC_TABLE)\n", - " .filter(expr(\"meta:phi_tag_report IS NOT NULL\"))\n", - " .withColumn(\"phi_tag_report_str\", expr(\"cast(meta:phi_tag_report AS STRING)\"))\n", - " .withColumn(\"tags\", from_json(\"phi_tag_report_str\", phi_schema))\n", - " .withColumn(\"elem\", explode(\"tags\"))\n", - " .select(\n", - " regexp_extract(\"path\", r\"[^/]+$\", 0).alias(\"filename\"),\n", - " col(\"elem.tag\").alias(\"tag\"),\n", - " col(\"elem.value\").alias(\"value\"),\n", - " col(\"elem.classification\").alias(\"classification\"),\n", - " )\n", - " .filter(col(\"classification\").isin(\"PHI\", \"QUESTIONABLE\"))\n", - " .orderBy(\"filename\", \"classification\", \"tag\")\n", - ")\n", - "print(f\"PHI/QUESTIONABLE findings: {phi_df.count()}\")\n", - "display(phi_df)" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "application/vnd.databricks.v1+cell": { - "cellMetadata": { - "byteLimit": 2048000, - "rowLimit": 10000 - }, - "inputWidgets": {}, - "nuid": "28f6bfb9-bec1-4cb0-9c80-5c395a1a9432", - "showTitle": true, - "tableResultSettingsMap": {}, - "title": "Cell 8: Extract label/macro sub-images" - } - }, - "outputs": [], - "source": [ - "# Extract label and macro sub-images from each SVS and save as PNGs.\n", - "# These are later submitted to the VLM (Cell 9) and used as audit artefacts.\n", - "# Uses a pandas_udf so extraction runs distributed across workers.\n", - "from pyspark.sql.functions import pandas_udf, regexp_extract, col\n", - "import pandas as pd\n", - "from pyspark.sql.types import StringType\n", - "\n", - "_LABEL_VOL = LABEL_VOLUME # captured in closure; serialised with the UDF\n", - "\n", - "@pandas_udf(StringType())\n", - "def extract_subimages_udf(paths: pd.Series, stems: pd.Series) -> pd.Series:\n", - " import openslide, os\n", - " results = []\n", - " for path, stem in zip(paths, stems):\n", - " try:\n", - " slide = openslide.OpenSlide(path)\n", - " saved = []\n", - " for name in (\"label\", \"macro\"):\n", - " if name in slide.associated_images:\n", - " img = slide.associated_images[name].convert(\"RGB\")\n", - " out = f\"{_LABEL_VOL}/{stem}_{name}.png\"\n", - " os.makedirs(os.path.dirname(out), exist_ok=True)\n", - " img.save(out)\n", - " saved.append(out)\n", - " slide.close()\n", - " results.append(\",\".join(saved))\n", - " except Exception as e:\n", - " results.append(f\"ERROR: {e}\")\n", - " return pd.Series(results)\n", - "\n", - "catalog_df = (\n", - " spark.table(UC_TABLE)\n", - " .withColumn(\"stem\", regexp_extract(col(\"path\"), r\"([^/]+)\\.svs$\", 1))\n", - ")\n", - "\n", - "extracted_df = catalog_df.withColumn(\n", - " \"extracted_images\",\n", - " extract_subimages_udf(col(\"local_path\"), col(\"stem\")),\n", - ")\n", - "\n", - "display(extracted_df.select(\"path\", \"stem\", \"extracted_images\"))\n", - "print(f\"Label/macro PNGs written to {LABEL_VOLUME}\")\n" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "application/vnd.databricks.v1+cell": { - "cellMetadata": { - "byteLimit": 2048000, - "rowLimit": 10000 - }, - "inputWidgets": {}, - "nuid": "556f24ea-d0c5-46ef-ac35-761ffed88820", - "showTitle": true, - "tableResultSettingsMap": {}, - "title": "Display first 10 label images" - } - }, - "outputs": [], - "source": [ - "# Display first 10 label sub-images extracted from SVS pathology slides\n", - "import os\n", - "from PIL import Image\n", - "import matplotlib.pyplot as plt\n", - "\n", - "label_dir = LABEL_VOLUME\n", - "label_files = sorted([f for f in os.listdir(label_dir) if f.endswith(\"_label.png\")])[:10]\n", - "\n", - "ncols = min(5, len(label_files))\n", - "nrows = (len(label_files) + ncols - 1) // ncols\n", - "fig, axes = plt.subplots(nrows, ncols, figsize=(4 * ncols, 5 * nrows))\n", - "if len(label_files) == 1:\n", - " axes = [axes]\n", - "else:\n", - " axes = axes.flatten()\n", - "\n", - "for i, fname in enumerate(label_files):\n", - " img = Image.open(os.path.join(label_dir, fname))\n", - " axes[i].imshow(img)\n", - " axes[i].set_title(fname.replace(\"_label.png\", \"\"), fontsize=9)\n", - " axes[i].axis(\"off\")\n", - "\n", - "for j in range(len(label_files), len(axes)):\n", - " axes[j].axis(\"off\")\n", - "\n", - "plt.suptitle(\"SVS Label Sub-Images (PHI candidates for VLM redaction)\", fontsize=13)\n", - "plt.tight_layout()\n", - "plt.show()" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "application/vnd.databricks.v1+cell": { - "cellMetadata": { - "byteLimit": 2048000, - "implicitDf": true, - "rowLimit": 10000 - }, - "height": "156", - "inputWidgets": {}, - "nuid": "23583e7a-dfc7-4da0-be2b-dbac0b061935", - "showTitle": true, - "tableResultSettingsMap": {}, - "title": "Cell 9: VLM PHI detection", - "width": "834" - } - }, - "outputs": [], - "source": [ - "%sql\n", - "-- Run VLM PHI detection on all label PNGs and insert results into object_catalog_redaction.\n", - "-- ai_query() submits the image binary directly via `files => content` (no base64 needed).\n", - "-- responseFormat => 'json_object' guarantees machine-parseable output.\n", - "INSERT INTO douglas_moore.pathology.object_catalog_redaction (\n", - " redaction_id, path, extension, modality,\n", - " has_phi, phi_elements, vlm_raw_response, model_endpoint,\n", - " output_file_paths, label_image_path, macro_image_path,\n", - " status, insert_timestamp, created_by\n", - ")\n", - "WITH vlm_raw AS (\n", - " SELECT\n", - " regexp_replace(f._metadata.file_name, '_label\\.png$', '') AS stem,\n", - " f._metadata.file_path AS label_image_path,\n", - " ai_query(\n", - " 'databricks-llama-4-maverick',\n", - " 'You are a HIPAA-compliant PHI detection system.\n", - "Analyze this pathology slide label image and identify all Protected Health Information:\n", - "patient names, dates, MRNs, accession numbers, barcodes, or other identifying text.\n", - "Return ONLY a json object — no prose, no markdown fences.\n", - "Schema: {\"has_phi\": bool, \"phi_elements\": [{\"type\": \"name|dob|mrn|accession|barcode|other\", \"value_hint\": \"\", \"bbox\": {\"x\": int, \"y\": int, \"w\": int, \"h\": int}, \"subimage\": \"label\"}]}\n", - "If no PHI found: {\"has_phi\": false, \"phi_elements\": []}',\n", - " files => content\n", - " ) AS vlm_raw_response\n", - " FROM READ_FILES(\n", - " '/Volumes/douglas_moore/pathology/label_images/',\n", - " format => 'binaryFile',\n", - " fileNamePattern => '*_label.png'\n", - " ) f\n", - "),\n", - "joined AS (\n", - " SELECT\n", - " v.stem,\n", - " v.label_image_path,\n", - " v.vlm_raw_response,\n", - " m.path AS obj_path,\n", - " concat('/Volumes/douglas_moore/pathology/label_images/', v.stem, '_macro.png') AS macro_image_path\n", - " FROM vlm_raw v\n", - " JOIN douglas_moore.pathology.object_catalog m\n", - " ON regexp_extract(m.path, '([^/]+)\\.svs$', 1) = v.stem\n", - ")\n", - "SELECT\n", - " uuid() AS redaction_id,\n", - " obj_path AS path,\n", - " 'svs' AS extension,\n", - " 'WSI' AS modality,\n", - " try_cast(get_json_object(vlm_raw_response, '$.has_phi') AS BOOLEAN) AS has_phi,\n", - " parse_json(get_json_object(vlm_raw_response, '$.phi_elements')) AS phi_elements,\n", - " vlm_raw_response,\n", - " 'databricks-llama-4-maverick' AS model_endpoint,\n", - " array(CAST(NULL AS STRING)) AS output_file_paths,\n", - " label_image_path,\n", - " macro_image_path,\n", - " 'PENDING' AS status,\n", - " current_timestamp() AS insert_timestamp,\n", - " current_user() AS created_by\n", - "FROM joined\n" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "application/vnd.databricks.v1+cell": { - "cellMetadata": { - "byteLimit": 2048000, - "implicitDf": true, - "rowLimit": 10000 - }, - "inputWidgets": {}, - "nuid": "6dae4df9-e494-403e-9c58-a933aa22052a", - "showTitle": false, - "tableResultSettingsMap": {}, - "title": "" - } - }, - "outputs": [], - "source": [ - "%sql\n", - "select * from douglas_moore.pathology.object_catalog_redaction" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "application/vnd.databricks.v1+cell": { - "cellMetadata": { - "byteLimit": 2048000, - "rowLimit": 10000 - }, - "inputWidgets": {}, - "nuid": "98cc5d83-bbdd-4f8c-9bd6-b61d259ad882", - "showTitle": false, - "tableResultSettingsMap": {}, - "title": "" - } - }, - "outputs": [], - "source": [ - "# Reload modules to pick up streaming TIFF writer\n", - "import importlib, sys\n", - "for mod_name in list(sys.modules):\n", - " if mod_name.startswith(\"dbx.pixels.svs\"):\n", - " del sys.modules[mod_name]\n", - "\n", - "# Join PENDING redaction rows (phi_elements from VLM) with object_catalog (local_path),\n", - "# run de-identification and produce pyramidal BigTIFFs.\n", - "redaction_df = spark.sql(f\"\"\"\n", - "SELECT\n", - " o.local_path,\n", - " o.path,\n", - " r.redaction_id,\n", - " to_json(r.phi_elements) AS phi_elements_json\n", - "FROM {CATALOG}.{SCHEMA}.object_catalog o\n", - "JOIN {CATALOG}.{SCHEMA}.object_catalog_redaction r\n", - " ON o.path = r.path\n", - "WHERE r.status = 'PENDING'\n", - "\"\"\")\n", - "print(f\"Files to de-identify: {redaction_df.count()}\")" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "application/vnd.databricks.v1+cell": { - "cellMetadata": { - "byteLimit": 2048000, - "rowLimit": 10000 - }, - "inputWidgets": {}, - "nuid": "a73b4474-3adc-4866-88cd-822dc0ad6c45", - "showTitle": true, - "tableResultSettingsMap": {}, - "title": "Cell 10: De-identified TIFF write" - } - }, - "outputs": [], - "source": [ - "# --- Distributed de-identification via mapInPandas (memory-safe for Serverless 1 GB) ---\n", - "#\n", - "# Design principles applied from review:\n", - "# • No inner ThreadPoolExecutor — mapInPandas already parallelizes across Spark\n", - "# partitions; nested threading doubles slide opens and memory pressure.\n", - "# • No to_dict(\"records\") — iterate rows via iloc to avoid duplicating the batch.\n", - "# • OpenSlide closed in a finally block so C-bindings are destroyed even on error.\n", - "# • gc.collect() after each slide reclaims PIL/OpenSlide C-level allocations.\n", - "# • Tile-based TIFF writing via write_pyramidal_bigtiff_streaming (256×256 read_region).\n", - "# • Temp files staged to /tmp (seek-capable), then shutil.copy2 to Volume (seq FUSE).\n", - "# • PID suffix on temp paths prevents collisions across retries/speculative tasks.\n", - "# • repartition(num_slides) ensures 1 row per partition — each executor handles\n", - "# exactly one slide. (arrow.maxRecordsPerBatch is NOT settable on Serverless.)\n", - "\n", - "import json\n", - "import pandas as pd\n", - "from pyspark.sql.types import StructType, StructField, StringType, ArrayType, IntegerType\n", - "\n", - "_TIFF_VOLUME = TIFF_VOLUME\n", - "_LABEL_VOLUME = LABEL_VOLUME\n", - "_SRC_ROOT = \"/Workspace/Users/douglas.moore@databricks.com/pixels-svs/src\"\n", - "\n", - "_result_schema = StructType([\n", - " StructField(\"path\", StringType(), True),\n", - " StructField(\"tiff_output_path\", StringType(), True),\n", - " StructField(\"label_image_path\", StringType(), True),\n", - " StructField(\"macro_image_path\", StringType(), True),\n", - " StructField(\"phi_tags_redacted\", ArrayType(StringType()), True),\n", - " StructField(\"pixel_regions_redacted\", IntegerType(), True),\n", - " StructField(\"error\", StringType(), True),\n", - "])\n", - "\n", - "\n", - "def _deidentify_batch(iterator):\n", - " \"\"\"mapInPandas worker: one slide per batch, streaming tile reads, no threading.\"\"\"\n", - " import sys, os, gc, shutil, time, logging, resource\n", - " from pathlib import Path\n", - "\n", - " # --- Memory debugging utilities ---\n", - " logging.basicConfig(level=logging.INFO, format=\"%(asctime)s [%(levelname)s] %(message)s\")\n", - " log = logging.getLogger(\"deidentify_worker\")\n", - "\n", - " def _mem_mb() -> dict:\n", - " \"\"\"Return RSS and VMS in MB from /proc/self/status (Linux) with fallback.\"\"\"\n", - " rss_mb = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss / 1024 # KB→MB on Linux\n", - " try:\n", - " with open(\"/proc/self/status\") as f:\n", - " status = f.read()\n", - " vmpeak = vmrss = vmsize = 0\n", - " for line in status.splitlines():\n", - " if line.startswith(\"VmPeak:\"):\n", - " vmpeak = int(line.split()[1]) / 1024\n", - " elif line.startswith(\"VmRSS:\"):\n", - " vmrss = int(line.split()[1]) / 1024\n", - " elif line.startswith(\"VmSize:\"):\n", - " vmsize = int(line.split()[1]) / 1024\n", - " return {\"rss_mb\": round(vmrss, 1), \"vms_mb\": round(vmsize, 1), \"peak_mb\": round(vmpeak, 1)}\n", - " except Exception:\n", - " return {\"rss_mb\": round(rss_mb, 1), \"vms_mb\": -1, \"peak_mb\": -1}\n", - "\n", - " def _log_mem(stage: str, stem: str, extra: str = \"\"):\n", - " mem = _mem_mb()\n", - " msg = f\"[{stem}] stage={stage} | RSS={mem['rss_mb']}MB VMS={mem['vms_mb']}MB Peak={mem['peak_mb']}MB\"\n", - " if extra:\n", - " msg += f\" | {extra}\"\n", - " log.info(msg)\n", - " # Warn if approaching the 1024 MB limit\n", - " if mem[\"rss_mb\"] > 800:\n", - " log.warning(f\"⚠️ HIGH MEMORY [{stem}] stage={stage} RSS={mem['rss_mb']}MB — approaching 1024MB limit!\")\n", - "\n", - " # Ensure src modules are importable on executors\n", - " if _SRC_ROOT not in sys.path:\n", - " sys.path.insert(0, _SRC_ROOT)\n", - "\n", - " import openslide\n", - " from dbx.pixels.svs.deidentify import redact_image, write_pyramidal_bigtiff_streaming\n", - " from dbx.pixels.svs.phi_tags import scrub_image_description\n", - "\n", - " for pdf in iterator:\n", - " results = []\n", - " log.info(f\"Batch received: {len(pdf)} row(s) | PID={os.getpid()}\")\n", - " _log_mem(\"batch_start\", \"batch\", f\"rows={len(pdf)}\")\n", - "\n", - " # Iterate rows directly via iloc — no to_dict(\"records\") memory copy\n", - " for idx in range(len(pdf)):\n", - " row = pdf.iloc[idx]\n", - " svs_path = row[\"local_path\"]\n", - " phi_json = row[\"phi_elements_json\"]\n", - " stem = Path(svs_path).stem\n", - " stage = \"init\"\n", - " slide = None\n", - " tmp_tiff = f\"/tmp/{stem}_{os.getpid()}.tiff\"\n", - " t0 = time.time()\n", - "\n", - " log.info(f\"=== Processing slide: {stem} ===\")\n", - " _log_mem(\"init\", stem, f\"svs_path={svs_path}\")\n", - "\n", - " try:\n", - " phi_elements = json.loads(phi_json) if phi_json else []\n", - " stage = \"open_slide\"\n", - " slide = openslide.OpenSlide(svs_path)\n", - " dims = slide.dimensions # (width, height) at level 0\n", - " levels = slide.level_count\n", - " _log_mem(\"open_slide\", stem, f\"dims={dims[0]}x{dims[1]} levels={levels}\")\n", - "\n", - " # 1. Redact label/macro sub-images (small RGBA → write PNGs to Volume)\n", - " label_path = macro_path = None\n", - " pixel_count = 0\n", - "\n", - " if \"label\" in slide.associated_images:\n", - " stage = \"redact_label\"\n", - " label_img = slide.associated_images[\"label\"]\n", - " label_phi = [e for e in phi_elements if e.get(\"subimage\", \"label\") != \"macro\"]\n", - " label_out = redact_image(label_img, label_phi)\n", - " pixel_count += len(label_phi)\n", - " label_path = f\"{_LABEL_VOLUME}/{stem}_label.png\"\n", - " os.makedirs(os.path.dirname(label_path), exist_ok=True)\n", - " label_out.save(label_path)\n", - " _log_mem(\"redact_label\", stem, f\"label_size={label_img.size} phi_count={len(label_phi)}\")\n", - " del label_img, label_out # free RGBA buffer immediately\n", - "\n", - " if \"macro\" in slide.associated_images:\n", - " stage = \"redact_macro\"\n", - " macro_img = slide.associated_images[\"macro\"]\n", - " macro_phi = [e for e in phi_elements if e.get(\"subimage\") == \"macro\"]\n", - " macro_out = redact_image(macro_img, macro_phi)\n", - " pixel_count += len(macro_phi)\n", - " macro_path = f\"{_LABEL_VOLUME}/{stem}_macro.png\"\n", - " os.makedirs(os.path.dirname(macro_path), exist_ok=True)\n", - " macro_out.save(macro_path)\n", - " _log_mem(\"redact_macro\", stem, f\"macro_size={macro_img.size} phi_count={len(macro_phi)}\")\n", - " del macro_img, macro_out\n", - "\n", - " # 2. Scrub metadata tags\n", - " stage = \"scrub_metadata\"\n", - " raw_desc = slide.properties.get(\"tiff.ImageDescription\", \"\")\n", - " scrubbed = scrub_image_description(raw_desc)\n", - " phi_tags_redacted = (\n", - " [\"tiff.ImageDescription\", \"openslide.comment\"]\n", - " if raw_desc != scrubbed else []\n", - " )\n", - " _log_mem(\"scrub_metadata\", stem)\n", - "\n", - " # 3. Write pyramidal BigTIFF — tile-streaming (256×256 read_region)\n", - " # Stage to /tmp (requires seek), then sequential copy to Volume.\n", - " stage = \"write_tiff_to_tmp\"\n", - " t_tiff_start = time.time()\n", - " write_pyramidal_bigtiff_streaming(tmp_tiff, slide, jpeg_quality=80)\n", - " t_tiff_elapsed = time.time() - t_tiff_start\n", - " tmp_size_mb = os.path.getsize(tmp_tiff) / (1024 * 1024) if os.path.exists(tmp_tiff) else 0\n", - " _log_mem(\"write_tiff_done\", stem, f\"tiff_size={tmp_size_mb:.1f}MB elapsed={t_tiff_elapsed:.1f}s\")\n", - "\n", - " # Close slide BEFORE copy to free C-level handles and mapped memory\n", - " slide.close()\n", - " slide = None\n", - " _log_mem(\"slide_closed\", stem)\n", - "\n", - " stage = \"copy_tiff_to_volume\"\n", - " t_copy_start = time.time()\n", - " tiff_path = f\"{_TIFF_VOLUME}/{stem}.tiff\"\n", - " shutil.copy2(tmp_tiff, tiff_path)\n", - " os.remove(tmp_tiff)\n", - " t_copy_elapsed = time.time() - t_copy_start\n", - " _log_mem(\"copy_done\", stem, f\"copy_elapsed={t_copy_elapsed:.1f}s\")\n", - "\n", - " total_elapsed = time.time() - t0\n", - " log.info(f\"✓ [{stem}] completed in {total_elapsed:.1f}s | tiff={tmp_size_mb:.1f}MB\")\n", - "\n", - " results.append({\n", - " \"path\": row[\"path\"],\n", - " \"tiff_output_path\": tiff_path,\n", - " \"label_image_path\": label_path,\n", - " \"macro_image_path\": macro_path,\n", - " \"phi_tags_redacted\": phi_tags_redacted,\n", - " \"pixel_regions_redacted\": pixel_count,\n", - " \"error\": None,\n", - " })\n", - "\n", - " except Exception as exc:\n", - " _log_mem(\"ERROR\", stem, f\"stage={stage} exc={type(exc).__name__}: {exc}\")\n", - " results.append({\n", - " \"path\": row[\"path\"],\n", - " \"tiff_output_path\": None,\n", - " \"label_image_path\": None,\n", - " \"macro_image_path\": None,\n", - " \"phi_tags_redacted\": [],\n", - " \"pixel_regions_redacted\": 0,\n", - " \"error\": f\"[stage={stage}] {type(exc).__name__}: {exc}\",\n", - " })\n", - "\n", - " finally:\n", - " # Ensure slide is always closed — destroy C-bindings immediately\n", - " if slide is not None:\n", - " try:\n", - " slide.close()\n", - " except Exception:\n", - " pass\n", - " # Clean up temp file on failure\n", - " if os.path.exists(tmp_tiff):\n", - " try:\n", - " os.remove(tmp_tiff)\n", - " except Exception:\n", - " pass\n", - " # Force GC to reclaim PIL/OpenSlide C-level allocations\n", - " gc.collect()\n", - " _log_mem(\"gc_complete\", stem)\n", - "\n", - " yield pd.DataFrame(results)" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "application/vnd.databricks.v1+cell": { - "cellMetadata": { - "byteLimit": 2048000, - "rowLimit": 10000 - }, - "inputWidgets": {}, - "nuid": "b449d29c-0651-41bc-afc8-bf83bdcf7f74", - "showTitle": false, - "tableResultSettingsMap": {}, - "title": "" - } - }, - "outputs": [], - "source": [ - "# Materialize the expensive mapInPandas UDF exactly ONCE.\n", - "# Strategy: persist() + count() forces a single execution pass.\n", - "# Downstream MERGE reads from the cached DataFrame via temp view.\n", - "TIFF_RESULTS_TABLE = f\"{CATALOG}.{SCHEMA}.tiff_results_staging\"\n", - "\n", - "num_slides = redaction_df.count()\n", - "print(f\"\"\"{num_slides}\"\"\")\n", - "assert num_slides > 0, \"No PENDING slides to de-identify — check object_catalog_redaction status\"\n" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "application/vnd.databricks.v1+cell": { - "cellMetadata": { - "byteLimit": 2048000, - "rowLimit": 10000 - }, - "inputWidgets": {}, - "nuid": "2581617a-cb08-4d46-b604-56b0c4d0400e", - "showTitle": true, - "tableResultSettingsMap": {}, - "title": "Dry run: 1 slide with memory logging" - } - }, - "outputs": [], - "source": [ - "# --- Dry run: 1 slide on DRIVER to capture full memory trace ---\n", - "# Runs the same logic outside mapInPandas so we can see exactly which stage\n", - "# exceeds 1024 MB without the executor being killed.\n", - "\n", - "import sys, os, gc, time, resource, json, shutil\n", - "from pathlib import Path\n", - "\n", - "if \"/Workspace/Users/douglas.moore@databricks.com/pixels-svs/src\" not in sys.path:\n", - " sys.path.insert(0, \"/Workspace/Users/douglas.moore@databricks.com/pixels-svs/src\")\n", - "\n", - "import openslide\n", - "from dbx.pixels.svs.deidentify import redact_image, write_pyramidal_bigtiff_streaming\n", - "from dbx.pixels.svs.phi_tags import scrub_image_description\n", - "\n", - "def _mem_mb():\n", - " \"\"\"RSS/VMS/Peak from /proc/self/status.\"\"\"\n", - " try:\n", - " with open(\"/proc/self/status\") as f:\n", - " status = f.read()\n", - " vals = {}\n", - " for line in status.splitlines():\n", - " for key in (\"VmPeak\", \"VmRSS\", \"VmSize\"):\n", - " if line.startswith(key + \":\"):\n", - " vals[key] = int(line.split()[1]) / 1024 # KB→MB\n", - " return {\"rss_mb\": round(vals.get(\"VmRSS\", 0), 1),\n", - " \"vms_mb\": round(vals.get(\"VmSize\", 0), 1),\n", - " \"peak_mb\": round(vals.get(\"VmPeak\", 0), 1)}\n", - " except Exception:\n", - " return {\"rss_mb\": -1, \"vms_mb\": -1, \"peak_mb\": -1}\n", - "\n", - "def log_mem(stage, extra=\"\"):\n", - " mem = _mem_mb()\n", - " warn = \" ⚠️ OVER 1GB!\" if mem[\"rss_mb\"] > 1024 else (\"⚠️ HIGH\" if mem[\"rss_mb\"] > 800 else \"\")\n", - " print(f\" [{stage:20s}] RSS={mem['rss_mb']:>7.1f}MB VMS={mem['vms_mb']:>7.1f}MB Peak={mem['peak_mb']:>7.1f}MB {warn} {extra}\")\n", - "\n", - "# Get one slide from the redaction dataframe\n", - "row = redaction_df.limit(1).collect()[0]\n", - "svs_path = row[\"local_path\"]\n", - "phi_json = row[\"phi_elements_json\"]\n", - "stem = Path(svs_path).stem\n", - "tmp_tiff = f\"/tmp/{stem}_dryrun.tiff\"\n", - "\n", - "print(f\"\\n{'='*80}\")\n", - "print(f\"DRY RUN MEMORY PROFILE: {stem}\")\n", - "print(f\"SVS path: {svs_path}\")\n", - "print(f\"{'='*80}\")\n", - "\n", - "gc.collect()\n", - "log_mem(\"baseline\")\n", - "\n", - "# Open slide\n", - "t0 = time.time()\n", - "slide = openslide.OpenSlide(svs_path)\n", - "dims = slide.dimensions\n", - "print(f\"\\n Slide: {dims[0]}x{dims[1]} pixels, {slide.level_count} levels\")\n", - "print(f\" Level dimensions: {[slide.level_dimensions[i] for i in range(slide.level_count)]}\")\n", - "print(f\" Associated images: {list(slide.associated_images.keys())}\")\n", - "log_mem(\"open_slide\", f\"file_size={os.path.getsize(svs_path)/(1024*1024):.1f}MB\")\n", - "\n", - "# Redact label\n", - "if \"label\" in slide.associated_images:\n", - " label_img = slide.associated_images[\"label\"]\n", - " print(f\"\\n Label image: {label_img.size} mode={label_img.mode}\")\n", - " phi_elements = json.loads(phi_json) if phi_json else []\n", - " label_phi = [e for e in phi_elements if e.get(\"subimage\", \"label\") != \"macro\"]\n", - " label_out = redact_image(label_img, label_phi)\n", - " log_mem(\"redact_label\", f\"phi_regions={len(label_phi)}\")\n", - " del label_img, label_out\n", - " gc.collect()\n", - " log_mem(\"label_freed\")\n", - "\n", - "# Redact macro\n", - "if \"macro\" in slide.associated_images:\n", - " macro_img = slide.associated_images[\"macro\"]\n", - " print(f\"\\n Macro image: {macro_img.size} mode={macro_img.mode}\")\n", - " phi_elements = json.loads(phi_json) if phi_json else []\n", - " macro_phi = [e for e in phi_elements if e.get(\"subimage\") == \"macro\"]\n", - " macro_out = redact_image(macro_img, macro_phi)\n", - " log_mem(\"redact_macro\", f\"phi_regions={len(macro_phi)}\")\n", - " del macro_img, macro_out\n", - " gc.collect()\n", - " log_mem(\"macro_freed\")\n", - "\n", - "# Scrub metadata\n", - "raw_desc = slide.properties.get(\"tiff.ImageDescription\", \"\")\n", - "scrubbed = scrub_image_description(raw_desc)\n", - "log_mem(\"scrub_metadata\")\n", - "\n", - "# Write pyramidal BigTIFF (this is the suspected memory hog)\n", - "print(f\"\\n Writing pyramidal BigTIFF to /tmp ...\")\n", - "t_tiff = time.time()\n", - "try:\n", - " write_pyramidal_bigtiff_streaming(tmp_tiff, slide, jpeg_quality=80)\n", - " tiff_elapsed = time.time() - t_tiff\n", - " tiff_size = os.path.getsize(tmp_tiff) / (1024 * 1024)\n", - " log_mem(\"write_tiff_done\", f\"size={tiff_size:.1f}MB elapsed={tiff_elapsed:.1f}s\")\n", - "except Exception as e:\n", - " log_mem(\"write_tiff_FAILED\", f\"{type(e).__name__}: {e}\")\n", - " tiff_size = 0\n", - "\n", - "# Close slide\n", - "slide.close()\n", - "log_mem(\"slide_closed\")\n", - "gc.collect()\n", - "log_mem(\"gc_after_close\")\n", - "\n", - "# Cleanup\n", - "if os.path.exists(tmp_tiff):\n", - " os.remove(tmp_tiff)\n", - "\n", - "total = time.time() - t0\n", - "print(f\"\\n{'='*80}\")\n", - "print(f\"COMPLETE: {stem} in {total:.1f}s | TIFF={tiff_size:.1f}MB\")\n", - "print(f\"Peak memory: {_mem_mb()['peak_mb']:.1f}MB\")\n", - "print(f\"{'='*80}\")\n", - "if _mem_mb()[\"peak_mb\"] > 1024:\n", - " print(\"\\n❌ Peak memory EXCEEDED 1024MB — this will OOM on Serverless executors.\")\n", - " print(\" → Investigate write_pyramidal_bigtiff_streaming tile buffer size.\")\n", - "else:\n", - " print(\"\\n✅ Peak memory stayed under 1024MB — safe for Serverless executors.\")" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "application/vnd.databricks.v1+cell": { - "cellMetadata": { - "byteLimit": 2048000, - "rowLimit": 10000 - }, - "inputWidgets": {}, - "nuid": "a8e83dad-0127-488c-b894-5d442508e404", - "showTitle": true, - "tableResultSettingsMap": {}, - "title": "Cell 26: Execute TIFF write (materialize once)" - } - }, - "outputs": [], - "source": [ - "\n", - "# 1 slide per partition to stay under 1GB UDF memory limit on serverless.\n", - "# Large SVS files (CMU-1 = 46000x32000) need sole access to executor RAM.\n", - "results_df = (\n", - " redaction_df\n", - " .repartition(num_slides)\n", - " .mapInPandas(_deidentify_batch, schema=_result_schema)\n", - " .limit(4)\n", - ")\n", - "\n", - "# Force single execution — UDF runs here and only here\n", - "results_df.select(\"path\", \"tiff_output_path\", \"label_image_path\", \"macro_image_path\", \"phi_tags_redacted\", \"pixel_regions_redacted\", \"error\").write.saveAsTable(TIFF_RESULTS_TABLE)\n", - "\n", - "\n", - "display(spark.sql(f\"SELECT * FROM {TIFF_RESULTS_TABLE}\"))" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "application/vnd.databricks.v1+cell": { - "cellMetadata": { - "byteLimit": 2048000, - "rowLimit": 10000 - }, - "inputWidgets": {}, - "nuid": "5c7b3af8-ded5-4207-92d1-42a0710e11e6", - "showTitle": true, - "tableResultSettingsMap": {}, - "title": "Cell 23" - } - }, - "outputs": [], - "source": [ - "# Merge output paths and status back into object_catalog_redaction\n", - "spark.sql(f\"\"\"\n", - "MERGE INTO {CATALOG}.{SCHEMA}.object_catalog_redaction AS tgt\n", - "USING (\n", - " SELECT * FROM (\n", - " SELECT *, ROW_NUMBER() OVER (PARTITION BY path ORDER BY path) AS rn\n", - " FROM tiff_results\n", - " ) WHERE rn = 1\n", - ") AS src\n", - " ON tgt.path = src.path\n", - "WHEN MATCHED THEN UPDATE SET\n", - " tgt.output_file_paths = array(src.tiff_output_path),\n", - " tgt.label_image_path = COALESCE(src.label_image_path, tgt.label_image_path),\n", - " tgt.macro_image_path = COALESCE(src.macro_image_path, tgt.macro_image_path),\n", - " tgt.phi_tags_redacted = src.phi_tags_redacted,\n", - " tgt.pixel_redactions_count = src.pixel_regions_redacted,\n", - " tgt.status = CASE WHEN src.error IS NULL THEN 'SUCCESS' ELSE 'FAILED' END,\n", - " tgt.error_messages = CASE WHEN src.error IS NOT NULL THEN array(src.error) ELSE NULL END,\n", - " tgt.update_timestamp = current_timestamp()\n", - "\"\"\")\n", - "print(\"TIFF write complete. Redaction records updated.\")" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "application/vnd.databricks.v1+cell": { - "cellMetadata": { - "byteLimit": 2048000, - "rowLimit": 10000 - }, - "inputWidgets": {}, - "nuid": "4ec2aac1-43ab-42ee-aae7-0fdf52c006ec", - "showTitle": true, - "tableResultSettingsMap": {}, - "title": "Cell 11: Audit summary" - } - }, - "outputs": [], - "source": [ - "# End-to-end audit: join object_catalog with object_catalog_redaction and summarise.\n", - "audit_df = spark.sql(f\"\"\"\n", - "SELECT\n", - " regexp_extract(o.path, '[^/]+$', 0) AS filename,\n", - " o.meta:width::int AS width_px,\n", - " o.meta:height::int AS height_px,\n", - " o.meta:level_count::int AS pyramid_levels,\n", - " r.has_phi,\n", - " r.status,\n", - " r.pixel_redactions_count,\n", - " size(r.phi_tags_redacted) AS tag_redactions,\n", - " r.output_file_paths[0] AS tiff_output_path,\n", - " r.label_image_path,\n", - " r.error_messages[0] AS error\n", - "FROM {CATALOG}.{SCHEMA}.object_catalog o\n", - "LEFT JOIN {CATALOG}.{SCHEMA}.object_catalog_redaction r\n", - " ON o.path = r.path\n", - "ORDER BY filename\n", - "\"\"\")\n", - "\n", - "total = audit_df.count()\n", - "phi_ct = audit_df.filter(\"has_phi = true\").count()\n", - "ok_ct = audit_df.filter(\"status = 'SUCCESS'\").count()\n", - "err_ct = audit_df.filter(\"status = 'FAILED'\").count()\n", - "\n", - "print(f\"Slides in catalog : {total}\")\n", - "print(f\"VLM-flagged with PHI : {phi_ct}\")\n", - "print(f\"Successfully written : {ok_ct}\")\n", - "print(f\"Errors : {err_ct}\")\n", - "\n", - "display(audit_df)\n" - ] - } - ], - "metadata": { - "application/vnd.databricks.v1+notebook": { - "computePreferences": null, - "dashboards": [], - "environmentMetadata": { - "base_environment": "", - "environment_version": "5" - }, - "inputWidgetPreferences": null, - "language": "python", - "notebookMetadata": { - "mostRecentlyExecutedCommandWithImplicitDF": { - "commandId": 8994411946469750, - "dataframes": [ - "_sqldf" - ] - }, - "pythonIndentUnit": 2 - }, - "notebookName": "SVS Pathology De-identification Pipeline", - "widgets": {} - }, - "language_info": { - "name": "python" - } - }, - "nbformat": 4, - "nbformat_minor": 0 -} diff --git a/notebooks/tiff/01-ingest-tiff.ipynb b/notebooks/tiff/01-ingest-tiff.ipynb deleted file mode 100644 index 778c3978..00000000 --- a/notebooks/tiff/01-ingest-tiff.ipynb +++ /dev/null @@ -1,471 +0,0 @@ -{ - "cells": [ - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "application/vnd.databricks.v1+cell": { - "cellMetadata": { - "byteLimit": 2048000, - "rowLimit": 10000 - }, - "inputWidgets": {}, - "nuid": "1783cadb-09c4-408b-934e-550998ae9631", - "showTitle": false, - "tableResultSettingsMap": {}, - "title": "" - } - }, - "outputs": [], - "source": [ - "%load_ext autoreload\n", - "%autoreload 2\n", - "# Enables autoreload; learn more at https://docs.databricks.com/en/files/workspace-modules.html#autoreload-for-python-modules\n", - "# To disable autoreload; run %autoreload 0" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "application/vnd.databricks.v1+cell": { - "cellMetadata": { - "byteLimit": 2048000, - "rowLimit": 10000 - }, - "inputWidgets": {}, - "nuid": "bf3db99b-7cb3-4c65-a428-96b93a9a5510", - "showTitle": true, - "tableResultSettingsMap": {}, - "title": "TIFF Ingest — Overview" - } - }, - "source": [ - "# TIFF File Ingest\n", - "\n", - "Ingests TIFF pathology slides into a Unity Catalog Delta table using the `dbx.pixels.Catalog` class.\n", - "\n", - "**Pipeline**\n", - "1. Load config from `config.yaml` (source path, pattern, table)\n", - "2. Initialise `Catalog` with the target Delta table and UC volume\n", - "3. `catalog.catalog()` — recursively discovers all `.tiff` files and enriches each row with path metadata\n", - "4. `catalog.save()` — writes the file catalog to the Delta table\n", - "5. SQL verification of ingested rows" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "application/vnd.databricks.v1+cell": { - "cellMetadata": { - "byteLimit": 2048000, - "rowLimit": 10000 - }, - "inputWidgets": {}, - "nuid": "d8ef4c9d-f0b5-468b-be2b-28b875926528", - "showTitle": true, - "tableResultSettingsMap": {}, - "title": "Install tifffile" - } - }, - "outputs": [], - "source": [ - "%pip install tifffile imagecodecs -q" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "application/vnd.databricks.v1+cell": { - "cellMetadata": { - "byteLimit": 2048000, - "rowLimit": 10000 - }, - "inputWidgets": {}, - "nuid": "612e3201-024b-4574-b46e-c456d555345f", - "showTitle": true, - "tableResultSettingsMap": {}, - "title": "Install / resolve dbx.pixels library" - } - }, - "outputs": [], - "source": [ - "import sys, pathlib\n", - "\n", - "# Use the editable source tree — no wheel build needed during development\n", - "SRC_PATH = \"/Workspace/Users/douglas.moore@databricks.com/pixels-tiff/src\"\n", - "\n", - "if SRC_PATH not in sys.path:\n", - " sys.path.insert(0, SRC_PATH)\n", - " print(f\"Added to sys.path: {SRC_PATH}\")\n", - "else:\n", - " print(f\"Already on sys.path: {SRC_PATH}\")" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "application/vnd.databricks.v1+cell": { - "cellMetadata": { - "byteLimit": 2048000, - "rowLimit": 10000 - }, - "inputWidgets": {}, - "nuid": "34ed423b-9b76-4818-befb-f8b6b5cfe83d", - "showTitle": true, - "tableResultSettingsMap": {}, - "title": "Load config from config.yaml" - } - }, - "outputs": [], - "source": [ - "import yaml, pathlib\n", - "\n", - "CONFIG_PATH = pathlib.Path(\"/Workspace/Users/douglas.moore@databricks.com/pixels-tiff/notebooks/tiff/config.yaml\")\n", - "\n", - "with CONFIG_PATH.open() as fh:\n", - " config = yaml.safe_load(fh)\n", - "\n", - "SOURCE_PATH = config[\"SOURCE_PATH\"]\n", - "PATTERN = config[\"PATTERN\"]\n", - "TABLE = config[\"INDEX\"]\n", - "\n", - "# Derive volume from catalog + schema of the index table (dmoore.tiff.)\n", - "_catalog, _schema, _ = TABLE.split(\".\")\n", - "VOLUME = f\"{_catalog}.{_schema}.tiff_volume\" # adjust if your volume name differs\n", - "\n", - "WRITE_MODE = \"overwrite\" # use 'append' for incremental runs\n", - "\n", - "print(f\"SOURCE_PATH : {SOURCE_PATH}\")\n", - "print(f\"PATTERN : {PATTERN}\")\n", - "print(f\"TABLE : {TABLE}\")\n", - "print(f\"VOLUME : {VOLUME}\")\n", - "print(f\"WRITE_MODE : {WRITE_MODE}\")" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "application/vnd.databricks.v1+cell": { - "cellMetadata": { - "byteLimit": 2048000, - "rowLimit": 10000 - }, - "inputWidgets": {}, - "nuid": "f636624f-94e6-4573-8863-4edfaed3ad86", - "showTitle": true, - "tableResultSettingsMap": {}, - "title": "Initialise Catalog" - } - }, - "outputs": [], - "source": [ - "from dbx.pixels import Catalog\n", - "\n", - "catalog = Catalog(spark, table=TABLE)\n", - "print(catalog)" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "application/vnd.databricks.v1+cell": { - "cellMetadata": { - "byteLimit": 2048000, - "rowLimit": 10000 - }, - "inputWidgets": {}, - "nuid": "aa563b95-dad5-4694-9c98-4b8f13c1607a", - "showTitle": true, - "tableResultSettingsMap": {}, - "title": "Catalog TIFF files" - } - }, - "outputs": [], - "source": [ - "# Recursively discover all TIFF files under SOURCE_PATH.\n", - "# .catalog() reads only file metadata (no pixel data is loaded).\n", - "catalog_df = catalog.catalog(\n", - " path=SOURCE_PATH,\n", - " pattern=PATTERN,\n", - " recurse=True,\n", - " streaming=False,\n", - ").repartition(4)\n", - "\n", - "\n", - "print(f\"Files found: {catalog_df.count()}\")\n", - "display(catalog_df)" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "application/vnd.databricks.v1+cell": { - "cellMetadata": { - "byteLimit": 2048000, - "rowLimit": 10000 - }, - "inputWidgets": {}, - "nuid": "aeba92f7-1d20-4767-a7d8-f000f744cf7d", - "showTitle": true, - "tableResultSettingsMap": {}, - "title": "Extract TIFF metadata" - } - }, - "outputs": [], - "source": [ - "from dbx.pixels.tiff import TiffMetaExtractor\n", - "\n", - "# Enrich the file catalog DataFrame with TIFF metadata.\n", - "# TiffMetaExtractor adds a `meta` VARIANT column containing all TIFF tags\n", - "# plus derived fields (page_count, is_ome, is_bigtiff, series info, phi_tag_report).\n", - "extractor = TiffMetaExtractor(catalog, inputCol=\"local_path\", outputCol=\"meta\")\n", - "enriched_df = extractor.transform(catalog_df)\n", - "\n", - "print(f\"Schema: {[f.name for f in enriched_df.schema.fields]}\")\n", - "display(enriched_df.select(\"local_path\", \"extension\", \"meta\"))" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "application/vnd.databricks.v1+cell": { - "cellMetadata": { - "byteLimit": 2048000, - "rowLimit": 10000 - }, - "inputWidgets": {}, - "nuid": "5acdc926-76ba-47c1-9108-8c4c0d483b3e", - "showTitle": true, - "tableResultSettingsMap": {}, - "title": "Verify TiffVLMPhiDetector import (no pydicom)" - } - }, - "outputs": [], - "source": [ - "import sys\n", - "for _m in [m for m in list(sys.modules) if m.startswith(\"dbx.pixels.tiff\")]:\n", - " sys.modules.pop(_m, None)\n", - "\n", - "from dbx.pixels.tiff import TiffVLMPhiDetector, TiffMetaExtractor\n", - "from dbx.pixels.tiff.tiff_vlm_phi_detector import VlmResult\n", - "\n", - "print(\"✓ TiffVLMPhiDetector imported — no pydicom dependency\")\n", - "print(f\" TiffVLMPhiDetector : {TiffVLMPhiDetector}\")\n", - "print(f\" VlmResult : {VlmResult}\")\n", - "\n", - "# Confirm pydicom is NOT on the import chain\n", - "import importlib, sys as _sys\n", - "assert \"pydicom\" not in _sys.modules, \"pydicom was unexpectedly imported\"\n", - "print(\"✓ pydicom not in sys.modules\")" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "application/vnd.databricks.v1+cell": { - "cellMetadata": { - "byteLimit": 2048000, - "rowLimit": 10000 - }, - "inputWidgets": {}, - "nuid": "09779984-1374-4bfe-86d6-b79aecdd1e8e", - "showTitle": true, - "tableResultSettingsMap": {}, - "title": "Diagnose tiff_to_image failure" - } - }, - "outputs": [], - "source": [ - "from dbx.pixels.tiff.tiff_utils import _tiff_to_array_tifffile, _tiff_to_array_pillow\n", - "\n", - "PROBE_PATH = \"/Volumes/hls_radiology_east/osuwmc/sample/Philips07_3b946eed-4d57-4d1a-9f27-32c559ecd07a_BIG.tiff\"\n", - "\n", - "print(\"--- tifffile ---\")\n", - "try:\n", - " arr = _tiff_to_array_tifffile(PROBE_PATH)\n", - " print(f\"OK shape={arr.shape} dtype={arr.dtype}\" if arr is not None else \"returned None\")\n", - "except Exception as e:\n", - " print(f\"ERROR: {type(e).__name__}: {e}\")\n", - "\n", - "print(\"--- Pillow ---\")\n", - "try:\n", - " arr = _tiff_to_array_pillow(PROBE_PATH)\n", - " print(f\"OK shape={arr.shape} dtype={arr.dtype}\" if arr is not None else \"returned None\")\n", - "except Exception as e:\n", - " print(f\"ERROR: {type(e).__name__}: {e}\")" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "application/vnd.databricks.v1+cell": { - "cellMetadata": { - "byteLimit": 2048000, - "rowLimit": 10000 - }, - "inputWidgets": {}, - "nuid": "f791598b-4aa2-4eda-80c9-c222ceddcc58", - "showTitle": true, - "tableResultSettingsMap": {}, - "title": "Detect PHI in TIFF images" - } - }, - "outputs": [], - "source": [ - "import sys\n", - "# Clear any stale module-cache entries so updated source files are picked up\n", - "for _m in [m for m in list(sys.modules) if m.startswith(\"dbx.pixels.tiff\")]:\n", - " sys.modules.pop(_m, None)\n", - "\n", - "from dbx.pixels.tiff import TiffVLMPhiDetector\n", - "\n", - "VLM_ENDPOINT = config.get(\"VLM_ENDPOINT\", \"\")\n", - "\n", - "detector = TiffVLMPhiDetector(\n", - " endpoint = VLM_ENDPOINT,\n", - " inputCol = \"local_path\",\n", - " outputCol = \"response\",\n", - " input_type = \"tiff\",\n", - " max_width = 768,\n", - ")\n", - "\n", - "phi_df = detector.transform(enriched_df)\n", - "\n", - "# Persist PHI assessment results\n", - "(\n", - " phi_df\n", - " .select(\"path\", \"local_path\", \"extension\", \"response\")\n", - " .write.format(\"delta\")\n", - " .mode(\"overwrite\")\n", - " .option(\"overwriteSchema\", \"true\")\n", - " .saveAsTable(config[\"PHI_ASSESSMENT_TABLE\"])\n", - ")\n", - "\n", - "display(phi_df.select(\"local_path\", \"response\"))" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "application/vnd.databricks.v1+cell": { - "cellMetadata": { - "byteLimit": 2048000, - "rowLimit": 10000 - }, - "inputWidgets": {}, - "nuid": "ab9f4a04-023e-4dc3-a4a8-cd083e59f793", - "showTitle": true, - "tableResultSettingsMap": {}, - "title": "Save file catalog to Delta table" - } - }, - "outputs": [], - "source": [ - "# Persist the file catalog to the Delta table defined in config.yaml (INDEX).\n", - "# Use write_mode='overwrite' for a full refresh, or 'append' for incremental.\n", - "catalog.save(phi_df, mode=WRITE_MODE)\n", - "\n", - "print(f\"Saved to: {TABLE} (mode={WRITE_MODE})\")" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "application/vnd.databricks.v1+cell": { - "cellMetadata": { - "byteLimit": 2048000, - "implicitDf": true, - "rowLimit": 10000 - }, - "inputWidgets": {}, - "nuid": "548d21b6-778e-4525-b7f0-b9fe09b8ca05", - "showTitle": true, - "tableResultSettingsMap": {}, - "title": "Verify ingested rows" - } - }, - "outputs": [], - "source": [ - "%sql\n", - "-- Quick verification: row count and sample paths\n", - "SELECT\n", - " COUNT(*) AS file_count,\n", - " SUM(length) / 1024 / 1024 AS total_size_mb,\n", - " COLLECT_SET(extension) AS extensions\n", - "FROM dmoore.tiff.object_catalog" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "application/vnd.databricks.v1+cell": { - "cellMetadata": { - "byteLimit": 2048000, - "implicitDf": true, - "rowLimit": 10000 - }, - "inputWidgets": {}, - "nuid": "a9616b94-823e-434b-8a72-52f0cd4adaef", - "showTitle": true, - "tableResultSettingsMap": {}, - "title": "Browse ingested catalog" - } - }, - "outputs": [], - "source": [ - "%sql\n", - "SELECT\n", - " path,\n", - " length,\n", - " modificationTime,\n", - " extension,\n", - " path_tags,\n", - " file_type,\n", - " meta\n", - "FROM dmoore.tiff.object_catalog\n", - "ORDER BY modificationTime DESC\n", - "LIMIT 50" - ] - } - ], - "metadata": { - "application/vnd.databricks.v1+notebook": { - "computePreferences": null, - "dashboards": [], - "environmentMetadata": { - "base_environment": "", - "environment_version": "5" - }, - "inputWidgetPreferences": null, - "language": "python", - "notebookMetadata": { - "mostRecentlyExecutedCommandWithImplicitDF": { - "commandId": 8636978234437395, - "dataframes": [ - "_sqldf" - ] - }, - "pythonIndentUnit": 4 - }, - "notebookName": "01-ingest-tiff", - "widgets": {} - }, - "language_info": { - "name": "python" - } - }, - "nbformat": 4, - "nbformat_minor": 0 -} diff --git a/notebooks/tiff/TIFF Pathology De-identification Pipeline.ipynb b/notebooks/tiff/TIFF Pathology De-identification Pipeline.ipynb deleted file mode 100644 index e9792a20..00000000 --- a/notebooks/tiff/TIFF Pathology De-identification Pipeline.ipynb +++ /dev/null @@ -1,1753 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": { - "application/vnd.databricks.v1+cell": { - "cellMetadata": {}, - "inputWidgets": {}, - "nuid": "a9402935-1770-4aa2-bec0-71e265bc53c1", - "showTitle": false, - "tableResultSettingsMap": {}, - "title": "" - } - }, - "source": [ - "\n", - "# Plan: TIFF Pathology De-identification Pipeline\n", - "## Architecture Plan — TIFF→ De-identified TIFF\n", - "\n", - "Extends [databricks-industry-solutions/pixels](https://github.com/databricks-industry-solutions/pixels) to treat `.tiff` as a first-class format alongside DICOM.\n", - "\n", - "---" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "application/vnd.databricks.v1+cell": { - "cellMetadata": {}, - "inputWidgets": {}, - "nuid": "b27e18db-0abf-446f-ba89-65d6906d1506", - "showTitle": false, - "tableResultSettingsMap": {}, - "title": "" - } - }, - "source": [ - "## 1. Confirmed Inputs & Outputs\n", - "\n", - "| Item | Value |\n", - "|---|---|\n", - "| Input SVS path | `/Volumes/hls_pathology/orthanc_demo/raw_images/Aperio/` |\n", - "| Demo scale | ~14 files; architecture targets **10 million** |\n", - "| Output catalog / schema | `douglas_moore.pathology` (to be created) |\n", - "| TIFF output volume | `/Volumes/douglas_moore/pathology/tiff_deidentified/` |\n", - "| Label images volume | `/Volumes/douglas_moore/pathology/label_images/` |\n", - "| VLM endpoint | `databricks-llama-4-maverick` (config param) |\n", - "| Redaction method | Black rectangle fill |\n", - "| Source SVS | **Read-only** — never modified |\n", - "\n", - "---" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "application/vnd.databricks.v1+cell": { - "cellMetadata": {}, - "inputWidgets": {}, - "nuid": "32ddb278-16b7-4719-a134-55360ad4bb26", - "showTitle": false, - "tableResultSettingsMap": {}, - "title": "" - } - }, - "source": [ - "## 2. Delta Table Schema\n", - "\n", - "> **No new DDL is needed for SVS ingest.** The `object_catalog` table is used exactly as defined in the base `CREATE_OBJECT_CATALOG.sql` — no columns are added or altered. All SVS-specific metadata (dimensions, pyramid levels, sub-image presence, PHI tag classification) is serialised into the existing `meta VARIANT` column and accessed via VARIANT path syntax. `SVSCatalog.init_tables()` calls `super().init_tables()` which runs the unmodified base DDL against the `douglas_moore.pathology` schema. The only new DDL is the `_redaction` table.\n", - "\n", - "### `douglas_moore.pathology.object_catalog` *(base DDL, unchanged)*\n", - "One row per SVS file. Populated by `SVSCatalog.catalog()` + `SVSMetaExtractor`.\n", - "\n", - "| Column | Type | Notes |\n", - "|---|---|---|\n", - "| `path` | STRING NOT NULL | Cloud storage path |\n", - "| `modificationTime` | TIMESTAMP NOT NULL | |\n", - "| `length` | BIGINT NOT NULL | File size bytes |\n", - "| `original_path` | STRING | |\n", - "| `relative_path` | STRING | |\n", - "| `local_path` | STRING NOT NULL | Worker-accessible path — **`inputCol` for all Transformers** |\n", - "| `extension` | STRING | `\"svs\"` |\n", - "| `file_type` | STRING | |\n", - "| `path_tags` | ARRAY\\ | From `TagExtractor` |\n", - "| `is_anon` | BOOLEAN | |\n", - "| `meta` | **VARIANT** | All OpenSlide properties + SVS-specific fields serialised together. Query with `meta:aperio.Date::string`, `meta:width::int`, `meta:has_label_image::boolean` |\n", - "\n", - "**SVS fields stored inside `meta VARIANT`** (no schema change required):\n", - "- `meta:width::int`, `meta:height::int` — level-0 pixel dimensions\n", - "- `meta:level_count::int` — pyramid depth\n", - "- `meta:has_label_image::boolean`, `meta:has_macro_image::boolean`\n", - "- `meta:phi_tag_report` — array of `{tag, value, classification}` structs\n", - "- All raw OpenSlide properties (e.g. `meta:\"aperio.AppMag\"::string`)\n", - "\n", - "### `douglas_moore.pathology.object_catalog_redaction` *(unified DICOM + SVS DDL)*\n", - "One row per redaction job, for any format. Created by `CREATE_SVS_CATALOG.sql`.\n", - "\n", - "Three DICOM columns are renamed to remove format-specific semantics; new columns cover VLM detection results and SVS artefacts. All new and renamed columns are nullable for backward compatibility.\n", - "\n", - "**Format-agnostic identifiers**\n", - "\n", - "| Column | Type | Notes |\n", - "|---|---|---|\n", - "| `redaction_id` | STRING NOT NULL | UUID per job |\n", - "| `path` | STRING | FK → `object_catalog.path` *(new — not in DICOM original)* |\n", - "| `extension` | STRING | Discriminator: `dcm`, `svs`, `czi` … *(new)* |\n", - "\n", - "**DICOM identifiers** *(NULL for SVS)*\n", - "\n", - "| Column | Type | Notes |\n", - "|---|---|---|\n", - "| `study_instance_uid` | STRING | DICOM Study UID |\n", - "| `series_instance_uid` | STRING | DICOM Series UID |\n", - "| `modality` | STRING | DICOM modality, or `WSI` for SVS |\n", - "| `new_series_instance_uid` | STRING | New UID for redacted DICOM series |\n", - "\n", - "**Redaction configuration** *(both formats)*\n", - "\n", - "| Column | Type | Change from DICOM original |\n", - "|---|---|---|\n", - "| `redaction_config` | VARIANT | **Renamed** from `redaction_json` |\n", - "| `metadata_redactions_count` | INT | **Renamed** from `global_redactions_count` |\n", - "| `pixel_redactions_count` | INT | **Renamed** from `frame_specific_redactions_count` |\n", - "| `total_redaction_areas` | INT | Unchanged |\n", - "| `phi_tags_redacted` | ARRAY\\ | Tag names scrubbed *(new)* |\n", - "\n", - "**VLM PHI detection results** *(new — both formats)*\n", - "\n", - "| Column | Type | Notes |\n", - "|---|---|---|\n", - "| `has_phi` | BOOLEAN | VLM verdict |\n", - "| `phi_elements` | VARIANT | Detected regions: type, value\\_hint, bbox |\n", - "| `vlm_raw_response` | STRING | Raw model output |\n", - "| `model_endpoint` | STRING | Endpoint name |\n", - "\n", - "**Output paths**\n", - "\n", - "| Column | Type | Notes |\n", - "|---|---|---|\n", - "| `output_file_paths` | ARRAY\\ | DICOM: one `.dcm` per slice. SVS: single TIFF at index 0 |\n", - "| `label_image_path` | STRING | De-identified label PNG *(SVS only, NULL for DICOM)* |\n", - "| `macro_image_path` | STRING | De-identified macro PNG *(SVS only, NULL for DICOM)* |\n", - "\n", - "**Processing status & audit** *(unchanged from DICOM original)*\n", - "`status`, `error_messages`, `insert_timestamp`, `update_timestamp`, `processing_start_timestamp`, `processing_end_timestamp`, `processing_duration_seconds`, `created_by`, `export_timestamp`\n", - "\n", - "---" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "application/vnd.databricks.v1+cell": { - "cellMetadata": {}, - "inputWidgets": {}, - "nuid": "207f4f4e-a49b-456e-b5d1-1165b1b1f8a9", - "showTitle": true, - "tableResultSettingsMap": {}, - "title": "Architecture Plan: SVS Pathology De-identification Pipeline" - } - }, - "source": [ - "\n", - "## 3. Python Package: `dbx.pixels.svs`\n", - "\n", - "> **Pattern source: actual repo code** — All transformers extend `pyspark.ml.pipeline.Transformer` (Spark ML, not a custom base). The main entry-point for file cataloguing is the `Catalog` class, not a `Processor`. There is no `Processor` in the repo. The CZI extractor (`src/dbx/pixels/czi/`) is a stub — SVS is genuinely the first completed non-DICOM format extension.\n", - "\n", - "Namespace-package extension of `dbx-pixels`. Created as workspace files under `svs-pixels/src/`, installed via `%pip install -e ./src`.\n", - "\n", - "```\n", - "svs-pixels/\n", - "├── src/\n", - "│ └── dbx/\n", - "│ └── pixels/\n", - "│ └── svs/\n", - "│ ├── __init__.py ← exports SVSCatalog, SVSMetaExtractor, SVSTiffWriter, SVSPhiPipeline\n", - "│ ├── catalog.py ← SVSCatalog(Catalog)\n", - "│ ├── svs_meta_extractor.py← SVSMetaExtractor(Transformer)\n", - "│ ├── svs_tiff_writer.py ← SVSTiffWriter(Transformer)\n", - "│ ├── phi_tags.py ← PHI classification lookup dict\n", - "│ └── deidentify.py ← pixel redaction helpers\n", - "│ └── resources/sql/\n", - "│ └── CREATE_SVS_CATALOG.sql ← creates object_catalog_redaction only\n", - "├── pyproject.toml\n", - "└── (this notebook)\n", - "```\n", - "\n", - "### `SVSCatalog` (extends `Catalog`)\n", - "- Calls `super().__init__(spark, table, volume)` — reuses all existing table management, volume, and Auto Loader infrastructure\n", - "- `catalog(path, pattern=\"*.svs\", ...)` → delegates to `Catalog.catalog()` with SVS glob pattern; callers never need to pass `pattern`\n", - "- `init_tables()` → calls `super().init_tables()` (creates `object_catalog` via unmodified base DDL), then executes one SVS-specific file — `resources/sql/CREATE_SVS_CATALOG.sql` — which creates only the `object_catalog_redaction` table with SVS-specific columns\n", - "\n", - "### `SVSMetaExtractor` (extends `pyspark.ml.pipeline.Transformer`)\n", - "Mirrors `DicomMetaExtractor`: uses `mapInPandas` with `ThreadPoolExecutor` for concurrent I/O (optimal for network-bound OpenSlide reads).\n", - "\n", - "```python\n", - "class SVSMetaExtractor(Transformer):\n", - " def __init__(self, catalog, inputCol=\"local_path\", outputCol=\"meta\",\n", - " maxWorkers=32, useVariant=True): ...\n", - "\n", - " def _transform(self, df): # Spark ML Transformer contract\n", - " # mapInPandas with ThreadPoolExecutor — same pattern as DicomMetaExtractor\n", - " ...\n", - "```\n", - "\n", - "**Single output column written to `object_catalog`:**\n", - "\n", - "| Column | Spark type | Notes |\n", - "|---|---|---|\n", - "| `meta` | `VARIANT` | OpenSlide properties dict merged with derived fields (`width`, `height`, `level_count`, `has_label_image`, `has_macro_image`, `phi_tag_report`) into one JSON object, then `parse_json()`'d into VARIANT |\n", - "\n", - "All SVS-specific fields are embedded inside `meta` before serialisation — no extra top-level columns are written, no `ALTER TABLE` or `mergeSchema` required. VARIANT path syntax handles all downstream access: `meta:width::int`, `meta:phi_tag_report[0].classification::string`, etc.\n", - "\n", - "### `SVSTiffWriter` (extends `pyspark.ml.pipeline.Transformer`)\n", - "Converts SVS → de-identified pyramidal BigTIFF. Wraps the write logic in `_transform(df)` operating on the output of `SVSMetaExtractor`.\n", - "\n", - "### `SVSPhiPipeline` (extends `pyspark.ml.Pipeline`)\n", - "Composed pipeline, mirrors `DicomPhiPipeline`:\n", - "```\n", - "Stage 1: SVSMetaExtractor → adds meta VARIANT + phi_tag_report\n", - "Stage 2: SVSVlmPhiDetector → adds phi_elements (VLM bboxes on label/macro)\n", - "Stage 3: SVSFilterTransformer → nullifies rows with no PHI detected\n", - "Stage 4: SVSTiffWriter → writes de-identified BigTIFF + audit log\n", - "```\n", - "\n", - "---\n", - "\n", - "\n" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "application/vnd.databricks.v1+cell": { - "cellMetadata": {}, - "inputWidgets": {}, - "nuid": "7e9389da-8946-4087-a9ac-3fe10773c829", - "showTitle": false, - "tableResultSettingsMap": {}, - "title": "" - } - }, - "source": [ - "\n", - "## 4. PHI Tag Classification (`phi_tags.py`)\n", - "\n", - "Research-hardcoded lookup covering all standard OpenSlide / Aperio TIFF properties:\n", - "\n", - "| Classification | Example Tags |\n", - "|---|---|\n", - "| `PHI` | `aperio.Patient`, `aperio.PatientID`, `aperio.DOB`, `aperio.MRN`, `aperio.AccessionNumber`, `aperio.ClinicID`, `aperio.ClinicalTrialID`, `aperio.Procedure`, `tiff.ImageDescription` (contains patient name in Aperio format) |\n", - "| `QUESTIONABLE` | `aperio.Date`, `aperio.Time`, `aperio.Clinic`, `aperio.Pathologist`, `tiff.Artist`, `tiff.Copyright`, `aperio.Title`, `aperio.Filename`, `aperio.User`, `aperio.ImageID` |\n", - "| `NOT_PHI` | `aperio.AppMag`, `aperio.MPP`, `aperio.ScanScope ID`, `openslide.level-count`, `openslide.mpp-x`, `openslide.mpp-y`, `openslide.objective-power`, `tiff.Make`, `tiff.Model`, `tiff.Software`, `openslide.vendor`, all `openslide.level[N].*` pyramid geometry tags |\n", - "\n", - "Function `classify_tags(properties: dict) → list[dict]` iterates all OpenSlide properties and returns the structured report stored in `phi_tag_report`.\n", - "\n", - "---" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "application/vnd.databricks.v1+cell": { - "cellMetadata": {}, - "inputWidgets": {}, - "nuid": "3f54bca4-4711-4cc7-8995-ce1bb3bfda99", - "showTitle": false, - "tableResultSettingsMap": {}, - "title": "" - } - }, - "source": [ - "## 5. VLM PHI Detection via `ai_query()`\n", - "\n", - "### What is Inspected\n", - "Aperio SVS files embed three sub-images accessible via `slide.associated_images` — **confirmed from real files**:\n", - "\n", - "| Sub-image | Dims (CMU-1) | Mode | PHI Risk | Action |\n", - "|---|---|---|---|---|\n", - "| `label` | 387×463 | RGBA | **HIGH** — physical paper label with patient name, barcode, accession | VLM analysis + black-box redaction |\n", - "| `macro` | 1280×431 | RGBA | **MEDIUM** — full-slide photo; label region visible at right edge | VLM analysis + label region redaction |\n", - "| `thumbnail` | 1024×732 | RGBA | LOW — auto-generated tissue preview | Excluded from output |\n", - "\n", - "The tissue scan (`level 0`: 46000×32914) is in a **completely separate coordinate space** from the label/macro sub-images. PHI in the tissue scan itself is rare but possible (e.g., handwriting on the glass).\n", - "\n", - "The label image is the **primary** VLM target. Macro is secondary.\n", - "\n", - "### Pipeline\n", - "1. `SVSTransformer.extract_embedded_images()` saves label and macro PNGs to `/Volumes/douglas_moore/pathology/label_images/` using the naming convention `{slide_name}_label.png` / `{slide_name}_macro.png`\n", - "2. Run `ai_query()` directly via `READ_FILES()` on the volume — **no binary column staging needed**:\n", - "\n", - "```sql\n", - "INSERT INTO douglas_moore.pathology.phi_pixel_audit\n", - "SELECT\n", - " m.path,\n", - " f._metadata.file_path AS label_image_path,\n", - " ai_query(\n", - " 'databricks-llama-4-maverick',\n", - " 'You are a medical PHI detection system analyzing a pathology slide label.\n", - " Return ONLY valid JSON:\n", - " {\"has_phi\": bool,\n", - " \"phi_elements\": [{\"type\": \"name|dob|mrn|accession|barcode|other\",\n", - " \"value_hint\": \"first 3 chars only\",\n", - " \"bbox\": {\"x\": int, \"y\": int, \"w\": int, \"h\": int}}]}\n", - " Bounding box coordinates are in the label image pixel space (origin top-left).',\n", - " files => f.content\n", - " ) AS vlm_raw_response,\n", - " 'databricks-llama-4-maverick' AS model_endpoint,\n", - " current_timestamp() AS inferred_at\n", - "FROM read_files(\n", - " '/Volumes/douglas_moore/pathology/label_images/',\n", - " format => 'binaryFile',\n", - " fileNamePattern => '*_label.png'\n", - ") f\n", - "JOIN douglas_moore.pathology.svs_metadata m\n", - " ON m.filename = regexp_replace(f._metadata.file_name, '_label\\.png\n", - "```\n", - "---\n" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "application/vnd.databricks.v1+cell": { - "cellMetadata": {}, - "inputWidgets": {}, - "nuid": "b3f28cd8-2465-4bfe-b153-664e929fe501", - "showTitle": false, - "tableResultSettingsMap": {}, - "title": "" - } - }, - "source": [ - "\n", - "## 6. De-identification & TIFF Output (`deidentify.py` / `SVSTiffWriter`)\n", - "\n", - "> **Pattern source: actual repo code** — `DicomPhiPipeline` uses a two-stage approach: (1) `VLMPhiDetector` returns a **pipe-separated list of PHI text strings** (`'John Smith'|'04-31-1954'`), NOT bboxes. (2) `OcrRedactor` then runs EasyOCR on the image to locate those strings and draw black rectangles. The VLM provides *what* is PHI; OCR provides *where*. SVS uses this same two-stage approach.\n", - "\n", - "### Revised De-identification Algorithm\n", - "\n", - "#### Stage 1 — VLM PHI Detection (`SVSVlmPhiDetector`, extends `Transformer`)\n", - "- Submit label/macro PNGs via `ai_query()` with `files => content`\n", - "- Prompt returns a **pipe-separated list of PHI entity strings** (consistent with the SA pattern)\n", - "- Optionally also request bboxes via `responseFormat => json_schema` (SVS-specific addition for direct redaction without a second OCR pass)\n", - "\n", - "#### Stage 2 — Pixel Redaction (`SVSTiffWriter._transform(df)`)\n", - "1. Open SVS with `openslide.OpenSlide(local_path)`\n", - "2. Read level-0 in 4096×4096 tiles using `slide.read_region()`\n", - "3. If bbox-only mode: draw filled black `PIL.ImageDraw.rectangle` over each detected region in label/macro\n", - "4. If text-only mode: run EasyOCR on label image to locate the strings from the VLM response, then black-out matching text (mirrors `OcrRedactor`)\n", - "5. Scrub PHI tags in `tiff.ImageDescription` using `phi_tags.scrub_image_description()`\n", - "6. Write pyramidal BigTIFF using `tifffile.TiffWriter(bigtiff=True)` with `subifds=level_count-1` and 256×256 JPEG tiles\n", - "7. Return `(tiff_output_path, phi_tags_redacted_list, pixel_regions_count)` for the audit row\n", - "\n", - "### VLM Implementation: Two Approaches\n", - "\n", - "| Approach | Used by | Library | Scale |\n", - "|---|---|---|---|\n", - "| OpenAI SDK + base64 | `VLMPhiExtractor` in pixels SA | `openai` Python SDK, `pandas_udf` | Single-node / moderate |\n", - "| `ai_query()` + `files => content` | **Our SVS pipeline** | Databricks SQL / Spark SQL | 10M images, serverless SQL |\n", - "\n", - "For the demo scale, both work. For 10M, `ai_query()` via SQL is the correct choice — it delegates throughput management to the Databricks SQL engine.\n" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "application/vnd.databricks.v1+cell": { - "cellMetadata": {}, - "inputWidgets": {}, - "nuid": "c5c5b775-4d44-415f-92f5-dd65275fa0bf", - "showTitle": false, - "tableResultSettingsMap": {}, - "title": "" - } - }, - "source": [ - "\n", - "## 7. Notebook Cell Structure\n", - "\n", - "| Cell | Purpose |\n", - "|---|---|\n", - "| **Cell 1** | This plan (markdown) |\n", - "| **Cell 2** | `%pip install dbx-pixels openslide-python openslide-bin tifffile Pillow easyocr` + `%pip install -e ./src` |\n", - "| **Cell 3** | Configuration: paths, catalog, schema, volume names, model endpoint |\n", - "| **Cell 4** | Storage bootstrap: `SVSCatalog(spark, ...).init_tables()` — `super().init_tables()` creates `object_catalog` (base DDL, unchanged); `CREATE_SVS_CATALOG.sql` creates `object_catalog_redaction` (SVS-specific columns only) |\n", - "| **Cell 5** | File discovery: `SVSCatalog.catalog(INPUT_PATH)` → writes `object_catalog` (pattern defaults to `\"*.svs\"`) |\n", - "| **Cell 6** | Metadata extraction: `SVSMetaExtractor(catalog)._transform(files_df)` → populates `meta VARIANT` (OpenSlide properties + derived SVS fields merged into one JSON object) |\n", - "| **Cell 7** | PHI tag report: SQL on `object_catalog` using VARIANT path syntax (`meta:aperio.Date::string`, `meta:phi_tag_report`) |\n", - "| **Cell 8** | Label/macro image extraction → `/Volumes/.../label_images/` |\n", - "| **Cell 9** | VLM inference: `ai_query()` SQL → `object_catalog_redaction` |\n", - "| **Cell 10** | De-identified TIFF write: `SVSTiffWriter._transform(df)` → BigTIFFs to volume |\n", - "| **Cell 11** | Audit summary: join `object_catalog` + `object_catalog_redaction`, show statistics |\n", - "\n", - "---" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "application/vnd.databricks.v1+cell": { - "cellMetadata": {}, - "inputWidgets": {}, - "nuid": "1e12f30f-e1ae-49f9-a136-8dd94032e7aa", - "showTitle": false, - "tableResultSettingsMap": {}, - "title": "" - } - }, - "source": [ - "\n", - "## 8. Scale Architecture Notes (10M Images)\n", - "\n", - "| Concern | Demo Approach | 10M Approach |\n", - "|---|---|---|\n", - "| File discovery | `dbutils.fs.ls` recursive | Auto Loader on the volume path |\n", - "| Metadata extraction | `SVSMetaExtractor` via `mapInPandas` + `ThreadPoolExecutor` | Same — already distributed |\n", - "| Label image storage | Written to volume as files | Stored as `BINARY` in Delta table (eliminates extra volume I/O) |\n", - "| VLM inference | Single SQL batch `ai_query()` | Incremental: `WHERE vlm_status='PENDING'` in a scheduled Lakeflow Job |\n", - "| TIFF conversion | `write_deidentified_tiff_udf` Spark UDF | Same — Photon-accelerated UDF dispatch |\n", - "| Checkpointing | `vlm_status` column | Same + Delta transaction log for idempotency |\n", - "| Cost control | Serverless interactive | SQL Serverless warehouse + compute-optimized clusters for UDF stages |\n", - "\n", - "---" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "application/vnd.databricks.v1+cell": { - "cellMetadata": {}, - "inputWidgets": {}, - "nuid": "9ff39799-03ad-4f51-a8a8-3d0823d8b3e2", - "showTitle": false, - "tableResultSettingsMap": {}, - "title": "" - } - }, - "source": [ - "## 9. Confirmed Findings & Resolved Design Decisions\n", - "\n", - "All four open questions are now resolved from direct inspection of the actual Aperio CMU-1.svs files.\n", - "\n", - "### Q1 — Bounding box coordinate space ✅ RESOLVED\n", - "\n", - "The label sub-image is **387×463 RGBA** — completely independent from the tissue scan (46000×32914). The two coordinate spaces share no relationship.\n", - "\n", - "**Decision:** Save and submit the label image to the VLM at **native resolution (no resizing)**. All VLM bboxes are in label-image pixel space (`0,0` = top-left). The tissue TIFF does **not** embed the label — it is automatically excluded when only the main pyramid is read. The de-identified label PNG (black rectangles applied) is written to the `label_images` volume as the audit artefact.\n", - "\n", - "**Macro image clarification:** The macro (1280×431) shows both tissue and the physical label (at one end). It is submitted to the VLM separately; its bboxes drive black-rectangle redaction of the label area in the macro output PNG.\n", - "\n", - "---\n", - "\n", - "### Q2 — `tiff.ImageDescription` scrubbing ✅ RESOLVED\n", - "\n", - "Exact format confirmed from the real file:\n", - "```\n", - "Aperio Image Library v10.0.51\\r\\n46920x33014 [0,100 46000x32914] (256x256) JPEG/RGB Q=30\n", - " |AppMag = 20|StripeWidth = 2040|ScanScope ID = CPAPERIOCS|Filename = CMU-1\n", - " |Date = 12/29/09|Time = 09:59:15|User = b414003d-...|ImageID = 1004486|...\n", - "```\n", - "\n", - "**Structure:** `{header_line}|key = val|key = val|...`  Header is technical-only — preserve as-is.\n", - "\n", - "**PHI classification of actual keys:**\n", - "\n", - "| Key | Classification | Notes |\n", - "|---|---|---|\n", - "| `Date`, `Time` | PHI | HIPAA date/time of service |\n", - "| `User` | QUESTIONABLE | GUID in demo; operator name in clinical use |\n", - "| `Filename` | QUESTIONABLE | May encode patient name or MRN |\n", - "| `ImageID` | QUESTIONABLE | Could be accession number |\n", - "| `ScanScope ID`, `AppMag`, `StripeWidth`, `Parmset`, `MPP`, all geometry/calibration, `Filtered`, `ICC Profile` | NOT_PHI | Pure scanner parameters |\n", - "\n", - "Clinical files may also contain: `Patient`, `DOB`, `MRN`, `AccessionNumber`, `Clinic`, `Pathologist`, `Procedure`, `Diagnosis`, `Id` — all PHI.\n", - "\n", - "**Scrubbing algorithm:**\n", - "1. `header, *kvs = image_desc.split('|')`\n", - "2. For each `kv`: `k, v = kv.split(' = ', 1)` — rebuild as `k = REDACTED` if `k.strip()` ∈ PHI/QUESTIONABLE set\n", - "3. Rejoin: `'|'.join([header] + rebuilt_kvs)`\n", - "4. Apply identical scrub to `openslide.comment` (same content) when writing TIFF metadata\n", - "\n", - "---\n", - "\n", - "### Q3 — Macro image redaction ✅ RESOLVED\n", - "\n", - "Macro (1280×431) shows the full physical slide including the affixed label. **Decision:** Include macro in the primary VLM pipeline alongside the label (not a follow-on phase). Naming: `{name}_label.png` / `{name}_macro.png`. Both de-identified PNGs go to the `label_images` volume.\n", - "\n", - "---\n", - "\n", - "### Q4 — Pyramidal TIFF output ✅ RESOLVED\n", - "\n", - "Flat TIFF is not viable for pathology — QuPath, OMERO, and DIGIPATH all require pyramidal. The source SVS has 3 levels with 256×256 tiles; match this in output.\n", - "\n", - "**Decision:** Write pyramidal **BigTIFF** via `tifffile`:\n", - "- `bigtiff=True` — mandatory (CMU-1 level-0 ~7.4 GB uncompressed, exceeds 4 GB TIFF limit)\n", - "- `tile=(256, 256)` — matches native Aperio tile size\n", - "- `compression='jpeg'` at quality 80; swap to `'lzw'` if lossless required\n", - "- `subifds=level_count - 1` — sub-IFDs are the QuPath/libvips-compatible pyramid convention\n", - "- Pyramid levels: 2× progressive downsampling with `PIL.Image.LANCZOS`\n", - "\n", - "```python\n", - "with tifffile.TiffWriter(output_path, bigtiff=True) as tif:\n", - " opts = dict(tile=(256, 256), compression='jpeg',\n", - " compressionargs={'level': 80}, photometric='rgb', metadata=None)\n", - " tif.write(level_0_rgb, subifds=level_count - 1, **opts) # main IFD\n", - " for lvl in range(1, level_count):\n", - " tif.write(level_arrays[lvl], subfiletype=1, **opts) # sub-IFDs\n", - "```\n", - "\n", - "OME-TIFF (`ome=True`) only if OMERO is a confirmed downstream consumer.\n", - "\n", - "---" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "application/vnd.databricks.v1+cell": { - "cellMetadata": {}, - "inputWidgets": {}, - "nuid": "308dda6c-953c-44e0-b8a3-4e20dafb9357", - "showTitle": false, - "tableResultSettingsMap": {}, - "title": "" - } - }, - "source": [ - "\n", - "## 10. More...\n", - "\n", - "### Additional: `openslide-bin` Required\n", - "\n", - "`openslide-python` alone fails at import on Databricks Serverless:\n", - "```\n", - "ModuleNotFoundError: Couldn't locate OpenSlide shared library. Try pip install openslide-bin.\n", - "```\n", - "**Cell 2 must install:** `openslide-python openslide-bin` (the `openslide-bin` wheel bundles `libopenslide.so` for environments without system package access).\n", - "\n", - "\n", - "> **Note**: `files => content` is the correct `ai_query()` API for binary image inputs — it passes the PNG bytes directly to the model without base64 encoding. Only JPEG and PNG inputs are supported.\n", - "\n", - "### Scale to 10M Images\n", - "- `vlm_status` column acts as a watermark: `PENDING → PROCESSING → COMPLETE / FAILED`\n", - "- The SQL above runs as a Databricks SQL batch job — `ai_query()` parallelizes across serverless SQL clusters automatically\n", - "- For throughput control: partition the batch by date/rack and run multiple concurrent SQL statements\n", - "- Auto Loader can feed new SVS arrivals into `svs_metadata` as `PENDING`, triggering incremental VLM runs via a scheduled Lakeflow Job\n", - "\n" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "application/vnd.databricks.v1+cell": { - "cellMetadata": {}, - "inputWidgets": {}, - "nuid": "dd0db54d-807b-4cfe-bca5-0b524fd6e636", - "showTitle": false, - "tableResultSettingsMap": {}, - "title": "" - } - }, - "source": [ - "# Code" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "application/vnd.databricks.v1+cell": { - "cellMetadata": {}, - "inputWidgets": {}, - "nuid": "6e4014e5-1ba3-4c5c-9a8c-fbe85432645b", - "showTitle": true, - "tableResultSettingsMap": {}, - "title": "Data Flow Diagram" - } - }, - "source": [ - "## Data Flow Diagram\n", - "\n", - "```mermaid\n", - "flowchart TD\n", - " %% ─── External Sources ───\n", - " SVS_INPUT[(\"/Volumes/hls_pathology/orthanc_demo/raw_images/Aperio/\\n~14 SVS files\")]\n", - " VLM_EP{{\"databricks-llama-4-maverick\\n(VLM Endpoint)\"}}\n", - "\n", - " %% ─── Delta Tables ───\n", - " OBJ_CAT[(\"douglas_moore.pathology\\n.object_catalog\")]\n", - " OBJ_RED[(\"douglas_moore.pathology\\n.object_catalog_redaction\")]\n", - " TIFF_STG[(\"douglas_moore.pathology\\n.tiff_results_staging\")]\n", - "\n", - " %% ─── Volumes (File Storage) ───\n", - " LABEL_VOL[(\"/Volumes/.../label_images/\\nPNG sub-images\")]\n", - " TIFF_VOL[(\"/Volumes/.../tiff_deidentified/\\nBigTIFF output\")]\n", - " TMP[\"/tmp/ (executor local)\\nBigTIFF staging\"]\n", - "\n", - " %% ─── Processing Steps ───\n", - " DISCOVER[\"Cell 17: File Discovery\\nSVSCatalog.catalog()\"]\n", - " META[\"Cell 18: Metadata Extraction\\nSVSMetaExtractor (mapInPandas)\\nOpenSlide properties → VARIANT\"]\n", - " PHI_TAGS[\"Cell 19: PHI Tag Report\\n(display only)\"]\n", - " EXTRACT[\"Cell 20: Extract Sub-images\\npandas_udf + OpenSlide\\nassociated_images → PNG\"]\n", - " VLM_DETECT[\"Cell 22: VLM PHI Detection\\nai_query(files => content)\\nREAD_FILES + INSERT\"]\n", - " BUILD_DF[\"Cell 24: Build redaction_df\\nJOIN catalog + redaction\\nWHERE status = PENDING\"]\n", - " UDF[\"Cell 25-27: De-identify UDF\\nmapInPandas + ThreadPoolExecutor\\nredact_image + write_pyramidal_bigtiff\"]\n", - " MERGE[\"Cell 28: MERGE Results\\nUPDATE status, paths, errors\"]\n", - " AUDIT[\"Cell 29: Audit Summary\\n(display only)\"]\n", - "\n", - " %% ─── Data Flows ───\n", - " SVS_INPUT -->|\"list files\"| DISCOVER\n", - " DISCOVER -->|\"files_df (in-memory)\"| META\n", - " SVS_INPUT -->|\"read OpenSlide props\"| META\n", - " META -->|\"mode=append\"| OBJ_CAT\n", - "\n", - " OBJ_CAT -->|\"read meta:phi_tag_report\"| PHI_TAGS\n", - "\n", - " SVS_INPUT -->|\"read associated_images\"| EXTRACT\n", - " EXTRACT -->|\"save PNG (sequential write)\"| LABEL_VOL\n", - "\n", - " LABEL_VOL -->|\"READ_FILES(binaryFile)\"| VLM_DETECT\n", - " VLM_DETECT -->|\"ai_query()\"| VLM_EP\n", - " VLM_EP -->|\"JSON response\"| VLM_DETECT\n", - " OBJ_CAT -->|\"JOIN for path\"| VLM_DETECT\n", - " VLM_DETECT -->|\"INSERT INTO\"| OBJ_RED\n", - "\n", - " OBJ_CAT -->|\"JOIN\"| BUILD_DF\n", - " OBJ_RED -->|\"WHERE PENDING\"| BUILD_DF\n", - "\n", - " BUILD_DF -->|\"redaction_df\"| UDF\n", - " SVS_INPUT -->|\"read tiles (OpenSlide)\"| UDF\n", - " UDF -->|\"redacted PNGs (seq write)\"| LABEL_VOL\n", - " UDF -->|\"write BigTIFF (seek+write)\"| TMP\n", - " TMP -->|\"shutil.copy2 (seq write)\"| TIFF_VOL\n", - " UDF -->|\"saveAsTable\"| TIFF_STG\n", - "\n", - " TIFF_STG -->|\"source for MERGE\"| MERGE\n", - " MERGE -->|\"UPDATE status/paths\"| OBJ_RED\n", - "\n", - " OBJ_CAT -->|\"LEFT JOIN\"| AUDIT\n", - " OBJ_RED -->|\"LEFT JOIN\"| AUDIT\n", - "\n", - " %% ─── Styling ───\n", - " classDef volume fill:#e8f5e9,stroke:#2e7d32\n", - " classDef table fill:#e3f2fd,stroke:#1565c0\n", - " classDef process fill:#fff3e0,stroke:#e65100\n", - " classDef external fill:#fce4ec,stroke:#c62828\n", - " classDef tmp fill:#f5f5f5,stroke:#616161,stroke-dasharray:5\n", - "\n", - " class SVS_INPUT,LABEL_VOL,TIFF_VOL volume\n", - " class OBJ_CAT,OBJ_RED,TIFF_STG table\n", - " class DISCOVER,META,PHI_TAGS,EXTRACT,VLM_DETECT,BUILD_DF,UDF,MERGE,AUDIT process\n", - " class VLM_EP external\n", - " class TMP tmp\n", - "```\n", - "\n", - "### Legend\n", - "| Color | Meaning |\n", - "|---|---|\n", - "| Green | UC Volumes (file storage) |\n", - "| Blue | Delta Tables (Unity Catalog) |\n", - "| Orange | Processing steps (notebook cells) |\n", - "| Pink | External service (model endpoint) |\n", - "| Dashed gray | Ephemeral local storage (/tmp) |\n", - "\n", - "### Key Write Patterns\n", - "| Target | Write Mode | Reason |\n", - "|---|---|---|\n", - "| `object_catalog` | `mode=append` | Idempotent cataloguing; dedup via path |\n", - "| `object_catalog_redaction` | `INSERT INTO` | One row per VLM detection run |\n", - "| `tiff_results_staging` | `mode=overwrite` | Ephemeral staging; replaced each run |\n", - "| `object_catalog_redaction` | `MERGE ... WHEN MATCHED UPDATE` | Update status after TIFF write |\n", - "| Label PNGs (volume) | Sequential FUSE write | PIL `img.save()` — no seek needed |\n", - "| BigTIFFs (volume) | `/tmp/` → `shutil.copy2` | tifffile needs seek; Volume FUSE does not support seek+write |" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "application/vnd.databricks.v1+cell": { - "cellMetadata": { - "byteLimit": 2048000, - "rowLimit": 10000 - }, - "inputWidgets": {}, - "nuid": "e1fbd42a-68b2-44e2-8f2a-d9210c5254f5", - "showTitle": true, - "tableResultSettingsMap": {}, - "title": "Cell 2: Install dependencies" - } - }, - "outputs": [], - "source": [ - "# Install core dependencies.\n", - "# databricks-pixels provides Catalog + Transformer base classes.\n", - "# openslide-bin bundles libopenslide.so so OpenSlide works on Serverless.\n", - "%pip install openslide-python openslide-bin tifffile imagecodecs Pillow easyocr -q" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "application/vnd.databricks.v1+cell": { - "cellMetadata": { - "byteLimit": 2048000, - "rowLimit": 10000 - }, - "inputWidgets": {}, - "nuid": "ec9caf79-4345-4b35-9b30-dc215d7885e4", - "showTitle": true, - "tableResultSettingsMap": {}, - "title": "Cell 3: Configuration" - } - }, - "outputs": [], - "source": [ - "# The full pixels source tree (including svs/) lives in the workspace.\n", - "# Add the src directory to sys.path so `dbx.pixels` and `dbx.pixels.svs`\n", - "# are importable without a separate pip install.\n", - "import sys\n", - "import types\n", - "import importlib\n", - "\n", - "_SRC_ROOT = \"/Workspace/Users/douglas.moore@databricks.com/pixels-svs/src\"\n", - "if _SRC_ROOT not in sys.path:\n", - " sys.path.insert(0, _SRC_ROOT)\n", - "importlib.invalidate_caches()\n", - "\n", - "# Load deidentify module (file is clean — no truncation needed)\n", - "_deident_path = f\"{_SRC_ROOT}/dbx/pixels/svs/deidentify.py\"\n", - "with open(_deident_path, \"r\") as _f:\n", - " _clean_src = _f.read()\n", - "_deident_mod = types.ModuleType(\"dbx.pixels.svs.deidentify\")\n", - "_deident_mod.__file__ = _deident_path\n", - "exec(compile(_clean_src, _deident_path, \"exec\"), _deident_mod.__dict__)\n", - "sys.modules[\"dbx.pixels.svs.deidentify\"] = _deident_mod\n", - "\n", - "from dbx.pixels.svs import SVSCatalog, SVSMetaExtractor, SVSTiffWriter\n", - "from dbx.pixels.svs.phi_tags import classify_tags, scrub_image_description\n", - "\n", - "# ── Pipeline configuration ────────────────────────────────────────────────────\n", - "CATALOG = \"douglas_moore\"\n", - "SCHEMA = \"pathology\"\n", - "UC_TABLE = f\"{CATALOG}.{SCHEMA}.object_catalog\"\n", - "UC_VOLUME = f\"{CATALOG}.{SCHEMA}.pixels_volume\"\n", - "INPUT_PATH = \"/Volumes/hls_pathology/orthanc_demo/raw_images/Aperio/\"\n", - "TIFF_VOLUME = f\"/Volumes/{CATALOG}/{SCHEMA}/tiff_deidentified\"\n", - "LABEL_VOLUME = f\"/Volumes/{CATALOG}/{SCHEMA}/label_images\"\n", - "VLM_ENDPOINT = \"databricks-llama-4-maverick\"\n", - "\n", - "print(f\"Input : {INPUT_PATH}\")\n", - "print(f\"Table : {UC_TABLE}\")\n", - "print(f\"TIFFs : {TIFF_VOLUME}\")\n", - "print(f\"Labels: {LABEL_VOLUME}\")\n", - "print(f\"VLM : {VLM_ENDPOINT}\")\n" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "application/vnd.databricks.v1+cell": { - "cellMetadata": { - "byteLimit": 2048000, - "implicitDf": true, - "rowLimit": 10000 - }, - "inputWidgets": {}, - "nuid": "47cec7fe-67e6-4928-9aa6-1c5947b011bc", - "showTitle": true, - "tableResultSettingsMap": {}, - "title": "Reset: Truncate pipeline tables" - } - }, - "outputs": [], - "source": [ - "%sql\n", - "-- Reset pipeline state for a clean end-to-end run.\n", - "-- Truncates data only; table structure and permissions preserved.\n", - "TRUNCATE TABLE douglas_moore.pathology.object_catalog;\n", - "TRUNCATE TABLE douglas_moore.pathology.object_catalog_redaction;" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "application/vnd.databricks.v1+cell": { - "cellMetadata": { - "byteLimit": 2048000, - "rowLimit": 10000 - }, - "inputWidgets": {}, - "nuid": "8c5e7ce2-bec6-48b8-987b-34b7b2a1eeb5", - "showTitle": true, - "tableResultSettingsMap": {}, - "title": "Cell 4: Storage bootstrap" - } - }, - "outputs": [], - "source": [ - "# Create schema and volumes (idempotent — safe to re-run)\n", - "spark.sql(f\"CREATE SCHEMA IF NOT EXISTS {CATALOG}.{SCHEMA}\")\n", - "spark.sql(f\"CREATE VOLUME IF NOT EXISTS {CATALOG}.{SCHEMA}.pixels_volume\")\n", - "spark.sql(f\"CREATE VOLUME IF NOT EXISTS {CATALOG}.{SCHEMA}.tiff_deidentified\")\n", - "spark.sql(f\"CREATE VOLUME IF NOT EXISTS {CATALOG}.{SCHEMA}.label_images\")\n", - "\n", - "# Initialise Delta tables:\n", - "# object_catalog — base DDL from databricks-pixels (unchanged)\n", - "# object_catalog_redaction — unified SVS/DICOM DDL from CREATE_SVS_CATALOG.sql\n", - "catalog = SVSCatalog(spark, table=UC_TABLE, volume=UC_VOLUME)\n", - "catalog.init_tables()\n", - "\n", - "print(\"Schema, volumes, and tables initialised.\")\n", - "display(spark.sql(f\"SHOW TABLES IN {CATALOG}.{SCHEMA}\"))\n" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "application/vnd.databricks.v1+cell": { - "cellMetadata": { - "byteLimit": 2048000, - "rowLimit": 10000 - }, - "inputWidgets": {}, - "nuid": "d0560972-3ca5-4f06-8f65-d7c621b688db", - "showTitle": true, - "tableResultSettingsMap": {}, - "title": "Cell 5: File discovery" - } - }, - "outputs": [], - "source": [ - "# Discover all SVS files under INPUT_PATH and register them in object_catalog.\n", - "# SVSCatalog.catalog() defaults pattern='*.svs'; also picks up sidecar .txt files.\n", - "files_df = catalog.catalog(INPUT_PATH)\n", - "print(f\"Discovered {files_df.count()} files\")\n", - "display(files_df.select(\"path\", \"local_path\", \"length\", \"modificationTime\", \"extension\"))\n" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "application/vnd.databricks.v1+cell": { - "cellMetadata": { - "byteLimit": 2048000, - "rowLimit": 10000 - }, - "inputWidgets": {}, - "nuid": "147b854b-61b3-4207-a172-9fc5f36649b9", - "showTitle": true, - "tableResultSettingsMap": {}, - "title": "Cell 6: Metadata extraction" - } - }, - "outputs": [], - "source": [ - "# SVSMetaExtractor reads every SVS via OpenSlide (ThreadPoolExecutor, 32 concurrent).\n", - "# All properties + derived fields (width, height, levels, phi_tag_report) are merged\n", - "# into one JSON dict → parse_json() → VARIANT. No schema changes to object_catalog.\n", - "extractor = SVSMetaExtractor(catalog, inputCol=\"local_path\", outputCol=\"meta\")\n", - "meta_df = extractor._transform(files_df)\n", - "\n", - "(\n", - " meta_df.write\n", - " .format(\"delta\")\n", - " .mode(\"append\")\n", - " .saveAsTable(UC_TABLE)\n", - ")\n", - "\n", - "print(f\"Wrote {spark.table(UC_TABLE).count()} rows to {UC_TABLE}\")\n", - "\n", - "display(spark.sql(f\"\"\"\n", - "SELECT\n", - " regexp_extract(path, '[^/]+$', 0) AS filename,\n", - " meta:width::int AS width,\n", - " meta:height::int AS height,\n", - " meta:level_count::int AS levels,\n", - " meta:has_label_image::boolean AS has_label,\n", - " meta:has_macro_image::boolean AS has_macro,\n", - " meta:`aperio.AppMag`::string AS app_mag,\n", - " meta:`aperio.MPP`::string AS mpp,\n", - " meta\n", - "FROM {UC_TABLE}\n", - "ORDER BY filename\n", - "\"\"\"))\n" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "application/vnd.databricks.v1+cell": { - "cellMetadata": { - "byteLimit": 2048000, - "rowLimit": 10000 - }, - "inputWidgets": {}, - "nuid": "541697a5-602a-40dd-9ac9-9f53624b4efa", - "showTitle": true, - "tableResultSettingsMap": { - "0": { - "dataGridStateBlob": "{\"version\":1,\"tableState\":{\"columnPinning\":{\"left\":[\"#row_number#\"],\"right\":[]},\"columnSizing\":{\"tag\":129},\"columnVisibility\":{}},\"settings\":{\"columns\":{}},\"syncTimestamp\":1781723212243}", - "filterBlob": null, - "queryPlanFiltersBlob": null, - "tableResultIndex": 0 - } - }, - "title": "Cell 7: PHI tag report" - } - }, - "outputs": [], - "source": [ - "# PHI / QUESTIONABLE tag values for every slide.\n", - "# Unpack the phi_tag_report VARIANT array stored in meta.\n", - "from pyspark.sql.functions import regexp_extract, col, explode, from_json, expr\n", - "from pyspark.sql.types import ArrayType, StructType, StructField, StringType\n", - "\n", - "phi_schema = ArrayType(StructType([\n", - " StructField(\"tag\", StringType()),\n", - " StructField(\"value\", StringType()),\n", - " StructField(\"classification\", StringType()),\n", - "]))\n", - "\n", - "phi_df = (\n", - " spark.table(UC_TABLE)\n", - " .filter(expr(\"meta:phi_tag_report IS NOT NULL\"))\n", - " .withColumn(\"phi_tag_report_str\", expr(\"cast(meta:phi_tag_report AS STRING)\"))\n", - " .withColumn(\"tags\", from_json(\"phi_tag_report_str\", phi_schema))\n", - " .withColumn(\"elem\", explode(\"tags\"))\n", - " .select(\n", - " regexp_extract(\"path\", r\"[^/]+$\", 0).alias(\"filename\"),\n", - " col(\"elem.tag\").alias(\"tag\"),\n", - " col(\"elem.value\").alias(\"value\"),\n", - " col(\"elem.classification\").alias(\"classification\"),\n", - " )\n", - " .filter(col(\"classification\").isin(\"PHI\", \"QUESTIONABLE\"))\n", - " .orderBy(\"filename\", \"classification\", \"tag\")\n", - ")\n", - "print(f\"PHI/QUESTIONABLE findings: {phi_df.count()}\")\n", - "display(phi_df)" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "application/vnd.databricks.v1+cell": { - "cellMetadata": { - "byteLimit": 2048000, - "rowLimit": 10000 - }, - "inputWidgets": {}, - "nuid": "28f6bfb9-bec1-4cb0-9c80-5c395a1a9432", - "showTitle": true, - "tableResultSettingsMap": {}, - "title": "Cell 8: Extract label/macro sub-images" - } - }, - "outputs": [], - "source": [ - "# Extract label and macro sub-images from each SVS and save as PNGs.\n", - "# These are later submitted to the VLM (Cell 9) and used as audit artefacts.\n", - "# Uses a pandas_udf so extraction runs distributed across workers.\n", - "from pyspark.sql.functions import pandas_udf, regexp_extract, col\n", - "import pandas as pd\n", - "from pyspark.sql.types import StringType\n", - "\n", - "_LABEL_VOL = LABEL_VOLUME # captured in closure; serialised with the UDF\n", - "\n", - "@pandas_udf(StringType())\n", - "def extract_subimages_udf(paths: pd.Series, stems: pd.Series) -> pd.Series:\n", - " import openslide, os\n", - " results = []\n", - " for path, stem in zip(paths, stems):\n", - " try:\n", - " slide = openslide.OpenSlide(path)\n", - " saved = []\n", - " for name in (\"label\", \"macro\"):\n", - " if name in slide.associated_images:\n", - " img = slide.associated_images[name].convert(\"RGB\")\n", - " out = f\"{_LABEL_VOL}/{stem}_{name}.png\"\n", - " os.makedirs(os.path.dirname(out), exist_ok=True)\n", - " img.save(out)\n", - " saved.append(out)\n", - " slide.close()\n", - " results.append(\",\".join(saved))\n", - " except Exception as e:\n", - " results.append(f\"ERROR: {e}\")\n", - " return pd.Series(results)\n", - "\n", - "catalog_df = (\n", - " spark.table(UC_TABLE)\n", - " .withColumn(\"stem\", regexp_extract(col(\"path\"), r\"([^/]+)\\.svs$\", 1))\n", - ")\n", - "\n", - "extracted_df = catalog_df.withColumn(\n", - " \"extracted_images\",\n", - " extract_subimages_udf(col(\"local_path\"), col(\"stem\")),\n", - ")\n", - "\n", - "display(extracted_df.select(\"path\", \"stem\", \"extracted_images\"))\n", - "print(f\"Label/macro PNGs written to {LABEL_VOLUME}\")\n" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "application/vnd.databricks.v1+cell": { - "cellMetadata": { - "byteLimit": 2048000, - "rowLimit": 10000 - }, - "inputWidgets": {}, - "nuid": "556f24ea-d0c5-46ef-ac35-761ffed88820", - "showTitle": true, - "tableResultSettingsMap": {}, - "title": "Display first 10 label images" - } - }, - "outputs": [], - "source": [ - "# Display first 10 label sub-images extracted from SVS pathology slides\n", - "import os\n", - "from PIL import Image\n", - "import matplotlib.pyplot as plt\n", - "\n", - "label_dir = LABEL_VOLUME\n", - "label_files = sorted([f for f in os.listdir(label_dir) if f.endswith(\"_label.png\")])[:10]\n", - "\n", - "ncols = min(5, len(label_files))\n", - "nrows = (len(label_files) + ncols - 1) // ncols\n", - "fig, axes = plt.subplots(nrows, ncols, figsize=(4 * ncols, 5 * nrows))\n", - "if len(label_files) == 1:\n", - " axes = [axes]\n", - "else:\n", - " axes = axes.flatten()\n", - "\n", - "for i, fname in enumerate(label_files):\n", - " img = Image.open(os.path.join(label_dir, fname))\n", - " axes[i].imshow(img)\n", - " axes[i].set_title(fname.replace(\"_label.png\", \"\"), fontsize=9)\n", - " axes[i].axis(\"off\")\n", - "\n", - "for j in range(len(label_files), len(axes)):\n", - " axes[j].axis(\"off\")\n", - "\n", - "plt.suptitle(\"SVS Label Sub-Images (PHI candidates for VLM redaction)\", fontsize=13)\n", - "plt.tight_layout()\n", - "plt.show()" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "application/vnd.databricks.v1+cell": { - "cellMetadata": { - "byteLimit": 2048000, - "implicitDf": true, - "rowLimit": 10000 - }, - "height": "156", - "inputWidgets": {}, - "nuid": "23583e7a-dfc7-4da0-be2b-dbac0b061935", - "showTitle": true, - "tableResultSettingsMap": {}, - "title": "Cell 9: VLM PHI detection", - "width": "834" - } - }, - "outputs": [], - "source": [ - "%sql\n", - "-- Run VLM PHI detection on all label PNGs and insert results into object_catalog_redaction.\n", - "-- ai_query() submits the image binary directly via `files => content` (no base64 needed).\n", - "-- responseFormat => 'json_object' guarantees machine-parseable output.\n", - "INSERT INTO douglas_moore.pathology.object_catalog_redaction (\n", - " redaction_id, path, extension, modality,\n", - " has_phi, phi_elements, vlm_raw_response, model_endpoint,\n", - " output_file_paths, label_image_path, macro_image_path,\n", - " status, insert_timestamp, created_by\n", - ")\n", - "WITH vlm_raw AS (\n", - " SELECT\n", - " regexp_replace(f._metadata.file_name, '_label\\.png$', '') AS stem,\n", - " f._metadata.file_path AS label_image_path,\n", - " ai_query(\n", - " 'databricks-llama-4-maverick',\n", - " 'You are a HIPAA-compliant PHI detection system.\n", - "Analyze this pathology slide label image and identify all Protected Health Information:\n", - "patient names, dates, MRNs, accession numbers, barcodes, or other identifying text.\n", - "Return ONLY a json object — no prose, no markdown fences.\n", - "Schema: {\"has_phi\": bool, \"phi_elements\": [{\"type\": \"name|dob|mrn|accession|barcode|other\", \"value_hint\": \"\", \"bbox\": {\"x\": int, \"y\": int, \"w\": int, \"h\": int}, \"subimage\": \"label\"}]}\n", - "If no PHI found: {\"has_phi\": false, \"phi_elements\": []}',\n", - " files => content\n", - " ) AS vlm_raw_response\n", - " FROM READ_FILES(\n", - " '/Volumes/douglas_moore/pathology/label_images/',\n", - " format => 'binaryFile',\n", - " fileNamePattern => '*_label.png'\n", - " ) f\n", - "),\n", - "joined AS (\n", - " SELECT\n", - " v.stem,\n", - " v.label_image_path,\n", - " v.vlm_raw_response,\n", - " m.path AS obj_path,\n", - " concat('/Volumes/douglas_moore/pathology/label_images/', v.stem, '_macro.png') AS macro_image_path\n", - " FROM vlm_raw v\n", - " JOIN douglas_moore.pathology.object_catalog m\n", - " ON regexp_extract(m.path, '([^/]+)\\.svs$', 1) = v.stem\n", - ")\n", - "SELECT\n", - " uuid() AS redaction_id,\n", - " obj_path AS path,\n", - " 'svs' AS extension,\n", - " 'WSI' AS modality,\n", - " try_cast(get_json_object(vlm_raw_response, '$.has_phi') AS BOOLEAN) AS has_phi,\n", - " parse_json(get_json_object(vlm_raw_response, '$.phi_elements')) AS phi_elements,\n", - " vlm_raw_response,\n", - " 'databricks-llama-4-maverick' AS model_endpoint,\n", - " array(CAST(NULL AS STRING)) AS output_file_paths,\n", - " label_image_path,\n", - " macro_image_path,\n", - " 'PENDING' AS status,\n", - " current_timestamp() AS insert_timestamp,\n", - " current_user() AS created_by\n", - "FROM joined\n" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "application/vnd.databricks.v1+cell": { - "cellMetadata": { - "byteLimit": 2048000, - "implicitDf": true, - "rowLimit": 10000 - }, - "inputWidgets": {}, - "nuid": "6dae4df9-e494-403e-9c58-a933aa22052a", - "showTitle": false, - "tableResultSettingsMap": {}, - "title": "" - } - }, - "outputs": [], - "source": [ - "%sql\n", - "select * from douglas_moore.pathology.object_catalog_redaction" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "application/vnd.databricks.v1+cell": { - "cellMetadata": { - "byteLimit": 2048000, - "rowLimit": 10000 - }, - "inputWidgets": {}, - "nuid": "98cc5d83-bbdd-4f8c-9bd6-b61d259ad882", - "showTitle": false, - "tableResultSettingsMap": {}, - "title": "" - } - }, - "outputs": [], - "source": [ - "# Reload modules to pick up streaming TIFF writer\n", - "import importlib, sys\n", - "for mod_name in list(sys.modules):\n", - " if mod_name.startswith(\"dbx.pixels.svs\"):\n", - " del sys.modules[mod_name]\n", - "\n", - "# Join PENDING redaction rows (phi_elements from VLM) with object_catalog (local_path),\n", - "# run de-identification and produce pyramidal BigTIFFs.\n", - "redaction_df = spark.sql(f\"\"\"\n", - "SELECT\n", - " o.local_path,\n", - " o.path,\n", - " r.redaction_id,\n", - " to_json(r.phi_elements) AS phi_elements_json\n", - "FROM {CATALOG}.{SCHEMA}.object_catalog o\n", - "JOIN {CATALOG}.{SCHEMA}.object_catalog_redaction r\n", - " ON o.path = r.path\n", - "WHERE r.status = 'PENDING'\n", - "\"\"\")\n", - "print(f\"Files to de-identify: {redaction_df.count()}\")" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "application/vnd.databricks.v1+cell": { - "cellMetadata": { - "byteLimit": 2048000, - "rowLimit": 10000 - }, - "inputWidgets": {}, - "nuid": "a73b4474-3adc-4866-88cd-822dc0ad6c45", - "showTitle": true, - "tableResultSettingsMap": {}, - "title": "Cell 10: De-identified TIFF write" - } - }, - "outputs": [], - "source": [ - "# --- Distributed de-identification via mapInPandas (memory-safe for Serverless 1 GB) ---\n", - "#\n", - "# Design principles applied from review:\n", - "# • No inner ThreadPoolExecutor — mapInPandas already parallelizes across Spark\n", - "# partitions; nested threading doubles slide opens and memory pressure.\n", - "# • No to_dict(\"records\") — iterate rows via iloc to avoid duplicating the batch.\n", - "# • OpenSlide closed in a finally block so C-bindings are destroyed even on error.\n", - "# • gc.collect() after each slide reclaims PIL/OpenSlide C-level allocations.\n", - "# • Tile-based TIFF writing via write_pyramidal_bigtiff_streaming (256×256 read_region).\n", - "# • Temp files staged to /tmp (seek-capable), then shutil.copy2 to Volume (seq FUSE).\n", - "# • PID suffix on temp paths prevents collisions across retries/speculative tasks.\n", - "# • repartition(num_slides) ensures 1 row per partition — each executor handles\n", - "# exactly one slide. (arrow.maxRecordsPerBatch is NOT settable on Serverless.)\n", - "\n", - "import json\n", - "import pandas as pd\n", - "from pyspark.sql.types import StructType, StructField, StringType, ArrayType, IntegerType\n", - "\n", - "_TIFF_VOLUME = TIFF_VOLUME\n", - "_LABEL_VOLUME = LABEL_VOLUME\n", - "_SRC_ROOT = \"/Workspace/Users/douglas.moore@databricks.com/pixels-svs/src\"\n", - "\n", - "_result_schema = StructType([\n", - " StructField(\"path\", StringType(), True),\n", - " StructField(\"tiff_output_path\", StringType(), True),\n", - " StructField(\"label_image_path\", StringType(), True),\n", - " StructField(\"macro_image_path\", StringType(), True),\n", - " StructField(\"phi_tags_redacted\", ArrayType(StringType()), True),\n", - " StructField(\"pixel_regions_redacted\", IntegerType(), True),\n", - " StructField(\"error\", StringType(), True),\n", - "])\n", - "\n", - "\n", - "def _deidentify_batch(iterator):\n", - " \"\"\"mapInPandas worker: one slide per batch, streaming tile reads, no threading.\"\"\"\n", - " import sys, os, gc, shutil, time, logging, resource\n", - " from pathlib import Path\n", - "\n", - " # --- Memory debugging utilities ---\n", - " logging.basicConfig(level=logging.INFO, format=\"%(asctime)s [%(levelname)s] %(message)s\")\n", - " log = logging.getLogger(\"deidentify_worker\")\n", - "\n", - " def _mem_mb() -> dict:\n", - " \"\"\"Return RSS and VMS in MB from /proc/self/status (Linux) with fallback.\"\"\"\n", - " rss_mb = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss / 1024 # KB→MB on Linux\n", - " try:\n", - " with open(\"/proc/self/status\") as f:\n", - " status = f.read()\n", - " vmpeak = vmrss = vmsize = 0\n", - " for line in status.splitlines():\n", - " if line.startswith(\"VmPeak:\"):\n", - " vmpeak = int(line.split()[1]) / 1024\n", - " elif line.startswith(\"VmRSS:\"):\n", - " vmrss = int(line.split()[1]) / 1024\n", - " elif line.startswith(\"VmSize:\"):\n", - " vmsize = int(line.split()[1]) / 1024\n", - " return {\"rss_mb\": round(vmrss, 1), \"vms_mb\": round(vmsize, 1), \"peak_mb\": round(vmpeak, 1)}\n", - " except Exception:\n", - " return {\"rss_mb\": round(rss_mb, 1), \"vms_mb\": -1, \"peak_mb\": -1}\n", - "\n", - " def _log_mem(stage: str, stem: str, extra: str = \"\"):\n", - " mem = _mem_mb()\n", - " msg = f\"[{stem}] stage={stage} | RSS={mem['rss_mb']}MB VMS={mem['vms_mb']}MB Peak={mem['peak_mb']}MB\"\n", - " if extra:\n", - " msg += f\" | {extra}\"\n", - " log.info(msg)\n", - " # Warn if approaching the 1024 MB limit\n", - " if mem[\"rss_mb\"] > 800:\n", - " log.warning(f\"⚠️ HIGH MEMORY [{stem}] stage={stage} RSS={mem['rss_mb']}MB — approaching 1024MB limit!\")\n", - "\n", - " # Ensure src modules are importable on executors\n", - " if _SRC_ROOT not in sys.path:\n", - " sys.path.insert(0, _SRC_ROOT)\n", - "\n", - " import openslide\n", - " from dbx.pixels.svs.deidentify import redact_image, write_pyramidal_bigtiff_streaming\n", - " from dbx.pixels.svs.phi_tags import scrub_image_description\n", - "\n", - " for pdf in iterator:\n", - " results = []\n", - " log.info(f\"Batch received: {len(pdf)} row(s) | PID={os.getpid()}\")\n", - " _log_mem(\"batch_start\", \"batch\", f\"rows={len(pdf)}\")\n", - "\n", - " # Iterate rows directly via iloc — no to_dict(\"records\") memory copy\n", - " for idx in range(len(pdf)):\n", - " row = pdf.iloc[idx]\n", - " svs_path = row[\"local_path\"]\n", - " phi_json = row[\"phi_elements_json\"]\n", - " stem = Path(svs_path).stem\n", - " stage = \"init\"\n", - " slide = None\n", - " tmp_tiff = f\"/tmp/{stem}_{os.getpid()}.tiff\"\n", - " t0 = time.time()\n", - "\n", - " log.info(f\"=== Processing slide: {stem} ===\")\n", - " _log_mem(\"init\", stem, f\"svs_path={svs_path}\")\n", - "\n", - " try:\n", - " phi_elements = json.loads(phi_json) if phi_json else []\n", - " stage = \"open_slide\"\n", - " slide = openslide.OpenSlide(svs_path)\n", - " dims = slide.dimensions # (width, height) at level 0\n", - " levels = slide.level_count\n", - " _log_mem(\"open_slide\", stem, f\"dims={dims[0]}x{dims[1]} levels={levels}\")\n", - "\n", - " # 1. Redact label/macro sub-images (small RGBA → write PNGs to Volume)\n", - " label_path = macro_path = None\n", - " pixel_count = 0\n", - "\n", - " if \"label\" in slide.associated_images:\n", - " stage = \"redact_label\"\n", - " label_img = slide.associated_images[\"label\"]\n", - " label_phi = [e for e in phi_elements if e.get(\"subimage\", \"label\") != \"macro\"]\n", - " label_out = redact_image(label_img, label_phi)\n", - " pixel_count += len(label_phi)\n", - " label_path = f\"{_LABEL_VOLUME}/{stem}_label.png\"\n", - " os.makedirs(os.path.dirname(label_path), exist_ok=True)\n", - " label_out.save(label_path)\n", - " _log_mem(\"redact_label\", stem, f\"label_size={label_img.size} phi_count={len(label_phi)}\")\n", - " del label_img, label_out # free RGBA buffer immediately\n", - "\n", - " if \"macro\" in slide.associated_images:\n", - " stage = \"redact_macro\"\n", - " macro_img = slide.associated_images[\"macro\"]\n", - " macro_phi = [e for e in phi_elements if e.get(\"subimage\") == \"macro\"]\n", - " macro_out = redact_image(macro_img, macro_phi)\n", - " pixel_count += len(macro_phi)\n", - " macro_path = f\"{_LABEL_VOLUME}/{stem}_macro.png\"\n", - " os.makedirs(os.path.dirname(macro_path), exist_ok=True)\n", - " macro_out.save(macro_path)\n", - " _log_mem(\"redact_macro\", stem, f\"macro_size={macro_img.size} phi_count={len(macro_phi)}\")\n", - " del macro_img, macro_out\n", - "\n", - " # 2. Scrub metadata tags\n", - " stage = \"scrub_metadata\"\n", - " raw_desc = slide.properties.get(\"tiff.ImageDescription\", \"\")\n", - " scrubbed = scrub_image_description(raw_desc)\n", - " phi_tags_redacted = (\n", - " [\"tiff.ImageDescription\", \"openslide.comment\"]\n", - " if raw_desc != scrubbed else []\n", - " )\n", - " _log_mem(\"scrub_metadata\", stem)\n", - "\n", - " # 3. Write pyramidal BigTIFF — tile-streaming (256×256 read_region)\n", - " # Stage to /tmp (requires seek), then sequential copy to Volume.\n", - " stage = \"write_tiff_to_tmp\"\n", - " t_tiff_start = time.time()\n", - " write_pyramidal_bigtiff_streaming(tmp_tiff, slide, jpeg_quality=80)\n", - " t_tiff_elapsed = time.time() - t_tiff_start\n", - " tmp_size_mb = os.path.getsize(tmp_tiff) / (1024 * 1024) if os.path.exists(tmp_tiff) else 0\n", - " _log_mem(\"write_tiff_done\", stem, f\"tiff_size={tmp_size_mb:.1f}MB elapsed={t_tiff_elapsed:.1f}s\")\n", - "\n", - " # Close slide BEFORE copy to free C-level handles and mapped memory\n", - " slide.close()\n", - " slide = None\n", - " _log_mem(\"slide_closed\", stem)\n", - "\n", - " stage = \"copy_tiff_to_volume\"\n", - " t_copy_start = time.time()\n", - " tiff_path = f\"{_TIFF_VOLUME}/{stem}.tiff\"\n", - " shutil.copy2(tmp_tiff, tiff_path)\n", - " os.remove(tmp_tiff)\n", - " t_copy_elapsed = time.time() - t_copy_start\n", - " _log_mem(\"copy_done\", stem, f\"copy_elapsed={t_copy_elapsed:.1f}s\")\n", - "\n", - " total_elapsed = time.time() - t0\n", - " log.info(f\"✓ [{stem}] completed in {total_elapsed:.1f}s | tiff={tmp_size_mb:.1f}MB\")\n", - "\n", - " results.append({\n", - " \"path\": row[\"path\"],\n", - " \"tiff_output_path\": tiff_path,\n", - " \"label_image_path\": label_path,\n", - " \"macro_image_path\": macro_path,\n", - " \"phi_tags_redacted\": phi_tags_redacted,\n", - " \"pixel_regions_redacted\": pixel_count,\n", - " \"error\": None,\n", - " })\n", - "\n", - " except Exception as exc:\n", - " _log_mem(\"ERROR\", stem, f\"stage={stage} exc={type(exc).__name__}: {exc}\")\n", - " results.append({\n", - " \"path\": row[\"path\"],\n", - " \"tiff_output_path\": None,\n", - " \"label_image_path\": None,\n", - " \"macro_image_path\": None,\n", - " \"phi_tags_redacted\": [],\n", - " \"pixel_regions_redacted\": 0,\n", - " \"error\": f\"[stage={stage}] {type(exc).__name__}: {exc}\",\n", - " })\n", - "\n", - " finally:\n", - " # Ensure slide is always closed — destroy C-bindings immediately\n", - " if slide is not None:\n", - " try:\n", - " slide.close()\n", - " except Exception:\n", - " pass\n", - " # Clean up temp file on failure\n", - " if os.path.exists(tmp_tiff):\n", - " try:\n", - " os.remove(tmp_tiff)\n", - " except Exception:\n", - " pass\n", - " # Force GC to reclaim PIL/OpenSlide C-level allocations\n", - " gc.collect()\n", - " _log_mem(\"gc_complete\", stem)\n", - "\n", - " yield pd.DataFrame(results)" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "application/vnd.databricks.v1+cell": { - "cellMetadata": { - "byteLimit": 2048000, - "rowLimit": 10000 - }, - "inputWidgets": {}, - "nuid": "b449d29c-0651-41bc-afc8-bf83bdcf7f74", - "showTitle": false, - "tableResultSettingsMap": {}, - "title": "" - } - }, - "outputs": [], - "source": [ - "# Materialize the expensive mapInPandas UDF exactly ONCE.\n", - "# Strategy: persist() + count() forces a single execution pass.\n", - "# Downstream MERGE reads from the cached DataFrame via temp view.\n", - "TIFF_RESULTS_TABLE = f\"{CATALOG}.{SCHEMA}.tiff_results_staging\"\n", - "\n", - "num_slides = redaction_df.count()\n", - "print(f\"\"\"{num_slides}\"\"\")\n", - "assert num_slides > 0, \"No PENDING slides to de-identify — check object_catalog_redaction status\"\n" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "application/vnd.databricks.v1+cell": { - "cellMetadata": { - "byteLimit": 2048000, - "rowLimit": 10000 - }, - "inputWidgets": {}, - "nuid": "2581617a-cb08-4d46-b604-56b0c4d0400e", - "showTitle": true, - "tableResultSettingsMap": {}, - "title": "Dry run: 1 slide with memory logging" - } - }, - "outputs": [], - "source": [ - "# --- Dry run: 1 slide on DRIVER to capture full memory trace ---\n", - "# Runs the same logic outside mapInPandas so we can see exactly which stage\n", - "# exceeds 1024 MB without the executor being killed.\n", - "\n", - "import sys, os, gc, time, resource, json, shutil\n", - "from pathlib import Path\n", - "\n", - "if \"/Workspace/Users/douglas.moore@databricks.com/pixels-svs/src\" not in sys.path:\n", - " sys.path.insert(0, \"/Workspace/Users/douglas.moore@databricks.com/pixels-svs/src\")\n", - "\n", - "import openslide\n", - "from dbx.pixels.svs.deidentify import redact_image, write_pyramidal_bigtiff_streaming\n", - "from dbx.pixels.svs.phi_tags import scrub_image_description\n", - "\n", - "def _mem_mb():\n", - " \"\"\"RSS/VMS/Peak from /proc/self/status.\"\"\"\n", - " try:\n", - " with open(\"/proc/self/status\") as f:\n", - " status = f.read()\n", - " vals = {}\n", - " for line in status.splitlines():\n", - " for key in (\"VmPeak\", \"VmRSS\", \"VmSize\"):\n", - " if line.startswith(key + \":\"):\n", - " vals[key] = int(line.split()[1]) / 1024 # KB→MB\n", - " return {\"rss_mb\": round(vals.get(\"VmRSS\", 0), 1),\n", - " \"vms_mb\": round(vals.get(\"VmSize\", 0), 1),\n", - " \"peak_mb\": round(vals.get(\"VmPeak\", 0), 1)}\n", - " except Exception:\n", - " return {\"rss_mb\": -1, \"vms_mb\": -1, \"peak_mb\": -1}\n", - "\n", - "def log_mem(stage, extra=\"\"):\n", - " mem = _mem_mb()\n", - " warn = \" ⚠️ OVER 1GB!\" if mem[\"rss_mb\"] > 1024 else (\"⚠️ HIGH\" if mem[\"rss_mb\"] > 800 else \"\")\n", - " print(f\" [{stage:20s}] RSS={mem['rss_mb']:>7.1f}MB VMS={mem['vms_mb']:>7.1f}MB Peak={mem['peak_mb']:>7.1f}MB {warn} {extra}\")\n", - "\n", - "# Get one slide from the redaction dataframe\n", - "row = redaction_df.limit(1).collect()[0]\n", - "svs_path = row[\"local_path\"]\n", - "phi_json = row[\"phi_elements_json\"]\n", - "stem = Path(svs_path).stem\n", - "tmp_tiff = f\"/tmp/{stem}_dryrun.tiff\"\n", - "\n", - "print(f\"\\n{'='*80}\")\n", - "print(f\"DRY RUN MEMORY PROFILE: {stem}\")\n", - "print(f\"SVS path: {svs_path}\")\n", - "print(f\"{'='*80}\")\n", - "\n", - "gc.collect()\n", - "log_mem(\"baseline\")\n", - "\n", - "# Open slide\n", - "t0 = time.time()\n", - "slide = openslide.OpenSlide(svs_path)\n", - "dims = slide.dimensions\n", - "print(f\"\\n Slide: {dims[0]}x{dims[1]} pixels, {slide.level_count} levels\")\n", - "print(f\" Level dimensions: {[slide.level_dimensions[i] for i in range(slide.level_count)]}\")\n", - "print(f\" Associated images: {list(slide.associated_images.keys())}\")\n", - "log_mem(\"open_slide\", f\"file_size={os.path.getsize(svs_path)/(1024*1024):.1f}MB\")\n", - "\n", - "# Redact label\n", - "if \"label\" in slide.associated_images:\n", - " label_img = slide.associated_images[\"label\"]\n", - " print(f\"\\n Label image: {label_img.size} mode={label_img.mode}\")\n", - " phi_elements = json.loads(phi_json) if phi_json else []\n", - " label_phi = [e for e in phi_elements if e.get(\"subimage\", \"label\") != \"macro\"]\n", - " label_out = redact_image(label_img, label_phi)\n", - " log_mem(\"redact_label\", f\"phi_regions={len(label_phi)}\")\n", - " del label_img, label_out\n", - " gc.collect()\n", - " log_mem(\"label_freed\")\n", - "\n", - "# Redact macro\n", - "if \"macro\" in slide.associated_images:\n", - " macro_img = slide.associated_images[\"macro\"]\n", - " print(f\"\\n Macro image: {macro_img.size} mode={macro_img.mode}\")\n", - " phi_elements = json.loads(phi_json) if phi_json else []\n", - " macro_phi = [e for e in phi_elements if e.get(\"subimage\") == \"macro\"]\n", - " macro_out = redact_image(macro_img, macro_phi)\n", - " log_mem(\"redact_macro\", f\"phi_regions={len(macro_phi)}\")\n", - " del macro_img, macro_out\n", - " gc.collect()\n", - " log_mem(\"macro_freed\")\n", - "\n", - "# Scrub metadata\n", - "raw_desc = slide.properties.get(\"tiff.ImageDescription\", \"\")\n", - "scrubbed = scrub_image_description(raw_desc)\n", - "log_mem(\"scrub_metadata\")\n", - "\n", - "# Write pyramidal BigTIFF (this is the suspected memory hog)\n", - "print(f\"\\n Writing pyramidal BigTIFF to /tmp ...\")\n", - "t_tiff = time.time()\n", - "try:\n", - " write_pyramidal_bigtiff_streaming(tmp_tiff, slide, jpeg_quality=80)\n", - " tiff_elapsed = time.time() - t_tiff\n", - " tiff_size = os.path.getsize(tmp_tiff) / (1024 * 1024)\n", - " log_mem(\"write_tiff_done\", f\"size={tiff_size:.1f}MB elapsed={tiff_elapsed:.1f}s\")\n", - "except Exception as e:\n", - " log_mem(\"write_tiff_FAILED\", f\"{type(e).__name__}: {e}\")\n", - " tiff_size = 0\n", - "\n", - "# Close slide\n", - "slide.close()\n", - "log_mem(\"slide_closed\")\n", - "gc.collect()\n", - "log_mem(\"gc_after_close\")\n", - "\n", - "# Cleanup\n", - "if os.path.exists(tmp_tiff):\n", - " os.remove(tmp_tiff)\n", - "\n", - "total = time.time() - t0\n", - "print(f\"\\n{'='*80}\")\n", - "print(f\"COMPLETE: {stem} in {total:.1f}s | TIFF={tiff_size:.1f}MB\")\n", - "print(f\"Peak memory: {_mem_mb()['peak_mb']:.1f}MB\")\n", - "print(f\"{'='*80}\")\n", - "if _mem_mb()[\"peak_mb\"] > 1024:\n", - " print(\"\\n❌ Peak memory EXCEEDED 1024MB — this will OOM on Serverless executors.\")\n", - " print(\" → Investigate write_pyramidal_bigtiff_streaming tile buffer size.\")\n", - "else:\n", - " print(\"\\n✅ Peak memory stayed under 1024MB — safe for Serverless executors.\")" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "application/vnd.databricks.v1+cell": { - "cellMetadata": { - "byteLimit": 2048000, - "rowLimit": 10000 - }, - "inputWidgets": {}, - "nuid": "a8e83dad-0127-488c-b894-5d442508e404", - "showTitle": true, - "tableResultSettingsMap": {}, - "title": "Cell 26: Execute TIFF write (materialize once)" - } - }, - "outputs": [], - "source": [ - "\n", - "# 1 slide per partition to stay under 1GB UDF memory limit on serverless.\n", - "# Large SVS files (CMU-1 = 46000x32000) need sole access to executor RAM.\n", - "results_df = (\n", - " redaction_df\n", - " .repartition(num_slides)\n", - " .mapInPandas(_deidentify_batch, schema=_result_schema)\n", - " .limit(4)\n", - ")\n", - "\n", - "# Force single execution — UDF runs here and only here\n", - "results_df.select(\"path\", \"tiff_output_path\", \"label_image_path\", \"macro_image_path\", \"phi_tags_redacted\", \"pixel_regions_redacted\", \"error\").write.saveAsTable(TIFF_RESULTS_TABLE)\n", - "\n", - "\n", - "display(spark.sql(f\"SELECT * FROM {TIFF_RESULTS_TABLE}\"))" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "application/vnd.databricks.v1+cell": { - "cellMetadata": { - "byteLimit": 2048000, - "rowLimit": 10000 - }, - "inputWidgets": {}, - "nuid": "5c7b3af8-ded5-4207-92d1-42a0710e11e6", - "showTitle": true, - "tableResultSettingsMap": {}, - "title": "Cell 23" - } - }, - "outputs": [], - "source": [ - "# Merge output paths and status back into object_catalog_redaction\n", - "spark.sql(f\"\"\"\n", - "MERGE INTO {CATALOG}.{SCHEMA}.object_catalog_redaction AS tgt\n", - "USING (\n", - " SELECT * FROM (\n", - " SELECT *, ROW_NUMBER() OVER (PARTITION BY path ORDER BY path) AS rn\n", - " FROM tiff_results\n", - " ) WHERE rn = 1\n", - ") AS src\n", - " ON tgt.path = src.path\n", - "WHEN MATCHED THEN UPDATE SET\n", - " tgt.output_file_paths = array(src.tiff_output_path),\n", - " tgt.label_image_path = COALESCE(src.label_image_path, tgt.label_image_path),\n", - " tgt.macro_image_path = COALESCE(src.macro_image_path, tgt.macro_image_path),\n", - " tgt.phi_tags_redacted = src.phi_tags_redacted,\n", - " tgt.pixel_redactions_count = src.pixel_regions_redacted,\n", - " tgt.status = CASE WHEN src.error IS NULL THEN 'SUCCESS' ELSE 'FAILED' END,\n", - " tgt.error_messages = CASE WHEN src.error IS NOT NULL THEN array(src.error) ELSE NULL END,\n", - " tgt.update_timestamp = current_timestamp()\n", - "\"\"\")\n", - "print(\"TIFF write complete. Redaction records updated.\")" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "application/vnd.databricks.v1+cell": { - "cellMetadata": { - "byteLimit": 2048000, - "rowLimit": 10000 - }, - "inputWidgets": {}, - "nuid": "4ec2aac1-43ab-42ee-aae7-0fdf52c006ec", - "showTitle": true, - "tableResultSettingsMap": {}, - "title": "Cell 11: Audit summary" - } - }, - "outputs": [], - "source": [ - "# End-to-end audit: join object_catalog with object_catalog_redaction and summarise.\n", - "audit_df = spark.sql(f\"\"\"\n", - "SELECT\n", - " regexp_extract(o.path, '[^/]+$', 0) AS filename,\n", - " o.meta:width::int AS width_px,\n", - " o.meta:height::int AS height_px,\n", - " o.meta:level_count::int AS pyramid_levels,\n", - " r.has_phi,\n", - " r.status,\n", - " r.pixel_redactions_count,\n", - " size(r.phi_tags_redacted) AS tag_redactions,\n", - " r.output_file_paths[0] AS tiff_output_path,\n", - " r.label_image_path,\n", - " r.error_messages[0] AS error\n", - "FROM {CATALOG}.{SCHEMA}.object_catalog o\n", - "LEFT JOIN {CATALOG}.{SCHEMA}.object_catalog_redaction r\n", - " ON o.path = r.path\n", - "ORDER BY filename\n", - "\"\"\")\n", - "\n", - "total = audit_df.count()\n", - "phi_ct = audit_df.filter(\"has_phi = true\").count()\n", - "ok_ct = audit_df.filter(\"status = 'SUCCESS'\").count()\n", - "err_ct = audit_df.filter(\"status = 'FAILED'\").count()\n", - "\n", - "print(f\"Slides in catalog : {total}\")\n", - "print(f\"VLM-flagged with PHI : {phi_ct}\")\n", - "print(f\"Successfully written : {ok_ct}\")\n", - "print(f\"Errors : {err_ct}\")\n", - "\n", - "display(audit_df)\n" - ] - } - ], - "metadata": { - "application/vnd.databricks.v1+notebook": { - "computePreferences": null, - "dashboards": [], - "environmentMetadata": { - "base_environment": "", - "environment_version": "5" - }, - "inputWidgetPreferences": null, - "language": "python", - "notebookMetadata": { - "mostRecentlyExecutedCommandWithImplicitDF": { - "commandId": 8994411946469750, - "dataframes": [ - "_sqldf" - ] - }, - "pythonIndentUnit": 2 - }, - "notebookName": "TIFF Pathology De-identification Pipeline", - "widgets": {} - }, - "language_info": { - "name": "python" - } - }, - "nbformat": 4, - "nbformat_minor": 0 -} diff --git a/notebooks/tiff/TIFF sample data.ipynb b/notebooks/tiff/TIFF sample data.ipynb deleted file mode 100644 index c2973c4f..00000000 --- a/notebooks/tiff/TIFF sample data.ipynb +++ /dev/null @@ -1,980 +0,0 @@ -{ - "cells": [ - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "application/vnd.databricks.v1+cell": { - "cellMetadata": { - "byteLimit": 2048000, - "rowLimit": 10000 - }, - "inputWidgets": {}, - "nuid": "86734738-8782-422b-9f72-1620460e8964", - "showTitle": true, - "tableResultSettingsMap": {}, - "title": "Install dependencies" - } - }, - "outputs": [], - "source": [ - "%pip install openslide-python openslide-bin tifffile imagecodecs -q" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "application/vnd.databricks.v1+cell": { - "cellMetadata": { - "byteLimit": 2048000, - "rowLimit": 10000 - }, - "inputWidgets": {}, - "nuid": "153c85b3-e046-4fda-b9e3-e54aca9c45a0", - "showTitle": true, - "tableResultSettingsMap": {}, - "title": "Configure" - } - }, - "outputs": [], - "source": [ - "# ── Shared utilities — run after pip install, before any other cell ──────────\n", - "import glob, os\n", - "import openslide\n", - "from PIL import Image\n", - "import matplotlib.pyplot as plt\n", - "\n", - "Image.MAX_IMAGE_PIXELS = None # safe — we read metadata or small sub-images only\n", - "\n", - "SOURCE = \"/Volumes/hls_pathology/osuwmc/sample\"\n", - "EXT = \".tiff\"\n", - "\n", - "\n", - "def discover_tiff_files(source=SOURCE, ext=EXT, synthetic=True):\n", - " \"\"\"Return sorted list of TIFF paths under *source*.\n", - " Pass synthetic=False to exclude files whose name contains '__synthetic'.\n", - " \"\"\"\n", - " files = sorted(set(\n", - " glob.glob(f\"{source}/**/*{ext}\", recursive=True) +\n", - " glob.glob(f\"{source}/*{ext}\")\n", - " ))\n", - " if not synthetic:\n", - " files = [f for f in files if \"__synthetic\" not in f]\n", - " return files\n", - "\n", - "\n", - "def to_rgb(img):\n", - " \"\"\"Flatten RGBA / palette PIL images to RGB on a white background.\"\"\"\n", - " if img.mode == \"RGBA\":\n", - " bg = Image.new(\"RGB\", img.size, (255, 255, 255))\n", - " bg.paste(img, mask=img.split()[3])\n", - " return bg\n", - " return img.convert(\"RGB\")\n", - "\n", - "\n", - "def fit_to(img, max_w=800, max_h=600):\n", - " \"\"\"Downsample *img* to fit within (max_w, max_h), preserving aspect ratio.\n", - " Never upscales.\"\"\"\n", - " w, h = img.size\n", - " scale = min(max_w / w, max_h / h, 1.0)\n", - " if scale < 1.0:\n", - " return img.resize((int(w * scale), int(h * scale)), Image.LANCZOS)\n", - " return img\n", - "\n", - "\n", - "def show_slide_layers(fpath, thumb_w=800, thumb_h=600, max_native=4_000_000):\n", - " \"\"\"Display all pyramid levels + associated images of a slide as a panel row.\"\"\"\n", - " fname = os.path.basename(fpath)\n", - " try:\n", - " slide = openslide.OpenSlide(fpath)\n", - " except Exception as exc:\n", - " print(f\" \\u26a0 Cannot open {fname}: {exc}\")\n", - " return\n", - " layers = []\n", - " mw, mh = slide.dimensions\n", - " layers.append({\n", - " \"title\": f\"main\\n{mw:,} \\u00d7 {mh:,} px\",\n", - " \"img\": slide.get_thumbnail((thumb_w, thumb_h)),\n", - " \"orig\": (mw, mh),\n", - " })\n", - " for lvl in range(1, slide.level_count):\n", - " lw, lh = slide.level_dimensions[lvl]\n", - " ds = slide.level_downsamples[lvl]\n", - " if lw * lh <= max_native:\n", - " img = slide.read_region((0, 0), lvl, (lw, lh))\n", - " else:\n", - " scale = min(thumb_w / lw, thumb_h / lh)\n", - " img = slide.get_thumbnail((int(lw * scale), int(lh * scale)))\n", - " layers.append({\n", - " \"title\": f\"level {lvl} (\\u00d7{ds:.0f}\\u2193)\\n{lw:,} \\u00d7 {lh:,} px\",\n", - " \"img\": img,\n", - " \"orig\": (lw, lh),\n", - " })\n", - " for name in sorted(slide.associated_images.keys()):\n", - " img = slide.associated_images[name]\n", - " aw, ah = img.size\n", - " layers.append({\"title\": f\"{name}\\n{aw} \\u00d7 {ah} px\", \"img\": img, \"orig\": (aw, ah)})\n", - " slide.close()\n", - " n = len(layers)\n", - " fig, axes = plt.subplots(1, n, figsize=(min(5 * n, 24), 4.5),\n", - " gridspec_kw={\"wspace\": 0.06})\n", - " if n == 1:\n", - " axes = [axes]\n", - " for ax, layer in zip(axes, layers):\n", - " disp = to_rgb(fit_to(layer[\"img\"], thumb_w, thumb_h))\n", - " dw, dh = disp.size\n", - " ax.imshow(disp)\n", - " ax.set_title(layer[\"title\"], fontsize=8, fontweight=\"bold\", pad=3, linespacing=1.4)\n", - " ax.set_xlabel(f\"displayed {dw}\\u00d7{dh}\", fontsize=7, labelpad=2)\n", - " ax.tick_params(left=False, bottom=False, labelleft=False, labelbottom=False)\n", - " for spine in ax.spines.values():\n", - " spine.set_linewidth(0.5)\n", - " fig.suptitle(fname, fontsize=10, fontweight=\"bold\", y=1.02)\n", - " plt.tight_layout()\n", - " plt.show()\n", - "\n", - "\n", - "def print_slide_metadata(fpath):\n", - " \"\"\"Print pyramid levels, associated images, and all openslide properties.\"\"\"\n", - " slide = openslide.OpenSlide(fpath)\n", - " print(f\"\\n Pyramid levels : {slide.level_count}\")\n", - " for lvl in range(slide.level_count):\n", - " lw, lh = slide.level_dimensions[lvl]\n", - " print(f\" [{lvl}] {lw:,} \\u00d7 {lh:,} px \"\n", - " f\"(downsample \\u00d7{slide.level_downsamples[lvl]:.2f})\")\n", - " assoc = sorted(slide.associated_images.keys())\n", - " print(f\"\\n Associated images : {assoc if assoc else '\\u2014'}\")\n", - " print(f\"\\n Properties ({len(slide.properties)}):\")\n", - " for k, v in sorted(slide.properties.items()):\n", - " print(f\" {k} = {v}\")\n", - " slide.close()\n", - "\n", - "\n", - "# Discover on load so tiff_files is available to all downstream cells\n", - "tiff_files = discover_tiff_files()\n", - "print(f\"Utilities loaded. {len(tiff_files)} {EXT} file(s) under {SOURCE}\")" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "application/vnd.databricks.v1+cell": { - "cellMetadata": { - "byteLimit": 2048000, - "rowLimit": 10000 - }, - "inputWidgets": {}, - "nuid": "e44640d5-5d8f-4e41-8afd-15e704984ab3", - "showTitle": true, - "tableResultSettingsMap": {}, - "title": "Display all TIFF layers" - } - }, - "outputs": [], - "source": [ - "# Uses show_slide_layers + print_slide_metadata from the Shared utilities cell.\n", - "# Shows original source TIFFs only; synthetic variants are shown in cell 8.\n", - "for fpath in discover_tiff_files(synthetic=False):\n", - " print(f\"{'─' * 72}\\n{os.path.basename(fpath)}\")\n", - " show_slide_layers(fpath)\n", - " print_slide_metadata(fpath)\n", - " print()\n" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "application/vnd.databricks.v1+cell": { - "cellMetadata": { - "byteLimit": 2048000, - "rowLimit": 10000 - }, - "inputWidgets": {}, - "nuid": "87ad871a-069e-47f2-b353-5e4698dd4892", - "showTitle": true, - "tableResultSettingsMap": {}, - "title": "Extract & save JPEGs — Philips07" - } - }, - "outputs": [], - "source": [ - "# 3rd TIFF file — display each layer large and save JPEGs back to the same volume.\n", - "import os\n", - "import openslide\n", - "from PIL import Image\n", - "import matplotlib.pyplot as plt\n", - "\n", - "TARGET_FILE_IDX = 2 # Philips07\n", - "MAX_NATIVE_DIM = 10_000 # read full level if max(w,h) ≤ this; else thumbnail\n", - "JPEG_THUMB_W = 4096 # max width when level must be thumbnailed\n", - "JPEG_QUALITY = 90\n", - "\n", - "\n", - "fpath = tiff_files[TARGET_FILE_IDX]\n", - "fname = os.path.basename(fpath)\n", - "fdir = os.path.dirname(fpath)\n", - "stem = os.path.splitext(fname)[0]\n", - "print(f\"Source : {fpath}\")\n", - "print(f\"Output : {fdir}\")\n", - "print(f\"Stem : {stem}\\n\")\n", - "\n", - "slide = openslide.OpenSlide(fpath)\n", - "\n", - "# ── collect layers ────────────────────────────────────────────────────────────\n", - "layers = {}\n", - "\n", - "# main: high-res thumbnail from full pyramid\n", - "mw, mh = slide.dimensions\n", - "scale_main = JPEG_THUMB_W / max(mw, mh)\n", - "layers[\"main\"] = {\n", - " \"img\" : slide.get_thumbnail((int(mw * scale_main), int(mh * scale_main))),\n", - " \"orig\" : (mw, mh),\n", - " \"label\": f\"main (level 0)\\nfull: {mw:,} × {mh:,} px\",\n", - "}\n", - "\n", - "# pyramid levels 1 – N\n", - "for lvl in range(1, slide.level_count):\n", - " lw, lh = slide.level_dimensions[lvl]\n", - " ds = slide.level_downsamples[lvl]\n", - " key = f\"level_{lvl}\"\n", - " if max(lw, lh) <= MAX_NATIVE_DIM: # safe to read entirely\n", - " img = slide.read_region((0, 0), lvl, (lw, lh))\n", - " else: # too large — thumbnail\n", - " sc = JPEG_THUMB_W / max(lw, lh)\n", - " img = slide.get_thumbnail((int(lw * sc), int(lh * sc)))\n", - " layers[key] = {\n", - " \"img\" : img,\n", - " \"orig\" : (lw, lh),\n", - " \"label\": f\"level {lvl} (×{ds:.0f}↓)\\n{lw:,} × {lh:,} px\",\n", - " }\n", - "\n", - "# associated sub-images (label / macro / thumbnail, if present)\n", - "for name in sorted(slide.associated_images.keys()):\n", - " img = slide.associated_images[name]\n", - " aw, ah = img.size\n", - " layers[f\"assoc_{name}\"] = {\n", - " \"img\" : img,\n", - " \"orig\" : (aw, ah),\n", - " \"label\": f\"{name}\\n{aw} × {ah} px\",\n", - " }\n", - "\n", - "slide.close()\n", - "\n", - "# ── display: one large figure per layer ──────────────────────────────────────\n", - "for key, layer in layers.items():\n", - " rgb = to_rgb(layer[\"img\"])\n", - " w, h = rgb.size\n", - " fig_w = 14\n", - " fig_h = fig_w * h / w\n", - " fig, ax = plt.subplots(figsize=(fig_w, fig_h))\n", - " ax.imshow(rgb)\n", - " ax.set_title(\n", - " f\"{fname} — {layer['label']}\",\n", - " fontsize=11, fontweight=\"bold\", pad=6, linespacing=1.5,\n", - " )\n", - " ax.set_xlabel(\n", - " f\"extracted at {w:,} × {h:,} px | original: {layer['orig'][0]:,} × {layer['orig'][1]:,} px\",\n", - " fontsize=9,\n", - " )\n", - " ax.tick_params(left=False, bottom=False, labelleft=False, labelbottom=False)\n", - " plt.tight_layout()\n", - " plt.show()\n", - "\n", - "# ── save JPEGs ────────────────────────────────────────────────────────────────\n", - "print(f\"\\nSaving JPEGs to {fdir}\\n\")\n", - "for key, layer in layers.items():\n", - " rgb = to_rgb(layer[\"img\"])\n", - " jpg_name = f\"{stem}__{key}.jpg\" # e.g. Philips07_3b946eed-...__level_3.jpg\n", - " jpg_path = os.path.join(fdir, jpg_name)\n", - " rgb.save(jpg_path, \"JPEG\", quality=JPEG_QUALITY)\n", - " ow, oh = layer[\"orig\"]\n", - " jw, jh = rgb.size\n", - " print(f\" {jpg_name}\")\n", - " print(f\" orig {ow:,}×{oh:,} → saved {jw:,}×{jh:,} ({jpg_path})\")\n", - "\n", - "print(f\"\\n{len(layers)} JPEG(s) written.\")\n" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "application/vnd.databricks.v1+cell": { - "cellMetadata": { - "byteLimit": 2048000, - "rowLimit": 10000 - }, - "inputWidgets": {}, - "nuid": "5c2565bb-fe30-412a-aa27-1554fb800a1d", - "showTitle": true, - "tableResultSettingsMap": {}, - "title": "Blood vessel zoom — Philips07 level 6" - } - }, - "outputs": [], - "source": [ - "import os\n", - "import openslide\n", - "from PIL import Image\n", - "import matplotlib.pyplot as plt\n", - "import matplotlib.patches as patches\n", - "\n", - "fpath = tiff_files[2] # Philips07\n", - "fname = os.path.basename(fpath)\n", - "slide = openslide.OpenSlide(fpath)\n", - "\n", - "# ── blood vessel location estimated from level-6 visual (1,440 × 800 px) ─────\n", - "# Bright pink/magenta spot, lower-centre of tissue. Adjust if needed.\n", - "VESSEL_X_L6 = 620 # centre x in level-6 pixels\n", - "VESSEL_Y_L6 = 540 # centre y in level-6 pixels\n", - "CROP_R_L6 = 160 # half-side of crop square in level-6 pixels\n", - "\n", - "DS = {lvl: int(slide.level_downsamples[lvl]) for lvl in range(slide.level_count)}\n", - "\n", - "# ── crop box in level-6 coordinates ─────────────────────────────────────────\n", - "l6_x0 = max(0, VESSEL_X_L6 - CROP_R_L6)\n", - "l6_y0 = max(0, VESSEL_Y_L6 - CROP_R_L6)\n", - "l6_w = min(CROP_R_L6 * 2, slide.level_dimensions[6][0] - l6_x0)\n", - "l6_h = min(CROP_R_L6 * 2, slide.level_dimensions[6][1] - l6_y0)\n", - "\n", - "# level-0 origin used by read_region for every pyramid level\n", - "loc0 = (l6_x0 * DS[6], l6_y0 * DS[6])\n", - "\n", - "# ── read four views of the same region ─────────────────────────────────────────\n", - "lvl6_full = slide.read_region((0, 0), 6, slide.level_dimensions[6]).convert(\"RGB\")\n", - "lvl6_crop = slide.read_region(loc0, 6, (l6_w, l6_h)).convert(\"RGB\")\n", - "\n", - "# level 4 (×16 downsample) — 4× more detail than level 6\n", - "l4_w = l6_w * DS[6] // DS[4]\n", - "l4_h = l6_h * DS[6] // DS[4]\n", - "lvl4_crop = slide.read_region(loc0, 4, (l4_w, l4_h)).convert(\"RGB\")\n", - "\n", - "# level 2 (×4 downsample) — 16× more detail than level 6\n", - "l2_w = l6_w * DS[6] // DS[2]\n", - "l2_h = l6_h * DS[6] // DS[2]\n", - "lvl2_crop = slide.read_region(loc0, 2, (l2_w, l2_h)).convert(\"RGB\")\n", - "\n", - "slide.close()\n", - "\n", - "# ── display ──────────────────────────────────────────────────────────────────\n", - "fig = plt.figure(figsize=(22, 7))\n", - "gs = fig.add_gridspec(1, 4, wspace=0.05)\n", - "axes = [fig.add_subplot(gs[i]) for i in range(4)]\n", - "\n", - "# Panel 1: level-6 overview with yellow crop-box annotation\n", - "axes[0].imshow(lvl6_full)\n", - "axes[0].add_patch(patches.Rectangle(\n", - " (l6_x0, l6_y0), l6_w, l6_h,\n", - " linewidth=2.5, edgecolor=\"yellow\", facecolor=\"none\",\n", - "))\n", - "axes[0].set_title(\"Level 6 — overview\\n1,440 × 800 px (×64↓)\",\n", - " fontsize=9, fontweight=\"bold\", pad=4)\n", - "axes[0].set_xlabel(\"yellow = crop region\", fontsize=7)\n", - "\n", - "# Panels 2-4: progressive zoom\n", - "for ax, img, lvl, label in [\n", - " (axes[1], lvl6_crop, 6, \"Level 6 crop\"),\n", - " (axes[2], lvl4_crop, 4, \"Level 4 — ×4 zoom\"),\n", - " (axes[3], lvl2_crop, 2, \"Level 2 — ×16 zoom\"),\n", - "]:\n", - " w, h = img.size\n", - " ax.imshow(img)\n", - " ax.set_title(\n", - " f\"{label}\\n{w:,} × {h:,} px (×{DS[lvl]}↓)\",\n", - " fontsize=9, fontweight=\"bold\", pad=4,\n", - " )\n", - " ax.set_xlabel(f\"origin L0: ({loc0[0]:,}, {loc0[1]:,})\", fontsize=7)\n", - "\n", - "for ax in axes:\n", - " ax.tick_params(left=False, bottom=False, labelleft=False, labelbottom=False)\n", - "\n", - "fig.suptitle(f\"{fname} — blood vessel zoom\",\n", - " fontsize=11, fontweight=\"bold\", y=1.01)\n", - "plt.tight_layout()\n", - "plt.show()\n", - "\n", - "print(f\"Level-0 origin : {loc0}\")\n", - "print(f\"Level 6 crop : {l6_w} × {l6_h} px\")\n", - "print(f\"Level 4 crop : {l4_w} × {l4_h} px (×4 more detail)\")\n", - "print(f\"Level 2 crop : {l2_w} × {l2_h} px (×16 more detail)\")\n" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "application/vnd.databricks.v1+cell": { - "cellMetadata": { - "byteLimit": 2048000, - "rowLimit": 10000 - }, - "inputWidgets": {}, - "nuid": "e4be9e45-e76a-4d51-887d-449ecf33df5f", - "showTitle": true, - "tableResultSettingsMap": {}, - "title": "Enumerate raw TIFF tags — all files" - } - }, - "outputs": [], - "source": [ - "from PIL.TiffTags import TAGS # to_rgb, discover_tiff_files, tiff_files from Shared utilities\n", - "\n", - "# Tags that could carry PHI\n", - "PHI_CANDIDATES = {\n", - " 270: \"ImageDescription\",\n", - " 305: \"Software\",\n", - " 315: \"Artist\",\n", - " 316: \"HostComputer\",\n", - " 33432: \"Copyright\",\n", - " 37510: \"UserComment\",\n", - " 40092: \"XPComment\",\n", - " 40094: \"XPKeywords\",\n", - " 40095: \"XPSubject\",\n", - "}\n", - "\n", - "for fpath in tiff_files:\n", - " fname = os.path.basename(fpath)\n", - " print(f\"\\n{'\\u2550' * 72}\")\n", - " print(f\"{fname}\")\n", - " print(f\"{'\\u2500' * 72}\")\n", - "\n", - " try:\n", - " img = Image.open(fpath)\n", - " n_frames = getattr(img, \"n_frames\", 1)\n", - " print(f\" IFDs (frames): {n_frames}\")\n", - "\n", - " # ── iterate all IFDs, deduplicate by tag-code signature ─────────────\n", - " sig_map = {}\n", - " for fi in range(n_frames):\n", - " try:\n", - " img.seek(fi)\n", - " except EOFError:\n", - " break\n", - " tags = dict(getattr(img, \"tag_v2\", {}))\n", - " sig = tuple(sorted(tags.keys()))\n", - " if sig not in sig_map:\n", - " sig_map[sig] = {\"frames\": [], \"sample\": tags}\n", - " sig_map[sig][\"frames\"].append(fi)\n", - "\n", - " # ── also expose SubIFDs (tag 330) from IFD 0 if present ─────────\n", - " img.seek(0)\n", - " subifd_tag = getattr(img, \"tag_v2\", {}).get(330)\n", - " if subifd_tag:\n", - " print(f\" SubIFDs (tag 330): {subifd_tag}\")\n", - "\n", - " img.close()\n", - "\n", - " # ── print each unique tag group ──────────────────────────────\n", - " for sig, info in sig_map.items():\n", - " frames = info[\"frames\"]\n", - " if len(frames) == 1:\n", - " f_label = f\"IFD {frames[0]}\"\n", - " elif frames == list(range(frames[0], frames[-1] + 1)):\n", - " f_label = f\"IFD {frames[0]}\\u2013{frames[-1]} ({len(frames)} frames)\"\n", - " else:\n", - " f_label = f\"{len(frames)} IFDs (non-contiguous)\"\n", - "\n", - " tags = info[\"sample\"]\n", - " print(f\"\\n \\u2500\\u2500 {f_label} ({len(sig)} tags) \\u2500\\u2500\")\n", - "\n", - " for code in sorted(tags.keys()):\n", - " name = TAGS.get(code, f\"Unknown_{code}\")\n", - " val = tags[code]\n", - "\n", - " if isinstance(val, bytes):\n", - " try:\n", - " val_str = val.decode(\"utf-8\", errors=\"replace\").strip()\n", - " except Exception:\n", - " val_str = f\"\"\n", - " elif isinstance(val, (tuple, list)) and len(val) > 8:\n", - " val_str = f\"{type(val).__name__}[{len(val)}] {repr(val[:4])} \\u2026\"\n", - " else:\n", - " val_str = repr(val)\n", - "\n", - " if len(val_str) > 300:\n", - " val_str = val_str[:300] + \" \\u2026\"\n", - "\n", - " phi = \" \\u26a0\\ufe0f PHI?\" if code in PHI_CANDIDATES else \"\"\n", - " print(f\" {code:6d} {name:<40s} {val_str}{phi}\")\n", - "\n", - " except Exception as exc:\n", - " print(f\" ERROR: {exc}\")\n", - "\n", - "print(\"\\nDone.\")\n" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "application/vnd.databricks.v1+cell": { - "cellMetadata": { - "byteLimit": 2048000, - "rowLimit": 10000 - }, - "inputWidgets": {}, - "nuid": "7ae34572-ed5a-4e29-a6f0-928ae7a064ac", - "showTitle": true, - "tableResultSettingsMap": {}, - "title": "Inject synthetic PHI — create test TIFFs" - } - }, - "outputs": [], - "source": [ - "# Writes one *__synthetic_phi.tiff per source file, containing:\n", - "# - Aperio-style ImageDescription (tag 270) with fake patient PHI\n", - "# - Software / Artist / HostComputer tags\n", - "# - Rendered label sub-image with visible PHI text + barcode\n", - "# - Macro sub-image (tissue thumbnail)\n", - "# Pyramid is built from source levels 5-8 (max 2,880 x 1,600) for speed.\n", - "# NOTE: tifffile requires random-access seeks; UC volumes don't support them.\n", - "# Strategy: write to /tmp, then shutil.copy2 to the volume.\n", - "\n", - "import random, shutil, tempfile\n", - "import tifffile\n", - "import numpy as np\n", - "from PIL import ImageDraw # to_rgb, discover_tiff_files, tiff_files from Shared utilities\n", - "\n", - "# ── synthetic patient records ────────────────────────────────────────────────\n", - "SYNTHETIC_PATIENTS = [\n", - " {\"Patient\": \"Smith, John A\", \"DOB\": \"1965-03-22\", \"MRN\": \"87654321\",\n", - " \"AccessionNumber\": \"ACC-2023-001\", \"Clinic\": \"Oncology\",\n", - " \"Pathologist\": \"Dr. Jane Doe\", \"User\": \"jdoe\"},\n", - " {\"Patient\": \"Johnson, Mary B\", \"DOB\": \"1978-11-15\", \"MRN\": \"12345678\",\n", - " \"AccessionNumber\": \"ACC-2023-002\", \"Clinic\": \"Pathology\",\n", - " \"Pathologist\": \"Dr. Robert Chen\", \"User\": \"rchen\"},\n", - " {\"Patient\": \"Williams, David C\", \"DOB\": \"1952-07-04\", \"MRN\": \"99887766\",\n", - " \"AccessionNumber\": \"ACC-2023-003\", \"Clinic\": \"Surgical\",\n", - " \"Pathologist\": \"Dr. Sarah Kim\", \"User\": \"skim\"},\n", - "]\n", - "\n", - "COPY_LEVELS = [5, 6, 7, 8] # 2,880×1,600 → 360×200 — fast to read\n", - "\n", - "\n", - "def make_aperio_description(phi: dict, stem: str) -> str:\n", - " \"\"\"Aperio pipe-delimited ImageDescription with embedded PHI (tag 270).\"\"\"\n", - " header = \"Aperio Image Library v12.1.0\\r\\n[0,0 92160x51200] (512x512) JPEG/YCC Q=80\"\n", - " fields = {\"Filename\": stem, \"Date\": \"2023-04-15\", \"Time\": \"10:23:45\", **phi}\n", - " body = \"|\".join(f\"{k} = {v}\" for k, v in fields.items())\n", - " return f\"{header}||{body}\"\n", - "\n", - "\n", - "def make_label_image(phi: dict, size=(387, 463)) -> np.ndarray:\n", - " \"\"\"Render a patient-sticker image with clearly legible PHI text.\"\"\"\n", - " img = Image.new(\"RGB\", size, (255, 255, 255))\n", - " draw = ImageDraw.Draw(img)\n", - " draw.rectangle([2, 2, size[0]-3, size[1]-3], outline=(0, 0, 0), width=2)\n", - " lines = [\n", - " f\"Patient : {phi['Patient']}\",\n", - " f\"DOB : {phi['DOB']}\",\n", - " f\"MRN : {phi['MRN']}\",\n", - " f\"Accn : {phi['AccessionNumber']}\",\n", - " f\"Clinic : {phi['Clinic']}\",\n", - " f\"Path : {phi['Pathologist']}\",\n", - " f\"Date : 2023-04-15\",\n", - " f\"User : {phi['User']}\",\n", - " \"\",\n", - " \"** SYNTHETIC PHI — TEST ONLY **\",\n", - " \"** NOT A REAL PATIENT **\",\n", - " ]\n", - " y = 16\n", - " for line in lines:\n", - " draw.text((10, y), line, fill=(0, 0, 0))\n", - " y += 34\n", - " # Fake barcode strip\n", - " random.seed(42)\n", - " for x in range(10, size[0]-10, 3):\n", - " h = random.randint(15, 50)\n", - " draw.rectangle([x, size[1]-70, x+1, size[1]-70+h], fill=(0, 0, 0))\n", - " return np.array(img)\n", - "\n", - "\n", - "# ── write one synthetic TIFF per source file ──────────────────────────────────────\n", - "for fi, fpath in enumerate(tiff_files):\n", - " # skip files that are already synthetic\n", - " if \"__synthetic_phi\" in fpath:\n", - " continue\n", - "\n", - " phi = SYNTHETIC_PATIENTS[fi % len(SYNTHETIC_PATIENTS)]\n", - " stem = os.path.splitext(os.path.basename(fpath))[0]\n", - " out_vol = os.path.join(os.path.dirname(fpath), f\"{stem}__synthetic_phi.tiff\")\n", - " tmp_out = os.path.join(tempfile.gettempdir(), f\"{stem}__synthetic_phi.tiff\")\n", - "\n", - " print(f\"\\n{'\\u2500'*72}\")\n", - " print(f\" source : {os.path.basename(fpath)}\")\n", - " print(f\" subject : {phi['Patient']} MRN={phi['MRN']}\")\n", - " print(f\" output : {out_vol}\")\n", - "\n", - " slide = openslide.OpenSlide(fpath)\n", - " pyramid = []\n", - " for lvl in COPY_LEVELS:\n", - " if lvl >= slide.level_count:\n", - " continue\n", - " lw, lh = slide.level_dimensions[lvl]\n", - " arr = np.array(slide.read_region((0, 0), lvl, (lw, lh)).convert(\"RGB\"))\n", - " pyramid.append((lvl, arr, lw, lh))\n", - " print(f\" level {lvl}: {lw}\\u00d7{lh}\")\n", - " slide.close()\n", - "\n", - " if not pyramid:\n", - " print(\" \\u26a0 no levels — skipping\")\n", - " continue\n", - "\n", - " image_desc = make_aperio_description(phi, stem)\n", - " label_arr = make_label_image(phi)\n", - " macro_sm = np.array(\n", - " Image.fromarray(pyramid[0][1]).resize((640, 356), Image.LANCZOS)\n", - " )\n", - " # metadata=None disables tifffile's auto-JSON shape override so our\n", - " # description= and extratags= values are written to tag 270 unchanged.\n", - " _write = dict(photometric=\"rgb\", tile=(512, 512),\n", - " compression=\"deflate\", compressionargs={\"level\": 6},\n", - " metadata=None)\n", - "\n", - " with tifffile.TiffWriter(tmp_out, bigtiff=True) as tif:\n", - "\n", - " # IFD 0 — main image (level 5 of source) + all PHI tags\n", - " _, arr, _, _ = pyramid[0]\n", - " tif.write(\n", - " arr, **_write,\n", - " subfiletype=0,\n", - " description=image_desc, # tag 270 — Aperio-style PHI\n", - " software=\"Philips IntelliSite 3.0\", # tag 305\n", - " extratags=[\n", - " (315, 2, 0, phi[\"User\"], True), # Artist — PHI\n", - " (316, 2, 0, \"SCANNER-PHI-01\", True), # HostComputer\n", - " ],\n", - " )\n", - "\n", - " # IFDs 1–N — reduced-resolution pyramid\n", - " for _, arr, _, _ in pyramid[1:]:\n", - " tif.write(arr, **_write, subfiletype=1)\n", - "\n", - " # label sub-image — PHI visible in pixel content (VLM/OCR target)\n", - " tif.write(\n", - " label_arr,\n", - " photometric=\"rgb\", compression=\"deflate\",\n", - " compressionargs={\"level\": 6},\n", - " metadata=None, subfiletype=1, description=\"label\",\n", - " )\n", - "\n", - " # macro sub-image — tissue overview\n", - " tif.write(\n", - " macro_sm,\n", - " photometric=\"rgb\", compression=\"deflate\",\n", - " compressionargs={\"level\": 6},\n", - " metadata=None, subfiletype=1, description=\"macro\",\n", - " )\n", - "\n", - " shutil.copy2(tmp_out, out_vol)\n", - " os.remove(tmp_out)\n", - "\n", - " fsize = os.path.getsize(out_vol) / 1024 / 1024\n", - " print(f\" written {fsize:.1f} MB (deflate-compressed)\")\n", - " print(f\" desc : {image_desc[:120]} \\u2026\")\n", - "\n", - " # ── verify: re-open with openslide and confirm PHI is readable ───────────\n", - " try:\n", - " chk = openslide.OpenSlide(out_vol)\n", - " desc = chk.properties.get(\"openslide.comment\",\n", - " chk.properties.get(\"tiff.ImageDescription\", \"(none)\"))\n", - " assoc = sorted(chk.associated_images.keys())\n", - " print(f\" verify : vendor={chk.properties.get('openslide.vendor')} \"\n", - " f\"assoc={assoc} desc_len={len(desc)}\")\n", - " print(f\" PHI ok : Patient={('Patient' in desc)} \"\n", - " f\"MRN={('MRN' in desc)} User={('User' in desc)}\")\n", - " chk.close()\n", - " except Exception as exc:\n", - " print(f\" verify error: {exc}\")\n", - "\n", - "print(\"\\nSynthetic PHI TIFFs complete.\")\n" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "application/vnd.databricks.v1+cell": { - "cellMetadata": { - "byteLimit": 2048000, - "rowLimit": 10000 - }, - "inputWidgets": {}, - "nuid": "a99cb8f2-daaf-4412-900e-de06f51610ff", - "showTitle": true, - "tableResultSettingsMap": {}, - "title": "Display synthetic PHI TIFFs" - } - }, - "outputs": [], - "source": [ - "# Displays each synthetic PHI TIFF in full:\n", - "# 1. Pyramid layers panel (via show_slide_layers)\n", - "# 2. Full ImageDescription (tag 270) with PHI fields flagged ⚠️\n", - "# 3. label + macro sub-images read from PIL IFDs — the VLM / OCR target\n", - "# All shared helpers come from the Shared utilities cell.\n", - "from PIL.TiffTags import TAGS\n", - "\n", - "PHI_KEYS = [\n", - " \"Patient\", \"DOB\", \"MRN\", \"AccessionNumber\", \"Clinic\",\n", - " \"Pathologist\", \"Date\", \"Time\", \"User\", \"Filename\", \"ImageID\",\n", - "]\n", - "\n", - "synthetic_files = [f for f in discover_tiff_files() if \"__synthetic_phi\" in f]\n", - "print(f\"Found {len(synthetic_files)} synthetic PHI TIFF(s)\\n\")\n", - "\n", - "for fpath in synthetic_files:\n", - " fname = os.path.basename(fpath)\n", - " print(f\"\\n{'\\u2550' * 72}\\n{fname}\\n{'\\u2500' * 72}\")\n", - "\n", - " # ── 1. Pyramid layers (openslide) ─────────────────────────────────────────\n", - " show_slide_layers(fpath)\n", - "\n", - " # ── 2. ImageDescription via PIL tag_v2 (full string, not truncated) ────────\n", - " pil_img = Image.open(fpath)\n", - " pil_img.seek(0)\n", - " raw = dict(getattr(pil_img, \"tag_v2\", {})).get(270, b\"\")\n", - " desc = raw.decode(\"utf-8\", errors=\"replace\").strip(\"\\x00\") if isinstance(raw, bytes) else str(raw)\n", - "\n", - " print(f\"\\n ImageDescription ({len(desc)} chars):\")\n", - " for part in desc.replace(\"\\r\\n\", \"||\").split(\"|\"):\n", - " part = part.strip()\n", - " if not part:\n", - " continue\n", - " is_phi = any(k in part for k in PHI_KEYS)\n", - " marker = \" \\u26a0\\ufe0f PHI\" if is_phi else \"\"\n", - " print(f\" {part}{marker}\")\n", - "\n", - " # ── 3. label / macro sub-images (PIL IFD walk) ─────────────────────────\n", - " n_frames = getattr(pil_img, \"n_frames\", 1)\n", - " sub_imgs = []\n", - " for fi in range(n_frames):\n", - " try:\n", - " pil_img.seek(fi)\n", - " except EOFError:\n", - " break\n", - " ifd_desc = dict(getattr(pil_img, \"tag_v2\", {})).get(270, b\"\")\n", - " if isinstance(ifd_desc, bytes):\n", - " ifd_desc = ifd_desc.decode(\"utf-8\", errors=\"replace\").strip(\"\\x00\")\n", - " if ifd_desc in (\"label\", \"macro\"):\n", - " sub_imgs.append({\"name\": ifd_desc, \"img\": pil_img.copy()})\n", - " pil_img.close()\n", - "\n", - " if sub_imgs:\n", - " n = len(sub_imgs)\n", - " fig, axes = plt.subplots(1, n, figsize=(9 * n, 9))\n", - " if n == 1:\n", - " axes = [axes]\n", - " for ax, si in zip(axes, sub_imgs):\n", - " rgb = to_rgb(si[\"img\"])\n", - " w, h = rgb.size\n", - " ax.imshow(rgb)\n", - " ax.set_title(\n", - " f\"{si['name']} ({w} \\u00d7 {h} px)\",\n", - " fontsize=12, fontweight=\"bold\", pad=6,\n", - " )\n", - " ax.set_xlabel(\n", - " \"PHI rendered in pixels — VLM / OCR redaction target\",\n", - " fontsize=9,\n", - " )\n", - " ax.tick_params(left=False, bottom=False, labelleft=False, labelbottom=False)\n", - " fig.suptitle(\n", - " f\"{fname} — sub-images\",\n", - " fontsize=11, fontweight=\"bold\", y=1.01,\n", - " )\n", - " plt.tight_layout()\n", - " plt.show()\n", - " else:\n", - " print(\"\\n (no label/macro found via PIL IFD walk)\")\n", - " print()\n", - "\n", - "print(\"Done.\")\n" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "application/vnd.databricks.v1+cell": { - "cellMetadata": { - "byteLimit": 2048000, - "rowLimit": 10000 - }, - "inputWidgets": {}, - "nuid": "249f2889-656a-4aad-9fd0-35e0130d13e9", - "showTitle": true, - "tableResultSettingsMap": {}, - "title": "Compare Philips TIFF vs Aperio SVS" - } - }, - "outputs": [], - "source": [ - "# Side-by-side structural and metadata comparison between:\n", - "# - Philips BIG.tiff (OSUWMC) — metadata-bare\n", - "# - Aperio SVS (orthanc_demo) — rich PHI metadata + associated sub-images\n", - "# Also shows the sub-images from SVS that are absent in the Philips files.\n", - "\n", - "import glob, os\n", - "import openslide\n", - "from PIL import Image\n", - "import matplotlib.pyplot as plt\n", - "from PIL.TiffTags import TAGS\n", - "\n", - "# to_rgb, discover_tiff_files, tiff_files from Shared utilities\n", - "orig_tiffs = discover_tiff_files(synthetic=False)\n", - "\n", - "# Locate SVS files\n", - "svs_files = sorted(\n", - " glob.glob(\"/Volumes/hls_pathology/orthanc_demo/raw_images/Aperio/**/*.svs\",\n", - " recursive=True) +\n", - " glob.glob(\"/Volumes/hls_pathology/orthanc_demo/raw_images/Aperio/*.svs\")\n", - ")[:3] # first 3 for comparison\n", - "\n", - "print(f\"Philips TIFFs : {len(orig_tiffs)}\")\n", - "print(f\"Aperio SVS : {len(svs_files)}\")\n", - "\n", - "\n", - "def slide_profile(fpath):\n", - " \"\"\"Return a dict of key attributes for one openslide-readable file.\"\"\"\n", - " fname = os.path.basename(fpath)\n", - " try:\n", - " s = openslide.OpenSlide(fpath)\n", - " desc = s.properties.get(\"openslide.comment\",\n", - " s.properties.get(\"tiff.ImageDescription\", \"\"))\n", - " phi_keys = [k for k in\n", - " [\"Date\",\"Time\",\"User\",\"Filename\",\n", - " \"Patient\",\"DOB\",\"MRN\",\"AccessionNumber\",\n", - " \"Clinic\",\"Pathologist\",\"Procedure\",\"Diagnosis\"]\n", - " if k in desc]\n", - " profile = {\n", - " \"fname\" : fname,\n", - " \"vendor\" : s.properties.get(\"openslide.vendor\", \"?\"),\n", - " \"dims\" : s.dimensions,\n", - " \"levels\" : s.level_count,\n", - " \"mpp\" : s.properties.get(\"openslide.mpp-x\",\n", - " s.properties.get(\"aperio.MPP\", \"n/a\")),\n", - " \"n_props\" : len(s.properties),\n", - " \"desc_len\" : len(desc),\n", - " \"phi_keys\" : phi_keys,\n", - " \"associated\" : sorted(s.associated_images.keys()),\n", - " \"assoc_images\": {k: s.associated_images[k]\n", - " for k in s.associated_images.keys()},\n", - " \"properties\" : dict(s.properties),\n", - " \"slide\" : s,\n", - " }\n", - " return profile\n", - " except Exception as exc:\n", - " return {\"fname\": fname, \"error\": str(exc)}\n", - "\n", - "\n", - "print(\"\\nProfiling files …\")\n", - "svs_profiles = [slide_profile(f) for f in svs_files]\n", - "tiff_profiles = [slide_profile(f) for f in orig_tiffs]\n", - "\n", - "\n", - "# ── text comparison table ────────────────────────────────────────────────────────────\n", - "vs = svs_profiles[0] if svs_profiles else {}\n", - "vt = tiff_profiles[0] if tiff_profiles else {}\n", - "\n", - "print(f\"\\n{'\\u2550'*90}\")\n", - "print(f\"{'Attribute':<28} {'Aperio SVS':<30} {'Philips BIG.tiff':<30}\")\n", - "print(f\"{'\\u2500'*90}\")\n", - "rows = [\n", - " (\"File\", lambda p: p.get(\"fname\",\"?\")[:40]),\n", - " (\"openslide.vendor\",lambda p: p.get(\"vendor\",\"?\")),\n", - " (\"Full resolution\", lambda p: f\"{p['dims'][0]:,}\\u00d7{p['dims'][1]:,}\" if \"dims\" in p else \"?\"),\n", - " (\"Pyramid levels\", lambda p: str(p.get(\"levels\",\"?\"))),\n", - " (\"MPP (microns/px)\",lambda p: str(p.get(\"mpp\",\"?\"))),\n", - " (\"# openslide props\",lambda p: str(p.get(\"n_props\",\"?\"))),\n", - " (\"ImageDescription len\",lambda p: str(p.get(\"desc_len\",0)) + \" chars\"),\n", - " (\"PHI keys in desc\", lambda p: str(p.get(\"phi_keys\",[]))),\n", - " (\"Associated images\",lambda p: str(p.get(\"associated\",[]))),\n", - "]\n", - "for name, fn in rows:\n", - " sv = fn(vs) if vs else \"N/A\"\n", - " tv = fn(vt) if vt else \"N/A\"\n", - " flag = \" \\u26a0\\ufe0f\" if name == \"PHI keys in desc\" and sv and sv != \"[]\" else \"\"\n", - " print(f\" {name:<26} {sv:<30} {tv:<30}{flag}\")\n", - "print(f\"{'\\u2550'*90}\")\n", - "\n", - "\n", - "# ── print SVS ImageDescription (show PHI fields) ──────────────────────────────────\n", - "if vs and vs.get(\"desc_len\", 0) > 0:\n", - " print(f\"\\nSVS ImageDescription ({vs['fname']})\")\n", - " desc = vs[\"properties\"].get(\"openslide.comment\",\n", - " vs[\"properties\"].get(\"tiff.ImageDescription\",\"\"))\n", - " for part in desc.split(\"|\"):\n", - " part = part.strip()\n", - " if not part:\n", - " continue\n", - " is_phi = any(k in part for k in\n", - " [\"Date\",\"Time\",\"User\",\"Patient\",\"DOB\",\"MRN\",\n", - " \"Accession\",\"Clinic\",\"Pathologist\",\"Filename\",\"ImageID\"])\n", - " marker = \" \\u26a0\\ufe0f PHI\" if is_phi else \"\"\n", - " print(f\" {part}{marker}\")\n", - "\n", - "\n", - "# ── display SVS associated sub-images (absent in Philips) ────────────────────────\n", - "for prof in svs_profiles:\n", - " if \"error\" in prof or not prof.get(\"associated\"):\n", - " continue\n", - " assoc = prof[\"assoc_images\"]\n", - " n = len(assoc)\n", - " fig, axes = plt.subplots(1, n, figsize=(5 * n, 4))\n", - " if n == 1:\n", - " axes = [axes]\n", - " for ax, (name, img) in zip(axes, assoc.items()):\n", - " rgb = to_rgb(img)\n", - " ax.imshow(rgb)\n", - " ax.set_title(\n", - " f\"{name}\\n{rgb.size[0]}\\u00d7{rgb.size[1]} px\",\n", - " fontsize=9, fontweight=\"bold\", pad=3,\n", - " )\n", - " ax.tick_params(left=False, bottom=False,\n", - " labelleft=False, labelbottom=False)\n", - " fig.suptitle(\n", - " f\"SVS associated images — {prof['fname']}\\n\"\n", - " f\"(PHI risk: label barcode + printed text; absent in Philips TIFFs)\",\n", - " fontsize=9, fontweight=\"bold\",\n", - " )\n", - " plt.tight_layout()\n", - " plt.show()\n", - "\n", - "# close openslide handles\n", - "for p in svs_profiles + tiff_profiles:\n", - " if \"slide\" in p:\n", - " try:\n", - " p[\"slide\"].close()\n", - " except Exception:\n", - " pass\n", - "\n", - "print(\"\\nComparison complete.\")\n" - ] - } - ], - "metadata": { - "application/vnd.databricks.v1+notebook": { - "computePreferences": null, - "dashboards": [], - "environmentMetadata": { - "base_environment": "", - "environment_version": "5" - }, - "inputWidgetPreferences": null, - "language": "python", - "notebookMetadata": { - "pythonIndentUnit": 2 - }, - "notebookName": "TIFF sample data", - "widgets": {} - }, - "language_info": { - "name": "python" - } - }, - "nbformat": 4, - "nbformat_minor": 0 -} diff --git a/notebooks/tiff/config.yaml-example b/notebooks/tiff/config.yaml-example deleted file mode 100644 index 01a78c03..00000000 --- a/notebooks/tiff/config.yaml-example +++ /dev/null @@ -1,11 +0,0 @@ -SOURCE_PATH: /Volumes/hls_radiology_east//sample -PATTERN: "*.tiff" -INDEX: .tiff.object_catalog -PHI_ASSESSMENT_TABLE: .tiff.phi_assessment - -# MLflow -MLFLOW_EXPERIMENT_NAME: /Users//tiff -MLFLOW_ARTIFACT_PATH: /Volumes//tiff//mlflow - -# VLM -VLM_ENDPOINT: databricks-llama-4-maverick diff --git a/src/dbx/pixels/svs/__init__.py b/src/dbx/pixels/svs/__init__.py deleted file mode 100644 index 143b325f..00000000 --- a/src/dbx/pixels/svs/__init__.py +++ /dev/null @@ -1,43 +0,0 @@ -"""dbx.pixels.svs — SVS (Aperio Whole Slide Image) extension for databricks-pixels. - -Install alongside ``databricks-pixels`` and extend the ``dbx.pixels`` namespace:: - - import dbx.pixels - _SVS_SRC = "/Workspace/Users//svs-pixels/src/dbx/pixels" - if _SVS_SRC not in dbx.pixels.__path__: - dbx.pixels.__path__.append(_SVS_SRC) - - from dbx.pixels.svs import SVSCatalog, SVSMetaExtractor, SVSTiffWriter -""" - -from dbx.pixels.svs.catalog import SVSCatalog -from dbx.pixels.svs.svs_meta_extractor import SVSMetaExtractor -from dbx.pixels.svs.phi_tags import ( - classify_tag, - classify_tags, - scrub_image_description, - PHI_TAGS, - QUESTIONABLE_TAGS, - NOT_PHI_TAGS, -) - -# Lazy import: SVSTiffWriter depends on deidentify.py which may not be clean -# on all environments. Import on first access only. -def __getattr__(name): - if name == "SVSTiffWriter": - from dbx.pixels.svs.svs_tiff_writer import SVSTiffWriter - globals()["SVSTiffWriter"] = SVSTiffWriter - return SVSTiffWriter - raise AttributeError(f"module {__name__!r} has no attribute {name!r}") - -__all__ = [ - "SVSCatalog", - "SVSMetaExtractor", - "SVSTiffWriter", # lazy-loaded - "classify_tag", - "classify_tags", - "scrub_image_description", - "PHI_TAGS", - "QUESTIONABLE_TAGS", - "NOT_PHI_TAGS", -] diff --git a/src/dbx/pixels/svs/catalog.py b/src/dbx/pixels/svs/catalog.py deleted file mode 100644 index 4fdb188a..00000000 --- a/src/dbx/pixels/svs/catalog.py +++ /dev/null @@ -1,74 +0,0 @@ -"""SVSCatalog — extends the base Catalog for Aperio SVS whole-slide images. - -Overrides: - - catalog() defaults to pattern='*.svs' - - init_tables() calls super().init_tables() then executes SVS-specific DDL - (CREATE_SVS_CATALOG.sql creates object_catalog_redaction) -""" - -from __future__ import annotations - -from dbx.pixels.catalog import Catalog - - -class SVSCatalog(Catalog): - """Object catalog for Aperio SVS whole-slide images. - - Extends :class:`dbx.pixels.Catalog` with SVS-specific defaults: - - ``catalog()`` uses ``pattern='*.svs'`` - - ``init_tables()`` also creates the ``object_catalog_redaction`` table - via ``CREATE_SVS_CATALOG.sql`` - - Args: - spark: Active SparkSession. - table: Fully qualified UC table name (e.g. ``douglas_moore.pathology.object_catalog``). - volume: Fully qualified UC volume name (e.g. ``douglas_moore.pathology.pixels_volume``). - """ - - def __init__(self, spark, table: str, volume: str): - super().__init__(spark, table=table, volume=volume) - - def init_tables(self): - """Create base tables via parent, then run SVS-specific DDL.""" - # Base DDL creates object_catalog - super().init_tables() - - # SVS-specific: create object_catalog_redaction - from pathlib import Path - - sql_path = Path(__file__).parent / "resources" / "sql" - - try: - sql_files = { - p.name: p.read_text() - for p in sql_path.iterdir() - if p.suffix == ".sql" and p.is_file() - } - except (PermissionError, OSError): - # Serverless may block non-Python file reads; fall back to SDK - from databricks.sdk import WorkspaceClient - - ws_path = str(sql_path) - sql_files = {} - w = WorkspaceClient() - for obj in w.workspace.list(ws_path): - if obj.path and obj.path.endswith(".sql"): - name = obj.path.rsplit("/", 1)[-1] - with w.workspace.download(obj.path) as f: - sql_files[name] = f.read().decode("utf-8") - - for file_name, content in sql_files.items(): - sql_commands = content.replace("{UC_TABLE}", self._table).replace( - "{UC_SCHEMA}", self._schema - ) - for sql_command in sql_commands.split(";"): - if sql_command.strip(): - self._spark.sql(sql_command) - - def catalog(self, path: str, pattern: str = "*.svs", **kwargs): - """Catalog SVS files at the given path. - - Delegates to :meth:`Catalog.catalog` with ``pattern='*.svs'`` default. - All other keyword arguments are forwarded unchanged. - """ - return super().catalog(path, pattern=pattern, **kwargs) diff --git a/src/dbx/pixels/svs/deidentify.py b/src/dbx/pixels/svs/deidentify.py deleted file mode 100644 index f2ac794b..00000000 --- a/src/dbx/pixels/svs/deidentify.py +++ /dev/null @@ -1,224 +0,0 @@ -"""De-identification image utilities for SVS pathology slides. - -Functions for redacting PHI from label/macro sub-images and writing -de-identified pyramidal BigTIFF outputs. -""" - -import os -import numpy as np -from PIL import Image, ImageDraw - - -def redact_image(img: "Image.Image", phi_elements: list) -> "Image.Image": - """Apply black-rectangle redaction to detected PHI bounding boxes. - - Parameters - ---------- - img : PIL.Image.Image - The source image (label or macro sub-image). - phi_elements : list[dict] - Each dict must have a "bbox" key with {"x", "y", "w", "h"} in pixels. - - Returns - ------- - PIL.Image.Image - Copy of the input image with PHI regions blacked out. - """ - out = img.convert("RGB").copy() - draw = ImageDraw.Draw(out) - for elem in phi_elements: - bbox = elem.get("bbox") - if not bbox: - continue - x, y, w, h = bbox.get("x", 0), bbox.get("y", 0), bbox.get("w", 0), bbox.get("h", 0) - draw.rectangle([x, y, x + w, y + h], fill=(0, 0, 0)) - return out - - -def read_level_tiled( - slide: "openslide.OpenSlide", - level: int, - tile_size: int = 4096, -) -> np.ndarray: - """Read a full pyramid level by tiling to limit peak allocation. - - Allocates ONE numpy array for the entire level, then fills it tile-by-tile. - Peak memory = full level array + one tile. - - Parameters - ---------- - slide : openslide.OpenSlide - level : int - tile_size : int - Tile edge in pixels at the target level (default 4096). - - Returns - ------- - np.ndarray shape (H, W, 3) uint8 - """ - w, h = slide.level_dimensions[level] - ds = slide.level_downsamples[level] - arr = np.zeros((h, w, 3), dtype=np.uint8) - for y in range(0, h, tile_size): - for x in range(0, w, tile_size): - tw = min(tile_size, w - x) - th = min(tile_size, h - y) - # read_region always uses level-0 coordinates - loc = (int(x * ds), int(y * ds)) - tile = np.array( - slide.read_region(loc, level, (tw, th)).convert("RGB") - ) - arr[y : y + th, x : x + tw] = tile - return arr - - -def build_pyramid(base: np.ndarray, min_dim: int = 256) -> list: - """Build a Gaussian-style pyramid by 2x downsampling. - - Parameters - ---------- - base : np.ndarray (H, W, 3) uint8 - min_dim : int - Stop when both dimensions are below this threshold. - - Returns - ------- - list[np.ndarray] — level 0 is `base`; subsequent are 2x smaller. - """ - from PIL import Image as _PILImage - - levels = [base] - current = base - while min(current.shape[0], current.shape[1]) > min_dim: - h, w = current.shape[0] // 2, current.shape[1] // 2 - if h == 0 or w == 0: - break - pil = _PILImage.fromarray(current).resize((w, h), _PILImage.LANCZOS) - current = np.array(pil) - levels.append(current) - return levels - - -def write_pyramidal_bigtiff( - path: str, - levels: list, - tile_size: int = 256, - jpeg_quality: int = 80, -) -> None: - """Write a multi-resolution pyramidal BigTIFF from pre-built level arrays. - - Parameters - ---------- - path : str - Output file path. - levels : list[np.ndarray] - Pyramid levels (index 0 = full resolution). - tile_size : int - jpeg_quality : int - """ - import tifffile - - _parent = os.path.dirname(path) - if _parent and not _parent.startswith("/Volumes"): - os.makedirs(_parent, exist_ok=True) - - opts = dict( - tile=(tile_size, tile_size), - compression="jpeg", - compressionargs={"level": jpeg_quality}, - photometric="rgb", - metadata=None, - ) - with tifffile.TiffWriter(path, bigtiff=True) as tif: - for i, arr in enumerate(levels): - if i == 0: - tif.write( - arr, - subifds=len(levels) - 1 if len(levels) > 1 else 0, - **opts, - ) - else: - tif.write(arr, subfiletype=1, **opts) - - -def write_pyramidal_bigtiff_streaming( - path: str, - slide: "openslide.OpenSlide", - tile_size: int = 256, - jpeg_quality: int = 80, -) -> None: - """Write a pyramidal BigTIFF tile-by-tile directly from an OpenSlide handle. - - This function never holds more than ONE tile in memory (~196 KB for 256x256x3). - Suitable for 1M-scale processing where each worker has limited RAM (e.g. 1 GB - serverless UDF limit or constrained cluster workers). - - The output contains all pyramid levels from the source slide, written as - JPEG-compressed tiles with SubIFD structure for multi-resolution readers. - - Parameters - ---------- - path : str - Output .tiff file path. If targeting a UC Volume (/Volumes/...), - the parent directory must already exist (no os.makedirs on Volumes). - slide : openslide.OpenSlide - Open slide handle — caller is responsible for closing it after. - tile_size : int - Tile edge in pixels (default 256). Both read and write use this size. - jpeg_quality : int - JPEG compression quality (default 80). - """ - import tifffile - - _parent = os.path.dirname(path) - if _parent and not _parent.startswith("/Volumes"): - os.makedirs(_parent, exist_ok=True) - - level_count = slide.level_count - opts = dict( - tile=(tile_size, tile_size), - compression="jpeg", - compressionargs={"level": jpeg_quality}, - photometric="rgb", - metadata=None, - ) - - def _tile_generator(level: int): - """Yield tiles row-by-row for a given pyramid level.""" - w, h = slide.level_dimensions[level] - ds = slide.level_downsamples[level] - for y in range(0, h, tile_size): - for x in range(0, w, tile_size): - tw = min(tile_size, w - x) - th = min(tile_size, h - y) - # read_region uses level-0 coordinates - loc = (int(x * ds), int(y * ds)) - tile = np.array( - slide.read_region(loc, level, (tw, th)).convert("RGB") - ) - # Pad to full tile_size if at edge (tifffile requires uniform tiles) - if tile.shape[0] < tile_size or tile.shape[1] < tile_size: - padded = np.zeros((tile_size, tile_size, 3), dtype=np.uint8) - padded[: tile.shape[0], : tile.shape[1]] = tile - tile = padded - yield tile - - with tifffile.TiffWriter(path, bigtiff=True) as tif: - for lvl in range(level_count): - w, h = slide.level_dimensions[lvl] - if lvl == 0: - tif.write( - _tile_generator(lvl), - shape=(h, w, 3), - dtype="uint8", - subifds=level_count - 1 if level_count > 1 else 0, - **opts, - ) - else: - tif.write( - _tile_generator(lvl), - shape=(h, w, 3), - dtype="uint8", - subfiletype=1, - **opts, - ) diff --git a/src/dbx/pixels/svs/phi_tags.py b/src/dbx/pixels/svs/phi_tags.py deleted file mode 100644 index d9088519..00000000 --- a/src/dbx/pixels/svs/phi_tags.py +++ /dev/null @@ -1,162 +0,0 @@ -"""PHI tag classification for Aperio SVS / OpenSlide metadata properties. - -Research-hardcoded lookup covering all standard OpenSlide / Aperio TIFF -properties. Each tag is classified as PHI, QUESTIONABLE, or NOT_PHI. - -Public API: - classify_tag(key) -> str - classify_tags(properties: dict) -> list[dict] - scrub_image_description(image_desc: str) -> str -""" - -from __future__ import annotations - -# -- PHI: Definite Protected Health Information -------------------------------- -PHI_TAGS: set[str] = { - "aperio.Patient", - "aperio.PatientID", - "aperio.DOB", - "aperio.MRN", - "aperio.AccessionNumber", - "aperio.ClinicID", - "aperio.ClinicalTrialID", - "aperio.Procedure", - "aperio.Diagnosis", - "aperio.Id", - # Short-key variants (inside tiff.ImageDescription pipe-delimited section) - "Patient", - "PatientID", - "DOB", - "MRN", - "AccessionNumber", - "ClinicID", - "ClinicalTrialID", - "Procedure", - "Diagnosis", - "Id", -} - -# -- QUESTIONABLE: May contain PHI depending on site configuration ------------- -QUESTIONABLE_TAGS: set[str] = { - "aperio.Date", - "aperio.Time", - "aperio.Clinic", - "aperio.Pathologist", - "aperio.Title", - "aperio.Filename", - "aperio.User", - "aperio.ImageID", - "tiff.Artist", - "tiff.Copyright", - # Short-key variants - "Date", - "Time", - "Clinic", - "Pathologist", - "Title", - "Filename", - "User", - "ImageID", -} - -# -- NOT_PHI: Pure scanner / technical parameters ------------------------------ -NOT_PHI_TAGS: set[str] = { - "aperio.AppMag", - "aperio.MPP", - "aperio.ScanScope ID", - "aperio.StripeWidth", - "aperio.Parmset", - "aperio.Filtered", - "aperio.ICC Profile", - "openslide.level-count", - "openslide.mpp-x", - "openslide.mpp-y", - "openslide.objective-power", - "openslide.vendor", - "openslide.quickhash-1", - "openslide.comment", - "tiff.Make", - "tiff.Model", - "tiff.Software", - "tiff.ResolutionUnit", - "tiff.XResolution", - "tiff.YResolution", - # Short-key variants - "AppMag", - "MPP", - "ScanScope ID", - "StripeWidth", - "Parmset", - "Filtered", - "ICC Profile", -} - - -def classify_tag(key: str) -> str: - """Classify a single OpenSlide property key. - - Returns one of: 'PHI', 'QUESTIONABLE', 'NOT_PHI'. - Tags not in any lookup default to 'NOT_PHI' (scanner geometry, level dims, etc.). - """ - if key in PHI_TAGS: - return "PHI" - if key in QUESTIONABLE_TAGS: - return "QUESTIONABLE" - return "NOT_PHI" - - -def classify_tags(properties: dict) -> list[dict]: - """Classify all OpenSlide properties and return the structured PHI report. - - Args: - properties: Dict of OpenSlide property key -> value strings. - - Returns: - List of dicts: [{"tag": key, "value": val, "classification": cls}, ...] - Only includes PHI and QUESTIONABLE entries (NOT_PHI are omitted for brevity). - """ - report = [] - for key, value in properties.items(): - cls = classify_tag(key) - if cls in ("PHI", "QUESTIONABLE"): - report.append({"tag": key, "value": value, "classification": cls}) - return report - - -def scrub_image_description(image_desc: str) -> str: - """Scrub PHI/QUESTIONABLE values from the Aperio ImageDescription string. - - Aperio format:: - - {header_line}|key = val|key = val|... - - Header line is technical-only (preserved as-is). Each key=value pair - whose key matches PHI or QUESTIONABLE is replaced with ``key = REDACTED``. - - Args: - image_desc: Raw tiff.ImageDescription string. - - Returns: - Scrubbed string with PHI values replaced. - """ - if not image_desc: - return image_desc - - parts = image_desc.split("|") - # First part is the header line -- always preserved - header = parts[0] - rebuilt = [header] - - for kv in parts[1:]: - if " = " in kv: - k, _v = kv.split(" = ", 1) - key_stripped = k.strip() - if key_stripped in PHI_TAGS or key_stripped in QUESTIONABLE_TAGS: - rebuilt.append(f"{k} = REDACTED") - else: - rebuilt.append(kv) - else: - # Continuation or malformed -- preserve as-is - rebuilt.append(kv) - - return "|".join(rebuilt) diff --git a/src/dbx/pixels/svs/resources/sql/CREATE_SVS_CATALOG.sql b/src/dbx/pixels/svs/resources/sql/CREATE_SVS_CATALOG.sql deleted file mode 100644 index 488db725..00000000 --- a/src/dbx/pixels/svs/resources/sql/CREATE_SVS_CATALOG.sql +++ /dev/null @@ -1,110 +0,0 @@ --- Unified redaction tracking table for all imaging formats (DICOM, SVS, CZI, ...) --- --- Extends CREATE_OBJECT_CATALOG_REDACTION.sql from the base dbx-pixels package: --- * Three DICOM columns renamed to remove format-specific semantics: --- redaction_json → redaction_config --- global_redactions_count → metadata_redactions_count --- frame_specific_redactions_count → pixel_redactions_count --- * New columns added (all nullable for backward compatibility with existing DICOM rows): --- path, extension — generic FK + format discriminator --- has_phi, phi_elements, --- vlm_raw_response, --- model_endpoint — VLM PHI detection results (applicable to all formats) --- phi_tags_redacted — metadata tags scrubbed (applicable to all formats) --- label_image_path, --- macro_image_path — SVS sub-image audit artefacts (NULL for DICOM) --- --- Format conventions: --- DICOM rows: populate study/series UIDs, output_file_paths has one entry per .dcm slice, --- label_image_path and macro_image_path are NULL --- SVS rows: study/series UIDs are NULL, output_file_paths[0] is the single TIFF output, --- modality = 'WSI', label_image_path / macro_image_path point to audit PNGs - -CREATE TABLE IF NOT EXISTS {UC_TABLE}_redaction ( - - -- ----------------------------------------------------------------------- - -- Primary identifiers (format-agnostic) - -- ----------------------------------------------------------------------- - redaction_id STRING NOT NULL COMMENT 'UUID assigned at job creation time', - path STRING COMMENT 'Source file path — FK to object_catalog.path', - extension STRING COMMENT 'Source format discriminator: dcm | svs | czi | …', - - -- ----------------------------------------------------------------------- - -- DICOM identifiers (NULL for non-DICOM formats) - -- ----------------------------------------------------------------------- - study_instance_uid STRING COMMENT 'DICOM Study Instance UID', - series_instance_uid STRING COMMENT 'DICOM Series Instance UID', - modality STRING COMMENT 'DICOM modality (CT, MR, US …) or WSI for whole-slide images', - - -- ----------------------------------------------------------------------- - -- Redaction configuration (format-agnostic VARIANT) - -- Renamed from redaction_json → redaction_config for cross-format clarity - -- ----------------------------------------------------------------------- - redaction_config VARIANT COMMENT 'Format-specific redaction instructions as VARIANT', - - -- ----------------------------------------------------------------------- - -- PHI detection results — VLM output, applicable to all formats - -- ----------------------------------------------------------------------- - has_phi BOOLEAN COMMENT 'True if VLM detected PHI in pixel data', - phi_elements VARIANT COMMENT 'Array of detected PHI regions: {type, value_hint, bbox}', - vlm_raw_response STRING COMMENT 'Raw response text from the VLM model', - model_endpoint STRING COMMENT 'Name of the model serving endpoint used', - phi_tags_redacted ARRAY COMMENT 'Metadata tag names whose values were scrubbed', - - -- ----------------------------------------------------------------------- - -- Redaction counts - -- Renamed from DICOM-specific frame model to generic pixel/metadata model - -- ----------------------------------------------------------------------- - metadata_redactions_count INT COMMENT 'Number of metadata tag values overwritten (was global_redactions_count)', - pixel_redactions_count INT COMMENT 'Number of pixel-level redaction regions applied (was frame_specific_redactions_count)', - total_redaction_areas INT COMMENT 'Total redaction areas across metadata and pixels', - - -- ----------------------------------------------------------------------- - -- Output paths - -- DICOM: one .dcm path per slice/frame - -- SVS: single-element array — output_file_paths[0] is the TIFF path - -- ----------------------------------------------------------------------- - output_file_paths ARRAY COMMENT 'Output file paths. DICOM: one per slice. SVS: [tiff_path].', - new_series_instance_uid STRING COMMENT 'New Series Instance UID for redacted DICOM series (NULL for SVS)', - - -- ----------------------------------------------------------------------- - -- SVS-specific audit artefacts (NULL for DICOM) - -- ----------------------------------------------------------------------- - label_image_path STRING COMMENT 'Path to de-identified label sub-image PNG (SVS only)', - macro_image_path STRING COMMENT 'Path to de-identified macro sub-image PNG (SVS only)', - - -- ----------------------------------------------------------------------- - -- Processing status (format-agnostic — unchanged from DICOM original) - -- ----------------------------------------------------------------------- - status STRING NOT NULL COMMENT 'PENDING | PROCESSING | SUCCESS | FAILED', - error_messages ARRAY COMMENT 'Error details if processing failed', - - -- ----------------------------------------------------------------------- - -- Timestamps (format-agnostic — unchanged from DICOM original) - -- ----------------------------------------------------------------------- - insert_timestamp TIMESTAMP NOT NULL COMMENT 'When the record was initially created', - update_timestamp TIMESTAMP COMMENT 'When the record was last updated', - processing_start_timestamp TIMESTAMP COMMENT 'When processing started', - processing_end_timestamp TIMESTAMP COMMENT 'When processing completed', - processing_duration_seconds DOUBLE COMMENT 'Wall-clock processing time in seconds', - - -- ----------------------------------------------------------------------- - -- Audit (format-agnostic — unchanged from DICOM original) - -- ----------------------------------------------------------------------- - created_by STRING COMMENT 'User who created the redaction job', - export_timestamp TIMESTAMP COMMENT 'When redaction annotations were exported' - -) -USING delta -CLUSTER BY (redaction_id) -COMMENT 'Unified redaction tracking table for all imaging formats (DICOM, SVS, CZI, …). Extends the base object_catalog_redaction schema with VLM PHI detection results and SVS sub-image artefacts.' -TBLPROPERTIES ( - 'delta.enableChangeDataFeed' = 'true', - 'delta.enableDeletionVectors' = 'true', - 'delta.feature.deletionVectors' = 'supported', - 'delta.minReaderVersion' = '3', - 'delta.minWriterVersion' = '7', - 'delta.targetFileSize' = '256mb', - 'delta.autoOptimize.autoCompact' = 'true', - 'delta.autoOptimize.optimizeWrite' = 'true' -); diff --git a/src/dbx/pixels/svs/svs_meta_extractor.py b/src/dbx/pixels/svs/svs_meta_extractor.py deleted file mode 100644 index 26a87dc6..00000000 --- a/src/dbx/pixels/svs/svs_meta_extractor.py +++ /dev/null @@ -1,115 +0,0 @@ -"""SVSMetaExtractor — Spark ML Transformer that reads OpenSlide metadata into meta VARIANT. - -Mirrors ``DicomMetaExtractor`` from the pixels SA: -- Extends ``pyspark.ml.pipeline.Transformer`` -- Implements ``_transform(df)`` -- Uses ``mapInPandas`` with ``ThreadPoolExecutor`` for concurrent network I/O -- Outputs a single ``meta`` column as a parsed VARIANT - -All SVS-specific derived fields (width, height, level_count, has_label_image, -has_macro_image, phi_tag_report) are merged into the OpenSlide properties dict -before JSON serialisation, so no schema change to ``object_catalog`` is required. -""" - -from __future__ import annotations - -import json -from concurrent.futures import ThreadPoolExecutor -from typing import Iterator - -import pandas as pd -import pyspark.sql.types as t -from pyspark.ml.pipeline import Transformer -from pyspark.sql.functions import expr - -from dbx.pixels.svs.phi_tags import classify_tags - - -class SVSMetaExtractor(Transformer): - """Extract OpenSlide metadata from SVS files into the ``meta VARIANT`` column. - - Args: - catalog: :class:`SVSCatalog` instance (used for ``is_anon`` flag). - inputCol: Column with worker-accessible file paths (default ``local_path``). - outputCol: Output column name (default ``meta``). - maxWorkers: ``ThreadPoolExecutor`` concurrency (default 32). - useVariant: Parse JSON string to VARIANT via ``parse_json()`` (default True). - """ - - MAX_WORKERS = 32 - - def __init__( - self, - catalog, - inputCol: str = "local_path", - outputCol: str = "meta", - maxWorkers: int = None, - useVariant: bool = True, - ): - self.catalog = catalog - self.inputCol = inputCol - self.outputCol = outputCol - self.maxWorkers = maxWorkers or self.MAX_WORKERS - self.useVariant = useVariant - - def _transform(self, df): - """Apply SVS metadata extraction using mapInPandas with concurrent I/O.""" - input_col = self.inputCol - output_col = self.outputCol - max_workers = self.maxWorkers - - out_schema = t.StructType( - list(df.schema.fields) - + [t.StructField(output_col, t.StringType(), True)] - ) - - def _extract_meta_batch( - iterator: Iterator[pd.DataFrame], - ) -> Iterator[pd.DataFrame]: - import openslide - from dbx.pixels.svs.phi_tags import classify_tags # noqa: direct import avoids __init__ chain - - def _process_file(path: str) -> str: - try: - slide = openslide.OpenSlide(path) - props = dict(slide.properties) - associated = list(slide.associated_images.keys()) - - meta = { - **props, - # --- derived SVS fields --- - "width": slide.dimensions[0], - "height": slide.dimensions[1], - "level_count": slide.level_count, - "level_dimensions": [ - list(d) for d in slide.level_dimensions - ], - "level_downsamples": list(slide.level_downsamples), - "has_label_image": "label" in associated, - "has_macro_image": "macro" in associated, - "associated_images": associated, - "phi_tag_report": classify_tags(props), - } - slide.close() - return json.dumps(meta) - except Exception as err: - return json.dumps( - {"error": str(err), "udf": "svs_meta_extractor", "path": path} - ) - - for pdf in iterator: - paths = pdf[input_col].tolist() - with ThreadPoolExecutor(max_workers=max_workers) as executor: - results = list(executor.map(_process_file, paths)) - pdf[output_col] = results - yield pdf - - df = df.mapInPandas(_extract_meta_batch, schema=out_schema) - - if self.useVariant: - df = df.withColumn( - self.outputCol, - expr(f"parse_json(`{self.outputCol}`)"), - ) - - return df diff --git a/src/dbx/pixels/svs/svs_tiff_writer.py b/src/dbx/pixels/svs/svs_tiff_writer.py deleted file mode 100644 index 70a9ab57..00000000 --- a/src/dbx/pixels/svs/svs_tiff_writer.py +++ /dev/null @@ -1,190 +0,0 @@ -"""SVSTiffWriter — Spark ML Transformer: SVS → de-identified pyramidal BigTIFF. - -For each SVS file: -1. Reads label and macro sub-images; applies black-rectangle redaction - over VLM-detected PHI bboxes; saves de-identified PNGs as audit artefacts. -2. Scrubs PHI metadata from ``tiff.ImageDescription`` / ``openslide.comment``. -3. Reads every pyramid level via tiled I/O to avoid OOM. -4. Writes a pyramidal BigTIFF (QuPath / libvips / OMERO compatible). -""" - -from __future__ import annotations - -import json -from concurrent.futures import ThreadPoolExecutor -from typing import Iterator - -import pandas as pd -import pyspark.sql.types as t -from pyspark.ml.pipeline import Transformer - -# deidentify imports moved inside _write_one (file is corrupt on disk; -# workers need the same truncation workaround as the driver in Cell 3). -from dbx.pixels.svs.phi_tags import scrub_image_description - - -# Schema of rows emitted by SVSTiffWriter -_OUTPUT_SCHEMA = t.StructType([ - t.StructField("path", t.StringType(), True), - t.StructField("tiff_output_path", t.StringType(), True), - t.StructField("label_image_path", t.StringType(), True), - t.StructField("macro_image_path", t.StringType(), True), - t.StructField("phi_tags_redacted", t.ArrayType(t.StringType()), True), - t.StructField("pixel_regions_redacted", t.IntegerType(), True), - t.StructField("error", t.StringType(), True), -]) - - -class SVSTiffWriter(Transformer): - """Convert SVS files to de-identified pyramidal BigTIFFs. - - Expects the input DataFrame to contain at minimum: - - ``local_path`` — worker-accessible SVS path - - *phi_col* — JSON-string array of PHI element dicts - ``[{type, value_hint, bbox:{x,y,w,h}, subimage}]`` - - Returns one row per input SVS with TIFF/PNG output paths and audit counts. - - Args: - output_volume: Volume path for de-identified TIFF output. - label_volume: Volume path for de-identified label/macro PNG artefacts. - inputCol: Path column name (default ``local_path``). - phiCol: Column with VLM phi_elements JSON (default - ``phi_elements_json``; may be absent — no redaction). - maxWorkers: ``ThreadPoolExecutor`` concurrency (default 4; - TIFF conversion is CPU + I/O bound). - jpeg_quality: Output JPEG tile quality (default 80). - """ - - def __init__( - self, - output_volume: str, - label_volume: str, - inputCol: str = "local_path", - phiCol: str = "phi_elements_json", - maxWorkers: int = 4, - jpeg_quality: int = 80, - ): - self.output_volume = output_volume.rstrip("/") - self.label_volume = label_volume.rstrip("/") - self.inputCol = inputCol - self.phiCol = phiCol - self.maxWorkers = maxWorkers - self.jpeg_quality = jpeg_quality - - def _transform(self, df): - input_col = self.inputCol - phi_col = self.phiCol - output_volume = self.output_volume - label_volume = self.label_volume - jpeg_quality = self.jpeg_quality - - def _write_one(path: str, phi_elements_json: str | None) -> dict: - import os - import sys - import types - import openslide - from pathlib import Path - - # Lazy import with corruption workaround (deidentify.py has 17K+ dup lines) - if "dbx.pixels.svs.deidentify" not in sys.modules: - _p = "/Workspace/Users/douglas.moore@databricks.com/pixels-svs/src/dbx/pixels/svs/deidentify.py" - with open(_p, "r") as _f: - _src = "".join(_f.readlines()[:200]) - _mod = types.ModuleType("dbx.pixels.svs.deidentify") - _mod.__file__ = _p - exec(compile(_src, _p, "exec"), _mod.__dict__) - sys.modules["dbx.pixels.svs.deidentify"] = _mod - from dbx.pixels.svs.deidentify import ( - redact_image, write_pyramidal_bigtiff_streaming, - ) - - try: - phi_elements: list[dict] = ( - json.loads(phi_elements_json) - if phi_elements_json - else [] - ) - slide = openslide.OpenSlide(path) - stem = Path(path).stem - - # ── 1. Extract + redact label / macro sub-images ────────── - label_path = macro_path = None - pixel_count = 0 - - if "label" in slide.associated_images: - label_img = slide.associated_images["label"] - label_phi = [ - e for e in phi_elements - if e.get("subimage", "label") != "macro" - ] - label_out = redact_image(label_img, label_phi) - pixel_count += len(label_phi) - label_path = f"{label_volume}/{stem}_label.png" - os.makedirs(os.path.dirname(label_path), exist_ok=True) - label_out.save(label_path) - - if "macro" in slide.associated_images: - macro_img = slide.associated_images["macro"] - macro_phi = [ - e for e in phi_elements - if e.get("subimage") == "macro" - ] - macro_out = redact_image(macro_img, macro_phi) - pixel_count += len(macro_phi) - macro_path = f"{label_volume}/{stem}_macro.png" - os.makedirs(os.path.dirname(macro_path), exist_ok=True) - macro_out.save(macro_path) - - # ── 2. Scrub ImageDescription metadata ──────────────────── - raw_desc = slide.properties.get("tiff.ImageDescription", "") - scrubbed = scrub_image_description(raw_desc) - phi_tags_redacted = ( - ["tiff.ImageDescription", "openslide.comment"] - if raw_desc != scrubbed - else [] - ) - - # ── 3. Write pyramidal BigTIFF (streaming, ~1 tile in RAM) ── - tiff_path = f"{output_volume}/{stem}.tiff" - write_pyramidal_bigtiff_streaming( - tiff_path, slide, jpeg_quality=jpeg_quality - ) - slide.close() - - return { - "path": path, - "tiff_output_path": tiff_path, - "label_image_path": label_path, - "macro_image_path": macro_path, - "phi_tags_redacted": phi_tags_redacted, - "pixel_regions_redacted": pixel_count, - "error": None, - } - - except Exception as exc: # noqa: BLE001 - return { - "path": path, - "tiff_output_path": None, - "label_image_path": None, - "macro_image_path": None, - "phi_tags_redacted": [], - "pixel_regions_redacted": 0, - "error": str(exc), - } - - def _batch( - iterator: Iterator[pd.DataFrame], - ) -> Iterator[pd.DataFrame]: - for pdf in iterator: - paths = pdf[input_col].tolist() - phi_jsns = ( - pdf[phi_col].tolist() - if phi_col in pdf.columns - else [None] * len(paths) - ) - with ThreadPoolExecutor(max_workers=self.maxWorkers) as ex: - results = list(ex.map(_write_one, paths, phi_jsns)) - yield pd.DataFrame(results) - - return df.mapInPandas(_batch, schema=_OUTPUT_SCHEMA) diff --git a/src/dbx/pixels/tiff/__init__.py b/src/dbx/pixels/tiff/__init__.py deleted file mode 100644 index c505be02..00000000 --- a/src/dbx/pixels/tiff/__init__.py +++ /dev/null @@ -1,4 +0,0 @@ -from dbx.pixels.tiff.tiff_meta_extractor import TiffMetaExtractor -from dbx.pixels.tiff.tiff_vlm_phi_detector import TiffVLMPhiDetector - -__all__ = ["TiffMetaExtractor", "TiffVLMPhiDetector"] diff --git a/src/dbx/pixels/tiff/tiff_meta_extractor.py b/src/dbx/pixels/tiff/tiff_meta_extractor.py deleted file mode 100644 index fda7e8cc..00000000 --- a/src/dbx/pixels/tiff/tiff_meta_extractor.py +++ /dev/null @@ -1,233 +0,0 @@ -"""TiffMetaExtractor — Spark ML Transformer that reads TIFF metadata into a ``meta`` VARIANT. - -Mirrors ``SVSMetaExtractor`` from the pixels SA: -- Extends ``pyspark.ml.pipeline.Transformer`` -- Implements ``_transform(df)`` -- Uses ``mapInPandas`` with ``ThreadPoolExecutor`` for concurrent I/O -- Outputs a single ``meta`` column as a parsed VARIANT - -Primary backend: ``tifffile`` (handles standard TIFF, BigTIFF, OME-TIFF, -Aperio SVS-style TIFF, NDPI). Falls back to ``Pillow`` if ``tifffile`` is -not installed. - -All derived fields (page_count, is_ome, is_bigtiff, series info, -phi_tag_report) are merged into the tag dict before JSON serialisation, so -no schema change to ``object_catalog`` is required. -""" - -from __future__ import annotations - -import json -from concurrent.futures import ThreadPoolExecutor -from typing import Iterator - -import pandas as pd -import pyspark.sql.types as t -from pyspark.ml.pipeline import Transformer -from pyspark.sql.functions import expr - -from dbx.pixels.tiff.tiff_phi_tags import classify_tags - - -class TiffMetaExtractor(Transformer): - """Extract TIFF metadata into the ``meta VARIANT`` column. - - Uses ``tifffile`` as the primary backend; falls back to ``Pillow`` when - ``tifffile`` is not available on the executor. - - Args: - catalog: Catalog instance (provides path/anon context). - inputCol: Column with worker-accessible file paths (default ``local_path``). - outputCol: Output column name (default ``meta``). - maxWorkers: ``ThreadPoolExecutor`` concurrency per Spark task (default 32). - useVariant: Parse JSON string to VARIANT via ``parse_json()`` (default True). - filterLargeTags: Drop tags in ``LARGE_TAGS`` (tile/strip offsets, JPEG tables, - ICC profile, XMP, etc.) before JSON serialisation (default True). - """ - - MAX_WORKERS = 32 - - def __init__( - self, - catalog, - inputCol: str = "local_path", - outputCol: str = "meta", - maxWorkers: int = None, - useVariant: bool = True, - filterLargeTags: bool = True, - ): - self.catalog = catalog - self.inputCol = inputCol - self.outputCol = outputCol - self.maxWorkers = maxWorkers or self.MAX_WORKERS - self.useVariant = useVariant - self.filterLargeTags = filterLargeTags - - # ------------------------------------------------------------------ - # Internal helpers (run on Spark executors inside mapInPandas) - # ------------------------------------------------------------------ - - @staticmethod - def _process_tifffile(path: str, filter_large: bool = True) -> str: - """Extract metadata with tifffile (primary backend).""" - import tifffile - - from dbx.pixels.tiff.tiff_phi_tags import LARGE_TAGS, classify_tags - - try: - with tifffile.TiffFile(path) as tif: - page = tif.pages[0] - - # All page-0 TIFF tags as plain strings; skip large binary/offset tags - tags: dict[str, str] = { - tag.name: str(tag.value) - for tag in page.tags.values() - if not filter_large or tag.name not in LARGE_TAGS - } - - # Per-series summary (multi-level WSI awareness) - series_info = [] - for s in tif.series: - entry: dict = { - "shape": list(s.shape), - "axes": s.axes, - "dtype": str(s.dtype), - } - if hasattr(s, "levels"): - entry["level_count"] = len(s.levels) - series_info.append(entry) - - meta = { - **tags, - # --- derived fields --- - "page_count": len(tif.pages), - "series_count": len(tif.series), - "is_bigtiff": tif.is_bigtiff, - "is_ome": tif.is_ome, - "is_svs": tif.is_svs, - "is_ndpi": getattr(tif, "is_ndpi", False), - "width": page.imagewidth, - "height": page.imagelength, - "bits_per_sample": page.bitspersample, - "samples_per_pixel": page.samplesperpixel, - "compression": str(page.compression), - "photometric": str(page.photometric), - "series": series_info, - "phi_tag_report": classify_tags(tags), - } - return json.dumps(meta) - - except Exception as err: - return json.dumps( - {"error": str(err), "udf": "tiff_meta_extractor_tifffile", "path": path} - ) - - @staticmethod - def _process_pillow(path: str, filter_large: bool = True) -> str: - """Extract metadata with Pillow (fallback when tifffile is absent).""" - from PIL import Image - - from dbx.pixels.tiff.tiff_phi_tags import LARGE_TAGS - - try: - # Lazy import: only available in Pillow >= 5.4 - try: - from PIL.TiffImagePlugin import IFDRational - except ImportError: - IFDRational = None - - with Image.open(path) as img: - raw_tags = img.tag_v2 if hasattr(img, "tag_v2") else {} - tags: dict = {} - - # Use TAGS mapping when available for human-readable names - try: - from PIL.ExifTags import TAGS as _TAGS - except ImportError: - _TAGS = {} - - for k, v in raw_tags.items(): - tag_name = _TAGS.get(k, str(k)) - if filter_large and tag_name in LARGE_TAGS: - continue - # Coerce non-JSON-serialisable types - if IFDRational is not None and isinstance(v, IFDRational): - tags[tag_name] = float(v) - elif isinstance(v, tuple): - tags[tag_name] = [ - float(x) if (IFDRational and isinstance(x, IFDRational)) else x - for x in v - ] - elif isinstance(v, bytes): - tags[tag_name] = v.decode("latin-1", errors="replace") - else: - tags[tag_name] = v - - meta = { - **tags, - "width": img.width, - "height": img.height, - "mode": img.mode, - "n_frames": getattr(img, "n_frames", 1), - "format": img.format, - "phi_tag_report": classify_tags( - {str(k): str(v) for k, v in tags.items()} - ), - } - return json.dumps(meta, default=str) - - except Exception as err: - return json.dumps( - {"error": str(err), "udf": "tiff_meta_extractor_pillow", "path": path} - ) - - @staticmethod - def _process_file(path: str, filter_large: bool = True) -> str: - """Dispatch to tifffile or Pillow, whichever is available.""" - try: - import tifffile # noqa: F401 - return TiffMetaExtractor._process_tifffile(path, filter_large) - except ImportError: - return TiffMetaExtractor._process_pillow(path, filter_large) - - # ------------------------------------------------------------------ - # Transformer entry point - # ------------------------------------------------------------------ - - def _transform(self, df): - """Apply TIFF metadata extraction using mapInPandas with concurrent I/O.""" - input_col = self.inputCol - output_col = self.outputCol - max_workers = self.maxWorkers - - out_schema = t.StructType( - list(df.schema.fields) - + [t.StructField(output_col, t.StringType(), True)] - ) - - filter_large = self.filterLargeTags - - def _extract_meta_batch( - iterator: Iterator[pd.DataFrame], - ) -> Iterator[pd.DataFrame]: - for pdf in iterator: - paths = pdf[input_col].tolist() - with ThreadPoolExecutor(max_workers=max_workers) as executor: - results = list( - executor.map( - lambda p: TiffMetaExtractor._process_file(p, filter_large), - paths, - ) - ) - pdf[output_col] = results - yield pdf - - df = df.mapInPandas(_extract_meta_batch, schema=out_schema) - - if self.useVariant: - df = df.withColumn( - self.outputCol, - expr(f"parse_json(`{self.outputCol}`)"), - ) - - return df diff --git a/src/dbx/pixels/tiff/tiff_phi_tags.py b/src/dbx/pixels/tiff/tiff_phi_tags.py deleted file mode 100644 index d5020471..00000000 --- a/src/dbx/pixels/tiff/tiff_phi_tags.py +++ /dev/null @@ -1,171 +0,0 @@ -"""PHI tag classification for standard TIFF / BigTIFF / OME-TIFF / Aperio TIFF files. - -Covers baseline TIFF tags (TIFF 6.0 spec) and common EXIF tags by their -string name as returned by tifffile (e.g. ``"Artist"``, ``"DateTime"``). - -Classification tiers --------------------- -PHI — Directly identifies a patient or operator. -QUESTIONABLE — May contain PHI depending on site configuration. -NOT_PHI — Pure scanner / geometry / technical parameters. - -Public API ----------- - classify_tag(key) -> str - classify_tags(properties: dict) -> list[dict] -""" - -from __future__ import annotations - -# --------------------------------------------------------------------------- -# PHI: Directly identifies a person -# --------------------------------------------------------------------------- -PHI_TAGS: set[str] = { - # Baseline TIFF 6.0 - "Artist", # tag 315 — person who created the image - "HostComputer", # tag 316 — workstation/operator ID - # Aperio SVS ImageDescription pipe-delimited sub-keys - "Patient", - "PatientID", - "DOB", - "MRN", - "AccessionNumber", - "ClinicID", - "ClinicalTrialID", - "Procedure", - "Diagnosis", - "Id", -} - -# --------------------------------------------------------------------------- -# QUESTIONABLE: May contain PHI depending on site / scanner configuration -# --------------------------------------------------------------------------- -QUESTIONABLE_TAGS: set[str] = { - # Baseline TIFF 6.0 — free-text / timestamp fields - "ImageDescription", # tag 270 — free-text; may embed patient info (Aperio, OME) - "DateTime", # tag 306 — image creation timestamp - "Copyright", # tag 33432 — may contain operator/institution name - # EXIF timestamps - "DateTimeOriginal", # EXIF 36867 - "DateTimeDigitized", # EXIF 36868 - # Aperio SVS ImageDescription sub-keys - "Date", - "Time", - "Clinic", - "Pathologist", - "Title", - "Filename", - "User", - "ImageID", -} - -# --------------------------------------------------------------------------- -# LARGE_TAGS: Binary / array tags that should be skipped or truncated during -# metadata extraction — their values are large byte blobs or offset arrays -# that add no textual metadata value and bloat the JSON output. -# --------------------------------------------------------------------------- -LARGE_TAGS: set[str] = { - # JPEG / compression tables - "JPEGTables", # tag 347 — JPEG quantisation + Huffman tables (binary) - "JPEGQTables", # tag 519 — old-style JPEG quantisation tables - "JPEGDCTables", # tag 520 — old-style DC Huffman tables - "JPEGACTables", # tag 521 — old-style AC Huffman tables - # Tile / strip index arrays (one entry per tile/strip — can be millions of entries) - "TileOffsets", # tag 324 — byte offset of every tile in the file - "TileByteCounts", # tag 325 — byte length of every tile - "StripOffsets", # tag 273 — byte offset of every strip - "StripByteCounts", # tag 279 — byte length of every strip - # Colour / profile data - "ICCProfile", # tag 34675 — ICC colour profile (often 400 B – 4 MB) - "ColorMap", # tag 320 — RGB palette for indexed-colour images - "TransferFunction", # tag 301 — transfer function curves - "ReferenceBlackWhite", # tag 532 — reference black/white for YCbCr - # Embedded metadata blobs - "XMP", # tag 700 — XMP metadata XML (can be 10s of KB) - "IPTCNAA", # tag 33723 — IPTC/NAA metadata record - "Photoshop", # tag 34377 — Photoshop ImageResources block - "ExifIFD", # tag 34665 — embedded EXIF IFD offset array - # GeoTIFF arrays - "GeoKeyDirectoryTag", # tag 34736 — GeoTIFF key directory - "GeoDoubleParamsTag", # tag 34736 — GeoTIFF double params - "GeoAsciiParamsTag", # tag 34737 — GeoTIFF ASCII params - # WSI / scanning - "ImageDepth", # tag 32997 — depth offset array in some WSI formats - "SubIFDs", # tag 330 — sub-IFD offset array (pyramid levels) -} - -# --------------------------------------------------------------------------- -# NOT_PHI: Scanner geometry / technical parameters (not exhaustive) -# --------------------------------------------------------------------------- -NOT_PHI_TAGS: set[str] = { - "ImageWidth", - "ImageLength", - "BitsPerSample", - "Compression", - "PhotometricInterpretation", - "StripOffsets", - "SamplesPerPixel", - "RowsPerStrip", - "StripByteCounts", - "XResolution", - "YResolution", - "PlanarConfiguration", - "ResolutionUnit", - "Software", # tag 305 — scanner software version (NOT PHI) - "Make", # tag 271 — scanner manufacturer - "Model", # tag 272 — scanner model - "TileWidth", - "TileLength", - "TileOffsets", - "TileByteCounts", - "NewSubfileType", - "SubfileType", - "Orientation", - "ExtraSamples", - "SampleFormat", - "JPEGTables", - "YCbCrSubSampling", - "ReferenceBlackWhite", - "ColorMap", - "GrayResponseUnit", - "GrayResponseCurve", - # Aperio / OpenSlide technical - "AppMag", - "MPP", - "ScanScope ID", - "StripeWidth", - "Parmset", - "Filtered", - "ICC Profile", -} - - -def classify_tag(key: str) -> str: - """Classify a single TIFF tag name. - - Returns one of: ``'PHI'``, ``'QUESTIONABLE'``, ``'NOT_PHI'``. - Tags not in any lookup default to ``'NOT_PHI'``. - """ - if key in PHI_TAGS: - return "PHI" - if key in QUESTIONABLE_TAGS: - return "QUESTIONABLE" - return "NOT_PHI" - - -def classify_tags(properties: dict) -> list[dict]: - """Classify all TIFF tag keys and return the structured PHI report. - - Args: - properties: Dict mapping TIFF tag name (str) -> value (str). - - Returns: - List of ``{"tag": key, "value": val, "classification": cls}`` dicts. - Only PHI and QUESTIONABLE entries are included (NOT_PHI omitted). - """ - report = [] - for key, value in properties.items(): - cls = classify_tag(key) - if cls in ("PHI", "QUESTIONABLE"): - report.append({"tag": key, "value": str(value), "classification": cls}) - return report diff --git a/src/dbx/pixels/tiff/tiff_utils.py b/src/dbx/pixels/tiff/tiff_utils.py deleted file mode 100644 index 6ffe0b55..00000000 --- a/src/dbx/pixels/tiff/tiff_utils.py +++ /dev/null @@ -1,191 +0,0 @@ -"""TIFF utility functions — image conversion for downstream processing. - -Provides ``tiff_to_image()``, the TIFF equivalent of -``dbx.pixels.dicom.dicom_utils.dicom_to_image()``. - -Handles standard TIFF, BigTIFF, and multi-level pyramidal WSI TIFFs -(Philips, Aperio SVS-style, NDPI) by reading the **smallest available -pyramid level** rather than the full-resolution page, so VLM callers -never load a 500 MB slide into memory. - -No dependency on ``dbx.pixels.dicom`` or ``pydicom``. - -Primary backend: ``tifffile``. Falls back to ``Pillow`` if absent. -""" - -from __future__ import annotations - -import io -from typing import Optional - -import numpy as np - -from dbx.pixels.logging import LoggerProvider - -logger = LoggerProvider() - - -# --------------------------------------------------------------------------- -# Internal helpers -# --------------------------------------------------------------------------- - -def _normalize_to_uint8_rgb(arr: np.ndarray) -> np.ndarray: - """Coerce any numpy array to uint8 RGB suitable for JPEG encoding. - - Handles: - - 16-bit / float arrays → normalise to 0-255 - - RGBA (4-channel) → drop alpha channel - - Grayscale (2-D) → replicate to 3 channels - - Single-channel 3-D → replicate to 3 channels - """ - # Drop alpha channel - if arr.ndim == 3 and arr.shape[-1] == 4: - arr = arr[:, :, :3] - - # Convert to uint8 - if arr.dtype != np.uint8: - lo, hi = float(arr.min()), float(arr.max()) - if hi > lo: - arr = ((arr.astype(np.float32) - lo) / (hi - lo) * 255).astype(np.uint8) - else: - arr = np.zeros_like(arr, dtype=np.uint8) - - # Grayscale → RGB - if arr.ndim == 2: - arr = np.stack([arr, arr, arr], axis=-1) - elif arr.ndim == 3 and arr.shape[-1] == 1: - arr = np.concatenate([arr, arr, arr], axis=-1) - - return arr - - -def _tiff_to_array_tifffile(path: str) -> Optional[np.ndarray]: - """Read the smallest pyramid level of a TIFF using ``tifffile``. - - For multi-level WSI TIFFs (e.g. Philips BigTIFF with 9 pyramid levels), - returns ``series[0].levels[-1].asarray()`` — the lowest-resolution level. - For single-level TIFFs, returns ``series[0].asarray()``. - """ - import tifffile - - try: - with tifffile.TiffFile(path) as tif: - if tif.series: - series = tif.series[0] - if hasattr(series, "levels") and len(series.levels) > 1: - # Multi-level pyramid — pick the smallest level - return series.levels[-1].asarray() - return series.asarray() - # No series metadata — fall back to page 0 - return tif.pages[0].asarray() - except Exception as e: - logger.exception(f"tifffile read failed for {path}: {e}") - return None - - -def _tiff_to_array_pillow(path: str) -> Optional[np.ndarray]: - """Read the last frame of a TIFF using ``Pillow`` (fallback). - - For pyramidal TIFFs the last IFD is the lowest-resolution page, - making it the best thumbnail candidate without tifffile. - - ``Image.MAX_IMAGE_PIXELS`` is temporarily disabled because WSI slides - legitimately exceed Pillow's default decompression-bomb limit (the full - resolution header triggers the guard even though we only decompress the - small last frame). - """ - from PIL import Image - - try: - _prev = Image.MAX_IMAGE_PIXELS - Image.MAX_IMAGE_PIXELS = None # suppress bomb check for large WSI - try: - with Image.open(path) as img: - n_frames = getattr(img, "n_frames", 1) - if n_frames > 1: - img.seek(n_frames - 1) - return np.array(img) - finally: - Image.MAX_IMAGE_PIXELS = _prev # always restore - except Exception as e: - logger.exception(f"Pillow read failed for {path}: {e}") - return None - - -# --------------------------------------------------------------------------- -# Public API -# --------------------------------------------------------------------------- - -def tiff_to_image( - path: str, - max_width: int = 768, - output_path: str = None, - return_type: str = "str", -) -> Optional[str | bytes]: - """Convert a TIFF file to a JPEG thumbnail. - - For multi-level pyramidal WSI TIFFs, reads the smallest available pyramid - level to avoid loading full-resolution pixel data. For single-level - TIFFs, reads the full image and resizes if needed. - - Primary backend: ``tifffile``. Falls back to ``Pillow`` if not installed. - No dependency on ``dbx.pixels.dicom`` or ``pydicom``. - - Args: - path: Local path to the TIFF file. - max_width: Resize thumbnail to this width (default 768). - Set to 0 to skip resize. - output_path: If set, also save the JPEG to this path. - return_type: ``"str"`` — base64-encoded JPEG string (default). - ``"binary"`` — raw JPEG bytes. - - Returns: - Base64 JPEG string, raw JPEG bytes, or ``None`` on failure. - """ - try: - # --- 1. Read pixel data (tifffile primary, Pillow fallback) --- - arr: Optional[np.ndarray] = None - - try: - import tifffile # noqa: F401 - arr = _tiff_to_array_tifffile(path) - except ImportError: - pass - - if arr is None: - arr = _tiff_to_array_pillow(path) - - if arr is None: - logger.error(f"tiff_to_image: could not read pixel data from {path}") - return None - - # --- 2. Normalise to uint8 RGB --- - arr = _normalize_to_uint8_rgb(arr) - - # --- 3. Resize + encode using PIL directly (no DICOM dependency) --- - import base64 as _base64 - from PIL import Image - - img = Image.fromarray(arr) - if max_width > 0 and img.width > max_width: - ratio = max_width / img.width - img = img.resize((max_width, int(img.height * ratio)), Image.LANCZOS) - - if output_path: - img.save(output_path, format="JPEG") - - buf = io.BytesIO() - img.save(buf, format="JPEG") - jpg_bytes = buf.getvalue() - - if return_type == "binary": - return jpg_bytes - if return_type == "str": - return _base64.b64encode(jpg_bytes).decode("utf-8") - - logger.warning(f"tiff_to_image: unknown return_type '{return_type}', returning None.") - return None - - except Exception as e: - logger.exception(f"tiff_to_image failed for {path}: {e}") - return None diff --git a/src/dbx/pixels/tiff/tiff_vlm_phi_detector.py b/src/dbx/pixels/tiff/tiff_vlm_phi_detector.py deleted file mode 100644 index d4f5c633..00000000 --- a/src/dbx/pixels/tiff/tiff_vlm_phi_detector.py +++ /dev/null @@ -1,208 +0,0 @@ -"""TiffVLMPhiDetector — Spark ML Transformer for pixel-level PHI detection in TIFF files. - -Fully self-contained: no dependency on ``dbx.pixels.dicom`` or ``pydicom``. - -- Extends ``pyspark.ml.base.Transformer`` -- Applies ``tiff_to_image()`` to render a JPEG thumbnail from the smallest - available pyramid level, then calls a Databricks VLM serving endpoint via - the OpenAI-compatible API. -- Output column schema mirrors ``VLMPhiDetector`` from the DICOM module - (content array, completion_tokens int, prompt_tokens int, - total_tokens int, error string). -""" - -from __future__ import annotations - -import base64 -from dataclasses import dataclass -from typing import Iterator, List, Optional - -import pandas as pd -from pyspark.ml.base import Transformer -from pyspark.sql.functions import col, pandas_udf - -from dbx.pixels.logging import LoggerProvider -from dbx.pixels.tiff.tiff_utils import tiff_to_image - -logger = LoggerProvider() - -__all__ = ["TiffVLMPhiDetector", "VlmResult"] - - -DEFAULT_SYSTEM_PROMPT = ( - "You are an expert in privacy and personal health information (PHI). " - "Per HIPAA rules there are 18 fields considered PHI. " - "Please identify all of the PHI fields found in the image and return a " - "list of pipe-separated named entities, e.g. 'John Smith'|'04-31-1954'|'123 Drury Lane' " - "and nothing else. " - "Don't be fooled by text fields especially acronyms that are not PHI. " - "If there's no PHI detected, return 'No PHI' and nothing else. " - "Answer concisely as requested without explanations." -) - - -@dataclass -class VlmResult: - """Per-image result returned by the VLM PHI detector.""" - content: Optional[List[str]] - completion_tokens: int - prompt_tokens: int - total_tokens: int - error: Optional[str] - - -# --------------------------------------------------------------------------- -# Module-level UDF factory -# --------------------------------------------------------------------------- -# Defined outside the class so Spark Connect can serialise the closure -# without capturing ``self``. Only primitive types are closed over. - -def _make_phi_detector_udf( - endpoint: str, - system_prompt: str, - temperature: float, - num_output_tokens: int, - input_type: str, - max_width: int, -): - """Return a ``pandas_udf`` configured with the given inference parameters.""" - - @pandas_udf( - "content array, completion_tokens int, " - "prompt_tokens int, total_tokens int, error string" - ) - def _extract_udf(paths: Iterator[pd.Series]) -> Iterator[pd.DataFrame]: - from dataclasses import replace as dc_replace - from mlflow.utils.databricks_utils import get_databricks_host_creds - from openai import OpenAI - - creds = get_databricks_host_creds("databricks") - client = OpenAI( - api_key=creds.token, - base_url=f"{creds.host}/serving-endpoints", - timeout=300, - max_retries=3, - ) - _null = VlmResult(None, 0, 0, 0, None) - - for batch in paths: - results = [] - for path in batch: - try: - # --- 1. Get image as base64 JPEG --- - if input_type == "tiff": - b64 = tiff_to_image(path, max_width=max_width, return_type="str") - if b64 is None: - results.append(dc_replace(_null, error=f"tiff_to_image returned None: {path}")) - continue - elif input_type == "image": - local = path[5:] if path.startswith("dbfs:") else path - with open(local, "rb") as fh: - b64 = base64.b64encode(fh.read()).decode("utf-8") - else: # "base64" — already encoded - b64 = path - - # --- 2. VLM inference --- - response = client.chat.completions.create( - model=endpoint, - messages=[ - {"role": "system", "content": system_prompt}, - { - "role": "user", - "content": [{ - "type": "image_url", - "image_url": { - "url": f"data:image/jpeg;base64,{b64}", - "detail": "low", - }, - }], - }, - ], - temperature=float(temperature), - max_tokens=int(num_output_tokens), - ) - - # --- 3. Parse response --- - content = response.choices[0].message.content - if "|" in content: - phi_list = content.split("|") - elif content.strip().lower() == "no phi": - phi_list = [] - else: - phi_list = [content] - - results.append(VlmResult( - phi_list, - response.usage.completion_tokens, - response.usage.prompt_tokens, - response.usage.total_tokens, - None, - )) - - except Exception as exc: - results.append(dc_replace(_null, error=str(exc))) - - yield pd.DataFrame(results) - - return _extract_udf - - -# --------------------------------------------------------------------------- -# Transformer -# --------------------------------------------------------------------------- - -class TiffVLMPhiDetector(Transformer): - """Detect pixel-level PHI in TIFF images using a Databricks VLM endpoint. - - No dependency on ``dbx.pixels.dicom`` or ``pydicom`` — fully self-contained. - - Converts TIFF files to JPEG thumbnails via ``tiff_to_image()`` (smallest - pyramid level for WSI) then calls a Databricks OpenAI-compatible VLM - serving endpoint. - - Args: - endpoint: Databricks serving endpoint name for the VLM. - system_prompt: Override the default HIPAA PHI detection prompt. - temperature: VLM sampling temperature (default 0.0). - num_output_tokens: Maximum tokens in the VLM response (default 200). - inputCol: Input column name (default ``local_path``). - outputCol: Output column name (default ``response``). - input_type: ``"tiff"`` — path to a TIFF file (default). - ``"image"`` — path to a JPEG/PNG file. - ``"base64"`` — already base64-encoded image string. - max_width: Resize thumbnail width before VLM (default 768). - Set to 0 to disable. - """ - - def __init__( - self, - endpoint: str, - system_prompt: str = None, - temperature: float = 0.0, - num_output_tokens: int = 200, - inputCol: str = "local_path", - outputCol: str = "response", - input_type: str = "tiff", - max_width: int = 768, - ): - super().__init__() - self.endpoint = endpoint - self.system_prompt = system_prompt - self.temperature = temperature - self.num_output_tokens = num_output_tokens - self.inputCol = inputCol - self.outputCol = outputCol - self.input_type = input_type - self.max_width = max_width - - def _transform(self, df): - """Apply VLM PHI detection via ``pandas_udf``.""" - _udf = _make_phi_detector_udf( - endpoint = self.endpoint, - system_prompt = self.system_prompt or DEFAULT_SYSTEM_PROMPT, - temperature = self.temperature, - num_output_tokens = self.num_output_tokens, - input_type = self.input_type, - max_width = self.max_width, - ) - return df.withColumn(self.outputCol, _udf(col(self.inputCol))) From 57cd266909b6b6c55001c7d99fa148d1dfb86c73 Mon Sep 17 00:00:00 2001 From: Douglas Moore Date: Tue, 14 Jul 2026 13:13:04 -0400 Subject: [PATCH 7/7] fix: run pre-commit and its hooks from the venv `make style` invoked `pre-commit` from PATH, which pyenv shims intercepted since it's only installed in `.venv/`. Same for the hooks' `language: system` entries (black, isort, autoflake). Prepend `.venv/bin` to PATH and call the venv's pre-commit explicitly. Applies the resulting reformats. Co-authored-by: Isaac --- Makefile | 2 +- .../pixels/dicom/cache/bot_cache_builder.py | 6 +- src/dbx/pixels/lakebase.py | 6 +- src/dbx/pixels/wsi/catalog.py | 1 - src/dbx/pixels/wsi/wsi_meta_extractor.py | 12 ++-- src/dbx/pixels/wsi/wsi_phi_tags.py | 57 +++++++++++-------- src/dbx/pixels/wsi/wsi_utils.py | 7 ++- tests/dbx/test_wsi.py | 44 +++++++++----- 8 files changed, 86 insertions(+), 49 deletions(-) diff --git a/Makefile b/Makefile index 5e7c20e6..0d4a2013 100644 --- a/Makefile +++ b/Makefile @@ -42,7 +42,7 @@ test: pytest -s tests --import-mode=importlib -W ignore::DeprecationWarning style: - pre-commit run --all-files + PATH="$(CURDIR)/.venv/bin:$$PATH" .venv/bin/pre-commit run --all-files check: style test diff --git a/src/dbx/pixels/dicom/cache/bot_cache_builder.py b/src/dbx/pixels/dicom/cache/bot_cache_builder.py index 97b2f105..e4b1b89e 100644 --- a/src/dbx/pixels/dicom/cache/bot_cache_builder.py +++ b/src/dbx/pixels/dicom/cache/bot_cache_builder.py @@ -655,7 +655,8 @@ def get_preload_list( hl_used = self.half_lives["last_used"] * 3600.0 # hours → seconds hl_ins = self.half_lives["inserted"] * 3600.0 - query = sql.SQL(""" + query = sql.SQL( + """ SELECT filename, COUNT(*) AS frame_count, @@ -685,7 +686,8 @@ def get_preload_list( GROUP BY filename ORDER BY priority_score DESC LIMIT %s - """).format( + """ + ).format( table=sql.Identifier(self.lb.schema, "dicom_frames"), w_used=sql.Literal(w_used), w_ins=sql.Literal(w_ins), diff --git a/src/dbx/pixels/lakebase.py b/src/dbx/pixels/lakebase.py index a5769e32..43f3f509 100644 --- a/src/dbx/pixels/lakebase.py +++ b/src/dbx/pixels/lakebase.py @@ -799,7 +799,8 @@ def get_preload_priority_list( # avoids the json_agg / OID-114 deserialization issue where psycopg2 # may return the json type as Python None if the JSON adapter is not # registered in the connection context. - query = sql.SQL(""" + query = sql.SQL( + """ SELECT filename, COUNT(*) AS frame_count, @@ -829,7 +830,8 @@ def get_preload_priority_list( GROUP BY filename ORDER BY priority_score DESC LIMIT %s - """).format(table=sql.Identifier(self.schema, table)) + """ + ).format(table=sql.Identifier(self.schema, table)) rows = self.execute_and_fetch_query(query, (uc_table_name, limit)) results = [] diff --git a/src/dbx/pixels/wsi/catalog.py b/src/dbx/pixels/wsi/catalog.py index 998f2d96..acda13db 100644 --- a/src/dbx/pixels/wsi/catalog.py +++ b/src/dbx/pixels/wsi/catalog.py @@ -21,7 +21,6 @@ from dbx.pixels.catalog import Catalog from dbx.pixels.wsi.wsi_phi_tags import OPENSLIDE_PATTERNS - # Default patterns for common WSI formats _DEFAULT_PATTERNS = OPENSLIDE_PATTERNS diff --git a/src/dbx/pixels/wsi/wsi_meta_extractor.py b/src/dbx/pixels/wsi/wsi_meta_extractor.py index 83c44b28..875e25e2 100644 --- a/src/dbx/pixels/wsi/wsi_meta_extractor.py +++ b/src/dbx/pixels/wsi/wsi_meta_extractor.py @@ -36,8 +36,6 @@ from pyspark.ml.pipeline import Transformer from pyspark.sql.functions import expr -from dbx.pixels.wsi.wsi_phi_tags import LARGE_TAGS, classify_tags - class WSIMetaExtractor(Transformer): """Extract metadata from any WSI file into the ``meta VARIANT`` column. @@ -164,7 +162,11 @@ def _process_tifffile(path: str, filter_large: bool = True) -> str: "_wsi_format": _detect_tifffile_format(tif), "_wsi_width": page.imagewidth, "_wsi_height": page.imagelength, - "_wsi_level_count": len(tif.series[0].levels) if tif.series and hasattr(tif.series[0], "levels") else len(tif.pages), + "_wsi_level_count": ( + len(tif.series[0].levels) + if tif.series and hasattr(tif.series[0], "levels") + else len(tif.pages) + ), "_wsi_level_dimensions": None, "_wsi_level_downsamples": None, "_wsi_mpp_x": None, @@ -229,8 +231,7 @@ def _transform(self, df): filter_large = self.filterLargeTags out_schema = t.StructType( - list(df.schema.fields) - + [t.StructField(output_col, t.StringType(), True)] + list(df.schema.fields) + [t.StructField(output_col, t.StringType(), True)] ) def _extract_meta_batch( @@ -263,6 +264,7 @@ def _extract_meta_batch( # Module-level helpers # --------------------------------------------------------------------------- + def _detect_format_name(path: str, vendor: str) -> str: """Return a human-readable format name based on vendor + extension.""" ext = path.rsplit(".", 1)[-1].lower() if "." in path else "" diff --git a/src/dbx/pixels/wsi/wsi_phi_tags.py b/src/dbx/pixels/wsi/wsi_phi_tags.py index 3241683f..b00edb1b 100644 --- a/src/dbx/pixels/wsi/wsi_phi_tags.py +++ b/src/dbx/pixels/wsi/wsi_phi_tags.py @@ -33,23 +33,31 @@ # Supported WSI file extensions (OpenSlide + tifffile fallback) # --------------------------------------------------------------------------- SUPPORTED_EXTENSIONS: set[str] = { - ".svs", # Aperio - ".tif", # Generic TIFF, Trestle, Ventana - ".tiff", # Generic TIFF, Philips - ".bif", # Ventana - ".ndpi", # Hamamatsu - ".mrxs", # MIRAX (index file) - ".vms", # Hamamatsu (virtual microscope slide) - ".vmu", # Hamamatsu (virtual microscope uncompressed) - ".scn", # Leica - ".svslide", # Sakura - ".dcm", # DICOM WSI + ".svs", # Aperio + ".tif", # Generic TIFF, Trestle, Ventana + ".tiff", # Generic TIFF, Philips + ".bif", # Ventana + ".ndpi", # Hamamatsu + ".mrxs", # MIRAX (index file) + ".vms", # Hamamatsu (virtual microscope slide) + ".vmu", # Hamamatsu (virtual microscope uncompressed) + ".scn", # Leica + ".svslide", # Sakura + ".dcm", # DICOM WSI } # Glob patterns for use with Spark binaryFile reader or Catalog.catalog() OPENSLIDE_PATTERNS: list[str] = [ - "*.svs", "*.tif", "*.tiff", "*.bif", "*.ndpi", - "*.mrxs", "*.vms", "*.vmu", "*.scn", "*.svslide", + "*.svs", + "*.tif", + "*.tiff", + "*.bif", + "*.ndpi", + "*.mrxs", + "*.vms", + "*.vmu", + "*.scn", + "*.svslide", ] # --------------------------------------------------------------------------- @@ -79,17 +87,17 @@ "Diagnosis", "Id", # --- Hamamatsu NDPI --- - "hamamatsu.SourceLens", # Operator-configured; may encode tech ID - "hamamatsu.Reference", # Patient/case reference + "hamamatsu.SourceLens", # Operator-configured; may encode tech ID + "hamamatsu.Reference", # Patient/case reference # --- Leica SCN --- - "leica.device-model", # Sometimes contains operator info + "leica.device-model", # Sometimes contains operator info # --- Philips --- - "philips.PIM_DP_UFS_BARCODE", # Barcode text (patient label) + "philips.PIM_DP_UFS_BARCODE", # Barcode text (patient label) # --- Sakura --- "sakura.NominalLensMagnification", # Technical, but barcode fields below # --- Generic TIFF tags (from tifffile) --- - "Artist", # tag 315 — person who created the image - "HostComputer", # tag 316 — workstation/operator ID + "Artist", # tag 315 — person who created the image + "HostComputer", # tag 316 — workstation/operator ID } # --------------------------------------------------------------------------- @@ -115,8 +123,8 @@ "User", "ImageID", # --- Hamamatsu --- - "hamamatsu.Created", # Scan timestamp - "hamamatsu.Updated", # Modification timestamp + "hamamatsu.Created", # Scan timestamp + "hamamatsu.Updated", # Modification timestamp # --- Leica --- "leica.creation-date", "leica.device-version", @@ -133,10 +141,10 @@ "mirax.GENERAL.SLIDE_NAME", # --- Generic TIFF --- "ImageDescription", # tag 270 — free-text; may embed patient info - "DateTime", # tag 306 — creation timestamp - "Copyright", # tag 33432 + "DateTime", # tag 306 — creation timestamp + "Copyright", # tag 33432 "DateTimeOriginal", # EXIF 36867 - "DateTimeDigitized", # EXIF 36868 + "DateTimeDigitized", # EXIF 36868 # --- OpenSlide common --- "openslide.comment", # May contain free-text with PHI # --- Artist/Copyright short keys --- @@ -230,6 +238,7 @@ # Public API # --------------------------------------------------------------------------- + def classify_tag(key: str) -> str: """Classify a single WSI metadata property key. diff --git a/src/dbx/pixels/wsi/wsi_utils.py b/src/dbx/pixels/wsi/wsi_utils.py index 6b68a4a0..2cfbe089 100644 --- a/src/dbx/pixels/wsi/wsi_utils.py +++ b/src/dbx/pixels/wsi/wsi_utils.py @@ -33,6 +33,7 @@ # Internal helpers # --------------------------------------------------------------------------- + def _normalize_to_uint8_rgb(arr: np.ndarray) -> np.ndarray: """Coerce any numpy array to uint8 RGB suitable for JPEG encoding. @@ -103,6 +104,7 @@ def _pil_to_output( # OpenSlide backend # --------------------------------------------------------------------------- + def _wsi_openslide( path: str, series: str = "tissue", @@ -145,6 +147,7 @@ def _wsi_openslide( # tifffile fallback # --------------------------------------------------------------------------- + def _wsi_tifffile( path: str, series: str = "tissue", @@ -205,6 +208,7 @@ def _wsi_tifffile( # Public API # --------------------------------------------------------------------------- + def wsi_to_image( path: str, max_width: int = 768, @@ -247,7 +251,6 @@ def wsi_to_image( "wsi_to_image: preferred openslide not available. " "Install openslide-python openslide-bin." ) - pass except Exception: pass @@ -270,6 +273,7 @@ def wsi_detect_format(path: str) -> Optional[str]: """ try: import openslide + return openslide.OpenSlide.detect_format(path) except ImportError: return None @@ -285,6 +289,7 @@ def wsi_get_properties(path: str) -> dict: """ try: import openslide + slide = openslide.OpenSlide(path) props = dict(slide.properties) slide.close() diff --git a/tests/dbx/test_wsi.py b/tests/dbx/test_wsi.py index 0106f761..768d7d87 100644 --- a/tests/dbx/test_wsi.py +++ b/tests/dbx/test_wsi.py @@ -79,11 +79,13 @@ def all_wsi_files(svs_files, tiff_files): # Test: phi_tags module # --------------------------------------------------------------------------- + class TestPhiTags: """Tests for wsi_phi_tags.py""" def test_classify_phi_tag(self): from dbx.pixels.wsi.wsi_phi_tags import classify_tag + assert classify_tag("aperio.Patient") == "PHI" assert classify_tag("Patient") == "PHI" assert classify_tag("Artist") == "PHI" @@ -91,6 +93,7 @@ def test_classify_phi_tag(self): def test_classify_questionable_tag(self): from dbx.pixels.wsi.wsi_phi_tags import classify_tag + assert classify_tag("aperio.Date") == "QUESTIONABLE" assert classify_tag("DateTime") == "QUESTIONABLE" assert classify_tag("ImageDescription") == "QUESTIONABLE" @@ -98,12 +101,14 @@ def test_classify_questionable_tag(self): def test_classify_not_phi_tag(self): from dbx.pixels.wsi.wsi_phi_tags import classify_tag + assert classify_tag("openslide.mpp-x") == "NOT_PHI" assert classify_tag("ImageWidth") == "NOT_PHI" assert classify_tag("SomeUnknownTag") == "NOT_PHI" def test_classify_tags_returns_only_phi_and_questionable(self): from dbx.pixels.wsi.wsi_phi_tags import classify_tags + props = { "aperio.Patient": "John Doe", "aperio.AppMag": "40", @@ -119,6 +124,7 @@ def test_classify_tags_returns_only_phi_and_questionable(self): def test_scrub_image_description_aperio_format(self): from dbx.pixels.wsi.wsi_phi_tags import scrub_image_description + desc = "Aperio Image Library v12.0.15|Patient = John Doe|AppMag = 40|Date = 2024-01-01" scrubbed = scrub_image_description(desc) assert "John Doe" not in scrubbed @@ -127,6 +133,7 @@ def test_scrub_image_description_aperio_format(self): def test_supported_extensions(self): from dbx.pixels.wsi.wsi_phi_tags import SUPPORTED_EXTENSIONS + assert ".svs" in SUPPORTED_EXTENSIONS assert ".tiff" in SUPPORTED_EXTENSIONS assert ".bif" in SUPPORTED_EXTENSIONS @@ -135,6 +142,7 @@ def test_supported_extensions(self): def test_openslide_patterns(self): from dbx.pixels.wsi.wsi_phi_tags import OPENSLIDE_PATTERNS + assert "*.svs" in OPENSLIDE_PATTERNS assert "*.tiff" in OPENSLIDE_PATTERNS @@ -143,11 +151,13 @@ def test_openslide_patterns(self): # Test: wsi_utils module # --------------------------------------------------------------------------- + class TestWSIUtils: """Tests for wsi_utils.py — requires real files.""" def test_detect_format_svs(self, svs_files): from dbx.pixels.wsi.wsi_utils import wsi_detect_format + # All SVS files should be detected as 'aperio' for f in svs_files: fmt = wsi_detect_format(f) @@ -155,6 +165,7 @@ def test_detect_format_svs(self, svs_files): def test_detect_format_tiff(self, tiff_files): from dbx.pixels.wsi.wsi_utils import wsi_detect_format + # TIFF files should be detected as some OpenSlide vendor or None for f in tiff_files: fmt = wsi_detect_format(f) @@ -163,6 +174,7 @@ def test_detect_format_tiff(self, tiff_files): def test_get_properties_svs(self, svs_files): from dbx.pixels.wsi.wsi_utils import wsi_get_properties + props = wsi_get_properties(svs_files[0]) assert len(props) > 0 assert "openslide.vendor" in props @@ -170,6 +182,7 @@ def test_get_properties_svs(self, svs_files): def test_wsi_to_image_tissue_svs(self, svs_files): from dbx.pixels.wsi.wsi_utils import wsi_to_image + result = wsi_to_image(svs_files[0], max_width=256, series="tissue") assert result is not None assert isinstance(result, str) # base64 string @@ -177,6 +190,7 @@ def test_wsi_to_image_tissue_svs(self, svs_files): def test_wsi_to_image_label_svs(self, svs_files): from dbx.pixels.wsi.wsi_utils import wsi_to_image + # CMU-1.svs has a label image cmu1 = next((f for f in svs_files if "CMU-1.svs" in f), svs_files[0]) result = wsi_to_image(cmu1, max_width=256, series="label") @@ -185,6 +199,7 @@ def test_wsi_to_image_label_svs(self, svs_files): def test_wsi_to_image_macro_svs(self, svs_files): from dbx.pixels.wsi.wsi_utils import wsi_to_image + cmu1 = next((f for f in svs_files if "CMU-1.svs" in f), svs_files[0]) result = wsi_to_image(cmu1, max_width=256, series="macro") assert result is not None @@ -192,6 +207,7 @@ def test_wsi_to_image_macro_svs(self, svs_files): def test_wsi_to_image_binary_output(self, svs_files): from dbx.pixels.wsi.wsi_utils import wsi_to_image + result = wsi_to_image(svs_files[0], max_width=128, return_type="binary") assert result is not None assert isinstance(result, bytes) @@ -199,6 +215,7 @@ def test_wsi_to_image_binary_output(self, svs_files): def test_wsi_to_image_tiff(self, tiff_files): from dbx.pixels.wsi.wsi_utils import wsi_to_image + result = wsi_to_image(tiff_files[0], max_width=256, series="tissue") assert result is not None assert isinstance(result, str) @@ -208,11 +225,13 @@ def test_wsi_to_image_tiff(self, tiff_files): # Test: WSIMetaExtractor module (standalone, no Spark) # --------------------------------------------------------------------------- + class TestWSIMetaExtractorStandalone: """Tests for the static extraction methods (no Spark required).""" def test_process_openslide_svs(self, svs_files): from dbx.pixels.wsi.wsi_meta_extractor import WSIMetaExtractor + result = WSIMetaExtractor._process_openslide(svs_files[0]) meta = json.loads(result) assert "error" not in meta, f"Error: {meta.get('error')}" @@ -227,6 +246,7 @@ def test_process_openslide_svs(self, svs_files): def test_process_openslide_has_mpp(self, svs_files): from dbx.pixels.wsi.wsi_meta_extractor import WSIMetaExtractor + result = WSIMetaExtractor._process_openslide(svs_files[0]) meta = json.loads(result) # CMU SVS files have MPP defined @@ -234,6 +254,7 @@ def test_process_openslide_has_mpp(self, svs_files): def test_process_openslide_svs_associated_images(self, svs_files): from dbx.pixels.wsi.wsi_meta_extractor import WSIMetaExtractor + # CMU-1.svs should have label and macro cmu1 = next((f for f in svs_files if "CMU-1.svs" in f), svs_files[0]) result = WSIMetaExtractor._process_openslide(cmu1) @@ -243,12 +264,14 @@ def test_process_openslide_svs_associated_images(self, svs_files): def test_process_file_dispatches_to_openslide_for_svs(self, svs_files): from dbx.pixels.wsi.wsi_meta_extractor import WSIMetaExtractor + result = WSIMetaExtractor._process_file(svs_files[0]) meta = json.loads(result) assert meta["_wsi_backend"] == "openslide" def test_process_tifffile_fallback(self, tiff_files): from dbx.pixels.wsi.wsi_meta_extractor import WSIMetaExtractor + # Force tifffile path result = WSIMetaExtractor._process_tifffile(tiff_files[0]) meta = json.loads(result) @@ -258,8 +281,10 @@ def test_process_tifffile_fallback(self, tiff_files): assert meta["_wsi_height"] > 0 def test_process_file_tiff_uses_openslide_when_possible(self, tiff_files): - from dbx.pixels.wsi.wsi_meta_extractor import WSIMetaExtractor import openslide + + from dbx.pixels.wsi.wsi_meta_extractor import WSIMetaExtractor + # Find a TIFF that OpenSlide recognizes for f in tiff_files: fmt = openslide.OpenSlide.detect_format(f) @@ -272,6 +297,7 @@ def test_process_file_tiff_uses_openslide_when_possible(self, tiff_files): def test_all_svs_files_extract_without_error(self, svs_files): from dbx.pixels.wsi.wsi_meta_extractor import WSIMetaExtractor + for f in svs_files: result = WSIMetaExtractor._process_file(f) meta = json.loads(result) @@ -279,6 +305,7 @@ def test_all_svs_files_extract_without_error(self, svs_files): def test_all_tiff_files_extract_without_error(self, tiff_files): from dbx.pixels.wsi.wsi_meta_extractor import WSIMetaExtractor + for f in tiff_files: result = WSIMetaExtractor._process_file(f) meta = json.loads(result) @@ -289,11 +316,13 @@ def test_all_tiff_files_extract_without_error(self, tiff_files): # Test: Format constants and patterns # --------------------------------------------------------------------------- + class TestConstants: """Verify module constants are consistent.""" def test_openslide_patterns_match_extensions(self): from dbx.pixels.wsi.wsi_phi_tags import OPENSLIDE_PATTERNS, SUPPORTED_EXTENSIONS + for pat in OPENSLIDE_PATTERNS: ext = "." + pat.lstrip("*.") assert ext in SUPPORTED_EXTENSIONS, f"Pattern {pat} not in SUPPORTED_EXTENSIONS" @@ -303,18 +332,7 @@ def test_imports(self): from dbx.pixels.wsi import ( WSICatalog, WSIMetaExtractor, - wsi_to_image, - wsi_detect_format, - wsi_get_properties, - classify_tag, - classify_tags, - scrub_image_description, - PHI_TAGS, - QUESTIONABLE_TAGS, - NOT_PHI_TAGS, - LARGE_TAGS, - SUPPORTED_EXTENSIONS, - OPENSLIDE_PATTERNS, ) + assert WSICatalog is not None assert WSIMetaExtractor is not None