Register CLEM colours (#739)

* Added logic to optionally insert collection mode and colour flags for the Atlas and GridSquare tables in ISPyB for the CLEM workflow
* Added column to CLEMImageSeries table to keep track of collection mode used for dataset
* Register colours and collection mode in CLEMImageSeries table upon arrival of message, then translate it across to Atlas and GridSquare tables in ISPyB
* Fixed error with calculating the location of ROIs on the atlas image

---------

Co-authored-by: Daniel Hatton <daniel.hatton@diamond.ac.uk>

Author: tieneupin

src/murfey/server/ispyb.py

@@ -186,13 +186,21 @@ def do_update_atlas(
         atlas_image: str,
         pixel_size: float,
         slot: int | None,
+        collection_mode: str | None = None,
+        color_flags: dict[str, str | int] | None = None,
     ):
+        color_flags = color_flags or {}
         try:
             with ISPyBSession() as db:
                 atlas = db.query(Atlas).filter(Atlas.atlasId == atlas_id).one()
                 atlas.atlasImage = atlas_image or atlas.atlasImage
                 atlas.pixelSize = pixel_size or atlas.pixelSize
                 atlas.cassetteSlot = slot or atlas.cassetteSlot
+                atlas.mode = collection_mode or atlas.mode
+                # Optionally insert colour flags if present
+                if color_flags:
+                    for col_name, value in color_flags.items():
+                        setattr(atlas, col_name, value)
                 db.add(atlas)
                 db.commit()
                 return {"success": True, "return_value": atlas.atlasId}
@@ -209,7 +217,9 @@ def do_insert_grid_square(
         atlas_id: int,
         grid_square_id: int,
         grid_square_parameters: GridSquareParameters,
+        color_flags: dict[str, int] | None = None,
     ):
+        color_flags = color_flags or {}
         # most of this is for mypy
         if (
             grid_square_parameters.pixel_size is not None
@@ -234,7 +244,12 @@ def do_insert_grid_square(
             stageLocationX=grid_square_parameters.x_stage_position,
             stageLocationY=grid_square_parameters.y_stage_position,
             pixelSize=grid_square_parameters.pixel_size,
+            mode=grid_square_parameters.collection_mode,
         )
+        # Optionally insert colour flags
+        if color_flags:
+            for col_name, value in color_flags.items():
+                setattr(record, col_name, value)
         try:
             with ISPyBSession() as db:
                 db.add(record)
@@ -250,8 +265,12 @@ def do_insert_grid_square(
         return {"success": False, "return_value": None}
 
     def do_update_grid_square(
-        self, grid_square_id: int, grid_square_parameters: GridSquareParameters
+        self,
+        grid_square_id: int,
+        grid_square_parameters: GridSquareParameters,
+        color_flags: dict[str, int] | None = None,
     ):
+        color_flags = color_flags or {}
         try:
             with ISPyBSession() as db:
                 grid_square: GridSquare = (
@@ -290,6 +309,12 @@ def do_update_grid_square(
                     grid_square.stageLocationY = grid_square_parameters.y_stage_position
                 if grid_square_parameters.pixel_size:
                     grid_square.pixelSize = grid_square_parameters.pixel_size
+                if grid_square_parameters.collection_mode:
+                    grid_square.mode = grid_square_parameters.collection_mode
+                # Optionally insert colour flags
+                if color_flags:
+                    for col_name, value in color_flags.items():
+                        setattr(grid_square, col_name, value)
                 db.add(grid_square)
                 db.commit()
                 return {"success": True, "return_value": grid_square.gridSquareId}

src/murfey/util/db.py

@@ -294,6 +294,14 @@ class CLEMImageSeries(SQLModel, table=True):  # type: ignore
         sa_relationship_kwargs={"cascade": "delete"},
     )  # One to many
     number_of_members: Optional[int] = Field(default=None)
+    has_grey: Optional[bool] = Field(default=None)
+    has_red: Optional[bool] = Field(default=None)
+    has_green: Optional[bool] = Field(default=None)
+    has_blue: Optional[bool] = Field(default=None)
+    has_cyan: Optional[bool] = Field(default=None)
+    has_magenta: Optional[bool] = Field(default=None)
+    has_yellow: Optional[bool] = Field(default=None)
+    collection_mode: Optional[str] = Field(default=None)
 
     # Shape and resolution information
     image_pixels_x: Optional[int] = Field(default=None)

src/murfey/util/models.py

@@ -157,6 +157,9 @@ class GridSquareParameters(BaseModel):
     pixel_size: Optional[float] = None
     angle: Optional[float] = None
 
+    # Collection mode
+    collection_mode: Optional[str] = None
+
 
 class FoilHoleParameters(BaseModel):
     tag: str

src/murfey/workflows/clem/register_preprocessing_results.py

@@ -11,6 +11,7 @@
 import logging
 import re
 import traceback
+from collections.abc import Collection
 from importlib.metadata import entry_points
 from pathlib import Path
 from typing import Literal, Optional
@@ -66,6 +67,27 @@ def _is_clem_atlas(result: CLEMPreprocessingResult):
     )
 
 
+COLOR_FLAGS_MURFEY = {
+    "gray": "has_grey",
+    "red": "has_red",
+    "green": "has_green",
+    "blue": "has_blue",
+    "cyan": "has_cyan",
+    "magenta": "has_magenta",
+    "yellow": "has_yellow",
+}
+
+
+def _get_color_flags(
+    colors: Collection[str] | None = None,
+):
+    colors = colors or []
+    color_flags = dict.fromkeys(COLOR_FLAGS_MURFEY.values(), False)
+    for color in colors:
+        color_flags[COLOR_FLAGS_MURFEY[color]] = True
+    return color_flags
+
+
 def _register_clem_image_series(
     session_id: int,
     result: CLEMPreprocessingResult,
@@ -159,6 +181,11 @@ def _register_clem_image_series(
     clem_img_series.image_search_string = str(output_file.parent / "*tiff")
     clem_img_series.data_type = "atlas" if _is_clem_atlas(result) else "grid_square"
     clem_img_series.number_of_members = result.number_of_members
+    for col_name, value in _get_color_flags(result.output_files.keys()).items():
+        setattr(clem_img_series, col_name, value)
+    clem_img_series.collection_mode = _determine_collection_mode(
+        result.output_files.keys()
+    )
     clem_img_series.image_pixels_x = result.pixels_x
     clem_img_series.image_pixels_y = result.pixels_y
     clem_img_series.image_pixel_size = result.pixel_size
@@ -186,6 +213,31 @@ def _register_clem_image_series(
     logger.info(f"CLEM preprocessing results registered for {result.series_name!r} ")
 
 
+def _determine_collection_mode(
+    colors: Collection[str] | None = None,
+):
+    if not colors:
+        logger.warning("No colours were present in returned result")
+        return None
+    if "gray" in colors:
+        if len(colors) == 1:
+            return "Bright Field"
+        else:
+            return "Bright Field and Fluorescent"
+    else:
+        return "Fluorescent"
+
+
+def _snake_to_camel_case(string: str):
+    parts = string.split("_")
+    return parts[0] + "".join(part.capitalize() for part in parts[1:])
+
+
+COLOR_FLAGS_MURFEY_TO_ISPYB = {
+    value: _snake_to_camel_case(value) for value in COLOR_FLAGS_MURFEY.values()
+}
+
+
 def _register_dcg_and_atlas(
     session_id: int,
     instrument_name: str,
@@ -225,9 +277,17 @@ def _register_dcg_and_atlas(
         else:
             atlas_name = str(output_file.parent / "*.tiff")
             atlas_pixel_size = result.pixel_size
+        # Translate colour flags into ISPyB convention
+        color_flags = {
+            COLOR_FLAGS_MURFEY_TO_ISPYB[key]: int(value)
+            for key, value in _get_color_flags(result.output_files.keys()).items()
+        }
+        collection_mode = _determine_collection_mode(result.output_files.keys())
     else:
         atlas_name = ""
         atlas_pixel_size = 0.0
+        color_flags = None
+        collection_mode = None
 
     if dcg_search := murfey_db.exec(
         select(MurfeyDB.DataCollectionGroup)
@@ -245,6 +305,8 @@ def _register_dcg_and_atlas(
                 "atlas": atlas_name,
                 "atlas_pixel_size": atlas_pixel_size,
                 "sample": dcg_entry.sample,
+                "color_flags": color_flags,
+                "collection_mode": collection_mode,
             }
             if entry_point_result := entry_points(
                 group="murfey.workflows", name="atlas_update"
@@ -269,6 +331,8 @@ def _register_dcg_and_atlas(
             "atlas": atlas_name,
             "atlas_pixel_size": atlas_pixel_size,
             "sample": None,
+            "color_flags": color_flags,
+            "collection_mode": collection_mode,
         }
         if entry_point_result := entry_points(
             group="murfey.workflows", name="data_collection_group"
@@ -342,6 +406,8 @@ def _register_grid_square(
             and atlas_entry.x1 is not None
             and atlas_entry.y0 is not None
             and atlas_entry.y1 is not None
+            and atlas_entry.thumbnail_pixels_x is not None
+            and atlas_entry.thumbnail_pixels_y is not None
         ):
             atlas_width_real = atlas_entry.x1 - atlas_entry.x0
             atlas_height_real = atlas_entry.y1 - atlas_entry.y0
@@ -356,34 +422,31 @@ def _register_grid_square(
                 and clem_img_series.x1 is not None
                 and clem_img_series.y0 is not None
                 and clem_img_series.y1 is not None
-                and clem_img_series.thumbnail_pixels_x is not None
-                and clem_img_series.thumbnail_pixels_y is not None
-                and clem_img_series.thumbnail_pixel_size is not None
             ):
                 # Find pixel corresponding to image midpoint on atlas
                 x_mid_real = (
                     0.5 * (clem_img_series.x0 + clem_img_series.x1) - atlas_entry.x0
                 )
                 x_mid_px = int(
-                    x_mid_real / atlas_width_real * clem_img_series.thumbnail_pixels_x
+                    x_mid_real / atlas_width_real * atlas_entry.thumbnail_pixels_x
                 )
                 y_mid_real = (
                     0.5 * (clem_img_series.y0 + clem_img_series.y1) - atlas_entry.y0
                 )
                 y_mid_px = int(
-                    y_mid_real / atlas_height_real * clem_img_series.thumbnail_pixels_y
+                    y_mid_real / atlas_height_real * atlas_entry.thumbnail_pixels_y
                 )
 
-                # Find the size of the image, in pixels, when overlaid the atlas
+                # Find the size of the image, in pixels, when overlaid on the atlas
                 width_scaled = int(
                     (clem_img_series.x1 - clem_img_series.x0)
                     / atlas_width_real
-                    * clem_img_series.thumbnail_pixels_x
+                    * atlas_entry.thumbnail_pixels_x
                 )
                 height_scaled = int(
                     (clem_img_series.y1 - clem_img_series.y0)
                     / atlas_height_real
-                    * clem_img_series.thumbnail_pixels_y
+                    * atlas_entry.thumbnail_pixels_y
                 )
             else:
                 logger.warning(
@@ -410,7 +473,13 @@ def _register_grid_square(
                 y_stage_position=0.5 * (clem_img_series.y0 + clem_img_series.y1),
                 pixel_size=clem_img_series.image_pixel_size,
                 image=clem_img_series.thumbnail_search_string,
+                collection_mode=clem_img_series.collection_mode,
             )
+            # Construct colour flags for ISPyB
+            color_flags = {
+                ispyb_color_flags: int(getattr(clem_img_series, murfey_color_flags, 0))
+                for murfey_color_flags, ispyb_color_flags in COLOR_FLAGS_MURFEY_TO_ISPYB.items()
+            }
             # Register or update the grid square entry as required
             if grid_square_result := murfey_db.exec(
                 select(MurfeyDB.GridSquare)
@@ -435,6 +504,7 @@ def _register_grid_square(
                 _transport_object.do_update_grid_square(
                     grid_square_id=grid_square_entry.id,
                     grid_square_parameters=grid_square_params,
+                    color_flags=color_flags,
                 )
             else:
                 # Look up data collection group for current series
@@ -448,6 +518,7 @@ def _register_grid_square(
                     atlas_id=dcg_entry.atlas_id,
                     grid_square_id=clem_img_series.id,
                     grid_square_parameters=grid_square_params,
+                    color_flags=color_flags,
                 )
                 # Register to Murfey
                 grid_square_entry = MurfeyDB.GridSquare(

src/murfey/workflows/register_atlas_update.py

@@ -19,10 +19,13 @@ def run(
     logger.info(f"Registering updated atlas: \n{message}")
 
     _transport_object.do_update_atlas(
-        message["atlas_id"],
-        message["atlas"],
-        message["atlas_pixel_size"],
-        message["sample"],
+        atlas_id=message["atlas_id"],
+        atlas_image=message["atlas"],
+        pixel_size=message["atlas_pixel_size"],
+        slot=message["sample"],
+        # Extract optional parameters
+        collection_mode=message.get("collection_mode"),
+        color_flags=message.get("color_flags", {}),
     )
     if dcg_hooks := entry_points(group="murfey.hooks", name="data_collection_group"):
         try:

src/murfey/workflows/register_data_collection_group.py

@@ -67,6 +67,12 @@ def run(message: dict, murfey_db: SQLModelSession) -> dict[str, bool]:
                 pixelSize=message.get("atlas_pixel_size", 0),
                 cassetteSlot=message.get("sample"),
             )
+            # Optionally set the collection mode and color flags
+            if collection_mode := message.get("collection_mode"):
+                atlas_record.mode = collection_mode
+            if color_flags := message.get("color_flags", {}):
+                for col_name, value in color_flags.items():
+                    setattr(atlas_record, col_name, value)
             atlas_id = _transport_object.do_insert_atlas(atlas_record).get(
                 "return_value", None
             )