From af16aaae25b1818875bd5aa480b8a913ab5e469a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joakim=20M=C3=B6ller?= Date: Mon, 4 May 2026 00:03:02 +0200 Subject: [PATCH 1/3] Fix pinning format propagation in scanner compilation Pass scan pinning_formats through to compile jobs via overwrite_pinning_matrices so downstream compilation uses the intended pinning matrices. Add regression tests for ScannerEffector setup and compile payload propagation. Bump project version to 3.1.1 and update uv.lock. --- pyproject.toml | 2 +- scanomatic/server/scanning_effector.py | 1 + tests/unit/server/test_scanning_effector.py | 107 ++++++++++++++++++++ uv.lock | 2 +- 4 files changed, 110 insertions(+), 2 deletions(-) create mode 100644 tests/unit/server/test_scanning_effector.py diff --git a/pyproject.toml b/pyproject.toml index 0e550095..21d7975c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "uv_build" [project] name = "scanomatic-standalone" -version = "3.1.0" +version = "3.1.1" description = "High Throughput Solid Media Image Phenotyping Platform" readme = "README.md" requires-python = ">=3.11,<3.14" diff --git a/scanomatic/server/scanning_effector.py b/scanomatic/server/scanning_effector.py index b2e0d203..bceb800e 100644 --- a/scanomatic/server/scanning_effector.py +++ b/scanomatic/server/scanning_effector.py @@ -315,6 +315,7 @@ def setup(self, job, redirect_logging=True): ), fixture_type=FIXTURE.Global, fixture_name=self._scanning_job.fixture, + overwrite_pinning_matrices=self._scanning_job.pinning_formats, cell_count_calibration_id=( self._scanning_job.cell_count_calibration_id ), diff --git a/tests/unit/server/test_scanning_effector.py b/tests/unit/server/test_scanning_effector.py new file mode 100644 index 00000000..bc87a873 --- /dev/null +++ b/tests/unit/server/test_scanning_effector.py @@ -0,0 +1,107 @@ +import os + +from scanomatic.io.jsonizer import dumps +from scanomatic.models.factories.rpc_job_factory import RPC_Job_Model_Factory +from scanomatic.models.factories.scanning_factory import ScanningModelFactory +from scanomatic.models.scanning_model import SCAN_STEP +from scanomatic.server.scanning_effector import ScannerEffector + + +class _FakeSaneBase: + def __init__(self, scan_mode, model): + self.scan_mode = scan_mode + self.model = model + + @staticmethod + def get_scan_program(): + return "fake-scan" + + @staticmethod + def get_program_version(): + return "0.0" + + def get_scan_instructions_as_tuple(self): + return ("--dpi", "600") + + +class _FakeRPCClient: + def __init__(self, compile_job_id="compile-job"): + self.compile_job_id = compile_job_id + self.payloads = [] + + def create_compile_project_job(self, payload): + self.payloads.append(payload) + return self.compile_job_id + + +def _make_scan_job(tmp_path, pinning_formats): + scan_model = ScanningModelFactory.create( + project_name="pinning-regression", + directory_containing_project=str(tmp_path), + fixture="fixture-1", + scanner=1, + scanner_hardware="EPSON V800", + pinning_formats=pinning_formats, + number_of_scans=2, + ) + return RPC_Job_Model_Factory.create(id="scan-job", content_model=scan_model) + + +def test_setup_copies_scan_pinning_formats_to_compile_model(tmp_path, monkeypatch): + fake_rpc_client = _FakeRPCClient() + monkeypatch.setattr( + "scanomatic.server.scanning_effector.rpc_client.get_client", + lambda: fake_rpc_client, + ) + monkeypatch.setattr( + "scanomatic.server.scanning_effector.sane.SaneBase", + _FakeSaneBase, + ) + + pinning_formats = ((8, 12), (16, 24), None) + job = _make_scan_job(tmp_path, pinning_formats) + effector = ScannerEffector(job) + + effector.setup(dumps(job)) + + assert ( + effector._scanning_effector_data.compile_project_model + .overwrite_pinning_matrices + == pinning_formats + ) + + +def test_compile_request_payload_contains_scan_pinning_formats( + tmp_path, + monkeypatch, +): + fake_rpc_client = _FakeRPCClient(compile_job_id="compile-123") + monkeypatch.setattr( + "scanomatic.server.scanning_effector.rpc_client.get_client", + lambda: fake_rpc_client, + ) + monkeypatch.setattr( + "scanomatic.server.scanning_effector.sane.SaneBase", + _FakeSaneBase, + ) + + pinning_formats = ((8, 12), None, (32, 48)) + job = _make_scan_job(tmp_path, pinning_formats) + effector = ScannerEffector(job) + + effector.setup(dumps(job)) + effector._scanning_effector_data.current_image = 0 + + step = effector._do_request_project_compilation() + + assert step == SCAN_STEP.NextMajor + assert len(fake_rpc_client.payloads) == 1 + assert ( + fake_rpc_client.payloads[0]["overwrite_pinning_matrices"] + == pinning_formats + ) + assert ( + effector._scanning_effector_data.compile_project_model.start_condition + == "compile-123" + ) + assert os.path.isdir(tmp_path / "pinning-regression") diff --git a/uv.lock b/uv.lock index 95976898..d3ce4c90 100644 --- a/uv.lock +++ b/uv.lock @@ -967,7 +967,7 @@ wheels = [ [[package]] name = "scanomatic-standalone" -version = "3.1.0" +version = "3.1.1" source = { editable = "." } dependencies = [ { name = "chardet" }, From 1af0465b81681eb06f2a2fbb132360cd3b0fb35f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joakim=20M=C3=B6ller?= Date: Mon, 4 May 2026 00:16:36 +0200 Subject: [PATCH 2/3] Harden scanner effector typing and runtime guards Run a focused ty-fix round for scan compilation flow and related models/tests. - Fix scanner mail helper to return str (not tuple) - Correct scan cycle disk-space step mapping - Add optional-safety guards for scanner, pipe effector, compile model and current image - Align model typing for optional current_image and mutable compile image list - Keep ScannerEffector/tests ty-clean while preserving scan behavior - Verify with pytest tests/unit/server/test_scanning_effector.py and focused ty checks --- scanomatic/models/compile_project_model.py | 5 +- scanomatic/models/scanning_model.py | 6 +- scanomatic/server/scanning_effector.py | 109 ++++++++++++-------- tests/unit/server/test_scanning_effector.py | 9 +- 4 files changed, 81 insertions(+), 48 deletions(-) diff --git a/scanomatic/models/compile_project_model.py b/scanomatic/models/compile_project_model.py index 464633eb..f08f573d 100644 --- a/scanomatic/models/compile_project_model.py +++ b/scanomatic/models/compile_project_model.py @@ -1,5 +1,6 @@ from enum import Enum, auto from collections.abc import Sequence +from typing import Optional from scanomatic.generics.model import Model from scanomatic.models.fixture_models import FixtureModel @@ -35,7 +36,7 @@ def __init__( self, compile_action=COMPILE_ACTION.InitiateAndSpawnAnalysis, start_time=0.0, - images=tuple(), + images: Optional[Sequence["CompileImageModel"]] = None, path="", start_condition="", fixture_type=FIXTURE.Local, @@ -45,7 +46,7 @@ def __init__( cell_count_calibration_id="default", ): self.compile_action: COMPILE_ACTION = compile_action - self.images: Sequence[CompileImageModel] = images + self.images: list[CompileImageModel] = list(images or ()) self.path: str = path self.start_time: float = start_time self.start_condition: str = start_condition diff --git a/scanomatic/models/scanning_model.py b/scanomatic/models/scanning_model.py index 91144c6c..2237bc0e 100644 --- a/scanomatic/models/scanning_model.py +++ b/scanomatic/models/scanning_model.py @@ -277,7 +277,7 @@ def __init__( *, current_cycle_step: SCAN_CYCLE = SCAN_CYCLE.Wait, current_step_start_time: float = -1, - current_image: int = -1, + current_image: Optional[int] = -1, current_image_path: str = "", current_image_path_pattern: str = "", previous_scan_cycle_start: float = -1.0, @@ -287,7 +287,7 @@ def __init__( usb_port: str = "", scanning_thread=None, scan_success: bool = False, - compile_project_model: CompileInstructionsModel, + compile_project_model: Optional[CompileInstructionsModel] = None, known_file_size=0, warned_file_size: bool = False, warned_scanner_error: bool = False, @@ -299,7 +299,7 @@ def __init__( ): self.current_cycle_step: SCAN_CYCLE = current_cycle_step self.current_step_start_time: float = current_step_start_time - self.current_image: int = current_image + self.current_image: Optional[int] = current_image self.current_image_path: str = current_image_path self.current_image_path_pattern: str = current_image_path_pattern self.previous_scan_cycle_start: float = previous_scan_cycle_start diff --git a/scanomatic/server/scanning_effector.py b/scanomatic/server/scanning_effector.py index bceb800e..12c7eedc 100644 --- a/scanomatic/server/scanning_effector.py +++ b/scanomatic/server/scanning_effector.py @@ -1,7 +1,7 @@ import os import time from threading import Thread -from typing import Union +from typing import Any, Optional, Union, cast from scanomatic.io import rpc_client from scanomatic.io import paths, sane, scanner_manager @@ -218,8 +218,7 @@ def som_mail_body_scanning_soon_done(seconds_left: float) -> str: All the best, -Scan-o-Matic""", - +Scan-o-Matic""" ) @@ -246,8 +245,8 @@ def __init__(self, job): self._specific_statuses['total'] = 'total_images' self._specific_statuses['currentImage'] = 'current_image' - self._allowed_calls['setup'] = self.setup - self._allowed_calls[JOBS_CALL_SET_USB] = self._set_usb_port + self._allowed_calls['setup'] = cast(Any, self.setup) + self._allowed_calls[JOBS_CALL_SET_USB] = cast(Any, self._set_usb_port) self._scanning_effector_data = ScanningModelEffectorData( compile_project_model=None @@ -268,7 +267,7 @@ def __init__(self, job): SCAN_CYCLE.WaitForScanComplete: self._do_wait_for_scan, SCAN_CYCLE.WaitForUSB: self._do_wait_for_usb, SCAN_CYCLE.VerifyImageSize: self._do_verify_image_size, - SCAN_CYCLE.VerifyDiskspace: self._do_verify_image_size + SCAN_CYCLE.VerifyDiskspace: self._do_verify_discspace, } @property @@ -284,10 +283,10 @@ def label(self) -> str: self._scanning_job.scanner, time_left) - def setup(self, job, redirect_logging=True): + def setup(self, job): job: RPCjobModel = loads(job) paths_object = paths.Paths() - self._scanning_job.id = job.id + self._scanning_job.id = job.id or "" self._scanning_job.computer = AppConfig().computer_human_name self._setup_directory() @@ -332,9 +331,10 @@ def setup(self, job, redirect_logging=True): ) self._scanning_job.scanning_program = sane.SaneBase.get_scan_program() - self._scanning_job.scanning_program_version = ( - sane.SaneBase.get_program_version() - ) + scan_program_version = sane.SaneBase.get_program_version() + if isinstance(scan_program_version, bytes): + scan_program_version = scan_program_version.decode("utf-8") + self._scanning_job.scanning_program_version = scan_program_version # NOTE: In actual scanning the scanner USB setting is prepended to the # settings @@ -404,11 +404,11 @@ def total_images(self) -> int: return self._scanning_job.number_of_scans @property - def current_image(self) -> int: + def current_image(self) -> Optional[int]: return self._scanning_effector_data.current_image - def __next__(self) -> Union[bool, SCAN_CYCLE]: + def __next__(self) -> bool: if self.waiting: return super().__next__() elif not self._stopping: @@ -485,11 +485,13 @@ def __next__(self) -> Union[bool, SCAN_CYCLE]: == SCAN_CYCLE.Wait ): self._stopping = True + compile_model = self._scanning_effector_data.compile_project_model if ( self._scanning_effector_data.compilation_state is not COMPILE_STATE.Finalized + and compile_model is not None ): - self._scanning_effector_data.compile_project_model.compile_action = ( # noqa: E501 + compile_model.compile_action = ( COMPILE_ACTION.AppendAndSpawnAnalysis if self._scanning_effector_data.compilation_state is COMPILE_STATE.Initialized @@ -500,13 +502,13 @@ def __next__(self) -> Union[bool, SCAN_CYCLE]: raise StopIteration else: - return self._scanning_effector_data.current_cycle_step + return True @property def _job_completed(self) -> bool: return ( - self.current_image >= self._scanning_job.number_of_scans - or self.current_image is None + self.current_image is None + or self.current_image >= self._scanning_job.number_of_scans ) def _get_step_to_next_scan_cycle_step(self) -> SCAN_STEP: @@ -521,7 +523,10 @@ def _get_step_to_next_scan_cycle_step(self) -> SCAN_STEP: def _update_scan_cycle_step(self, step_action): if step_action is SCAN_STEP.NextMajor: self._scanning_effector_data.current_cycle_step = ( - self._scanning_effector_data.current_cycle_step.next_major + cast( + SCAN_CYCLE, + self._scanning_effector_data.current_cycle_step.next_major, + ) ) self._logger.info( "Entering step {0}".format( @@ -531,7 +536,10 @@ def _update_scan_cycle_step(self, step_action): elif step_action is SCAN_STEP.NextMinor: self._scanning_effector_data.current_cycle_step = ( - self._scanning_effector_data.current_cycle_step.next_minor + cast( + SCAN_CYCLE, + self._scanning_effector_data.current_cycle_step.next_minor, + ) ) elif step_action is SCAN_STEP.TruncateIteration: @@ -551,7 +559,7 @@ def _update_scan_cycle_step(self, step_action): self._scanning_effector_data.current_step_start_time = time.time() def _do_wait(self): - if self.current_image < 0: + if self.current_image is None or self.current_image < 0: self._start_time = time.time() self._scanning_effector_data.previous_scan_cycle_start = ( self.run_time @@ -630,7 +638,7 @@ def _do_wait_for_scan(self): self._logger.warning("Scan completed, but not successfully.") self._mail( "Scan-o-Matic: '{project_name}' error while scanning", - som_mail_body_scan_fail(self.current_image), + som_mail_body_scan_fail(self.current_image or 0), self._scanning_job, ) @@ -652,7 +660,8 @@ def _do_wait_for_scan(self): @property def _scan_completed(self) -> bool: - return not self._scanning_effector_data.scanning_thread.is_alive() + scanning_thread = self._scanning_effector_data.scanning_thread + return bool(scanning_thread and not scanning_thread.is_alive()) def _do_report_error_scanning(self): @@ -771,12 +780,12 @@ def get_size_of_last_image(): self._scanning_effector_data.warned_file_size = True self._mail( "Scan-o-Matic: Project '{project_name}' got suspicious image", # noqa: E501 - self._scanning_job, som_mail_body_image_suspicious( self._scanning_effector_data.current_image_path, current_size, self._scanning_effector_data.known_file_size, - ) + ), + self._scanning_job, ) return SCAN_STEP.TruncateIteration @@ -793,7 +802,9 @@ def get_size_of_last_image(): return SCAN_STEP.NextMinor def _removed_current_image(self): - del self._scanning_effector_data.compile_project_model.images[-1] + compile_model = self._scanning_effector_data.compile_project_model + if compile_model is not None and compile_model.images: + compile_model.images.pop() try: os.remove(self._scanning_effector_data.current_image_path) except OSError: @@ -814,6 +825,7 @@ def get_free_space(): if ( self._scanning_effector_data.known_file_size and not self._scanning_effector_data.warned_discspace + and self._scanning_effector_data.current_image is not None ): bytes_needed = ( @@ -846,6 +858,9 @@ def _do_request_scanner_on(self) -> SCAN_STEP: self._logger.info("Job {0} requested scanner on".format( self._scanning_job.id, )) + if self.pipe_effector is None: + self._logger.error("No pipe effector available to request scanner") + return SCAN_STEP.TruncateIteration self.pipe_effector.send( scanner_manager.JOB_CALL_SCANNER_REQUEST_ON, self._scanning_job.id, @@ -857,11 +872,16 @@ def _do_request_scanner_off(self) -> SCAN_STEP: self._logger.info("Job {0} requested scanner off".format( self._scanning_job.id, )) + if self.pipe_effector is None: + self._logger.error("No pipe effector available to release scanner") + return SCAN_STEP.TruncateIteration self.pipe_effector.send( scanner_manager.JOB_CALL_SCANNER_REQUEST_OFF, self._scanning_job.id, ) self._scanning_effector_data.usb_port = "" + if self._scanning_effector_data.current_image is None: + self._scanning_effector_data.current_image = 0 self._scanning_effector_data.current_image += 1 return SCAN_STEP.NextMajor @@ -871,32 +891,35 @@ def _do_request_project_compilation(self) -> SCAN_STEP: If it is the first request of compilation, the COMPILE_ACTION is set to initiate from the setup-method. """ + compile_model = self._scanning_effector_data.compile_project_model + if ( self._scanning_job.fixture and self._scanning_effector_data.compilation_state is not COMPILE_STATE.Finalized + and compile_model is not None ): - self._scanning_effector_data.compile_project_model.email = ( + compile_model.email = ( self._scanning_job.email - if self._scanning_effector_data.compile_project_model.compile_action # noqa: E501 + if compile_model.compile_action in ( COMPILE_ACTION.AppendAndSpawnAnalysis, COMPILE_ACTION.InitiateAndSpawnAnalysis ) - else [] + else "" ) compile_job_id = self._rpc_client.create_compile_project_job( compile_project_factory.CompileProjectFactory.to_dict( - self._scanning_effector_data.compile_project_model, + compile_model, ), ) if compile_job_id: if ( - self._scanning_effector_data.compile_project_model.compile_action # noqa: E501 + compile_model.compile_action in ( COMPILE_ACTION.AppendAndSpawnAnalysis, COMPILE_ACTION.InitiateAndSpawnAnalysis @@ -917,20 +940,18 @@ def _do_request_project_compilation(self) -> SCAN_STEP: ) if next_image_is_last: - self._scanning_effector_data.compile_project_model.compile_action = ( # noqa: E501 + compile_model.compile_action = ( COMPILE_ACTION.AppendAndSpawnAnalysis ) else: - self._scanning_effector_data.compile_project_model.compile_action = ( # noqa: E501 + compile_model.compile_action = ( COMPILE_ACTION.Append ) - self._scanning_effector_data.compile_project_model.start_condition = ( # noqa: E501 + compile_model.start_condition = ( compile_job_id ) - while ( - self._scanning_effector_data.compile_project_model.images - ): - self._scanning_effector_data.compile_project_model.images.pop() # noqa: E501 + while compile_model.images: + compile_model.images.pop() self._logger.info( f"Job {self._scanning_job.id} created compile project job", ) @@ -970,6 +991,10 @@ def _do_scan(self) -> SCAN_STEP: return SCAN_STEP.NextMajor def _scan_thread(self): + if self._scanner is None: + self._scanning_effector_data.scan_success = False + return + self._scanning_effector_data.scan_success = ( self._scanner.AcquireByFile( scanner=self._scanning_effector_data.usb_port, @@ -990,7 +1015,7 @@ def _project_directory(self) -> str: def _set_usb_port(self, port, scanner_model): self._logger.info("Got an usb port '{0}'".format(port)) self._scanning_effector_data.scanner_model = scanner_model - if scanner_model: + if scanner_model and self._scanner is not None: self._scanner.model = scanner_model self._scanning_effector_data.usb_port = port @@ -1001,6 +1026,8 @@ def _add_scanned_image(self, index, time_stamp, path): path=path, ) - self._scanning_effector_data.compile_project_model.images.append( - image_model, - ) + compile_model = self._scanning_effector_data.compile_project_model + if compile_model is not None: + compile_model.images.append( + image_model, + ) diff --git a/tests/unit/server/test_scanning_effector.py b/tests/unit/server/test_scanning_effector.py index bc87a873..b0120ceb 100644 --- a/tests/unit/server/test_scanning_effector.py +++ b/tests/unit/server/test_scanning_effector.py @@ -63,10 +63,12 @@ def test_setup_copies_scan_pinning_formats_to_compile_model(tmp_path, monkeypatc effector = ScannerEffector(job) effector.setup(dumps(job)) + compile_model = effector._scanning_effector_data.compile_project_model + + assert compile_model is not None assert ( - effector._scanning_effector_data.compile_project_model - .overwrite_pinning_matrices + compile_model.overwrite_pinning_matrices == pinning_formats ) @@ -100,6 +102,9 @@ def test_compile_request_payload_contains_scan_pinning_formats( fake_rpc_client.payloads[0]["overwrite_pinning_matrices"] == pinning_formats ) + assert ( + effector._scanning_effector_data.compile_project_model is not None + ) assert ( effector._scanning_effector_data.compile_project_model.start_condition == "compile-123" From cb3d003fe4f3266e9636217357ec227b98f991da Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joakim=20M=C3=B6ller?= Date: Mon, 4 May 2026 00:28:57 +0200 Subject: [PATCH 3/3] style: fix Co-authored-by: Copilot --- scanomatic/server/scanning_effector.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scanomatic/server/scanning_effector.py b/scanomatic/server/scanning_effector.py index 12c7eedc..9f084be0 100644 --- a/scanomatic/server/scanning_effector.py +++ b/scanomatic/server/scanning_effector.py @@ -1,7 +1,7 @@ import os import time from threading import Thread -from typing import Any, Optional, Union, cast +from typing import Any, Optional, cast from scanomatic.io import rpc_client from scanomatic.io import paths, sane, scanner_manager