From f02b0ecafdbe4a576635a3218cd4487313d37b17 Mon Sep 17 00:00:00 2001 From: Eu Pin Tien Date: Mon, 2 Feb 2026 19:01:26 +0000 Subject: [PATCH 01/20] Added logic to optionally insert collection mode and colour flags for the Atlas and GridSquare tables in ISPyB for the CLEM workflow --- src/murfey/server/ispyb.py | 26 ++++++++++- src/murfey/util/models.py | 3 ++ .../clem/register_preprocessing_results.py | 43 +++++++++++++++++++ src/murfey/workflows/register_atlas_update.py | 11 +++-- .../register_data_collection_group.py | 6 +++ 5 files changed, 84 insertions(+), 5 deletions(-) diff --git a/src/murfey/server/ispyb.py b/src/murfey/server/ispyb.py index 47860c891..46e88e506 100644 --- a/src/murfey/server/ispyb.py +++ b/src/murfey/server/ispyb.py @@ -186,6 +186,8 @@ def do_update_atlas( atlas_image: str, pixel_size: float, slot: int | None, + collection_mode: str | None = None, + color_flags: dict[str, str | int] = {}, ): try: with ISPyBSession() as db: @@ -193,6 +195,14 @@ def do_update_atlas( atlas.atlasImage = atlas_image or atlas.atlasImage atlas.pixelSize = pixel_size or atlas.pixelSize atlas.cassetteSlot = slot or atlas.cassetteSlot + atlas.mode = collection_mode or atlas.mode + # Optionally insert colour flags if present + if color_flags: + for ( + col_name, + value, + ) in color_flags.items(): + setattr(atlas, col_name, value) db.add(atlas) db.commit() return {"success": True, "return_value": atlas.atlasId} @@ -209,6 +219,7 @@ def do_insert_grid_square( atlas_id: int, grid_square_id: int, grid_square_parameters: GridSquareParameters, + color_flags: dict[str, int] = {}, ): # most of this is for mypy if ( @@ -235,6 +246,10 @@ def do_insert_grid_square( stageLocationY=grid_square_parameters.y_stage_position, pixelSize=grid_square_parameters.pixel_size, ) + # Optionally insert colour flags + if color_flags: + for col_name, value in color_flags.items(): + setattr(record, col_name, value) try: with ISPyBSession() as db: db.add(record) @@ -250,7 +265,10 @@ def do_insert_grid_square( return {"success": False, "return_value": None} def do_update_grid_square( - self, grid_square_id: int, grid_square_parameters: GridSquareParameters + self, + grid_square_id: int, + grid_square_parameters: GridSquareParameters, + color_flags: dict[str, int] = {}, ): try: with ISPyBSession() as db: @@ -290,6 +308,12 @@ def do_update_grid_square( grid_square.stageLocationY = grid_square_parameters.y_stage_position if grid_square_parameters.pixel_size: grid_square.pixelSize = grid_square_parameters.pixel_size + if grid_square_parameters.collection_mode: + grid_square.mode = grid_square_parameters.collection_mode + # Optionally insert colour flags + if color_flags: + for col_name, value in color_flags.items(): + setattr(grid_square, col_name, value) db.add(grid_square) db.commit() return {"success": True, "return_value": grid_square.gridSquareId} diff --git a/src/murfey/util/models.py b/src/murfey/util/models.py index 4d079dbd9..1d59a6b25 100644 --- a/src/murfey/util/models.py +++ b/src/murfey/util/models.py @@ -157,6 +157,9 @@ class GridSquareParameters(BaseModel): pixel_size: Optional[float] = None angle: Optional[float] = None + # Collection mode + collection_mode: Optional[str] = None + class FoilHoleParameters(BaseModel): tag: str diff --git a/src/murfey/workflows/clem/register_preprocessing_results.py b/src/murfey/workflows/clem/register_preprocessing_results.py index d72859b4a..5e3a5c315 100644 --- a/src/murfey/workflows/clem/register_preprocessing_results.py +++ b/src/murfey/workflows/clem/register_preprocessing_results.py @@ -11,6 +11,7 @@ import logging import re import traceback +from collections.abc import Collection from importlib.metadata import entry_points from pathlib import Path from typing import Literal, Optional @@ -186,6 +187,41 @@ def _register_clem_image_series( logger.info(f"CLEM preprocessing results registered for {result.series_name!r} ") +color_columns = { + "gray": "hasGrey", + "red": "hasRed", + "green": "hasGreen", + "blue": "hasBlue", + "cyan": "hasCyan", + "magenta": "hasMagenta", + "yellow": "hasYellow", +} + + +def _get_color_flags( + colors: Collection[str] = [], +): + return { + color_columns[color]: (1 if color in color_columns.keys() else 0) + for color in colors + } + + +def _determine_collection_mode( + colors: Collection[str] = [], +): + if not colors: + logger.warning("No colours were present in returned result") + return None + if "gray" in colors: + if len(colors) == 1: + return "Bright Field" + else: + return "Bright Field and Fluorescent" + else: + return "Fluorescent" + + def _register_dcg_and_atlas( session_id: int, instrument_name: str, @@ -245,6 +281,10 @@ def _register_dcg_and_atlas( "atlas": atlas_name, "atlas_pixel_size": atlas_pixel_size, "sample": dcg_entry.sample, + "color_flags": _get_color_flags(result.output_files.keys()), + "collection_mode": _determine_collection_mode( + result.output_files.keys() + ), } if entry_point_result := entry_points( group="murfey.workflows", name="atlas_update" @@ -269,6 +309,8 @@ def _register_dcg_and_atlas( "atlas": atlas_name, "atlas_pixel_size": atlas_pixel_size, "sample": None, + "color_flags": _get_color_flags(result.output_files.keys()), + "collection_mode": _determine_collection_mode(result.output_files.keys()), } if entry_point_result := entry_points( group="murfey.workflows", name="data_collection_group" @@ -410,6 +452,7 @@ def _register_grid_square( y_stage_position=0.5 * (clem_img_series.y0 + clem_img_series.y1), pixel_size=clem_img_series.image_pixel_size, image=clem_img_series.thumbnail_search_string, + collection_mode=_determine_collection_mode(result.output_files.keys()), ) # Register or update the grid square entry as required if grid_square_result := murfey_db.exec( diff --git a/src/murfey/workflows/register_atlas_update.py b/src/murfey/workflows/register_atlas_update.py index 6ff68cc44..28bd77c9f 100644 --- a/src/murfey/workflows/register_atlas_update.py +++ b/src/murfey/workflows/register_atlas_update.py @@ -19,10 +19,13 @@ def run( logger.info(f"Registering updated atlas: \n{message}") _transport_object.do_update_atlas( - message["atlas_id"], - message["atlas"], - message["atlas_pixel_size"], - message["sample"], + atlas_id=message["atlas_id"], + atlas_image=message["atlas"], + pixel_size=message["atlas_pixel_size"], + slot=message["sample"], + # Extract optional parameters + collection_mode=message.get("collection_mode"), + color_flags=message.get("color_flags", {}), ) if dcg_hooks := entry_points(group="murfey.hooks", name="data_collection_group"): try: diff --git a/src/murfey/workflows/register_data_collection_group.py b/src/murfey/workflows/register_data_collection_group.py index a225936fc..135575500 100644 --- a/src/murfey/workflows/register_data_collection_group.py +++ b/src/murfey/workflows/register_data_collection_group.py @@ -67,6 +67,12 @@ def run(message: dict, murfey_db: SQLModelSession) -> dict[str, bool]: pixelSize=message.get("atlas_pixel_size", 0), cassetteSlot=message.get("sample"), ) + # Optionally set the collection mode and color flags + if collection_mode := message.get("collection_mode"): + atlas_record.mode = collection_mode + if color_flags := message.get("color_flags", {}): + for col_name, value in color_flags.items(): + setattr(atlas_record, col_name, value) atlas_id = _transport_object.do_insert_atlas(atlas_record).get( "return_value", None ) From 07f9d400a4c34619b0520c6d0a39de1851f8acc0 Mon Sep 17 00:00:00 2001 From: Eu Pin Tien Date: Fri, 6 Feb 2026 03:22:16 +0000 Subject: [PATCH 02/20] Fixed 'register_atlas_update' test --- tests/workflows/test_register_atlas_update.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/tests/workflows/test_register_atlas_update.py b/tests/workflows/test_register_atlas_update.py index 983c739b8..adfdd628b 100644 --- a/tests/workflows/test_register_atlas_update.py +++ b/tests/workflows/test_register_atlas_update.py @@ -25,9 +25,11 @@ def test_run( # Run the function and check the results and calls made result = run(message, mock_murfey_db) mock_transport_object.do_update_atlas.assert_called_once_with( - message["atlas_id"], - message["atlas"], - message["atlas_pixel_size"], - message["sample"], + atlas_id=message["atlas_id"], + atlas_image=message["atlas"], + pixel_size=message["atlas_pixel_size"], + slot=message["sample"], + collection_mode=message.get("collection_mode"), + color_flags=message.get("color_flags", {}), ) assert result == {"success": True} From 762d03365dbb4683f1a10ac982a8946c7d367726 Mon Sep 17 00:00:00 2001 From: Eu Pin Tien Date: Fri, 6 Feb 2026 03:28:25 +0000 Subject: [PATCH 03/20] Test call contents of 'register_data_collection_group' --- .../test_register_data_collection_group.py | 60 ++++++++++++------- 1 file changed, 37 insertions(+), 23 deletions(-) diff --git a/tests/workflows/test_register_data_collection_group.py b/tests/workflows/test_register_data_collection_group.py index 3324dec9c..e349120dd 100644 --- a/tests/workflows/test_register_data_collection_group.py +++ b/tests/workflows/test_register_data_collection_group.py @@ -1,33 +1,35 @@ from unittest.mock import MagicMock +import ispyb.sqlalchemy._auto_db_schema as ISPyBDB import pytest from pytest_mock import MockerFixture from murfey.workflows.register_data_collection_group import run from tests.conftest import ExampleVisit -register_data_collection_group_params_matrix = ( - # ISPyB session ID | # DCG search result | # DCG insert result | # Atlas insert result - (0, 0, 0, 0), - (0, 0, 0, None), - (0, 0, None, 0), - (0, 0, None, None), - (0, None, 0, 0), - (0, None, 0, None), - (0, None, None, 0), - (0, None, None, None), - (None, 0, 0, 0), - (None, 0, 0, None), - (None, 0, None, 0), - (None, 0, None, None), - (None, None, 0, 0), - (None, None, 0, None), - (None, None, None, 0), - (None, None, None, None), -) - -@pytest.mark.parametrize("test_params", register_data_collection_group_params_matrix) +@pytest.mark.parametrize( + "test_params", + ( + # ISPyB session ID | # DCG search result | # DCG insert result | # Atlas insert result + (0, 0, 0, 0), + (0, 0, 0, None), + (0, 0, None, 0), + (0, 0, None, None), + (0, None, 0, 0), + (0, None, 0, None), + (0, None, None, 0), + (0, None, None, None), + (None, 0, 0, 0), + (None, 0, 0, None), + (None, 0, None, 0), + (None, 0, None, None), + (None, None, 0, 0), + (None, None, 0, None), + (None, None, None, 0), + (None, None, None, None), + ), +) def test_run( mocker: MockerFixture, test_params: tuple[int | None, int | None, int | None, int | None], @@ -76,9 +78,21 @@ def test_run( assert result == {"success": True} else: if ispyb_session_id is not None: - mock_transport_object.do_insert_data_collection_group.assert_called_once() + mock_transport_object.do_insert_data_collection_group.assert_called_once_with( + ISPyBDB.DataCollectionGroup( + sessionId=ispyb_session_id, + experimentTypeId=message["experiment_type_id"], + ) + ) if insert_dcg is not None: - mock_transport_object.do_insert_atlas.assert_called_once() + mock_transport_object.do_insert_atlas.assert_called_once_with( + ISPyBDB.Atlas( + dataCollectionGroupId=dcg_result, + atlasImage=message.get("atlas", ""), + pixelSize=message.get("atlas_pixel_size", 0), + cassetteSlot=message.get("sample"), + ) + ) assert result == {"success": True} else: assert result == {"success": False, "requeue": True} From 0c1461d7187bedec2a3d5629def3c8eb3fe913ce Mon Sep 17 00:00:00 2001 From: Eu Pin Tien Date: Fri, 6 Feb 2026 03:29:40 +0000 Subject: [PATCH 04/20] Fixed 'register_preprocessing_results' tests --- .../test_register_preprocessing_results.py | 82 ++++++++++++++----- 1 file changed, 63 insertions(+), 19 deletions(-) diff --git a/tests/workflows/clem/test_register_preprocessing_results.py b/tests/workflows/clem/test_register_preprocessing_results.py index 3353be984..8e180fbc5 100644 --- a/tests/workflows/clem/test_register_preprocessing_results.py +++ b/tests/workflows/clem/test_register_preprocessing_results.py @@ -12,6 +12,8 @@ import murfey.util.db as MurfeyDB from murfey.workflows.clem.register_preprocessing_results import ( + _determine_collection_mode, + _get_color_flags, _register_clem_image_series, _register_dcg_and_atlas, _register_grid_square, @@ -22,7 +24,6 @@ visit_name = f"{ExampleVisit.proposal_code}{ExampleVisit.proposal_number}-{ExampleVisit.visit_number}" processed_dir_name = "processed" grid_name = "Grid_1" -colors = ("gray", "green", "red") @pytest.fixture @@ -33,6 +34,7 @@ def rsync_basepath(tmp_path: Path): def generate_preprocessing_messages( rsync_basepath: Path, session_id: int, + colors: list[str], ): # Make directory to where data for current grid is stored visit_dir = rsync_basepath / "2020" / visit_name @@ -116,25 +118,35 @@ def generate_preprocessing_messages( return messages -@pytest.mark.skip def test_register_clem_image_series(): assert _register_clem_image_series -@pytest.mark.skip def test_register_dcg_and_atlas(): assert _register_dcg_and_atlas -@pytest.mark.skip def test_register_grid_square(): assert _register_grid_square +@pytest.mark.parametrize( + "test_params", + ( # Colors + (["gray"],), + (["gray", "green"],), + (["red", "green", "blue"],), + (["cyan", "magenta", "blue"],), + ), +) def test_run( mocker: MockerFixture, rsync_basepath: Path, + test_params: tuple[list[str]], ): + # Unpack test params + (colors,) = test_params + # Mock the MurfeyDB connection mock_murfey_session_entry = MagicMock() mock_murfey_session_entry.instrument_name = ExampleVisit.instrument_name @@ -161,6 +173,7 @@ def test_run( preprocessing_messages = generate_preprocessing_messages( rsync_basepath=rsync_basepath, session_id=ExampleVisit.murfey_session_id, + colors=colors, ) for message in preprocessing_messages: result = run( @@ -171,29 +184,44 @@ def test_run( assert mock_register_clem_series.call_count == len(preprocessing_messages) assert mock_register_dcg_and_atlas.call_count == len(preprocessing_messages) assert mock_register_grid_square.call_count == len(preprocessing_messages) - assert mock_align_and_merge_call.call_count == len(preprocessing_messages) * len( - colors - ) + if ("gray" not in colors) or ("gray" in colors and len(colors) == 1): + assert mock_align_and_merge_call.call_count == len(preprocessing_messages) + else: + assert mock_align_and_merge_call.call_count == len(preprocessing_messages) * 3 -test_matrix = ( - # Reverse order of list - (False,), - (True,), +@pytest.mark.parametrize( + "test_params", + ( + # Reverse list order? | Colors + ( + False, + [ + "gray", + ], + ), + ( + True, + [ + "gray", + ], + ), + (False, ["red", "green", "blue"]), + (True, ["cyan", "magenta", "yellow"]), + (False, ["gray", "red", "green", "blue"]), + (True, ["gray", "cyan", "magenta", "yellow"]), + ), ) - - -@pytest.mark.parametrize("test_params", test_matrix) def test_run_with_db( mocker: MockerFixture, rsync_basepath: Path, mock_ispyb_credentials, murfey_db_session: SQLModelSession, ispyb_db_session: SQLAlchemySession, - test_params: tuple[bool], + test_params: tuple[bool, list[str]], ): # Unpack test params - (shuffle_message,) = test_params + (shuffle_message, colors) = test_params # Create a session to insert for this test murfey_session: MurfeyDB.Session = get_or_create_db_entry( @@ -258,6 +286,7 @@ def test_run_with_db( preprocessing_messages = generate_preprocessing_messages( rsync_basepath=rsync_basepath, session_id=murfey_session.id, + colors=colors, ) if shuffle_message: preprocessing_messages.reverse() @@ -270,9 +299,10 @@ def test_run_with_db( # Each message should call the align-and-merge workflow thrice # if gray and colour channels are both present - assert mock_align_and_merge_call.call_count == len(preprocessing_messages) * len( - colors - ) + if ("gray" not in colors) or ("gray" in colors and len(colors) == 1): + assert mock_align_and_merge_call.call_count == len(preprocessing_messages) + else: + assert mock_align_and_merge_call.call_count == len(preprocessing_messages) * 3 # Both databases should have entries for data collection group, and grid squares # ISPyB database should additionally have an atlas entry @@ -313,7 +343,16 @@ def test_run_with_db( ) assert len(ispyb_atlas_search) == 1 + # Determine the color flags and collection mode + color_flags = _get_color_flags(colors) + collection_mode = _determine_collection_mode(colors) + ispyb_atlas = ispyb_atlas_search[0] + # Check that the Atlas color flags and collection mode are set correctly + for flag, value in color_flags.items(): + assert getattr(ispyb_atlas, flag) == value + assert ispyb_atlas.mode == collection_mode + ispyb_gs_search = ( ispyb_db_session.execute( sa_select(ISPyBDB.GridSquare).where( @@ -324,6 +363,11 @@ def test_run_with_db( .all() ) assert len(ispyb_gs_search) == len(preprocessing_messages) - 1 + for gs in ispyb_gs_search: + # Check that the Atlas color flags and collection mode are set correctly + for flag, value in color_flags.items(): + assert getattr(gs, flag) == value + assert gs.mode == collection_mode murfey_db_session.close() ispyb_db_session.close() From b5ab82cba2985bd8e561c21c9e57dc7dbbc6fcb3 Mon Sep 17 00:00:00 2001 From: Eu Pin Tien Date: Fri, 6 Feb 2026 04:01:00 +0000 Subject: [PATCH 05/20] Assign color flags to attributes directly instead of using 'setattr' --- src/murfey/server/ispyb.py | 30 +++++++++++++++++++++--------- 1 file changed, 21 insertions(+), 9 deletions(-) diff --git a/src/murfey/server/ispyb.py b/src/murfey/server/ispyb.py index 46e88e506..99075c7b7 100644 --- a/src/murfey/server/ispyb.py +++ b/src/murfey/server/ispyb.py @@ -198,11 +198,13 @@ def do_update_atlas( atlas.mode = collection_mode or atlas.mode # Optionally insert colour flags if present if color_flags: - for ( - col_name, - value, - ) in color_flags.items(): - setattr(atlas, col_name, value) + atlas.hasGrey = color_flags.get("hasGrey") + atlas.hasRed = color_flags.get("hasRed") + atlas.hasGreen = color_flags.get("hasGreen") + atlas.hasBlue = color_flags.get("hasBlue") + atlas.hasCyan = color_flags.get("hasCyan") + atlas.hasYellow = color_flags.get("hasYellow") + atlas.hasMagenta = color_flags.get("hasMagenta") db.add(atlas) db.commit() return {"success": True, "return_value": atlas.atlasId} @@ -248,8 +250,13 @@ def do_insert_grid_square( ) # Optionally insert colour flags if color_flags: - for col_name, value in color_flags.items(): - setattr(record, col_name, value) + record.hasGrey = color_flags.get("hasGrey") + record.hasRed = color_flags.get("hasRed") + record.hasGreen = color_flags.get("hasGreen") + record.hasBlue = color_flags.get("hasBlue") + record.hasCyan = color_flags.get("hasCyan") + record.hasYellow = color_flags.get("hasYellow") + record.hasMagenta = color_flags.get("hasMagenta") try: with ISPyBSession() as db: db.add(record) @@ -312,8 +319,13 @@ def do_update_grid_square( grid_square.mode = grid_square_parameters.collection_mode # Optionally insert colour flags if color_flags: - for col_name, value in color_flags.items(): - setattr(grid_square, col_name, value) + grid_square.hasGrey = color_flags.get("hasGrey") + grid_square.hasRed = color_flags.get("hasRed") + grid_square.hasGreen = color_flags.get("hasGreen") + grid_square.hasBlue = color_flags.get("hasBlue") + grid_square.hasCyan = color_flags.get("hasCyan") + grid_square.hasYellow = color_flags.get("hasYellow") + grid_square.hasMagenta = color_flags.get("hasMagenta") db.add(grid_square) db.commit() return {"success": True, "return_value": grid_square.gridSquareId} From 71473819da9f8388e33896700b81bb13de17877e Mon Sep 17 00:00:00 2001 From: Eu Pin Tien Date: Fri, 6 Feb 2026 04:02:09 +0000 Subject: [PATCH 06/20] Store color flags and collection mode in variables instead of repeating the 'get' call --- .../clem/register_preprocessing_results.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/src/murfey/workflows/clem/register_preprocessing_results.py b/src/murfey/workflows/clem/register_preprocessing_results.py index 5e3a5c315..2427675c6 100644 --- a/src/murfey/workflows/clem/register_preprocessing_results.py +++ b/src/murfey/workflows/clem/register_preprocessing_results.py @@ -265,6 +265,9 @@ def _register_dcg_and_atlas( atlas_name = "" atlas_pixel_size = 0.0 + color_flags = _get_color_flags(result.output_files.keys()) + collection_mode = _determine_collection_mode(result.output_files.keys()) + if dcg_search := murfey_db.exec( select(MurfeyDB.DataCollectionGroup) .where(MurfeyDB.DataCollectionGroup.session_id == session_id) @@ -281,10 +284,8 @@ def _register_dcg_and_atlas( "atlas": atlas_name, "atlas_pixel_size": atlas_pixel_size, "sample": dcg_entry.sample, - "color_flags": _get_color_flags(result.output_files.keys()), - "collection_mode": _determine_collection_mode( - result.output_files.keys() - ), + "color_flags": color_flags, + "collection_mode": collection_mode, } if entry_point_result := entry_points( group="murfey.workflows", name="atlas_update" @@ -309,8 +310,8 @@ def _register_dcg_and_atlas( "atlas": atlas_name, "atlas_pixel_size": atlas_pixel_size, "sample": None, - "color_flags": _get_color_flags(result.output_files.keys()), - "collection_mode": _determine_collection_mode(result.output_files.keys()), + "color_flags": color_flags, + "collection_mode": collection_mode, } if entry_point_result := entry_points( group="murfey.workflows", name="data_collection_group" From fffcee390a4ebda8411a27ad1645d9f5225ad248 Mon Sep 17 00:00:00 2001 From: Eu Pin Tien Date: Fri, 6 Feb 2026 04:03:57 +0000 Subject: [PATCH 07/20] Assign color flags to attributes directly instead of using 'setattr' --- src/murfey/workflows/register_data_collection_group.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/murfey/workflows/register_data_collection_group.py b/src/murfey/workflows/register_data_collection_group.py index 135575500..d4a9b79dc 100644 --- a/src/murfey/workflows/register_data_collection_group.py +++ b/src/murfey/workflows/register_data_collection_group.py @@ -71,8 +71,13 @@ def run(message: dict, murfey_db: SQLModelSession) -> dict[str, bool]: if collection_mode := message.get("collection_mode"): atlas_record.mode = collection_mode if color_flags := message.get("color_flags", {}): - for col_name, value in color_flags.items(): - setattr(atlas_record, col_name, value) + atlas_record.hasGrey = color_flags.get("hasGrey") + atlas_record.hasRed = color_flags.get("hasRed") + atlas_record.hasGreen = color_flags.get("hasGreen") + atlas_record.hasBlue = color_flags.get("hasBlue") + atlas_record.hasCyan = color_flags.get("hasCyan") + atlas_record.hasYellow = color_flags.get("hasYellow") + atlas_record.hasMagenta = color_flags.get("hasMagenta") atlas_id = _transport_object.do_insert_atlas(atlas_record).get( "return_value", None ) From 4492e4f7ab76933a4705305d5e4286b4cfa81d16 Mon Sep 17 00:00:00 2001 From: Eu Pin Tien Date: Fri, 6 Feb 2026 04:07:17 +0000 Subject: [PATCH 08/20] Database table classes get instantiated with different IDs, and do not match --- .../test_register_data_collection_group.py | 17 ++--------------- 1 file changed, 2 insertions(+), 15 deletions(-) diff --git a/tests/workflows/test_register_data_collection_group.py b/tests/workflows/test_register_data_collection_group.py index e349120dd..9d2783f70 100644 --- a/tests/workflows/test_register_data_collection_group.py +++ b/tests/workflows/test_register_data_collection_group.py @@ -1,6 +1,5 @@ from unittest.mock import MagicMock -import ispyb.sqlalchemy._auto_db_schema as ISPyBDB import pytest from pytest_mock import MockerFixture @@ -78,21 +77,9 @@ def test_run( assert result == {"success": True} else: if ispyb_session_id is not None: - mock_transport_object.do_insert_data_collection_group.assert_called_once_with( - ISPyBDB.DataCollectionGroup( - sessionId=ispyb_session_id, - experimentTypeId=message["experiment_type_id"], - ) - ) + mock_transport_object.do_insert_data_collection_group.assert_called_once() if insert_dcg is not None: - mock_transport_object.do_insert_atlas.assert_called_once_with( - ISPyBDB.Atlas( - dataCollectionGroupId=dcg_result, - atlasImage=message.get("atlas", ""), - pixelSize=message.get("atlas_pixel_size", 0), - cassetteSlot=message.get("sample"), - ) - ) + mock_transport_object.do_insert_atlas.assert_called_once() assert result == {"success": True} else: assert result == {"success": False, "requeue": True} From b5c905b6708b2b25cad47c699a32d1f4c196d183 Mon Sep 17 00:00:00 2001 From: Eu Pin Tien Date: Fri, 6 Feb 2026 04:37:12 +0000 Subject: [PATCH 09/20] Forgot to pass 'color_flags' parameter to 'insert_grid_square' function --- src/murfey/workflows/clem/register_preprocessing_results.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/murfey/workflows/clem/register_preprocessing_results.py b/src/murfey/workflows/clem/register_preprocessing_results.py index 2427675c6..b0a3ec46c 100644 --- a/src/murfey/workflows/clem/register_preprocessing_results.py +++ b/src/murfey/workflows/clem/register_preprocessing_results.py @@ -492,6 +492,7 @@ def _register_grid_square( atlas_id=dcg_entry.atlas_id, grid_square_id=clem_img_series.id, grid_square_parameters=grid_square_params, + color_flags=_get_color_flags(result.output_files.keys()), ) # Register to Murfey grid_square_entry = MurfeyDB.GridSquare( From 94f73d0931f6c3016ba5d75d126970ca5bd8001d Mon Sep 17 00:00:00 2001 From: Eu Pin Tien Date: Fri, 6 Feb 2026 05:46:30 +0000 Subject: [PATCH 10/20] Iteratively update color flaggs using 'setattr' --- src/murfey/server/ispyb.py | 30 +++++-------------- .../register_data_collection_group.py | 9 ++---- 2 files changed, 10 insertions(+), 29 deletions(-) diff --git a/src/murfey/server/ispyb.py b/src/murfey/server/ispyb.py index 99075c7b7..dcd08f594 100644 --- a/src/murfey/server/ispyb.py +++ b/src/murfey/server/ispyb.py @@ -7,7 +7,7 @@ import ispyb import workflows.transport from fastapi import Depends -from ispyb.sqlalchemy import ( +from ispyb.sqlalchemy._auto_db_schema import ( Atlas, AutoProcProgram, BLSample, @@ -198,13 +198,8 @@ def do_update_atlas( atlas.mode = collection_mode or atlas.mode # Optionally insert colour flags if present if color_flags: - atlas.hasGrey = color_flags.get("hasGrey") - atlas.hasRed = color_flags.get("hasRed") - atlas.hasGreen = color_flags.get("hasGreen") - atlas.hasBlue = color_flags.get("hasBlue") - atlas.hasCyan = color_flags.get("hasCyan") - atlas.hasYellow = color_flags.get("hasYellow") - atlas.hasMagenta = color_flags.get("hasMagenta") + for col_name, value in color_flags.items(): + setattr(atlas, col_name, value) db.add(atlas) db.commit() return {"success": True, "return_value": atlas.atlasId} @@ -247,16 +242,12 @@ def do_insert_grid_square( stageLocationX=grid_square_parameters.x_stage_position, stageLocationY=grid_square_parameters.y_stage_position, pixelSize=grid_square_parameters.pixel_size, + mode=grid_square_parameters.collection_mode, ) # Optionally insert colour flags if color_flags: - record.hasGrey = color_flags.get("hasGrey") - record.hasRed = color_flags.get("hasRed") - record.hasGreen = color_flags.get("hasGreen") - record.hasBlue = color_flags.get("hasBlue") - record.hasCyan = color_flags.get("hasCyan") - record.hasYellow = color_flags.get("hasYellow") - record.hasMagenta = color_flags.get("hasMagenta") + for col_name, value in color_flags.items(): + setattr(record, col_name, value) try: with ISPyBSession() as db: db.add(record) @@ -319,13 +310,8 @@ def do_update_grid_square( grid_square.mode = grid_square_parameters.collection_mode # Optionally insert colour flags if color_flags: - grid_square.hasGrey = color_flags.get("hasGrey") - grid_square.hasRed = color_flags.get("hasRed") - grid_square.hasGreen = color_flags.get("hasGreen") - grid_square.hasBlue = color_flags.get("hasBlue") - grid_square.hasCyan = color_flags.get("hasCyan") - grid_square.hasYellow = color_flags.get("hasYellow") - grid_square.hasMagenta = color_flags.get("hasMagenta") + for col_name, value in color_flags.items(): + setattr(grid_square, col_name, value) db.add(grid_square) db.commit() return {"success": True, "return_value": grid_square.gridSquareId} diff --git a/src/murfey/workflows/register_data_collection_group.py b/src/murfey/workflows/register_data_collection_group.py index d4a9b79dc..135575500 100644 --- a/src/murfey/workflows/register_data_collection_group.py +++ b/src/murfey/workflows/register_data_collection_group.py @@ -71,13 +71,8 @@ def run(message: dict, murfey_db: SQLModelSession) -> dict[str, bool]: if collection_mode := message.get("collection_mode"): atlas_record.mode = collection_mode if color_flags := message.get("color_flags", {}): - atlas_record.hasGrey = color_flags.get("hasGrey") - atlas_record.hasRed = color_flags.get("hasRed") - atlas_record.hasGreen = color_flags.get("hasGreen") - atlas_record.hasBlue = color_flags.get("hasBlue") - atlas_record.hasCyan = color_flags.get("hasCyan") - atlas_record.hasYellow = color_flags.get("hasYellow") - atlas_record.hasMagenta = color_flags.get("hasMagenta") + for col_name, value in color_flags.items(): + setattr(atlas_record, col_name, value) atlas_id = _transport_object.do_insert_atlas(atlas_record).get( "return_value", None ) From b315684db01c6229585d68987603a90b0817de7b Mon Sep 17 00:00:00 2001 From: Eu Pin Tien Date: Fri, 6 Feb 2026 05:54:05 +0000 Subject: [PATCH 11/20] Reverted import --- src/murfey/server/ispyb.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/murfey/server/ispyb.py b/src/murfey/server/ispyb.py index dcd08f594..072e97dd4 100644 --- a/src/murfey/server/ispyb.py +++ b/src/murfey/server/ispyb.py @@ -7,7 +7,7 @@ import ispyb import workflows.transport from fastapi import Depends -from ispyb.sqlalchemy._auto_db_schema import ( +from ispyb.sqlalchemy import ( Atlas, AutoProcProgram, BLSample, From e6b340aaf9ea8b0714ed87e156e8837af8686ccd Mon Sep 17 00:00:00 2001 From: Eu Pin Tien Date: Fri, 6 Feb 2026 06:33:48 +0000 Subject: [PATCH 12/20] Missed inserting a 'color_flag' field in an if-else block branch --- src/murfey/workflows/clem/register_preprocessing_results.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/murfey/workflows/clem/register_preprocessing_results.py b/src/murfey/workflows/clem/register_preprocessing_results.py index b0a3ec46c..fa0f9913d 100644 --- a/src/murfey/workflows/clem/register_preprocessing_results.py +++ b/src/murfey/workflows/clem/register_preprocessing_results.py @@ -479,6 +479,7 @@ def _register_grid_square( _transport_object.do_update_grid_square( grid_square_id=grid_square_entry.id, grid_square_parameters=grid_square_params, + color_flags=_get_color_flags(result.output_files.keys()), ) else: # Look up data collection group for current series From 473937dccad0f5fc397878a9749ef5c222f06624 Mon Sep 17 00:00:00 2001 From: Daniel Hatton Date: Fri, 6 Feb 2026 09:42:05 +0000 Subject: [PATCH 13/20] remove empty dict and list defaults --- src/murfey/server/ispyb.py | 9 ++++++--- .../workflows/clem/register_preprocessing_results.py | 5 +++-- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/src/murfey/server/ispyb.py b/src/murfey/server/ispyb.py index 072e97dd4..f6cebb30f 100644 --- a/src/murfey/server/ispyb.py +++ b/src/murfey/server/ispyb.py @@ -187,8 +187,9 @@ def do_update_atlas( pixel_size: float, slot: int | None, collection_mode: str | None = None, - color_flags: dict[str, str | int] = {}, + color_flags: dict[str, str | int] | None = None, ): + color_flags = color_flags or {} try: with ISPyBSession() as db: atlas = db.query(Atlas).filter(Atlas.atlasId == atlas_id).one() @@ -216,8 +217,9 @@ def do_insert_grid_square( atlas_id: int, grid_square_id: int, grid_square_parameters: GridSquareParameters, - color_flags: dict[str, int] = {}, + color_flags: dict[str, int] | None = None, ): + color_flags = color_flags or {} # most of this is for mypy if ( grid_square_parameters.pixel_size is not None @@ -266,8 +268,9 @@ def do_update_grid_square( self, grid_square_id: int, grid_square_parameters: GridSquareParameters, - color_flags: dict[str, int] = {}, + color_flags: dict[str, int] | None = None, ): + color_flags = color_flags or {} try: with ISPyBSession() as db: grid_square: GridSquare = ( diff --git a/src/murfey/workflows/clem/register_preprocessing_results.py b/src/murfey/workflows/clem/register_preprocessing_results.py index fa0f9913d..319f72707 100644 --- a/src/murfey/workflows/clem/register_preprocessing_results.py +++ b/src/murfey/workflows/clem/register_preprocessing_results.py @@ -199,8 +199,9 @@ def _register_clem_image_series( def _get_color_flags( - colors: Collection[str] = [], + colors: Collection[str] | None = None, ): + colors = colors or [] return { color_columns[color]: (1 if color in color_columns.keys() else 0) for color in colors @@ -208,7 +209,7 @@ def _get_color_flags( def _determine_collection_mode( - colors: Collection[str] = [], + colors: Collection[str] | None = None, ): if not colors: logger.warning("No colours were present in returned result") From 15185caea4eab5e561887d65559597f18bbb8526 Mon Sep 17 00:00:00 2001 From: Eu Pin Tien Date: Fri, 6 Feb 2026 10:22:06 +0000 Subject: [PATCH 14/20] Added unit test for '_get_color_flags' --- .../test_register_preprocessing_results.py | 40 +++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/tests/workflows/clem/test_register_preprocessing_results.py b/tests/workflows/clem/test_register_preprocessing_results.py index 8e180fbc5..d0a6d1d68 100644 --- a/tests/workflows/clem/test_register_preprocessing_results.py +++ b/tests/workflows/clem/test_register_preprocessing_results.py @@ -122,6 +122,46 @@ def test_register_clem_image_series(): assert _register_clem_image_series +@pytest.mark.parametrize( + "test_params", + ( + ( + ["gray"], + { + "hasGrey": 1, + }, + ), + ( + ["gray", "red"], + { + "hasGrey": 1, + "hasRed": 1, + }, + ), + ( + ["red", "green", "blue"], + { + "hasRed": 1, + "hasGreen": 1, + "hasBlue": 1, + }, + ), + ( + ["cyan", "magenta", "yellow"], + { + "hasCyan": 1, + "hasMagenta": 1, + "hasYellow": 1, + }, + ), + ), +) +def test_get_color_flags(test_params: tuple[list[str], dict[str, int]]): + # Unpack test params + colors, expected_result = test_params + assert _get_color_flags(colors) == expected_result + + def test_register_dcg_and_atlas(): assert _register_dcg_and_atlas From 1c36fc2d907b57b5ebc00a5c630aaa5fd2eb2c4b Mon Sep 17 00:00:00 2001 From: Eu Pin Tien Date: Fri, 6 Feb 2026 11:40:53 +0000 Subject: [PATCH 15/20] Changed '_get_color_flags' to overwrite all the colour flags instead of just activating the ones it receives; fixed colour flag and collection mode insertion logic for the Atlas --- .../clem/register_preprocessing_results.py | 15 +++---- .../test_register_preprocessing_results.py | 41 +++++++++++++++++-- 2 files changed, 45 insertions(+), 11 deletions(-) diff --git a/src/murfey/workflows/clem/register_preprocessing_results.py b/src/murfey/workflows/clem/register_preprocessing_results.py index 319f72707..3779743c6 100644 --- a/src/murfey/workflows/clem/register_preprocessing_results.py +++ b/src/murfey/workflows/clem/register_preprocessing_results.py @@ -202,10 +202,10 @@ def _get_color_flags( colors: Collection[str] | None = None, ): colors = colors or [] - return { - color_columns[color]: (1 if color in color_columns.keys() else 0) - for color in colors - } + color_flags = dict.fromkeys(color_columns.values(), 0) + for color in colors: + color_flags[color_columns[color]] = 1 + return color_flags def _determine_collection_mode( @@ -262,12 +262,13 @@ def _register_dcg_and_atlas( else: atlas_name = str(output_file.parent / "*.tiff") atlas_pixel_size = result.pixel_size + color_flags = _get_color_flags(result.output_files.keys()) + collection_mode = _determine_collection_mode(result.output_files.keys()) else: atlas_name = "" atlas_pixel_size = 0.0 - - color_flags = _get_color_flags(result.output_files.keys()) - collection_mode = _determine_collection_mode(result.output_files.keys()) + color_flags = None + collection_mode = None if dcg_search := murfey_db.exec( select(MurfeyDB.DataCollectionGroup) diff --git a/tests/workflows/clem/test_register_preprocessing_results.py b/tests/workflows/clem/test_register_preprocessing_results.py index d0a6d1d68..75925211d 100644 --- a/tests/workflows/clem/test_register_preprocessing_results.py +++ b/tests/workflows/clem/test_register_preprocessing_results.py @@ -119,7 +119,7 @@ def generate_preprocessing_messages( def test_register_clem_image_series(): - assert _register_clem_image_series + _register_clem_image_series @pytest.mark.parametrize( @@ -158,16 +158,49 @@ def test_register_clem_image_series(): ) def test_get_color_flags(test_params: tuple[list[str], dict[str, int]]): # Unpack test params - colors, expected_result = test_params + colors, positive_flags = test_params + expected_result = dict.fromkeys( + ( + "hasGrey", + "hasRed", + "hasGreen", + "hasBlue", + "hasCyan", + "hasMagenta", + "hasYellow", + ), + 0, + ) + for flag, value in positive_flags.items(): + expected_result[flag] = value assert _get_color_flags(colors) == expected_result +@pytest.mark.parametrize( + "test_params", + ( + ( + ["gray"], + "Bright Field", + ), + ( + ["gray", "blue"], + "Bright Field and Fluorescent", + ), + (["red", "green", "blue"], "Fluorescent"), + ), +) +def test_determine_collection_mode(test_params: tuple[list[str], str]): + colors, expected_result = test_params + assert _determine_collection_mode(colors) == expected_result + + def test_register_dcg_and_atlas(): - assert _register_dcg_and_atlas + _register_dcg_and_atlas def test_register_grid_square(): - assert _register_grid_square + _register_grid_square @pytest.mark.parametrize( From 04310edf2f2c0e82d0d280f744ffec2b1ebadaf1 Mon Sep 17 00:00:00 2001 From: Eu Pin Tien Date: Fri, 6 Feb 2026 12:19:24 +0000 Subject: [PATCH 16/20] Add columns to CLEMImageSeries table to keep track of colours present in the dataset --- src/murfey/util/db.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/murfey/util/db.py b/src/murfey/util/db.py index 193e0ff2a..f03365694 100644 --- a/src/murfey/util/db.py +++ b/src/murfey/util/db.py @@ -294,6 +294,13 @@ class CLEMImageSeries(SQLModel, table=True): # type: ignore sa_relationship_kwargs={"cascade": "delete"}, ) # One to many number_of_members: Optional[int] = Field(default=None) + has_grey: Optional[bool] = Field(default=None) + has_red: Optional[bool] = Field(default=None) + has_green: Optional[bool] = Field(default=None) + has_blue: Optional[bool] = Field(default=None) + has_cyan: Optional[bool] = Field(default=None) + has_magenta: Optional[bool] = Field(default=None) + has_yellow: Optional[bool] = Field(default=None) # Shape and resolution information image_pixels_x: Optional[int] = Field(default=None) From b1ba65452bef804cc4966e77287f63add46240a3 Mon Sep 17 00:00:00 2001 From: Eu Pin Tien Date: Fri, 6 Feb 2026 12:26:38 +0000 Subject: [PATCH 17/20] Added column to CLEMImageSeries table to keep track of collection mode used for dataset --- src/murfey/util/db.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/murfey/util/db.py b/src/murfey/util/db.py index f03365694..a46a1267a 100644 --- a/src/murfey/util/db.py +++ b/src/murfey/util/db.py @@ -301,6 +301,7 @@ class CLEMImageSeries(SQLModel, table=True): # type: ignore has_cyan: Optional[bool] = Field(default=None) has_magenta: Optional[bool] = Field(default=None) has_yellow: Optional[bool] = Field(default=None) + collection_mode: Optional[str] = Field(default=None) # Shape and resolution information image_pixels_x: Optional[int] = Field(default=None) From 119b489a7c0c0accbef229f98f45500af651c738 Mon Sep 17 00:00:00 2001 From: Eu Pin Tien Date: Fri, 6 Feb 2026 12:40:32 +0000 Subject: [PATCH 18/20] Register colours and collection mode in CLEMImageSeries table upon arrival of message, then translate it across to Atlas and GridSquare tables in ISPyB --- .../clem/register_preprocessing_results.py | 74 ++++++++++----- .../test_register_preprocessing_results.py | 95 ++++++++++--------- 2 files changed, 100 insertions(+), 69 deletions(-) diff --git a/src/murfey/workflows/clem/register_preprocessing_results.py b/src/murfey/workflows/clem/register_preprocessing_results.py index 3779743c6..dd4c3a58d 100644 --- a/src/murfey/workflows/clem/register_preprocessing_results.py +++ b/src/murfey/workflows/clem/register_preprocessing_results.py @@ -67,6 +67,27 @@ def _is_clem_atlas(result: CLEMPreprocessingResult): ) +COLOR_FLAGS_MURFEY = { + "gray": "has_grey", + "red": "has_red", + "green": "has_green", + "blue": "has_blue", + "cyan": "has_cyan", + "magenta": "has_magenta", + "yellow": "has_yellow", +} + + +def _get_color_flags( + colors: Collection[str] | None = None, +): + colors = colors or [] + color_flags = dict.fromkeys(COLOR_FLAGS_MURFEY.values(), False) + for color in colors: + color_flags[COLOR_FLAGS_MURFEY[color]] = True + return color_flags + + def _register_clem_image_series( session_id: int, result: CLEMPreprocessingResult, @@ -160,6 +181,11 @@ def _register_clem_image_series( clem_img_series.image_search_string = str(output_file.parent / "*tiff") clem_img_series.data_type = "atlas" if _is_clem_atlas(result) else "grid_square" clem_img_series.number_of_members = result.number_of_members + for col_name, value in _get_color_flags(result.output_files.keys()): + setattr(clem_img_series, col_name, value) + clem_img_series.collection_mode = _determine_collection_mode( + result.output_files.keys() + ) clem_img_series.image_pixels_x = result.pixels_x clem_img_series.image_pixels_y = result.pixels_y clem_img_series.image_pixel_size = result.pixel_size @@ -187,27 +213,6 @@ def _register_clem_image_series( logger.info(f"CLEM preprocessing results registered for {result.series_name!r} ") -color_columns = { - "gray": "hasGrey", - "red": "hasRed", - "green": "hasGreen", - "blue": "hasBlue", - "cyan": "hasCyan", - "magenta": "hasMagenta", - "yellow": "hasYellow", -} - - -def _get_color_flags( - colors: Collection[str] | None = None, -): - colors = colors or [] - color_flags = dict.fromkeys(color_columns.values(), 0) - for color in colors: - color_flags[color_columns[color]] = 1 - return color_flags - - def _determine_collection_mode( colors: Collection[str] | None = None, ): @@ -223,6 +228,16 @@ def _determine_collection_mode( return "Fluorescent" +def _snake_to_camel_case(string: str): + parts = string.split("_") + return parts[0] + "".join(part.capitalize() for part in parts[1:]) + + +COLOR_FLAGS_MURFEY_TO_ISPYB = { + value: _snake_to_camel_case(value) for value in COLOR_FLAGS_MURFEY.values() +} + + def _register_dcg_and_atlas( session_id: int, instrument_name: str, @@ -262,7 +277,11 @@ def _register_dcg_and_atlas( else: atlas_name = str(output_file.parent / "*.tiff") atlas_pixel_size = result.pixel_size - color_flags = _get_color_flags(result.output_files.keys()) + # Translate colour flags into ISPyB convention + color_flags = { + COLOR_FLAGS_MURFEY_TO_ISPYB[key]: int(value) + for key, value in _get_color_flags(result.output_files.keys()).items() + } collection_mode = _determine_collection_mode(result.output_files.keys()) else: atlas_name = "" @@ -455,8 +474,13 @@ def _register_grid_square( y_stage_position=0.5 * (clem_img_series.y0 + clem_img_series.y1), pixel_size=clem_img_series.image_pixel_size, image=clem_img_series.thumbnail_search_string, - collection_mode=_determine_collection_mode(result.output_files.keys()), + collection_mode=clem_img_series.collection_mode, ) + # Construct colour flags for ISPyB + color_flags = { + ispyb_color_flags: int(getattr(clem_img_series, murfey_color_flags, 0)) + for murfey_color_flags, ispyb_color_flags in COLOR_FLAGS_MURFEY_TO_ISPYB.items() + } # Register or update the grid square entry as required if grid_square_result := murfey_db.exec( select(MurfeyDB.GridSquare) @@ -481,7 +505,7 @@ def _register_grid_square( _transport_object.do_update_grid_square( grid_square_id=grid_square_entry.id, grid_square_parameters=grid_square_params, - color_flags=_get_color_flags(result.output_files.keys()), + color_flags=color_flags, ) else: # Look up data collection group for current series @@ -495,7 +519,7 @@ def _register_grid_square( atlas_id=dcg_entry.atlas_id, grid_square_id=clem_img_series.id, grid_square_parameters=grid_square_params, - color_flags=_get_color_flags(result.output_files.keys()), + color_flags=color_flags, ) # Register to Murfey grid_square_entry = MurfeyDB.GridSquare( diff --git a/tests/workflows/clem/test_register_preprocessing_results.py b/tests/workflows/clem/test_register_preprocessing_results.py index 75925211d..b701a6b6d 100644 --- a/tests/workflows/clem/test_register_preprocessing_results.py +++ b/tests/workflows/clem/test_register_preprocessing_results.py @@ -12,11 +12,13 @@ import murfey.util.db as MurfeyDB from murfey.workflows.clem.register_preprocessing_results import ( + COLOR_FLAGS_MURFEY_TO_ISPYB, _determine_collection_mode, _get_color_flags, _register_clem_image_series, _register_dcg_and_atlas, _register_grid_square, + _snake_to_camel_case, run, ) from tests.conftest import ExampleVisit, get_or_create_db_entry @@ -118,75 +120,68 @@ def generate_preprocessing_messages( return messages -def test_register_clem_image_series(): - _register_clem_image_series - - @pytest.mark.parametrize( "test_params", ( ( ["gray"], { - "hasGrey": 1, + "has_grey": True, }, ), ( ["gray", "red"], { - "hasGrey": 1, - "hasRed": 1, + "has_grey": True, + "has_red": True, }, ), ( ["red", "green", "blue"], { - "hasRed": 1, - "hasGreen": 1, - "hasBlue": 1, + "has_red": True, + "has_green": True, + "has_blue": True, }, ), ( ["cyan", "magenta", "yellow"], { - "hasCyan": 1, - "hasMagenta": 1, - "hasYellow": 1, + "has_cyan": True, + "has_magenta": True, + "has_yellow": True, }, ), ), ) -def test_get_color_flags(test_params: tuple[list[str], dict[str, int]]): - # Unpack test params +def test_get_color_flags(test_params: tuple[list[str], dict[str, bool]]): colors, positive_flags = test_params expected_result = dict.fromkeys( ( - "hasGrey", - "hasRed", - "hasGreen", - "hasBlue", - "hasCyan", - "hasMagenta", - "hasYellow", + "has_grey", + "has_red", + "has_green", + "has_blue", + "has_cyan", + "has_magenta", + "has_yellow", ), - 0, + False, ) for flag, value in positive_flags.items(): expected_result[flag] = value assert _get_color_flags(colors) == expected_result +def test_register_clem_image_series(): + _register_clem_image_series + + @pytest.mark.parametrize( "test_params", ( - ( - ["gray"], - "Bright Field", - ), - ( - ["gray", "blue"], - "Bright Field and Fluorescent", - ), + (["gray"], "Bright Field"), + (["gray", "blue"], "Bright Field and Fluorescent"), (["red", "green", "blue"], "Fluorescent"), ), ) @@ -195,6 +190,25 @@ def test_determine_collection_mode(test_params: tuple[list[str], str]): assert _determine_collection_mode(colors) == expected_result +@pytest.mark.parametrize( + "test_params", + ( + ("has_grey", "hasGrey"), + ("has_red", "hasRed"), + ("has_green", "hasGreen"), + ("has_blue", "hasBlue"), + ("has_cyan", "hasCyan"), + ("has_magenta", "hasMagenta"), + ("has_yellow", "hasYellow"), + ), +) +def test_snake_to_camel_case( + test_params: tuple[str, str], +): + string, expected_result = test_params + assert _snake_to_camel_case(string) == expected_result + + def test_register_dcg_and_atlas(): _register_dcg_and_atlas @@ -267,18 +281,8 @@ def test_run( "test_params", ( # Reverse list order? | Colors - ( - False, - [ - "gray", - ], - ), - ( - True, - [ - "gray", - ], - ), + (False, ["gray"]), + (True, ["gray"]), (False, ["red", "green", "blue"]), (True, ["cyan", "magenta", "yellow"]), (False, ["gray", "red", "green", "blue"]), @@ -417,7 +421,10 @@ def test_run_with_db( assert len(ispyb_atlas_search) == 1 # Determine the color flags and collection mode - color_flags = _get_color_flags(colors) + color_flags = { + COLOR_FLAGS_MURFEY_TO_ISPYB[flag]: int(value) + for flag, value in _get_color_flags(colors).items() + } collection_mode = _determine_collection_mode(colors) ispyb_atlas = ispyb_atlas_search[0] From 365afd707d06c718ffc2b58767fc8b6a3b9c8547 Mon Sep 17 00:00:00 2001 From: Eu Pin Tien Date: Fri, 6 Feb 2026 12:47:27 +0000 Subject: [PATCH 19/20] Unpacked dictionary incorrectly --- src/murfey/workflows/clem/register_preprocessing_results.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/murfey/workflows/clem/register_preprocessing_results.py b/src/murfey/workflows/clem/register_preprocessing_results.py index dd4c3a58d..e127e5815 100644 --- a/src/murfey/workflows/clem/register_preprocessing_results.py +++ b/src/murfey/workflows/clem/register_preprocessing_results.py @@ -181,7 +181,7 @@ def _register_clem_image_series( clem_img_series.image_search_string = str(output_file.parent / "*tiff") clem_img_series.data_type = "atlas" if _is_clem_atlas(result) else "grid_square" clem_img_series.number_of_members = result.number_of_members - for col_name, value in _get_color_flags(result.output_files.keys()): + for col_name, value in _get_color_flags(result.output_files.keys()).items(): setattr(clem_img_series, col_name, value) clem_img_series.collection_mode = _determine_collection_mode( result.output_files.keys() From d7320d457640c695236a0cffd5ca5c623d801e8d Mon Sep 17 00:00:00 2001 From: Eu Pin Tien Date: Tue, 10 Feb 2026 12:41:34 +0000 Subject: [PATCH 20/20] Fixed error with calculating the location of ROIs on the atlas image --- .../clem/register_preprocessing_results.py | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/src/murfey/workflows/clem/register_preprocessing_results.py b/src/murfey/workflows/clem/register_preprocessing_results.py index e127e5815..aa8d8dda0 100644 --- a/src/murfey/workflows/clem/register_preprocessing_results.py +++ b/src/murfey/workflows/clem/register_preprocessing_results.py @@ -406,6 +406,8 @@ def _register_grid_square( and atlas_entry.x1 is not None and atlas_entry.y0 is not None and atlas_entry.y1 is not None + and atlas_entry.thumbnail_pixels_x is not None + and atlas_entry.thumbnail_pixels_y is not None ): atlas_width_real = atlas_entry.x1 - atlas_entry.x0 atlas_height_real = atlas_entry.y1 - atlas_entry.y0 @@ -420,34 +422,31 @@ def _register_grid_square( and clem_img_series.x1 is not None and clem_img_series.y0 is not None and clem_img_series.y1 is not None - and clem_img_series.thumbnail_pixels_x is not None - and clem_img_series.thumbnail_pixels_y is not None - and clem_img_series.thumbnail_pixel_size is not None ): # Find pixel corresponding to image midpoint on atlas x_mid_real = ( 0.5 * (clem_img_series.x0 + clem_img_series.x1) - atlas_entry.x0 ) x_mid_px = int( - x_mid_real / atlas_width_real * clem_img_series.thumbnail_pixels_x + x_mid_real / atlas_width_real * atlas_entry.thumbnail_pixels_x ) y_mid_real = ( 0.5 * (clem_img_series.y0 + clem_img_series.y1) - atlas_entry.y0 ) y_mid_px = int( - y_mid_real / atlas_height_real * clem_img_series.thumbnail_pixels_y + y_mid_real / atlas_height_real * atlas_entry.thumbnail_pixels_y ) - # Find the size of the image, in pixels, when overlaid the atlas + # Find the size of the image, in pixels, when overlaid on the atlas width_scaled = int( (clem_img_series.x1 - clem_img_series.x0) / atlas_width_real - * clem_img_series.thumbnail_pixels_x + * atlas_entry.thumbnail_pixels_x ) height_scaled = int( (clem_img_series.y1 - clem_img_series.y0) / atlas_height_real - * clem_img_series.thumbnail_pixels_y + * atlas_entry.thumbnail_pixels_y ) else: logger.warning(