From 26bdbcfda17293d7d406d949d65dab0937d7a7ca Mon Sep 17 00:00:00 2001 From: Martin Hjelmare Date: Sat, 28 Dec 2019 16:03:19 +0100 Subject: [PATCH 1/4] Fix imports --- camacqplugins/gain/__init__.py | 10 ++++++---- camacqplugins/production/__init__.py | 15 +++++++++------ 2 files changed, 15 insertions(+), 10 deletions(-) diff --git a/camacqplugins/gain/__init__.py b/camacqplugins/gain/__init__.py index 961ebb4..7c77e33 100644 --- a/camacqplugins/gain/__init__.py +++ b/camacqplugins/gain/__init__.py @@ -11,9 +11,7 @@ import voluptuous as vol from scipy.optimize import curve_fit -from camacq.const import CHANNEL_ID, WELL, WELL_NAME from camacq.event import Event -from camacq.plugins.sample import Channel from camacq.helper import BASE_ACTION_SCHEMA from camacq.image import make_proj from camacq.util import write_csv @@ -26,6 +24,7 @@ BOX = "box" COUNT = "count" VALID = "valid" +CHANNEL_ID = "C{:02d}" CONF_CHANNEL = "channel" CONF_CHANNELS = "channels" CONF_GAIN = "gain" @@ -34,6 +33,8 @@ COUNT_CLOSE_TO_ZERO = 2 GAIN_CALC_EVENT = "gain_calc_event" SAVED_GAINS = "saved_gains" +WELL = "well" +WELL_NAME = "U{:02d}--V{:02d}" ACTION_CALC_GAIN = "calc_gain" CALC_GAIN_ACTION_SCHEMA = BASE_ACTION_SCHEMA.extend( @@ -60,6 +61,7 @@ GAIN = "gain" Data = namedtuple("Data", [BOX, GAIN, VALID]) # pylint: disable=invalid-name +Channel = namedtuple("Channel", ["name", GAIN]) # pylint: disable=invalid-name async def setup_module(center, config): @@ -108,7 +110,7 @@ async def calc_gain( ] # This should be a path to a base file name, not to a dir or file. - plot_path = plot_dir / f"U{well_x:02}--V{well_y:02}" + plot_path = plot_dir / WELL_NAME.format(well_x, well_y) gains = await center.add_executor_job( partial(_calc_gain, projs, init_gain, plot=make_plots, save_path=plot_path) ) @@ -209,7 +211,7 @@ def _calc_gain(projs, init_gain, plot=True, save_path=""): y_data = roi[BOX].astype(float).values coeffs, _ = curve_fit(_power_func, x_data, y_data, p0=(1000, -1)) if plot: - _save_path = "{}{}.ome.png".format(save_path, CHANNEL_ID.format(c_id)) + _save_path = "{}_{}.ome.png".format(save_path, CHANNEL_ID.format(c_id)) _create_plot( _save_path, hist_data[COUNT], hist_data[BOX], coeffs, "count-box" ) diff --git a/camacqplugins/production/__init__.py b/camacqplugins/production/__init__.py index 9bbb997..f116cb3 100644 --- a/camacqplugins/production/__init__.py +++ b/camacqplugins/production/__init__.py @@ -4,11 +4,15 @@ import voluptuous as vol -from camacq.const import CAMACQ_START_EVENT, CHANNEL_EVENT, IMAGE_EVENT, WELL_EVENT +from camacq.const import CAMACQ_START_EVENT, IMAGE_EVENT from camacq.event import match_event from camacq.plugins.leica.command import cam_com, del_com, gain_com -from camacq.plugins.sample import ACTION_TO_METHOD, ACTION_SET_PLATE -from camacq.plugins.sample.helper import next_well_xy +from camacq.plugins.leica.sample import ( + CHANNEL_EVENT, + SET_SAMPLE_SCHEMA, + WELL_EVENT, + next_well_xy, +) from camacq.util import read_csv _LOGGER = logging.getLogger(__name__) @@ -49,10 +53,9 @@ def is_sample_state(value): for idx, data in enumerate(value): valid = False error = None - for action, settings in ACTION_TO_METHOD.items(): - if action == ACTION_SET_PLATE: + for schema in SET_SAMPLE_SCHEMA.validators: + if schema.schema["name"] == "plate": continue - schema = settings["schema"] try: data.update(schema(data)) except vol.Invalid as exc: From 16a176f9a962ae74f3e18581646feca597d98ce6 Mon Sep 17 00:00:00 2001 From: Martin Hjelmare Date: Sun, 29 Dec 2019 14:29:31 +0100 Subject: [PATCH 2/4] Use new sample api and fix tests --- camacqplugins/gain/__init__.py | 12 +++--- camacqplugins/production/README.md | 18 ++++---- camacqplugins/production/__init__.py | 61 ++++++++++++++++------------ tests/conftest.py | 11 ++++- tests/gain/test_gain.py | 2 +- tests/production/test_production.py | 48 +++++++++++++++------- 6 files changed, 97 insertions(+), 55 deletions(-) diff --git a/camacqplugins/gain/__init__.py b/camacqplugins/gain/__init__.py index 7c77e33..71bc2be 100644 --- a/camacqplugins/gain/__init__.py +++ b/camacqplugins/gain/__init__.py @@ -69,19 +69,21 @@ async def setup_module(center, config): async def handle_calc_gain(**kwargs): """Handle call to calc_gain action.""" - well_x = kwargs.get("well_x") - well_y = kwargs.get("well_y") - plate_name = kwargs.get("plate_name") + well_x = kwargs["well_x"] + well_y = kwargs["well_y"] + plate_name = kwargs["plate_name"] paths = kwargs.get("images") # list of paths to calculate gain for if not paths: - well = center.sample.get_well(plate_name, well_x, well_y) + well = center.samples.leica.get_sample( + "well", plate_name=plate_name, well_x=well_x, well_y=well_y + ) if not well: return images = {path: image.channel_id for path, image in well.images.items()} else: images = { path: image.channel_id - for path, image in center.sample.images.items() + for path, image in center.samples.leica.images.items() if path in paths } projs = await center.add_executor_job(make_proj, images) diff --git a/camacqplugins/production/README.md b/camacqplugins/production/README.md index c3566b1..9a934e2 100644 --- a/camacqplugins/production/README.md +++ b/camacqplugins/production/README.md @@ -35,20 +35,22 @@ production: ``` Each row in the csv file should represent at least one state of a sample container, -ie well, field or channel. A plate name must also be included. The csv file should have a -header. See below. +ie well, field, channel or z_slice. A plate name must also be included. The csv file should have a +header. The first column should have the name of the most low level container to create. +Eg a field must be part of a well which must be part of a plate, so field is the most low level +container of those containers. See below. ```csv -plate_name,well_x,well_y,channel_name,gain -00,1,1,blue,600 +name,plate_name,well_x,well_y,channel_id +channel,00,1,1,0 ``` -This example will set a plate '00', a well (1, 1), a blue channel -and set the gain of the blue channel to 600. +This example will set a plate '00', a well (1, 1), and a channel +with channel id 0. ```csv -plate_name,well_x,well_y,field_x,field_y -00,1,1,1,1 +name,plate_name,well_x,well_y,field_x,field_y +field,00,1,1,1,1 ``` This example will create a plate '00' a well (1, 1) and a field (1, 1) diff --git a/camacqplugins/production/__init__.py b/camacqplugins/production/__init__.py index f116cb3..03d5bee 100644 --- a/camacqplugins/production/__init__.py +++ b/camacqplugins/production/__init__.py @@ -13,6 +13,7 @@ WELL_EVENT, next_well_xy, ) +from camacq.plugins.sample import get_matched_samples from camacq.util import read_csv _LOGGER = logging.getLogger(__name__) @@ -50,19 +51,24 @@ def is_sample_state(value): At least one sample action must validate per sample data item. """ + schemas = list(SET_SAMPLE_SCHEMA.validators) for idx, data in enumerate(value): valid = False error = None - for schema in SET_SAMPLE_SCHEMA.validators: - if schema.schema["name"] == "plate": + sample_name = data.get("name") + for schema in schemas: + if ( + schema.schema["name"] in ("plate", "image") + or schema.schema["name"] != sample_name + ): continue try: data.update(schema(data)) except vol.Invalid as exc: error = exc continue - else: - valid = True + valid = True + break if not valid: _LOGGER.error( @@ -70,7 +76,9 @@ def is_sample_state(value): idx + 2, error, ) - raise error + if error: + raise error + raise vol.Invalid("Invalid sample state file.") return value @@ -158,24 +166,16 @@ async def setup(self, state_data): async def load_sample(self, state_data): """Load sample state.""" for data in state_data: - if data[SAMPLE_PLATE_NAME] not in self._center.sample.plates: - await self._center.actions.sample.set_plate(silent=True, **data) well_coord = data[SAMPLE_WELL_X], data[SAMPLE_WELL_Y] self.wells_left.add(well_coord) - if ( - well_coord - not in self._center.sample.plates[data[SAMPLE_PLATE_NAME]].wells - ): - await self._center.actions.sample.set_well(silent=True, **data) - await self._center.actions.sample.set_channel(silent=True, **data) - await self._center.actions.sample.set_field(silent=True, **data) + await self._center.actions.sample.set_sample(silent=True, **data) def image_next_well_on_sample(self): """Image next well in existing sample.""" async def send_cam_job(center, event): """Run on well event.""" - next_well_x, next_well_y = next_well_xy(center.sample, PLATE_NAME) + next_well_x, next_well_y = next_well_xy(center.samples.leica, PLATE_NAME) if ( not match_event(event, event_type=CAMACQ_START_EVENT) @@ -190,7 +190,7 @@ async def send_cam_job(center, event): ): return - if center.sample.images: + if center.samples.leica.images: await center.actions.command.stop_imaging() await self.send_gain_jobs( next_well_x, next_well_y, @@ -237,10 +237,10 @@ def set_exp_gain(self): async def set_gain(center, event): """Set pmt gain.""" - channel = next( + channel_id, channel = next( ( - channel - for channel in self.channels + (channel_id, channel) + for channel_id, channel in enumerate(self.channels) if event.channel_name == channel[CONF_CHANNEL] ) ) @@ -253,12 +253,13 @@ async def set_gain(center, event): # Set the gain at the microscope. await center.actions.command.send(command=command) # Set the gain in the sample state. - await center.actions.sample.set_channel( + await center.actions.sample.set_sample( + name="channel", plate_name=event.plate_name, well_x=event.well_x, well_y=event.well_y, - channel_name=event.channel_name, - gain=gain, + channel_id=channel_id, + values={"channel_name": event.channel_name, "gain": gain}, ) return self._center.bus.register("gain_calc_event", set_gain) @@ -269,8 +270,17 @@ def add_exp_job(self): async def add_cam_job(center, event): """Add an experiment job to the cam list.""" last_channel = self.channels[-1] + channels = get_matched_samples( + center.samples.leica, + "channel", + { + "plate_name": event.plate_name, + "well_x": event.well_x, + "well_y": event.well_y, + }, + ) if not match_event(event, channel_name=last_channel[CONF_CHANNEL]) or len( - event.well.channels + channels ) != len(self.channels): return @@ -377,13 +387,14 @@ async def set_sample_img_ok(self, center, event): if not match_event(event, job_id=self.exp_job_ids[-1]): return - await center.actions.sample.set_field( + await center.actions.sample.set_sample( + name="field", plate_name=event.plate_name, well_x=event.well_x, well_y=event.well_y, field_x=event.field_x, field_y=event.field_y, - img_ok=True, + values={"img_ok": True}, ) diff --git a/tests/conftest.py b/tests/conftest.py index 3ae48e2..cd2116f 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -2,11 +2,18 @@ import pytest from camacq.control import Center +from camacq.plugins.leica import sample as leica_sample_mod -@pytest.fixture -def center(event_loop): +@pytest.fixture(name="center") +def center_fixture(event_loop): """Give access to center via fixture.""" _center = Center(loop=event_loop) _center._track_tasks = True # pylint: disable=protected-access yield _center + + +@pytest.fixture(name="leica_sample") +async def leica_sample_fixture(center): + """Mock leica sample.""" + await leica_sample_mod.setup_module(center, {}) diff --git a/tests/gain/test_gain.py b/tests/gain/test_gain.py index 3933d66..5eb8c53 100644 --- a/tests/gain/test_gain.py +++ b/tests/gain/test_gain.py @@ -28,7 +28,7 @@ def load_image_fixture(): yield load_image -async def test_gain(center, load_image): +async def test_gain(center, leica_sample, load_image): """Run gain calculation test.""" config = { "gain": { diff --git a/tests/production/test_production.py b/tests/production/test_production.py index 061c0bd..827f0b1 100644 --- a/tests/production/test_production.py +++ b/tests/production/test_production.py @@ -8,6 +8,7 @@ from camacq import plugins from camacq.plugins.api import ImageEvent +from camacq.plugins.sample import get_matched_samples from camacqplugins.gain import GainCalcEvent # All test coroutines will be treated as marked. @@ -50,12 +51,12 @@ """ SAMPLE_STATE = """ -plate_name,well_x,well_y,channel_name,gain,field_x,field_y -00,0,0,,,0,0 -00,0,0,,,0,1 -00,0,1 -00,1,0 -00,1,1 +name,plate_name,well_x,well_y,field_x,field_y +field,00,0,0,0,0 +field,00,0,0,0,1 +well,00,0,1 +well,00,1,0 +well,00,1,1 """.strip() @@ -70,7 +71,7 @@ def job_id(self): return self.data.get("job_id") -async def test_image_events(center): +async def test_image_events(center, leica_sample): """Test image events.""" config = YAML(typ="safe").load(CONFIG) plate_name = "00" @@ -118,6 +119,7 @@ async def fire_gain_event(**kwargs): "field_x": 1, "field_y": 1, "job_id": 3, + "z_slice_id": 0, "channel_id": 31, } ) @@ -128,12 +130,24 @@ async def fire_gain_event(**kwargs): assert calc_gain.call_args == call( action_id="calc_gain", plate_name=plate_name, well_x=well_x, well_y=well_y, ) + + channels = { + channel["channel"]: channel_id + for channel_id, channel in enumerate(config["production"]["channels"]) + } + for channel_name, gain in gains.items(): - channel = center.sample.get_channel(plate_name, well_x, well_y, channel_name) - assert channel.gain == gain + channel = center.samples.leica.get_sample( + "channel", + plate_name=plate_name, + well_x=well_x, + well_y=well_y, + channel_id=channels[channel_name], + ) + assert channel.values["gain"] == gain -async def test_load_sample(center, tmp_path): +async def test_load_sample(center, leica_sample, tmp_path): """Test loading sample state from file.""" state_file = tmp_path / "state_file.csv" state_file.write_text(SAMPLE_STATE) @@ -143,7 +157,13 @@ async def test_load_sample(center, tmp_path): await plugins.setup_module(center, config) await center.wait_for() - plate = center.sample.get_plate(plate_name) - assert len(plate.wells) == 4 - well = center.sample.get_well(plate_name, 0, 0) - assert len(well.fields) == 2 + wells = get_matched_samples( + center.samples.leica, "well", {"plate_name": plate_name} + ) + assert len(wells) == 4 + fields = get_matched_samples( + center.samples.leica, + "field", + {"plate_name": plate_name, "well_x": 0, "well_y": 0}, + ) + assert len(fields) == 2 From e4b2fbc045e3522b38812fbd5516abce15bb114f Mon Sep 17 00:00:00 2001 From: Martin Hjelmare Date: Fri, 14 Feb 2020 17:44:05 +0100 Subject: [PATCH 3/4] Fix csv read and write --- camacqplugins/gain/__init__.py | 5 +++-- camacqplugins/production/__init__.py | 15 +++++++++++++-- 2 files changed, 16 insertions(+), 4 deletions(-) diff --git a/camacqplugins/gain/__init__.py b/camacqplugins/gain/__init__.py index 71bc2be..f2038dd 100644 --- a/camacqplugins/gain/__init__.py +++ b/camacqplugins/gain/__init__.py @@ -14,7 +14,6 @@ from camacq.event import Event from camacq.helper import BASE_ACTION_SCHEMA from camacq.image import make_proj -from camacq.util import write_csv matplotlib.use("AGG") # use noninteractive default backend # pylint: disable=wrong-import-order, wrong-import-position, ungrouped-imports @@ -266,7 +265,9 @@ def _calc_gain(projs, init_gain, plot=True, save_path=""): def save_gain(save_dir, saved_gains, header): """Save a csv file with gain values per image channel.""" path = os.path.normpath(os.path.join(save_dir, "output_gains.csv")) - write_csv(path, saved_gains, header) + data = pd.DataFrame.from_dict(saved_gains, orient="index", columns=[header[1:]]) + data.index.name = header[0] + data.to_csv(path) def ensure_plot_dir(plot_dir): diff --git a/camacqplugins/production/__init__.py b/camacqplugins/production/__init__.py index 03d5bee..10583db 100644 --- a/camacqplugins/production/__init__.py +++ b/camacqplugins/production/__init__.py @@ -2,6 +2,7 @@ import logging from math import ceil +import pandas as pd import voluptuous as vol from camacq.const import CAMACQ_START_EVENT, IMAGE_EVENT @@ -14,7 +15,6 @@ next_well_xy, ) from camacq.plugins.sample import get_matched_samples -from camacq.util import read_csv _LOGGER = logging.getLogger(__name__) @@ -40,6 +40,17 @@ SAMPLE_WELL_Y = "well_y" +def read_csv(path): + """Return a list where each item is a row and dict.""" + try: + data = pd.read_csv(path, dtype=str) + data = data.fillna(value="") + except Exception as exc: # pylint: disable=broad-except + _LOGGER.error("Failed to read csv file: %s", exc) + raise vol.Invalid from exc + return data.to_dict(orient="records") + + @vol.truth def is_csv(value): """Return true if value ends with .csv.""" @@ -78,7 +89,7 @@ def is_sample_state(value): ) if error: raise error - raise vol.Invalid("Invalid sample state file.") + raise vol.Invalid("Invalid sample state file") return value From 5dabcddaeebda50f67fb2b2f73b9c776fa6d23bc Mon Sep 17 00:00:00 2001 From: Martin Hjelmare Date: Fri, 14 Feb 2020 17:44:47 +0100 Subject: [PATCH 4/4] Update camacq to 0.6.0 --- requirements.txt | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index 4c66360..4b3af0f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -camacq==0.5.0 +camacq==0.6.0 matplotlib==3.1.2 pandas==0.25.3 scipy==1.3.3 diff --git a/setup.py b/setup.py index 86e3518..a054d92 100644 --- a/setup.py +++ b/setup.py @@ -7,7 +7,7 @@ VERSION = (PROJECT_DIR / "camacqplugins" / "VERSION").read_text().strip() README_FILE = PROJECT_DIR / "README.md" LONG_DESCR = README_FILE.read_text(encoding="utf-8") -REQUIRES = ["camacq>=0.5.0", "matplotlib", "pandas", "scipy"] +REQUIRES = ["camacq>=0.6.0", "matplotlib", "pandas", "scipy"] setuptools.setup(